tesseract 4.1.1
Loading...
Searching...
No Matches
ambigs.cpp
Go to the documentation of this file.
1
2// File: ambigs.cpp
3// Description: Functions for dealing with ambiguities
4// (training and recognition).
5// Author: Daria Antonova
6//
7// (C) Copyright 2008, Google Inc.
8// Licensed under the Apache License, Version 2.0 (the "License");
9// you may not use this file except in compliance with the License.
10// You may obtain a copy of the License at
11// http://www.apache.org/licenses/LICENSE-2.0
12// Unless required by applicable law or agreed to in writing, software
13// distributed under the License is distributed on an "AS IS" BASIS,
14// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15// See the License for the specific language governing permissions and
16// limitations under the License.
17//
19
20#include "ambigs.h"
21
22#include <cstdio>
23#include "helpers.h"
24#include "universalambigs.h"
25
26#if defined(_WIN32) && !defined(__GNUC__)
27#define strtok_r(str, delim, saveptr) strtok_s(str, delim, saveptr)
28#endif /* _WIN32 && !__GNUC__ */
29
30namespace tesseract {
31
32static const char kAmbigDelimiters[] = "\t ";
33static const char kIllegalMsg[] =
34 "Illegal ambiguity specification on line %d\n";
35static const char kIllegalUnicharMsg[] =
36 "Illegal unichar %s in ambiguity specification\n";
37
38// Maximum line size:
39// 10 for sizes of ambigs, tabs, abmig type and newline
40// UNICHAR_LEN * (MAX_AMBIG_SIZE + 1) for each part of the ambig
42
44 wrong_ngram[0] = INVALID_UNICHAR_ID;
45 correct_fragments[0] = INVALID_UNICHAR_ID;
46 correct_ngram_id = INVALID_UNICHAR_ID;
49}
50
52
53// Initializes the ambigs by adding a nullptr pointer to each table.
54void UnicharAmbigs::InitUnicharAmbigs(const UNICHARSET& unicharset,
55 bool use_ambigs_for_adaption) {
56 for (int i = 0; i < unicharset.size(); ++i) {
57 replace_ambigs_.push_back(nullptr);
58 dang_ambigs_.push_back(nullptr);
59 one_to_one_definite_ambigs_.push_back(nullptr);
60 if (use_ambigs_for_adaption) {
61 ambigs_for_adaption_.push_back(nullptr);
62 reverse_ambigs_for_adaption_.push_back(nullptr);
63 }
64 }
65}
66
67// Loads the universal ambigs that are useful for any language.
69 UNICHARSET* unicharset) {
70 TFile file;
72 LoadUnicharAmbigs(encoder_set, &file, 0, false, unicharset);
73}
74
76 TFile *ambig_file,
77 int debug_level,
78 bool use_ambigs_for_adaption,
79 UNICHARSET *unicharset) {
80 int i, j;
81 UnicharIdVector *adaption_ambigs_entry;
82 if (debug_level) tprintf("Reading ambiguities\n");
83
84 int test_ambig_part_size;
85 int replacement_ambig_part_size;
86 // The space for buffer is allocated on the heap to avoid
87 // GCC frame size warning.
88 const int kBufferSize = 10 + 2 * kMaxAmbigStringSize;
89 char *buffer = new char[kBufferSize];
90 char replacement_string[kMaxAmbigStringSize];
91 UNICHAR_ID test_unichar_ids[MAX_AMBIG_SIZE + 1];
92 int line_num = 0;
93 int type = NOT_AMBIG;
94
95 // Determine the version of the ambigs file.
96 int version = 0;
97 ASSERT_HOST(ambig_file->FGets(buffer, kBufferSize) != nullptr &&
98 strlen(buffer) > 0);
99 if (*buffer == 'v') {
100 version = static_cast<int>(strtol(buffer+1, nullptr, 10));
101 ++line_num;
102 } else {
103 ambig_file->Rewind();
104 }
105 while (ambig_file->FGets(buffer, kBufferSize) != nullptr) {
106 chomp_string(buffer);
107 if (debug_level > 2) tprintf("read line %s\n", buffer);
108 ++line_num;
109 if (!ParseAmbiguityLine(line_num, version, debug_level, encoder_set,
110 buffer, &test_ambig_part_size, test_unichar_ids,
111 &replacement_ambig_part_size,
112 replacement_string, &type)) continue;
113 // Construct AmbigSpec and add it to the appropriate AmbigSpec_LIST.
114 auto *ambig_spec = new AmbigSpec();
115 if (!InsertIntoTable((type == REPLACE_AMBIG) ? replace_ambigs_
116 : dang_ambigs_,
117 test_ambig_part_size, test_unichar_ids,
118 replacement_ambig_part_size, replacement_string, type,
119 ambig_spec, unicharset))
120 continue;
121
122 // Update one_to_one_definite_ambigs_.
123 if (test_ambig_part_size == 1 &&
124 replacement_ambig_part_size == 1 && type == DEFINITE_AMBIG) {
125 if (one_to_one_definite_ambigs_[test_unichar_ids[0]] == nullptr) {
126 one_to_one_definite_ambigs_[test_unichar_ids[0]] = new UnicharIdVector();
127 }
128 one_to_one_definite_ambigs_[test_unichar_ids[0]]->push_back(
129 ambig_spec->correct_ngram_id);
130 }
131 // Update ambigs_for_adaption_.
132 if (use_ambigs_for_adaption) {
134 // Silently ignore invalid strings, as before, so it is safe to use a
135 // universal ambigs file.
136 if (unicharset->encode_string(replacement_string, true, &encoding,
137 nullptr, nullptr)) {
138 for (i = 0; i < test_ambig_part_size; ++i) {
139 if (ambigs_for_adaption_[test_unichar_ids[i]] == nullptr) {
140 ambigs_for_adaption_[test_unichar_ids[i]] = new UnicharIdVector();
141 }
142 adaption_ambigs_entry = ambigs_for_adaption_[test_unichar_ids[i]];
143 for (int r = 0; r < encoding.size(); ++r) {
144 UNICHAR_ID id_to_insert = encoding[r];
145 ASSERT_HOST(id_to_insert != INVALID_UNICHAR_ID);
146 // Add the new unichar id to adaption_ambigs_entry (only if the
147 // vector does not already contain it) keeping it in sorted order.
148 for (j = 0; j < adaption_ambigs_entry->size() &&
149 (*adaption_ambigs_entry)[j] > id_to_insert; ++j);
150 if (j < adaption_ambigs_entry->size()) {
151 if ((*adaption_ambigs_entry)[j] != id_to_insert) {
152 adaption_ambigs_entry->insert(id_to_insert, j);
153 }
154 } else {
155 adaption_ambigs_entry->push_back(id_to_insert);
156 }
157 }
158 }
159 }
160 }
161 }
162 delete[] buffer;
163
164 // Fill in reverse_ambigs_for_adaption from ambigs_for_adaption vector.
165 if (use_ambigs_for_adaption) {
166 for (i = 0; i < ambigs_for_adaption_.size(); ++i) {
167 adaption_ambigs_entry = ambigs_for_adaption_[i];
168 if (adaption_ambigs_entry == nullptr) continue;
169 for (j = 0; j < adaption_ambigs_entry->size(); ++j) {
170 UNICHAR_ID ambig_id = (*adaption_ambigs_entry)[j];
171 if (reverse_ambigs_for_adaption_[ambig_id] == nullptr) {
172 reverse_ambigs_for_adaption_[ambig_id] = new UnicharIdVector();
173 }
174 reverse_ambigs_for_adaption_[ambig_id]->push_back(i);
175 }
176 }
177 }
178
179 // Print what was read from the input file.
180 if (debug_level > 1) {
181 for (int tbl = 0; tbl < 2; ++tbl) {
182 const UnicharAmbigsVector &print_table =
183 (tbl == 0) ? replace_ambigs_ : dang_ambigs_;
184 for (i = 0; i < print_table.size(); ++i) {
185 AmbigSpec_LIST *lst = print_table[i];
186 if (lst == nullptr) continue;
187 if (!lst->empty()) {
188 tprintf("%s Ambiguities for %s:\n",
189 (tbl == 0) ? "Replaceable" : "Dangerous",
190 unicharset->debug_str(i).string());
191 }
192 AmbigSpec_IT lst_it(lst);
193 for (lst_it.mark_cycle_pt(); !lst_it.cycled_list(); lst_it.forward()) {
194 AmbigSpec *ambig_spec = lst_it.data();
195 tprintf("wrong_ngram:");
196 UnicharIdArrayUtils::print(ambig_spec->wrong_ngram, *unicharset);
197 tprintf("correct_fragments:");
198 UnicharIdArrayUtils::print(ambig_spec->correct_fragments, *unicharset);
199 }
200 }
201 }
202 if (use_ambigs_for_adaption) {
203 for (int vec_id = 0; vec_id < 2; ++vec_id) {
204 const GenericVector<UnicharIdVector *> &vec = (vec_id == 0) ?
205 ambigs_for_adaption_ : reverse_ambigs_for_adaption_;
206 for (i = 0; i < vec.size(); ++i) {
207 adaption_ambigs_entry = vec[i];
208 if (adaption_ambigs_entry != nullptr) {
209 tprintf("%sAmbigs for adaption for %s:\n",
210 (vec_id == 0) ? "" : "Reverse ",
211 unicharset->debug_str(i).string());
212 for (j = 0; j < adaption_ambigs_entry->size(); ++j) {
213 tprintf("%s ", unicharset->debug_str(
214 (*adaption_ambigs_entry)[j]).string());
215 }
216 tprintf("\n");
217 }
218 }
219 }
220 }
221 }
222}
223
224bool UnicharAmbigs::ParseAmbiguityLine(
225 int line_num, int version, int debug_level, const UNICHARSET &unicharset,
226 char *buffer, int *test_ambig_part_size, UNICHAR_ID *test_unichar_ids,
227 int *replacement_ambig_part_size, char *replacement_string, int *type) {
228 if (version > 1) {
229 // Simpler format is just wrong-string correct-string type\n.
230 STRING input(buffer);
232 input.split(' ', &fields);
233 if (fields.size() != 3) {
234 if (debug_level) tprintf(kIllegalMsg, line_num);
235 return false;
236 }
237 // Encode wrong-string.
239 if (!unicharset.encode_string(fields[0].string(), true, &unichars, nullptr,
240 nullptr)) {
241 return false;
242 }
243 *test_ambig_part_size = unichars.size();
244 if (*test_ambig_part_size > MAX_AMBIG_SIZE) {
245 if (debug_level)
246 tprintf("Too many unichars in ambiguity on line %d\n", line_num);
247 return false;
248 }
249 // Copy encoded string to output.
250 for (int i = 0; i < unichars.size(); ++i)
251 test_unichar_ids[i] = unichars[i];
252 test_unichar_ids[unichars.size()] = INVALID_UNICHAR_ID;
253 // Encode replacement-string to check validity.
254 if (!unicharset.encode_string(fields[1].string(), true, &unichars, nullptr,
255 nullptr)) {
256 return false;
257 }
258 *replacement_ambig_part_size = unichars.size();
259 if (*replacement_ambig_part_size > MAX_AMBIG_SIZE) {
260 if (debug_level)
261 tprintf("Too many unichars in ambiguity on line %d\n", line_num);
262 return false;
263 }
264 if (sscanf(fields[2].string(), "%d", type) != 1) {
265 if (debug_level) tprintf(kIllegalMsg, line_num);
266 return false;
267 }
268 snprintf(replacement_string, kMaxAmbigStringSize, "%s", fields[1].string());
269 return true;
270 }
271 int i;
272 char *token;
273 char *next_token;
274 if (!(token = strtok_r(buffer, kAmbigDelimiters, &next_token)) ||
275 !sscanf(token, "%d", test_ambig_part_size) ||
276 *test_ambig_part_size <= 0) {
277 if (debug_level) tprintf(kIllegalMsg, line_num);
278 return false;
279 }
280 if (*test_ambig_part_size > MAX_AMBIG_SIZE) {
281 if (debug_level)
282 tprintf("Too many unichars in ambiguity on line %d\n", line_num);
283 return false;
284 }
285 for (i = 0; i < *test_ambig_part_size; ++i) {
286 if (!(token = strtok_r(nullptr, kAmbigDelimiters, &next_token))) break;
287 if (!unicharset.contains_unichar(token)) {
288 if (debug_level) tprintf(kIllegalUnicharMsg, token);
289 break;
290 }
291 test_unichar_ids[i] = unicharset.unichar_to_id(token);
292 }
293 test_unichar_ids[i] = INVALID_UNICHAR_ID;
294
295 if (i != *test_ambig_part_size ||
296 !(token = strtok_r(nullptr, kAmbigDelimiters, &next_token)) ||
297 !sscanf(token, "%d", replacement_ambig_part_size) ||
298 *replacement_ambig_part_size <= 0) {
299 if (debug_level) tprintf(kIllegalMsg, line_num);
300 return false;
301 }
302 if (*replacement_ambig_part_size > MAX_AMBIG_SIZE) {
303 if (debug_level)
304 tprintf("Too many unichars in ambiguity on line %d\n", line_num);
305 return false;
306 }
307 replacement_string[0] = '\0';
308 for (i = 0; i < *replacement_ambig_part_size; ++i) {
309 if (!(token = strtok_r(nullptr, kAmbigDelimiters, &next_token))) break;
310 strcat(replacement_string, token);
311 if (!unicharset.contains_unichar(token)) {
312 if (debug_level) tprintf(kIllegalUnicharMsg, token);
313 break;
314 }
315 }
316 if (i != *replacement_ambig_part_size) {
317 if (debug_level) tprintf(kIllegalMsg, line_num);
318 return false;
319 }
320 if (version > 0) {
321 // The next field being true indicates that the abiguity should
322 // always be substituted (e.g. '' should always be changed to ").
323 // For such "certain" n -> m ambigs tesseract will insert character
324 // fragments for the n pieces in the unicharset. AmbigsFound()
325 // will then replace the incorrect ngram with the character
326 // fragments of the correct character (or ngram if m > 1).
327 // Note that if m > 1, an ngram will be inserted into the
328 // modified word, not the individual unigrams. Tesseract
329 // has limited support for ngram unichar (e.g. dawg permuter).
330 if (!(token = strtok_r(nullptr, kAmbigDelimiters, &next_token)) ||
331 !sscanf(token, "%d", type)) {
332 if (debug_level) tprintf(kIllegalMsg, line_num);
333 return false;
334 }
335 }
336 return true;
337}
338
339bool UnicharAmbigs::InsertIntoTable(
340 UnicharAmbigsVector &table, int test_ambig_part_size,
341 UNICHAR_ID *test_unichar_ids, int replacement_ambig_part_size,
342 const char *replacement_string, int type,
343 AmbigSpec *ambig_spec, UNICHARSET *unicharset) {
344 ambig_spec->type = static_cast<AmbigType>(type);
345 if (test_ambig_part_size == 1 && replacement_ambig_part_size == 1 &&
346 unicharset->to_lower(test_unichar_ids[0]) ==
347 unicharset->to_lower(unicharset->unichar_to_id(replacement_string))) {
348 ambig_spec->type = CASE_AMBIG;
349 }
350
351 ambig_spec->wrong_ngram_size =
352 UnicharIdArrayUtils::copy(test_unichar_ids, ambig_spec->wrong_ngram);
353
354 // Since we need to maintain a constant number of unichar positions in
355 // order to construct ambig_blob_choices vector in NoDangerousAmbig(), for
356 // each n->m ambiguity we will have to place n character fragments of the
357 // correct ngram into the corresponding positions in the vector (e.g. given
358 // "vvvvw" and vvvv->ww we will place v and |ww|0|4 into position 0, v and
359 // |ww|1|4 into position 1 and so on. The correct ngram is reconstructed
360 // from fragments by dawg_permute_and_select().
361
362 // Insert the corresponding correct ngram into the unicharset.
363 // Unicharset code assumes that the "base" ngram is inserted into
364 // the unicharset before fragments of this ngram are inserted.
365 unicharset->unichar_insert(replacement_string, OldUncleanUnichars::kTrue);
366 ambig_spec->correct_ngram_id =
367 unicharset->unichar_to_id(replacement_string);
368 if (replacement_ambig_part_size > 1) {
369 unicharset->set_isngram(ambig_spec->correct_ngram_id, true);
370 }
371 // Add the corresponding fragments of the wrong ngram to unicharset.
372 int i;
373 for (i = 0; i < test_ambig_part_size; ++i) {
374 UNICHAR_ID unichar_id;
375 if (test_ambig_part_size == 1) {
376 unichar_id = ambig_spec->correct_ngram_id;
377 } else {
379 replacement_string, i, test_ambig_part_size, false);
380 unicharset->unichar_insert(frag_str.string(), OldUncleanUnichars::kTrue);
381 unichar_id = unicharset->unichar_to_id(frag_str.string());
382 }
383 ambig_spec->correct_fragments[i] = unichar_id;
384 }
385 ambig_spec->correct_fragments[i] = INVALID_UNICHAR_ID;
386
387 // Add AmbigSpec for this ambiguity to the corresponding AmbigSpec_LIST.
388 // Keep AmbigSpec_LISTs sorted by AmbigSpec.wrong_ngram.
389 if (table[test_unichar_ids[0]] == nullptr) {
390 table[test_unichar_ids[0]] = new AmbigSpec_LIST();
391 }
392 if (table[test_unichar_ids[0]]->add_sorted(
393 AmbigSpec::compare_ambig_specs, true, ambig_spec))
394 return true;
395 delete ambig_spec;
396 return false;
397}
398
399} // namespace tesseract
#define MAX_AMBIG_SIZE
Definition: ambigs.h:31
#define ELISTIZE(CLASSNAME)
Definition: elst.h:931
#define ASSERT_HOST(x)
Definition: errcode.h:88
void chomp_string(char *str)
Definition: helpers.h:77
DLLSYM void tprintf(const char *format,...)
Definition: tprintf.cpp:35
#define UNICHAR_LEN
Definition: unichar.h:30
int UNICHAR_ID
Definition: unichar.h:34
const char kUniversalAmbigsFile[]
GenericVector< UNICHAR_ID > UnicharIdVector
Definition: ambigs.h:35
GenericVector< AmbigSpec_LIST * > UnicharAmbigsVector
Definition: ambigs.h:134
const int ksizeofUniversalAmbigsFile
AmbigType
Definition: ambigs.h:37
@ CASE_AMBIG
Definition: ambigs.h:42
@ DEFINITE_AMBIG
Definition: ambigs.h:40
@ REPLACE_AMBIG
Definition: ambigs.h:39
@ NOT_AMBIG
Definition: ambigs.h:38
const int kMaxAmbigStringSize
Definition: ambigs.cpp:41
int push_back(T object)
int size() const
Definition: genericvector.h:72
void insert(const T &t, int index)
static void print(const UNICHAR_ID array[], const UNICHARSET &unicharset)
Definition: ambigs.h:91
static int copy(const UNICHAR_ID src[], UNICHAR_ID dst[])
Definition: ambigs.h:81
UNICHAR_ID correct_ngram_id
Definition: ambigs.h:126
UNICHAR_ID wrong_ngram[MAX_AMBIG_SIZE+1]
Definition: ambigs.h:124
static int compare_ambig_specs(const void *spec1, const void *spec2)
Definition: ambigs.h:115
UNICHAR_ID correct_fragments[MAX_AMBIG_SIZE+1]
Definition: ambigs.h:125
AmbigType type
Definition: ambigs.h:127
void LoadUniversal(const UNICHARSET &encoder_set, UNICHARSET *unicharset)
Definition: ambigs.cpp:68
void LoadUnicharAmbigs(const UNICHARSET &encoder_set, TFile *ambigs_file, int debug_level, bool use_ambigs_for_adaption, UNICHARSET *unicharset)
Definition: ambigs.cpp:75
char * FGets(char *buffer, int buffer_size)
Definition: serialis.cpp:249
bool Open(const STRING &filename, FileReader reader)
Definition: serialis.cpp:197
Definition: strngs.h:45
const char * string() const
Definition: strngs.cpp:194
STRING to_string() const
Definition: unicharset.h:79
void set_isngram(UNICHAR_ID unichar_id, bool value)
Definition: unicharset.h:456
UNICHAR_ID to_lower(UNICHAR_ID unichar_id) const
Definition: unicharset.h:704
void unichar_insert(const char *const unichar_repr, OldUncleanUnichars old_style)
Definition: unicharset.cpp:626
bool contains_unichar(const char *const unichar_repr) const
Definition: unicharset.cpp:671
bool encode_string(const char *str, bool give_up_on_failure, GenericVector< UNICHAR_ID > *encoding, GenericVector< char > *lengths, int *encoded_length) const
Definition: unicharset.cpp:259
STRING debug_str(UNICHAR_ID id) const
Definition: unicharset.cpp:343
UNICHAR_ID unichar_to_id(const char *const unichar_repr) const
Definition: unicharset.cpp:210