79 {
80 int i, j;
82 if (debug_level)
tprintf(
"Reading ambiguities\n");
83
84 int test_ambig_part_size;
85 int replacement_ambig_part_size;
86
87
89 char *buffer = new char[kBufferSize];
92 int line_num = 0;
94
95
96 int version = 0;
97 ASSERT_HOST(ambig_file->FGets(buffer, kBufferSize) !=
nullptr &&
98 strlen(buffer) > 0);
99 if (*buffer == 'v') {
100 version = static_cast<int>(strtol(buffer+1, nullptr, 10));
101 ++line_num;
102 } else {
103 ambig_file->Rewind();
104 }
105 while (ambig_file->FGets(buffer, kBufferSize) != nullptr) {
107 if (debug_level > 2)
tprintf(
"read line %s\n", buffer);
108 ++line_num;
109 if (!ParseAmbiguityLine(line_num, version, debug_level, encoder_set,
110 buffer, &test_ambig_part_size, test_unichar_ids,
111 &replacement_ambig_part_size,
112 replacement_string, &type)) continue;
113
114 auto *ambig_spec = new AmbigSpec();
115 if (!InsertIntoTable((type ==
REPLACE_AMBIG) ? replace_ambigs_
116 : dang_ambigs_,
117 test_ambig_part_size, test_unichar_ids,
118 replacement_ambig_part_size, replacement_string, type,
119 ambig_spec, unicharset))
120 continue;
121
122
123 if (test_ambig_part_size == 1 &&
125 if (one_to_one_definite_ambigs_[test_unichar_ids[0]] == nullptr) {
126 one_to_one_definite_ambigs_[test_unichar_ids[0]] =
new UnicharIdVector();
127 }
128 one_to_one_definite_ambigs_[test_unichar_ids[0]]->push_back(
129 ambig_spec->correct_ngram_id);
130 }
131
132 if (use_ambigs_for_adaption) {
134
135
136 if (unicharset->
encode_string(replacement_string,
true, &encoding,
137 nullptr, nullptr)) {
138 for (i = 0; i < test_ambig_part_size; ++i) {
139 if (ambigs_for_adaption_[test_unichar_ids[i]] == nullptr) {
141 }
142 adaption_ambigs_entry = ambigs_for_adaption_[test_unichar_ids[i]];
143 for (
int r = 0; r < encoding.
size(); ++r) {
146
147
148 for (j = 0; j < adaption_ambigs_entry->size() &&
149 (*adaption_ambigs_entry)[j] > id_to_insert; ++j);
150 if (j < adaption_ambigs_entry->size()) {
151 if ((*adaption_ambigs_entry)[j] != id_to_insert) {
152 adaption_ambigs_entry->insert(id_to_insert, j);
153 }
154 } else {
155 adaption_ambigs_entry->push_back(id_to_insert);
156 }
157 }
158 }
159 }
160 }
161 }
162 delete[] buffer;
163
164
165 if (use_ambigs_for_adaption) {
166 for (i = 0; i < ambigs_for_adaption_.size(); ++i) {
167 adaption_ambigs_entry = ambigs_for_adaption_[i];
168 if (adaption_ambigs_entry == nullptr) continue;
169 for (j = 0; j < adaption_ambigs_entry->size(); ++j) {
170 UNICHAR_ID ambig_id = (*adaption_ambigs_entry)[j];
171 if (reverse_ambigs_for_adaption_[ambig_id] == nullptr) {
173 }
174 reverse_ambigs_for_adaption_[ambig_id]->push_back(i);
175 }
176 }
177 }
178
179
180 if (debug_level > 1) {
181 for (int tbl = 0; tbl < 2; ++tbl) {
183 (tbl == 0) ? replace_ambigs_ : dang_ambigs_;
184 for (i = 0; i < print_table.size(); ++i) {
185 AmbigSpec_LIST *lst = print_table[i];
186 if (lst == nullptr) continue;
187 if (!lst->empty()) {
188 tprintf(
"%s Ambiguities for %s:\n",
189 (tbl == 0) ? "Replaceable" : "Dangerous",
191 }
192 AmbigSpec_IT lst_it(lst);
193 for (lst_it.mark_cycle_pt(); !lst_it.cycled_list(); lst_it.forward()) {
194 AmbigSpec *ambig_spec = lst_it.data();
199 }
200 }
201 }
202 if (use_ambigs_for_adaption) {
203 for (int vec_id = 0; vec_id < 2; ++vec_id) {
205 ambigs_for_adaption_ : reverse_ambigs_for_adaption_;
206 for (i = 0; i < vec.
size(); ++i) {
207 adaption_ambigs_entry = vec[i];
208 if (adaption_ambigs_entry != nullptr) {
209 tprintf(
"%sAmbigs for adaption for %s:\n",
210 (vec_id == 0) ? "" : "Reverse ",
212 for (j = 0; j < adaption_ambigs_entry->size(); ++j) {
214 (*adaption_ambigs_entry)[j]).
string());
215 }
217 }
218 }
219 }
220 }
221 }
222}
void chomp_string(char *str)
DLLSYM void tprintf(const char *format,...)
GenericVector< UNICHAR_ID > UnicharIdVector
GenericVector< AmbigSpec_LIST * > UnicharAmbigsVector
const int kMaxAmbigStringSize
static void print(const UNICHAR_ID array[], const UNICHARSET &unicharset)
const char * string() const
bool encode_string(const char *str, bool give_up_on_failure, GenericVector< UNICHAR_ID > *encoding, GenericVector< char > *lengths, int *encoded_length) const
STRING debug_str(UNICHAR_ID id) const