22#include "config_auto.h"
57static const char kPermuterTypeNoPerm[] =
"None";
58static const char kPermuterTypePuncPerm[] =
"Punctuation";
59static const char kPermuterTypeTopPerm[] =
"Top Choice";
60static const char kPermuterTypeLowerPerm[] =
"Top Lower Case";
61static const char kPermuterTypeUpperPerm[] =
"Top Upper Case";
62static const char kPermuterTypeNgramPerm[] =
"Ngram";
63static const char kPermuterTypeNumberPerm[] =
"Number";
64static const char kPermuterTypeUserPatPerm[] =
"User Pattern";
65static const char kPermuterTypeSysDawgPerm[] =
"System Dictionary";
66static const char kPermuterTypeDocDawgPerm[] =
"Document Dictionary";
67static const char kPermuterTypeUserDawgPerm[] =
"User Dictionary";
68static const char kPermuterTypeFreqDawgPerm[] =
"Frequent Words Dictionary";
69static const char kPermuterTypeCompoundPerm[] =
"Compound";
71static const char *
const kPermuterTypeNames[] = {
73 kPermuterTypePuncPerm,
75 kPermuterTypeLowerPerm,
76 kPermuterTypeUpperPerm,
77 kPermuterTypeNgramPerm,
78 kPermuterTypeNumberPerm,
79 kPermuterTypeUserPatPerm,
80 kPermuterTypeSysDawgPerm,
81 kPermuterTypeDocDawgPerm,
82 kPermuterTypeUserDawgPerm,
83 kPermuterTypeFreqDawgPerm,
84 kPermuterTypeCompoundPerm
100 unichar_id_ = src_unichar_id;
101 rating_ = src_rating;
102 certainty_ = src_cert;
105 script_id_ = src_script_id;
124 matrix_cell_ = other.matrix_cell_;
125 min_xheight_ = other.min_xheight_;
126 max_xheight_ = other.max_xheight_;
128 classifier_ = other.classifier_;
129#ifndef DISABLED_LEGACY_ENGINE
130 fonts_ = other.fonts_;
143 matrix_cell_ = other.matrix_cell_;
144 min_xheight_ = other.min_xheight_;
145 max_xheight_ = other.max_xheight_;
147 classifier_ = other.classifier_;
148#ifndef DISABLED_LEGACY_ENGINE
149 fonts_ = other.fonts_;
161 tprintf(
"Baseline diff %g for %d v %d\n",
162 baseline_diff, unichar_id_, other.unichar_id_);
168 double denominator =
ClipToRange(std::min(this_range, other_range),
172 overlap /= denominator;
174 tprintf(
"PosAndSize for %d v %d: bl diff = %g, ranges %g, %g / %g ->%g\n",
175 unichar_id_, other.unichar_id_, baseline_diff,
176 this_range, other_range, denominator, overlap);
185 BLOB_CHOICE_LIST* bc_list) {
187 BLOB_CHOICE_IT choice_it(bc_list);
188 for (choice_it.mark_cycle_pt(); !choice_it.cycled_list();
189 choice_it.forward()) {
199 return kPermuterTypeNames[
permuter];
205 switch (script_pos) {
224 : unicharset_(&unicharset){
231 STRING src_lengths = &lengths[0];
250 const char *src_lengths,
253 uint8_t src_permuter) {
254 int src_string_len = strlen(src_string);
255 if (src_string_len == 0) {
258 this->
init(src_lengths ? strlen(src_lengths): src_string_len);
261 for (
int i = 0; i < length_; ++i) {
262 int unichar_length = src_lengths ? src_lengths[i] : 1;
264 unicharset_->
unichar_to_id(src_string+offset, unichar_length);
266 certainties_[i] = src_certainty;
267 offset += unichar_length;
270 adjust_factor_ = 1.0f;
271 rating_ = src_rating;
272 certainty_ = src_certainty;
273 permuter_ = src_permuter;
274 dangerous_ambig_found_ =
false;
281 delete[] unichar_ids_;
282 delete[] script_pos_;
284 delete[] certainties_;
288 return kPermuterTypeNames[permuter_];
296 BLOB_CHOICE_LIST* result = ratings->
get(coord.
col, coord.
row);
297 if (result ==
nullptr) {
298 result =
new BLOB_CHOICE_LIST;
299 ratings->
put(coord.
col, coord.
row, result);
308 for (
int i = 0; i < index; ++i)
310 int row = col + state_[index] - 1;
318 unichar_ids_[index] = blob_choice->
unichar_id();
320 state_[index] = blob_count;
321 certainties_[index] = blob_choice->
certainty();
331 for (
int i = 0; i < length_; ++i) {
349 for (
int i = 0; i < num; ++i) {
351 state_[start - 1] += state_[start + i];
352 else if (start + num < length_)
353 state_[start + num] += state_[start + i];
355 for (
int i = start; i + num < length_; ++i) {
356 unichar_ids_[i] = unichar_ids_[i + num];
357 script_pos_[i] = script_pos_[i + num];
358 state_[i] = state_[i + num];
359 certainties_[i] = certainties_[i + num];
370 for (
int i = 0; i < length_ / 2; ++i) {
372 unichar_ids_[i] = unicharset_->
get_mirror(unichar_ids_[length_-1-i]);
373 unichar_ids_[length_-1-i] = unicharset_->
get_mirror(tmp_id);
375 if (length_ % 2 != 0) {
376 unichar_ids_[length_/2] = unicharset_->
get_mirror(unichar_ids_[length_/2]);
390 while (*start <
length() &&
409 while (start < end &&
421 if (end < start) { end = start; }
423 for (
int i = start; i < end; i++) {
425 unichar_ids_[i], state_[i], 0.0f, certainties_[i]);
437 for (i = 0; i < length_; ++i) {
454 STRING *word_lengths_str)
const {
456 if (word_lengths_str !=
nullptr) *word_lengths_str =
"";
457 for (
int i = 0; i < length_; ++i) {
460 if (word_lengths_str !=
nullptr) {
461 *word_lengths_str += strlen(ch);
474 float rating,
float certainty) {
475 if (length_ == reserved_) {
491 while (reserved_ < length_ + second.
length()) {
495 for (
int i = 0; i < second.
length(); ++i) {
496 unichar_ids_[length_ + i] = other_unichar_ids[i];
497 state_[length_ + i] = second.state_[i];
498 certainties_[length_ + i] = second.certainties_[i];
501 length_ += second.
length();
502 if (second.adjust_factor_ > adjust_factor_)
503 adjust_factor_ = second.adjust_factor_;
504 rating_ += second.
rating();
507 if (second.dangerous_ambig_found_)
508 dangerous_ambig_found_ =
true;
526 while (reserved_ < source.
length()) {
530 unicharset_ = source.unicharset_;
532 for (
int i = 0; i < source.
length(); ++i) {
533 unichar_ids_[i] = other_unichar_ids[i];
534 state_[i] = source.state_[i];
535 certainties_[i] = source.certainties_[i];
538 length_ = source.
length();
539 adjust_factor_ = source.adjust_factor_;
540 rating_ = source.
rating();
545 dangerous_ambig_found_ = source.dangerous_ambig_found_;
556 for (
int i = 0; i < length_; ++i)
562 int position_counts[4] = { 0, 0, 0, 0 };
565 for (
int blob_index = 0; blob_index < length_; ++blob_index, ++chunk_index) {
569 if (state_ !=
nullptr) {
570 for (
int i = 1; i < state_[blob_index]; ++i) {
572 tblob = word->
blobs[chunk_index];
581 position_counts[script_pos_[blob_index]]++;
588 tprintf(
"Most characters of %s are subscript or superscript.\n"
589 "That seems wrong, so I'll assume we got the baseline wrong\n",
592 for (
int i = 0; i < length_; i++) {
593 ScriptPos sp = script_pos_[i];
595 position_counts[sp]--;
606 for (
int blob_index = 0; blob_index < length_; ++blob_index) {
612 chunk_index += state_ !=
nullptr ? state_[blob_index] : 1;
620 if (positions != script_pos_) {
621 delete [] script_pos_;
622 script_pos_ =
new ScriptPos[
length];
623 memcpy(script_pos_, positions,
sizeof(positions[0]) *
length);
628 for (
int i = 0; i < length_; ++i)
629 script_pos_[i] = position;
635 const TBOX& blob_box,
638 int top = blob_box.
top();
639 int bottom = blob_box.
bottom();
640 int min_bottom, max_bottom, min_top, max_top;
642 &min_bottom, &max_bottom,
650 }
else if (top < sub_thresh_top && bottom < sub_thresh_bot) {
652 }
else if (bottom > sup_thresh_bot) {
657 const char *pos = ScriptPosToString(retval);
658 tprintf(
"%s Character %s[bot:%d top: %d] "
659 "bot_range[%d,%d] top_range[%d, %d] "
660 "sub_thresh[bot:%d top:%d] sup_thresh_bot %d\n",
663 min_bottom, max_bottom, min_top, max_top,
664 sub_thresh_bot, sub_thresh_top,
673 int *sid =
new int[max_script];
675 for (x = 0; x < max_script; x++) sid[x] = 0;
676 for (x = 0; x < length_; ++x) {
694 for (x = 1; x < max_script; x++)
695 if (sid[x] >= sid[max_sid]) max_sid = x;
696 if (sid[max_sid] < length_ / 2)
704 int total_chunks = 0;
705 for (
int i = 0; i < length_; ++i) {
706 total_chunks += state_[i];
707 if (total_chunks > blob_position) {
716 int total_chunks = 0;
717 for (
int i = 0; i < length_; ++i) {
718 total_chunks += state_[i];
730 for (
int i = 0; i < length_; ++i) {
733 tprintf(
" : R=%g, C=%g, F=%g, Perm=%d, xht=[%g,%g], ambig=%d\n",
734 rating_, certainty_, adjust_factor_, permuter_,
735 min_x_height_, max_x_height_, dangerous_ambig_found_);
737 for (
int i = 0; i < length_; ++i) {
738 tprintf(
"\t%s", ScriptPosToString(script_pos_[i]));
741 for (
int i = 0; i < length_; ++i) {
745 for (
int i = 0; i < length_; ++i) {
749 for (
int i = 0; i < length_; ++i) {
750 tprintf(
"\t%.3f", certainties_[i]);
758 for (
int i = 0; i < length_; ++i)
766#ifndef GRAPHICS_DISABLED
768 const int kNumColors = 6;
772 bool already_done = prev_drawn_state.
size() == length_;
773 if (!already_done) prev_drawn_state.
init_to_size(length_, 0);
774 for (
int i = 0; i < length_; ++i) {
775 if (prev_drawn_state[i] != state_[i]) {
776 already_done =
false;
778 prev_drawn_state[i] = state_[i];
780 if (already_done || word->
blobs.
empty())
return;
783 if (segm_window ==
nullptr) {
784 segm_window =
new ScrollView(
"Segmentation", 5, 10, 500, 256,
785 2000.0, 256.0,
true);
787 segm_window->
Clear();
792 for (
int c = 0; c < length_; ++c) {
795 for (
int i = 0; i < state_[c]; ++i, ++blob_index) {
798 blob->
plot(segm_window, color, color);
812 if (word2.
unicharset() != uchset)
return false;
817 if (w1end - w1start != w2end - w2start)
return false;
818 for (
int i = 0; i < w1end - w1start; i++) {
838 BLOB_CHOICE_LIST *ratings,
840 if (ratings->length() == 0) {
848 c_it.set_to_list(ratings);
849 for (c_it.mark_cycle_pt(); !c_it.cycled_list(); c_it.forward()) {
850 c_it.data()->print(¤t_unicharset);
851 if (!c_it.at_last())
tprintf(
"\n");
const int kBlnBaselineOffset
const int kMinSubscriptOffset
void print_ratings_list(const char *msg, BLOB_CHOICE_LIST *ratings, const UNICHARSET ¤t_unicharset)
const int kMinSuperscriptOffset
const double kMaxBaselineDrift
const double kMaxOverlapDenominator
const double kMinXHeightMatch
bool EqualIgnoringCaseAndTerminalPunct(const WERD_CHOICE &word1, const WERD_CHOICE &word2)
const int kMaxDropCapBottom
BLOB_CHOICE * FindMatchingChoice(UNICHAR_ID char_id, BLOB_CHOICE_LIST *bc_list)
#define ELISTIZE(CLASSNAME)
T ClipToRange(const T &x, const T &lower_bound, const T &upper_bound)
DLLSYM void tprintf(const char *format,...)
char window_wait(ScrollView *win)
const char * ScriptPosToString(enum ScriptPos script_pos)
void init_to_size(int size, const T &t)
void put(ICOORD pos, const T &thing)
TBOX bounding_box() const
void plot(ScrollView *window, ScrollView::Color color, ScrollView::Color child_color)
GenericVector< TBLOB * > blobs
float max_xheight() const
int16_t fontinfo_id2() const
bool PosAndSizeAgree(const BLOB_CHOICE &other, float x_height, bool debug) const
int16_t fontinfo_id() const
UNICHAR_ID unichar_id() const
float min_xheight() const
MATRIX_COORD MatrixCoord(int index) const
WERD_CHOICE shallow_copy(int start, int end) const
void SetScriptPositions(bool small_caps, TWERD *word, int debug=0)
void UpdateStateForSplit(int blob_position)
const STRING & unichar_string() const
int TotalOfStates() const
void double_the_size()
Make more space in unichar_id_ and fragment_lengths_ arrays.
static tesseract::ScriptPos ScriptPositionOf(bool print_debug, const UNICHARSET &unicharset, const TBOX &blob_box, UNICHAR_ID unichar_id)
void string_and_lengths(STRING *word_str, STRING *word_lengths_str) const
int GetTopScriptID() const
WERD_CHOICE & operator=(const WERD_CHOICE &source)
void punct_stripped(int *start_core, int *end_core) const
void print_state(const char *msg) const
UNICHAR_ID unichar_id(int index) const
bool contains_unichar_id(UNICHAR_ID unichar_id) const
const UNICHARSET * unicharset() const
const char * permuter_name() const
WERD_CHOICE & operator+=(const WERD_CHOICE &second)
void DisplaySegmentation(TWERD *word)
void SetAllScriptPositions(tesseract::ScriptPos position)
BLOB_CHOICE_LIST * blob_choices(int index, MATRIX *ratings) const
static const float kBadRating
void make_bad()
Set the fields in this choice to be default (bad) values.
bool has_rtl_unichar_id() const
void remove_unichar_ids(int index, int num)
void set_blob_choice(int index, int blob_count, const BLOB_CHOICE *blob_choice)
void reverse_and_mirror_unichar_ids()
float min_x_height() const
WERD_CHOICE(const UNICHARSET *unicharset)
tesseract::ScriptPos BlobPosition(int index) const
float max_x_height() const
const UNICHAR_ID * unichar_ids() const
void GetNonSuperscriptSpan(int *start, int *end) const
void append_unichar_id(UNICHAR_ID unichar_id, int blob_count, float rating, float certainty)
void append_unichar_id_space_allocated(UNICHAR_ID unichar_id, int blob_count, float rating, float certainty)
void operator=(const ELIST_LINK &)
const char * string() const
const char * id_to_unichar_ext(UNICHAR_ID id) const
Direction get_direction(UNICHAR_ID unichar_id) const
UNICHAR_ID to_lower(UNICHAR_ID unichar_id) const
int get_script(UNICHAR_ID unichar_id) const
bool get_isdigit(UNICHAR_ID unichar_id) const
const char * id_to_unichar(UNICHAR_ID id) const
bool encode_string(const char *str, bool give_up_on_failure, GenericVector< UNICHAR_ID > *encoding, GenericVector< char > *lengths, int *encoded_length) const
UNICHAR_ID get_mirror(UNICHAR_ID unichar_id) const
int get_script_table_size() const
static std::string CleanupString(const char *utf8_str)
UNICHAR_ID unichar_to_id(const char *const unichar_repr) const
void get_top_bottom(UNICHAR_ID unichar_id, int *min_bottom, int *max_bottom, int *min_top, int *max_top) const
void ZoomToRectangle(int x1, int y1, int x2, int y2)