tesseract 4.1.1
Loading...
Searching...
No Matches
tesseract::Validator Class Referenceabstract

#include <validator.h>

Inheritance diagram for tesseract::Validator:
tesseract::ValidateGrapheme tesseract::ValidateIndic tesseract::ValidateJavanese tesseract::ValidateKhmer tesseract::ValidateMyanmar

Public Member Functions

virtual ~Validator ()
 

Static Public Member Functions

static bool ValidateCleanAndSegment (GraphemeNormMode g_mode, bool report_errors, const std::vector< char32 > &src, std::vector< std::vector< char32 > > *dest)
 
static bool IsZeroWidthMark (char32 ch)
 

Static Public Attributes

static const char32 kZeroWidthSpace = 0x200B
 
static const char32 kZeroWidthNonJoiner = 0x200C
 
static const char32 kZeroWidthJoiner = 0x200D
 
static const char32 kLeftToRightMark = 0x200E
 
static const char32 kRightToLeftMark = 0x200F
 
static const char32 kInvalid = 0xfffd
 

Protected Types

enum class  CharClass {
  kConsonant = 'C' , kVowel = 'V' , kVirama = 'H' , kMatra = 'M' ,
  kMatraPiece = 'P' , kVowelModifier = 'D' , kZeroWidthNonJoiner = 'z' , kZeroWidthJoiner = 'Z' ,
  kVedicMark = 'v' , kNukta = 'N' , kRobat = 'R' , kOther = 'O' ,
  kWhitespace = ' ' , kCombiner = 'c'
}
 
using IndicPair = std::pair< CharClass, char32 >
 

Protected Member Functions

 Validator (ViramaScript script, bool report_errors)
 
bool ValidateCleanAndSegmentInternal (GraphemeNormMode g_mode, const std::vector< char32 > &src, std::vector< std::vector< char32 > > *dest)
 
void MoveResultsToDest (GraphemeNormMode g_mode, std::vector< std::vector< char32 > > *dest)
 
bool IsSubscriptScript () const
 
bool CodeOnlyToOutput ()
 
void MultiCodePart (unsigned length)
 
bool UseMultiCode (unsigned length)
 
virtual bool ConsumeGraphemeIfValid ()=0
 
void ComputeClassCodes (const std::vector< char32 > &text)
 
virtual CharClass UnicodeToCharClass (char32 ch) const =0
 
void Clear ()
 

Static Protected Member Functions

static std::unique_ptr< ValidatorScriptValidator (ViramaScript script, bool report_errors)
 
static ViramaScript MostFrequentViramaScript (const std::vector< char32 > &utf32)
 
static bool IsVirama (char32 unicode)
 
static bool IsVedicAccent (char32 unicode)
 

Protected Attributes

ViramaScript script_
 
std::vector< IndicPaircodes_
 
std::vector< std::vector< char32 > > parts_
 
std::vector< char32output_
 
unsigned codes_used_
 
unsigned output_used_
 
bool report_errors_
 

Static Protected Attributes

static const int kIndicCodePageSize = 128
 
static const char32 kMinIndicUnicode = 0x900
 
static const char32 kMaxSinhalaUnicode = 0xdff
 
static const char32 kMaxViramaScriptUnicode = 0x17ff
 
static const char32 kSinhalaVirama = 0xdca
 
static const char32 kMyanmarVirama = 0x1039
 
static const char32 kKhmerVirama = 0x17d2
 
static const char32 kJavaneseVirama = 0xa9c0
 
static const char32 kMaxJavaneseUnicode = 0xa9df
 

Detailed Description

Definition at line 71 of file validator.h.

Member Typedef Documentation

◆ IndicPair

using tesseract::Validator::IndicPair = std::pair<CharClass, char32>
protected

Definition at line 133 of file validator.h.

Member Enumeration Documentation

◆ CharClass

enum class tesseract::Validator::CharClass
strongprotected
Enumerator
kConsonant 
kVowel 
kVirama 
kMatra 
kMatraPiece 
kVowelModifier 
kZeroWidthNonJoiner 
kZeroWidthJoiner 
kVedicMark 
kNukta 
kRobat 
kOther 
kWhitespace 
kCombiner 

Definition at line 111 of file validator.h.

111 {
112 // NOTE: The values of the enum members are meaningless and arbitrary, ie
113 // they are not used for sorting, or any other risky application.
114 // The reason they are what they are is they are a single character
115 // abbreviation that can be used in a regexp/BNF definition of a grammar,
116 // IN A COMMENT, and still not relied upon in the code.
117 kConsonant = 'C',
118 kVowel = 'V',
119 kVirama = 'H', // (aka Halant)
120 kMatra = 'M', // (aka Dependent Vowel)
121 kMatraPiece = 'P', // unicode provides pieces of Matras.
122 kVowelModifier = 'D', // (candrabindu, anusvara, visarga, other marks)
123 kZeroWidthNonJoiner = 'z', // Unicode Zero Width Non-Joiner U+200C
124 kZeroWidthJoiner = 'Z', // Unicode Zero Width Joiner U+200D
125 kVedicMark = 'v', // Modifiers can come modify any indic syllable.
126 kNukta = 'N', // Occurs only immediately after consonants.
127 kRobat = 'R', // Khmer only.
128 kOther = 'O', // (digits, measures, non-Indic, etc)
129 // Additional classes used only by ValidateGrapheme.
130 kWhitespace = ' ',
131 kCombiner = 'c', // Combiners other than virama.
132 };
static const char32 kZeroWidthNonJoiner
Definition: validator.h:95
static const char32 kZeroWidthJoiner
Definition: validator.h:96

Constructor & Destructor Documentation

◆ ~Validator()

tesseract::Validator::~Validator ( )
virtualdefault

◆ Validator()

tesseract::Validator::Validator ( ViramaScript  script,
bool  report_errors 
)
inlineprotected

Definition at line 135 of file validator.h.

136 : script_(script),
137 codes_used_(0),
138 output_used_(0),
139 report_errors_(report_errors) {}
ViramaScript script_
Definition: validator.h:229
unsigned output_used_
Definition: validator.h:239
unsigned codes_used_
Definition: validator.h:237

Member Function Documentation

◆ Clear()

void tesseract::Validator::Clear ( )
protected

Definition at line 214 of file validator.cpp.

214 {
215 codes_.clear();
216 parts_.clear();
217 output_.clear();
218 codes_used_ = 0;
219 output_used_ = 0;
220}
std::vector< char32 > output_
Definition: validator.h:235
std::vector< IndicPair > codes_
Definition: validator.h:231
std::vector< std::vector< char32 > > parts_
Definition: validator.h:233

◆ CodeOnlyToOutput()

bool tesseract::Validator::CodeOnlyToOutput ( )
inlineprotected

Definition at line 171 of file validator.h.

171 {
172 output_.push_back(codes_[codes_used_].second);
173 return ++codes_used_ == codes_.size();
174 }

◆ ComputeClassCodes()

void tesseract::Validator::ComputeClassCodes ( const std::vector< char32 > &  text)
protected

Definition at line 206 of file validator.cpp.

206 {
207 codes_.reserve(text.size());
208 for (char32 c : text) {
209 codes_.push_back(std::make_pair(UnicodeToCharClass(c), c));
210 }
211}
signed int char32
virtual CharClass UnicodeToCharClass(char32 ch) const =0

◆ ConsumeGraphemeIfValid()

virtual bool tesseract::Validator::ConsumeGraphemeIfValid ( )
protectedpure virtual

◆ IsSubscriptScript()

bool tesseract::Validator::IsSubscriptScript ( ) const
protected

◆ IsVedicAccent()

bool tesseract::Validator::IsVedicAccent ( char32  unicode)
staticprotected

Definition at line 191 of file validator.cpp.

191 {
192 return (0x1cd0 <= unicode && unicode < 0x1d00) ||
193 (0xa8e0 <= unicode && unicode <= 0xa8f7) ||
194 (0x951 <= unicode && unicode <= 0x954);
195}

◆ IsVirama()

bool tesseract::Validator::IsVirama ( char32  unicode)
staticprotected

Definition at line 180 of file validator.cpp.

180 {
181 return (kMinIndicUnicode <= unicode && unicode <= kMaxSinhalaUnicode &&
182 (unicode & 0x7f) == 0x4d) ||
183 unicode == kSinhalaVirama ||
184 unicode == kJavaneseVirama ||
185 unicode == kMyanmarVirama ||
186 unicode == kKhmerVirama;
187}
static const char32 kSinhalaVirama
Definition: validator.h:221
static const char32 kKhmerVirama
Definition: validator.h:223
static const char32 kJavaneseVirama
Definition: validator.h:225
static const char32 kMaxSinhalaUnicode
Definition: validator.h:217
static const char32 kMyanmarVirama
Definition: validator.h:222
static const char32 kMinIndicUnicode
Definition: validator.h:215

◆ IsZeroWidthMark()

static bool tesseract::Validator::IsZeroWidthMark ( char32  ch)
inlinestatic

Definition at line 87 of file validator.h.

87 {
88 return ch == kZeroWidthSpace || ch == kLeftToRightMark ||
89 ch == kRightToLeftMark || ch == kInvalid;
90 }
static const char32 kInvalid
Definition: validator.h:99
static const char32 kRightToLeftMark
Definition: validator.h:98
static const char32 kLeftToRightMark
Definition: validator.h:97
static const char32 kZeroWidthSpace
Definition: validator.h:94

◆ MostFrequentViramaScript()

ViramaScript tesseract::Validator::MostFrequentViramaScript ( const std::vector< char32 > &  utf32)
staticprotected

Definition at line 143 of file validator.cpp.

144 {
145 std::unordered_map<int, int> histogram;
146 for (char32 ch : utf32) {
147 // Determine the codepage base. For the Indic scripts, Khmer and Javanese, it is
148 // sufficient to divide by kIndicCodePageSize but Myanmar is all over the
149 // unicode code space, so use its script id.
150 int base = ch / kIndicCodePageSize;
151 IcuErrorCode err;
152 UScriptCode script_code = uscript_getScript(ch, err);
153 if ((kMinIndicUnicode <= ch && ch <= kMaxJavaneseUnicode &&
154 script_code != USCRIPT_COMMON) ||
155 script_code == USCRIPT_MYANMAR) {
156 if (script_code == USCRIPT_MYANMAR)
157 base = static_cast<char32>(ViramaScript::kMyanmar) / kIndicCodePageSize;
158 ++histogram[base];
159 }
160 }
161 if (!histogram.empty()) {
162 int base =
163 std::max_element(histogram.begin(), histogram.end(), CmpPairSecond)
164 ->first;
165 char32 codebase = static_cast<char32>(base * kIndicCodePageSize);
166 // Check for validity.
167 if (codebase == static_cast<char32>(ViramaScript::kMyanmar) ||
168 codebase == static_cast<char32>(ViramaScript::kJavanese) ||
169 codebase == static_cast<char32>(ViramaScript::kKhmer) ||
170 (static_cast<char32>(ViramaScript::kDevanagari) <= codebase &&
171 codebase <= static_cast<char32>(ViramaScript::kSinhala))) {
172 return static_cast<ViramaScript>(codebase);
173 }
174 }
176}
static const int kIndicCodePageSize
Definition: validator.h:213
static const char32 kMaxJavaneseUnicode
Definition: validator.h:226

◆ MoveResultsToDest()

void tesseract::Validator::MoveResultsToDest ( GraphemeNormMode  g_mode,
std::vector< std::vector< char32 > > *  dest 
)
protected

Definition at line 114 of file validator.cpp.

115 {
117 // Append each element of the combined output_ that we made as a new vector
118 // in dest.
119 dest->reserve(dest->size() + output_.size());
120 for (char32 ch : output_) dest->push_back({ch});
121 } else if (g_mode == GraphemeNormMode::kGlyphSplit) {
122 // Append all the parts_ that we made onto dest.
123 std::move(parts_.begin(), parts_.end(), std::back_inserter(*dest));
124 } else if (g_mode == GraphemeNormMode::kCombined || dest->empty()) {
125 // Append the combined output_ that we made onto dest as one new vector.
126 dest->push_back(std::vector<char32>());
127 output_.swap(dest->back());
128 } else { // kNone.
129 // Append the combined output_ that we made onto the last existing element
130 // of dest.
131 dest->back().insert(dest->back().end(), output_.begin(), output_.end());
132 }
133}

◆ MultiCodePart()

void tesseract::Validator::MultiCodePart ( unsigned  length)
inlineprotected

Definition at line 181 of file validator.h.

181 {
182 while (output_used_ + length < output_.size()) {
183 parts_.emplace_back(
184 std::initializer_list<char32>{output_[output_used_++]});
185 }
186 parts_.emplace_back(std::initializer_list<char32>{output_[output_used_]});
187 while (++output_used_ < output_.size()) {
188 parts_.back().push_back(output_[output_used_]);
189 }
190 }

◆ ScriptValidator()

std::unique_ptr< Validator > tesseract::Validator::ScriptValidator ( ViramaScript  script,
bool  report_errors 
)
staticprotected

Definition at line 71 of file validator.cpp.

72 {
73 switch (script) {
75 return std::unique_ptr<Validator>(
76 new ValidateGrapheme(script, report_errors));
78 return std::unique_ptr<Validator>(
79 new ValidateJavanese(script, report_errors));
81 return std::unique_ptr<Validator>(
82 new ValidateMyanmar(script, report_errors));
84 return std::unique_ptr<Validator>(
85 new ValidateKhmer(script, report_errors));
86 default:
87 return std::unique_ptr<Validator>(
88 new ValidateIndic(script, report_errors));
89 }
90}

◆ UnicodeToCharClass()

virtual CharClass tesseract::Validator::UnicodeToCharClass ( char32  ch) const
protectedpure virtual

◆ UseMultiCode()

bool tesseract::Validator::UseMultiCode ( unsigned  length)
inlineprotected

Definition at line 195 of file validator.h.

195 {
196 output_.push_back(codes_[codes_used_].second);
197 MultiCodePart(length);
198 return ++codes_used_ == codes_.size();
199 }
void MultiCodePart(unsigned length)
Definition: validator.h:181

◆ ValidateCleanAndSegment()

bool tesseract::Validator::ValidateCleanAndSegment ( GraphemeNormMode  g_mode,
bool  report_errors,
const std::vector< char32 > &  src,
std::vector< std::vector< char32 > > *  dest 
)
static

Definition at line 40 of file validator.cpp.

42 {
43 ValidateGrapheme g_validator(ViramaScript::kNonVirama, report_errors);
44 std::vector<std::vector<char32>> graphemes;
46 bool success = true;
47 if (script == ViramaScript::kNonVirama) {
48 // The grapheme segmenter's maximum segmentation is the grapheme unit, so
49 // up the mode by 1 to get the desired effect.
50 if (g_mode == GraphemeNormMode::kCombined)
52 else if (g_mode == GraphemeNormMode::kGlyphSplit)
54 // Just do grapheme segmentation.
55 success = g_validator.ValidateCleanAndSegmentInternal(g_mode, src, dest);
56 } else {
57 success = g_validator.ValidateCleanAndSegmentInternal(
58 GraphemeNormMode::kGlyphSplit, src, &graphemes);
59 std::unique_ptr<Validator> validator(
60 ScriptValidator(script, report_errors));
61 for (const auto& grapheme : graphemes) {
62 if (!validator->ValidateCleanAndSegmentInternal(g_mode, grapheme, dest)) {
63 success = false;
64 }
65 }
66 }
67 return success;
68}
static ViramaScript MostFrequentViramaScript(const std::vector< char32 > &utf32)
Definition: validator.cpp:143
static std::unique_ptr< Validator > ScriptValidator(ViramaScript script, bool report_errors)
Definition: validator.cpp:71

◆ ValidateCleanAndSegmentInternal()

bool tesseract::Validator::ValidateCleanAndSegmentInternal ( GraphemeNormMode  g_mode,
const std::vector< char32 > &  src,
std::vector< std::vector< char32 > > *  dest 
)
protected

Definition at line 97 of file validator.cpp.

99 {
100 Clear();
102 bool success = true;
103 for (codes_used_ = 0; codes_used_ < codes_.size();) {
104 if (!ConsumeGraphemeIfValid()) {
105 success = false;
106 ++codes_used_;
107 }
108 }
109 MoveResultsToDest(g_mode, dest);
110 return success;
111}
void MoveResultsToDest(GraphemeNormMode g_mode, std::vector< std::vector< char32 > > *dest)
Definition: validator.cpp:114
void ComputeClassCodes(const std::vector< char32 > &text)
Definition: validator.cpp:206
virtual bool ConsumeGraphemeIfValid()=0

Member Data Documentation

◆ codes_

std::vector<IndicPair> tesseract::Validator::codes_
protected

Definition at line 231 of file validator.h.

◆ codes_used_

unsigned tesseract::Validator::codes_used_
protected

Definition at line 237 of file validator.h.

◆ kIndicCodePageSize

const int tesseract::Validator::kIndicCodePageSize = 128
staticprotected

Definition at line 213 of file validator.h.

◆ kInvalid

const char32 tesseract::Validator::kInvalid = 0xfffd
static

Definition at line 99 of file validator.h.

◆ kJavaneseVirama

const char32 tesseract::Validator::kJavaneseVirama = 0xa9c0
staticprotected

Definition at line 225 of file validator.h.

◆ kKhmerVirama

const char32 tesseract::Validator::kKhmerVirama = 0x17d2
staticprotected

Definition at line 223 of file validator.h.

◆ kLeftToRightMark

const char32 tesseract::Validator::kLeftToRightMark = 0x200E
static

Definition at line 97 of file validator.h.

◆ kMaxJavaneseUnicode

const char32 tesseract::Validator::kMaxJavaneseUnicode = 0xa9df
staticprotected

Definition at line 226 of file validator.h.

◆ kMaxSinhalaUnicode

const char32 tesseract::Validator::kMaxSinhalaUnicode = 0xdff
staticprotected

Definition at line 217 of file validator.h.

◆ kMaxViramaScriptUnicode

const char32 tesseract::Validator::kMaxViramaScriptUnicode = 0x17ff
staticprotected

Definition at line 219 of file validator.h.

◆ kMinIndicUnicode

const char32 tesseract::Validator::kMinIndicUnicode = 0x900
staticprotected

Definition at line 215 of file validator.h.

◆ kMyanmarVirama

const char32 tesseract::Validator::kMyanmarVirama = 0x1039
staticprotected

Definition at line 222 of file validator.h.

◆ kRightToLeftMark

const char32 tesseract::Validator::kRightToLeftMark = 0x200F
static

Definition at line 98 of file validator.h.

◆ kSinhalaVirama

const char32 tesseract::Validator::kSinhalaVirama = 0xdca
staticprotected

Definition at line 221 of file validator.h.

◆ kZeroWidthJoiner

const char32 tesseract::Validator::kZeroWidthJoiner = 0x200D
static

Definition at line 96 of file validator.h.

◆ kZeroWidthNonJoiner

const char32 tesseract::Validator::kZeroWidthNonJoiner = 0x200C
static

Definition at line 95 of file validator.h.

◆ kZeroWidthSpace

const char32 tesseract::Validator::kZeroWidthSpace = 0x200B
static

Definition at line 94 of file validator.h.

◆ output_

std::vector<char32> tesseract::Validator::output_
protected

Definition at line 235 of file validator.h.

◆ output_used_

unsigned tesseract::Validator::output_used_
protected

Definition at line 239 of file validator.h.

◆ parts_

std::vector<std::vector<char32> > tesseract::Validator::parts_
protected

Definition at line 233 of file validator.h.

◆ report_errors_

bool tesseract::Validator::report_errors_
protected

Definition at line 241 of file validator.h.

◆ script_

ViramaScript tesseract::Validator::script_
protected

Definition at line 229 of file validator.h.


The documentation for this class was generated from the following files: