tesseract 4.1.1
Loading...
Searching...
No Matches
unicharset_extractor.cpp
Go to the documentation of this file.
1
2// File: unicharset_extractor.cpp
3// Description: Unicode character/ligature set extractor.
4// Author: Thomas Kielbus
5//
6// (C) Copyright 2006, Google Inc.
7// Licensed under the Apache License, Version 2.0 (the "License");
8// you may not use this file except in compliance with the License.
9// You may obtain a copy of the License at
10// http://www.apache.org/licenses/LICENSE-2.0
11// Unless required by applicable law or agreed to in writing, software
12// distributed under the License is distributed on an "AS IS" BASIS,
13// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14// See the License for the specific language governing permissions and
15// limitations under the License.
16//
18
19// Given a list of box files or text files on the command line, this program
20// normalizes the text according to command-line options and generates
21// a unicharset.
22
23#include <cstdlib>
24#include "boxread.h"
25#include "commandlineflags.h"
26#include "commontraining.h" // CheckSharedLibraryVersion
27#include "genericvector.h"
28#include "lang_model_helpers.h"
29#include "normstrngs.h"
30#include "strngs.h"
31#include "unicharset.h"
33
34static STRING_PARAM_FLAG(output_unicharset, "unicharset", "Output file path");
35static INT_PARAM_FLAG(norm_mode, 1,
36 "Normalization mode: 1=Combine graphemes, "
37 "2=Split graphemes, 3=Pure unicode");
38
39namespace tesseract {
40
41// Helper normalizes and segments the given strings according to norm_mode, and
42// adds the segmented parts to unicharset.
43static void AddStringsToUnicharset(const GenericVector<STRING>& strings,
44 int norm_mode, UNICHARSET* unicharset) {
45 for (int i = 0; i < strings.size(); ++i) {
46 std::vector<std::string> normalized;
48 static_cast<GraphemeNormMode>(norm_mode),
49 /*report_errors*/ true,
50 strings[i].string(), &normalized)) {
51 for (const std::string& normed : normalized) {
52
53 // normed is a UTF-8 encoded string
54 if (normed.empty() || IsUTF8Whitespace(normed.c_str())) continue;
55 unicharset->unichar_insert(normed.c_str());
56 }
57 } else {
58 tprintf("Normalization failed for string '%s'\n", strings[i].c_str());
59 }
60 }
61}
62
63static int Main(int argc, char** argv) {
64 UNICHARSET unicharset;
65 // Load input files
66 for (int arg = 1; arg < argc; ++arg) {
67 STRING file_data = tesseract::ReadFile(argv[arg], /*reader*/ nullptr);
68 if (file_data.length() == 0) continue;
70 if (ReadMemBoxes(-1, /*skip_blanks*/ true, &file_data[0],
71 /*continue_on_failure*/ false, /*boxes*/ nullptr,
72 &texts, /*box_texts*/ nullptr, /*pages*/ nullptr)) {
73 tprintf("Extracting unicharset from box file %s\n", argv[arg]);
74 } else {
75 tprintf("Extracting unicharset from plain text file %s\n", argv[arg]);
76 texts.truncate(0);
77 file_data.split('\n', &texts);
78 }
79 AddStringsToUnicharset(texts, FLAGS_norm_mode, &unicharset);
80 }
81 SetupBasicProperties(/*report_errors*/ true, /*decompose*/ false,
82 &unicharset);
83 // Write unicharset file.
84 if (unicharset.save_to_file(FLAGS_output_unicharset.c_str())) {
85 tprintf("Wrote unicharset file %s\n", FLAGS_output_unicharset.c_str());
86 } else {
87 tprintf("Cannot save unicharset file %s\n",
88 FLAGS_output_unicharset.c_str());
89 return EXIT_FAILURE;
90 }
91 return EXIT_SUCCESS;
92}
93
94} // namespace tesseract
95
96int main(int argc, char** argv) {
97 tesseract::CheckSharedLibraryVersion();
98 if (argc > 1) {
99 tesseract::ParseCommandLineFlags(argv[0], &argc, &argv, true);
100 }
101 if (argc < 2) {
102 tprintf(
103 "Usage: %s [--output_unicharset filename] [--norm_mode mode]"
104 " box_or_text_file [...]\n",
105 argv[0]);
106 tprintf("Where mode means:\n");
107 tprintf(" 1=combine graphemes (use for Latin and other simple scripts)\n");
108 tprintf(" 2=split graphemes (use for Indic/Khmer/Myanmar)\n");
109 tprintf(" 3=pure unicode (use for Arabic/Hebrew/Thai/Tibetan)\n");
110 tprintf("Reads box or plain text files to extract the unicharset.\n");
111 return EXIT_FAILURE;
112 }
113 return tesseract::Main(argc, argv);
114}
bool ReadMemBoxes(int target_page, bool skip_blanks, const char *box_data, bool continue_on_failure, GenericVector< TBOX > *boxes, GenericVector< STRING > *texts, GenericVector< STRING > *box_texts, GenericVector< int > *pages)
Definition: boxread.cpp:69
DLLSYM void tprintf(const char *format,...)
Definition: tprintf.cpp:35
#define INT_PARAM_FLAG(name, val, comment)
#define STRING_PARAM_FLAG(name, val, comment)
int main(int argc, char **argv)
void ParseCommandLineFlags(const char *usage, int *argc, char ***argv, const bool remove_flags)
GraphemeNormMode
Definition: validator.h:33
bool IsUTF8Whitespace(const char *text)
Definition: normstrngs.cpp:229
bool NormalizeCleanAndSegmentUTF8(UnicodeNormMode u_mode, OCRNorm ocr_normalize, GraphemeNormMode g_mode, bool report_errors, const char *str8, std::vector< std::string > *graphemes)
Definition: normstrngs.cpp:172
void SetupBasicProperties(bool report_errors, bool decompose, UNICHARSET *unicharset)
STRING ReadFile(const std::string &filename, FileReader reader)
int size() const
Definition: genericvector.h:72
void truncate(int size)
Definition: strngs.h:45
int32_t length() const
Definition: strngs.cpp:189
void split(char c, GenericVector< STRING > *splited)
Definition: strngs.cpp:282
void unichar_insert(const char *const unichar_repr, OldUncleanUnichars old_style)
Definition: unicharset.cpp:626
bool save_to_file(const char *const filename) const
Definition: unicharset.h:350