tesseract 4.1.1
Loading...
Searching...
No Matches
combine_lang_model.cpp
Go to the documentation of this file.
1// Copyright 2017 Google Inc. All Rights Reserved.
2// Author: rays@google.com (Ray Smith)
3// Purpose: Program to generate a traineddata file that can be used to train an
4// LSTM-based neural network model from a unicharset and an optional
5// set of wordlists. Eliminates the need to run
6// set_unicharset_properties, wordlist2dawg, some non-existent binary
7// to generate the recoder, and finally combine_tessdata.
8
9// Licensed under the Apache License, Version 2.0 (the "License");
10// you may not use this file except in compliance with the License.
11// You may obtain a copy of the License at
12// http://www.apache.org/licenses/LICENSE-2.0
13// Unless required by applicable law or agreed to in writing, software
14// distributed under the License is distributed on an "AS IS" BASIS,
15// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16// See the License for the specific language governing permissions and
17// limitations under the License.
18
19#include "commandlineflags.h"
20#include "commontraining.h" // CheckSharedLibraryVersion
21#include "lang_model_helpers.h"
22#include "tprintf.h"
24
25static STRING_PARAM_FLAG(input_unicharset, "",
26 "Filename with unicharset to complete and use in encoding");
27static STRING_PARAM_FLAG(script_dir, "",
28 "Directory name for input script unicharsets");
29static STRING_PARAM_FLAG(words, "",
30 "File listing words to use for the system dictionary");
31static STRING_PARAM_FLAG(puncs, "", "File listing punctuation patterns");
32static STRING_PARAM_FLAG(numbers, "", "File listing number patterns");
33static STRING_PARAM_FLAG(output_dir, "", "Root directory for output files");
34static STRING_PARAM_FLAG(version_str, "", "Version string to add to traineddata file");
35static STRING_PARAM_FLAG(lang, "", "Name of language being processed");
36static BOOL_PARAM_FLAG(lang_is_rtl, false,
37 "True if lang being processed is written right-to-left");
38static BOOL_PARAM_FLAG(pass_through_recoder, false,
39 "If true, the recoder is a simple pass-through of the "
40 "unicharset. Otherwise, potentially a compression of it");
41
42int main(int argc, char** argv) {
43 // Sets properties on the input unicharset file, and writes:
44 // rootdir/lang/lang.charset_size=ddd.txt
45 // rootdir/lang/lang.traineddata
46 // rootdir/lang/lang.unicharset
47 // If the 3 word lists are provided, the dawgs are also added
48 // to the traineddata file.
49 // The output unicharset and charset_size files are just for
50 // human readability.
51 tesseract::CheckSharedLibraryVersion();
52 tesseract::ParseCommandLineFlags(argv[0], &argc, &argv, true);
53
54 GenericVector<STRING> words, puncs, numbers;
55 // If these reads fail, we get a warning message and an empty list of words.
56 tesseract::ReadFile(FLAGS_words.c_str(), nullptr).split('\n', &words);
57 tesseract::ReadFile(FLAGS_puncs.c_str(), nullptr).split('\n', &puncs);
58 tesseract::ReadFile(FLAGS_numbers.c_str(), nullptr).split('\n', &numbers);
59 // Load the input unicharset
60 UNICHARSET unicharset;
61 if (!unicharset.load_from_file(FLAGS_input_unicharset.c_str(), false)) {
62 tprintf("Failed to load unicharset from %s\n",
63 FLAGS_input_unicharset.c_str());
64 return 1;
65 }
66 tprintf("Loaded unicharset of size %d from file %s\n", unicharset.size(),
67 FLAGS_input_unicharset.c_str());
68
69 // Set unichar properties
70 tprintf("Setting unichar properties\n");
71 tesseract::SetupBasicProperties(/*report_errors*/ true,
72 /*decompose (NFD)*/ false, &unicharset);
73 tprintf("Setting script properties\n");
74 tesseract::SetScriptProperties(FLAGS_script_dir.c_str(), &unicharset);
75 // Combine everything into a traineddata file.
77 unicharset, FLAGS_script_dir.c_str(), FLAGS_version_str.c_str(),
78 FLAGS_output_dir.c_str(), FLAGS_lang.c_str(), FLAGS_pass_through_recoder,
79 words, puncs, numbers, FLAGS_lang_is_rtl, /*reader*/ nullptr,
80 /*writer*/ nullptr);
81}
DLLSYM void tprintf(const char *format,...)
Definition: tprintf.cpp:35
int main(int argc, char **argv)
#define BOOL_PARAM_FLAG(name, val, comment)
#define STRING_PARAM_FLAG(name, val, comment)
void ParseCommandLineFlags(const char *usage, int *argc, char ***argv, const bool remove_flags)
int CombineLangModel(const UNICHARSET &unicharset, const std::string &script_dir, const std::string &version_str, const std::string &output_dir, const std::string &lang, bool pass_through_recoder, const GenericVector< STRING > &words, const GenericVector< STRING > &puncs, const GenericVector< STRING > &numbers, bool lang_is_rtl, FileReader reader, FileWriter writer)
void SetScriptProperties(const std::string &script_dir, UNICHARSET *unicharset)
void SetupBasicProperties(bool report_errors, bool decompose, UNICHARSET *unicharset)
STRING ReadFile(const std::string &filename, FileReader reader)
void split(char c, GenericVector< STRING > *splited)
Definition: strngs.cpp:282
int size() const
Definition: unicharset.h:341
bool load_from_file(const char *const filename, bool skip_fragments)
Definition: unicharset.h:388