tesseract 4.1.1
Loading...
Searching...
No Matches
lang_model_helpers.h
Go to the documentation of this file.
1// Copyright 2017 Google Inc. All Rights Reserved.
2// Author: rays@google.com (Ray Smith)
3// Purpose: Collection of convenience functions to simplify creation of the
4// unicharset, recoder, and dawgs for an LSTM model.
5
6// Licensed under the Apache License, Version 2.0 (the "License");
7// you may not use this file except in compliance with the License.
8// You may obtain a copy of the License at
9// http://www.apache.org/licenses/LICENSE-2.0
10// Unless required by applicable law or agreed to in writing, software
11// distributed under the License is distributed on an "AS IS" BASIS,
12// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13// See the License for the specific language governing permissions and
14// limitations under the License.
15#ifndef TESSERACT_TRAINING_LANG_MODEL_HELPERS_H_
16#define TESSERACT_TRAINING_LANG_MODEL_HELPERS_H_
17
18#include <string>
19#include "genericvector.h"
20#include "serialis.h"
21#include "strngs.h"
22#include "tessdatamanager.h"
23#include "unicharset.h"
24
25namespace tesseract {
26
27// Helper makes a filename (<output_dir>/<lang>/<lang><suffix>) and writes data
28// to the file, using writer if not null, otherwise, a default writer.
29// Default writer will overwrite any existing file, but a supplied writer
30// can do its own thing. If lang is empty, returns true but does nothing.
31// NOTE that suffix should contain any required . for the filename.
32bool WriteFile(const std::string& output_dir, const std::string& lang,
33 const std::string& suffix, const GenericVector<char>& data,
34 FileWriter writer);
35// Helper reads a file with optional reader and returns a STRING.
36// On failure emits a warning message and returns and empty STRING.
37STRING ReadFile(const std::string& filename, FileReader reader);
38
39// Helper writes the unicharset to file and to the traineddata.
40bool WriteUnicharset(const UNICHARSET& unicharset, const std::string& output_dir,
41 const std::string& lang, FileWriter writer,
42 TessdataManager* traineddata);
43// Helper creates the recoder from the unicharset and writes it to the
44// traineddata, with a human-readable form to file at:
45// <output_dir>/<lang>/<lang>.charset_size=<num> for some num being the size
46// of the re-encoded character set. The charset_size file is written using
47// writer if not null, or using a default file writer otherwise, overwriting
48// any existing content.
49// If pass_through is true, then the recoder will be a no-op, passing the
50// unicharset codes through unchanged. Otherwise, the recoder will "compress"
51// the unicharset by encoding Hangul in Jamos, decomposing multi-unicode
52// symbols into sequences of unicodes, and encoding Han using the data in the
53// radical_table_data, which must be the content of the file:
54// langdata/radical-stroke.txt.
55bool WriteRecoder(const UNICHARSET& unicharset, bool pass_through,
56 const std::string& output_dir, const std::string& lang,
57 FileWriter writer, STRING* radical_table_data,
58 TessdataManager* traineddata);
59
60// The main function for combine_lang_model.cpp.
61// Returns EXIT_SUCCESS or EXIT_FAILURE for error.
62// unicharset: can be a hand-created file with incomplete fields. Its basic
63// and script properties will be set before it is used.
64// script_dir: should point to the langdata (github repo) directory.
65// version_str: arbitrary version label.
66// Output files will be written to <output_dir>/<lang>/<lang>.*
67// If pass_through_recoder is true, the unicharset will be used unchanged as
68// labels in the classifier, otherwise, the unicharset will be "compressed" to
69// make the recognition task simpler and faster.
70// The words/puncs/numbers lists may be all empty. If any are non-empty then
71// puncs must be non-empty.
72// lang_is_rtl indicates that the language is generally written from right
73// to left (eg Arabic/Hebrew).
74int CombineLangModel(const UNICHARSET& unicharset, const std::string& script_dir,
75 const std::string& version_str, const std::string& output_dir,
76 const std::string& lang, bool pass_through_recoder,
77 const GenericVector<STRING>& words,
78 const GenericVector<STRING>& puncs,
79 const GenericVector<STRING>& numbers, bool lang_is_rtl,
80 FileReader reader, FileWriter writer);
81
82} // namespace tesseract
83
84#endif // TESSERACT_TRAINING_LANG_MODEL_HELPERS_H_
bool(*)(const STRING &, GenericVector< char > *) FileReader
Definition: serialis.h:49
int CombineLangModel(const UNICHARSET &unicharset, const std::string &script_dir, const std::string &version_str, const std::string &output_dir, const std::string &lang, bool pass_through_recoder, const GenericVector< STRING > &words, const GenericVector< STRING > &puncs, const GenericVector< STRING > &numbers, bool lang_is_rtl, FileReader reader, FileWriter writer)
bool WriteRecoder(const UNICHARSET &unicharset, bool pass_through, const std::string &output_dir, const std::string &lang, FileWriter writer, STRING *radical_table_data, TessdataManager *traineddata)
bool WriteFile(const std::string &output_dir, const std::string &lang, const std::string &suffix, const GenericVector< char > &data, FileWriter writer)
bool(*)(const GenericVector< char > &, const STRING &) FileWriter
Definition: serialis.h:52
bool WriteUnicharset(const UNICHARSET &unicharset, const std::string &output_dir, const std::string &lang, FileWriter writer, TessdataManager *traineddata)
STRING ReadFile(const std::string &filename, FileReader reader)
Definition: strngs.h:45