correction, ajout gitignore

This commit is contained in:
Thibaud
2019-07-06 13:25:00 +02:00
commit 28d1936dee
182 changed files with 28900 additions and 0 deletions
@@ -0,0 +1,17 @@
#!/bin/sh
# -*- coding: utf-8 -*-
# sh compileWASM.sh hyphenEngine
SRCFILE="$1.asm.js"
WASTNAME="$1.wast"
WASMNAME="$1.wasm"
echo 'running asm2wasm...'
./third-party/binaryen/bin/asm2wasm $SRCFILE -O4 -m 2097152 -mm 16777216 > $WASTNAME
echo 'optimize > WASM...'
./third-party/binaryen/bin/wasm-opt $WASTNAME -O4 -o $WASMNAME
wc -c $WASMNAME
rm $WASTNAME
@@ -0,0 +1,109 @@
#!/bin/sh
# -*- coding: utf-8 -*-
# license.txt characters.txt patterns.txt [exceptions.txt]
#node tex2hpb ../texPatterns/hyph-en-us.lic.txt ../texPatterns/hyph-en-us.chr.txt ../texPatterns/hyph-en-us.pat.txt ../texPatterns/hyph-en-us.hyp.txt en
#node tex2hpb ../texPatterns/hyph-de-1996.lic.txt ../texPatterns/hyph-de-1996.chr.txt ../texPatterns/hyph-de-1996.pat.txt ../texPatterns/hyph-de-1996.hyp.txt de
texName[1]='de-1996'
hpbName[1]='de'
texName[2]='en-us'
hpbName[2]='en-us'
texName[3]='hy'
hpbName[3]='hy'
texName[4]='hu'
hpbName[4]='hu'
texName[5]='cs'
hpbName[5]='cs'
texName[6]='da'
hpbName[6]='da'
texName[7]='fr'
hpbName[7]='fr'
texName[8]='nl'
hpbName[8]='nl'
texName[9]='es'
hpbName[9]='es'
texName[10]='sv'
hpbName[10]='sv'
texName[11]='it'
hpbName[11]='it'
texName[12]='fi'
hpbName[12]='fi'
texName[13]='pl'
hpbName[13]='pl'
texName[14]='pt'
hpbName[14]='pt'
texName[15]='tr'
hpbName[15]='tr'
texName[16]='lt'
hpbName[16]='lt'
texName[17]='sl'
hpbName[17]='sl'
texName[18]='la-x-classic'
hpbName[18]='la'
texName[19]='nb'
hpbName[19]='nb-no'
texName[20]='lv'
hpbName[20]='lv'
texName[21]='ca'
hpbName[21]='ca'
texName[22]='sk'
hpbName[22]='sk'
texName[23]='eo'
hpbName[23]='eo'
texName[24]='sh-latn'
hpbName[24]='sh-latn'
texName[25]='ro'
hpbName[25]='ro'
texName[26]='et'
hpbName[26]='et'
texName[27]='ga'
hpbName[27]='ga'
texName[28]='be'
hpbName[28]='be'
texName[29]='ru'
hpbName[29]='ru'
texName[30]='sr-cyrl'
hpbName[30]='sr-cyrl'
texName[31]='uk'
hpbName[31]='uk'
pt='../texPatterns/'
px='hyph-'
for i in {1..1}; do {
node tex2hpb.js ${pt}${px}${texName[${i}]}.lic.txt ${pt}${px}${texName[${i}]}.chr.txt ${pt}${px}${texName[${i}]}.pat.txt ${pt}${px}${texName[${i}]}.hyp.txt ${hpbName[${i}]}
}
done
@@ -0,0 +1,17 @@
#!/bin/sh
# -*- coding: utf-8 -*-
path_in='../texPatterns/'
path_out='./convertedPatterns/'
prefix='hyph-'
rm -R $path_out
mkdir $path_out
for filename in ${path_in}*.chr.txt; do {
#filename="de.chr.txt"
base=$(basename "$filename" .chr.txt)
langcode=${base#$prefix}
node tex2hpb.js $path_in$base.lic.txt $path_in$base.chr.txt $path_in$base.pat.txt $path_in$base.hyp.txt $path_out$langcode
}
done
@@ -0,0 +1,12 @@
#!/bin/sh
# -*- coding: utf-8 -*-
mkdir -p min
terser Hyphenopoly_Loader.js -o min/Hyphenopoly_Loader.js --comments -c -m --warn
terser Hyphenopoly.js -o min/Hyphenopoly.js --comments -c -m --warn
terser hyphenEngine.asm.js -o min/hyphenEngine.asm.js --comments -c -m --warn --verbose
cp hyphenEngine.wasm min/hyphenEngine.wasm
cp -R patterns/ min/patterns
cp -R testsuite/ min/testsuite
@@ -0,0 +1,711 @@
/* eslint-disable complexity */
/* eslint-disable security/detect-non-literal-fs-filename */
/* eslint-disable no-console, no-sync */
/* eslint-env node */
/*
* Convert TeX-patterns to hpb: tex2hpb – Version 1.0
*
* This tool converts hyphenation patterns from TeX
* (https://ctan.org/tex-archive/language/hyph-utf8)
* to the binary format used in Hyphenopoly.js
* (https://github.com/mnater/Hyphenopoly)
*
* Usage:
* # node tex2hpb.js lic.txt chars.txt pat.txt [exc.txt | null] outname
*
* This creates a new file called input.hpb in pwd
*
* All input files must be utf-8 encoded files.
*
* license.txt
* Some licenses require to be included in every distribution of the work.
* If not empty the license.txt file must contain the license of the patterns.
* newline (0x0a).
*
* characters.txt
* When creating the pattern trie, characters of the languages alphabet are
* mapped to internal small integers. The program thus need to know which
* characters are used in the language.
* in the patterns followed by other representations of the same
* character, if any (e.g. its uppercase form):
* Example:
* aA
* äÄ
* sSſ
* ß
* Character groups are not supported, yet (e.g. german ßSS is invalid).
*
* patterns.txt must be a utf-8 encoded file of two parts:
* rightmin. Typically: 22
* For many patterns you'll find these numbers in the license text or on
* http://www.hyphenation.org/#languages
* If this information is missing, 22 is assumed.
* 2: TeX hyphenation patterns where each pattern is separated by a
* newline (0x0a).
*
* The optional exceptions.txt contains exceptional hyphenations that are
* hyphenation points.
* Example:
* ta-ble
* project
* Internally exceptions are converted to special patterns. The examples above
* will become:
* _10t10a11b10l10e10_
* _10p10r10o10j10e10c10t10_
* So "tables" will not by hyphenated by this pattern.
*
* If there's no exceptions file, use "null" as a placeholder.
*
* outname
* outname (typically the language code) is the filename where patterns will
* be stored. The .hpb ending is added automatically.
*/
/*
* Binary format: .hpb (hyphenopoly patterns binary)
* The hyphenopoly patterns binary stores hyphenation patterns
* for one language in a tight format shaped for fast loading
* and execution by Hyphenopoly.js
* Unlike in other hyphenation binaries (like e.g. .hyb files) the
* patterns are not stored as a trie. The trie is (even when packed)
* slightly larger then the raw patterns.
* The trie has to be built by the consumer of the patterns.
* The binary file consists of four parts: HEADER, LICENSE, TRANSLATE and
* PATTERNS. All data is little endian.
*
* HEADER
* Uint32Array of length 8
* [0]: magic number 0x01627068 (\hpb1, 1 is the version)
* [1]: TRANSLATE offset (to skip LICENSE)
* [2]: PATTERNS offset (skip LICENSE + TRANSLATE)
* [3]: patternlength (bytes)
* [4]: leftmin
* [5]: rightmin
* [6]: Trie Array Size (needed to preallocate memory)
* [7]: Values Size (needed to preallocate memory)
*
* LICENSE
* UTF-8 encoded license text, padded to 4 bytes
*
* TRANSLATE
* When creating a trie, the characters of the alphabet have to be
* mapped to uInts from 1 to the lenght of the alphabet.
* (Since wasm memory is initialized with zeros, 0 is reserved here).
* Also upon creating the trie, the number of characters has to be
* known.
* The mapping from utf16 characters to these internal uInts is stored
* in the TRANSLATE Table which is an Uint16Array of variable length.
* The characters of the patterns are stored on locations with odd
* adresses (starting at 1) while there other cases are stored on even
* adresses. Adress 0 denotes the length of the alphabet.
* By using 16bits the characters are restricted to the BMP.
* Characters in the TRANSLATE table are sorted by their Unicode code
* point in increasing order (except substitutions, see below)
* Characters that don't have a upperCase are followed by 0.
* Characters that are a SUBSTITUTION for an other character in the
* alphabet are stored at the end of the list, preceded by their
* substituted character.
* The underline character (_) is reserved to mark the beginning and
* the end of the word (TeX patterns use the dot (.) for this purpose)
* and is always the first character in the TRANSLATE.
* Example:
* For the characters '_rst' and 'ſ' (LATIN SMALL LETTER LONG S) the
* TRANSLATE is '4_\0rRsStTſs' (substitutes dont count for the length)
* The TRANSLATE Table has several impacts on the behaviour of the
* hyphenation algorithm:
* - words containing a character that is not in the translate are
* not hyphenated
* - words don't need to be lowerCase'd before hyphenation
* - substitute characters are not needed to be substituted before
* hyphenation
*
* PATTERNS
* A Uint8Array of variable length.
* Patterns from the input file are mapped by the TRANSLATE Table.
* TeX patterns contain (a) numbers from 1 to 9 to indicate hyphenation points
* and (b) the characters of the alphabet.
* The numbers are directly stored with their value. The word boundary
* marker (_) has always the value 12 (0xC). The other characters are
* stored with values >= 13 (0xD). Thus the maximum alphabet length is
* 255 - 12 = 243 which should be enough for most use cases.
* Example 3:
* Given the TRANSLATE '_\0aAbBcC'
* (0x0400 0x5f00 0x0000 0x6100 0x4100 0x6200 0x4200 0x6300 0x4300)
* the characters for the pattern '1ba' are stored as
* '0x01 0x0e 0x0d' = '01 14 13'
* Individual patterns are not separated. Instead patterns of the same
* length and the same prefix are grouped. 0 (zero) marks the beginning of a new
* group with same length, 255 marks a new prefix group.
* Example 4:
* The patterns '1ba 1be 1abd 1abf 1bba' are grouped as
* follows
* '0 3 255 1 b a e 0 4 255 1 a b d b f 255 1 b b a'
*/
"use strict";
const fs = require("fs");
const VERSION = 2;
const licenseFileName = process.argv[2];
const charactersFileName = process.argv[3];
const patternsFileName = process.argv[4];
const exceptionsFileName = process.argv[5];
const saveFileName = process.argv[6];
let leftmin = 2;
let rightmin = 2;
const logger = (function createLogger() {
let msgNr = 1;
/**
* Logs text to console.
* @param {string} text - The text to be logged
* @param {boolean} indent - If true, indents log by 4 spaces
*/
function log(text, indent) {
if (indent) {
console.log(` \x1b[34m${text}\x1b[0m`);
} else {
console.log(`\x1b[33m(${msgNr.toString(16)})\x1b[0m: \x1b[34m${text}\x1b[0m`);
msgNr += 1;
}
}
return {
"log": log
};
}());
/**
* Create the magic number (the first 32Bits of the .hpb-file).
* The MagicNumber contains the utf8 code for the letters "hpb" and
* the version as digit (e.g. hpb1 -> 104,112,98,1 -> 68,70,62,01 in hex)
*/
function createMagicNumber() {
const mnstring = "hpb";
const mnarray = [];
let i = 0;
while (i < mnstring.length) {
mnarray.push(mnstring.codePointAt(i));
i += 1;
}
mnarray.push(VERSION);
const mnui8 = Uint8Array.from(mnarray);
const mnui32 = new Uint32Array(mnui8.buffer);
return mnui32[0];
}
/**
* Create the header of the hpb file. The header contains 8 32bit values:
* [0]: magic number 0x01627068 (\hpb1, 1 is the version)
* [1]: TRANSLATE offset (to skip LICENSE)
* [2]: PATTERNS offset (skip LICENSE + TRANSLATE)
* [3]: patternlength (bytes)
* [4]: leftmin
* [5]: rightmin
* [6]: Trie Array Size (needed to preallocate memory)
* [7]: Values Size (needed to preallocate memory)
* @param {number} licenseLength - Length of the licence
* @param {number} translateLength - Length of the translate
* @param {number} trieLength - Length of the pattern trie
* @param {number} valueLength - Length of the value list
* @param {number} patternLength - Length of the raw patterns
*/
function createHeader(
licenseLength,
translateLength,
trieLength,
valueLength,
patternLength
) {
const headerui32 = new Uint32Array(8);
const translateByteOffset = headerui32.byteLength + licenseLength;
const patternByteOffset = translateByteOffset + translateLength;
headerui32[0] = createMagicNumber();
headerui32[1] = translateByteOffset;
headerui32[2] = patternByteOffset;
headerui32[3] = patternLength;
headerui32[4] = leftmin;
headerui32[5] = rightmin;
headerui32[6] = trieLength;
headerui32[7] = valueLength;
return headerui32;
}
/**
* Read .lic.txt File
*/
function getLicenseFileBuffer() {
logger.log(`read license file: ${licenseFileName} (${fs.statSync(licenseFileName).size} Bytes)`);
const licensefile = fs.readFileSync("./" + licenseFileName);
return licensefile;
}
/**
* Read .chr.txt File
*/
function getCharactersFile() {
logger.log(`read characters file: ${charactersFileName} (${fs.statSync(charactersFileName).size} Bytes)`);
let charactersfile = fs.readFileSync("./" + charactersFileName, "utf8");
charactersfile = charactersfile.trim();
return charactersfile;
}
/**
* Read .pat.txt File
*/
function getPatternsFile() {
logger.log(`read patterns file: ${patternsFileName} (${fs.statSync(patternsFileName).size} Bytes)`);
let patternsfile = fs.readFileSync("./" + patternsFileName, "utf8");
patternsfile = patternsfile.trim();
// eslint-disable-next-line prefer-named-capture-group
patternsfile = patternsfile.replace(/(\d{2})\n/, function repl(ignore, p1) {
const digits = p1.split("");
leftmin = parseInt(digits[0], 10);
rightmin = parseInt(digits[1], 10);
logger.log(`set leftmin: ${leftmin}, rightmin: ${rightmin}`);
return "";
});
patternsfile = patternsfile.replace(/\./g, "_");
patternsfile = patternsfile.replace(/\n/g, " ");
return patternsfile;
}
/**
* Read .hyp.txt File
*/
function getExceptionsFile() {
if (exceptionsFileName && exceptionsFileName !== "null") {
logger.log(`read exceptions file: ${exceptionsFileName} (${fs.statSync(exceptionsFileName).size} Bytes)`);
const exceptionsfile = fs.readFileSync("./" + exceptionsFileName, "utf8");
return exceptionsfile;
}
logger.log("no exceptions");
return null;
}
/**
* Create translateTable
* @param {string} characters - List of chars
*/
function createTranslate(characters) {
const lines = characters.split("\n");
// At index 0: alphabet length
const translateTable = [0];
const substitutions = [];
const logalpha = [];
const logsubst = [];
const wordDelim = "_";
translateTable.push(wordDelim.charCodeAt(0));
translateTable.push(0);
translateTable[0] += 1;
lines.forEach(function eachLine(value) {
translateTable[0] += 1;
if (value.length === 2) {
translateTable.push(value.charCodeAt(0));
translateTable.push(value.charCodeAt(1));
logalpha.push(value.charAt(0));
logalpha.push(value.charAt(1));
} else if (value.length === 1) {
translateTable.push(value.charCodeAt(0));
translateTable.push(0);
logalpha.push(value.charAt(0));
logalpha.push("⎵");
} else if (value.length > 2) {
// Substitutions
translateTable.push(value.charCodeAt(0));
translateTable.push(value.charCodeAt(1));
logalpha.push(value.charAt(0));
logalpha.push(value.charAt(1));
let i = 2;
while (i < value.length) {
substitutions.push(value.charCodeAt(i));
substitutions.push(value.charCodeAt(0));
logsubst.push(value.charAt(i));
logsubst.push(value.charAt(0));
i += 1;
}
}
});
logger.log(`collected alphabet of length ${logalpha.length} (${logalpha.length / 2}):`);
logger.log(`${logalpha.join("")}`, true);
logger.log(`collected substitutions: ${logsubst.join("")}`);
const ui16 = Uint16Array.from(translateTable.concat(substitutions));
return ui16;
}
/**
* Create a lookup table from the translate table
* @param {Uint16Array} translate - Translate table
*/
function createTranslateLookUpTable(translate) {
// eslint-disable-next-line no-bitwise
const lookuptable = new Uint16Array(2 << 15);
let i = 1;
let k = 12;
while (i < translate.length) {
if (lookuptable[translate[i + 1]] === 0) {
// eslint-disable-next-line security/detect-object-injection
lookuptable[translate[i]] = k;
if (translate[i + 1] !== 0) {
lookuptable[translate[i + 1]] = k;
}
k += 1;
} else {
// Substitute
// eslint-disable-next-line security/detect-object-injection
lookuptable[translate[i]] = lookuptable[translate[i + 1]];
}
i += 2;
}
logger.log(`mapped chars of alphabet to internal numbers in range [12, ${k})`);
return lookuptable;
}
/**
* Create special patterns from exceptions
* @param {string} exceptions - List of exceptions
*/
function createExceptionPatterns(exceptions) {
const lines = exceptions.split("\n");
const ret = [];
lines.forEach(function eachLine(value, index) {
if (value !== "") {
// eslint-disable-next-line security/detect-object-injection
ret[index] = "_" + value.split("").map(function mapper(c) {
if (c === "-") {
return "11";
}
return "10" + c;
}).
join("").
replace(/1110/gi, "11") + "10_";
}
});
return ret.join(" ");
}
/**
* Convert TeX-patterns to hpb-patterns
* @param {Uint16Array} translate - The translate table
* @param {string} patterns - The TeX-patterns
* @param {string} exceptionsfile - The content of the .hyp.txt
*/
function createPatterns(translate, patterns, exceptionsfile) {
/* eslint-disable security/detect-object-injection */
const lookuptable = createTranslateLookUpTable(translate);
const allExceptions = createExceptionPatterns(exceptionsfile);
if (allExceptions !== "") {
patterns = patterns + " " + allExceptions;
}
const allPatterns = patterns.split(" ");
const exceptions = [];
// eslint-disable-next-line complexity
const translatedPatterns = allPatterns.map(function mapper(pat) {
let i = 0;
let cP1 = 0;
let cP2 = 0;
const ret = [];
let isException = false;
while (i < pat.length) {
cP1 = pat.codePointAt(i);
if (cP1 > 57 || cP1 < 49) {
ret.push(lookuptable[cP1]);
} else {
cP2 = pat.codePointAt(i + 1);
if (cP2 && (cP2 < 57 && cP2 > 47)) {
isException = true;
ret.push((10 * (cP1 - 48)) + (cP2 - 48));
i += 1;
} else {
ret.push(cP1 - 48);
}
}
i += 1;
}
if (isException) {
exceptions.push(pat);
}
return ret;
});
logger.log(`found ${exceptions.length} pattern exceptions`);
const groupedPatterns = {};
let patternLength = 0;
let i = 0;
let longestP = 0;
let shortestP = Number.MAX_SAFE_INTEGER;
while (i < translatedPatterns.length) {
patternLength = translatedPatterns[i].length;
// eslint-disable-next-line no-prototype-builtins
if (groupedPatterns.hasOwnProperty(patternLength)) {
groupedPatterns[patternLength].push(translatedPatterns[i]);
} else {
groupedPatterns[patternLength] = [translatedPatterns[i]];
}
i += 1;
}
const outPatterns = [];
Object.keys(groupedPatterns).forEach(function eachPatternLength(k) {
groupedPatterns[k].sort();
let l = 0;
let j = 0;
const currentLength = parseInt(k, 10);
let currentFirst = 0;
let currentSecond = 0;
longestP = Math.max(longestP, parseInt(k, 10));
shortestP = Math.min(shortestP, parseInt(k, 10));
outPatterns.push(0);
outPatterns.push(currentLength);
while (l < groupedPatterns[k].length) {
j = 2;
if (currentFirst !== groupedPatterns[k][l][0] ||
currentSecond !== groupedPatterns[k][l][1]
) {
currentFirst = groupedPatterns[k][l][0];
currentSecond = groupedPatterns[k][l][1];
outPatterns.push(255);
outPatterns.push(currentFirst);
outPatterns.push(currentSecond);
}
while (j < groupedPatterns[k][l].length) {
outPatterns.push(groupedPatterns[k][l][j]);
j += 1;
}
l += 1;
}
});
/*
* Object.keys(groupedPatterns).forEach(function eachPatternLength(k) {
* groupedPatterns[k].sort();
* outPatterns.push(58);
* outPatterns.push(parseInt(k, 10));
* outPatterns.push(58);
* let l = 0;
* let j = 0;
* longestP = Math.max(longestP, parseInt(k, 10));
* shortestP = Math.min(shortestP, parseInt(k, 10));
* while (l < groupedPatterns[k].length) {
* j = 0;
* while (j < groupedPatterns[k][l].length) {
* outPatterns.push(groupedPatterns[k][l][j]);
* j += 1;
* }
* l += 1;
* }
* });
*/
logger.log(`grouped and sorted patterns: shortest: ${shortestP}, longest: ${longestP}`);
return Uint8Array.from(outPatterns);
// eslint-enable security/detect-object-injection
}
/**
* Create the patterns trie
* @param {Uint8Array} patterns - hpb-patterns
* @param {number} trieRowLength - number of characters
*/
function TrieCreator(patterns, trieRowLength) {
let i = 0;
let patternlength = 0;
let count = 0;
let rowStart = 0;
let nextRowStart = 0;
let prevWasDigit = false;
let trieNextEmptyRow = 0;
let rowOffset = 0;
let valueStoreNextStartIndex = 0;
let valueStoreCurrentIdx = 0;
let valueStorePrevIdx = 0;
const patternTrie = [];
const valueStore = [];
/**
* Add 0 to value store
*/
function add0ToValueStore() {
valueStore[valueStoreCurrentIdx] = 0;
valueStoreCurrentIdx += 1;
}
/**
* Add a value to value store
* @param {number} p - Value to be added
*/
function addToValueStore(p) {
valueStore[valueStoreCurrentIdx] = p;
valueStorePrevIdx = valueStoreCurrentIdx;
valueStoreCurrentIdx += 1;
}
/**
* Get link to value store index
*/
function getLinkToValueStore() {
const start = valueStoreNextStartIndex;
// Mark end of pattern:
valueStore[valueStorePrevIdx + 1] = 255;
valueStoreNextStartIndex = valueStorePrevIdx + 2;
valueStoreCurrentIdx = valueStoreNextStartIndex;
return start;
}
/**
* Add a new Row filled with 0
* @param {number} startIndex - From this index
*/
function makeRow(startIndex) {
let s = startIndex;
while (s < (trieRowLength + startIndex)) {
patternTrie[s] = 0;
s += 1;
}
return startIndex;
}
/**
* Add a codePoint to the Trie
* @param {number} codePoint - Translated code Point
*/
function addToTrie(codePoint) {
if (codePoint > 11) {
// It's a char
if (!prevWasDigit) {
add0ToValueStore();
}
prevWasDigit = false;
if (nextRowStart === -1) {
// Start a new row
trieNextEmptyRow = trieNextEmptyRow + trieRowLength + 1;
nextRowStart = trieNextEmptyRow;
patternTrie[rowStart + rowOffset] = makeRow(nextRowStart);
}
rowOffset = (codePoint - 12) * 2;
rowStart = nextRowStart;
nextRowStart = patternTrie[rowStart + rowOffset];
if (nextRowStart === 0) {
patternTrie[rowStart + rowOffset] = -1;
nextRowStart = -1;
}
} else {
// It's a digit
addToValueStore(codePoint);
prevWasDigit = true;
}
}
/**
* Add last codePoint of a pattern to the Trie
* @param {number} codePoint - Translated code Point
*/
function terminateTrie() {
patternTrie[rowStart + rowOffset + 1] = getLinkToValueStore();
}
makeRow(0);
let first = 0;
let second = 0;
while (i < patterns.length) {
if (patterns[i] === 0) {
patternlength = patterns[i + 1];
i += 2;
} else {
if (patterns[i] === 255) {
first = patterns[i + 1];
second = patterns[i + 2];
i += 3;
}
while (count < patternlength) {
switch (count) {
case 0:
addToTrie(first);
count += 1;
break;
case 1:
addToTrie(second);
count += 1;
break;
default:
addToTrie(patterns[i]);
count += 1;
i += 1;
}
}
terminateTrie();
// Reset indizes
count = 0;
rowStart = 0;
nextRowStart = 0;
prevWasDigit = 0;
}
}
logger.log("created Trie.");
logger.log(`trieLength: ${patternTrie.length}`, true);
logger.log(`valueStoreLength: ${valueStore.length}`, true);
return {
"trieLength": patternTrie.length,
"valueStoreLength": valueStore.length
};
}
/**
* The one function to rule them all...
*/
function main() {
const start = process.hrtime();
console.log(`\x1b[35mRunning tex2hbp.js (v${VERSION}) on node.js (${process.version})\x1b[0m`);
const licenseBuf = getLicenseFileBuffer();
const paddedLicenseBuf = licenseBuf.byteLength + 4 -
(licenseBuf.byteLength % 4);
const charactersfile = getCharactersFile();
const patternsfile = getPatternsFile();
const exceptionsfile = getExceptionsFile();
const translate = createTranslate(charactersfile);
const patterns = createPatterns(translate, patternsfile, exceptionsfile);
const dummyTrie = new TrieCreator(patterns, translate[0] * 2);
const header = createHeader(
paddedLicenseBuf,
translate.byteLength,
dummyTrie.trieLength,
dummyTrie.valueStoreLength,
patterns.byteLength
);
let fileBufferSize = header.byteLength + paddedLicenseBuf +
translate.byteLength + patterns.byteLength;
const pad = 4 - (fileBufferSize % 4);
fileBufferSize += pad;
const fileBuffer = new ArrayBuffer(fileBufferSize);
const fileBufferui32 = new Uint32Array(fileBuffer);
const fileBufferui16 = new Uint16Array(fileBuffer);
const fileBufferui8 = new Uint8Array(fileBuffer);
fileBufferui32.set(header, 0);
fileBufferui8.set(licenseBuf, header.byteLength);
// eslint-disable-next-line no-bitwise
fileBufferui16.set(translate, (header.byteLength + paddedLicenseBuf) >> 1);
fileBufferui8.set(
patterns,
header.byteLength + paddedLicenseBuf + translate.byteLength
);
fs.writeFile(saveFileName + ".hpb", fileBufferui8, function cb(err) {
if (err) {
console.log(err);
} else {
logger.log(`Finish: file saved to '${saveFileName + ".hpb"}' (${fileBufferSize} Bytes)`);
console.log(`\x1b[35mtook ${process.hrtime(start)} seconds\x1b[0m`);
}
});
}
main();