source: trip-planner-front/node_modules/chardet/encoding/sbcs.js@ 76712b2

Last change on this file since 76712b2 was 6a3a178, checked in by Ema <ema_spirova@…>, 3 years ago

initial commit
  • Property mode set to 100644
File size: 40.1 KB
Line 
1var util = require('util'),
2 Match = require ('../match');
3
4/**
5 * This class recognizes single-byte encodings. Because the encoding scheme is so
6 * simple, language statistics are used to do the matching.
7 */
8
9function NGramParser(theNgramList, theByteMap) {
10 var N_GRAM_MASK = 0xFFFFFF;
11
12 this.byteIndex = 0;
13 this.ngram = 0;
14
15 this.ngramList = theNgramList;
16 this.byteMap = theByteMap;
17
18 this.ngramCount = 0;
19 this.hitCount = 0;
20
21 this.spaceChar;
22
23 /*
24 * Binary search for value in table, which must have exactly 64 entries.
25 */
26 this.search = function(table, value) {
27 var index = 0;
28
29 if (table[index + 32] <= value) index += 32;
30 if (table[index + 16] <= value) index += 16;
31 if (table[index + 8] <= value) index += 8;
32 if (table[index + 4] <= value) index += 4;
33 if (table[index + 2] <= value) index += 2;
34 if (table[index + 1] <= value) index += 1;
35 if (table[index] > value) index -= 1;
36
37 if (index < 0 || table[index] != value)
38 return -1;
39
40 return index;
41 };
42
43 this.lookup = function(thisNgram) {
44 this.ngramCount += 1;
45 if (this.search(this.ngramList, thisNgram) >= 0) {
46 this.hitCount += 1;
47 }
48 };
49
50 this.addByte = function(b) {
51 this.ngram = ((this.ngram << 8) + (b & 0xFF)) & N_GRAM_MASK;
52 this.lookup(this.ngram);
53 }
54
55 this.nextByte = function(det) {
56 if (this.byteIndex >= det.fInputLen)
57 return -1;
58
59 return det.fInputBytes[this.byteIndex++] & 0xFF;
60 }
61
62 this.parse = function(det, spaceCh) {
63 var b, ignoreSpace = false;
64 this.spaceChar = spaceCh;
65
66 while ((b = this.nextByte(det)) >= 0) {
67 var mb = this.byteMap[b];
68
69 // TODO: 0x20 might not be a space in all character sets...
70 if (mb != 0) {
71 if (!(mb == this.spaceChar && ignoreSpace)) {
72 this.addByte(mb);
73 }
74
75 ignoreSpace = (mb == this.spaceChar);
76 }
77 }
78
79 // TODO: Is this OK? The buffer could have ended in the middle of a word...
80 this.addByte(this.spaceChar);
81
82 var rawPercent = this.hitCount / this.ngramCount;
83
84 // TODO - This is a bit of a hack to take care of a case
85 // were we were getting a confidence of 135...
86 if (rawPercent > 0.33)
87 return 98;
88
89 return Math.floor(rawPercent * 300.0);
90 };
91};
92
93function NGramsPlusLang(la, ng) {
94 this.fLang = la;
95 this.fNGrams = ng;
96};
97
98function sbcs() {};
99sbcs.prototype.spaceChar = 0x20;
100sbcs.prototype.ngrams = function() {};
101sbcs.prototype.byteMap = function() {};
102sbcs.prototype.match = function(det) {
103
104 var ngrams = this.ngrams();
105 var multiple = (Array.isArray(ngrams) && ngrams[0] instanceof NGramsPlusLang);
106
107 if (!multiple) {
108 var parser = new NGramParser(ngrams, this.byteMap());
109 var confidence = parser.parse(det, this.spaceChar);
110 return confidence <= 0 ? null : new Match(det, this, confidence);
111 }
112
113 var bestConfidenceSoFar = -1;
114 var lang = null;
115
116 for (var i = ngrams.length - 1; i >= 0; i--) {
117 var ngl = ngrams[i];
118
119 var parser = new NGramParser(ngl.fNGrams, this.byteMap());
120 var confidence = parser.parse(det, this.spaceChar);
121 if (confidence > bestConfidenceSoFar) {
122 bestConfidenceSoFar = confidence;
123 lang = ngl.fLang;
124 }
125 }
126
127 var name = this.name(det);
128 return bestConfidenceSoFar <= 0 ? null : new Match(det, this, bestConfidenceSoFar, name, lang);
129};
130
131
132module.exports.ISO_8859_1 = function() {
133 this.byteMap = function() {
134 return [
135 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
136 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
137 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
138 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
139 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00,
140 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
141 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
142 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
143 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
144 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
145 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
146 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
147 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
148 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
149 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
150 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
151 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
152 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
153 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
154 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
155 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
156 0x20, 0x20, 0xAA, 0x20, 0x20, 0x20, 0x20, 0x20,
157 0x20, 0x20, 0x20, 0x20, 0x20, 0xB5, 0x20, 0x20,
158 0x20, 0x20, 0xBA, 0x20, 0x20, 0x20, 0x20, 0x20,
159 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
160 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
161 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0x20,
162 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xDF,
163 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
164 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
165 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0x20,
166 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF
167 ];
168 };
169
170 this.ngrams = function() {
171 return [
172 new NGramsPlusLang('da', [
173 0x206166, 0x206174, 0x206465, 0x20656E, 0x206572, 0x20666F, 0x206861, 0x206920,
174 0x206D65, 0x206F67, 0x2070E5, 0x207369, 0x207374, 0x207469, 0x207669, 0x616620,
175 0x616E20, 0x616E64, 0x617220, 0x617420, 0x646520, 0x64656E, 0x646572, 0x646574,
176 0x652073, 0x656420, 0x656465, 0x656E20, 0x656E64, 0x657220, 0x657265, 0x657320,
177 0x657420, 0x666F72, 0x676520, 0x67656E, 0x676572, 0x696765, 0x696C20, 0x696E67,
178 0x6B6520, 0x6B6B65, 0x6C6572, 0x6C6967, 0x6C6C65, 0x6D6564, 0x6E6465, 0x6E6520,
179 0x6E6720, 0x6E6765, 0x6F6720, 0x6F6D20, 0x6F7220, 0x70E520, 0x722064, 0x722065,
180 0x722073, 0x726520, 0x737465, 0x742073, 0x746520, 0x746572, 0x74696C, 0x766572
181 ]),
182 new NGramsPlusLang('de', [
183 0x20616E, 0x206175, 0x206265, 0x206461, 0x206465, 0x206469, 0x206569, 0x206765,
184 0x206861, 0x20696E, 0x206D69, 0x207363, 0x207365, 0x20756E, 0x207665, 0x20766F,
185 0x207765, 0x207A75, 0x626572, 0x636820, 0x636865, 0x636874, 0x646173, 0x64656E,
186 0x646572, 0x646965, 0x652064, 0x652073, 0x65696E, 0x656974, 0x656E20, 0x657220,
187 0x657320, 0x67656E, 0x68656E, 0x687420, 0x696368, 0x696520, 0x696E20, 0x696E65,
188 0x697420, 0x6C6963, 0x6C6C65, 0x6E2061, 0x6E2064, 0x6E2073, 0x6E6420, 0x6E6465,
189 0x6E6520, 0x6E6720, 0x6E6765, 0x6E7465, 0x722064, 0x726465, 0x726569, 0x736368,
190 0x737465, 0x742064, 0x746520, 0x74656E, 0x746572, 0x756E64, 0x756E67, 0x766572
191 ]),
192 new NGramsPlusLang('en', [
193 0x206120, 0x20616E, 0x206265, 0x20636F, 0x20666F, 0x206861, 0x206865, 0x20696E,
194 0x206D61, 0x206F66, 0x207072, 0x207265, 0x207361, 0x207374, 0x207468, 0x20746F,
195 0x207768, 0x616964, 0x616C20, 0x616E20, 0x616E64, 0x617320, 0x617420, 0x617465,
196 0x617469, 0x642061, 0x642074, 0x652061, 0x652073, 0x652074, 0x656420, 0x656E74,
197 0x657220, 0x657320, 0x666F72, 0x686174, 0x686520, 0x686572, 0x696420, 0x696E20,
198 0x696E67, 0x696F6E, 0x697320, 0x6E2061, 0x6E2074, 0x6E6420, 0x6E6720, 0x6E7420,
199 0x6F6620, 0x6F6E20, 0x6F7220, 0x726520, 0x727320, 0x732061, 0x732074, 0x736169,
200 0x737420, 0x742074, 0x746572, 0x746861, 0x746865, 0x74696F, 0x746F20, 0x747320
201 ]),
202 new NGramsPlusLang('es', [
203 0x206120, 0x206361, 0x20636F, 0x206465, 0x20656C, 0x20656E, 0x206573, 0x20696E,
204 0x206C61, 0x206C6F, 0x207061, 0x20706F, 0x207072, 0x207175, 0x207265, 0x207365,
205 0x20756E, 0x207920, 0x612063, 0x612064, 0x612065, 0x61206C, 0x612070, 0x616369,
206 0x61646F, 0x616C20, 0x617220, 0x617320, 0x6369F3, 0x636F6E, 0x646520, 0x64656C,
207 0x646F20, 0x652064, 0x652065, 0x65206C, 0x656C20, 0x656E20, 0x656E74, 0x657320,
208 0x657374, 0x69656E, 0x69F36E, 0x6C6120, 0x6C6F73, 0x6E2065, 0x6E7465, 0x6F2064,
209 0x6F2065, 0x6F6E20, 0x6F7220, 0x6F7320, 0x706172, 0x717565, 0x726120, 0x726573,
210 0x732064, 0x732065, 0x732070, 0x736520, 0x746520, 0x746F20, 0x756520, 0xF36E20
211 ]),
212 new NGramsPlusLang('fr', [
213 0x206175, 0x20636F, 0x206461, 0x206465, 0x206475, 0x20656E, 0x206574, 0x206C61,
214 0x206C65, 0x207061, 0x20706F, 0x207072, 0x207175, 0x207365, 0x20736F, 0x20756E,
215 0x20E020, 0x616E74, 0x617469, 0x636520, 0x636F6E, 0x646520, 0x646573, 0x647520,
216 0x652061, 0x652063, 0x652064, 0x652065, 0x65206C, 0x652070, 0x652073, 0x656E20,
217 0x656E74, 0x657220, 0x657320, 0x657420, 0x657572, 0x696F6E, 0x697320, 0x697420,
218 0x6C6120, 0x6C6520, 0x6C6573, 0x6D656E, 0x6E2064, 0x6E6520, 0x6E7320, 0x6E7420,
219 0x6F6E20, 0x6F6E74, 0x6F7572, 0x717565, 0x72206C, 0x726520, 0x732061, 0x732064,
220 0x732065, 0x73206C, 0x732070, 0x742064, 0x746520, 0x74696F, 0x756520, 0x757220
221 ]),
222 new NGramsPlusLang('it', [
223 0x20616C, 0x206368, 0x20636F, 0x206465, 0x206469, 0x206520, 0x20696C, 0x20696E,
224 0x206C61, 0x207065, 0x207072, 0x20756E, 0x612063, 0x612064, 0x612070, 0x612073,
225 0x61746F, 0x636865, 0x636F6E, 0x64656C, 0x646920, 0x652061, 0x652063, 0x652064,
226 0x652069, 0x65206C, 0x652070, 0x652073, 0x656C20, 0x656C6C, 0x656E74, 0x657220,
227 0x686520, 0x692061, 0x692063, 0x692064, 0x692073, 0x696120, 0x696C20, 0x696E20,
228 0x696F6E, 0x6C6120, 0x6C6520, 0x6C6920, 0x6C6C61, 0x6E6520, 0x6E6920, 0x6E6F20,
229 0x6E7465, 0x6F2061, 0x6F2064, 0x6F2069, 0x6F2073, 0x6F6E20, 0x6F6E65, 0x706572,
230 0x726120, 0x726520, 0x736920, 0x746120, 0x746520, 0x746920, 0x746F20, 0x7A696F
231 ]),
232 new NGramsPlusLang('nl', [
233 0x20616C, 0x206265, 0x206461, 0x206465, 0x206469, 0x206565, 0x20656E, 0x206765,
234 0x206865, 0x20696E, 0x206D61, 0x206D65, 0x206F70, 0x207465, 0x207661, 0x207665,
235 0x20766F, 0x207765, 0x207A69, 0x61616E, 0x616172, 0x616E20, 0x616E64, 0x617220,
236 0x617420, 0x636874, 0x646520, 0x64656E, 0x646572, 0x652062, 0x652076, 0x65656E,
237 0x656572, 0x656E20, 0x657220, 0x657273, 0x657420, 0x67656E, 0x686574, 0x696520,
238 0x696E20, 0x696E67, 0x697320, 0x6E2062, 0x6E2064, 0x6E2065, 0x6E2068, 0x6E206F,
239 0x6E2076, 0x6E6465, 0x6E6720, 0x6F6E64, 0x6F6F72, 0x6F7020, 0x6F7220, 0x736368,
240 0x737465, 0x742064, 0x746520, 0x74656E, 0x746572, 0x76616E, 0x766572, 0x766F6F
241 ]),
242 new NGramsPlusLang('no', [
243 0x206174, 0x206176, 0x206465, 0x20656E, 0x206572, 0x20666F, 0x206861, 0x206920,
244 0x206D65, 0x206F67, 0x2070E5, 0x207365, 0x20736B, 0x20736F, 0x207374, 0x207469,
245 0x207669, 0x20E520, 0x616E64, 0x617220, 0x617420, 0x646520, 0x64656E, 0x646574,
246 0x652073, 0x656420, 0x656E20, 0x656E65, 0x657220, 0x657265, 0x657420, 0x657474,
247 0x666F72, 0x67656E, 0x696B6B, 0x696C20, 0x696E67, 0x6B6520, 0x6B6B65, 0x6C6520,
248 0x6C6C65, 0x6D6564, 0x6D656E, 0x6E2073, 0x6E6520, 0x6E6720, 0x6E6765, 0x6E6E65,
249 0x6F6720, 0x6F6D20, 0x6F7220, 0x70E520, 0x722073, 0x726520, 0x736F6D, 0x737465,
250 0x742073, 0x746520, 0x74656E, 0x746572, 0x74696C, 0x747420, 0x747465, 0x766572
251 ]),
252 new NGramsPlusLang('pt', [
253 0x206120, 0x20636F, 0x206461, 0x206465, 0x20646F, 0x206520, 0x206573, 0x206D61,
254 0x206E6F, 0x206F20, 0x207061, 0x20706F, 0x207072, 0x207175, 0x207265, 0x207365,
255 0x20756D, 0x612061, 0x612063, 0x612064, 0x612070, 0x616465, 0x61646F, 0x616C20,
256 0x617220, 0x617261, 0x617320, 0x636F6D, 0x636F6E, 0x646120, 0x646520, 0x646F20,
257 0x646F73, 0x652061, 0x652064, 0x656D20, 0x656E74, 0x657320, 0x657374, 0x696120,
258 0x696361, 0x6D656E, 0x6E7465, 0x6E746F, 0x6F2061, 0x6F2063, 0x6F2064, 0x6F2065,
259 0x6F2070, 0x6F7320, 0x706172, 0x717565, 0x726120, 0x726573, 0x732061, 0x732064,
260 0x732065, 0x732070, 0x737461, 0x746520, 0x746F20, 0x756520, 0xE36F20, 0xE7E36F
261 ]),
262 new NGramsPlusLang('sv', [
263 0x206174, 0x206176, 0x206465, 0x20656E, 0x2066F6, 0x206861, 0x206920, 0x20696E,
264 0x206B6F, 0x206D65, 0x206F63, 0x2070E5, 0x20736B, 0x20736F, 0x207374, 0x207469,
265 0x207661, 0x207669, 0x20E472, 0x616465, 0x616E20, 0x616E64, 0x617220, 0x617474,
266 0x636820, 0x646520, 0x64656E, 0x646572, 0x646574, 0x656420, 0x656E20, 0x657220,
267 0x657420, 0x66F672, 0x67656E, 0x696C6C, 0x696E67, 0x6B6120, 0x6C6C20, 0x6D6564,
268 0x6E2073, 0x6E6120, 0x6E6465, 0x6E6720, 0x6E6765, 0x6E696E, 0x6F6368, 0x6F6D20,
269 0x6F6E20, 0x70E520, 0x722061, 0x722073, 0x726120, 0x736B61, 0x736F6D, 0x742073,
270 0x746120, 0x746520, 0x746572, 0x74696C, 0x747420, 0x766172, 0xE47220, 0xF67220,
271 ])
272 ];
273 };
274
275 this.name = function(det) {
276 return (det && det.fC1Bytes) ? 'windows-1252' : 'ISO-8859-1';
277 };
278};
279util.inherits(module.exports.ISO_8859_1, sbcs);
280
281
282module.exports.ISO_8859_2 = function() {
283 this.byteMap = function() {
284 return [
285 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
286 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
287 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
288 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
289 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00,
290 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
291 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
292 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
293 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
294 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
295 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
296 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
297 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
298 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
299 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
300 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
301 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
302 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
303 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
304 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
305 0x20, 0xB1, 0x20, 0xB3, 0x20, 0xB5, 0xB6, 0x20,
306 0x20, 0xB9, 0xBA, 0xBB, 0xBC, 0x20, 0xBE, 0xBF,
307 0x20, 0xB1, 0x20, 0xB3, 0x20, 0xB5, 0xB6, 0xB7,
308 0x20, 0xB9, 0xBA, 0xBB, 0xBC, 0x20, 0xBE, 0xBF,
309 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
310 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
311 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0x20,
312 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xDF,
313 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
314 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
315 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0x20,
316 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0x20
317 ];
318 }
319
320 this.ngrams = function() {
321 return [
322 new NGramsPlusLang('cs', [
323 0x206120, 0x206279, 0x20646F, 0x206A65, 0x206E61, 0x206E65, 0x206F20, 0x206F64,
324 0x20706F, 0x207072, 0x2070F8, 0x20726F, 0x207365, 0x20736F, 0x207374, 0x20746F,
325 0x207620, 0x207679, 0x207A61, 0x612070, 0x636520, 0x636820, 0x652070, 0x652073,
326 0x652076, 0x656D20, 0x656EED, 0x686F20, 0x686F64, 0x697374, 0x6A6520, 0x6B7465,
327 0x6C6520, 0x6C6920, 0x6E6120, 0x6EE920, 0x6EEC20, 0x6EED20, 0x6F2070, 0x6F646E,
328 0x6F6A69, 0x6F7374, 0x6F7520, 0x6F7661, 0x706F64, 0x706F6A, 0x70726F, 0x70F865,
329 0x736520, 0x736F75, 0x737461, 0x737469, 0x73746E, 0x746572, 0x746EED, 0x746F20,
330 0x752070, 0xBE6520, 0xE16EED, 0xE9686F, 0xED2070, 0xED2073, 0xED6D20, 0xF86564,
331 ]),
332 new NGramsPlusLang('hu', [
333 0x206120, 0x20617A, 0x206265, 0x206567, 0x20656C, 0x206665, 0x206861, 0x20686F,
334 0x206973, 0x206B65, 0x206B69, 0x206BF6, 0x206C65, 0x206D61, 0x206D65, 0x206D69,
335 0x206E65, 0x20737A, 0x207465, 0x20E973, 0x612061, 0x61206B, 0x61206D, 0x612073,
336 0x616B20, 0x616E20, 0x617A20, 0x62616E, 0x62656E, 0x656779, 0x656B20, 0x656C20,
337 0x656C65, 0x656D20, 0x656E20, 0x657265, 0x657420, 0x657465, 0x657474, 0x677920,
338 0x686F67, 0x696E74, 0x697320, 0x6B2061, 0x6BF67A, 0x6D6567, 0x6D696E, 0x6E2061,
339 0x6E616B, 0x6E656B, 0x6E656D, 0x6E7420, 0x6F6779, 0x732061, 0x737A65, 0x737A74,
340 0x737AE1, 0x73E967, 0x742061, 0x747420, 0x74E173, 0x7A6572, 0xE16E20, 0xE97320,
341 ]),
342 new NGramsPlusLang('pl', [
343 0x20637A, 0x20646F, 0x206920, 0x206A65, 0x206B6F, 0x206D61, 0x206D69, 0x206E61,
344 0x206E69, 0x206F64, 0x20706F, 0x207072, 0x207369, 0x207720, 0x207769, 0x207779,
345 0x207A20, 0x207A61, 0x612070, 0x612077, 0x616E69, 0x636820, 0x637A65, 0x637A79,
346 0x646F20, 0x647A69, 0x652070, 0x652073, 0x652077, 0x65207A, 0x65676F, 0x656A20,
347 0x656D20, 0x656E69, 0x676F20, 0x696120, 0x696520, 0x69656A, 0x6B6120, 0x6B6920,
348 0x6B6965, 0x6D6965, 0x6E6120, 0x6E6961, 0x6E6965, 0x6F2070, 0x6F7761, 0x6F7769,
349 0x706F6C, 0x707261, 0x70726F, 0x70727A, 0x727A65, 0x727A79, 0x7369EA, 0x736B69,
350 0x737461, 0x776965, 0x796368, 0x796D20, 0x7A6520, 0x7A6965, 0x7A7920, 0xF37720,
351 ]),
352 new NGramsPlusLang('ro', [
353 0x206120, 0x206163, 0x206361, 0x206365, 0x20636F, 0x206375, 0x206465, 0x206469,
354 0x206C61, 0x206D61, 0x207065, 0x207072, 0x207365, 0x2073E3, 0x20756E, 0x20BA69,
355 0x20EE6E, 0x612063, 0x612064, 0x617265, 0x617420, 0x617465, 0x617520, 0x636172,
356 0x636F6E, 0x637520, 0x63E320, 0x646520, 0x652061, 0x652063, 0x652064, 0x652070,
357 0x652073, 0x656120, 0x656920, 0x656C65, 0x656E74, 0x657374, 0x692061, 0x692063,
358 0x692064, 0x692070, 0x696520, 0x696920, 0x696E20, 0x6C6120, 0x6C6520, 0x6C6F72,
359 0x6C7569, 0x6E6520, 0x6E7472, 0x6F7220, 0x70656E, 0x726520, 0x726561, 0x727520,
360 0x73E320, 0x746520, 0x747275, 0x74E320, 0x756920, 0x756C20, 0xBA6920, 0xEE6E20,
361 ])
362 ];
363 };
364
365 this.name = function(det) {
366 return (det && det.fC1Bytes) ? 'windows-1250' : 'ISO-8859-2';
367 };
368};
369util.inherits(module.exports.ISO_8859_2, sbcs);
370
371
372module.exports.ISO_8859_5 = function() {
373 this.byteMap = function() {
374 return [
375 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
376 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
377 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
378 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
379 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00,
380 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
381 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
382 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
383 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
384 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
385 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
386 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
387 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
388 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
389 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
390 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
391 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
392 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
393 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
394 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
395 0x20, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7,
396 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0x20, 0xFE, 0xFF,
397 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7,
398 0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF,
399 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
400 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
401 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7,
402 0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF,
403 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
404 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
405 0x20, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7,
406 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0x20, 0xFE, 0xFF
407 ];
408 };
409
410 this.ngrams = function() {
411 return [
412 0x20D220, 0x20D2DE, 0x20D4DE, 0x20D7D0, 0x20D820, 0x20DAD0, 0x20DADE, 0x20DDD0,
413 0x20DDD5, 0x20DED1, 0x20DFDE, 0x20DFE0, 0x20E0D0, 0x20E1DE, 0x20E1E2, 0x20E2DE,
414 0x20E7E2, 0x20EDE2, 0xD0DDD8, 0xD0E2EC, 0xD3DE20, 0xD5DBEC, 0xD5DDD8, 0xD5E1E2,
415 0xD5E220, 0xD820DF, 0xD8D520, 0xD8D820, 0xD8EF20, 0xDBD5DD, 0xDBD820, 0xDBECDD,
416 0xDDD020, 0xDDD520, 0xDDD8D5, 0xDDD8EF, 0xDDDE20, 0xDDDED2, 0xDE20D2, 0xDE20DF,
417 0xDE20E1, 0xDED220, 0xDED2D0, 0xDED3DE, 0xDED920, 0xDEDBEC, 0xDEDC20, 0xDEE1E2,
418 0xDFDEDB, 0xDFE0D5, 0xDFE0D8, 0xDFE0DE, 0xE0D0D2, 0xE0D5D4, 0xE1E2D0, 0xE1E2D2,
419 0xE1E2D8, 0xE1EF20, 0xE2D5DB, 0xE2DE20, 0xE2DEE0, 0xE2EC20, 0xE7E2DE, 0xEBE520
420 ];
421 };
422
423 this.name = function(det) {
424 return 'ISO-8859-5';
425 };
426
427 this.language = function() {
428 return 'ru';
429 };
430};
431util.inherits(module.exports.ISO_8859_5, sbcs);
432
433
434module.exports.ISO_8859_6 = function() {
435 this.byteMap = function() {
436 return [
437 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
438 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
439 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
440 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
441 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00,
442 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
443 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
444 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
445 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
446 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
447 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
448 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
449 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
450 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
451 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
452 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
453 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
454 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
455 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
456 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
457 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
458 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
459 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
460 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
461 0x20, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7,
462 0xC8, 0xC9, 0xCA, 0xCB, 0xCC, 0xCD, 0xCE, 0xCF,
463 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7,
464 0xD8, 0xD9, 0xDA, 0x20, 0x20, 0x20, 0x20, 0x20,
465 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
466 0xE8, 0xE9, 0xEA, 0x20, 0x20, 0x20, 0x20, 0x20,
467 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
468 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20
469 ];
470 };
471
472 this.ngrams = function() {
473 return [
474 0x20C7E4, 0x20C7E6, 0x20C8C7, 0x20D9E4, 0x20E1EA, 0x20E4E4, 0x20E5E6, 0x20E8C7,
475 0xC720C7, 0xC7C120, 0xC7CA20, 0xC7D120, 0xC7E420, 0xC7E4C3, 0xC7E4C7, 0xC7E4C8,
476 0xC7E4CA, 0xC7E4CC, 0xC7E4CD, 0xC7E4CF, 0xC7E4D3, 0xC7E4D9, 0xC7E4E2, 0xC7E4E5,
477 0xC7E4E8, 0xC7E4EA, 0xC7E520, 0xC7E620, 0xC7E6CA, 0xC820C7, 0xC920C7, 0xC920E1,
478 0xC920E4, 0xC920E5, 0xC920E8, 0xCA20C7, 0xCF20C7, 0xCFC920, 0xD120C7, 0xD1C920,
479 0xD320C7, 0xD920C7, 0xD9E4E9, 0xE1EA20, 0xE420C7, 0xE4C920, 0xE4E920, 0xE4EA20,
480 0xE520C7, 0xE5C720, 0xE5C920, 0xE5E620, 0xE620C7, 0xE720C7, 0xE7C720, 0xE8C7E4,
481 0xE8E620, 0xE920C7, 0xEA20C7, 0xEA20E5, 0xEA20E8, 0xEAC920, 0xEAD120, 0xEAE620
482 ];
483 };
484
485 this.name = function(det) {
486 return 'ISO-8859-6';
487 };
488
489 this.language = function() {
490 return 'ar';
491 };
492};
493util.inherits(module.exports.ISO_8859_6, sbcs);
494
495
496module.exports.ISO_8859_7 = function() {
497 this.byteMap = function() {
498 return [
499 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
500 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
501 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
502 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
503 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00,
504 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
505 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
506 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
507 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
508 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
509 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
510 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
511 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
512 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
513 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
514 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
515 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
516 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
517 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
518 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
519 0x20, 0xA1, 0xA2, 0x20, 0x20, 0x20, 0x20, 0x20,
520 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
521 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0xDC, 0x20,
522 0xDD, 0xDE, 0xDF, 0x20, 0xFC, 0x20, 0xFD, 0xFE,
523 0xC0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
524 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
525 0xF0, 0xF1, 0x20, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7,
526 0xF8, 0xF9, 0xFA, 0xFB, 0xDC, 0xDD, 0xDE, 0xDF,
527 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
528 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
529 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7,
530 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0x20
531 ];
532 };
533
534 this.ngrams = function() {
535 return [
536 0x20E1ED, 0x20E1F0, 0x20E3E9, 0x20E4E9, 0x20E5F0, 0x20E720, 0x20EAE1, 0x20ECE5,
537 0x20EDE1, 0x20EF20, 0x20F0E1, 0x20F0EF, 0x20F0F1, 0x20F3F4, 0x20F3F5, 0x20F4E7,
538 0x20F4EF, 0xDFE120, 0xE120E1, 0xE120F4, 0xE1E920, 0xE1ED20, 0xE1F0FC, 0xE1F220,
539 0xE3E9E1, 0xE5E920, 0xE5F220, 0xE720F4, 0xE7ED20, 0xE7F220, 0xE920F4, 0xE9E120,
540 0xE9EADE, 0xE9F220, 0xEAE1E9, 0xEAE1F4, 0xECE520, 0xED20E1, 0xED20E5, 0xED20F0,
541 0xEDE120, 0xEFF220, 0xEFF520, 0xF0EFF5, 0xF0F1EF, 0xF0FC20, 0xF220E1, 0xF220E5,
542 0xF220EA, 0xF220F0, 0xF220F4, 0xF3E520, 0xF3E720, 0xF3F4EF, 0xF4E120, 0xF4E1E9,
543 0xF4E7ED, 0xF4E7F2, 0xF4E9EA, 0xF4EF20, 0xF4EFF5, 0xF4F9ED, 0xF9ED20, 0xFEED20
544 ];
545 };
546
547 this.name = function(det) {
548 return (det && det.fC1Bytes) ? 'windows-1253' : 'ISO-8859-7';
549 };
550
551 this.language = function() {
552 return 'el';
553 };
554};
555util.inherits(module.exports.ISO_8859_7, sbcs);
556
557module.exports.ISO_8859_8 = function() {
558
559 this.byteMap = function() {
560 return [
561 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
562 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
563 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
564 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
565 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00,
566 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
567 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
568 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
569 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
570 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
571 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
572 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
573 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
574 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
575 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
576 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
577 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
578 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
579 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
580 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
581 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
582 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
583 0x20, 0x20, 0x20, 0x20, 0x20, 0xB5, 0x20, 0x20,
584 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
585 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
586 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
587 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
588 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
589 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
590 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
591 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7,
592 0xF8, 0xF9, 0xFA, 0x20, 0x20, 0x20, 0x20, 0x20
593 ];
594 };
595
596 this.ngrams = function() {
597 return [
598 new NGramsPlusLang('he', [
599 0x20E0E5, 0x20E0E7, 0x20E0E9, 0x20E0FA, 0x20E1E9, 0x20E1EE, 0x20E4E0, 0x20E4E5,
600 0x20E4E9, 0x20E4EE, 0x20E4F2, 0x20E4F9, 0x20E4FA, 0x20ECE0, 0x20ECE4, 0x20EEE0,
601 0x20F2EC, 0x20F9EC, 0xE0FA20, 0xE420E0, 0xE420E1, 0xE420E4, 0xE420EC, 0xE420EE,
602 0xE420F9, 0xE4E5E0, 0xE5E020, 0xE5ED20, 0xE5EF20, 0xE5F820, 0xE5FA20, 0xE920E4,
603 0xE9E420, 0xE9E5FA, 0xE9E9ED, 0xE9ED20, 0xE9EF20, 0xE9F820, 0xE9FA20, 0xEC20E0,
604 0xEC20E4, 0xECE020, 0xECE420, 0xED20E0, 0xED20E1, 0xED20E4, 0xED20EC, 0xED20EE,
605 0xED20F9, 0xEEE420, 0xEF20E4, 0xF0E420, 0xF0E920, 0xF0E9ED, 0xF2EC20, 0xF820E4,
606 0xF8E9ED, 0xF9EC20, 0xFA20E0, 0xFA20E1, 0xFA20E4, 0xFA20EC, 0xFA20EE, 0xFA20F9,
607 ]),
608 new NGramsPlusLang('he', [
609 0x20E0E5, 0x20E0EC, 0x20E4E9, 0x20E4EC, 0x20E4EE, 0x20E4F0, 0x20E9F0, 0x20ECF2,
610 0x20ECF9, 0x20EDE5, 0x20EDE9, 0x20EFE5, 0x20EFE9, 0x20F8E5, 0x20F8E9, 0x20FAE0,
611 0x20FAE5, 0x20FAE9, 0xE020E4, 0xE020EC, 0xE020ED, 0xE020FA, 0xE0E420, 0xE0E5E4,
612 0xE0EC20, 0xE0EE20, 0xE120E4, 0xE120ED, 0xE120FA, 0xE420E4, 0xE420E9, 0xE420EC,
613 0xE420ED, 0xE420EF, 0xE420F8, 0xE420FA, 0xE4EC20, 0xE5E020, 0xE5E420, 0xE7E020,
614 0xE9E020, 0xE9E120, 0xE9E420, 0xEC20E4, 0xEC20ED, 0xEC20FA, 0xECF220, 0xECF920,
615 0xEDE9E9, 0xEDE9F0, 0xEDE9F8, 0xEE20E4, 0xEE20ED, 0xEE20FA, 0xEEE120, 0xEEE420,
616 0xF2E420, 0xF920E4, 0xF920ED, 0xF920FA, 0xF9E420, 0xFAE020, 0xFAE420, 0xFAE5E9,
617 ])
618 ];
619 };
620
621 this.name = function(det) {
622 return (det && det.fC1Bytes) ? 'windows-1255' : 'ISO-8859-8';
623 };
624
625 this.language = function() {
626 return 'he';
627 };
628
629};
630util.inherits(module.exports.ISO_8859_8, sbcs);
631
632
633module.exports.ISO_8859_9 = function() {
634 this.byteMap = function() {
635 return [
636 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
637 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
638 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
639 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
640 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00,
641 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
642 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
643 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
644 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
645 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
646 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
647 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
648 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
649 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
650 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
651 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
652 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
653 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
654 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
655 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
656 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
657 0x20, 0x20, 0xAA, 0x20, 0x20, 0x20, 0x20, 0x20,
658 0x20, 0x20, 0x20, 0x20, 0x20, 0xB5, 0x20, 0x20,
659 0x20, 0x20, 0xBA, 0x20, 0x20, 0x20, 0x20, 0x20,
660 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
661 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
662 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0x20,
663 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0x69, 0xFE, 0xDF,
664 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
665 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
666 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0x20,
667 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF
668 ];
669 };
670
671 this.ngrams = function() {
672 return [
673 0x206261, 0x206269, 0x206275, 0x206461, 0x206465, 0x206765, 0x206861, 0x20696C,
674 0x206B61, 0x206B6F, 0x206D61, 0x206F6C, 0x207361, 0x207461, 0x207665, 0x207961,
675 0x612062, 0x616B20, 0x616C61, 0x616D61, 0x616E20, 0x616EFD, 0x617220, 0x617261,
676 0x6172FD, 0x6173FD, 0x617961, 0x626972, 0x646120, 0x646520, 0x646920, 0x652062,
677 0x65206B, 0x656469, 0x656E20, 0x657220, 0x657269, 0x657369, 0x696C65, 0x696E20,
678 0x696E69, 0x697220, 0x6C616E, 0x6C6172, 0x6C6520, 0x6C6572, 0x6E2061, 0x6E2062,
679 0x6E206B, 0x6E6461, 0x6E6465, 0x6E6520, 0x6E6920, 0x6E696E, 0x6EFD20, 0x72696E,
680 0x72FD6E, 0x766520, 0x796120, 0x796F72, 0xFD6E20, 0xFD6E64, 0xFD6EFD, 0xFDF0FD
681 ];
682 };
683
684 this.name = function(det) {
685 return (det && det.fC1Bytes) ? 'windows-1254' : 'ISO-8859-9';
686 };
687
688 this.language = function() {
689 return 'tr';
690 };
691};
692util.inherits(module.exports.ISO_8859_9, sbcs);
693
694
695module.exports.windows_1251 = function() {
696 this.byteMap = function() {
697 return [
698 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
699 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
700 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
701 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
702 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00,
703 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
704 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
705 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
706 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
707 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
708 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
709 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
710 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
711 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
712 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
713 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
714 0x90, 0x83, 0x20, 0x83, 0x20, 0x20, 0x20, 0x20,
715 0x20, 0x20, 0x9A, 0x20, 0x9C, 0x9D, 0x9E, 0x9F,
716 0x90, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
717 0x20, 0x20, 0x9A, 0x20, 0x9C, 0x9D, 0x9E, 0x9F,
718 0x20, 0xA2, 0xA2, 0xBC, 0x20, 0xB4, 0x20, 0x20,
719 0xB8, 0x20, 0xBA, 0x20, 0x20, 0x20, 0x20, 0xBF,
720 0x20, 0x20, 0xB3, 0xB3, 0xB4, 0xB5, 0x20, 0x20,
721 0xB8, 0x20, 0xBA, 0x20, 0xBC, 0xBE, 0xBE, 0xBF,
722 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
723 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
724 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7,
725 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF,
726 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
727 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
728 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7,
729 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF
730 ];
731 };
732
733 this.ngrams = function() {
734 return [
735 0x20E220, 0x20E2EE, 0x20E4EE, 0x20E7E0, 0x20E820, 0x20EAE0, 0x20EAEE, 0x20EDE0,
736 0x20EDE5, 0x20EEE1, 0x20EFEE, 0x20EFF0, 0x20F0E0, 0x20F1EE, 0x20F1F2, 0x20F2EE,
737 0x20F7F2, 0x20FDF2, 0xE0EDE8, 0xE0F2FC, 0xE3EE20, 0xE5EBFC, 0xE5EDE8, 0xE5F1F2,
738 0xE5F220, 0xE820EF, 0xE8E520, 0xE8E820, 0xE8FF20, 0xEBE5ED, 0xEBE820, 0xEBFCED,
739 0xEDE020, 0xEDE520, 0xEDE8E5, 0xEDE8FF, 0xEDEE20, 0xEDEEE2, 0xEE20E2, 0xEE20EF,
740 0xEE20F1, 0xEEE220, 0xEEE2E0, 0xEEE3EE, 0xEEE920, 0xEEEBFC, 0xEEEC20, 0xEEF1F2,
741 0xEFEEEB, 0xEFF0E5, 0xEFF0E8, 0xEFF0EE, 0xF0E0E2, 0xF0E5E4, 0xF1F2E0, 0xF1F2E2,
742 0xF1F2E8, 0xF1FF20, 0xF2E5EB, 0xF2EE20, 0xF2EEF0, 0xF2FC20, 0xF7F2EE, 0xFBF520
743 ];
744 };
745
746 this.name = function(det) {
747 return 'windows-1251';
748 };
749
750 this.language = function() {
751 return 'ru';
752 };
753};
754util.inherits(module.exports.windows_1251, sbcs);
755
756
757module.exports.windows_1256 = function() {
758 this.byteMap = function() {
759 return [
760 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
761 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
762 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
763 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
764 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00,
765 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
766 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
767 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
768 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
769 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
770 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
771 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
772 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
773 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
774 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
775 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
776 0x20, 0x81, 0x20, 0x83, 0x20, 0x20, 0x20, 0x20,
777 0x88, 0x20, 0x8A, 0x20, 0x9C, 0x8D, 0x8E, 0x8F,
778 0x90, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
779 0x98, 0x20, 0x9A, 0x20, 0x9C, 0x20, 0x20, 0x9F,
780 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
781 0x20, 0x20, 0xAA, 0x20, 0x20, 0x20, 0x20, 0x20,
782 0x20, 0x20, 0x20, 0x20, 0x20, 0xB5, 0x20, 0x20,
783 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
784 0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7,
785 0xC8, 0xC9, 0xCA, 0xCB, 0xCC, 0xCD, 0xCE, 0xCF,
786 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0x20,
787 0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF,
788 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
789 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
790 0x20, 0x20, 0x20, 0x20, 0xF4, 0x20, 0x20, 0x20,
791 0x20, 0xF9, 0x20, 0xFB, 0xFC, 0x20, 0x20, 0xFF
792 ];
793 };
794
795 this.ngrams = function() {
796 return [
797 0x20C7E1, 0x20C7E4, 0x20C8C7, 0x20DAE1, 0x20DDED, 0x20E1E1, 0x20E3E4, 0x20E6C7,
798 0xC720C7, 0xC7C120, 0xC7CA20, 0xC7D120, 0xC7E120, 0xC7E1C3, 0xC7E1C7, 0xC7E1C8,
799 0xC7E1CA, 0xC7E1CC, 0xC7E1CD, 0xC7E1CF, 0xC7E1D3, 0xC7E1DA, 0xC7E1DE, 0xC7E1E3,
800 0xC7E1E6, 0xC7E1ED, 0xC7E320, 0xC7E420, 0xC7E4CA, 0xC820C7, 0xC920C7, 0xC920DD,
801 0xC920E1, 0xC920E3, 0xC920E6, 0xCA20C7, 0xCF20C7, 0xCFC920, 0xD120C7, 0xD1C920,
802 0xD320C7, 0xDA20C7, 0xDAE1EC, 0xDDED20, 0xE120C7, 0xE1C920, 0xE1EC20, 0xE1ED20,
803 0xE320C7, 0xE3C720, 0xE3C920, 0xE3E420, 0xE420C7, 0xE520C7, 0xE5C720, 0xE6C7E1,
804 0xE6E420, 0xEC20C7, 0xED20C7, 0xED20E3, 0xED20E6, 0xEDC920, 0xEDD120, 0xEDE420
805 ];
806 };
807
808 this.name = function(det) {
809 return 'windows-1256';
810 };
811
812 this.language = function() {
813 return 'ar';
814 };
815};
816util.inherits(module.exports.windows_1256, sbcs);
817
818
819module.exports.KOI8_R = function() {
820 this.byteMap = function() {
821 return [
822 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
823 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
824 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
825 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
826 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00,
827 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
828 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
829 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
830 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
831 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
832 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
833 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
834 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
835 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
836 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
837 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
838 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
839 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
840 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
841 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
842 0x20, 0x20, 0x20, 0xA3, 0x20, 0x20, 0x20, 0x20,
843 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
844 0x20, 0x20, 0x20, 0xA3, 0x20, 0x20, 0x20, 0x20,
845 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
846 0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7,
847 0xC8, 0xC9, 0xCA, 0xCB, 0xCC, 0xCD, 0xCE, 0xCF,
848 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7,
849 0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF,
850 0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7,
851 0xC8, 0xC9, 0xCA, 0xCB, 0xCC, 0xCD, 0xCE, 0xCF,
852 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7,
853 0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF
854 ];
855 };
856
857 this.ngrams = function() {
858 return [
859 0x20C4CF, 0x20C920, 0x20CBC1, 0x20CBCF, 0x20CEC1, 0x20CEC5, 0x20CFC2, 0x20D0CF,
860 0x20D0D2, 0x20D2C1, 0x20D3CF, 0x20D3D4, 0x20D4CF, 0x20D720, 0x20D7CF, 0x20DAC1,
861 0x20DCD4, 0x20DED4, 0xC1CEC9, 0xC1D4D8, 0xC5CCD8, 0xC5CEC9, 0xC5D3D4, 0xC5D420,
862 0xC7CF20, 0xC920D0, 0xC9C520, 0xC9C920, 0xC9D120, 0xCCC5CE, 0xCCC920, 0xCCD8CE,
863 0xCEC120, 0xCEC520, 0xCEC9C5, 0xCEC9D1, 0xCECF20, 0xCECFD7, 0xCF20D0, 0xCF20D3,
864 0xCF20D7, 0xCFC7CF, 0xCFCA20, 0xCFCCD8, 0xCFCD20, 0xCFD3D4, 0xCFD720, 0xCFD7C1,
865 0xD0CFCC, 0xD0D2C5, 0xD0D2C9, 0xD0D2CF, 0xD2C1D7, 0xD2C5C4, 0xD3D120, 0xD3D4C1,
866 0xD3D4C9, 0xD3D4D7, 0xD4C5CC, 0xD4CF20, 0xD4CFD2, 0xD4D820, 0xD9C820, 0xDED4CF
867 ];
868 };
869
870 this.name = function(det) {
871 return 'KOI8-R';
872 };
873
874 this.language = function() {
875 return 'ru';
876 };
877};
878util.inherits(module.exports.KOI8_R, sbcs);
879
880
881/*
882module.exports.ISO_8859_7 = function() {
883 this.byteMap = function() {
884 return [
885
886 ];
887 };
888
889 this.ngrams = function() {
890 return [
891
892 ];
893 };
894
895 this.name = function(det) {
896 if (typeof det == 'undefined')
897 return 'ISO-8859-7';
898 return det.fC1Bytes ? 'windows-1253' : 'ISO-8859-7';
899 };
900
901 this.language = function() {
902 return 'el';
903 };
904};
905util.inherits(module.exports.ISO_8859_7, sbcs);
906*/
907
Note: See TracBrowser for help on using the repository browser.