Reduce bundle size by compressing conversion tables

src/config.js

@@ -1,5 +1,4 @@
 var util = require('./util');
-var EncodingTable = require('./encoding-table');
 
 // Fallback character when a character can't be represented
 exports.FALLBACK_CHARACTER = 63; // '?'
@@ -107,33 +106,3 @@ exports.EncodingOrders = (function() {
 
   return orders;
 }());
-
-function init_JIS_TO_UTF8_TABLE() {
-  if (EncodingTable.JIS_TO_UTF8_TABLE === null) {
-    EncodingTable.JIS_TO_UTF8_TABLE = {};
-
-    var keys = util.objectKeys(EncodingTable.UTF8_TO_JIS_TABLE);
-    var i = 0;
-    var len = keys.length;
-    var key, value;
-
-    for (; i < len; i++) {
-      key = keys[i];
-      value = EncodingTable.UTF8_TO_JIS_TABLE[key];
-      if (value > 0x5F) {
-        EncodingTable.JIS_TO_UTF8_TABLE[value] = key | 0;
-      }
-    }
-
-    EncodingTable.JISX0212_TO_UTF8_TABLE = {};
-    keys = util.objectKeys(EncodingTable.UTF8_TO_JISX0212_TABLE);
-    len = keys.length;
-
-    for (i = 0; i < len; i++) {
-      key = keys[i];
-      value = EncodingTable.UTF8_TO_JISX0212_TABLE[key];
-      EncodingTable.JISX0212_TO_UTF8_TABLE[value] = key | 0;
-    }
-  }
-}
-exports.init_JIS_TO_UTF8_TABLE = init_JIS_TO_UTF8_TABLE;

src/encoding-convert.js

@@ -404,7 +404,7 @@ exports.EUCJPToSJIS = EUCJPToSJIS;
  * SJIS To UTF-8
  */
 function SJISToUTF8(data) {
-  config.init_JIS_TO_UTF8_TABLE();
+  EncodingTable.initConversionTables();
 
   var results = [];
   var i = 0;
@@ -485,7 +485,7 @@ exports.SJISToUTF8 = SJISToUTF8;
  * EUC-JP to UTF-8
  */
 function EUCJPToUTF8(data) {
-  config.init_JIS_TO_UTF8_TABLE();
+  EncodingTable.initConversionTables();
 
   var results = [];
   var i = 0;
@@ -551,7 +551,7 @@ exports.EUCJPToUTF8 = EUCJPToUTF8;
  * JIS to UTF-8
  */
 function JISToUTF8(data) {
-  config.init_JIS_TO_UTF8_TABLE();
+  EncodingTable.initConversionTables();
 
   var results = [];
   var index = 0;
@@ -635,6 +635,8 @@ exports.JISToUTF8 = JISToUTF8;
  * UTF-8 to SJIS
  */
 function UTF8ToSJIS(data, options) {
+  EncodingTable.initConversionTables();
+
   var results = [];
   var i = 0;
   var len = data && data.length;
@@ -717,6 +719,8 @@ exports.UTF8ToSJIS = UTF8ToSJIS;
  * UTF-8 to EUC-JP
  */
 function UTF8ToEUCJP(data, options) {
+  EncodingTable.initConversionTables();
+
   var results = [];
   var i = 0;
   var len = data && data.length;
@@ -784,6 +788,8 @@ exports.UTF8ToEUCJP = UTF8ToEUCJP;
  * UTF-8 to JIS
  */
 function UTF8ToJIS(data, options) {
+  EncodingTable.initConversionTables();
+
   var results = [];
   var index = 0;
   var len = data && data.length;

src/encoding-table.js

@@ -1,5 +1,108 @@
-exports.UTF8_TO_JIS_TABLE = require('./utf8-to-jis-table');
-exports.UTF8_TO_JIS_ALIAS_TABLE = require('./utf8-to-jis-alias-table');
-exports.UTF8_TO_JISX0212_TABLE = require('./utf8-to-jisx0212-table');
-exports.JIS_TO_UTF8_TABLE = require('./jis-to-utf8-table');
-exports.JISX0212_TO_UTF8_TABLE = require('./jisx0212-to-utf8-table');
+/**
+ * Conversion tables between UTF-8 and JIS
+ *
+ * The tables are stored in a compressed text format
+ * (see `src/utf8-to-jis-table.js`) and expanded into the *_TABLE objects.
+ */
+var jisTableData = require('./utf8-to-jis-table');
+var jisx0212TableData = require('./utf8-to-jisx0212-table');
+var UTF8_TO_JIS_ALIAS_TABLE = require('./utf8-to-jis-alias-table');
+
+var UTF8_TO_JIS_TABLE = {};
+var JIS_TO_UTF8_TABLE = {};
+var UTF8_TO_JISX0212_TABLE = {};
+var JISX0212_TO_UTF8_TABLE = {};
+
+// Unicode code point (BMP only) -> UTF-8 bytes packed into a number
+// Same encoding as UNICODEToUTF8, but returns a number instead of an array
+function toUTF8(c) {
+  if (c < 0x80) {
+    return c;
+  }
+  if (c < 0x800) {
+    return (0xC0 | (c >> 6)) << 8 |
+           (0x80 | (c & 0x3F));
+  }
+  return (0xE0 | (c >> 12)) << 16 |
+         (0x80 | ((c >> 6) & 0x3F)) << 8 |
+         (0x80 | (c & 0x3F));
+}
+
+/**
+ * Expand the compressed table data into the forward and reverse tables.
+ *
+ * Tokens are separated by ",":
+ *   "LB="    start of lead byte 0xLB (hex): the trail byte restarts at 0x21
+ *   "UUUU"   Unicode code point (hex) for the current trail byte
+ *   "UUUU+N" N consecutive code points starting at UUUU
+ *   "-N"     skip N trail bytes (unassigned)
+ *
+ * Example of compressed table data:
+ *   '21=3000+3,ff0c,ff0e,30fb, ...' : lead byte 0x21
+ *   '22=25c6,25a1,25a0,25b3,25b2, ...' : lead byte 0x22
+ *   '23=-15,ff10+10,-7,ff21+26,-6, ...' : lead byte 0x23
+ */
+function expandTable(data, utf8ToJisTable, jisToUtf8Table) {
+  var tokens = data.split(',');
+  var lead, trail, token, pos, cp, n, jis, utf8;
+
+  for (var i = 0, len = tokens.length; i < len; i++) {
+    token = tokens[i];
+
+    // The first token always contains "LB="
+    pos = token.indexOf('=');
+    if (pos !== -1) {
+      lead = parseInt(token.slice(0, pos), 16);
+      trail = 0x21; // the trail byte restarts at 0x21
+      token = token.slice(pos + 1);
+    }
+
+    if (token.charAt(0) === '-') {
+      // skip N trail bytes (e.g., "23=-15,ff10+10,-7, ...")
+      trail += parseInt(token.slice(1), 10);
+      continue;
+    }
+
+    // "UUUU" or "UUUU+N"
+    pos = token.indexOf('+');
+    if (pos === -1) {
+      cp = parseInt(token, 16);
+      n = 1;
+    } else {
+      cp = parseInt(token.slice(0, pos), 16);
+      n = parseInt(token.slice(pos + 1), 10);
+    }
+
+    while (n-- > 0) {
+      jis = (lead << 8) | trail++;
+      utf8 = toUTF8(cp++);
+      utf8ToJisTable[utf8] = jis;
+      jisToUtf8Table[jis] = utf8;
+    }
+  }
+}
+
+var expanded = false;
+
+function initConversionTables() {
+  if (expanded) {
+    return;
+  }
+
+  expandTable(jisTableData, UTF8_TO_JIS_TABLE, JIS_TO_UTF8_TABLE);
+  expandTable(jisx0212TableData, UTF8_TO_JISX0212_TABLE, JISX0212_TO_UTF8_TABLE);
+
+  // Add Half-width katakana (not stored in the compressed data)
+  // U+FF61..U+FF9F -> 0x21..0x5F (forward table only)
+  for (var kana = 0xFF61, jisKana = 0x21; jisKana <= 0x5F; kana++, jisKana++) {
+    UTF8_TO_JIS_TABLE[toUTF8(kana)] = jisKana;
+  }
+  expanded = true;
+}
+exports.initConversionTables = initConversionTables;
+
+exports.UTF8_TO_JIS_TABLE = UTF8_TO_JIS_TABLE;
+exports.UTF8_TO_JIS_ALIAS_TABLE = UTF8_TO_JIS_ALIAS_TABLE;
+exports.UTF8_TO_JISX0212_TABLE = UTF8_TO_JISX0212_TABLE;
+exports.JIS_TO_UTF8_TABLE = JIS_TO_UTF8_TABLE;
+exports.JISX0212_TO_UTF8_TABLE = JISX0212_TO_UTF8_TABLE;

src/utf8-to-jis-alias-table.js

@@ -10,5 +10,7 @@ module.exports = {
   0xE28892: 0x215D, // − U+2212 MINUS SIGN same cell as － U+FF0D (0xEFBC8D) -> SJIS 0x817C
   0xE3809C: 0x2141, // 〜 U+301C WAVE DASH same cell as ～ U+FF5E (0xEFBD9E) -> SJIS 0x8160
   0xC2A2: 0x2171,   // ¢ U+00A2 CENT SIGN  same cell as ￠ U+FFE0 (0xEFBFA0) -> SJIS 0x8191
-  0xC2A3: 0x2172    // £ U+00A3 POUND SIGN same cell as ￡ U+FFE1 (0xEFBFA1) -> SJIS 0x8192
+  0xC2A3: 0x2172,   // £ U+00A3 POUND SIGN same cell as ￡ U+FFE1 (0xEFBFA1) -> SJIS 0x8192
+  0xC2AC: 0x224C,   // ¬ U+00AC NOT SIGN same cell as ￢ U+FFE2 (0xEFBFA2) -> SJIS 0x81CA
+  0xE28096: 0x2142  // ‖ U+2016 DOUBLE VERTICAL LINE same cell as ∥ U+2225 (0xE288A5) -> SJIS 0x8161
 };