diff options
author | isaacs <i@izs.me> | 2020-07-23 20:58:04 +0300 |
---|---|---|
committer | isaacs <i@izs.me> | 2020-07-29 21:53:42 +0300 |
commit | ad5e07d8bd86d1dbe2b03dc142f8c8d6f4828ffe (patch) | |
tree | 97b66f97d77f35774f10a5e3e9957b1897d150bb /node_modules/iconv-lite | |
parent | a16994cfdd2f255016f3d8ee60d03473d80eabd8 (diff) |
Full dependency reboot
Reinstall everything from a clean node_modules and package-lock.json
state.
Re-generate list of bundleDependencies and node_modules/.gitignore with
a script that does the right thing based on actual dependency state.
Diffstat (limited to 'node_modules/iconv-lite')
-rw-r--r-- | node_modules/iconv-lite/.github/dependabot.yml | 11 | ||||
-rw-r--r-- | node_modules/iconv-lite/.travis.yml | 23 | ||||
-rw-r--r-- | node_modules/iconv-lite/Changelog.md | 98 | ||||
-rw-r--r-- | node_modules/iconv-lite/README.md | 68 | ||||
-rw-r--r-- | node_modules/iconv-lite/encodings/dbcs-codec.js | 144 | ||||
-rw-r--r-- | node_modules/iconv-lite/encodings/index.js | 3 | ||||
-rw-r--r-- | node_modules/iconv-lite/encodings/internal.js | 14 | ||||
-rw-r--r-- | node_modules/iconv-lite/encodings/sbcs-codec.js | 4 | ||||
-rw-r--r-- | node_modules/iconv-lite/encodings/sbcs-data.js | 10 | ||||
-rw-r--r-- | node_modules/iconv-lite/encodings/tables/gbk-added.json | 5 | ||||
-rw-r--r-- | node_modules/iconv-lite/encodings/utf16.js | 96 | ||||
-rw-r--r-- | node_modules/iconv-lite/encodings/utf32.js | 319 | ||||
-rw-r--r-- | node_modules/iconv-lite/encodings/utf7.js | 8 | ||||
-rw-r--r-- | node_modules/iconv-lite/lib/extend-node.js | 217 | ||||
-rw-r--r-- | node_modules/iconv-lite/lib/index.d.ts | 17 | ||||
-rw-r--r-- | node_modules/iconv-lite/lib/index.js | 53 | ||||
-rw-r--r-- | node_modules/iconv-lite/lib/streams.js | 190 | ||||
-rw-r--r-- | node_modules/iconv-lite/package.json | 49 |
18 files changed, 778 insertions, 551 deletions
diff --git a/node_modules/iconv-lite/.github/dependabot.yml b/node_modules/iconv-lite/.github/dependabot.yml new file mode 100644 index 000000000..e4a0e0afd --- /dev/null +++ b/node_modules/iconv-lite/.github/dependabot.yml @@ -0,0 +1,11 @@ +# Please see the documentation for all configuration options: +# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates + +version: 2 +updates: + - package-ecosystem: "npm" + directory: "/" + schedule: + interval: "daily" + allow: + - dependency-type: production diff --git a/node_modules/iconv-lite/.travis.yml b/node_modules/iconv-lite/.travis.yml deleted file mode 100644 index 3eab7fdb3..000000000 --- a/node_modules/iconv-lite/.travis.yml +++ /dev/null @@ -1,23 +0,0 @@ - sudo: false - language: node_js - node_js: - - "0.10" - - "0.11" - - "0.12" - - "iojs" - - "4" - - "6" - - "8" - - "node" - - - env: - - CXX=g++-4.8 - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - gcc-4.8 - - g++-4.8 - diff --git a/node_modules/iconv-lite/Changelog.md b/node_modules/iconv-lite/Changelog.md index e31cd0c24..c299cc06a 100644 --- a/node_modules/iconv-lite/Changelog.md +++ b/node_modules/iconv-lite/Changelog.md @@ -1,46 +1,96 @@ +## 0.6.2 / 2020-07-08 + * Support Uint8Array-s decoding without conversion to Buffers, plus fix an edge case. -# 0.4.23 / 2018-05-07 + +## 0.6.1 / 2020-06-28 + * Support Uint8Array-s directly when decoding (#246, by @gyzerok) + * Unify package.json version ranges to be strictly semver-compatible (#241) + * Fix minor issue in UTF-32 decoder's endianness detection code. + + +## 0.6.0 / 2020-06-08 + * Updated 'gb18030' encoding to :2005 edition (see https://github.com/whatwg/encoding/issues/22). + * Removed `iconv.extendNodeEncodings()` mechanism. It was deprecated 5 years ago and didn't work + in recent Node versions. + * Reworked Streaming API behavior in browser environments to fix #204. Streaming API will be + excluded by default in browser packs, saving ~100Kb bundle size, unless enabled explicitly using + `iconv.enableStreamingAPI(require('stream'))`. + * Updates to development environment & tests: + * Added ./test/webpack private package to test complex new use cases that need custom environment. + It's tested as a separate job in Travis CI. + * Updated generation code for the new EUC-KR index file format from Encoding Standard. + * Removed Buffer() constructor in tests (#197 by @gabrielschulhof). + + +## 0.5.2 / 2020-06-08 + * Added `iconv.getEncoder()` and `iconv.getDecoder()` methods to typescript definitions (#229). + * Fixed semver version to 6.1.2 to support Node 8.x (by @tanandara). + * Capped iconv version to 2.x as 3.x has dropped support for older Node versions. + * Switched from instanbul to c8 for code coverage. + + +## 0.5.1 / 2020-01-18 + + * Added cp720 encoding (#221, by @kr-deps) + * (minor) Changed Changelog.md formatting to use h2. + + +## 0.5.0 / 2019-06-26 + + * Added UTF-32 encoding, both little-endian and big-endian variants (UTF-32LE, UTF32-BE). If endianness + is not provided for decoding, it's deduced automatically from the stream using a heuristic similar to + what we use in UTF-16. (great work in #216 by @kshetline) + * Several minor updates to README (#217 by @oldj, plus some more) + * Added Node versions 10 and 12 to Travis test harness. + + +## 0.4.24 / 2018-08-22 + + * Added MIK encoding (#196, by @Ivan-Kalatchev) + + +## 0.4.23 / 2018-05-07 * Fix deprecation warning in Node v10 due to the last usage of `new Buffer` (#185, by @felixbuenemann) * Switched from NodeBuffer to Buffer in typings (#155 by @felixfbecker, #186 by @larssn) -# 0.4.22 / 2018-05-05 +## 0.4.22 / 2018-05-05 * Use older semver style for dependencies to be compatible with Node version 0.10 (#182, by @dougwilson) * Fix tests to accomodate fixes in Node v10 (#182, by @dougwilson) -# 0.4.21 / 2018-04-06 +## 0.4.21 / 2018-04-06 * Fix encoding canonicalization (#156) * Fix the paths in the "browser" field in package.json (#174 by @LMLB) * Removed "contributors" section in package.json - see Git history instead. -# 0.4.20 / 2018-04-06 +## 0.4.20 / 2018-04-06 * Updated `new Buffer()` usages with recommended replacements as it's being deprecated in Node v10 (#176, #178 by @ChALkeR) -# 0.4.19 / 2017-09-09 +## 0.4.19 / 2017-09-09 * Fixed iso8859-1 codec regression in handling untranslatable characters (#162, caused by #147) * Re-generated windows1255 codec, because it was updated in iconv project * Fixed grammar in error message when iconv-lite is loaded with encoding other than utf8 -# 0.4.18 / 2017-06-13 +## 0.4.18 / 2017-06-13 * Fixed CESU-8 regression in Node v8. -# 0.4.17 / 2017-04-22 +## 0.4.17 / 2017-04-22 * Updated typescript definition file to support Angular 2 AoT mode (#153 by @larssn) -# 0.4.16 / 2017-04-22 +## 0.4.16 / 2017-04-22 * Added support for React Native (#150) * Changed iso8859-1 encoding to usine internal 'binary' encoding, as it's the same thing (#147 by @mscdex) @@ -49,12 +99,12 @@ * Added a warning if iconv-lite is loaded not as utf-8 (see #142) -# 0.4.15 / 2016-11-21 +## 0.4.15 / 2016-11-21 * Fixed typescript type definition (#137) -# 0.4.14 / 2016-11-20 +## 0.4.14 / 2016-11-20 * Preparation for v1.0 * Added Node v6 and latest Node versions to Travis CI test rig @@ -64,12 +114,12 @@ * Add ms prefix to dbcs windows encodings (@rokoroku) -# 0.4.13 / 2015-10-01 +## 0.4.13 / 2015-10-01 * Fix silly mistake in deprecation notice. -# 0.4.12 / 2015-09-26 +## 0.4.12 / 2015-09-26 * Node v4 support: * Added CESU-8 decoding (#106) @@ -77,18 +127,18 @@ * Added Travis tests for Node v4 and io.js latest (#105 by @Mithgol) -# 0.4.11 / 2015-07-03 +## 0.4.11 / 2015-07-03 * Added CESU-8 encoding. -# 0.4.10 / 2015-05-26 +## 0.4.10 / 2015-05-26 * Changed UTF-16 endianness heuristic to take into account any ASCII chars, not just spaces. This should minimize the importance of "default" endianness. -# 0.4.9 / 2015-05-24 +## 0.4.9 / 2015-05-24 * Streamlined BOM handling: strip BOM by default, add BOM when encoding if addBOM: true. Added docs to Readme. @@ -100,12 +150,12 @@ * Use strict mode in all files. -# 0.4.8 / 2015-04-14 +## 0.4.8 / 2015-04-14 * added alias UNICODE-1-1-UTF-7 for UTF-7 encoding (#94) -# 0.4.7 / 2015-02-05 +## 0.4.7 / 2015-02-05 * stop official support of Node.js v0.8. Should still work, but no guarantees. reason: Packages needed for testing are hard to get on Travis CI. @@ -113,40 +163,40 @@ props (#89). -# 0.4.6 / 2015-01-12 +## 0.4.6 / 2015-01-12 * fix rare aliases of single-byte encodings (thanks @mscdex) * double the timeout for dbcs tests to make them less flaky on travis -# 0.4.5 / 2014-11-20 +## 0.4.5 / 2014-11-20 * fix windows-31j and x-sjis encoding support (@nleush) * minor fix: undefined variable reference when internal error happens -# 0.4.4 / 2014-07-16 +## 0.4.4 / 2014-07-16 * added encodings UTF-7 (RFC2152) and UTF-7-IMAP (RFC3501 Section 5.1.3) * fixed streaming base64 encoding -# 0.4.3 / 2014-06-14 +## 0.4.3 / 2014-06-14 * added encodings UTF-16BE and UTF-16 with BOM -# 0.4.2 / 2014-06-12 +## 0.4.2 / 2014-06-12 * don't throw exception if `extendNodeEncodings()` is called more than once -# 0.4.1 / 2014-06-11 +## 0.4.1 / 2014-06-11 * codepage 808 added -# 0.4.0 / 2014-06-10 +## 0.4.0 / 2014-06-10 * code is rewritten from scratch * all widespread encodings are supported diff --git a/node_modules/iconv-lite/README.md b/node_modules/iconv-lite/README.md index c981c3708..3c97f8730 100644 --- a/node_modules/iconv-lite/README.md +++ b/node_modules/iconv-lite/README.md @@ -1,38 +1,40 @@ -## Pure JS character encoding conversion [![Build Status](https://travis-ci.org/ashtuchkin/iconv-lite.svg?branch=master)](https://travis-ci.org/ashtuchkin/iconv-lite) +## iconv-lite: Pure JS character encoding conversion - * Doesn't need native code compilation. Works on Windows and in sandboxed environments like [Cloud9](http://c9.io). + * No need for native code compilation. Quick to install, works on Windows and in sandboxed environments like [Cloud9](http://c9.io). * Used in popular projects like [Express.js (body_parser)](https://github.com/expressjs/body-parser), [Grunt](http://gruntjs.com/), [Nodemailer](http://www.nodemailer.com/), [Yeoman](http://yeoman.io/) and others. * Faster than [node-iconv](https://github.com/bnoordhuis/node-iconv) (see below for performance comparison). - * Intuitive encode/decode API - * Streaming support for Node v0.10+ - * [Deprecated] Can extend Node.js primitives (buffers, streams) to support all iconv-lite encodings. - * In-browser usage via [Browserify](https://github.com/substack/node-browserify) (~180k gzip compressed with Buffer shim included). + * Intuitive encode/decode API, including Streaming support. + * In-browser usage via [browserify](https://github.com/substack/node-browserify) or [webpack](https://webpack.js.org/) (~180kb gzip compressed with Buffer shim included). * Typescript [type definition file](https://github.com/ashtuchkin/iconv-lite/blob/master/lib/index.d.ts) included. - * React Native is supported (need to explicitly `npm install` two more modules: `buffer` and `stream`). + * React Native is supported (need to install `stream` module to enable Streaming API). * License: MIT. -[![NPM Stats](https://nodei.co/npm/iconv-lite.png?downloads=true&downloadRank=true)](https://npmjs.org/packages/iconv-lite/) +[![NPM Stats](https://nodei.co/npm/iconv-lite.png)](https://npmjs.org/package/iconv-lite/) +[![Build Status](https://travis-ci.org/ashtuchkin/iconv-lite.svg?branch=master)](https://travis-ci.org/ashtuchkin/iconv-lite) +[![npm](https://img.shields.io/npm/v/iconv-lite.svg)](https://npmjs.org/package/iconv-lite/) +[![npm downloads](https://img.shields.io/npm/dm/iconv-lite.svg)](https://npmjs.org/package/iconv-lite/) +[![npm bundle size](https://img.shields.io/bundlephobia/min/iconv-lite.svg)](https://npmjs.org/package/iconv-lite/) ## Usage ### Basic API ```javascript var iconv = require('iconv-lite'); -// Convert from an encoded buffer to js string. +// Convert from an encoded buffer to a js string. str = iconv.decode(Buffer.from([0x68, 0x65, 0x6c, 0x6c, 0x6f]), 'win1251'); -// Convert from js string to an encoded buffer. +// Convert from a js string to an encoded buffer. buf = iconv.encode("Sample input string", 'win1251'); // Check if encoding is supported iconv.encodingExists("us-ascii") ``` -### Streaming API (Node v0.10+) +### Streaming API ```javascript -// Decode stream (from binary stream to js strings) +// Decode stream (from binary data stream to js strings) http.createServer(function(req, res) { var converterStream = iconv.decodeStream('win1251'); req.pipe(converterStream); @@ -57,44 +59,10 @@ http.createServer(function(req, res) { }); ``` -### [Deprecated] Extend Node.js own encodings -> NOTE: This doesn't work on latest Node versions. See [details](https://github.com/ashtuchkin/iconv-lite/wiki/Node-v4-compatibility). - -```javascript -// After this call all Node basic primitives will understand iconv-lite encodings. -iconv.extendNodeEncodings(); - -// Examples: -buf = new Buffer(str, 'win1251'); -buf.write(str, 'gbk'); -str = buf.toString('latin1'); -assert(Buffer.isEncoding('iso-8859-15')); -Buffer.byteLength(str, 'us-ascii'); - -http.createServer(function(req, res) { - req.setEncoding('big5'); - req.collect(function(err, body) { - console.log(body); - }); -}); - -fs.createReadStream("file.txt", "shift_jis"); - -// External modules are also supported (if they use Node primitives, which they probably do). -request = require('request'); -request({ - url: "http://github.com/", - encoding: "cp932" -}); - -// To remove extensions -iconv.undoExtendNodeEncodings(); -``` - ## Supported encodings * All node.js native encodings: utf8, ucs2 / utf16-le, ascii, binary, base64, hex. - * Additional unicode encodings: utf16, utf16-be, utf-7, utf-7-imap. + * Additional unicode encodings: utf16, utf16-be, utf-7, utf-7-imap, utf32, utf32-le, and utf32-be. * All widespread singlebyte encodings: Windows 125x family, ISO-8859 family, IBM/DOS codepages, Macintosh family, KOI8 family, all others supported by iconv library. Aliases like 'latin1', 'us-ascii' also supported. @@ -133,6 +101,12 @@ smart about endianness in the following ways: overridden with `defaultEncoding: 'utf-16be'` option. Strips BOM unless `stripBOM: false`. * Encoding: uses UTF-16LE and writes BOM by default. Use `addBOM: false` to override. +## UTF-32 Encodings + +This library supports UTF-32LE, UTF-32BE and UTF-32 encodings. Like the UTF-16 encoding above, UTF-32 defaults to UTF-32LE, but uses BOM and 'spaces heuristics' to determine input endianness. + * The default of UTF-32LE can be overridden with the `defaultEncoding: 'utf-32be'` option. Strips BOM unless `stripBOM: false`. + * Encoding: uses UTF-32LE and writes BOM by default. Use `addBOM: false` to override. (`defaultEncoding: 'utf-32be'` can also be used here to change encoding.) + ## Other notes When decoding, be sure to supply a Buffer to decode() method, otherwise [bad things usually happen](https://github.com/ashtuchkin/iconv-lite/wiki/Use-Buffers-when-decoding). diff --git a/node_modules/iconv-lite/encodings/dbcs-codec.js b/node_modules/iconv-lite/encodings/dbcs-codec.js index 1fe3e1601..fa8391703 100644 --- a/node_modules/iconv-lite/encodings/dbcs-codec.js +++ b/node_modules/iconv-lite/encodings/dbcs-codec.js @@ -49,6 +49,48 @@ function DBCSCodec(codecOptions, iconv) { for (var i = 0; i < mappingTable.length; i++) this._addDecodeChunk(mappingTable[i]); + // Load & create GB18030 tables when needed. + if (typeof codecOptions.gb18030 === 'function') { + this.gb18030 = codecOptions.gb18030(); // Load GB18030 ranges. + + // Add GB18030 common decode nodes. + var commonThirdByteNodeIdx = this.decodeTables.length; + this.decodeTables.push(UNASSIGNED_NODE.slice(0)); + + var commonFourthByteNodeIdx = this.decodeTables.length; + this.decodeTables.push(UNASSIGNED_NODE.slice(0)); + + // Fill out the tree + var firstByteNode = this.decodeTables[0]; + for (var i = 0x81; i <= 0xFE; i++) { + var secondByteNode = this.decodeTables[NODE_START - firstByteNode[i]]; + for (var j = 0x30; j <= 0x39; j++) { + if (secondByteNode[j] === UNASSIGNED) { + secondByteNode[j] = NODE_START - commonThirdByteNodeIdx; + } else if (secondByteNode[j] > NODE_START) { + throw new Error("gb18030 decode tables conflict at byte 2"); + } + + var thirdByteNode = this.decodeTables[NODE_START - secondByteNode[j]]; + for (var k = 0x81; k <= 0xFE; k++) { + if (thirdByteNode[k] === UNASSIGNED) { + thirdByteNode[k] = NODE_START - commonFourthByteNodeIdx; + } else if (thirdByteNode[k] === NODE_START - commonFourthByteNodeIdx) { + continue; + } else if (thirdByteNode[k] > NODE_START) { + throw new Error("gb18030 decode tables conflict at byte 3"); + } + + var fourthByteNode = this.decodeTables[NODE_START - thirdByteNode[k]]; + for (var l = 0x30; l <= 0x39; l++) { + if (fourthByteNode[l] === UNASSIGNED) + fourthByteNode[l] = GB18030_CODE; + } + } + } + } + } + this.defaultCharUnicode = iconv.defaultCharUnicode; @@ -92,30 +134,6 @@ function DBCSCodec(codecOptions, iconv) { this.defCharSB = this.encodeTable[0][iconv.defaultCharSingleByte.charCodeAt(0)]; if (this.defCharSB === UNASSIGNED) this.defCharSB = this.encodeTable[0]['?']; if (this.defCharSB === UNASSIGNED) this.defCharSB = "?".charCodeAt(0); - - - // Load & create GB18030 tables when needed. - if (typeof codecOptions.gb18030 === 'function') { - this.gb18030 = codecOptions.gb18030(); // Load GB18030 ranges. - - // Add GB18030 decode tables. - var thirdByteNodeIdx = this.decodeTables.length; - var thirdByteNode = this.decodeTables[thirdByteNodeIdx] = UNASSIGNED_NODE.slice(0); - - var fourthByteNodeIdx = this.decodeTables.length; - var fourthByteNode = this.decodeTables[fourthByteNodeIdx] = UNASSIGNED_NODE.slice(0); - - for (var i = 0x81; i <= 0xFE; i++) { - var secondByteNodeIdx = NODE_START - this.decodeTables[0][i]; - var secondByteNode = this.decodeTables[secondByteNodeIdx]; - for (var j = 0x30; j <= 0x39; j++) - secondByteNode[j] = NODE_START - thirdByteNodeIdx; - } - for (var i = 0x81; i <= 0xFE; i++) - thirdByteNode[i] = NODE_START - fourthByteNodeIdx; - for (var i = 0x30; i <= 0x39; i++) - fourthByteNode[i] = GB18030_CODE - } } DBCSCodec.prototype.encoder = DBCSEncoder; @@ -124,7 +142,7 @@ DBCSCodec.prototype.decoder = DBCSDecoder; // Decoder helpers DBCSCodec.prototype._getDecodeTrieNode = function(addr) { var bytes = []; - for (; addr > 0; addr >>= 8) + for (; addr > 0; addr >>>= 8) bytes.push(addr & 0xFF); if (bytes.length == 0) bytes.push(0); @@ -249,19 +267,32 @@ DBCSCodec.prototype._setEncodeSequence = function(seq, dbcsCode) { DBCSCodec.prototype._fillEncodeTable = function(nodeIdx, prefix, skipEncodeChars) { var node = this.decodeTables[nodeIdx]; + var hasValues = false; + var subNodeEmpty = {}; for (var i = 0; i < 0x100; i++) { var uCode = node[i]; var mbCode = prefix + i; if (skipEncodeChars[mbCode]) continue; - if (uCode >= 0) + if (uCode >= 0) { this._setEncodeChar(uCode, mbCode); - else if (uCode <= NODE_START) - this._fillEncodeTable(NODE_START - uCode, mbCode << 8, skipEncodeChars); - else if (uCode <= SEQ_START) + hasValues = true; + } else if (uCode <= NODE_START) { + var subNodeIdx = NODE_START - uCode; + if (!subNodeEmpty[subNodeIdx]) { // Skip empty subtrees (they are too large in gb18030). + var newPrefix = (mbCode << 8) >>> 0; // NOTE: '>>> 0' keeps 32-bit num positive. + if (this._fillEncodeTable(subNodeIdx, newPrefix, skipEncodeChars)) + hasValues = true; + else + subNodeEmpty[subNodeIdx] = true; + } + } else if (uCode <= SEQ_START) { this._setEncodeSequence(this.decodeTableSeq[SEQ_START - uCode], mbCode); + hasValues = true; + } } + return hasValues; } @@ -388,10 +419,15 @@ DBCSEncoder.prototype.write = function(str) { newBuf[j++] = dbcsCode >> 8; // high byte newBuf[j++] = dbcsCode & 0xFF; // low byte } - else { + else if (dbcsCode < 0x1000000) { newBuf[j++] = dbcsCode >> 16; newBuf[j++] = (dbcsCode >> 8) & 0xFF; newBuf[j++] = dbcsCode & 0xFF; + } else { + newBuf[j++] = dbcsCode >>> 24; + newBuf[j++] = (dbcsCode >>> 16) & 0xFF; + newBuf[j++] = (dbcsCode >>> 8) & 0xFF; + newBuf[j++] = dbcsCode & 0xFF; } } @@ -440,7 +476,7 @@ DBCSEncoder.prototype.findIdx = findIdx; function DBCSDecoder(options, codec) { // Decoder state this.nodeIdx = 0; - this.prevBuf = Buffer.alloc(0); + this.prevBytes = []; // Static data this.decodeTables = codec.decodeTables; @@ -452,15 +488,12 @@ function DBCSDecoder(options, codec) { DBCSDecoder.prototype.write = function(buf) { var newBuf = Buffer.alloc(buf.length*2), nodeIdx = this.nodeIdx, - prevBuf = this.prevBuf, prevBufOffset = this.prevBuf.length, - seqStart = -this.prevBuf.length, // idx of the start of current parsed sequence. + prevBytes = this.prevBytes, prevOffset = this.prevBytes.length, + seqStart = -this.prevBytes.length, // idx of the start of current parsed sequence. uCode; - if (prevBufOffset > 0) // Make prev buf overlap a little to make it easier to slice later. - prevBuf = Buffer.concat([prevBuf, buf.slice(0, 10)]); - for (var i = 0, j = 0; i < buf.length; i++) { - var curByte = (i >= 0) ? buf[i] : prevBuf[i + prevBufOffset]; + var curByte = (i >= 0) ? buf[i] : prevBytes[i + prevOffset]; // Lookup in current trie node. var uCode = this.decodeTables[nodeIdx][curByte]; @@ -470,13 +503,18 @@ DBCSDecoder.prototype.write = function(buf) { } else if (uCode === UNASSIGNED) { // Unknown char. // TODO: Callback with seq. - //var curSeq = (seqStart >= 0) ? buf.slice(seqStart, i+1) : prevBuf.slice(seqStart + prevBufOffset, i+1 + prevBufOffset); - i = seqStart; // Try to parse again, after skipping first byte of the sequence ('i' will be incremented by 'for' cycle). uCode = this.defaultCharUnicode.charCodeAt(0); + i = seqStart; // Skip one byte ('i' will be incremented by the for loop) and try to parse again. } else if (uCode === GB18030_CODE) { - var curSeq = (seqStart >= 0) ? buf.slice(seqStart, i+1) : prevBuf.slice(seqStart + prevBufOffset, i+1 + prevBufOffset); - var ptr = (curSeq[0]-0x81)*12600 + (curSeq[1]-0x30)*1260 + (curSeq[2]-0x81)*10 + (curSeq[3]-0x30); + if (i >= 3) { + var ptr = (buf[i-3]-0x81)*12600 + (buf[i-2]-0x30)*1260 + (buf[i-1]-0x81)*10 + (curByte-0x30); + } else { + var ptr = (prevBytes[i-3+prevOffset]-0x81)*12600 + + (((i-2 >= 0) ? buf[i-2] : prevBytes[i-2+prevOffset])-0x30)*1260 + + (((i-1 >= 0) ? buf[i-1] : prevBytes[i-1+prevOffset])-0x81)*10 + + (curByte-0x30); + } var idx = findIdx(this.gb18030.gbChars, ptr); uCode = this.gb18030.uChars[idx] + ptr - this.gb18030.gbChars[idx]; } @@ -497,13 +535,13 @@ DBCSDecoder.prototype.write = function(buf) { throw new Error("iconv-lite internal error: invalid decoding table value " + uCode + " at " + nodeIdx + "/" + curByte); // Write the character to buffer, handling higher planes using surrogate pair. - if (uCode > 0xFFFF) { + if (uCode >= 0x10000) { uCode -= 0x10000; - var uCodeLead = 0xD800 + Math.floor(uCode / 0x400); + var uCodeLead = 0xD800 | (uCode >> 10); newBuf[j++] = uCodeLead & 0xFF; newBuf[j++] = uCodeLead >> 8; - uCode = 0xDC00 + uCode % 0x400; + uCode = 0xDC00 | (uCode & 0x3FF); } newBuf[j++] = uCode & 0xFF; newBuf[j++] = uCode >> 8; @@ -513,7 +551,10 @@ DBCSDecoder.prototype.write = function(buf) { } this.nodeIdx = nodeIdx; - this.prevBuf = (seqStart >= 0) ? buf.slice(seqStart) : prevBuf.slice(seqStart + prevBufOffset); + this.prevBytes = (seqStart >= 0) + ? Array.prototype.slice.call(buf, seqStart) + : prevBytes.slice(seqStart + prevOffset).concat(Array.prototype.slice.call(buf)); + return newBuf.slice(0, j).toString('ucs2'); } @@ -521,18 +562,19 @@ DBCSDecoder.prototype.end = function() { var ret = ''; // Try to parse all remaining chars. - while (this.prevBuf.length > 0) { + while (this.prevBytes.length > 0) { // Skip 1 character in the buffer. ret += this.defaultCharUnicode; - var buf = this.prevBuf.slice(1); + var bytesArr = this.prevBytes.slice(1); // Parse remaining as usual. - this.prevBuf = Buffer.alloc(0); + this.prevBytes = []; this.nodeIdx = 0; - if (buf.length > 0) - ret += this.write(buf); + if (bytesArr.length > 0) + ret += this.write(bytesArr); } + this.prevBytes = []; this.nodeIdx = 0; return ret; } @@ -544,7 +586,7 @@ function findIdx(table, val) { var l = 0, r = table.length; while (l < r-1) { // always table[l] <= val < table[r] - var mid = l + Math.floor((r-l+1)/2); + var mid = l + ((r-l+1) >> 1); if (table[mid] <= val) l = mid; else diff --git a/node_modules/iconv-lite/encodings/index.js b/node_modules/iconv-lite/encodings/index.js index e30400317..d95c24411 100644 --- a/node_modules/iconv-lite/encodings/index.js +++ b/node_modules/iconv-lite/encodings/index.js @@ -4,6 +4,7 @@ // We support Browserify by skipping automatic module discovery and requiring modules directly. var modules = [ require("./internal"), + require("./utf32"), require("./utf16"), require("./utf7"), require("./sbcs-codec"), @@ -13,7 +14,7 @@ var modules = [ require("./dbcs-data"), ]; -// Put all encoding/alias/codec definitions to single object and export it. +// Put all encoding/alias/codec definitions to single object and export it. for (var i = 0; i < modules.length; i++) { var module = modules[i]; for (var enc in module) diff --git a/node_modules/iconv-lite/encodings/internal.js b/node_modules/iconv-lite/encodings/internal.js index 05ce38b27..dc1074f04 100644 --- a/node_modules/iconv-lite/encodings/internal.js +++ b/node_modules/iconv-lite/encodings/internal.js @@ -53,10 +53,20 @@ if (!StringDecoder.prototype.end) // Node v0.8 doesn't have this method. function InternalDecoder(options, codec) { - StringDecoder.call(this, codec.enc); + this.decoder = new StringDecoder(codec.enc); } -InternalDecoder.prototype = StringDecoder.prototype; +InternalDecoder.prototype.write = function(buf) { + if (!Buffer.isBuffer(buf)) { + buf = Buffer.from(buf); + } + + return this.decoder.write(buf); +} + +InternalDecoder.prototype.end = function() { + return this.decoder.end(); +} //------------------------------------------------------------------------------ diff --git a/node_modules/iconv-lite/encodings/sbcs-codec.js b/node_modules/iconv-lite/encodings/sbcs-codec.js index f2258237b..abac5ffaa 100644 --- a/node_modules/iconv-lite/encodings/sbcs-codec.js +++ b/node_modules/iconv-lite/encodings/sbcs-codec.js @@ -20,10 +20,10 @@ function SBCSCodec(codecOptions, iconv) { codecOptions.chars = asciiString + codecOptions.chars; } - this.decodeBuf = new Buffer.from(codecOptions.chars, 'ucs2'); + this.decodeBuf = Buffer.from(codecOptions.chars, 'ucs2'); // Encoding buffer. - var encodeBuf = new Buffer.alloc(65536, iconv.defaultCharSingleByte.charCodeAt(0)); + var encodeBuf = Buffer.alloc(65536, iconv.defaultCharSingleByte.charCodeAt(0)); for (var i = 0; i < codecOptions.chars.length; i++) encodeBuf[codecOptions.chars.charCodeAt(i)] = i; diff --git a/node_modules/iconv-lite/encodings/sbcs-data.js b/node_modules/iconv-lite/encodings/sbcs-data.js index 2d6f846ad..066f904e5 100644 --- a/node_modules/iconv-lite/encodings/sbcs-data.js +++ b/node_modules/iconv-lite/encodings/sbcs-data.js @@ -17,6 +17,16 @@ module.exports = { "chars": "АБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯабвгдежзийклмноп░▒▓│┤╡╢╖╕╣║╗╝╜╛┐└┴┬├─┼╞╟╚╔╩╦╠═╬╧╨╤╥╙╘╒╓╫╪┘┌█▄▌▐▀рстуфхцчшщъыьэюяЁёЄєЇїЎў°∙·√№€■ " }, + "mik": { + "type": "_sbcs", + "chars": "АБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯабвгдежзийклмнопрстуфхцчшщъыьэюя└┴┬├─┼╣║╚╔╩╦╠═╬┐░▒▓│┤№§╗╝┘┌█▄▌▐▀αßΓπΣσµτΦΘΩδ∞φε∩≡±≥≤⌠⌡÷≈°∙·√ⁿ²■ " + }, + + "cp720": { + "type": "_sbcs", + "chars": "\x80\x81éâ\x84à\x86çêëèïî\x8d\x8e\x8f\x90\u0651\u0652ô¤ـûùءآأؤ£إئابةتثجحخدذرزسشص«»░▒▓│┤╡╢╖╕╣║╗╝╜╛┐└┴┬├─┼╞╟╚╔╩╦╠═╬╧╨╤╥╙╘╒╓╫╪┘┌█▄▌▐▀ضطظعغفµقكلمنهوىي≡\u064b\u064c\u064d\u064e\u064f\u0650≈°∙·√ⁿ²■\u00a0" + }, + // Aliases of generated encodings. "ascii8bit": "ascii", "usascii": "ascii", diff --git a/node_modules/iconv-lite/encodings/tables/gbk-added.json b/node_modules/iconv-lite/encodings/tables/gbk-added.json index 8abfa9f7b..b742e368f 100644 --- a/node_modules/iconv-lite/encodings/tables/gbk-added.json +++ b/node_modules/iconv-lite/encodings/tables/gbk-added.json @@ -27,7 +27,7 @@ ["a7c2","",14], ["a7f2","",12], ["a896","",10], -["a8bc",""], +["a8bc","ḿ"], ["a8bf","ǹ"], ["a8c1",""], ["a8ea","",20], @@ -51,5 +51,6 @@ ["fca1","",93], ["fda1","",93], ["fe50","⺁⺄㑳㑇⺈⺋㖞㘚㘎⺌⺗㥮㤘㧏㧟㩳㧐㭎㱮㳠⺧⺪䁖䅟⺮䌷⺳⺶⺷䎱䎬⺻䏝䓖䙡䙌"], -["fe80","䜣䜩䝼䞍⻊䥇䥺䥽䦂䦃䦅䦆䦟䦛䦷䦶䲣䲟䲠䲡䱷䲢䴓",6,"䶮",93] +["fe80","䜣䜩䝼䞍⻊䥇䥺䥽䦂䦃䦅䦆䦟䦛䦷䦶䲣䲟䲠䲡䱷䲢䴓",6,"䶮",93], +["8135f437",""] ] diff --git a/node_modules/iconv-lite/encodings/utf16.js b/node_modules/iconv-lite/encodings/utf16.js index 54765aeee..97d066925 100644 --- a/node_modules/iconv-lite/encodings/utf16.js +++ b/node_modules/iconv-lite/encodings/utf16.js @@ -61,6 +61,7 @@ Utf16BEDecoder.prototype.write = function(buf) { } Utf16BEDecoder.prototype.end = function() { + this.overflowByte = -1; } @@ -103,8 +104,8 @@ Utf16Encoder.prototype.end = function() { function Utf16Decoder(options, codec) { this.decoder = null; - this.initialBytes = []; - this.initialBytesLen = 0; + this.initialBufs = []; + this.initialBufsLen = 0; this.options = options || {}; this.iconv = codec.iconv; @@ -113,17 +114,22 @@ function Utf16Decoder(options, codec) { Utf16Decoder.prototype.write = function(buf) { if (!this.decoder) { // Codec is not chosen yet. Accumulate initial bytes. - this.initialBytes.push(buf); - this.initialBytesLen += buf.length; + this.initialBufs.push(buf); + this.initialBufsLen += buf.length; - if (this.initialBytesLen < 16) // We need more bytes to use space heuristic (see below) + if (this.initialBufsLen < 16) // We need more bytes to use space heuristic (see below) return ''; // We have enough bytes -> detect endianness. - var buf = Buffer.concat(this.initialBytes), - encoding = detectEncoding(buf, this.options.defaultEncoding); + var encoding = detectEncoding(this.initialBufs, this.options.defaultEncoding); this.decoder = this.iconv.getDecoder(encoding, this.options); - this.initialBytes.length = this.initialBytesLen = 0; + + var resStr = ''; + for (var i = 0; i < this.initialBufs.length; i++) + resStr += this.decoder.write(this.initialBufs[i]); + + this.initialBufs.length = this.initialBufsLen = 0; + return resStr; } return this.decoder.write(buf); @@ -131,47 +137,61 @@ Utf16Decoder.prototype.write = function(buf) { Utf16Decoder.prototype.end = function() { if (!this.decoder) { - var buf = Buffer.concat(this.initialBytes), - encoding = detectEncoding(buf, this.options.defaultEncoding); + var encoding = detectEncoding(this.initialBufs, this.options.defaultEncoding); this.decoder = this.iconv.getDecoder(encoding, this.options); - var res = this.decoder.write(buf), - trail = this.decoder.end(); + var resStr = ''; + for (var i = 0; i < this.initialBufs.length; i++) + resStr += this.decoder.write(this.initialBufs[i]); - return trail ? (res + trail) : res; + var trail = this.decoder.end(); + if (trail) + resStr += trail; + + this.initialBufs.length = this.initialBufsLen = 0; + return resStr; } return this.decoder.end(); } -function detectEncoding(buf, defaultEncoding) { - var enc = defaultEncoding || 'utf-16le'; - - if (buf.length >= 2) { - // Check BOM. - if (buf[0] == 0xFE && buf[1] == 0xFF) // UTF-16BE BOM - enc = 'utf-16be'; - else if (buf[0] == 0xFF && buf[1] == 0xFE) // UTF-16LE BOM - enc = 'utf-16le'; - else { - // No BOM found. Try to deduce encoding from initial content. - // Most of the time, the content has ASCII chars (U+00**), but the opposite (U+**00) is uncommon. - // So, we count ASCII as if it was LE or BE, and decide from that. - var asciiCharsLE = 0, asciiCharsBE = 0, // Counts of chars in both positions - _len = Math.min(buf.length - (buf.length % 2), 64); // Len is always even. - - for (var i = 0; i < _len; i += 2) { - if (buf[i] === 0 && buf[i+1] !== 0) asciiCharsBE++; - if (buf[i] !== 0 && buf[i+1] === 0) asciiCharsLE++; +function detectEncoding(bufs, defaultEncoding) { + var b = []; + var charsProcessed = 0; + var asciiCharsLE = 0, asciiCharsBE = 0; // Number of ASCII chars when decoded as LE or BE. + + outer_loop: + for (var i = 0; i < bufs.length; i++) { + var buf = bufs[i]; + for (var j = 0; j < buf.length; j++) { + b.push(buf[j]); + if (b.length === 2) { + if (charsProcessed === 0) { + // Check BOM first. + if (b[0] === 0xFF && b[1] === 0xFE) return 'utf-16le'; + if (b[0] === 0xFE && b[1] === 0xFF) return 'utf-16be'; + } + + if (b[0] === 0 && b[1] !== 0) asciiCharsBE++; + if (b[0] !== 0 && b[1] === 0) asciiCharsLE++; + + b.length = 0; + charsProcessed++; + + if (charsProcessed >= 100) { + break outer_loop; + } } - - if (asciiCharsBE > asciiCharsLE) - enc = 'utf-16be'; - else if (asciiCharsBE < asciiCharsLE) - enc = 'utf-16le'; } } - return enc; + // Make decisions. + // Most of the time, the content has ASCII chars (U+00**), but the opposite (U+**00) is uncommon. + // So, we count ASCII as if it was LE or BE, and decide from that. + if (asciiCharsBE > asciiCharsLE) return 'utf-16be'; + if (asciiCharsBE < asciiCharsLE) return 'utf-16le'; + + // Couldn't decide (likely all zeros or not enough data). + return defaultEncoding || 'utf-16le'; } diff --git a/node_modules/iconv-lite/encodings/utf32.js b/node_modules/iconv-lite/encodings/utf32.js new file mode 100644 index 000000000..2fa900a12 --- /dev/null +++ b/node_modules/iconv-lite/encodings/utf32.js @@ -0,0 +1,319 @@ +'use strict'; + +var Buffer = require('safer-buffer').Buffer; + +// == UTF32-LE/BE codec. ========================================================== + +exports._utf32 = Utf32Codec; + +function Utf32Codec(codecOptions, iconv) { + this.iconv = iconv; + this.bomAware = true; + this.isLE = codecOptions.isLE; +} + +exports.utf32le = { type: '_utf32', isLE: true }; +exports.utf32be = { type: '_utf32', isLE: false }; + +// Aliases +exports.ucs4le = 'utf32le'; +exports.ucs4be = 'utf32be'; + +Utf32Codec.prototype.encoder = Utf32Encoder; +Utf32Codec.prototype.decoder = Utf32Decoder; + +// -- Encoding + +function Utf32Encoder(options, codec) { + this.isLE = codec.isLE; + this.highSurrogate = 0; +} + +Utf32Encoder.prototype.write = function(str) { + var src = Buffer.from(str, 'ucs2'); + var dst = Buffer.alloc(src.length * 2); + var write32 = this.isLE ? dst.writeUInt32LE : dst.writeUInt32BE; + var offset = 0; + + for (var i = 0; i < src.length; i += 2) { + var code = src.readUInt16LE(i); + var isHighSurrogate = (0xD800 <= code && code < 0xDC00); + var isLowSurrogate = (0xDC00 <= code && code < 0xE000); + + if (this.highSurrogate) { + if (isHighSurrogate || !isLowSurrogate) { + // There shouldn't be two high surrogates in a row, nor a high surrogate which isn't followed by a low + // surrogate. If this happens, keep the pending high surrogate as a stand-alone semi-invalid character + // (technically wrong, but expected by some applications, like Windows file names). + write32.call(dst, this.highSurrogate, offset); + offset += 4; + } + else { + // Create 32-bit value from high and low surrogates; + var codepoint = (((this.highSurrogate - 0xD800) << 10) | (code - 0xDC00)) + 0x10000; + + write32.call(dst, codepoint, offset); + offset += 4; + this.highSurrogate = 0; + + continue; + } + } + + if (isHighSurrogate) + this.highSurrogate = code; + else { + // Even if the current character is a low surrogate, with no previous high surrogate, we'll + // encode it as a semi-invalid stand-alone character for the same reasons expressed above for + // unpaired high surrogates. + write32.call(dst, code, offset); + offset += 4; + this.highSurrogate = 0; + } + } + + if (offset < dst.length) + dst = dst.slice(0, offset); + + return dst; +}; + +Utf32Encoder.prototype.end = function() { + // Treat any leftover high surrogate as a semi-valid independent character. + if (!this.highSurrogate) + return; + + var buf = Buffer.alloc(4); + + if (this.isLE) + buf.writeUInt32LE(this.highSurrogate, 0); + else + buf.writeUInt32BE(this.highSurrogate, 0); + + this.highSurrogate = 0; + + return buf; +}; + +// -- Decoding + +function Utf32Decoder(options, codec) { + this.isLE = codec.isLE; + this.badChar = codec.iconv.defaultCharUnicode.charCodeAt(0); + this.overflow = []; +} + +Utf32Decoder.prototype.write = function(src) { + if (src.length === 0) + return ''; + + var i = 0; + var codepoint = 0; + var dst = Buffer.alloc(src.length + 4); + var offset = 0; + var isLE = this.isLE; + var overflow = this.overflow; + var badChar = this.badChar; + + if (overflow.length > 0) { + for (; i < src.length && overflow.length < 4; i++) + overflow.push(src[i]); + + if (overflow.length === 4) { + // NOTE: codepoint is a signed int32 and can be negative. + // NOTE: We copied this block from below to help V8 optimize it (it works with array, not buffer). + if (isLE) { + codepoint = overflow[i] | (overflow[i+1] << 8) | (overflow[i+2] << 16) | (overflow[i+3] << 24); + } else { + codepoint = overflow[i+3] | (overflow[i+2] << 8) | (overflow[i+1] << 16) | (overflow[i] << 24); + } + overflow.length = 0; + + offset = _writeCodepoint(dst, offset, codepoint, badChar); + } + } + + // Main loop. Should be as optimized as possible. + for (; i < src.length - 3; i += 4) { + // NOTE: codepoint is a signed int32 and can be negative. + if (isLE) { + codepoint = src[i] | (src[i+1] << 8) | (src[i+2] << 16) | (src[i+3] << 24); + } else { + codepoint = src[i+3] | (src[i+2] << 8) | (src[i+1] << 16) | (src[i] << 24); + } + offset = _writeCodepoint(dst, offset, codepoint, badChar); + } + + // Keep overflowing bytes. + for (; i < src.length; i++) { + overflow.push(src[i]); + } + + return dst.slice(0, offset).toString('ucs2'); +}; + +function _writeCodepoint(dst, offset, codepoint, badChar) { + // NOTE: codepoint is signed int32 and can be negative. We keep it that way to help V8 with optimizations. + if (codepoint < 0 || codepoint > 0x10FFFF) { + // Not a valid Unicode codepoint + codepoint = badChar; + } + + // Ephemeral Planes: Write high surrogate. + if (codepoint >= 0x10000) { + codepoint -= 0x10000; + + var high = 0xD800 | (codepoint >> 10); + dst[offset++] = high & 0xff; + dst[offset++] = high >> 8; + + // Low surrogate is written below. + var codepoint = 0xDC00 | (codepoint & 0x3FF); + } + + // Write BMP char or low surrogate. + dst[offset++] = codepoint & 0xff; + dst[offset++] = codepoint >> 8; + + return offset; +}; + +Utf32Decoder.prototype.end = function() { + this.overflow.length = 0; +}; + +// == UTF-32 Auto codec ============================================================= +// Decoder chooses automatically from UTF-32LE and UTF-32BE using BOM and space-based heuristic. +// Defaults to UTF-32LE. http://en.wikipedia.org/wiki/UTF-32 +// Encoder/decoder default can be changed: iconv.decode(buf, 'utf32', {defaultEncoding: 'utf-32be'}); + +// Encoder prepends BOM (which can be overridden with (addBOM: false}). + +exports.utf32 = Utf32AutoCodec; +exports.ucs4 = 'utf32'; + +function Utf32AutoCodec(options, iconv) { + this.iconv = iconv; +} + +Utf32AutoCodec.prototype.encoder = Utf32AutoEncoder; +Utf32AutoCodec.prototype.decoder = Utf32AutoDecoder; + +// -- Encoding + +function Utf32AutoEncoder(options, codec) { + options = options || {}; + + if (options.addBOM === undefined) + options.addBOM = true; + + this.encoder = codec.iconv.getEncoder(options.defaultEncoding || 'utf-32le', options); +} + +Utf32AutoEncoder.prototype.write = function(str) { + return this.encoder.write(str); +}; + +Utf32AutoEncoder.prototype.end = function() { + return this.encoder.end(); +}; + +// -- Decoding + +function Utf32AutoDecoder(options, codec) { + this.decoder = null; + this.initialBufs = []; + this.initialBufsLen = 0; + this.options = options || {}; + this.iconv = codec.iconv; +} + +Utf32AutoDecoder.prototype.write = function(buf) { + if (!this.decoder) { + // Codec is not chosen yet. Accumulate initial bytes. + this.initialBufs.push(buf); + this.initialBufsLen += buf.length; + + if (this.initialBufsLen < 32) // We need more bytes to use space heuristic (see below) + return ''; + + // We have enough bytes -> detect endianness. + var encoding = detectEncoding(this.initialBufs, this.options.defaultEncoding); + this.decoder = this.iconv.getDecoder(encoding, this.options); + + var resStr = ''; + for (var i = 0; i < this.initialBufs.length; i++) + resStr += this.decoder.write(this.initialBufs[i]); + + this.initialBufs.length = this.initialBufsLen = 0; + return resStr; + } + + return this.decoder.write(buf); +}; + +Utf32AutoDecoder.prototype.end = function() { + if (!this.decoder) { + var encoding = detectEncoding(this.initialBufs, this.options.defaultEncoding); + this.decoder = this.iconv.getDecoder(encoding, this.options); + + var resStr = ''; + for (var i = 0; i < this.initialBufs.length; i++) + resStr += this.decoder.write(this.initialBufs[i]); + + var trail = this.decoder.end(); + if (trail) + resStr += trail; + + this.initialBufs.length = this.initialBufsLen = 0; + return resStr; + } + + return this.decoder.end(); +}; + +function detectEncoding(bufs, defaultEncoding) { + var b = []; + var charsProcessed = 0; + var invalidLE = 0, invalidBE = 0; // Number of invalid chars when decoded as LE or BE. + var bmpCharsLE = 0, bmpCharsBE = 0; // Number of BMP chars when decoded as LE or BE. + + outer_loop: + for (var i = 0; i < bufs.length; i++) { + var buf = bufs[i]; + for (var j = 0; j < buf.length; j++) { + b.push(buf[j]); + if (b.length === 4) { + if (charsProcessed === 0) { + // Check BOM first. + if (b[0] === 0xFF && b[1] === 0xFE && b[2] === 0 && b[3] === 0) { + return 'utf-32le'; + } + if (b[0] === 0 && b[1] === 0 && b[2] === 0xFE && b[3] === 0xFF) { + return 'utf-32be'; + } + } + + if (b[0] !== 0 || b[1] > 0x10) invalidBE++; + if (b[3] !== 0 || b[2] > 0x10) invalidLE++; + + if (b[0] === 0 && b[1] === 0 && (b[2] !== 0 || b[3] !== 0)) bmpCharsBE++; + if ((b[0] !== 0 || b[1] !== 0) && b[2] === 0 && b[3] === 0) bmpCharsLE++; + + b.length = 0; + charsProcessed++; + + if (charsProcessed >= 100) { + break outer_loop; + } + } + } + } + + // Make decisions. + if (bmpCharsBE - invalidBE > bmpCharsLE - invalidLE) return 'utf-32be'; + if (bmpCharsBE - invalidBE < bmpCharsLE - invalidLE) return 'utf-32le'; + + // Couldn't decide (likely all zeros or not enough data). + return defaultEncoding || 'utf-32le'; +} diff --git a/node_modules/iconv-lite/encodings/utf7.js b/node_modules/iconv-lite/encodings/utf7.js index b7631c23a..eacae34d5 100644 --- a/node_modules/iconv-lite/encodings/utf7.js +++ b/node_modules/iconv-lite/encodings/utf7.js @@ -74,7 +74,7 @@ Utf7Decoder.prototype.write = function(buf) { if (i == lastI && buf[i] == minusChar) {// "+-" -> "+" res += "+"; } else { - var b64str = base64Accum + buf.slice(lastI, i).toString(); + var b64str = base64Accum + this.iconv.decode(buf.slice(lastI, i), "ascii"); res += this.iconv.decode(Buffer.from(b64str, 'base64'), "utf16-be"); } @@ -91,7 +91,7 @@ Utf7Decoder.prototype.write = function(buf) { if (!inBase64) { res += this.iconv.decode(buf.slice(lastI), "ascii"); // Write direct chars. } else { - var b64str = base64Accum + buf.slice(lastI).toString(); + var b64str = base64Accum + this.iconv.decode(buf.slice(lastI), "ascii"); var canBeDecoded = b64str.length - (b64str.length % 8); // Minimal chunk: 2 quads -> 2x3 bytes -> 3 chars. base64Accum = b64str.slice(canBeDecoded); // The rest will be decoded in future. @@ -245,7 +245,7 @@ Utf7IMAPDecoder.prototype.write = function(buf) { if (i == lastI && buf[i] == minusChar) { // "&-" -> "&" res += "&"; } else { - var b64str = base64Accum + buf.slice(lastI, i).toString().replace(/,/g, '/'); + var b64str = base64Accum + this.iconv.decode(buf.slice(lastI, i), "ascii").replace(/,/g, '/'); res += this.iconv.decode(Buffer.from(b64str, 'base64'), "utf16-be"); } @@ -262,7 +262,7 @@ Utf7IMAPDecoder.prototype.write = function(buf) { if (!inBase64) { res += this.iconv.decode(buf.slice(lastI), "ascii"); // Write direct chars. } else { - var b64str = base64Accum + buf.slice(lastI).toString().replace(/,/g, '/'); + var b64str = base64Accum + this.iconv.decode(buf.slice(lastI), "ascii").replace(/,/g, '/'); var canBeDecoded = b64str.length - (b64str.length % 8); // Minimal chunk: 2 quads -> 2x3 bytes -> 3 chars. base64Accum = b64str.slice(canBeDecoded); // The rest will be decoded in future. diff --git a/node_modules/iconv-lite/lib/extend-node.js b/node_modules/iconv-lite/lib/extend-node.js deleted file mode 100644 index 87f5394a4..000000000 --- a/node_modules/iconv-lite/lib/extend-node.js +++ /dev/null @@ -1,217 +0,0 @@ -"use strict"; -var Buffer = require("buffer").Buffer; -// Note: not polyfilled with safer-buffer on a purpose, as overrides Buffer - -// == Extend Node primitives to use iconv-lite ================================= - -module.exports = function (iconv) { - var original = undefined; // Place to keep original methods. - - // Node authors rewrote Buffer internals to make it compatible with - // Uint8Array and we cannot patch key functions since then. - // Note: this does use older Buffer API on a purpose - iconv.supportsNodeEncodingsExtension = !(Buffer.from || new Buffer(0) instanceof Uint8Array); - - iconv.extendNodeEncodings = function extendNodeEncodings() { - if (original) return; - original = {}; - - if (!iconv.supportsNodeEncodingsExtension) { - console.error("ACTION NEEDED: require('iconv-lite').extendNodeEncodings() is not supported in your version of Node"); - console.error("See more info at https://github.com/ashtuchkin/iconv-lite/wiki/Node-v4-compatibility"); - return; - } - - var nodeNativeEncodings = { - 'hex': true, 'utf8': true, 'utf-8': true, 'ascii': true, 'binary': true, - 'base64': true, 'ucs2': true, 'ucs-2': true, 'utf16le': true, 'utf-16le': true, - }; - - Buffer.isNativeEncoding = function(enc) { - return enc && nodeNativeEncodings[enc.toLowerCase()]; - } - - // -- SlowBuffer ----------------------------------------------------------- - var SlowBuffer = require('buffer').SlowBuffer; - - original.SlowBufferToString = SlowBuffer.prototype.toString; - SlowBuffer.prototype.toString = function(encoding, start, end) { - encoding = String(encoding || 'utf8').toLowerCase(); - - // Use native conversion when possible - if (Buffer.isNativeEncoding(encoding)) - return original.SlowBufferToString.call(this, encoding, start, end); - - // Otherwise, use our decoding method. - if (typeof start == 'undefined') start = 0; - if (typeof end == 'undefined') end = this.length; - return iconv.decode(this.slice(start, end), encoding); - } - - original.SlowBufferWrite = SlowBuffer.prototype.write; - SlowBuffer.prototype.write = function(string, offset, length, encoding) { - // Support both (string, offset, length, encoding) - // and the legacy (string, encoding, offset, length) - if (isFinite(offset)) { - if (!isFinite(length)) { - encoding = length; - length = undefined; - } - } else { // legacy - var swap = encoding; - encoding = offset; - offset = length; - length = swap; - } - - offset = +offset || 0; - var remaining = this.length - offset; - if (!length) { - length = remaining; - } else { - length = +length; - if (length > remaining) { - length = remaining; - } - } - encoding = String(encoding || 'utf8').toLowerCase(); - - // Use native conversion when possible - if (Buffer.isNativeEncoding(encoding)) - return original.SlowBufferWrite.call(this, string, offset, length, encoding); - - if (string.length > 0 && (length < 0 || offset < 0)) - throw new RangeError('attempt to write beyond buffer bounds'); - - // Otherwise, use our encoding method. - var buf = iconv.encode(string, encoding); - if (buf.length < length) length = buf.length; - buf.copy(this, offset, 0, length); - return length; - } - - // -- Buffer --------------------------------------------------------------- - - original.BufferIsEncoding = Buffer.isEncoding; - Buffer.isEncoding = function(encoding) { - return Buffer.isNativeEncoding(encoding) || iconv.encodingExists(encoding); - } - - original.BufferByteLength = Buffer.byteLength; - Buffer.byteLength = SlowBuffer.byteLength = function(str, encoding) { - encoding = String(encoding || 'utf8').toLowerCase(); - - // Use native conversion when possible - if (Buffer.isNativeEncoding(encoding)) - return original.BufferByteLength.call(this, str, encoding); - - // Slow, I know, but we don't have a better way yet. - return iconv.encode(str, encoding).length; - } - - original.BufferToString = Buffer.prototype.toString; - Buffer.prototype.toString = function(encoding, start, end) { - encoding = String(encoding || 'utf8').toLowerCase(); - - // Use native conversion when possible - if (Buffer.isNativeEncoding(encoding)) - return original.BufferToString.call(this, encoding, start, end); - - // Otherwise, use our decoding method. - if (typeof start == 'undefined') start = 0; - if (typeof end == 'undefined') end = this.length; - return iconv.decode(this.slice(start, end), encoding); - } - - original.BufferWrite = Buffer.prototype.write; - Buffer.prototype.write = function(string, offset, length, encoding) { - var _offset = offset, _length = length, _encoding = encoding; - // Support both (string, offset, length, encoding) - // and the legacy (string, encoding, offset, length) - if (isFinite(offset)) { - if (!isFinite(length)) { - encoding = length; - length = undefined; - } - } else { // legacy - var swap = encoding; - encoding = offset; - offset = length; - length = swap; - } - - encoding = String(encoding || 'utf8').toLowerCase(); - - // Use native conversion when possible - if (Buffer.isNativeEncoding(encoding)) - return original.BufferWrite.call(this, string, _offset, _length, _encoding); - - offset = +offset || 0; - var remaining = this.length - offset; - if (!length) { - length = remaining; - } else { - length = +length; - if (length > remaining) { - length = remaining; - } - } - - if (string.length > 0 && (length < 0 || offset < 0)) - throw new RangeError('attempt to write beyond buffer bounds'); - - // Otherwise, use our encoding method. - var buf = iconv.encode(string, encoding); - if (buf.length < length) length = buf.length; - buf.copy(this, offset, 0, length); - return length; - - // TODO: Set _charsWritten. - } - - - // -- Readable ------------------------------------------------------------- - if (iconv.supportsStreams) { - var Readable = require('stream').Readable; - - original.ReadableSetEncoding = Readable.prototype.setEncoding; - Readable.prototype.setEncoding = function setEncoding(enc, options) { - // Use our own decoder, it has the same interface. - // We cannot use original function as it doesn't handle BOM-s. - this._readableState.decoder = iconv.getDecoder(enc, options); - this._readableState.encoding = enc; - } - - Readable.prototype.collect = iconv._collect; - } - } - - // Remove iconv-lite Node primitive extensions. - iconv.undoExtendNodeEncodings = function undoExtendNodeEncodings() { - if (!iconv.supportsNodeEncodingsExtension) - return; - if (!original) - throw new Error("require('iconv-lite').undoExtendNodeEncodings(): Nothing to undo; extendNodeEncodings() is not called.") - - delete Buffer.isNativeEncoding; - - var SlowBuffer = require('buffer').SlowBuffer; - - SlowBuffer.prototype.toString = original.SlowBufferToString; - SlowBuffer.prototype.write = original.SlowBufferWrite; - - Buffer.isEncoding = original.BufferIsEncoding; - Buffer.byteLength = original.BufferByteLength; - Buffer.prototype.toString = original.BufferToString; - Buffer.prototype.write = original.BufferWrite; - - if (iconv.supportsStreams) { - var Readable = require('stream').Readable; - - Readable.prototype.setEncoding = original.ReadableSetEncoding; - delete Readable.prototype.collect; - } - - original = undefined; - } -} diff --git a/node_modules/iconv-lite/lib/index.d.ts b/node_modules/iconv-lite/lib/index.d.ts index 0547eb346..99f200f4a 100644 --- a/node_modules/iconv-lite/lib/index.d.ts +++ b/node_modules/iconv-lite/lib/index.d.ts @@ -6,15 +6,22 @@ *--------------------------------------------------------------------------------------------*/ declare module 'iconv-lite' { + // Basic API export function decode(buffer: Buffer, encoding: string, options?: Options): string; export function encode(content: string, encoding: string, options?: Options): Buffer; export function encodingExists(encoding: string): boolean; + // Stream API export function decodeStream(encoding: string, options?: Options): NodeJS.ReadWriteStream; export function encodeStream(encoding: string, options?: Options): NodeJS.ReadWriteStream; + + // Low-level stream APIs + export function getEncoder(encoding: string, options?: Options): EncoderStream; + + export function getDecoder(encoding: string, options?: Options): DecoderStream; } export interface Options { @@ -22,3 +29,13 @@ export interface Options { addBOM?: boolean; defaultEncoding?: string; } + +export interface EncoderStream { + write(str: string): Buffer; + end(): Buffer | undefined; +} + +export interface DecoderStream { + write(buf: Buffer): string; + end(): string | undefined; +} diff --git a/node_modules/iconv-lite/lib/index.js b/node_modules/iconv-lite/lib/index.js index 5391919ca..657701c38 100644 --- a/node_modules/iconv-lite/lib/index.js +++ b/node_modules/iconv-lite/lib/index.js @@ -1,7 +1,5 @@ "use strict"; -// Some environments don't have global Buffer (e.g. React Native). -// Solution would be installing npm modules "buffer" and "stream" explicitly. var Buffer = require("safer-buffer").Buffer; var bomHandling = require("./bom-handling"), @@ -133,21 +131,50 @@ iconv.getDecoder = function getDecoder(encoding, options) { return decoder; } +// Streaming API +// NOTE: Streaming API naturally depends on 'stream' module from Node.js. Unfortunately in browser environments this module can add +// up to 100Kb to the output bundle. To avoid unnecessary code bloat, we don't enable Streaming API in browser by default. +// If you would like to enable it explicitly, please add the following code to your app: +// > iconv.enableStreamingAPI(require('stream')); +iconv.enableStreamingAPI = function enableStreamingAPI(stream_module) { + if (iconv.supportsStreams) + return; + + // Dependency-inject stream module to create IconvLite stream classes. + var streams = require("./streams")(stream_module); + + // Not public API yet, but expose the stream classes. + iconv.IconvLiteEncoderStream = streams.IconvLiteEncoderStream; + iconv.IconvLiteDecoderStream = streams.IconvLiteDecoderStream; + + // Streaming API. + iconv.encodeStream = function encodeStream(encoding, options) { + return new iconv.IconvLiteEncoderStream(iconv.getEncoder(encoding, options), options); + } -// Load extensions in Node. All of them are omitted in Browserify build via 'browser' field in package.json. -var nodeVer = typeof process !== 'undefined' && process.versions && process.versions.node; -if (nodeVer) { - - // Load streaming support in Node v0.10+ - var nodeVerArr = nodeVer.split(".").map(Number); - if (nodeVerArr[0] > 0 || nodeVerArr[1] >= 10) { - require("./streams")(iconv); + iconv.decodeStream = function decodeStream(encoding, options) { + return new iconv.IconvLiteDecoderStream(iconv.getDecoder(encoding, options), options); } - // Load Node primitive extensions. - require("./extend-node")(iconv); + iconv.supportsStreams = true; +} + +// Enable Streaming API automatically if 'stream' module is available and non-empty (the majority of environments). +var stream_module; +try { + stream_module = require("stream"); +} catch (e) {} + +if (stream_module && stream_module.Transform) { + iconv.enableStreamingAPI(stream_module); + +} else { + // In rare cases where 'stream' module is not available by default, throw a helpful exception. + iconv.encodeStream = iconv.decodeStream = function() { + throw new Error("iconv-lite Streaming API is not enabled. Use iconv.enableStreamingAPI(require('stream')); to enable it."); + }; } if ("Ā" != "\u0100") { - console.error("iconv-lite warning: javascript files use encoding different from utf-8. See https://github.com/ashtuchkin/iconv-lite/wiki/Javascript-source-file-encodings for more info."); + console.error("iconv-lite warning: js files use non-utf8 encoding. See https://github.com/ashtuchkin/iconv-lite/wiki/Javascript-source-file-encodings for more info."); } diff --git a/node_modules/iconv-lite/lib/streams.js b/node_modules/iconv-lite/lib/streams.js index 440955295..a1506482f 100644 --- a/node_modules/iconv-lite/lib/streams.js +++ b/node_modules/iconv-lite/lib/streams.js @@ -1,121 +1,109 @@ "use strict"; -var Buffer = require("buffer").Buffer, - Transform = require("stream").Transform; +var Buffer = require("safer-buffer").Buffer; +// NOTE: Due to 'stream' module being pretty large (~100Kb, significant in browser environments), +// we opt to dependency-inject it instead of creating a hard dependency. +module.exports = function(stream_module) { + var Transform = stream_module.Transform; -// == Exports ================================================================== -module.exports = function(iconv) { - - // Additional Public API. - iconv.encodeStream = function encodeStream(encoding, options) { - return new IconvLiteEncoderStream(iconv.getEncoder(encoding, options), options); + // == Encoder stream ======================================================= + + function IconvLiteEncoderStream(conv, options) { + this.conv = conv; + options = options || {}; + options.decodeStrings = false; // We accept only strings, so we don't need to decode them. + Transform.call(this, options); } - iconv.decodeStream = function decodeStream(encoding, options) { - return new IconvLiteDecoderStream(iconv.getDecoder(encoding, options), options); + IconvLiteEncoderStream.prototype = Object.create(Transform.prototype, { + constructor: { value: IconvLiteEncoderStream } + }); + + IconvLiteEncoderStream.prototype._transform = function(chunk, encoding, done) { + if (typeof chunk != 'string') + return done(new Error("Iconv encoding stream needs strings as its input.")); + try { + var res = this.conv.write(chunk); + if (res && res.length) this.push(res); + done(); + } + catch (e) { + done(e); + } } - iconv.supportsStreams = true; + IconvLiteEncoderStream.prototype._flush = function(done) { + try { + var res = this.conv.end(); + if (res && res.length) this.push(res); + done(); + } + catch (e) { + done(e); + } + } + IconvLiteEncoderStream.prototype.collect = function(cb) { + var chunks = []; + this.on('error', cb); + this.on('data', function(chunk) { chunks.push(chunk); }); + this.on('end', function() { + cb(null, Buffer.concat(chunks)); + }); + return this; + } - // Not published yet. - iconv.IconvLiteEncoderStream = IconvLiteEncoderStream; - iconv.IconvLiteDecoderStream = IconvLiteDecoderStream; - iconv._collect = IconvLiteDecoderStream.prototype.collect; -}; + // == Decoder stream ======================================================= -// == Encoder stream ======================================================= -function IconvLiteEncoderStream(conv, options) { - this.conv = conv; - options = options || {}; - options.decodeStrings = false; // We accept only strings, so we don't need to decode them. - Transform.call(this, options); -} - -IconvLiteEncoderStream.prototype = Object.create(Transform.prototype, { - constructor: { value: IconvLiteEncoderStream } -}); - -IconvLiteEncoderStream.prototype._transform = function(chunk, encoding, done) { - if (typeof chunk != 'string') - return done(new Error("Iconv encoding stream needs strings as its input.")); - try { - var res = this.conv.write(chunk); - if (res && res.length) this.push(res); - done(); - } - catch (e) { - done(e); + function IconvLiteDecoderStream(conv, options) { + this.conv = conv; + options = options || {}; + options.encoding = this.encoding = 'utf8'; // We output strings. + Transform.call(this, options); } -} -IconvLiteEncoderStream.prototype._flush = function(done) { - try { - var res = this.conv.end(); - if (res && res.length) this.push(res); - done(); - } - catch (e) { - done(e); - } -} - -IconvLiteEncoderStream.prototype.collect = function(cb) { - var chunks = []; - this.on('error', cb); - this.on('data', function(chunk) { chunks.push(chunk); }); - this.on('end', function() { - cb(null, Buffer.concat(chunks)); + IconvLiteDecoderStream.prototype = Object.create(Transform.prototype, { + constructor: { value: IconvLiteDecoderStream } }); - return this; -} - - -// == Decoder stream ======================================================= -function IconvLiteDecoderStream(conv, options) { - this.conv = conv; - options = options || {}; - options.encoding = this.encoding = 'utf8'; // We output strings. - Transform.call(this, options); -} - -IconvLiteDecoderStream.prototype = Object.create(Transform.prototype, { - constructor: { value: IconvLiteDecoderStream } -}); - -IconvLiteDecoderStream.prototype._transform = function(chunk, encoding, done) { - if (!Buffer.isBuffer(chunk)) - return done(new Error("Iconv decoding stream needs buffers as its input.")); - try { - var res = this.conv.write(chunk); - if (res && res.length) this.push(res, this.encoding); - done(); - } - catch (e) { - done(e); + + IconvLiteDecoderStream.prototype._transform = function(chunk, encoding, done) { + if (!Buffer.isBuffer(chunk) && !(chunk instanceof Uint8Array)) + return done(new Error("Iconv decoding stream needs buffers as its input.")); + try { + var res = this.conv.write(chunk); + if (res && res.length) this.push(res, this.encoding); + done(); + } + catch (e) { + done(e); + } } -} -IconvLiteDecoderStream.prototype._flush = function(done) { - try { - var res = this.conv.end(); - if (res && res.length) this.push(res, this.encoding); - done(); + IconvLiteDecoderStream.prototype._flush = function(done) { + try { + var res = this.conv.end(); + if (res && res.length) this.push(res, this.encoding); + done(); + } + catch (e) { + done(e); + } } - catch (e) { - done(e); + + IconvLiteDecoderStream.prototype.collect = function(cb) { + var res = ''; + this.on('error', cb); + this.on('data', function(chunk) { res += chunk; }); + this.on('end', function() { + cb(null, res); + }); + return this; } -} - -IconvLiteDecoderStream.prototype.collect = function(cb) { - var res = ''; - this.on('error', cb); - this.on('data', function(chunk) { res += chunk; }); - this.on('end', function() { - cb(null, res); - }); - return this; -} + return { + IconvLiteEncoderStream: IconvLiteEncoderStream, + IconvLiteDecoderStream: IconvLiteDecoderStream, + }; +}; diff --git a/node_modules/iconv-lite/package.json b/node_modules/iconv-lite/package.json index 07f6f1a8e..603426531 100644 --- a/node_modules/iconv-lite/package.json +++ b/node_modules/iconv-lite/package.json @@ -1,54 +1,52 @@ { - "_from": "iconv-lite@~0.4.13", - "_id": "iconv-lite@0.4.23", + "_from": "iconv-lite@^0.6.2", + "_id": "iconv-lite@0.6.2", "_inBundle": false, - "_integrity": "sha512-neyTUVFtahjf0mB3dZT77u+8O0QB89jFdnBkd5P1JgYPbPaia3gXXOVL2fq8VyU2gMMD7SaN7QukTB/pmXYvDA==", + "_integrity": "sha512-2y91h5OpQlolefMPmUlivelittSWy0rP+oYVpn6A7GwVHNE8AWzoYOBNmlwks3LobaJxgHCYZAnyNo2GgpNRNQ==", "_location": "/iconv-lite", "_phantomChildren": {}, "_requested": { "type": "range", "registry": true, - "raw": "iconv-lite@~0.4.13", + "raw": "iconv-lite@^0.6.2", "name": "iconv-lite", "escapedName": "iconv-lite", - "rawSpec": "~0.4.13", + "rawSpec": "^0.6.2", "saveSpec": null, - "fetchSpec": "~0.4.13" + "fetchSpec": "^0.6.2" }, "_requiredBy": [ - "/encoding", - "/external-editor" + "/encoding" ], - "_resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.23.tgz", - "_shasum": "297871f63be507adcfbfca715d0cd0eed84e9a63", - "_spec": "iconv-lite@~0.4.13", - "_where": "/Users/rebecca/code/npm/node_modules/encoding", + "_resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.2.tgz", + "_shasum": "ce13d1875b0c3a674bd6a04b7f76b01b1b6ded01", + "_spec": "iconv-lite@^0.6.2", + "_where": "/Users/isaacs/dev/npm/cli/node_modules/encoding", "author": { "name": "Alexander Shtuchkin", "email": "ashtuchkin@gmail.com" }, "browser": { - "./lib/extend-node": false, - "./lib/streams": false + "stream": false }, "bugs": { "url": "https://github.com/ashtuchkin/iconv-lite/issues" }, "bundleDependencies": false, "dependencies": { - "safer-buffer": ">= 2.1.2 < 3" + "safer-buffer": ">= 2.1.2 < 3.0.0" }, "deprecated": false, "description": "Convert character encodings in pure javascript.", "devDependencies": { - "async": "*", - "errto": "*", - "iconv": "*", - "istanbul": "*", - "mocha": "^3.1.0", - "request": "~2.81.0", - "semver": "*", - "unorm": "*" + "async": "^3.2.0", + "c8": "^7.2.0", + "errto": "^0.2.1", + "iconv": "^2.3.5", + "mocha": "^3.5.3", + "request": "^2.88.2", + "semver": "^6.3.0", + "unorm": "^1.6.0" }, "engines": { "node": ">=0.10.0" @@ -68,10 +66,9 @@ "url": "git://github.com/ashtuchkin/iconv-lite.git" }, "scripts": { - "coverage": "istanbul cover _mocha -- --grep .", - "coverage-open": "open coverage/lcov-report/index.html", + "coverage": "c8 _mocha --grep .", "test": "mocha --reporter spec --grep ." }, "typings": "./lib/index.d.ts", - "version": "0.4.23" + "version": "0.6.2" } |