Add memory benchmark tests for streamed decryption of large files (#1462)

Also, provide more details about `config.allowUnauthenticatedStream`.

2 files changed, 145 insertions, 19 deletions

diff --git a/src/config/config.js b/src/config/config.js
index 4ed4050a..285f8988 100644
--- a/src/config/config.js
+++ b/src/config/config.js
@@ -92,7 +92,10 @@ export default {
    */
   allowUnauthenticatedMessages: false,
   /**
-   * Allow streaming unauthenticated data before its integrity has been checked.
+   * Allow streaming unauthenticated data before its integrity has been checked. This would allow the application to
+   * process large streams while limiting memory usage by releasing the decrypted chunks as soon as possible
+   * and deferring checking their integrity until the decrypted stream has been read in full.
+   *
    * This setting is **insecure** if the partially decrypted message is processed further or displayed to the user.
    * @memberof module:config
    * @property {Boolean} allowUnauthenticatedStream
diff --git a/test/benchmarks/memory_usage.js b/test/benchmarks/memory_usage.js
index d00aa17b..58166105 100644
--- a/test/benchmarks/memory_usage.js
+++ b/test/benchmarks/memory_usage.js
@@ -1,6 +1,5 @@
 /* eslint-disable no-console */
 const assert = require('assert');
-const stream = require('@openpgp/web-stream-tools');
 const path = require('path');
 const { writeFileSync, unlinkSync } = require('fs');
 const { fork } = require('child_process');
@@ -17,7 +16,6 @@ const benchmark = async function(fn) {
   // the code to execute must be written to a file
   writeFileSync(tmpFileName, `
 const assert = require('assert');
-const stream = require('@openpgp/web-stream-tools');
 const openpgp = require('../..');
 let maxMemoryComsumption;
 let activeSampling = false;
@@ -111,9 +109,10 @@ class MemoryBenchamrkSuite {
   suite.add('empty test (baseline)', () => {});
 
   suite.add('openpgp.encrypt/decrypt (CFB, binary)', async () => {
+    const ONE_MEGABYTE = 1000000;
     const passwords = 'password';
     const config = { aeadProtect: false, preferredCompressionAlgorithm: openpgp.enums.compression.uncompressed };
-    const plaintextMessage = await openpgp.createMessage({ binary: new Uint8Array(1000000).fill(1) });
+    const plaintextMessage = await openpgp.createMessage({ binary: new Uint8Array(ONE_MEGABYTE) });
 
     const armoredEncryptedMessage = await openpgp.encrypt({ message: plaintextMessage, passwords, config });
     const encryptedMessage = await openpgp.readMessage({ armoredMessage: armoredEncryptedMessage });
@@ -122,9 +121,10 @@ class MemoryBenchamrkSuite {
   });
 
   suite.add('openpgp.encrypt/decrypt (CFB, text)', async () => {
+    const ONE_MEGABYTE = 1000000;
     const passwords = 'password';
     const config = { aeadProtect: false, preferredCompressionAlgorithm: openpgp.enums.compression.uncompressed };
-    const plaintextMessage = await openpgp.createMessage({ text: 'a'.repeat(10000000 / 2) }); // two bytes per character
+    const plaintextMessage = await openpgp.createMessage({ text: 'a'.repeat(ONE_MEGABYTE / 2) }); // two bytes per character
 
     const armoredEncryptedMessage = await openpgp.encrypt({ message: plaintextMessage, passwords, config });
     const encryptedMessage = await openpgp.readMessage({ armoredMessage: armoredEncryptedMessage });
@@ -133,9 +133,10 @@ class MemoryBenchamrkSuite {
   });
 
   suite.add('openpgp.encrypt/decrypt (AEAD, binary)', async () => {
+    const ONE_MEGABYTE = 1000000;
     const passwords = 'password';
     const config = { aeadProtect: true, preferredCompressionAlgorithm: openpgp.enums.compression.uncompressed };
-    const plaintextMessage = await openpgp.createMessage({ binary: new Uint8Array(1000000).fill(1) });
+    const plaintextMessage = await openpgp.createMessage({ binary: new Uint8Array(ONE_MEGABYTE) });
 
     const armoredEncryptedMessage = await openpgp.encrypt({ message: plaintextMessage, passwords, config });
     const encryptedMessage = await openpgp.readMessage({ armoredMessage: armoredEncryptedMessage });
@@ -144,9 +145,10 @@ class MemoryBenchamrkSuite {
   });
 
   suite.add('openpgp.encrypt/decrypt (AEAD, text)', async () => {
+    const ONE_MEGABYTE = 1000000;
     const passwords = 'password';
     const config = { aeadProtect: true, preferredCompressionAlgorithm: openpgp.enums.compression.uncompressed };
-    const plaintextMessage = await openpgp.createMessage({ text: 'a'.repeat(10000000 / 2) }); // two bytes per character
+    const plaintextMessage = await openpgp.createMessage({ text: 'a'.repeat(ONE_MEGABYTE / 2) }); // two bytes per character
 
     const armoredEncryptedMessage = await openpgp.encrypt({ message: plaintextMessage, passwords, config });
     const encryptedMessage = await openpgp.readMessage({ armoredMessage: armoredEncryptedMessage });
@@ -156,61 +158,182 @@ class MemoryBenchamrkSuite {
 
   // streaming tests
   suite.add('openpgp.encrypt/decrypt (CFB, binary, with streaming)', async () => {
-    await stream.loadStreamsPonyfill();
+    const ONE_MEGABYTE = 1000000;
+    function* largeDataGenerator({ chunk, numberOfChunks }) {
+      for (let chunkNumber = 0; chunkNumber < numberOfChunks; chunkNumber++) {
+        yield chunk;
+      }
+    }
 
     const passwords = 'password';
     const config = { aeadProtect: false, preferredCompressionAlgorithm: openpgp.enums.compression.uncompressed };
-    const plaintextMessage = await openpgp.createMessage({ binary: stream.toStream(new Uint8Array(1000000).fill(1)) });
+    const inputStream = require('stream').Readable.from(largeDataGenerator({ chunk: new Uint8Array(ONE_MEGABYTE), numberOfChunks: 1 }));
+    const plaintextMessage = await openpgp.createMessage({ binary: inputStream });
     assert(plaintextMessage.fromStream);
 
     const armoredEncryptedMessage = await openpgp.encrypt({ message: plaintextMessage, passwords, config });
     const encryptedMessage = await openpgp.readMessage({ armoredMessage: armoredEncryptedMessage });
     assert.ok(encryptedMessage.packets[1] instanceof openpgp.SymEncryptedIntegrityProtectedDataPacket);
     const { data: decryptedData } = await openpgp.decrypt({ message: encryptedMessage, passwords, config });
-    await stream.readToEnd(decryptedData);
+    // read out output stream to trigger decryption
+    await new Promise(resolve => {
+      decryptedData.pipe(require('fs').createWriteStream('/dev/null'));
+      decryptedData.on('end', resolve);
+    });
   });
 
   suite.add('openpgp.encrypt/decrypt (CFB, text, with streaming)', async () => {
-    await stream.loadStreamsPonyfill();
+    const ONE_MEGABYTE = 1000000;
+    function* largeDataGenerator({ chunk, numberOfChunks }) {
+      for (let chunkNumber = 0; chunkNumber < numberOfChunks; chunkNumber++) {
+        yield chunk;
+      }
+    }
 
     const passwords = 'password';
     const config = { aeadProtect: false, preferredCompressionAlgorithm: openpgp.enums.compression.uncompressed };
-    const plaintextMessage = await openpgp.createMessage({ text: stream.toStream('a'.repeat(10000000 / 2)) });
+    const inputStream = require('stream').Readable.from(largeDataGenerator({ chunk: 'a'.repeat(ONE_MEGABYTE / 2), numberOfChunks: 1 }));
+    const plaintextMessage = await openpgp.createMessage({ text: inputStream });
     assert(plaintextMessage.fromStream);
 
     const armoredEncryptedMessage = await openpgp.encrypt({ message: plaintextMessage, passwords, config });
     const encryptedMessage = await openpgp.readMessage({ armoredMessage: armoredEncryptedMessage });
     assert.ok(encryptedMessage.packets[1] instanceof openpgp.SymEncryptedIntegrityProtectedDataPacket);
     const { data: decryptedData } = await openpgp.decrypt({ message: encryptedMessage, passwords, config });
-    await stream.readToEnd(decryptedData);
+    // read out output stream to trigger decryption
+    await new Promise(resolve => {
+      decryptedData.pipe(require('fs').createWriteStream('/dev/null'));
+      decryptedData.on('end', resolve);
+    });
   });
 
   suite.add('openpgp.encrypt/decrypt (AEAD, binary, with streaming)', async () => {
-    await stream.loadStreamsPonyfill();
+    const ONE_MEGABYTE = 1000000;
+    function* largeDataGenerator({ chunk, numberOfChunks }) {
+      for (let chunkNumber = 0; chunkNumber < numberOfChunks; chunkNumber++) {
+        yield chunk;
+      }
+    }
 
     const passwords = 'password';
     const config = { aeadProtect: true, preferredCompressionAlgorithm: openpgp.enums.compression.uncompressed };
-    const plaintextMessage = await openpgp.createMessage({ binary: stream.toStream(new Uint8Array(1000000).fill(1)) });
+    const inputStream = require('stream').Readable.from(largeDataGenerator({ chunk: new Uint8Array(ONE_MEGABYTE), numberOfChunks: 1 }));
+    const plaintextMessage = await openpgp.createMessage({ binary:inputStream });
     assert(plaintextMessage.fromStream);
 
     const armoredEncryptedMessage = await openpgp.encrypt({ message: plaintextMessage, passwords, config });
     const encryptedMessage = await openpgp.readMessage({ armoredMessage: armoredEncryptedMessage });
     assert.ok(encryptedMessage.packets[1] instanceof openpgp.AEADEncryptedDataPacket);
-    await openpgp.decrypt({ message: encryptedMessage, passwords, config });
+    const { data: decryptedData } = await openpgp.decrypt({ message: encryptedMessage, passwords, config });
+    // read out output stream to trigger decryption
+    await new Promise(resolve => {
+      decryptedData.pipe(require('fs').createWriteStream('/dev/null'));
+      decryptedData.on('end', resolve);
+    });
   });
 
   suite.add('openpgp.encrypt/decrypt (AEAD, text, with streaming)', async () => {
-    await stream.loadStreamsPonyfill();
+    const ONE_MEGABYTE = 1000000;
+    function* largeDataGenerator({ chunk, numberOfChunks }) {
+      for (let chunkNumber = 0; chunkNumber < numberOfChunks; chunkNumber++) {
+        yield chunk;
+      }
+    }
 
     const passwords = 'password';
     const config = { aeadProtect: true, preferredCompressionAlgorithm: openpgp.enums.compression.uncompressed };
-    const plaintextMessage = await openpgp.createMessage({ text: stream.toStream('a'.repeat(10000000 / 2)) });
+    const inputStream = require('stream').Readable.from(largeDataGenerator({ chunk: 'a'.repeat(ONE_MEGABYTE / 2), numberOfChunks: 1 }));
+    const plaintextMessage = await openpgp.createMessage({ text: inputStream });
     assert(plaintextMessage.fromStream);
 
     const armoredEncryptedMessage = await openpgp.encrypt({ message: plaintextMessage, passwords, config });
     const encryptedMessage = await openpgp.readMessage({ armoredMessage: armoredEncryptedMessage });
     assert.ok(encryptedMessage.packets[1] instanceof openpgp.AEADEncryptedDataPacket);
-    await openpgp.decrypt({ message: encryptedMessage, passwords, config });
+    const { data: decryptedData } = await openpgp.decrypt({ message: encryptedMessage, passwords, config });
+    // read out output stream to trigger decryption
+    await new Promise(resolve => {
+      decryptedData.pipe(require('fs').createWriteStream('/dev/null'));
+      decryptedData.on('end', resolve);
+    });
+  });
+
+  suite.add('openpgp.encrypt/decrypt (CFB, text @ 10MB, with streaming)', async () => {
+    const ONE_MEGABYTE = 1000000;
+    function* largeDataGenerator({ chunk, numberOfChunks }) {
+      for (let chunkNumber = 0; chunkNumber < numberOfChunks; chunkNumber++) {
+        yield chunk;
+      }
+    }
+
+    const passwords = 'password';
+    const config = { aeadProtect: false, preferredCompressionAlgorithm: openpgp.enums.compression.uncompressed };
+    const inputStream = require('stream').Readable.from(largeDataGenerator({ chunk: 'a'.repeat(ONE_MEGABYTE / 2), numberOfChunks: 20 }));
+    const plaintextMessage = await openpgp.createMessage({ text: inputStream });
+    assert(plaintextMessage.fromStream);
+
+    const armoredEncryptedMessage = await openpgp.encrypt({ message: plaintextMessage, passwords, config });
+    const encryptedMessage = await openpgp.readMessage({ armoredMessage: armoredEncryptedMessage });
+    assert.ok(encryptedMessage.packets[1] instanceof openpgp.SymEncryptedIntegrityProtectedDataPacket);
+    const { data: decryptedData } = await openpgp.decrypt({ message: encryptedMessage, passwords, config });
+    // read out output stream to trigger decryption
+    await new Promise(resolve => {
+      decryptedData.pipe(require('fs').createWriteStream('/dev/null'));
+      decryptedData.on('end', resolve);
+    });
+  });
+
+  suite.add('openpgp.encrypt/decrypt (CFB, text @ 10MB, with unauthenticated streaming)', async () => {
+    const ONE_MEGABYTE = 1000000;
+    function* largeDataGenerator({ chunk, numberOfChunks }) {
+      for (let chunkNumber = 0; chunkNumber < numberOfChunks; chunkNumber++) {
+        yield chunk;
+      }
+    }
+
+    const passwords = 'password';
+    const config = { aeadProtect: false, preferredCompressionAlgorithm: openpgp.enums.compression.uncompressed };
+    const inputStream = require('stream').Readable.from(largeDataGenerator({ chunk: 'a'.repeat(ONE_MEGABYTE / 2), numberOfChunks: 20 }));
+    const plaintextMessage = await openpgp.createMessage({ text: inputStream });
+    assert(plaintextMessage.fromStream);
+
+    const armoredEncryptedMessage = await openpgp.encrypt({ message: plaintextMessage, passwords, config });
+    const encryptedMessage = await openpgp.readMessage({ armoredMessage: armoredEncryptedMessage });
+    assert.ok(encryptedMessage.packets[1] instanceof openpgp.SymEncryptedIntegrityProtectedDataPacket);
+    const { data: decryptedData } = await openpgp.decrypt({
+      message: encryptedMessage,
+      passwords,
+      config: { ...config, allowUnauthenticatedStream: true }
+    });
+    // read out output stream to trigger decryption
+    await new Promise(resolve => {
+      decryptedData.pipe(require('fs').createWriteStream('/dev/null'));
+      decryptedData.on('end', resolve);
+    });
+  });
+
+  suite.add('openpgp.encrypt/decrypt (AEAD, text @ 10MB, with streaming)', async () => {
+    const ONE_MEGABYTE = 1000000;
+    function* largeDataGenerator({ chunk, numberOfChunks }) {
+      for (let chunkNumber = 0; chunkNumber < numberOfChunks; chunkNumber++) {
+        yield chunk;
+      }
+    }
+
+    const passwords = 'password';
+    const config = { aeadProtect: true, preferredCompressionAlgorithm: openpgp.enums.compression.uncompressed };
+    const inputStream = require('stream').Readable.from(largeDataGenerator({ chunk: 'a'.repeat(ONE_MEGABYTE / 2), numberOfChunks: 20 }));
+    const plaintextMessage = await openpgp.createMessage({ text: inputStream });
+    assert(plaintextMessage.fromStream);
+
+    const armoredEncryptedMessage = await openpgp.encrypt({ message: plaintextMessage, passwords, config });
+    const encryptedMessage = await openpgp.readMessage({ armoredMessage: armoredEncryptedMessage });
+    assert.ok(encryptedMessage.packets[1] instanceof openpgp.AEADEncryptedDataPacket);
+    const { data: decryptedData } = await openpgp.decrypt({ message: encryptedMessage, passwords, config });
+    // read out output stream to trigger decryption
+    await new Promise(resolve => {
+      decryptedData.pipe(require('fs').createWriteStream('/dev/null'));
+      decryptedData.on('end', resolve);
+    });
   });
 
   const stats = await suite.run();