src: add --heapsnapshot-near-heap-limit option

This patch adds a --heapsnapshot-near-heap-limit CLI option
that takes heap snapshots when the V8 heap is approaching
the heap size limit. It will try to write the snapshots
to disk before the program crashes due to OOM.

PR-URL: https://github.com/nodejs/node/pull/33010
Refs: https://github.com/nodejs/node/issues/27552
Reviewed-By: Anna Henningsen <anna@addaleax.net>
Reviewed-By: Richard Lau <rlau@redhat.com>
Reviewed-By: Gireesh Punathil <gpunathi@in.ibm.com>

6 files changed, 241 insertions, 0 deletions

diff --git a/test/fixtures/workload/bounded.js b/test/fixtures/workload/bounded.js
new file mode 100644
index 00000000000..ddf288d034b
--- /dev/null
+++ b/test/fixtures/workload/bounded.js
@@ -0,0 +1,22 @@
+'use strict';
+
+const total = parseInt(process.env.TEST_ALLOCATION) || 5000;
+const chunk = parseInt(process.env.TEST_CHUNK) || 1000;
+const cleanInterval = parseInt(process.env.TEST_CLEAN_INTERVAL) || 100;
+let count = 0;
+let arr = [];
+function runAllocation() {
+  count++;
+  if (count < total) {
+    if (count % cleanInterval === 0) {
+      arr.splice(0, arr.length);
+      setImmediate(runAllocation);
+    } else {
+      const str = JSON.stringify(process.config).slice(0, chunk);
+      arr.push(str);
+      setImmediate(runAllocation);
+    }
+  }
+}
+
+setImmediate(runAllocation);
diff --git a/test/fixtures/workload/grow-worker.js b/test/fixtures/workload/grow-worker.js
new file mode 100644
index 00000000000..092d8f27751
--- /dev/null
+++ b/test/fixtures/workload/grow-worker.js
@@ -0,0 +1,14 @@
+'use strict';
+
+const { Worker } = require('worker_threads');
+const path = require('path');
+const max_snapshots = parseInt(process.env.TEST_SNAPSHOTS) || 1;
+new Worker(path.join(__dirname, 'grow.js'), {
+  execArgv: [
+    `--heapsnapshot-near-heap-limit=${max_snapshots}`,
+  ],
+  resourceLimits: {
+    maxOldGenerationSizeMb:
+      parseInt(process.env.TEST_OLD_SPACE_SIZE) || 20
+  }
+});
diff --git a/test/fixtures/workload/grow.js b/test/fixtures/workload/grow.js
new file mode 100644
index 00000000000..9ac0139b332
--- /dev/null
+++ b/test/fixtures/workload/grow.js
@@ -0,0 +1,12 @@
+'use strict';
+
+const chunk = parseInt(process.env.TEST_CHUNK) || 1000;
+
+let arr = [];
+function runAllocation() {
+  const str = JSON.stringify(process.config).slice(0, chunk);
+  arr.push(str);
+  setImmediate(runAllocation);
+}
+
+setImmediate(runAllocation);
diff --git a/test/parallel/test-heapsnapshot-near-heap-limit-bounded.js b/test/parallel/test-heapsnapshot-near-heap-limit-bounded.js
new file mode 100644
index 00000000000..16d1f915fee
--- /dev/null
+++ b/test/parallel/test-heapsnapshot-near-heap-limit-bounded.js
@@ -0,0 +1,36 @@
+'use strict';
+
+require('../common');
+const tmpdir = require('../common/tmpdir');
+const assert = require('assert');
+const { spawnSync } = require('child_process');
+const fixtures = require('../common/fixtures');
+const fs = require('fs');
+const env = {
+  ...process.env,
+  TEST_ALLOCATION: 50000,
+  TEST_CHUNK: 1000,
+  TEST_CLEAN_INTERVAL: 500,
+  NODE_DEBUG_NATIVE: 'diagnostics'
+};
+
+{
+  console.log('\nTesting limit = 1');
+  tmpdir.refresh();
+  const child = spawnSync(process.execPath, [
+    '--trace-gc',
+    '--heapsnapshot-near-heap-limit=1',
+    '--max-old-space-size=50',
+    fixtures.path('workload', 'bounded.js')
+  ], {
+    cwd: tmpdir.path,
+    env,
+  });
+  console.log(child.stdout.toString());
+  console.log(child.stderr.toString());
+  assert.strictEqual(child.signal, null);
+  assert.strictEqual(child.status, 0);
+  const list = fs.readdirSync(tmpdir.path)
+    .filter((file) => file.endsWith('.heapsnapshot'));
+  assert.strictEqual(list.length, 0);
+}
diff --git a/test/parallel/test-heapsnapshot-near-heap-limit.js b/test/parallel/test-heapsnapshot-near-heap-limit.js
new file mode 100644
index 00000000000..db75da221ab
--- /dev/null
+++ b/test/parallel/test-heapsnapshot-near-heap-limit.js
@@ -0,0 +1,114 @@
+'use strict';
+
+const common = require('../common');
+const tmpdir = require('../common/tmpdir');
+const assert = require('assert');
+const { spawnSync } = require('child_process');
+const fixtures = require('../common/fixtures');
+const fs = require('fs');
+const env = {
+  ...process.env,
+  NODE_DEBUG_NATIVE: 'diagnostics'
+};
+
+{
+  tmpdir.refresh();
+  const child = spawnSync(process.execPath, [
+    '--heapsnapshot-near-heap-limit=-15',
+    '--max-old-space-size=50',
+    fixtures.path('workload', 'grow.js')
+  ], {
+    cwd: tmpdir.path,
+    env,
+  });
+  assert.strictEqual(child.status, 9);
+}
+
+{
+  console.log('\nTesting limit = 0');
+  tmpdir.refresh();
+  const child = spawnSync(process.execPath, [
+    '--trace-gc',
+    '--heapsnapshot-near-heap-limit=0',
+    '--max-old-space-size=50',
+    fixtures.path('workload', 'grow.js')
+  ], {
+    cwd: tmpdir.path,
+    env,
+  });
+  console.log(child.stdout.toString());
+  console.log(child.stderr.toString());
+  assert(common.nodeProcessAborted(child.status, child.signal),
+         'process should have aborted, but did not');
+  const list = fs.readdirSync(tmpdir.path)
+    .filter((file) => file.endsWith('.heapsnapshot'));
+  assert.strictEqual(list.length, 0);
+}
+
+{
+  console.log('\nTesting limit = 1');
+  tmpdir.refresh();
+  const child = spawnSync(process.execPath, [
+    '--trace-gc',
+    '--heapsnapshot-near-heap-limit=1',
+    '--max-old-space-size=50',
+    fixtures.path('workload', 'grow.js')
+  ], {
+    cwd: tmpdir.path,
+    env,
+  });
+  console.log(child.stdout.toString());
+  console.log(child.stderr.toString());
+  assert(common.nodeProcessAborted(child.status, child.signal),
+         'process should have aborted, but did not');
+  const list = fs.readdirSync(tmpdir.path)
+    .filter((file) => file.endsWith('.heapsnapshot'));
+  assert.strictEqual(list.length, 1);
+}
+
+{
+  console.log('\nTesting limit = 3');
+  tmpdir.refresh();
+  const child = spawnSync(process.execPath, [
+    '--trace-gc',
+    '--heapsnapshot-near-heap-limit=3',
+    '--max-old-space-size=50',
+    fixtures.path('workload', 'grow.js')
+  ], {
+    cwd: tmpdir.path,
+    env,
+  });
+  console.log(child.stdout.toString());
+  console.log(child.stderr.toString());
+  assert(common.nodeProcessAborted(child.status, child.signal),
+         'process should have aborted, but did not');
+  const list = fs.readdirSync(tmpdir.path)
+    .filter((file) => file.endsWith('.heapsnapshot'));
+  assert(list.length > 0 && list.length <= 3);
+}
+
+
+{
+  console.log('\nTesting worker');
+  tmpdir.refresh();
+  const child = spawnSync(process.execPath, [
+    fixtures.path('workload', 'grow-worker.js')
+  ], {
+    cwd: tmpdir.path,
+    env: {
+      TEST_SNAPSHOTS: 1,
+      TEST_OLD_SPACE_SIZE: 50,
+      ...env
+    }
+  });
+  console.log(child.stdout.toString());
+  const stderr = child.stderr.toString();
+  console.log(stderr);
+  // There should be one snapshot taken and then after the
+  // snapshot heap limit callback is popped, the OOM callback
+  // becomes effective.
+  assert(stderr.includes('ERR_WORKER_OUT_OF_MEMORY'));
+  const list = fs.readdirSync(tmpdir.path)
+    .filter((file) => file.endsWith('.heapsnapshot'));
+  assert.strictEqual(list.length, 1);
+}
diff --git a/test/pummel/test-heapsnapshot-near-heap-limit-big.js b/test/pummel/test-heapsnapshot-near-heap-limit-big.js
new file mode 100644
index 00000000000..f70d562c1d1
--- /dev/null
+++ b/test/pummel/test-heapsnapshot-near-heap-limit-big.js
@@ -0,0 +1,43 @@
+'use strict';
+
+const common = require('../common');
+const tmpdir = require('../common/tmpdir');
+const assert = require('assert');
+const { spawnSync } = require('child_process');
+const fixtures = require('../common/fixtures');
+const fs = require('fs');
+const env = {
+  ...process.env,
+  NODE_DEBUG_NATIVE: 'diagnostics'
+};
+
+if (!common.enoughTestMem)
+  common.skip('Insufficient memory for snapshot test');
+
+{
+  console.log('\nTesting limit = 3');
+  tmpdir.refresh();
+  const child = spawnSync(process.execPath, [
+    '--heapsnapshot-near-heap-limit=3',
+    '--max-old-space-size=512',
+    fixtures.path('workload', 'grow.js')
+  ], {
+    cwd: tmpdir.path,
+    env: {
+      ...env,
+      TEST_CHUNK: 2000,
+    }
+  });
+  const stderr = child.stderr.toString();
+  console.log(stderr);
+  assert(common.nodeProcessAborted(child.status, child.signal),
+         'process should have aborted, but did not');
+  const list = fs.readdirSync(tmpdir.path)
+    .filter((file) => file.endsWith('.heapsnapshot'));
+  if (list.length === 0) {
+    assert(stderr.includes(
+      'Not generating snapshots because it\'s too risky'));
+  } else {
+    assert(list.length > 0 && list.length <= 3);
+  }
+}