diff options
author | claudiahdz <cghr1990@gmail.com> | 2020-02-06 23:25:33 +0300 |
---|---|---|
committer | isaacs <i@izs.me> | 2020-05-08 04:11:51 +0300 |
commit | 0a45514d8beba749bebbd973ca9750ee0dc13b40 (patch) | |
tree | eec0c989c38fcb7395c9fd565f2a44c79369c59e /node_modules/cacache/lib/verify.js | |
parent | 019e6c41a85edf62727b517e1dfe2d5b9e53ba0e (diff) |
cacache@15.0.0
Diffstat (limited to 'node_modules/cacache/lib/verify.js')
-rw-r--r-- | node_modules/cacache/lib/verify.js | 276 |
1 files changed, 168 insertions, 108 deletions
diff --git a/node_modules/cacache/lib/verify.js b/node_modules/cacache/lib/verify.js index 617d38db1..5a011a3f1 100644 --- a/node_modules/cacache/lib/verify.js +++ b/node_modules/cacache/lib/verify.js @@ -1,35 +1,39 @@ 'use strict' -const BB = require('bluebird') +const util = require('util') +const pMap = require('p-map') const contentPath = require('./content/path') -const figgyPudding = require('figgy-pudding') -const finished = BB.promisify(require('mississippi').finished) const fixOwner = require('./util/fix-owner') -const fs = require('graceful-fs') -const glob = BB.promisify(require('glob')) +const fs = require('fs') +const fsm = require('fs-minipass') +const glob = util.promisify(require('glob')) const index = require('./entry-index') const path = require('path') -const rimraf = BB.promisify(require('rimraf')) +const rimraf = util.promisify(require('rimraf')) const ssri = require('ssri') -BB.promisifyAll(fs) +const hasOwnProperty = (obj, key) => + Object.prototype.hasOwnProperty.call(obj, key) -const VerifyOpts = figgyPudding({ - concurrency: { - default: 20 - }, - filter: {}, - log: { - default: { silly () {} } - } +const stat = util.promisify(fs.stat) +const truncate = util.promisify(fs.truncate) +const writeFile = util.promisify(fs.writeFile) +const readFile = util.promisify(fs.readFile) + +const verifyOpts = (opts) => ({ + concurrency: 20, + log: { silly () {} }, + ...opts }) module.exports = verify + function verify (cache, opts) { - opts = VerifyOpts(opts) + opts = verifyOpts(opts) opts.log.silly('verify', 'verifying cache at', cache) - return BB.reduce([ + + const steps = [ markStartTime, fixPerms, garbageCollect, @@ -37,38 +41,57 @@ function verify (cache, opts) { cleanTmp, writeVerifile, markEndTime - ], (stats, step, i) => { - const label = step.name || `step #${i}` - const start = new Date() - return BB.resolve(step(cache, opts)).then(s => { - s && Object.keys(s).forEach(k => { - stats[k] = s[k] + ] + + return steps + .reduce((promise, step, i) => { + const label = step.name + const start = new Date() + return promise.then((stats) => { + return step(cache, opts).then((s) => { + s && + Object.keys(s).forEach((k) => { + stats[k] = s[k] + }) + const end = new Date() + if (!stats.runTime) { + stats.runTime = {} + } + stats.runTime[label] = end - start + return Promise.resolve(stats) + }) }) - const end = new Date() - if (!stats.runTime) { stats.runTime = {} } - stats.runTime[label] = end - start + }, Promise.resolve({})) + .then((stats) => { + stats.runTime.total = stats.endTime - stats.startTime + opts.log.silly( + 'verify', + 'verification finished for', + cache, + 'in', + `${stats.runTime.total}ms` + ) return stats }) - }, {}).tap(stats => { - stats.runTime.total = stats.endTime - stats.startTime - opts.log.silly('verify', 'verification finished for', cache, 'in', `${stats.runTime.total}ms`) - }) } function markStartTime (cache, opts) { - return { startTime: new Date() } + return Promise.resolve({ startTime: new Date() }) } function markEndTime (cache, opts) { - return { endTime: new Date() } + return Promise.resolve({ endTime: new Date() }) } function fixPerms (cache, opts) { opts.log.silly('verify', 'fixing cache permissions') - return fixOwner.mkdirfix(cache, cache).then(() => { - // TODO - fix file permissions too - return fixOwner.chownr(cache, cache) - }).then(() => null) + return fixOwner + .mkdirfix(cache, cache) + .then(() => { + // TODO - fix file permissions too + return fixOwner.chownr(cache, cache) + }) + .then(() => null) } // Implements a naive mark-and-sweep tracing garbage collector. @@ -84,85 +107,105 @@ function garbageCollect (cache, opts) { opts.log.silly('verify', 'garbage collecting content') const indexStream = index.lsStream(cache) const liveContent = new Set() - indexStream.on('data', entry => { - if (opts.filter && !opts.filter(entry)) { return } + indexStream.on('data', (entry) => { + if (opts.filter && !opts.filter(entry)) { + return + } liveContent.add(entry.integrity.toString()) }) - return finished(indexStream).then(() => { - const contentDir = contentPath._contentDir(cache) + return new Promise((resolve, reject) => { + indexStream.on('end', resolve).on('error', reject) + }).then(() => { + const contentDir = contentPath.contentDir(cache) return glob(path.join(contentDir, '**'), { follow: false, nodir: true, nosort: true - }).then(files => { - return BB.resolve({ + }).then((files) => { + return Promise.resolve({ verifiedContent: 0, reclaimedCount: 0, reclaimedSize: 0, badContentCount: 0, keptSize: 0 - }).tap((stats) => BB.map(files, (f) => { - const split = f.split(/[/\\]/) - const digest = split.slice(split.length - 3).join('') - const algo = split[split.length - 4] - const integrity = ssri.fromHex(digest, algo) - if (liveContent.has(integrity.toString())) { - return verifyContent(f, integrity).then(info => { - if (!info.valid) { - stats.reclaimedCount++ - stats.badContentCount++ - stats.reclaimedSize += info.size + }).then((stats) => + pMap( + files, + (f) => { + const split = f.split(/[/\\]/) + const digest = split.slice(split.length - 3).join('') + const algo = split[split.length - 4] + const integrity = ssri.fromHex(digest, algo) + if (liveContent.has(integrity.toString())) { + return verifyContent(f, integrity).then((info) => { + if (!info.valid) { + stats.reclaimedCount++ + stats.badContentCount++ + stats.reclaimedSize += info.size + } else { + stats.verifiedContent++ + stats.keptSize += info.size + } + return stats + }) } else { - stats.verifiedContent++ - stats.keptSize += info.size + // No entries refer to this content. We can delete. + stats.reclaimedCount++ + return stat(f).then((s) => { + return rimraf(f).then(() => { + stats.reclaimedSize += s.size + return stats + }) + }) } - return stats - }) - } else { - // No entries refer to this content. We can delete. - stats.reclaimedCount++ - return fs.statAsync(f).then(s => { - return rimraf(f).then(() => { - stats.reclaimedSize += s.size - return stats - }) - }) - } - }, { concurrency: opts.concurrency })) + }, + { concurrency: opts.concurrency } + ).then(() => stats) + ) }) }) } function verifyContent (filepath, sri) { - return fs.statAsync(filepath).then(stat => { - const contentInfo = { - size: stat.size, - valid: true - } - return ssri.checkStream( - fs.createReadStream(filepath), - sri - ).catch(err => { - if (err.code !== 'EINTEGRITY') { throw err } - return rimraf(filepath).then(() => { - contentInfo.valid = false - }) - }).then(() => contentInfo) - }).catch({ code: 'ENOENT' }, () => ({ size: 0, valid: false })) + return stat(filepath) + .then((s) => { + const contentInfo = { + size: s.size, + valid: true + } + return ssri + .checkStream(new fsm.ReadStream(filepath), sri) + .catch((err) => { + if (err.code !== 'EINTEGRITY') { + throw err + } + return rimraf(filepath).then(() => { + contentInfo.valid = false + }) + }) + .then(() => contentInfo) + }) + .catch((err) => { + if (err.code === 'ENOENT') { + return { size: 0, valid: false } + } + throw err + }) } function rebuildIndex (cache, opts) { opts.log.silly('verify', 'rebuilding index') - return index.ls(cache).then(entries => { + return index.ls(cache).then((entries) => { const stats = { missingContent: 0, rejectedEntries: 0, totalEntries: 0 } const buckets = {} - for (let k in entries) { - if (entries.hasOwnProperty(k)) { - const hashed = index._hashKey(k) + for (const k in entries) { + /* istanbul ignore else */ + if (hasOwnProperty(entries, k)) { + const hashed = index.hashKey(k) const entry = entries[k] const excluded = opts.filter && !opts.filter(entry) excluded && stats.rejectedEntries++ @@ -172,35 +215,51 @@ function rebuildIndex (cache, opts) { // skip } else if (excluded) { buckets[hashed] = [] - buckets[hashed]._path = index._bucketPath(cache, k) + buckets[hashed]._path = index.bucketPath(cache, k) } else { buckets[hashed] = [entry] - buckets[hashed]._path = index._bucketPath(cache, k) + buckets[hashed]._path = index.bucketPath(cache, k) } } } - return BB.map(Object.keys(buckets), key => { - return rebuildBucket(cache, buckets[key], stats, opts) - }, { concurrency: opts.concurrency }).then(() => stats) + return pMap( + Object.keys(buckets), + (key) => { + return rebuildBucket(cache, buckets[key], stats, opts) + }, + { concurrency: opts.concurrency } + ).then(() => stats) }) } function rebuildBucket (cache, bucket, stats, opts) { - return fs.truncateAsync(bucket._path).then(() => { + return truncate(bucket._path).then(() => { // This needs to be serialized because cacache explicitly // lets very racy bucket conflicts clobber each other. - return BB.mapSeries(bucket, entry => { - const content = contentPath(cache, entry.integrity) - return fs.statAsync(content).then(() => { - return index.insert(cache, entry.key, entry.integrity, { - metadata: entry.metadata, - size: entry.size - }).then(() => { stats.totalEntries++ }) - }).catch({ code: 'ENOENT' }, () => { - stats.rejectedEntries++ - stats.missingContent++ + return bucket.reduce((promise, entry) => { + return promise.then(() => { + const content = contentPath(cache, entry.integrity) + return stat(content) + .then(() => { + return index + .insert(cache, entry.key, entry.integrity, { + metadata: entry.metadata, + size: entry.size + }) + .then(() => { + stats.totalEntries++ + }) + }) + .catch((err) => { + if (err.code === 'ENOENT') { + stats.rejectedEntries++ + stats.missingContent++ + return + } + throw err + }) }) - }) + }, Promise.resolve()) }) } @@ -213,15 +272,16 @@ function writeVerifile (cache, opts) { const verifile = path.join(cache, '_lastverified') opts.log.silly('verify', 'writing verifile to ' + verifile) try { - return fs.writeFileAsync(verifile, '' + (+(new Date()))) + return writeFile(verifile, '' + +new Date()) } finally { fixOwner.chownr.sync(cache, verifile) } } module.exports.lastRun = lastRun + function lastRun (cache) { - return fs.readFileAsync( - path.join(cache, '_lastverified'), 'utf8' - ).then(data => new Date(+data)) + return readFile(path.join(cache, '_lastverified'), 'utf8').then( + (data) => new Date(+data) + ) } |