diff options
| -rw-r--r-- | lib/search.js | 2 | ||||
| -rw-r--r-- | lib/utils/npm-registry-client/get.js | 55 |
2 files changed, 56 insertions, 1 deletions
diff --git a/lib/search.js b/lib/search.js index e5ad155f8..162a1ee3c 100644 --- a/lib/search.js +++ b/lib/search.js @@ -59,7 +59,7 @@ function search (args, silent, staleness, cb_) { } function getFilteredData (staleness, args, notArgs, cb) { - registry.get( "/-/all?stale=update_after", null, staleness, false + registry.get( "/-/all", null, staleness, false , true, function (er, data) { if (er) return cb(er) return cb(null, filter(data, args, notArgs)) diff --git a/lib/utils/npm-registry-client/get.js b/lib/utils/npm-registry-client/get.js index c1eca93d4..de3677949 100644 --- a/lib/utils/npm-registry-client/get.js +++ b/lib/utils/npm-registry-client/get.js @@ -28,6 +28,14 @@ function get (project, version, timeout, nofollow, staleOk, cb) { uri.push(project || "") if (version) uri.push(version) uri = uri.join("/") + + // /-/all is special. + // It uses timestamp-based caching and partial updates, + // because it is a monster. + if (uri === "/-/all") { + return requestAll(cb) + } + var cache = path.join(npm.cache, uri, ".cache.json") fs.stat(cache, function (er, stat) { if (!er) fs.readFile(cache, function (er, data) { @@ -39,6 +47,53 @@ function get (project, version, timeout, nofollow, staleOk, cb) { }) } +function requestAll (cb) { + var cache = path.join(npm.cache, "/-/all", ".cache.json") + + mkdir(path.join(npm.cache, "-", "all"), function (er) { + fs.readFile(cache, function (er, data) { + if (er) return requestAll_(0, {}, cb) + try { + data = JSON.parse(data) + } catch (ex) { + fs.writeFile(cache, "{}", function (er) { + if (er) return cb(new Error("Broken cache. " + +"Please run 'npm cache clean' " + +"and try again.")) + return requestAll_(0, {}, cb) + }) + } + var t = +data._updated || 0 + requestAll_(t, data, cb) + }) + }) +} + +function requestAll_ (c, data, cb) { + // use the cache and update in the background if it's not too old + if (Date.now() - c < 60000) { + cb(null, data) + cb = function () {} + } + + var uri = "/-/all/since?startkey=" + c + , cache = path.join(npm.cache, "-/all", ".cache.json") + GET(uri, function (er, updates, _, res) { + if (er) return cb(er, data) + var headers = res.headers + , updated = Date.parse(headers.date) + Object.keys(updates).forEach(function (p) { + data[p] = updates[p] + }) + data._updated = updated + fs.writeFile( cache, JSON.stringify(data) + , function (er) { + delete data._updated + return cb(er, data) + }) + }) +} + function get_ (uri, timeout, cache, stat, data, nofollow, staleOk, cb) { var etag if (data && data._etag) etag = data._etag |