diff options
author | Forrest L Norvell <forrest@npmjs.com> | 2014-10-31 06:32:32 +0300 |
---|---|---|
committer | Forrest L Norvell <forrest@npmjs.com> | 2014-10-31 11:18:12 +0300 |
commit | ed619714c93718b6c1922b8c286f4b6cd2b97c80 (patch) | |
tree | 738d6648c45cb8f092ae40f3f0e4b41e42082f03 | |
parent | 239c3086cc1cbee651f9ec85609b8dd3ab8258d7 (diff) |
directly cache search metadata
-rw-r--r-- | lib/cache/update-index.js | 100 | ||||
-rw-r--r-- | lib/search.js | 3 | ||||
-rw-r--r-- | test/tap/update-index.js | 143 |
3 files changed, 245 insertions, 1 deletions
diff --git a/lib/cache/update-index.js b/lib/cache/update-index.js new file mode 100644 index 000000000..564ec957b --- /dev/null +++ b/lib/cache/update-index.js @@ -0,0 +1,100 @@ +module.exports = updateIndex + +var fs = require("graceful-fs") + , assert = require("assert") + , path = require("path") + , mkdir = require("mkdirp") + , chownr = require("chownr") + , url = require("url") + , npm = require("../npm.js") + , log = require("npmlog") + , cacheFile = require("npm-cache-filename") + , getCacheStat = require("./get-stat.js") + +/* /-/all is special. + * It uses timestamp-based caching and partial updates, + * because it is a monster. + */ +function updateIndex (uri, params, cb) { + assert(typeof uri === "string", "must pass registry URI to getAll") + assert(params && typeof params === "object", "must pass params to getAll") + assert(typeof cb === "function", "must pass callback to getAll") + + var parsed = url.parse(uri) + assert( + parsed.protocol === "http:" || parsed.protocol === "https:", + "must have a URL that starts with http: or https:" + ) + + var cacheBase = cacheFile(npm.config.get("cache"))(uri) + var cachePath = path.join(cacheBase, ".cache.json") + log.info("updateIndex", cachePath) + + getCacheStat(function (er, st) { + if (er) return cb(er) + + mkdir(cacheBase, function (er, made) { + if (er) return cb(er) + + fs.readFile(cachePath, function (er, data) { + if (er) return requestAll_(uri, 0, {}, cachePath, cb) + + try { + data = JSON.parse(data) + } + catch (ex) { + fs.writeFile(cachePath, "{}", function (er) { + if (er) return cb(new Error("Broken cache.")) + + return requestAll_(uri, 0, {}, cachePath, cb) + }) + } + var t = +data._updated || 0 + chownr(made || cachePath, st.uid, st.gid, function (er) { + if (er) return cb(er) + + requestAll_(uri, t, data, cachePath, cb) + }) + }) + }) + }) +} + +function requestAll_ (uri, t, data, cachePath, cb) { + // use the cache and update in the background if it's not too old + if (Date.now() - t < 60000) { + cb(null, data) + cb = function () {} + } + + var full + if (t === 0) { + log.warn("", "Building the local index for the first time, please be patient") + full = url.resolve(uri, "/-/all") + } + else { + full = url.resolve(uri, "/-/all/since?stale=update_after&startkey=" + t) + } + + npm.registry.request(full, {}, function (er, updates, _, res) { + if (er) return cb(er, data) + + var headers = res.headers + var updated = updates._updated || Date.parse(headers.date) + + Object.keys(updates).forEach(function (p) { data[p] = updates[p] }) + + data._updated = updated + getCacheStat(function (er, st) { + if (er) return cb(er) + + fs.writeFile(cachePath, JSON.stringify(data), function (er) { + delete data._updated + if (er) return cb(er) + chownr(cachePath, st.uid, st.gid, function (er) { + cb(er, data) + }) + }) + }) + }) +} diff --git a/lib/search.js b/lib/search.js index 68139a7ac..ad3f312e5 100644 --- a/lib/search.js +++ b/lib/search.js @@ -4,6 +4,7 @@ module.exports = exports = search var npm = require("./npm.js") , columnify = require("columnify") , mapToRegistry = require("./utils/map-to-registry.js") + , updateIndex = require("./cache/update-index.js") search.usage = "npm search [some search terms ...]" @@ -66,7 +67,7 @@ function getFilteredData (staleness, args, notArgs, cb) { staleOk : true, auth : auth } - npm.registry.getAll(uri, params, function (er, data) { + updateIndex(uri, params, function (er, data) { if (er) return cb(er) return cb(null, filter(data, args, notArgs)) }) diff --git a/test/tap/update-index.js b/test/tap/update-index.js new file mode 100644 index 000000000..423adfa42 --- /dev/null +++ b/test/tap/update-index.js @@ -0,0 +1,143 @@ +var common = require("../common-tap.js") +var test = require("tap").test +var npm = require("../../") +var mkdirp = require("mkdirp") +var rimraf = require("rimraf") +var path = require("path") +var mr = require("npm-registry-mock") + +var updateIndex = require("../../lib/cache/update-index.js") + +var PKG_DIR = path.resolve(__dirname, "get-basic") +var CACHE_DIR = path.resolve(PKG_DIR, "cache") + +var server + +test("setup", function (t) { + mkdirp.sync(CACHE_DIR) + mr({ port: common.port, mocks: mocks.all }, function (s) { + npm.load({cache: CACHE_DIR, registry: common.registry}, function (err) { + t.ifError(err) + server = s + t.end() + }) + }) +}) + +test("basic request", function (t) { + updateIndex("http://localhost:1337/-/all", {}, function (er) { + t.ifError(er, "no error") + t.end() + }) +}) + +test("cleanup", function (t) { + server.close() + rimraf.sync(PKG_DIR) + + t.end() +}) + +var mocks = { + all: function(server) { + server.get("/-/all").reply(200, allMock) + } +} + +var allMock = { + "_updated": 1411727900+25, + "generator-frontcow": { + "name": "generator-frontcow", + "description": "f36b6a6123da50959741e2ce4d634f96ec668c56 This is a fake description to ensure we do not accidentally search the real npm registry or use some kind of cache", + "dist-tags": { + "latest": "0.1.19" + }, + "maintainers": [ + { + "name": "bcabanes", + "email": "contact@benjamincabanes.com" + } + ], + "homepage": "https://github.com/bcabanes/generator-frontcow", + "keywords": [ + "sass", + "frontend", + "yeoman-generator", + "atomic", + "design", + "sass", + "foundation", + "foundation5", + "atomic design", + "bourbon", + "polyfill", + "font awesome" + ], + "repository": { + "type": "git", + "url": "https://github.com/bcabanes/generator-frontcow" + }, + "author": { + "name": "ben", + "email": "contact@benjamincabanes.com", + "url": "https://github.com/bcabanes" + }, + "bugs": { + "url": "https://github.com/bcabanes/generator-frontcow/issues" + }, + "license": "MIT", + "readmeFilename": "README.md", + "time": { + "modified": "2014-10-03T02:26:18.406Z" + }, + "versions": { + "0.1.19": "latest" + } + }, + "marko": { + "name": "marko", + "description": "Marko is an extensible, streaming, asynchronous, high performance, HTML-based templating language that can be used in Node.js or in the browser.", + "dist-tags": { + "latest": "1.2.16" + }, + "maintainers": [ + { + "name": "pnidem", + "email": "pnidem@gmail.com" + }, + { + "name": "philidem", + "email": "phillip.idem@gmail.com" + } + ], + "homepage": "https://github.com/raptorjs/marko", + "keywords": [ + "templating", + "template", + "async", + "streaming" + ], + "repository": { + "type": "git", + "url": "https://github.com/raptorjs/marko.git" + }, + "author": { + "name": "Patrick Steele-Idem", + "email": "pnidem@gmail.com" + }, + "bugs": { + "url": "https://github.com/raptorjs/marko/issues" + }, + "license": "Apache License v2.0", + "readmeFilename": "README.md", + "users": { + "pnidem": true + }, + "time": { + "modified": "2014-10-03T02:27:31.775Z" + }, + "versions": { + "1.2.16": "latest" + } + } +} |