directly cache search metadata

3 files changed, 245 insertions, 1 deletions

diff --git a/lib/cache/update-index.js b/lib/cache/update-index.js
new file mode 100644
index 000000000..564ec957b
--- /dev/null
+++ b/lib/cache/update-index.js
@@ -0,0 +1,100 @@
+module.exports = updateIndex
+
+var fs = require("graceful-fs")
+  , assert = require("assert")
+  , path = require("path")
+  , mkdir = require("mkdirp")
+  , chownr = require("chownr")
+  , url = require("url")
+  , npm = require("../npm.js")
+  , log = require("npmlog")
+  , cacheFile = require("npm-cache-filename")
+  , getCacheStat = require("./get-stat.js")
+
+/* /-/all is special.
+ * It uses timestamp-based caching and partial updates,
+ * because it is a monster.
+ */
+function updateIndex (uri, params, cb) {
+  assert(typeof uri === "string", "must pass registry URI to getAll")
+  assert(params && typeof params === "object", "must pass params to getAll")
+  assert(typeof cb === "function", "must pass callback to getAll")
+
+  var parsed = url.parse(uri)
+  assert(
+    parsed.protocol === "http:" || parsed.protocol === "https:",
+    "must have a URL that starts with http: or https:"
+  )
+
+  var cacheBase = cacheFile(npm.config.get("cache"))(uri)
+  var cachePath = path.join(cacheBase, ".cache.json")
+  log.info("updateIndex", cachePath)
+
+  getCacheStat(function (er, st) {
+    if (er) return cb(er)
+
+    mkdir(cacheBase, function (er, made) {
+      if (er) return cb(er)
+
+      fs.readFile(cachePath, function (er, data) {
+        if (er) return requestAll_(uri, 0, {}, cachePath, cb)
+
+        try {
+          data = JSON.parse(data)
+        }
+        catch (ex) {
+          fs.writeFile(cachePath, "{}", function (er) {
+            if (er) return cb(new Error("Broken cache."))
+
+            return requestAll_(uri, 0, {}, cachePath, cb)
+          })
+        }
+        var t = +data._updated || 0
+        chownr(made || cachePath, st.uid, st.gid, function (er) {
+          if (er) return cb(er)
+
+          requestAll_(uri, t, data, cachePath, cb)
+        })
+      })
+    })
+  })
+}
+
+function requestAll_ (uri, t, data, cachePath, cb) {
+  // use the cache and update in the background if it's not too old
+  if (Date.now() - t < 60000) {
+    cb(null, data)
+    cb = function () {}
+  }
+
+  var full
+  if (t === 0) {
+    log.warn("", "Building the local index for the first time, please be patient")
+    full = url.resolve(uri, "/-/all")
+  }
+  else {
+    full = url.resolve(uri, "/-/all/since?stale=update_after&startkey=" + t)
+  }
+
+  npm.registry.request(full, {}, function (er, updates, _, res) {
+    if (er) return cb(er, data)
+
+    var headers = res.headers
+    var updated = updates._updated || Date.parse(headers.date)
+
+    Object.keys(updates).forEach(function (p) { data[p] = updates[p] })
+
+    data._updated = updated
+    getCacheStat(function (er, st) {
+      if (er) return cb(er)
+
+      fs.writeFile(cachePath, JSON.stringify(data), function (er) {
+        delete data._updated
+        if (er) return cb(er)
+        chownr(cachePath, st.uid, st.gid, function (er) {
+          cb(er, data)
+        })
+      })
+    })
+  })
+}
diff --git a/lib/search.js b/lib/search.js
index 68139a7ac..ad3f312e5 100644
--- a/lib/search.js
+++ b/lib/search.js
@@ -4,6 +4,7 @@ module.exports = exports = search
 var npm = require("./npm.js")
   , columnify = require("columnify")
   , mapToRegistry = require("./utils/map-to-registry.js")
+  , updateIndex = require("./cache/update-index.js")
 
 search.usage = "npm search [some search terms ...]"
 
@@ -66,7 +67,7 @@ function getFilteredData (staleness, args, notArgs, cb) {
       staleOk : true,
       auth    : auth
     }
-    npm.registry.getAll(uri, params, function (er, data) {
+    updateIndex(uri, params, function (er, data) {
       if (er) return cb(er)
       return cb(null, filter(data, args, notArgs))
     })
diff --git a/test/tap/update-index.js b/test/tap/update-index.js
new file mode 100644
index 000000000..423adfa42
--- /dev/null
+++ b/test/tap/update-index.js
@@ -0,0 +1,143 @@
+var common = require("../common-tap.js")
+var test = require("tap").test
+var npm = require("../../")
+var mkdirp = require("mkdirp")
+var rimraf = require("rimraf")
+var path = require("path")
+var mr = require("npm-registry-mock")
+
+var updateIndex = require("../../lib/cache/update-index.js")
+
+var PKG_DIR = path.resolve(__dirname, "get-basic")
+var CACHE_DIR = path.resolve(PKG_DIR, "cache")
+
+var server
+
+test("setup", function (t) {
+  mkdirp.sync(CACHE_DIR)
+  mr({ port: common.port, mocks: mocks.all }, function (s) {
+    npm.load({cache: CACHE_DIR, registry: common.registry}, function (err) {
+      t.ifError(err)
+      server = s
+      t.end()
+    })
+  })
+})
+
+test("basic request", function (t) {
+  updateIndex("http://localhost:1337/-/all", {}, function (er) {
+    t.ifError(er, "no error")
+    t.end()
+  })
+})
+
+test("cleanup", function (t) {
+  server.close()
+  rimraf.sync(PKG_DIR)
+
+  t.end()
+})
+
+var mocks = {
+  all: function(server) {
+    server.get("/-/all").reply(200, allMock)
+  }
+}
+
+var allMock = {
+  "_updated": 1411727900+25,
+  "generator-frontcow": {
+      "name": "generator-frontcow",
+      "description": "f36b6a6123da50959741e2ce4d634f96ec668c56 This is a fake description to ensure we do not accidentally search the real npm registry or use some kind of cache",
+      "dist-tags": {
+          "latest": "0.1.19"
+      },
+      "maintainers": [
+          {
+              "name": "bcabanes",
+              "email": "contact@benjamincabanes.com"
+          }
+      ],
+      "homepage": "https://github.com/bcabanes/generator-frontcow",
+      "keywords": [
+          "sass",
+          "frontend",
+          "yeoman-generator",
+          "atomic",
+          "design",
+          "sass",
+          "foundation",
+          "foundation5",
+          "atomic design",
+          "bourbon",
+          "polyfill",
+          "font awesome"
+      ],
+      "repository": {
+          "type": "git",
+          "url": "https://github.com/bcabanes/generator-frontcow"
+      },
+      "author": {
+          "name": "ben",
+          "email": "contact@benjamincabanes.com",
+          "url": "https://github.com/bcabanes"
+      },
+      "bugs": {
+          "url": "https://github.com/bcabanes/generator-frontcow/issues"
+      },
+      "license": "MIT",
+      "readmeFilename": "README.md",
+      "time": {
+          "modified": "2014-10-03T02:26:18.406Z"
+      },
+      "versions": {
+          "0.1.19": "latest"
+      }
+  },
+  "marko": {
+      "name": "marko",
+      "description": "Marko is an extensible, streaming, asynchronous, high performance, HTML-based templating language that can be used in Node.js or in the browser.",
+      "dist-tags": {
+          "latest": "1.2.16"
+      },
+      "maintainers": [
+          {
+              "name": "pnidem",
+              "email": "pnidem@gmail.com"
+          },
+          {
+              "name": "philidem",
+              "email": "phillip.idem@gmail.com"
+          }
+      ],
+      "homepage": "https://github.com/raptorjs/marko",
+      "keywords": [
+          "templating",
+          "template",
+          "async",
+          "streaming"
+      ],
+      "repository": {
+          "type": "git",
+          "url": "https://github.com/raptorjs/marko.git"
+      },
+      "author": {
+          "name": "Patrick Steele-Idem",
+          "email": "pnidem@gmail.com"
+      },
+      "bugs": {
+          "url": "https://github.com/raptorjs/marko/issues"
+      },
+      "license": "Apache License v2.0",
+      "readmeFilename": "README.md",
+      "users": {
+          "pnidem": true
+      },
+      "time": {
+          "modified": "2014-10-03T02:27:31.775Z"
+      },
+      "versions": {
+          "1.2.16": "latest"
+      }
+  }
+}