merge `bittorrent-client` into this module

When I started the WebTorrent project I thought there were going to
need to be two separate client implementations (bittorrent-client and
webtorrent-client) that would get tied together in a higher-level
module.

Fortunately, this was not necessary because of the awesome “browser”
field support in browserify. By substituting just a few modules, we can
make the same module (webtorrent) work in node AND the browser, with
the same codebase!

So, from now on, you can just `require(‘webtorrent’)` in node or the
browser, and it will just work. You can also `npm install webtorrent`
if you want to use bittorrent in a node app or script. Lastly, you can
`npm install webtorrent -g` if you want to use webtorrent as a command
line app (it installs a `webtorrent` command).

9 files changed, 594 insertions, 15 deletions

diff --git a/test/basic.js b/test/cmd.js
index 0801452..b7c11c0 100644
--- a/test/basic.js
+++ b/test/cmd.js
@@ -1,20 +1,5 @@
 var cp = require('child_process')
 var test = require('tape')
-var WebTorrent = require('../')
-
-/**
- * Extensive bittorrent functionality tests are contained within dependencies like
- * `bittorrent-client`, `bitorrent-protocol`, etc.
- */
-
-test('Module usage (sanity check)', function (t) {
-  var client = new WebTorrent()
-  t.equal(typeof client.add, 'function', 'client.add exists')
-  client.destroy(function () {
-    t.pass('client.destroy works')
-    t.end()
-  })
-})
 
 test('Command line: --help', function (t) {
   t.plan(2)
diff --git a/test/download.js b/test/download.js
new file mode 100644
index 0000000..34997e7
--- /dev/null
+++ b/test/download.js
@@ -0,0 +1,307 @@
+var auto = require('run-auto')
+var BitTorrentClient = require('../')
+var BlockStream = require('block-stream')
+var DHT = require('bittorrent-dht/client')
+var fs = require('fs')
+var parseTorrent = require('parse-torrent')
+var test = require('tape')
+var TrackerServer = require('bittorrent-tracker').Server
+
+var leavesFile = __dirname + '/torrents/Leaves of Grass by Walt Whitman.epub'
+var leavesTorrent = fs.readFileSync(__dirname + '/torrents/leaves.torrent')
+var leavesParsed = parseTorrent(leavesTorrent)
+
+var BLOCK_LENGTH = 16 * 1024
+function writeToStorage (storage, file, cb) {
+  var pieceIndex = 0
+  fs.createReadStream(file)
+    .pipe(new BlockStream(leavesParsed.pieceLength, { nopad: true }))
+    .on('data', function (piece) {
+      var index = pieceIndex
+      pieceIndex += 1
+
+      var blockIndex = 0
+      var s = new BlockStream(BLOCK_LENGTH, { nopad: true })
+      s.on('data', function (block) {
+        var offset = blockIndex * BLOCK_LENGTH
+        blockIndex += 1
+
+        storage.writeBlock(index, offset, block)
+      })
+      s.write(piece)
+      s.end()
+    })
+    .on('end', function () {
+      cb(null)
+    })
+    .on('error', function (err) {
+      cb(err)
+    })
+}
+
+function downloadTrackerTest (t, serverType) {
+  t.plan(8)
+
+  var trackerStartCount = 0
+
+  auto({
+    tracker: function (cb) {
+      var tracker = new TrackerServer(
+        serverType === 'udp' ? { http: false } : { udp: false }
+      )
+
+      tracker.on('error', function (err) {
+        t.fail(err)
+      })
+
+      tracker.on('start', function () {
+        trackerStartCount += 1
+      })
+
+      tracker.listen(function (port) {
+        var announceUrl = serverType === 'http'
+          ? 'http://127.0.0.1:' + port + '/announce'
+          : 'udp://127.0.0.1:' + port
+
+        // Overwrite announce with our local tracker
+        leavesParsed.announce = [ announceUrl ]
+        leavesParsed.announceList = [[ announceUrl ]]
+
+        cb(null, tracker)
+      })
+    },
+
+    client1: ['tracker', function (cb) {
+      var client1 = new BitTorrentClient({ dht: false })
+      client1.on('error', function (err) { t.fail(err) })
+
+      client1.add(leavesParsed)
+
+      client1.on('torrent', function (torrent) {
+        // torrent metadata has been fetched -- sanity check it
+        t.equal(torrent.name, 'Leaves of Grass by Walt Whitman.epub')
+
+        var names = [
+          'Leaves of Grass by Walt Whitman.epub'
+        ]
+
+        t.deepEqual(torrent.files.map(function (file) { return file.name }), names)
+
+        writeToStorage(torrent.storage, leavesFile, function (err) {
+          cb(err, client1)
+        })
+      })
+    }],
+
+    client2: ['client1', function (cb) {
+      var client2 = new BitTorrentClient({ dht: false })
+      client2.on('error', function (err) { t.fail(err) })
+
+      client2.add(leavesParsed)
+
+      client2.on('torrent', function (torrent) {
+        torrent.files.forEach(function (file) {
+          file.createReadStream()
+        })
+
+        torrent.once('done', function () {
+          t.pass('client2 downloaded torrent from client1')
+          cb(null, client2)
+        })
+      })
+    }]
+
+  }, function (err, r) {
+    t.error(err)
+    t.equal(trackerStartCount, 2)
+
+    r.tracker.close(function () {
+      t.pass('tracker closed')
+    })
+    r.client1.destroy(function () {
+      t.pass('client1 destroyed')
+    })
+    r.client2.destroy(function () {
+      t.pass('client2 destroyed')
+    })
+  })
+}
+
+test('Simple download using UDP tracker', function (t) {
+  downloadTrackerTest(t, 'udp')
+})
+
+test('Simple download using HTTP tracker', function (t) {
+  downloadTrackerTest(t, 'http')
+})
+
+test('Simple download using a tracker (only) via a magnet uri', function (t) {
+  t.plan(8)
+
+  var trackerStartCount = 0
+
+  var magnetUri
+  auto({
+    tracker: function (cb) {
+      var tracker = new TrackerServer('udp')
+
+      tracker.on('error', function (err) {
+        t.fail(err)
+      })
+
+      tracker.on('start', function () {
+        trackerStartCount += 1
+      })
+
+      tracker.listen(function (port) {
+        var announceUrl = 'udp://127.0.0.1:' + port
+        leavesParsed.announce = [ announceUrl ]
+        leavesParsed.announceList = [[ announceUrl ]]
+        magnetUri = 'magnet:?xt=urn:btih:' + leavesParsed.infoHash + '&tr=' + encodeURIComponent(announceUrl)
+        cb(null, tracker)
+      })
+    },
+
+    client1: ['tracker', function (cb) {
+      var client1 = new BitTorrentClient({ dht: false })
+      client1.on('error', function (err) { t.fail(err) })
+
+      client1.add(leavesParsed)
+
+      client1.on('torrent', function (torrent) {
+        // torrent metadata has been fetched -- sanity check it
+        t.equal(torrent.name, 'Leaves of Grass by Walt Whitman.epub')
+
+        var names = [
+          'Leaves of Grass by Walt Whitman.epub'
+        ]
+
+        t.deepEqual(torrent.files.map(function (file) { return file.name }), names)
+
+        writeToStorage(torrent.storage, leavesFile, function (err) {
+          cb(err, client1)
+        })
+      })
+    }],
+
+    client2: ['client1', function (cb) {
+      var client2 = new BitTorrentClient({ dht: false })
+      client2.on('error', function (err) { t.fail(err) })
+
+      client2.add(magnetUri)
+
+      client2.on('torrent', function (torrent) {
+        torrent.files.forEach(function (file) {
+          file.createReadStream()
+        })
+
+        torrent.once('done', function () {
+          t.pass('client2 downloaded torrent from client1')
+          cb(null, client2)
+        })
+      })
+    }]
+
+  }, function (err, r) {
+    t.error(err)
+    t.equal(trackerStartCount, 2)
+
+    r.tracker.close(function () {
+      t.pass('tracker closed')
+    })
+    r.client1.destroy(function () {
+      t.pass('client1 destroyed')
+    })
+    r.client2.destroy(function () {
+      t.pass('client2 destroyed')
+    })
+  })
+})
+
+test('Simple download using DHT', function (t) {
+  t.plan(7)
+
+  // no trackers
+  leavesParsed.announce = []
+  leavesParsed.announceList = []
+
+  // TODO: use actual DHT server here, instead of client
+  var dhtServer = new DHT({ bootstrap: false })
+
+  dhtServer.on('error', function (err) {
+    t.fail(err)
+  })
+
+  auto({
+    dhtPort: function (cb) {
+      dhtServer.listen(function (port) {
+        cb(null, port)
+      })
+    },
+    client1: ['dhtPort', function (cb, r) {
+      var client1 = new BitTorrentClient({
+        trackers: false,
+        dht: { bootstrap: '127.0.0.1:' + r.dhtPort }
+      })
+      client1.on('error', function (err) { t.fail(err) })
+
+      client1.add(leavesParsed)
+
+      var announced, wroteStorage
+      function maybeDone (err) {
+        if ((announced && wroteStorage) || err) cb(err, client1)
+      }
+
+      client1.on('torrent', function (torrent) {
+        // torrent metadata has been fetched -- sanity check it
+        t.equal(torrent.name, 'Leaves of Grass by Walt Whitman.epub')
+
+        var names = [ 'Leaves of Grass by Walt Whitman.epub' ]
+        t.deepEqual(torrent.files.map(function (file) { return file.name }), names)
+
+        torrent.on('dhtAnnounce', function () {
+          announced = true
+          maybeDone(null)
+        })
+
+        writeToStorage(torrent.storage, leavesFile, function (err) {
+          wroteStorage = true
+          maybeDone(err)
+        })
+      })
+    }],
+
+    client2: ['client1', function (cb, r) {
+      var client2 = new BitTorrentClient({
+        trackers: false,
+        dht: { bootstrap: '127.0.0.1:' + r.dhtPort }
+      })
+      client2.on('error', function (err) { t.fail(err) })
+
+      client2.add(leavesParsed)
+
+      client2.on('torrent', function (torrent) {
+        torrent.files.forEach(function (file) {
+          file.createReadStream()
+        })
+
+        torrent.once('done', function () {
+          t.pass('client2 downloaded torrent from client1')
+          cb(null, client2)
+        })
+      })
+    }],
+
+  }, function (err, r) {
+    t.error(err)
+    r.client1.destroy(function () {
+      t.pass('client1 destroyed')
+    })
+    r.client2.destroy(function () {
+      t.pass('client2 destroyed')
+    })
+    dhtServer.destroy(function () {
+      t.pass('dht server destroyed')
+    })
+  })
+})
diff --git a/test/metadata.js b/test/metadata.js
new file mode 100644
index 0000000..5c86b79
--- /dev/null
+++ b/test/metadata.js
@@ -0,0 +1,47 @@
+var BitTorrentClient = require('../')
+var parseTorrent = require('parse-torrent')
+var test = require('tape')
+var fs = require('fs')
+
+var leaves = fs.readFileSync(__dirname + '/torrents/leaves.torrent')
+var leavesTorrent = parseTorrent(leaves)
+
+test('ut_metadata transfer', function (t) {
+  t.plan(5)
+
+  var client1 = new BitTorrentClient({ dht: false, trackers: false })
+  var client2 = new BitTorrentClient({ dht: false, trackers: false })
+
+  client1.on('torrent', function (torrent) {
+    t.pass('client1 emits torrent event') // even though it started with metadata
+  })
+
+  // client1 starts with metadata from torrent file
+  client1.add(leaves)
+
+  client1.on('error', function (err) { t.fail(err) })
+  client2.on('error', function (err) { t.fail(err) })
+
+  client1.on('torrent', function (torrent1) {
+    t.deepEqual(torrent1.parsedTorrent.info, leavesTorrent.info)
+
+    // client2 starts with infohash
+    client2.add(leavesTorrent.infoHash)
+
+    client2.on('listening', function (port, torrent2) {
+      // manually add the peer
+      torrent2.addPeer('127.0.0.1:' + client1.torrentPort)
+
+      client2.on('torrent', function () {
+        t.deepEqual(torrent1.parsedTorrent.info, torrent2.parsedTorrent.info)
+
+        client1.destroy(function () {
+          t.pass('client1 destroyed')
+        })
+        client2.destroy(function () {
+          t.pass('client2 destroyed')
+        })
+      })
+    })
+  })
+})
diff --git a/test/multiple.js b/test/multiple.js
new file mode 100644
index 0000000..cf92266
--- /dev/null
+++ b/test/multiple.js
@@ -0,0 +1,60 @@
+/*
+var BitTorrentClient = require('../')
+var test = require('tape')
+var fs = require('fs')
+
+var torrents = [ 'leaves', 'pride' ].map(function (name) {
+  return fs.readFileSync(__dirname + '/torrents/' + name + '.torrent')
+})
+
+// TODO: replace this with a test that can run offline
+test('two simultaneous downloads with dht disabled', function (t) {
+  t.plan(torrents.length * 2)
+
+  var client = new BitTorrentClient({ dht: false })
+  var numDone = 0
+
+  client.on('error', function (err) { t.fail(err.message) })
+
+  torrents.forEach(function (torrent) {
+    client.add(torrent)
+  })
+
+  client.on('torrent', function (torrent) {
+    t.pass('received metadata for torrent ' + torrent.name)
+
+    torrent.once('done', function () {
+      t.pass('done downloading torrent ' + torrent.name)
+
+      if (++numDone >= torrents.length) {
+        client.destroy()
+      }
+    })
+  })
+})
+
+test('two simultaneous downloads with dht enabled', function (t) {
+  t.plan(torrents.length * 2)
+
+  var client = new BitTorrentClient()
+  var numDone = 0
+
+  client.on('error', function (err) { t.fail(err.message) })
+
+  torrents.forEach(function (torrent) {
+    client.add(torrent)
+  })
+
+  client.on('torrent', function (torrent) {
+    t.pass('received metadata for torrent ' + torrent.name)
+
+    torrent.once('done', function () {
+      t.pass('done downloading torrent ' + torrent.name)
+
+      if (++numDone >= torrents.length) {
+        client.destroy()
+      }
+    })
+  })
+})
+*/
diff --git a/test/package.json b/test/package.json
new file mode 100644
index 0000000..3464324
--- /dev/null
+++ b/test/package.json
@@ -0,0 +1,7 @@
+{
+  "name": "test",
+  "version": "0.0.0",
+  "browserify": {
+    "transform": ["brfs"]
+  }
+}
diff --git a/test/rarity-map.js b/test/rarity-map.js
new file mode 100644
index 0000000..5e04ddc
--- /dev/null
+++ b/test/rarity-map.js
@@ -0,0 +1,113 @@
+var RarityMap = require('../lib/rarity-map')
+var BitField = require('bitfield')
+var Swarm = require('bittorrent-swarm')
+var EventEmitter = require('events').EventEmitter
+var test = require('tape')
+var hat = require('hat')
+
+var infoHash = 'd2474e86c95b19b8bcfdb92bc12c9d44667cfa36'
+var peerId1 = '-WW0001-' + hat(48)
+
+test('Rarity map usage', function (t) {
+  t.plan(16)
+
+  var swarm = new Swarm(infoHash, peerId1)
+  var numPieces = 4
+  swarm.wires = [ new EventEmitter(), new EventEmitter() ]
+  swarm.wires.forEach(function (wire) {
+    wire.peerPieces = new BitField(numPieces)
+  })
+  var rarityMap = new RarityMap(swarm, numPieces)
+
+  function validateInitial () {
+    // note that getRarestPiece will return a random piece since they're all equal
+    // so repeat the test several times to reasonably ensure its correctness.
+    var piece = rarityMap.getRarestPiece()
+    t.ok(piece >= 0 && piece < numPieces)
+
+    piece = rarityMap.getRarestPiece()
+    t.ok(piece >= 0 && piece < numPieces)
+
+    piece = rarityMap.getRarestPiece()
+    t.ok(piece >= 0 && piece < numPieces)
+
+    piece = rarityMap.getRarestPiece()
+    t.ok(piece >= 0 && piece < numPieces)
+  }
+
+  // test initial / empty case
+  validateInitial()
+
+  rarityMap.recalculate()
+
+  // test initial / empty case after recalc
+  validateInitial()
+
+  function setPiece (wire, index) {
+    wire.peerPieces.set(index)
+    wire.emit('have', index)
+  }
+
+  setPiece(swarm.wires[0], 0)
+  setPiece(swarm.wires[1], 0)
+
+  setPiece(swarm.wires[0], 1)
+  setPiece(swarm.wires[1], 3)
+
+  // test rarest piece after setting pieces and handling 'have' events
+  var piece = rarityMap.getRarestPiece()
+  t.equal(piece, 2)
+
+  rarityMap.recalculate()
+
+  // test rarest piece after recalc to ensure its the same
+  piece = rarityMap.getRarestPiece()
+  t.equal(piece, 2)
+
+  function addWire () {
+    var wire = new EventEmitter()
+    wire.peerPieces = new BitField(numPieces)
+    wire.peerPieces.set(1)
+    wire.peerPieces.set(2)
+    swarm.wires.push(wire)
+    swarm.emit('wire', wire)
+  }
+
+  addWire()
+  addWire()
+
+  // test rarest piece after adding wires
+  piece = rarityMap.getRarestPiece()
+  t.equal(piece, 3)
+
+  rarityMap.recalculate()
+
+  // test rarest piece after adding wires and recalc
+  piece = rarityMap.getRarestPiece()
+  t.equal(piece, 3)
+
+  function removeWire (index) {
+    var wire = swarm.wires.splice(index, 1)[0]
+    wire.emit('close')
+  }
+
+  removeWire(3)
+  removeWire(1)
+
+  // test rarest piece after removing wires
+  piece = rarityMap.getRarestPiece()
+  t.equal(piece, 3)
+
+  rarityMap.recalculate()
+
+  // test rarest piece after removing wires and recalc
+  piece = rarityMap.getRarestPiece()
+  t.equal(piece, 3)
+
+  // test piece filter func
+  piece = rarityMap.getRarestPiece(function (i) { return i <= 1 })
+  t.equal(piece, 0)
+
+  piece = rarityMap.getRarestPiece(function (i) { return i === 1 || i === 2 })
+  t.equal(piece, 2)
+})
diff --git a/test/storage.js b/test/storage.js
new file mode 100644
index 0000000..6dfc04b
--- /dev/null
+++ b/test/storage.js
@@ -0,0 +1,60 @@
+var Storage = require('../lib/storage')
+var parseTorrent = require('parse-torrent')
+var test = require('tape')
+var fs = require('fs')
+
+var torrents = [ 'leaves', 'pride' ].map(function (name) {
+  var torrent = fs.readFileSync(__dirname + '/torrents/' + name + '.torrent')
+
+  return {
+    name : name,
+    torrent : torrent,
+    parsedTorrent : parseTorrent(torrent)
+  }
+})
+
+torrents.forEach(function (torrent) {
+  test('sanity check backing storage for ' + torrent.name + ' torrent', function (t) {
+    var parsedTorrent = torrent.parsedTorrent
+    var storage = new Storage(parsedTorrent)
+
+    t.equal(storage.files.length, parsedTorrent.files.length)
+    t.equal(storage.pieces.length, parsedTorrent.pieces.length)
+
+    var length = 0, pieces = 0
+
+    storage.pieces.forEach(function (piece, index) {
+      t.notOk(piece.verified)
+      length += piece.length
+
+      // ensure all blocks start out empty
+      for (var i = 0; i < piece.blocks.length; ++i) {
+        t.equal(piece.blocks[i], 0)
+      }
+    })
+
+    t.equal(length, parsedTorrent.length)
+    length = 0
+
+    storage.files.forEach(function (file, index) {
+      t.notOk(file.done)
+      length += file.length
+      pieces += file.pieces.length
+
+      t.assert(file.length >= 0)
+      t.assert(file.pieces.length >= 0)
+    })
+
+    t.equal(length, parsedTorrent.length)
+
+    if (parsedTorrent.files.length > 1) {
+      // if the torrent contains multiple files, the pieces may overlap file boundaries,
+      // so the aggregate number of file pieces will be at least the number of pieces.
+      t.assert(pieces >= parsedTorrent.pieces.length)
+    } else {
+      t.equal(pieces, parsedTorrent.pieces.length)
+    }
+
+    t.end()
+  })
+})
diff --git a/test/torrents/Leaves of Grass by Walt Whitman.epub b/test/torrents/Leaves of Grass by Walt Whitman.epub
new file mode 100644
index 0000000..66791ed
--- /dev/null
+++ b/test/torrents/Leaves of Grass by Walt Whitman.epub
diff --git a/test/torrents/pride.torrent b/test/torrents/pride.torrent
new file mode 100644
index 0000000..a9bf635
--- /dev/null
+++ b/test/torrents/pride.torrent