Fix nightly_build_scripts to include OctoPi publish againCustomPiOS

5 files changed, 412 insertions, 0 deletions

diff --git a/.gitignore b/.gitignore
index 585909e..caa2b02 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,6 +3,8 @@ src/custompios_path
 src/image/*.zip
 src/image-variants/*.zip
 **/key.json
+src/nightly_build_scripts/index.html
+src/nightly_build_scripts/node_modules
 src/workspace-*
 src/workspace
 src/build.log
diff --git a/src/nightly_build_scripts/cleanup_storage.js b/src/nightly_build_scripts/cleanup_storage.js
new file mode 100644
index 0000000..971be31
--- /dev/null
+++ b/src/nightly_build_scripts/cleanup_storage.js
@@ -0,0 +1,108 @@
+/**
+ * Usage: node cleanup_storage.js <action> [<keyfile>]
+ *
+ * action:
+ *     "print" or "delete"
+ * keyfile:
+ *     The key.json file to use for authentication
+ *
+ * Setup:
+ *     npm install pkgcloud
+ */
+
+//~~ setup
+
+// imports
+
+var pkgcloud = require('pkgcloud'),
+    fs = require('fs'),
+    path = require('path');
+
+// polyfills
+
+if (!String.prototype.startsWith) {
+  String.prototype.startsWith = function (str) {
+      return !this.indexOf(str);
+  }
+}
+
+if (!String.prototype.endsWith) {
+  String.prototype.endsWith = function(searchString, position) {
+      var subjectString = this.toString();
+      if (typeof position !== 'number' || !isFinite(position) || Math.floor(position) !== position || position > subjectString.length) {
+        position = subjectString.length;
+      }
+      position -= searchString.length;
+      var lastIndex = subjectString.indexOf(searchString, position);
+      return lastIndex !== -1 && lastIndex === position;
+  };
+}
+
+//~~ argument parsing
+
+// "delete" -> delete, "print" -> only print
+if (process.argv.length < 3) {
+  console.log("Missing mandatory action parameter");
+  process.exit();
+}
+var action = process.argv[2];
+
+// key file to use => ./key.json or second command line argument
+var keyfile = path.join(__dirname, 'key.json');
+if (process.argv.length >= 4) {
+  keyfile = process.argv[3];
+}
+
+//~~ helpers
+
+var sortByDate = function(a, b) {
+  if (a.timeCreated < b.timeCreated) return 1;
+  if (a.timeCreated > b.timeCreated) return -1;
+  return 0;
+}
+
+//~~ action and go
+
+// construct client
+var client = require('pkgcloud').storage.createClient({
+  provider: 'google',
+  keyFilename: keyfile, // path to a JSON key file
+});
+var container = "octoprint";
+
+// fetch our files and render our page
+var matchers = [
+  {
+    matcher: function(obj) { return !obj.name.startsWith("stable/") && !obj.name.startsWith("bananapi-m1/") && /octopi-(wheezy|jessie)-/.test(obj.name); },
+    limit: 14
+  },
+  {
+    matcher: function(obj) { return /^bananapi-m1\//.test(obj.name); },
+    limit: 14
+  }
+]
+
+var now = new Date();
+client.getFiles(container, function (err, files) {
+  matchers.forEach(function(m) {
+    var cutoff = new Date();
+    cutoff.setDate(now.getDate() - m.limit);
+
+    var filesToDelete = files.filter(m.matcher)
+                             .filter(function(obj) { return new Date(Date.parse(obj.timeCreated)) < cutoff });
+
+    filesToDelete.forEach(function (file) {
+      if (action == "delete") {
+        client.removeFile(container, encodeURIComponent(file.name), function(err) {
+          if (err) {
+            console.log("Error deleting " + file.name + ": " + err);
+          } else {
+            console.log("Deleted " + file.name + " on " + container);
+          }
+        });
+      } else {
+        console.log("Would now delete " + file.name + " on " + container);
+      }
+    });
+  });
+});
diff --git a/src/nightly_build_scripts/generate_nightly_page.js b/src/nightly_build_scripts/generate_nightly_page.js
new file mode 100644
index 0000000..445f26f
--- /dev/null
+++ b/src/nightly_build_scripts/generate_nightly_page.js
@@ -0,0 +1,214 @@
+/**
+ * Usage: node generate_nightly_page.js [<keyfile> [<outputfile> [<templatefile>]]]
+ *
+ * keyfile:
+ *     The key.json file to use for authentication
+ * outputfile:
+ *     The file where to write the output to
+ * templatefile:
+ *     The HTML template to use, supports the following placeholders:
+ *     - "{{ title }}" - will be replaced with page title
+ *     - "{{ description }}" - will be replaced with page description
+ *     - "{{ content }}" - will be replaced with page content
+ *
+ * Setup:
+ *     npm install pkgcloud
+ * For NodeJS < 0.10 also
+ *     npm install readable-stream
+ */
+
+//~~ setup
+
+// imports
+
+var pkgcloud = require('pkgcloud'),
+    fs = require('fs'),
+    path = require('path'),
+    stream = require('stream'),
+    util = require('util');
+
+// polyfills
+
+if (!String.prototype.startsWith) {
+  String.prototype.startsWith = function (str) {
+      return !this.indexOf(str);
+  }
+}
+
+if (!String.prototype.endsWith) {
+  String.prototype.endsWith = function(searchString, position) {
+      var subjectString = this.toString();
+      if (typeof position !== 'number' || !isFinite(position) || Math.floor(position) !== position || position > subjectString.length) {
+        position = subjectString.length;
+      }
+      position -= searchString.length;
+      var lastIndex = subjectString.indexOf(searchString, position);
+      return lastIndex !== -1 && lastIndex === position;
+  };
+}
+
+//~~ argument parsing
+
+// key file to use => ./key.json or first command line argument
+var keyfile = path.join(__dirname, 'key.json');
+if (process.argv.length >= 3) {
+  keyfile = process.argv[2];
+}
+
+// output file => ./index.html or second command line argument
+var outputfile = path.join(__dirname, 'index.html');
+if (process.argv.length >= 4) {
+  outputfile = process.argv[3];
+}
+
+// template file ==> ./template.html or third command line argument
+var templatefile = path.join(__dirname, 'template.html');
+if (process.argv.length >= 5) {
+  templatefile = process.argv[4];
+}
+
+//~~ helpers
+
+var filterByExtension = function(fileObjs, extensions) {
+  return fileObjs.filter(function (obj) {
+    var name = obj.name;
+    return extensions.some(function (extension) { return name.endsWith(extension); })
+  });
+}
+
+var filterByName = function(fileObjs, name) {
+  return fileObjs.filter(function (obj) { return obj.name.startsWith(name) });
+}
+
+var filterNameByRegex = function(fileObjs, regex) {
+  return fileObjs.filter(function (obj) { return regex.test(obj.name) });
+}
+
+var stripLeading = function(name, toStrip) {
+  return name.substring(toStrip.length);
+}
+
+var sortByDate = function(a, b) {
+  if (a.timeCreated < b.timeCreated) return 1;
+  if (a.timeCreated > b.timeCreated) return -1;
+  return 0;
+}
+
+var formatDate = function(date) {
+  return date.replace(/T/, ' ').replace(/\..+/, '') + " UTC";
+}
+
+var formatSize = function(bytes) {
+  // Formats the given file size in bytes
+  if (!bytes) return "-";
+
+  var units = ["bytes", "KB", "MB"];
+  for (var i = 0; i < units.length; i++) {
+      if (bytes < 1024) {
+          return bytes.toFixed(1) + units[i];
+      }
+      bytes /= 1024;
+  }
+  return bytes.toFixed(1) + "GB";
+}
+
+var convertHash = function(hash) {
+  // Converts a hash from base64 to hex
+  return new Buffer(hash, 'base64').toString('hex');
+}
+
+var outputTable = function(fileObjs, s, nameProcessor, limit) {
+  // Outputs an HTML table to <s> for the provided <fileObjs>, limiting them to <limit>
+  // and preprocessing the filename with <nameProcessor>
+
+  limit = limit || 20;
+
+  s.write('<table class="table table-hover table-bordered">\n');
+  s.write('<tr><th class="name">Name</th><th class="date">Creation Date</th><th class="size">Size</th><th class="md5sum">MD5 Hash</th></tr>\n');
+
+  // sort by date and limit
+  fileObjs.sort(sortByDate).slice(0, limit).forEach(function(fileObj) {
+    console.log("Processing file object: %j", fileObj);
+
+    var url = "https://storage.googleapis.com/octoprint/" + fileObj.name;
+    var name = nameProcessor(fileObj.name);
+
+    s.write('<tr>');
+    s.write('<td class="name"><a href="' + url + '">' + name + "</a></td>");
+    s.write('<td class="date">' + formatDate(fileObj.timeCreated) + "</td>");
+    s.write("<td class='size'>" + formatSize(fileObj.size) + "</td>");
+    s.write("<td class='md5sum'><code>" + convertHash(fileObj.md5Hash) + "</code></td>");
+    s.write("</tr>\n");
+  });
+
+  s.write('</table>\n');
+}
+
+var outputPage = function(files, s) {
+  // Outputs the page for <files> to stream <s>, using the template.
+  var title = "OctoPi Downloads";
+  var description = "OctoPi Downloads";
+
+  var Writable = stream.Writable || require('readable-stream').Writable;
+  function StringStream(options) {
+    Writable.call(this, options);
+    this.buffer = "";
+  }
+  util.inherits(StringStream, Writable);
+  StringStream.prototype._write = function (chunk, enc, cb) {
+    this.buffer += chunk;
+    cb();
+  };
+
+  var output = new StringStream();
+
+  output.write("<ul><li><a href='#rpi'>Raspberry Pi</a><ul><li><a href='#rpi-stable'>Stable Builds</li><li><a href='#rpi-nightly'>Nightly Builds</a></li></ul></li><li><a href='#banana'>Banana Pi M1</a><ul><li><a href='#banana-nightly'>Nightly Builds</a></li></ul></li></ul>");
+
+  output.write("<h2 id='rpi'>Raspberry Pi</h2>\n");
+
+  output.write("<h3 id='rpi-stable'>Stable Builds</h3>\n")
+  outputTable(filterNameByRegex(files, /^stable\/.*octopi-(wheezy|jessie)-.*/),
+              output,
+              function(name) { return stripLeading(name, "stable/") },
+              3);
+
+  output.write("<h3 id='rpi-nightly'>Nightly Builds</h3>\n");
+  output.write("<small>Warning: These builds are untested and can be unstable and/or broken. If in doubt use a stable build.</small>");
+  outputTable(filterNameByRegex(files.filter(function (obj) { return !obj.name.startsWith("stable/") && !obj.name.startsWith("bananapi-m1/") }), /octopi-(wheezy|jessie)-/),
+              output,
+              function(name) { return name },
+              14);
+
+  output.write("<h2 id='banana'>Banana Pi M1</h2>\n")
+
+  output.write("<h3 id='banana-nightly'>Nightly Builds</h3>\n");
+  output.write("<small>Warning: These builds are untested and can be unstable and/or broken.</small>");
+  outputTable(filterNameByRegex(files, /^bananapi-m1\//),
+              output,
+              function(name) { return stripLeading(name, "bananapi-m1/") },
+              14);
+
+  var content = output.buffer;
+  fs.readFile(templatefile, "utf8", function (err, template) {
+    var result = template.replace(/{{ content }}/g, content)
+                         .replace(/{{ title }}/g, title)
+                         .replace(/{{ description }}/g, description);
+    s.write(result);
+  })
+
+}
+
+//~~ action and go
+
+// construct client
+var client = require('pkgcloud').storage.createClient({
+  provider: 'google',
+  keyFilename: keyfile, // path to a JSON key file
+});
+var container = "octoprint";
+
+// fetch our files and render our page
+client.getFiles(container, function (err, files) {
+  var stream = fs.createWriteStream(outputfile);
+  outputPage(filterByExtension(files, [".zip"]), stream);
+});
diff --git a/src/nightly_build_scripts/template.html b/src/nightly_build_scripts/template.html
new file mode 100644
index 0000000..88989d1
--- /dev/null
+++ b/src/nightly_build_scripts/template.html
@@ -0,0 +1,76 @@
+<!DOCTYPE html>
+<!--[if lt IE 7 ]><html class="ie ie6" lang="en"> <![endif]-->
+<!--[if IE 7 ]><html class="ie ie7" lang="en"> <![endif]-->
+<!--[if IE 8 ]><html class="ie ie8" lang="en"> <![endif]-->
+<!--[if (gte IE 9)|!(IE)]><!-->
+<html lang="en" xmlns="http://www.w3.org/1999/html"> <!--<![endif]-->
+<head>
+    <meta charset="utf-8" />
+    <title>{{ title }}</title>
+    <meta name="description" content="{{ description }}">
+
+    <!--[if lt IE 9]>
+    <script src="https://html5shim.googlecode.com/svn/trunk/html5.js"></script>
+    <![endif]-->
+
+    <!-- Enable responsive viewport -->
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+    <!-- Le styles -->
+    <link rel="stylesheet" href="https://octopi.octoprint.org/assets/css/site.css">
+    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/2.3.2/css/bootstrap-responsive.min.css">
+
+    <!-- Le fav and touch icons -->
+    <link rel="shortcut icon" href="https://octopi.octoprint.org/assets/img/tentacle-32x32.png">
+
+
+    <style>
+      .table .name {
+        width: 45%;
+      }
+      .table .size {
+        width: 10%;
+      }
+      .table .date {
+        width: 20%;
+      }
+      .table .md5sum {
+        width: 25%;
+      }
+      .table .size,
+      .table .date,
+      .table .md5sum {
+        text-align: center
+      }
+    </style>
+
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/1.9.1/jquery.min.js"></script>
+</head>
+
+<body>
+
+<div id="wrap">
+    <div id="main">
+      <div class="page-header">
+          <div class="container">
+              <div class="row">
+                  <div class="span12">
+                      <h1>{{ title }}</h1>
+                  </div>
+              </div>
+          </div>
+      </div>
+      <div class="container">
+          <div class="content row-fluid">
+              <div class="span12">
+                  {{ content }}
+              </div>
+          </div>
+      </div>
+    </div>
+</div>
+
+<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/2.3.2/js/bootstrap.min.js"></script>
+
+</body>
+</html>
diff --git a/src/nightly_build_scripts/update_git_mirrors b/src/nightly_build_scripts/update_git_mirrors
new file mode 100755
index 0000000..03346a0
--- /dev/null
+++ b/src/nightly_build_scripts/update_git_mirrors
@@ -0,0 +1,12 @@
+#!/bin/bash
+MIRROR_LOCATION=/var/www/git
+mkdir $MIRROR_LOCATION
+pushd MIRROR_LOCATION
+    for repo in `ls`
+    do
+        pushd $repo
+            git fetch --prune
+            git update-server-info
+        popd
+    done
+popd