Add connection test for pods in the network

* add a class for checking pod connectivity
* extend pod model to handle new functionality
* add an admin frontend to list pods and re-trigger checks manually
* add a daily worker to run through all the pods
* add unit tests for most of the new code

29 files changed, 1196 insertions, 28 deletions

diff --git a/app/assets/javascripts/app/collections/pods.js b/app/assets/javascripts/app/collections/pods.js
new file mode 100644
index 000000000..18573be99
--- /dev/null
+++ b/app/assets/javascripts/app/collections/pods.js
@@ -0,0 +1,9 @@
+// @license magnet:?xt=urn:btih:0b31508aeb0634b347b8270c7bee4d411b5d4109&dn=agpl-3.0.txt AGPL-v3-or-Later
+app.collections.Pods = Backbone.Collection.extend({
+  model: app.models.Pod,
+
+  comparator: function(model) {
+    return model.get("host").toLowerCase();
+  }
+});
+// @license-end
diff --git a/app/assets/javascripts/app/models/pod.js b/app/assets/javascripts/app/models/pod.js
new file mode 100644
index 000000000..5ea497df5
--- /dev/null
+++ b/app/assets/javascripts/app/models/pod.js
@@ -0,0 +1,15 @@
+// @license magnet:?xt=urn:btih:0b31508aeb0634b347b8270c7bee4d411b5d4109&dn=agpl-3.0.txt AGPL-v3-or-Later
+app.models.Pod = Backbone.Model.extend({
+  urlRoot: Routes.adminPods(),
+
+  recheck: function() {
+    var self = this,
+        url  = Routes.adminPodRecheck(this.id).toString();
+
+    return $.ajax({url: url, method: "POST", dataType: "json"})
+      .done(function(newAttributes) {
+        self.set(newAttributes);
+      });
+  }
+});
+// @license-end
diff --git a/app/assets/javascripts/app/pages/admin_pods.js b/app/assets/javascripts/app/pages/admin_pods.js
new file mode 100644
index 000000000..b91204e69
--- /dev/null
+++ b/app/assets/javascripts/app/pages/admin_pods.js
@@ -0,0 +1,45 @@
+// @license magnet:?xt=urn:btih:0b31508aeb0634b347b8270c7bee4d411b5d4109&dn=agpl-3.0.txt AGPL-v3-or-Later
+
+app.pages.AdminPods = app.views.Base.extend({
+  templateName: "pod_table",
+
+  initialize: function() {
+    this.pods = new app.collections.Pods(app.parsePreload("pods"));
+    this.rows = []; // contains the table row views
+  },
+
+  postRenderTemplate: function() {
+    var self = this;
+    this._showMessages();
+
+    // avoid reflowing the page for every entry
+    var fragment = document.createDocumentFragment();
+    this.pods.each(function(pod) {
+      self.rows.push(new app.views.PodEntry({
+        parent: fragment,
+        model: pod
+      }).render());
+    });
+    this.$("tbody").append(fragment);
+
+    return this;
+  },
+
+  _showMessages: function() {
+    var msgs = document.createDocumentFragment();
+    if( gon.uncheckedCount && gon.uncheckedCount > 0 ) {
+      var unchecked = $("<div class='alert alert-info' />")
+        .append(Diaspora.I18n.t("admin.pods.unchecked", {count: gon.uncheckedCount}));
+      msgs.appendChild(unchecked[0]);
+    }
+    if( gon.errorCount && gon.errorCount > 0 ) {
+      var errors = $("<div class='alert alert-danger' />")
+        .append(Diaspora.I18n.t("admin.pods.errors", {count: gon.errorCount}));
+        msgs.appendChild(errors[0]);
+    }
+
+    $("#pod-alerts").html(msgs);
+  }
+});
+
+// @license-end
diff --git a/app/assets/javascripts/app/router.js b/app/assets/javascripts/app/router.js
index 8bce2b284..ee19dbd68 100644
--- a/app/assets/javascripts/app/router.js
+++ b/app/assets/javascripts/app/router.js
@@ -11,6 +11,7 @@ app.Router = Backbone.Router.extend({
     "users/sign_up": "registration",
     "profile/edit": "settings",
     "admins/dashboard": "adminDashboard",
+    "admin/pods": "adminPods",
 
     //new hotness
     "posts/:id": "singlePost",
@@ -52,6 +53,14 @@ app.Router = Backbone.Router.extend({
     app.page = new app.pages.AdminDashboard();
   },
 
+  adminPods: function() {
+    this.renderPage(function() {
+      return new app.pages.AdminPods({
+        el: $("#pod-list")
+      });
+    });
+  },
+
   contacts: function() {
     app.aspect = new app.models.Aspect(gon.preloads.aspect);
     app.contacts = new app.collections.Contacts(app.parsePreload("contacts"));
diff --git a/app/assets/javascripts/app/views.js b/app/assets/javascripts/app/views.js
index 043800d08..36d85aeb7 100644
--- a/app/assets/javascripts/app/views.js
+++ b/app/assets/javascripts/app/views.js
@@ -119,12 +119,14 @@ app.views.Base = Backbone.View.extend({
     });
   },
 
+  destroyConfirmMsg: function() { return Diaspora.I18n.t("confirm_dialog"); },
+
   destroyModel: function(evt) {
     evt && evt.preventDefault();
     var self = this;
     var url = this.model.urlRoot + '/' + this.model.id;
 
-    if (confirm(Diaspora.I18n.t("confirm_dialog"))) {
+    if( confirm(_.result(this, "destroyConfirmMsg")) ) {
       this.$el.addClass('deleting');
       this.model.destroy({ url: url })
         .done(function() {
diff --git a/app/assets/javascripts/app/views/pod_entry_view.js b/app/assets/javascripts/app/views/pod_entry_view.js
new file mode 100644
index 000000000..01ca99490
--- /dev/null
+++ b/app/assets/javascripts/app/views/pod_entry_view.js
@@ -0,0 +1,88 @@
+// @license magnet:?xt=urn:btih:0b31508aeb0634b347b8270c7bee4d411b5d4109&dn=agpl-3.0.txt AGPL-v3-or-Later
+
+app.views.PodEntry = app.views.Base.extend({
+  templateName: "pod_table_entry",
+
+  tagName: "tr",
+
+  events: {
+    "click .more": "toggleMore",
+    "click .recheck": "recheckPod"
+  },
+
+  className: function() {
+    if( this.model.get("offline") ) { return "bg-danger"; }
+    if( this.model.get("status")==="version_failed" ) { return "bg-warning"; }
+    if( this.model.get("status")==="no_errors" ) { return "bg-success"; }
+  },
+
+  initialize: function(opts) {
+    this.parent = opts.parent;
+    this.rendered = false;
+    this.model.on("change", this.render, this);
+  },
+
+  presenter: function() {
+    return _.extend({}, this.defaultPresenter(), {
+      /* jshint camelcase: false */
+      is_unchecked: (this.model.get("status")==="unchecked"),
+      has_no_errors: (this.model.get("status")==="no_errors"),
+      has_errors: (this.model.get("status")!=="no_errors"),
+      status_text: Diaspora.I18n.t("admin.pods.states."+this.model.get("status")),
+      pod_url: (this.model.get("ssl") ? "https" : "http") + "://" + this.model.get("host"),
+      response_time_fmt: this._fmtResponseTime()
+      /* jshint camelcase: true */
+    });
+  },
+
+  postRenderTemplate: function() {
+    if( !this.rendered ) {
+      this.parent.appendChild(this.el);
+    }
+
+    this.rendered = true;
+    return this;
+  },
+
+  toggleMore: function() {
+    this.$(".details").toggle();
+    return false;
+  },
+
+  recheckPod: function() {
+    var self  = this,
+        flash = new Diaspora.Widgets.FlashMessages();
+    this.$el.addClass("checking");
+
+    this.model.recheck()
+      .done(function(){
+        flash.render({
+          success: true,
+          notice: Diaspora.I18n.t("admin.pods.recheck.success")
+        });
+      })
+      .fail(function(){
+        flash.render({
+          success: false,
+          notice: Diaspora.I18n.t("admin.pods.recheck.failure")
+        });
+      })
+      .always(function(){
+        self.$el
+          .removeClass("bg-danger bg-warning bg-success")
+          .addClass(_.result(self, "className"))
+          .removeClass("checking");
+      });
+
+    return false;
+  },
+
+  _fmtResponseTime: function() {
+    if( this.model.get("response_time")===-1 ) {
+      return Diaspora.I18n.t("admin.pods.not_available");
+    }
+    return Diaspora.I18n.t("admin.pods.ms", {count: this.model.get("response_time")});
+  }
+});
+
+// @license-end
diff --git a/app/assets/stylesheets/admin.scss b/app/assets/stylesheets/admin.scss
index 76006defc..f796777a6 100644
--- a/app/assets/stylesheets/admin.scss
+++ b/app/assets/stylesheets/admin.scss
@@ -1,4 +1,5 @@
 @import 'colors';
+@import 'new_styles/animations';
 
 /** ADMIN STYlES **/
 /** user search **/
@@ -33,3 +34,33 @@
 /** reported posts **/
 
 @import 'report';
+
+/** pod list **/
+
+#pod-list {
+  th.added,
+  td.added,
+  td.actions { white-space: nowrap; }
+
+  tr.deleting {
+    opacity: .5;
+  }
+
+  tr.checking .recheck i {
+    animation-duration: .4s;
+    animation-name: pulsate;
+    animation-iteration-count: infinite;
+    animation-direction: alternate;
+  }
+
+  td.actions {
+    text-align: right;
+
+    a { color: inherit; }
+  }
+
+  pre.details {
+    margin-top: 1em;
+    white-space: normal;
+  }
+}
diff --git a/app/assets/stylesheets/new_styles/_animations.scss b/app/assets/stylesheets/new_styles/_animations.scss
index 0c65e1a83..64b47cf20 100644
--- a/app/assets/stylesheets/new_styles/_animations.scss
+++ b/app/assets/stylesheets/new_styles/_animations.scss
@@ -11,3 +11,12 @@
   0%   { transform: rotate(0deg); }
   100% { transform: rotate(360deg); }
 }
+
+@keyframes pulsate {
+  from {
+    opacity: 1;
+  }
+  to {
+    opacity: .1;
+  }
+}
diff --git a/app/assets/templates/pod_table_entry_tpl.jst.hbs b/app/assets/templates/pod_table_entry_tpl.jst.hbs
new file mode 100644
index 000000000..a7ebfe6ae
--- /dev/null
+++ b/app/assets/templates/pod_table_entry_tpl.jst.hbs
@@ -0,0 +1,41 @@
+
+<td>
+  {{#if ssl}}
+    <i title="{{t 'admin.pods.ssl_enabled'}}" class="entypo-check">
+  {{else}}
+    <i title="{{t 'admin.pods.ssl_disabled'}}" class="entypo-block">
+  {{/if}}
+  </i>
+</td>
+<td>{{host}}</td>
+<td class="added">
+  <small><time datetime="{{created_at}}" title="{{localTime created_at}}" /></small>
+</td>
+<td>
+  {{#if has_no_errors}}
+    <i title="{{status_text}}" class="glyphicon glyphicon-ok"></i>
+  {{else}}
+    {{status_text}}
+  {{/if}}
+  {{#unless is_unchecked}}
+    <br><small>{{t 'admin.pods.last_check'}} <time datetime="{{checked_at}}" title="{{localTime checked_at}}" /></small>
+  {{/unless}}
+  {{#if offline}}
+    | <small>{{t 'admin.pods.offline_since'}} <time datetime="{{offline_since}}" title="{{localTime offline_since}}" /></small>
+  {{/if}}
+  {{#if is_unchecked}}<br><small class="text-muted">{{t 'admin.pods.no_info'}}</small>{{/if}}
+  <pre class="details" style="display: none;">
+    {{#unless is_unchecked}}
+      {{t 'admin.pods.server_software'}} {{#if software}}{{software}}{{else}}{{t 'admin.pods.unknown'}}{{/if}}
+      <br>{{t 'admin.pods.response_time'}} {{response_time_fmt}}
+      {{#if has_errors}}<br>{{error}}{{/if}}
+    {{/unless}}
+  </pre>
+</td>
+<td class="actions">
+  {{#unless is_unchecked}}
+    <a class="more" href="#"><i title="{{t 'admin.pods.more_info'}}" class="entypo-circled-help"></i></a>
+  {{/unless}}
+  <a class="recheck" href="{{urlTo 'adminPodRecheck' id}}"><i title="{{t 'admin.pods.check'}}" class="entypo-cycle"></i></a>
+  <a href="{{pod_url}}" target="_blank"><i title="{{t 'admin.pods.follow_link'}}" class="entypo-forward"></i></a>
+</td>
diff --git a/app/assets/templates/pod_table_tpl.jst.hbs b/app/assets/templates/pod_table_tpl.jst.hbs
new file mode 100644
index 000000000..a37fea525
--- /dev/null
+++ b/app/assets/templates/pod_table_tpl.jst.hbs
@@ -0,0 +1,14 @@
+
+<table class="table">
+  <thead>
+    <tr>
+      <th><i title="{{t 'admin.pods.ssl'}}" class="glyphicon glyphicon-lock"></i></th>
+      <th>{{t 'admin.pods.pod'}}</th>
+      <th class="added">{{t 'admin.pods.added'}}</th>
+      <th>{{t 'admin.pods.status'}}</th>
+      <th><i title="{{t 'admin.pods.actions'}}" class="entypo-tools pull-right"></i></th>
+    </tr>
+  </thead>
+  <tbody>
+  </tbody>
+</table>
diff --git a/app/controllers/admin/pods_controller.rb b/app/controllers/admin/pods_controller.rb
new file mode 100644
index 000000000..3f81dd582
--- /dev/null
+++ b/app/controllers/admin/pods_controller.rb
@@ -0,0 +1,31 @@
+
+module Admin
+  class PodsController < AdminController
+    respond_to :html, :json
+
+    def index
+      pods_json = PodPresenter.as_collection(Pod.all)
+
+      respond_with do |format|
+        format.html do
+          gon.preloads[:pods] = pods_json
+          gon.unchecked_count = Pod.unchecked.count
+          gon.error_count = Pod.check_failed.count
+
+          render "admins/pods"
+        end
+        format.json { render json: pods_json }
+      end
+    end
+
+    def recheck
+      pod = Pod.find(params[:pod_id])
+      pod.test_connection!
+
+      respond_with do |format|
+        format.html { redirect_to admin_pods_path }
+        format.json { render json: PodPresenter.new(pod).as_json }
+      end
+    end
+  end
+end
diff --git a/app/models/pod.rb b/app/models/pod.rb
index dcd23b310..1d57aec58 100644
--- a/app/models/pod.rb
+++ b/app/models/pod.rb
@@ -1,11 +1,100 @@
 class Pod < ActiveRecord::Base
-  def self.find_or_create_by(opts) # Rename this method to not override an AR method
-    u = URI.parse(opts.fetch(:url))
-    pod = self.find_or_initialize_by(host: u.host)
-    unless pod.persisted?
-      pod.ssl = (u.scheme == 'https')? true : false
-      pod.save
-    end
-    pod
+  enum status: %i(
+    unchecked
+    no_errors
+    dns_failed
+    net_failed
+    ssl_failed
+    http_failed
+    version_failed
+    unknown_error
+  )
+
+  ERROR_MAP = {
+    ConnectionTester::AddressFailure  => :dns_failed,
+    ConnectionTester::DNSFailure      => :dns_failed,
+    ConnectionTester::NetFailure      => :net_failed,
+    ConnectionTester::SSLFailure      => :ssl_failed,
+    ConnectionTester::HTTPFailure     => :http_failed,
+    ConnectionTester::NodeInfoFailure => :version_failed
+  }
+
+  scope :check_failed, lambda {
+    where(arel_table[:status].gt(Pod.statuses[:no_errors]))
+  }
+
+  class << self
+    def find_or_create_by(opts) # Rename this method to not override an AR method
+      u = URI.parse(opts.fetch(:url))
+      find_or_initialize_by(host: u.host).tap do |pod|
+        unless pod.persisted?
+          pod.ssl = (u.scheme == "https")
+          pod.save
+        end
+      end
+    end
+
+    # don't consider a failed version reading to be fatal
+    def offline_statuses
+      [Pod.statuses[:dns_failed],
+       Pod.statuses[:net_failed],
+       Pod.statuses[:ssl_failed],
+       Pod.statuses[:http_failed],
+       Pod.statuses[:unknown_error]]
+    end
+
+    def check_all!
+      Pod.find_in_batches(batch_size: 20).each(&:test_connection!)
+    end
+  end
+
+  def offline?
+    Pod.offline_statuses.include?(Pod.statuses[status])
+  end
+
+  def was_offline?
+    Pod.offline_statuses.include?(Pod.statuses[status_was])
+  end
+
+  def test_connection!
+    url = "#{ssl ? 'https' : 'http'}://#{host}"
+    result = ConnectionTester.check url
+    logger.info "testing pod: '#{url}' - #{result.inspect}"
+
+    transaction do
+      update_from_result(result)
+    end
+  end
+
+  private
+
+  def update_from_result(result)
+    self.status = status_from_result(result)
+
+    if offline?
+      touch(:offline_since) unless was_offline?
+      logger.warn "OFFLINE #{result.failure_message}"
+    else
+      self.offline_since = nil
+    end
+
+    attributes_from_result(result)
+    touch(:checked_at)
+
+    save
+  end
+
+  def attributes_from_result(result)
+    self.error = result.failure_message[0..254] if result.error?
+    self.software = result.software_version[0..254] if result.software_version.present?
+    self.response_time = result.rt
+  end
+
+  def status_from_result(result)
+    if result.error?
+      ERROR_MAP.fetch(result.error.class, :unknown_error)
+    else
+      :no_errors
+    end
   end
 end
diff --git a/app/presenters/pod_presenter.rb b/app/presenters/pod_presenter.rb
new file mode 100644
index 000000000..c59823c39
--- /dev/null
+++ b/app/presenters/pod_presenter.rb
@@ -0,0 +1,19 @@
+class PodPresenter < BasePresenter
+  def base_hash(*_arg)
+    {
+      id:            id,
+      host:          host,
+      ssl:           ssl,
+      status:        status,
+      checked_at:    checked_at,
+      response_time: response_time,
+      offline:       offline?,
+      offline_since: offline_since,
+      created_at:    created_at,
+      software:      software,
+      error:         error
+    }
+  end
+
+  alias_method :as_json, :base_hash
+end
diff --git a/app/views/admins/_admin_bar.haml b/app/views/admins/_admin_bar.haml
index b6c27d428..c2187bf4e 100644
--- a/app/views/admins/_admin_bar.haml
+++ b/app/views/admins/_admin_bar.haml
@@ -5,15 +5,17 @@
 
 %ul#admin_nav.nav.nav-pills.nav-stacked
   %li{role: "presentation", class: current_page?(admin_dashboard_path) && "active"}
-    = link_to t('.dashboard'), admin_dashboard_path
+    = link_to t(".dashboard"), admin_dashboard_path
   %li{role: "presentation", class: current_page?(user_search_path) && "active"}
-    = link_to t('.user_search'), user_search_path
+    = link_to t(".user_search"), user_search_path
   %li{role: "presentation", class: current_page?(weekly_user_stats_path) && "active"}
-    = link_to t('.weekly_user_stats'), weekly_user_stats_path
+    = link_to t(".weekly_user_stats"), weekly_user_stats_path
   %li{role: "presentation", class: current_page?(pod_stats_path) && "active"}
-    = link_to t('.pod_stats'), pod_stats_path
+    = link_to t(".pod_stats"), pod_stats_path
   %li{role: "presentation", class: current_page?(report_index_path) && "active"}
-    = link_to t('.report'), report_index_path
+    = link_to t(".report"), report_index_path
+  %li{role: "presentation", class: current_page?(admin_pods_path) && "active"}
+    = link_to t(".pod_network"), admin_pods_path
   %li{role: "presentation", class: current_page?(sidekiq_path) && "active"}
-    = link_to t('.sidekiq_monitor'), sidekiq_path
+    = link_to t(".sidekiq_monitor"), sidekiq_path
 
diff --git a/app/views/admins/pods.html.haml b/app/views/admins/pods.html.haml
new file mode 100644
index 000000000..cb7058543
--- /dev/null
+++ b/app/views/admins/pods.html.haml
@@ -0,0 +1,14 @@
+.container
+  .row
+    .col-md-3
+      = render partial: "admins/admin_bar"
+
+    .col-md-9
+      %h2
+        = t(".pod_network")
+
+      #pod-alerts
+        / filled by backbonejs
+
+      #pod-list
+        / filled by backbonejs
diff --git a/app/workers/recurring_pod_check.rb b/app/workers/recurring_pod_check.rb
new file mode 100644
index 000000000..3bf2d79e8
--- /dev/null
+++ b/app/workers/recurring_pod_check.rb
@@ -0,0 +1,14 @@
+
+module Workers
+  class RecurringPodCheck < Base
+    include Sidetiq::Schedulable
+
+    sidekiq_options queue: :maintenance
+
+    recurrence { daily }
+
+    def perform
+      Pod.check_all!
+    end
+  end
+end
diff --git a/config/locales/diaspora/en.yml b/config/locales/diaspora/en.yml
index 20f166c0c..6071abcef 100644
--- a/config/locales/diaspora/en.yml
+++ b/config/locales/diaspora/en.yml
@@ -104,6 +104,7 @@ en:
       pod_stats: "Pod stats"
       report: "Reports"
       sidekiq_monitor: "Sidekiq monitor"
+      pod_network: "Pod network"
     dashboard:
       pod_status: "Pod status"
       fetching_diaspora_version: "Determining latest diaspora* version..."
@@ -173,6 +174,8 @@ en:
       current_segment: "The current segment is averaging <b>%{post_yest}</b> posts per user, from <b>%{post_day}</b>"
       50_most: "50 most popular tags"
       tag_name: "Tag name: <b>%{name_tag}</b> Count: <b>%{count_tag}</b>"
+    pods:
+      pod_network: "Pod network"
   application:
     helper:
       unknown_person: "Unknown person"
diff --git a/config/locales/javascript/javascript.en.yml b/config/locales/javascript/javascript.en.yml
index eb2c0efb5..695a00c89 100644
--- a/config/locales/javascript/javascript.en.yml
+++ b/config/locales/javascript/javascript.en.yml
@@ -35,6 +35,46 @@ en:
         outdated: "Your pod is outdated."
         compare_versions: "The latest diaspora* release is <%= latestVersion %>, your pod is running <%= podVersion %>."
         error: "Unable to determine latest diaspora* version."
+    admin:
+      pods:
+        pod: "Pod"
+        ssl: "SSL"
+        ssl_enabled: "SSL enabled"
+        ssl_disabled: "SSL disabled"
+        added: "Added"
+        status: "Status"
+        states:
+          unchecked: "Unchecked"
+          no_errors: "OK"
+          dns_failed: "Name resolution (DNS) failed"
+          net_failed: "Connection attempt failed"
+          ssl_failed: "Secure connection (SSL) failed"
+          http_failed: "HTTP connection failed"
+          version_failed: "Unable to retrieve software version"
+          unknown_error: "An unspecified error has happened during the check"
+        actions: "Actions"
+        offline_since: "offline since:"
+        last_check: "last check:"
+        more_info: "show more information"
+        check: "perform connection test"
+        recheck:
+          success: "The pod was just checked again."
+          failure: "The check was not performed."
+        follow_link: "open link in browser"
+        no_info: "No additional information available at this point"
+        server_software: "Server software:"
+        response_time: "Response time:"
+        ms:
+          one: "<%= count %>ms"
+          other: "<%= count %>ms"
+        unknown: "unknown"
+        not_available: "not available"
+        unchecked:
+          one: "There is still one pod that hasn't been checked at all."
+          other: "There are still <%= count %> pods that haven't been checked at all."
+        errors:
+          one: "The connection test returned an error for one pod."
+          other: "The connection test returned an error for <%= count %> pods."
 
     aspects:
       make_aspect_list_visible: "Make contacts in this aspect visible to each other?"
diff --git a/config/routes.rb b/config/routes.rb
index a1bc5b14a..585e3c804 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -148,6 +148,10 @@ Diaspora::Application.routes.draw do
   end
 
   namespace :admin do
+    resources :pods, only: :index do
+      post :recheck
+    end
+
     post 'users/:id/close_account' => 'users#close_account', :as => 'close_account'
     post 'users/:id/lock_account' => 'users#lock_account', :as => 'lock_account'
     post 'users/:id/unlock_account' => 'users#unlock_account', :as => 'unlock_account'
diff --git a/db/migrate/20150731123114_add_status_to_pods.rb b/db/migrate/20150731123114_add_status_to_pods.rb
new file mode 100644
index 000000000..b53051f06
--- /dev/null
+++ b/db/migrate/20150731123114_add_status_to_pods.rb
@@ -0,0 +1,14 @@
+class AddStatusToPods < ActiveRecord::Migration
+  def change
+    add_column :pods, :status, :integer, default: 0
+    add_column :pods, :checked_at, :datetime, default: Time.zone.at(0)
+    add_column :pods, :offline_since, :datetime, default: nil
+    add_column :pods, :response_time, :integer, default: -1
+    add_column :pods, :software, :string, limit: 255
+    add_column :pods, :error, :string, limit: 255
+
+    add_index :pods, :status
+    add_index :pods, :checked_at
+    add_index :pods, :offline_since
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index a0dffc8dc..284aa9957 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20150724152052) do
+ActiveRecord::Schema.define(version: 20150731123114) do
 
   create_table "account_deletions", force: :cascade do |t|
     t.string   "diaspora_handle", limit: 255
@@ -306,11 +306,21 @@ ActiveRecord::Schema.define(version: 20150724152052) do
   add_index "photos", ["status_message_guid"], name: "index_photos_on_status_message_guid", length: {"status_message_guid"=>191}, using: :btree
 
   create_table "pods", force: :cascade do |t|
-    t.string   "host",       limit: 255
+    t.string   "host",          limit: 255
     t.boolean  "ssl"
-    t.datetime "created_at",             null: false
-    t.datetime "updated_at",             null: false
-  end
+    t.datetime "created_at",                                                null: false
+    t.datetime "updated_at",                                                null: false
+    t.integer  "status",        limit: 4,   default: 0
+    t.datetime "checked_at",                default: '1970-01-01 00:00:00'
+    t.datetime "offline_since"
+    t.integer  "response_time", limit: 4,   default: -1
+    t.string   "software",      limit: 255
+    t.string   "error",         limit: 255
+  end
+
+  add_index "pods", ["checked_at"], name: "index_pods_on_checked_at", using: :btree
+  add_index "pods", ["offline_since"], name: "index_pods_on_offline_since", using: :btree
+  add_index "pods", ["status"], name: "index_pods_on_status", using: :btree
 
   create_table "poll_answers", force: :cascade do |t|
     t.string  "answer",     limit: 255,             null: false
diff --git a/lib/connection_tester.rb b/lib/connection_tester.rb
new file mode 100644
index 000000000..dd940c761
--- /dev/null
+++ b/lib/connection_tester.rb
@@ -0,0 +1,262 @@
+
+class ConnectionTester
+  NODEINFO_SCHEMA   = "http://nodeinfo.diaspora.software/ns/schema/1.0"
+  NODEINFO_FRAGMENT = "/.well-known/nodeinfo"
+
+  class << self
+    # Test the reachability of a server by the given HTTP/S URL.
+    # In the first step, a DNS query is performed to check whether the
+    # given name even resolves correctly.
+    # The second step is to send a HTTP request and look at the returned
+    # status code or any returned errors.
+    # This function isn't intended to check for the availability of a
+    # specific page, instead a GET request is sent to the root directory
+    # of the server.
+    # In the third step an attempt is made to determine the software version
+    # used on the server, via the nodeinfo page.
+    #
+    # @api This is the entry point you're supposed to use for testing
+    #   connections to other diaspora-compatible servers.
+    # @param [String] server URL
+    # @return [Result] result object containing information about the
+    #   server and to what point the connection was successful
+    def check(url)
+      url = "http://#{url}" unless url.include?("://")
+      result = Result.new
+
+      begin
+        ct = ConnectionTester.new(url, result)
+
+        # test DNS resolving
+        ct.resolve
+
+        # test HTTP request
+        ct.request
+
+        # test for the diaspora* version
+        ct.nodeinfo
+
+      rescue Failure => e
+        result_from_failure(result, e)
+      end
+
+      result.freeze
+    end
+
+    private
+
+    # infer some attributes of the result object based on the failure
+    def result_from_failure(result, error)
+      result.error = error
+
+      case error
+      when AddressFailure, DNSFailure, NetFailure
+        result.reachable = false
+      when SSLFailure
+        result.reachable = true
+        result.ssl_status = false
+      when HTTPFailure
+        result.reachable = true
+      when NodeInfoFailure
+        result.software_version = ""
+      end
+    end
+  end
+
+  # @raise [AddressFailure] if the specified url is not http(s)
+  def initialize(url, result=Result.new)
+    @url ||= url
+    @result ||= result
+    @uri ||= URI.parse(@url)
+    raise AddressFailure,
+          "invalid protocol: '#{@uri.scheme.upcase}'" unless http_uri?(@uri)
+
+    result.hostname = @uri.host
+  rescue URI::InvalidURIError => e
+    raise AddressFailure, e.message
+  end
+
+  # Perform the DNS query, the IP address will be stored in the result
+  # @raise [DNSFailure] caused by a failure to resolve or a timeout
+  def resolve
+    with_dns_resolver do |dns|
+      addr = dns.getaddress(@uri.host)
+      @result.ip = addr.to_s
+    end
+  rescue Resolv::ResolvError, Resolv::ResolvTimeout => e
+    raise DNSFailure, "'#{@uri.host}' - #{e.message}"
+  end
+
+  # Perform a HTTP GET request to determine the following information
+  # * is the host reachable
+  # * is port 80/443 open
+  # * is the SSL certificate valid (only on HTTPS)
+  # * does the server return a successful HTTP status code
+  # * is there a reasonable amount of redirects (3 by default)
+  # (can't do a HEAD request, since that's not a defined route in the app)
+  #
+  # @raise [NetFailure, SSLFailure, HTTPFailure] if any of the checks fail
+  # @return [Integer] HTTP status code
+  def request
+    with_http_connection do |http|
+      response = capture_response_time { http.get("/") }
+      handle_http_response(response)
+    end
+  rescue Faraday::ConnectionFailed, Faraday::TimeoutError => e
+    raise NetFailure, e.message
+  rescue Faraday::SSLError => e
+    raise SSLFailure, e.message
+  rescue ArgumentError, FaradayMiddleware::RedirectLimitReached, Faraday::ClientError => e
+    raise HTTPFailure, e.message
+  end
+
+  # Try to find out the version of the other servers software.
+  # Assuming the server speaks nodeinfo
+  #
+  # @raise [NodeInfoFailure] if the document can't be fetched
+  #   or the attempt to parse it failed
+  def nodeinfo
+    with_http_connection do |http|
+      ni_resp = http.get(NODEINFO_FRAGMENT)
+      nd_resp = http.get(find_nodeinfo_url(ni_resp.body))
+      find_software_version(nd_resp.body)
+    end
+  rescue Faraday::ResourceNotFound, KeyError, JSON::JSONError => e
+    raise NodeInfoFailure, e.message
+  end
+
+  private
+
+  def with_http_connection
+    @http ||= Faraday.new(@url) do |c|
+      c.use Faraday::Response::RaiseError
+      c.use FaradayMiddleware::FollowRedirects, limit: 3
+      c.adapter(Faraday.default_adapter)
+      c.headers[:user_agent] = "diaspora-connection-tester"
+      c.options.timeout = 12
+      c.options.open_timeout = 6
+      # use the configured CA
+      c.ssl.ca_file = Faraday.default_connection.ssl.ca_file
+    end
+    yield(@http) if block_given?
+  end
+
+  def with_dns_resolver
+    dns = Resolv::DNS.new
+    yield(dns) if block_given?
+  ensure
+    dns.close
+  end
+
+  def http_uri?(uri)
+    uri.is_a?(URI::HTTP) || uri.is_a?(URI::HTTPS)
+  end
+
+  def uses_ssl?
+    @uses_ssl
+  end
+
+  # request root path, measure response time
+  # measured time may be skewed, if there are redirects
+  #
+  # @return [Faraday::Response]
+  def capture_response_time
+    start     = Time.zone.now
+    resp = yield if block_given?
+    @result.rt = ((Time.zone.now - start) * 1000.0).to_i # milliseconds
+    resp
+  end
+
+  def handle_http_response(response)
+    @uses_ssl = (response.env.url.scheme == "https")
+    @result.status_code = Integer(response.status)
+
+    if response.success?
+      @result.reachable  = true
+      @result.ssl_status = @uses_ssl
+    else
+      raise HTTPFailure, "unsuccessful response code: #{response.status}"
+    end
+  end
+
+  # walk the JSON document, get the actual document location
+  def find_nodeinfo_url(body)
+    links = JSON.parse(body)
+    links.fetch("links").find { |entry|
+      entry.fetch("rel") == NODEINFO_SCHEMA
+    }.fetch("href")
+  end
+
+  # walk the JSON document, find the version string
+  def find_software_version(body)
+    info = JSON.parse(body)
+    sw = info.fetch("software")
+    @result.software_version = "#{sw.fetch('name')} #{sw.fetch('version')}"
+  end
+
+  class Failure < StandardError
+  end
+
+  class AddressFailure < Failure
+  end
+
+  class DNSFailure < Failure
+  end
+
+  class NetFailure < Failure
+  end
+
+  class SSLFailure < Failure
+  end
+
+  class HTTPFailure < Failure
+  end
+
+  class NodeInfoFailure < Failure
+  end
+
+  Result = Struct.new(
+    :hostname, :ip, :reachable, :ssl_status, :status_code, :rt, :software_version, :error
+  ) do
+    # @!attribute hostname
+    #   @return [String] hostname derived from the URL
+
+    # @!attribute ip
+    #   @return [String] resolved IP address from DNS query
+
+    # @!attribute reachable
+    #   @return [Boolean] whether the host was reachable over the network
+
+    # @!attribute ssl_status
+    #   @return [Boolean] indicating how the SSL verification went
+
+    # @!attribute status_code
+    #   @return [Integer] HTTP status code that was returned for the HEAD request
+
+    # @!attribute rt
+    #   @return [Integer] response time for the HTTP request
+
+    # @!attribute software_version
+    #   @return [String] version of diaspora* as reported by nodeinfo
+
+    # @!attribute error
+    #   @return [Exception] if the test is unsuccessful, this will contain
+    #                       an exception of type {ConnectionTester::Failure}
+
+    def initialize
+      self.rt = -1
+    end
+
+    def success?
+      error.nil?
+    end
+
+    def error?
+      !error.nil?
+    end
+
+    def failure_message
+      "#{error.class.name}: #{error.message}" if error?
+    end
+  end
+end
diff --git a/spec/controllers/admin/pods_controller_spec.rb b/spec/controllers/admin/pods_controller_spec.rb
new file mode 100644
index 000000000..621836916
--- /dev/null
+++ b/spec/controllers/admin/pods_controller_spec.rb
@@ -0,0 +1,52 @@
+
+require "spec_helper"
+
+describe Admin::PodsController, type: :controller do
+  before do
+    @user = FactoryGirl.create :user
+    Role.add_admin(@user.person)
+
+    sign_in :user, @user
+  end
+
+  describe "#index" do
+    it "renders the pod list template" do
+      get :index
+      expect(response).to render_template("admins/pods")
+      expect(response.body).to match(/id='pod-alerts'/im)
+      expect(response.body).to match(/id='pod-list'/im)
+    end
+
+    it "contains the preloads" do
+      get :index
+      expect(response.body).to match(/uncheckedCount=/im)
+      expect(response.body).to match(/errorCount=/im)
+      expect(response.body).to match(/preloads.*"pods"\s?\:/im)
+    end
+
+    it "returns the json data" do
+      @pods = (0..2).map { FactoryGirl.create(:pod).reload } # normalize timestamps
+      get :index, format: :json
+
+      expect(response.body).to eql(PodPresenter.as_collection(@pods).to_json)
+    end
+  end
+
+  describe "#recheck" do
+    before do
+      @pod = FactoryGirl.create(:pod).reload
+      allow(Pod).to receive(:find) { @pod }
+      expect(@pod).to receive(:test_connection!)
+    end
+
+    it "performs a connection test" do
+      post :recheck, pod_id: 1
+      expect(response).to be_redirect
+    end
+
+    it "performs a connection test (format: json)" do
+      post :recheck, pod_id: 1, format: :json
+      expect(response.body).to eql(PodPresenter.new(@pod).to_json)
+    end
+  end
+end
diff --git a/spec/factories.rb b/spec/factories.rb
index 02bfa2fc0..d3470e3be 100644
--- a/spec/factories.rb
+++ b/spec/factories.rb
@@ -210,6 +210,11 @@ FactoryGirl.define do
     photo_url "/assets/user/adams.jpg"
   end
 
+  factory :pod do
+    host "pod.example.com"
+    ssl true
+  end
+
   factory(:comment) do
     sequence(:text) {|n| "#{n} cats"}
     association(:author, :factory => :person)
diff --git a/spec/javascripts/app/models/pod_spec.js b/spec/javascripts/app/models/pod_spec.js
new file mode 100644
index 000000000..a848d3225
--- /dev/null
+++ b/spec/javascripts/app/models/pod_spec.js
@@ -0,0 +1,45 @@
+describe("app.model.Pod", function() {
+  var podId = 123;
+
+  beforeEach(function() {
+    this.pod = new app.models.Pod({
+      id: podId,
+      host: "pod.example.com",
+      status: "unchecked",
+      /* jshint camelcase: false */
+      checked_at: null
+      /* jshint camelcase: true */
+    });
+  });
+
+  describe("recheck", function() {
+    var newAttributes = {
+      id: podId,
+      status: "no_errors",
+      /* jshint camelcase: false */
+      checked_at: new Date()
+      /* jshint camelcase: true */
+    };
+    var ajaxSuccess = {
+      status: 200,
+      responseText: JSON.stringify(newAttributes)
+    };
+
+    it("calls the recheck action on the server", function() {
+      var expected = Routes.adminPodRecheck(podId);
+      this.pod.recheck();
+      expect(jasmine.Ajax.requests.mostRecent().url).toEqual(expected);
+    });
+
+    it("updates the model attributes from the response", function() {
+      spyOn(this.pod, "set").and.callThrough();
+      expect(this.pod.get("status")).toEqual("unchecked");
+      this.pod.recheck();
+      jasmine.Ajax.requests.mostRecent().respondWith(ajaxSuccess);
+
+      expect(this.pod.set).toHaveBeenCalled();
+      expect(this.pod.get("status")).toEqual("no_errors");
+      expect(this.pod.get("checked_at")).not.toEqual(null);
+    });
+  });
+});
diff --git a/spec/javascripts/app/views/pod_entry_view_spec.js b/spec/javascripts/app/views/pod_entry_view_spec.js
new file mode 100644
index 000000000..81777b6af
--- /dev/null
+++ b/spec/javascripts/app/views/pod_entry_view_spec.js
@@ -0,0 +1,91 @@
+
+describe("app.views.PodEntry", function() {
+  beforeEach(function() {
+    this.pod = new app.models.Pod({id : 123});
+    this.view = new app.views.PodEntry({
+      model: this.pod,
+      parent: document.createDocumentFragment()
+    });
+  });
+
+  describe("className", function() {
+    it("returns danger bg when offline", function() {
+      this.pod.set("offline", true);
+      expect(this.view.className()).toEqual("bg-danger");
+    });
+
+    it("returns warning bg when version unknown", function() {
+      this.pod.set("status", "version_failed");
+      expect(this.view.className()).toEqual("bg-warning");
+    });
+
+    it("returns success bg for no errors", function() {
+      this.pod.set("status", "no_errors");
+      expect(this.view.className()).toEqual("bg-success");
+    });
+  });
+
+  describe("presenter", function() {
+    it("contains calculated attributes", function() {
+      this.pod.set({
+        status: "no_errors",
+        ssl: true,
+        host: "pod.example.com"
+      });
+      var actual = this.view.presenter();
+      expect(actual).toEqual(jasmine.objectContaining({
+        /* jshint camelcase: false */
+        is_unchecked: false,
+        has_no_errors: true,
+        has_errors: false,
+        status_text: jasmine.anything(),
+        response_time_fmt: jasmine.anything(),
+        pod_url: "https://pod.example.com"
+        /* jshint camelcase: true */
+      }));
+    });
+  });
+
+  describe("postRenderTemplate", function() {
+    it("appends itself to the parent", function() {
+      var childCount = $(this.view.parent).children().length;
+      this.view.render();
+      expect($(this.view.parent).children().length).toEqual(childCount+1);
+    });
+  });
+
+  describe("recheckPod", function() {
+    var ajaxSuccess = { status: 200, responseText: "{}" };
+    var ajaxFail = { status: 400 };
+
+    it("calls .recheck() on the model", function() {
+      spyOn(this.pod, "recheck").and.returnValue($.Deferred());
+      this.view.recheckPod();
+      expect(this.pod.recheck).toHaveBeenCalled();
+    });
+
+    it("renders a success flash message", function() {
+      this.view.recheckPod();
+      jasmine.Ajax.requests.mostRecent().respondWith(ajaxSuccess);
+      expect($("[id^=\"flash\"]")).toBeSuccessFlashMessage();
+    });
+
+    it("renders an error flash message", function() {
+      this.view.recheckPod();
+      jasmine.Ajax.requests.mostRecent().respondWith(ajaxFail);
+      expect($("[id^=\"flash\"]")).toBeErrorFlashMessage();
+    });
+
+    it("sets the appropriate CSS class", function() {
+      this.view.$el.addClass("bg-danger");
+      this.pod.set({ offline: false, status: "no_errors" });
+
+      this.view.recheckPod();
+      expect(this.view.$el.attr("class")).toContain("checking");
+      jasmine.Ajax.requests.mostRecent().respondWith(ajaxSuccess);
+      expect(this.view.$el.attr("class")).toContain("bg-success");
+      expect(this.view.$el.attr("class")).not.toContain("checking");
+      expect(this.view.$el.attr("class")).not.toContain("bg-danger");
+    });
+  });
+});
diff --git a/spec/lib/connection_tester_spec.rb b/spec/lib/connection_tester_spec.rb
new file mode 100644
index 000000000..737003a80
--- /dev/null
+++ b/spec/lib/connection_tester_spec.rb
@@ -0,0 +1,130 @@
+
+require "spec_helper"
+
+describe ConnectionTester do
+  describe "::check" do
+    it "takes a http url and returns a result object" do
+      res = ConnectionTester.check("https://pod.example.com")
+      expect(res).to be_a(ConnectionTester::Result)
+    end
+
+    it "still returns a result object, even for invalid urls" do
+      res = ConnectionTester.check("i:am/not)a+url")
+      expect(res).to be_a(ConnectionTester::Result)
+      expect(res.error).to be_a(ConnectionTester::Failure)
+    end
+  end
+
+  describe "#initialize" do
+    it "accepts the http protocol" do
+      expect {
+        ConnectionTester.new("https://pod.example.com")
+      }.not_to raise_error
+    end
+    it "rejects unexpected protocols" do
+      expect {
+        ConnectionTester.new("xmpp:user@example.com")
+      }.to raise_error(ConnectionTester::AddressFailure)
+    end
+  end
+
+  describe "#resolve" do
+    before do
+      @result = ConnectionTester::Result.new
+      @dns = instance_double("Resolv::DNS")
+      allow(@dns).to receive(:close).once
+    end
+
+    it "resolves the IP address" do
+      tester = ConnectionTester.new("https://pod.example.com", @result)
+      expect(tester).to receive(:with_dns_resolver).and_yield(@dns)
+      expect(@dns).to receive(:getaddress).and_return("192.168.1.2")
+
+      tester.resolve
+      expect(@result.ip).to eq("192.168.1.2")
+    end
+  end
+
+  describe "#request" do
+    before do
+      @url = "https://pod.example.com"
+      @stub =
+      @result = ConnectionTester::Result.new
+      @tester = ConnectionTester.new(@url, @result)
+    end
+
+    it "performs a successful GET request on '/'" do
+      stub_request(:get, @url).to_return(status: 200, body: "Hello World!")
+
+      @tester.request
+      expect(@result.rt).to be > -1
+      expect(@result.reachable).to be_truthy
+      expect(@result.ssl_status).to be_truthy
+    end
+
+    it "receives a 'normal' 301 redirect" do
+      stub_request(:get, @url).to_return(status: 301, headers: {"Location" => "#{@url}/redirect"})
+      stub_request(:get, "#{@url}/redirect").to_return(status: 200, body: "Hello World!")
+
+      @tester.request
+    end
+
+    it "receives too many 301 redirects" do
+      stub_request(:get, @url).to_return(status: 301, headers: {"Location" => "#{@url}/redirect"})
+      stub_request(:get, "#{@url}/redirect").to_return(status: 301, headers: {"Location" => "#{@url}/redirect1"})
+      stub_request(:get, "#{@url}/redirect1").to_return(status: 301, headers: {"Location" => "#{@url}/redirect2"})
+      stub_request(:get, "#{@url}/redirect2").to_return(status: 301, headers: {"Location" => "#{@url}/redirect3"})
+      stub_request(:get, "#{@url}/redirect3").to_return(status: 200, body: "Hello World!")
+
+      expect { @tester.request }.to raise_error(ConnectionTester::HTTPFailure)
+    end
+
+    it "receives a 404 not found" do
+      stub_request(:get, @url).to_return(status: 404, body: "Not Found!")
+      expect { @tester.request }.to raise_error(ConnectionTester::HTTPFailure)
+    end
+
+    it "cannot connect" do
+      stub_request(:get, @url).to_raise(Faraday::ConnectionFailed.new("Error!"))
+      expect { @tester.request }.to raise_error(ConnectionTester::NetFailure)
+    end
+
+    it "encounters an invalid SSL setup" do
+      stub_request(:get, @url).to_raise(Faraday::SSLError.new("Error!"))
+      expect { @tester.request }.to raise_error(ConnectionTester::SSLFailure)
+    end
+  end
+
+  describe "#nodeinfo" do
+    before do
+      @url = "https://diaspora.example.com"
+      @result = ConnectionTester::Result.new
+      @tester = ConnectionTester.new(@url, @result)
+
+      @ni_wellknown = {links: [{rel:  ConnectionTester::NODEINFO_SCHEMA,
+                                href: "/nodeinfo"}]}
+      @ni_document = {software: {name: "diaspora", version: "a.b.c.d"}}
+    end
+
+    it "reads the version from the nodeinfo document" do
+      stub_request(:get, "#{@url}#{ConnectionTester::NODEINFO_FRAGMENT}")
+        .to_return(status: 200, body: JSON.generate(@ni_wellknown))
+      stub_request(:get, "#{@url}/nodeinfo").to_return(status: 200, body: JSON.generate(@ni_document))
+
+      @tester.nodeinfo
+      expect(@result.software_version).to eq("diaspora a.b.c.d")
+    end
+
+    it "handles a missing nodeinfo document gracefully" do
+      stub_request(:get, "#{@url}#{ConnectionTester::NODEINFO_FRAGMENT}")
+        .to_return(status: 404, body: "Not Found")
+      expect { @tester.nodeinfo }.to raise_error(ConnectionTester::NodeInfoFailure)
+    end
+
+    it "handles a malformed document gracefully" do
+      stub_request(:get, "#{@url}#{ConnectionTester::NODEINFO_FRAGMENT}")
+        .to_return(status: 200, body: '{"json"::::"malformed"}')
+      expect { @tester.nodeinfo }.to raise_error(ConnectionTester::NodeInfoFailure)
+    end
+  end
+end
diff --git a/spec/models/pod_spec.rb b/spec/models/pod_spec.rb
index 7aee2dfa2..fd9d148e3 100644
--- a/spec/models/pod_spec.rb
+++ b/spec/models/pod_spec.rb
@@ -1,15 +1,78 @@
-require 'spec_helper'
+require "spec_helper"
 
-describe Pod, :type => :model do
-  describe '.find_or_create_by' do
-    it 'takes a url, and makes one by host' do
-      pod = Pod.find_or_create_by(url: 'https://joindiaspora.com/maxwell')
-      expect(pod.host).to eq('joindiaspora.com')
+describe Pod, type: :model do
+  describe "::find_or_create_by" do
+    it "takes a url, and makes one by host" do
+      pod = Pod.find_or_create_by(url: "https://joindiaspora.com/maxwell")
+      expect(pod.host).to eq("joindiaspora.com")
     end
 
-    it 'sets ssl boolean(side-effect)' do
-      pod = Pod.find_or_create_by(url: 'https://joindiaspora.com/maxwell')
+    it "sets ssl boolean (side-effect)" do
+      pod = Pod.find_or_create_by(url: "https://joindiaspora.com/maxwell")
       expect(pod.ssl).to be true
     end
   end
+
+  describe "::check_all!" do
+    before do
+      @pods = (0..4).map do
+        double("pod").tap do |pod|
+          expect(pod).to receive(:test_connection!)
+        end
+      end
+      allow(Pod).to receive(:find_in_batches) { @pods }
+    end
+
+    it "calls #test_connection! on every pod" do
+      Pod.check_all!
+    end
+  end
+
+  describe "#test_connection!" do
+    before do
+      @pod = FactoryGirl.create(:pod)
+      @result = double("result")
+
+      allow(@result).to receive(:rt) { 123 }
+      allow(@result).to receive(:software_version) { "diaspora a.b.c.d" }
+      allow(@result).to receive(:failure_message) { "hello error!" }
+
+      expect(ConnectionTester).to receive(:check).at_least(:once).and_return(@result)
+    end
+
+    it "updates the connectivity values" do
+      allow(@result).to receive(:error)
+      allow(@result).to receive(:error?)
+      @pod.test_connection!
+
+      expect(@pod.status).to eq("no_errors")
+      expect(@pod.offline?).to be_falsy
+      expect(@pod.response_time).to eq(123)
+      expect(@pod.checked_at).to be_within(1.second).of Time.zone.now
+    end
+
+    it "handles a failed check" do
+      expect(@result).to receive(:error?).at_least(:once) { true }
+      expect(@result).to receive(:error).at_least(:once) { ConnectionTester::NetFailure.new }
+      @pod.test_connection!
+
+      expect(@pod.offline?).to be_truthy
+      expect(@pod.offline_since).to be_within(1.second).of Time.zone.now
+    end
+
+    it "preserves the original offline timestamp" do
+      expect(@result).to receive(:error?).at_least(:once) { true }
+      expect(@result).to receive(:error).at_least(:once) { ConnectionTester::NetFailure.new }
+      @pod.test_connection!
+
+      now = Time.zone.now
+      expect(@pod.offline_since).to be_within(1.second).of now
+
+      Timecop.travel(Time.zone.today + 30.days) do
+        @pod.test_connection!
+        expect(@pod.offline_since).to be_within(1.second).of now
+        expect(Time.zone.now).to be_within(1.day).of(now + 30.days)
+      end
+    end
+  end
 end
diff --git a/spec/workers/recurring_pod_check_spec.rb b/spec/workers/recurring_pod_check_spec.rb
new file mode 100644
index 000000000..b7f240f1e
--- /dev/null
+++ b/spec/workers/recurring_pod_check_spec.rb
@@ -0,0 +1,17 @@
+
+require "spec_helper"
+
+describe Workers::RecurringPodCheck do
+  before do
+    @pods = (0..4).map do
+      FactoryGirl.create(:pod).tap { |pod|
+        expect(pod).to receive(:test_connection!)
+      }
+    end
+    allow(Pod).to receive(:find_in_batches) { @pods }
+  end
+
+  it "performs a connection test on all existing pods" do
+    Workers::RecurringPodCheck.new.perform
+  end
+end