Fix replication import

Signed-off-by: Roeland Jago Douma <roeland@famdouma.nl>

4 files changed, 113 insertions, 38 deletions

diff --git a/server/cronjob.php b/server/cronjob.php
index 1bac221..626034d 100755
--- a/server/cronjob.php
+++ b/server/cronjob.php
@@ -1,38 +1,20 @@
-#!/usr/bin/php
 <?php
 
-/**
-* Cronjob. Please call this script every 10min
-*
-* @author Frank Karlitschek
-* @copyright 2016 Frank Karlitschek frank@karlitschek.de
-*
-* This library is free software; you can redistribute it and/or
-* modify it under the terms of the GNU AFFERO GENERAL PUBLIC LICENSE
-* License as published by the Free Software Foundation; either
-* version 3 of the License, or any later version.
-*
-* This library is distributed in the hope that it will be useful,
-* but WITHOUT ANY WARRANTY; without even the implied warranty of
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-* GNU AFFERO GENERAL PUBLIC LICENSE for more details.
-*
-* You should have received a copy of the GNU Affero General Public
-* License along with this library.  If not, see <http://www.gnu.org/licenses/>.
-*
-*/
+require __DIR__ . '/vendor/autoload.php';
 
-// enable the full error reporting for easier debugging
-error_reporting(E_ERROR | E_WARNING | E_PARSE | E_NOTICE | E_ALL);
+if (PHP_SAPI !== 'cli') {
+	return;
+}
 
-// include the main contribook lib
-require('vendor/autoload.php');
+$env = \Slim\Http\Environment::mock(['REQUEST_URI' => '/import']);
 
-// Cronjob
-$s = new \LookupServer\Server();
+$settings = require __DIR__ . '/src/config.php';
+$settings['environment'] = $env;
+$container = new \Slim\Container($settings);
+require __DIR__ . '/src/dependencies.php';
 
-// Cleanup the API Traffic Limit
-$s->cleanup();
+$app = new \Slim\App($container);
+
+$app->map(['GET'], '/import', 'Replication:import');
+$app->run();
 
-// Import from other replication hosts
-$s->importReplication();
diff --git a/server/lib/Replication.php b/server/lib/Replication.php
index a7c5cef..002cd47 100644
--- a/server/lib/Replication.php
+++ b/server/lib/Replication.php
@@ -2,6 +2,7 @@
 
 namespace LookupServer;
 
+use GuzzleHttp\Client;
 use Slim\Http\Request;
 use Slim\Http\Response;
 
@@ -13,9 +14,13 @@ class Replication {
 	/** @var string */
 	private $auth;
 
-	public function __construct(\PDO $db, $auth) {
+	/** @var string[] */
+	private $replicationHosts;
+
+	public function __construct(\PDO $db, $auth, $replicationHosts) {
 		$this->db = $db;
 		$this->auth = $auth;
+		$this->replicationHosts = $replicationHosts;
 	}
 
 	public function export(Request $request, Response $response) {
@@ -38,9 +43,9 @@ class Replication {
 		$timestamp = (int)$params['timestamp'];
 		$page = (int)$params['page'];
 
-		$stmt = $this->db->prepare('SELECT * 
+		$stmt = $this->db->prepare('SELECT id, federationId, UNIX_TIMESTAMP(timestamp) AS timestamp
 			FROM users 
-			WHERE timestamp >= :timestamp
+			WHERE UNIX_TIMESTAMP(timestamp) >= :timestamp
 			ORDER BY timestamp, id
 			LIMIT :limit
 			OFFSET :offset');
@@ -53,8 +58,8 @@ class Replication {
 		$result = [];
 		while($data = $stmt->fetch()) {
 			$user = [
-				'federationId' => $data['federationId'],
-				'timestamp' => $data['timestamp'],
+				'cloudId' => $data['federationId'],
+				'timestamp' => (int)$data['timestamp'],
 				'data' => [],
 			];
 
@@ -68,7 +73,7 @@ class Replication {
 				$user['data'][] = [
 					'key' => $userData['k'],
 					'value' => $userData['v'],
-					'validated' => $userData['valid'],
+					'validated' => (int)$userData['valid'],
 				];
 			}
 			$stmt2->closeCursor();
@@ -79,4 +84,91 @@ class Replication {
 		$response->getBody()->write(json_encode($result));
 		return $response;
 	}
+
+	public function import(Request $request, Response $response) {
+		$replicationStatus = [];
+
+		if (file_exists(__DIR__ . '/../config/replication.json')) {
+			$replicationStatus = json_decode(file_get_contents(__DIR__ . '/../config/replication.json'), true);
+		}
+
+		foreach ($this->replicationHosts as $replicationHost) {
+			$timestamp = 0;
+
+			if (isset($replicationStatus[$replicationHost])) {
+				$timestamp = $replicationStatus[$replicationHost];
+			}
+
+			$page = 0;
+			while(true) {
+				// Retrieve public key && store
+				$req = new \GuzzleHttp\Psr7\Request('GET', $replicationHost . '?timestamp=' . $timestamp . '&page=' . $page);
+
+				$client = new Client();
+				$resp = $client->send($req, [
+					'timeout' => 5,
+				]);
+
+				$data = json_decode($resp->getBody(), true);
+				if (count($data) === 0) {
+					break;
+				}
+
+				foreach ($data as $user) {
+					$this->parseUser($user);
+					$replicationStatus[$replicationHost] = $user['timestamp'];
+				}
+
+				$page++;
+			}
+
+			file_put_contents(__DIR__. '/../config/replication.json', json_encode($replicationStatus, JSON_PRETTY_PRINT));
+		}
+
+		return $response;
+	}
+
+	private function parseUser($user) {
+		$stmt = $this->db->prepare('SELECT id, UNIX_TIMESTAMP(timestamp) AS timestamp
+			FROM users
+			WHERE federationId = :id');
+		$stmt->bindParam('id', $user['cloudId']);
+
+		$stmt->execute();
+
+		// New
+		if ($stmt->rowCount() === 1) {
+			$data = $stmt->fetch();
+			if ($data['timestamp'] > $user['timestamp']) {
+				$stmt->closeCursor();
+				return;
+			}
+
+			$stmt2 = $this->db->prepare('DELETE FROM users
+				WHERE federationId = :id');
+			$stmt2->bindParam('id', $user['cloudId']);
+			$stmt2->execute();
+			$stmt2->closeCursor();
+		}
+
+		$stmt->closeCursor();
+
+		$stmt = $this->db->prepare('INSERT INTO users (federationId, timestamp) VALUES (:federationId, FROM_UNIXTIME(:timestamp))');
+		$stmt->bindParam(':federationId', $user['cloudId'], \PDO::PARAM_STR);
+		$stmt->bindParam(':timestamp', $user['timestamp'], \PDO::PARAM_INT);
+		$stmt->execute();
+		$id = $this->db->lastInsertId();
+		$stmt->closeCursor();
+
+		foreach ($user['data'] as $data) {
+			$stmt = $this->db->prepare('INSERT INTO store (userId, k, v, valid) VALUES (:userId, :k, :v, :valid)');
+			$stmt->bindParam(':userId', $id, \PDO::PARAM_INT);
+			$stmt->bindParam(':k', $data['key'], \PDO::PARAM_STR);
+			$stmt->bindParam(':v', $data['value'], \PDO::PARAM_STR);
+			$stmt->bindParam(':valid', $data['validated'], \PDO::PARAM_INT);
+
+			$stmt->execute();
+			$stmt->closeCursor();
+		}
+	}
 }
diff --git a/server/src/config.php b/server/src/config.php
index 534f700..dc53e67 100644
--- a/server/src/config.php
+++ b/server/src/config.php
@@ -15,5 +15,6 @@ return [
 		'host' => $CONFIG['PUBLIC_URL'],
 		'emailfrom' => $CONFIG['EMAIL_SENDER'],
 		'replication_auth' => $CONFIG['REPLICATION_AUTH'],
+		'replication_hosts' => $CONFIG['REPLICATION_HOSTS'],
 	]
 ];
diff --git a/server/src/dependencies.php b/server/src/dependencies.php
index d08f920..0e95274 100644
--- a/server/src/dependencies.php
+++ b/server/src/dependencies.php
@@ -23,5 +23,5 @@ $container['Status'] = function($c) {
 	return new \LookupServer\Status();
 };
 $container['Replication'] = function ($c) {
-	return new \LookupServer\Replication($c->db, $c->settings['replication_auth']);
+	return new \LookupServer\Replication($c->db, $c->settings['replication_auth'], $c->settings['replication_hosts']);
 };