Merge pull request #10 from nextcloud/replication

Improve replication handling

14 files changed, 208 insertions, 92 deletions

diff --git a/doc/architecture.md b/doc/architecture.md
index 32f0b82..44660c9 100755
--- a/doc/architecture.md
+++ b/doc/architecture.md
@@ -117,7 +117,7 @@ curl -X GET http://dev/nextcloud/lookup-server/server/users?search=searchstring
 ### Get replication log
 This call is used for master-master replication between different nodes.
 Example:
-curl -X GET http://lookup:foobar@dev/nextcloud/lookup-server/server/replication.php/?timestamp=123456\&page=0  
+curl -X GET http://lookup:foobar@dev/nextcloud/lookup-server/replication?timestamp=123456\&page=0  
 
 ## High availability
 Several Lookup-Server can do master-master replication and sync their data. 
diff --git a/server/config/config.sample.php b/server/config/config.sample.php
index 0f5cd8a..5cc6adb 100755
--- a/server/config/config.sample.php
+++ b/server/config/config.sample.php
@@ -34,12 +34,9 @@ $CONFIG = [
 
 	// the list of remote replication servers that should be queried in the cronjob
 	'REPLICATION_HOSTS' => [
-		'https://lookup:slavefoobar@example.com'
+		'https://lookup:slavefoobar@example.com/replication'
 	],
 
-	// replication interval. The number of seconds into the past that should be used when fetching the replication log from a remote server. Should be a bit higher then the cronjob intervall
-	'REPLICATION_INTERVAL' => 900, // 15min
-
 	// ip black list. usefull to block spammers.
 	'IP_BLACKLIST' => [
 		'333.444.555.',
diff --git a/server/cronjob.php b/server/cronjob.php
index 1bac221..626034d 100755
--- a/server/cronjob.php
+++ b/server/cronjob.php
@@ -1,38 +1,20 @@
-#!/usr/bin/php
 <?php
 
-/**
-* Cronjob. Please call this script every 10min
-*
-* @author Frank Karlitschek
-* @copyright 2016 Frank Karlitschek frank@karlitschek.de
-*
-* This library is free software; you can redistribute it and/or
-* modify it under the terms of the GNU AFFERO GENERAL PUBLIC LICENSE
-* License as published by the Free Software Foundation; either
-* version 3 of the License, or any later version.
-*
-* This library is distributed in the hope that it will be useful,
-* but WITHOUT ANY WARRANTY; without even the implied warranty of
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-* GNU AFFERO GENERAL PUBLIC LICENSE for more details.
-*
-* You should have received a copy of the GNU Affero General Public
-* License along with this library.  If not, see <http://www.gnu.org/licenses/>.
-*
-*/
+require __DIR__ . '/vendor/autoload.php';
 
-// enable the full error reporting for easier debugging
-error_reporting(E_ERROR | E_WARNING | E_PARSE | E_NOTICE | E_ALL);
+if (PHP_SAPI !== 'cli') {
+	return;
+}
 
-// include the main contribook lib
-require('vendor/autoload.php');
+$env = \Slim\Http\Environment::mock(['REQUEST_URI' => '/import']);
 
-// Cronjob
-$s = new \LookupServer\Server();
+$settings = require __DIR__ . '/src/config.php';
+$settings['environment'] = $env;
+$container = new \Slim\Container($settings);
+require __DIR__ . '/src/dependencies.php';
 
-// Cleanup the API Traffic Limit
-$s->cleanup();
+$app = new \Slim\App($container);
+
+$app->map(['GET'], '/import', 'Replication:import');
+$app->run();
 
-// Import from other replication hosts
-$s->importReplication();
diff --git a/server/index.php b/server/index.php
index de9c028..457d2ba 100644
--- a/server/index.php
+++ b/server/index.php
@@ -22,4 +22,6 @@ $app->delete('/users', 'UserManager:delete');
 $app->get('/validate/email/{token}', 'EmailValidator:validate')->setName('validateEmail');
 $app->get('/status', 'Status:status');
 
+$app->get('/replication', 'Replication:export');
+
 $app->run();
diff --git a/server/lib/Replication.php b/server/lib/Replication.php
new file mode 100644
index 0000000..002cd47
--- /dev/null
+++ b/server/lib/Replication.php
@@ -0,0 +1,174 @@
+<?php
+
+namespace LookupServer;
+
+use GuzzleHttp\Client;
+use Slim\Http\Request;
+use Slim\Http\Response;
+
+class Replication {
+
+	/** @var \PDO */
+	private $db;
+
+	/** @var string */
+	private $auth;
+
+	/** @var string[] */
+	private $replicationHosts;
+
+	public function __construct(\PDO $db, $auth, $replicationHosts) {
+		$this->db = $db;
+		$this->auth = $auth;
+		$this->replicationHosts = $replicationHosts;
+	}
+
+	public function export(Request $request, Response $response) {
+		$userInfo = $request->getUri()->getUserInfo();
+
+		$userInfo = explode(':', $userInfo, 2);
+
+		if (count($userInfo) !== 2 || $userInfo[0] !== 'lookup' || $userInfo[1] !== $this->auth)  {
+			$response = $response->withStatus(401);
+			return $response;
+		}
+
+		$params = $request->getQueryParams();
+		if (!isset($params['timestamp'], $params['page']) || !ctype_digit($params['timestamp']) ||
+		    !ctype_digit($params['page'])) {
+			$response = $response->withStatus(400);
+			return $response;
+		}
+
+		$timestamp = (int)$params['timestamp'];
+		$page = (int)$params['page'];
+
+		$stmt = $this->db->prepare('SELECT id, federationId, UNIX_TIMESTAMP(timestamp) AS timestamp
+			FROM users 
+			WHERE UNIX_TIMESTAMP(timestamp) >= :timestamp
+			ORDER BY timestamp, id
+			LIMIT :limit
+			OFFSET :offset');
+		$stmt->bindParam('timestamp', $timestamp);
+		$stmt->bindValue('limit', 100, \PDO::PARAM_INT);
+		$stmt->bindValue('offset', 100 * $page, \PDO::PARAM_INT);
+
+		$stmt->execute();
+
+		$result = [];
+		while($data = $stmt->fetch()) {
+			$user = [
+				'cloudId' => $data['federationId'],
+				'timestamp' => (int)$data['timestamp'],
+				'data' => [],
+			];
+
+			$stmt2 = $this->db->prepare('SELECT *
+				FROM store
+				WHERE userId = :uid');
+			$stmt2->bindValue('uid', $data['id']);
+			$stmt2->execute();
+
+			while($userData = $stmt2->fetch()) {
+				$user['data'][] = [
+					'key' => $userData['k'],
+					'value' => $userData['v'],
+					'validated' => (int)$userData['valid'],
+				];
+			}
+			$stmt2->closeCursor();
+
+			$result[] = $user;
+		}
+
+		$response->getBody()->write(json_encode($result));
+		return $response;
+	}
+
+	public function import(Request $request, Response $response) {
+		$replicationStatus = [];
+
+		if (file_exists(__DIR__ . '/../config/replication.json')) {
+			$replicationStatus = json_decode(file_get_contents(__DIR__ . '/../config/replication.json'), true);
+		}
+
+		foreach ($this->replicationHosts as $replicationHost) {
+			$timestamp = 0;
+
+			if (isset($replicationStatus[$replicationHost])) {
+				$timestamp = $replicationStatus[$replicationHost];
+			}
+
+			$page = 0;
+			while(true) {
+				// Retrieve public key && store
+				$req = new \GuzzleHttp\Psr7\Request('GET', $replicationHost . '?timestamp=' . $timestamp . '&page=' . $page);
+
+				$client = new Client();
+				$resp = $client->send($req, [
+					'timeout' => 5,
+				]);
+
+				$data = json_decode($resp->getBody(), true);
+				if (count($data) === 0) {
+					break;
+				}
+
+				foreach ($data as $user) {
+					$this->parseUser($user);
+					$replicationStatus[$replicationHost] = $user['timestamp'];
+				}
+
+				$page++;
+			}
+
+			file_put_contents(__DIR__. '/../config/replication.json', json_encode($replicationStatus, JSON_PRETTY_PRINT));
+		}
+
+		return $response;
+	}
+
+	private function parseUser($user) {
+		$stmt = $this->db->prepare('SELECT id, UNIX_TIMESTAMP(timestamp) AS timestamp
+			FROM users
+			WHERE federationId = :id');
+		$stmt->bindParam('id', $user['cloudId']);
+
+		$stmt->execute();
+
+		// New
+		if ($stmt->rowCount() === 1) {
+			$data = $stmt->fetch();
+			if ($data['timestamp'] > $user['timestamp']) {
+				$stmt->closeCursor();
+				return;
+			}
+
+			$stmt2 = $this->db->prepare('DELETE FROM users
+				WHERE federationId = :id');
+			$stmt2->bindParam('id', $user['cloudId']);
+			$stmt2->execute();
+			$stmt2->closeCursor();
+		}
+
+		$stmt->closeCursor();
+
+		$stmt = $this->db->prepare('INSERT INTO users (federationId, timestamp) VALUES (:federationId, FROM_UNIXTIME(:timestamp))');
+		$stmt->bindParam(':federationId', $user['cloudId'], \PDO::PARAM_STR);
+		$stmt->bindParam(':timestamp', $user['timestamp'], \PDO::PARAM_INT);
+		$stmt->execute();
+		$id = $this->db->lastInsertId();
+		$stmt->closeCursor();
+
+		foreach ($user['data'] as $data) {
+			$stmt = $this->db->prepare('INSERT INTO store (userId, k, v, valid) VALUES (:userId, :k, :v, :valid)');
+			$stmt->bindParam(':userId', $id, \PDO::PARAM_INT);
+			$stmt->bindParam(':k', $data['key'], \PDO::PARAM_STR);
+			$stmt->bindParam(':v', $data['value'], \PDO::PARAM_STR);
+			$stmt->bindParam(':valid', $data['validated'], \PDO::PARAM_INT);
+
+			$stmt->execute();
+			$stmt->closeCursor();
+		}
+	}
+}
diff --git a/server/lib/UserManager.php b/server/lib/UserManager.php
index abd20f8..6727199 100644
--- a/server/lib/UserManager.php
+++ b/server/lib/UserManager.php
@@ -25,7 +25,8 @@ class UserManager {
 
 		if (!isset($params['search']) || $params['search'] === '') {
 			$response->withStatus(404);
-			return $response;		}
+			return $response;
+		}
 
 		$search = $params['search'];
 		$searchCloudId = $params['exactCloudId'];
diff --git a/server/replication.php b/server/replication.php
deleted file mode 100755
index 6040b59..0000000
--- a/server/replication.php
+++ /dev/null
@@ -1,35 +0,0 @@
-<?php
-
-/**
-* Lookup Server replication
-*
-* @author Frank Karlitschek
-* @copyright 2016 Frank Karlitschek frank@karlitschek.de
-*
-* This library is free software; you can redistribute it and/or
-* modify it under the terms of the GNU AFFERO GENERAL PUBLIC LICENSE
-* License as published by the Free Software Foundation; either
-* version 3 of the License, or any later version.
-*
-* This library is distributed in the hope that it will be useful,
-* but WITHOUT ANY WARRANTY; without even the implied warranty of
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-* GNU AFFERO GENERAL PUBLIC LICENSE for more details.
-*
-* You should have received a copy of the GNU Affero General Public
-* License along with this library.  If not, see <http://www.gnu.org/licenses/>.
-*
-*/
-
-//makes it easier to debug
-error_reporting(E_ERROR | E_WARNING | E_PARSE | E_NOTICE);
-
-//set the default timezone to use.
-date_default_timezone_set('Europe/Berlin');
-
-//you have to include lib_lookup.
-require('vendor/autoload.php');
-
-//process the request.
-$s = new \LookupServer\Server();
-$s->handleReplication();
diff --git a/server/src/config.php b/server/src/config.php
index 9cba346..dc53e67 100644
--- a/server/src/config.php
+++ b/server/src/config.php
@@ -14,5 +14,7 @@ return [
 		],
 		'host' => $CONFIG['PUBLIC_URL'],
 		'emailfrom' => $CONFIG['EMAIL_SENDER'],
+		'replication_auth' => $CONFIG['REPLICATION_AUTH'],
+		'replication_hosts' => $CONFIG['REPLICATION_HOSTS'],
 	]
 ];
diff --git a/server/src/config.sample.php b/server/src/config.sample.php
deleted file mode 100644
index 435baee..0000000
--- a/server/src/config.sample.php
+++ /dev/null
@@ -1,16 +0,0 @@
-<?php
-
-return [
-	'settings' => [
-		'displayErrorDetails' => true,
-		'addContentLengthHeader' => true,
-		'db' => [
-			'host' => "localhost",
-			'user' => "lookup",
-			'pass' => "lookup",
-			'dbname' => "lookup",
-		],
-		'host' => 'https://lookup.nextcloud.com',
-		'emailfrom' => 'no-reply@lookup.nextcloud.com',
-	]
-];
diff --git a/server/src/dependencies.php b/server/src/dependencies.php
index 5d186ca..0e95274 100644
--- a/server/src/dependencies.php
+++ b/server/src/dependencies.php
@@ -22,3 +22,6 @@ $container['EmailValidator'] = function($c) {
 $container['Status'] = function($c) {
 	return new \LookupServer\Status();
 };
+$container['Replication'] = function ($c) {
+	return new \LookupServer\Replication($c->db, $c->settings['replication_auth'], $c->settings['replication_hosts']);
+};
diff --git a/server/vendor/composer/ClassLoader.php b/server/vendor/composer/ClassLoader.php
index 4626994..2c72175 100644
--- a/server/vendor/composer/ClassLoader.php
+++ b/server/vendor/composer/ClassLoader.php
@@ -374,9 +374,13 @@ class ClassLoader
 
         $first = $class[0];
         if (isset($this->prefixLengthsPsr4[$first])) {
-            foreach ($this->prefixLengthsPsr4[$first] as $prefix => $length) {
-                if (0 === strpos($class, $prefix)) {
-                    foreach ($this->prefixDirsPsr4[$prefix] as $dir) {
+            $subPath = $class;
+            while (false !== $lastPos = strrpos($subPath, '\\')) {
+                $subPath = substr($subPath, 0, $lastPos);
+                $search = $subPath.'\\';
+                if (isset($this->prefixDirsPsr4[$search])) {
+                    foreach ($this->prefixDirsPsr4[$search] as $dir) {
+                        $length = $this->prefixLengthsPsr4[$first][$search];
                         if (file_exists($file = $dir . DIRECTORY_SEPARATOR . substr($logicalPathPsr4, $length))) {
                             return $file;
                         }
diff --git a/server/vendor/composer/LICENSE b/server/vendor/composer/LICENSE
index 1a28124..f27399a 100644
--- a/server/vendor/composer/LICENSE
+++ b/server/vendor/composer/LICENSE
@@ -1,5 +1,5 @@
 
-Copyright (c) 2016 Nils Adermann, Jordi Boggiano
+Copyright (c) Nils Adermann, Jordi Boggiano
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
diff --git a/server/vendor/composer/autoload_classmap.php b/server/vendor/composer/autoload_classmap.php
index e6c8d48..8505a8b 100644
--- a/server/vendor/composer/autoload_classmap.php
+++ b/server/vendor/composer/autoload_classmap.php
@@ -91,6 +91,7 @@ return array(
     'Interop\\Container\\Exception\\ContainerException' => $vendorDir . '/container-interop/container-interop/src/Interop/Container/Exception/ContainerException.php',
     'Interop\\Container\\Exception\\NotFoundException' => $vendorDir . '/container-interop/container-interop/src/Interop/Container/Exception/NotFoundException.php',
     'LookupServer\\BruteForceMiddleware' => $baseDir . '/lib/BruteForceMiddleware.php',
+    'LookupServer\\Replication' => $baseDir . '/lib/Replication.php',
     'LookupServer\\Status' => $baseDir . '/lib/Status.php',
     'LookupServer\\UserManager' => $baseDir . '/lib/UserManager.php',
     'LookupServer\\Validator\\Email' => $baseDir . '/lib/Validator/Email.php',
diff --git a/server/vendor/composer/autoload_static.php b/server/vendor/composer/autoload_static.php
index 11fc612..22ec6d1 100644
--- a/server/vendor/composer/autoload_static.php
+++ b/server/vendor/composer/autoload_static.php
@@ -173,6 +173,7 @@ class ComposerStaticInit509ee4e79733fbe3199b97373b795eca
         'Interop\\Container\\Exception\\ContainerException' => __DIR__ . '/..' . '/container-interop/container-interop/src/Interop/Container/Exception/ContainerException.php',
         'Interop\\Container\\Exception\\NotFoundException' => __DIR__ . '/..' . '/container-interop/container-interop/src/Interop/Container/Exception/NotFoundException.php',
         'LookupServer\\BruteForceMiddleware' => __DIR__ . '/../..' . '/lib/BruteForceMiddleware.php',
+        'LookupServer\\Replication' => __DIR__ . '/../..' . '/lib/Replication.php',
         'LookupServer\\Status' => __DIR__ . '/../..' . '/lib/Status.php',
         'LookupServer\\UserManager' => __DIR__ . '/../..' . '/lib/UserManager.php',
         'LookupServer\\Validator\\Email' => __DIR__ . '/../..' . '/lib/Validator/Email.php',