From 5153616b59f85b467a46e1356ff2dda208a5c57a Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Mon, 10 Apr 2017 10:33:14 +0200 Subject: Do not show errors by default --- server/src/config.php | 2 +- server/src/config.sample.php | 16 ---------------- 2 files changed, 1 insertion(+), 17 deletions(-) delete mode 100644 server/src/config.sample.php diff --git a/server/src/config.php b/server/src/config.php index 9cba346..960bae7 100644 --- a/server/src/config.php +++ b/server/src/config.php @@ -4,7 +4,7 @@ require __DIR__ . '/../config/config.php'; return [ 'settings' => [ - 'displayErrorDetails' => true, + 'displayErrorDetails' => false, 'addContentLengthHeader' => true, 'db' => [ 'host' => $CONFIG['DB']['host'], diff --git a/server/src/config.sample.php b/server/src/config.sample.php deleted file mode 100644 index 435baee..0000000 --- a/server/src/config.sample.php +++ /dev/null @@ -1,16 +0,0 @@ - [ - 'displayErrorDetails' => true, - 'addContentLengthHeader' => true, - 'db' => [ - 'host' => "localhost", - 'user' => "lookup", - 'pass' => "lookup", - 'dbname' => "lookup", - ], - 'host' => 'https://lookup.nextcloud.com', - 'emailfrom' => 'no-reply@lookup.nextcloud.com', - ] -]; -- cgit v1.2.3 From fe81493f6ba17a2c876b8f1e0463e8156948f4c4 Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Mon, 10 Apr 2017 12:34:50 +0200 Subject: Fix replication export Signed-off-by: Roeland Jago Douma --- doc/architecture.md | 2 +- server/index.php | 2 + server/lib/Replication.php | 82 ++++++++++++++++++++++++++++ server/lib/UserManager.php | 3 +- server/src/config.php | 3 +- server/src/dependencies.php | 3 + server/vendor/composer/ClassLoader.php | 10 +++- server/vendor/composer/LICENSE | 2 +- server/vendor/composer/autoload_classmap.php | 1 + server/vendor/composer/autoload_static.php | 1 + 10 files changed, 102 insertions(+), 7 deletions(-) create mode 100644 server/lib/Replication.php diff --git a/doc/architecture.md b/doc/architecture.md index 32f0b82..44660c9 100755 --- a/doc/architecture.md +++ b/doc/architecture.md @@ -117,7 +117,7 @@ curl -X GET http://dev/nextcloud/lookup-server/server/users?search=searchstring ### Get replication log This call is used for master-master replication between different nodes. Example: -curl -X GET http://lookup:foobar@dev/nextcloud/lookup-server/server/replication.php/?timestamp=123456\&page=0 +curl -X GET http://lookup:foobar@dev/nextcloud/lookup-server/replication?timestamp=123456\&page=0 ## High availability Several Lookup-Server can do master-master replication and sync their data. diff --git a/server/index.php b/server/index.php index de9c028..457d2ba 100644 --- a/server/index.php +++ b/server/index.php @@ -22,4 +22,6 @@ $app->delete('/users', 'UserManager:delete'); $app->get('/validate/email/{token}', 'EmailValidator:validate')->setName('validateEmail'); $app->get('/status', 'Status:status'); +$app->get('/replication', 'Replication:export'); + $app->run(); diff --git a/server/lib/Replication.php b/server/lib/Replication.php new file mode 100644 index 0000000..a7c5cef --- /dev/null +++ b/server/lib/Replication.php @@ -0,0 +1,82 @@ +db = $db; + $this->auth = $auth; + } + + public function export(Request $request, Response $response) { + $userInfo = $request->getUri()->getUserInfo(); + + $userInfo = explode(':', $userInfo, 2); + + if (count($userInfo) !== 2 || $userInfo[0] !== 'lookup' || $userInfo[1] !== $this->auth) { + $response = $response->withStatus(401); + return $response; + } + + $params = $request->getQueryParams(); + if (!isset($params['timestamp'], $params['page']) || !ctype_digit($params['timestamp']) || + !ctype_digit($params['page'])) { + $response = $response->withStatus(400); + return $response; + } + + $timestamp = (int)$params['timestamp']; + $page = (int)$params['page']; + + $stmt = $this->db->prepare('SELECT * + FROM users + WHERE timestamp >= :timestamp + ORDER BY timestamp, id + LIMIT :limit + OFFSET :offset'); + $stmt->bindParam('timestamp', $timestamp); + $stmt->bindValue('limit', 100, \PDO::PARAM_INT); + $stmt->bindValue('offset', 100 * $page, \PDO::PARAM_INT); + + $stmt->execute(); + + $result = []; + while($data = $stmt->fetch()) { + $user = [ + 'federationId' => $data['federationId'], + 'timestamp' => $data['timestamp'], + 'data' => [], + ]; + + $stmt2 = $this->db->prepare('SELECT * + FROM store + WHERE userId = :uid'); + $stmt2->bindValue('uid', $data['id']); + $stmt2->execute(); + + while($userData = $stmt2->fetch()) { + $user['data'][] = [ + 'key' => $userData['k'], + 'value' => $userData['v'], + 'validated' => $userData['valid'], + ]; + } + $stmt2->closeCursor(); + + $result[] = $user; + } + + $response->getBody()->write(json_encode($result)); + return $response; + } +} diff --git a/server/lib/UserManager.php b/server/lib/UserManager.php index 8c995d5..fd8f181 100644 --- a/server/lib/UserManager.php +++ b/server/lib/UserManager.php @@ -25,7 +25,8 @@ class UserManager { if (!isset($params['search']) || $params['search'] === '') { $response->withStatus(404); - return $response; } + return $response; + } $search = $params['search']; $stmt = $this->db->prepare('SELECT * diff --git a/server/src/config.php b/server/src/config.php index 960bae7..534f700 100644 --- a/server/src/config.php +++ b/server/src/config.php @@ -4,7 +4,7 @@ require __DIR__ . '/../config/config.php'; return [ 'settings' => [ - 'displayErrorDetails' => false, + 'displayErrorDetails' => true, 'addContentLengthHeader' => true, 'db' => [ 'host' => $CONFIG['DB']['host'], @@ -14,5 +14,6 @@ return [ ], 'host' => $CONFIG['PUBLIC_URL'], 'emailfrom' => $CONFIG['EMAIL_SENDER'], + 'replication_auth' => $CONFIG['REPLICATION_AUTH'], ] ]; diff --git a/server/src/dependencies.php b/server/src/dependencies.php index 5d186ca..d08f920 100644 --- a/server/src/dependencies.php +++ b/server/src/dependencies.php @@ -22,3 +22,6 @@ $container['EmailValidator'] = function($c) { $container['Status'] = function($c) { return new \LookupServer\Status(); }; +$container['Replication'] = function ($c) { + return new \LookupServer\Replication($c->db, $c->settings['replication_auth']); +}; diff --git a/server/vendor/composer/ClassLoader.php b/server/vendor/composer/ClassLoader.php index 4626994..2c72175 100644 --- a/server/vendor/composer/ClassLoader.php +++ b/server/vendor/composer/ClassLoader.php @@ -374,9 +374,13 @@ class ClassLoader $first = $class[0]; if (isset($this->prefixLengthsPsr4[$first])) { - foreach ($this->prefixLengthsPsr4[$first] as $prefix => $length) { - if (0 === strpos($class, $prefix)) { - foreach ($this->prefixDirsPsr4[$prefix] as $dir) { + $subPath = $class; + while (false !== $lastPos = strrpos($subPath, '\\')) { + $subPath = substr($subPath, 0, $lastPos); + $search = $subPath.'\\'; + if (isset($this->prefixDirsPsr4[$search])) { + foreach ($this->prefixDirsPsr4[$search] as $dir) { + $length = $this->prefixLengthsPsr4[$first][$search]; if (file_exists($file = $dir . DIRECTORY_SEPARATOR . substr($logicalPathPsr4, $length))) { return $file; } diff --git a/server/vendor/composer/LICENSE b/server/vendor/composer/LICENSE index 1a28124..f27399a 100644 --- a/server/vendor/composer/LICENSE +++ b/server/vendor/composer/LICENSE @@ -1,5 +1,5 @@ -Copyright (c) 2016 Nils Adermann, Jordi Boggiano +Copyright (c) Nils Adermann, Jordi Boggiano Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/server/vendor/composer/autoload_classmap.php b/server/vendor/composer/autoload_classmap.php index e6c8d48..8505a8b 100644 --- a/server/vendor/composer/autoload_classmap.php +++ b/server/vendor/composer/autoload_classmap.php @@ -91,6 +91,7 @@ return array( 'Interop\\Container\\Exception\\ContainerException' => $vendorDir . '/container-interop/container-interop/src/Interop/Container/Exception/ContainerException.php', 'Interop\\Container\\Exception\\NotFoundException' => $vendorDir . '/container-interop/container-interop/src/Interop/Container/Exception/NotFoundException.php', 'LookupServer\\BruteForceMiddleware' => $baseDir . '/lib/BruteForceMiddleware.php', + 'LookupServer\\Replication' => $baseDir . '/lib/Replication.php', 'LookupServer\\Status' => $baseDir . '/lib/Status.php', 'LookupServer\\UserManager' => $baseDir . '/lib/UserManager.php', 'LookupServer\\Validator\\Email' => $baseDir . '/lib/Validator/Email.php', diff --git a/server/vendor/composer/autoload_static.php b/server/vendor/composer/autoload_static.php index 11fc612..22ec6d1 100644 --- a/server/vendor/composer/autoload_static.php +++ b/server/vendor/composer/autoload_static.php @@ -173,6 +173,7 @@ class ComposerStaticInit509ee4e79733fbe3199b97373b795eca 'Interop\\Container\\Exception\\ContainerException' => __DIR__ . '/..' . '/container-interop/container-interop/src/Interop/Container/Exception/ContainerException.php', 'Interop\\Container\\Exception\\NotFoundException' => __DIR__ . '/..' . '/container-interop/container-interop/src/Interop/Container/Exception/NotFoundException.php', 'LookupServer\\BruteForceMiddleware' => __DIR__ . '/../..' . '/lib/BruteForceMiddleware.php', + 'LookupServer\\Replication' => __DIR__ . '/../..' . '/lib/Replication.php', 'LookupServer\\Status' => __DIR__ . '/../..' . '/lib/Status.php', 'LookupServer\\UserManager' => __DIR__ . '/../..' . '/lib/UserManager.php', 'LookupServer\\Validator\\Email' => __DIR__ . '/../..' . '/lib/Validator/Email.php', -- cgit v1.2.3 From c66d942a8a6fd973c6ebc1daf9331066986f5d48 Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Mon, 10 Apr 2017 12:35:55 +0200 Subject: Kill old replocation Signed-off-by: Roeland Jago Douma --- server/replication.php | 35 ----------------------------------- 1 file changed, 35 deletions(-) delete mode 100755 server/replication.php diff --git a/server/replication.php b/server/replication.php deleted file mode 100755 index 6040b59..0000000 --- a/server/replication.php +++ /dev/null @@ -1,35 +0,0 @@ -. -* -*/ - -//makes it easier to debug -error_reporting(E_ERROR | E_WARNING | E_PARSE | E_NOTICE); - -//set the default timezone to use. -date_default_timezone_set('Europe/Berlin'); - -//you have to include lib_lookup. -require('vendor/autoload.php'); - -//process the request. -$s = new \LookupServer\Server(); -$s->handleReplication(); -- cgit v1.2.3 From 1061de869a493ec2a12d242e2ef718a2ccd57ffa Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Mon, 10 Apr 2017 14:25:14 +0200 Subject: Fix replication import Signed-off-by: Roeland Jago Douma --- server/cronjob.php | 44 ++++++------------- server/lib/Replication.php | 104 +++++++++++++++++++++++++++++++++++++++++--- server/src/config.php | 1 + server/src/dependencies.php | 2 +- 4 files changed, 113 insertions(+), 38 deletions(-) diff --git a/server/cronjob.php b/server/cronjob.php index 1bac221..626034d 100755 --- a/server/cronjob.php +++ b/server/cronjob.php @@ -1,38 +1,20 @@ -#!/usr/bin/php . -* -*/ +require __DIR__ . '/vendor/autoload.php'; -// enable the full error reporting for easier debugging -error_reporting(E_ERROR | E_WARNING | E_PARSE | E_NOTICE | E_ALL); +if (PHP_SAPI !== 'cli') { + return; +} -// include the main contribook lib -require('vendor/autoload.php'); +$env = \Slim\Http\Environment::mock(['REQUEST_URI' => '/import']); -// Cronjob -$s = new \LookupServer\Server(); +$settings = require __DIR__ . '/src/config.php'; +$settings['environment'] = $env; +$container = new \Slim\Container($settings); +require __DIR__ . '/src/dependencies.php'; -// Cleanup the API Traffic Limit -$s->cleanup(); +$app = new \Slim\App($container); + +$app->map(['GET'], '/import', 'Replication:import'); +$app->run(); -// Import from other replication hosts -$s->importReplication(); diff --git a/server/lib/Replication.php b/server/lib/Replication.php index a7c5cef..002cd47 100644 --- a/server/lib/Replication.php +++ b/server/lib/Replication.php @@ -2,6 +2,7 @@ namespace LookupServer; +use GuzzleHttp\Client; use Slim\Http\Request; use Slim\Http\Response; @@ -13,9 +14,13 @@ class Replication { /** @var string */ private $auth; - public function __construct(\PDO $db, $auth) { + /** @var string[] */ + private $replicationHosts; + + public function __construct(\PDO $db, $auth, $replicationHosts) { $this->db = $db; $this->auth = $auth; + $this->replicationHosts = $replicationHosts; } public function export(Request $request, Response $response) { @@ -38,9 +43,9 @@ class Replication { $timestamp = (int)$params['timestamp']; $page = (int)$params['page']; - $stmt = $this->db->prepare('SELECT * + $stmt = $this->db->prepare('SELECT id, federationId, UNIX_TIMESTAMP(timestamp) AS timestamp FROM users - WHERE timestamp >= :timestamp + WHERE UNIX_TIMESTAMP(timestamp) >= :timestamp ORDER BY timestamp, id LIMIT :limit OFFSET :offset'); @@ -53,8 +58,8 @@ class Replication { $result = []; while($data = $stmt->fetch()) { $user = [ - 'federationId' => $data['federationId'], - 'timestamp' => $data['timestamp'], + 'cloudId' => $data['federationId'], + 'timestamp' => (int)$data['timestamp'], 'data' => [], ]; @@ -68,7 +73,7 @@ class Replication { $user['data'][] = [ 'key' => $userData['k'], 'value' => $userData['v'], - 'validated' => $userData['valid'], + 'validated' => (int)$userData['valid'], ]; } $stmt2->closeCursor(); @@ -79,4 +84,91 @@ class Replication { $response->getBody()->write(json_encode($result)); return $response; } + + public function import(Request $request, Response $response) { + $replicationStatus = []; + + if (file_exists(__DIR__ . '/../config/replication.json')) { + $replicationStatus = json_decode(file_get_contents(__DIR__ . '/../config/replication.json'), true); + } + + foreach ($this->replicationHosts as $replicationHost) { + $timestamp = 0; + + if (isset($replicationStatus[$replicationHost])) { + $timestamp = $replicationStatus[$replicationHost]; + } + + $page = 0; + while(true) { + // Retrieve public key && store + $req = new \GuzzleHttp\Psr7\Request('GET', $replicationHost . '?timestamp=' . $timestamp . '&page=' . $page); + + $client = new Client(); + $resp = $client->send($req, [ + 'timeout' => 5, + ]); + + $data = json_decode($resp->getBody(), true); + if (count($data) === 0) { + break; + } + + foreach ($data as $user) { + $this->parseUser($user); + $replicationStatus[$replicationHost] = $user['timestamp']; + } + + $page++; + } + + file_put_contents(__DIR__. '/../config/replication.json', json_encode($replicationStatus, JSON_PRETTY_PRINT)); + } + + return $response; + } + + private function parseUser($user) { + $stmt = $this->db->prepare('SELECT id, UNIX_TIMESTAMP(timestamp) AS timestamp + FROM users + WHERE federationId = :id'); + $stmt->bindParam('id', $user['cloudId']); + + $stmt->execute(); + + // New + if ($stmt->rowCount() === 1) { + $data = $stmt->fetch(); + if ($data['timestamp'] > $user['timestamp']) { + $stmt->closeCursor(); + return; + } + + $stmt2 = $this->db->prepare('DELETE FROM users + WHERE federationId = :id'); + $stmt2->bindParam('id', $user['cloudId']); + $stmt2->execute(); + $stmt2->closeCursor(); + } + + $stmt->closeCursor(); + + $stmt = $this->db->prepare('INSERT INTO users (federationId, timestamp) VALUES (:federationId, FROM_UNIXTIME(:timestamp))'); + $stmt->bindParam(':federationId', $user['cloudId'], \PDO::PARAM_STR); + $stmt->bindParam(':timestamp', $user['timestamp'], \PDO::PARAM_INT); + $stmt->execute(); + $id = $this->db->lastInsertId(); + $stmt->closeCursor(); + + foreach ($user['data'] as $data) { + $stmt = $this->db->prepare('INSERT INTO store (userId, k, v, valid) VALUES (:userId, :k, :v, :valid)'); + $stmt->bindParam(':userId', $id, \PDO::PARAM_INT); + $stmt->bindParam(':k', $data['key'], \PDO::PARAM_STR); + $stmt->bindParam(':v', $data['value'], \PDO::PARAM_STR); + $stmt->bindParam(':valid', $data['validated'], \PDO::PARAM_INT); + + $stmt->execute(); + $stmt->closeCursor(); + } + } } diff --git a/server/src/config.php b/server/src/config.php index 534f700..dc53e67 100644 --- a/server/src/config.php +++ b/server/src/config.php @@ -15,5 +15,6 @@ return [ 'host' => $CONFIG['PUBLIC_URL'], 'emailfrom' => $CONFIG['EMAIL_SENDER'], 'replication_auth' => $CONFIG['REPLICATION_AUTH'], + 'replication_hosts' => $CONFIG['REPLICATION_HOSTS'], ] ]; diff --git a/server/src/dependencies.php b/server/src/dependencies.php index d08f920..0e95274 100644 --- a/server/src/dependencies.php +++ b/server/src/dependencies.php @@ -23,5 +23,5 @@ $container['Status'] = function($c) { return new \LookupServer\Status(); }; $container['Replication'] = function ($c) { - return new \LookupServer\Replication($c->db, $c->settings['replication_auth']); + return new \LookupServer\Replication($c->db, $c->settings['replication_auth'], $c->settings['replication_hosts']); }; -- cgit v1.2.3 From f1cbda1b4ec7cfde88d99397e9387cb9a81938ef Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Mon, 10 Apr 2017 14:25:58 +0200 Subject: Fix sample config We now keep track of old replication. So we always continue where we left of. Signed-off-by: Roeland Jago Douma --- server/config/config.sample.php | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/server/config/config.sample.php b/server/config/config.sample.php index 0f5cd8a..5cc6adb 100755 --- a/server/config/config.sample.php +++ b/server/config/config.sample.php @@ -34,12 +34,9 @@ $CONFIG = [ // the list of remote replication servers that should be queried in the cronjob 'REPLICATION_HOSTS' => [ - 'https://lookup:slavefoobar@example.com' + 'https://lookup:slavefoobar@example.com/replication' ], - // replication interval. The number of seconds into the past that should be used when fetching the replication log from a remote server. Should be a bit higher then the cronjob intervall - 'REPLICATION_INTERVAL' => 900, // 15min - // ip black list. usefull to block spammers. 'IP_BLACKLIST' => [ '333.444.555.', -- cgit v1.2.3