JS Offline tracking (#15970)

* JS Offline tracking

* minor tweaks

* add some tests

* add some tests

* apply review feedback

4 files changed, 231 insertions, 1 deletions

diff --git a/core/Tracker/Request.php b/core/Tracker/Request.php
index 93d64d0781..0d6c449ab7 100644
--- a/core/Tracker/Request.php
+++ b/core/Tracker/Request.php
@@ -398,6 +398,7 @@ class Request
             // some visitor attributes can be overwritten
             'cip'          => array('', 'string'),
             'cdt'          => array('', 'string'),
+            'cdo'          => array('', 'int'),
             'cid'          => array('', 'string'),
             'uid'          => array('', 'string'),
 
@@ -484,11 +485,16 @@ class Request
 
     protected function getCustomTimestamp()
     {
-        if (!$this->hasParam('cdt')) {
+        if (!$this->hasParam('cdt') && !$this->hasParam('cdo')) {
             return false;
         }
 
         $cdt = $this->getParam('cdt');
+        $cdo = $this->getParam('cdo');
+
+        if (empty($cdt) && $cdo) {
+            $cdt = $this->timestamp;
+        }
 
         if (empty($cdt)) {
             return false;
@@ -498,6 +504,10 @@ class Request
             $cdt = strtotime($cdt);
         }
 
+        if (!empty($cdo)) {
+            $cdt = $cdt - abs($cdo);
+        }
+
         if (!$this->isTimestampValid($cdt, $this->timestamp)) {
             Common::printDebug(sprintf("Datetime %s is not valid", date("Y-m-d H:i:m", $cdt)));
             return false;
diff --git a/js/piwik.js b/js/piwik.js
index 2014d9dc7f..272dd7324a 100644
--- a/js/piwik.js
+++ b/js/piwik.js
@@ -6855,6 +6855,13 @@ if (typeof window.Matomo !== 'object') {
 
         // initialize the Matomo singleton
         addEventListener(windowAlias, 'beforeunload', beforeUnloadHandler, false);
+        addEventListener(windowAlias, 'online', function () {
+            if (isDefined(navigatorAlias.serviceWorker) && isDefined(navigatorAlias.serviceWorker.ready)) {
+                navigatorAlias.serviceWorker.ready.then(function(swRegistration) {
+                    return swRegistration.sync.register('matomoSync');
+                });
+            }
+        }, false);
 
         addEventListener(windowAlias,'message', function(e) {
             if (!e || !e.origin) {
diff --git a/offline-service-worker.js b/offline-service-worker.js
new file mode 100644
index 0000000000..a00fe1a906
--- /dev/null
+++ b/offline-service-worker.js
@@ -0,0 +1,175 @@
+var matomoAnalytics = {initialize: function (options) {
+    if ('object' !== typeof options) {
+        options = {};
+    }
+
+    var maxLimitQueue = options.queueLimit || 50;
+    var maxTimeLimit = options.timeLimit || (60 * 60 * 24); // in seconds...
+    // same as configured in in tracking_requests_require_authentication_when_custom_timestamp_newer_than
+
+    function getQueue()
+    {
+        return new Promise(function(resolve, reject) {
+            // do a thing, possibly async, then...
+
+            if (!indexedDB) {
+                reject(new Error('No support for IndexedDB'));
+                return;
+            }
+            var request = indexedDB.open("matomo", 1);
+
+            request.onerror = function() {
+                console.error("Error", request.error);
+                reject(new Error(request.error));
+            };
+            request.onupgradeneeded = function(event) {
+                console.log('onupgradeneeded')
+                var db = event.target.result;
+
+                if (!db.objectStoreNames.contains('requests')) {
+                    db.createObjectStore('requests', {autoIncrement : true, keyPath: 'id'});
+                }
+
+            };
+            request.onsuccess = function(event) {
+                var db = event.target.result;
+                let transaction = db.transaction("requests", "readwrite");
+                let requests =transaction.objectStore("requests");
+                resolve(requests);
+
+
+            };
+        });
+    }
+
+    function syncQueue () {
+        // check something in indexdb
+        return getQueue().then(function (queue) {
+            queue.openCursor().onsuccess = function(event) {
+                var cursor = event.target.result;
+                if (cursor && navigator.onLine) {
+                    cursor.continue();
+                    var queueId = cursor.value.id;
+
+                    var secondsQueuedAgo = ((Date.now() - cursor.value.created) / 1000);
+                    secondsQueuedAgo = parseInt(secondsQueuedAgo, 10);
+                    if (secondsQueuedAgo > maxTimeLimit) {
+                        // too old
+                        getQueue().then(function (queue) {
+                            queue.delete(queueId);
+                        });
+                        return;
+                    }
+
+                    console.log("Cursor " + cursor.key);
+
+                    var init = {
+                        headers: cursor.value.headers,
+                        method: cursor.value.method,
+                    }
+                    if (cursor.value.body) {
+                        init.body = cursor.value.body;
+                    }
+
+                    if (cursor.value.url.includes('?')) {
+                        cursor.value.url += '&cdo=' + secondsQueuedAgo;
+                    } else if (init.body) {
+                        // todo test if this actually works for bulk requests
+                        init.body = init.body.replace('&idsite=', '&cdo=' + secondsQueuedAgo + '&idsite=');
+                    }
+
+                    fetch(cursor.value.url, init).then(function (response) {
+                        console.log('server response', response);
+                        if (response.status < 400) {
+                            getQueue().then(function (queue) {
+                                queue.delete(queueId);
+                            });
+                        }
+                    }).catch(function (error) {
+                        console.error('Send to Server failed:', error);
+                        throw error
+                    })
+                }
+                else {
+                    console.log("No more entries!");
+                }
+            };
+        });
+    }
+
+    function limitQueueIfNeeded(queue)
+    {
+        var countRequest = queue.count();
+        countRequest.onsuccess = function(event) {
+            if (event.result > maxLimitQueue) {
+                // we delete only one at a time because of concurrency some other process might delete data too
+                queue.openCursor().onsuccess = function(event) {
+                    var cursor = event.target.result;
+                    if (cursor) {
+                        queue.delete(cursor.value.id);
+                        limitQueueIfNeeded(queue);
+                    }
+                }
+            }
+        }
+    }
+
+    self.addEventListener('sync', function(event) {
+        if (event.tag === 'matomoSync') {
+            syncQueue();
+        }
+    });
+
+    self.addEventListener('fetch', function (event)  {
+        let isOnline = navigator.onLine;
+
+        let isTrackingRequest = (event.request.url.includes('/matomo.php')
+                            || event.request.url.includes('/piwik.php'));
+        let isTrackerRequest = event.request.url.endsWith('/matomo.js')
+                            || event.request.url.endsWith('/piwik.js');
+
+        if (isTrackerRequest) {
+            if (isOnline) {
+                syncQueue();
+            }
+            caches.open('matomo').then(function(cache) {
+                return cache.match(event.request).then(function (response) {
+                    return response || fetch(event.request).then(function(response) {
+                        cache.put(event.request, response.clone());
+                        return response;
+                    });
+                });
+            })
+        } else if (isTrackingRequest && isOnline) {
+            syncQueue();
+            event.respondWith(fetch(event.request));
+        } else if (isTrackingRequest && !isOnline) {
+
+            var headers = {};
+            for (const [header, value] of event.request.headers) {
+                headers[header] = value;
+            }
+
+            let requestInfo = {
+                url: event.request.url,
+                referrer : event.request.referrer,
+                method : event.request.method,
+                referrerPolicy : event.request.referrerPolicy,
+                headers : headers,
+                created: Date.now()
+            };
+            event.request.text().then(function (postData) {
+                requestInfo.body = postData;
+
+                getQueue().then(function (queue) {
+                    queue.add(requestInfo);
+                    limitQueueIfNeeded(queue);
+
+                    return queue;
+                });
+            });
+
+        }
+    });
+}
+};
\ No newline at end of file
diff --git a/tests/PHPUnit/Unit/Tracker/RequestTest.php b/tests/PHPUnit/Unit/Tracker/RequestTest.php
index c934565a28..4a0aeaaed4 100644
--- a/tests/PHPUnit/Unit/Tracker/RequestTest.php
+++ b/tests/PHPUnit/Unit/Tracker/RequestTest.php
@@ -48,6 +48,44 @@ class RequestTest extends UnitTestCase
         $this->assertSame($this->time, $request->getCurrentTimestamp());
     }
 
+    public function test_getCurrentTimestamp_ShouldReturnTheCurrentTimestamp_IfRelativeOffsetIsUsed()
+    {
+        $request = $this->buildRequest(array('cdo' => '10'));
+        $this->assertSame($this->time - 10, $request->getCurrentTimestamp());
+    }
+
+    public function test_getCurrentTimestamp_ShouldReturnTheCurrentTimestamp_IfRelativeOffsetIsUsedIsTooMuchInPastShouldReturnFalseWhenNotAuthenticated()
+    {
+        $this->expectException(\Exception::class);
+        $this->expectExceptionMessage('Custom timestamp is 99990 seconds old, requires &token_auth');
+        $request = $this->buildRequest(array('cdo' => '99990'));
+        $this->assertSame($this->time - 10, $request->getCurrentTimestamp());
+    }
+
+    public function test_getCurrentTimestamp_CanUseRelativeOffsetAndCustomTimestamp()
+    {
+        $time = time() - 20;
+        $request = $this->buildRequest(array('cdo' => '10', 'cdt' => $time));
+        $request->setCurrentTimestamp(time());
+        $this->assertSame($time - 10, $request->getCurrentTimestamp());
+    }
+
+    public function test_getCurrentTimestamp_CanUseNegativeRelativeOffsetAndCustomTimestamp()
+    {
+        $time = time() - 20;
+        $request = $this->buildRequest(array('cdo' => '-10', 'cdt' => $time));
+        $request->setCurrentTimestamp(time());
+        $this->assertSame($time - 10, $request->getCurrentTimestamp());
+    }
+
+    public function test_getCurrentTimestamp_WithCustomTimestamp()
+    {
+        $time = time() - 20;
+        $request = $this->buildRequest(array('cdt' => $time));
+        $request->setCurrentTimestamp(time());
+        $this->assertEquals($time, $request->getCurrentTimestamp());
+    }
+
     public function test_isEmptyRequest_ShouldReturnTrue_InCaseNoParamsSet()
     {
         $request = $this->buildRequest(array());