In log importer, allow filtering logs by host & datetime (new submodule commit + tests). (#12267)

* In log importer, allow filtering logs by host & datetime (new submodule commit + tests).

* Add tests for replay tracking filtering & check that filtered line is in output.

* More log filtering tests for log replay.

1 files changed, 51 insertions, 2 deletions

diff --git a/tests/PHPUnit/Fixtures/ManySitesImportedLogs.php b/tests/PHPUnit/Fixtures/ManySitesImportedLogs.php
index f89fc4948b..8dea041717 100644
--- a/tests/PHPUnit/Fixtures/ManySitesImportedLogs.php
+++ b/tests/PHPUnit/Fixtures/ManySitesImportedLogs.php
@@ -141,6 +141,43 @@ class ManySitesImportedLogs extends Fixture
         if ($this->includeApiCustomVarMapping) {
             $this->logIisWithCustomFormat($mapToCustom = true);
         }
+
+        $this->logWithIncludeFilters();
+        $this->logWithExcludeFilters();
+    }
+
+    private function logWithExcludeFilters()
+    {
+        $logFile = PIWIK_INCLUDE_PATH . '/tests/resources/access-logs/filter_exclude_logs.log'; # log file
+
+        $opts = array('--idsite'                    => $this->idSite,
+                      '--enable-testmode'           => false,
+                      '--recorders'                 => '1',
+                      '--recorder-max-payload-size' => '1',
+                      '--exclude-host' => ['filtered1.com', 'www.filtered2.com'],
+                      '--exclude-older-than' => '2012-08-09 08:10:39 +0000',
+                      '--exclude-newer-than' => '2012-08-31 00:00:00 +0000');
+
+        $output = self::executeLogImporter($logFile, $opts);
+        $output = implode("\n", $output);
+
+        $this->assertContains('4 filtered log lines', $output);
+    }
+
+    private function logWithIncludeFilters()
+    {
+        $logFile = PIWIK_INCLUDE_PATH . '/tests/resources/access-logs/filter_include_logs.log'; # log file
+
+        $opts = array('--idsite'                    => $this->idSite,
+                      '--enable-testmode'           => false,
+                      '--recorders'                 => '1',
+                      '--recorder-max-payload-size' => '1',
+                      '--include-host' => ['included3.com', 'www.included4.com']);
+
+        $output = self::executeLogImporter($logFile, $opts);
+        $output = implode("\n", $output);
+
+        $this->assertContains('2 filtered log lines', $output);
     }
 
     private function setupSegments()
@@ -238,16 +275,28 @@ class ManySitesImportedLogs extends Fixture
     private function replayLogFile($additonalOptions = array())
     {
         $logFile = PIWIK_INCLUDE_PATH . '/tests/resources/access-logs/fake_logs_replay.log';
+        $logFileWithHost = PIWIK_INCLUDE_PATH . '/tests/resources/access-logs/fake_logs_replay_host.log';
 
         $opts = array('--login'                     => 'superUserLogin',
                       '--password'                  => 'superUserPass',
                       '--recorders'                 => '1',
                       '--recorder-max-payload-size' => '1',
-                      '--replay-tracking'           => false);
+                      '--replay-tracking'           => false,
+                      '--exclude-older-than' => '2012-01-01 08:10:39 +0000',
+                      '--exclude-host' => ['excluded.com']);
 
         $opts = array_merge($opts, $additonalOptions);
 
-        self::executeLogImporter($logFile, $opts);
+        $output = self::executeLogImporter($logFile, $opts);
+        $output = implode("\n", $output);
+
+        $this->assertContains('1 filtered log lines', $output);
+
+        // test that correct logs are excluded when the host is in the log file
+        $output = self::executeLogImporter($logFileWithHost, $opts);
+        $output = implode("\n", $output);
+
+        $this->assertContains('2 filtered log lines', $output);
     }
 
     /**