Adds some more tests for log importer (#17700)

* Adds some tests for log importer with some basic options

* test some more options

* Adds tests for some more options

3 files changed, 239 insertions, 0 deletions

diff --git a/tests/PHPUnit/Integration/LogImporterTest.php b/tests/PHPUnit/Integration/LogImporterTest.php
new file mode 100644
index 0000000000..a83233c792
--- /dev/null
+++ b/tests/PHPUnit/Integration/LogImporterTest.php
@@ -0,0 +1,230 @@
+<?php
+/**
+ * Matomo - free/libre analytics platform
+ *
+ * @link    https://matomo.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+
+namespace Piwik\Tests\Integration;
+
+use Piwik\Common;
+use Piwik\Container\StaticContainer;
+use Piwik\Db;
+use Piwik\Filesystem;
+use Piwik\Tests\Framework\Fixture;
+use Piwik\Tests\Framework\TestCase\IntegrationTestCase;
+
+/**
+ * @group Core
+ * @group Integration
+ * @group LogImporter
+ */
+class LogImporterTest extends IntegrationTestCase
+{
+    public static $dateTime = '2012-08-09 11:22:33';
+
+    public function setUp(): void
+    {
+        parent::setUp();
+        Fixture::createSuperUser(false);
+        Fixture::createWebsite(self::$dateTime, 0, false, 'https://matomo.org/', 1, null, null, null, null, 0);
+        Fixture::createWebsite(self::$dateTime, 1, false, 'https://shop.matomo.org/', 1, null, null, null, null, 0);
+        Fixture::createWebsite(self::$dateTime, 0, false, 'https://piwik.org/', 1, null, null, null, null, 1);
+    }
+
+    /**
+     * @dataProvider getParserOptionsToTest
+     */
+    public function testParserOptions($options, $expectedVisitAndActionCountPerSite, $outputToMatch = null)
+    {
+        // ensure to send testmode option
+        $options['--enable-testmode'] = false;
+
+        $result = Fixture::executeLogImporter(PIWIK_PATH_TEST_TO_ROOT . '/tests/resources/access-logs/different_hosts.log', $options, true);
+
+        if ($outputToMatch) {
+            $this->assertStringContainsString($outputToMatch, implode("\n", $result));
+        }
+
+        foreach ($expectedVisitAndActionCountPerSite as $expectedVisitAndActionCount) {
+            $this->assertVisitAndActionCount($expectedVisitAndActionCount[0], $expectedVisitAndActionCount[1], $expectedVisitAndActionCount[2] ?? null);
+        }
+    }
+
+    public function getParserOptionsToTest(): array
+    {
+        return [
+            [
+                [],
+                [[3, 3], [1, 1, 1], [1, 1, 2], [1, 1, 3]],
+            ],
+            [
+                ['--token-auth' => Fixture::ADMIN_USER_TOKEN,],
+                [[3, 3], [1, 1, 1], [1, 1, 2], [1, 1, 3]],
+            ],
+            [
+                ['--login' => Fixture::ADMIN_USER_LOGIN, '--password' => Fixture::ADMIN_USER_PASSWORD,],
+                [[3, 3], [1, 1, 1], [1, 1, 2], [1, 1, 3]],
+            ],
+            [
+                ['--idsite-fallback' => '1',],
+                [[4, 4], [2, 2, 1], [1, 1, 2], [1, 1, 3]],
+            ],
+            [
+                ['--add-sites-new-hosts' => false,],
+                [[4, 4], [1, 1, 1], [1, 1, 2], [1, 1, 3], [1, 1, 4]],
+            ],
+            [
+                // --idsite that allows all hosts
+                ['--idsite' => '1',],
+                [[4, 4], [4, 4, 1], [0, 0, 2], [0, 0, 3]],
+            ],
+            [
+                // --idsite that only allows specific hosts
+                ['--idsite' => '3',],
+                [[1, 1], [0, 0, 1], [0, 0, 2], [1, 1, 3]],
+            ],
+            [
+                ['--skip' => '3',],
+                [[1, 1], [0, 0, 1], [1, 1, 2], [0, 0, 3]],
+            ],
+            [
+                ['--dry-run' => false,],
+                [[0, 0], [0, 0, 1], [0, 0, 2], [0, 0, 3]],
+                '4 requests imported successfully',
+            ],
+            [
+                ['--log-hostname' => 'matomo.org',],
+                [[4, 4], [4, 4, 1], [0, 0, 2], [0, 0, 3]],
+            ],
+            [
+                ['--hostname' => '*matomo.org',],
+                [[2, 2], [1, 1, 1], [1, 1, 2], [0, 0, 3]],
+            ],
+            [
+                ['--hostname' => ['matomo.org', 'piwik.org'],],
+                [[2, 2], [1, 1, 1], [0, 0, 2], [1, 1, 3]],
+            ],
+            [
+                ['--include-host' => 'matomo.org',],
+                [[1, 1], [1, 1, 1], [0, 0, 2], [0, 0, 3]],
+            ],
+            [
+                ['--include-host' => ['matomo.org', 'shop.matomo.org'],],
+                [[2, 2], [1, 1, 1], [1, 1, 2], [0, 0, 3]],
+            ],
+            [
+                ['--exclude-host' => 'matomo.org',],
+                [[2, 2], [0, 0, 1], [1, 1, 2], [1, 1, 3]],
+            ],
+            [
+                ['--exclude-host' => ['matomo.org', 'shop.matomo.org'],],
+                [[1, 1], [0, 0, 1], [0, 0, 2], [1, 1, 3]],
+            ],
+            [
+                ['--exclude-older-than' => '2020-08-11 09:00:00 +0000',],
+                [[1, 1], [0, 0, 1], [1, 1, 2], [0, 0, 3]],
+            ],
+            [
+                ['--exclude-newer-than' => '2020-08-11 09:00:00 +0000',],
+                [[2, 2], [1, 1, 1], [0, 0, 2], [1, 1, 3]],
+            ],
+            [
+                ['--enable-static' => false,],
+                [[3, 4], [1, 1, 1], [1, 1, 2], [1, 2, 3]],
+            ],
+            [
+                ['--enable-bots' => false,],
+                [[4, 4], [2, 2, 1], [1, 1, 2], [1, 1, 3]],
+            ],
+            [
+                ['--useragent-exclude' => 'Android',],
+                [[2, 2], [1, 1, 1], [1, 1, 2], [0, 0, 3]],
+            ],
+            [
+                ['--useragent-exclude' => ['Android', 'Linux'],],
+                [[1, 1], [0, 0, 1], [1, 1, 2], [0, 0, 3]],
+            ],
+            [
+                ['--enable-http-errors' => false,],
+                [[3, 4], [1, 1, 1], [1, 2, 2], [1, 1, 3]],
+            ],
+            [
+                ['--enable-http-redirects' => false,],
+                [[4, 4], [2, 2, 1], [1, 1, 2], [1, 1, 3]],
+            ],
+            [
+                ['--exclude-path' => '*/guides/*',],
+                [[2, 2], [1, 1, 1], [1, 1, 2], [0, 0, 3]],
+            ],
+            [
+                ['--include-path' => '*/guides/*',],
+                [[1, 1], [0, 0, 1], [0, 0, 2], [1, 1, 3]],
+            ],
+        ];
+    }
+
+    public function testEncodingOption()
+    {
+        $options = [
+            '--encoding' => 'windows-1252',
+            '--enable-testmode' => false,
+        ];
+
+        $result = Fixture::executeLogImporter(PIWIK_PATH_TEST_TO_ROOT . '/tests/resources/access-logs/windows-1252.log', $options, true);
+
+        $this->assertVisitCount(1, 1);
+        $this->assertActionCount(1, 1);
+
+        $name = Db::fetchOne('SELECT `name` FROM ' . Common::prefixTable('log_action'));
+        $this->assertEquals('matomo.org/äöüߧ$%', $name);
+    }
+
+    /**
+     * @group bla
+     */
+    public function testOutputOption()
+    {
+        $file = StaticContainer::get('path.tmp') . '/logs/import_log.log';
+
+        Filesystem::deleteFileIfExists($file);
+
+        $options = [
+            '--output' => $file,
+            '--enable-testmode' => false,
+        ];
+
+        Fixture::executeLogImporter(PIWIK_PATH_TEST_TO_ROOT . '/tests/resources/access-logs/different_hosts.log', $options, true);
+
+        $this->assertVisitCount(3);
+
+        $output = file_get_contents($file);
+        $this->assertStringContainsString('4 requests imported successfully', $output);
+    }
+
+    protected function assertVisitAndActionCount($visitCount, $actionCount, $idSite = null)
+    {
+        $this->assertVisitCount($visitCount, $idSite);
+        $this->assertActionCount($actionCount, $idSite);
+    }
+
+    protected function assertVisitCount($expectedCount, $idSite = null)
+    {
+        $where = $idSite ? ' AND idsite = ' . (int)$idSite : '';
+
+        $visitCount = Db::fetchOne('SELECT count(*) FROM ' . Common::prefixTable('log_visit') . ' WHERE 1 ' . $where);
+
+        self::assertEquals($expectedCount, $visitCount);
+    }
+
+    protected function assertActionCount($expectedCount, $idSite = null)
+    {
+        $where = $idSite ? ' AND idsite = ' . (int)$idSite : '';
+
+        $actionCount = Db::fetchOne('SELECT count(*) FROM ' . Common::prefixTable('log_link_visit_action') . ' WHERE 1 ' . $where);
+
+        self::assertEquals($expectedCount, $actionCount);
+    }
+}
\ No newline at end of file
diff --git a/tests/resources/access-logs/different_hosts.log b/tests/resources/access-logs/different_hosts.log
new file mode 100644
index 0000000000..114b11734d
--- /dev/null
+++ b/tests/resources/access-logs/different_hosts.log
@@ -0,0 +1,8 @@
+piwik.org 175.12.22.40 - - [09/Aug/2020:13:11:30 +0200] "GET /guides/ HTTP/1.1" 200 1221 "-" "Mozilla/5.0 (Linux; U; Android 2.3.5; en-us; HTC Vision Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1"
+piwik.org 175.12.22.40 - - [09/Aug/2020:13:11:31 +0200] "GET /guides/logo.jpeg HTTP/1.1" 200 255 "-" "Mozilla/5.0 (Linux; U; Android 2.3.5; en-us; HTC Vision Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1"
+matomo.org 175.22.92.40 - - [10/Aug/2020:13:20:30 +0200] "GET /faq/ HTTP/1.1" 200 3574 "-" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.1.963.56 Safari/535.11"
+matomo.org 166.55.44.00 - - [10/Aug/2020:19:20:30 +0200] "GET /faq/ HTTP/1.1" 200 3574 "-" "Googlebot/2.1 (http://www.googlebot.com/bot.html)"
+shop.matomo.org 185.41.192.40 - - [11/Aug/2020:13:55:30 +0200] "GET /cart/ HTTP/1.1" 200 4455 "-" "Mozilla/5.0 (Windows NT 10.0.16299.125; osmeta 10.3.3308) AppleWebKit/602.1.1 (KHTML, like Gecko) Version/9.0 Safari/602.1.1 osmeta/10.3.3308 Build/3308 [FBAN/FBW;FBAV/140.0.0.232.179;FBBV/83145113;FBDV/WindowsDevice;FBMD/80VR;FBSN/Windows;FBSV/10.0.16299.371;FBSS/1;FBCR/;FBID/desktop;FBLC/ru_RU;FBOP/45;FBRV/0]"
+shop.matomo.org 185.41.192.40 - - [11/Aug/2020:13:57:30 +0200] "GET /invalid/ HTTP/1.1" 404 235 "-" "Mozilla/5.0 (Windows NT 10.0.16299.125; osmeta 10.3.3308) AppleWebKit/602.1.1 (KHTML, like Gecko) Version/9.0 Safari/602.1.1 osmeta/10.3.3308 Build/3308 [FBAN/FBW;FBAV/140.0.0.232.179;FBBV/83145113;FBDV/WindowsDevice;FBMD/80VR;FBSN/Windows;FBSV/10.0.16299.371;FBSS/1;FBCR/;FBID/desktop;FBLC/ru_RU;FBOP/45;FBRV/0]"
+www.example.fr 105.41.102.40 - - [12/Aug/2020:14:05:30 +0200] "GET / HTTP/1.1" 200 252 "-" "Mozilla/5.0 (Linux; Android 6.0; BRAVIS S500) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.116 Mobile Safari/537.36 OPR/55.1.2719.50626"
+matomo.org 112.45.45.45 - - [14/Aug/2020:10:20:30 +0200] "GET /faqs/ HTTP/1.1" 302 122 "-" "Mozilla/5.0 (Linux; Android 4.2.2; DEXP Ursus 9EV 3G Build/JDQ39) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.68 Safari/537.36"
diff --git a/tests/resources/access-logs/windows-1252.log b/tests/resources/access-logs/windows-1252.log
new file mode 100644
index 0000000000..410267dcfb
--- /dev/null
+++ b/tests/resources/access-logs/windows-1252.log
@@ -0,0 +1 @@
+matomo.org 1.2.3.4 - theuser [10/Feb/2012:16:42:07 -0500] "GET /äöüߧ$% HTTP/1.0" 200 368 "-" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11"