In log importer, allow filtering logs by host & datetime (new submodule commit + tests). (#12267)

* In log importer, allow filtering logs by host & datetime (new submodule commit + tests).

* Add tests for replay tracking filtering & check that filtered line is in output.

* More log filtering tests for log replay.

4 files changed, 23 insertions, 0 deletions

diff --git a/tests/resources/access-logs/fake_logs_replay.log b/tests/resources/access-logs/fake_logs_replay.log
index 0a162381b7..e604c860bb 100644
--- a/tests/resources/access-logs/fake_logs_replay.log
+++ b/tests/resources/access-logs/fake_logs_replay.log
@@ -28,3 +28,6 @@
 193.159.20.129 - - [13/Mar/2014:01:34:42 +0100] "GET /piwik.php?action_name=demo.piwik.org%2FPiwik%20Forums%20-%20Piwik%20%E2%80%BA%20Web%20Analytics%20Reports&idsite=1&rec=1&r=546849&h=1&m=34&s=37&url=http%3A%2F%2Fdemo.piwik.org%2Findex.php%3Fmodule%3DCoreHome%26action%3Dindex%26date%3Dyesterday%26period%3Dday%26idSite%3D7%23%2Fmodule%3DLive%26action%3DindexVisitorLog%26date%3Dyesterday%26period%3Dday%26idSite%3D7&urlref=http%3A%2F%2Fdemo.piwik.org%2Findex.php%3Fmodule%3DMultiSites%26action%3Dindex%26date%3Dyesterday%26period%3Dday%26idSite%3D32&_id=d80e3396f1a4c2ac&_idts=1394665308&_idvc=1&_idn=0&_refts=0&_viewts=1394365308&pdf=1&qt=1&realp=0&wma=0&dir=0&fla=1&java=1&gears=0&ag=1&cookie=1&res=1920x1200&_cvar=%7B%221%22%3A%5B%22Domain%20landed%22%2C%22piwik.org%22%5D%2C%222%22%3A%5B%22Demo%20language%22%2C%22English%22%5D%7D&gt_ms=413 HTTP/1.1" 200 43 "http://demo.piwik.org/index.php?module=CoreHome&action=index&date=yesterday&period=day&idSite=7" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.45 Safari/537.36"
 206.190.75.8 - - [13/Mar/2014:01:34:46 +0100] "GET /piwik.php?action_name=Contact%20the%20Piwik%20team%20-%20Analytics%20-%20Piwik&idsite=1&rec=1&r=456386&h=17&m=34&s=45&url=http%3A%2F%2Fpiwik.org%2Fcontact%2F&urlref=http%3A%2F%2Fpiwik.org%2Fchangelog%2F&_id=e0eb35c4fae2aa0a&_idts=1379033580&_idvc=2&_idn=0&_refts=1394169945&_viewts=1379033693&_ref=http%3A%2F%2Ftrends.builtwith.com%2Fanalytics%2FPiwik-Web-Analytics&res=1536x864&_cvar=%7B%221%22%3A%5B%22Domain%20landed%22%2C%22piwik.org%22%5D%7D HTTP/1.1" 200 43 "http://piwik.org/contact/" "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; .NET4.0C; InfoPath.2)"
 174.97.139.63 - - [13/Mar/2014:01:34:53 +0100] "GET /piwik.php?action_name=Liberate%20Web%20Analytics%20-%20Analytics%20-%20Piwik&idsite=1&rec=1&r=022131&h=20&m=34&s=49&url=http%3A%2F%2Fpiwik.org%2F&urlref=http%3A%2F%2Fblog.comperiosearch.com%2Fblog%2F2014%2F02%2F05%2Fdynamic-search-ranking-using-elasticsearch-neo4j-and-piwik%2F&_id=d98b53d7fcf5a082&_idts=1394670890&_idvc=77&_idn=1&_refts=1394670890&_viewts=1394670890&_ref=http%3A%2F%2Fblog.comperiosearch.com%2Fblog%2F2014%2F02%2F05%2Fdynamic-search-ranking-using-elasticsearch-neo4j-and-piwik%2F&pdf=1&qt=0&realp=0&wma=0&dir=0&fla=1&java=1&gears=0&ag=0&cookie=1&res=1366x768&_cvar=%7B%221%22%3A%5B%22Domain%20landed%22%2C%22piwik.org%22%5D%7D&gt_ms=292 HTTP/1.1" 200 43 "http://piwik.org/" "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.146 Safari/537.36"
+
+# excluded logs
+174.97.139.63 - - [13/Mar/2010:01:34:53 +0100] "GET /piwik.php?action_name=Liberate%20Web%20Analytics%20-%20Analytics%20-%20Piwik&idsite=1&rec=1&r=022131&h=20&m=34&s=49&url=http%3A%2F%2Fpiwik.org%2F&urlref=http%3A%2F%2Fblog.comperiosearch.com%2Fblog%2F2014%2F02%2F05%2Fdynamic-search-ranking-using-elasticsearch-neo4j-and-piwik%2F&_id=d98b53d7fcf5a082&_idts=1394670890&_idvc=77&_idn=1&_refts=1394670890&_viewts=1394670890&_ref=http%3A%2F%2Fblog.comperiosearch.com%2Fblog%2F2014%2F02%2F05%2Fdynamic-search-ranking-using-elasticsearch-neo4j-and-piwik%2F&pdf=1&qt=0&realp=0&wma=0&dir=0&fla=1&java=1&gears=0&ag=0&cookie=1&res=1366x768&_cvar=%7B%221%22%3A%5B%22Domain%20landed%22%2C%22piwik.org%22%5D%7D&gt_ms=292 HTTP/1.1" 200 43 "http://piwik.org/" "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.146 Safari/537.36"
diff --git a/tests/resources/access-logs/fake_logs_replay_host.log b/tests/resources/access-logs/fake_logs_replay_host.log
new file mode 100644
index 0000000000..31dfce6c6f
--- /dev/null
+++ b/tests/resources/access-logs/fake_logs_replay_host.log
@@ -0,0 +1,3 @@
+# excluded logs
+piwik.org 174.97.139.63 - - [13/Mar/2010:01:34:53 +0100] "GET /piwik.php?action_name=Liberate%20Web%20Analytics%20-%20Analytics%20-%20Piwik&idsite=1&rec=1&r=022131&h=20&m=34&s=49&url=http%3A%2F%2Fpiwik.org%2F&urlref=http%3A%2F%2Fblog.comperiosearch.com%2Fblog%2F2014%2F02%2F05%2Fdynamic-search-ranking-using-elasticsearch-neo4j-and-piwik%2F&_id=d98b53d7fcf5a082&_idts=1394670890&_idvc=77&_idn=1&_refts=1394670890&_viewts=1394670890&_ref=http%3A%2F%2Fblog.comperiosearch.com%2Fblog%2F2014%2F02%2F05%2Fdynamic-search-ranking-using-elasticsearch-neo4j-and-piwik%2F&pdf=1&qt=0&realp=0&wma=0&dir=0&fla=1&java=1&gears=0&ag=0&cookie=1&res=1366x768&_cvar=%7B%221%22%3A%5B%22Domain%20landed%22%2C%22piwik.org%22%5D%7D&gt_ms=292 HTTP/1.1" 200 43 "http://piwik.org/" "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.146 Safari/537.36"
+excluded.com 174.97.139.63 - - [13/Mar/2016:01:34:53 +0100] "GET /piwik.php?action_name=Liberate%20Web%20Analytics%20-%20Analytics%20-%20Piwik&idsite=1&rec=1&r=022131&h=20&m=34&s=49&url=http%3A%2F%2Fpiwik.org%2F&urlref=http%3A%2F%2Fblog.comperiosearch.com%2Fblog%2F2014%2F02%2F05%2Fdynamic-search-ranking-using-elasticsearch-neo4j-and-piwik%2F&_id=d98b53d7fcf5a082&_idts=1394670890&_idvc=77&_idn=1&_refts=1394670890&_viewts=1394670890&_ref=http%3A%2F%2Fblog.comperiosearch.com%2Fblog%2F2014%2F02%2F05%2Fdynamic-search-ranking-using-elasticsearch-neo4j-and-piwik%2F&pdf=1&qt=0&realp=0&wma=0&dir=0&fla=1&java=1&gears=0&ag=0&cookie=1&res=1366x768&_cvar=%7B%221%22%3A%5B%22Domain%20landed%22%2C%22piwik.org%22%5D%7D&gt_ms=292 HTTP/1.1" 200 43 "http://piwik.org" "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.146 Safari/537.36"
diff --git a/tests/resources/access-logs/filter_exclude_logs.log b/tests/resources/access-logs/filter_exclude_logs.log
new file mode 100644
index 0000000000..3040abed06
--- /dev/null
+++ b/tests/resources/access-logs/filter_exclude_logs.log
@@ -0,0 +1,10 @@
+# excluded logs
+175.41.191.41 - - [09/Aug/2012:10:10:38 +0200] "GET http://filtered1.com/blog/category/meta/ HTTP/1.1" 200 3574 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_7) AppleWebKit/534.24 (KHTML, like Gecko) RockMelt/0.9.58.494 Chrome/11.0.696.71 Safari/534.24"
+175.41.191.42 - - [09/Aug/2012:10:10:38 +0200] "GET http://www.filtered2.com/blog/category/meta/ HTTP/1.1" 200 3574 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_7) AppleWebKit/534.24 (KHTML, like Gecko) RockMelt/0.9.58.494 Chrome/11.0.696.71 Safari/534.24"
+175.41.191.43 - - [09/Jul/2012:10:10:38 +0200] "GET http://www.datefiltered.com/blog/category/meta/ HTTP/1.1" 200 3574 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_7) AppleWebKit/534.24 (KHTML, like Gecko) RockMelt/0.9.58.494 Chrome/11.0.696.71 Safari/534.24"
+175.41.191.44 - - [09/Sep/2012:10:10:38 +0200] "GET http://www.datefiltered.com/blog/category/meta/ HTTP/1.1" 200 3574 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_7) AppleWebKit/534.24 (KHTML, like Gecko) RockMelt/0.9.58.494 Chrome/11.0.696.71 Safari/534.24"
+
+# included logs
+175.41.191.45 - - [10/Aug/2012:10:10:38 +0200] "GET http://included1.com/blog/category/meta/ HTTP/1.1" 200 3574 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_7) AppleWebKit/534.24 (KHTML, like Gecko) RockMelt/0.9.58.494 Chrome/11.0.696.71 Safari/534.24"
+175.41.191.46 - - [09/Aug/2012:10:10:39 +0200] "GET http://www.included2.com/blog/category/meta/ HTTP/1.1" 200 3574 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_7) AppleWebKit/534.24 (KHTML, like Gecko) RockMelt/0.9.58.494 Chrome/11.0.696.71 Safari/534.24"
+175.41.191.47 - - [19/Aug/2012:10:10:38 +0200] "GET http://www.notdatefiltered.com/blog/category/meta/ HTTP/1.1" 200 3574 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_7) AppleWebKit/534.24 (KHTML, like Gecko) RockMelt/0.9.58.494 Chrome/11.0.696.71 Safari/534.24"
diff --git a/tests/resources/access-logs/filter_include_logs.log b/tests/resources/access-logs/filter_include_logs.log
new file mode 100644
index 0000000000..99ac478a54
--- /dev/null
+++ b/tests/resources/access-logs/filter_include_logs.log
@@ -0,0 +1,7 @@
+# excluded logs
+175.41.193.41 - - [09/Aug/2012:10:10:38 +0200] "GET http://filtered1.com/blog/category/meta/ HTTP/1.1" 200 3574 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_7) AppleWebKit/534.24 (KHTML, like Gecko) RockMelt/0.9.58.494 Chrome/11.0.696.71 Safari/534.24"
+175.41.193.42 - - [09/Aug/2012:10:10:38 +0200] "GET http://www.filtered2.com/blog/category/meta/ HTTP/1.1" 200 3574 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_7) AppleWebKit/534.24 (KHTML, like Gecko) RockMelt/0.9.58.494 Chrome/11.0.696.71 Safari/534.24"
+
+# included logs
+175.41.193.45 - - [10/Aug/2012:10:10:38 +0200] "GET http://included3.com/blog/category/meta/ HTTP/1.1" 200 3574 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_7) AppleWebKit/534.24 (KHTML, like Gecko) RockMelt/0.9.58.494 Chrome/11.0.696.71 Safari/534.24"
+175.41.193.46 - - [09/Aug/2012:10:10:39 +0200] "GET http://www.included4.com/blog/category/meta/ HTTP/1.1" 200 3574 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_7) AppleWebKit/534.24 (KHTML, like Gecko) RockMelt/0.9.58.494 Chrome/11.0.696.71 Safari/534.24"