diff options
85 files changed, 2861 insertions, 0 deletions
diff --git a/cid.log b/cid.log new file mode 100644 index 0000000000..5577db17e3 --- /dev/null +++ b/cid.log @@ -0,0 +1,4 @@ + +5348954:180.153.201.215 - - [02/Aug/2015:21:15:38 +0000] "GET /piwik.php?action_name=Kavos aparatai nuo 36 ??\xAC&idsite=1&rec=1&r=199956&h=0&m=18&s=24&url=http://www.kavosdraugas.lt/kavos-aparatai?gclid=CL6m2O6oi8cCFY_JtAod_tYIPQ&price=216,456&urlref=http://www.kavosdraugas.lt/kavos-aparatai?gclid=CL6m2O6oi8cCFY_JtAod_tYIPQ&_id=54f474c0c50144df&_idts=1438550142&_idvc=1&_idn=0&_refts=1438550142&_viewts=1438550142&_ref=http://www.googleadservices.com/pagead/aclk?sa=L&ai=Cii7VcIi-VeDRGIaRjAaPqK-QBZr23PYFwq3-29QBtcrnsxUIABABIL_gsAMoA2C5A6ABrqKPywPIAQGpAtRzbKu7bLI-qgQiT9Dly94npwbCg5D7SkUsuqSK-SLDbgVFTQ55B4AxabBQloAFkE6IBgGAB7rd8DSQBwOoB6LCG6gHpr4b2AcB&ohost=www.google.com&cid=5GjGWDPimmkdeh1FrgcuUImc3pojqtIChK-EDtUjr4JZmw&sig=AOD64_3GESvwtzmjkBhlZRJcFCvb4QN70g&clui=0&rct=j&q=&ved=0CBoQ0QxqFQoTCIz3keuoi8cCFcKyFAodIQ0Ktw&adurl=http://www.kavosdraugas.lt/kavos-aparatai&pdf=1&qt=0&realp=0&wma=1&dir=0&fla=1&java=1&gears=0&ag=0&cookie=1&res=1366x768>_ms=7972 HTTP/1.1" 200 54 "http://www.kavosdraugas.lt/kavos-aparatai?gclid=CL6m2O6oi8cCFY_JtAod_tYIPQ&price=216,456" "Mozilla/5.0 (Linux; U; Android 4.4.2; zh-cn; GT-I9500 Build/KOT49H) AppleWebKit/537.36 (KHTML, like Gecko)Version/4.0 MQQBrowser/5.0 QQ-Manager Mobile Safari/537.36" + + diff --git a/cid.log~ b/cid.log~ new file mode 100644 index 0000000000..d266b0dce5 --- /dev/null +++ b/cid.log~ @@ -0,0 +1,21 @@ +5333697:87.247.69.4 - - [02/Aug/2015:21:12:54 +0000] "GET /piwik.php?action_name=Kavos%20aparatai%20nuo%2036%20%E2%82%AC&idsite=1&rec=1&r=006963&h=0&m=15&s=41&url=http%3A%2F%2Fwww.kavosdraugas.lt%2Fkavos-aparatai%3Fgclid%3DCL6m2O6oi8cCFY_JtAod_tYIPQ&urlref=http%3A%2F%2Fwww.googleadservices.com%2Fpagead%2Faclk%3Fsa%3DL%26ai%3DCii7VcIi-VeDRGIaRjAaPqK-QBZr23PYFwq3-29QBtcrnsxUIABABIL_gsAMoA2C5A6ABrqKPywPIAQGpAtRzbKu7bLI-qgQiT9Dly94npwbCg5D7SkUsuqSK-SLDbgVFTQ55B4AxabBQloAFkE6IBgGAB7rd8DSQBwOoB6LCG6gHpr4b2AcB%26ohost%3Dwww.google.com%26cid%3D5GjGWDPimmkdeh1FrgcuUImc3pojqtIChK-EDtUjr4JZmw%26sig%3DAOD64_3GESvwtzmjkBhlZRJcFCvb4QN70g%26clui%3D0%26rct%3Dj%26q%3D%26ved%3D0CBoQ0QxqFQoTCIz3keuoi8cCFcKyFAodIQ0Ktw%26adurl%3Dhttp%3A%2F%2Fwww.kavosdraugas.lt%2Fkavos-aparatai&_id=54f474c0c50144df&_idts=1438550142&_idvc=1&_idn=1&_refts=1438550142&_viewts=1438550142&_ref=http%3A%2F%2Fwww.googleadservices.com%2Fpagead%2Faclk%3Fsa%3DL%26ai%3DCii7VcIi-VeDRGIaRjAaPqK-QBZr23PYFwq3-29QBtcrnsxUIABABIL_gsAMoA2C5A6ABrqKPywPIAQGpAtRzbKu7bLI-qgQiT9Dly94npwbCg5D7SkUsuqSK-SLDbgVFTQ55B4AxabBQloAFkE6IBgGAB7rd8DSQBwOoB6LCG6gHpr4b2AcB%26ohost%3Dwww.google.com%26cid%3D5GjGWDPimmkdeh1FrgcuUImc3pojqtIChK-EDtUjr4JZmw%26sig%3DAOD64_3GESvwtzmjkBhlZRJcFCvb4QN70g%26clui%3D0%26rct%3Dj%26q%3D%26ved%3D0CBoQ0QxqFQoTCIz3keuoi8cCFcKyFAodIQ0Ktw%26adurl%3Dhttp%3A%2F%2Fwww.kavosdraugas.lt%2Fkavos-aparatai&pdf=1&qt=0&realp=0&wma=1&dir=0&fla=1&java=1&gears=0&ag=0&cookie=1&res=1366x768>_ms=3068 HTTP/1.1" 200 54 "http://www.kavosdraugas.lt/kavos-aparatai?gclid=CL6m2O6oi8cCFY_JtAod_tYIPQ" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.130 Safari/537.36 OPR/30.0.1835.125" + +5334309:87.247.69.4 - - [02/Aug/2015:21:13:01 +0000] "GET /piwik.php?e_c=997&e_a=8218%2Fshown&idsite=1&rec=1&r=680202&h=0&m=15&s=48&url=http%3A%2F%2Fwww.kavosdraugas.lt%2Fkavos-aparatai%3Fgclid%3DCL6m2O6oi8cCFY_JtAod_tYIPQ&urlref=http%3A%2F%2Fwww.googleadservices.com%2Fpagead%2Faclk%3Fsa%3DL%26ai%3DCii7VcIi-VeDRGIaRjAaPqK-QBZr23PYFwq3-29QBtcrnsxUIABABIL_gsAMoA2C5A6ABrqKPywPIAQGpAtRzbKu7bLI-qgQiT9Dly94npwbCg5D7SkUsuqSK-SLDbgVFTQ55B4AxabBQloAFkE6IBgGAB7rd8DSQBwOoB6LCG6gHpr4b2AcB%26ohost%3Dwww.google.com%26cid%3D5GjGWDPimmkdeh1FrgcuUImc3pojqtIChK-EDtUjr4JZmw%26sig%3DAOD64_3GESvwtzmjkBhlZRJcFCvb4QN70g%26clui%3D0%26rct%3Dj%26q%3D%26ved%3D0CBoQ0QxqFQoTCIz3keuoi8cCFcKyFAodIQ0Ktw%26adurl%3Dhttp%3A%2F%2Fwww.kavosdraugas.lt%2Fkavos-aparatai&_id=54f474c0c50144df&_idts=1438550142&_idvc=1&_idn=0&_refts=1438550142&_viewts=1438550142&_ref=http%3A%2F%2Fwww.googleadservices.com%2Fpagead%2Faclk%3Fsa%3DL%26ai%3DCii7VcIi-VeDRGIaRjAaPqK-QBZr23PYFwq3-29QBtcrnsxUIABABIL_gsAMoA2C5A6ABrqKPywPIAQGpAtRzbKu7bLI-qgQiT9Dly94npwbCg5D7SkUsuqSK-SLDbgVFTQ55B4AxabBQloAFkE6IBgGAB7rd8DSQBwOoB6LCG6gHpr4b2AcB%26ohost%3Dwww.google.com%26cid%3D5GjGWDPimmkdeh1FrgcuUImc3pojqtIChK-EDtUjr4JZmw%26sig%3DAOD64_3GESvwtzmjkBhlZRJcFCvb4QN70g%26clui%3D0%26rct%3Dj%26q%3D%26ved%3D0CBoQ0QxqFQoTCIz3keuoi8cCFcKyFAodIQ0Ktw%26adurl%3Dhttp%3A%2F%2Fwww.kavosdraugas.lt%2Fkavos-aparatai&pdf=1&qt=0&realp=0&wma=1&dir=0&fla=1&java=1&gears=0&ag=0&cookie=1&res=1366x768>_ms=3068 HTTP/1.1" 200 54 "http://www.kavosdraugas.lt/kavos-aparatai?gclid=CL6m2O6oi8cCFY_JtAod_tYIPQ" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.130 Safari/537.36 OPR/30.0.1835.125" + +5348878:87.247.69.4 - - [02/Aug/2015:21:15:37 +0000] "GET /piwik.php?action_name=Kavos%20aparatai%20nuo%2036%20%E2%82%AC&idsite=1&rec=1&r=199956&h=0&m=18&s=24&url=http%3A%2F%2Fwww.kavosdraugas.lt%2Fkavos-aparatai%3Fgclid%3DCL6m2O6oi8cCFY_JtAod_tYIPQ%26price%3D216%2C456&urlref=http%3A%2F%2Fwww.kavosdraugas.lt%2Fkavos-aparatai%3Fgclid%3DCL6m2O6oi8cCFY_JtAod_tYIPQ&_id=54f474c0c50144df&_idts=1438550142&_idvc=1&_idn=0&_refts=1438550142&_viewts=1438550142&_ref=http%3A%2F%2Fwww.googleadservices.com%2Fpagead%2Faclk%3Fsa%3DL%26ai%3DCii7VcIi-VeDRGIaRjAaPqK-QBZr23PYFwq3-29QBtcrnsxUIABABIL_gsAMoA2C5A6ABrqKPywPIAQGpAtRzbKu7bLI-qgQiT9Dly94npwbCg5D7SkUsuqSK-SLDbgVFTQ55B4AxabBQloAFkE6IBgGAB7rd8DSQBwOoB6LCG6gHpr4b2AcB%26ohost%3Dwww.google.com%26cid%3D5GjGWDPimmkdeh1FrgcuUImc3pojqtIChK-EDtUjr4JZmw%26sig%3DAOD64_3GESvwtzmjkBhlZRJcFCvb4QN70g%26clui%3D0%26rct%3Dj%26q%3D%26ved%3D0CBoQ0QxqFQoTCIz3keuoi8cCFcKyFAodIQ0Ktw%26adurl%3Dhttp%3A%2F%2Fwww.kavosdraugas.lt%2Fkavos-aparatai&pdf=1&qt=0&realp=0&wma=1&dir=0&fla=1&java=1&gears=0&ag=0&cookie=1&res=1366x768>_ms=7972 HTTP/1.1" 200 54 "http://www.kavosdraugas.lt/kavos-aparatai?gclid=CL6m2O6oi8cCFY_JtAod_tYIPQ&price=216,456" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.130 Safari/537.36 OPR/30.0.1835.125" + +5348954:180.153.201.215 - - [02/Aug/2015:21:15:38 +0000] "GET /piwik.php?action_name=Kavos aparatai nuo 36 ??\xAC&idsite=1&rec=1&r=199956&h=0&m=18&s=24&url=http://www.kavosdraugas.lt/kavos-aparatai?gclid=CL6m2O6oi8cCFY_JtAod_tYIPQ&price=216,456&urlref=http://www.kavosdraugas.lt/kavos-aparatai?gclid=CL6m2O6oi8cCFY_JtAod_tYIPQ&_id=54f474c0c50144df&_idts=1438550142&_idvc=1&_idn=0&_refts=1438550142&_viewts=1438550142&_ref=http://www.googleadservices.com/pagead/aclk?sa=L&ai=Cii7VcIi-VeDRGIaRjAaPqK-QBZr23PYFwq3-29QBtcrnsxUIABABIL_gsAMoA2C5A6ABrqKPywPIAQGpAtRzbKu7bLI-qgQiT9Dly94npwbCg5D7SkUsuqSK-SLDbgVFTQ55B4AxabBQloAFkE6IBgGAB7rd8DSQBwOoB6LCG6gHpr4b2AcB&ohost=www.google.com&cid=5GjGWDPimmkdeh1FrgcuUImc3pojqtIChK-EDtUjr4JZmw&sig=AOD64_3GESvwtzmjkBhlZRJcFCvb4QN70g&clui=0&rct=j&q=&ved=0CBoQ0QxqFQoTCIz3keuoi8cCFcKyFAodIQ0Ktw&adurl=http://www.kavosdraugas.lt/kavos-aparatai&pdf=1&qt=0&realp=0&wma=1&dir=0&fla=1&java=1&gears=0&ag=0&cookie=1&res=1366x768>_ms=7972 HTTP/1.1" 200 54 "http://www.kavosdraugas.lt/kavos-aparatai?gclid=CL6m2O6oi8cCFY_JtAod_tYIPQ&price=216,456" "Mozilla/5.0 (Linux; U; Android 4.4.2; zh-cn; GT-I9500 Build/KOT49H) AppleWebKit/537.36 (KHTML, like Gecko)Version/4.0 MQQBrowser/5.0 QQ-Manager Mobile Safari/537.36" + +5349667:87.247.69.4 - - [02/Aug/2015:21:15:46 +0000] "GET /piwik.php?action_name=Kavos%20aparatai%20nuo%2036%20%E2%82%AC&idsite=1&rec=1&r=298865&h=0&m=18&s=33&url=http%3A%2F%2Fwww.kavosdraugas.lt%2Fkavos-aparatai%3Fgclid%3DCL6m2O6oi8cCFY_JtAod_tYIPQ%26price%3D216%2C456%2615%3D97&urlref=http%3A%2F%2Fwww.kavosdraugas.lt%2Fkavos-aparatai%3Fgclid%3DCL6m2O6oi8cCFY_JtAod_tYIPQ%26price%3D216%2C456&_id=54f474c0c50144df&_idts=1438550142&_idvc=1&_idn=0&_refts=1438550142&_viewts=1438550142&_ref=http%3A%2F%2Fwww.googleadservices.com%2Fpagead%2Faclk%3Fsa%3DL%26ai%3DCii7VcIi-VeDRGIaRjAaPqK-QBZr23PYFwq3-29QBtcrnsxUIABABIL_gsAMoA2C5A6ABrqKPywPIAQGpAtRzbKu7bLI-qgQiT9Dly94npwbCg5D7SkUsuqSK-SLDbgVFTQ55B4AxabBQloAFkE6IBgGAB7rd8DSQBwOoB6LCG6gHpr4b2AcB%26ohost%3Dwww.google.com%26cid%3D5GjGWDPimmkdeh1FrgcuUImc3pojqtIChK-EDtUjr4JZmw%26sig%3DAOD64_3GESvwtzmjkBhlZRJcFCvb4QN70g%26clui%3D0%26rct%3Dj%26q%3D%26ved%3D0CBoQ0QxqFQoTCIz3keuoi8cCFcKyFAodIQ0Ktw%26adurl%3Dhttp%3A%2F%2Fwww.kavosdraugas.lt%2Fkavos-aparatai&pdf=1&qt=0&realp=0&wma=1&dir=0&fla=1&java=1&gears=0&ag=0&cookie=1&res=1366x768>_ms=2467 HTTP/1.1" 200 54 "http://www.kavosdraugas.lt/kavos-aparatai?gclid=CL6m2O6oi8cCFY_JtAod_tYIPQ&price=216,456&15=97" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.130 Safari/537.36 OPR/30.0.1835.125" + +5350042:112.64.235.91 - - [02/Aug/2015:21:15:50 +0000] "GET /piwik.php?action_name=Kavos aparatai nuo 36 ??\xAC&idsite=1&rec=1&r=298865&h=0&m=18&s=33&url=http://www.kavosdraugas.lt/kavos-aparatai?gclid=CL6m2O6oi8cCFY_JtAod_tYIPQ&price=216,456&15=97&urlref=http://www.kavosdraugas.lt/kavos-aparatai?gclid=CL6m2O6oi8cCFY_JtAod_tYIPQ&price=216,456&_id=54f474c0c50144df&_idts=1438550142&_idvc=1&_idn=0&_refts=1438550142&_viewts=1438550142&_ref=http://www.googleadservices.com/pagead/aclk?sa=L&ai=Cii7VcIi-VeDRGIaRjAaPqK-QBZr23PYFwq3-29QBtcrnsxUIABABIL_gsAMoA2C5A6ABrqKPywPIAQGpAtRzbKu7bLI-qgQiT9Dly94npwbCg5D7SkUsuqSK-SLDbgVFTQ55B4AxabBQloAFkE6IBgGAB7rd8DSQBwOoB6LCG6gHpr4b2AcB&ohost=www.google.com&cid=5GjGWDPimmkdeh1FrgcuUImc3pojqtIChK-EDtUjr4JZmw&sig=AOD64_3GESvwtzmjkBhlZRJcFCvb4QN70g&clui=0&rct=j&q=&ved=0CBoQ0QxqFQoTCIz3keuoi8cCFcKyFAodIQ0Ktw&adurl=http://www.kavosdraugas.lt/kavos-aparatai&pdf=1&qt=0&realp=0&wma=1&dir=0&fla=1&java=1&gears=0&ag=0&cookie=1&res=1366x768>_ms=2467 HTTP/1.1" 200 54 "-" "Mozilla/5.0 (Linux; U; Android 4.4.2; zh-cn; GT-I9500 Build/KOT49H) AppleWebKit/537.36 (KHTML, like Gecko)Version/4.0 MQQBrowser/5.0 QQ-Manager Mobile Safari/537.36" + +5354857:87.247.69.4 - - [02/Aug/2015:21:16:40 +0000] "GET /piwik.php?action_name=Kavos%20aparatas%20De%27Longhi%20%E2%80%9EECAM%2022.110%E2%80%9C%20-%20Kavos%20draugas&idsite=1&rec=1&r=863518&h=0&m=19&s=27&url=http%3A%2F%2Fwww.kavosdraugas.lt%2Fkavos-aparatas-delonghi-ecam-22-110&urlref=http%3A%2F%2Fwww.kavosdraugas.lt%2Fkavos-aparatai%3Fgclid%3DCL6m2O6oi8cCFY_JtAod_tYIPQ%26price%3D216%2C456%2615%3D97&_id=54f474c0c50144df&_idts=1438550142&_idvc=1&_idn=0&_refts=1438550142&_viewts=1438550142&_ref=http%3A%2F%2Fwww.googleadservices.com%2Fpagead%2Faclk%3Fsa%3DL%26ai%3DCii7VcIi-VeDRGIaRjAaPqK-QBZr23PYFwq3-29QBtcrnsxUIABABIL_gsAMoA2C5A6ABrqKPywPIAQGpAtRzbKu7bLI-qgQiT9Dly94npwbCg5D7SkUsuqSK-SLDbgVFTQ55B4AxabBQloAFkE6IBgGAB7rd8DSQBwOoB6LCG6gHpr4b2AcB%26ohost%3Dwww.google.com%26cid%3D5GjGWDPimmkdeh1FrgcuUImc3pojqtIChK-EDtUjr4JZmw%26sig%3DAOD64_3GESvwtzmjkBhlZRJcFCvb4QN70g%26clui%3D0%26rct%3Dj%26q%3D%26ved%3D0CBoQ0QxqFQoTCIz3keuoi8cCFcKyFAodIQ0Ktw%26adurl%3Dhttp%3A%2F%2Fwww.kavosdraugas.lt%2Fkavos-aparatai&pdf=1&qt=0&realp=0&wma=1&dir=0&fla=1&java=1&gears=0&ag=0&cookie=1&res=1366x768>_ms=1226 HTTP/1.1" 200 54 "http://www.kavosdraugas.lt/kavos-aparatas-delonghi-ecam-22-110" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.130 Safari/537.36 OPR/30.0.1835.125" + +5354859:87.247.69.4 - - [02/Aug/2015:21:16:40 +0000] "GET /piwik.php?action_name=Kavos%20aparatas%20De%27Longhi%20%E2%80%9EEC%20850%20M%E2%80%9C%20-%20Kavos%20draugas&idsite=1&rec=1&r=751500&h=0&m=19&s=27&url=http%3A%2F%2Fwww.kavosdraugas.lt%2Fkavos-aparatas-delonghi-ec-850m&urlref=http%3A%2F%2Fwww.kavosdraugas.lt%2Fkavos-aparatai%3Fgclid%3DCL6m2O6oi8cCFY_JtAod_tYIPQ%26price%3D216%2C456%2615%3D97&_id=54f474c0c50144df&_idts=1438550142&_idvc=1&_idn=0&_refts=1438550142&_viewts=1438550142&_ref=http%3A%2F%2Fwww.googleadservices.com%2Fpagead%2Faclk%3Fsa%3DL%26ai%3DCii7VcIi-VeDRGIaRjAaPqK-QBZr23PYFwq3-29QBtcrnsxUIABABIL_gsAMoA2C5A6ABrqKPywPIAQGpAtRzbKu7bLI-qgQiT9Dly94npwbCg5D7SkUsuqSK-SLDbgVFTQ55B4AxabBQloAFkE6IBgGAB7rd8DSQBwOoB6LCG6gHpr4b2AcB%26ohost%3Dwww.google.com%26cid%3D5GjGWDPimmkdeh1FrgcuUImc3pojqtIChK-EDtUjr4JZmw%26sig%3DAOD64_3GESvwtzmjkBhlZRJcFCvb4QN70g%26clui%3D0%26rct%3Dj%26q%3D%26ved%3D0CBoQ0QxqFQoTCIz3keuoi8cCFcKyFAodIQ0Ktw%26adurl%3Dhttp%3A%2F%2Fwww.kavosdraugas.lt%2Fkavos-aparatai&pdf=1&qt=0&realp=0&wma=1&dir=0&fla=1&java=1&gears=0&ag=0&cookie=1&res=1366x768>_ms=1140 HTTP/1.1" 200 54 "http://www.kavosdraugas.lt/kavos-aparatas-delonghi-ec-850m" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.130 Safari/537.36 OPR/30.0.1835.125" + +5354946:112.65.193.14 - - [02/Aug/2015:21:16:41 +0000] "GET /piwik.php?action_name=Kavos aparatas De'Longhi ???ECAM 22.110??\x9C - Kavos draugas&idsite=1&rec=1&r=863518&h=0&m=19&s=27&url=http://www.kavosdraugas.lt/kavos-aparatas-delonghi-ecam-22-110&urlref=http://www.kavosdraugas.lt/kavos-aparatai?gclid=CL6m2O6oi8cCFY_JtAod_tYIPQ&price=216,456&15=97&_id=54f474c0c50144df&_idts=1438550142&_idvc=1&_idn=0&_refts=1438550142&_viewts=1438550142&_ref=http://www.googleadservices.com/pagead/aclk?sa=L&ai=Cii7VcIi-VeDRGIaRjAaPqK-QBZr23PYFwq3-29QBtcrnsxUIABABIL_gsAMoA2C5A6ABrqKPywPIAQGpAtRzbKu7bLI-qgQiT9Dly94npwbCg5D7SkUsuqSK-SLDbgVFTQ55B4AxabBQloAFkE6IBgGAB7rd8DSQBwOoB6LCG6gHpr4b2AcB&ohost=www.google.com&cid=5GjGWDPimmkdeh1FrgcuUImc3pojqtIChK-EDtUjr4JZmw&sig=AOD64_3GESvwtzmjkBhlZRJcFCvb4QN70g&clui=0&rct=j&q=&ved=0CBoQ0QxqFQoTCIz3keuoi8cCFcKyFAodIQ0Ktw&adurl=http://www.kavosdraugas.lt/kavos-aparatai&pdf=1&qt=0&realp=0&wma=1&dir=0&fla=1&java=1&gears=0&ag=0&cookie=1&res=1366x768>_ms=1226 HTTP/1.1" 200 54 "http://www.kavosdraugas.lt/kavos-aparatas-delonghi-ecam-22-110" "Mozilla/5.0 (Linux; U; Android 4.4.2; zh-cn; GT-I9500 Build/KOT49H) AppleWebKit/537.36 (KHTML, like Gecko)Version/4.0 MQQBrowser/5.0 QQ-Manager Mobile Safari/537.36" + +5354983:180.153.163.210 - - [02/Aug/2015:21:16:41 +0000] "GET /piwik.php?action_name=Kavos aparatas De'Longhi ???EC 850 M??\x9C - Kavos draugas&idsite=1&rec=1&r=751500&h=0&m=19&s=27&url=http://www.kavosdraugas.lt/kavos-aparatas-delonghi-ec-850m&urlref=http://www.kavosdraugas.lt/kavos-aparatai?gclid=CL6m2O6oi8cCFY_JtAod_tYIPQ&price=216,456&15=97&_id=54f474c0c50144df&_idts=1438550142&_idvc=1&_idn=0&_refts=1438550142&_viewts=1438550142&_ref=http://www.googleadservices.com/pagead/aclk?sa=L&ai=Cii7VcIi-VeDRGIaRjAaPqK-QBZr23PYFwq3-29QBtcrnsxUIABABIL_gsAMoA2C5A6ABrqKPywPIAQGpAtRzbKu7bLI-qgQiT9Dly94npwbCg5D7SkUsuqSK-SLDbgVFTQ55B4AxabBQloAFkE6IBgGAB7rd8DSQBwOoB6LCG6gHpr4b2AcB&ohost=www.google.com&cid=5GjGWDPimmkdeh1FrgcuUImc3pojqtIChK-EDtUjr4JZmw&sig=AOD64_3GESvwtzmjkBhlZRJcFCvb4QN70g&clui=0&rct=j&q=&ved=0CBoQ0QxqFQoTCIz3keuoi8cCFcKyFAodIQ0Ktw&adurl=http://www.kavosdraugas.lt/kavos-aparatai&pdf=1&qt=0&realp=0&wma=1&dir=0&fla=1&java=1&gears=0&ag=0&cookie=1&res=1366x768>_ms=1140 HTTP/1.1" 200 54 "http://www.kavosdraugas.lt/kavos-aparatas-delonghi-ec-850m" "Mozilla/5.0 (Linux; U; Android 4.4.2; zh-cn; GT-I9500 Build/KOT49H) AppleWebKit/537.36 (KHTML, like Gecko)Version/4.0 MQQBrowser/5.0 QQ-Manager Mobile Safari/537.36" + + diff --git a/cloud-archive.sh b/cloud-archive.sh new file mode 100644 index 0000000000..4f7cc696e7 --- /dev/null +++ b/cloud-archive.sh @@ -0,0 +1,20 @@ +source cloud-config.sh + +for I in $(seq 1 $END); +do + echo "Running enterprise:dashboard..." + echo "" + + ./console enterprise:dashboard --piwik-domain=http://test$I.piwik.pro + + echo "Running archiving for: test$I.piwik.pro... with --force-idsites" + echo "" + echo "" + ./console enterprise:archive --piwik-domain=http://test$I.piwik.pro --force-idsites=1 + echo "" + echo "" + +done + + + diff --git a/cloud-config.sh b/cloud-config.sh new file mode 100644 index 0000000000..56deaef82c --- /dev/null +++ b/cloud-config.sh @@ -0,0 +1,2 @@ +#!/bin/bash +END=5 diff --git a/cloud-generate.sh b/cloud-generate.sh new file mode 100644 index 0000000000..df160bdc27 --- /dev/null +++ b/cloud-generate.sh @@ -0,0 +1,17 @@ + +source cloud-config.sh + +for I in $(seq 1 $END); do + for ((n=1;n<$(( ( RANDOM % 5 ) + 1 ));n++)); + do + echo "Generating traffic for Piwik instance $I and website id = $n..." + echo "" + CMD="./console visitorgenerator:generate-visits --idsite=$n --limit-fake-visits=$(( ( RANDOM % 50 ) + 1 )) --no-logs --piwik-domain=http://test$I.piwik.pro" + echo $CMD + $CMD + echo "" + + done + +done + diff --git a/cloud-setup.sh b/cloud-setup.sh new file mode 100644 index 0000000000..beebfd35b6 --- /dev/null +++ b/cloud-setup.sh @@ -0,0 +1,43 @@ +#!/bin/bash +set -o xtrace +source cloud-config.sh + +rm config/common.config.ini.php +rm config/test*.piwik.pro.config.ini.php + +cat plugins/EnterpriseAdmin/ConfigTemplate/common.config.ini.php | sed "s/force_ssl = 1/force_ssl = 0/" > config/common.config.ini.php +cat plugins/{LoginAdmin,WhitelistAdmin}/ConfigTemplate/common.config.ini.php >> config/common.config.ini.php + + +for I in $(seq 1 $END); do + mysql -uroot -e "DROP DATABASE dbname$I;" + + sudo chmod 777 -R tmp/test$I.piwik.pro/ + + # test no db prefix + ./console enterprise:install \ + --db-host="localhost" --db-login="root" --db-password="" --db-name="dbname$I" --db-adapter="PDO\\MYSQL" \ + --website-name="Main Shop $I" --website-url="http://shop$I.enterprise.com" --website-timezone="Africa/Accra" --website-ecommerce \ + --superuser-login="admin" --superuser-password="secure" --superuser-email="admin@enterpriseanalytics.com" \ + --piwik-domain=http://test$I.piwik.pro \ + --skip-writable-check --skip-geoip-check --skip-https-check --skip-http-check #--db-dropexistingtables + + echo "Generating websites, users and goals for cloud instance = $I..." + ./console enterprise:plugin activate VisitorGenerator --piwik-domain=http://test$I.piwik.pro + + echo "Generating websites for cloud instance ID = $I..." + ./console visitorgenerator:generate-websites --piwik-domain=test$I.piwik.pro --limit=$(( ( RANDOM % 5 ) + 10 )) + + echo "Generating users for cloud instance ID = $I..." + ./console visitorgenerator:generate-users --piwik-domain=test$I.piwik.pro --limit=5 + + echo "Generating goals for cloud instance ID = $I..." + for ((n=1;n<5;n++)); do (./console visitorgenerator:generate-goals --idsite=$n --piwik-domain=test$I.piwik.pro); done + +done + +chmod 755 config/*.config.ini.php + +echo "Setting master instance to test1.piwik.pro..." +./console enterprise:setmasterinstance --piwik-domain=test1.piwik.pro -vvv + diff --git a/config/common.config.ini.php.not b/config/common.config.ini.php.not new file mode 100644 index 0000000000..e8b2775483 --- /dev/null +++ b/config/common.config.ini.php.not @@ -0,0 +1,62 @@ +; <?php exit; ?> DO NOT REMOVE THIS LINE +; Configuration settings which are applied to all Piwik instances. +; This file here is not read by Piwik: the EnterpriseAdmin installer will ask to copy it to config/common.config.ini.php +; +; The config files are read in this order: +; 1) global.ini.php +; 2) common.config.ini.php +; 3) $instanceHostname.config.ini.php +; +; Settings found here will apply to all instances, unless they are overriden in the customer config file + +[General] +; Disable Super User features or features requiring internet access +enable_marketplace = 0 +enable_plugins_admin = 0 +enable_geolocation_admin = 0 +enable_delete_old_data_settings_admin = 0 +enable_general_settings_admin = 0 +enable_auto_update = 0 +enable_update_communication = 0 + +; Disable other features that are enabled in global.ini.php +enable_load_data_infile = 0 + +; Bypass the Custom Logo "is writable" check as we are writing the logo file to another backend +enable_custom_logo_check = 0 + +; Default timeout for archives (overrides the default 10 seconds) +time_before_today_archive_considered_outdated = 60 + +; Disable browser trigger archiving for all users by default +enable_browser_archiving_triggering = 0 + +; Uncomment to make sure that API requests with a &segment= parameter will not trigger archiving +; browser_archiving_disabled_enforce = 1 + +; Uncomment to make sure that all new Custom Segments created in the future +; will be set to "Pre-processed (faster, requires cron core:archive command)" +; enable_create_realtime_segments = 0 + +; Email addresses +noreply_email_address = "robot@piwik.pro" +login_password_recovery_email_address = "robot@piwik.pro" +login_password_recovery_email_name = "Password recovery (Piwik PRO)" +login_password_recovery_replyto_email_address = "noreply@{DOMAIN}" +login_password_recovery_replyto_email_name = "Password recovery (Piwik PRO)" + +feedback_email_address = contact@piwik.pro +scheduled_reports_replyto_is_user_email_and_alias = 1 + +; Enable Database storage of sessions +session_save_handler = dbtable + +; Force SSL everywhere +force_ssl = 1 + +; Disable OPTIMIZE queries +enable_sql_optimize_queries = 0 + +process_new_segments_from = "segment_creation_time" + + diff --git a/info.php b/info.php new file mode 100644 index 0000000000..c4837a32b6 --- /dev/null +++ b/info.php @@ -0,0 +1 @@ +<?php phpinfo(); diff --git a/misc/user/example.com/logo-header.png b/misc/user/example.com/logo-header.png Binary files differnew file mode 100644 index 0000000000..0c3bd89f8a --- /dev/null +++ b/misc/user/example.com/logo-header.png diff --git a/misc/user/example.com/logo.png b/misc/user/example.com/logo.png Binary files differnew file mode 100644 index 0000000000..58a38ad82d --- /dev/null +++ b/misc/user/example.com/logo.png diff --git a/plugins/AdvancedCampaignReporting b/plugins/AdvancedCampaignReporting new file mode 160000 +Subproject 13a6ddea85f7398bcfdf7ded16bd6187f72f651 diff --git a/plugins/ApiGetWithSitesInfo b/plugins/ApiGetWithSitesInfo new file mode 160000 +Subproject c77f273ceedf8e93f5cd537c880faf1a6f3b37a diff --git a/plugins/AutoLogImporter/.gitignore b/plugins/AutoLogImporter/.gitignore new file mode 100644 index 0000000000..c8c9480010 --- /dev/null +++ b/plugins/AutoLogImporter/.gitignore @@ -0,0 +1 @@ +tests/System/processed/*xml
\ No newline at end of file diff --git a/plugins/AutoLogImporter/.travis.yml b/plugins/AutoLogImporter/.travis.yml new file mode 100644 index 0000000000..1fc816a0d0 --- /dev/null +++ b/plugins/AutoLogImporter/.travis.yml @@ -0,0 +1,132 @@ +# do not edit this file manually, instead run the generate:travis-yml console command + +language: php + +php: + - 5.6 + - 5.3.3 +# - hhvm + +services: + - redis-server + +# Separate different test suites +env: + global: + - PLUGIN_NAME=AutoLogImporter + - PIWIK_ROOT_DIR=$TRAVIS_BUILD_DIR/piwik + - PIWIK_LATEST_STABLE_TEST_TARGET=2.15.0-b4 + - secure: "uSQcAHxwIABymCg+L6ZzEasENSAIuRGXlpnhr7Rt+GrLLtSYnviBSsgGYuafLMs6AcnA7OQEv1e97N20B2JKPfG9TArpwFxGG6tqipZlV58bdd/HZ2MmJIzSf8bnkvvCOxNdzKR4myL07v9nCE9qvJef5lx9JHX+MH6pZzPX9QZmei4DoR+UweyW86AGpNVwNhRcCXzw6GQ/32n8kpj3fUEr7aZ6jRLtxxQMxTnSJq1Yr8hI7TgprzHq5Da36zVxzjxNPJdgUKreROqjuqA4zgB4QgYSNXNUbe4cto8HPFteRxyw5OqWHTAWV/wrK7yo1Vq4kU8jrUKjbzVPudR21NY4teicra2qANtyPVrqTXRkeUwcBEEeWYByEq84gz/bsk9luiwAKZHH4/26zT8RGoLVKRWI16cuBN8WQxvrvZBgoiJ9n2gxerfeKqZizzzY+67RrzYwjDTtYyAwXkp8H4TELKZO/oEimrEy2oEVu4R7UwijwugHgg8d65S1B/UaWtdUHFIhcMP/FjSwdNRikk24a/koTnqVKh0LWBHxPnmV9lLW6CVgSVN2xiVPQe4RWggnNcVlhkLTNkJzpN5ISYALN20mmaKq/PpdOdPbhsRWXObmCFhv89Ot54HaHixUB99B60I0zvwxX9j2C1nHUxWXzwxkePBLAZxNRBeJR2U=" + - secure: "VYNfBVkaKl2AbmmW6jAp6UVFQA1z0acKyPrP7IQN2eFGXVn/mCgNBKwpVcNxVbrGre02MQhRI/8TCsPdZDCEa74mobozZ0ef87qF/eVgRQQjgxcVZ69QBL32K2VBg/RCwuC/WlnaFBTOn/ElkpJNYRJNUjWTLtw3lgIdi6zN4ZfBfvELuczi//n++KYH75MYnF//yX1+BD/4BQyC+yYLvc+NcdErTxQUS81C3nlp/VqKpqA0OQ+aNqtKb+MSNrHDH9K286oha8R2fJBR3YZvjCzrK8I2/+iF26lnnLaOfalMJjrdJXVV1w/OGps1hLYev5FMrNmBjgbKZP5E7SOJlfLbvbHtDI140ZXZbBLD4anMO0Uvc0cCfnk8QAu9kUvVq+nBszmBmArEvU4LTuvPqxJVU92kw8WYOA+AoSrT34mTq58rpo1sECQxtx8ivCKLRA6SMYlE7+m0quwhn/CnAeBfsOViEYhK6nkp/N/tyk39rgQ+F+JVm/AtVm/AVuKWlzxbYkI3m7eQ6oB5/BD4l1rI4ZWkYCyHAQkUAHCGsX8c2mQGOWJIOdz7Tm+Is+vFaGA6mS0JwxqXN8USs/3iUHh0VYAO1jp16QZ2/1IeO9hzx9Bj1oH/q7/JXpVeyuWMevqyQ9bWbYQvM1eOLXPIMyDgFOj9yL43ebe9leU3lQ0=" + matrix: + - TEST_SUITE=PluginTests MYSQL_ADAPTER=PDO_MYSQL TEST_AGAINST_PIWIK_BRANCH=$PIWIK_LATEST_STABLE_TEST_TARGET + - TEST_SUITE=PluginTests MYSQL_ADAPTER=PDO_MYSQL TEST_AGAINST_CORE=minimum_required_piwik + - TEST_SUITE=UITests MYSQL_ADAPTER=PDO_MYSQL TEST_AGAINST_PIWIK_BRANCH=$PIWIK_LATEST_STABLE_TEST_TARGET + +matrix: + exclude: + # execute latest stable tests only w/ PHP 5.5 + - php: 5.3.3 + env: TEST_SUITE=PluginTests MYSQL_ADAPTER=PDO_MYSQL TEST_AGAINST_CORE=minimum_required_piwik + # execute UI tests only w/ PHP 5.6 + - php: 5.3.3 + env: TEST_SUITE=UITests MYSQL_ADAPTER=PDO_MYSQL TEST_AGAINST_PIWIK_BRANCH=$PIWIK_LATEST_STABLE_TEST_TARGET + +sudo: required + +script: $PIWIK_ROOT_DIR/tests/travis/travis.sh + +before_install: + # do not use the Zend allocator on PHP 5.3 since it will randomly segfault after program execution + - '[[ "$TRAVIS_PHP_VERSION" == 5.3* ]] && export USE_ZEND_ALLOC=0 || true' + +install: + # move all contents of current repo (which contains the plugin) to a new directory + - mkdir $PLUGIN_NAME + - cp -R !($PLUGIN_NAME) $PLUGIN_NAME + - cp -R .git/ $PLUGIN_NAME/ + - cp .travis.yml $PLUGIN_NAME + # checkout piwik in the current directory + - git clone -q https://github.com/piwik/piwik.git piwik + - cd piwik + - git fetch -q --all + - git submodule update + + # make sure travis-scripts repo is latest for initial travis setup + - '[ -d ./tests/travis/.git ] || sh -c "rm -rf ./tests/travis && git clone https://github.com/piwik/travis-scripts.git ./tests/travis"' + - cd ./tests/travis ; git checkout master ; cd ../.. + + - export GENERATE_TRAVIS_YML_COMMAND="php ./tests/travis/generator/main.php generate:travis-yml --plugin=\"AutoLogImporter\" --verbose" + - '[[ "$TRAVIS_JOB_NUMBER" != *.1 || "$TRAVIS_PULL_REQUEST" != "false" ]] || ./tests/travis/autoupdate_travis_yml.sh' + + - ./tests/travis/checkout_test_against_branch.sh + + - '[ ! -f ./tests/travis/install_mysql_5.6.sh ] || ./tests/travis/install_mysql_5.6.sh' + + # Make sure we use Python 2.6 + - '[ ! -f ./tests/travis/install_python_2.6.sh ] || ./tests/travis/install_python_2.6.sh' + + - ./tests/travis/configure_git.sh + + # travis now complains about this failing 9 times out of 10, so removing it + #- travis_retry composer self-update + + - '[ "$SKIP_COMPOSER_INSTALL" == "1" ] || travis_retry composer install' + + # move plugin contents to folder in the plugins subdirectory + - rm -rf plugins/$PLUGIN_NAME + - mv ../$PLUGIN_NAME plugins + + # clone dependent repos + - ./tests/travis/checkout_dependent_plugins.sh + +before_script: + - phpenv config-rm xdebug.ini; + + # add always_populate_raw_post_data=-1 to php.ini + - echo "always_populate_raw_post_data=-1" >> ~/.phpenv/versions/$(phpenv version-name)/etc/php.ini + + # disable opcache to avoid random failures on travis + - echo "opcache.enable=0" >> ~/.phpenv/versions/$(phpenv version-name)/etc/php.ini + + # print out mysql information + - mysql --version + - mysql -e "SELECT VERSION();" + + # configure mysql + - mysql -e "SET GLOBAL sql_mode = 'NO_ENGINE_SUBSTITUTION,STRICT_TRANS_TABLES'" # Travis default + # try to avoid mysql has gone away errors + - mysql -e "SET GLOBAL wait_timeout = 36000;" + - mysql -e "SET GLOBAL max_allowed_packet = 134209536;" + - mysql -e "SHOW VARIABLES LIKE 'max_allowed_packet';" + - mysql -e "SHOW VARIABLES LIKE 'wait_timeout';" + + - mysql -e "SELECT @@sql_mode;" + # - mysql -e "SHOW GLOBAL VARIABLES;" + + # print out more debugging info + - uname -a + - date + - php -r "var_dump(gd_info());" + - mysql -e 'create database piwik_tests;' + + - ./tests/travis/prepare.sh + - ./tests/travis/setup_webserver.sh + + - cd tests/PHPUnit + +after_script: + # change directory back to root travis dir + - cd $PIWIK_ROOT_DIR + + # output contents of files w/ debugging info to screen + - cat /var/log/nginx/error.log + - cat $PIWIK_ROOT_DIR/tmp/php-fpm.log + - cat $PIWIK_ROOT_DIR/tmp/logs/piwik.log + - cat $PIWIK_ROOT_DIR/config/config.ini.php + + # upload test artifacts (for debugging travis failures) + - ./tests/travis/upload_artifacts.sh + +after_success: + - cd $PIWIK_ROOT_DIR diff --git a/plugins/AutoLogImporter/API.php b/plugins/AutoLogImporter/API.php new file mode 100644 index 0000000000..fa3e09cc55 --- /dev/null +++ b/plugins/AutoLogImporter/API.php @@ -0,0 +1,93 @@ +<?php +/** + * Copyright (C) Piwik PRO - All rights reserved. + * + * Using this code requires that you first get a license from Piwik PRO. + * Unauthorized copying of this file, via any medium is strictly prohibited. + * + * @link http://piwik.pro + */ + +namespace Piwik\Plugins\AutoLogImporter; + +use Piwik\Container\StaticContainer; +use Piwik\DataTable; +use Piwik\DataTable\Row; +use Piwik\Piwik; + +/** + * API for plugin AutoLogImporter + * + * @method static \Piwik\Plugins\AutoLogImporter\API getInstance() + */ +class API extends \Piwik\Plugin\API +{ + /** + * Get all imported files no matter if the import was successful or not. + * + * Latest imported files are listed first. + * @return array + */ + public function getAllImportedFiles() + { + Piwik::checkUserHasSuperUserAccess(); + + $dao = new Dao(); + $files = $dao->getAllImportedFiles(); + + usort($files, function ($a, $b){ + return $a['idlogimport'] > $b['idlogimport'] ? -1 : 1; + }); + + return $files; + } + + /** + * Get all files that are currently in process importing logs. + * + * Latest importing files are listed first. + * @return array + */ + public function getAllImportingFiles() + { + Piwik::checkUserHasSuperUserAccess(); + + $dao = new Dao(); + $files = $dao->getAllImportingFiles(); + + usort($files, function ($a, $b){ + return $a['idlogimport'] > $b['idlogimport'] ? -1 : 1; + }); + + return $files; + } + + /** + * Get a list of files that will be imported as soon as the task runs the next time. + * + * @return string[] + */ + public function getFilesThatCanBeImported() + { + Piwik::checkUserHasSuperUserAccess(); + + $importer = StaticContainer::get('Piwik\Plugins\AutoLogImporter\LogImporter'); + return $importer->getFilesThatCanBeImported(); + } + + + /** + * Get a list of files that have a hash file but the verify hash doesn't match. This usually indicates a not + * sucessfully copied log file. + * + * @return array + */ + public function getFilesHavingInvalidHash() + { + Piwik::checkUserHasSuperUserAccess(); + + $logFileList = StaticContainer::get('Piwik\Plugins\AutoLogImporter\LogImporter\LogFileList'); + return $logFileList->findFilesHavingWrongHashFile(); + } + +} diff --git a/plugins/AutoLogImporter/AutoLogImporter.php b/plugins/AutoLogImporter/AutoLogImporter.php new file mode 100644 index 0000000000..4c7261b95b --- /dev/null +++ b/plugins/AutoLogImporter/AutoLogImporter.php @@ -0,0 +1,38 @@ +<?php +/** + * Copyright (C) Piwik PRO - All rights reserved. + * + * Using this code requires that you first get a license from Piwik PRO. + * Unauthorized copying of this file, via any medium is strictly prohibited. + * + * @link http://piwik.pro + */ + +namespace Piwik\Plugins\AutoLogImporter; + +class AutoLogImporter extends \Piwik\Plugin +{ + public function getListHooksRegistered() + { + return array( + 'AssetManager.getStylesheetFiles' => 'getStylesheetFiles', + ); + } + + public function getStylesheetFiles(&$stylesheets) + { + $stylesheets[] = "plugins/AutoLogImporter/stylesheets/status.less"; + } + + public function install() + { + $dao = new Dao(); + $dao->install(); + } + + public function uninstall() + { + $dao = new Dao(); + $dao->uninstall(); + } +} diff --git a/plugins/AutoLogImporter/Controller.php b/plugins/AutoLogImporter/Controller.php new file mode 100644 index 0000000000..1d6c5d0a95 --- /dev/null +++ b/plugins/AutoLogImporter/Controller.php @@ -0,0 +1,56 @@ +<?php +/** + * Copyright (C) Piwik PRO - All rights reserved. + * + * Using this code requires that you first get a license from Piwik PRO. + * Unauthorized copying of this file, via any medium is strictly prohibited. + * + * @link http://piwik.pro + */ + +namespace Piwik\Plugins\AutoLogImporter; +use Piwik\API\Request; +use Piwik\Common; +use Piwik\Piwik; + +/** + * A controller let's you for example create a page that can be added to a menu. For more information read our guide + * http://developer.piwik.org/guides/mvc-in-piwik or have a look at the our API references for controller and view: + * http://developer.piwik.org/api-reference/Piwik/Plugin/Controller and + * http://developer.piwik.org/api-reference/Piwik/View + */ +class Controller extends \Piwik\Plugin\ControllerAdmin +{ + public function getDefaultAction() + { + return 'status'; + } + + public function status() + { + Piwik::checkUserHasSuperUserAccess(); + + $limit = Common::getRequestVar('limit', 200, 'int'); + + $importingFiles = Request::processRequest('AutoLogImporter.getAllImportingFiles'); + $importingSoon = Request::processRequest('AutoLogImporter.getFilesThatCanBeImported'); + $filesWithInvalidHashes = Request::processRequest('AutoLogImporter.getFilesHavingInvalidHash'); + $importedFiles = Request::processRequest('AutoLogImporter.getAllImportedFiles', array( + 'filter_limit' => $limit + )); + + if (count($importedFiles) < $limit) { + $limit = count($importedFiles); + } + + $formatter = new Formatter(); + + return $this->renderTemplate('status', array( + 'importingFiles' => $formatter->addTimeAgoAsSentence($formatter->formatImportedFiles($importingFiles)), + 'importedFiles' => $formatter->formatImportedFiles($importedFiles), + 'scheduledFiles' => $importingSoon, + 'filesWithInvalidHashes' => $filesWithInvalidHashes, + 'limit' => $limit + )); + } +} diff --git a/plugins/AutoLogImporter/Dao.php b/plugins/AutoLogImporter/Dao.php new file mode 100644 index 0000000000..ec4412fc41 --- /dev/null +++ b/plugins/AutoLogImporter/Dao.php @@ -0,0 +1,106 @@ +<?php +/** + * Copyright (C) Piwik PRO - All rights reserved. + * + * Using this code requires that you first get a license from Piwik PRO. + * Unauthorized copying of this file, via any medium is strictly prohibited. + * + * @link http://piwik.pro + */ + +namespace Piwik\Plugins\AutoLogImporter; + +use Piwik\Common; +use Piwik\Date; +use Piwik\Db; +use Piwik\DbHelper; + +class Dao +{ + /** + * @var Db + */ + private $db; + + private $tableName = 'auto_log_importer'; + private $tableNamePrefixed; + + public function __construct() + { + $this->db = Db::get(); + $this->tableNamePrefixed = Common::prefixTable($this->tableName); + } + + public function getAllImportedFiles() + { + return Db::fetchAll('SELECT * FROM ' . $this->tableNamePrefixed . ' WHERE end_date is not null'); + } + + public function getAllImportingFiles() + { + return Db::fetchAll('SELECT * FROM ' . $this->tableNamePrefixed . ' WHERE end_date is null'); + } + + public function isAlreadyImported($fileHash) + { + return (bool) $this->db->fetchOne("SELECT 1 FROM " . $this->tableNamePrefixed . " + WHERE `file_hash` = ? LIMIT 1", array($fileHash)); + } + + public function startLogImport($logFile, $fileHash, $fileSize, $numLogLines, $startDateTime) + { + $this->db->insert($this->tableNamePrefixed, array( + 'file' => $logFile, + 'file_hash' => $fileHash, + 'file_size' => $fileSize, + 'num_log_lines' => $numLogLines, + 'start_date' => $startDateTime, + )); + + return (int) $this->db->lastInsertId(); + } + + public function setImportFinished($id, $command, $output, $exitCode, $duration, $endDateTime) + { + // we cannot store unlimited characters in TEXT field + if ($posSummary = strpos($output, 'Logs import summary')) { + // if it contains a summary, only store the actual summary + $output = substr($output, strpos($output, 'Logs import summary')); + } elseif (strlen($output) > 10000) { + // only store last 10k characters + $output = substr($output, strlen($output) - 10000); + } + + $this->db->update($this->tableNamePrefixed, array( + 'output' => $output, + 'command' => $command, + 'exit_code' => $exitCode, + 'end_date' => $endDateTime, + 'duration' => $duration, + ), 'idlogimport = ' . (int) $id); + } + + public function install() + { + $table = "`idlogimport` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY , + `start_date` datetime NOT NULL , + `end_date` datetime NULL , + `duration` MEDIUMINT UNSIGNED NULL , + `file` TEXT NOT NULL , + `file_size` INT UNSIGNED NOT NULL DEFAULT 0, + `file_hash` VARCHAR(32) NOT NULL , + `num_log_lines` INT UNSIGNED NOT NULL DEFAULT 0, + `command` TEXT NULL, + `output` TEXT NULL, + `exit_code` TINYINT UNSIGNED NULL, + UNIQUE KEY uniq_hash(file_hash)"; + + DbHelper::createTable($this->tableName, $table); + } + + public function uninstall() + { + Db::dropTables(array($this->tableNamePrefixed)); + } + +} diff --git a/plugins/AutoLogImporter/Formatter.php b/plugins/AutoLogImporter/Formatter.php new file mode 100644 index 0000000000..0872c7a275 --- /dev/null +++ b/plugins/AutoLogImporter/Formatter.php @@ -0,0 +1,44 @@ +<?php +/** + * Copyright (C) Piwik PRO - All rights reserved. + * + * Using this code requires that you first get a license from Piwik PRO. + * Unauthorized copying of this file, via any medium is strictly prohibited. + * + * @link http://piwik.pro + */ + +namespace Piwik\Plugins\AutoLogImporter; + +use Piwik\Container\StaticContainer; +use Piwik\Date; +use Piwik\Metrics\Formatter as MetricsFormatter; + +class Formatter +{ + public function addTimeAgoAsSentence($files) + { + $formatter = new MetricsFormatter(); + $now = StaticContainer::get('AutoLogImporter.currentTimestamp'); + + foreach ($files as &$file) { + $diff = $now - Date::factory($file['start_date'])->getTimestamp(); + $file['time_ago'] = $formatter->getPrettyTimeFromSeconds($diff, true); + } + + return $files; + } + + public function formatImportedFiles($files) + { + $formatter = new MetricsFormatter(); + + foreach ($files as &$file) { + $file['filename'] = basename($file['file']); + $file['date'] = substr($file['start_date'], 0, 10); + $file['size_human'] = $formatter->getPrettySizeFromBytes((int) $file['file_size']); + } + + return $files; + } +} diff --git a/plugins/AutoLogImporter/LogImporter.php b/plugins/AutoLogImporter/LogImporter.php new file mode 100644 index 0000000000..594eecd746 --- /dev/null +++ b/plugins/AutoLogImporter/LogImporter.php @@ -0,0 +1,90 @@ +<?php +/** + * Copyright (C) Piwik PRO - All rights reserved. + * + * Using this code requires that you first get a license from Piwik PRO. + * Unauthorized copying of this file, via any medium is strictly prohibited. + * + * @link http://piwik.pro + */ + +namespace Piwik\Plugins\AutoLogImporter; + +use Piwik\Date; +use Piwik\Plugins\AutoLogImporter\LogImporter\Import; +use Piwik\Plugins\AutoLogImporter\LogImporter\LogFileList; +use Piwik\Plugins\AutoLogImporter\LogImporter\File; + +class LogImporter +{ + /** + * @var Dao + */ + private $dao; + + /** + * @var LogFileList + */ + private $logFileList; + + /** + * @var File + */ + private $file; + + /** + * @var Import + */ + private $import; + + public function __construct(Dao $dao, LogFileList $logFileList, File $file, Import $import) + { + $this->dao = $dao; + $this->logFileList = $logFileList; + $this->file = $file; + $this->import = $import; + } + + public function getFilesThatCanBeImported() + { + $files = array(); + + foreach ($this->logFileList->findLogFiles() as $logFile) { + $hash = $this->file->getHash($logFile); + + if (!$this->dao->isAlreadyImported($hash) + && $this->file->wasFileSuccessfullyCopiedViaFileSynchronizerPlugin($logFile, $hash)) { + $files[$hash] = $logFile; + } + } + + return $files; + } + + public function importFiles() + { + $files = $this->getFilesThatCanBeImported(); + + foreach ($files as $hash => $logFile) { + $this->importFile($logFile, $hash); + } + } + + private function importFile($logFile, $hash) + { + $size = $this->file->getSize($logFile); + $lines = $this->file->getNumberOfLines($logFile); + $id = $this->dao->startLogImport($logFile, $hash, $size, $lines, $this->getCurrentDateTime()); + + $start = time(); + $result = $this->import->import($logFile); + $duration = time() - $start; + + $this->dao->setImportFinished($id, $result->getCommand(), $result->getOutput(), $result->getExitCode(), $duration, $this->getCurrentDateTime()); + } + + private function getCurrentDateTime() + { + return Date::now()->getDatetime(); + } +} diff --git a/plugins/AutoLogImporter/LogImporter/File.php b/plugins/AutoLogImporter/LogImporter/File.php new file mode 100644 index 0000000000..e43c3564a7 --- /dev/null +++ b/plugins/AutoLogImporter/LogImporter/File.php @@ -0,0 +1,75 @@ +<?php +/** + * Copyright (C) Piwik PRO - All rights reserved. + * + * Using this code requires that you first get a license from Piwik PRO. + * Unauthorized copying of this file, via any medium is strictly prohibited. + * + * @link http://piwik.pro + */ + +namespace Piwik\Plugins\AutoLogImporter\LogImporter; + +class File +{ + public function getNumberOfLines($file) + { + if (!file_exists($file) || !is_readable($file)) { + return 0; + } + + $numLines = 0; + $fileHandle = fopen($file, 'rb'); + + while (!feof($fileHandle)) { + $numLines += substr_count(fread($fileHandle, 8192), "\n"); + } + + // +1 for fist line + $numLines++; + + fclose($fileHandle); + + return $numLines; + } + + public function getSize($file) + { + return filesize($file); + } + + public function getHash($file) + { + return md5_file($file); + } + + public function getPathToVerifyHashFile($file) + { + return $file . '.hash'; + } + + public function getVerifyHash($file) + { + if (!$this->hasVerifyHash($file)) { + return; + } + + $hash = file_get_contents($this->getPathToVerifyHashFile($file)); + + if (!empty($hash)) { + return trim($hash); + } + } + + public function hasVerifyHash($file) + { + return file_exists($this->getPathToVerifyHashFile($file)); + } + + public function wasFileSuccessfullyCopiedViaFileSynchronizerPlugin($logFile, $hash) + { + $verifyHash = $this->getVerifyHash($logFile); + + return !empty($verifyHash) && $verifyHash === trim($hash); + } +} diff --git a/plugins/AutoLogImporter/LogImporter/Import.php b/plugins/AutoLogImporter/LogImporter/Import.php new file mode 100644 index 0000000000..c1a67acd0f --- /dev/null +++ b/plugins/AutoLogImporter/LogImporter/Import.php @@ -0,0 +1,61 @@ +<?php +/** + * Copyright (C) Piwik PRO - All rights reserved. + * + * Using this code requires that you first get a license from Piwik PRO. + * Unauthorized copying of this file, via any medium is strictly prohibited. + * + * @link http://piwik.pro + */ + +namespace Piwik\Plugins\AutoLogImporter\LogImporter; + +use Piwik\Container\StaticContainer; +use Piwik\SettingsPiwik; +use Psr\Log\LoggerInterface; + +class Import +{ + /** + * @var LoggerInterface + */ + private $logger; + + public function __construct(LoggerInterface $logger) + { + $this->logger = $logger; + } + + public function import($logFile) + { + $options = StaticContainer::get('AutoLogImporter.logImportOptions'); + $options[] = '--url=' . escapeshellarg(SettingsPiwik::getPiwikUrl()); + $options[] = '--replay-tracking'; + $options[] = escapeshellarg($logFile); + + if (preg_match('/_idsite_(\d+)/', $logFile, $matches)) { + $options[] = '--idsite=' . (int) $matches[1]; + } + + $command = PIWIK_INCLUDE_PATH . '/misc/log-analytics/import_logs.py'; + foreach ($options as $option) { + $command .= ' ' . $option; + } + + $command .= ' 2>&1'; + + $this->logger->debug("Executing command '$command' to import '$logFile'"); + + try { + exec($command, $output, $exitCode); + $output = implode("\n", $output); + } catch (\Exception $e) { + $output = $e->getMessage() . ' Trace: ' . $e->getTraceAsString(); + $exitCode = 255; + } + + $this->logger->debug("Finished command with exit code '$exitCode' and output '$output'"); + + return new Result($command, $output, $exitCode); + } +} diff --git a/plugins/AutoLogImporter/LogImporter/LogFileList.php b/plugins/AutoLogImporter/LogImporter/LogFileList.php new file mode 100644 index 0000000000..defb51145b --- /dev/null +++ b/plugins/AutoLogImporter/LogImporter/LogFileList.php @@ -0,0 +1,80 @@ +<?php +/** + * Copyright (C) Piwik PRO - All rights reserved. + * + * Using this code requires that you first get a license from Piwik PRO. + * Unauthorized copying of this file, via any medium is strictly prohibited. + * + * @link http://piwik.pro + */ + +namespace Piwik\Plugins\AutoLogImporter\LogImporter; + +use Piwik\Db; +use Piwik\Filesystem; +use Piwik\Plugins\AutoLogImporter\Settings; + +class LogFileList +{ + /** + * @var Settings + */ + private $settings; + + /** + * @var File + */ + private $file; + + public function __construct(Settings $settings, File $file) + { + $this->settings = $settings; + $this->file = $file; + } + + /** + * @return string[] + */ + public function findLogFiles() + { + $path = $this->settings->logFilesPath->getValue(); + $pattern = $this->settings->filePattern->getValue(); + + if (!$this->settings->enabled->getValue()) { + return array(); + } + + if (empty($path) || !file_exists($path) || !is_readable($path) || !is_dir($path)) { + // case when plugin not configured yet + return array(); + } + + if (empty($pattern)) { + $pattern = '*'; + } + + return Filesystem::globr($path, $pattern); + } + + public function findFilesHavingWrongHashFile() + { + $files = $this->findLogFiles(); + + $wrong = array(); + foreach ($files as $file) { + $hash = $this->file->getHash($file); + + if ($this->file->hasVerifyHash($file) + && !$this->file->wasFileSuccessfullyCopiedViaFileSynchronizerPlugin($file, $hash)) { + $wrong[] = array( + 'file' => $file, + 'hash' => $hash, + 'verify_file' => $this->file->getPathToVerifyHashFile($file), + 'verify_hash' => $this->file->getVerifyHash($file) + ); + } + } + + return $wrong; + } +} diff --git a/plugins/AutoLogImporter/LogImporter/Result.php b/plugins/AutoLogImporter/LogImporter/Result.php new file mode 100644 index 0000000000..18333e92b9 --- /dev/null +++ b/plugins/AutoLogImporter/LogImporter/Result.php @@ -0,0 +1,51 @@ +<?php +/** + * Copyright (C) Piwik PRO - All rights reserved. + * + * Using this code requires that you first get a license from Piwik PRO. + * Unauthorized copying of this file, via any medium is strictly prohibited. + * + * @link http://piwik.pro + */ + +namespace Piwik\Plugins\AutoLogImporter\LogImporter; + +use Piwik\Db; + +class Result +{ + private $command; + private $output; + private $exitCode; + + public function __construct($command, $output, $exitCode) + { + $this->command = $command; + $this->output = $output; + $this->exitCode = $exitCode; + } + + /** + * @return string + */ + public function getCommand() + { + return $this->command; + } + + /** + * @return string + */ + public function getOutput() + { + return $this->output; + } + + /** + * @return string + */ + public function getExitCode() + { + return $this->exitCode; + } +} diff --git a/plugins/AutoLogImporter/Menu.php b/plugins/AutoLogImporter/Menu.php new file mode 100644 index 0000000000..ae6b5fa3b1 --- /dev/null +++ b/plugins/AutoLogImporter/Menu.php @@ -0,0 +1,31 @@ +<?php +/** + * Copyright (C) Piwik PRO - All rights reserved. + * + * Using this code requires that you first get a license from Piwik PRO. + * Unauthorized copying of this file, via any medium is strictly prohibited. + * + * @link http://piwik.pro + */ + +namespace Piwik\Plugins\AutoLogImporter; + +use Piwik\Menu\MenuAdmin; +use Piwik\Piwik; + +/** + * This class allows you to add, remove or rename menu items. + * To configure a menu (such as Admin Menu, Top Menu, User Menu...) simply call the corresponding methods as + * described in the API-Reference http://developer.piwik.org/api-reference/Piwik/Menu/MenuAbstract + */ +class Menu extends \Piwik\Plugin\Menu +{ + + public function configureAdminMenu(MenuAdmin $menu) + { + if (Piwik::hasUserSuperUserAccess()) { + $menu->addDiagnosticItem('Auto Log Import', $this->urlForAction('status', array('limit' => 200))); + } + } + +} diff --git a/plugins/AutoLogImporter/README.md b/plugins/AutoLogImporter/README.md new file mode 100644 index 0000000000..45dcc04a06 --- /dev/null +++ b/plugins/AutoLogImporter/README.md @@ -0,0 +1,100 @@ +# Piwik AutoLogImporter Plugin + +[](https://magnum.travis-ci.com/PiwikPRO/plugin-AutoLogImporter) + +## Description + +Automatically imports all log files within a specified directory. It remembers which files of a directory have been +imported and only imports new files. Important: If a file was changed after an import the file is considered as a +new file and will be imported again. + +Features: +* Automatically imports files from a specified directory +* Makes sure to not import the same file twice +* Gives you detailed output the log import result +* Shows which files will be imported next +* Shows which files are currently imported +* Shows which files have been imported in the past +* Custom log importer options can be configured to make it work with different log formats etc +* Imported files won't be deleted after importing but marked as imported in the database + +## FAQ + +__What are the requirements?__ + +* You need to have Python installed under `/usr/bin/python` see https://github.com/piwik/piwik-log-analytics. +* It currently works only out of the box if the log format can be detected automatically (eg Apache). A different log format can be configured in `config.php` see further below. +* We currently assume `--replay-tracking`, meaning the log file should contain requests to `piwik.php`. +* The PHP method `exec()` must be enabled. +* This plugin has not been tested with very large log files. +* Beside each log file there has to be a `.log.hash` file containing the md5 sum of the log file, otherwise a log file will be skipped. This is to make sure the log file was copied correctly via the `FileSynchronizer` plugin. + +__How do I setup this plugin?__ + +* Install this plugin +* Configure it via "Administration => Plugin Settings". + +**Warning**: +Make sure to not specify for example the path to the Apache log files directly as we would possibly import the same file +multiple times. Imagine there's an `access.log` and Apache writes each request into it as they are coming. We would +import this file every hour (as the task to import log files runs every hour). We cannot detect that we imported this +file already has the MD5 hash changes every time the file is imported. Unfortunately, the +[Log importer](https://github.com/piwik/piwik-log-analytics) cannot detect which lines of a file were already imported. + +The easiest way to setup this plugin is to install the plugin `FileSynchronizer` as well. Specify the Apache log files +directory as source directory and `*-*-*.log` as file pattern. This makes sure to not copy `access.log` but log files +of the previous day, eg `2015-02-10.log`. Specify any target directory in the `FileSynchronizer`, just make sure to also +configure this directory for this plugin. + +This will make sure only (finished) log files of previous days will be copied and imported, and it makes sure to create +a `.hash` file for each log file to make sure the log file will be actually imported. + +__Why is a ".hash" file needed for each log file?__ + +Short version: To prevent "race conditions". Long version: Imagine the `FileSynchronizer` copies log files to the +target directory, possibly from a remote server. At the same time this plugin is starting to import log files. We would +import a log file that is currently still being copied and possibly even import twice as the MD5 hash would change once +the file is fully copied. The `.hash` file is created after the file was copied so we can verify whether the file was +copied successfully. + +__How can I access details about imported files?__ + +Either by accessing the diagnostic status page under "Administration => Auto Log Import" or by having a look at the +database table `auto_log_importer`. + +__Can I configure the log parameters passed to the log importer?__ + +Yes! Create the file `/config/config.php` in case it does not exist and dfine additional parameters like this: + +``` +<?php + +return array( + 'AutoLogImporter.logImportOptions' => array('--token-auth=FOOBAR', '--enable-static') +); +``` + +__How can I trigger a log import manually?__ + +You can trigger a file import manually by executing the following command: + +`./console core:run-scheduled-tasks "Piwik\Plugins\AutoLogImporter\Tasks.importLogFiles"` + +__Can I force a certain log file to be written into a specific file?__ + +If the filename contains the pattern `_idsite_$idSite` (eg "2015-01-01_idsite_1.log"), the log importer will +automatically set the option `--idsite=$idSite`. + +__I want to see more than the latest 200 imported files, is it possible?__ + +Yes, there is a `limit` URL parameter that you can change to any number. + + +## Changelog + +* 0.1.0 Initial version + +## Contact +To get a license for one of the Enterprise plugins, or to access the latest updates, contact us. +If you have any suggestion, code review, or feedback please email contact@piwik.pro + diff --git a/plugins/AutoLogImporter/Settings.php b/plugins/AutoLogImporter/Settings.php new file mode 100644 index 0000000000..e6a2d84013 --- /dev/null +++ b/plugins/AutoLogImporter/Settings.php @@ -0,0 +1,108 @@ +<?php +/** + * Copyright (C) Piwik PRO - All rights reserved. + * + * Using this code requires that you first get a license from Piwik PRO. + * Unauthorized copying of this file, via any medium is strictly prohibited. + * + * @link http://piwik.pro + */ + +namespace Piwik\Plugins\AutoLogImporter; + +use Piwik\Common; +use Piwik\Settings\SystemSetting; + +/** + * Defines Settings for AutoLogImporter. + * + * Usage like this: + * $settings = new Settings('AutoLogImporter'); + * $settings->autoRefresh->getValue(); + * $settings->metric->getValue(); + */ +class Settings extends \Piwik\Plugin\Settings +{ + + /** @var SystemSetting */ + public $enabled; + + /** @var SystemSetting */ + public $logFilesPath; + + /** @var SystemSetting */ + public $filePattern; + + protected function init() + { + $this->createEnabledSetting(); + $this->createLogFilesPathSetting(); + $this->createFilePatternSetting(); + } + + private function createEnabledSetting() + { + $this->enabled = new SystemSetting('enabled', 'Enabled'); + $this->enabled->type = static::TYPE_BOOL; + $this->enabled->uiControlType = static::CONTROL_CHECKBOX; + $this->enabled->defaultValue = false; + + $this->addSetting($this->enabled); + } + + private function createLogFilesPathSetting() + { + $this->logFilesPath = new SystemSetting('logFilesPath', 'Path to log files'); + $this->logFilesPath->type = static::TYPE_STRING; + $this->logFilesPath->uiControlType = static::CONTROL_TEXT; + + $self = $this; + $this->logFilesPath->validate = function ($value, $setting) use ($self) { + if (empty($value) && !$self->enabled->getValue()) { + // it is not enabled, an empty value is okay. + return; + } + if (empty($value)) { + throw new \Exception('A value must be specified'); + } + if (!file_exists($value)) { + throw new \Exception('This path does not exist'); + } + if (!is_dir($value)) { + throw new \Exception('Path is not a directory'); + } + if (!is_readable($value)) { + throw new \Exception('This directory is not readable'); + } + }; + $this->logFilesPath->transform = function ($value) { + if (!empty($value) && Common::stringEndsWith($value, '/')) { + $value = substr($value, 0, -1); + } + + return $value; + }; + $this->logFilesPath->description = 'Defines the path to the log files.'; + + $this->addSetting($this->logFilesPath); + } + + private function createFilePatternSetting() + { + $this->filePattern = new SystemSetting('filePattern', 'File Pattern'); + $this->filePattern->type = static::TYPE_STRING; + $this->filePattern->uiControlType = static::CONTROL_TEXT; + $this->filePattern->defaultValue = '*.log'; + $this->filePattern->description = 'If specified, only files matching this pattern will be imported.'; + $this->filePattern->transform = function ($value) { + if (!empty($value)) { + return trim($value); + } + + return '*'; + }; + + $this->addSetting($this->filePattern); + } + +} diff --git a/plugins/AutoLogImporter/Tasks.php b/plugins/AutoLogImporter/Tasks.php new file mode 100644 index 0000000000..fd8c0cdd66 --- /dev/null +++ b/plugins/AutoLogImporter/Tasks.php @@ -0,0 +1,44 @@ +<?php +/** + * Copyright (C) Piwik PRO - All rights reserved. + * + * Using this code requires that you first get a license from Piwik PRO. + * Unauthorized copying of this file, via any medium is strictly prohibited. + * + * @link http://piwik.pro + */ + +namespace Piwik\Plugins\AutoLogImporter; + +use Piwik\Container\StaticContainer; +use Piwik\Settings\Setting; + +class Tasks extends \Piwik\Plugin\Tasks +{ + public function schedule() + { + $this->hourly('importLogFiles'); + } + + public function importLogFiles() + { + $settings = StaticContainer::get('Piwik\Plugins\AutoLogImporter\Settings'); + + if (!$settings->enabled->getValue()) { + return; + } + + // we validate to make sure directory is still readable etc, if not an exception will be thrown + $this->validateSetting($settings->logFilesPath); + + $importer = StaticContainer::get('Piwik\Plugins\AutoLogImporter\LogImporter'); + $importer->importFiles(); + } + + private function validateSetting(Setting $setting) + { + if (isset($setting->validate)) { + call_user_func($setting->validate, $setting->getValue(), $setting); + } + } +} diff --git a/plugins/AutoLogImporter/config/config.php b/plugins/AutoLogImporter/config/config.php new file mode 100644 index 0000000000..08340534d2 --- /dev/null +++ b/plugins/AutoLogImporter/config/config.php @@ -0,0 +1,10 @@ +<?php + +return array( + + 'AutoLogImporter.logImportOptions' => array(), + + 'AutoLogImporter.currentTimestamp' => DI\factory(function () { + return \Piwik\Date::now()->getTimestamp(); + }), +); diff --git a/plugins/AutoLogImporter/config/test.php b/plugins/AutoLogImporter/config/test.php new file mode 100644 index 0000000000..5d479ddac8 --- /dev/null +++ b/plugins/AutoLogImporter/config/test.php @@ -0,0 +1,26 @@ +<?php + +return array( + + 'Piwik\Plugins\AutoLogImporter\Settings' => DI\decorate(function (\Piwik\Plugins\AutoLogImporter\Settings $settings) { + if ($settings->enabled->isWritableByCurrentUser()) { + $path = PIWIK_INCLUDE_PATH; + if (\Piwik\Common::stringEndsWith($path, '/')) { + $path = substr($path, 0, -1); + } + $settings->enabled->setValue(true); + $settings->logFilesPath->setValue($path . '/plugins/AutoLogImporter/tests/resources'); + + if (!empty($_GET['no_autologimporter_match'])) { + $settings->filePattern->setValue('*.nothing'); + } else { + $settings->filePattern->setValue('*.log'); + } + } + + return $settings; + }), + + 'AutoLogImporter.logImportOptions' => array('--token-auth=9ad1de7f8b329ab919d854c556f860c1'), + 'AutoLogImporter.currentTimestamp' => '1440592473' +); diff --git a/plugins/AutoLogImporter/lang/en.json b/plugins/AutoLogImporter/lang/en.json new file mode 100644 index 0000000000..2becd8e937 --- /dev/null +++ b/plugins/AutoLogImporter/lang/en.json @@ -0,0 +1,26 @@ +{ + "AutoLogImporter": { + "ScheduledFiles": "Log files that will be imported next", + "ScheduledFilesDescription": "These files should be imported as soon as the import task runs the next time.", + "NoScheduledFiles": "No file is scheduled to be imported.", + "ImportingFiles": "Log files currently importing", + "NoImportingFiles": "No log file is currently being imported.", + "ImportedFiles": "Imported log files", + "ImportedFilesDescription": "The following table shows the latest %d imported files.", + "NoImportedFile": "No log file has been imported yet.", + "ToConfigurePluginGoToSettings": "To configure this plugin go to %s%s%s.", + "File": "File", + "StartDate": "Start date", + "InProcessSince": "In process since", + "NumLogLines": "Log lines count", + "FileSize": "File size", + "Imported": "Imported", + "TimeTaken": "Time taken", + "ImportDetails": "Exit code: '%s', output: '%s'", + "HashFile": "Hash file", + "ExitCode": "Exit Code", + "FileDetails": "'%s' via '%s'. Hash of file is '%s'", + "InvalidFilesDescription": "The following files do not match the expected hash. This usually indicates a not successfully copied log file. These files won't be imported. If a file was already imported either remove the hash file, update the expected hash file with the actual hash or copy the log file again if it was not copied correctly.", + "InvalidFileDetails": "'%s' should match hash '%s' in '%s' but matches '%s'." + } +}
\ No newline at end of file diff --git a/plugins/AutoLogImporter/plugin.json b/plugins/AutoLogImporter/plugin.json new file mode 100644 index 0000000000..e4fe65aa49 --- /dev/null +++ b/plugins/AutoLogImporter/plugin.json @@ -0,0 +1,16 @@ +{ + "name": "AutoLogImporter", + "version": "0.1.0", + "description": "Imports log files automatically", + "theme": false, + "require": { + "piwik": ">=2.15.0-b4" + }, + "authors": [ + { + "name": "Piwik", + "email": "", + "homepage": "" + } + ] +}
\ No newline at end of file diff --git a/plugins/AutoLogImporter/screenshots/.gitkeep b/plugins/AutoLogImporter/screenshots/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/plugins/AutoLogImporter/screenshots/.gitkeep diff --git a/plugins/AutoLogImporter/screenshots/Error_If_File_Was_Not_Copied_Successfully.png b/plugins/AutoLogImporter/screenshots/Error_If_File_Was_Not_Copied_Successfully.png Binary files differnew file mode 100644 index 0000000000..05a1703479 --- /dev/null +++ b/plugins/AutoLogImporter/screenshots/Error_If_File_Was_Not_Copied_Successfully.png diff --git a/plugins/AutoLogImporter/screenshots/Error_Import_Details.png b/plugins/AutoLogImporter/screenshots/Error_Import_Details.png Binary files differnew file mode 100644 index 0000000000..393814a107 --- /dev/null +++ b/plugins/AutoLogImporter/screenshots/Error_Import_Details.png diff --git a/plugins/AutoLogImporter/screenshots/Plugin_Settings.png b/plugins/AutoLogImporter/screenshots/Plugin_Settings.png Binary files differnew file mode 100644 index 0000000000..53b6853063 --- /dev/null +++ b/plugins/AutoLogImporter/screenshots/Plugin_Settings.png diff --git a/plugins/AutoLogImporter/screenshots/Status_Initially.png b/plugins/AutoLogImporter/screenshots/Status_Initially.png Binary files differnew file mode 100644 index 0000000000..6965d640b3 --- /dev/null +++ b/plugins/AutoLogImporter/screenshots/Status_Initially.png diff --git a/plugins/AutoLogImporter/screenshots/Status_Report.png b/plugins/AutoLogImporter/screenshots/Status_Report.png Binary files differnew file mode 100644 index 0000000000..50808568ab --- /dev/null +++ b/plugins/AutoLogImporter/screenshots/Status_Report.png diff --git a/plugins/AutoLogImporter/screenshots/Successful_Import_Details.png b/plugins/AutoLogImporter/screenshots/Successful_Import_Details.png Binary files differnew file mode 100644 index 0000000000..6ee327f1ba --- /dev/null +++ b/plugins/AutoLogImporter/screenshots/Successful_Import_Details.png diff --git a/plugins/AutoLogImporter/stylesheets/status.less b/plugins/AutoLogImporter/stylesheets/status.less new file mode 100644 index 0000000000..c8a04c3dfe --- /dev/null +++ b/plugins/AutoLogImporter/stylesheets/status.less @@ -0,0 +1,29 @@ +ul.autoLogImportInvalidFiles { + list-style: circle; + margin-top: 20px; +} + +#listImportingFiles, +#listImportedFiles { + table { + max-width: 1000px; + min-width: 600px; + } + + .date_field { + white-space: nowrap + } + + .filename { + width: 300px; + word-break: break-all; + } + + .icon-success { + color: green; + } + + .icon-error { + color: #D4291F; + } +}
\ No newline at end of file diff --git a/plugins/AutoLogImporter/templates/status.twig b/plugins/AutoLogImporter/templates/status.twig new file mode 100644 index 0000000000..0f9698433b --- /dev/null +++ b/plugins/AutoLogImporter/templates/status.twig @@ -0,0 +1,117 @@ +{% extends 'admin.twig' %} + +{% block content %} + +{% if filesWithInvalidHashes|length %} + {% set invalidFiles %} + {{ 'AutoLogImporter_InvalidFilesDescription'|translate }} + + <ul class="autoLogImportInvalidFiles"> + {% for fileWithInvalidHashes in filesWithInvalidHashes %} + <li>{{ 'AutoLogImporter_InvalidFileDetails'|translate(fileWithInvalidHashes.file, fileWithInvalidHashes.verify_hash, fileWithInvalidHashes.verify_file, fileWithInvalidHashes.hash) }}</li> + {% endfor %} + </ul> + {% endset %} + + {{ invalidFiles|notification({'noclear': true, 'raw': true, 'context': 'error'}) }} +{% endif %} + + +<h2>{{ 'AutoLogImporter_ImportingFiles'|translate }}</h2> + +<p>{{ 'AutoLogImporter_ToConfigurePluginGoToSettings'|translate('<a href="' ~ linkTo({'module': 'CoreAdminHome', 'action': 'adminPluginSettings'}) ~ '#AutoLogImporter">', ('CoreAdminHome_PluginSettings'|translate), '</a>')|raw }}</p> + +<div id="listImportingFiles"> + <table class="entityTable dataTable"> + <thead> + <tr> + <th class="filename">{{ 'AutoLogImporter_File'|translate }}</th> + <th>{{ 'AutoLogImporter_StartDate'|translate }}</th> + <th>{{ 'AutoLogImporter_InProcessSince'|translate }}</th> + <th>{{ 'AutoLogImporter_FileSize'|translate }}</th> + <th>{{ 'AutoLogImporter_NumLogLines'|translate }}</th> + </tr> + </thead> + <tbody> + {% if importingFiles|length %} + {% for importingFile in importingFiles %} + <tr> + <td class="filename" title="{{ 'AutoLogImporter_FileDetails'|translate(importingFile.file, importingFile.command, importingFile.file_hash) }}">{{ importingFile.filename }}</td> + <td class="date_field">{{ importingFile.start_date }}</td> + <td>{{ importingFile.time_ago }}</td> + <td>{{ importingFile.size_human }}</td> + <td>{{ importingFile.num_log_lines }}</td> + </tr> + {% endfor %} + {% else %} + <tr><td colspan="5">{{ 'AutoLogImporter_NoImportingFiles'|translate }}</td></tr> + {% endif %} + </tbody> + </table> +</div> + +<h2>{{ 'AutoLogImporter_ScheduledFiles'|translate }}</h2> + +<p>{{ 'AutoLogImporter_ScheduledFilesDescription'|translate }}</p> + +<div id="listScheduledFiles"> + <table class="entityTable dataTable"> + <thead> + <tr> + <th class="filename">{{ 'AutoLogImporter_File'|translate }}</th> + </tr> + </thead> + <tbody> + {% if scheduledFiles|length %} + {% for scheduledFile in scheduledFiles %} + <tr> + <td>{{ scheduledFile }}</td> + </tr> + {% endfor %} + {% else %} + <tr><td colspan="5">{{ 'AutoLogImporter_NoScheduledFiles'|translate }}</td></tr> + {% endif %} + </tbody> + </table> +</div> + + +<h2>{{ 'AutoLogImporter_ImportedFiles'|translate }}</h2> +<p>{{ 'AutoLogImporter_ImportedFilesDescription'|translate(limit) }}</p> +<div id="listImportedFiles"> + <table class="entityTable dataTable"> + <thead> + <tr> + <th class="filename">{{ 'AutoLogImporter_File'|translate }}</th> + <th>{{ 'General_Date'|translate }}</th> + <th>{{ 'AutoLogImporter_TimeTaken'|translate }}</th> + <th>{{ 'AutoLogImporter_FileSize'|translate }}</th> + <th>{{ 'AutoLogImporter_NumLogLines'|translate }}</th> + <th>{{ 'AutoLogImporter_Imported'|translate }}</th> + </tr> + </thead> + <tbody> + {% if importedFiles|length %} + {% for importedFile in importedFiles %} + <tr> + <td class="filename" title="{{ 'AutoLogImporter_FileDetails'|translate(importedFile.file, importedFile.command, importedFile.file_hash) }}">{{ importedFile.filename }}</td> + <td class="date_field" title="{{ importedFile.start_date|e('html_attr') }} - {{ importedFile.end_date|e('html_attr') }}">{{ importedFile.date }}</td> + <td>{{ importedFile.duration }}s</td> + <td>{{ importedFile.size_human }}</td> + <td>{{ importedFile.num_log_lines }}</td> + <td title="{{ 'AutoLogImporter_ImportDetails'|translate(importedFile.exit_code, importedFile.output)|e('html_attr') }}"> + {% if importedFile.exit_code == 0 %} + <span class="icon-success"></span> + {% else %} + <span class="icon-error"></span> + {% endif %} + </td> + </tr> + {% endfor %} + {% else %} + <tr><td colspan="6">{{ 'AutoLogImporter_NoImportedFile'|translate }}</td></tr> + {% endif %} + </tbody> + </table> +</div> +{% endblock %}
\ No newline at end of file diff --git a/plugins/Bandwidth b/plugins/Bandwidth new file mode 160000 +Subproject a046ad794f992f4b3c5cdd33cd56ee9d604ce1f diff --git a/plugins/ComparisonDashboard b/plugins/ComparisonDashboard new file mode 160000 +Subproject eadb0ea480b0037aea75d3d0f5e6b34dc92394c diff --git a/plugins/ConcurrentVisits b/plugins/ConcurrentVisits new file mode 160000 +Subproject 45280cb3ba4831d1a164696dd68eb07087b2b65 diff --git a/plugins/CustomTrackerJs b/plugins/CustomTrackerJs new file mode 160000 +Subproject aa296ff7f76db00aa8e2c41b332b58f9b73d3ba diff --git a/plugins/DFOCanada b/plugins/DFOCanada new file mode 160000 +Subproject c6ab17d70785114904616584c3f62212c5e6983 diff --git a/plugins/FileSynchronizer/.gitignore b/plugins/FileSynchronizer/.gitignore new file mode 100644 index 0000000000..c8c9480010 --- /dev/null +++ b/plugins/FileSynchronizer/.gitignore @@ -0,0 +1 @@ +tests/System/processed/*xml
\ No newline at end of file diff --git a/plugins/FileSynchronizer/.travis.yml b/plugins/FileSynchronizer/.travis.yml new file mode 100644 index 0000000000..ecc6f3996a --- /dev/null +++ b/plugins/FileSynchronizer/.travis.yml @@ -0,0 +1,132 @@ +# do not edit this file manually, instead run the generate:travis-yml console command + +language: php + +php: + - 5.6 + - 5.3.3 +# - hhvm + +services: + - redis-server + +# Separate different test suites +env: + global: + - PLUGIN_NAME=FileSynchronizer + - PIWIK_ROOT_DIR=$TRAVIS_BUILD_DIR/piwik + - PIWIK_LATEST_STABLE_TEST_TARGET=2.15.0-b3 + - secure: "GwPMT+B1vLoyQLVcbvWPU3FZl1UvMFufQhlVKxnAy+FSmMamMCiNRd/XpYhB/zER1TuKQvSgWmZjFXrab6ih98MRJEvc+ae0nqCduSRvlPOWWOWkerUdeKJOis0tjaPdzBzdgIiP2lrw/TF+7Mo4CNPE0i8XDcoGjBQRT9dmgQpg130zasTdxkdp484NKFb8busQEBXZmAHYDLkF6u+hDVvcf1zR7ebOGPyEfTJCItvN/CCUvznj02zPUouknpdAuMGmlQmQD4KzhlV68NlBurB2dE9KG7LlIlX6fvnpxK/DiTNDrUvbAgBGUMmu1wKgC7mE9j1jYAwbWMldttBGQa7G81AP1l73w8SAyTXbyKLzf0g2hfaoQV/IEG2I6tQ6SicWkyapjJ8pGsHWhr7JS6z5LO/OQpHaytegEXga8CuUMPxj3U9BBQ7XEiGyMvVF2qBtL5KDvSZXPbDUlNzYCaiNJBZKuBLqaKowU7r91GkcSph8KSVg8lKq9DouUi7m65x21zAjLEEtTqtM5vRPcUFq3Nmvs1vYH+q5Su/35kXnH78sx8W2R/0oBgp6SaHOMKLw5l7KpZrL8Y5BL//mhnPs2llUnSYVKwyLiyCEG1QjiS9KfGYOD1WJ4UkXsbBWtSuwM0loV9VWa+DXSuE+zIA3Zu0jP9gbwqEWZEYBXZ0=" + - secure: "ABmkRR2PtfFy2+ebTdaBzekuydJ7VyEE9z2DeO9e1FUVD2BS0oJ0blf3iJDXxrb50ypMRcfjnipM8UZFcHIC+H949gV/9OWyaV1r2iog0XvbouTZHDWG2DN4fAqnghMNgS5yavQGPTyiYZTgPKW85jE6PTvx0mj9sOg3fur1djb4f8V+CHl0bIO84udhpAXHDlu0/UK20vfhF02QwI9HPsFWqsej8FQFxzJHevYjGJGgHWOf/m8phd0r0Ag6K52S4bkmoLIL4bfU9KuujU76k6DFzAyoKEgxlpRXfomRutj4vRcjOX4vLy7PnOXkYc1fAI+Evstb+Bsczqa8C2BQe17VRaevx0wy6XXpoLkbvMJzZNapOFqnZmYxB2b1N1M4gKJF1seFsJ5WECSP2IVX/e0CY/rTC7RhXUO5c+MhqajXv/9rdPfy4dDCKQ/Ag1NhlYjRL2ybtKS/vFO3Kt5ImI0fLr6sibqfCnrddkhSZ2zlyvi4IA596EUSqfZZNa5j//j3gllAuDMyCzc8MEb10mMl6vrYiU8mGVOcG4xyHssXHaUSas9wTV2bmEVcjXc1rkn/f7pvMAwlMv9K5a8t3Zm+8k5sF2DwHguzO9g4UqJd9QBAnO5oKtPcdN6yQS30768YFI1ojXwn+YCRGDXjbn/DtRzRwHSyWHnPlNoDPh4=" + matrix: + - TEST_SUITE=PluginTests MYSQL_ADAPTER=PDO_MYSQL TEST_AGAINST_PIWIK_BRANCH=$PIWIK_LATEST_STABLE_TEST_TARGET + - TEST_SUITE=PluginTests MYSQL_ADAPTER=PDO_MYSQL TEST_AGAINST_CORE=minimum_required_piwik + - TEST_SUITE=UITests MYSQL_ADAPTER=PDO_MYSQL TEST_AGAINST_PIWIK_BRANCH=$PIWIK_LATEST_STABLE_TEST_TARGET + +matrix: + exclude: + # execute latest stable tests only w/ PHP 5.5 + - php: 5.3.3 + env: TEST_SUITE=PluginTests MYSQL_ADAPTER=PDO_MYSQL TEST_AGAINST_CORE=minimum_required_piwik + # execute UI tests only w/ PHP 5.6 + - php: 5.3.3 + env: TEST_SUITE=UITests MYSQL_ADAPTER=PDO_MYSQL TEST_AGAINST_PIWIK_BRANCH=$PIWIK_LATEST_STABLE_TEST_TARGET + +sudo: required + +script: $PIWIK_ROOT_DIR/tests/travis/travis.sh + +before_install: + # do not use the Zend allocator on PHP 5.3 since it will randomly segfault after program execution + - '[[ "$TRAVIS_PHP_VERSION" == 5.3* ]] && export USE_ZEND_ALLOC=0 || true' + +install: + # move all contents of current repo (which contains the plugin) to a new directory + - mkdir $PLUGIN_NAME + - cp -R !($PLUGIN_NAME) $PLUGIN_NAME + - cp -R .git/ $PLUGIN_NAME/ + - cp .travis.yml $PLUGIN_NAME + # checkout piwik in the current directory + - git clone -q https://github.com/piwik/piwik.git piwik + - cd piwik + - git fetch -q --all + - git submodule update + + # make sure travis-scripts repo is latest for initial travis setup + - '[ -d ./tests/travis/.git ] || sh -c "rm -rf ./tests/travis && git clone https://github.com/piwik/travis-scripts.git ./tests/travis"' + - cd ./tests/travis ; git checkout master ; cd ../.. + + - export GENERATE_TRAVIS_YML_COMMAND="php ./tests/travis/generator/main.php generate:travis-yml --plugin=\"FileSynchronizer\" --verbose" + - '[[ "$TRAVIS_JOB_NUMBER" != *.1 || "$TRAVIS_PULL_REQUEST" != "false" ]] || ./tests/travis/autoupdate_travis_yml.sh' + + - ./tests/travis/checkout_test_against_branch.sh + + - '[ ! -f ./tests/travis/install_mysql_5.6.sh ] || ./tests/travis/install_mysql_5.6.sh' + + # Make sure we use Python 2.6 + - '[ ! -f ./tests/travis/install_python_2.6.sh ] || ./tests/travis/install_python_2.6.sh' + + - ./tests/travis/configure_git.sh + + # travis now complains about this failing 9 times out of 10, so removing it + #- travis_retry composer self-update + + - '[ "$SKIP_COMPOSER_INSTALL" == "1" ] || travis_retry composer install' + + # move plugin contents to folder in the plugins subdirectory + - rm -rf plugins/$PLUGIN_NAME + - mv ../$PLUGIN_NAME plugins + + # clone dependent repos + - ./tests/travis/checkout_dependent_plugins.sh + +before_script: + - phpenv config-rm xdebug.ini; + + # add always_populate_raw_post_data=-1 to php.ini + - echo "always_populate_raw_post_data=-1" >> ~/.phpenv/versions/$(phpenv version-name)/etc/php.ini + + # disable opcache to avoid random failures on travis + - echo "opcache.enable=0" >> ~/.phpenv/versions/$(phpenv version-name)/etc/php.ini + + # print out mysql information + - mysql --version + - mysql -e "SELECT VERSION();" + + # configure mysql + - mysql -e "SET GLOBAL sql_mode = 'NO_ENGINE_SUBSTITUTION,STRICT_TRANS_TABLES'" # Travis default + # try to avoid mysql has gone away errors + - mysql -e "SET GLOBAL wait_timeout = 36000;" + - mysql -e "SET GLOBAL max_allowed_packet = 134209536;" + - mysql -e "SHOW VARIABLES LIKE 'max_allowed_packet';" + - mysql -e "SHOW VARIABLES LIKE 'wait_timeout';" + + - mysql -e "SELECT @@sql_mode;" + # - mysql -e "SHOW GLOBAL VARIABLES;" + + # print out more debugging info + - uname -a + - date + - php -r "var_dump(gd_info());" + - mysql -e 'create database piwik_tests;' + + - ./tests/travis/prepare.sh + - ./tests/travis/setup_webserver.sh + + - cd tests/PHPUnit + +after_script: + # change directory back to root travis dir + - cd $PIWIK_ROOT_DIR + + # output contents of files w/ debugging info to screen + - cat /var/log/nginx/error.log + - cat $PIWIK_ROOT_DIR/tmp/php-fpm.log + - cat $PIWIK_ROOT_DIR/tmp/logs/piwik.log + - cat $PIWIK_ROOT_DIR/config/config.ini.php + + # upload test artifacts (for debugging travis failures) + - ./tests/travis/upload_artifacts.sh + +after_success: + - cd $PIWIK_ROOT_DIR diff --git a/plugins/FileSynchronizer/API.php b/plugins/FileSynchronizer/API.php new file mode 100644 index 0000000000..5663a5d540 --- /dev/null +++ b/plugins/FileSynchronizer/API.php @@ -0,0 +1,73 @@ +<?php +/** + * Copyright (C) Piwik PRO - All rights reserved. + * + * Using this code requires that you first get a license from Piwik PRO. + * Unauthorized copying of this file, via any medium is strictly prohibited. + * + * @link http://piwik.pro + */ + +namespace Piwik\Plugins\FileSynchronizer; + +use Piwik\Container\StaticContainer; +use Piwik\DataTable; +use Piwik\DataTable\Row; +use Piwik\Piwik; + +/** + * API for plugin FileSynchronizer + * + * @method static \Piwik\Plugins\FileSynchronizer\API getInstance() + */ +class API extends \Piwik\Plugin\API +{ + /** + * Get all synced files whether they were successful or not. + * + * Latest synced files are listed first. + * @return array + */ + public function getAllSyncedFiles() + { + Piwik::checkUserHasSuperUserAccess(); + + $dao = new Dao(); + $files = $dao->getAllSyncedFiles(); + + usort($files, function ($a, $b){ + return $a['idfilesync'] > $b['idfilesync'] ? -1 : 1; + }); + + return $files; + } + + /** + * Get all files that are currently syncing. + * + * @return array + */ + public function getAllSyncingFiles() + { + Piwik::checkUserHasSuperUserAccess(); + + $dao = new Dao(); + $files = $dao->getAllSyncingFiles(); + + return $files; + } + + /** + * Get a list of all files that can be synced and will be synced as soon as the next tasks run. + * + * @return array + */ + public function getFilesThatCanBeSynced() + { + Piwik::checkUserHasSuperUserAccess(); + + $syncFiles = StaticContainer::get('Piwik\Plugins\FileSynchronizer\SyncFiles'); + return $syncFiles->getFilesThatCanBeSynced(); + } + +} diff --git a/plugins/FileSynchronizer/Controller.php b/plugins/FileSynchronizer/Controller.php new file mode 100644 index 0000000000..b0d8fee335 --- /dev/null +++ b/plugins/FileSynchronizer/Controller.php @@ -0,0 +1,50 @@ +<?php +/** + * Copyright (C) Piwik PRO - All rights reserved. + * + * Using this code requires that you first get a license from Piwik PRO. + * Unauthorized copying of this file, via any medium is strictly prohibited. + * + * @link http://piwik.pro + */ + +namespace Piwik\Plugins\FileSynchronizer; +use Piwik\API\Request; +use Piwik\Common; +use Piwik\Piwik; + +/** + * A controller let's you for example create a page that can be added to a menu. For more information read our guide + * http://developer.piwik.org/guides/mvc-in-piwik or have a look at the our API references for controller and view: + * http://developer.piwik.org/api-reference/Piwik/Plugin/Controller and + * http://developer.piwik.org/api-reference/Piwik/View + */ +class Controller extends \Piwik\Plugin\ControllerAdmin +{ + public function status() + { + Piwik::checkUserHasSuperUserAccess(); + + $limit = Common::getRequestVar('limit', 200, 'int'); + + $syncingFiles = Request::processRequest('FileSynchronizer.getAllSyncingFiles'); + $scheduledFiles = Request::processRequest('FileSynchronizer.getFilesThatCanBeSynced'); + $syncedFiles = Request::processRequest('FileSynchronizer.getAllSyncedFiles', array( + 'filter_limit' => $limit + )); + + if (count($syncedFiles) < $limit) { + $limit = count($syncedFiles); + } + + $formatter = new Formatter(); + + return $this->renderTemplate('status', array( + 'syncedFiles' => $formatter->formatSyncedFiles($syncedFiles), + 'syncingFiles' => $formatter->addTimeAgoAsSentence($formatter->formatSyncedFiles($syncingFiles)), + 'scheduledFiles' => $scheduledFiles, + 'limit' => $limit + )); + } + +} diff --git a/plugins/FileSynchronizer/Dao.php b/plugins/FileSynchronizer/Dao.php new file mode 100644 index 0000000000..68f46b4823 --- /dev/null +++ b/plugins/FileSynchronizer/Dao.php @@ -0,0 +1,112 @@ +<?php +/** + * Copyright (C) Piwik PRO - All rights reserved. + * + * Using this code requires that you first get a license from Piwik PRO. + * Unauthorized copying of this file, via any medium is strictly prohibited. + * + * @link http://piwik.pro + */ + +namespace Piwik\Plugins\FileSynchronizer; + +use Piwik\Common; +use Piwik\Date; +use Piwik\Db; +use Piwik\DbHelper; + +class Dao +{ + /** + * @var Db + */ + private $db; + + private $tableName = 'file_synchronizer'; + private $tableNamePrefixed; + + public function __construct() + { + $this->db = Db::get(); + $this->tableNamePrefixed = Common::prefixTable($this->tableName); + } + + public function getAllSyncedFiles() + { + return Db::fetchAll('SELECT * FROM ' . $this->tableNamePrefixed . ' WHERE end_date is not null'); + } + + public function getAllSyncingFiles() + { + return Db::fetchAll('SELECT * FROM ' . $this->tableNamePrefixed . ' WHERE end_date is null'); + } + + public function isSynced($fileHash) + { + // "end_date is null" => probably in process by another server + // "exit_code = 0" => file was already imported successfully + return (bool) $this->db->fetchOne("SELECT 1 FROM " . $this->tableNamePrefixed . " + WHERE `file_hash` = ? and (end_date is null or exit_code = 0) LIMIT 1", array($fileHash)); + } + + public function isHashFileCreated($fileHash) + { + return (bool) $this->db->fetchOne("SELECT 1 FROM " . $this->tableNamePrefixed . " + WHERE `file_hash` = ? and hash_file_created = 1 LIMIT 1", array($fileHash)); + } + + public function markHashFileCreated($fileHash) + { + $this->db->update($this->tableNamePrefixed, array( + 'hash_file_created' => 1, + ), '`file_hash` = "' . $fileHash . '"'); + } + + public function logFileSyncStart($source, $target, $fileHash, $fileSize, $startDate) + { + $this->db->insert($this->tableNamePrefixed, array( + 'source' => $source, + 'target' => $target, + 'file_hash' => $fileHash, + 'file_size' => $fileSize, + 'start_date' => $startDate, + )); + + return (int) $this->db->lastInsertId(); + } + + public function logFileSyncFinished($id, $command, $output, $exitCode, $duration, $endDate) + { + $this->db->update($this->tableNamePrefixed, array( + 'output' => $output, + 'command' => $command, + 'exit_code' => $exitCode, + 'end_date' => $endDate, + 'duration_in_ms' => $duration, + ), 'idfilesync = ' . (int) $id); + } + + public function install() + { + $table = "`idfilesync` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY , + `start_date` datetime NOT NULL , + `end_date` datetime NULL , + `duration_in_ms` INT UNSIGNED NULL , + `source` TEXT NOT NULL , + `target` TEXT NOT NULL , + `file_size` INT UNSIGNED NOT NULL DEFAULT 0, + `file_hash` VARCHAR(32) NOT NULL , + `hash_file_created` TINYINT UNSIGNED NOT NULL DEFAULT 0, + `command` TEXT NULL, + `output` TEXT NULL , + `exit_code` TINYINT UNSIGNED NULL"; + + DbHelper::createTable($this->tableName, $table); + } + + public function uninstall() + { + Db::dropTables(array($this->tableNamePrefixed)); + } + +} diff --git a/plugins/FileSynchronizer/FileSynchronizer.php b/plugins/FileSynchronizer/FileSynchronizer.php new file mode 100644 index 0000000000..3e585132f2 --- /dev/null +++ b/plugins/FileSynchronizer/FileSynchronizer.php @@ -0,0 +1,39 @@ +<?php +/** + * Copyright (C) Piwik PRO - All rights reserved. + * + * Using this code requires that you first get a license from Piwik PRO. + * Unauthorized copying of this file, via any medium is strictly prohibited. + * + * @link http://piwik.pro + */ + +namespace Piwik\Plugins\FileSynchronizer; + +class FileSynchronizer extends \Piwik\Plugin +{ + public function getListHooksRegistered() + { + return array( + 'AssetManager.getStylesheetFiles' => 'getStylesheetFiles', + ); + } + + public function getStylesheetFiles(&$stylesheets) + { + $stylesheets[] = "plugins/FileSynchronizer/stylesheets/status.less"; + } + + public function install() + { + $dao = new Dao(); + $dao->install(); + } + + public function uninstall() + { + $dao = new Dao(); + $dao->uninstall(); + } + +} diff --git a/plugins/FileSynchronizer/Formatter.php b/plugins/FileSynchronizer/Formatter.php new file mode 100644 index 0000000000..1f10bbf8c1 --- /dev/null +++ b/plugins/FileSynchronizer/Formatter.php @@ -0,0 +1,48 @@ +<?php +/** + * Copyright (C) Piwik PRO - All rights reserved. + * + * Using this code requires that you first get a license from Piwik PRO. + * Unauthorized copying of this file, via any medium is strictly prohibited. + * + * @link http://piwik.pro + */ + +namespace Piwik\Plugins\FileSynchronizer; + +use Piwik\Container\StaticContainer; +use Piwik\Date; +use Piwik\Db; +use Piwik\Metrics\Formatter as MetricsFormatter; + +class Formatter +{ + + public function addTimeAgoAsSentence($files) + { + $formatter = new MetricsFormatter(); + $now = StaticContainer::get('FileSynchronizer.currentTimestamp'); + + foreach ($files as &$file) { + $diff = $now - Date::factory($file['start_date'])->getTimestamp(); + $file['time_ago'] = $formatter->getPrettyTimeFromSeconds($diff, true); + } + + return $files; + } + + public function formatSyncedFiles($files) + { + $formatter = new MetricsFormatter(); + + foreach ($files as &$syncedFile) { + $syncedFile['source_filename'] = basename($syncedFile['source']); + $syncedFile['date'] = substr($syncedFile['start_date'], 0, 10); + $syncedFile['duration'] = $syncedFile['duration_in_ms'] . 'ms'; + $syncedFile['size_human'] = $formatter->getPrettySizeFromBytes($syncedFile['file_size']); + } + + return $files; + } + +} diff --git a/plugins/FileSynchronizer/Menu.php b/plugins/FileSynchronizer/Menu.php new file mode 100644 index 0000000000..608f52ebc7 --- /dev/null +++ b/plugins/FileSynchronizer/Menu.php @@ -0,0 +1,31 @@ +<?php +/** + * Copyright (C) Piwik PRO - All rights reserved. + * + * Using this code requires that you first get a license from Piwik PRO. + * Unauthorized copying of this file, via any medium is strictly prohibited. + * + * @link http://piwik.pro + */ + +namespace Piwik\Plugins\FileSynchronizer; + +use Piwik\Menu\MenuAdmin; +use Piwik\Piwik; + +/** + * This class allows you to add, remove or rename menu items. + * To configure a menu (such as Admin Menu, Top Menu, User Menu...) simply call the corresponding methods as + * described in the API-Reference http://developer.piwik.org/api-reference/Piwik/Menu/MenuAbstract + */ +class Menu extends \Piwik\Plugin\Menu +{ + + public function configureAdminMenu(MenuAdmin $menu) + { + if (Piwik::hasUserSuperUserAccess()) { + $menu->addDiagnosticItem('File Synchronizer', $this->urlForAction('status', array('limit' => 200))); + } + } + +} diff --git a/plugins/FileSynchronizer/README.md b/plugins/FileSynchronizer/README.md new file mode 100644 index 0000000000..42ab1e3281 --- /dev/null +++ b/plugins/FileSynchronizer/README.md @@ -0,0 +1,66 @@ +# Piwik FileSynchronizer Plugin + +[](https://magnum.travis-ci.com/PiwikPRO/plugin-FileSynchronizer) + +## Description + +Synchronizes any files of a directory with another directory. Useful for example to copy log files to another directory. + +Features: + +* Syncs any files from one directory to another +* Can copy files to a remote computer by changing the copy template (eg `scp $source user@host:$target`) +* Specify a shell wildcard to sync only files matching this pattern +* Modify the filename while syncing +* Shows which files will be synced next +* Shows which files are currently syncing +* Shows which files were synced in the past +* Gives detailed error output if a sync failed and will retry automatically +* Syncs files once per hour + +## FAQ + +__Can I sync files to another server?__ + +Yes, you can specify a copy template like `scp $source user@host:$target`. By default files are simply copied from +`$source` to `$target` via the linux command `cp`. + +__How often are files synchronized?__ + +Once per hour. + +__Is Windows supported?__ + +No. + +__How can I verify the file was correctly copied?__ + +We create a hash file for each copied file containing the MD5 hash of the content. If the file name is `access.log`, +there will be a `access.log.hash` in the configured target directory once the file was copied successfully. + +__What happens when a file having the same name already exists in the target directory?__ + +The file in the target directory will be overwritten. + +__How can I trigger a file sync manually?__ + +You can trigger a file sync manually by executing the following command: + +`./console core:run-scheduled-tasks "Piwik\Plugins\FileSynchronizer\Tasks.syncFiles"` + +__How can I access details about previously synced files?__ + +Either by accessing the diagnostic status page under "Administration => File Synchronizer" or by having a look at the +database table `file_synchronizer`. + +__I want to see more than the latest 200 synced files, is it possible?__ + +Yes, there is a `limit` URL parameter that you can change to any number. + +## Changelog + +* 0.1.0 Initial version + +## Contact +To get a license for one of the Enterprise plugins, or to access the latest updates, contact us. +If you have any suggestion, code review, or feedback please email contact@piwik.pro diff --git a/plugins/FileSynchronizer/Settings.php b/plugins/FileSynchronizer/Settings.php new file mode 100644 index 0000000000..37f2e5485e --- /dev/null +++ b/plugins/FileSynchronizer/Settings.php @@ -0,0 +1,207 @@ +<?php +/** + * Copyright (C) Piwik PRO - All rights reserved. + * + * Using this code requires that you first get a license from Piwik PRO. + * Unauthorized copying of this file, via any medium is strictly prohibited. + * + * @link http://piwik.pro + */ + +namespace Piwik\Plugins\FileSynchronizer; + +use Piwik\Common; +use Piwik\Settings\Setting; +use Piwik\Settings\SystemSetting; + +/** + * Defines Settings for FileSynchronizer. + * + * Usage like this: + * $settings = new Settings('FileSynchronizer'); + * $settings->enabled->getValue(); + * $settings->sourceDirectory->getValue(); + */ +class Settings extends \Piwik\Plugin\Settings +{ + /** @var SystemSetting */ + public $enabled; + + /** @var SystemSetting */ + public $sourceDirectory; + + /** @var SystemSetting */ + public $filePattern; + + /** @var SystemSetting */ + public $targetDirectory; + + /** @var SystemSetting */ + public $targetFilenameTemplate; + + /** @var SystemSetting */ + public $copyCommandTemplate; + + protected function init() + { + $this->createEnabledSetting(); + $this->createSourceDirectorySetting(); + $this->createFilePatternSetting(); + $this->createTargetDirectorySetting(); + $this->createTargetFilenameSetting(); + $this->createCopyTemplateSetting(); + } + + private function createEnabledSetting() + { + $this->enabled = new SystemSetting('enabled', 'Enabled'); + $this->enabled->type = static::TYPE_BOOL; + $this->enabled->uiControlType = static::CONTROL_CHECKBOX; + $this->enabled->defaultValue = false; + + $this->addSetting($this->enabled); + } + + private function createSourceDirectorySetting() + { + $this->sourceDirectory = new SystemSetting('sourceDirectory', 'Source directory'); + $this->configureDirectorySetting($this->sourceDirectory, $checkDir = true); + $this->sourceDirectory->description = 'Defines the source directory that contains the files that need to be synced.'; + + $this->addSetting($this->sourceDirectory); + } + + private function createFilePatternSetting() + { + $this->filePattern = new SystemSetting('filePattern', 'File Pattern'); + $this->filePattern->type = static::TYPE_STRING; + $this->filePattern->uiControlType = static::CONTROL_TEXT; + $this->filePattern->defaultValue = '*'; + $this->filePattern->description = 'If specified, only files matching this pattern will be synced.'; + $this->filePattern->inlineHelp = 'Any shell wildcards can be specified, for example "*.log".'; + $this->filePattern->transform = function ($value) { + if (!empty($value)) { + return trim($value); + } + + return '*'; + }; + + $this->addSetting($this->filePattern); + } + + private function createTargetDirectorySetting() + { + $this->targetDirectory = new SystemSetting('targetDirectory', 'Target directory'); + $this->configureDirectorySetting($this->targetDirectory, $checkDir = false); + // we do not check dir as it might be on a remote computer + $this->targetDirectory->description = 'Defines the target directory the files should be copied to.'; + + $this->addSetting($this->targetDirectory); + } + + private function createTargetFilenameSetting() + { + $this->targetFilenameTemplate = new SystemSetting('targetFilenameTemplate', 'Target filename template'); + $this->targetFilenameTemplate->type = static::TYPE_STRING; + $this->targetFilenameTemplate->uiControlType = static::CONTROL_TEXT; + $this->targetFilenameTemplate->defaultValue = '$basename'; + $this->targetFilenameTemplate->description = 'Allows to modify the filename. "$basename" or "$filename" must be specified, "$extension" can be specified optionally.'; + $this->targetFilenameTemplate->inlineHelp = '"$basename" will be replaced with the filename including extension (eg "apache.log"), "$filename" will be replaced by the filename excluding extension (eg "apache") and "$extension" will be replaced by the file extension (eg "log"). This allows to modify the filename in the target directory for example like this: "$filename_idsite_1.$extension"'; + + $self = $this; + $this->targetFilenameTemplate->validate = function ($value, $setting) use ($self) { + if (empty($value) && !$self->enabled->getValue()) { + // it is not enabled, an empty value is okay. + return; + } + + if (empty($value)) { + throw new \Exception('A value must be specified'); + } + + if (false === strpos($value, '$filename') && false === strpos($value, '$basename')) { + throw new \Exception('$filename or $basename must be specified'); + } + }; + $this->targetFilenameTemplate->transform = function ($value) { + if (!empty($value)) { + return trim($value); + } + + return $value; + }; + + $this->addSetting($this->targetFilenameTemplate); + } + + private function configureDirectorySetting(Setting $setting, $checkDir) + { + $setting->type = static::TYPE_STRING; + $setting->uiControlType = static::CONTROL_TEXT; + + $self = $this; + $setting->validate = function ($value, $setting) use ($self, $checkDir) { + if (empty($value) && !$self->enabled->getValue()) { + // it is not enabled, an empty value is okay. + return; + } + if (empty($value)) { + throw new \Exception('A value must be specified'); + } + if ($checkDir && !file_exists($value)) { + throw new \Exception('This path does not exist'); + } + if ($checkDir && !is_dir($value)) { + throw new \Exception('Path is not a directory'); + } + if ($checkDir && !is_readable($value)) { + throw new \Exception('This directory is not readable'); + } + }; + $setting->transform = function ($value) { + if (!empty($value) && Common::stringEndsWith($value, '/')) { + $value = substr($value, 0, -1); + } + + return $value; + }; + } + + private function createCopyTemplateSetting() + { + $this->copyCommandTemplate = new SystemSetting('copyCommandTemplate', 'Copy Command Template'); + $this->copyCommandTemplate->type = static::TYPE_STRING; + $this->copyCommandTemplate->uiControlType = static::CONTROL_TEXT; + $this->copyCommandTemplate->description = 'The shell command to sync files. "$source" and "$target" must be specified.'; + $this->copyCommandTemplate->inlineHelp = '"$source" will be replaced by the path of the source file, "$target" by the path to the target file.'; + $this->copyCommandTemplate->defaultValue = 'cp $source $target'; + + $self = $this; + $this->copyCommandTemplate->validate = function ($value, $setting) use ($self) { + if (empty($value) && !$self->enabled->getValue()) { + // it is not enabled, an empty value is okay. + return; + } + + if (empty($value)) { + throw new \Exception('A value must be specified'); + } + if (false === strpos($value, '$source')) { + throw new \Exception('$source is not specified'); + } + if (false === strpos($value, '$target')) { + throw new \Exception('$target is not specified'); + } + }; + $this->copyCommandTemplate->transform = function ($value) { + if (!empty($value)) { + return trim($value); + } + + return $value; + }; + + $this->addSetting($this->copyCommandTemplate); + } +} diff --git a/plugins/FileSynchronizer/SyncFiles.php b/plugins/FileSynchronizer/SyncFiles.php new file mode 100644 index 0000000000..44f144ddd4 --- /dev/null +++ b/plugins/FileSynchronizer/SyncFiles.php @@ -0,0 +1,121 @@ +<?php +/** + * Copyright (C) Piwik PRO - All rights reserved. + * + * Using this code requires that you first get a license from Piwik PRO. + * Unauthorized copying of this file, via any medium is strictly prohibited. + * + * @link http://piwik.pro + */ + +namespace Piwik\Plugins\FileSynchronizer; + +use Piwik\Container\StaticContainer; +use Piwik\Date; +use Piwik\Db; +use Piwik\Plugins\FileSynchronizer\SyncFiles\Copy; +use Piwik\Plugins\FileSynchronizer\SyncFiles\File; +use Piwik\Plugins\FileSynchronizer\SyncFiles\FileList; + +class SyncFiles +{ + /** + * @var Dao + */ + private $dao; + + /** + * @var File + */ + private $file; + + /** + * @var string + */ + private $copyCommandTemplate; + + /** + * @var string[] + */ + private $sourceFiles; + + /** + * @var string + */ + private $targetDirectory; + + /** + * @var string + */ + private $targetFilenameTemplate; + + public function __construct(Dao $dao, File $file, Settings $settings, FileList $fileList) + { + $this->dao = $dao; + $this->file = $file; + $this->copyCommandTemplate = $settings->copyCommandTemplate->getValue(); + $this->targetDirectory = $settings->targetDirectory->getValue(); + $this->targetFilenameTemplate = $settings->targetFilenameTemplate->getValue(); + $this->sourceFiles = $fileList->findFilesToSync(); + } + + public function getFilesThatCanBeSynced() + { + $files = array(); + + foreach ($this->sourceFiles as $sourceFile) { + $hash = $this->file->getHash($sourceFile); + + if (!$this->dao->isSynced($hash)) { + $files[$hash] = $sourceFile; + } + } + + return $files; + } + + public function sync() + { + $copy = StaticContainer::get('Piwik\Plugins\FileSynchronizer\SyncFiles\Copy'); + + foreach ($this->sourceFiles as $sourceFile) { + $hash = $this->file->getHash($sourceFile); + + if (!$this->dao->isSynced($hash)) { + $this->copyFile($copy, $sourceFile, $hash); + } + + if ($this->dao->isSynced($hash) && !$this->dao->isHashFileCreated($hash)) { + $this->copyHashFile($copy, $sourceFile, $hash); + } + } + } + + private function copyHashFile(Copy $copy, $sourceFile, $fileHash) + { + $targetFile = $this->file->buildTargetFileName($sourceFile, $this->targetFilenameTemplate); + $hashFile = $targetFile . '.hash'; + + $result = $copy->copyContent($hashFile, $fileHash, $this->targetDirectory, $this->copyCommandTemplate); + + if ($result->getExitCode() == 0) { + $this->dao->markHashFileCreated($fileHash); + } + } + + private function copyFile(Copy $copy, $sourceFile, $fileHash) + { + $fileSize = $this->file->getSize($sourceFile); + $targetFile = $this->file->buildTargetFilePath($sourceFile, $this->targetDirectory, $this->targetFilenameTemplate); + + $id = $this->dao->logFileSyncStart($sourceFile, $targetFile, $fileHash, $fileSize, Date::now()->getDatetime()); + + $startTime = microtime(true); + $result = $copy->copy($sourceFile, $targetFile, $this->copyCommandTemplate); + + $durationInMs = (microtime(true) - $startTime) * 1000; + + $this->dao->logFileSyncFinished($id, $result->getCommand(), $result->getOutput(), $result->getExitCode(), + $durationInMs, Date::now()->getDatetime()); + } +} diff --git a/plugins/FileSynchronizer/SyncFiles/Copy.php b/plugins/FileSynchronizer/SyncFiles/Copy.php new file mode 100644 index 0000000000..8d3ffc1b75 --- /dev/null +++ b/plugins/FileSynchronizer/SyncFiles/Copy.php @@ -0,0 +1,89 @@ +<?php +/** + * Copyright (C) Piwik PRO - All rights reserved. + * + * Using this code requires that you first get a license from Piwik PRO. + * Unauthorized copying of this file, via any medium is strictly prohibited. + * + * @link http://piwik.pro + */ + +namespace Piwik\Plugins\FileSynchronizer\SyncFiles; + +use Piwik\Container\StaticContainer; +use Piwik\Db; +use Piwik\Filechecks; +use Piwik\Filesystem; +use Psr\Log\LoggerInterface; + +class Copy +{ + /** + * @var LoggerInterface + */ + private $logger; + + public function __construct(LoggerInterface $logger) + { + $this->logger = $logger; + } + + /** + * Copies the given source file to the target directory. + * + * @param string $sourceFile + * @param string $targetDir + * @param string $copyCommandTemplate eg 'cp $source $directory' or 'scp $source user@host:$target' + * @return Result + */ + public function copy($sourceFile, $targetDir, $copyCommandTemplate) + { + $command = $this->buildCommandToCopyFile($sourceFile, $targetDir, $copyCommandTemplate); + + $this->logger->debug("Executing command '$command' to copy '$sourceFile' to '$targetDir'"); + + try { + exec($command . ' 2>&1', $output, $exitCode); + $output = implode("\n", $output); + } catch (\Exception $e) { + $output = $e->getMessage() . ' Trace: ' . $e->getTraceAsString(); + $exitCode = 255; + } + + $this->logger->debug("Finished copying '$sourceFile'. Exit code: '$exitCode', output: '$output'"); + + return new Result($command, $output, $exitCode); + } + + /** + * Similar to {@link copy()} but should be used if there is no file that can be copied but content instead. + * + * @param string $filename The name of the file that shall be created in the target directory. eg 'test.hash' + * @param string $content The content of the file, eg '123456' + * @param string $targetDir + * @param string $copyCommandTemplate + * @return Result + */ + public function copyContent($filename, $content, $targetDir, $copyCommandTemplate) + { + $tmp = StaticContainer::get('path.tmp'); + Filechecks::dieIfDirectoriesNotWritable(array($tmp)); + + $file = $tmp . DIRECTORY_SEPARATOR . $filename; + file_put_contents($file, $content); + + $result = $this->copy($file, $targetDir, $copyCommandTemplate); + + Filesystem::remove($file); + + return $result; + } + + private function buildCommandToCopyFile($sourceFile, $targetDir, $copyCommandTemplate) + { + $search = array('$source', '$target'); + $replace = array(escapeshellarg($sourceFile), escapeshellarg($targetDir)); + + return str_replace($search, $replace, $copyCommandTemplate); + } +} diff --git a/plugins/FileSynchronizer/SyncFiles/File.php b/plugins/FileSynchronizer/SyncFiles/File.php new file mode 100644 index 0000000000..6256707ee0 --- /dev/null +++ b/plugins/FileSynchronizer/SyncFiles/File.php @@ -0,0 +1,45 @@ +<?php +/** + * Copyright (C) Piwik PRO - All rights reserved. + * + * Using this code requires that you first get a license from Piwik PRO. + * Unauthorized copying of this file, via any medium is strictly prohibited. + * + * @link http://piwik.pro + */ + +namespace Piwik\Plugins\FileSynchronizer\SyncFiles; + +class File +{ + public function getSize($file) + { + return filesize($file); + } + + public function getHash($file) + { + return md5_file($file); + } + + public function buildTargetFileName($sourceFile, $targetFilenameTemplate) + { + $basename = basename($sourceFile); + $extension = pathinfo($sourceFile, PATHINFO_EXTENSION); + $filename = pathinfo($sourceFile, PATHINFO_FILENAME); + + $search = array('$basename', '$filename', '$extension'); + $replace = array($basename, $filename, $extension); + + $targetBasename = str_replace($search, $replace, $targetFilenameTemplate); + + return $targetBasename; + } + + public function buildTargetFilePath($sourceFile, $targetDirectory, $targetFilenameTemplate) + { + $targetBasename = $this->buildTargetFileName($sourceFile, $targetFilenameTemplate); + + return $targetDirectory . DIRECTORY_SEPARATOR . $targetBasename; + } +} diff --git a/plugins/FileSynchronizer/SyncFiles/FileList.php b/plugins/FileSynchronizer/SyncFiles/FileList.php new file mode 100644 index 0000000000..aa3a7b8dfb --- /dev/null +++ b/plugins/FileSynchronizer/SyncFiles/FileList.php @@ -0,0 +1,52 @@ +<?php +/** + * Copyright (C) Piwik PRO - All rights reserved. + * + * Using this code requires that you first get a license from Piwik PRO. + * Unauthorized copying of this file, via any medium is strictly prohibited. + * + * @link http://piwik.pro + */ + +namespace Piwik\Plugins\FileSynchronizer\SyncFiles; + +use Piwik\Db; +use Piwik\Filesystem; +use Piwik\Plugins\FileSynchronizer\Settings; + +class FileList +{ + /** + * @var Settings + */ + private $settings; + + public function __construct(Settings $settings) + { + $this->settings = $settings; + } + + /** + * @return string[] + */ + public function findFilesToSync() + { + $path = $this->settings->sourceDirectory->getValue(); + $pattern = $this->settings->filePattern->getValue(); + + if (!$this->settings->enabled->getValue()) { + return array(); + } + + if (empty($path) || !file_exists($path) || !is_readable($path) || !is_dir($path)) { + // case when plugin not configured yet + return array(); + } + + if (empty($pattern)) { + $pattern = '*'; + } + + return Filesystem::globr($path, $pattern); + } +} diff --git a/plugins/FileSynchronizer/SyncFiles/Result.php b/plugins/FileSynchronizer/SyncFiles/Result.php new file mode 100644 index 0000000000..1b01621063 --- /dev/null +++ b/plugins/FileSynchronizer/SyncFiles/Result.php @@ -0,0 +1,51 @@ +<?php +/** + * Copyright (C) Piwik PRO - All rights reserved. + * + * Using this code requires that you first get a license from Piwik PRO. + * Unauthorized copying of this file, via any medium is strictly prohibited. + * + * @link http://piwik.pro + */ + +namespace Piwik\Plugins\FileSynchronizer\SyncFiles; + +use Piwik\Db; + +class Result +{ + private $command; + private $output; + private $exitCode; + + public function __construct($command, $output, $exitCode) + { + $this->command = $command; + $this->output = $output; + $this->exitCode = $exitCode; + } + + /** + * @return string + */ + public function getCommand() + { + return $this->command; + } + + /** + * @return string + */ + public function getOutput() + { + return $this->output; + } + + /** + * @return string + */ + public function getExitCode() + { + return $this->exitCode; + } +} diff --git a/plugins/FileSynchronizer/Tasks.php b/plugins/FileSynchronizer/Tasks.php new file mode 100644 index 0000000000..b269ef691c --- /dev/null +++ b/plugins/FileSynchronizer/Tasks.php @@ -0,0 +1,46 @@ +<?php +/** + * Copyright (C) Piwik PRO - All rights reserved. + * + * Using this code requires that you first get a license from Piwik PRO. + * Unauthorized copying of this file, via any medium is strictly prohibited. + * + * @link http://piwik.pro + */ + +namespace Piwik\Plugins\FileSynchronizer; + +use Piwik\Container\StaticContainer; +use Piwik\Settings\Setting; + +class Tasks extends \Piwik\Plugin\Tasks +{ + public function schedule() + { + $this->hourly('syncFiles'); + } + + public function syncFiles() + { + $settings = StaticContainer::get('Piwik\Plugins\FileSynchronizer\Settings'); + + if (!$settings->enabled->getValue()) { + return; + } + + // we validate to make sure it is still readable etc. + $this->validateSetting($settings->copyCommandTemplate); + $this->validateSetting($settings->sourceDirectory); + $this->validateSetting($settings->targetDirectory); + + $sync = StaticContainer::get('Piwik\Plugins\FileSynchronizer\SyncFiles'); + $sync->sync(); + } + + private function validateSetting(Setting $setting) + { + if (isset($setting->validate)) { + call_user_func($setting->validate, $setting->getValue(), $setting); + } + } +} diff --git a/plugins/FileSynchronizer/config/config.php b/plugins/FileSynchronizer/config/config.php new file mode 100644 index 0000000000..c2590057ec --- /dev/null +++ b/plugins/FileSynchronizer/config/config.php @@ -0,0 +1,8 @@ +<?php + +return array( + + 'FileSynchronizer.currentTimestamp' => DI\factory(function () { + return \Piwik\Date::now()->getTimestamp(); + }), +); diff --git a/plugins/FileSynchronizer/config/test.php b/plugins/FileSynchronizer/config/test.php new file mode 100644 index 0000000000..98de70b86e --- /dev/null +++ b/plugins/FileSynchronizer/config/test.php @@ -0,0 +1,22 @@ +<?php + +return array( + + 'Piwik\Plugins\FileSynchronizer\Settings' => DI\decorate(function (\Piwik\Plugins\FileSynchronizer\Settings $settings) { + if ($settings->enabled->isWritableByCurrentUser()) { + $path = PIWIK_INCLUDE_PATH; + if (\Piwik\Common::stringEndsWith($path, '/')) { + $path = substr($path, 0, -1); + } + $settings->enabled->setValue(true); + $settings->filePattern->setValue('*.log'); + $settings->sourceDirectory->setValue($path . '/plugins/FileSynchronizer/tests/resources/source'); + $settings->targetDirectory->setValue($path . '/plugins/FileSynchronizer/tests/resources/target'); + $settings->targetFilenameTemplate->setValue('$basename_$filename_idsite_1_.$extension'); + } + + return $settings; + }), + + 'FileSynchronizer.currentTimestamp' => '1440592473' +); diff --git a/plugins/FileSynchronizer/lang/en.json b/plugins/FileSynchronizer/lang/en.json new file mode 100644 index 0000000000..f6801c25eb --- /dev/null +++ b/plugins/FileSynchronizer/lang/en.json @@ -0,0 +1,25 @@ +{ + "FileSynchronizer": { + "SynchronizingFiles": "Synchronizing files", + "ToConfigurePluginGoToSettings": "To configure this plugin go to %s%s%s.", + "ScheduledFiles": "Files that will be synchronized next", + "ScheduledFilesDescription": "These files should be synced as soon as the sync task runs the next time.", + "SynchronizedFiles": "Synchronized files", + "SynchronizedFilesDescription": "The following table shows the latest %d synchronized files.", + "NoSynchronizedFile": "No file has been synchronized yet.", + "NoSynchronizingFile": "No file is currently being synchronized.", + "NoScheduledFiles": "No file is scheduled to be synced.", + "File": "File", + "FileSize": "File size", + "Synced": "Synced", + "StartDate": "Start Date", + "TimeTaken": "Time taken", + "InProcessSince": "In process since", + "SyncDetails": "Exit code: '%s', output: '%s'", + "HashFileCreated": "Hash file was successfully created", + "HashFileNotCreated": "Hash file not created", + "HashFile": "Hash file", + "ExitCode": "Exit Code", + "FileDetails": "'%s' to '%s' via '%s'. Hash of file is '%s'" + } +}
\ No newline at end of file diff --git a/plugins/FileSynchronizer/plugin.json b/plugins/FileSynchronizer/plugin.json new file mode 100644 index 0000000000..d677352462 --- /dev/null +++ b/plugins/FileSynchronizer/plugin.json @@ -0,0 +1,16 @@ +{ + "name": "FileSynchronizer", + "version": "0.1.0", + "description": "Synchronizes any files of a directory with another directory.", + "theme": false, + "require": { + "piwik": ">=2.15.0-b3" + }, + "authors": [ + { + "name": "Piwik", + "email": "", + "homepage": "" + } + ] +}
\ No newline at end of file diff --git a/plugins/FileSynchronizer/screenshots/.gitkeep b/plugins/FileSynchronizer/screenshots/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/plugins/FileSynchronizer/screenshots/.gitkeep diff --git a/plugins/FileSynchronizer/screenshots/Error_Sync_Details.png b/plugins/FileSynchronizer/screenshots/Error_Sync_Details.png Binary files differnew file mode 100644 index 0000000000..faf7fcb196 --- /dev/null +++ b/plugins/FileSynchronizer/screenshots/Error_Sync_Details.png diff --git a/plugins/FileSynchronizer/screenshots/Plugin_Settings.png b/plugins/FileSynchronizer/screenshots/Plugin_Settings.png Binary files differnew file mode 100644 index 0000000000..b44d51486b --- /dev/null +++ b/plugins/FileSynchronizer/screenshots/Plugin_Settings.png diff --git a/plugins/FileSynchronizer/screenshots/Status_Report.png b/plugins/FileSynchronizer/screenshots/Status_Report.png Binary files differnew file mode 100644 index 0000000000..50a0b7e88a --- /dev/null +++ b/plugins/FileSynchronizer/screenshots/Status_Report.png diff --git a/plugins/FileSynchronizer/stylesheets/status.less b/plugins/FileSynchronizer/stylesheets/status.less new file mode 100644 index 0000000000..b197666db5 --- /dev/null +++ b/plugins/FileSynchronizer/stylesheets/status.less @@ -0,0 +1,23 @@ +#listSyncedFiles { + table { + max-width: 1000px; + min-width: 600px; + } + + .date_field { + white-space: nowrap + } + + .filename { + max-width: 300px; + word-break: break-all; + } + + .icon-success { + color: green; + } + + .icon-error { + color: #D4291F; + } +}
\ No newline at end of file diff --git a/plugins/FileSynchronizer/templates/status.twig b/plugins/FileSynchronizer/templates/status.twig new file mode 100644 index 0000000000..abbe8e5d78 --- /dev/null +++ b/plugins/FileSynchronizer/templates/status.twig @@ -0,0 +1,100 @@ +{% extends 'admin.twig' %} + +{% block content %} +<h2>{{ 'FileSynchronizer_SynchronizingFiles'|translate }}</h2> +<p>{{ 'FileSynchronizer_ToConfigurePluginGoToSettings'|translate('<a href="' ~ linkTo({'module': 'CoreAdminHome', 'action': 'adminPluginSettings'}) ~ '#FileSynchronizer">', ('CoreAdminHome_PluginSettings'|translate), '</a>')|raw }}</p> +<div id="listSyncedFiles"> + <table class="entityTable dataTable"> + <thead> + <tr> + <th class="filename">{{ 'FileSynchronizer_File'|translate }}</th> + <th>{{ 'FileSynchronizer_StartDate'|translate }}</th> + <th>{{ 'FileSynchronizer_InProcessSince'|translate }}</th> + <th>{{ 'FileSynchronizer_FileSize'|translate }}</th> + </tr> + </thead> + <tbody> + {% if syncingFiles|length %} + {% for syncingFile in syncingFiles %} + <tr> + <td class="filename" title="{{ 'FileSynchronizer_FileDetails'|translate(syncingFile.source, syncingFile.target, syncingFile.command, syncingFile.file_hash) }}">{{ syncingFile.source_filename }}</td> + <td class="date_field">{{ syncingFile.start_date }}</td> + <td>{{ syncingFile.time_ago }}</td> + <td>{{ syncingFile.size_human }}</td> + </tr> + {% endfor %} + {% else %} + <tr><td colspan="6">{{ 'FileSynchronizer_NoSynchronizingFile'|translate }}</td></tr> + {% endif %} + </tbody> + </table> +</div> + +<h2>{{ 'FileSynchronizer_ScheduledFiles'|translate }}</h2> + +<p>{{ 'FileSynchronizer_ScheduledFilesDescription'|translate }}</p> + +<div id="listSyncingFiles"> + <table class="entityTable dataTable"> + <thead> + <tr> + <th class="filename">{{ 'FileSynchronizer_File'|translate }}</th> + </tr> + </thead> + <tbody> + {% if scheduledFiles|length %} + {% for hash, scheduledFile in scheduledFiles %} + <tr> + <td>{{ scheduledFile }}</td> + </tr> + {% endfor %} + {% else %} + <tr><td colspan="5">{{ 'FileSynchronizer_NoScheduledFiles'|translate }}</td></tr> + {% endif %} + </tbody> + </table> +</div> + +<h2>{{ 'FileSynchronizer_SynchronizedFiles'|translate }}</h2> +<p>{{ 'FileSynchronizer_SynchronizedFilesDescription'|translate(limit) }}</p> +<div id="listSyncedFiles"> + <table class="entityTable dataTable"> + <thead> + <tr> + <th class="filename">{{ 'FileSynchronizer_File'|translate }}</th> + <th>{{ 'General_Date'|translate }}</th> + <th>{{ 'FileSynchronizer_FileSize'|translate }}</th> + <th>{{ 'FileSynchronizer_TimeTaken'|translate }}</th> + <th>{{ 'FileSynchronizer_Synced'|translate }}</th> + <th>{{ 'FileSynchronizer_HashFile'|translate }}</th> + </tr> + </thead> + <tbody> + {% if syncedFiles|length %} + {% for syncedFile in syncedFiles %} + <tr> + <td class="filename" title="{{ 'FileSynchronizer_FileDetails'|translate(syncedFile.source, syncedFile.target, syncedFile.command, syncedFile.file_hash) }}">{{ syncedFile.source_filename }}</td> + <td class="date_field" title="{{ syncedFile.start_date|e('html_attr') }} - {{ syncedFile.end_date|e('html_attr') }}">{{ syncedFile.date }}</td> + <td>{{ syncedFile.size_human }}</td> + <td>{{ syncedFile.duration }}</td> + <td title="{{ 'FileSynchronizer_SyncDetails'|translate(syncedFile.exit_code, syncedFile.output)|e('html_attr') }}"> + {% if syncedFile.exit_code == 0 %} + <span class="icon-success"></span> + {% else %} + <span class="icon-error"></span> + {% endif %} + </td> + {% if syncedFile.hash_file_created == 0 %} + <td title="{{ 'FileSynchronizer_HashFileNotCreated'|translate }}"><span class="icon-error"></span></td> + {% else %} + <td title="{{ 'FileSynchronizer_HashFileCreated'|translate }}"><span class="icon-success"></span></td> + {% endif %} + </tr> + {% endfor %} + {% else %} + <tr><td colspan="6">{{ 'FileSynchronizer_NoSynchronizedFile'|translate }}</td></tr> + {% endif %} + </tbody> + </table> +</div> +{% endblock %}
\ No newline at end of file diff --git a/plugins/Funnel b/plugins/Funnel new file mode 160000 +Subproject 363cc5b7a888e8be97d4b5b9fb62fe45df57c86 diff --git a/plugins/InterSites b/plugins/InterSites new file mode 160000 +Subproject e2268b0beac1d0bacfb24794ee4be94d7ea7ebd diff --git a/plugins/LogViewer b/plugins/LogViewer -Subproject e51a8fd6b3d4d133b9588caf3971d44bcad85be +Subproject 345dbecb786de105df7ee190be8282706de411d diff --git a/plugins/LoginHttpAuth b/plugins/LoginHttpAuth new file mode 160000 +Subproject 9044930813f45882eddd8e80878417d0a10b266 diff --git a/plugins/MediaPlayer b/plugins/MediaPlayer new file mode 160000 +Subproject e837d26f1f9140b146470d53c5fc2b979d928c7 diff --git a/plugins/MetaSites b/plugins/MetaSites new file mode 160000 +Subproject 20c5cc14ab28c4cef06496dd2a17fec503a1bb7 diff --git a/plugins/RawDataExporter b/plugins/RawDataExporter new file mode 160000 +Subproject 459375ff10a7253963af7c8432156e45ef10cfc diff --git a/plugins/SimplePageBuilder b/plugins/SimplePageBuilder new file mode 160000 +Subproject ea18cae3b7fbc29605717afb3af72cef4c646f8 diff --git a/plugins/SiteMigration b/plugins/SiteMigration new file mode 160000 +Subproject bc7f03f6ea2c2e173bba4c501ce29e3b13bb878 diff --git a/plugins/TrafficMonitor b/plugins/TrafficMonitor new file mode 160000 +Subproject 4cf14c91afafc15c2ff578651d5714c6da67d11 diff --git a/plugins/WebsiteGroups b/plugins/WebsiteGroups new file mode 160000 +Subproject 74cad4473ec63ccc5721d28d3a62dc4beb6b483 diff --git a/plugins/WhiteLabel b/plugins/WhiteLabel new file mode 160000 +Subproject 0c0a6dc8701b56ef92b5448766af68839d58275 diff --git a/plugins/WhitelistAdmin b/plugins/WhitelistAdmin new file mode 160000 +Subproject 53b5f26133ece499f427ccc70d9ea1c5483fb9b |
