From 48c9f2c676b4178d8526197af0059bdd7a0f6a72 Mon Sep 17 00:00:00 2001 From: Thomas Steur Date: Fri, 16 Aug 2019 14:47:33 +1200 Subject: Faster segment archiving (#14761) * use temporary table when segmenting * still apply original where * do not apply where twice * make sure to sort tables correctly * performance tweak, no longer a where needed in most queries * fix notice * have a better cache per date * small tweaks * potentially fix an error where we received too many rows * do not apply any custom force group by * better handling of log tables to fix some tests * enable new feature through config * remove comment * fix some system tests * drop table before starting to archive in case archiving fails * fix more tests * trying to fix ui test * save the config change * use reader when executing ranking queries * apply review feedback * drop table only if it actually exists to prevent problems on a reader * Update JoinGenerator.php --- ...ntInteractionMatch__Actions.getPageUrls_day.xml | 31 +++++++++- ...eractionMatch__Contents.getContentNames_day.xml | 33 +++++++++- ...ractionMatch__Contents.getContentPieces_day.xml | 72 +++++++++++++++++++++- ...contentTargetMatch__Actions.getPageUrls_day.xml | 31 +++++++++- ...ntTargetMatch__Contents.getContentNames_day.xml | 28 +++++++-- ...tTargetMatch__Contents.getContentPieces_day.xml | 49 +++++++++++++++ ...ntNameOrPieceMatch__Actions.getPageUrls_day.xml | 31 +++++++++- ...eOrPieceMatch__Contents.getContentNames_day.xml | 22 +++++-- ...OrPieceMatch__Contents.getContentPieces_day.xml | 20 ++++++ 9 files changed, 302 insertions(+), 15 deletions(-) mode change 100644 => 100755 plugins/Contents/tests/System/expected/test_Contents_contentInteractionMatch__Actions.getPageUrls_day.xml mode change 100644 => 100755 plugins/Contents/tests/System/expected/test_Contents_contentInteractionMatch__Contents.getContentNames_day.xml mode change 100644 => 100755 plugins/Contents/tests/System/expected/test_Contents_contentInteractionMatch__Contents.getContentPieces_day.xml mode change 100644 => 100755 plugins/Contents/tests/System/expected/test_Contents_contentTargetMatch__Actions.getPageUrls_day.xml mode change 100644 => 100755 plugins/Contents/tests/System/expected/test_Contents_contentTargetMatch__Contents.getContentNames_day.xml mode change 100644 => 100755 plugins/Contents/tests/System/expected/test_Contents_contentTargetMatch__Contents.getContentPieces_day.xml mode change 100644 => 100755 plugins/Contents/tests/System/expected/test_ContentscontentNameOrPieceMatch__Actions.getPageUrls_day.xml mode change 100644 => 100755 plugins/Contents/tests/System/expected/test_ContentscontentNameOrPieceMatch__Contents.getContentNames_day.xml mode change 100644 => 100755 plugins/Contents/tests/System/expected/test_ContentscontentNameOrPieceMatch__Contents.getContentPieces_day.xml (limited to 'plugins/Contents') diff --git a/plugins/Contents/tests/System/expected/test_Contents_contentInteractionMatch__Actions.getPageUrls_day.xml b/plugins/Contents/tests/System/expected/test_Contents_contentInteractionMatch__Actions.getPageUrls_day.xml old mode 100644 new mode 100755 index c234bed59e..97b0082789 --- a/plugins/Contents/tests/System/expected/test_Contents_contentInteractionMatch__Actions.getPageUrls_day.xml +++ b/plugins/Contents/tests/System/expected/test_Contents_contentInteractionMatch__Actions.getPageUrls_day.xml @@ -1,2 +1,31 @@ - \ No newline at end of file + + + + 2 + 2 + 2 + 540 + 2 + 0.333 + 0.333 + 0 + 0 + + + 2 + 2 + 2 + 542 + 2 + 2 + 2 + 0 + 270 + 100% + 100% + 0.333 + http://www.example.org/page + pageUrl==http%3A%2F%2Fwww.example.org%2Fpage + + \ No newline at end of file diff --git a/plugins/Contents/tests/System/expected/test_Contents_contentInteractionMatch__Contents.getContentNames_day.xml b/plugins/Contents/tests/System/expected/test_Contents_contentInteractionMatch__Contents.getContentNames_day.xml old mode 100644 new mode 100755 index c234bed59e..aab05b8c9c --- a/plugins/Contents/tests/System/expected/test_Contents_contentInteractionMatch__Contents.getContentNames_day.xml +++ b/plugins/Contents/tests/System/expected/test_Contents_contentInteractionMatch__Contents.getContentNames_day.xml @@ -1,2 +1,33 @@ - \ No newline at end of file + + + + 8 + 8 + 8 + 2 + 25% + http://www.example.com + contentName==ImageAd + + + + 6 + 6 + 6 + 4 + 66.67% + http://piwik.org/ + contentName==Text+Ad + + + + 2 + 2 + 4 + 0 + 0% + + contentName==Video+Ad + + \ No newline at end of file diff --git a/plugins/Contents/tests/System/expected/test_Contents_contentInteractionMatch__Contents.getContentPieces_day.xml b/plugins/Contents/tests/System/expected/test_Contents_contentInteractionMatch__Contents.getContentPieces_day.xml old mode 100644 new mode 100755 index c234bed59e..ccdd21b1d9 --- a/plugins/Contents/tests/System/expected/test_Contents_contentInteractionMatch__Contents.getContentPieces_day.xml +++ b/plugins/Contents/tests/System/expected/test_Contents_contentInteractionMatch__Contents.getContentPieces_day.xml @@ -1,2 +1,72 @@ - \ No newline at end of file + + + + 4 + 4 + 4 + 2 + 50% + http://piwik.org/download + contentPiece==Click+to+download+Piwik+now + + + + 2 + 2 + 2 + 2 + 100% + http://www.example.com + contentPiece==%2Fpath%2Fad.jpg + + + + 2 + 2 + 2 + 0 + 0% + http://www.example.com + contentPiece==%2Fpath%2Fad2.jpg + + + + 2 + 2 + 2 + 2 + 100% + http://piwik.org/ + contentPiece==Click+NOW + + + + 2 + 2 + 2 + 0 + 0% + + + + + 2 + 2 + 4 + 0 + 0% + + contentPiece==movie.mov + + + + 2 + 2 + 2 + 0 + 0% + + contentPiece==Unknown + + \ No newline at end of file diff --git a/plugins/Contents/tests/System/expected/test_Contents_contentTargetMatch__Actions.getPageUrls_day.xml b/plugins/Contents/tests/System/expected/test_Contents_contentTargetMatch__Actions.getPageUrls_day.xml old mode 100644 new mode 100755 index c234bed59e..97b0082789 --- a/plugins/Contents/tests/System/expected/test_Contents_contentTargetMatch__Actions.getPageUrls_day.xml +++ b/plugins/Contents/tests/System/expected/test_Contents_contentTargetMatch__Actions.getPageUrls_day.xml @@ -1,2 +1,31 @@ - \ No newline at end of file + + + + 2 + 2 + 2 + 540 + 2 + 0.333 + 0.333 + 0 + 0 + + + 2 + 2 + 2 + 542 + 2 + 2 + 2 + 0 + 270 + 100% + 100% + 0.333 + http://www.example.org/page + pageUrl==http%3A%2F%2Fwww.example.org%2Fpage + + \ No newline at end of file diff --git a/plugins/Contents/tests/System/expected/test_Contents_contentTargetMatch__Contents.getContentNames_day.xml b/plugins/Contents/tests/System/expected/test_Contents_contentTargetMatch__Contents.getContentNames_day.xml old mode 100644 new mode 100755 index f0c23afff4..aab05b8c9c --- a/plugins/Contents/tests/System/expected/test_Contents_contentTargetMatch__Contents.getContentNames_day.xml +++ b/plugins/Contents/tests/System/expected/test_Contents_contentTargetMatch__Contents.getContentNames_day.xml @@ -2,12 +2,32 @@ - 4 - 4 - 4 + 8 + 8 + 8 2 - 50% + 25% http://www.example.com contentName==ImageAd + + + 6 + 6 + 6 + 4 + 66.67% + http://piwik.org/ + contentName==Text+Ad + + + + 2 + 2 + 4 + 0 + 0% + + contentName==Video+Ad + \ No newline at end of file diff --git a/plugins/Contents/tests/System/expected/test_Contents_contentTargetMatch__Contents.getContentPieces_day.xml b/plugins/Contents/tests/System/expected/test_Contents_contentTargetMatch__Contents.getContentPieces_day.xml old mode 100644 new mode 100755 index ae70087a09..ccdd21b1d9 --- a/plugins/Contents/tests/System/expected/test_Contents_contentTargetMatch__Contents.getContentPieces_day.xml +++ b/plugins/Contents/tests/System/expected/test_Contents_contentTargetMatch__Contents.getContentPieces_day.xml @@ -1,5 +1,15 @@ + + + 4 + 4 + 4 + 2 + 50% + http://piwik.org/download + contentPiece==Click+to+download+Piwik+now + 2 @@ -20,4 +30,43 @@ http://www.example.com contentPiece==%2Fpath%2Fad2.jpg + + + 2 + 2 + 2 + 2 + 100% + http://piwik.org/ + contentPiece==Click+NOW + + + + 2 + 2 + 2 + 0 + 0% + + + + + 2 + 2 + 4 + 0 + 0% + + contentPiece==movie.mov + + + + 2 + 2 + 2 + 0 + 0% + + contentPiece==Unknown + \ No newline at end of file diff --git a/plugins/Contents/tests/System/expected/test_ContentscontentNameOrPieceMatch__Actions.getPageUrls_day.xml b/plugins/Contents/tests/System/expected/test_ContentscontentNameOrPieceMatch__Actions.getPageUrls_day.xml old mode 100644 new mode 100755 index c234bed59e..97b0082789 --- a/plugins/Contents/tests/System/expected/test_ContentscontentNameOrPieceMatch__Actions.getPageUrls_day.xml +++ b/plugins/Contents/tests/System/expected/test_ContentscontentNameOrPieceMatch__Actions.getPageUrls_day.xml @@ -1,2 +1,31 @@ - \ No newline at end of file + + + + 2 + 2 + 2 + 540 + 2 + 0.333 + 0.333 + 0 + 0 + + + 2 + 2 + 2 + 542 + 2 + 2 + 2 + 0 + 270 + 100% + 100% + 0.333 + http://www.example.org/page + pageUrl==http%3A%2F%2Fwww.example.org%2Fpage + + \ No newline at end of file diff --git a/plugins/Contents/tests/System/expected/test_ContentscontentNameOrPieceMatch__Contents.getContentNames_day.xml b/plugins/Contents/tests/System/expected/test_ContentscontentNameOrPieceMatch__Contents.getContentNames_day.xml old mode 100644 new mode 100755 index 1f75bfaa1b..aab05b8c9c --- a/plugins/Contents/tests/System/expected/test_ContentscontentNameOrPieceMatch__Contents.getContentNames_day.xml +++ b/plugins/Contents/tests/System/expected/test_ContentscontentNameOrPieceMatch__Contents.getContentNames_day.xml @@ -12,12 +12,22 @@ - 4 - 4 - 4 - 2 - 50% - http://piwik.org/download + 6 + 6 + 6 + 4 + 66.67% + http://piwik.org/ contentName==Text+Ad + + + 2 + 2 + 4 + 0 + 0% + + contentName==Video+Ad + \ No newline at end of file diff --git a/plugins/Contents/tests/System/expected/test_ContentscontentNameOrPieceMatch__Contents.getContentPieces_day.xml b/plugins/Contents/tests/System/expected/test_ContentscontentNameOrPieceMatch__Contents.getContentPieces_day.xml old mode 100644 new mode 100755 index 98fbf6ad4a..ccdd21b1d9 --- a/plugins/Contents/tests/System/expected/test_ContentscontentNameOrPieceMatch__Contents.getContentPieces_day.xml +++ b/plugins/Contents/tests/System/expected/test_ContentscontentNameOrPieceMatch__Contents.getContentPieces_day.xml @@ -30,6 +30,16 @@ http://www.example.com contentPiece==%2Fpath%2Fad2.jpg + + + 2 + 2 + 2 + 2 + 100% + http://piwik.org/ + contentPiece==Click+NOW + 2 @@ -39,6 +49,16 @@ 0% + + + 2 + 2 + 4 + 0 + 0% + + contentPiece==movie.mov + 2 -- cgit v1.2.3