Refs #4878, create update script for converting *_returning metrics to new segmented metrics. Removed Archiver and API. Old segmented VisitFrequency metrics will not be accessible.

8 files changed, 215 insertions, 217 deletions

diff --git a/core/ArchiveProcessor/Rules.php b/core/ArchiveProcessor/Rules.php
index 02a8357ac6..13f2a34610 100644
--- a/core/ArchiveProcessor/Rules.php
+++ b/core/ArchiveProcessor/Rules.php
@@ -98,7 +98,7 @@ class Rules
         return $segmentsToProcess;
     }
 
-    private static function getDoneFlagArchiveContainsOnePlugin(Segment $segment, $plugin, $isSkipAggregationOfSubTables = false)
+    public static function getDoneFlagArchiveContainsOnePlugin(Segment $segment, $plugin, $isSkipAggregationOfSubTables = false)
     {
         $partial = self::isFlagArchivePartial($plugin, $isSkipAggregationOfSubTables);
         return 'done' . $segment->getHash() . '.' . $plugin . $partial ;
diff --git a/core/Updater.php b/core/Updater.php
index 59587cb19b..a1f39fb71b 100644
--- a/core/Updater.php
+++ b/core/Updater.php
@@ -287,16 +287,41 @@ class Updater
     static function updateDatabase($file, $sqlarray)
     {
         foreach ($sqlarray as $update => $ignoreError) {
-            try {
-                Db::exec($update);
-            } catch (\Exception $e) {
-                if (($ignoreError === false)
-                    || !Db::get()->isErrNo($e, $ignoreError)
-                ) {
-                    $message = $file . ":\nError trying to execute the query '" . $update . "'.\nThe error was: " . $e->getMessage();
-                    throw new UpdaterErrorException($message);
-                }
-            }
+            self::executeMigrationQuery($update, $ignoreError, $file);
+        }
+    }
+
+    /**
+     * Executes a database update query.
+     *
+     * @param string $updateSql Update SQL query.
+     * @param int|false $errorToIgnore A MySQL error code to ignore.
+     * @param string $file The Update file that's calling this method.
+     */
+    public static function executeMigrationQuery($updateSql, $errorToIgnore, $file)
+    {
+        try {
+            Db::exec($updateSql);
+        } catch (\Exception $e) {
+            self::handleQueryError($e, $updateSql, $errorToIgnore, $file);
+        }
+    }
+
+    /**
+     * Handle an error that is thrown from a database query.
+     *
+     * @param \Exception $e the exception thrown.
+     * @param string $updateSql Update SQL query.
+     * @param int|false $errorToIgnore A MySQL error code to ignore.
+     * @param string $file The Update file that's calling this method.
+     */
+    public static function handleQueryError($e, $updateSql, $errorToIgnore, $file)
+    {
+        if (($errorToIgnore === false)
+            || !Db::get()->isErrNo($e, $errorToIgnore)
+        ) {
+            $message = $file . ":\nError trying to execute the query '" . $updateSql . "'.\nThe error was: " . $e->getMessage();
+            throw new UpdaterErrorException($message);
         }
     }
 }
diff --git a/core/Updates/2.1.1-b11.php b/core/Updates/2.1.1-b11.php
new file mode 100644
index 0000000000..b38f92e331
--- /dev/null
+++ b/core/Updates/2.1.1-b11.php
@@ -0,0 +1,127 @@
+<?php
+/**
+ * Piwik - Open source web analytics
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+namespace Piwik\Updates;
+
+use Piwik\Db;
+use Piwik\Updates;
+use Piwik\Updater;
+use Piwik\Date;
+use Piwik\Segment;
+use Piwik\ArchiveProcessor\Rules;
+use Piwik\Db\BatchInsert;
+use Piwik\DataAccess\ArchiveWriter;
+use Piwik\Plugins\VisitFrequency\API as VisitFrequencyApi;
+
+/**
+ */
+class Updates_2_1_1_b11 extends Updates
+{
+    static function update()
+    {
+        $returningMetrics = array(
+            'nb_visits_returning',
+            'nb_actions_returning',
+            'max_actions_returning',
+            'sum_visit_length_returning',
+            'bounce_count_returning',
+            'nb_visits_converted_returning',
+            'nb_uniq_visitors_returning'
+        );
+
+        $now = Date::factory('now')->getDatetime();
+
+        $archiveNumericTables = Db::get()->fetchCol("SHOW TABLES LIKE '%archive_numeric%'");
+
+        // for each numeric archive table, copy *_returning metrics to VisitsSummary metrics w/ the appropriate
+        // returning visit segment
+        foreach ($archiveNumericTables as $table) {
+            // get archives w/ *._returning
+            $sql = "SELECT idarchive, idsite, period, date1, date2
+                      FROM $table
+                     WHERE name IN ('" . implode("','", $returningMetrics) . "')
+                  GROUP BY idarchive";
+            $idArchivesWithReturning = Db::fetchAll($sql);
+
+            // get archives for visitssummary returning visitor segment
+            $sql = "SELECT idarchive, idsite, period, date1, date2
+                      FROM $table
+                     WHERE name = ?
+                  GROUP BY idarchive";
+            $visitSummaryReturningSegmentDone = Rules::getDoneFlagArchiveContainsOnePlugin(
+                new Segment(VisitFrequencyApi::RETURNING_VISITOR_SEGMENT, $idSites = array()), 'VisitsSummary');
+            $idArchivesWithVisitReturningSegment = Db::fetchAll($sql, array($visitSummaryReturningSegmentDone));
+
+            // collect info for new visitssummary archives have to be created to match archives w/ *._returning
+            // metrics
+            $missingIdArchives = array();
+            $idArchiveMappings = array();
+            foreach ($idArchivesWithReturning as $row) {
+                $withMetricsIdArchive = $row['idarchive'];
+                foreach ($idArchivesWithVisitReturningSegment as $segmentRow) {
+                    if ($row['idsite'] == $segmentRow['idsite']
+                        && $row['period'] == $segmentRow['period']
+                        && $row['date1'] == $segmentRow['date1']
+                        && $row['date2'] == $segmentRow['date2']
+                    ) {
+                        $idArchiveMappings[$withMetricsIdArchive] = $segmentRow['idarchive'];
+                    }
+                }
+
+                if (!isset($idArchiveMappings[$withMetricsIdArchive])) {
+                    $missingIdArchives[$withMetricsIdArchive] = $row;
+                }
+            }
+
+            // if there are missing idarchives, fill out new archive row values
+            if (!empty($missingIdArchives)) {
+                $newIdArchiveStart = Db::fetchOne("SELECT MAX(idarchive) FROM $table") + 1;
+                foreach ($missingIdArchives as $withMetricsIdArchive => &$rowToInsert) {
+                    $idArchiveMappings[$withMetricsIdArchive] = $newIdArchiveStart;
+
+                    $rowToInsert['idarchive'] = $newIdArchiveStart;
+                    $rowToInsert['ts_archived'] = $now;
+                    $rowToInsert['name'] = $visitSummaryReturningSegmentDone;
+                    $rowToInsert['value'] = ArchiveWriter::DONE_OK;
+
+                    ++$newIdArchiveStart;
+                }
+
+                // add missing archives
+                try {
+                    BatchInsert::tableInsertBatch($table, array_keys(reset($missingIdArchives)), $missingIdArchives, $throwException = false);
+                } catch (\Exception $ex) {
+                    Updater::handleQueryError($ex, "<batch insert>", false, __FILE__);
+                }
+            }
+
+            // update idarchive & name columns in rows with *._returning metrics
+            $updateSqlPrefix = "UPDATE $table
+                                   SET idarchive = CASE idarchive ";
+            $updateSqlSuffix = " END, name = CASE name ";
+            foreach ($returningMetrics as $metric) {
+                $newMetricName = substr($metric, 0, strlen($metric) - strlen(VisitFrequencyApi::COLUMN_SUFFIX));
+                $updateSqlSuffix .= "WHEN '$metric' THEN '" . $newMetricName . "' ";
+            }
+            $updateSqlSuffix .= " END WHERE idarchive IN (" . implode(',', array_keys($idArchiveMappings)) . ")
+                                        AND name IN ('" . implode("','", $returningMetrics) . "')";
+
+            // update only 1000 rows at a time so we don't send too large an SQL query to MySQL
+            foreach (array_chunk($missingIdArchives, 1000, $preserveKeys = true) as $chunk) {
+                $idArchives = array();
+
+                $updateSql = $updateSqlPrefix;
+                foreach ($chunk as $withMetricsIdArchive => $row) {
+                    $updateSql .= "WHEN $withMetricsIdArchive THEN {$row['idarchive']} ";
+                }
+                $updateSql .= $updateSqlSuffix;
+
+                Updater::executeMigrationQuery($updateSql, false, __FILE__);
+            }
+        }
+    }
+}
diff --git a/core/Version.php b/core/Version.php
index 889a5cd065..fbe16426b5 100644
--- a/core/Version.php
+++ b/core/Version.php
@@ -21,5 +21,5 @@ final class Version
      * The current Piwik version.
      * @var string
      */
-    const VERSION = '2.1.1-b10';
+    const VERSION = '2.1.1-b11';
 }
diff --git a/plugins/CustomAlerts b/plugins/CustomAlerts
-Subproject a1125e96827e12c7bd5c3d6107b9c162aace32f
+Subproject c9edd879003ca7a9f75d81fe07ec6ff0c2340f9
diff --git a/plugins/Goals/Controller.php b/plugins/Goals/Controller.php
index ba8a68efed..2d64135948 100644
--- a/plugins/Goals/Controller.php
+++ b/plugins/Goals/Controller.php
@@ -145,7 +145,7 @@ class Controller extends \Piwik\Plugin\Controller
         $view->topDimensions = $this->getTopDimensions($idGoal);
 
         // conversion rate for new and returning visitors
-        $segment = urldecode(\Piwik\Plugins\VisitFrequency\Archiver::RETURNING_VISITOR_SEGMENT);
+        $segment = urldecode(\Piwik\Plugins\VisitFrequency\API::RETURNING_VISITOR_SEGMENT);
         $conversionRateReturning = API::getInstance()->getConversionRate($this->idSite, Common::getRequestVar('period'), Common::getRequestVar('date'), $segment, $idGoal);
         $view->conversion_rate_returning = $this->formatConversionRate($conversionRateReturning);
         $segment = 'visitorType==new';
diff --git a/plugins/VisitFrequency/API.php b/plugins/VisitFrequency/API.php
index 4c0f724398..e837fee01d 100644
--- a/plugins/VisitFrequency/API.php
+++ b/plugins/VisitFrequency/API.php
@@ -15,6 +15,7 @@ use Piwik\Common;
 use Piwik\Archive;
 use Piwik\SegmentExpression;
 use Piwik\SettingsPiwik;
+use Piwik\Plugins\VisitsSummary\API as APIVisitsSummary;
 
 /**
  * VisitFrequency API lets you access a list of metrics related to Returning Visitors.
@@ -22,86 +23,62 @@ use Piwik\SettingsPiwik;
  */
 class API extends \Piwik\Plugin\API
 {
+    // visitorType==returning,visitorType==returningCustomer
+    const RETURNING_VISITOR_SEGMENT = "visitorType%3D%3Dreturning%2CvisitorType%3D%3DreturningCustomer";
+    const COLUMN_SUFFIX = "_returning";
+
+    /**
+     * @param int $idSite
+     * @param string $period
+     * @param string $date
+     * @param bool|string $segment
+     * @param bool|array $columns
+     * @return mixed
+     */
     public function get($idSite, $period, $date, $segment = false, $columns = false)
     {
-        $archive = Archive::build($idSite, $period, $date, $segment);
-
-        // array values are comma separated
-        $columns = Piwik::getArrayFromApiParameter($columns);
-        $tempColumns = array();
-
-        $bounceRateReturningRequested = $averageVisitDurationReturningRequested = $actionsPerVisitReturningRequested = false;
-        if (!empty($columns)) {
-            // make sure base metrics are there for processed metrics
-            if (false !== ($bounceRateReturningRequested = array_search('bounce_rate_returning', $columns))) {
-                if (!in_array('nb_visits_returning', $columns)) {
-                    $tempColumns[] = 'nb_visits_returning';
-                }
-
-                if (!in_array('bounce_count_returning', $columns)) {
-                    $tempColumns[] = 'bounce_count_returning';
-                }
-
-                unset($columns[$bounceRateReturningRequested]);
-            }
-
-            if (false !== ($actionsPerVisitReturningRequested = array_search('nb_actions_per_visit_returning', $columns))) {
-                if (!in_array('nb_actions_returning', $columns)) {
-                    $tempColumns[] = 'nb_actions_returning';
-                }
-
-                if (!in_array('nb_visits_returning', $columns)) {
-                    $tempColumns[] = 'nb_visits_returning';
-                }
-
-                unset($columns[$actionsPerVisitReturningRequested]);
-            }
-
-            if (false !== ($averageVisitDurationReturningRequested = array_search('avg_time_on_site_returning', $columns))) {
-                if (!in_array('sum_visit_length_returning', $columns)) {
-                    $tempColumns[] = 'sum_visit_length_returning';
-                }
-
-                if (!in_array('nb_visits_returning', $columns)) {
-                    $tempColumns[] = 'nb_visits_returning';
-                }
-
-                unset($columns[$averageVisitDurationReturningRequested]);
-            }
+        $segment = $this->appendReturningVisitorSegment($segment);
+
+        $this->unprefixColumns($columns);
+        $params = array(
+            'idSite'    => $idSite,
+            'period'    => $period,
+            'date'      => $date,
+            'segment'   => $segment,
+            'columns'   => implode(',', $columns),
+            'format'    => 'original',
+            'serialize' => 0 // tests set this to 1
+        );
+        $table = Request::processRequest('VisitsSummary.get', $params);
+        $this->prefixColumns($table, $period);
+        return $table;
+    }
 
-            $tempColumns = array_unique($tempColumns);
-            $columns = array_merge($columns, $tempColumns);
+    protected function appendReturningVisitorSegment($segment)
+    {
+        if (empty($segment)) {
+            $segment = '';
         } else {
-            $bounceRateReturningRequested = $averageVisitDurationReturningRequested = $actionsPerVisitReturningRequested = true;
-            $columns = array(
-                'nb_visits_returning',
-                'nb_actions_returning',
-                'nb_visits_converted_returning',
-                'bounce_count_returning',
-                'sum_visit_length_returning',
-                'max_actions_returning',
-            );
-
-            if (SettingsPiwik::isUniqueVisitorsEnabled($period)) {
-                $columns = array_merge(array('nb_uniq_visitors_returning'), $columns);
-            }
+            $segment .= urlencode(SegmentExpression::AND_DELIMITER);
         }
-        $dataTable = $archive->getDataTableFromNumeric($columns);
+        $segment .= self::RETURNING_VISITOR_SEGMENT;
+        return $segment;
+    }
 
-        // Process ratio metrics
-        if ($bounceRateReturningRequested !== false) {
-            $dataTable->filter('ColumnCallbackAddColumnPercentage', array('bounce_rate_returning', 'bounce_count_returning', 'nb_visits_returning', 0));
-        }
-        if ($actionsPerVisitReturningRequested !== false) {
-            $dataTable->filter('ColumnCallbackAddColumnQuotient', array('nb_actions_per_visit_returning', 'nb_actions_returning', 'nb_visits_returning', 1));
-        }
-        if ($averageVisitDurationReturningRequested !== false) {
-            $dataTable->filter('ColumnCallbackAddColumnQuotient', array('avg_time_on_site_returning', 'sum_visit_length_returning', 'nb_visits_returning', 0));
+    protected function unprefixColumns(&$columns)
+    {
+        $columns = Piwik::getArrayFromApiParameter($columns);
+        foreach ($columns as &$column) {
+            $column = str_replace(self::COLUMN_SUFFIX, "", $column);
         }
+    }
 
-        // remove temporary metrics that were used to compute processed metrics
-        $dataTable->deleteColumns($tempColumns);
-
-        return $dataTable;
+    protected function prefixColumns($table, $period)
+    {
+        $rename = array();
+        foreach (APIVisitsSummary::getInstance()->getColumns($period) as $oldColumn) {
+            $rename[$oldColumn] = $oldColumn . self::COLUMN_SUFFIX;
+        }
+        $table->filter('ReplaceColumnNames', array($rename));
     }
 }
\ No newline at end of file
diff --git a/plugins/VisitFrequency/Archiver.php b/plugins/VisitFrequency/Archiver.php
deleted file mode 100644
index b6f9913346..0000000000
--- a/plugins/VisitFrequency/Archiver.php
+++ /dev/null
@@ -1,131 +0,0 @@
-<?php
-/**
- * Piwik - Open source web analytics
- *
- * @link http://piwik.org
- * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
- */
-
-namespace Piwik\Plugins\VisitFrequency;
-
-use Piwik\ArchiveProcessor;
-use Piwik\ArchiveProcessor\Parameters as ArchiveProcessorParams;
-use Piwik\DataAccess\LogAggregator;
-use Piwik\API\Request;
-use Piwik\Piwik;
-use Piwik\Segment;
-use Piwik\SegmentExpression;
-use Piwik\Plugins\VisitsSummary\API as APIVisitsSummary;
-use Piwik\SettingsPiwik;
-use Piwik\Metrics;
-
-/**
- * Introduced to provide backwards compatibility for pre-2.0 data. Uses a segment to archive
- * data for day periods and aggregates this data for non-day periods.
- *
- * We normally would want to just forward requests to the VisitsSummary API w/ the correctly
- * modified segment, but in order to combine pre-2.0 data with post-2.0 data, there has
- * to be a VisitFrequency Archiver. Otherwise, the VisitsSummary metrics archiving will
- * be called, and the pre-2.0 VisitFrequency data (which is not retrieved by VisitsSummary) will
- * be ignored.
- */
-class Archiver extends \Piwik\Plugin\Archiver
-{
-    // visitorType==returning,visitorType==returningCustomer
-    const RETURNING_VISITOR_SEGMENT = "visitorType%3D%3Dreturning%2CvisitorType%3D%3DreturningCustomer";
-    const COLUMN_SUFFIX = "_returning";
-
-    public static $visitFrequencyPeriodMetrics = array(
-        'nb_visits_returning',
-        'nb_actions_returning',
-        'max_actions_returning',
-        'sum_visit_length_returning',
-        'bounce_count_returning',
-        'nb_visits_converted_returning'
-    );
-
-    public function aggregateDayReport()
-    {
-        $this->callVisitsSummaryApiAndArchive();
-    }
-
-    public function aggregateMultipleReports()
-    {
-        if (SettingsPiwik::isUniqueVisitorsEnabled($this->getProcessor()->getParams()->getPeriod()->getLabel())) {
-            $this->computeUniqueVisitsForNonDay();
-        }
-
-        $this->getProcessor()->aggregateNumericMetrics(self::$visitFrequencyPeriodMetrics);
-    }
-
-    private function callVisitsSummaryApiAndArchive($columns = false)
-    {
-        $archiveParams = $this->getProcessor()->getParams();
-        $periodLabel = $archiveParams->getPeriod()->getLabel();
-
-        $params = array(
-            'idSite'     => $archiveParams->getSite()->getId(),
-            'period'     => $periodLabel,
-            'date'       => $archiveParams->getPeriod()->getDateStart()->toString(),
-            'segment'    => $this->appendReturningVisitorSegment($archiveParams->getSegment()->getString()),
-            'format'     => 'original',
-            'serialize'  => 0 // make sure we don't serialize (in case serialize is in the query parameters)
-        );
-        if ($columns) {
-            $params['columns'] = implode(",", $columns);
-        }
-
-        $table = Request::processRequest('VisitsSummary.get', $params);
-        $this->suffixColumns($table, $periodLabel);
-
-        if ($table->getRowsCount() > 0) {
-            $this->getProcessor()->insertNumericRecords($table->getFirstRow()->getColumns());
-        }
-    }
-
-    private function computeUniqueVisitsForNonDay()
-    {
-        // NOTE: we cannot call the VisitsSummary API from within period archiving for some reason. it results in
-        //       a very hard to trace bug that breaks the OmniFixture severely (causes lots of data to not be shown).
-        //       no idea why it causes such an error.
-        $oldParams = $this->getProcessor()->getParams();
-        $site = $oldParams->getSite();
-        $newSegment = $this->appendReturningVisitorSegment($oldParams->getSegment()->getString());
-        $newParams = new ArchiveProcessorParams($site, $oldParams->getPeriod(), new Segment($newSegment, array($site->getId())));
-
-        $logAggregator = new LogAggregator($newParams);
-        $query = $logAggregator->queryVisitsByDimension(array(), false, array(), array(Metrics::INDEX_NB_UNIQ_VISITORS));
-        $data = $query->fetch();
-        $nbUniqVisitors = (float)$data[Metrics::INDEX_NB_UNIQ_VISITORS];
-
-        $this->getProcessor()->insertNumericRecord('nb_uniq_visitors_returning', $nbUniqVisitors);
-    }
-
-    protected function appendReturningVisitorSegment($segment)
-    {
-        if (empty($segment)) {
-            $segment = '';
-        } else {
-            $segment .= urlencode(SegmentExpression::AND_DELIMITER);
-        }
-        $segment .= self::RETURNING_VISITOR_SEGMENT;
-        return $segment;
-    }
-
-    protected function unsuffixColumns(&$columns)
-    {
-        $columns = Piwik::getArrayFromApiParameter($columns);
-        foreach ($columns as &$column) {
-            $column = str_replace(self::COLUMN_SUFFIX, "", $column);
-        }
-    }
-
-    protected function suffixColumns($table, $period)
-    {
-        $rename = array();
-        foreach (APIVisitsSummary::getInstance()->getColumns($period) as $oldColumn) {
-            $rename[$oldColumn] = $oldColumn . self::COLUMN_SUFFIX;
-        }
-        $table->filter('ReplaceColumnNames', array($rename));
-    }
-}
\ No newline at end of file