refs #4996 actually archive the tracked data and display the actual data in a report

7 files changed, 305 insertions, 12 deletions

diff --git a/core/Metrics.php b/core/Metrics.php
index 4af30600d3..bb7d5d430f 100644
--- a/core/Metrics.php
+++ b/core/Metrics.php
@@ -78,6 +78,9 @@ class Metrics
     const INDEX_EVENT_MAX_EVENT_VALUE = 37;
     const INDEX_EVENT_NB_HITS_WITH_VALUE = 38;
 
+    // Contents
+    const INDEX_CONTENT_NB_IMPRESSIONS = 39;
+
     // Goal reports
     const INDEX_GOAL_NB_CONVERSIONS = 1;
     const INDEX_GOAL_REVENUE = 2;
@@ -133,8 +136,10 @@ class Metrics
         Metrics::INDEX_EVENT_SUM_EVENT_VALUE                 => 'sum_event_value',
         Metrics::INDEX_EVENT_MIN_EVENT_VALUE                 => 'min_event_value',
         Metrics::INDEX_EVENT_MAX_EVENT_VALUE                 => 'max_event_value',
-        Metrics::INDEX_EVENT_NB_HITS_WITH_VALUE              => 'nb_events_with_value'
+        Metrics::INDEX_EVENT_NB_HITS_WITH_VALUE              => 'nb_events_with_value',
 
+        // Contents
+        Metrics::INDEX_CONTENT_NB_IMPRESSIONS                => 'nb_impressions'
     );
 
     public static $mappingFromIdToNameGoal = array(
diff --git a/core/Tracker/TableLogAction.php b/core/Tracker/TableLogAction.php
index 6ebc6d60ff..40acd89f70 100644
--- a/core/Tracker/TableLogAction.php
+++ b/core/Tracker/TableLogAction.php
@@ -232,6 +232,9 @@ class TableLogAction
             'eventAction' => Action::TYPE_EVENT_ACTION,
             'eventCategory' => Action::TYPE_EVENT_CATEGORY,
             'eventName' => Action::TYPE_EVENT_NAME,
+            'contentPiece' => Action::TYPE_CONTENT_PIECE,
+            'contentTarget' => Action::TYPE_CONTENT_TARGET,
+            'contentName' => Action::TYPE_CONTENT_NAME,
         );
         if(!empty($exactMatch[$segmentName])) {
             return $exactMatch[$segmentName];
diff --git a/misc/internal-docs/content-tracking.md b/misc/internal-docs/content-tracking.md
index 09fb9d65a5..63dc1762ee 100644
--- a/misc/internal-docs/content-tracking.md
+++ b/misc/internal-docs/content-tracking.md
@@ -6,7 +6,13 @@ This is the technical concept for implementing content tracking. We won't plan a
 * Plugin name: Content
 * Content name - The name of the content visible in reports
 * Content piece - eg a video file, image file, text, ...
-* Content target - a clicked url, a started video, any "conversion"... Are we always assuming it is a click or can it be a hover or drag/drop, ...?
+* Content target - a clicked url, a started video, any "conversion"... 
+
+## Further Questions
+1. Can the same piece have different names / targets? Can the same content name have different targets? How are they presented?
+2. Are we always assuming the "conversion" or "target URL" is caused by a click or can it be a hover or drag/drop, ...?
+3. Would a piece of content have maybe custom variables etc?
+4. How do we present the data in a report? Similar to events with second dimensions? Probably depends on 1)
 
 ## Tagging of the content piece declarative
 In HTML...
@@ -24,7 +30,6 @@ Impressions are logically not really events and I don't think it makes sense to
 * New url parameters like `c_p`, `c_n` and `c_u` for piece of content, name and url. Maybe instead of `c_u` would be better `c_t` for target which is more generic. Sending a JSON array would not work since we cannot log multiple actions in one tracking request. They have to be sent using bulk tracking instead.
  * `c_c` and `c_n` would be required, `c_t` not as for instance a piece of content does not necessarily have a target (hard to measure a click ratio in this case?)
 
-Would a piece of content have maybe custom variables etc?
 
 ## Tracking the clicks
 Contrary to impressions, clicks are actually events and it would be nice to use events here. Maybe we can link an event with a piece of content?
diff --git a/plugins/Actions/Archiver.php b/plugins/Actions/Archiver.php
index 14d4214744..d5aafd6f6e 100644
--- a/plugins/Actions/Archiver.php
+++ b/plugins/Actions/Archiver.php
@@ -123,7 +123,7 @@ class Archiver extends \Piwik\Plugin\Archiver
      */
     public static function getWhereClauseActionIsNotEvent()
     {
-        return " AND log_link_visit_action.idaction_event_category IS NULL";
+        return " AND log_link_visit_action.idaction_event_category IS NULL AND log_link_visit_action.idaction_content_piece IS NULL";
     }
 
     /**
diff --git a/plugins/Contents/API.php b/plugins/Contents/API.php
index da5c6f1081..f9c2b8d9ab 100644
--- a/plugins/Contents/API.php
+++ b/plugins/Contents/API.php
@@ -8,8 +8,11 @@
  */
 namespace Piwik\Plugins\Contents;
 
+use Piwik\Archive;
 use Piwik\DataTable;
 use Piwik\DataTable\Row;
+use Piwik\Metrics;
+use Piwik\Piwik;
 
 /**
  * API for plugin Contents
@@ -19,6 +22,10 @@ use Piwik\DataTable\Row;
 class API extends \Piwik\Plugin\API
 {
 
+    protected $mappingApiToRecord = array(
+        'getContents' => Archiver::CONTENTS_NAME_RECORD_NAME
+    );
+
     /**
      * Another example method that returns a data table.
      * @param int    $idSite
@@ -29,15 +36,39 @@ class API extends \Piwik\Plugin\API
      */
     public function getContents($idSite, $period, $date, $segment = false)
     {
-        $table = new DataTable();
+        return $this->getDataTable(__FUNCTION__, $idSite, $period, $date, $segment);
+    }
+
+    protected function getDataTable($name, $idSite, $period, $date, $segment)
+    {
+        Piwik::checkUserHasViewAccess($idSite);
+        $recordName = $this->getRecordNameForAction($name);
+        $dataTable = Archive::getDataTableFromArchive($recordName, $idSite, $period, $date, $segment, false);
+        $this->filterDataTable($dataTable);
+        return $dataTable;
+    }
 
-        $table->addRowFromArray(array(Row::COLUMNS => array(
-            'label' => 'My banner',
-            'nb_impressions' => 50,
-            'nb_conversions' => 5,
-            'conversion_rate' => '10%'
-        )));
+    protected function getRecordNameForAction($apiMethod, $secondaryDimension = false)
+    {
+        return $this->mappingApiToRecord[$apiMethod];
+    }
+
+    /**
+     * @param DataTable $dataTable
+     */
+    protected function filterDataTable($dataTable)
+    {
+        $dataTable->filter('Sort', array(Metrics::INDEX_NB_VISITS));
+        $dataTable->queueFilter('ReplaceColumnNames');
+        $dataTable->queueFilter('ReplaceSummaryRowLabel');
+        $dataTable->filter(function (DataTable $table) {
+            $row = $table->getRowFromLabel(Archiver::CONTENT_TARGET_NOT_SET);
+            if ($row) {
+                $row->setColumn('label', Piwik::translate('General_NotDefined', Piwik::translate('Contents_ContentTarget')));
+            }
+        });
 
-        return $table;
+        // Content conversion rate = conversions / impressions
+        $dataTable->queueFilter('ColumnCallbackAddColumnPercentage', array('conversion_rate', 'nb_conversions', 'nb_impressions', $precision = 2));
     }
 }
diff --git a/plugins/Contents/Archiver.php b/plugins/Contents/Archiver.php
new file mode 100644
index 0000000000..81abbe75d6
--- /dev/null
+++ b/plugins/Contents/Archiver.php
@@ -0,0 +1,188 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\Plugins\Contents;
+
+use Piwik\DataTable;
+use Piwik\Metrics;
+use Piwik\Plugins\Actions\ArchivingHelper;
+use Piwik\RankingQuery;
+
+/**
+ * Processing reports for Contents
+
+ */
+class Archiver extends \Piwik\Plugin\Archiver
+{
+    const CONTENTS_PIECE_RECORD_NAME  = 'Contents_piece';
+    const CONTENTS_TARGET_RECORD_NAME = 'Contents_target';
+    const CONTENTS_NAME_RECORD_NAME   = 'Contents_name';
+    const CONTENT_TARGET_NOT_SET      = 'Piwik_ContentNameNotSet';
+
+    /**
+     * @var DataArray[]
+     */
+    protected $arrays = array();
+
+    function __construct($processor)
+    {
+        parent::__construct($processor);
+        $this->columnToSortByBeforeTruncation = Metrics::INDEX_NB_VISITS;
+        $this->maximumRowsInDataTable         = ArchivingHelper::$maximumRowsInDataTableLevelZero;
+        $this->maximumRowsInSubDataTable      = ArchivingHelper::$maximumRowsInSubDataTable;
+    }
+
+    protected function getRecordToDimensions()
+    {
+        return array(
+            self::CONTENTS_PIECE_RECORD_NAME  => array('contentPiece'),
+            self::CONTENTS_TARGET_RECORD_NAME => array('contentTarget'),
+            self::CONTENTS_NAME_RECORD_NAME   => array('contentName')
+        );
+    }
+
+    public function aggregateMultipleReports()
+    {
+        $dataTableToSum = $this->getRecordNames();
+        $this->getProcessor()->aggregateDataTableRecords($dataTableToSum, $this->maximumRowsInDataTable, $this->maximumRowsInSubDataTable, $this->columnToSortByBeforeTruncation);
+    }
+
+    protected function getRecordNames()
+    {
+        $mapping = $this->getRecordToDimensions();
+        return array_keys($mapping);
+    }
+
+    public function aggregateDayReport()
+    {
+        $this->aggregateDayContents();
+        $this->insertDayReports();
+    }
+
+    protected function aggregateDayContents()
+    {
+        $select = "
+                log_action_content_piece.name as contentPiece,
+                log_action_content_target.name as contentTarget,
+                log_action_content_name.name as contentName,
+
+				count(distinct log_link_visit_action.idvisit) as `" . Metrics::INDEX_NB_VISITS . "`,
+				count(distinct log_link_visit_action.idvisitor) as `" . Metrics::INDEX_NB_UNIQ_VISITORS . "`,
+				count(*) as `" . Metrics::INDEX_CONTENT_NB_IMPRESSIONS . "`
+        ";
+
+        $from = array(
+            "log_link_visit_action",
+            array(
+                "table"      => "log_action",
+                "tableAlias" => "log_action_content_piece",
+                "joinOn"     => "log_link_visit_action.idaction_content_piece = log_action_content_piece.idaction"
+            ),
+            array(
+                "table"      => "log_action",
+                "tableAlias" => "log_action_content_target",
+                "joinOn"     => "log_link_visit_action.idaction_content_target = log_action_content_target.idaction"
+            ),
+            array(
+                "table"      => "log_action",
+                "tableAlias" => "log_action_content_name",
+                "joinOn"     => "log_link_visit_action.idaction_name = log_action_content_name.idaction"
+            )
+        );
+
+        $where = "log_link_visit_action.server_time >= ?
+                    AND log_link_visit_action.server_time <= ?
+                    AND log_link_visit_action.idsite = ?
+                    AND log_link_visit_action.idaction_content_piece IS NOT NULL";
+
+        $groupBy = "log_action_content_piece.idaction,
+                    log_action_content_target.idaction,
+                    log_action_content_name.idaction";
+
+        $orderBy = "`" . Metrics::INDEX_NB_VISITS . "` DESC";
+
+        $rankingQueryLimit = ArchivingHelper::getRankingQueryLimit();
+        $rankingQuery = null;
+        if ($rankingQueryLimit > 0) {
+            $rankingQuery = new RankingQuery($rankingQueryLimit);
+            $rankingQuery->setOthersLabel(DataTable::LABEL_SUMMARY_ROW);
+            $rankingQuery->addLabelColumn(array('contentPiece', 'contentTarget', 'contentName'));
+            $rankingQuery->addColumn(array(Metrics::INDEX_NB_UNIQ_VISITORS));
+            $rankingQuery->addColumn(array(Metrics::INDEX_CONTENT_NB_IMPRESSIONS, Metrics::INDEX_NB_VISITS), 'sum');
+        }
+
+        $this->archiveDayQueryProcess($select, $from, $where, $orderBy, $groupBy, $rankingQuery);
+    }
+
+    protected function archiveDayQueryProcess($select, $from, $where, $orderBy, $groupBy, RankingQuery $rankingQuery)
+    {
+        // get query with segmentation
+        $query = $this->getLogAggregator()->generateQuery($select, $from, $where, $groupBy, $orderBy);
+
+        // apply ranking query
+        if ($rankingQuery) {
+            $query['sql'] = $rankingQuery->generateQuery($query['sql']);
+        }
+
+        // get result
+        $resultSet = $this->getLogAggregator()->getDb()->query($query['sql'], $query['bind']);
+
+        if ($resultSet === false) {
+            return;
+        }
+
+        while ($row = $resultSet->fetch()) {
+            $this->aggregateContentRow($row);
+        }
+    }
+
+    /**
+     * Records the daily datatables
+     */
+    protected function insertDayReports()
+    {
+        foreach ($this->arrays as $recordName => $dataArray) {
+            $dataTable = $dataArray->asDataTable();
+            $blob = $dataTable->getSerialized(
+                $this->maximumRowsInDataTable,
+                $this->maximumRowsInSubDataTable,
+                $this->columnToSortByBeforeTruncation);
+            $this->getProcessor()->insertBlobRecord($recordName, $blob);
+        }
+    }
+
+    /**
+     * @param string $name
+     * @return DataArray
+     */
+    protected function getDataArray($name)
+    {
+        if(empty($this->arrays[$name])) {
+            $this->arrays[$name] = new DataArray();
+        }
+        return $this->arrays[$name];
+    }
+
+    protected function aggregateContentRow($row)
+    {
+        foreach ($this->getRecordToDimensions() as $record => $dimensions) {
+            $dataArray = $this->getDataArray($record);
+
+            $mainDimension = $dimensions[0];
+            $mainLabel = $row[$mainDimension];
+
+            // Content target is optional
+            if ($mainDimension == 'contentTarget'
+                && empty($mainLabel)) {
+                $mainLabel = self::CONTENT_TARGET_NOT_SET;
+            }
+            $dataArray->sumMetricsContents($mainLabel, $row);
+        }
+    }
+
+}
diff --git a/plugins/Contents/DataArray.php b/plugins/Contents/DataArray.php
new file mode 100644
index 0000000000..d506567513
--- /dev/null
+++ b/plugins/Contents/DataArray.php
@@ -0,0 +1,61 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\Plugins\Contents;
+
+use Exception;
+use Piwik\Tracker\GoalManager;
+use Piwik\Metrics;
+
+/**
+ * The DataArray is a data structure used to aggregate datasets,
+ * ie. sum arrays made of rows made of columns,
+ * data from the logs is stored in a DataArray before being converted in a DataTable
+ *
+ */
+
+class DataArray extends \Piwik\DataArray
+{
+    public function sumMetricsContents($label, $row)
+    {
+        if (!isset($this->data[$label])) {
+            $this->data[$label] = self::makeEmptyContentsRow();
+        }
+        $this->doSumContentsMetrics($row, $this->data[$label], $onlyMetricsAvailableInActionsTable = true);
+    }
+
+    protected static function makeEmptyContentsRow()
+    {
+        return array(
+            Metrics::INDEX_NB_UNIQ_VISITORS        => 0,
+            Metrics::INDEX_NB_VISITS               => 0,
+            Metrics::INDEX_CONTENT_NB_IMPRESSIONS  => 0
+        );
+    }
+
+    /**
+     * @param array $newRowToAdd
+     * @param array $oldRowToUpdate
+     * @return void
+     */
+    protected function doSumContentsMetrics($newRowToAdd, &$oldRowToUpdate)
+    {
+        $oldRowToUpdate[Metrics::INDEX_NB_VISITS] += $newRowToAdd[Metrics::INDEX_NB_VISITS];
+        $oldRowToUpdate[Metrics::INDEX_NB_UNIQ_VISITORS] += $newRowToAdd[Metrics::INDEX_NB_UNIQ_VISITORS];
+        $oldRowToUpdate[Metrics::INDEX_CONTENT_NB_IMPRESSIONS] += $newRowToAdd[Metrics::INDEX_CONTENT_NB_IMPRESSIONS];
+    }
+
+    public function sumMetricsContentsPivot($parentLabel, $label, $row)
+    {
+        if (!isset($this->dataTwoLevels[$parentLabel][$label])) {
+            $this->dataTwoLevels[$parentLabel][$label] = $this->makeEmptyEventRow();
+        }
+        $this->doSumContentsMetrics($row, $this->dataTwoLevels[$parentLabel][$label]);
+    }
+
+}