PDF rausgenommen

msd2/tracking/piwik/core/API/ApiRenderer.php

@@ -0,0 +1,143 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\API;
+
+use Exception;
+use Piwik\Common;
+use Piwik\DataTable\Renderer;
+use Piwik\DataTable;
+use Piwik\Piwik;
+use Piwik\Plugin;
+
+/**
+ * API renderer
+ */
+abstract class ApiRenderer
+{
+    protected $request;
+
+    final public function __construct($request)
+    {
+        $this->request = $request;
+        $this->init();
+    }
+
+    protected function init()
+    {
+    }
+
+    abstract public function sendHeader();
+
+    public function renderSuccess($message)
+    {
+        return 'Success:' . $message;
+    }
+
+    /**
+     * @param $message
+     * @param Exception|\Throwable $exception
+     * @return mixed
+     */
+    public function renderException($message, $exception)
+    {
+        return $message;
+    }
+
+    public function renderScalar($scalar)
+    {
+        $dataTable = new DataTable\Simple();
+        $dataTable->addRowsFromArray(array($scalar));
+        return $this->renderDataTable($dataTable);
+    }
+
+    public function renderDataTable($dataTable)
+    {
+        $renderer = $this->buildDataTableRenderer($dataTable);
+        return $renderer->render();
+    }
+
+    public function renderArray($array)
+    {
+        $renderer = $this->buildDataTableRenderer($array);
+        return $renderer->render();
+    }
+
+    public function renderObject($object)
+    {
+        $exception = new Exception('The API cannot handle this data structure.');
+        return $this->renderException($exception->getMessage(), $exception);
+    }
+
+    public function renderResource($resource)
+    {
+        $exception = new Exception('The API cannot handle this data structure.');
+        return $this->renderException($exception->getMessage(), $exception);
+    }
+
+    /**
+     * @param $dataTable
+     * @return Renderer
+     */
+    protected function buildDataTableRenderer($dataTable)
+    {
+        $format   = self::getFormatFromClass(get_class($this));
+        if ($format == 'json2') {
+            $format = 'json';
+        }
+
+        $idSite = Common::getRequestVar('idSite', 0, 'int', $this->request);
+
+        if (empty($idSite)) {
+            $idSite = 'all';
+        }
+
+        $renderer = Renderer::factory($format);
+        $renderer->setTable($dataTable);
+        $renderer->setIdSite($idSite);
+        $renderer->setRenderSubTables(Common::getRequestVar('expanded', false, 'int', $this->request));
+        $renderer->setHideIdSubDatableFromResponse(Common::getRequestVar('hideIdSubDatable', false, 'int', $this->request));
+
+        return $renderer;
+    }
+
+    /**
+     * @param string $format
+     * @param array  $request
+     * @return ApiRenderer
+     * @throws Exception
+     */
+    public static function factory($format, $request)
+    {
+        $formatToCheck = '\\' . ucfirst(strtolower($format));
+
+        $rendererClassnames = Plugin\Manager::getInstance()->findMultipleComponents('Renderer', 'Piwik\\API\\ApiRenderer');
+
+        foreach ($rendererClassnames as $klassName) {
+            if (Common::stringEndsWith($klassName, $formatToCheck)) {
+                return new $klassName($request);
+            }
+        }
+
+        $availableRenderers = array();
+        foreach ($rendererClassnames as $rendererClassname) {
+            $availableRenderers[] = self::getFormatFromClass($rendererClassname);
+        }
+
+        $availableRenderers = implode(', ', $availableRenderers);
+        Common::sendHeader('Content-Type: text/plain; charset=utf-8');
+        throw new Exception(Piwik::translate('General_ExceptionInvalidRendererFormat', array($format, $availableRenderers)));
+    }
+
+    private static function getFormatFromClass($klassname)
+    {
+        $klass = explode('\\', $klassname);
+
+        return strtolower(end($klass));
+    }
+}

msd2/tracking/piwik/core/API/CORSHandler.php

@@ -0,0 +1,57 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\API;
+
+use Piwik\Common;
+use Piwik\Url;
+
+class CORSHandler
+{
+    /**
+     * @var array
+     */
+    protected $domains;
+
+    public function __construct()
+    {
+        $this->domains = Url::getCorsHostsFromConfig();
+    }
+
+    public function handle()
+    {
+        if (empty($this->domains)) {
+            return;
+        }
+        
+        Common::sendHeader('Vary: Origin');
+        
+        // allow Piwik to serve data to all domains
+        if (in_array("*", $this->domains)) {
+            
+            Common::sendHeader('Access-Control-Allow-Credentials: true');
+            
+            if (!empty($_SERVER['HTTP_ORIGIN'])) {
+                Common::sendHeader('Access-Control-Allow-Origin: ' . $_SERVER['HTTP_ORIGIN']);
+                return;
+            }
+            
+            Common::sendHeader('Access-Control-Allow-Origin: *');
+            return;
+        }
+
+        // specifically allow if it is one of the whitelisted CORS domains
+        if (!empty($_SERVER['HTTP_ORIGIN'])) {
+            $origin = $_SERVER['HTTP_ORIGIN'];
+            if (in_array($origin, $this->domains, true)) {
+                Common::sendHeader('Access-Control-Allow-Credentials: true');
+                Common::sendHeader('Access-Control-Allow-Origin: ' . $_SERVER['HTTP_ORIGIN']);
+            }
+        }
+    }
+}

msd2/tracking/piwik/core/API/DataTableGenericFilter.php

@@ -0,0 +1,245 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\API;
+
+use Exception;
+use Piwik\Common;
+use Piwik\DataTable;
+use Piwik\Plugin\ProcessedMetric;
+use Piwik\Plugin\Report;
+
+class DataTableGenericFilter
+{
+    /**
+     * List of filter names not to run.
+     *
+     * @var string[]
+     */
+    private $disabledFilters = array();
+
+    /**
+     * @var Report
+     */
+    private $report;
+
+    /**
+     * @var array
+     */
+    private $request;
+
+    /**
+     * Constructor
+     *
+     * @param $request
+     */
+    public function __construct($request, $report)
+    {
+        $this->request = $request;
+        $this->report  = $report;
+    }
+
+    /**
+     * Filters the given data table
+     *
+     * @param DataTable $table
+     */
+    public function filter($table)
+    {
+        $this->applyGenericFilters($table);
+    }
+
+    /**
+     * Makes sure a set of filters are not run.
+     *
+     * @param string[] $filterNames The name of each filter to disable.
+     */
+    public function disableFilters($filterNames)
+    {
+        $this->disabledFilters = array_unique(array_merge($this->disabledFilters, $filterNames));
+    }
+
+    /**
+     * Returns an array containing the information of the generic Filter
+     * to be applied automatically to the data resulting from the API calls.
+     *
+     * Order to apply the filters:
+     * 1 - Filter that remove filtered rows
+     * 2 - Filter that sort the remaining rows
+     * 3 - Filter that keep only a subset of the results
+     * 4 - Presentation filters
+     *
+     * @return array  See the code for spec
+     */
+    public static function getGenericFiltersInformation()
+    {
+        return array(
+            array('Pattern',
+                  array(
+                      'filter_column'  => array('string', 'label'),
+                      'filter_pattern' => array('string')
+                  )),
+            array('PatternRecursive',
+                  array(
+                      'filter_column_recursive'  => array('string', 'label'),
+                      'filter_pattern_recursive' => array('string'),
+                  )),
+            array('ExcludeLowPopulation',
+                  array(
+                      'filter_excludelowpop'       => array('string'),
+                      'filter_excludelowpop_value' => array('float', '0'),
+                  )),
+            array('Sort',
+                  array(
+                      'filter_sort_column' => array('string'),
+                      'filter_sort_order'  => array('string', 'desc'),
+                      $naturalSort = true,
+                      $recursiveSort = true,
+                      'filter_sort_column_secondary' => true
+                  )),
+            array('Truncate',
+                  array(
+                      'filter_truncate' => array('integer'),
+                  )),
+            array('Limit',
+                  array(
+                      'filter_offset'    => array('integer', '0'),
+                      'filter_limit'     => array('integer'),
+                      'keep_summary_row' => array('integer', '0'),
+                  ))
+        );
+    }
+
+    private function getGenericFiltersHavingDefaultValues()
+    {
+        $filters = self::getGenericFiltersInformation();
+
+        if ($this->report && $this->report->getDefaultSortColumn()) {
+            foreach ($filters as $index => $filter) {
+                if ($filter[0] === 'Sort') {
+                    $filters[$index][1]['filter_sort_column'] = array('string', $this->report->getDefaultSortColumn());
+                    $filters[$index][1]['filter_sort_order']  = array('string', $this->report->getDefaultSortOrder());
+
+                    $callback = $this->report->getSecondarySortColumnCallback();
+
+                    if (is_callable($callback)) {
+                        $filters[$index][1]['filter_sort_column_secondary'] = $callback;
+                    }
+
+                }
+            }
+        }
+
+        return $filters;
+    }
+
+    /**
+     * Apply generic filters to the DataTable object resulting from the API Call.
+     * Disable this feature by setting the parameter disable_generic_filters to 1 in the API call request.
+     *
+     * @param DataTable $datatable
+     * @return bool
+     */
+    protected function applyGenericFilters($datatable)
+    {
+        if ($datatable instanceof DataTable\Map) {
+            $tables = $datatable->getDataTables();
+            foreach ($tables as $table) {
+                $this->applyGenericFilters($table);
+            }
+            return;
+        }
+
+        $tableDisabledFilters = $datatable->getMetadata(DataTable::GENERIC_FILTERS_TO_DISABLE_METADATA_NAME) ?: [];
+        $genericFilters = $this->getGenericFiltersHavingDefaultValues();
+
+        $filterApplied = false;
+        foreach ($genericFilters as $filterMeta) {
+            $filterName = $filterMeta[0];
+            $filterParams = $filterMeta[1];
+            $filterParameters = array();
+            $exceptionRaised = false;
+
+            if (in_array($filterName, $this->disabledFilters)
+                || in_array($filterName, $tableDisabledFilters)
+            ) {
+                continue;
+            }
+
+            foreach ($filterParams as $name => $info) {
+                if (!is_array($info)) {
+                    // hard coded value that cannot be changed via API, see eg $naturalSort = true in 'Sort'
+                    $filterParameters[] = $info;
+                } else {
+                    // parameter type to cast to
+                    $type = $info[0];
+
+                    // default value if specified, when the parameter doesn't have a value
+                    $defaultValue = null;
+                    if (isset($info[1])) {
+                        $defaultValue = $info[1];
+                    }
+
+                    try {
+                        $value = Common::getRequestVar($name, $defaultValue, $type, $this->request);
+                        settype($value, $type);
+                        $filterParameters[] = $value;
+                    } catch (Exception $e) {
+                        $exceptionRaised = true;
+                        break;
+                    }
+                }
+            }
+
+            if (!$exceptionRaised) {
+                $datatable->filter($filterName, $filterParameters);
+                $filterApplied = true;
+            }
+        }
+
+        return $filterApplied;
+    }
+
+    public function areProcessedMetricsNeededFor($metrics)
+    {
+        $columnQueryParameters = array(
+            'filter_column',
+            'filter_column_recursive',
+            'filter_excludelowpop',
+            'filter_sort_column'
+        );
+
+        foreach ($columnQueryParameters as $queryParamName) {
+            $queryParamValue = Common::getRequestVar($queryParamName, false, $type = null, $this->request);
+            if (!empty($queryParamValue)
+                && $this->containsProcessedMetric($metrics, $queryParamValue)
+            ) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    /**
+     * @param ProcessedMetric[] $metrics
+     * @param string $name
+     * @return bool
+     */
+    private function containsProcessedMetric($metrics, $name)
+    {
+        foreach ($metrics as $metric) {
+            if ($metric instanceof ProcessedMetric
+                && $metric->getName() == $name
+            ) {
+                return true;
+            }
+        }
+        return false;
+    }
+}

msd2/tracking/piwik/core/API/DataTableManipulator.php

@@ -0,0 +1,207 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\API;
+
+use Exception;
+use Piwik\Archive\DataTableFactory;
+use Piwik\Container\StaticContainer;
+use Piwik\DataTable\Row;
+use Piwik\DataTable;
+use Piwik\Period\Range;
+use Piwik\Plugins\API\API;
+
+/**
+ * Base class for manipulating data tables.
+ * It provides generic mechanisms like iteration and loading subtables.
+ *
+ * The manipulators are used in ResponseBuilder and are triggered by
+ * API parameters. They are not filters because they don't work on the pre-
+ * fetched nested data tables. Instead, they load subtables using this base
+ * class. This way, they can only load the tables they really need instead
+ * of using expanded=1. Another difference between manipulators and filters
+ * is that filters keep the overall structure of the table intact while
+ * manipulators can change the entire thing.
+ */
+abstract class DataTableManipulator
+{
+    protected $apiModule;
+    protected $apiMethod;
+    protected $request;
+    protected $apiMethodForSubtable;
+
+    /**
+     * Constructor
+     *
+     * @param bool $apiModule
+     * @param bool $apiMethod
+     * @param array $request
+     */
+    public function __construct($apiModule = false, $apiMethod = false, $request = array())
+    {
+        $this->apiModule = $apiModule;
+        $this->apiMethod = $apiMethod;
+        $this->request = $request;
+    }
+
+    /**
+     * This method can be used by subclasses to iterate over data tables that might be
+     * data table maps. It calls back the template method self::doManipulate for each table.
+     * This way, data table arrays can be handled in a transparent fashion.
+     *
+     * @param DataTable\Map|DataTable $dataTable
+     * @throws Exception
+     * @return DataTable\Map|DataTable
+     */
+    protected function manipulate($dataTable)
+    {
+        if ($dataTable instanceof DataTable\Map) {
+            return $this->manipulateDataTableMap($dataTable);
+        } elseif ($dataTable instanceof DataTable) {
+            return $this->manipulateDataTable($dataTable);
+        } else {
+            return $dataTable;
+        }
+    }
+
+    /**
+     * Manipulates child DataTables of a DataTable\Map. See @manipulate for more info.
+     *
+     * @param DataTable\Map $dataTable
+     * @return DataTable\Map
+     */
+    protected function manipulateDataTableMap($dataTable)
+    {
+        $result = $dataTable->getEmptyClone();
+        foreach ($dataTable->getDataTables() as $tableLabel => $childTable) {
+            $newTable = $this->manipulate($childTable);
+            $result->addTable($newTable, $tableLabel);
+        }
+        return $result;
+    }
+
+    /**
+     * Manipulates a single DataTable instance. Derived classes must define
+     * this function.
+     */
+    abstract protected function manipulateDataTable($dataTable);
+
+    /**
+     * Load the subtable for a row.
+     * Returns null if none is found.
+     *
+     * @param DataTable $dataTable
+     * @param Row $row
+     *
+     * @return DataTable
+     */
+    protected function loadSubtable($dataTable, $row)
+    {
+        if (!($this->apiModule && $this->apiMethod && count($this->request))) {
+            return null;
+        }
+
+        $request = $this->request;
+
+        $idSubTable = $row->getIdSubDataTable();
+        if ($idSubTable === null) {
+            return null;
+        }
+
+        $request['idSubtable'] = $idSubTable;
+        if ($dataTable) {
+            $period = $dataTable->getMetadata(DataTableFactory::TABLE_METADATA_PERIOD_INDEX);
+            if ($period instanceof Range) {
+                $request['date'] = $period->getDateStart() . ',' . $period->getDateEnd();
+            } else {
+                $request['date'] = $period->getDateStart()->toString();
+            }
+        }
+
+        $method = $this->getApiMethodForSubtable($request);
+        return $this->callApiAndReturnDataTable($this->apiModule, $method, $request);
+    }
+
+    /**
+     * In this method, subclasses can clean up the request array for loading subtables
+     * in order to make ResponseBuilder behave correctly (e.g. not trigger the
+     * manipulator again).
+     *
+     * @param $request
+     * @return
+     */
+    abstract protected function manipulateSubtableRequest($request);
+
+    /**
+     * Extract the API method for loading subtables from the meta data
+     *
+     * @throws Exception
+     * @return string
+     */
+    protected function getApiMethodForSubtable($request)
+    {
+        if (!$this->apiMethodForSubtable) {
+            if (!empty($request['idSite'])) {
+                $idSite = $request['idSite'];
+            } else {
+                $idSite = 'all';
+            }
+
+            $apiParameters = array();
+            $entityNames = StaticContainer::get('entities.idNames');
+            foreach ($entityNames as $idName) {
+                if (!empty($request[$idName])) {
+                    $apiParameters[$idName] = $request[$idName];
+                }
+            }
+
+            $meta = API::getInstance()->getMetadata($idSite, $this->apiModule, $this->apiMethod, $apiParameters);
+
+            if (empty($meta) && array_key_exists('idGoal', $apiParameters)) {
+                unset($apiParameters['idGoal']);
+                $meta = API::getInstance()->getMetadata($idSite, $this->apiModule, $this->apiMethod, $apiParameters);
+            }
+
+            if (empty($meta)) {
+                throw new Exception(sprintf(
+                    "The DataTable cannot be manipulated: Metadata for report %s.%s could not be found. You can define the metadata in a hook, see example at: https://developer.matomo.org/api-reference/events#apigetreportmetadata",
+                    $this->apiModule, $this->apiMethod
+                ));
+            }
+
+            if (isset($meta[0]['actionToLoadSubTables'])) {
+                $this->apiMethodForSubtable = $meta[0]['actionToLoadSubTables'];
+            } else {
+                $this->apiMethodForSubtable = $this->apiMethod;
+            }
+        }
+        return $this->apiMethodForSubtable;
+    }
+
+    protected function callApiAndReturnDataTable($apiModule, $method, $request)
+    {
+        $class = Request::getClassNameAPI($apiModule);
+
+        $request = $this->manipulateSubtableRequest($request);
+        $request['serialize'] = 0;
+        $request['expanded'] = 0;
+        $request['format'] = 'original';
+        $request['format_metrics'] = 0;
+
+        // don't want to run recursive filters on the subtables as they are loaded,
+        // otherwise the result will be empty in places (or everywhere). instead we
+        // run it on the flattened table.
+        unset($request['filter_pattern_recursive']);
+
+        $dataTable = Proxy::getInstance()->call($class, $method, $request);
+        $response = new ResponseBuilder($format = 'original', $request);
+        $response->disableSendHeader();
+        $dataTable = $response->getResponse($dataTable, $apiModule, $method);
+        return $dataTable;
+    }
+}

msd2/tracking/piwik/core/API/DataTableManipulator/Flattener.php

@@ -0,0 +1,209 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\API\DataTableManipulator;
+
+use Piwik\API\DataTableManipulator;
+use Piwik\Common;
+use Piwik\DataTable;
+use Piwik\DataTable\Row;
+use Piwik\Plugin\ReportsProvider;
+
+/**
+ * This class is responsible for flattening data tables.
+ *
+ * It loads subtables and combines them into a single table by concatenating the labels.
+ * This manipulator is triggered by using flat=1 in the API request.
+ */
+class Flattener extends DataTableManipulator
+{
+
+    private $includeAggregateRows = false;
+
+    /**
+     * If the flattener is used after calling this method, aggregate rows will
+     * be included in the result. This can be useful when they contain data that
+     * the leafs don't have (e.g. conversion stats in some cases).
+     */
+    public function includeAggregateRows()
+    {
+        $this->includeAggregateRows = true;
+    }
+
+    /**
+     * Separator for building recursive labels (or paths)
+     * @var string
+     */
+    public $recursiveLabelSeparator = '';
+
+    /**
+     * @param  DataTable $dataTable
+     * @param string $recursiveLabelSeparator
+     * @return DataTable|DataTable\Map
+     */
+    public function flatten($dataTable, $recursiveLabelSeparator)
+    {
+        $this->recursiveLabelSeparator = $recursiveLabelSeparator;
+
+        return $this->manipulate($dataTable);
+    }
+
+    /**
+     * Template method called from self::manipulate.
+     * Flatten each data table.
+     *
+     * @param DataTable $dataTable
+     * @return DataTable
+     */
+    protected function manipulateDataTable($dataTable)
+    {
+        $newDataTable = $dataTable->getEmptyClone($keepFilters = true);
+        if ($dataTable->getTotalsRow()) {
+            $newDataTable->setTotalsRow($dataTable->getTotalsRow());
+        }
+
+        // this recursive filter will be applied to subtables
+        $dataTable->filter('ReplaceSummaryRowLabel');
+        $dataTable->filter('ReplaceColumnNames');
+
+        $report        = ReportsProvider::factory($this->apiModule, $this->apiMethod);
+        if (!empty($report)) {
+            $dimension = $report->getDimension();
+        }
+
+        $dimensionName = !empty($dimension) ? str_replace('.', '_', $dimension->getId()) : 'label1';
+
+        $this->flattenDataTableInto($dataTable, $newDataTable, $level = 1, $dimensionName);
+
+        return $newDataTable;
+    }
+
+    /**
+     * @param $dataTable DataTable
+     * @param $newDataTable
+     * @param $dimensionName
+     */
+    protected function flattenDataTableInto($dataTable, $newDataTable, $level, $dimensionName, $prefix = '', $logo = false)
+    {
+        foreach ($dataTable->getRows() as $rowId => $row) {
+            $this->flattenRow($row, $rowId, $newDataTable, $level, $dimensionName, $prefix, $logo);
+        }
+    }
+
+    /**
+     * @param Row $row
+     * @param DataTable $dataTable
+     * @param string $labelPrefix
+     * @param string $dimensionName
+     * @param bool $parentLogo
+     */
+    private function flattenRow
+    (Row $row, $rowId, DataTable $dataTable, $level, $dimensionName,
+                                $labelPrefix = '', $parentLogo = false)
+    {
+        $origLabel = $label = $row->getColumn('label');
+
+        if ($label !== false) {
+            $label = trim($label);
+
+            if ($this->recursiveLabelSeparator == '/') {
+                if (substr($label, 0, 1) == '/' && substr($labelPrefix, -1) == '/') {
+                    $label = substr($label, 1);
+                } elseif ($rowId === DataTable::ID_SUMMARY_ROW && $labelPrefix && $label != DataTable::LABEL_SUMMARY_ROW) {
+                    $label = ' - ' . $label;
+                }
+            }
+
+            $origLabel = $label;
+
+            $label = $labelPrefix . $label;
+            $row->setColumn('label', $label);
+
+            if ($row->getMetadata($dimensionName)) {
+                $origLabel = $row->getMetadata($dimensionName) . $this->recursiveLabelSeparator . $origLabel;
+            }
+
+            $row->setMetadata($dimensionName, $origLabel);
+        }
+
+        $logo = $row->getMetadata('logo');
+        if ($logo === false && $parentLogo !== false) {
+            $logo = $parentLogo;
+            $row->setMetadata('logo', $logo);
+        }
+
+        /** @var DataTable $subTable */
+        $subTable = $row->getSubtable();
+
+        if ($subTable) {
+            $subTable->applyQueuedFilters();
+            $row->deleteMetadata('idsubdatatable_in_db');
+        } else {
+            $subTable = $this->loadSubtable($dataTable, $row);
+        }
+
+        $row->removeSubtable();
+
+        if ($subTable === null) {
+            if ($this->includeAggregateRows) {
+                $row->setMetadata('is_aggregate', 0);
+            }
+            $dataTable->addRow($row);
+        } else {
+            if ($this->includeAggregateRows) {
+                $row->setMetadata('is_aggregate', 1);
+                $dataTable->addRow($row);
+            }
+            $prefix = $label . $this->recursiveLabelSeparator;
+
+            $report        = ReportsProvider::factory($this->apiModule, $this->apiMethod);
+            if (!empty($report)) {
+                $subDimension = $report->getSubtableDimension();
+            }
+
+            if ($level === 2) {
+                $subDimension = $report->getThirdLeveltableDimension();
+            }
+
+            if (empty($subDimension)) {
+                $report           = ReportsProvider::factory($this->apiModule, $this->getApiMethodForSubtable($this->request));
+                $subDimension     = $report->getDimension();
+            }
+
+            $subDimensionName = $subDimension ? str_replace('.', '_', $subDimension->getId()) : 'label' . (substr_count($prefix, $this->recursiveLabelSeparator) + 1);
+
+            if ($origLabel !== false) {
+                foreach ($subTable->getRows() as $subRow) {
+                    foreach ($row->getMetadata() as $name => $value) {
+                        if ($subRow->getMetadata($name) === false) {
+                            $subRow->setMetadata($name, $value);
+                        }
+                    }
+
+                    $subRow->setMetadata($dimensionName, $origLabel);
+                }
+            }
+
+            $this->flattenDataTableInto($subTable, $dataTable, $level + 1, $subDimensionName, $prefix, $logo);
+        }
+    }
+
+    /**
+     * Remove the flat parameter from the subtable request
+     *
+     * @param array $request
+     * @return array
+     */
+    protected function manipulateSubtableRequest($request)
+    {
+        unset($request['flat']);
+
+        return $request;
+    }
+
+}

msd2/tracking/piwik/core/API/DataTableManipulator/LabelFilter.php

@@ -0,0 +1,188 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\API\DataTableManipulator;
+
+use Piwik\API\DataTableManipulator;
+use Piwik\Common;
+use Piwik\DataTable;
+use Piwik\DataTable\Row;
+
+/**
+ * This class is responsible for handling the label parameter that can be
+ * added to every API call. If the parameter is set, only the row with the matching
+ * label is returned.
+ *
+ * The labels passed to this class should be urlencoded.
+ * Some reports use recursive labels (e.g. action reports). Use > to join them.
+ */
+class LabelFilter extends DataTableManipulator
+{
+    const SEPARATOR_RECURSIVE_LABEL = '>';
+    const TERMINAL_OPERATOR = '@';
+
+    private $labels;
+    private $addLabelIndex;
+    const FLAG_IS_ROW_EVOLUTION = 'label_index';
+
+    /**
+     * Filter a data table by label.
+     * The filtered table is returned, which might be a new instance.
+     *
+     * $apiModule, $apiMethod and $request are needed load sub-datatables
+     * for the recursive search. If the label is not recursive, these parameters
+     * are not needed.
+     *
+     * @param string $labels the labels to search for
+     * @param DataTable $dataTable the data table to be filtered
+     * @param bool $addLabelIndex Whether to add label_index metadata describing which
+     *                            label a row corresponds to.
+     * @return DataTable
+     */
+    public function filter($labels, $dataTable, $addLabelIndex = false)
+    {
+        if (!is_array($labels)) {
+            $labels = array($labels);
+        }
+
+        $this->labels = $labels;
+        $this->addLabelIndex = (bool)$addLabelIndex;
+        return $this->manipulate($dataTable);
+    }
+
+    /**
+     * Method for the recursive descend
+     *
+     * @param array $labelParts
+     * @param DataTable $dataTable
+     * @return Row|bool
+     */
+    private function doFilterRecursiveDescend($labelParts, $dataTable)
+    {
+        // we need to make sure to rebuild the index as some filters change the label column directly via
+        // $row->setColumn('label', '') which would not be noticed in the label index otherwise.
+        $dataTable->rebuildIndex();
+
+        // search for the first part of the tree search
+        $labelPart = array_shift($labelParts);
+
+        $row = false;
+        foreach ($this->getLabelVariations($labelPart) as $labelPart) {
+            $row = $dataTable->getRowFromLabel($labelPart);
+            if ($row !== false) {
+                break;
+            }
+        }
+
+        if ($row === false) {
+            // not found
+            return false;
+        }
+
+        // end of tree search reached
+        if (count($labelParts) == 0) {
+            return $row;
+        }
+
+        $subTable = $this->loadSubtable($dataTable, $row);
+        if ($subTable === null) {
+            // no more subtables but label parts left => no match found
+            return false;
+        }
+
+        return $this->doFilterRecursiveDescend($labelParts, $subTable);
+    }
+
+    /**
+     * Clean up request for ResponseBuilder to behave correctly
+     *
+     * @param $request
+     */
+    protected function manipulateSubtableRequest($request)
+    {
+        unset($request['label']);
+        unset($request['flat']);
+        $request['totals'] = 0;
+        $request['filter_sort_column'] = ''; // do not sort, we only want to find a matching column
+
+        return $request;
+    }
+
+    /**
+     * Use variations of the label to make it easier to specify the desired label
+     *
+     * Note: The HTML Encoded version must be tried first, since in ResponseBuilder the $label is unsanitized
+     * via Common::unsanitizeLabelParameter.
+     *
+     * @param string $originalLabel
+     * @return array
+     */
+    private function getLabelVariations($originalLabel)
+    {
+        static $pageTitleReports = array('getPageTitles', 'getEntryPageTitles', 'getExitPageTitles');
+
+        $originalLabel = trim($originalLabel);
+
+        $isTerminal = substr($originalLabel, 0, 1) == self::TERMINAL_OPERATOR;
+        if ($isTerminal) {
+            $originalLabel = substr($originalLabel, 1);
+        }
+
+        $variations = array();
+        $label = trim(urldecode($originalLabel));
+
+        $sanitizedLabel = Common::sanitizeInputValue($label);
+        $variations[] = $sanitizedLabel;
+
+        if ($this->apiModule == 'Actions'
+            && in_array($this->apiMethod, $pageTitleReports)
+        ) {
+            if ($isTerminal) {
+                array_unshift($variations, ' ' . $sanitizedLabel);
+                array_unshift($variations, ' ' . $label);
+            } else {
+                // special case: the Actions.getPageTitles report prefixes some labels with a blank.
+                // the blank might be passed by the user but is removed in Request::getRequestArrayFromString.
+                $variations[] = ' ' . $sanitizedLabel;
+                $variations[] = ' ' . $label;
+            }
+        }
+        $variations[] = $label;
+
+        $variations = array_unique($variations);
+
+        return $variations;
+    }
+
+    /**
+     * Filter a DataTable instance. See @filter for more info.
+     *
+     * @param DataTable\Simple|DataTable\Map $dataTable
+     * @return mixed
+     */
+    protected function manipulateDataTable($dataTable)
+    {
+        $result = $dataTable->getEmptyClone();
+        foreach ($this->labels as $labelIndex => $label) {
+            $row = null;
+            foreach ($this->getLabelVariations($label) as $labelVariation) {
+                $labelVariation = explode(self::SEPARATOR_RECURSIVE_LABEL, $labelVariation);
+
+                $row = $this->doFilterRecursiveDescend($labelVariation, $dataTable);
+                if ($row) {
+                    if ($this->addLabelIndex) {
+                        $row->setMetadata(self::FLAG_IS_ROW_EVOLUTION, $labelIndex);
+                    }
+                    $result->addRow($row);
+                    break;
+                }
+            }
+        }
+        return $result;
+    }
+}

msd2/tracking/piwik/core/API/DataTableManipulator/ReportTotalsCalculator.php

@@ -0,0 +1,249 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\API\DataTableManipulator;
+
+use Piwik\API\DataTableManipulator;
+use Piwik\API\DataTablePostProcessor;
+use Piwik\Common;
+use Piwik\DataTable;
+use Piwik\Metrics;
+use Piwik\Period;
+use Piwik\Piwik;
+use Piwik\Plugin\Report;
+use Piwik\Plugin\ReportsProvider;
+
+/**
+ * This class is responsible for setting the metadata property 'totals' on each dataTable if the report
+ * has a dimension. 'Totals' means it tries to calculate the total report value for each metric. For each
+ * the total number of visits, actions, ... for a given report / dataTable.
+ */
+class ReportTotalsCalculator extends DataTableManipulator
+{
+    /**
+     * @var Report
+     */
+    private $report;
+
+    /**
+     * Constructor
+     *
+     * @param bool $apiModule
+     * @param bool $apiMethod
+     * @param array $request
+     * @param Report $report
+     */
+    public function __construct($apiModule = false, $apiMethod = false, $request = array(), $report = null)
+    {
+        parent::__construct($apiModule, $apiMethod, $request);
+        $this->report = $report;
+    }
+
+    /**
+     * @param  DataTable $table
+     * @return \Piwik\DataTable|\Piwik\DataTable\Map
+     */
+    public function calculate($table)
+    {
+        // apiModule and/or apiMethod is empty for instance in case when flat=1 is called. Basically whenever a
+        // datamanipulator calls the API and wants the dataTable in return, see callApiAndReturnDataTable().
+        // it is also not set for some settings API request etc.
+        if (empty($this->apiModule) || empty($this->apiMethod)) {
+            return $table;
+        }
+
+        try {
+            return $this->manipulate($table);
+        } catch (\Exception $e) {
+            // eg. requests with idSubtable may trigger this exception
+            // (where idSubtable was removed in
+            // ?module=API&method=Events.getNameFromCategoryId&idSubtable=1&secondaryDimension=eventName&format=XML&idSite=1&period=day&date=yesterday&flat=0
+            return $table;
+        }
+    }
+
+    /**
+     * Adds ratio metrics if possible.
+     *
+     * @param  DataTable $dataTable
+     * @return DataTable
+     */
+    protected function manipulateDataTable($dataTable)
+    {
+        if (!empty($this->report) && !$this->report->getDimension() && !$this->isAllMetricsReport()) {
+            // we currently do not calculate the total value for reports having no dimension
+            return $dataTable;
+        }
+
+        if (1 != Common::getRequestVar('totals', 1, 'integer', $this->request)) {
+            return $dataTable;
+        }
+
+        $firstLevelTable = $this->makeSureToWorkOnFirstLevelDataTable($dataTable);
+
+        if (!$firstLevelTable->getRowsCount()
+            || $dataTable->getTotalsRow()
+            || $dataTable->getMetadata('totals')
+        ) {
+            return $dataTable;
+        }
+
+        // keeping queued filters would not only add various metadata but also break the totals calculator for some reports
+        // eg when needed metadata is missing to get site information (multisites.getall) etc
+        $clone = $firstLevelTable->getEmptyClone($keepFilters = false);
+        foreach ($firstLevelTable->getQueuedFilters() as $queuedFilter) {
+            if (is_array($queuedFilter) && 'ReplaceColumnNames' === $queuedFilter['className']) {
+                $clone->queueFilter($queuedFilter['className'], $queuedFilter['parameters']);
+            }
+        }
+        $tableMeta = $firstLevelTable->getMetadata(DataTable::COLUMN_AGGREGATION_OPS_METADATA_NAME);
+
+        /** @var DataTable\Row $totalRow */
+        $totalRow = null;
+        foreach ($firstLevelTable->getRows() as $row) {
+            if (!isset($totalRow)) {
+                $columns = $row->getColumns();
+                $columns['label'] = DataTable::LABEL_TOTALS_ROW;
+                $totalRow = new DataTable\Row(array(DataTable\Row::COLUMNS => $columns));
+            } else {
+                $totalRow->sumRow($row, $copyMetadata = false, $tableMeta);
+            }
+        }
+        $clone->addRow($totalRow);
+
+        if ($this->report
+            && $this->report->getProcessedMetrics()
+            && array_keys($this->report->getProcessedMetrics()) === array('nb_actions_per_visit', 'avg_time_on_site', 'bounce_rate', 'conversion_rate')) {
+            // hack for AllColumns table or default processed metrics
+            $clone->filter('AddColumnsProcessedMetrics', array($deleteRowsWithNoVisit = false));
+        }
+
+        $processor = new DataTablePostProcessor($this->apiModule, $this->apiMethod, $this->request);
+        $processor->applyComputeProcessedMetrics($clone);
+        $clone = $processor->applyQueuedFilters($clone);
+        $clone = $processor->applyMetricsFormatting($clone);
+
+        $totalRow = null;
+        foreach ($clone->getRows() as $row) {
+            /** * @var DataTable\Row $row */
+            if ($row->getColumn('label') === DataTable::LABEL_TOTALS_ROW) {
+                $totalRow = $row;
+                break;
+            }
+        }
+
+        if (!isset($totalRow) && $clone->getRowsCount() === 1) {
+            // if for some reason the processor renamed the totals row,
+            $totalRow = $clone->getFirstRow();
+        }
+
+        if (isset($totalRow)) {
+            $totals = $row->getColumns();
+            unset($totals['label']);
+            $dataTable->setMetadata('totals', $totals);
+
+            if (1 === Common::getRequestVar('keep_totals_row', 0, 'integer', $this->request)) {
+                $row->deleteMetadata(false);
+                $row->setColumn('label', Piwik::translate('General_Totals'));
+                $dataTable->setTotalsRow($row);
+            }
+        }
+
+        return $dataTable;
+    }
+
+    private function makeSureToWorkOnFirstLevelDataTable($table)
+    {
+        if (!array_key_exists('idSubtable', $this->request)) {
+            return $table;
+        }
+
+        $firstLevelReport = $this->findFirstLevelReport();
+
+        if (empty($firstLevelReport)) {
+            // it is not a subtable report
+            $module = $this->apiModule;
+            $action = $this->apiMethod;
+        } else {
+            $module = $firstLevelReport->getModule();
+            $action = $firstLevelReport->getAction();
+        }
+
+        $request = $this->request;
+        unset($request['idSubtable']); // to make sure we work on first level table
+
+        /** @var \Piwik\Period $period */
+        $period = $table->getMetadata('period');
+
+        if (!empty($period)) {
+            // we want a dataTable, not a dataTable\map
+            if (Period::isMultiplePeriod($request['date'], $request['period']) || 'range' == $period->getLabel()) {
+                $request['date']   = $period->getRangeString();
+                $request['period'] = 'range';
+            } else {
+                $request['date']   = $period->getDateStart()->toString();
+                $request['period'] = $period->getLabel();
+            }
+        }
+
+        $table = $this->callApiAndReturnDataTable($module, $action, $request);
+
+        if ($table instanceof DataTable\Map) {
+            $table = $table->mergeChildren();
+        }
+
+        return $table;
+    }
+
+    /**
+     * Make sure to get all rows of the first level table.
+     *
+     * @param array $request
+     * @return array
+     */
+    protected function manipulateSubtableRequest($request)
+    {
+        $request['totals']        = 0;
+        $request['expanded']      = 0;
+        $request['filter_limit']  = -1;
+        $request['filter_offset'] = 0;
+        $request['filter_sort_column'] = '';
+
+        $parametersToRemove = array('flat');
+
+        if (!array_key_exists('idSubtable', $this->request)) {
+            $parametersToRemove[] = 'idSubtable';
+        }
+
+        foreach ($parametersToRemove as $param) {
+            if (array_key_exists($param, $request)) {
+                unset($request[$param]);
+            }
+        }
+        return $request;
+    }
+
+    private function findFirstLevelReport()
+    {
+        $reports = new ReportsProvider();
+        foreach ($reports->getAllReports() as $report) {
+            $actionToLoadSubtables = $report->getActionToLoadSubTables();
+            if ($actionToLoadSubtables == $this->apiMethod
+                && $this->apiModule == $report->getModule()
+            ) {
+                return $report;
+            }
+        }
+        return null;
+    }
+
+    private function isAllMetricsReport()
+    {
+        return $this->report->getModule() == 'API' && $this->report->getAction() == 'get';
+    }
+}

msd2/tracking/piwik/core/API/DataTablePostProcessor.php

@@ -0,0 +1,459 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+
+namespace Piwik\API;
+
+use Exception;
+use Piwik\API\DataTableManipulator\Flattener;
+use Piwik\API\DataTableManipulator\LabelFilter;
+use Piwik\API\DataTableManipulator\ReportTotalsCalculator;
+use Piwik\Common;
+use Piwik\DataTable;
+use Piwik\DataTable\DataTableInterface;
+use Piwik\DataTable\Filter\PivotByDimension;
+use Piwik\Metrics\Formatter;
+use Piwik\Plugin\ProcessedMetric;
+use Piwik\Plugin\Report;
+use Piwik\Plugin\ReportsProvider;
+
+/**
+ * Processes DataTables that should be served through Piwik's APIs. This processing handles
+ * special query parameters and computes processed metrics. It does not included rendering to
+ * output formats (eg, 'xml').
+ */
+class DataTablePostProcessor
+{
+    const PROCESSED_METRICS_COMPUTED_FLAG = 'processed_metrics_computed';
+
+    /**
+     * @var null|Report
+     */
+    private $report;
+
+    /**
+     * @var string[]
+     */
+    private $request;
+
+    /**
+     * @var string
+     */
+    private $apiModule;
+
+    /**
+     * @var string
+     */
+    private $apiMethod;
+
+    /**
+     * @var Inconsistencies
+     */
+    private $apiInconsistencies;
+
+    /**
+     * @var Formatter
+     */
+    private $formatter;
+
+    private $callbackBeforeGenericFilters;
+    private $callbackAfterGenericFilters;
+
+    /**
+     * Constructor.
+     */
+    public function __construct($apiModule, $apiMethod, $request)
+    {
+        $this->apiModule = $apiModule;
+        $this->apiMethod = $apiMethod;
+        $this->setRequest($request);
+
+        $this->report = ReportsProvider::factory($apiModule, $apiMethod);
+        $this->apiInconsistencies = new Inconsistencies();
+        $this->setFormatter(new Formatter());
+    }
+
+    public function setFormatter(Formatter $formatter)
+    {
+        $this->formatter = $formatter;
+    }
+
+    public function setRequest($request)
+    {
+        $this->request = $request;
+    }
+
+    public function setCallbackBeforeGenericFilters($callbackBeforeGenericFilters)
+    {
+        $this->callbackBeforeGenericFilters = $callbackBeforeGenericFilters;
+    }
+
+    public function setCallbackAfterGenericFilters($callbackAfterGenericFilters)
+    {
+        $this->callbackAfterGenericFilters = $callbackAfterGenericFilters;
+    }
+
+    /**
+     * Apply post-processing logic to a DataTable of a report for an API request.
+     *
+     * @param DataTableInterface $dataTable The data table to process.
+     * @return DataTableInterface A new data table.
+     */
+    public function process(DataTableInterface $dataTable)
+    {
+        // TODO: when calculating metrics before hand, only calculate for needed metrics, not all. NOTE:
+        //       this is non-trivial since it will require, eg, to make sure processed metrics aren't added
+        //       after pivotBy is handled.
+        $dataTable = $this->applyPivotByFilter($dataTable);
+        $dataTable = $this->applyTotalsCalculator($dataTable);
+        $dataTable = $this->applyFlattener($dataTable);
+
+        if ($this->callbackBeforeGenericFilters) {
+            call_user_func($this->callbackBeforeGenericFilters, $dataTable);
+        }
+
+        $dataTable = $this->applyGenericFilters($dataTable);
+        $this->applyComputeProcessedMetrics($dataTable);
+
+        if ($this->callbackAfterGenericFilters) {
+            call_user_func($this->callbackAfterGenericFilters, $dataTable);
+        }
+
+        // we automatically safe decode all datatable labels (against xss)
+        $dataTable->queueFilter('SafeDecodeLabel');
+        $dataTable = $this->convertSegmentValueToSegment($dataTable);
+        $dataTable = $this->applyQueuedFilters($dataTable);
+        $dataTable = $this->applyRequestedColumnDeletion($dataTable);
+        $dataTable = $this->applyLabelFilter($dataTable);
+        $dataTable = $this->applyMetricsFormatting($dataTable);
+        return $dataTable;
+    }
+
+    private function convertSegmentValueToSegment(DataTableInterface $dataTable)
+    {
+        $dataTable->filter('AddSegmentBySegmentValue', array($this->report));
+        $dataTable->filter('ColumnCallbackDeleteMetadata', array('segmentValue'));
+
+        return $dataTable;
+    }
+
+    /**
+     * @param DataTableInterface $dataTable
+     * @return DataTableInterface
+     */
+    public function applyPivotByFilter(DataTableInterface $dataTable)
+    {
+        $pivotBy = Common::getRequestVar('pivotBy', false, 'string', $this->request);
+        if (!empty($pivotBy)) {
+            $this->applyComputeProcessedMetrics($dataTable);
+            $dataTable = $this->convertSegmentValueToSegment($dataTable);
+
+            $pivotByColumn = Common::getRequestVar('pivotByColumn', false, 'string', $this->request);
+            $pivotByColumnLimit = Common::getRequestVar('pivotByColumnLimit', false, 'int', $this->request);
+
+            $dataTable->filter('PivotByDimension', array($this->report, $pivotBy, $pivotByColumn, $pivotByColumnLimit,
+                PivotByDimension::isSegmentFetchingEnabledInConfig()));
+            $dataTable->filter('ColumnCallbackDeleteMetadata', array('segment'));
+        }
+        return $dataTable;
+    }
+
+    /**
+     * @param DataTableInterface $dataTable
+     * @return DataTable|DataTableInterface|DataTable\Map
+     */
+    public function applyFlattener($dataTable)
+    {
+        if (Common::getRequestVar('flat', '0', 'string', $this->request) == '1') {
+            // skip flattening if not supported by report and remove subtables only
+            if ($this->report && !$this->report->supportsFlatten()) {
+                $dataTable->filter('RemoveSubtables');
+                return $dataTable;
+            }
+
+            $flattener = new Flattener($this->apiModule, $this->apiMethod, $this->request);
+            if (Common::getRequestVar('include_aggregate_rows', '0', 'string', $this->request) == '1') {
+                $flattener->includeAggregateRows();
+            }
+
+            $recursiveLabelSeparator = ' - ';
+            if ($this->report) {
+                $recursiveLabelSeparator = $this->report->getRecursiveLabelSeparator();
+            }
+
+            $dataTable = $flattener->flatten($dataTable, $recursiveLabelSeparator);
+        }
+        return $dataTable;
+    }
+
+    /**
+     * @param DataTableInterface $dataTable
+     * @return DataTableInterface
+     */
+    public function applyTotalsCalculator($dataTable)
+    {
+        if (1 == Common::getRequestVar('totals', '1', 'integer', $this->request)) {
+            $calculator = new ReportTotalsCalculator($this->apiModule, $this->apiMethod, $this->request, $this->report);
+            $dataTable  = $calculator->calculate($dataTable);
+        }
+        return $dataTable;
+    }
+
+    /**
+     * @param DataTableInterface $dataTable
+     * @return DataTableInterface
+     */
+    public function applyGenericFilters($dataTable)
+    {
+        // if the flag disable_generic_filters is defined we skip the generic filters
+        if (0 == Common::getRequestVar('disable_generic_filters', '0', 'string', $this->request)) {
+            $this->applyProcessedMetricsGenericFilters($dataTable);
+
+            $genericFilter = new DataTableGenericFilter($this->request, $this->report);
+
+            $self = $this;
+            $report = $this->report;
+            $dataTable->filter(function (DataTable $table) use ($genericFilter, $report, $self) {
+                $processedMetrics = Report::getProcessedMetricsForTable($table, $report);
+                if ($genericFilter->areProcessedMetricsNeededFor($processedMetrics)) {
+                    $self->computeProcessedMetrics($table);
+                }
+            });
+
+            $label = self::getLabelFromRequest($this->request);
+            if (!empty($label)) {
+                $genericFilter->disableFilters(array('Limit', 'Truncate'));
+            }
+
+            $genericFilter->filter($dataTable);
+        }
+
+        return $dataTable;
+    }
+
+    /**
+     * @param DataTableInterface $dataTable
+     * @return DataTableInterface
+     */
+    public function applyProcessedMetricsGenericFilters($dataTable)
+    {
+        $addNormalProcessedMetrics = null;
+        try {
+            $addNormalProcessedMetrics = Common::getRequestVar(
+                'filter_add_columns_when_show_all_columns', null, 'integer', $this->request);
+        } catch (Exception $ex) {
+            // ignore
+        }
+
+        if ($addNormalProcessedMetrics !== null) {
+            $dataTable->filter('AddColumnsProcessedMetrics', array($addNormalProcessedMetrics));
+        }
+
+        $addGoalProcessedMetrics = null;
+        try {
+            $addGoalProcessedMetrics = Common::getRequestVar(
+                'filter_update_columns_when_show_all_goals', null, 'integer', $this->request);
+        } catch (Exception $ex) {
+            // ignore
+        }
+
+        if ($addGoalProcessedMetrics !== null) {
+            $idGoal = Common::getRequestVar(
+                'idGoal', DataTable\Filter\AddColumnsProcessedMetricsGoal::GOALS_OVERVIEW, 'string', $this->request);
+
+            $dataTable->filter('AddColumnsProcessedMetricsGoal', array($ignore = true, $idGoal));
+        }
+
+        return $dataTable;
+    }
+
+    /**
+     * @param DataTableInterface $dataTable
+     * @return DataTableInterface
+     */
+    public function applyQueuedFilters($dataTable)
+    {
+        // if the flag disable_queued_filters is defined we skip the filters that were queued
+        if (Common::getRequestVar('disable_queued_filters', 0, 'int', $this->request) == 0) {
+            $dataTable->applyQueuedFilters();
+        }
+        return $dataTable;
+    }
+
+    /**
+     * @param DataTableInterface $dataTable
+     * @return DataTableInterface
+     */
+    public function applyRequestedColumnDeletion($dataTable)
+    {
+        // use the ColumnDelete filter if hideColumns/showColumns is provided (must be done
+        // after queued filters are run so processed metrics can be removed, too)
+        $hideColumns = Common::getRequestVar('hideColumns', '', 'string', $this->request);
+        $showColumns = Common::getRequestVar('showColumns', '', 'string', $this->request);
+        $showRawMetrics = Common::getRequestVar('showRawMetrics', 0, 'int', $this->request);
+        if (!empty($hideColumns)
+            || !empty($showColumns)
+        ) {
+            $dataTable->filter('ColumnDelete', array($hideColumns, $showColumns));
+        } else if ($showRawMetrics !== 1) {
+            $this->removeTemporaryMetrics($dataTable);
+        }
+
+        return $dataTable;
+    }
+
+    /**
+     * @param DataTableInterface $dataTable
+     */
+    public function removeTemporaryMetrics(DataTableInterface $dataTable)
+    {
+        $allColumns = !empty($this->report) ? $this->report->getAllMetrics() : array();
+
+        $report = $this->report;
+        $dataTable->filter(function (DataTable $table) use ($report, $allColumns) {
+            $processedMetrics = Report::getProcessedMetricsForTable($table, $report);
+
+            $allTemporaryMetrics = array();
+            foreach ($processedMetrics as $metric) {
+                $allTemporaryMetrics = array_merge($allTemporaryMetrics, $metric->getTemporaryMetrics());
+            }
+
+            if (!empty($allTemporaryMetrics)) {
+                $table->filter('ColumnDelete', array($allTemporaryMetrics));
+            }
+        });
+    }
+
+    /**
+     * @param DataTableInterface $dataTable
+     * @return DataTableInterface
+     */
+    public function applyLabelFilter($dataTable)
+    {
+        $label = self::getLabelFromRequest($this->request);
+
+        // apply label filter: only return rows matching the label parameter (more than one if more than one label)
+        if (!empty($label)) {
+            $addLabelIndex = Common::getRequestVar('labelFilterAddLabelIndex', 0, 'int', $this->request) == 1;
+
+            $filter = new LabelFilter($this->apiModule, $this->apiMethod, $this->request);
+            $dataTable = $filter->filter($label, $dataTable, $addLabelIndex);
+        }
+        return $dataTable;
+    }
+
+    /**
+     * @param DataTableInterface $dataTable
+     * @return DataTableInterface
+     */
+    public function applyMetricsFormatting($dataTable)
+    {
+        $formatMetrics = Common::getRequestVar('format_metrics', 0, 'string', $this->request);
+        if ($formatMetrics == '0') {
+            return $dataTable;
+        }
+
+        // in Piwik 2.X & below, metrics are not formatted in API responses except for percents.
+        // this code implements this inconsistency
+        $onlyFormatPercents = $formatMetrics === 'bc';
+
+        $metricsToFormat = null;
+        if ($onlyFormatPercents) {
+            $metricsToFormat = $this->apiInconsistencies->getPercentMetricsToFormat();
+        }
+
+        // 'all' is a special value that indicates we should format non-processed metrics that are identified
+        // by string, like 'revenue'. this should be removed when all metrics are using the `Metric` class.
+        $formatAll = $formatMetrics === 'all';
+
+        $dataTable->filter(array($this->formatter, 'formatMetrics'), array($this->report, $metricsToFormat, $formatAll));
+        return $dataTable;
+    }
+
+    /**
+     * Returns the value for the label query parameter which can be either a string
+     * (ie, label=...) or array (ie, label[]=...).
+     *
+     * @param array $request
+     * @return array
+     */
+    public static function getLabelFromRequest($request)
+    {
+        $label = Common::getRequestVar('label', array(), 'array', $request);
+        if (empty($label)) {
+            $label = Common::getRequestVar('label', '', 'string', $request);
+            if (!empty($label)) {
+                $label = array($label);
+            }
+        }
+
+        $label = self::unsanitizeLabelParameter($label);
+        return $label;
+    }
+
+    public static function unsanitizeLabelParameter($label)
+    {
+        // this is needed because Proxy uses Common::getRequestVar which in turn
+        // uses Common::sanitizeInputValue. This causes the > that separates recursive labels
+        // to become &gt; and we need to undo that here.
+        $label = str_replace( htmlentities('>', ENT_COMPAT | ENT_HTML401, 'UTF-8'), '>', $label);
+        return $label;
+    }
+
+    public function computeProcessedMetrics(DataTable $dataTable)
+    {
+        if ($dataTable->getMetadata(self::PROCESSED_METRICS_COMPUTED_FLAG)) {
+            return;
+        }
+
+        /** @var ProcessedMetric[] $processedMetrics */
+        $processedMetrics = Report::getProcessedMetricsForTable($dataTable, $this->report);
+        if (empty($processedMetrics)) {
+            return;
+        }
+
+        $dataTable->setMetadata(self::PROCESSED_METRICS_COMPUTED_FLAG, true);
+
+        foreach ($processedMetrics as $name => $processedMetric) {
+            if (!$processedMetric->beforeCompute($this->report, $dataTable)) {
+                continue;
+            }
+
+            foreach ($dataTable->getRows() as $row) {
+                if ($row->getColumn($name) !== false) { // only compute the metric if it has not been computed already
+                    continue;
+                }
+
+                $computedValue = $processedMetric->compute($row);
+                if ($computedValue !== false) {
+                    $row->addColumn($name, $computedValue);
+                }
+            }
+        }
+
+        foreach ($dataTable->getRows() as $row) {
+            $subtable = $row->getSubtable();
+            if (!empty($subtable)) {
+                foreach ($processedMetrics as $name => $processedMetric) {
+                    $processedMetric->beforeComputeSubtable($row);
+                }
+
+                $this->computeProcessedMetrics($subtable);
+
+                foreach ($processedMetrics as $name => $processedMetric) {
+                    $processedMetric->afterComputeSubtable($row);
+                }
+            }
+        }
+    }
+
+    public function applyComputeProcessedMetrics(DataTableInterface $dataTable)
+    {
+        $dataTable->filter(array($this, 'computeProcessedMetrics'));
+    }
+}
+

msd2/tracking/piwik/core/API/DocumentationGenerator.php

@@ -0,0 +1,390 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\API;
+
+use Exception;
+use Piwik\Common;
+use Piwik\Container\StaticContainer;
+use Piwik\Piwik;
+use Piwik\Url;
+use ReflectionClass;
+
+/**
+ * Possible tags to use in APIs
+ *
+ * @hide -> Won't be shown in list of all APIs but is also not possible to be called via HTTP API
+ * @hideForAll Same as @hide
+ * @hideExceptForSuperUser Same as @hide but still shown and possible to be called by a user with super user access
+ * @internal -> Won't be shown in list of all APIs but is possible to be called via HTTP API
+ */
+class DocumentationGenerator
+{
+    protected $countPluginsLoaded = 0;
+
+    /**
+     * trigger loading all plugins with an API.php file in the Proxy
+     */
+    public function __construct()
+    {
+        $plugins = \Piwik\Plugin\Manager::getInstance()->getLoadedPluginsName();
+        foreach ($plugins as $plugin) {
+            try {
+                $className = Request::getClassNameAPI($plugin);
+                Proxy::getInstance()->registerClass($className);
+            } catch (Exception $e) {
+            }
+        }
+    }
+
+    /**
+     * Returns a HTML page containing help for all the successfully loaded APIs.
+     *
+     * @param bool $outputExampleUrls
+     * @return string
+     */
+    public function getApiDocumentationAsString($outputExampleUrls = true)
+    {
+        list($toc, $str) = $this->generateDocumentation($outputExampleUrls, $prefixUrls = '', $displayTitlesAsAngularDirective = true);
+
+        return "<div piwik-content-block content-title='Quick access to APIs' id='topApiRef' name='topApiRef'>
+				$toc</div>
+				$str";
+    }
+
+    /**
+     * Used on developer.piwik.org
+     *
+     * @param bool|true $outputExampleUrls
+     * @param string $prefixUrls
+     * @return string
+     */
+    public function getApiDocumentationAsStringForDeveloperReference($outputExampleUrls = true, $prefixUrls = '')
+    {
+        list($toc, $str) = $this->generateDocumentation($outputExampleUrls, $prefixUrls, $displayTitlesAsAngularDirective = false);
+
+        return "<h2 id='topApiRef' name='topApiRef'>Quick access to APIs</h2>
+				$toc
+				$str";
+    }
+
+    protected function prepareModuleToDisplay($moduleName)
+    {
+        return "<a href='#$moduleName'>$moduleName</a><br/>";
+    }
+
+    protected function prepareMethodToDisplay($moduleName, $info, $methods, $class, $outputExampleUrls, $prefixUrls, $displayTitlesAsAngularDirective)
+    {
+        $str = '';
+        $str .= "\n<a name='$moduleName' id='$moduleName'></a>";
+        if($displayTitlesAsAngularDirective) {
+            $str .= "<div piwik-content-block content-title='Module " . $moduleName . "'>";
+        } else {
+            $str .= "<h2>Module " . $moduleName . "</h2>";
+        }
+        $info['__documentation'] = $this->checkDocumentation($info['__documentation']);
+        $str .= "<div class='apiDescription'> " . $info['__documentation'] . " </div>";
+        foreach ($methods as $methodName) {
+            if (Proxy::getInstance()->isDeprecatedMethod($class, $methodName)) {
+                continue;
+            }
+
+            $params = $this->getParametersString($class, $methodName);
+
+            $str .= "\n <div class='apiMethod'>- <b>$moduleName.$methodName </b>" . $params . "";
+            $str .= '<small>';
+            if ($outputExampleUrls) {
+                $str .= $this->addExamples($class, $methodName, $prefixUrls);
+            }
+            $str .= '</small>';
+            $str .= "</div>\n";
+        }
+
+        if($displayTitlesAsAngularDirective) {
+            $str .= "</div>";
+        }
+
+        return $str;
+    }
+
+    protected function prepareModulesAndMethods($info, $moduleName)
+    {
+        $toDisplay = array();
+
+        foreach ($info as $methodName => $infoMethod) {
+            if ($methodName == '__documentation') {
+                continue;
+            }
+            $toDisplay[$moduleName][] = $methodName;
+        }
+
+        return $toDisplay;
+    }
+
+    protected function addExamples($class, $methodName, $prefixUrls)
+    {
+        $token_auth = "&token_auth=" . Piwik::getCurrentUserTokenAuth();
+        $parametersToSet = array(
+            'idSite' => Common::getRequestVar('idSite', 1, 'int'),
+            'period' => Common::getRequestVar('period', 'day', 'string'),
+            'date' => Common::getRequestVar('date', 'today', 'string')
+        );
+        $str = '';
+        $str .= "<span class=\"example\">";
+        $exampleUrl = $this->getExampleUrl($class, $methodName, $parametersToSet);
+        if ($exampleUrl !== false) {
+            $lastNUrls = '';
+            if (preg_match('/(&period)|(&date)/', $exampleUrl)) {
+                $exampleUrlRss = $prefixUrls . $this->getExampleUrl($class, $methodName, array('date' => 'last10', 'period' => 'day') + $parametersToSet);
+                $lastNUrls = ", RSS of the last <a target='_blank' href='$exampleUrlRss&format=rss$token_auth&translateColumnNames=1'>10 days</a>";
+            }
+            $exampleUrl = $prefixUrls . $exampleUrl;
+            $str .= " [ Example in
+                                                                    <a target='_blank' href='$exampleUrl&format=xml$token_auth'>XML</a>,
+                                                                    <a target='_blank' href='$exampleUrl&format=JSON$token_auth'>Json</a>,
+                                                                    <a target='_blank' href='$exampleUrl&format=Tsv$token_auth&translateColumnNames=1'>Tsv (Excel)</a>
+                                                                    $lastNUrls
+                                                                    ]";
+        } else {
+            $str .= " [ No example available ]";
+        }
+        $str .= "</span>";
+        return $str;
+    }
+
+    /**
+     * Check if Class contains @hide
+     *
+     * @param ReflectionClass $rClass instance of ReflectionMethod
+     * @return bool
+     */
+    public function checkIfClassCommentContainsHideAnnotation(ReflectionClass $rClass)
+    {
+        return false !== strstr($rClass->getDocComment(), '@hide');
+    }
+
+    /**
+     * Check if Class contains @internal
+     *
+     * @param ReflectionClass|\ReflectionMethod $rClass instance of ReflectionMethod
+     * @return bool
+     */
+    private function checkIfCommentContainsInternalAnnotation($rClass)
+    {
+        return false !== strstr($rClass->getDocComment(), '@internal');
+    }
+
+    /**
+     * Check if documentation contains @hide annotation and deletes it
+     *
+     * @param $moduleToCheck
+     * @return mixed
+     */
+    public function checkDocumentation($moduleToCheck)
+    {
+        if (strpos($moduleToCheck, '@hide') == true) {
+            $moduleToCheck = str_replace(strtok(strstr($moduleToCheck, '@hide'), "\n"), "", $moduleToCheck);
+        }
+        return $moduleToCheck;
+    }
+
+    /**
+     * Returns a string containing links to examples on how to call a given method on a given API
+     * It will export links to XML, CSV, HTML, JSON, PHP, etc.
+     * It will not export links for methods such as deleteSite or deleteUser
+     *
+     * @param string $class the class
+     * @param string $methodName the method
+     * @param array $parametersToSet parameters to set
+     * @return string|bool when not possible
+     */
+    public function getExampleUrl($class, $methodName, $parametersToSet = array())
+    {
+        $knowExampleDefaultParametersValues = array(
+            'access'         => 'view',
+            'userLogin'      => 'test',
+            'passwordMd5ied' => 'passwordExample',
+            'email'          => 'test@example.org',
+
+            'languageCode'   => 'fr',
+            'url'            => 'http://forum.piwik.org/',
+            'pageUrl'        => 'http://forum.piwik.org/',
+            'apiModule'      => 'UserCountry',
+            'apiAction'      => 'getCountry',
+            'lastMinutes'    => '30',
+            'abandonedCarts' => '0',
+            'segmentName'    => 'pageTitle',
+            'ip'             => '194.57.91.215',
+            'idSites'             => '1,2',
+            'idAlert'             => '1',
+            'seconds'        => '3600',
+//            'segmentName'    => 'browserCode',
+        );
+
+        foreach ($parametersToSet as $name => $value) {
+            $knowExampleDefaultParametersValues[$name] = $value;
+        }
+
+        // no links for these method names
+        $doNotPrintExampleForTheseMethods = array(
+            //Sites
+            'deleteSite',
+            'addSite',
+            'updateSite',
+            'addSiteAliasUrls',
+            //Users
+            'deleteUser',
+            'addUser',
+            'updateUser',
+            'setUserAccess',
+            //Goals
+            'addGoal',
+            'updateGoal',
+            'deleteGoal',
+            //Marketplace
+            'deleteLicenseKey'
+        );
+
+        if (in_array($methodName, $doNotPrintExampleForTheseMethods)) {
+            return false;
+        }
+
+        // we try to give an URL example to call the API
+        $aParameters = Proxy::getInstance()->getParametersList($class, $methodName);
+        $aParameters['format'] = false;
+        $aParameters['hideIdSubDatable'] = false;
+        $aParameters['serialize'] = false;
+        $aParameters['language'] = false;
+        $aParameters['translateColumnNames'] = false;
+        $aParameters['label'] = false;
+        $aParameters['flat'] = false;
+        $aParameters['include_aggregate_rows'] = false;
+        $aParameters['filter_offset'] = false; 
+        $aParameters['filter_limit'] = false; 
+        $aParameters['filter_sort_column'] = false; 
+        $aParameters['filter_sort_order'] = false; 
+        $aParameters['filter_excludelowpop'] = false; 
+        $aParameters['filter_excludelowpop_value'] = false; 
+        $aParameters['filter_column_recursive'] = false; 
+        $aParameters['filter_pattern'] = false; 
+        $aParameters['filter_pattern_recursive'] = false; 
+        $aParameters['filter_truncate'] = false;
+        $aParameters['hideColumns'] = false;
+        $aParameters['showColumns'] = false;
+        $aParameters['filter_pattern_recursive'] = false;
+        $aParameters['pivotBy'] = false;
+        $aParameters['pivotByColumn'] = false;
+        $aParameters['pivotByColumnLimit'] = false;
+        $aParameters['disable_queued_filters'] = false;
+        $aParameters['disable_generic_filters'] = false;
+        $aParameters['expanded'] = false;
+        $aParameters['idDimenson'] = false;
+        $aParameters['format_metrics'] = false;
+
+        $entityNames = StaticContainer::get('entities.idNames');
+        foreach ($entityNames as $entityName) {
+            if (isset($aParameters[$entityName])) {
+                continue;
+            }
+            $aParameters[$entityName] = false;
+        }
+
+        $moduleName = Proxy::getInstance()->getModuleNameFromClassName($class);
+        $aParameters = array_merge(array('module' => 'API', 'method' => $moduleName . '.' . $methodName), $aParameters);
+
+        foreach ($aParameters as $nameVariable => &$defaultValue) {
+            if (isset($knowExampleDefaultParametersValues[$nameVariable])) {
+                $defaultValue = $knowExampleDefaultParametersValues[$nameVariable];
+            } // if there isn't a default value for a given parameter,
+            // we need a 'know default value' or we can't generate the link
+            elseif ($defaultValue instanceof NoDefaultValue) {
+                return false;
+            }
+        }
+        return '?' . Url::getQueryStringFromParameters($aParameters);
+    }
+
+    /**
+     * Returns the methods $class.$name parameters (and default value if provided) as a string.
+     *
+     * @param string $class The class name
+     * @param string $name The method name
+     * @return string  For example "(idSite, period, date = 'today')"
+     */
+    protected function getParametersString($class, $name)
+    {
+        $aParameters = Proxy::getInstance()->getParametersList($class, $name);
+        $asParameters = array();
+        foreach ($aParameters as $nameVariable => $defaultValue) {
+            // Do not show API parameters starting with _
+            // They are supposed to be used only in internal API calls
+            if (strpos($nameVariable, '_') === 0) {
+                continue;
+            }
+            $str = $nameVariable;
+            if (!($defaultValue instanceof NoDefaultValue)) {
+                if (is_array($defaultValue)) {
+                    $str .= " = 'Array'";
+                } else {
+                    $str .= " = '$defaultValue'";
+                }
+            }
+            $asParameters[] = $str;
+        }
+        $sParameters = implode(", ", $asParameters);
+        return "($sParameters)";
+    }
+
+    /**
+     * @param $outputExampleUrls
+     * @param $prefixUrls
+     * @param $displayTitlesAsAngularDirective
+     * @return array
+     */
+    protected function generateDocumentation($outputExampleUrls, $prefixUrls, $displayTitlesAsAngularDirective)
+    {
+        $str = $toc = '';
+
+        foreach (Proxy::getInstance()->getMetadata() as $class => $info) {
+            $moduleName = Proxy::getInstance()->getModuleNameFromClassName($class);
+            $rClass = new ReflectionClass($class);
+
+            if (!Piwik::hasUserSuperUserAccess() && $this->checkIfClassCommentContainsHideAnnotation($rClass)) {
+                continue;
+            }
+
+            if ($this->checkIfCommentContainsInternalAnnotation($rClass)) {
+                continue;
+            }
+
+            $toDisplay = $this->prepareModulesAndMethods($info, $moduleName);
+
+            foreach ($toDisplay as $moduleName => $methods) {
+                foreach ($methods as $index => $method) {
+                    if (!method_exists($class, $method)) { // method is handled through API.Request.intercept event
+                        continue;
+                    }
+
+                    $reflectionMethod = new \ReflectionMethod($class, $method);
+                    if ($this->checkIfCommentContainsInternalAnnotation($reflectionMethod)) {
+                        unset($toDisplay[$moduleName][$index]);
+                    }
+                }
+                if (empty($toDisplay[$moduleName])) {
+                    unset($toDisplay[$moduleName]);
+                }
+            }
+
+            foreach ($toDisplay as $moduleName => $methods) {
+                $toc .= $this->prepareModuleToDisplay($moduleName);
+                $str .= $this->prepareMethodToDisplay($moduleName, $info, $methods, $class, $outputExampleUrls, $prefixUrls, $displayTitlesAsAngularDirective);
+            }
+        }
+        return array($toc, $str);
+    }
+}

msd2/tracking/piwik/core/API/Inconsistencies.php

@@ -0,0 +1,45 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+namespace Piwik\API;
+
+/**
+ * Contains logic to replicate inconsistencies in Piwik's API. This class exists
+ * to provide a way to clean up existing Piwik code and behavior without breaking
+ * backwards compatibility immediately.
+ *
+ * Code that handles the case when the 'format_metrics' query parameter value is
+ * 'bc' should be removed as well. This code is in API\Request and DataTablePostProcessor.
+ *
+ * Should be removed before releasing Piwik 3.0.
+ */
+class Inconsistencies
+{
+    /**
+     * In Piwik 2.X and below, the "raw" API would format percent values but no others.
+     * This method returns the list of percent metrics that were returned from the API
+     * formatted so we can maintain BC.
+     *
+     * Used by DataTablePostProcessor.
+     */
+    public function getPercentMetricsToFormat()
+    {
+        return array(
+            'bounce_rate',
+            'conversion_rate',
+            'abandoned_rate',
+            'interaction_rate',
+            'exit_rate',
+            'bounce_rate_returning',
+            'nb_visits_percentage',
+            '/.*_evolution/',
+            '/goal_.*_conversion_rate/',
+            '/form_.*_rate/',
+            '/field_.*_rate/',
+        );
+    }
+}

msd2/tracking/piwik/core/API/Proxy.php

@@ -0,0 +1,582 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+
+namespace Piwik\API;
+
+use Exception;
+use Piwik\Common;
+use Piwik\Container\StaticContainer;
+use Piwik\Context;
+use Piwik\Piwik;
+use Piwik\Plugin\Manager;
+use Piwik\Singleton;
+use ReflectionClass;
+use ReflectionMethod;
+
+/**
+ * Proxy is a singleton that has the knowledge of every method available, their parameters
+ * and default values.
+ * Proxy receives all the API calls requests via call() and forwards them to the right
+ * object, with the parameters in the right order.
+ *
+ * It will also log the performance of API calls (time spent, parameter values, etc.) if logger available
+ */
+class Proxy
+{
+    // array of already registered plugins names
+    protected $alreadyRegistered = array();
+
+    protected $metadataArray = array();
+    private $hideIgnoredFunctions = true;
+
+    // when a parameter doesn't have a default value we use this
+    private $noDefaultValue;
+
+    public function __construct()
+    {
+        $this->noDefaultValue = new NoDefaultValue();
+    }
+
+    public static function getInstance()
+    {
+        return StaticContainer::get(self::class);
+    }
+
+    /**
+     * Returns array containing reflection meta data for all the loaded classes
+     * eg. number of parameters, method names, etc.
+     *
+     * @return array
+     */
+    public function getMetadata()
+    {
+        ksort($this->metadataArray);
+        return $this->metadataArray;
+    }
+
+    /**
+     * Registers the API information of a given module.
+     *
+     * The module to be registered must be
+     * - a singleton (providing a getInstance() method)
+     * - the API file must be located in plugins/ModuleName/API.php
+     *   for example plugins/Referrers/API.php
+     *
+     * The method will introspect the methods, their parameters, etc.
+     *
+     * @param string $className ModuleName eg. "API"
+     */
+    public function registerClass($className)
+    {
+        if (isset($this->alreadyRegistered[$className])) {
+            return;
+        }
+        $this->includeApiFile($className);
+        $this->checkClassIsSingleton($className);
+
+        $rClass = new ReflectionClass($className);
+        if (!$this->shouldHideAPIMethod($rClass->getDocComment())) {
+            foreach ($rClass->getMethods() as $method) {
+                $this->loadMethodMetadata($className, $method);
+            }
+
+            $this->setDocumentation($rClass, $className);
+            $this->alreadyRegistered[$className] = true;
+        }
+    }
+
+    /**
+     * Will be displayed in the API page
+     *
+     * @param ReflectionClass $rClass Instance of ReflectionClass
+     * @param string $className Name of the class
+     */
+    private function setDocumentation($rClass, $className)
+    {
+        // Doc comment
+        $doc = $rClass->getDocComment();
+        $doc = str_replace(" * " . PHP_EOL, "<br>", $doc);
+
+        // boldify the first line only if there is more than one line, otherwise too much bold
+        if (substr_count($doc, '<br>') > 1) {
+            $firstLineBreak = strpos($doc, "<br>");
+            $doc = "<div class='apiFirstLine'>" . substr($doc, 0, $firstLineBreak) . "</div>" . substr($doc, $firstLineBreak + strlen("<br>"));
+        }
+        $doc = preg_replace("/(@package)[a-z _A-Z]*/", "", $doc);
+        $doc = preg_replace("/(@method).*/", "", $doc);
+        $doc = str_replace(array("\t", "\n", "/**", "*/", " * ", " *", "  ", "\t*", "  *  @package"), " ", $doc);
+
+        // replace 'foo' and `bar` and "foobar" with code blocks... much magic
+        $doc = preg_replace('/`(.*?)`/', '<code>$1</code>', $doc);
+        $this->metadataArray[$className]['__documentation'] = $doc;
+    }
+
+    /**
+     * Returns number of classes already loaded
+     * @return int
+     */
+    public function getCountRegisteredClasses()
+    {
+        return count($this->alreadyRegistered);
+    }
+
+    /**
+     * Will execute $className->$methodName($parametersValues)
+     * If any error is detected (wrong number of parameters, method not found, class not found, etc.)
+     * it will throw an exception
+     *
+     * It also logs the API calls, with the parameters values, the returned value, the performance, etc.
+     * You can enable logging in config/global.ini.php (log_api_call)
+     *
+     * @param string $className The class name (eg. API)
+     * @param string $methodName The method name
+     * @param array $parametersRequest The parameters pairs (name=>value)
+     *
+     * @return mixed|null
+     * @throws Exception|\Piwik\NoAccessException
+     */
+    public function call($className, $methodName, $parametersRequest)
+    {
+        // Temporarily sets the Request array to this API call context
+        return Context::executeWithQueryParameters($parametersRequest, function () use ($className, $methodName, $parametersRequest) {
+            $returnedValue = null;
+
+            $this->registerClass($className);
+
+            // instanciate the object
+            $object = $className::getInstance();
+
+            // check method exists
+            $this->checkMethodExists($className, $methodName);
+
+            // get the list of parameters required by the method
+            $parameterNamesDefaultValues = $this->getParametersList($className, $methodName);
+
+            // load parameters in the right order, etc.
+            $finalParameters = $this->getRequestParametersArray($parameterNamesDefaultValues, $parametersRequest);
+
+            // allow plugins to manipulate the value
+            $pluginName = $this->getModuleNameFromClassName($className);
+
+            $returnedValue = null;
+
+            /**
+             * Triggered before an API request is dispatched.
+             *
+             * This event can be used to modify the arguments passed to one or more API methods.
+             *
+             * **Example**
+             *
+             *     Piwik::addAction('API.Request.dispatch', function (&$parameters, $pluginName, $methodName) {
+             *         if ($pluginName == 'Actions') {
+             *             if ($methodName == 'getPageUrls') {
+             *                 // ... do something ...
+             *             } else {
+             *                 // ... do something else ...
+             *             }
+             *         }
+             *     });
+             *
+             * @param array &$finalParameters List of parameters that will be passed to the API method.
+             * @param string $pluginName The name of the plugin the API method belongs to.
+             * @param string $methodName The name of the API method that will be called.
+             */
+            Piwik::postEvent('API.Request.dispatch', array(&$finalParameters, $pluginName, $methodName));
+
+            /**
+             * Triggered before an API request is dispatched.
+             *
+             * This event exists for convenience and is triggered directly after the {@hook API.Request.dispatch}
+             * event is triggered. It can be used to modify the arguments passed to a **single** API method.
+             *
+             * _Note: This is can be accomplished with the {@hook API.Request.dispatch} event as well, however
+             * event handlers for that event will have to do more work._
+             *
+             * **Example**
+             *
+             *     Piwik::addAction('API.Actions.getPageUrls', function (&$parameters) {
+             *         // force use of a single website. for some reason.
+             *         $parameters['idSite'] = 1;
+             *     });
+             *
+             * @param array &$finalParameters List of parameters that will be passed to the API method.
+             */
+            Piwik::postEvent(sprintf('API.%s.%s', $pluginName, $methodName), array(&$finalParameters));
+
+            /**
+             * Triggered before an API request is dispatched.
+             *
+             * Use this event to intercept an API request and execute your own code instead. If you set
+             * `$returnedValue` in a handler for this event, the original API method will not be executed,
+             * and the result will be what you set in the event handler.
+             *
+             * @param mixed &$returnedValue Set this to set the result and preempt normal API invocation.
+             * @param array &$finalParameters List of parameters that will be passed to the API method.
+             * @param string $pluginName The name of the plugin the API method belongs to.
+             * @param string $methodName The name of the API method that will be called.
+             * @param array $parametersRequest The query parameters for this request.
+             */
+            Piwik::postEvent('API.Request.intercept', [&$returnedValue, $finalParameters, $pluginName, $methodName, $parametersRequest]);
+
+            $apiParametersInCorrectOrder = array();
+
+            foreach ($parameterNamesDefaultValues as $name => $defaultValue) {
+                if (isset($finalParameters[$name]) || array_key_exists($name, $finalParameters)) {
+                    $apiParametersInCorrectOrder[] = $finalParameters[$name];
+                }
+            }
+
+            // call the method if a hook hasn't already set an output variable
+            if ($returnedValue === null) {
+                $returnedValue = call_user_func_array(array($object, $methodName), $apiParametersInCorrectOrder);
+            }
+
+            $endHookParams = array(
+                &$returnedValue,
+                array('className'  => $className,
+                    'module'     => $pluginName,
+                    'action'     => $methodName,
+                    'parameters' => $finalParameters)
+            );
+
+            /**
+             * Triggered directly after an API request is dispatched.
+             *
+             * This event exists for convenience and is triggered immediately before the
+             * {@hook API.Request.dispatch.end} event. It can be used to modify the output of a **single**
+             * API method.
+             *
+             * _Note: This can be accomplished with the {@hook API.Request.dispatch.end} event as well,
+             * however event handlers for that event will have to do more work._
+             *
+             * **Example**
+             *
+             *     // append (0 hits) to the end of row labels whose row has 0 hits
+             *     Piwik::addAction('API.Actions.getPageUrls', function (&$returnValue, $info)) {
+             *         $returnValue->filter('ColumnCallbackReplace', 'label', function ($label, $hits) {
+             *             if ($hits === 0) {
+             *                 return $label . " (0 hits)";
+             *             } else {
+             *                 return $label;
+             *             }
+             *         }, null, array('nb_hits'));
+             *     }
+             *
+             * @param mixed &$returnedValue The API method's return value. Can be an object, such as a
+             *                              {@link Piwik\DataTable DataTable} instance.
+             *                              could be a {@link Piwik\DataTable DataTable}.
+             * @param array $extraInfo An array holding information regarding the API request. Will
+             *                         contain the following data:
+             *
+             *                         - **className**: The namespace-d class name of the API instance
+             *                                          that's being called.
+             *                         - **module**: The name of the plugin the API request was
+             *                                       dispatched to.
+             *                         - **action**: The name of the API method that was executed.
+             *                         - **parameters**: The array of parameters passed to the API
+             *                                           method.
+             */
+            Piwik::postEvent(sprintf('API.%s.%s.end', $pluginName, $methodName), $endHookParams);
+
+            /**
+             * Triggered directly after an API request is dispatched.
+             *
+             * This event can be used to modify the output of any API method.
+             *
+             * **Example**
+             *
+             *     // append (0 hits) to the end of row labels whose row has 0 hits for any report that has the 'nb_hits' metric
+             *     Piwik::addAction('API.Actions.getPageUrls.end', function (&$returnValue, $info)) {
+             *         // don't process non-DataTable reports and reports that don't have the nb_hits column
+             *         if (!($returnValue instanceof DataTableInterface)
+             *             || in_array('nb_hits', $returnValue->getColumns())
+             *         ) {
+             *             return;
+             *         }
+             *
+             *         $returnValue->filter('ColumnCallbackReplace', 'label', function ($label, $hits) {
+             *             if ($hits === 0) {
+             *                 return $label . " (0 hits)";
+             *             } else {
+             *                 return $label;
+             *             }
+             *         }, null, array('nb_hits'));
+             *     }
+             *
+             * @param mixed &$returnedValue The API method's return value. Can be an object, such as a
+             *                              {@link Piwik\DataTable DataTable} instance.
+             * @param array $extraInfo An array holding information regarding the API request. Will
+             *                         contain the following data:
+             *
+             *                         - **className**: The namespace-d class name of the API instance
+             *                                          that's being called.
+             *                         - **module**: The name of the plugin the API request was
+             *                                       dispatched to.
+             *                         - **action**: The name of the API method that was executed.
+             *                         - **parameters**: The array of parameters passed to the API
+             *                                           method.
+             */
+            Piwik::postEvent('API.Request.dispatch.end', $endHookParams);
+
+            return $returnedValue;
+        });
+    }
+
+    /**
+     * Returns the parameters names and default values for the method $name
+     * of the class $class
+     *
+     * @param string $class The class name
+     * @param string $name The method name
+     * @return array  Format array(
+     *                            'testParameter' => null, // no default value
+     *                            'life'          => 42, // default value = 42
+     *                            'date'          => 'yesterday',
+     *                       );
+     */
+    public function getParametersList($class, $name)
+    {
+        return $this->metadataArray[$class][$name]['parameters'];
+    }
+
+    /**
+     * Check if given method name is deprecated or not.
+     */
+    public function isDeprecatedMethod($class, $methodName)
+    {
+        return $this->metadataArray[$class][$methodName]['isDeprecated'];
+    }
+
+    /**
+     * Returns the 'moduleName' part of '\\Piwik\\Plugins\\moduleName\\API'
+     *
+     * @param string $className "API"
+     * @return string "Referrers"
+     */
+    public function getModuleNameFromClassName($className)
+    {
+        return str_replace(array('\\Piwik\\Plugins\\', '\\API'), '', $className);
+    }
+
+    public function isExistingApiAction($pluginName, $apiAction)
+    {
+        $namespacedApiClassName = "\\Piwik\\Plugins\\$pluginName\\API";
+        $api = $namespacedApiClassName::getInstance();
+
+        return method_exists($api, $apiAction);
+    }
+
+    public function buildApiActionName($pluginName, $apiAction)
+    {
+        return sprintf("%s.%s", $pluginName, $apiAction);
+    }
+
+    /**
+     * Sets whether to hide '@ignore'd functions from method metadata or not.
+     *
+     * @param bool $hideIgnoredFunctions
+     */
+    public function setHideIgnoredFunctions($hideIgnoredFunctions)
+    {
+        $this->hideIgnoredFunctions = $hideIgnoredFunctions;
+
+        // make sure metadata gets reloaded
+        $this->alreadyRegistered = array();
+        $this->metadataArray = array();
+    }
+
+    /**
+     * Returns an array containing the values of the parameters to pass to the method to call
+     *
+     * @param array $requiredParameters array of (parameter name, default value)
+     * @param array $parametersRequest
+     * @throws Exception
+     * @return array values to pass to the function call
+     */
+    private function getRequestParametersArray($requiredParameters, $parametersRequest)
+    {
+        $finalParameters = array();
+        foreach ($requiredParameters as $name => $defaultValue) {
+            try {
+                if ($defaultValue instanceof NoDefaultValue) {
+                    $requestValue = Common::getRequestVar($name, null, null, $parametersRequest);
+                } else {
+                    try {
+                        if ($name == 'segment' && !empty($parametersRequest['segment'])) {
+                            // segment parameter is an exception: we do not want to sanitize user input or it would break the segment encoding
+                            $requestValue = ($parametersRequest['segment']);
+                        } else {
+                            $requestValue = Common::getRequestVar($name, $defaultValue, null, $parametersRequest);
+                        }
+                    } catch (Exception $e) {
+                        // Special case: empty parameter in the URL, should return the empty string
+                        if (isset($parametersRequest[$name])
+                            && $parametersRequest[$name] === ''
+                        ) {
+                            $requestValue = '';
+                        } else {
+                            $requestValue = $defaultValue;
+                        }
+                    }
+                }
+            } catch (Exception $e) {
+                throw new Exception(Piwik::translate('General_PleaseSpecifyValue', array($name)));
+            }
+            $finalParameters[$name] = $requestValue;
+        }
+        return $finalParameters;
+    }
+
+    /**
+     * Includes the class API by looking up plugins/xxx/API.php
+     *
+     * @param string $fileName api class name eg. "API"
+     * @throws Exception
+     */
+    private function includeApiFile($fileName)
+    {
+        $module = self::getModuleNameFromClassName($fileName);
+        $path = Manager::getPluginDirectory($module) . '/API.php';
+
+        if (is_readable($path)) {
+            require_once $path; // prefixed by PIWIK_INCLUDE_PATH
+        } else {
+            throw new Exception("API module $module not found.");
+        }
+    }
+
+    /**
+     * @param string $class name of a class
+     * @param ReflectionMethod $method instance of ReflectionMethod
+     */
+    private function loadMethodMetadata($class, $method)
+    {
+        if (!$this->checkIfMethodIsAvailable($method)) {
+            return;
+        }
+        $name = $method->getName();
+        $parameters = $method->getParameters();
+        $docComment = $method->getDocComment();
+
+        $aParameters = array();
+        foreach ($parameters as $parameter) {
+            $nameVariable = $parameter->getName();
+
+            $defaultValue = $this->noDefaultValue;
+            if ($parameter->isDefaultValueAvailable()) {
+                $defaultValue = $parameter->getDefaultValue();
+            }
+
+            $aParameters[$nameVariable] = $defaultValue;
+        }
+        $this->metadataArray[$class][$name]['parameters'] = $aParameters;
+        $this->metadataArray[$class][$name]['numberOfRequiredParameters'] = $method->getNumberOfRequiredParameters();
+        $this->metadataArray[$class][$name]['isDeprecated'] = false !== strstr($docComment, '@deprecated');
+    }
+
+    /**
+     * Checks that the method exists in the class
+     *
+     * @param string $className The class name
+     * @param string $methodName The method name
+     * @throws Exception If the method is not found
+     */
+    private function checkMethodExists($className, $methodName)
+    {
+        if (!$this->isMethodAvailable($className, $methodName)) {
+            throw new Exception(Piwik::translate('General_ExceptionMethodNotFound', array($methodName, $className)));
+        }
+    }
+
+    /**
+     * @param $docComment
+     * @return bool
+     */
+    public function shouldHideAPIMethod($docComment)
+    {
+        $hideLine = strstr($docComment, '@hide');
+
+        if ($hideLine === false) {
+            return false;
+        }
+
+        $hideLine = trim($hideLine);
+        $hideLine .= ' ';
+
+        $token = trim(strtok($hideLine, " "), "\n");
+
+        $hide = false;
+
+        if (!empty($token)) {
+            /**
+             * This event exists for checking whether a Plugin API class or a Plugin API method tagged
+             * with a `@hideXYZ` should be hidden in the API listing.
+             *
+             * @param bool &$hide whether to hide APIs tagged with $token should be displayed.
+             */
+            Piwik::postEvent(sprintf('API.DocumentationGenerator.%s', $token), array(&$hide));
+        }
+
+        return $hide;
+    }
+
+    /**
+     * @param ReflectionMethod $method
+     * @return bool
+     */
+    protected function checkIfMethodIsAvailable(ReflectionMethod $method)
+    {
+        if (!$method->isPublic() || $method->isConstructor() || $method->getName() === 'getInstance') {
+            return false;
+        }
+
+        if ($this->hideIgnoredFunctions && false !== strstr($method->getDocComment(), '@ignore')) {
+            return false;
+        }
+
+        if ($this->shouldHideAPIMethod($method->getDocComment())) {
+            return false;
+        }
+
+        return true;
+    }
+
+    /**
+     * Returns true if the method is found in the API of the given class name.
+     *
+     * @param string $className The class name
+     * @param string $methodName The method name
+     * @return bool
+     */
+    private function isMethodAvailable($className, $methodName)
+    {
+        return isset($this->metadataArray[$className][$methodName]);
+    }
+
+    /**
+     * Checks that the class is a Singleton (presence of the getInstance() method)
+     *
+     * @param string $className The class name
+     * @throws Exception If the class is not a Singleton
+     */
+    private function checkClassIsSingleton($className)
+    {
+        if (!method_exists($className, "getInstance")) {
+            throw new Exception("$className that provide an API must be Singleton and have a 'public static function getInstance()' method.");
+        }
+    }
+}
+
+/**
+ * To differentiate between "no value" and default value of null
+ *
+ */
+class NoDefaultValue
+{
+}

msd2/tracking/piwik/core/API/Request.php

@@ -0,0 +1,643 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\API;
+
+use Exception;
+use Piwik\Access;
+use Piwik\Cache;
+use Piwik\Common;
+use Piwik\Container\StaticContainer;
+use Piwik\Context;
+use Piwik\DataTable;
+use Piwik\Exception\PluginDeactivatedException;
+use Piwik\IP;
+use Piwik\Log;
+use Piwik\Piwik;
+use Piwik\Plugin\Manager as PluginManager;
+use Piwik\Plugins\CoreHome\LoginWhitelist;
+use Piwik\SettingsServer;
+use Piwik\Url;
+use Piwik\UrlHelper;
+use Psr\Log\LoggerInterface;
+
+/**
+ * Dispatches API requests to the appropriate API method.
+ *
+ * The Request class is used throughout Piwik to call API methods. The difference
+ * between using Request and calling API methods directly is that Request
+ * will do more after calling the API including: applying generic filters, applying queued filters,
+ * and handling the **flat** and **label** query parameters.
+ *
+ * Additionally, the Request class will **forward current query parameters** to the request
+ * which is more convenient than calling {@link Piwik\Common::getRequestVar()} many times over.
+ *
+ * In most cases, using a Request object to query the API is the correct approach.
+ *
+ * ### Post-processing
+ *
+ * The return value of API methods undergo some extra processing before being returned by Request.
+ *
+ * ### Output Formats
+ *
+ * The value returned by Request will be serialized to a certain format before being returned.
+ *
+ * ### Examples
+ *
+ * **Basic Usage**
+ *
+ *     $request = new Request('method=UserLanguage.getLanguage&idSite=1&date=yesterday&period=week'
+ *                          . '&format=xml&filter_limit=5&filter_offset=0')
+ *     $result = $request->process();
+ *     echo $result;
+ *
+ * **Getting a unrendered DataTable**
+ *
+ *     // use the convenience method 'processRequest'
+ *     $dataTable = Request::processRequest('UserLanguage.getLanguage', array(
+ *         'idSite' => 1,
+ *         'date' => 'yesterday',
+ *         'period' => 'week',
+ *         'filter_limit' => 5,
+ *         'filter_offset' => 0
+ *
+ *         'format' => 'original', // this is the important bit
+ *     ));
+ *     echo "This DataTable has " . $dataTable->getRowsCount() . " rows.";
+ *
+ * @see http://piwik.org/docs/analytics-api
+ * @api
+ */
+class Request
+{
+    /**
+     * The count of nested API request invocations. Used to determine if the currently executing request is the root or not.
+     *
+     * @var int
+     */
+    private static $nestedApiInvocationCount = 0;
+
+    private $request = null;
+
+    /**
+     * Converts the supplied request string into an array of query paramater name/value
+     * mappings. The current query parameters (everything in `$_GET` and `$_POST`) are
+     * forwarded to request array before it is returned.
+     *
+     * @param string|array|null $request The base request string or array, eg,
+     *                                   `'module=UserLanguage&action=getLanguage'`.
+     * @param array $defaultRequest Default query parameters. If a query parameter is absent in `$request`, it will be loaded
+     *                              from this. Defaults to `$_GET + $_POST`.
+     * @return array
+     */
+    public static function getRequestArrayFromString($request, $defaultRequest = null)
+    {
+        if ($defaultRequest === null) {
+            $defaultRequest = self::getDefaultRequest();
+
+            $requestRaw = self::getRequestParametersGET();
+            if (!empty($requestRaw['segment'])) {
+                $defaultRequest['segment'] = $requestRaw['segment'];
+            }
+
+            if (!isset($defaultRequest['format_metrics'])) {
+                $defaultRequest['format_metrics'] = 'bc';
+            }
+        }
+
+        $requestArray = $defaultRequest;
+
+        if (!is_null($request)) {
+            if (is_array($request)) {
+                $requestParsed = $request;
+            } else {
+                $request = trim($request);
+                $request = str_replace(array("\n", "\t"), '', $request);
+
+                $requestParsed = UrlHelper::getArrayFromQueryString($request);
+            }
+
+            $requestArray = $requestParsed + $defaultRequest;
+        }
+
+        foreach ($requestArray as &$element) {
+            if (!is_array($element)) {
+                $element = trim($element);
+            }
+        }
+        return $requestArray;
+    }
+
+    /**
+     * Constructor.
+     *
+     * @param string|array $request Query string that defines the API call (must at least contain a **method** parameter),
+     *                              eg, `'method=UserLanguage.getLanguage&idSite=1&date=yesterday&period=week&format=xml'`
+     *                              If a request is not provided, then we use the values in the `$_GET` and `$_POST`
+     *                              superglobals.
+     * @param array $defaultRequest Default query parameters. If a query parameter is absent in `$request`, it will be loaded
+     *                              from this. Defaults to `$_GET + $_POST`.
+     */
+    public function __construct($request = null, $defaultRequest = null)
+    {
+        $this->request = self::getRequestArrayFromString($request, $defaultRequest);
+        $this->sanitizeRequest();
+        $this->renameModuleAndActionInRequest();
+    }
+
+    /**
+     * For backward compatibility: Piwik API still works if module=Referers,
+     * we rewrite to correct renamed plugin: Referrers
+     *
+     * @param $module
+     * @param $action
+     * @return array( $module, $action )
+     * @ignore
+     */
+    public static function getRenamedModuleAndAction($module, $action)
+    {
+        /**
+         * This event is posted in the Request dispatcher and can be used
+         * to overwrite the Module and Action to dispatch.
+         * This is useful when some Controller methods or API methods have been renamed or moved to another plugin.
+         *
+         * @param $module string
+         * @param $action string
+         */
+        Piwik::postEvent('Request.getRenamedModuleAndAction', array(&$module, &$action));
+
+        return array($module, $action);
+    }
+
+    /**
+     * Make sure that the request contains no logical errors
+     */
+    private function sanitizeRequest()
+    {
+        // The label filter does not work with expanded=1 because the data table IDs have a different meaning
+        // depending on whether the table has been loaded yet. expanded=1 causes all tables to be loaded, which
+        // is why the label filter can't descend when a recursive label has been requested.
+        // To fix this, we remove the expanded parameter if a label parameter is set.
+        if (isset($this->request['label']) && !empty($this->request['label'])
+            && isset($this->request['expanded']) && $this->request['expanded']
+        ) {
+            unset($this->request['expanded']);
+        }
+    }
+
+    /**
+     * Dispatches the API request to the appropriate API method and returns the result
+     * after post-processing.
+     *
+     * Post-processing includes:
+     *
+     * - flattening if **flat** is 0
+     * - running generic filters unless **disable_generic_filters** is set to 1
+     * - URL decoding label column values
+     * - running queued filters unless **disable_queued_filters** is set to 1
+     * - removing columns based on the values of the **hideColumns** and **showColumns** query parameters
+     * - filtering rows if the **label** query parameter is set
+     * - converting the result to the appropriate format (ie, XML, JSON, etc.)
+     *
+     * If `'original'` is supplied for the output format, the result is returned as a PHP
+     * object.
+     *
+     * @throws PluginDeactivatedException if the module plugin is not activated.
+     * @throws Exception if the requested API method cannot be called, if required parameters for the
+     *                   API method are missing or if the API method throws an exception and the **format**
+     *                   query parameter is **original**.
+     * @return DataTable|Map|string The data resulting from the API call.
+     */
+    public function process()
+    {
+        // read the format requested for the output data
+        $outputFormat = strtolower(Common::getRequestVar('format', 'xml', 'string', $this->request));
+
+        $disablePostProcessing = $this->shouldDisablePostProcessing();
+
+        // create the response
+        $response = new ResponseBuilder($outputFormat, $this->request);
+        if ($disablePostProcessing) {
+            $response->disableDataTablePostProcessor();
+        }
+
+        $corsHandler = new CORSHandler();
+        $corsHandler->handle();
+
+        $tokenAuth = Common::getRequestVar('token_auth', '', 'string', $this->request);
+        $shouldReloadAuth = false;
+
+        try {
+            ++self::$nestedApiInvocationCount;
+
+            // IP check is needed here as we cannot listen to API.Request.authenticate as it would then not return proper API format response.
+            // We can also not do it by listening to API.Request.dispatch as by then the user is already authenticated and we want to make sure
+            // to not expose any information in case the IP is not whitelisted.
+            $whitelist = new LoginWhitelist();
+            if ($whitelist->shouldCheckWhitelist() && $whitelist->shouldWhitelistApplyToAPI()) {
+                $ip = IP::getIpFromHeader();
+                $whitelist->checkIsWhitelisted($ip);
+            }
+
+            // read parameters
+            $moduleMethod = Common::getRequestVar('method', null, 'string', $this->request);
+
+            list($module, $method) = $this->extractModuleAndMethod($moduleMethod);
+            list($module, $method) = self::getRenamedModuleAndAction($module, $method);
+
+            PluginManager::getInstance()->checkIsPluginActivated($module);
+
+            $apiClassName = self::getClassNameAPI($module);
+
+            if ($shouldReloadAuth = self::shouldReloadAuthUsingTokenAuth($this->request)) {
+                $access = Access::getInstance();
+                $tokenAuthToRestore = $access->getTokenAuth();
+                $hadSuperUserAccess = $access->hasSuperUserAccess();
+                self::forceReloadAuthUsingTokenAuth($tokenAuth);
+            }
+
+            // call the method
+            $returnedValue = Proxy::getInstance()->call($apiClassName, $method, $this->request);
+
+            // get the response with the request query parameters loaded, since DataTablePost processor will use the Report
+            // class instance, which may inspect the query parameters. (eg, it may look for the idCustomReport parameters
+            // which may only exist in $this->request, if the request was called programatically)
+            $toReturn = Context::executeWithQueryParameters($this->request, function () use ($response, $returnedValue, $module, $method) {
+                return $response->getResponse($returnedValue, $module, $method);
+            });
+        } catch (Exception $e) {
+            StaticContainer::get(LoggerInterface::class)->error('Uncaught exception in API: {exception}', [
+                'exception' => $e,
+                'ignoreInScreenWriter' => true,
+            ]);
+
+            $toReturn = $response->getResponseException($e);
+        } finally {
+            --self::$nestedApiInvocationCount;
+        }
+
+        if ($shouldReloadAuth) {
+            $this->restoreAuthUsingTokenAuth($tokenAuthToRestore, $hadSuperUserAccess);
+        }
+
+        return $toReturn;
+    }
+
+    private function restoreAuthUsingTokenAuth($tokenToRestore, $hadSuperUserAccess)
+    {
+        // if we would not make sure to unset super user access, the tokenAuth would be not authenticated and any
+        // token would just keep super user access (eg if the token that was reloaded before had super user access)
+        Access::getInstance()->setSuperUserAccess(false);
+
+        // we need to restore by reloading the tokenAuth as some permissions could have been removed in the API
+        // request etc. Otherwise we could just store a clone of Access::getInstance() and restore here
+        self::forceReloadAuthUsingTokenAuth($tokenToRestore);
+
+        if ($hadSuperUserAccess && !Access::getInstance()->hasSuperUserAccess()) {
+            // we are in context of `doAsSuperUser()` and need to restore this behaviour
+            Access::getInstance()->setSuperUserAccess(true);
+        }
+    }
+
+    /**
+     * Returns the name of a plugin's API class by plugin name.
+     *
+     * @param string $plugin The plugin name, eg, `'Referrers'`.
+     * @return string The fully qualified API class name, eg, `'\Piwik\Plugins\Referrers\API'`.
+     */
+    public static function getClassNameAPI($plugin)
+    {
+        return sprintf('\Piwik\Plugins\%s\API', $plugin);
+    }
+
+    /**
+     * @ignore
+     * @internal
+     * @param string $currentApiMethod
+     */
+    public static function setIsRootRequestApiRequest($currentApiMethod)
+    {
+        Cache::getTransientCache()->save('API.setIsRootRequestApiRequest', $currentApiMethod);
+    }
+
+    /**
+     * @ignore
+     * @internal
+     * @return string current Api Method if it is an api request
+     */
+    public static function getRootApiRequestMethod()
+    {
+        return Cache::getTransientCache()->fetch('API.setIsRootRequestApiRequest');
+    }
+
+    /**
+     * Detect if the root request (the actual request) is an API request or not. To detect whether an API is currently
+     * request within any request, have a look at {@link isApiRequest()}.
+     *
+     * @return bool
+     * @throws Exception
+     */
+    public static function isRootRequestApiRequest()
+    {
+        $apiMethod = Cache::getTransientCache()->fetch('API.setIsRootRequestApiRequest');
+        return !empty($apiMethod);
+    }
+
+    /**
+     * Checks if the currently executing API request is the root API request or not.
+     *
+     * Note: the "root" API request is the first request made. Within that request, further API methods
+     * can be called programmatically. These requests are considered "child" API requests.
+     *
+     * @return bool
+     * @throws Exception
+     */
+    public static function isCurrentApiRequestTheRootApiRequest()
+    {
+        return self::$nestedApiInvocationCount == 1;
+    }
+
+    /**
+     * Detect if request is an API request. Meaning the module is 'API' and an API method having a valid format was
+     * specified. Note that this method will return true even if the actual request is for example a regular UI
+     * reporting page request but within this request we are currently processing an API request (eg a
+     * controller calls Request::processRequest('API.getMatomoVersion')). To find out if the root request is an API
+     * request or not, call {@link isRootRequestApiRequest()}
+     *
+     * @param array $request  eg array('module' => 'API', 'method' => 'Test.getMethod')
+     * @return bool
+     * @throws Exception
+     */
+    public static function isApiRequest($request)
+    {
+        $method = self::getMethodIfApiRequest($request);
+        return !empty($method);
+    }
+
+    /**
+     * Returns the current API method being executed, if the current request is an API request.
+     *
+     * @param array $request  eg array('module' => 'API', 'method' => 'Test.getMethod')
+     * @return string|null
+     * @throws Exception
+     */
+    public static function getMethodIfApiRequest($request)
+    {
+        $module = Common::getRequestVar('module', '', 'string', $request);
+        $method = Common::getRequestVar('method', '', 'string', $request);
+
+        $isApi = $module === 'API' && !empty($method) && (count(explode('.', $method)) === 2);
+        return $isApi ? $method : null;
+    }
+
+    /**
+     * If the token_auth is found in the $request parameter,
+     * the current session will be authenticated using this token_auth.
+     * It will overwrite the previous Auth object.
+     *
+     * @param array $request If null, uses the default request ($_GET)
+     * @return void
+     * @ignore
+     */
+    public static function reloadAuthUsingTokenAuth($request = null)
+    {
+        // if a token_auth is specified in the API request, we load the right permissions
+        $token_auth = Common::getRequestVar('token_auth', '', 'string', $request);
+
+        if (self::shouldReloadAuthUsingTokenAuth($request)) {
+            self::forceReloadAuthUsingTokenAuth($token_auth);
+        }
+    }
+
+    /**
+     * The current session will be authenticated using this token_auth.
+     * It will overwrite the previous Auth object.
+     *
+     * @param string $tokenAuth
+     * @return void
+     */
+    private static function forceReloadAuthUsingTokenAuth($tokenAuth)
+    {
+        /**
+         * Triggered when authenticating an API request, but only if the **token_auth**
+         * query parameter is found in the request.
+         *
+         * Plugins that provide authentication capabilities should subscribe to this event
+         * and make sure the global authentication object (the object returned by `StaticContainer::get('Piwik\Auth')`)
+         * is setup to use `$token_auth` when its `authenticate()` method is executed.
+         *
+         * @param string $token_auth The value of the **token_auth** query parameter.
+         */
+        Piwik::postEvent('API.Request.authenticate', array($tokenAuth));
+        if (!Access::getInstance()->reloadAccess() && $tokenAuth && $tokenAuth !== 'anonymous') {
+            /**
+             * @ignore
+             * @internal
+             */
+            Piwik::postEvent('API.Request.authenticate.failed');
+        }
+        SettingsServer::raiseMemoryLimitIfNecessary();
+    }
+
+    private static function shouldReloadAuthUsingTokenAuth($request)
+    {
+        if (is_null($request)) {
+            $request = self::getDefaultRequest();
+        }
+
+        if (!isset($request['token_auth'])) {
+            // no token is given so we just keep the current loaded user
+            return false;
+        }
+
+        // a token is specified, we need to reload auth in case it is different than the current one, even if it is empty
+        $tokenAuth = Common::getRequestVar('token_auth', '', 'string', $request);
+
+        // not using !== is on purpose as getTokenAuth() might return null whereas $tokenAuth is '' . In this case
+        // we do not need to reload.
+
+        return $tokenAuth != Access::getInstance()->getTokenAuth();
+    }
+
+    /**
+     * Returns array($class, $method) from the given string $class.$method
+     *
+     * @param string $parameter
+     * @throws Exception
+     * @return array
+     */
+    private function extractModuleAndMethod($parameter)
+    {
+        $a = explode('.', $parameter);
+        if (count($a) != 2) {
+            throw new Exception("The method name is invalid. Expected 'module.methodName'");
+        }
+        return $a;
+    }
+
+    /**
+     * Helper method that processes an API request in one line using the variables in `$_GET`
+     * and `$_POST`.
+     *
+     * @param string $method The API method to call, ie, `'Actions.getPageTitles'`.
+     * @param array $paramOverride The parameter name-value pairs to use instead of what's
+     *                             in `$_GET` & `$_POST`.
+     * @param array $defaultRequest Default query parameters. If a query parameter is absent in `$request`, it will be loaded
+     *                              from this. Defaults to `$_GET + $_POST`.
+     *
+     *                              To avoid using any parameters from $_GET or $_POST, set this to an empty `array()`.
+     * @return mixed The result of the API request. See {@link process()}.
+     */
+    public static function processRequest($method, $paramOverride = array(), $defaultRequest = null)
+    {
+        $params = array();
+        $params['format'] = 'original';
+        $params['serialize'] = '0';
+        $params['module'] = 'API';
+        $params['method'] = $method;
+        $params = $paramOverride + $params;
+
+        // process request
+        $request = new Request($params, $defaultRequest);
+        return $request->process();
+    }
+
+    /**
+     * Returns the original request parameters in the current query string as an array mapping
+     * query parameter names with values. The result of this function will not be affected
+     * by any modifications to `$_GET` and will not include parameters in `$_POST`.
+     *
+     * @return array
+     */
+    public static function getRequestParametersGET()
+    {
+        if (empty($_SERVER['QUERY_STRING'])) {
+            return array();
+        }
+        $GET = UrlHelper::getArrayFromQueryString($_SERVER['QUERY_STRING']);
+        return $GET;
+    }
+
+    /**
+     * Returns the URL for the current requested report w/o any filter parameters.
+     *
+     * @param string $module The API module.
+     * @param string $action The API action.
+     * @param array $queryParams Query parameter overrides.
+     * @return string
+     */
+    public static function getBaseReportUrl($module, $action, $queryParams = array())
+    {
+        $params = array_merge($queryParams, array('module' => $module, 'action' => $action));
+        return Request::getCurrentUrlWithoutGenericFilters($params);
+    }
+
+    /**
+     * Returns the current URL without generic filter query parameters.
+     *
+     * @param array $params Query parameter values to override in the new URL.
+     * @return string
+     */
+    public static function getCurrentUrlWithoutGenericFilters($params)
+    {
+        // unset all filter query params so the related report will show up in its default state,
+        // unless the filter param was in $queryParams
+        $genericFiltersInfo = DataTableGenericFilter::getGenericFiltersInformation();
+        foreach ($genericFiltersInfo as $filter) {
+            foreach ($filter[1] as $queryParamName => $queryParamInfo) {
+                if (!isset($params[$queryParamName])) {
+                    $params[$queryParamName] = null;
+                }
+            }
+        }
+
+        return Url::getCurrentQueryStringWithParametersModified($params);
+    }
+
+    /**
+     * Returns whether the DataTable result will have to be expanded for the
+     * current request before rendering.
+     *
+     * @return bool
+     * @ignore
+     */
+    public static function shouldLoadExpanded()
+    {
+        // if filter_column_recursive & filter_pattern_recursive are supplied, and flat isn't supplied
+        // we have to load all the child subtables.
+        return Common::getRequestVar('filter_column_recursive', false) !== false
+            && Common::getRequestVar('filter_pattern_recursive', false) !== false
+            && !self::shouldLoadFlatten();
+    }
+
+    /**
+     * @return bool
+     */
+    public static function shouldLoadFlatten()
+    {
+        return Common::getRequestVar('flat', false) == 1;
+    }
+
+    /**
+     * Returns the segment query parameter from the original request, without modifications.
+     *
+     * @return array|bool
+     */
+    public static function getRawSegmentFromRequest()
+    {
+        // we need the URL encoded segment parameter, we fetch it from _SERVER['QUERY_STRING'] instead of default URL decoded _GET
+        $segmentRaw = false;
+        $segment = Common::getRequestVar('segment', '', 'string');
+        if (!empty($segment)) {
+            $request = Request::getRequestParametersGET();
+            if (!empty($request['segment'])) {
+                $segmentRaw = $request['segment'];
+            }
+        }
+        return $segmentRaw;
+    }
+
+    private function renameModuleAndActionInRequest()
+    {
+        if (empty($this->request['apiModule'])) {
+            return;
+        }
+        if (empty($this->request['apiAction'])) {
+            $this->request['apiAction'] = null;
+        }
+        list($this->request['apiModule'], $this->request['apiAction']) = $this->getRenamedModuleAndAction($this->request['apiModule'], $this->request['apiAction']);
+    }
+
+    /**
+     * @return array
+     */
+    private static function getDefaultRequest()
+    {
+        return $_GET + $_POST;
+    }
+
+    private function shouldDisablePostProcessing()
+    {
+        $shouldDisable = false;
+
+        /**
+         * After an API method returns a value, the value is post processed (eg, rows are sorted
+         * based on the `filter_sort_column` query parameter, rows are truncated based on the
+         * `filter_limit`/`filter_offset` parameters, amongst other things).
+         *
+         * If you're creating a plugin that needs to disable post processing entirely for
+         * certain requests, use this event.
+         *
+         * @param bool &$shouldDisable Set this to true to disable datatable post processing for a request.
+         * @param array $request The request parameters.
+         */
+        Piwik::postEvent('Request.shouldDisablePostProcessing', [&$shouldDisable, $this->request]);
+
+        return $shouldDisable;
+    }
+}

msd2/tracking/piwik/core/API/ResponseBuilder.php

@@ -0,0 +1,233 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\API;
+
+use Exception;
+use Piwik\Common;
+use Piwik\DataTable;
+use Piwik\DataTable\Renderer;
+use Piwik\DataTable\DataTableInterface;
+use Piwik\DataTable\Filter\ColumnDelete;
+use Piwik\DataTable\Filter\Pattern;
+use Piwik\Plugins\Monolog\Processor\ExceptionToTextProcessor;
+
+/**
+ */
+class ResponseBuilder
+{
+    private $outputFormat = null;
+    private $apiRenderer  = null;
+    private $request      = null;
+    private $sendHeader   = true;
+    private $postProcessDataTable = true;
+
+    private $apiModule = false;
+    private $apiMethod = false;
+    private $shouldPrintBacktrace = false;
+
+    /**
+     * @param string $outputFormat
+     * @param array $request
+     */
+    public function __construct($outputFormat, $request = array(), $shouldPrintBacktrace = null)
+    {
+        $this->outputFormat = $outputFormat;
+        $this->request      = $request;
+        $this->apiRenderer  = ApiRenderer::factory($outputFormat, $request);
+        $this->shouldPrintBacktrace = $shouldPrintBacktrace === null ? \Piwik_ShouldPrintBackTraceWithMessage() : $shouldPrintBacktrace;
+    }
+
+    public function disableSendHeader()
+    {
+        $this->sendHeader = false;
+    }
+
+    public function disableDataTablePostProcessor()
+    {
+        $this->postProcessDataTable = false;
+    }
+
+    /**
+     * This method processes the data resulting from the API call.
+     *
+     * - If the data resulted from the API call is a DataTable then
+     *         - we apply the standard filters if the parameters have been found
+     *           in the URL. For example to offset,limit the Table you can add the following parameters to any API
+     *        call that returns a DataTable: filter_limit=10&filter_offset=20
+     *         - we apply the filters that have been previously queued on the DataTable
+     * @see DataTable::queueFilter()
+     *         - we apply the renderer that generate the DataTable in a given format (XML, PHP, HTML, JSON, etc.)
+     *           the format can be changed using the 'format' parameter in the request.
+     *        Example: format=xml
+     *
+     * - If there is nothing returned (void) we display a standard success message
+     *
+     * - If there is a PHP array returned, we try to convert it to a dataTable
+     *   It is then possible to convert this datatable to any requested format (xml/etc)
+     *
+     * - If a bool is returned we convert to a string (true is displayed as 'true' false as 'false')
+     *
+     * - If an integer / float is returned, we simply return it
+     *
+     * @param mixed $value The initial returned value, before post process. If set to null, success response is returned.
+     * @param bool|string $apiModule The API module that was called
+     * @param bool|string $apiMethod The API method that was called
+     * @return mixed  Usually a string, but can still be a PHP data structure if the format requested is 'original'
+     */
+    public function getResponse($value = null, $apiModule = false, $apiMethod = false)
+    {
+        $this->apiModule = $apiModule;
+        $this->apiMethod = $apiMethod;
+
+        $this->sendHeaderIfEnabled();
+
+        // when null or void is returned from the api call, we handle it as a successful operation
+        if (!isset($value)) {
+            if (ob_get_contents()) {
+                return null;
+            }
+
+            return $this->apiRenderer->renderSuccess('ok');
+        }
+
+        // If the returned value is an object DataTable we
+        // apply the set of generic filters if asked in the URL
+        // and we render the DataTable according to the format specified in the URL
+        if ($value instanceof DataTableInterface) {
+            return $this->handleDataTable($value);
+        }
+
+        // Case an array is returned from the API call, we convert it to the requested format
+        // - if calling from inside the application (format = original)
+        //    => the data stays unchanged (ie. a standard php array or whatever data structure)
+        // - if any other format is requested, we have to convert this data structure (which we assume
+        //   to be an array) to a DataTable in order to apply the requested DataTable_Renderer (for example XML)
+        if (is_array($value)) {
+            return $this->handleArray($value);
+        }
+
+        if (is_object($value)) {
+            return $this->apiRenderer->renderObject($value);
+        }
+
+        if (is_resource($value)) {
+            return $this->apiRenderer->renderResource($value);
+        }
+
+        return $this->apiRenderer->renderScalar($value);
+    }
+
+    /**
+     * Returns an error $message in the requested $format
+     *
+     * @param Exception|\Throwable $e
+     * @throws Exception
+     * @return string
+     */
+    public function getResponseException($e)
+    {
+        $e       = $this->decorateExceptionWithDebugTrace($e);
+        $message = $this->formatExceptionMessage($e);
+
+        $this->sendHeaderIfEnabled();
+
+        return $this->apiRenderer->renderException($message, $e);
+    }
+
+    /**
+     * @param Exception|\Throwable $e
+     * @return Exception
+     */
+    private function decorateExceptionWithDebugTrace($e)
+    {
+        // If we are in tests, show full backtrace
+        if (defined('PIWIK_PATH_TEST_TO_ROOT')) {
+            if ($this->shouldPrintBacktrace) {
+                $message = $e->getMessage() . " in \n " . $e->getFile() . ":" . $e->getLine() . " \n " . $e->getTraceAsString();
+            } else {
+                $message = $e->getMessage() . "\n \n --> To temporarily debug this error further, set const PIWIK_PRINT_ERROR_BACKTRACE=true; in index.php";
+            }
+
+            return new Exception($message);
+        }
+
+        return $e;
+    }
+
+    /**
+     * @param Exception|\Throwable $exception
+     * @return string
+     */
+    private function formatExceptionMessage($exception)
+    {
+        $message = ExceptionToTextProcessor::getWholeBacktrace($exception, $this->shouldPrintBacktrace);
+
+        return Renderer::formatValueXml($message);
+    }
+
+    private function handleDataTable(DataTableInterface $datatable)
+    {
+        if ($this->postProcessDataTable) {
+            $postProcessor = new DataTablePostProcessor($this->apiModule, $this->apiMethod, $this->request);
+            $datatable = $postProcessor->process($datatable);
+        }
+
+        return $this->apiRenderer->renderDataTable($datatable);
+    }
+
+    private function handleArray($array)
+    {
+        $firstArray = null;
+        $firstKey   = null;
+        if (!empty($array)) {
+            $firstArray = reset($array);
+            $firstKey   = key($array);
+        }
+
+        $isAssoc = !empty($firstArray) && is_numeric($firstKey) && is_array($firstArray) && count(array_filter(array_keys($firstArray), 'is_string'));
+
+        if (is_numeric($firstKey)) {
+            $columns = Common::getRequestVar('filter_column', false, 'array', $this->request);
+            $pattern = Common::getRequestVar('filter_pattern', '', 'string', $this->request);
+
+            if ($columns != array(false) && $pattern !== '') {
+                $pattern = new Pattern(new DataTable(), $columns, $pattern);
+                $array   = $pattern->filterArray($array);
+            }
+
+            $limit  = Common::getRequestVar('filter_limit', -1, 'integer', $this->request);
+            $offset = Common::getRequestVar('filter_offset', '0', 'integer', $this->request);
+
+            if ($limit >= 0 || $offset > 0) {
+                if ($limit < 0) {
+                    $limit = null; // make sure to return all results from offset
+                }
+                $array = array_slice($array, $offset, $limit, $preserveKeys = false);
+            }
+        }
+
+        if ($isAssoc) {
+            $hideColumns = Common::getRequestVar('hideColumns', '', 'string', $this->request);
+            $showColumns = Common::getRequestVar('showColumns', '', 'string', $this->request);
+            if ($hideColumns !== '' || $showColumns !== '') {
+                $columnDelete = new ColumnDelete(new DataTable(), $hideColumns, $showColumns);
+                $array = $columnDelete->filter($array);
+            }
+        }
+
+        return $this->apiRenderer->renderArray($array);
+    }
+
+    private function sendHeaderIfEnabled()
+    {
+        if ($this->sendHeader) {
+            $this->apiRenderer->sendHeader();
+        }
+    }
+}

msd2/tracking/piwik/core/Access.php

@@ -0,0 +1,700 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik;
+
+use Exception;
+use Piwik\Access\CapabilitiesProvider;
+use Piwik\Access\RolesProvider;
+use Piwik\Container\StaticContainer;
+use Piwik\Plugins\SitesManager\API as SitesManagerApi;
+
+/**
+ * Singleton that manages user access to Piwik resources.
+ *
+ * To check whether a user has access to a resource, use one of the {@link Piwik Piwik::checkUser...}
+ * methods.
+ *
+ * In Piwik there are four different access levels:
+ *
+ * - **no access**: Users with this access level cannot view the resource.
+ * - **view access**: Users with this access level can view the resource, but cannot modify it.
+ * - **admin access**: Users with this access level can view and modify the resource.
+ * - **Super User access**: Only the Super User has this access level. It means the user can do
+ *                          whatever they want.
+ *
+ *                          Super user access is required to set some configuration options.
+ *                          All other options are specific to the user or to a website.
+ *
+ * Access is granted per website. Uses with access for a website can view all
+ * data associated with that website.
+ *
+ */
+class Access
+{
+    /**
+     * Array of idsites available to the current user, indexed by permission level
+     * @see getSitesIdWith*()
+     *
+     * @var array
+     */
+    protected $idsitesByAccess = null;
+
+    /**
+     * Login of the current user
+     *
+     * @var string
+     */
+    protected $login = null;
+
+    /**
+     * token_auth of the current user
+     *
+     * @var string
+     */
+    protected $token_auth = null;
+
+    /**
+     * Defines if the current user is the Super User
+     * @see hasSuperUserAccess()
+     *
+     * @var bool
+     */
+    protected $hasSuperUserAccess = false;
+
+    /**
+     * Authentification object (see Auth)
+     *
+     * @var Auth
+     */
+    private $auth = null;
+
+    /**
+     * Gets the singleton instance. Creates it if necessary.
+     *
+     * @return self
+     */
+    public static function getInstance()
+    {
+        return StaticContainer::get('Piwik\Access');
+    }
+
+    /**
+     * @var CapabilitiesProvider
+     */
+    protected $capabilityProvider;
+
+    /**
+     * @var RolesProvider
+     */
+    private $roleProvider;
+
+    /**
+     * Constructor
+     */
+    public function __construct(RolesProvider $roleProvider = null, CapabilitiesProvider $capabilityProvider = null)
+    {
+        if (!isset($roleProvider)) {
+            $roleProvider = StaticContainer::get('Piwik\Access\RolesProvider');
+        }
+        if (!isset($capabilityProvider)) {
+            $capabilityProvider = StaticContainer::get('Piwik\Access\CapabilitiesProvider');
+        }
+        $this->roleProvider = $roleProvider;
+        $this->capabilityProvider = $capabilityProvider;
+
+        $this->resetSites();
+    }
+
+    private function resetSites()
+    {
+        $this->idsitesByAccess = array(
+            'view'      => array(),
+            'write'     => array(),
+            'admin'     => array(),
+            'superuser' => array()
+        );
+    }
+
+    /**
+     * Loads the access levels for the current user.
+     *
+     * Calls the authentication method to try to log the user in the system.
+     * If the user credentials are not correct we don't load anything.
+     * If the login/password is correct the user is either the SuperUser or a normal user.
+     * We load the access levels for this user for all the websites.
+     *
+     * @param null|Auth $auth Auth adapter
+     * @return bool  true on success, false if reloading access failed (when auth object wasn't specified and user is not enforced to be Super User)
+     */
+    public function reloadAccess(Auth $auth = null)
+    {
+        $this->resetSites();
+
+        if (isset($auth)) {
+            $this->auth = $auth;
+        }
+
+        if ($this->hasSuperUserAccess()) {
+            $this->makeSureLoginNameIsSet();
+            return true;
+        }
+
+        $this->token_auth = null;
+        $this->login = null;
+
+        // if the Auth wasn't set, we may be in the special case of setSuperUser(), otherwise we fail TODO: docs + review
+        if (!isset($this->auth)) {
+            return false;
+        }
+
+        // access = array ( idsite => accessIdSite, idsite2 => accessIdSite2)
+        $result = $this->auth->authenticate();
+
+        if (!$result->wasAuthenticationSuccessful()) {
+            return false;
+        }
+
+        $this->login = $result->getIdentity();
+        $this->token_auth = $result->getTokenAuth();
+
+        // case the superUser is logged in
+        if ($result->hasSuperUserAccess()) {
+            $this->setSuperUserAccess(true);
+        }
+
+        return true;
+    }
+
+    public function getRawSitesWithSomeViewAccess($login)
+    {
+        $sql = self::getSqlAccessSite("access, t2.idsite");
+
+        return Db::fetchAll($sql, $login);
+    }
+
+    /**
+     * Returns the SQL query joining sites and access table for a given login
+     *
+     * @param string $select Columns or expression to SELECT FROM table, eg. "MIN(ts_created)"
+     * @return string  SQL query
+     */
+    public static function getSqlAccessSite($select)
+    {
+        $access    = Common::prefixTable('access');
+        $siteTable = Common::prefixTable('site');
+
+        return "SELECT " . $select . " FROM " . $access . " as t1
+				JOIN " . $siteTable . " as t2 USING (idsite) WHERE login = ?";
+    }
+
+    /**
+     * Make sure a login name is set
+     *
+     * @return true
+     */
+    protected function makeSureLoginNameIsSet()
+    {
+        if (empty($this->login)) {
+            // flag to force non empty login so Super User is not mistaken for anonymous
+            $this->login = 'super user was set';
+        }
+    }
+
+    protected function loadSitesIfNeeded()
+    {
+        if ($this->hasSuperUserAccess) {
+            if (empty($this->idsitesByAccess['superuser'])) {
+                try {
+                    $api = SitesManagerApi::getInstance();
+                    $allSitesId = $api->getAllSitesId();
+                } catch (\Exception $e) {
+                    $allSitesId = array();
+                }
+                $this->idsitesByAccess['superuser'] = $allSitesId;
+            }
+        } elseif (isset($this->login)) {
+            if (empty($this->idsitesByAccess['view'])
+                && empty($this->idsitesByAccess['write'])
+                && empty($this->idsitesByAccess['admin'])
+            ) {
+                // we join with site in case there are rows in access for an idsite that doesn't exist anymore
+                // (backward compatibility ; before we deleted the site without deleting rows in _access table)
+                $accessRaw = $this->getRawSitesWithSomeViewAccess($this->login);
+
+                foreach ($accessRaw as $access) {
+                    $accessType = $access['access'];
+                    $this->idsitesByAccess[$accessType][] = $access['idsite'];
+
+                    if ($this->roleProvider->isValidRole($accessType)) {
+                        foreach ($this->capabilityProvider->getAllCapabilities() as $capability) {
+                            if ($capability->hasRoleCapability($accessType)) {
+                                // we automatically add this capability
+                                if (!isset($this->idsitesByAccess[$capability->getId()])) {
+                                    $this->idsitesByAccess[$capability->getId()] = array();
+                                }
+                                $this->idsitesByAccess[$capability->getId()][] = $access['idsite'];
+                            }
+                        }
+                    }
+                }
+
+                /**
+                 * Triggered after the initial access levels and permissions for the current user are loaded. Use this
+                 * event to modify the current user's permissions (for example, making sure every user has view access
+                 * to a specific site).
+                 *
+                 * **Example**
+                 *
+                 *     function (&$idsitesByAccess, $login) {
+                 *         if ($login == 'somespecialuser') {
+                 *             return;
+                 *         }
+                 *
+                 *         $idsitesByAccess['view'][] = $mySpecialIdSite;
+                 *     }
+                 *
+                 * @param array[] &$idsitesByAccess The current user's access levels for individual sites. Maps role and
+                 *                                  capability IDs to list of site IDs, eg:
+                 *
+                 *                                  ```
+                 *                                  [
+                 *                                      'view' => [1, 2, 3],
+                 *                                      'write' => [4, 5],
+                 *                                      'admin' => [],
+                 *                                  ]
+                 *                                  ```
+                 * @param string $login The current user's login.
+                 */
+                Piwik::postEvent('Access.modifyUserAccess', [&$this->idsitesByAccess, $this->login]);
+            }
+        }
+    }
+
+    /**
+     * We bypass the normal auth method and give the current user Super User rights.
+     * This should be very carefully used.
+     *
+     * @param bool $bool
+     */
+    public function setSuperUserAccess($bool = true)
+    {
+        $this->hasSuperUserAccess = (bool) $bool;
+
+        if ($bool) {
+            $this->makeSureLoginNameIsSet();
+        } else {
+            $this->resetSites();
+        }
+    }
+
+    /**
+     * Returns true if the current user is logged in as the Super User
+     *
+     * @return bool
+     */
+    public function hasSuperUserAccess()
+    {
+        return $this->hasSuperUserAccess;
+    }
+
+    /**
+     * Returns the current user login
+     *
+     * @return string|null
+     */
+    public function getLogin()
+    {
+        return $this->login;
+    }
+
+    /**
+     * Returns the token_auth used to authenticate this user in the API
+     *
+     * @return string|null
+     */
+    public function getTokenAuth()
+    {
+        return $this->token_auth;
+    }
+
+    /**
+     * Returns an array of ID sites for which the user has at least a VIEW access.
+     * Which means VIEW OR WRITE or ADMIN or SUPERUSER.
+     *
+     * @return array  Example if the user is ADMIN for 4
+     *                and has VIEW access for 1 and 7, it returns array(1, 4, 7);
+     */
+    public function getSitesIdWithAtLeastViewAccess()
+    {
+        $this->loadSitesIfNeeded();
+
+        return array_unique(array_merge(
+                $this->idsitesByAccess['view'],
+                $this->idsitesByAccess['write'],
+                $this->idsitesByAccess['admin'],
+                $this->idsitesByAccess['superuser'])
+        );
+    }
+
+    /**
+     * Returns an array of ID sites for which the user has at least a WRITE access.
+     * Which means WRITE or ADMIN or SUPERUSER.
+     *
+     * @return array  Example if the user is WRITE for 4 and 8
+     *                and has VIEW access for 1 and 7, it returns array(4, 8);
+     */
+    public function getSitesIdWithAtLeastWriteAccess()
+    {
+        $this->loadSitesIfNeeded();
+
+        return array_unique(array_merge(
+                $this->idsitesByAccess['write'],
+                $this->idsitesByAccess['admin'],
+                $this->idsitesByAccess['superuser'])
+        );
+    }
+
+    /**
+     * Returns an array of ID sites for which the user has an ADMIN access.
+     *
+     * @return array  Example if the user is ADMIN for 4 and 8
+     *                and has VIEW access for 1 and 7, it returns array(4, 8);
+     */
+    public function getSitesIdWithAdminAccess()
+    {
+        $this->loadSitesIfNeeded();
+
+        return array_unique(array_merge(
+                $this->idsitesByAccess['admin'],
+                $this->idsitesByAccess['superuser'])
+        );
+    }
+
+    /**
+     * Returns an array of ID sites for which the user has a VIEW access only.
+     *
+     * @return array  Example if the user is ADMIN for 4
+     *                and has VIEW access for 1 and 7, it returns array(1, 7);
+     * @see getSitesIdWithAtLeastViewAccess()
+     */
+    public function getSitesIdWithViewAccess()
+    {
+        $this->loadSitesIfNeeded();
+
+        return $this->idsitesByAccess['view'];
+    }
+
+    /**
+     * Returns an array of ID sites for which the user has a WRITE access only.
+     *
+     * @return array  Example if the user is ADMIN for 4
+     *                and has WRITE access for 1 and 7, it returns array(1, 7);
+     * @see getSitesIdWithAtLeastWriteAccess()
+     */
+    public function getSitesIdWithWriteAccess()
+    {
+        $this->loadSitesIfNeeded();
+
+        return $this->idsitesByAccess['write'];
+    }
+
+    /**
+     * Throws an exception if the user is not the SuperUser
+     *
+     * @throws \Piwik\NoAccessException
+     */
+    public function checkUserHasSuperUserAccess()
+    {
+        if (!$this->hasSuperUserAccess()) {
+            throw new NoAccessException(Piwik::translate('General_ExceptionPrivilege', array("'superuser'")));
+        }
+    }
+
+    /**
+     * Returns `true` if the current user has admin access to at least one site.
+     *
+     * @return bool
+     */
+    public function isUserHasSomeWriteAccess()
+    {
+        if ($this->hasSuperUserAccess()) {
+            return true;
+        }
+
+        $idSitesAccessible = $this->getSitesIdWithAtLeastWriteAccess();
+
+        return count($idSitesAccessible) > 0;
+    }
+
+    /**
+     * Returns `true` if the current user has admin access to at least one site.
+     *
+     * @return bool
+     */
+    public function isUserHasSomeAdminAccess()
+    {
+        if ($this->hasSuperUserAccess()) {
+            return true;
+        }
+
+        $idSitesAccessible = $this->getSitesIdWithAdminAccess();
+
+        return count($idSitesAccessible) > 0;
+    }
+
+    /**
+     * If the user doesn't have an WRITE access for at least one website, throws an exception
+     *
+     * @throws \Piwik\NoAccessException
+     */
+    public function checkUserHasSomeWriteAccess()
+    {
+        if (!$this->isUserHasSomeWriteAccess()) {
+            throw new NoAccessException(Piwik::translate('General_ExceptionPrivilegeAtLeastOneWebsite', array('write')));
+        }
+    }
+
+    /**
+     * If the user doesn't have an ADMIN access for at least one website, throws an exception
+     *
+     * @throws \Piwik\NoAccessException
+     */
+    public function checkUserHasSomeAdminAccess()
+    {
+        if (!$this->isUserHasSomeAdminAccess()) {
+            throw new NoAccessException(Piwik::translate('General_ExceptionPrivilegeAtLeastOneWebsite', array('admin')));
+        }
+    }
+
+    /**
+     * If the user doesn't have any view permission, throw exception
+     *
+     * @throws \Piwik\NoAccessException
+     */
+    public function checkUserHasSomeViewAccess()
+    {
+        if ($this->hasSuperUserAccess()) {
+            return;
+        }
+
+        $idSitesAccessible = $this->getSitesIdWithAtLeastViewAccess();
+
+        if (count($idSitesAccessible) == 0) {
+            throw new NoAccessException(Piwik::translate('General_ExceptionPrivilegeAtLeastOneWebsite', array('view')));
+        }
+    }
+
+    /**
+     * This method checks that the user has ADMIN access for the given list of websites.
+     * If the user doesn't have ADMIN access for at least one website of the list, we throw an exception.
+     *
+     * @param int|array $idSites List of ID sites to check
+     * @throws \Piwik\NoAccessException If for any of the websites the user doesn't have an ADMIN access
+     */
+    public function checkUserHasAdminAccess($idSites)
+    {
+        if ($this->hasSuperUserAccess()) {
+            return;
+        }
+
+        $idSites = $this->getIdSites($idSites);
+        $idSitesAccessible = $this->getSitesIdWithAdminAccess();
+
+        foreach ($idSites as $idsite) {
+            if (!in_array($idsite, $idSitesAccessible)) {
+                throw new NoAccessException(Piwik::translate('General_ExceptionPrivilegeAccessWebsite', array("'admin'", $idsite)));
+            }
+        }
+    }
+
+    /**
+     * This method checks that the user has VIEW or ADMIN access for the given list of websites.
+     * If the user doesn't have VIEW or ADMIN access for at least one website of the list, we throw an exception.
+     *
+     * @param int|array|string $idSites List of ID sites to check (integer, array of integers, string comma separated list of integers)
+     * @throws \Piwik\NoAccessException  If for any of the websites the user doesn't have an VIEW or ADMIN access
+     */
+    public function checkUserHasViewAccess($idSites)
+    {
+        if ($this->hasSuperUserAccess()) {
+            return;
+        }
+
+        $idSites = $this->getIdSites($idSites);
+        $idSitesAccessible = $this->getSitesIdWithAtLeastViewAccess();
+
+        foreach ($idSites as $idsite) {
+            if (!in_array($idsite, $idSitesAccessible)) {
+                throw new NoAccessException(Piwik::translate('General_ExceptionPrivilegeAccessWebsite', array("'view'", $idsite)));
+            }
+        }
+    }
+
+    /**
+     * This method checks that the user has VIEW or ADMIN access for the given list of websites.
+     * If the user doesn't have VIEW or ADMIN access for at least one website of the list, we throw an exception.
+     *
+     * @param int|array|string $idSites List of ID sites to check (integer, array of integers, string comma separated list of integers)
+     * @throws \Piwik\NoAccessException  If for any of the websites the user doesn't have an VIEW or ADMIN access
+     */
+    public function checkUserHasWriteAccess($idSites)
+    {
+        if ($this->hasSuperUserAccess()) {
+            return;
+        }
+
+        $idSites = $this->getIdSites($idSites);
+        $idSitesAccessible = $this->getSitesIdWithAtLeastWriteAccess();
+
+        foreach ($idSites as $idsite) {
+            if (!in_array($idsite, $idSitesAccessible)) {
+                throw new NoAccessException(Piwik::translate('General_ExceptionPrivilegeAccessWebsite', array("'write'", $idsite)));
+            }
+        }
+    }
+
+    private function getSitesIdWithCapability($capability)
+    {
+        if (!empty($this->idsitesByAccess[$capability])) {
+            return $this->idsitesByAccess[$capability];
+        }
+        return array();
+    }
+
+    public function checkUserHasCapability($idSites, $capability)
+    {
+        if ($this->hasSuperUserAccess()) {
+            return;
+        }
+
+        $idSites = $this->getIdSites($idSites);
+        $idSitesAccessible = $this->getSitesIdWithCapability($capability);
+
+        foreach ($idSites as $idsite) {
+            if (!in_array($idsite, $idSitesAccessible)) {
+                throw new NoAccessException(Piwik::translate('ExceptionCapabilityAccessWebsite', array("'" . $capability ."'", $idsite)));
+            }
+        }
+
+        // a capability applies only when the user also has at least view access
+        $this->checkUserHasViewAccess($idSites);
+    }
+
+    /**
+     * @param int|array|string $idSites
+     * @return array
+     * @throws \Piwik\NoAccessException
+     */
+    protected function getIdSites($idSites)
+    {
+        if ($idSites === 'all') {
+            $idSites = $this->getSitesIdWithAtLeastViewAccess();
+        }
+
+        $idSites = Site::getIdSitesFromIdSitesString($idSites);
+
+        if (empty($idSites)) {
+            throw new NoAccessException("The parameter 'idSite=' is missing from the request.");
+        }
+
+        return $idSites;
+    }
+
+    /**
+     * Executes a callback with superuser privileges, making sure those privileges are rescinded
+     * before this method exits. Privileges will be rescinded even if an exception is thrown.
+     *
+     * @param callback $function The callback to execute. Should accept no arguments.
+     * @return mixed The result of `$function`.
+     * @throws Exception rethrows any exceptions thrown by `$function`.
+     * @api
+     */
+    public static function doAsSuperUser($function)
+    {
+        $isSuperUser = self::getInstance()->hasSuperUserAccess();
+
+        $access = self::getInstance();
+        $login = $access->getLogin();
+        $shouldResetLogin = empty($login); // make sure to reset login if a login was set by "makeSureLoginNameIsSet()"
+        $access->setSuperUserAccess(true);
+
+        try {
+            $result = $function();
+        } catch (Exception $ex) {
+            $access->setSuperUserAccess($isSuperUser);
+            if ($shouldResetLogin) {
+                $access->login = null;
+            }
+
+            throw $ex;
+        }
+
+        if ($shouldResetLogin) {
+            $access->login = null;
+        }
+        $access->setSuperUserAccess($isSuperUser);
+
+        return $result;
+    }
+
+    /**
+     * Returns the level of access the current user has to the given site.
+     *
+     * @param int $idSite The site to check.
+     * @return string The access level, eg, 'view', 'admin', 'noaccess'.
+     */
+    public function getRoleForSite($idSite)
+    {
+        if ($this->hasSuperUserAccess
+            || in_array($idSite, $this->getSitesIdWithAdminAccess())
+        ) {
+            return 'admin';
+        }
+
+        if (in_array($idSite, $this->getSitesIdWithWriteAccess())) {
+            return 'write';
+        }
+
+        if (in_array($idSite, $this->getSitesIdWithViewAccess())) {
+            return 'view';
+        }
+
+        return 'noaccess';
+    }
+
+    /**
+     * Returns the capabilities the current user has for a given site.
+     *
+     * @param int $idSite The site to check.
+     * @return string[] The capabilities the user has.
+     */
+    public function getCapabilitiesForSite($idSite)
+    {
+        $result = [];
+        foreach ($this->capabilityProvider->getAllCapabilityIds() as $capabilityId) {
+            if (empty($this->idsitesByAccess[$capabilityId])) {
+                continue;
+            }
+
+            if (in_array($idSite, $this->idsitesByAccess[$capabilityId])) {
+                $result[] = $capabilityId;
+            }
+        }
+        return $result;
+    }
+}
+
+/**
+ * Exception thrown when a user doesn't have sufficient access to a resource.
+ *
+ * @api
+ */
+class NoAccessException extends \Exception
+{
+}

msd2/tracking/piwik/core/Access/CapabilitiesProvider.php

@@ -0,0 +1,123 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\Access;
+
+use Exception;
+use Piwik\CacheId;
+use Piwik\Piwik;
+use Piwik\Cache as PiwikCache;
+
+class CapabilitiesProvider
+{
+    /**
+     * @return Capability[]
+     */
+    public function getAllCapabilities()
+    {
+        $cacheId = CacheId::siteAware(CacheId::languageAware('Capabilities'));
+        $cache   = PiwikCache::getTransientCache();
+
+        if (!$cache->contains($cacheId)) {
+            $capabilities = array();
+
+            /**
+             * Triggered to add new capabilities.
+             *
+             * **Example**
+             *
+             *     public function addCapabilities(&$capabilities)
+             *     {
+             *         $capabilities[] = new MyNewCapabilitiy();
+             *     }
+             *
+             * @param Capability[] $reports An array of reports
+             * @internal
+             */
+            Piwik::postEvent('Access.Capability.addCapabilities', array(&$capabilities));
+
+            /**
+             * Triggered to filter / restrict capabilities.
+             *
+             * **Example**
+             *
+             *     public function filterCapabilities(&$capabilities)
+             *     {
+             *         foreach ($capabilities as $index => $capability) {
+             *              if ($capability->getId() === 'tagmanager_write') {}
+             *                  unset($capabilities[$index]); // remove the given capability
+             *              }
+             *         }
+             *     }
+             *
+             * @param Capability[] $reports An array of reports
+             * @internal
+             */
+            Piwik::postEvent('Access.Capability.filterCapabilities', array(&$capabilities));
+
+            $capabilities = array_values($capabilities);
+
+            $this->checkCapabilityIds($capabilities);
+
+            $cache->save($cacheId, $capabilities);
+            return $capabilities;
+        }
+
+        return $cache->fetch($cacheId);
+    }
+
+    /**
+     * @param $capabilityId
+     * @return Capability|null
+     */
+    public function getCapability($capabilityId)
+    {
+        foreach ($this->getAllCapabilities() as $capability) {
+            if ($capabilityId === $capability->getId()) {
+                return $capability;
+            }
+        }
+    }
+
+    public function getAllCapabilityIds()
+    {
+        $ids = array();
+        foreach ($this->getAllCapabilities() as $capability) {
+            $ids[] = $capability->getId();
+        }
+        return $ids;
+    }
+
+    public function isValidCapability($capabilityId)
+    {
+        $capabilities = $this->getAllCapabilityIds();
+
+        return in_array($capabilityId, $capabilities, true);
+    }
+
+    public function checkValidCapability($capabilityId)
+    {
+        if (!$this->isValidCapability($capabilityId)) {
+            $capabilities = $this->getAllCapabilityIds();
+            throw new Exception(Piwik::translate("UsersManager_ExceptionAccessValues", implode(", ", $capabilities)));
+        }
+    }
+
+    /**
+     * @param Capability[] $capabilities
+     */
+    private function checkCapabilityIds($capabilities)
+    {
+        foreach ($capabilities as $capability) {
+            $id = $capability->getId();
+            if (preg_match('/[^a-zA-Z0-9_-]/', $id)) {
+                throw new \Exception("Capability with invalid ID found: '$id'. Valid characters are 'a-zA-Z0-9_-'.");
+            }
+        }
+    }
+}

msd2/tracking/piwik/core/Access/Capability.php

@@ -0,0 +1,29 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\Access;
+
+abstract class Capability
+{
+    abstract public function getId();
+    abstract public function getName();
+    abstract public function getCategory();
+    abstract public function getDescription();
+    abstract public function getIncludedInRoles();
+
+    public function getHelpUrl()
+    {
+        return '';
+    }
+
+    public function hasRoleCapability($idRole)
+    {
+        return in_array($idRole, $this->getIncludedInRoles(), true);
+    }
+
+}

msd2/tracking/piwik/core/Access/Role.php

@@ -0,0 +1,22 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\Access;
+
+abstract class Role
+{
+    abstract public function getName();
+    abstract public function getId();
+    abstract public function getDescription();
+
+    public function getHelpUrl()
+    {
+        return '';
+    }
+
+}

msd2/tracking/piwik/core/Access/Role/Admin.php

@@ -0,0 +1,40 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\Access\Role;
+
+use Piwik\Access\Role;
+use Piwik\Piwik;
+
+class Admin extends Role
+{
+    const ID = 'admin';
+
+    public function getName()
+    {
+        return Piwik::translate('UsersManager_PrivAdmin');
+    }
+
+    public function getId()
+    {
+        return self::ID;
+    }
+
+    public function getDescription()
+    {
+        return Piwik::translate('UsersManager_PrivAdminDescription', array(
+            Piwik::translate('UsersManager_PrivWrite')
+        ));
+    }
+
+    public function getHelpUrl()
+    {
+        return 'https://matomo.org/faq/general/faq_69/';
+    }
+
+}

msd2/tracking/piwik/core/Access/Role/View.php

@@ -0,0 +1,39 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\Access\Role;
+
+use Piwik\Access\Role;
+use Piwik\Piwik;
+
+class View extends Role
+{
+    const ID = 'view';
+
+    public function getName()
+    {
+        return Piwik::translate('UsersManager_PrivView');
+    }
+
+    public function getId()
+    {
+        return self::ID;
+    }
+
+    public function getDescription()
+    {
+        return Piwik::translate('UsersManager_PrivViewDescription');
+    }
+
+    public function getHelpUrl()
+    {
+        return 'https://matomo.org/faq/general/faq_70/';
+    }
+
+
+}

msd2/tracking/piwik/core/Access/Role/Write.php

@@ -0,0 +1,38 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\Access\Role;
+
+use Piwik\Access\Role;
+use Piwik\Piwik;
+
+class Write extends Role
+{
+    const ID = 'write';
+
+    public function getName()
+    {
+        return Piwik::translate('UsersManager_PrivWrite');
+    }
+
+    public function getId()
+    {
+        return self::ID;
+    }
+
+    public function getDescription()
+    {
+        return Piwik::translate('UsersManager_PrivWriteDescription');
+    }
+
+    public function getHelpUrl()
+    {
+        return '';
+    }
+
+}

msd2/tracking/piwik/core/Access/RolesProvider.php

@@ -0,0 +1,62 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\Access;
+
+use Piwik\Access\Role\Admin;
+use Piwik\Access\Role\View;
+use Piwik\Access\Role\Write;
+use Piwik\Piwik;
+use Exception;
+
+class RolesProvider
+{
+    /**
+     * @return Role[]
+     */
+    public function getAllRoles()
+    {
+        return array(
+            new View(),
+            new Write(),
+            new Admin()
+        );
+    }
+
+    /**
+     * Returns the list of the existing Access level.
+     * Useful when a given API method requests a given acccess Level.
+     * We first check that the required access level exists.
+     *
+     * @return array
+     */
+    public function getAllRoleIds()
+    {
+        $ids = array();
+        foreach ($this->getAllRoles() as $role) {
+            $ids[] = $role->getId();
+        }
+        return $ids;
+    }
+
+    public function isValidRole($roleId)
+    {
+        $roles = $this->getAllRoleIds();
+
+        return in_array($roleId, $roles, true);
+    }
+
+    public function checkValidRole($roleId)
+    {
+        if (!$this->isValidRole($roleId)) {
+            $roles = $this->getAllRoleIds();
+            throw new Exception(Piwik::translate("UsersManager_ExceptionAccessValues", implode(", ", $roles)));
+        }
+    }
+
+}

msd2/tracking/piwik/core/Application/Environment.php

@@ -0,0 +1,246 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+
+namespace Piwik\Application;
+
+use DI\Container;
+use Piwik\Application\Kernel\EnvironmentValidator;
+use Piwik\Application\Kernel\GlobalSettingsProvider;
+use Piwik\Application\Kernel\PluginList;
+use Piwik\Container\ContainerFactory;
+use Piwik\Container\StaticContainer;
+use Piwik\Piwik;
+
+/**
+ * Encapsulates Piwik environment setup and access.
+ *
+ * The Piwik environment consists of two main parts: the kernel and the DI container.
+ *
+ * The 'kernel' is the core part of Piwik that cannot be modified / extended through the DI container.
+ * It includes components that are required to create the DI container.
+ *
+ * Currently the only objects in the 'kernel' are a GlobalSettingsProvider object and a
+ * PluginList object. The GlobalSettingsProvider object is required for the current PluginList
+ * implementation and for checking whether Development mode is enabled. The PluginList is
+ * needed in order to determine what plugins are activated, since plugins can provide their
+ * own DI configuration.
+ *
+ * The DI container contains every other Piwik object, including the Plugin\Manager,
+ * plugin API instances, dependent services, etc. Plugins and users can override/extend
+ * the objects in this container.
+ *
+ * NOTE: DI support in Piwik is currently a work in process; not everything is currently
+ * stored in the DI container, but we are working towards this.
+ */
+class Environment
+{
+    /**
+     * @internal
+     * @var EnvironmentManipulator
+     */
+    private static $globalEnvironmentManipulator = null;
+
+    /**
+     * @var string
+     */
+    private $environment;
+
+    /**
+     * @var array
+     */
+    private $definitions;
+
+    /**
+     * @var Container
+     */
+    private $container;
+
+    /**
+     * @var GlobalSettingsProvider
+     */
+    private $globalSettingsProvider;
+
+    /**
+     * @var PluginList
+     */
+    private $pluginList;
+
+    /**
+     * @param string $environment
+     * @param array $definitions
+     */
+    public function __construct($environment, array $definitions = array())
+    {
+        $this->environment = $environment;
+        $this->definitions = $definitions;
+    }
+
+    /**
+     * Initializes the kernel globals and DI container.
+     */
+    public function init()
+    {
+        $this->invokeBeforeContainerCreatedHook();
+
+        $this->container = $this->createContainer();
+
+        StaticContainer::push($this->container);
+
+        $this->validateEnvironment();
+
+        $this->invokeEnvironmentBootstrappedHook();
+
+        Piwik::postEvent('Environment.bootstrapped'); // this event should be removed eventually
+    }
+
+    /**
+     * Destroys an environment. MUST be called when embedding environments.
+     */
+    public function destroy()
+    {
+        StaticContainer::pop();
+    }
+
+    /**
+     * Returns the DI container. All Piwik objects for a specific Piwik instance should be stored
+     * in this container.
+     *
+     * @return Container
+     */
+    public function getContainer()
+    {
+        return $this->container;
+    }
+
+    /**
+     * @link http://php-di.org/doc/container-configuration.html
+     */
+    private function createContainer()
+    {
+        $pluginList = $this->getPluginListCached();
+        $settings = $this->getGlobalSettingsCached();
+
+        $extraDefinitions = $this->getExtraDefinitionsFromManipulators();
+        $definitions = array_merge(StaticContainer::getDefinitions(), $extraDefinitions, array($this->definitions));
+
+        $environments = array($this->environment);
+        $environments = array_merge($environments, $this->getExtraEnvironmentsFromManipulators());
+
+        $containerFactory = new ContainerFactory($pluginList, $settings, $environments, $definitions);
+        return $containerFactory->create();
+    }
+
+    protected function getGlobalSettingsCached()
+    {
+        if ($this->globalSettingsProvider === null) {
+            $original = $this->getGlobalSettings();
+            $globalSettingsProvider = $this->getGlobalSettingsProviderOverride($original);
+
+            $this->globalSettingsProvider = $globalSettingsProvider ?: $original;
+        }
+        return $this->globalSettingsProvider;
+    }
+
+    protected function getPluginListCached()
+    {
+        if ($this->pluginList === null) {
+            $pluginList = $this->getPluginListOverride();
+            $this->pluginList = $pluginList ?: $this->getPluginList();
+        }
+        return $this->pluginList;
+    }
+
+    /**
+     * Returns the kernel global GlobalSettingsProvider object. Derived classes can override this method
+     * to provide a different implementation.
+     *
+     * @return null|GlobalSettingsProvider
+     */
+    protected function getGlobalSettings()
+    {
+        return new GlobalSettingsProvider();
+    }
+
+    /**
+     * Returns the kernel global PluginList object. Derived classes can override this method to
+     * provide a different implementation.
+     *
+     * @return PluginList
+     */
+    protected function getPluginList()
+    {
+        // TODO: in tracker should only load tracker plugins. can't do properly until tracker entrypoint is encapsulated.
+        return new PluginList($this->getGlobalSettingsCached());
+    }
+
+    private function validateEnvironment()
+    {
+        /** @var EnvironmentValidator $validator */
+        $validator = $this->container->get('Piwik\Application\Kernel\EnvironmentValidator');
+        $validator->validate();
+    }
+
+    /**
+     * @param EnvironmentManipulator $manipulator
+     * @internal
+     */
+    public static function setGlobalEnvironmentManipulator(EnvironmentManipulator $manipulator)
+    {
+        self::$globalEnvironmentManipulator = $manipulator;
+    }
+
+    private function getGlobalSettingsProviderOverride(GlobalSettingsProvider $original)
+    {
+        if (self::$globalEnvironmentManipulator) {
+            return self::$globalEnvironmentManipulator->makeGlobalSettingsProvider($original);
+        } else {
+            return null;
+        }
+    }
+
+    private function invokeBeforeContainerCreatedHook()
+    {
+        if (self::$globalEnvironmentManipulator) {
+            return self::$globalEnvironmentManipulator->beforeContainerCreated();
+        }
+    }
+
+    private function getExtraDefinitionsFromManipulators()
+    {
+        if (self::$globalEnvironmentManipulator) {
+            return self::$globalEnvironmentManipulator->getExtraDefinitions();
+        } else {
+            return array();
+        }
+    }
+
+    private function invokeEnvironmentBootstrappedHook()
+    {
+        if (self::$globalEnvironmentManipulator) {
+            self::$globalEnvironmentManipulator->onEnvironmentBootstrapped();
+        }
+    }
+
+    private function getExtraEnvironmentsFromManipulators()
+    {
+        if (self::$globalEnvironmentManipulator) {
+            return self::$globalEnvironmentManipulator->getExtraEnvironments();
+        } else {
+            return array();
+        }
+    }
+
+    private function getPluginListOverride()
+    {
+        if (self::$globalEnvironmentManipulator) {
+            return self::$globalEnvironmentManipulator->makePluginList($this->getGlobalSettingsCached());
+        } else {
+            return null;
+        }
+    }
+}

msd2/tracking/piwik/core/Application/EnvironmentManipulator.php

@@ -0,0 +1,59 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+
+namespace Piwik\Application;
+
+use Piwik\Application\Kernel\GlobalSettingsProvider;
+use Piwik\Application\Kernel\PluginList;
+
+/**
+ * Used to manipulate Environment instances before the container is created.
+ * Only used by the testing environment setup code, shouldn't be used anywhere
+ * else.
+ */
+interface EnvironmentManipulator
+{
+    /**
+     * Create a custom GlobalSettingsProvider kernel object, overriding the default behavior.
+     *
+     * @return GlobalSettingsProvider
+     */
+    public function makeGlobalSettingsProvider(GlobalSettingsProvider $original);
+
+    /**
+     * Create a custom PluginList kernel object, overriding the default behavior.@deprecated
+     *
+     * @param GlobalSettingsProvider $globalSettingsProvider
+     * @return PluginList
+     */
+    public function makePluginList(GlobalSettingsProvider $globalSettingsProvider);
+
+    /**
+     * Invoked before the container is created.
+     */
+    public function beforeContainerCreated();
+
+    /**
+     * Return an array of definition arrays that override DI config specified in PHP config files.
+     *
+     * @return array[]
+     */
+    public function getExtraDefinitions();
+
+    /**
+     * Invoked after the container is created and the environment is considered bootstrapped.
+     */
+    public function onEnvironmentBootstrapped();
+
+    /**
+     * Return an array of environment names to apply after the normal environment.
+     *
+     * @return string[]
+     */
+    public function getExtraEnvironments();
+}

msd2/tracking/piwik/core/Application/Kernel/EnvironmentValidator.php

@@ -0,0 +1,143 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+
+namespace Piwik\Application\Kernel;
+
+use Piwik\Common;
+use Piwik\Exception\NotYetInstalledException;
+use Piwik\Filechecks;
+use Piwik\Piwik;
+use Piwik\SettingsPiwik;
+use Piwik\SettingsServer;
+use Piwik\Translation\Translator;
+
+/**
+ * Validates the Piwik environment. This includes making sure the required config files
+ * are present, and triggering the correct behaviour if otherwise.
+ */
+class EnvironmentValidator
+{
+    /**
+     * @var GlobalSettingsProvider
+     */
+    protected $settingsProvider;
+
+    /**
+     * @var Translator
+     */
+    protected $translator;
+
+    public function __construct(GlobalSettingsProvider $settingsProvider, Translator $translator)
+    {
+        $this->settingsProvider = $settingsProvider;
+        $this->translator = $translator;
+    }
+
+    public function validate()
+    {
+        $this->checkConfigFileExists($this->settingsProvider->getPathGlobal());
+
+        if(SettingsPiwik::isPiwikInstalled()) {
+            $this->checkConfigFileExists($this->settingsProvider->getPathLocal(), $startInstaller = false);
+            return;
+        }
+
+        $startInstaller = true;
+
+        if(SettingsServer::isTrackerApiRequest()) {
+            // if Piwik is not installed yet, the piwik.php should do nothing and not return an error
+            throw new NotYetInstalledException("As Matomo is not installed yet, the Tracking API cannot proceed and will exit without error.");
+        }
+
+        if(Common::isPhpCliMode()) {
+            // in CLI, do not start/redirect to installer, simply output the exception at the top
+            $startInstaller = false;
+        }
+
+        // Start the installation when config file not found
+        $this->checkConfigFileExists($this->settingsProvider->getPathLocal(), $startInstaller);
+
+    }
+
+    /**
+     * @param $path
+     * @param bool $startInstaller
+     * @throws \Exception
+     */
+    private function checkConfigFileExists($path, $startInstaller = false)
+    {
+        if (is_readable($path)) {
+            return;
+        }
+
+        $message = $this->getSpecificMessageWhetherFileExistsOrNot($path);
+
+        $exception = new NotYetInstalledException($message);
+
+        if ($startInstaller) {
+            $this->startInstallation($exception);
+        } else {
+            throw $exception;
+        }
+    }
+
+    /**
+     * @param $exception
+     */
+    private function startInstallation($exception)
+    {
+        /**
+         * Triggered when the configuration file cannot be found or read, which usually
+         * means Piwik is not installed yet.
+         *
+         * This event can be used to start the installation process or to display a custom error message.
+         *
+         * @param \Exception $exception The exception that was thrown by `Config::getInstance()`.
+         */
+        Piwik::postEvent('Config.NoConfigurationFile', array($exception), $pending = true);
+    }
+
+    /**
+     * @param $path
+     * @return string
+     */
+    private function getMessageWhenFileExistsButNotReadable($path)
+    {
+        $format = " \n<b>» %s </b>";
+        if(Common::isPhpCliMode()) {
+            $format = "\n » %s \n";
+        }
+
+        return sprintf($format,
+            $this->translator->translate('General_ExceptionConfigurationFilePleaseCheckReadableByUser',
+                array($path, Filechecks::getUser())));
+    }
+
+    /**
+     * @param $path
+     * @return string
+     */
+    private function getSpecificMessageWhetherFileExistsOrNot($path)
+    {
+        if (!file_exists($path)) {
+            $message = $this->translator->translate('General_ExceptionConfigurationFileNotFound', array($path));
+            if (Common::isPhpCliMode()) {
+                $message .= $this->getMessageWhenFileExistsButNotReadable($path);
+            }
+        } else {
+            $message = $this->translator->translate('General_ExceptionConfigurationFileExistsButNotReadable',
+                array($path));
+            $message .= $this->getMessageWhenFileExistsButNotReadable($path);
+        }
+
+        if (Common::isPhpCliMode()) {
+            $message = "\n" . $message;
+        }
+        return $message;
+    }
+}

msd2/tracking/piwik/core/Application/Kernel/GlobalSettingsProvider.php

@@ -0,0 +1,111 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+
+namespace Piwik\Application\Kernel;
+
+use Piwik\Config;
+use Piwik\Config\IniFileChain;
+
+/**
+ * Provides global settings. Global settings are organized in sections where
+ * each section contains a list of name => value pairs. Setting values can
+ * be primitive values or arrays of primitive values.
+ *
+ * Uses the config.ini.php, common.ini.php and global.ini.php files to provide global settings.
+ *
+ * At the moment a singleton instance of this class is used in order to get tests to pass.
+ */
+class GlobalSettingsProvider
+{
+    /**
+     * @var IniFileChain
+     */
+    protected $iniFileChain;
+
+    /**
+     * @var string
+     */
+    protected $pathGlobal = null;
+
+    /**
+     * @var string
+     */
+    protected $pathCommon = null;
+
+    /**
+     * @var string
+     */
+    protected $pathLocal = null;
+
+    /**
+     * @param string|null $pathGlobal Path to the global.ini.php file. Or null to use the default.
+     * @param string|null $pathLocal Path to the config.ini.php file. Or null to use the default.
+     * @param string|null $pathCommon Path to the common.ini.php file. Or null to use the default.
+     */
+    public function __construct($pathGlobal = null, $pathLocal = null, $pathCommon = null)
+    {
+        $this->pathGlobal = $pathGlobal ?: Config::getGlobalConfigPath();
+        $this->pathCommon = $pathCommon ?: Config::getCommonConfigPath();
+        $this->pathLocal = $pathLocal ?: Config::getLocalConfigPath();
+
+        $this->iniFileChain = new IniFileChain();
+        $this->reload();
+    }
+
+    public function reload($pathGlobal = null, $pathLocal = null, $pathCommon = null)
+    {
+        $this->pathGlobal = $pathGlobal ?: $this->pathGlobal;
+        $this->pathCommon = $pathCommon ?: $this->pathCommon;
+        $this->pathLocal = $pathLocal ?: $this->pathLocal;
+
+        $this->iniFileChain->reload(array($this->pathGlobal, $this->pathCommon), $this->pathLocal);
+    }
+
+    /**
+     * Returns a settings section.
+     *
+     * @param string $name
+     * @return array
+     */
+    public function &getSection($name)
+    {
+        $section =& $this->iniFileChain->get($name);
+        return $section;
+    }
+
+    /**
+     * Sets a settings section.
+     *
+     * @param string $name
+     * @param array $value
+     */
+    public function setSection($name, $value)
+    {
+        $this->iniFileChain->set($name, $value);
+    }
+
+    public function getIniFileChain()
+    {
+        return $this->iniFileChain;
+    }
+
+    public function getPathGlobal()
+    {
+        return $this->pathGlobal;
+    }
+
+    public function getPathLocal()
+    {
+        return $this->pathLocal;
+    }
+
+    public function getPathCommon()
+    {
+        return $this->pathCommon;
+    }
+}

msd2/tracking/piwik/core/Application/Kernel/PluginList.php

@@ -0,0 +1,193 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+
+namespace Piwik\Application\Kernel;
+
+use Piwik\Plugin\MetadataLoader;
+
+/**
+ * Lists the currently activated plugins. Used when setting up Piwik's environment before
+ * initializing the DI container.
+ *
+ * Uses the [Plugins] section in Piwik's INI config to get the activated plugins.
+ *
+ * Depends on GlobalSettingsProvider being used.
+ *
+ * TODO: parts of Plugin\Manager edit the plugin list; maybe PluginList implementations should be mutable?
+ */
+class PluginList
+{
+    /**
+     * @var GlobalSettingsProvider
+     */
+    private $settings;
+
+    /**
+     * Plugins bundled with core package, disabled by default
+     * @var array
+     */
+    private $corePluginsDisabledByDefault = array(
+        'DBStats',
+        'ExampleCommand',
+        'ExampleSettingsPlugin',
+        'ExampleUI',
+        'ExampleVisualization',
+        'ExamplePluginTemplate',
+        'ExampleTracker',
+        'ExampleLogTables',
+        'ExampleReport',
+        'MobileAppMeasurable',
+        'Provider',
+        'TagManager'
+    );
+
+    // Themes bundled with core package, disabled by default
+    private $coreThemesDisabledByDefault = array(
+        'ExampleTheme'
+    );
+
+    public function __construct(GlobalSettingsProvider $settings)
+    {
+        $this->settings = $settings;
+    }
+
+    /**
+     * Returns the list of plugins that should be loaded. Used by the container factory to
+     * load plugin specific DI overrides.
+     *
+     * @return string[]
+     */
+    public function getActivatedPlugins()
+    {
+        $section = $this->settings->getSection('Plugins');
+        return @$section['Plugins'] ?: array();
+    }
+
+    /**
+     * Returns the list of plugins that are bundled with Piwik.
+     *
+     * @return string[]
+     */
+    public function getPluginsBundledWithPiwik()
+    {
+        $pathGlobal = $this->settings->getPathGlobal();
+
+        $section = $this->settings->getIniFileChain()->getFrom($pathGlobal, 'Plugins');
+        return $section['Plugins'];
+    }
+
+    /**
+     * Returns the plugins bundled with core package that are disabled by default.
+     *
+     * @return string[]
+     */
+    public function getCorePluginsDisabledByDefault()
+    {
+        return array_merge($this->corePluginsDisabledByDefault, $this->coreThemesDisabledByDefault);
+    }
+
+    /**
+     * Sorts an array of plugins in the order they should be loaded. We cannot use DI here as DI is not initialized
+     * at this stage.
+     *
+     * @params string[] $plugins
+     * @return \string[]
+     */
+    public function sortPlugins(array $plugins)
+    {
+        $global = $this->getPluginsBundledWithPiwik();
+        if (empty($global)) {
+            return $plugins;
+        }
+
+        // we need to make sure a possibly disabled plugin will be still loaded before any 3rd party plugin
+        $global = array_merge($global, $this->corePluginsDisabledByDefault);
+
+        $global = array_values($global);
+        $plugins = array_values($plugins);
+
+        $defaultPluginsLoadedFirst = array_intersect($global, $plugins);
+
+        $otherPluginsToLoadAfterDefaultPlugins = array_diff($plugins, $defaultPluginsLoadedFirst);
+
+        // sort by name to have a predictable order for those extra plugins
+        natcasesort($otherPluginsToLoadAfterDefaultPlugins);
+
+        $sorted = array_merge($defaultPluginsLoadedFirst, $otherPluginsToLoadAfterDefaultPlugins);
+
+        return $sorted;
+    }
+
+    /**
+     * Sorts an array of plugins in the order they should be saved in config.ini.php. This basically influences
+     * the order of the plugin config.php and which config will be loaded first. We want to make sure to require the
+     * config or a required plugin first before loading the plugin that requires it.
+     *
+     * We do not sort using this logic on each request since it is much slower than `sortPlugins()`. The order
+     * of plugins in config.ini.php is only important for the ContainerFactory. During a regular request it is otherwise
+     * fine to load the plugins in the order of `sortPlugins()` since we will make sure that required plugins will be
+     * loaded first in plugin manager.
+     *
+     * @param string[] $plugins
+     * @param array[] $pluginJsonCache  For internal testing only
+     * @return \string[]
+     */
+    public function sortPluginsAndRespectDependencies(array $plugins, $pluginJsonCache = array())
+    {
+        $global = $this->getPluginsBundledWithPiwik();
+
+        if (empty($global)) {
+            return $plugins;
+        }
+
+        // we need to make sure a possibly disabled plugin will be still loaded before any 3rd party plugin
+        $global = array_merge($global, $this->corePluginsDisabledByDefault);
+
+        $global = array_values($global);
+        $plugins = array_values($plugins);
+
+        $defaultPluginsLoadedFirst = array_intersect($global, $plugins);
+
+        $otherPluginsToLoadAfterDefaultPlugins = array_diff($plugins, $defaultPluginsLoadedFirst);
+
+        // we still want to sort alphabetically by default
+        natcasesort($otherPluginsToLoadAfterDefaultPlugins);
+
+        $sorted = array();
+        foreach ($otherPluginsToLoadAfterDefaultPlugins as $pluginName) {
+            $sorted = $this->sortRequiredPlugin($pluginName, $pluginJsonCache, $otherPluginsToLoadAfterDefaultPlugins, $sorted);
+        }
+
+        $sorted = array_merge($defaultPluginsLoadedFirst, $sorted);
+
+        return $sorted;
+    }
+
+    private function sortRequiredPlugin($pluginName, &$pluginJsonCache, $toBeSorted, $sorted)
+    {
+        if (!isset($pluginJsonCache[$pluginName])) {
+            $loader = new MetadataLoader($pluginName);
+            $pluginJsonCache[$pluginName] = $loader->loadPluginInfoJson();
+        }
+
+        if (!empty($pluginJsonCache[$pluginName]['require'])) {
+            $dependencies = $pluginJsonCache[$pluginName]['require'];
+            foreach ($dependencies as $possiblePluginName => $key) {
+                if (in_array($possiblePluginName, $toBeSorted, true) && !in_array($possiblePluginName, $sorted, true)) {
+                    $sorted = $this->sortRequiredPlugin($possiblePluginName, $pluginJsonCache, $toBeSorted, $sorted);
+                }
+            }
+        }
+
+        if (!in_array($pluginName, $sorted, true)) {
+            $sorted[] = $pluginName;
+        }
+
+        return $sorted;
+    }
+}

msd2/tracking/piwik/core/Archive.php

@@ -0,0 +1,893 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik;
+
+use Piwik\Archive\ArchiveQuery;
+use Piwik\Archive\ArchiveQueryFactory;
+use Piwik\Archive\Parameters;
+use Piwik\ArchiveProcessor\Rules;
+use Piwik\Archive\ArchiveInvalidator;
+use Piwik\Container\StaticContainer;
+use Piwik\DataAccess\ArchiveSelector;
+
+/**
+ * The **Archive** class is used to query cached analytics statistics
+ * (termed "archive data").
+ *
+ * You can use **Archive** instances to get data that was archived for one or more sites,
+ * for one or more periods and one optional segment.
+ *
+ * If archive data is not found, this class will initiate the archiving process. [1](#footnote-1)
+ *
+ * **Archive** instances must be created using the {@link build()} factory method;
+ * they cannot be constructed.
+ *
+ * You can search for metrics (such as `nb_visits`) using the {@link getNumeric()} and
+ * {@link getDataTableFromNumeric()} methods. You can search for
+ * reports using the {@link getBlob()}, {@link getDataTable()} and {@link getDataTableExpanded()} methods.
+ *
+ * If you're creating an API that returns report data, you may want to use the
+ * {@link createDataTableFromArchive()} helper function.
+ *
+ * ### Learn more
+ *
+ * Learn more about _archiving_ [here](/guides/all-about-analytics-data).
+ *
+ * ### Limitations
+ *
+ * - You cannot get data for multiple range periods in a single query.
+ * - You cannot get data for periods of different types in a single query.
+ *
+ * ### Examples
+ *
+ * **_Querying metrics for an API method_**
+ *
+ *     // one site and one period
+ *     $archive = Archive::build($idSite = 1, $period = 'week', $date = '2013-03-08');
+ *     return $archive->getDataTableFromNumeric(array('nb_visits', 'nb_actions'));
+ *
+ *     // all sites and multiple dates
+ *     $archive = Archive::build($idSite = 'all', $period = 'month', $date = '2013-01-02,2013-03-08');
+ *     return $archive->getDataTableFromNumeric(array('nb_visits', 'nb_actions'));
+ *
+ * **_Querying and using metrics immediately_**
+ *
+ *     // one site and one period
+ *     $archive = Archive::build($idSite = 1, $period = 'week', $date = '2013-03-08');
+ *     $data = $archive->getNumeric(array('nb_visits', 'nb_actions'));
+ *
+ *     $visits = $data['nb_visits'];
+ *     $actions = $data['nb_actions'];
+ *
+ *     // ... do something w/ metric data ...
+ *
+ *     // multiple sites and multiple dates
+ *     $archive = Archive::build($idSite = '1,2,3', $period = 'month', $date = '2013-01-02,2013-03-08');
+ *     $data = $archive->getNumeric('nb_visits');
+ *
+ *     $janSite1Visits = $data['1']['2013-01-01,2013-01-31']['nb_visits'];
+ *     $febSite1Visits = $data['1']['2013-02-01,2013-02-28']['nb_visits'];
+ *     // ... etc.
+ *
+ * **_Querying for reports_**
+ *
+ *     $archive = Archive::build($idSite = 1, $period = 'week', $date = '2013-03-08');
+ *     $dataTable = $archive->getDataTable('MyPlugin_MyReport');
+ *     // ... manipulate $dataTable ...
+ *     return $dataTable;
+ *
+ * **_Querying a report for an API method_**
+ *
+ *     public function getMyReport($idSite, $period, $date, $segment = false, $expanded = false)
+ *     {
+ *         $dataTable = Archive::createDataTableFromArchive('MyPlugin_MyReport', $idSite, $period, $date, $segment, $expanded);
+ *         return $dataTable;
+ *     }
+ *
+ * **_Querying data for multiple range periods_**
+ *
+ *     // get data for first range
+ *     $archive = Archive::build($idSite = 1, $period = 'range', $date = '2013-03-08,2013-03-12');
+ *     $dataTable = $archive->getDataTableFromNumeric(array('nb_visits', 'nb_actions'));
+ *
+ *     // get data for second range
+ *     $archive = Archive::build($idSite = 1, $period = 'range', $date = '2013-03-15,2013-03-20');
+ *     $dataTable = $archive->getDataTableFromNumeric(array('nb_visits', 'nb_actions'));
+ *
+ * <a name="footnote-1"></a>
+ * [1]: The archiving process will not be launched if browser archiving is disabled
+ *      and the current request came from a browser.
+ *
+ *
+ * @api
+ */
+class Archive implements ArchiveQuery
+{
+    const REQUEST_ALL_WEBSITES_FLAG = 'all';
+    const ARCHIVE_ALL_PLUGINS_FLAG = 'all';
+    const ID_SUBTABLE_LOAD_ALL_SUBTABLES = 'all';
+
+    /**
+     * List of archive IDs for the site, periods and segment we are querying with.
+     * Archive IDs are indexed by done flag and period, ie:
+     *
+     * array(
+     *     'done.Referrers' => array(
+     *         '2010-01-01' => 1,
+     *         '2010-01-02' => 2,
+     *     ),
+     *     'done.VisitsSummary' => array(
+     *         '2010-01-01' => 3,
+     *         '2010-01-02' => 4,
+     *     ),
+     * )
+     *
+     * or,
+     *
+     * array(
+     *     'done.all' => array(
+     *         '2010-01-01' => 1,
+     *         '2010-01-02' => 2
+     *     )
+     * )
+     *
+     * @var array
+     */
+    private $idarchives = array();
+
+    /**
+     * If set to true, the result of all get functions (ie, getNumeric, getBlob, etc.)
+     * will be indexed by the site ID, even if we're only querying data for one site.
+     *
+     * @var bool
+     */
+    private $forceIndexedBySite;
+
+    /**
+     * If set to true, the result of all get functions (ie, getNumeric, getBlob, etc.)
+     * will be indexed by the period, even if we're only querying data for one period.
+     *
+     * @var bool
+     */
+    private $forceIndexedByDate;
+
+    /**
+     * @var Parameters
+     */
+    private $params;
+
+    /**
+     * @var \Piwik\Cache\Cache
+     */
+    private static $cache;
+
+    /**
+     * @var ArchiveInvalidator
+     */
+    private $invalidator;
+
+    /**
+     * @param Parameters $params
+     * @param bool $forceIndexedBySite Whether to force index the result of a query by site ID.
+     * @param bool $forceIndexedByDate Whether to force index the result of a query by period.
+     */
+    public function __construct(Parameters $params, $forceIndexedBySite = false,
+                                   $forceIndexedByDate = false)
+    {
+        $this->params = $params;
+        $this->forceIndexedBySite = $forceIndexedBySite;
+        $this->forceIndexedByDate = $forceIndexedByDate;
+
+        $this->invalidator = StaticContainer::get('Piwik\Archive\ArchiveInvalidator');
+    }
+
+    /**
+     * Returns a new Archive instance that will query archive data for the given set of
+     * sites and periods, using an optional Segment.
+     *
+     * This method uses data that is found in query parameters, so the parameters to this
+     * function can be string values.
+     *
+     * If you want to create an Archive instance with an array of Period instances, use
+     * {@link Archive::factory()}.
+     *
+     * @param string|int|array $idSites A single ID (eg, `'1'`), multiple IDs (eg, `'1,2,3'` or `array(1, 2, 3)`),
+     *                                  or `'all'`.
+     * @param string $period 'day', `'week'`, `'month'`, `'year'` or `'range'`
+     * @param Date|string $strDate 'YYYY-MM-DD', magic keywords (ie, 'today'; {@link Date::factory()}
+     *                             or date range (ie, 'YYYY-MM-DD,YYYY-MM-DD').
+     * @param bool|false|string $segment Segment definition or false if no segment should be used. {@link Piwik\Segment}
+     * @param bool|false|string $_restrictSitesToLogin Used only when running as a scheduled task.
+     * @return ArchiveQuery
+     */
+    public static function build($idSites, $period, $strDate, $segment = false, $_restrictSitesToLogin = false)
+    {
+        return StaticContainer::get(ArchiveQueryFactory::class)->build($idSites, $period, $strDate, $segment,
+            $_restrictSitesToLogin);
+    }
+
+    /**
+     * Returns a new Archive instance that will query archive data for the given set of
+     * sites and periods, using an optional segment.
+     *
+     * This method uses an array of Period instances and a Segment instance, instead of strings
+     * like {@link build()}.
+     *
+     * If you want to create an Archive instance using data found in query parameters,
+     * use {@link build()}.
+     *
+     * @param Segment $segment The segment to use. For no segment, use `new Segment('', $idSites)`.
+     * @param array $periods An array of Period instances.
+     * @param array $idSites An array of site IDs (eg, `array(1, 2, 3)`).
+     * @param bool $idSiteIsAll Whether `'all'` sites are being queried or not. If true, then
+     *                          the result of querying functions will be indexed by site, regardless
+     *                          of whether `count($idSites) == 1`.
+     * @param bool $isMultipleDate Whether multiple dates are being queried or not. If true, then
+     *                             the result of querying functions will be indexed by period,
+     *                             regardless of whether `count($periods) == 1`.
+     *
+     * @return ArchiveQuery
+     */
+    public static function factory(Segment $segment, array $periods, array $idSites, $idSiteIsAll = false,
+                                   $isMultipleDate = false)
+    {
+        return StaticContainer::get(ArchiveQueryFactory::class)->factory($segment, $periods, $idSites, $idSiteIsAll,
+            $isMultipleDate);
+    }
+
+    /**
+     * Queries and returns metric data in an array.
+     *
+     * If multiple sites were requested in {@link build()} or {@link factory()} the result will
+     * be indexed by site ID.
+     *
+     * If multiple periods were requested in {@link build()} or {@link factory()} the result will
+     * be indexed by period.
+     *
+     * The site ID index is always first, so if multiple sites & periods were requested, the result
+     * will be indexed by site ID first, then period.
+     *
+     * @param string|array $names One or more archive names, eg, `'nb_visits'`, `'Referrers_distinctKeywords'`,
+     *                            etc.
+     * @return false|integer|array `false` if there is no data to return, a single numeric value if we're not querying
+     *                             for multiple sites/periods, or an array if multiple sites, periods or names are
+     *                             queried for.
+     */
+    public function getNumeric($names)
+    {
+        $data = $this->get($names, 'numeric');
+
+        $resultIndices = $this->getResultIndices();
+        $result = $data->getIndexedArray($resultIndices);
+
+        // if only one metric is returned, just return it as a numeric value
+        if (empty($resultIndices)
+            && count($result) <= 1
+            && (!is_array($names) || count($names) == 1)
+        ) {
+            $result = (float)reset($result); // convert to float in case $result is empty
+        }
+
+        return $result;
+    }
+
+    /**
+     * Queries and returns metric data in a DataTable instance.
+     *
+     * If multiple sites were requested in {@link build()} or {@link factory()} the result will
+     * be a DataTable\Map that is indexed by site ID.
+     *
+     * If multiple periods were requested in {@link build()} or {@link factory()} the result will
+     * be a {@link DataTable\Map} that is indexed by period.
+     *
+     * The site ID index is always first, so if multiple sites & periods were requested, the result
+     * will be a {@link DataTable\Map} indexed by site ID which contains {@link DataTable\Map} instances that are
+     * indexed by period.
+     *
+     * _Note: Every DataTable instance returned will have at most one row in it. The contents of each
+     *        row will be the requested metrics for the appropriate site and period._
+     *
+     * @param string|array $names One or more archive names, eg, 'nb_visits', 'Referrers_distinctKeywords',
+     *                            etc.
+     * @return DataTable|DataTable\Map A DataTable if multiple sites and periods were not requested.
+     *                                 An appropriately indexed DataTable\Map if otherwise.
+     */
+    public function getDataTableFromNumeric($names)
+    {
+        $data = $this->get($names, 'numeric');
+        return $data->getDataTable($this->getResultIndices());
+    }
+
+    /**
+     * Similar to {@link getDataTableFromNumeric()} but merges all children on the created DataTable.
+     *
+     * This is the same as doing `$this->getDataTableFromNumeric()->mergeChildren()` but this way it is much faster.
+     *
+     * @return DataTable|DataTable\Map
+     *
+     * @internal Currently only used by MultiSites.getAll plugin. Feel free to remove internal tag if needed somewhere
+     *           else. If no longer needed by MultiSites.getAll please remove this method. If you need this to work in
+     *           a bit different way feel free to refactor as always.
+     */
+    public function getDataTableFromNumericAndMergeChildren($names)
+    {
+        $data  = $this->get($names, 'numeric');
+        $resultIndexes = $this->getResultIndices();
+        return $data->getMergedDataTable($resultIndexes);
+    }
+
+    /**
+     * Queries and returns one or more reports as DataTable instances.
+     *
+     * This method will query blob data that is a serialized array of of {@link DataTable\Row}'s and
+     * unserialize it.
+     *
+     * If multiple sites were requested in {@link build()} or {@link factory()} the result will
+     * be a {@link DataTable\Map} that is indexed by site ID.
+     *
+     * If multiple periods were requested in {@link build()} or {@link factory()} the result will
+     * be a DataTable\Map that is indexed by period.
+     *
+     * The site ID index is always first, so if multiple sites & periods were requested, the result
+     * will be a {@link DataTable\Map} indexed by site ID which contains {@link DataTable\Map} instances that are
+     * indexed by period.
+     *
+     * @param string $name The name of the record to get. This method can only query one record at a time.
+     * @param int|string|null $idSubtable The ID of the subtable to get (if any).
+     * @return DataTable|DataTable\Map A DataTable if multiple sites and periods were not requested.
+     *                                 An appropriately indexed {@link DataTable\Map} if otherwise.
+     */
+    public function getDataTable($name, $idSubtable = null)
+    {
+        $data = $this->get($name, 'blob', $idSubtable);
+        return $data->getDataTable($this->getResultIndices());
+    }
+
+    /**
+     * Queries and returns one report with all of its subtables loaded.
+     *
+     * If multiple sites were requested in {@link build()} or {@link factory()} the result will
+     * be a DataTable\Map that is indexed by site ID.
+     *
+     * If multiple periods were requested in {@link build()} or {@link factory()} the result will
+     * be a DataTable\Map that is indexed by period.
+     *
+     * The site ID index is always first, so if multiple sites & periods were requested, the result
+     * will be a {@link DataTable\Map indexed} by site ID which contains {@link DataTable\Map} instances that are
+     * indexed by period.
+     *
+     * @param string $name The name of the record to get.
+     * @param int|string|null $idSubtable The ID of the subtable to get (if any). The subtable will be expanded.
+     * @param int|null $depth The maximum number of subtable levels to load. If null, all levels are loaded.
+     *                        For example, if `1` is supplied, then the DataTable returned will have its subtables
+     *                        loaded. Those subtables, however, will NOT have their subtables loaded.
+     * @param bool $addMetadataSubtableId Whether to add the database subtable ID as metadata to each datatable,
+     *                                    or not.
+     * @return DataTable|DataTable\Map
+     */
+    public function getDataTableExpanded($name, $idSubtable = null, $depth = null, $addMetadataSubtableId = true)
+    {
+        $data = $this->get($name, 'blob', self::ID_SUBTABLE_LOAD_ALL_SUBTABLES);
+        return $data->getExpandedDataTable($this->getResultIndices(), $idSubtable, $depth, $addMetadataSubtableId);
+    }
+
+    /**
+     * Returns the list of plugins that archive the given reports.
+     *
+     * @param array $archiveNames
+     * @return array
+     */
+    private function getRequestedPlugins($archiveNames)
+    {
+        $result = array();
+
+        foreach ($archiveNames as $name) {
+            $result[] = self::getPluginForReport($name);
+        }
+
+        return array_unique($result);
+    }
+
+    /**
+     * Returns an object describing the set of sites, the set of periods and the segment
+     * this Archive will query data for.
+     *
+     * @return Parameters
+     */
+    public function getParams()
+    {
+        return $this->params;
+    }
+
+    /**
+     * Helper function that creates an Archive instance and queries for report data using
+     * query parameter data. API methods can use this method to reduce code redundancy.
+     *
+     * @param string $recordName The name of the report to return.
+     * @param int|string|array $idSite @see {@link build()}
+     * @param string $period @see {@link build()}
+     * @param string $date @see {@link build()}
+     * @param string $segment @see {@link build()}
+     * @param bool $expanded If true, loads all subtables. See {@link getDataTableExpanded()}
+     * @param bool $flat If true, loads all subtables and disabled all recursive filters.
+     * @param int|null $idSubtable See {@link getDataTableExpanded()}
+     * @param int|null $depth See {@link getDataTableExpanded()}
+     * @return DataTable|DataTable\Map
+     */
+    public static function createDataTableFromArchive($recordName, $idSite, $period, $date, $segment, $expanded = false, $flat = false, $idSubtable = null, $depth = null)
+    {
+        Piwik::checkUserHasViewAccess($idSite);
+
+        if ($flat && !$idSubtable) {
+            $expanded = true;
+        }
+
+        $archive = Archive::build($idSite, $period, $date, $segment, $_restrictSitesToLogin = false);
+        if ($idSubtable === false) {
+            $idSubtable = null;
+        }
+
+        if ($expanded) {
+            $dataTable = $archive->getDataTableExpanded($recordName, $idSubtable, $depth);
+        } else {
+            $dataTable = $archive->getDataTable($recordName, $idSubtable);
+        }
+
+        $dataTable->queueFilter('ReplaceSummaryRowLabel');
+        $dataTable->queueFilter('ReplaceColumnNames');
+
+        if ($expanded) {
+            $dataTable->queueFilterSubtables('ReplaceColumnNames');
+        }
+
+        if ($flat) {
+            $dataTable->disableRecursiveFilters();
+        }
+
+        return $dataTable;
+    }
+
+    private function getSiteIdsThatAreRequestedInThisArchiveButWereNotInvalidatedYet()
+    {
+        if (is_null(self::$cache)) {
+            self::$cache = Cache::getTransientCache();
+        }
+
+        $id = 'Archive.SiteIdsOfRememberedReportsInvalidated';
+
+        if (!self::$cache->contains($id)) {
+            self::$cache->save($id, array());
+        }
+
+        $siteIdsAlreadyHandled = self::$cache->fetch($id);
+        $siteIdsRequested      = $this->params->getIdSites();
+
+        foreach ($siteIdsRequested as $index => $siteIdRequested) {
+            $siteIdRequested = (int) $siteIdRequested;
+
+            if (in_array($siteIdRequested, $siteIdsAlreadyHandled)) {
+                unset($siteIdsRequested[$index]); // was already handled previously, do not do it again
+            } else {
+                $siteIdsAlreadyHandled[] = $siteIdRequested; // we will handle this id this time
+            }
+        }
+
+        self::$cache->save($id, $siteIdsAlreadyHandled);
+
+        return $siteIdsRequested;
+    }
+
+    private function invalidatedReportsIfNeeded()
+    {
+        $siteIdsRequested = $this->getSiteIdsThatAreRequestedInThisArchiveButWereNotInvalidatedYet();
+
+        if (empty($siteIdsRequested)) {
+            return; // all requested site ids were already handled
+        }
+
+        $sitesPerDays = $this->invalidator->getRememberedArchivedReportsThatShouldBeInvalidated();
+
+        foreach ($sitesPerDays as $date => $siteIds) {
+            if (empty($siteIds)) {
+                continue;
+            }
+
+            $siteIdsToActuallyInvalidate = array_intersect($siteIds, $siteIdsRequested);
+
+            if (empty($siteIdsToActuallyInvalidate)) {
+                continue; // all site ids that should be handled are already handled
+            }
+
+            try {
+                $this->invalidator->markArchivesAsInvalidated($siteIdsToActuallyInvalidate, array(Date::factory($date)), false);
+            } catch (\Exception $e) {
+                Site::clearCache();
+                throw $e;
+            }
+        }
+
+        Site::clearCache();
+    }
+
+    /**
+     * Queries archive tables for data and returns the result.
+     * @param array|string $archiveNames
+     * @param $archiveDataType
+     * @param null|int $idSubtable
+     * @return Archive\DataCollection
+     */
+    protected function get($archiveNames, $archiveDataType, $idSubtable = null)
+    {
+        if (!is_array($archiveNames)) {
+            $archiveNames = array($archiveNames);
+        }
+
+        // apply idSubtable
+        if ($idSubtable !== null
+            && $idSubtable != self::ID_SUBTABLE_LOAD_ALL_SUBTABLES
+        ) {
+            // this is also done in ArchiveSelector. It should be actually only done in ArchiveSelector but DataCollection
+            // does require to have the subtableId appended. Needs to be changed in refactoring to have it only in one
+            // place.
+            $dataNames = array();
+            foreach ($archiveNames as $name) {
+                $dataNames[] = ArchiveSelector::appendIdsubtable($name, $idSubtable);
+            }
+        } else {
+            $dataNames = $archiveNames;
+        }
+
+        $result = new Archive\DataCollection(
+            $dataNames, $archiveDataType, $this->params->getIdSites(), $this->params->getPeriods(), $defaultRow = null);
+
+        $archiveIds = $this->getArchiveIds($archiveNames);
+        if (empty($archiveIds)) {
+            /**
+             * Triggered when no archive data is found in an API request.
+             * @ignore
+             */
+            Piwik::postEvent('Archive.noArchivedData');
+            return $result;
+        }
+
+        $archiveData = ArchiveSelector::getArchiveData($archiveIds, $archiveNames, $archiveDataType, $idSubtable);
+
+        $isNumeric = $archiveDataType == 'numeric';
+
+        foreach ($archiveData as $row) {
+            // values are grouped by idsite (site ID), date1-date2 (date range), then name (field name)
+            $periodStr = $row['date1'] . ',' . $row['date2'];
+
+            if ($isNumeric) {
+                $row['value'] = $this->formatNumericValue($row['value']);
+            } else {
+                $result->addMetadata($row['idsite'], $periodStr, DataTable::ARCHIVED_DATE_METADATA_NAME, $row['ts_archived']);
+            }
+
+            $result->set($row['idsite'], $periodStr, $row['name'], $row['value']);
+        }
+
+        return $result;
+    }
+
+    /**
+     * Returns archive IDs for the sites, periods and archive names that are being
+     * queried. This function will use the idarchive cache if it has the right data,
+     * query archive tables for IDs w/o launching archiving, or launch archiving and
+     * get the idarchive from ArchiveProcessor instances.
+     *
+     * @param string $archiveNames
+     * @return array
+     */
+    private function getArchiveIds($archiveNames)
+    {
+        $plugins = $this->getRequestedPlugins($archiveNames);
+
+        // figure out which archives haven't been processed (if an archive has been processed,
+        // then we have the archive IDs in $this->idarchives)
+        $doneFlags     = array();
+        $archiveGroups = array();
+        foreach ($plugins as $plugin) {
+            $doneFlag = $this->getDoneStringForPlugin($plugin, $this->params->getIdSites());
+
+            $doneFlags[$doneFlag] = true;
+            if (!isset($this->idarchives[$doneFlag])) {
+                $archiveGroup = $this->getArchiveGroupOfPlugin($plugin);
+
+                if ($archiveGroup == self::ARCHIVE_ALL_PLUGINS_FLAG) {
+                    $archiveGroup = reset($plugins);
+                }
+                $archiveGroups[] = $archiveGroup;
+            }
+
+            $globalDoneFlag = Rules::getDoneFlagArchiveContainsAllPlugins($this->params->getSegment());
+            if ($globalDoneFlag !== $doneFlag) {
+                $doneFlags[$globalDoneFlag] = true;
+            }
+        }
+
+        $archiveGroups = array_unique($archiveGroups);
+
+        // cache id archives for plugins we haven't processed yet
+        if (!empty($archiveGroups)) {
+            if (!Rules::isArchivingDisabledFor($this->params->getIdSites(), $this->params->getSegment(), $this->getPeriodLabel())) {
+                $this->cacheArchiveIdsAfterLaunching($archiveGroups, $plugins);
+            } else {
+                $this->cacheArchiveIdsWithoutLaunching($plugins);
+            }
+        }
+
+        $idArchivesByMonth = $this->getIdArchivesByMonth($doneFlags);
+
+        return $idArchivesByMonth;
+    }
+
+    /**
+     * Gets the IDs of the archives we're querying for and stores them in $this->archives.
+     * This function will launch the archiving process for each period/site/plugin if
+     * metrics/reports have not been calculated/archived already.
+     *
+     * @param array $archiveGroups @see getArchiveGroupOfReport
+     * @param array $plugins List of plugin names to archive.
+     */
+    private function cacheArchiveIdsAfterLaunching($archiveGroups, $plugins)
+    {
+        $this->invalidatedReportsIfNeeded();
+
+        $today = Date::today();
+
+        foreach ($this->params->getPeriods() as $period) {
+            $twoDaysBeforePeriod = $period->getDateStart()->subDay(2);
+            $twoDaysAfterPeriod = $period->getDateEnd()->addDay(2);
+
+            foreach ($this->params->getIdSites() as $idSite) {
+                $site = new Site($idSite);
+
+                // if the END of the period is BEFORE the website creation date
+                // we already know there are no stats for this period
+                // we add one day to make sure we don't miss the day of the website creation
+                if ($twoDaysAfterPeriod->isEarlier($site->getCreationDate())) {
+                    Log::debug("Archive site %s, %s (%s) skipped, archive is before the website was created.",
+                        $idSite, $period->getLabel(), $period->getPrettyString());
+                    continue;
+                }
+
+                // if the starting date is in the future we know there is no visiidsite = ?t
+                if ($twoDaysBeforePeriod->isLater($today)) {
+                    Log::debug("Archive site %s, %s (%s) skipped, archive is after today.",
+                        $idSite, $period->getLabel(), $period->getPrettyString());
+                    continue;
+                }
+
+                $this->prepareArchive($archiveGroups, $site, $period);
+            }
+        }
+    }
+
+    /**
+     * Gets the IDs of the archives we're querying for and stores them in $this->archives.
+     * This function will not launch the archiving process (and is thus much, much faster
+     * than cacheArchiveIdsAfterLaunching).
+     *
+     * @param array $plugins List of plugin names from which data is being requested.
+     */
+    private function cacheArchiveIdsWithoutLaunching($plugins)
+    {
+        $idarchivesByReport = ArchiveSelector::getArchiveIds(
+            $this->params->getIdSites(), $this->params->getPeriods(), $this->params->getSegment(), $plugins);
+
+        // initialize archive ID cache for each report
+        foreach ($plugins as $plugin) {
+            $doneFlag = $this->getDoneStringForPlugin($plugin, $this->params->getIdSites());
+            $this->initializeArchiveIdCache($doneFlag);
+            $globalDoneFlag = Rules::getDoneFlagArchiveContainsAllPlugins($this->params->getSegment());
+            $this->initializeArchiveIdCache($globalDoneFlag);
+        }
+
+        foreach ($idarchivesByReport as $doneFlag => $idarchivesByDate) {
+            foreach ($idarchivesByDate as $dateRange => $idarchives) {
+                foreach ($idarchives as $idarchive) {
+                    $this->idarchives[$doneFlag][$dateRange][] = $idarchive;
+                }
+            }
+        }
+    }
+
+    /**
+     * Returns the done string flag for a plugin using this instance's segment & periods.
+     * @param string $plugin
+     * @return string
+     */
+    private function getDoneStringForPlugin($plugin, $idSites)
+    {
+        return Rules::getDoneStringFlagFor(
+                    $idSites,
+                    $this->params->getSegment(),
+                    $this->getPeriodLabel(),
+                    $plugin
+        );
+    }
+
+    private function getPeriodLabel()
+    {
+        $periods = $this->params->getPeriods();
+        return reset($periods)->getLabel();
+    }
+
+    /**
+     * Returns an array describing what metadata to use when indexing a query result.
+     * For use with DataCollection.
+     *
+     * @return array
+     */
+    private function getResultIndices()
+    {
+        $indices = array();
+
+        if (count($this->params->getIdSites()) > 1
+            || $this->forceIndexedBySite
+        ) {
+            $indices['site'] = 'idSite';
+        }
+
+        if (count($this->params->getPeriods()) > 1
+            || $this->forceIndexedByDate
+        ) {
+            $indices['period'] = 'date';
+        }
+
+        return $indices;
+    }
+
+    private function formatNumericValue($value)
+    {
+        // If there is no dot, we return as is
+        // Note: this could be an integer bigger than 32 bits
+        if (strpos($value, '.') === false) {
+            if ($value === false) {
+                return 0;
+            }
+            return (float)$value;
+        }
+
+        // Round up the value with 2 decimals
+        // we cast the result as float because returns false when no visitors
+        return round((float)$value, 2);
+    }
+
+    /**
+     * Initializes the archive ID cache ($this->idarchives) for a particular 'done' flag.
+     *
+     * It is necessary that each archive ID caching function call this method for each
+     * unique 'done' flag it encounters, since the getArchiveIds function determines
+     * whether archiving should be launched based on whether $this->idarchives has a
+     * an entry for a specific 'done' flag.
+     *
+     * If this  function is not called, then periods with no visits will not add
+     * entries to the cache. If the archive is used again, SQL will be executed to
+     * try and find the archive IDs even though we know there are none.
+     *
+     * @param string $doneFlag
+     */
+    private function initializeArchiveIdCache($doneFlag)
+    {
+        if (!isset($this->idarchives[$doneFlag])) {
+            $this->idarchives[$doneFlag] = array();
+        }
+    }
+
+    /**
+     * Returns the archiving group identifier given a plugin.
+     *
+     * More than one plugin can be called at once when archiving. In such a case
+     * we don't want to launch archiving three times for three plugins if doing
+     * it once is enough, so getArchiveIds makes sure to get the archive group of
+     * all reports.
+     *
+     * If the period isn't a range, then all plugins' archiving code is executed.
+     * If the period is a range, then archiving code is executed individually for
+     * each plugin.
+     */
+    private function getArchiveGroupOfPlugin($plugin)
+    {
+        $periods = $this->params->getPeriods();
+        $periodLabel = reset($periods)->getLabel();
+        
+        if (Rules::shouldProcessReportsAllPlugins($this->params->getIdSites(), $this->params->getSegment(), $periodLabel)) {
+            return self::ARCHIVE_ALL_PLUGINS_FLAG;
+        }
+
+        return $plugin;
+    }
+
+    /**
+     * Returns the name of the plugin that archives a given report.
+     *
+     * @param string $report Archive data name, eg, `'nb_visits'`, `'DevicesDetection_...'`, etc.
+     * @return string Plugin name.
+     * @throws \Exception If a plugin cannot be found or if the plugin for the report isn't
+     *                    activated.
+     */
+    public static function getPluginForReport($report)
+    {
+        // Core metrics are always processed in Core, for the requested date/period/segment
+        if (in_array($report, Metrics::getVisitsMetricNames())) {
+            $report = 'VisitsSummary_CoreMetrics';
+        } // Goal_* metrics are processed by the Goals plugin (HACK)
+        elseif (strpos($report, 'Goal_') === 0) {
+            $report = 'Goals_Metrics';
+        } elseif (strrpos($report, '_returning') === strlen($report) - strlen('_returning')) { // HACK
+            $report = 'VisitFrequency_Metrics';
+        }
+
+        $plugin = substr($report, 0, strpos($report, '_'));
+        if (empty($plugin)
+            || !\Piwik\Plugin\Manager::getInstance()->isPluginActivated($plugin)
+        ) {
+            throw new \Exception("Error: The report '$report' was requested but it is not available at this stage."
+                               . " (Plugin '$plugin' is not activated.)");
+        }
+        return $plugin;
+    }
+
+    /**
+     * @param $archiveGroups
+     * @param $site
+     * @param $period
+     */
+    private function prepareArchive(array $archiveGroups, Site $site, Period $period)
+    {
+        $parameters = new ArchiveProcessor\Parameters($site, $period, $this->params->getSegment());
+        $archiveLoader = new ArchiveProcessor\Loader($parameters);
+
+        $periodString = $period->getRangeString();
+
+        $idSites = array($site->getId());
+        
+        // process for each plugin as well
+        foreach ($archiveGroups as $plugin) {
+            $doneFlag = $this->getDoneStringForPlugin($plugin, $idSites);
+            $this->initializeArchiveIdCache($doneFlag);
+
+            $idArchive = $archiveLoader->prepareArchive($plugin);
+
+            if ($idArchive) {
+                $this->idarchives[$doneFlag][$periodString][] = $idArchive;
+            }
+        }
+    }
+
+    private function getIdArchivesByMonth($doneFlags)
+    {
+        // order idarchives by the table month they belong to
+        $idArchivesByMonth = array();
+
+        foreach (array_keys($doneFlags) as $doneFlag) {
+            if (empty($this->idarchives[$doneFlag])) {
+                continue;
+            }
+
+            foreach ($this->idarchives[$doneFlag] as $dateRange => $idarchives) {
+                foreach ($idarchives as $id) {
+                    $idArchivesByMonth[$dateRange][] = $id;
+                }
+            }
+        }
+
+        return $idArchivesByMonth;
+    }
+
+    /**
+     * @internal
+     */
+    public static function clearStaticCache()
+    {
+        self::$cache = null;
+    }
+}

msd2/tracking/piwik/core/Archive/ArchiveInvalidator.php

@@ -0,0 +1,362 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+
+namespace Piwik\Archive;
+
+use Piwik\Archive\ArchiveInvalidator\InvalidationResult;
+use Piwik\CronArchive\SitesToReprocessDistributedList;
+use Piwik\DataAccess\ArchiveTableCreator;
+use Piwik\DataAccess\Model;
+use Piwik\Date;
+use Piwik\Option;
+use Piwik\Piwik;
+use Piwik\Plugins\CoreAdminHome\Tasks\ArchivesToPurgeDistributedList;
+use Piwik\Plugins\PrivacyManager\PrivacyManager;
+use Piwik\Period;
+use Piwik\Segment;
+
+/**
+ * Service that can be used to invalidate archives or add archive references to a list so they will
+ * be invalidated later.
+ *
+ * Archives are put in an "invalidated" state by setting the done flag to `ArchiveWriter::DONE_INVALIDATED`.
+ * This class also adds the archive's associated site to the a distributed list and adding the archive's year month to another
+ * distributed list.
+ *
+ * CronArchive will reprocess the archive data for all sites in the first list, and a scheduled task
+ * will purge the old, invalidated data in archive tables identified by the second list.
+ *
+ * Until CronArchive, or browser triggered archiving, re-processes data for an invalidated archive, the invalidated
+ * archive data will still be displayed in the UI and API.
+ *
+ * ### Deferred Invalidation
+ *
+ * Invalidating archives means running queries on one or more archive tables. In some situations, like during
+ * tracking, this is not desired. In such cases, archive references can be added to a list via the
+ * rememberToInvalidateArchivedReportsLater method, which will add the reference to a distributed list
+ *
+ * Later, during Piwik's normal execution, the list will be read and every archive it references will
+ * be invalidated.
+ */
+class ArchiveInvalidator
+{
+    private $rememberArchivedReportIdStart = 'report_to_invalidate_';
+
+    /**
+     * @var Model
+     */
+    private $model;
+
+    public function __construct(Model $model)
+    {
+        $this->model = $model;
+    }
+
+    public function rememberToInvalidateArchivedReportsLater($idSite, Date $date)
+    {
+        // To support multiple transactions at once, look for any other process to have set (and committed)
+        // this report to be invalidated.
+        $key   = $this->buildRememberArchivedReportIdForSiteAndDate($idSite, $date->toString());
+
+        // we do not really have to get the value first. we could simply always try to call set() and it would update or
+        // insert the record if needed but we do not want to lock the table (especially since there are still some
+        // MyISAM installations)
+        $value = Option::getLike($key . '%');
+
+        // In order to support multiple concurrent transactions, add our pid to the end of the key so that it will just insert
+        // rather than waiting on some other process to commit before proceeding.The issue is that with out this, more than
+        // one process is trying to add the exact same value to the table, which causes contention. With the pid suffixed to
+        // the value, each process can successfully enter its own row in the table. The net result will be the same. We could
+        // always just set this, but it would result in a lot of rows in the options table.. more than needed.  With this
+        // change you'll have at most N rows per date/site, where N is the number of parallel requests on this same idsite/date
+        // that happen to run in overlapping transactions.
+        $mykey = $this->buildRememberArchivedReportIdProcessSafe($idSite, $date->toString());
+        // getLike() returns an empty array rather than 'false'
+        if (empty($value)) {
+            Option::set($mykey, '1');
+        }
+    }
+
+    public function getRememberedArchivedReportsThatShouldBeInvalidated()
+    {
+        $reports = Option::getLike($this->rememberArchivedReportIdStart . '%_%');
+
+        $sitesPerDay = array();
+
+        foreach ($reports as $report => $value) {
+            $report = str_replace($this->rememberArchivedReportIdStart, '', $report);
+            $report = explode('_', $report);
+            $siteId = (int) $report[0];
+            $date   = $report[1];
+
+            if (empty($sitesPerDay[$date])) {
+                $sitesPerDay[$date] = array();
+            }
+
+            $sitesPerDay[$date][] = $siteId;
+        }
+
+        return $sitesPerDay;
+    }
+
+    private function buildRememberArchivedReportIdForSite($idSite)
+    {
+        return $this->rememberArchivedReportIdStart . (int) $idSite;
+    }
+    
+    private function buildRememberArchivedReportIdForSiteAndDate($idSite, $date)
+    {
+        $id  = $this->buildRememberArchivedReportIdForSite($idSite);
+        $id .= '_' . trim($date);
+
+        return $id;
+    }
+
+    // This version is multi process safe on the insert of a new date to invalidate.
+    private function buildRememberArchivedReportIdProcessSafe($idSite, $date)
+    {
+        $id  = $this->buildRememberArchivedReportIdForSiteAndDate($idSite, $date);
+        $id .= '_' . getmypid();
+        return $id;
+    }
+
+    public function forgetRememberedArchivedReportsToInvalidateForSite($idSite)
+    {
+        $id = $this->buildRememberArchivedReportIdForSite($idSite) . '_%';
+        Option::deleteLike($id);
+    }
+
+    /**
+     * @internal
+     */
+    public function forgetRememberedArchivedReportsToInvalidate($idSite, Date $date)
+    {
+        $id = $this->buildRememberArchivedReportIdForSiteAndDate($idSite, $date->toString());
+
+        // The process pid is added to the end of the entry in order to support multiple concurrent transactions.
+        //  So this must be a deleteLike call to get all the entries, where there used to only be one.
+        Option::deleteLike($id . '%');
+    }
+
+    /**
+     * @param $idSites int[]
+     * @param $dates Date[]
+     * @param $period string
+     * @param $segment Segment
+     * @param bool $cascadeDown
+     * @return InvalidationResult
+     * @throws \Exception
+     */
+    public function markArchivesAsInvalidated(array $idSites, array $dates, $period, Segment $segment = null, $cascadeDown = false)
+    {
+        $invalidationInfo = new InvalidationResult();
+
+        /**
+         * Triggered when a Matomo user requested the invalidation of some reporting archives. Using this event, plugin
+         * developers can automatically invalidate another site, when a site is being invalidated. A plugin may even
+         * remove an idSite from the list of sites that should be invalidated to prevent it from ever being
+         * invalidated.
+         *
+         * **Example**
+         *
+         *     public function getIdSitesToMarkArchivesAsInvalidates(&$idSites)
+         *     {
+         *         if (in_array(1, $idSites)) {
+         *             $idSites[] = 5; // when idSite 1 is being invalidated, also invalidate idSite 5
+         *         }
+         *     }
+         *
+         * @param array &$idSites An array containing a list of site IDs which are requested to be invalidated.
+         */
+        Piwik::postEvent('Archiving.getIdSitesToMarkArchivesAsInvalidated', array(&$idSites));
+        // we trigger above event on purpose here and it is good that the segment was created like
+        // `new Segment($segmentString, $idSites)` because when a user adds a site via this event, the added idSite
+        // might not have this segment meaning we avoid a possible error. For the workflow to work, any added or removed
+        // idSite does not need to be added to $segment.
+
+        $datesToInvalidate = $this->removeDatesThatHaveBeenPurged($dates, $invalidationInfo);
+
+        if (empty($period)) {
+            // if the period is empty, we don't need to cascade in any way, since we'll remove all periods
+            $periodDates = $this->getDatesByYearMonthAndPeriodType($dates);
+        } else {
+            $periods = $this->getPeriodsToInvalidate($datesToInvalidate, $period, $cascadeDown);
+            $periodDates = $this->getPeriodDatesByYearMonthAndPeriodType($periods);
+        }
+
+        $periodDates = $this->getUniqueDates($periodDates);
+        $this->markArchivesInvalidated($idSites, $periodDates, $segment);
+
+        $yearMonths = array_keys($periodDates);
+        $this->markInvalidatedArchivesForReprocessAndPurge($idSites, $yearMonths);
+
+        foreach ($idSites as $idSite) {
+            foreach ($dates as $date) {
+                $this->forgetRememberedArchivedReportsToInvalidate($idSite, $date);
+            }
+        }
+
+        return $invalidationInfo;
+    }
+
+    /**
+     * @param string[][][] $periodDates
+     * @return string[][][]
+     */
+    private function getUniqueDates($periodDates)
+    {
+        $result = array();
+        foreach ($periodDates as $yearMonth => $periodsByYearMonth) {
+            foreach ($periodsByYearMonth as $periodType => $periods) {
+                $result[$yearMonth][$periodType] = array_unique($periods);
+            }
+        }
+        return $result;
+    }
+
+    /**
+     * @param Date[] $dates
+     * @param string $periodType
+     * @param bool $cascadeDown
+     * @return Period[]
+     */
+    private function getPeriodsToInvalidate($dates, $periodType, $cascadeDown)
+    {
+        $periodsToInvalidate = array();
+
+        foreach ($dates as $date) {
+            if ($periodType == 'range') {
+                $date = $date . ',' . $date;
+            }
+
+            $period = Period\Factory::build($periodType, $date);
+            $periodsToInvalidate[] = $period;
+
+            if ($cascadeDown) {
+                $periodsToInvalidate = array_merge($periodsToInvalidate, $period->getAllOverlappingChildPeriods());
+            }
+
+            if ($periodType != 'year'
+                && $periodType != 'range'
+            ) {
+                $periodsToInvalidate[] = Period\Factory::build('year', $date);
+            }
+        }
+
+        return $periodsToInvalidate;
+    }
+
+    /**
+     * @param Period[] $periods
+     * @return string[][][]
+     */
+    private function getPeriodDatesByYearMonthAndPeriodType($periods)
+    {
+        $result = array();
+        foreach ($periods as $period) {
+            $date = $period->getDateStart();
+            $periodType = $period->getId();
+
+            $yearMonth = ArchiveTableCreator::getTableMonthFromDate($date);
+            $result[$yearMonth][$periodType][] = $date->toString();
+        }
+        return $result;
+    }
+
+    /**
+     * Called when deleting all periods.
+     *
+     * @param Date[] $dates
+     * @return string[][][]
+     */
+    private function getDatesByYearMonthAndPeriodType($dates)
+    {
+        $result = array();
+        foreach ($dates as $date) {
+            $yearMonth = ArchiveTableCreator::getTableMonthFromDate($date);
+            $result[$yearMonth][null][] = $date->toString();
+
+            // since we're removing all periods, we must make sure to remove year periods as well.
+            // this means we have to make sure the january table is processed.
+            $janYearMonth = $date->toString('Y') . '_01';
+            $result[$janYearMonth][null][] = $date->toString();
+        }
+        return $result;
+    }
+
+    /**
+     * @param int[] $idSites
+     * @param string[][][] $dates
+     * @throws \Exception
+     */
+    private function markArchivesInvalidated($idSites, $dates, Segment $segment = null)
+    {
+        $archiveNumericTables = ArchiveTableCreator::getTablesArchivesInstalled($type = ArchiveTableCreator::NUMERIC_TABLE);
+        foreach ($archiveNumericTables as $table) {
+            $tableDate = ArchiveTableCreator::getDateFromTableName($table);
+            if (empty($dates[$tableDate])) {
+                continue;
+            }
+
+            $this->model->updateArchiveAsInvalidated($table, $idSites, $dates[$tableDate], $segment);
+        }
+    }
+
+    /**
+     * @param Date[] $dates
+     * @param InvalidationResult $invalidationInfo
+     * @return \Piwik\Date[]
+     */
+    private function removeDatesThatHaveBeenPurged($dates, InvalidationResult $invalidationInfo)
+    {
+        $this->findOlderDateWithLogs($invalidationInfo);
+
+        $result = array();
+        foreach ($dates as $date) {
+            // we should only delete reports for dates that are more recent than N days
+            if ($invalidationInfo->minimumDateWithLogs
+                && $date->isEarlier($invalidationInfo->minimumDateWithLogs)
+            ) {
+                $invalidationInfo->warningDates[] = $date->toString();
+                continue;
+            }
+
+            $result[] = $date;
+            $invalidationInfo->processedDates[] = $date->toString();
+        }
+        return $result;
+    }
+
+    private function findOlderDateWithLogs(InvalidationResult $info)
+    {
+        // If using the feature "Delete logs older than N days"...
+        $purgeDataSettings = PrivacyManager::getPurgeDataSettings();
+        $logsDeletedWhenOlderThanDays = (int)$purgeDataSettings['delete_logs_older_than'];
+        $logsDeleteEnabled = $purgeDataSettings['delete_logs_enable'];
+
+        if ($logsDeleteEnabled
+            && $logsDeletedWhenOlderThanDays
+        ) {
+            $info->minimumDateWithLogs = Date::factory('today')->subDay($logsDeletedWhenOlderThanDays);
+        }
+    }
+
+    /**
+     * @param array $idSites
+     * @param array $yearMonths
+     */
+    private function markInvalidatedArchivesForReprocessAndPurge(array $idSites, $yearMonths)
+    {
+        $store = new SitesToReprocessDistributedList();
+        $store->add($idSites);
+
+        $archivesToPurge = new ArchivesToPurgeDistributedList();
+        $archivesToPurge->add($yearMonths);
+    }
+}

msd2/tracking/piwik/core/Archive/ArchiveInvalidator/InvalidationResult.php

@@ -0,0 +1,56 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+
+namespace Piwik\Archive\ArchiveInvalidator;
+
+use Piwik\Date;
+
+/**
+ * Information about the result of an archive invalidation operation.
+ */
+class InvalidationResult
+{
+    /**
+     * Dates that couldn't be invalidated because they are earlier than the configured log
+     * deletion limit.
+     *
+     * @var array
+     */
+    public $warningDates = array();
+
+    /**
+     * Dates that were successfully invalidated.
+     *
+     * @var array
+     */
+    public $processedDates = array();
+
+    /**
+     * The day of the oldest log entry.
+     *
+     * @var Date|bool
+     */
+    public $minimumDateWithLogs = false;
+
+    /**
+     * @return string[]
+     */
+    public function makeOutputLogs()
+    {
+        $output = array();
+        if ($this->warningDates) {
+            $output[] = 'Warning: the following Dates have not been invalidated, because they are earlier than your Log Deletion limit: ' .
+                implode(", ", $this->warningDates) .
+                "\n The last day with logs is " . $this->minimumDateWithLogs . ". " .
+                "\n Please disable 'Delete old Logs' or set it to a higher deletion threshold (eg. 180 days or 365 years).'.";
+        }
+
+        $output[] = "Success. The following dates were invalidated successfully: " . implode(", ", $this->processedDates);
+        return $output;
+    }
+}

msd2/tracking/piwik/core/Archive/ArchivePurger.php

@@ -0,0 +1,272 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\Archive;
+
+use Piwik\ArchiveProcessor\Rules;
+use Piwik\Config;
+use Piwik\Container\StaticContainer;
+use Piwik\DataAccess\ArchiveTableCreator;
+use Piwik\DataAccess\Model;
+use Piwik\Date;
+use Piwik\Piwik;
+use Psr\Log\LoggerInterface;
+use Psr\Log\LogLevel;
+
+/**
+ * Service that purges temporary, error-ed, invalid and custom range archives from archive tables.
+ *
+ * Temporary archives are purged if they were archived before a specific time. The time is dependent
+ * on whether browser triggered archiving is enabled or not.
+ *
+ * Error-ed archives are purged w/o constraint.
+ *
+ * Invalid archives are purged if a new, valid, archive exists w/ the same site, date, period combination.
+ * Archives are marked as invalid via Piwik\Archive\ArchiveInvalidator.
+ */
+class ArchivePurger
+{
+    /**
+     * @var Model
+     */
+    private $model;
+
+    /**
+     * Date threshold for purging custom range archives. Archives that are older than this date
+     * are purged unconditionally from the requested archive table.
+     *
+     * @var Date
+     */
+    private $purgeCustomRangesOlderThan;
+
+    /**
+     * Date to use for 'yesterday'. Exists so tests can override this value.
+     *
+     * @var Date
+     */
+    private $yesterday;
+
+    /**
+     * Date to use for 'today'. Exists so tests can override this value.
+     *
+     * @var $today
+     */
+    private $today;
+
+    /**
+     * Date to use for 'now'. Exists so tests can override this value.
+     *
+     * @var int
+     */
+    private $now;
+
+    /**
+     * @var LoggerInterface
+     */
+    private $logger;
+
+    public function __construct(Model $model = null, Date $purgeCustomRangesOlderThan = null, LoggerInterface $logger = null)
+    {
+        $this->model = $model ?: new Model();
+
+        $this->purgeCustomRangesOlderThan = $purgeCustomRangesOlderThan ?: self::getDefaultCustomRangeToPurgeAgeThreshold();
+
+        $this->yesterday = Date::factory('yesterday');
+        $this->today = Date::factory('today');
+        $this->now = time();
+        $this->logger = $logger ?: StaticContainer::get('Psr\Log\LoggerInterface');
+    }
+
+    /**
+     * Purge all invalidate archives for whom there are newer, valid archives from the archive
+     * table that stores data for `$date`.
+     *
+     * @param Date $date The date identifying the archive table.
+     * @return int The total number of archive rows deleted (from both the blog & numeric tables).
+     */
+    public function purgeInvalidatedArchivesFrom(Date $date)
+    {
+        $numericTable = ArchiveTableCreator::getNumericTable($date);
+
+        // we don't want to do an INNER JOIN on every row in a archive table that can potentially have tens to hundreds of thousands of rows,
+        // so we first look for sites w/ invalidated archives, and use this as a constraint in getInvalidatedArchiveIdsSafeToDelete() below.
+        // the constraint will hit an INDEX and speed up the inner join that happens in getInvalidatedArchiveIdsSafeToDelete().
+        $idSites = $this->model->getSitesWithInvalidatedArchive($numericTable);
+        if (empty($idSites)) {
+            $this->logger->debug("No sites with invalidated archives found in {table}.", array('table' => $numericTable));
+            return 0;
+        }
+
+        $archiveIds = $this->model->getInvalidatedArchiveIdsSafeToDelete($numericTable, $idSites);
+        if (empty($archiveIds)) {
+            $this->logger->debug("No invalidated archives found in {table} with newer, valid archives.", array('table' => $numericTable));
+            return 0;
+        }
+
+        $this->logger->info("Found {countArchiveIds} invalidated archives safe to delete in {table}.", array(
+            'table' => $numericTable, 'countArchiveIds' => count($archiveIds)
+        ));
+
+        $deletedRowCount = $this->deleteArchiveIds($date, $archiveIds);
+
+        $this->logger->debug("Deleted {count} rows in {table} and its associated blob table.", array(
+            'table' => $numericTable, 'count' => $deletedRowCount
+        ));
+
+        return $deletedRowCount;
+    }
+
+    /**
+     * Removes the outdated archives for the given month.
+     * (meaning they are marked with a done flag of ArchiveWriter::DONE_OK_TEMPORARY or ArchiveWriter::DONE_ERROR)
+     *
+     * @param Date $dateStart Only the month will be used
+     * @return int Returns the total number of rows deleted.
+     */
+    public function purgeOutdatedArchives(Date $dateStart)
+    {
+        $purgeArchivesOlderThan = $this->getOldestTemporaryArchiveToKeepThreshold();
+        $deletedRowCount = 0;
+
+        $idArchivesToDelete = $this->getOutdatedArchiveIds($dateStart, $purgeArchivesOlderThan);
+        if (!empty($idArchivesToDelete)) {
+            $deletedRowCount = $this->deleteArchiveIds($dateStart, $idArchivesToDelete);
+
+            $this->logger->info("Deleted {count} rows in archive tables (numeric + blob) for {date}.", array(
+                'count' => $deletedRowCount,
+                'date' => $dateStart
+            ));
+        } else {
+            $this->logger->debug("No outdated archives found in archive numeric table for {date}.", array('date' => $dateStart));
+        }
+
+        $this->logger->debug("Purging temporary archives: done [ purged archives older than {date} in {yearMonth} ] [Deleted IDs: {deletedIds}]", array(
+            'date' => $purgeArchivesOlderThan,
+            'yearMonth' => $dateStart->toString('Y-m'),
+            'deletedIds' => implode(',', $idArchivesToDelete)
+        ));
+
+        return $deletedRowCount;
+    }
+
+    protected function getOutdatedArchiveIds(Date $date, $purgeArchivesOlderThan)
+    {
+        $archiveTable = ArchiveTableCreator::getNumericTable($date);
+
+        $result = $this->model->getTemporaryArchivesOlderThan($archiveTable, $purgeArchivesOlderThan);
+
+        $idArchivesToDelete = array();
+        if (!empty($result)) {
+            foreach ($result as $row) {
+                $idArchivesToDelete[] = $row['idarchive'];
+            }
+        }
+
+        return $idArchivesToDelete;
+    }
+
+    /**
+     * Deleting "Custom Date Range" reports after 1 day, since they can be re-processed and would take up un-necessary space.
+     *
+     * @param $date Date
+     * @return int The total number of rows deleted from both the numeric & blob table.
+     */
+    public function purgeArchivesWithPeriodRange(Date $date)
+    {
+        $numericTable = ArchiveTableCreator::getNumericTable($date);
+        $blobTable    = ArchiveTableCreator::getBlobTable($date);
+
+        $deletedCount = $this->model->deleteArchivesWithPeriod(
+            $numericTable, $blobTable, Piwik::$idPeriods['range'], $this->purgeCustomRangesOlderThan);
+
+        $level = $deletedCount == 0 ? LogLevel::DEBUG : LogLevel::INFO;
+        $this->logger->log($level, "Purged {count} range archive rows from {numericTable} & {blobTable}.", array(
+            'count' => $deletedCount,
+            'numericTable' => $numericTable,
+            'blobTable' => $blobTable
+        ));
+
+        $this->logger->debug("  [ purged archives older than {threshold} ]", array('threshold' => $this->purgeCustomRangesOlderThan));
+
+        return $deletedCount;
+    }
+
+    /**
+     * Deletes by batches Archive IDs in the specified month,
+     *
+     * @param Date $date
+     * @param $idArchivesToDelete
+     * @return int Number of rows deleted from both numeric + blob table.
+     */
+    protected function deleteArchiveIds(Date $date, $idArchivesToDelete)
+    {
+        $batches      = array_chunk($idArchivesToDelete, 1000);
+        $numericTable = ArchiveTableCreator::getNumericTable($date);
+        $blobTable    = ArchiveTableCreator::getBlobTable($date);
+
+        $deletedCount = 0;
+        foreach ($batches as $idsToDelete) {
+            $deletedCount += $this->model->deleteArchiveIds($numericTable, $blobTable, $idsToDelete);
+        }
+        return $deletedCount;
+    }
+
+    /**
+     * Returns a timestamp indicating outdated archives older than this timestamp (processed before) can be purged.
+     *
+     * @return int|bool  Outdated archives older than this timestamp should be purged
+     */
+    protected function getOldestTemporaryArchiveToKeepThreshold()
+    {
+        $temporaryArchivingTimeout = Rules::getTodayArchiveTimeToLive();
+        if (Rules::isBrowserTriggerEnabled()) {
+            // If Browser Archiving is enabled, it is likely there are many more temporary archives
+            // We delete more often which is safe, since reports are re-processed on demand
+            return Date::factory($this->now - 2 * $temporaryArchivingTimeout)->getDateTime();
+        }
+
+        // If cron core:archive command is building the reports, we should keep all temporary reports from today
+        return $this->yesterday->getDateTime();
+    }
+
+    private static function getDefaultCustomRangeToPurgeAgeThreshold()
+    {
+        $daysRangesValid = Config::getInstance()->General['purge_date_range_archives_after_X_days'];
+        return Date::factory('today')->subDay($daysRangesValid)->getDateTime();
+    }
+
+    /**
+     * For tests.
+     *
+     * @param Date $yesterday
+     */
+    public function setYesterdayDate(Date $yesterday)
+    {
+        $this->yesterday = $yesterday;
+    }
+
+    /**
+     * For tests.
+     *
+     * @param Date $today
+     */
+    public function setTodayDate(Date $today)
+    {
+        $this->today = $today;
+    }
+
+    /**
+     * For tests.
+     *
+     * @param int $now
+     */
+    public function setNow($now)
+    {
+        $this->now = $now;
+    }
+}

msd2/tracking/piwik/core/Archive/ArchiveQuery.php

@@ -0,0 +1,49 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\Archive;
+
+
+use Piwik\DataTable;
+
+interface ArchiveQuery
+{
+    /**
+     * @param string|string[] $names
+     * @return false|number|array
+     */
+    public function getNumeric($names);
+
+    /**
+     * @param string|string[] $names
+     * @return DataTable|DataTable\Map
+     */
+    public function getDataTableFromNumeric($names);
+
+    /**
+     * @param $names
+     * @return mixed
+     */
+    public function getDataTableFromNumericAndMergeChildren($names);
+
+    /**
+     * @param string $name
+     * @param int|string|null $idSubtable
+     * @return DataTable|DataTable\Map
+     */
+    public function getDataTable($name, $idSubtable = null);
+
+    /**
+     * @param string $name
+     * @param int|string|null $idSubtable
+     * @param int|null $depth
+     * @param bool $addMetadataSubtableId
+     * @return DataTable|DataTable\Map
+     */
+    public function getDataTableExpanded($name, $idSubtable = null, $depth = null, $addMetadataSubtableId = true);
+}

msd2/tracking/piwik/core/Archive/ArchiveQueryFactory.php

@@ -0,0 +1,127 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+
+namespace Piwik\Archive;
+
+use Piwik\Archive;
+use Piwik\Period;
+use Piwik\Segment;
+use Piwik\Site;
+use Piwik\Period\Factory as PeriodFactory;
+
+class ArchiveQueryFactory
+{
+    public function __construct()
+    {
+        // empty
+    }
+
+    /**
+     * @see \Piwik\Archive::build()
+     */
+    public function build($idSites, $strPeriod, $strDate, $strSegment = false, $_restrictSitesToLogin = false)
+    {
+        list($websiteIds, $timezone, $idSiteIsAll) = $this->getSiteInfoFromQueryParam($idSites, $_restrictSitesToLogin);
+        list($allPeriods, $isMultipleDate) = $this->getPeriodInfoFromQueryParam($strDate, $strPeriod, $timezone);
+        $segment = $this->getSegmentFromQueryParam($strSegment, $websiteIds);
+
+        return $this->factory($segment, $allPeriods, $websiteIds, $idSiteIsAll, $isMultipleDate);
+    }
+
+    /**
+     * @see \Piwik\Archive::factory()
+     */
+    public function factory(Segment $segment, array $periods, array $idSites, $idSiteIsAll = false, $isMultipleDate = false)
+    {
+        $forceIndexedBySite = false;
+        $forceIndexedByDate = false;
+
+        if ($idSiteIsAll || count($idSites) > 1) {
+            $forceIndexedBySite = true;
+        }
+
+        if (count($periods) > 1 || $isMultipleDate) {
+            $forceIndexedByDate = true;
+        }
+
+        $params = new Parameters($idSites, $periods, $segment);
+
+        return $this->newInstance($params, $forceIndexedBySite, $forceIndexedByDate);
+    }
+
+    public function newInstance(Parameters $params, $forceIndexedBySite, $forceIndexedByDate)
+    {
+        return new Archive($params, $forceIndexedBySite, $forceIndexedByDate);
+    }
+
+    /**
+     * Parses the site ID string provided in the 'idSite' query parameter to a list of
+     * website IDs.
+     *
+     * @param string $idSites the value of the 'idSite' query parameter
+     * @param bool $_restrictSitesToLogin
+     * @return array an array containing three elements:
+     *     - an array of website IDs
+     *     - string timezone to use (or false to use no timezone) when creating periods.
+     *     - true if the request was for all websites (this forces the archive result to
+     *       be indexed by site, even if there is only one site in Piwik)
+     */
+    protected function getSiteInfoFromQueryParam($idSites, $_restrictSitesToLogin)
+    {
+        $websiteIds = Site::getIdSitesFromIdSitesString($idSites, $_restrictSitesToLogin);
+
+        $timezone = false;
+        if (count($websiteIds) == 1) {
+            $timezone = Site::getTimezoneFor($websiteIds[0]);
+        }
+
+        $idSiteIsAll = $idSites == Archive::REQUEST_ALL_WEBSITES_FLAG;
+
+        return [$websiteIds, $timezone, $idSiteIsAll];
+    }
+
+    /**
+     * Parses the date & period query parameters into a list of periods.
+     *
+     * @param string $strDate the value of the 'date' query parameter
+     * @param string $strPeriod the value of the 'period' query parameter
+     * @param string $timezone the timezone to use when constructing periods.
+     * @return array an array containing two elements:
+     *     - the list of period objects to query archive data for
+     *     - true if the request was for multiple periods (ie, two months, two weeks, etc.), false if otherwise.
+     *       (this forces the archive result to be indexed by period, even if the list of periods
+     *       has only one period).
+     */
+    protected function getPeriodInfoFromQueryParam($strDate, $strPeriod, $timezone)
+    {
+        if (Period::isMultiplePeriod($strDate, $strPeriod)) {
+            $oPeriod    = PeriodFactory::build($strPeriod, $strDate, $timezone);
+            $allPeriods = $oPeriod->getSubperiods();
+        } else {
+            $oPeriod    = PeriodFactory::makePeriodFromQueryParams($timezone, $strPeriod, $strDate);
+            $allPeriods = array($oPeriod);
+        }
+
+        $isMultipleDate = Period::isMultiplePeriod($strDate, $strPeriod);
+
+        return [$allPeriods, $isMultipleDate];
+    }
+
+    /**
+     * Parses the segment query parameter into a Segment object.
+     *
+     * @param string $strSegment the value of the 'segment' query parameter.
+     * @param int[] $websiteIds the list of sites being queried.
+     * @return Segment
+     */
+    protected function getSegmentFromQueryParam($strSegment, $websiteIds)
+    {
+        return new Segment($strSegment, $websiteIds);
+    }
+}

msd2/tracking/piwik/core/Archive/Chunk.php

@@ -0,0 +1,144 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+
+namespace Piwik\Archive;
+
+use Piwik\DataTable;
+
+/**
+ * This class is used to split blobs of DataTables into chunks. Each blob used to be stored under one blob in the
+ * archive table. For better efficiency we do now combine multiple DataTable into one blob entry.
+ *
+ * Chunks are identified by having the recordName $recordName_chunk_0_99, $recordName_chunk_100_199 (this chunk stores
+ * the subtable 100-199).
+ */
+class Chunk
+{
+    const ARCHIVE_APPENDIX_SUBTABLES = 'chunk';
+    const NUM_TABLES_IN_CHUNK = 100;
+
+    /**
+     * Get's the record name to use for a given tableId/subtableId.
+     *
+     * @param string $recordName eg 'Actions_ActionsUrl'
+     * @param int    $tableId    eg '5' for tableId '5'
+     * @return string            eg 'Actions_ActionsUrl_chunk_0_99' as the table should be stored under this blob id.
+     */
+    public function getRecordNameForTableId($recordName, $tableId)
+    {
+        $chunk = (floor($tableId / self::NUM_TABLES_IN_CHUNK));
+        $start = $chunk * self::NUM_TABLES_IN_CHUNK;
+        $end   = $start + self::NUM_TABLES_IN_CHUNK - 1;
+
+        return $recordName . $this->getAppendix() . $start . '_' . $end;
+    }
+
+    /**
+     * Moves the given blobs into chunks and assigns a proper record name containing the chunk number.
+     *
+     * @param string $recordName The original archive record name, eg 'Actions_ActionsUrl'
+     * @param array  $blobs  An array containg a mapping of tableIds to blobs. Eg array(0 => 'blob', 1 => 'subtableBlob', ...)
+     * @return array        An array where each blob is moved into a chunk, indexed by recordNames.
+     *                      eg array('Actions_ActionsUrl_chunk_0_99'    => array(0 => 'blob', 1 => 'subtableBlob', ...),
+     *                               'Actions_ActionsUrl_chunk_100_199' => array(...))
+     */
+    public function moveArchiveBlobsIntoChunks($recordName, $blobs)
+    {
+        $chunks = array();
+
+        foreach ($blobs as $tableId => $blob) {
+            $name = $this->getRecordNameForTableId($recordName, $tableId);
+
+            if (!array_key_exists($name, $chunks)) {
+                $chunks[$name] = array();
+            }
+
+            $chunks[$name][$tableId] = $blob;
+        }
+
+        return $chunks;
+    }
+
+    /**
+     * Detects whether a recordName like 'Actions_ActionUrls_chunk_0_99' or 'Actions_ActionUrls' belongs to a
+     * chunk or not.
+     *
+     * To be a valid recordName that belongs to a chunk it must end with '_chunk_NUMERIC_NUMERIC'.
+     *
+     * @param string $recordName
+     * @return bool
+     */
+    public function isRecordNameAChunk($recordName)
+    {
+        $posAppendix = $this->getEndPosOfChunkAppendix($recordName);
+
+        if (false === $posAppendix) {
+            return false;
+        }
+
+        // will contain "0_99" of "chunk_0_99"
+        $blobId = substr($recordName, $posAppendix);
+
+        return $this->isChunkRange($blobId);
+    }
+
+    private function isChunkRange($blobId)
+    {
+        $blobId = explode('_', $blobId);
+
+        return 2 === count($blobId) && is_numeric($blobId[0]) && is_numeric($blobId[1]);
+    }
+
+    /**
+     * When having a record like 'Actions_ActionUrls_chunk_0_99" it will return the raw recordName 'Actions_ActionUrls'.
+     *
+     * @param  string $recordName
+     * @return string
+     */
+    public function getRecordNameWithoutChunkAppendix($recordName)
+    {
+        if (!$this->isRecordNameAChunk($recordName)) {
+            return $recordName;
+        }
+
+        $posAppendix = $this->getStartPosOfChunkAppendix($recordName);
+
+        if (false === $posAppendix) {
+            return $recordName;
+        }
+
+        return substr($recordName, 0, $posAppendix);
+    }
+
+    /**
+     * Returns the string that is appended to the original record name. This appendix identifes a record name is a
+     * chunk.
+     * @return string
+     */
+    public function getAppendix()
+    {
+        return '_' . self::ARCHIVE_APPENDIX_SUBTABLES . '_';
+    }
+
+    private function getStartPosOfChunkAppendix($recordName)
+    {
+        return strpos($recordName, $this->getAppendix());
+    }
+
+    private function getEndPosOfChunkAppendix($recordName)
+    {
+        $pos = strpos($recordName, $this->getAppendix());
+
+        if ($pos === false) {
+            return false;
+        }
+
+        return $pos + strlen($this->getAppendix());
+    }
+}

msd2/tracking/piwik/core/Archive/DataCollection.php

@@ -0,0 +1,375 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+
+namespace Piwik\Archive;
+
+use Exception;
+use Piwik\DataTable;
+
+/**
+ * This class is used to hold and transform archive data for the Archive class.
+ *
+ * Archive data is loaded into an instance of this type, can be indexed by archive
+ * metadata (such as the site ID, period string, etc.), and can be transformed into
+ * DataTable and Map instances.
+ */
+class DataCollection
+{
+    const METADATA_CONTAINER_ROW_KEY = '_metadata';
+
+    /**
+     * The archive data, indexed first by site ID and then by period date range. Eg,
+     *
+     * array(
+     *     '0' => array(
+     *         array(
+     *             '2012-01-01,2012-01-01' => array(...),
+     *             '2012-01-02,2012-01-02' => array(...),
+     *         )
+     *     ),
+     *     '1' => array(
+     *         array(
+     *             '2012-01-01,2012-01-01' => array(...),
+     *         )
+     *     )
+     * )
+     *
+     * Archive data can be either a numeric value or a serialized string blob. Every
+     * piece of archive data is associated by it's archive name. For example,
+     * the array(...) above could look like:
+     *
+     * array(
+     *    'nb_visits' => 1,
+     *    'nb_actions' => 2
+     * )
+     *
+     * There is a special element '_metadata' in data rows that holds values treated
+     * as DataTable metadata.
+     */
+    private $data = array();
+
+    /**
+     * The whole list of metric/record names that were used in the archive query.
+     *
+     * @var array
+     */
+    private $dataNames;
+
+    /**
+     * The type of data that was queried for (ie, "blob" or "numeric").
+     *
+     * @var string
+     */
+    private $dataType;
+
+    /**
+     * The default values to use for each metric/record name that's being queried
+     * for.
+     *
+     * @var array
+     */
+    private $defaultRow;
+
+    /**
+     * The list of all site IDs that were queried for.
+     *
+     * @var array
+     */
+    private $sitesId;
+
+    /**
+     * The list of all periods that were queried for. Each period is associated with
+     * the period's range string. Eg,
+     *
+     * array(
+     *     '2012-01-01,2012-01-31' => new Period(...),
+     *     '2012-02-01,2012-02-28' => new Period(...),
+     * )
+     *
+     * @var \Piwik\Period[]
+     */
+    private $periods;
+
+    /**
+     * Constructor.
+     *
+     * @param array $dataNames @see $this->dataNames
+     * @param string $dataType @see $this->dataType
+     * @param array $sitesId @see $this->sitesId
+     * @param \Piwik\Period[] $periods @see $this->periods
+     * @param array $defaultRow @see $this->defaultRow
+     */
+    public function __construct($dataNames, $dataType, $sitesId, $periods, $defaultRow = null)
+    {
+        $this->dataNames = $dataNames;
+        $this->dataType = $dataType;
+
+        if ($defaultRow === null) {
+            $defaultRow = array_fill_keys($dataNames, 0);
+        }
+
+        $this->sitesId = $sitesId;
+
+        foreach ($periods as $period) {
+            $this->periods[$period->getRangeString()] = $period;
+        }
+        $this->defaultRow = $defaultRow;
+    }
+
+    /**
+     * Returns a reference to the data for a specific site & period. If there is
+     * no data for the given site ID & period, it is set to the default row.
+     *
+     * @param int $idSite
+     * @param string $period eg, '2012-01-01,2012-01-31'
+     */
+    public function &get($idSite, $period)
+    {
+        if (!isset($this->data[$idSite][$period])) {
+            $this->data[$idSite][$period] = $this->defaultRow;
+        }
+        return $this->data[$idSite][$period];
+    }
+
+    /**
+     * Set data for a specific site & period. If there is no data for the given site ID & period,
+     * it is set to the default row.
+     *
+     * @param int $idSite
+     * @param string $period eg, '2012-01-01,2012-01-31'
+     * @param string $name eg 'nb_visits'
+     * @param string $value eg 5
+     */
+    public function set($idSite, $period, $name, $value)
+    {
+        $row = & $this->get($idSite, $period);
+        $row[$name] = $value;
+    }
+
+    /**
+     * Adds a new metadata to the data for specific site & period. If there is no
+     * data for the given site ID & period, it is set to the default row.
+     *
+     * Note: Site ID and period range string are two special types of metadata. Since
+     * the data stored in this class is indexed by site & period, this metadata is not
+     * stored in individual data rows.
+     *
+     * @param int $idSite
+     * @param string $period eg, '2012-01-01,2012-01-31'
+     * @param string $name The metadata name.
+     * @param mixed $value The metadata name.
+     */
+    public function addMetadata($idSite, $period, $name, $value)
+    {
+        $row = & $this->get($idSite, $period);
+        $row[self::METADATA_CONTAINER_ROW_KEY][$name] = $value;
+    }
+
+    /**
+     * Returns archive data as an array indexed by metadata.
+     *
+     * @param array $resultIndices An array mapping metadata names to pretty labels
+     *                             for them. Each archive data row will be indexed
+     *                             by the metadata specified here.
+     *
+     *                             Eg, array('site' => 'idSite', 'period' => 'Date')
+     * @return array
+     */
+    public function getIndexedArray($resultIndices)
+    {
+        $indexKeys = array_keys($resultIndices);
+
+        $result = $this->createOrderedIndex($indexKeys);
+        foreach ($this->data as $idSite => $rowsByPeriod) {
+            foreach ($rowsByPeriod as $period => $row) {
+                // FIXME: This hack works around a strange bug that occurs when getting
+                //         archive IDs through ArchiveProcessing instances. When a table
+                //         does not already exist, for some reason the archive ID for
+                //         today (or from two days ago) will be added to the Archive
+                //         instances list. The Archive instance will then select data
+                //         for periods outside of the requested set.
+                //         working around the bug here, but ideally, we need to figure
+                //         out why incorrect idarchives are being selected.
+                if (empty($this->periods[$period])) {
+                    continue;
+                }
+
+                $this->putRowInIndex($result, $indexKeys, $row, $idSite, $period);
+            }
+        }
+
+        return $result;
+    }
+
+    /**
+     * Returns archive data as a DataTable indexed by metadata. Indexed data will
+     * be represented by Map instances.
+     *
+     * @param array $resultIndices An array mapping metadata names to pretty labels
+     *                             for them. Each archive data row will be indexed
+     *                             by the metadata specified here.
+     *
+     *                             Eg, array('site' => 'idSite', 'period' => 'Date')
+     * @return DataTable|DataTable\Map
+     */
+    public function getDataTable($resultIndices)
+    {
+        $dataTableFactory = new DataTableFactory(
+            $this->dataNames, $this->dataType, $this->sitesId, $this->periods, $this->defaultRow);
+
+        $index = $this->getIndexedArray($resultIndices);
+
+        return $dataTableFactory->make($index, $resultIndices);
+    }
+
+    /**
+     * See {@link DataTableFactory::makeMerged()}
+     *
+     * @param array $resultIndices
+     * @return DataTable|DataTable\Map
+     * @throws Exception
+     */
+    public function getMergedDataTable($resultIndices)
+    {
+        $dataTableFactory = new DataTableFactory(
+            $this->dataNames, $this->dataType, $this->sitesId, $this->periods, $this->defaultRow);
+
+        $index = $this->getIndexedArray($resultIndices);
+
+        return $dataTableFactory->makeMerged($index, $resultIndices);
+    }
+
+    /**
+     * Returns archive data as a DataTable indexed by metadata. Indexed data will
+     * be represented by Map instances. Each DataTable will have
+     * its subtable IDs set.
+     *
+     * This function will only work if blob data was loaded and only one record
+     * was loaded (not including subtables of the record).
+     *
+     * @param array $resultIndices An array mapping metadata names to pretty labels
+     *                             for them. Each archive data row will be indexed
+     *                             by the metadata specified here.
+     *
+     *                             Eg, array('site' => 'idSite', 'period' => 'Date')
+     * @param int|null $idSubTable The subtable to return.
+     * @param int|null $depth max depth for subtables.
+     * @param bool $addMetadataSubTableId Whether to add the DB subtable ID as metadata
+     *                                    to each datatable, or not.
+     * @throws Exception
+     * @return DataTable|DataTable\Map
+     */
+    public function getExpandedDataTable($resultIndices, $idSubTable = null, $depth = null, $addMetadataSubTableId = false)
+    {
+        if ($this->dataType != 'blob') {
+            throw new Exception("DataCollection: cannot call getExpandedDataTable with "
+                . "{$this->dataType} data types. Only works with blob data.");
+        }
+
+        if (count($this->dataNames) !== 1) {
+            throw new Exception("DataCollection: cannot call getExpandedDataTable with "
+                . "more than one record.");
+        }
+
+        $dataTableFactory = new DataTableFactory(
+            $this->dataNames, 'blob', $this->sitesId, $this->periods, $this->defaultRow);
+        $dataTableFactory->expandDataTable($depth, $addMetadataSubTableId);
+        $dataTableFactory->useSubtable($idSubTable);
+
+        $index = $this->getIndexedArray($resultIndices);
+
+        return $dataTableFactory->make($index, $resultIndices);
+    }
+
+    /**
+     * Returns metadata for a data row.
+     *
+     * @param array $data The data row.
+     * @return array
+     */
+    public static function getDataRowMetadata($data)
+    {
+        if (isset($data[self::METADATA_CONTAINER_ROW_KEY])) {
+            return $data[self::METADATA_CONTAINER_ROW_KEY];
+        } else {
+            return array();
+        }
+    }
+
+    /**
+     * Removes all table metadata from a data row.
+     *
+     * @param array $data The data row.
+     */
+    public static function removeMetadataFromDataRow(&$data)
+    {
+        unset($data[self::METADATA_CONTAINER_ROW_KEY]);
+    }
+
+    /**
+     * Creates an empty index using a list of metadata names. If the 'site' and/or
+     * 'period' metadata names are supplied, empty rows are added for every site/period
+     * that was queried for.
+     *
+     * Using this function ensures consistent ordering in the indexed result.
+     *
+     * @param array $metadataNamesToIndexBy List of metadata names to index archive data by.
+     * @return array
+     */
+    private function createOrderedIndex($metadataNamesToIndexBy)
+    {
+        $result = array();
+
+        if (!empty($metadataNamesToIndexBy)) {
+            $metadataName = array_shift($metadataNamesToIndexBy);
+
+            if ($metadataName == DataTableFactory::TABLE_METADATA_SITE_INDEX) {
+                $indexKeyValues = array_values($this->sitesId);
+            } elseif ($metadataName == DataTableFactory::TABLE_METADATA_PERIOD_INDEX) {
+                $indexKeyValues = array_keys($this->periods);
+            }
+
+            if (empty($metadataNamesToIndexBy)) {
+                $result = array_fill_keys($indexKeyValues, array());
+            } else {
+                foreach ($indexKeyValues as $key) {
+                    $result[$key] = $this->createOrderedIndex($metadataNamesToIndexBy);
+                }
+            }
+        }
+
+        return $result;
+    }
+
+    /**
+     * Puts an archive data row in an index.
+     */
+    private function putRowInIndex(&$index, $metadataNamesToIndexBy, $row, $idSite, $period)
+    {
+        $currentLevel = & $index;
+
+        foreach ($metadataNamesToIndexBy as $metadataName) {
+            if ($metadataName == DataTableFactory::TABLE_METADATA_SITE_INDEX) {
+                $key = $idSite;
+            } elseif ($metadataName == DataTableFactory::TABLE_METADATA_PERIOD_INDEX) {
+                $key = $period;
+            } else {
+                $key = $row[self::METADATA_CONTAINER_ROW_KEY][$metadataName];
+            }
+
+            if (!isset($currentLevel[$key])) {
+                $currentLevel[$key] = array();
+            }
+
+            $currentLevel = & $currentLevel[$key];
+        }
+
+        $currentLevel = $row;
+    }
+}

msd2/tracking/piwik/core/Archive/DataTableFactory.php

@@ -0,0 +1,552 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+
+namespace Piwik\Archive;
+
+use Piwik\DataTable;
+use Piwik\DataTable\Row;
+use Piwik\Period\Week;
+use Piwik\Site;
+
+/**
+ * Creates a DataTable or Set instance based on an array
+ * index created by DataCollection.
+ *
+ * This class is only used by DataCollection.
+ */
+class DataTableFactory
+{
+    /**
+     * @see DataCollection::$dataNames.
+     */
+    private $dataNames;
+
+    /**
+     * @see DataCollection::$dataType.
+     */
+    private $dataType;
+
+    /**
+     * Whether to expand the DataTables that're created or not. Expanding a DataTable
+     * means creating DataTables using subtable blobs and correctly setting the subtable
+     * IDs of all DataTables.
+     *
+     * @var bool
+     */
+    private $expandDataTable = false;
+
+    /**
+     * Whether to add the subtable ID used in the database to the in-memory DataTables
+     * as metadata or not.
+     *
+     * @var bool
+     */
+    private $addMetadataSubtableId = false;
+
+    /**
+     * The maximum number of subtable levels to create when creating an expanded
+     * DataTable.
+     *
+     * @var int
+     */
+    private $maxSubtableDepth = null;
+
+    /**
+     * @see DataCollection::$sitesId.
+     */
+    private $sitesId;
+
+    /**
+     * @see DataCollection::$periods.
+     */
+    private $periods;
+
+    /**
+     * The ID of the subtable to create a DataTable for. Only relevant for blob data.
+     *
+     * @var int|null
+     */
+    private $idSubtable = null;
+
+    /**
+     * @see DataCollection::$defaultRow.
+     */
+    private $defaultRow;
+
+    const TABLE_METADATA_SITE_INDEX = 'site';
+    const TABLE_METADATA_PERIOD_INDEX = 'period';
+
+    /**
+     * Constructor.
+     */
+    public function __construct($dataNames, $dataType, $sitesId, $periods, $defaultRow)
+    {
+        $this->dataNames = $dataNames;
+        $this->dataType = $dataType;
+        $this->sitesId = $sitesId;
+
+        //here index period by string only
+        $this->periods = $periods;
+        $this->defaultRow = $defaultRow;
+    }
+
+    /**
+     * Returns the ID of the site a table is related to based on the 'site' metadata entry,
+     * or null if there is none.
+     *
+     * @param DataTable $table
+     * @return int|null
+     */
+    public static function getSiteIdFromMetadata(DataTable $table)
+    {
+        $site = $table->getMetadata('site');
+        if (empty($site)) {
+            return null;
+        } else {
+            return $site->getId();
+        }
+    }
+
+    /**
+     * Tells the factory instance to expand the DataTables that are created by
+     * creating subtables and setting the subtable IDs of rows w/ subtables correctly.
+     *
+     * @param null|int $maxSubtableDepth max depth for subtables.
+     * @param bool $addMetadataSubtableId Whether to add the subtable ID used in the
+     *                                    database to the in-memory DataTables as
+     *                                    metadata or not.
+     */
+    public function expandDataTable($maxSubtableDepth = null, $addMetadataSubtableId = false)
+    {
+        $this->expandDataTable = true;
+        $this->maxSubtableDepth = $maxSubtableDepth;
+        $this->addMetadataSubtableId = $addMetadataSubtableId;
+    }
+
+    /**
+     * Tells the factory instance to create a DataTable using a blob with the
+     * supplied subtable ID.
+     *
+     * @param int $idSubtable An in-database subtable ID.
+     * @throws \Exception
+     */
+    public function useSubtable($idSubtable)
+    {
+        if (count($this->dataNames) !== 1) {
+            throw new \Exception("DataTableFactory: Getting subtables for multiple records in one"
+                . " archive query is not currently supported.");
+        }
+
+        $this->idSubtable = $idSubtable;
+    }
+
+    private function isNumericDataType()
+    {
+        return $this->dataType == 'numeric';
+    }
+
+    /**
+     * Creates a DataTable|Set instance using an index of
+     * archive data.
+     *
+     * @param array $index @see DataCollection
+     * @param array $resultIndices an array mapping metadata names with pretty metadata
+     *                             labels.
+     * @return DataTable|DataTable\Map
+     */
+    public function make($index, $resultIndices)
+    {
+        $keyMetadata = $this->getDefaultMetadata();
+
+        if (empty($resultIndices)) {
+            // for numeric data, if there's no index (and thus only 1 site & period in the query),
+            // we want to display every queried metric name
+            if (empty($index)
+                && $this->isNumericDataType()
+            ) {
+                $index = $this->defaultRow;
+            }
+
+            $dataTable = $this->createDataTable($index, $keyMetadata);
+        } else {
+            $dataTable = $this->createDataTableMapFromIndex($index, $resultIndices, $keyMetadata);
+        }
+
+        return $dataTable;
+    }
+
+    /**
+     * Creates a merged DataTable|Map instance using an index of archive data similar to {@link make()}.
+     *
+     * Whereas {@link make()} creates a Map for each result index (period and|or site), this will only create a Map
+     * for a period result index and move all site related indices into one dataTable. This is the same as doing
+     * `$dataTableFactory->make()->mergeChildren()` just much faster. It is mainly useful for reports across many sites
+     * eg `MultiSites.getAll`. Was done as part of https://github.com/piwik/piwik/issues/6809
+     *
+     * @param array $index @see DataCollection
+     * @param array $resultIndices an array mapping metadata names with pretty metadata labels.
+     *
+     * @return DataTable|DataTable\Map
+     * @throws \Exception
+     */
+    public function makeMerged($index, $resultIndices)
+    {
+        if (!$this->isNumericDataType()) {
+            throw new \Exception('This method is supposed to work with non-numeric data types but it is not tested. To use it, remove this exception and write tests to be sure it works.');
+        }
+
+        $hasSiteIndex   = isset($resultIndices[self::TABLE_METADATA_SITE_INDEX]);
+        $hasPeriodIndex = isset($resultIndices[self::TABLE_METADATA_PERIOD_INDEX]);
+
+        $isNumeric = $this->isNumericDataType();
+        // to be backwards compatible use a Simple table if needed as it will be formatted differently
+        $useSimpleDataTable = !$hasSiteIndex && $isNumeric;
+
+        if (!$hasSiteIndex) {
+            $firstIdSite = reset($this->sitesId);
+            $index = array($firstIdSite => $index);
+        }
+
+        if ($hasPeriodIndex) {
+            $dataTable = $this->makeMergedTableWithPeriodAndSiteIndex($index, $resultIndices, $useSimpleDataTable, $isNumeric);
+        } else {
+            $dataTable = $this->makeMergedWithSiteIndex($index, $useSimpleDataTable, $isNumeric);
+        }
+
+        return $dataTable;
+    }
+
+    /**
+     * Creates a DataTable|Set instance using an array
+     * of blobs.
+     *
+     * If only one record is being queried, a single DataTable will
+     * be returned. Otherwise, a DataTable\Map is returned that indexes
+     * DataTables by record name.
+     *
+     * If expandDataTable was called, and only one record is being queried,
+     * the created DataTable's subtables will be expanded.
+     *
+     * @param array $blobRow
+     * @return DataTable|DataTable\Map
+     */
+    private function makeFromBlobRow($blobRow, $keyMetadata)
+    {
+        if ($blobRow === false) {
+            return new DataTable();
+        }
+
+        if (count($this->dataNames) === 1) {
+            return $this->makeDataTableFromSingleBlob($blobRow, $keyMetadata);
+        } else {
+            return $this->makeIndexedByRecordNameDataTable($blobRow, $keyMetadata);
+        }
+    }
+
+    /**
+     * Creates a DataTable for one record from an archive data row.
+     *
+     * @see makeFromBlobRow
+     *
+     * @param array $blobRow
+     * @return DataTable
+     */
+    private function makeDataTableFromSingleBlob($blobRow, $keyMetadata)
+    {
+        $recordName = reset($this->dataNames);
+        if ($this->idSubtable !== null) {
+            $recordName .= '_' . $this->idSubtable;
+        }
+
+        if (!empty($blobRow[$recordName])) {
+            $table = DataTable::fromSerializedArray($blobRow[$recordName]);
+        } else {
+            $table = new DataTable();
+        }
+
+        // set table metadata
+        $table->setAllTableMetadata(array_merge(DataCollection::getDataRowMetadata($blobRow), $keyMetadata));
+
+        if ($this->expandDataTable) {
+            $table->enableRecursiveFilters();
+            $this->setSubtables($table, $blobRow);
+        }
+
+        return $table;
+    }
+
+    /**
+     * Creates a DataTable for every record in an archive data row and puts them
+     * in a DataTable\Map instance.
+     *
+     * @param array $blobRow
+     * @return DataTable\Map
+     */
+    private function makeIndexedByRecordNameDataTable($blobRow, $keyMetadata)
+    {
+        $table = new DataTable\Map();
+        $table->setKeyName('recordName');
+
+        $tableMetadata = array_merge(DataCollection::getDataRowMetadata($blobRow), $keyMetadata);
+
+        foreach ($blobRow as $name => $blob) {
+            $newTable = DataTable::fromSerializedArray($blob);
+            $newTable->setAllTableMetadata($tableMetadata);
+
+            $table->addTable($newTable, $name);
+        }
+
+        return $table;
+    }
+
+    /**
+     * Creates a Set from an array index.
+     *
+     * @param array $index @see DataCollection
+     * @param array $resultIndices @see make
+     * @param array $keyMetadata The metadata to add to the table when it's created.
+     * @return DataTable\Map
+     */
+    private function createDataTableMapFromIndex($index, $resultIndices, $keyMetadata)
+    {
+        $result = new DataTable\Map();
+        $result->setKeyName(reset($resultIndices));
+        $resultIndex = key($resultIndices);
+
+        array_shift($resultIndices);
+
+        $hasIndices = !empty($resultIndices);
+
+        foreach ($index as $label => $value) {
+            $keyMetadata[$resultIndex] = $this->createTableIndexMetadata($resultIndex, $label);
+
+            if ($hasIndices) {
+                $newTable = $this->createDataTableMapFromIndex($value, $resultIndices, $keyMetadata);
+            } else {
+                $newTable = $this->createDataTable($value, $keyMetadata);
+            }
+
+            $result->addTable($newTable, $this->prettifyIndexLabel($resultIndex, $label));
+        }
+
+        return $result;
+    }
+
+    private function createTableIndexMetadata($resultIndex, $label)
+    {
+        if ($resultIndex === DataTableFactory::TABLE_METADATA_SITE_INDEX) {
+            return new Site($label);
+        } elseif ($resultIndex === DataTableFactory::TABLE_METADATA_PERIOD_INDEX) {
+            return $this->periods[$label];
+        }
+    }
+
+    /**
+     * Creates a DataTable instance from an index row.
+     *
+     * @param array $data An archive data row.
+     * @param array $keyMetadata The metadata to add to the table(s) when created.
+     * @return DataTable|DataTable\Map
+     */
+    private function createDataTable($data, $keyMetadata)
+    {
+        if ($this->dataType == 'blob') {
+            $result = $this->makeFromBlobRow($data, $keyMetadata);
+        } else {
+            $result = $this->makeFromMetricsArray($data, $keyMetadata);
+        }
+
+        return $result;
+    }
+
+    /**
+     * Creates DataTables from $dataTable's subtable blobs (stored in $blobRow) and sets
+     * the subtable IDs of each DataTable row.
+     *
+     * @param DataTable $dataTable
+     * @param array $blobRow An array associating record names (w/ subtable if applicable)
+     *                           with blob values. This should hold every subtable blob for
+     *                           the loaded DataTable.
+     * @param int $treeLevel
+     */
+    private function setSubtables($dataTable, $blobRow, $treeLevel = 0)
+    {
+        if ($this->maxSubtableDepth
+            && $treeLevel >= $this->maxSubtableDepth
+        ) {
+            // unset the subtables so DataTableManager doesn't throw
+            foreach ($dataTable->getRowsWithoutSummaryRow() as $row) {
+                $row->removeSubtable();
+            }
+
+            return;
+        }
+
+        $dataName = reset($this->dataNames);
+
+        foreach ($dataTable->getRowsWithoutSummaryRow() as $row) {
+            $sid = $row->getIdSubDataTable();
+            if ($sid === null) {
+                continue;
+            }
+
+            $blobName = $dataName . "_" . $sid;
+            if (isset($blobRow[$blobName])) {
+                $subtable = DataTable::fromSerializedArray($blobRow[$blobName]);
+                $this->setSubtables($subtable, $blobRow, $treeLevel + 1);
+
+                // we edit the subtable ID so that it matches the newly table created in memory
+                // NB: we don't overwrite the datatableid in the case we are displaying the table expanded.
+                if ($this->addMetadataSubtableId) {
+                    // this will be written back to the column 'idsubdatatable' just before rendering,
+                    // see Renderer/Php.php
+                    $row->addMetadata('idsubdatatable_in_db', $row->getIdSubDataTable());
+                }
+
+                $row->setSubtable($subtable);
+            }
+        }
+    }
+
+    private function getDefaultMetadata()
+    {
+        return array(
+            DataTableFactory::TABLE_METADATA_SITE_INDEX => new Site(reset($this->sitesId)),
+            DataTableFactory::TABLE_METADATA_PERIOD_INDEX => reset($this->periods),
+        );
+    }
+
+    /**
+     * Returns the pretty version of an index label.
+     *
+     * @param string $labelType eg, 'site', 'period', etc.
+     * @param string $label eg, '0', '1', '2012-01-01,2012-01-31', etc.
+     * @return string
+     */
+    private function prettifyIndexLabel($labelType, $label)
+    {
+        if ($labelType == self::TABLE_METADATA_PERIOD_INDEX) { // prettify period labels
+            $period = $this->periods[$label];
+            $label = $period->getLabel();
+            if ($label === 'week' || $label === 'range') {
+                return $period->getRangeString();
+            }
+
+            return $period->getPrettyString();
+        }
+        return $label;
+    }
+
+    /**
+     * @param $data
+     * @return DataTable\Simple
+     */
+    private function makeFromMetricsArray($data, $keyMetadata)
+    {
+        $table = new DataTable\Simple();
+
+        if (!empty($data)) {
+            $table->setAllTableMetadata(array_merge(DataCollection::getDataRowMetadata($data), $keyMetadata));
+
+            DataCollection::removeMetadataFromDataRow($data);
+
+            $table->addRow(new Row(array(Row::COLUMNS => $data)));
+        } else {
+            // if we're querying numeric data, we couldn't find any, and we're only
+            // looking for one metric, add a row w/ one column w/ value 0. this is to
+            // ensure that the PHP renderer outputs 0 when only one column is queried.
+            // w/o this code, an empty array would be created, and other parts of Piwik
+            // would break.
+            if (count($this->dataNames) == 1
+                && $this->isNumericDataType()
+            ) {
+                $name = reset($this->dataNames);
+                $table->addRow(new Row(array(Row::COLUMNS => array($name => 0))));
+            }
+
+            $table->setAllTableMetadata($keyMetadata);
+        }
+
+        $result = $table;
+        return $result;
+    }
+
+    private function makeMergedTableWithPeriodAndSiteIndex($index, $resultIndices, $useSimpleDataTable, $isNumeric)
+    {
+        $map = new DataTable\Map();
+        $map->setKeyName($resultIndices[self::TABLE_METADATA_PERIOD_INDEX]);
+
+        // we save all tables of the map in this array to be able to add rows fast
+        $tables = array();
+
+        foreach ($this->periods as $range => $period) {
+            // as the resulting table is "merged", we do only set Period metedata and no metadata for site. Instead each
+            // row will have an idsite metadata entry.
+            $metadata = array(self::TABLE_METADATA_PERIOD_INDEX => $period);
+
+            if ($useSimpleDataTable) {
+                $table = new DataTable\Simple();
+            } else {
+                $table = new DataTable();
+            }
+
+            $table->setAllTableMetadata($metadata);
+            $map->addTable($table, $this->prettifyIndexLabel(self::TABLE_METADATA_PERIOD_INDEX, $range));
+
+            $tables[$range] = $table;
+        }
+
+        foreach ($index as $idsite => $table) {
+            $rowMeta = array('idsite' => $idsite);
+
+            foreach ($table as $range => $row) {
+                if (!empty($row)) {
+                    $tables[$range]->addRow(new Row(array(
+                        Row::COLUMNS  => $row,
+                        Row::METADATA => $rowMeta)
+                    ));
+                } elseif ($isNumeric) {
+                    $tables[$range]->addRow(new Row(array(
+                        Row::COLUMNS  => $this->defaultRow,
+                        Row::METADATA => $rowMeta)
+                    ));
+                }
+            }
+        }
+
+        return $map;
+    }
+
+    private function makeMergedWithSiteIndex($index, $useSimpleDataTable, $isNumeric)
+    {
+        if ($useSimpleDataTable) {
+            $table = new DataTable\Simple();
+        } else {
+            $table = new DataTable();
+        }
+
+        $table->setAllTableMetadata(array(DataTableFactory::TABLE_METADATA_PERIOD_INDEX => reset($this->periods)));
+
+        foreach ($index as $idsite => $row) {
+            if (!empty($row)) {
+                $table->addRow(new Row(array(
+                    Row::COLUMNS  => $row,
+                    Row::METADATA => array('idsite' => $idsite))
+                ));
+            } elseif ($isNumeric) {
+                $table->addRow(new Row(array(
+                    Row::COLUMNS  => $this->defaultRow,
+                    Row::METADATA => array('idsite' => $idsite))
+                ));
+            }
+        }
+
+        return $table;
+    }
+}

msd2/tracking/piwik/core/Archive/Parameters.php

@@ -0,0 +1,59 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+
+namespace Piwik\Archive;
+
+use Piwik\Period;
+use Piwik\Segment;
+
+class Parameters
+{
+    /**
+     * The list of site IDs to query archive data for.
+     *
+     * @var array
+     */
+    private $idSites = array();
+
+    /**
+     * The list of Period's to query archive data for.
+     *
+     * @var Period[]
+     */
+    private $periods = array();
+
+    /**
+     * Segment applied to the visits set.
+     *
+     * @var Segment
+     */
+    private $segment;
+
+    public function getSegment()
+    {
+        return $this->segment;
+    }
+
+    public function __construct($idSites, $periods, Segment $segment)
+    {
+        $this->idSites = $idSites;
+        $this->periods = $periods;
+        $this->segment = $segment;
+    }
+
+    public function getPeriods()
+    {
+        return $this->periods;
+    }
+
+    public function getIdSites()
+    {
+        return $this->idSites;
+    }
+}

msd2/tracking/piwik/core/ArchiveProcessor.php

@@ -0,0 +1,640 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik;
+
+use Exception;
+use Piwik\Archive\DataTableFactory;
+use Piwik\ArchiveProcessor\Parameters;
+use Piwik\ArchiveProcessor\Rules;
+use Piwik\DataAccess\ArchiveWriter;
+use Piwik\DataAccess\LogAggregator;
+use Piwik\DataTable\Manager;
+use Piwik\DataTable\Map;
+use Piwik\DataTable\Row;
+use Piwik\Segment\SegmentExpression;
+
+/**
+ * Used by {@link Piwik\Plugin\Archiver} instances to insert and aggregate archive data.
+ *
+ * ### See also
+ *
+ * - **{@link Piwik\Plugin\Archiver}** - to learn how plugins should implement their own analytics
+ *                                       aggregation logic.
+ * - **{@link Piwik\DataAccess\LogAggregator}** - to learn how plugins can perform data aggregation
+ *                                                across Piwik's log tables.
+ *
+ * ### Examples
+ *
+ * **Inserting numeric data**
+ *
+ *     // function in an Archiver descendant
+ *     public function aggregateDayReport()
+ *     {
+ *         $archiveProcessor = $this->getProcessor();
+ *
+ *         $myFancyMetric = // ... calculate the metric value ...
+ *         $archiveProcessor->insertNumericRecord('MyPlugin_myFancyMetric', $myFancyMetric);
+ *     }
+ *
+ * **Inserting serialized DataTables**
+ *
+ *     // function in an Archiver descendant
+ *     public function aggregateDayReport()
+ *     {
+ *         $archiveProcessor = $this->getProcessor();
+ *
+ *         $maxRowsInTable = Config::getInstance()->General['datatable_archiving_maximum_rows_standard'];j
+ *
+ *         $dataTable = // ... build by aggregating visits ...
+ *         $serializedData = $dataTable->getSerialized($maxRowsInTable, $maxRowsInSubtable = $maxRowsInTable,
+ *                                                     $columnToSortBy = Metrics::INDEX_NB_VISITS);
+ *
+ *         $archiveProcessor->insertBlobRecords('MyPlugin_myFancyReport', $serializedData);
+ *     }
+ *
+ * **Aggregating archive data**
+ *
+ *     // function in Archiver descendant
+ *     public function aggregateMultipleReports()
+ *     {
+ *         $archiveProcessor = $this->getProcessor();
+ *
+ *         // aggregate a metric
+ *         $archiveProcessor->aggregateNumericMetrics('MyPlugin_myFancyMetric');
+ *         $archiveProcessor->aggregateNumericMetrics('MyPlugin_mySuperFancyMetric', 'max');
+ *
+ *         // aggregate a report
+ *         $archiveProcessor->aggregateDataTableRecords('MyPlugin_myFancyReport');
+ *     }
+ *
+ */
+class ArchiveProcessor
+{
+    /**
+     * @var \Piwik\DataAccess\ArchiveWriter
+     */
+    private $archiveWriter;
+
+    /**
+     * @var \Piwik\DataAccess\LogAggregator
+     */
+    private $logAggregator;
+
+    /**
+     * @var Archive
+     */
+    public $archive = null;
+
+    /**
+     * @var Parameters
+     */
+    private $params;
+
+    /**
+     * @var int
+     */
+    private $numberOfVisits = false;
+
+    private $numberOfVisitsConverted = false;
+
+    /**
+     * If true, unique visitors are not calculated when we are aggregating data for multiple sites.
+     * The `[General] enable_processing_unique_visitors_multiple_sites` INI config option controls
+     * the value of this variable.
+     *
+     * @var bool
+     */
+    private $skipUniqueVisitorsCalculationForMultipleSites = true;
+
+    public function __construct(Parameters $params, ArchiveWriter $archiveWriter, LogAggregator $logAggregator)
+    {
+        $this->params = $params;
+        $this->logAggregator = $logAggregator;
+        $this->archiveWriter = $archiveWriter;
+
+        $this->skipUniqueVisitorsCalculationForMultipleSites = Rules::shouldSkipUniqueVisitorsCalculationForMultipleSites();
+    }
+
+    protected function getArchive()
+    {
+        if (empty($this->archive)) {
+            $subPeriods = $this->params->getSubPeriods();
+            $idSites    = $this->params->getIdSites();
+            $this->archive = Archive::factory($this->params->getSegment(), $subPeriods, $idSites);
+        }
+
+        return $this->archive;
+    }
+
+    public function setNumberOfVisits($visits, $visitsConverted)
+    {
+        $this->numberOfVisits = $visits;
+        $this->numberOfVisitsConverted = $visitsConverted;
+    }
+
+    /**
+     * Returns the {@link Parameters} object containing the site, period and segment we're archiving
+     * data for.
+     *
+     * @return Parameters
+     * @api
+     */
+    public function getParams()
+    {
+        return $this->params;
+    }
+
+    /**
+     * Returns a `{@link Piwik\DataAccess\LogAggregator}` instance for the site, period and segment this
+     * ArchiveProcessor will insert archive data for.
+     *
+     * @return LogAggregator
+     * @api
+     */
+    public function getLogAggregator()
+    {
+        return $this->logAggregator;
+    }
+
+    /**
+     * Array of (column name before => column name renamed) of the columns for which sum operation is invalid.
+     * These columns will be renamed as per this mapping.
+     * @var array
+     */
+    protected static $columnsToRenameAfterAggregation = array(
+        Metrics::INDEX_NB_UNIQ_VISITORS => Metrics::INDEX_SUM_DAILY_NB_UNIQ_VISITORS,
+        Metrics::INDEX_NB_USERS         => Metrics::INDEX_SUM_DAILY_NB_USERS,
+    );
+
+    /**
+     * Sums records for every subperiod of the current period and inserts the result as the record
+     * for this period.
+     *
+     * DataTables are summed recursively so subtables will be summed as well.
+     *
+     * @param string|array $recordNames Name(s) of the report we are aggregating, eg, `'Referrers_type'`.
+     * @param int $maximumRowsInDataTableLevelZero Maximum number of rows allowed in the top level DataTable.
+     * @param int $maximumRowsInSubDataTable Maximum number of rows allowed in each subtable.
+     * @param string $columnToSortByBeforeTruncation The name of the column to sort by before truncating a DataTable.
+     * @param array $columnsAggregationOperation Operations for aggregating columns, see {@link Row::sumRow()}.
+     * @param array $columnsToRenameAfterAggregation Columns mapped to new names for columns that must change names
+     *                                               when summed because they cannot be summed, eg,
+     *                                               `array('nb_uniq_visitors' => 'sum_daily_nb_uniq_visitors')`.
+     * @param bool|array $countRowsRecursive if set to true, will calculate the recursive rows count for all record names
+     *                                       which makes it slower. If you only need it for some records pass an array of
+     *                                       recordNames that defines for which ones you need a recursive row count.
+     * @return array Returns the row counts of each aggregated report before truncation, eg,
+     *
+     *                   array(
+     *                       'report1' => array('level0' => $report1->getRowsCount,
+     *                                          'recursive' => $report1->getRowsCountRecursive()),
+     *                       'report2' => array('level0' => $report2->getRowsCount,
+     *                                          'recursive' => $report2->getRowsCountRecursive()),
+     *                       ...
+     *                   )
+     * @api
+     */
+    public function aggregateDataTableRecords($recordNames,
+                                              $maximumRowsInDataTableLevelZero = null,
+                                              $maximumRowsInSubDataTable = null,
+                                              $columnToSortByBeforeTruncation = null,
+                                              &$columnsAggregationOperation = null,
+                                              $columnsToRenameAfterAggregation = null,
+                                              $countRowsRecursive = true)
+    {
+        if (!is_array($recordNames)) {
+            $recordNames = array($recordNames);
+        }
+
+        $nameToCount = array();
+        foreach ($recordNames as $recordName) {
+            $latestUsedTableId = Manager::getInstance()->getMostRecentTableId();
+
+            $table = $this->aggregateDataTableRecord($recordName, $columnsAggregationOperation, $columnsToRenameAfterAggregation);
+
+            $nameToCount[$recordName]['level0'] = $table->getRowsCount();
+            if ($countRowsRecursive === true || (is_array($countRowsRecursive) && in_array($recordName, $countRowsRecursive))) {
+                $nameToCount[$recordName]['recursive'] = $table->getRowsCountRecursive();
+            }
+
+            $blob = $table->getSerialized($maximumRowsInDataTableLevelZero, $maximumRowsInSubDataTable, $columnToSortByBeforeTruncation);
+            Common::destroy($table);
+            $this->insertBlobRecord($recordName, $blob);
+
+            unset($blob);
+            DataTable\Manager::getInstance()->deleteAll($latestUsedTableId);
+        }
+
+        return $nameToCount;
+    }
+
+    /**
+     * Aggregates one or more metrics for every subperiod of the current period and inserts the results
+     * as metrics for the current period.
+     *
+     * @param array|string $columns Array of metric names to aggregate.
+     * @param bool|string $operationToApply The operation to apply to the metric. Either `'sum'`, `'max'` or `'min'`.
+     * @return array|int Returns the array of aggregate values. If only one metric was aggregated,
+     *                   the aggregate value will be returned as is, not in an array.
+     *                   For example, if `array('nb_visits', 'nb_hits')` is supplied for `$columns`,
+     *
+     *                       array(
+     *                           'nb_visits' => 3040,
+     *                           'nb_hits' => 405
+     *                       )
+     *
+     *                   could be returned. If `array('nb_visits')` or `'nb_visits'` is used for `$columns`,
+     *                   then `3040` would be returned.
+     * @api
+     */
+    public function aggregateNumericMetrics($columns, $operationToApply = false)
+    {
+        $metrics = $this->getAggregatedNumericMetrics($columns, $operationToApply);
+
+        foreach ($metrics as $column => $value) {
+            $value = Common::forceDotAsSeparatorForDecimalPoint($value);
+            $this->archiveWriter->insertRecord($column, $value);
+        }
+        // if asked for only one field to sum
+        if (count($metrics) == 1) {
+            return reset($metrics);
+        }
+
+        // returns the array of records once summed
+        return $metrics;
+    }
+
+    public function getNumberOfVisits()
+    {
+        if ($this->numberOfVisits === false) {
+            throw new Exception("visits should have been set here");
+        }
+        return $this->numberOfVisits;
+    }
+
+    public function getNumberOfVisitsConverted()
+    {
+        return $this->numberOfVisitsConverted;
+    }
+
+    /**
+     * Caches multiple numeric records in the archive for this processor's site, period
+     * and segment.
+     *
+     * @param array $numericRecords A name-value mapping of numeric values that should be
+     *                              archived, eg,
+     *
+     *                                  array('Referrers_distinctKeywords' => 23, 'Referrers_distinctCampaigns' => 234)
+     * @api
+     */
+    public function insertNumericRecords($numericRecords)
+    {
+        foreach ($numericRecords as $name => $value) {
+            $this->insertNumericRecord($name, $value);
+        }
+    }
+
+    /**
+     * Caches a single numeric record in the archive for this processor's site, period and
+     * segment.
+     *
+     * Numeric values are not inserted if they equal `0`.
+     *
+     * @param string $name The name of the numeric value, eg, `'Referrers_distinctKeywords'`.
+     * @param float $value The numeric value.
+     * @api
+     */
+    public function insertNumericRecord($name, $value)
+    {
+        $value = round($value, 2);
+        $value = Common::forceDotAsSeparatorForDecimalPoint($value);
+
+        $this->archiveWriter->insertRecord($name, $value);
+    }
+
+    /**
+     * Caches one or more blob records in the archive for this processor's site, period
+     * and segment.
+     *
+     * @param string $name The name of the record, eg, 'Referrers_type'.
+     * @param string|array $values A blob string or an array of blob strings. If an array
+     *                             is used, the first element in the array will be inserted
+     *                             with the `$name` name. The others will be inserted with
+     *                             `$name . '_' . $index` as the record name (where $index is
+     *                             the index of the blob record in `$values`).
+     * @api
+     */
+    public function insertBlobRecord($name, $values)
+    {
+        $this->archiveWriter->insertBlobRecord($name, $values);
+    }
+
+    /**
+     * This method selects all DataTables that have the name $name over the period.
+     * All these DataTables are then added together, and the resulting DataTable is returned.
+     *
+     * @param string $name
+     * @param array $columnsAggregationOperation Operations for aggregating columns, @see Row::sumRow()
+     * @param array $columnsToRenameAfterAggregation columns in the array (old name, new name) to be renamed as the sum operation is not valid on them (eg. nb_uniq_visitors->sum_daily_nb_uniq_visitors)
+     * @return DataTable
+     */
+    protected function aggregateDataTableRecord($name, $columnsAggregationOperation = null, $columnsToRenameAfterAggregation = null)
+    {
+        try {
+            ErrorHandler::pushFatalErrorBreadcrumb(__CLASS__, ['name' => $name]);
+
+            // By default we shall aggregate all sub-tables.
+            $dataTable = $this->getArchive()->getDataTableExpanded($name, $idSubTable = null, $depth = null, $addMetadataSubtableId = false);
+
+            $columnsRenamed = false;
+
+            if ($dataTable instanceof Map) {
+                $columnsRenamed = true;
+                // see https://github.com/piwik/piwik/issues/4377
+                $self = $this;
+                $dataTable->filter(function ($table) use ($self, $columnsToRenameAfterAggregation) {
+
+                    if ($self->areColumnsNotAlreadyRenamed($table)) {
+                        /**
+                         * This makes archiving and range dates a lot faster. Imagine we archive a week, then we will
+                         * rename all columns of each 7 day archives. Afterwards we know the columns will be replaced in a
+                         * week archive. When generating month archives, which uses mostly week archives, we do not have
+                         * to replace those columns for the week archives again since we can be sure they were already
+                         * replaced. Same when aggregating year and range archives. This can save up 10% or more when
+                         * aggregating Month, Year and Range archives.
+                         */
+                        $self->renameColumnsAfterAggregation($table, $columnsToRenameAfterAggregation);
+                    }
+                });
+            }
+
+            $dataTable = $this->getAggregatedDataTableMap($dataTable, $columnsAggregationOperation);
+
+            if (!$columnsRenamed) {
+                $this->renameColumnsAfterAggregation($dataTable, $columnsToRenameAfterAggregation);
+            }
+        } finally {
+            ErrorHandler::popFatalErrorBreadcrumb();
+        }
+
+        return $dataTable;
+    }
+
+    /**
+     * Note: public only for use in closure in PHP 5.3.
+     *
+     * @param $table
+     * @return \Piwik\Period
+     */
+    public function areColumnsNotAlreadyRenamed($table)
+    {
+        $period = $table->getMetadata(DataTableFactory::TABLE_METADATA_PERIOD_INDEX);
+
+        return !$period || $period->getLabel() === 'day';
+    }
+
+    protected function getOperationForColumns($columns, $defaultOperation)
+    {
+        $operationForColumn = array();
+        foreach ($columns as $name) {
+            $operation = $defaultOperation;
+            if (empty($operation)) {
+                $operation = $this->guessOperationForColumn($name);
+            }
+            $operationForColumn[$name] = $operation;
+        }
+        return $operationForColumn;
+    }
+
+    protected function enrichWithUniqueVisitorsMetric(Row $row)
+    {
+        // skip unique visitors metrics calculation if calculating for multiple sites is disabled
+        if (!$this->getParams()->isSingleSite()
+            && $this->skipUniqueVisitorsCalculationForMultipleSites
+        ) {
+            return;
+        }
+
+        if ($row->getColumn('nb_uniq_visitors') === false
+            && $row->getColumn('nb_users') === false
+        ) {
+            return;
+        }
+
+        if (!SettingsPiwik::isUniqueVisitorsEnabled($this->getParams()->getPeriod()->getLabel())) {
+            $row->deleteColumn('nb_uniq_visitors');
+            $row->deleteColumn('nb_users');
+            return;
+        }
+
+        $metrics = array(
+            Metrics::INDEX_NB_USERS
+        );
+
+        if ($this->getParams()->isSingleSite()) {
+            $uniqueVisitorsMetric = Metrics::INDEX_NB_UNIQ_VISITORS;
+        } else {
+            if (!SettingsPiwik::isSameFingerprintAcrossWebsites()) {
+                throw new Exception("Processing unique visitors across websites is enabled for this instance,
+                            but to process this metric you must first set enable_fingerprinting_across_websites=1
+                            in the config file, under the [Tracker] section.");
+            }
+            $uniqueVisitorsMetric = Metrics::INDEX_NB_UNIQ_FINGERPRINTS;
+        }
+        $metrics[] = $uniqueVisitorsMetric;
+
+        $uniques = $this->computeNbUniques($metrics);
+
+        // see edge case as described in https://github.com/piwik/piwik/issues/9357 where uniq_visitors might be higher
+        // than visits because we archive / process it after nb_visits. Between archiving nb_visits and nb_uniq_visitors
+        // there could have been a new visit leading to a higher nb_unique_visitors than nb_visits which is not possible
+        // by definition. In this case we simply use the visits metric instead of unique visitors metric.
+        $visits = $row->getColumn('nb_visits');
+        if ($visits !== false && $uniques[$uniqueVisitorsMetric] !== false) {
+            $uniques[$uniqueVisitorsMetric] = min($uniques[$uniqueVisitorsMetric], $visits);
+        }
+
+        $row->setColumn('nb_uniq_visitors', $uniques[$uniqueVisitorsMetric]);
+        $row->setColumn('nb_users', $uniques[Metrics::INDEX_NB_USERS]);
+    }
+
+    protected function guessOperationForColumn($column)
+    {
+        if (strpos($column, 'max_') === 0) {
+            return 'max';
+        }
+        if (strpos($column, 'min_') === 0) {
+            return 'min';
+        }
+        return 'sum';
+    }
+
+    /**
+     * Processes number of unique visitors for the given period
+     *
+     * This is the only Period metric (ie. week/month/year/range) that we process from the logs directly,
+     * since unique visitors cannot be summed like other metrics.
+     *
+     * @param array Metrics Ids for which to aggregates count of values
+     * @return array of metrics, where the key is metricid and the value is the metric value
+     */
+    protected function computeNbUniques($metrics)
+    {
+        $logAggregator = $this->getLogAggregator();
+        $query = $logAggregator->queryVisitsByDimension(array(), false, array(), $metrics);
+        $data = $query->fetch();
+        return $data;
+    }
+
+    /**
+     * If the DataTable is a Map, sums all DataTable in the map and return the DataTable.
+     *
+     *
+     * @param $data DataTable|DataTable\Map
+     * @param $columnsToRenameAfterAggregation array
+     * @return DataTable
+     */
+    protected function getAggregatedDataTableMap($data, $columnsAggregationOperation)
+    {
+        $table = new DataTable();
+
+        if (!empty($columnsAggregationOperation)) {
+            $table->setMetadata(DataTable::COLUMN_AGGREGATION_OPS_METADATA_NAME, $columnsAggregationOperation);
+        }
+
+        if ($data instanceof DataTable\Map) {
+            // as $date => $tableToSum
+            $this->aggregatedDataTableMapsAsOne($data, $table);
+        } else {
+            $table->addDataTable($data);
+        }
+
+        return $table;
+    }
+
+    /**
+     * Aggregates the DataTable\Map into the destination $aggregated
+     * @param $map
+     * @param $aggregated
+     */
+    protected function aggregatedDataTableMapsAsOne(Map $map, DataTable $aggregated)
+    {
+        foreach ($map->getDataTables() as $tableToAggregate) {
+            if ($tableToAggregate instanceof Map) {
+                $this->aggregatedDataTableMapsAsOne($tableToAggregate, $aggregated);
+            } else {
+                $aggregated->addDataTable($tableToAggregate);
+            }
+        }
+    }
+
+    /**
+     * Note: public only for use in closure in PHP 5.3.
+     */
+    public function renameColumnsAfterAggregation(DataTable $table, $columnsToRenameAfterAggregation = null)
+    {
+        // Rename columns after aggregation
+        if (is_null($columnsToRenameAfterAggregation)) {
+            $columnsToRenameAfterAggregation = self::$columnsToRenameAfterAggregation;
+        }
+
+        if (empty($columnsToRenameAfterAggregation)) {
+            return;
+        }
+        
+        foreach ($table->getRows() as $row) {
+            foreach ($columnsToRenameAfterAggregation as $oldName => $newName) {
+                $row->renameColumn($oldName, $newName);
+            }
+
+            $subTable = $row->getSubtable();
+            if ($subTable) {
+                $this->renameColumnsAfterAggregation($subTable, $columnsToRenameAfterAggregation);
+            }
+        }
+    }
+
+    protected function getAggregatedNumericMetrics($columns, $operationToApply)
+    {
+        if (!is_array($columns)) {
+            $columns = array($columns);
+        }
+
+        $operationForColumn = $this->getOperationForColumns($columns, $operationToApply);
+
+        $dataTable = $this->getArchive()->getDataTableFromNumeric($columns);
+
+        $results = $this->getAggregatedDataTableMap($dataTable, $operationForColumn);
+        if ($results->getRowsCount() > 1) {
+            throw new Exception("A DataTable is an unexpected state:" . var_export($results, true));
+        }
+
+        $rowMetrics = $results->getFirstRow();
+        if ($rowMetrics === false) {
+            $rowMetrics = new Row;
+        }
+        $this->enrichWithUniqueVisitorsMetric($rowMetrics);
+        $this->renameColumnsAfterAggregation($results, self::$columnsToRenameAfterAggregation);
+
+        $metrics = $rowMetrics->getColumns();
+
+        foreach ($columns as $name) {
+            if (!isset($metrics[$name])) {
+                $metrics[$name] = 0;
+            }
+        }
+
+        return $metrics;
+    }
+
+    /**
+     * Initiate archiving for a plugin during an ongoing archiving. The plugin can be another
+     * plugin or the same plugin.
+     *
+     * This method should be called during archiving when one plugin uses the report of another
+     * plugin with a segment. It will ensure reports for that segment & plugin will be archived
+     * without initiating archiving for every plugin with that segment (which would be a performance
+     * killer).
+     *
+     * @param string $plugin
+     * @param string $segment
+     */
+    public function processDependentArchive($plugin, $segment)
+    {
+        $params = $this->getParams();
+        if (!$params->isRootArchiveRequest()) { // prevent all recursion
+            return;
+        }
+
+        $idSites = [$params->getSite()->getId()];
+
+        $newSegment = Segment::combine($params->getSegment()->getString(), SegmentExpression::AND_DELIMITER, $segment);
+        if ($newSegment === $segment && $params->getRequestedPlugin() === $plugin) { // being processed now
+            return;
+        }
+
+        $newSegment = new Segment($newSegment, $idSites);
+        if (ArchiveProcessor\Rules::isSegmentPreProcessed($idSites, $newSegment)) {
+            // will be processed anyway
+            return;
+        }
+
+        $parameters = new ArchiveProcessor\Parameters($params->getSite(), $params->getPeriod(), $newSegment);
+        $parameters->onlyArchiveRequestedPlugin();
+        $parameters->setIsRootArchiveRequest(false);
+
+        $archiveLoader = new ArchiveProcessor\Loader($parameters);
+        $archiveLoader->prepareArchive($plugin);
+    }
+
+    public function getArchiveWriter()
+    {
+        return $this->archiveWriter;
+    }
+}

msd2/tracking/piwik/core/ArchiveProcessor/Loader.php

@@ -0,0 +1,260 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\ArchiveProcessor;
+
+use Piwik\Archive;
+use Piwik\Cache;
+use Piwik\CacheId;
+use Piwik\Common;
+use Piwik\Config;
+use Piwik\Context;
+use Piwik\DataAccess\ArchiveSelector;
+use Piwik\Date;
+use Piwik\Period;
+use Piwik\Piwik;
+
+/**
+ * This class uses PluginsArchiver class to trigger data aggregation and create archives.
+ */
+class Loader
+{
+    /**
+     * Is the current archive temporary. ie.
+     * - today
+     * - current week / month / year
+     */
+    protected $temporaryArchive;
+
+    /**
+     * Idarchive in the DB for the requested archive
+     *
+     * @var int
+     */
+    protected $idArchive;
+
+    /**
+     * @var Parameters
+     */
+    protected $params;
+
+    public function __construct(Parameters $params)
+    {
+        $this->params = $params;
+    }
+
+    /**
+     * @return bool
+     */
+    protected function isThereSomeVisits($visits)
+    {
+        return $visits > 0;
+    }
+
+    /**
+     * @return bool
+     */
+    protected function mustProcessVisitCount($visits)
+    {
+        return $visits === false;
+    }
+
+    public function prepareArchive($pluginName)
+    {
+        return Context::changeIdSite($this->params->getSite()->getId(), function () use ($pluginName) {
+            return $this->prepareArchiveImpl($pluginName);
+        });
+    }
+
+    private function prepareArchiveImpl($pluginName)
+    {
+        $this->params->setRequestedPlugin($pluginName);
+
+        list($idArchive, $visits, $visitsConverted) = $this->loadExistingArchiveIdFromDb();
+        if (!empty($idArchive)) {
+            return $idArchive;
+        }
+
+        list($visits, $visitsConverted) = $this->prepareCoreMetricsArchive($visits, $visitsConverted);
+        list($idArchive, $visits) = $this->prepareAllPluginsArchive($visits, $visitsConverted);
+
+        if ($this->isThereSomeVisits($visits) || PluginsArchiver::doesAnyPluginArchiveWithoutVisits()) {
+            return $idArchive;
+        }
+        return false;
+    }
+
+    /**
+     * Prepares the core metrics if needed.
+     *
+     * @param $visits
+     * @return array
+     */
+    protected function prepareCoreMetricsArchive($visits, $visitsConverted)
+    {
+        $createSeparateArchiveForCoreMetrics = $this->mustProcessVisitCount($visits)
+                                && !$this->doesRequestedPluginIncludeVisitsSummary();
+
+        if ($createSeparateArchiveForCoreMetrics) {
+            $requestedPlugin = $this->params->getRequestedPlugin();
+
+            $this->params->setRequestedPlugin('VisitsSummary');
+
+            $pluginsArchiver = new PluginsArchiver($this->params, $this->isArchiveTemporary());
+            $metrics = $pluginsArchiver->callAggregateCoreMetrics();
+            $pluginsArchiver->finalizeArchive();
+
+            $this->params->setRequestedPlugin($requestedPlugin);
+
+            $visits = $metrics['nb_visits'];
+            $visitsConverted = $metrics['nb_visits_converted'];
+        }
+
+        return array($visits, $visitsConverted);
+    }
+
+    protected function prepareAllPluginsArchive($visits, $visitsConverted)
+    {
+        $pluginsArchiver = new PluginsArchiver($this->params, $this->isArchiveTemporary());
+
+        if ($this->mustProcessVisitCount($visits)
+            || $this->doesRequestedPluginIncludeVisitsSummary()
+        ) {
+            $metrics = $pluginsArchiver->callAggregateCoreMetrics();
+            $visits = $metrics['nb_visits'];
+            $visitsConverted = $metrics['nb_visits_converted'];
+        }
+
+        $forceArchivingWithoutVisits = !$this->isThereSomeVisits($visits) && $this->shouldArchiveForSiteEvenWhenNoVisits();
+        $pluginsArchiver->callAggregateAllPlugins($visits, $visitsConverted, $forceArchivingWithoutVisits);
+
+        $idArchive = $pluginsArchiver->finalizeArchive();
+
+        return array($idArchive, $visits);
+    }
+
+    protected function doesRequestedPluginIncludeVisitsSummary()
+    {
+        $processAllReportsIncludingVisitsSummary =
+                Rules::shouldProcessReportsAllPlugins($this->params->getIdSites(), $this->params->getSegment(), $this->params->getPeriod()->getLabel());
+        $doesRequestedPluginIncludeVisitsSummary = $processAllReportsIncludingVisitsSummary
+                                                        || $this->params->getRequestedPlugin() == 'VisitsSummary';
+        return $doesRequestedPluginIncludeVisitsSummary;
+    }
+
+    protected function isArchivingForcedToTrigger()
+    {
+        $period = $this->params->getPeriod()->getLabel();
+        $debugSetting = 'always_archive_data_period'; // default
+
+        if ($period == 'day') {
+            $debugSetting = 'always_archive_data_day';
+        } elseif ($period == 'range') {
+            $debugSetting = 'always_archive_data_range';
+        }
+
+        return (bool) Config::getInstance()->Debug[$debugSetting];
+    }
+
+    /**
+     * Returns the idArchive if the archive is available in the database for the requested plugin.
+     * Returns false if the archive needs to be processed.
+     *
+     * @return array
+     */
+    protected function loadExistingArchiveIdFromDb()
+    {
+        $noArchiveFound = array(false, false, false);
+
+        // see isArchiveTemporary()
+        $minDatetimeArchiveProcessedUTC = $this->getMinTimeArchiveProcessed();
+
+        if ($this->isArchivingForcedToTrigger()) {
+            return $noArchiveFound;
+        }
+
+        $idAndVisits = ArchiveSelector::getArchiveIdAndVisits($this->params, $minDatetimeArchiveProcessedUTC);
+
+        if (!$idAndVisits) {
+            return $noArchiveFound;
+        }
+
+        return $idAndVisits;
+    }
+
+    /**
+     * Returns the minimum archive processed datetime to look at. Only public for tests.
+     *
+     * @return int|bool  Datetime timestamp, or false if must look at any archive available
+     */
+    protected function getMinTimeArchiveProcessed()
+    {
+        $endDateTimestamp = self::determineIfArchivePermanent($this->params->getDateEnd());
+        $isArchiveTemporary = ($endDateTimestamp === false);
+        $this->temporaryArchive = $isArchiveTemporary;
+
+        if ($endDateTimestamp) {
+            // Permanent archive
+            return $endDateTimestamp;
+        }
+
+        $dateStart = $this->params->getDateStart();
+        $period    = $this->params->getPeriod();
+        $segment   = $this->params->getSegment();
+        $site      = $this->params->getSite();
+
+        // Temporary archive
+        return Rules::getMinTimeProcessedForTemporaryArchive($dateStart, $period, $segment, $site);
+    }
+
+    protected static function determineIfArchivePermanent(Date $dateEnd)
+    {
+        $now = time();
+        $endTimestampUTC = strtotime($dateEnd->getDateEndUTC());
+
+        if ($endTimestampUTC <= $now) {
+            // - if the period we are looking for is finished, we look for a ts_archived that
+            //   is greater than the last day of the archive
+            return $endTimestampUTC;
+        }
+
+        return false;
+    }
+
+    protected function isArchiveTemporary()
+    {
+        if (is_null($this->temporaryArchive)) {
+            throw new \Exception("getMinTimeArchiveProcessed() should be called prior to isArchiveTemporary()");
+        }
+
+        return $this->temporaryArchive;
+    }
+
+    private function shouldArchiveForSiteEvenWhenNoVisits()
+    {
+        $idSitesToArchive = $this->getIdSitesToArchiveWhenNoVisits();
+        return in_array($this->params->getSite()->getId(), $idSitesToArchive);
+    }
+
+    private function getIdSitesToArchiveWhenNoVisits()
+    {
+        $cache = Cache::getTransientCache();
+        $cacheKey = 'Archiving.getIdSitesToArchiveWhenNoVisits';
+
+        if (!$cache->contains($cacheKey)) {
+            $idSites = array();
+
+            // leaving undocumented unless decided otherwise
+            Piwik::postEvent('Archiving.getIdSitesToArchiveWhenNoVisits', array(&$idSites));
+
+            $cache->save($cacheKey, $idSites);
+        }
+
+        return $cache->fetch($cacheKey);
+    }
+}

msd2/tracking/piwik/core/ArchiveProcessor/Parameters.php

@@ -0,0 +1,264 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+
+namespace Piwik\ArchiveProcessor;
+
+use Piwik\Date;
+use Piwik\Log;
+use Piwik\Period;
+use Piwik\Piwik;
+use Piwik\Segment;
+use Piwik\Site;
+
+/**
+ * Contains the analytics parameters for the reports that are currently being archived. The analytics
+ * parameters include the **website** the reports describe, the **period** of time the reports describe
+ * and the **segment** used to limit the visit set.
+ */
+class Parameters
+{
+    /**
+     * @var Site
+     */
+    private $site = null;
+
+    /**
+     * @var Period
+     */
+    private $period = null;
+
+    /**
+     * @var Segment
+     */
+    private $segment = null;
+
+    /**
+     * @var string Plugin name which triggered this archive processor
+     */
+    private $requestedPlugin = false;
+
+    private $onlyArchiveRequestedPlugin = false;
+
+    /**
+     * @var bool
+     */
+    private $isRootArchiveRequest = true;
+
+    /**
+     * Constructor.
+     *
+     * @ignore
+     */
+    public function __construct(Site $site, Period $period, Segment $segment)
+    {
+        $this->site = $site;
+        $this->period = $period;
+        $this->segment = $segment;
+    }
+
+    /**
+     * @ignore
+     */
+    public function setRequestedPlugin($plugin)
+    {
+        $this->requestedPlugin = $plugin;
+    }
+
+    /**
+     * @ignore
+     */
+    public function onlyArchiveRequestedPlugin()
+    {
+        $this->onlyArchiveRequestedPlugin = true;
+    }
+
+    /**
+     * @ignore
+     */
+    public function shouldOnlyArchiveRequestedPlugin()
+    {
+        return $this->onlyArchiveRequestedPlugin;
+    }
+
+    /**
+     * @ignore
+     */
+    public function getRequestedPlugin()
+    {
+        return $this->requestedPlugin;
+    }
+
+    /**
+     * Returns the period we are computing statistics for.
+     *
+     * @return Period
+     * @api
+     */
+    public function getPeriod()
+    {
+        return $this->period;
+    }
+
+    /**
+     * Returns the array of Period which make up this archive.
+     *
+     * @return \Piwik\Period[]
+     * @ignore
+     */
+    public function getSubPeriods()
+    {
+        if ($this->getPeriod()->getLabel() == 'day') {
+            return array( $this->getPeriod() );
+        }
+        return $this->getPeriod()->getSubperiods();
+    }
+
+    /**
+     * @return array
+     * @ignore
+     */
+    public function getIdSites()
+    {
+        $idSite = $this->getSite()->getId();
+
+        $idSites = array($idSite);
+
+        Piwik::postEvent('ArchiveProcessor.Parameters.getIdSites', array(&$idSites, $this->getPeriod()));
+
+        return $idSites;
+    }
+
+    /**
+     * Returns the site we are computing statistics for.
+     *
+     * @return Site
+     * @api
+     */
+    public function getSite()
+    {
+        return $this->site;
+    }
+
+    /**
+     * The Segment used to limit the set of visits that are being aggregated.
+     *
+     * @return Segment
+     * @api
+     */
+    public function getSegment()
+    {
+        return $this->segment;
+    }
+
+    /**
+     * Returns the end day of the period in the site's timezone.
+     *
+     * @return Date
+     */
+    public function getDateEnd()
+    {
+        return $this->getPeriod()->getDateEnd()->setTimezone($this->getSite()->getTimezone());
+    }
+
+    /**
+     * Returns the start day of the period in the site's timezone.
+     *
+     * @return Date
+     */
+    public function getDateStart()
+    {
+        return $this->getPeriod()->getDateStart()->setTimezone($this->getSite()->getTimezone());
+    }
+
+    /**
+     * Returns the start day of the period in the site's timezone (includes the time of day).
+     *
+     * @return Date
+     */
+    public function getDateTimeStart()
+    {
+        return $this->getPeriod()->getDateTimeStart()->setTimezone($this->getSite()->getTimezone());
+    }
+
+    /**
+     * Returns the end day of the period in the site's timezone (includes the time of day).
+     *
+     * @return Date
+     */
+    public function getDateTimeEnd()
+    {
+        return $this->getPeriod()->getDateTimeEnd()->setTimezone($this->getSite()->getTimezone());
+    }
+
+    /**
+     * @return bool
+     */
+    public function isSingleSiteDayArchive()
+    {
+        return $this->isDayArchive() && $this->isSingleSite();
+    }
+
+    /**
+     * @return bool
+     */
+    public function isDayArchive()
+    {
+        $period = $this->getPeriod();
+        $secondsInPeriod = $period->getDateEnd()->getTimestampUTC() - $period->getDateStart()->getTimestampUTC();
+        $oneDay = $secondsInPeriod < Date::NUM_SECONDS_IN_DAY;
+
+        return $oneDay;
+    }
+
+    public function isSingleSite()
+    {
+        return count($this->getIdSites()) == 1;
+    }
+
+    public function logStatusDebug($isTemporary)
+    {
+        $temporary = 'definitive archive';
+        if ($isTemporary) {
+            $temporary = 'temporary archive';
+        }
+        Log::debug(
+            "%s archive, idSite = %d (%s), segment '%s', report = '%s', UTC datetime [%s -> %s]",
+            $this->getPeriod()->getLabel(),
+            $this->getSite()->getId(),
+            $temporary,
+            $this->getSegment()->getString(),
+            $this->getRequestedPlugin(),
+            $this->getDateStart()->getDateStartUTC(),
+            $this->getDateEnd()->getDateEndUTC()
+        );
+    }
+
+    /**
+     * Returns `true` if these parameters are part of an initial archiving request.
+     * Returns `false` if these parameters are for an archiving request that was initiated
+     * during archiving.
+     *
+     * @return bool
+     */
+    public function isRootArchiveRequest()
+    {
+        return $this->isRootArchiveRequest;
+    }
+
+    /**
+     * Sets whether these parameters are part of the initial archiving request or if they are
+     * for a request that was initiated during archiving.
+     *
+     * @param $isRootArchiveRequest
+     */
+    public function setIsRootArchiveRequest($isRootArchiveRequest)
+    {
+        $this->isRootArchiveRequest = $isRootArchiveRequest;
+    }
+}

msd2/tracking/piwik/core/ArchiveProcessor/PluginsArchiver.php

@@ -0,0 +1,318 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+
+namespace Piwik\ArchiveProcessor;
+
+use Piwik\ArchiveProcessor;
+use Piwik\Container\StaticContainer;
+use Piwik\CronArchive\Performance\Logger;
+use Piwik\DataAccess\ArchiveWriter;
+use Piwik\DataAccess\LogAggregator;
+use Piwik\DataTable\Manager;
+use Piwik\Metrics;
+use Piwik\Piwik;
+use Piwik\Plugin\Archiver;
+use Piwik\Log;
+use Piwik\Timer;
+use Exception;
+
+/**
+ * This class creates the Archiver objects found in plugins and will trigger aggregation,
+ * so each plugin can process their reports.
+ */
+class PluginsArchiver
+{
+    /**
+     * @param ArchiveProcessor $archiveProcessor
+     */
+    public $archiveProcessor;
+
+    /**
+     * @var Parameters
+     */
+    protected $params;
+
+    /**
+     * @var LogAggregator
+     */
+    private $logAggregator;
+
+    /**
+     * Public only for tests. Won't be necessary after DI changes are complete.
+     *
+     * @var Archiver[] $archivers
+     */
+    public static $archivers = array();
+
+    /**
+     * Defines if we should aggregate from raw data by using MySQL queries (when true) or aggregate archives (when false)
+     * @var bool
+     */
+    private $shouldAggregateFromRawData;
+
+    public function __construct(Parameters $params, $isTemporaryArchive, ArchiveWriter $archiveWriter = null)
+    {
+        $this->params = $params;
+        $this->isTemporaryArchive = $isTemporaryArchive;
+        $this->archiveWriter = $archiveWriter ?: new ArchiveWriter($this->params, $this->isTemporaryArchive);
+        $this->archiveWriter->initNewArchive();
+
+        $this->logAggregator = new LogAggregator($params);
+
+        $this->archiveProcessor = new ArchiveProcessor($this->params, $this->archiveWriter, $this->logAggregator);
+
+        $shouldAggregateFromRawData = $this->params->isSingleSiteDayArchive();
+
+        /**
+         * Triggered to detect if the archiver should aggregate from raw data by using MySQL queries (when true)
+         * or by aggregate archives (when false). Typically, data is aggregated from raw data for "day" period, and
+         * aggregregated from archives for all other periods.
+         *
+         * @param bool $shouldAggregateFromRawData  Set to true, to aggregate from raw data, or false to aggregate multiple reports.
+         * @param Parameters $params
+         * @ignore
+         * @deprecated
+         *
+         * In Matomo 4.0 we should maybe remove this event, and instead maybe always archive from raw data when it is daily archive,
+         * no matter if single site or not. We cannot do this in Matomo 3.X as some custom plugin archivers may not be able to handle multiple sites.
+         */
+        Piwik::postEvent('ArchiveProcessor.shouldAggregateFromRawData', array(&$shouldAggregateFromRawData, $this->params));
+
+        $this->shouldAggregateFromRawData = $shouldAggregateFromRawData;
+    }
+
+    /**
+     * If period is day, will get the core metrics (including visits) from the logs.
+     * If period is != day, will sum the core metrics from the existing archives.
+     * @return array Core metrics
+     */
+    public function callAggregateCoreMetrics()
+    {
+        $this->logAggregator->setQueryOriginHint('Core');
+
+        if ($this->shouldAggregateFromRawData) {
+            $metrics = $this->aggregateDayVisitsMetrics();
+        } else {
+            $metrics = $this->aggregateMultipleVisitsMetrics();
+        }
+
+        if (empty($metrics)) {
+            return array(
+                'nb_visits' => false,
+                'nb_visits_converted' => false
+            );
+        }
+        return array(
+            'nb_visits' => $metrics['nb_visits'],
+            'nb_visits_converted' => $metrics['nb_visits_converted']
+        );
+    }
+
+    /**
+     * Instantiates the Archiver class in each plugin that defines it,
+     * and triggers Aggregation processing on these plugins.
+     */
+    public function callAggregateAllPlugins($visits, $visitsConverted, $forceArchivingWithoutVisits = false)
+    {
+        Log::debug("PluginsArchiver::%s: Initializing archiving process for all plugins [visits = %s, visits converted = %s]",
+            __FUNCTION__, $visits, $visitsConverted);
+
+        /** @var Logger $performanceLogger */
+        $performanceLogger = StaticContainer::get(Logger::class);
+
+        $this->archiveProcessor->setNumberOfVisits($visits, $visitsConverted);
+
+        $archivers = static::getPluginArchivers();
+
+        foreach ($archivers as $pluginName => $archiverClass) {
+            // We clean up below all tables created during this function call (and recursive calls)
+            $latestUsedTableId = Manager::getInstance()->getMostRecentTableId();
+
+            /** @var Archiver $archiver */
+            $archiver = $this->makeNewArchiverObject($archiverClass, $pluginName);
+
+            if (!$archiver->isEnabled()) {
+                Log::debug("PluginsArchiver::%s: Skipping archiving for plugin '%s' (disabled).", __FUNCTION__, $pluginName);
+                continue;
+            }
+
+            if (!$forceArchivingWithoutVisits && !$visits && !$archiver->shouldRunEvenWhenNoVisits()) {
+                Log::debug("PluginsArchiver::%s: Skipping archiving for plugin '%s' (no visits).", __FUNCTION__, $pluginName);
+                continue;
+            }
+
+            if ($this->shouldProcessReportsForPlugin($pluginName)) {
+
+                $this->logAggregator->setQueryOriginHint($pluginName);
+
+                try {
+                    $timer = new Timer();
+                    if ($this->shouldAggregateFromRawData) {
+                        Log::debug("PluginsArchiver::%s: Archiving day reports for plugin '%s'.", __FUNCTION__, $pluginName);
+
+                        $archiver->callAggregateDayReport();
+                    } else {
+                        Log::debug("PluginsArchiver::%s: Archiving period reports for plugin '%s'.", __FUNCTION__, $pluginName);
+
+                        $archiver->callAggregateMultipleReports();
+                    }
+
+                    $this->logAggregator->setQueryOriginHint('');
+
+                    $performanceLogger->logMeasurement('plugin', $pluginName, $this->params, $timer);
+
+                    Log::debug("PluginsArchiver::%s: %s while archiving %s reports for plugin '%s' %s.",
+                        __FUNCTION__,
+                        $timer->getMemoryLeak(),
+                        $this->params->getPeriod()->getLabel(),
+                        $pluginName,
+                        $this->params->getSegment() ? sprintf("(for segment = '%s')", $this->params->getSegment()->getString()) : ''
+                    );
+                } catch (Exception $e) {
+                    throw new PluginsArchiverException($e->getMessage() . " - in plugin $pluginName", $e->getCode(), $e);
+                }
+            } else {
+                Log::debug("PluginsArchiver::%s: Not archiving reports for plugin '%s'.", __FUNCTION__, $pluginName);
+            }
+
+            Manager::getInstance()->deleteAll($latestUsedTableId);
+            unset($archiver);
+        }
+    }
+
+    public function finalizeArchive()
+    {
+        $this->params->logStatusDebug($this->archiveWriter->isArchiveTemporary);
+        $this->archiveWriter->finalizeArchive();
+        return $this->archiveWriter->getIdArchive();
+    }
+
+    /**
+     * Returns if any plugin archiver archives without visits
+     */
+    public static function doesAnyPluginArchiveWithoutVisits()
+    {
+        $archivers = static::getPluginArchivers();
+
+        foreach ($archivers as $pluginName => $archiverClass) {
+            if ($archiverClass::shouldRunEvenWhenNoVisits()) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    /**
+     * Loads Archiver class from any plugin that defines one.
+     *
+     * @return \Piwik\Plugin\Archiver[]
+     */
+    protected static function getPluginArchivers()
+    {
+        if (empty(static::$archivers)) {
+            $pluginNames = \Piwik\Plugin\Manager::getInstance()->getActivatedPlugins();
+            $archivers = array();
+            foreach ($pluginNames as $pluginName) {
+                $archivers[$pluginName] = self::getPluginArchiverClass($pluginName);
+            }
+            static::$archivers = array_filter($archivers);
+        }
+        return static::$archivers;
+    }
+
+    private static function getPluginArchiverClass($pluginName)
+    {
+        $klassName = 'Piwik\\Plugins\\' . $pluginName . '\\Archiver';
+        if (class_exists($klassName)
+            && is_subclass_of($klassName, 'Piwik\\Plugin\\Archiver')) {
+            return $klassName;
+        }
+        return false;
+    }
+
+    /**
+     * Whether the specified plugin's reports should be archived
+     * @param string $pluginName
+     * @return bool
+     */
+    protected function shouldProcessReportsForPlugin($pluginName)
+    {
+        if ($this->params->getRequestedPlugin() == $pluginName) {
+            return true;
+        }
+
+        if ($this->params->shouldOnlyArchiveRequestedPlugin()) {
+            return false;
+        }
+
+        if (Rules::shouldProcessReportsAllPlugins(
+            $this->params->getIdSites(),
+            $this->params->getSegment(),
+            $this->params->getPeriod()->getLabel())) {
+            return true;
+        }
+
+        if (!\Piwik\Plugin\Manager::getInstance()->isPluginLoaded($this->params->getRequestedPlugin())) {
+            return true;
+        }
+        return false;
+    }
+
+    protected function aggregateDayVisitsMetrics()
+    {
+        $query = $this->archiveProcessor->getLogAggregator()->queryVisitsByDimension();
+        $data = $query->fetch();
+
+        $metrics = $this->convertMetricsIdToName($data);
+        $this->archiveProcessor->insertNumericRecords($metrics);
+        return $metrics;
+    }
+
+    protected function convertMetricsIdToName($data)
+    {
+        $metrics = array();
+        foreach ($data as $metricId => $value) {
+            $readableMetric = Metrics::$mappingFromIdToName[$metricId];
+            $metrics[$readableMetric] = $value;
+        }
+        return $metrics;
+    }
+
+    protected function aggregateMultipleVisitsMetrics()
+    {
+        $toSum = Metrics::getVisitsMetricNames();
+        $metrics = $this->archiveProcessor->aggregateNumericMetrics($toSum);
+        return $metrics;
+    }
+
+
+    /**
+     * @param $archiverClass
+     * @return Archiver
+     */
+    private function makeNewArchiverObject($archiverClass, $pluginName)
+    {
+        $archiver = new $archiverClass($this->archiveProcessor);
+
+        /**
+         * Triggered right after a new **plugin archiver instance** is created.
+         * Subscribers to this event can configure the plugin archiver, for example prevent the archiving of a plugin's data
+         * by calling `$archiver->disable()` method.
+         *
+         * @param \Piwik\Plugin\Archiver &$archiver The newly created plugin archiver instance.
+         * @param string $pluginName The name of plugin of which archiver instance was created.
+         * @param array $this->params Array containing archive parameters (Site, Period, Date and Segment)
+         * @param bool $this->isTemporaryArchive Flag indicating whether the archive being processed is temporary (ie. the period isn't finished yet) or final (the period is already finished and in the past).
+         */
+        Piwik::postEvent('Archiving.makeNewArchiverObject', array($archiver, $pluginName, $this->params, $this->isTemporaryArchive));
+
+        return $archiver;
+    }
+}

msd2/tracking/piwik/core/ArchiveProcessor/PluginsArchiverException.php

@@ -0,0 +1,15 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+
+namespace Piwik\ArchiveProcessor;
+
+
+class PluginsArchiverException extends \Exception
+{
+}

msd2/tracking/piwik/core/ArchiveProcessor/Rules.php

@@ -0,0 +1,307 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\ArchiveProcessor;
+
+use Exception;
+use Piwik\Config;
+use Piwik\DataAccess\ArchiveWriter;
+use Piwik\Date;
+use Piwik\Log;
+use Piwik\Option;
+use Piwik\Piwik;
+use Piwik\Plugins\CoreAdminHome\Controller;
+use Piwik\Segment;
+use Piwik\SettingsPiwik;
+use Piwik\SettingsServer;
+use Piwik\Site;
+use Piwik\Tracker\Cache;
+
+/**
+ * This class contains Archiving rules/logic which are used when creating and processing Archives.
+ *
+ */
+class Rules
+{
+    const OPTION_TODAY_ARCHIVE_TTL = 'todayArchiveTimeToLive';
+
+    const OPTION_BROWSER_TRIGGER_ARCHIVING = 'enableBrowserTriggerArchiving';
+
+    const FLAG_TABLE_PURGED = 'lastPurge_';
+
+    /** Flag that will forcefully disable the archiving process (used in tests only) */
+    public static $archivingDisabledByTests = false;
+
+    /**
+     * Returns the name of the archive field used to tell the status of an archive, (ie,
+     * whether the archive was created successfully or not).
+     *
+     * @param array $idSites
+     * @param Segment $segment
+     * @param string $periodLabel
+     * @param string $plugin
+     * @return string
+     */
+    public static function getDoneStringFlagFor(array $idSites, $segment, $periodLabel, $plugin)
+    {
+        if (!self::shouldProcessReportsAllPlugins($idSites, $segment, $periodLabel)) {
+            return self::getDoneFlagArchiveContainsOnePlugin($segment, $plugin);
+        }
+        return self::getDoneFlagArchiveContainsAllPlugins($segment);
+    }
+
+    public static function shouldProcessReportsAllPlugins(array $idSites, Segment $segment, $periodLabel)
+    {
+        if ($segment->isEmpty() && $periodLabel != 'range') {
+            return true;
+        }
+
+        return self::isSegmentPreProcessed($idSites, $segment);
+    }
+
+    /**
+     * @param $idSites
+     * @return array
+     */
+    public static function getSegmentsToProcess($idSites)
+    {
+        $knownSegmentsToArchiveAllSites = SettingsPiwik::getKnownSegmentsToArchive();
+
+        $segmentsToProcess = $knownSegmentsToArchiveAllSites;
+        foreach ($idSites as $idSite) {
+            $segmentForThisWebsite = SettingsPiwik::getKnownSegmentsToArchiveForSite($idSite);
+            $segmentsToProcess = array_merge($segmentsToProcess, $segmentForThisWebsite);
+        }
+        $segmentsToProcess = array_unique($segmentsToProcess);
+        return $segmentsToProcess;
+    }
+
+    public static function getDoneFlagArchiveContainsOnePlugin(Segment $segment, $plugin)
+    {
+        return 'done' . $segment->getHash() . '.' . $plugin ;
+    }
+
+    public static function getDoneFlagArchiveContainsAllPlugins(Segment $segment)
+    {
+        return 'done' . $segment->getHash();
+    }
+
+    /**
+     * Return done flags used to tell how the archiving process for a specific archive was completed,
+     *
+     * @param array $plugins
+     * @param $segment
+     * @return array
+     */
+    public static function getDoneFlags(array $plugins, Segment $segment)
+    {
+        $doneFlags = array();
+        $doneAllPlugins = self::getDoneFlagArchiveContainsAllPlugins($segment);
+        $doneFlags[$doneAllPlugins] = $doneAllPlugins;
+
+        $plugins = array_unique($plugins);
+        foreach ($plugins as $plugin) {
+            $doneOnePlugin = self::getDoneFlagArchiveContainsOnePlugin($segment, $plugin);
+            $doneFlags[$plugin] = $doneOnePlugin;
+        }
+        return $doneFlags;
+    }
+
+    public static function getMinTimeProcessedForTemporaryArchive(
+        Date $dateStart, \Piwik\Period $period, Segment $segment, Site $site)
+    {
+        $todayArchiveTimeToLive = self::getPeriodArchiveTimeToLiveDefault($period->getLabel());
+
+        $now = time();
+        $minimumArchiveTime = $now - $todayArchiveTimeToLive;
+
+        $idSites = array($site->getId());
+        $isArchivingDisabled = Rules::isArchivingDisabledFor($idSites, $segment, $period->getLabel());
+        if ($isArchivingDisabled) {
+            if ($period->getNumberOfSubperiods() == 0
+                && $dateStart->getTimestamp() <= $now
+            ) {
+                // Today: accept any recent enough archive
+                $minimumArchiveTime = false;
+            } else {
+                // This week, this month, this year:
+                // accept any archive that was processed today after 00:00:01 this morning
+                $timezone = $site->getTimezone();
+                $minimumArchiveTime = Date::factory(Date::factory('now', $timezone)->getDateStartUTC())->setTimezone($timezone)->getTimestamp();
+            }
+        }
+        return $minimumArchiveTime;
+    }
+
+    public static function setTodayArchiveTimeToLive($timeToLiveSeconds)
+    {
+        $timeToLiveSeconds = (int)$timeToLiveSeconds;
+        if ($timeToLiveSeconds <= 0) {
+            throw new Exception(Piwik::translate('General_ExceptionInvalidArchiveTimeToLive'));
+        }
+        Option::set(self::OPTION_TODAY_ARCHIVE_TTL, $timeToLiveSeconds, $autoLoad = true);
+    }
+
+    public static function getTodayArchiveTimeToLive()
+    {
+        $uiSettingIsEnabled = Controller::isGeneralSettingsAdminEnabled();
+
+        if ($uiSettingIsEnabled) {
+            $timeToLive = Option::get(self::OPTION_TODAY_ARCHIVE_TTL);
+            if ($timeToLive !== false) {
+                return $timeToLive;
+            }
+        }
+        return self::getTodayArchiveTimeToLiveDefault();
+    }
+
+    public static function getPeriodArchiveTimeToLiveDefault($periodLabel)
+    {
+        if (empty($periodLabel) || strtolower($periodLabel) === 'day') {
+            return self::getTodayArchiveTimeToLive();
+        }
+
+        $config = Config::getInstance();
+        $general = $config->General;
+
+        $key = sprintf('time_before_%s_archive_considered_outdated', $periodLabel);
+        if (isset($general[$key]) && is_numeric($general[$key]) && $general[$key] > 0) {
+            return $general[$key];
+        }
+
+        return self::getTodayArchiveTimeToLive();
+    }
+
+    public static function getTodayArchiveTimeToLiveDefault()
+    {
+        return Config::getInstance()->General['time_before_today_archive_considered_outdated'];
+    }
+
+    public static function isBrowserArchivingAvailableForSegments()
+    {
+        $generalConfig = Config::getInstance()->General;
+        return !$generalConfig['browser_archiving_disabled_enforce'];
+    }
+
+    public static function isArchivingDisabledFor(array $idSites, Segment $segment, $periodLabel)
+    {
+        $generalConfig = Config::getInstance()->General;
+
+        if ($periodLabel == 'range') {
+            if (!isset($generalConfig['archiving_range_force_on_browser_request'])
+                || $generalConfig['archiving_range_force_on_browser_request'] != false
+            ) {
+                return false;
+            }
+
+            Log::debug("Not forcing archiving for range period.");
+            $processOneReportOnly = false;
+
+        } else {
+            $processOneReportOnly = !self::shouldProcessReportsAllPlugins($idSites, $segment, $periodLabel);
+        }
+
+        $isArchivingEnabled = self::isRequestAuthorizedToArchive() && !self::$archivingDisabledByTests;
+
+        if ($processOneReportOnly)  {
+            // When there is a segment, we disable archiving when browser_archiving_disabled_enforce applies
+            if (!$segment->isEmpty()
+                && !$isArchivingEnabled
+                && !self::isBrowserArchivingAvailableForSegments()
+                && !SettingsServer::isArchivePhpTriggered() // Only applies when we are not running core:archive command
+            ) {
+                Log::debug("Archiving is disabled because of config setting browser_archiving_disabled_enforce=1");
+                return true;
+            }
+
+            // Always allow processing one report
+            return false;
+        }
+
+        return !$isArchivingEnabled;
+    }
+
+    public static function isRequestAuthorizedToArchive()
+    {
+        return Rules::isBrowserTriggerEnabled() || SettingsServer::isArchivePhpTriggered();
+    }
+
+    public static function isBrowserTriggerEnabled()
+    {
+        $uiSettingIsEnabled = Controller::isGeneralSettingsAdminEnabled();
+
+        if ($uiSettingIsEnabled) {
+            $browserArchivingEnabled = Option::get(self::OPTION_BROWSER_TRIGGER_ARCHIVING);
+            if ($browserArchivingEnabled !== false) {
+                return (bool)$browserArchivingEnabled;
+            }
+        }
+        return (bool)Config::getInstance()->General['enable_browser_archiving_triggering'];
+    }
+
+    public static function setBrowserTriggerArchiving($enabled)
+    {
+        if (!is_bool($enabled)) {
+            throw new Exception('Browser trigger archiving must be set to true or false.');
+        }
+        Option::set(self::OPTION_BROWSER_TRIGGER_ARCHIVING, (int)$enabled, $autoLoad = true);
+        Cache::clearCacheGeneral();
+    }
+
+    /**
+     * Returns true if the archiving process should skip the calculation of unique visitors
+     * across several sites. The `[General] enable_processing_unique_visitors_multiple_sites`
+     * INI config option controls the value of this variable.
+     *
+     * @return bool
+     */
+    public static function shouldSkipUniqueVisitorsCalculationForMultipleSites()
+    {
+        return Config::getInstance()->General['enable_processing_unique_visitors_multiple_sites'] != 1;
+    }
+
+    /**
+     * @param array $idSites
+     * @param Segment $segment
+     * @return bool
+     */
+    public static function isSegmentPreProcessed(array $idSites, Segment $segment)
+    {
+        $segmentsToProcess = self::getSegmentsToProcess($idSites);
+
+        if (empty($segmentsToProcess)) {
+            return false;
+        }
+        // If the requested segment is one of the segments to pre-process
+        // we ensure that any call to the API will trigger archiving of all reports for this segment
+        $segment = $segment->getString();
+
+        // Turns out the getString() above returns the URL decoded segment string
+        $segmentsToProcessUrlDecoded = array_map('urldecode', $segmentsToProcess);
+
+        return in_array($segment, $segmentsToProcess)
+            || in_array($segment, $segmentsToProcessUrlDecoded);
+    }
+
+    /**
+     * Returns done flag values allowed to be selected
+     *
+     * @return string
+     */
+    public static function getSelectableDoneFlagValues()
+    {
+        $possibleValues = array(ArchiveWriter::DONE_OK, ArchiveWriter::DONE_OK_TEMPORARY);
+
+        if (!Rules::isRequestAuthorizedToArchive()) {
+            //If request is not authorized to archive then fetch also invalidated archives
+            $possibleValues[] = ArchiveWriter::DONE_INVALIDATED;
+        }
+
+        return $possibleValues;
+    }
+}

msd2/tracking/piwik/core/Archiver/Request.php

@@ -0,0 +1,53 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\Archiver;
+
+class Request
+{
+    /**
+     * If a request is aborted, the response of a CliMutli job will be a serialized array containing the
+     * key/value "aborted => 1".
+     */
+    const ABORT = 'abort';
+
+    /**
+     * @var string
+     */
+    private $url;
+
+    /**
+     * @var callable|null
+     */
+    private $before;
+
+    /**
+     * @param string $url
+     */
+    public function __construct($url)
+    {
+        $this->url = $url;
+    }
+
+    public function before($callable)
+    {
+        $this->before = $callable;
+    }
+
+    public function start()
+    {
+        if ($this->before) {
+            return call_user_func($this->before);
+        }
+    }
+
+    public function __toString()
+    {
+        return $this->url;
+    }
+}

msd2/tracking/piwik/core/AssetManager.php

@@ -0,0 +1,427 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik;
+
+use Exception;
+use Piwik\AssetManager\UIAsset;
+use Piwik\AssetManager\UIAsset\InMemoryUIAsset;
+use Piwik\AssetManager\UIAsset\OnDiskUIAsset;
+use Piwik\AssetManager\UIAssetCacheBuster;
+use Piwik\AssetManager\UIAssetFetcher\JScriptUIAssetFetcher;
+use Piwik\AssetManager\UIAssetFetcher\StaticUIAssetFetcher;
+use Piwik\AssetManager\UIAssetFetcher\StylesheetUIAssetFetcher;
+use Piwik\AssetManager\UIAssetFetcher;
+use Piwik\AssetManager\UIAssetMerger\JScriptUIAssetMerger;
+use Piwik\AssetManager\UIAssetMerger\StylesheetUIAssetMerger;
+use Piwik\Container\StaticContainer;
+use Piwik\Plugin\Manager;
+
+/**
+ * AssetManager is the class used to manage the inclusion of UI assets:
+ * JavaScript and CSS files.
+ *
+ * It performs the following actions:
+ *  - Identifies required assets
+ *  - Includes assets in the rendered HTML page
+ *  - Manages asset merging and minifying
+ *  - Manages server-side cache
+ *
+ * Whether assets are included individually or as merged files is defined by
+ * the global option 'disable_merged_assets'. See the documentation in the global
+ * config for more information.
+ *
+ * @method static AssetManager getInstance()
+ */
+class AssetManager extends Singleton
+{
+    const MERGED_CSS_FILE = "asset_manager_global_css.css";
+    const MERGED_CORE_JS_FILE = "asset_manager_core_js.js";
+    const MERGED_NON_CORE_JS_FILE = "asset_manager_non_core_js.js";
+
+    const CSS_IMPORT_DIRECTIVE = "<link rel=\"stylesheet\" type=\"text/css\" href=\"%s\" />\n";
+    const JS_IMPORT_DIRECTIVE = "<script type=\"text/javascript\" src=\"%s\"></script>\n";
+    const GET_CSS_MODULE_ACTION = "index.php?module=Proxy&action=getCss";
+    const GET_CORE_JS_MODULE_ACTION = "index.php?module=Proxy&action=getCoreJs";
+    const GET_NON_CORE_JS_MODULE_ACTION = "index.php?module=Proxy&action=getNonCoreJs";
+
+    /**
+     * @var UIAssetCacheBuster
+     */
+    private $cacheBuster;
+
+    /**
+     * @var UIAssetFetcher
+     */
+    private $minimalStylesheetFetcher;
+
+    /**
+     * @var Theme
+     */
+    private $theme;
+
+    public function __construct()
+    {
+        $this->cacheBuster = UIAssetCacheBuster::getInstance();
+
+        $this->minimalStylesheetFetcher = new StaticUIAssetFetcher(array(), array(), $this->theme);
+
+        $theme = Manager::getInstance()->getThemeEnabled();
+        if (!empty($theme)) {
+            $this->theme = new Theme();
+        }
+    }
+
+    /**
+     * @param UIAssetCacheBuster $cacheBuster
+     */
+    public function setCacheBuster($cacheBuster)
+    {
+        $this->cacheBuster = $cacheBuster;
+    }
+
+    /**
+     * @param UIAssetFetcher $minimalStylesheetFetcher
+     */
+    public function setMinimalStylesheetFetcher($minimalStylesheetFetcher)
+    {
+        $this->minimalStylesheetFetcher = $minimalStylesheetFetcher;
+    }
+
+    /**
+     * @param Theme $theme
+     */
+    public function setTheme($theme)
+    {
+        $this->theme = $theme;
+    }
+
+    /**
+     * Return CSS file inclusion directive(s) using the markup <link>
+     *
+     * @return string
+     */
+    public function getCssInclusionDirective()
+    {
+        return sprintf(self::CSS_IMPORT_DIRECTIVE, self::GET_CSS_MODULE_ACTION);
+    }
+
+    /**
+     * Return JS file inclusion directive(s) using the markup <script>
+     *
+     * @return string
+     */
+    public function getJsInclusionDirective()
+    {
+        $result = "<script type=\"text/javascript\">\n" . Translate::getJavascriptTranslations() . "\n</script>";
+
+        if ($this->isMergedAssetsDisabled()) {
+            $this->getMergedCoreJSAsset()->delete();
+            $this->getMergedNonCoreJSAsset()->delete();
+
+            $result .= $this->getIndividualCoreAndNonCoreJsIncludes();
+        } else {
+            $result .= sprintf(self::JS_IMPORT_DIRECTIVE, self::GET_CORE_JS_MODULE_ACTION);
+            $result .= sprintf(self::JS_IMPORT_DIRECTIVE, self::GET_NON_CORE_JS_MODULE_ACTION);
+        }
+
+        return $result;
+    }
+
+    /**
+     * Return the base.less compiled to css
+     *
+     * @return UIAsset
+     */
+    public function getCompiledBaseCss()
+    {
+        $mergedAsset = new InMemoryUIAsset();
+
+        $assetMerger = new StylesheetUIAssetMerger($mergedAsset, $this->minimalStylesheetFetcher, $this->cacheBuster);
+
+        $assetMerger->generateFile();
+
+        return $mergedAsset;
+    }
+
+    /**
+     * Return the css merged file absolute location.
+     * If there is none, the generation process will be triggered.
+     *
+     * @return UIAsset
+     */
+    public function getMergedStylesheet()
+    {
+        $mergedAsset = $this->getMergedStylesheetAsset();
+
+        $assetFetcher = new StylesheetUIAssetFetcher(Manager::getInstance()->getLoadedPluginsName(), $this->theme);
+
+        $assetMerger = new StylesheetUIAssetMerger($mergedAsset, $assetFetcher, $this->cacheBuster);
+
+        $assetMerger->generateFile();
+
+        return $mergedAsset;
+    }
+
+    /**
+     * Return the core js merged file absolute location.
+     * If there is none, the generation process will be triggered.
+     *
+     * @return UIAsset
+     */
+    public function getMergedCoreJavaScript()
+    {
+        return $this->getMergedJavascript($this->getCoreJScriptFetcher(), $this->getMergedCoreJSAsset());
+    }
+
+    /**
+     * Return the non core js merged file absolute location.
+     * If there is none, the generation process will be triggered.
+     *
+     * @return UIAsset
+     */
+    public function getMergedNonCoreJavaScript()
+    {
+        return $this->getMergedJavascript($this->getNonCoreJScriptFetcher(), $this->getMergedNonCoreJSAsset());
+    }
+
+    /**
+     * @param boolean $core
+     * @return string[]
+     */
+    public function getLoadedPlugins($core)
+    {
+        $loadedPlugins = array();
+
+        foreach (Manager::getInstance()->getPluginsLoadedAndActivated() as $plugin) {
+            $pluginName = $plugin->getPluginName();
+            $pluginIsCore = Manager::getInstance()->isPluginBundledWithCore($pluginName);
+
+            if (($pluginIsCore && $core) || (!$pluginIsCore && !$core)) {
+                $loadedPlugins[] = $pluginName;
+            }
+        }
+
+        return $loadedPlugins;
+    }
+
+    /**
+     * Remove previous merged assets
+     */
+    public function removeMergedAssets($pluginName = false)
+    {
+        $assetsToRemove = array($this->getMergedStylesheetAsset());
+
+        if ($pluginName) {
+            if ($this->pluginContainsJScriptAssets($pluginName)) {
+                if (Manager::getInstance()->isPluginBundledWithCore($pluginName)) {
+                    $assetsToRemove[] = $this->getMergedCoreJSAsset();
+                } else {
+                    $assetsToRemove[] = $this->getMergedNonCoreJSAsset();
+                }
+            }
+        } else {
+            $assetsToRemove[] = $this->getMergedCoreJSAsset();
+            $assetsToRemove[] = $this->getMergedNonCoreJSAsset();
+        }
+
+        $this->removeAssets($assetsToRemove);
+    }
+
+    /**
+     * Check if the merged file directory exists and is writable.
+     *
+     * @return string The directory location
+     * @throws Exception if directory is not writable.
+     */
+    public function getAssetDirectory()
+    {
+        $mergedFileDirectory = StaticContainer::get('path.tmp') . '/assets';
+
+        if (!is_dir($mergedFileDirectory)) {
+            Filesystem::mkdir($mergedFileDirectory);
+        }
+
+        if (!is_writable($mergedFileDirectory)) {
+            throw new Exception("Directory " . $mergedFileDirectory . " has to be writable.");
+        }
+
+        return $mergedFileDirectory;
+    }
+
+    /**
+     * Return the global option disable_merged_assets
+     *
+     * @return boolean
+     */
+    public function isMergedAssetsDisabled()
+    {
+        if (Config::getInstance()->Development['disable_merged_assets'] == 1) {
+            return true;
+        }
+        
+        if (isset($_GET['disable_merged_assets']) && $_GET['disable_merged_assets'] == 1) {
+            return true;
+        }
+        
+        return false;
+    }
+
+    /**
+     * @param UIAssetFetcher $assetFetcher
+     * @param UIAsset $mergedAsset
+     * @return UIAsset
+     */
+    private function getMergedJavascript($assetFetcher, $mergedAsset)
+    {
+        $assetMerger = new JScriptUIAssetMerger($mergedAsset, $assetFetcher, $this->cacheBuster);
+
+        $assetMerger->generateFile();
+
+        return $mergedAsset;
+    }
+
+    /**
+     * Return individual JS file inclusion directive(s) using the markup <script>
+     *
+     * @return string
+     */
+    private function getIndividualCoreAndNonCoreJsIncludes()
+    {
+        return
+            $this->getIndividualJsIncludesFromAssetFetcher($this->getCoreJScriptFetcher()) .
+            $this->getIndividualJsIncludesFromAssetFetcher($this->getNonCoreJScriptFetcher());
+    }
+
+    /**
+     * @param UIAssetFetcher $assetFetcher
+     * @return string
+     */
+    private function getIndividualJsIncludesFromAssetFetcher($assetFetcher)
+    {
+        $jsIncludeString = '';
+
+        $assets = $assetFetcher->getCatalog()->getAssets();
+
+        foreach ($assets as $jsFile) {
+            $jsFile->validateFile();
+            $jsIncludeString = $jsIncludeString . sprintf(self::JS_IMPORT_DIRECTIVE, $jsFile->getRelativeLocation());
+        }
+
+        return $jsIncludeString;
+    }
+
+    private function getCoreJScriptFetcher()
+    {
+        return new JScriptUIAssetFetcher($this->getLoadedPlugins(true), $this->theme);
+    }
+
+    private function getNonCoreJScriptFetcher()
+    {
+        return new JScriptUIAssetFetcher($this->getLoadedPlugins(false), $this->theme);
+    }
+
+    /**
+     * @param string $pluginName
+     * @return boolean
+     */
+    private function pluginContainsJScriptAssets($pluginName)
+    {
+        $fetcher = new JScriptUIAssetFetcher(array($pluginName), $this->theme);
+
+        try {
+            $assets = $fetcher->getCatalog()->getAssets();
+        } catch (\Exception $e) {
+            // This can happen when a plugin is not valid (eg. Piwik 1.x format)
+            // When posting the event to the plugin, it returns an exception "Plugin has not been loaded"
+            return false;
+        }
+
+        $pluginManager = Manager::getInstance();
+        $plugin = $pluginManager->getLoadedPlugin($pluginName);
+
+        if ($plugin->isTheme()) {
+            $theme = $pluginManager->getTheme($pluginName);
+
+            $javaScriptFiles = $theme->getJavaScriptFiles();
+
+            if (!empty($javaScriptFiles)) {
+                $assets = array_merge($assets, $javaScriptFiles);
+            }
+        }
+
+        return !empty($assets);
+    }
+
+    /**
+     * @param UIAsset[] $uiAssets
+     */
+    public function removeAssets($uiAssets)
+    {
+        foreach ($uiAssets as $uiAsset) {
+            $uiAsset->delete();
+        }
+    }
+
+    /**
+     * @return UIAsset
+     */
+    public function getMergedStylesheetAsset()
+    {
+        return $this->getMergedUIAsset(self::MERGED_CSS_FILE);
+    }
+
+    /**
+     * @return UIAsset
+     */
+    private function getMergedCoreJSAsset()
+    {
+        return $this->getMergedUIAsset(self::MERGED_CORE_JS_FILE);
+    }
+
+    /**
+     * @return UIAsset
+     */
+    private function getMergedNonCoreJSAsset()
+    {
+        return $this->getMergedUIAsset(self::MERGED_NON_CORE_JS_FILE);
+    }
+
+    /**
+     * @param string $fileName
+     * @return UIAsset
+     */
+    private function getMergedUIAsset($fileName)
+    {
+        return new OnDiskUIAsset($this->getAssetDirectory(), $fileName);
+    }
+
+    public static function compileCustomStylesheets($files)
+    {
+        $assetManager = new AssetManager();
+
+        $fetcher = new StaticUIAssetFetcher($files, $priorityOrder = array(), $theme = null);
+
+        $assetManager->setMinimalStylesheetFetcher($fetcher);
+
+        return $assetManager->getCompiledBaseCss()->getContent();
+    }
+
+    public static function compileCustomJs($files)
+    {
+        $mergedAsset = new InMemoryUIAsset();
+        $fetcher = new StaticUIAssetFetcher($files, $priorityOrder = array(), $theme = null);
+        
+        $cacheBuster = UIAssetCacheBuster::getInstance();
+
+        $assetMerger = new JScriptUIAssetMerger($mergedAsset, $fetcher, $cacheBuster);
+        $assetMerger->generateFile();
+        
+        return $mergedAsset->getContent();
+    }
+}

msd2/tracking/piwik/core/AssetManager/UIAsset.php

@@ -0,0 +1,61 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\AssetManager;
+
+use Exception;
+
+abstract class UIAsset
+{
+    abstract public function validateFile();
+
+    /**
+     * @return string
+     */
+    abstract public function getAbsoluteLocation();
+
+    /**
+     * @return string
+     */
+    abstract public function getRelativeLocation();
+
+    /**
+     * @return string
+     */
+    abstract public function getBaseDirectory();
+
+    /**
+     * Removes the previous file if it exists.
+     * Also tries to remove compressed version of the file.
+     *
+     * @see ProxyStaticFile::serveStaticFile(serveFile
+     * @throws Exception if the file couldn't be deleted
+     */
+    abstract public function delete();
+
+    /**
+     * @param string $content
+     * @throws \Exception
+     */
+    abstract public function writeContent($content);
+
+    /**
+     * @return string
+     */
+    abstract public function getContent();
+
+    /**
+     * @return boolean
+     */
+    abstract public function exists();
+
+    /**
+     * @return int
+     */
+    abstract public function getModificationDate();
+}

msd2/tracking/piwik/core/AssetManager/UIAsset/InMemoryUIAsset.php

@@ -0,0 +1,62 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\AssetManager\UIAsset;
+
+use Exception;
+use Piwik\AssetManager\UIAsset;
+
+class InMemoryUIAsset extends UIAsset
+{
+    private $content;
+
+    public function validateFile()
+    {
+        return;
+    }
+
+    public function getAbsoluteLocation()
+    {
+        throw new Exception('invalid operation');
+    }
+
+    public function getRelativeLocation()
+    {
+        throw new Exception('invalid operation');
+    }
+
+    public function getBaseDirectory()
+    {
+        throw new Exception('invalid operation');
+    }
+
+    public function delete()
+    {
+        $this->content = null;
+    }
+
+    public function exists()
+    {
+        return false;
+    }
+
+    public function writeContent($content)
+    {
+        $this->content = $content;
+    }
+
+    public function getContent()
+    {
+        return $this->content;
+    }
+
+    public function getModificationDate()
+    {
+        throw new Exception('invalid operation');
+    }
+}

msd2/tracking/piwik/core/AssetManager/UIAsset/OnDiskUIAsset.php

@@ -0,0 +1,135 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\AssetManager\UIAsset;
+
+use Exception;
+use Piwik\AssetManager\UIAsset;
+use Piwik\Common;
+use Piwik\Filesystem;
+
+class OnDiskUIAsset extends UIAsset
+{
+    /**
+     * @var string
+     */
+    private $baseDirectory;
+
+    /**
+     * @var string
+     */
+    private $relativeLocation;
+
+    /**
+     * @var string
+     */
+    private $relativeRootDir;
+
+    /**
+     * @param string $baseDirectory
+     * @param string $fileLocation
+     */
+    public function __construct($baseDirectory, $fileLocation, $relativeRootDir = '')
+    {
+        $this->baseDirectory = $baseDirectory;
+        $this->relativeLocation = $fileLocation;
+
+        if (!empty($relativeRootDir)
+            && is_string($relativeRootDir)
+            && !Common::stringEndsWith($relativeRootDir, '/')) {
+            $relativeRootDir .= '/';
+        }
+
+        $this->relativeRootDir = $relativeRootDir;
+    }
+
+    public function getAbsoluteLocation()
+    {
+        return $this->baseDirectory . '/' . $this->relativeLocation;
+    }
+
+    public function getRelativeLocation()
+    {
+        if (isset($this->relativeRootDir)) {
+            return $this->relativeRootDir . $this->relativeLocation;
+        }
+        return $this->relativeLocation;
+    }
+
+    public function getBaseDirectory()
+    {
+        return $this->baseDirectory;
+    }
+
+    public function validateFile()
+    {
+        if (!$this->assetIsReadable()) {
+            throw new Exception("The ui asset with 'href' = " . $this->getAbsoluteLocation() . " is not readable");
+        }
+    }
+
+    public function delete()
+    {
+        if ($this->exists()) {
+            try {
+                Filesystem::remove($this->getAbsoluteLocation());
+            } catch (Exception $e) {
+                throw new Exception("Unable to delete merged file : " . $this->getAbsoluteLocation() . ". Please delete the file and refresh");
+            }
+
+            // try to remove compressed version of the merged file.
+            Filesystem::remove($this->getAbsoluteLocation() . ".deflate", true);
+            Filesystem::remove($this->getAbsoluteLocation() . ".gz", true);
+        }
+    }
+
+    /**
+     * @param string $content
+     * @throws \Exception
+     */
+    public function writeContent($content)
+    {
+        $this->delete();
+
+        $newFile = @fopen($this->getAbsoluteLocation(), "w");
+
+        if (!$newFile) {
+            throw new Exception("The file : " . $newFile . " can not be opened in write mode.");
+        }
+
+        fwrite($newFile, $content);
+
+        fclose($newFile);
+    }
+
+    /**
+     * @return string
+     */
+    public function getContent()
+    {
+        return file_get_contents($this->getAbsoluteLocation());
+    }
+
+    public function exists()
+    {
+        return $this->assetIsReadable();
+    }
+
+    /**
+     * @return boolean
+     */
+    private function assetIsReadable()
+    {
+        return is_readable($this->getAbsoluteLocation());
+    }
+
+    public function getModificationDate()
+    {
+        return filemtime($this->getAbsoluteLocation());
+    }
+}

msd2/tracking/piwik/core/AssetManager/UIAssetCacheBuster.php

@@ -0,0 +1,66 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ * @method static \Piwik\AssetManager\UIAssetCacheBuster getInstance()
+ */
+namespace Piwik\AssetManager;
+
+use Piwik\Plugin\Manager;
+use Piwik\Singleton;
+use Piwik\Version;
+
+class UIAssetCacheBuster extends Singleton
+{
+    /**
+     * Cache buster based on
+     *  - Piwik version
+     *  - Loaded plugins (name and version)
+     *  - Super user salt
+     *  - Latest
+     *
+     * @param string[] $pluginNames
+     * @return string
+     */
+    public function piwikVersionBasedCacheBuster($pluginNames = false)
+    {
+        static $cachedCacheBuster = null;
+
+        if (empty($cachedCacheBuster) || $pluginNames !== false) {
+
+            $masterFile     = PIWIK_INCLUDE_PATH . '/.git/refs/heads/master';
+            $currentGitHash = file_exists($masterFile) ? @file_get_contents($masterFile) : null;
+
+            $plugins = !$pluginNames ? Manager::getInstance()->getLoadedPluginsName() : $pluginNames;
+            sort($plugins);
+
+            $pluginsInfo = '';
+            foreach ($plugins as $pluginName) {
+                $plugin      = Manager::getInstance()->getLoadedPlugin($pluginName);
+                $pluginsInfo .= $plugin->getPluginName() . $plugin->getVersion() . ',';
+            }
+
+            $cacheBuster = md5($pluginsInfo . PHP_VERSION . Version::VERSION . trim($currentGitHash));
+
+            if ($pluginNames !== false) {
+                return $cacheBuster;
+            }
+
+            $cachedCacheBuster = $cacheBuster;
+        }
+
+        return $cachedCacheBuster;
+    }
+
+    /**
+     * @param string $content
+     * @return string
+     */
+    public function md5BasedCacheBuster($content)
+    {
+        return md5($content);
+    }
+}

msd2/tracking/piwik/core/AssetManager/UIAssetCatalog.php

@@ -0,0 +1,73 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\AssetManager;
+
+class UIAssetCatalog
+{
+    /**
+     * @var UIAsset[]
+     */
+    private $uiAssets = array();
+
+    /**
+     * @var UIAssetCatalogSorter
+     */
+    private $catalogSorter;
+
+    /**
+     * @var string[]  Absolute file locations
+     */
+    private $existingAssetLocations = array();
+
+    /**
+     * @param UIAssetCatalogSorter $catalogSorter
+     */
+    public function __construct($catalogSorter)
+    {
+        $this->catalogSorter = $catalogSorter;
+    }
+
+    /**
+     * @param UIAsset $uiAsset
+     */
+    public function addUIAsset($uiAsset)
+    {
+        $location = $uiAsset->getAbsoluteLocation();
+
+        if (!$this->assetAlreadyInCatalog($location)) {
+            $this->existingAssetLocations[] = $location;
+            $this->uiAssets[] = $uiAsset;
+        }
+    }
+
+    /**
+     * @return UIAsset[]
+     */
+    public function getAssets()
+    {
+        return $this->uiAssets;
+    }
+
+    /**
+     * @return UIAssetCatalog
+     */
+    public function getSortedCatalog()
+    {
+        return $this->catalogSorter->sortUIAssetCatalog($this);
+    }
+
+    /**
+     * @param UIAsset $uiAsset
+     * @return boolean
+     */
+    private function assetAlreadyInCatalog($location)
+    {
+        return in_array($location, $this->existingAssetLocations);
+    }
+}

msd2/tracking/piwik/core/AssetManager/UIAssetCatalogSorter.php

@@ -0,0 +1,58 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\AssetManager;
+
+class UIAssetCatalogSorter
+{
+    /**
+     * @var string[]
+     */
+    private $priorityOrder;
+
+    /**
+     * @param string[] $priorityOrder
+     */
+    public function __construct($priorityOrder)
+    {
+        $this->priorityOrder = $priorityOrder;
+    }
+
+    /**
+     * @param UIAssetCatalog $uiAssetCatalog
+     * @return UIAssetCatalog
+     */
+    public function sortUIAssetCatalog($uiAssetCatalog)
+    {
+        $sortedCatalog = new UIAssetCatalog($this);
+        foreach ($this->priorityOrder as $filePattern) {
+            $assetsMatchingPattern = array_filter($uiAssetCatalog->getAssets(), function ($uiAsset) use ($filePattern) {
+                return preg_match('~^' . $filePattern . '~', $uiAsset->getRelativeLocation());
+            });
+
+            foreach ($assetsMatchingPattern as $assetMatchingPattern) {
+                $sortedCatalog->addUIAsset($assetMatchingPattern);
+            }
+        }
+
+        $this->addUnmatchedAssets($uiAssetCatalog, $sortedCatalog);
+
+        return $sortedCatalog;
+    }
+
+    /**
+     * @param UIAssetCatalog $uiAssetCatalog
+     * @param UIAssetCatalog $sortedCatalog
+     */
+    private function addUnmatchedAssets($uiAssetCatalog, $sortedCatalog)
+    {
+        foreach ($uiAssetCatalog->getAssets() as $uiAsset) {
+            $sortedCatalog->addUIAsset($uiAsset);
+        }
+    }
+}

msd2/tracking/piwik/core/AssetManager/UIAssetFetcher.php

@@ -0,0 +1,150 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\AssetManager;
+
+use Piwik\AssetManager\UIAsset\OnDiskUIAsset;
+use Piwik\Plugin\Manager;
+use Piwik\Theme;
+
+abstract class UIAssetFetcher
+{
+    /**
+     * @var UIAssetCatalog
+     */
+    protected $catalog;
+
+    /**
+     * @var string[]
+     */
+    protected $fileLocations = array();
+
+    /**
+     * @var string[]
+     */
+    protected $plugins;
+
+    /**
+     * @var Theme
+     */
+    private $theme;
+
+    /**
+     * @param string[] $plugins
+     * @param Theme $theme
+     */
+    public function __construct($plugins, $theme)
+    {
+        $this->plugins = $plugins;
+        $this->theme = $theme;
+    }
+
+    /**
+     * @return string[]
+     */
+    public function getPlugins()
+    {
+        return $this->plugins;
+    }
+
+    /**
+     * $return UIAssetCatalog
+     */
+    public function getCatalog()
+    {
+        if ($this->catalog == null) {
+            $this->createCatalog();
+        }
+
+        return $this->catalog;
+    }
+
+    abstract protected function retrieveFileLocations();
+
+    /**
+     * @return string[]
+     */
+    abstract protected function getPriorityOrder();
+
+    private function createCatalog()
+    {
+        $this->retrieveFileLocations();
+
+        $this->initCatalog();
+
+        $this->populateCatalog();
+
+        $this->sortCatalog();
+    }
+
+    private function initCatalog()
+    {
+        $catalogSorter = new UIAssetCatalogSorter($this->getPriorityOrder());
+        $this->catalog = new UIAssetCatalog($catalogSorter);
+    }
+
+    private function populateCatalog()
+    {
+        $pluginBaseDir = Manager::getPluginsDirectory();
+        $pluginWebDirectories = Manager::getAlternativeWebRootDirectories();
+        $matomoRootDir = $this->getBaseDirectory();
+
+        foreach ($this->fileLocations as $fileLocation) {
+            $fileAbsolute = $matomoRootDir . '/' . $fileLocation;
+
+            $newUIAsset = new OnDiskUIAsset($this->getBaseDirectory(), $fileLocation);
+            if ($newUIAsset->exists()) {
+                $this->catalog->addUIAsset($newUIAsset);
+                continue;
+            }
+
+            $found = false;
+
+            if (strpos($fileAbsolute, $pluginBaseDir) === 0) {
+                // we iterate over all custom plugin directories only for plugin files, not libs files (not needed there)
+                foreach ($pluginWebDirectories as $pluginDirectory => $relative) {
+                    $fileTest = str_replace($pluginBaseDir, $pluginDirectory, $fileAbsolute);
+                    $newFileRelative = str_replace($pluginDirectory, '', $fileTest);
+                    $testAsset = new OnDiskUIAsset($pluginDirectory, $newFileRelative, $relative);
+                    if ($testAsset->exists()) {
+                        $this->catalog->addUIAsset($testAsset);
+                        $found = true;
+                        break;
+                    }
+                }
+            }
+
+            if (!$found) {
+                // we add it anyway so it'll trigger an error about the missing file
+                $this->catalog->addUIAsset($newUIAsset);
+            }
+        }
+    }
+
+    private function sortCatalog()
+    {
+        $this->catalog = $this->catalog->getSortedCatalog();
+    }
+
+    /**
+     * @return string
+     */
+    private function getBaseDirectory()
+    {
+        // served by web server directly, so must be a public path
+        return PIWIK_DOCUMENT_ROOT;
+    }
+
+    /**
+     * @return Theme
+     */
+    public function getTheme()
+    {
+        return $this->theme;
+    }
+}

msd2/tracking/piwik/core/AssetManager/UIAssetFetcher/JScriptUIAssetFetcher.php

@@ -0,0 +1,88 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\AssetManager\UIAssetFetcher;
+
+use Piwik\AssetManager\UIAssetFetcher;
+use Piwik\Piwik;
+
+class JScriptUIAssetFetcher extends UIAssetFetcher
+{
+
+    protected function retrieveFileLocations()
+    {
+        if (!empty($this->plugins)) {
+
+            /**
+             * Triggered when gathering the list of all JavaScript files needed by Piwik
+             * and its plugins.
+             *
+             * Plugins that have their own JavaScript should use this event to make those
+             * files load in the browser.
+             *
+             * JavaScript files should be placed within a **javascripts** subdirectory in your
+             * plugin's root directory.
+             *
+             * _Note: While you are developing your plugin you should enable the config setting
+             * `[Development] disable_merged_assets` so JavaScript files will be reloaded immediately
+             * after every change._
+             *
+             * **Example**
+             *
+             *     public function getJsFiles(&$jsFiles)
+             *     {
+             *         $jsFiles[] = "plugins/MyPlugin/javascripts/myfile.js";
+             *         $jsFiles[] = "plugins/MyPlugin/javascripts/anotherone.js";
+             *     }
+             *
+             * @param string[] $jsFiles The JavaScript files to load.
+             */
+             Piwik::postEvent('AssetManager.getJavaScriptFiles', array(&$this->fileLocations), null, $this->plugins);
+        }
+
+        $this->addThemeFiles();
+    }
+
+    protected function addThemeFiles()
+    {
+        $theme = $this->getTheme();
+        if (!$theme) {
+            return;
+        }
+        if (in_array($theme->getThemeName(), $this->plugins)) {
+            $jsInThemes = $this->getTheme()->getJavaScriptFiles();
+
+            if (!empty($jsInThemes)) {
+                foreach ($jsInThemes as $jsFile) {
+                    $this->fileLocations[] = $jsFile;
+                }
+            }
+        }
+    }
+
+    protected function getPriorityOrder()
+    {
+        return array(
+            'libs/bower_components/jquery/dist/jquery.min.js',
+            'libs/bower_components/jquery-ui/ui/minified/jquery-ui.min.js',
+            'libs/jquery/jquery.browser.js',
+            'libs/',
+            'js/',
+            'piwik.js',
+            'matomo.js',
+            'plugins/CoreHome/javascripts/require.js',
+            'plugins/Morpheus/javascripts/piwikHelper.js',
+            'plugins/Morpheus/javascripts/',
+            'plugins/CoreHome/javascripts/uiControl.js',
+            'plugins/CoreHome/javascripts/broadcast.js',
+            'plugins/CoreHome/javascripts/', // load CoreHome JS before other plugins
+            'plugins/',
+            'tests/',
+        );
+    }
+}

msd2/tracking/piwik/core/AssetManager/UIAssetFetcher/StaticUIAssetFetcher.php

@@ -0,0 +1,36 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\AssetManager\UIAssetFetcher;
+
+use Piwik\AssetManager\UIAssetFetcher;
+
+class StaticUIAssetFetcher extends UIAssetFetcher
+{
+    /**
+     * @var string[]
+     */
+    private $priorityOrder;
+
+    public function __construct($fileLocations, $priorityOrder, $theme)
+    {
+        parent::__construct(array(), $theme);
+
+        $this->fileLocations = $fileLocations;
+        $this->priorityOrder = $priorityOrder;
+    }
+
+    protected function retrieveFileLocations()
+    {
+    }
+
+    protected function getPriorityOrder()
+    {
+        return $this->priorityOrder;
+    }
+}

msd2/tracking/piwik/core/AssetManager/UIAssetFetcher/StylesheetUIAssetFetcher.php

@@ -0,0 +1,85 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\AssetManager\UIAssetFetcher;
+
+use Piwik\AssetManager\UIAssetFetcher;
+use Piwik\Piwik;
+
+class StylesheetUIAssetFetcher extends UIAssetFetcher
+{
+    protected function getPriorityOrder()
+    {
+        $theme = $this->getTheme();
+        $themeName = $theme->getThemeName();
+
+        $order = array(
+            'plugins/Morpheus/stylesheets/base/bootstrap.css',
+            'plugins/Morpheus/stylesheets/base/icons.css',
+            'libs/',
+            'plugins/CoreHome/stylesheets/color_manager.css', // must be before other Piwik stylesheets
+            'plugins/Morpheus/stylesheets/base.less',
+        );
+
+        if ($themeName === 'Morpheus') {
+            $order[] = 'plugins\/((?!Morpheus).)*\/';
+        } else {
+            $order[] = sprintf('plugins\/((?!(Morpheus)|(%s)).)*\/', $themeName);
+        }
+
+        $order = array_merge(
+            $order,
+            array(
+                'plugins/Dashboard/stylesheets/dashboard.less',
+                'tests/',
+            )
+        );
+
+        return $order;
+    }
+
+    protected function retrieveFileLocations()
+    {
+        /**
+         * Triggered when gathering the list of all stylesheets (CSS and LESS) needed by
+         * Piwik and its plugins.
+         *
+         * Plugins that have stylesheets should use this event to make those stylesheets
+         * load.
+         *
+         * Stylesheets should be placed within a **stylesheets** subdirectory in your plugin's
+         * root directory.
+         *
+         * **Example**
+         *
+         *     public function getStylesheetFiles(&$stylesheets)
+         *     {
+         *         $stylesheets[] = "plugins/MyPlugin/stylesheets/myfile.less";
+         *         $stylesheets[] = "plugins/MyPlugin/stylesheets/myotherfile.css";
+         *     }
+         *
+         * @param string[] &$stylesheets The list of stylesheet paths.
+         */
+        Piwik::postEvent('AssetManager.getStylesheetFiles', array(&$this->fileLocations));
+
+        $this->addThemeFiles();
+    }
+
+    protected function addThemeFiles()
+    {
+        $theme = $this->getTheme();
+        if (!$theme) {
+            return;
+        }
+        $themeStylesheet = $theme->getStylesheet();
+
+        if ($themeStylesheet) {
+            $this->fileLocations[] = $themeStylesheet;
+        }
+    }
+}

msd2/tracking/piwik/core/AssetManager/UIAssetMerger.php

@@ -0,0 +1,193 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\AssetManager;
+
+
+abstract class UIAssetMerger
+{
+    /**
+     * @var UIAssetFetcher
+     */
+    private $assetFetcher;
+
+    /**
+     * @var UIAsset
+     */
+    private $mergedAsset;
+
+    /**
+     * @var string
+     */
+    protected $mergedContent;
+
+    /**
+     * @var UIAssetCacheBuster
+     */
+    protected $cacheBuster;
+
+    /**
+     * @param UIAsset $mergedAsset
+     * @param UIAssetFetcher $assetFetcher
+     * @param UIAssetCacheBuster $cacheBuster
+     */
+    public function __construct($mergedAsset, $assetFetcher, $cacheBuster)
+    {
+        $this->mergedAsset = $mergedAsset;
+        $this->assetFetcher = $assetFetcher;
+        $this->cacheBuster = $cacheBuster;
+    }
+
+    public function generateFile()
+    {
+        if (!$this->shouldGenerate()) {
+            return;
+        }
+
+        $this->mergedContent = $this->getMergedAssets();
+
+        $this->postEvent($this->mergedContent);
+
+        $this->adjustPaths();
+
+        $this->addPreamble();
+
+        $this->writeContentToFile();
+    }
+
+    /**
+     * @return string
+     */
+    abstract protected function getMergedAssets();
+
+    /**
+     * @return string
+     */
+    abstract protected function generateCacheBuster();
+
+    /**
+     * @return string
+     */
+    abstract protected function getPreamble();
+
+    /**
+     * @return string
+     */
+    abstract protected function getFileSeparator();
+
+    /**
+     * @param UIAsset $uiAsset
+     * @return string
+     */
+    abstract protected function processFileContent($uiAsset);
+
+    /**
+     * @param string $mergedContent
+     */
+    abstract protected function postEvent(&$mergedContent);
+
+    protected function getConcatenatedAssets()
+    {
+        if (empty($this->mergedContent)) {
+            $this->concatenateAssets();
+        }
+
+        return $this->mergedContent;
+    }
+
+    protected function concatenateAssets()
+    {
+        $mergedContent = '';
+
+        foreach ($this->getAssetCatalog()->getAssets() as $uiAsset) {
+            $uiAsset->validateFile();
+            $content = $this->processFileContent($uiAsset);
+
+            $mergedContent .= $this->getFileSeparator() . $content;
+        }
+
+        $this->mergedContent = $mergedContent;
+    }
+
+    /**
+     * @return string[]
+     */
+    protected function getPlugins()
+    {
+        return $this->assetFetcher->getPlugins();
+    }
+
+    /**
+     * @return UIAssetCatalog
+     */
+    protected function getAssetCatalog()
+    {
+        return $this->assetFetcher->getCatalog();
+    }
+
+    /**
+     * @return boolean
+     */
+    private function shouldGenerate()
+    {
+        if (!$this->mergedAsset->exists()) {
+            return true;
+        }
+
+        return !$this->isFileUpToDate();
+    }
+
+    /**
+     * @return boolean
+     */
+    private function isFileUpToDate()
+    {
+        $f = fopen($this->mergedAsset->getAbsoluteLocation(), 'r');
+        $firstLine = fgets($f);
+        fclose($f);
+
+        if (!empty($firstLine) && trim($firstLine) == trim($this->getCacheBusterValue())) {
+            return true;
+        }
+
+        // Some CSS file in the merge, has changed since last merged asset was generated
+        // Note: we do not detect changes in @import'ed LESS files
+        return false;
+    }
+
+    private function adjustPaths()
+    {
+        $theme = $this->assetFetcher->getTheme();
+        // During installation theme is not yet ready
+        if ($theme) {
+            $this->mergedContent = $this->assetFetcher->getTheme()->rewriteAssetsPathToTheme($this->mergedContent);
+        }
+    }
+
+    private function writeContentToFile()
+    {
+        $this->mergedAsset->writeContent($this->mergedContent);
+    }
+
+    /**
+     * @return string
+     */
+    protected function getCacheBusterValue()
+    {
+        if (empty($this->cacheBusterValue)) {
+            $this->cacheBusterValue = $this->generateCacheBuster();
+        }
+
+        return $this->cacheBusterValue;
+    }
+
+    private function addPreamble()
+    {
+        $this->mergedContent = $this->getPreamble() . $this->mergedContent;
+    }
+}

msd2/tracking/piwik/core/AssetManager/UIAssetMerger/JScriptUIAssetMerger.php

@@ -0,0 +1,88 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\AssetManager\UIAssetMerger;
+
+use Piwik\AssetManager\UIAsset;
+use Piwik\AssetManager\UIAssetCacheBuster;
+use Piwik\AssetManager\UIAssetFetcher\JScriptUIAssetFetcher;
+use Piwik\AssetManager\UIAssetMerger;
+use Piwik\AssetManager;
+use Piwik\AssetManager\UIAssetMinifier;
+use Piwik\Piwik;
+
+class JScriptUIAssetMerger extends UIAssetMerger
+{
+    /**
+     * @var UIAssetMinifier
+     */
+    private $assetMinifier;
+
+    /**
+     * @param UIAsset $mergedAsset
+     * @param JScriptUIAssetFetcher $assetFetcher
+     * @param UIAssetCacheBuster $cacheBuster
+     */
+    public function __construct($mergedAsset, $assetFetcher, $cacheBuster)
+    {
+        parent::__construct($mergedAsset, $assetFetcher, $cacheBuster);
+
+        $this->assetMinifier = UIAssetMinifier::getInstance();
+    }
+
+    protected function getMergedAssets()
+    {
+        return $this->getConcatenatedAssets();
+    }
+
+    protected function generateCacheBuster()
+    {
+        $cacheBuster = $this->cacheBuster->piwikVersionBasedCacheBuster($this->getPlugins());
+        return "/* Matomo Javascript - cb=" . $cacheBuster . "*/\n";
+    }
+
+    protected function getPreamble()
+    {
+        return $this->getCacheBusterValue();
+    }
+
+    protected function postEvent(&$mergedContent)
+    {
+        $plugins = $this->getPlugins();
+
+        if (!empty($plugins)) {
+
+            /**
+             * Triggered after all the JavaScript files Piwik uses are minified and merged into a
+             * single file, but before the merged JavaScript is written to disk.
+             *
+             * Plugins can use this event to modify merged JavaScript or do something else
+             * with it.
+             *
+             * @param string $mergedContent The minified and merged JavaScript.
+             */
+            Piwik::postEvent('AssetManager.filterMergedJavaScripts', array(&$mergedContent), null, $plugins);
+        }
+    }
+
+    public function getFileSeparator()
+    {
+        return "\n";
+    }
+
+    protected function processFileContent($uiAsset)
+    {
+        $content = $uiAsset->getContent();
+
+        if (!$this->assetMinifier->isMinifiedJs($content)) {
+            $content = $this->assetMinifier->minifyJs($content);
+        }
+
+        return $content;
+    }
+}

msd2/tracking/piwik/core/AssetManager/UIAssetMerger/StylesheetUIAssetMerger.php

@@ -0,0 +1,260 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\AssetManager\UIAssetMerger;
+
+use Exception;
+use lessc;
+use Piwik\AssetManager\UIAsset;
+use Piwik\AssetManager\UIAssetMerger;
+use Piwik\Common;
+use Piwik\Exception\StylesheetLessCompileException;
+use Piwik\Piwik;
+use Piwik\Plugin\Manager;
+
+class StylesheetUIAssetMerger extends UIAssetMerger
+{
+    /**
+     * @var lessc
+     */
+    private $lessCompiler;
+
+    /**
+     * @var UIAsset[]
+     */
+    private $cssAssetsToReplace = array();
+
+    public function __construct($mergedAsset, $assetFetcher, $cacheBuster)
+    {
+        parent::__construct($mergedAsset, $assetFetcher, $cacheBuster);
+
+        $this->lessCompiler = self::getLessCompiler();
+    }
+
+    protected function getMergedAssets()
+    {
+        // note: we're using setImportDir on purpose (not addImportDir)
+        $this->lessCompiler->setImportDir(PIWIK_DOCUMENT_ROOT);
+        $concatenatedAssets = $this->getConcatenatedAssets();
+
+        $this->lessCompiler->setFormatter('classic');
+        try {
+            $compiled = $this->lessCompiler->compile($concatenatedAssets);
+        } catch(\Exception $e) {
+            throw new StylesheetLessCompileException($e->getMessage());
+        }
+
+        foreach ($this->cssAssetsToReplace as $asset) {
+            // to fix #10173
+            $cssPath = $asset->getAbsoluteLocation();
+            $cssContent = $this->processFileContent($asset);
+            $compiled = str_replace($this->getCssStatementForReplacement($cssPath), $cssContent, $compiled);
+        }
+
+        $this->mergedContent = $compiled;
+        $this->cssAssetsToReplace = array();
+
+        return $compiled;
+    }
+    
+    private function getCssStatementForReplacement($path)
+    {
+        return '.nonExistingSelectorOnlyForReplacementOfCssFiles { display:"' . $path . '"; }';
+    }
+
+    protected function concatenateAssets()
+    {
+        $concatenatedContent = '';
+
+        foreach ($this->getAssetCatalog()->getAssets() as $uiAsset) {
+            $uiAsset->validateFile();
+
+            try {
+                $path = $uiAsset->getAbsoluteLocation();
+            } catch (Exception $e) {
+                $path = null;
+            }
+
+            if (!empty($path) && Common::stringEndsWith($path, '.css')) {
+                // to fix #10173
+                $concatenatedContent .= "\n" . $this->getCssStatementForReplacement($path) . "\n";
+                $this->cssAssetsToReplace[] = $uiAsset;
+            } else {
+                $content = $this->processFileContent($uiAsset);
+                $concatenatedContent .= $this->getFileSeparator() . $content;
+            }
+        }
+
+        /**
+         * Triggered after all less stylesheets are concatenated into one long string but before it is
+         * minified and merged into one file.
+         *
+         * This event can be used to add less stylesheets that are not located in a file on the disc.
+         *
+         * @param string $concatenatedContent The content of all concatenated less files.
+         */
+        Piwik::postEvent('AssetManager.addStylesheets', array(&$concatenatedContent));
+
+        $this->mergedContent = $concatenatedContent;
+    }
+    
+    /**
+     * @return lessc
+     * @throws Exception
+     */
+    private static function getLessCompiler()
+    {
+        if (!class_exists("lessc")) {
+            throw new Exception("Less was added to composer during 2.0. ==> Execute this command to update composer packages: \$ php composer.phar install");
+        }
+        $less = new lessc();
+        return $less;
+    }
+
+    protected function generateCacheBuster()
+    {
+        $fileHash = $this->cacheBuster->md5BasedCacheBuster($this->getConcatenatedAssets());
+        return "/* compile_me_once=$fileHash */";
+    }
+
+    protected function getPreamble()
+    {
+        return $this->getCacheBusterValue() . "\n"
+        . "/* Matomo CSS file is compiled with Less. You may be interested in writing a custom Theme for Matomo! */\n";
+    }
+
+    protected function postEvent(&$mergedContent)
+    {
+        /**
+         * Triggered after all less stylesheets are compiled to CSS, minified and merged into
+         * one file, but before the generated CSS is written to disk.
+         *
+         * This event can be used to modify merged CSS.
+         *
+         * @param string $mergedContent The merged and minified CSS.
+         */
+        Piwik::postEvent('AssetManager.filterMergedStylesheets', array(&$mergedContent));
+    }
+
+    public function getFileSeparator()
+    {
+        return '';
+    }
+
+    protected function processFileContent($uiAsset)
+    {
+        $pathsRewriter = $this->getCssPathsRewriter($uiAsset);
+        $content = $uiAsset->getContent();
+        $content = $this->rewriteCssImagePaths($content, $pathsRewriter);
+        $content = $this->rewriteCssImportPaths($content, $pathsRewriter);
+        return $content;
+    }
+
+    /**
+     * Rewrite CSS url() directives
+     *
+     * @param string $content
+     * @param callable $pathsRewriter
+     * @return string
+     */
+    private function rewriteCssImagePaths($content, $pathsRewriter)
+    {
+        $content = preg_replace_callback("/(url\(['\"]?)([^'\")]*)/", $pathsRewriter, $content);
+        return $content;
+    }
+
+    /**
+     * Rewrite CSS import directives
+     *
+     * @param string $content
+     * @param callable $pathsRewriter
+     * @return string
+     */
+    private function rewriteCssImportPaths($content, $pathsRewriter)
+    {
+        $content = preg_replace_callback("/(@import \")([^\")]*)/", $pathsRewriter, $content);
+        return $content;
+    }
+
+    /**
+     * Rewrite CSS url directives
+     * - rewrites paths defined relatively to their css/less definition file
+     * - rewrite windows directory separator \\ to /
+     *
+     * @param UIAsset $uiAsset
+     * @return \Closure
+     */
+    private function getCssPathsRewriter($uiAsset)
+    {
+        $baseDirectory = dirname($uiAsset->getRelativeLocation());
+        $webDirs = Manager::getAlternativeWebRootDirectories();
+
+        return function ($matches) use ($baseDirectory, $webDirs) {
+            $absolutePath = PIWIK_DOCUMENT_ROOT . "/$baseDirectory/" . $matches[2];
+
+            // Allow to import extension less file
+            if (strpos($matches[2], '.') === false) {
+                $absolutePath .= '.less';
+            }
+
+            // Prevent from rewriting full path
+            $absolutePathReal = realpath($absolutePath);
+            if ($absolutePathReal) {
+                $relativePath = $baseDirectory . "/" . $matches[2];
+                $relativePath = str_replace('\\', '/', $relativePath);
+                $publicPath   = $matches[1] . $relativePath;
+            } else {
+                foreach ($webDirs as $absPath => $relativePath) {
+                    if (strpos($baseDirectory, $relativePath) === 0) {
+                        if (strpos($matches[2], '.') === 0) {
+                            // eg ../images/ok.png
+                            $fileRelative = $baseDirectory . '/' . $matches[2];
+                            $fileAbsolute = $absPath . str_replace($relativePath, '', $fileRelative);
+                            if (file_exists($fileAbsolute)) {
+                                return $matches[1] . $fileRelative;
+                            }
+                        } elseif (strpos($matches[2], 'plugins/') === 0) {
+                            // eg plugins/Foo/images/ok.png
+                            $fileRelative = substr($matches[2], strlen('plugins/'));
+                            $fileAbsolute = $absPath . $fileRelative;
+                            if (file_exists($fileAbsolute)) {
+                                return $matches[1] . $relativePath . $fileRelative;
+                            }
+                        } elseif ($matches[1] === '@import "') {
+                            $fileRelative = $baseDirectory . '/' . $matches[2];
+                            $fileAbsolute = $absPath . str_replace($relativePath, '', $fileRelative);
+                            if (file_exists($fileAbsolute)) {
+                                return $matches[1] . $baseDirectory . '/' . $matches[2];
+                            }
+                        }
+                    }
+                }
+
+                $publicPath = $matches[1] . $matches[2];
+            }
+
+            return $publicPath;
+        };
+    }
+
+    /**
+     * @param UIAsset $uiAsset
+     * @return int
+     */
+    protected function countDirectoriesInPathToRoot($uiAsset)
+    {
+        $rootDirectory = realpath($uiAsset->getBaseDirectory());
+
+        if ($rootDirectory != PATH_SEPARATOR
+            && substr($rootDirectory, -strlen(PATH_SEPARATOR)) !== PATH_SEPARATOR) {
+            $rootDirectory .= PATH_SEPARATOR;
+        }
+        $rootDirectoryLen = strlen($rootDirectory);
+        return $rootDirectoryLen;
+    }
+}

msd2/tracking/piwik/core/AssetManager/UIAssetMinifier.php

@@ -0,0 +1,66 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ * @method static \Piwik\AssetManager\UIAssetMinifier getInstance()
+ */
+namespace Piwik\AssetManager;
+
+use Exception;
+use JShrink\Minifier;
+use Piwik\Singleton;
+
+class UIAssetMinifier extends Singleton
+{
+    const MINIFIED_JS_RATIO = 100;
+
+    protected function __construct()
+    {
+        self::validateDependency();
+        parent::__construct();
+    }
+
+    /**
+     * Indicates if the provided JavaScript content has already been minified or not.
+     * The heuristic is based on a custom ratio : (size of file) / (number of lines).
+     * The threshold (100) has been found empirically on existing files :
+     * - the ratio never exceeds 50 for non-minified content and
+     * - it never goes under 150 for minified content.
+     *
+     * @param string $content Contents of the JavaScript file
+     * @return boolean
+     */
+    public function isMinifiedJs($content)
+    {
+        $lineCount = substr_count($content, "\n");
+
+        if ($lineCount == 0) {
+            return true;
+        }
+
+        $contentSize = strlen($content);
+
+        $ratio = $contentSize / $lineCount;
+
+        return $ratio > self::MINIFIED_JS_RATIO;
+    }
+
+    /**
+     * @param string $content
+     * @return string
+     */
+    public function minifyJs($content)
+    {
+        return Minifier::minify($content);
+    }
+
+    private static function validateDependency()
+    {
+        if (!class_exists("JShrink\\Minifier")) {
+            throw new Exception("JShrink could not be found, maybe you are using Matomo from git and need to update Composer. $ php composer.phar update");
+        }
+    }
+}

msd2/tracking/piwik/core/Auth.php

@@ -0,0 +1,221 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+
+namespace Piwik;
+
+use Exception;
+
+/**
+ * Base interface for authentication implementations.
+ *
+ * Plugins that provide Auth implementations must provide a class that implements
+ * this interface. Additionally, an instance of that class must be set in the
+ * container with the 'Piwik\Auth' key during the
+ * [Request.initAuthenticationObject](http://developer.piwik.org/api-reference/events#requestinitauthenticationobject)
+ * event.
+ *
+ * Authentication implementations must support authentication via username and
+ * clear-text password and authentication via username and token auth. They can
+ * additionally support authentication via username and an MD5 hash of a password. If
+ * they don't support it, then [formless authentication](http://piwik.org/faq/how-to/faq_30/) will fail.
+ *
+ * Derived implementations should favor authenticating by password over authenticating
+ * by token auth. That is to say, if a token auth and a password are set, password
+ * authentication should be used.
+ *
+ * ### Examples
+ *
+ * **How an Auth implementation will be used**
+ *
+ *     // authenticating by password
+ *     $auth = StaticContainer::get('Piwik\Auth');
+ *     $auth->setLogin('user');
+ *     $auth->setPassword('password');
+ *     $result = $auth->authenticate();
+ *
+ *     // authenticating by token auth
+ *     $auth = StaticContainer::get('Piwik\Auth');
+ *     $auth->setLogin('user');
+ *     $auth->setTokenAuth('...');
+ *     $result = $auth->authenticate();
+ *
+ * @api
+ */
+interface Auth
+{
+    /**
+     * Must return the Authentication module's name, e.g., `"Login"`.
+     *
+     * @return string
+     */
+    public function getName();
+
+    /**
+     * Sets the authentication token to authenticate with.
+     *
+     * @param string $token_auth authentication token
+     */
+    public function setTokenAuth($token_auth);
+
+    /**
+     * Returns the login of the user being authenticated.
+     *
+     * @return string
+     */
+    public function getLogin();
+
+    /**
+     * Returns the secret used to calculate a user's token auth.
+     *
+     * A users token auth is generated using the user's login and this secret. The secret
+     * should be specific to the user and not easily guessed. Piwik's default Auth implementation
+     * uses an MD5 hash of a user's password.
+     *
+     * @return string
+     * @throws Exception if the token auth secret does not exist or cannot be obtained.
+     */
+    public function getTokenAuthSecret();
+
+    /**
+     * Sets the login name to authenticate with.
+     *
+     * @param string $login The username.
+     */
+    public function setLogin($login);
+
+    /**
+     * Sets the password to authenticate with.
+     *
+     * @param string $password Password (not hashed).
+     */
+    public function setPassword($password);
+
+    /**
+     * Sets the hash of the password to authenticate with. The hash will be an MD5 hash.
+     *
+     * @param string $passwordHash The hashed password.
+     * @throws Exception if authentication by hashed password is not supported.
+     */
+    public function setPasswordHash($passwordHash);
+
+    /**
+     * Authenticates a user using the login and password set using the setters. Can also authenticate
+     * via token auth if one is set and no password is set.
+     *
+     * Note: this method must successfully authenticate if the token auth supplied is a special hash
+     * of the user's real token auth. This is because the SessionInitializer class stores a
+     * hash of the token auth in the session cookie. You can calculate the token auth hash using the
+     * {@link Piwik\Plugins\Login\SessionInitializer::getHashTokenAuth()} method.
+     *
+     * @return AuthResult
+     * @throws Exception if the Auth implementation has an invalid state (ie, no login
+     *                   was specified). Note: implementations are not **required** to throw
+     *                   exceptions for invalid state, but they are allowed to.
+     */
+    public function authenticate();
+}
+
+/**
+ * Authentication result. This is what is returned by authentication attempts using {@link Auth}
+ * implementations.
+ *
+ * @api
+ */
+class AuthResult
+{
+    const FAILURE = 0;
+    const SUCCESS = 1;
+    const SUCCESS_SUPERUSER_AUTH_CODE = 42;
+
+    /**
+     * token_auth parameter used to authenticate in the API
+     *
+     * @var string
+     */
+    protected $tokenAuth = null;
+
+    /**
+     * The login used to authenticate.
+     *
+     * @var string
+     */
+    protected $login = null;
+
+    /**
+     * The authentication result code. Can be self::FAILURE, self::SUCCESS, or
+     * self::SUCCESS_SUPERUSER_AUTH_CODE.
+     *
+     * @var int
+     */
+    protected $code = null;
+
+    /**
+     * Constructor for AuthResult
+     *
+     * @param int $code
+     * @param string $login identity
+     * @param string $tokenAuth
+     */
+    public function __construct($code, $login, $tokenAuth)
+    {
+        $this->code = (int)$code;
+        $this->login = $login;
+        $this->tokenAuth = $tokenAuth;
+    }
+
+    /**
+     * Returns the login used to authenticate.
+     *
+     * @return string
+     */
+    public function getIdentity()
+    {
+        return $this->login;
+    }
+
+    /**
+     * Returns the token_auth to authenticate the current user in the API
+     *
+     * @return string
+     */
+    public function getTokenAuth()
+    {
+        return $this->tokenAuth;
+    }
+
+    /**
+     * Returns the authentication result code.
+     *
+     * @return int
+     */
+    public function getCode()
+    {
+        return $this->code;
+    }
+
+    /**
+     * Returns true if the user has Super User access, false otherwise.
+     *
+     * @return bool
+     */
+    public function hasSuperUserAccess()
+    {
+        return $this->getCode() == self::SUCCESS_SUPERUSER_AUTH_CODE;
+    }
+
+    /**
+     * Returns true if this result was successfully authentication.
+     *
+     * @return bool
+     */
+    public function wasAuthenticationSuccessful()
+    {
+        return $this->code > self::FAILURE;
+    }
+}

msd2/tracking/piwik/core/Auth/Password.php

@@ -0,0 +1,69 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+namespace Piwik\Auth;
+
+/**
+ * Main class to handle actions related to password hashing and verification.
+ *
+ * @api
+ */
+class Password
+{
+    /**
+     * Hashes a password with the configured algorithm.
+     *
+     * @param string $password
+     * @return string
+     */
+    public function hash($password)
+    {
+        return password_hash($password, PASSWORD_BCRYPT);
+    }
+
+    /**
+     * Returns information about a hashed password (algo, options, ...).
+     *
+     * Can be used to verify whether a string is compatible with password_hash().
+     *
+     * @param string
+     * @return array
+     */
+    public function info($hash)
+    {
+        return password_get_info($hash);
+    }
+
+    /**
+     * Rehashes a user's password if necessary.
+     *
+     * This method expects the password to be pre-hashed by
+     * \Piwik\Plugins\UsersManager\UsersManager::getPasswordHash().
+     *
+     * @param string $hash
+     * @return boolean
+     */
+    public function needsRehash($hash)
+    {
+        return password_needs_rehash($hash, PASSWORD_BCRYPT);
+    }
+
+    /**
+     * Verifies a user's password against the provided hash.
+     *
+     * This method expects the password to be pre-hashed by
+     * \Piwik\Plugins\UsersManager\UsersManager::getPasswordHash().
+     *
+     * @param string $password
+     * @param string $hash
+     * @return boolean
+     */
+    public function verify($password, $hash)
+    {
+        return password_verify($password, $hash);
+    }
+}

msd2/tracking/piwik/core/BaseFactory.php

@@ -0,0 +1,65 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik;
+
+use Exception;
+
+/**
+ * Base class for all factory types.
+ *
+ * Factory types are base classes that contain a **factory** method. This method is used to instantiate
+ * concrete instances by a specified string ID. Fatal errors do not occur if a class does not exist.
+ * Instead an exception is thrown.
+ *
+ * Derived classes should override the **getClassNameFromClassId** and **getInvalidClassIdExceptionMessage**
+ * static methods.
+ */
+abstract class BaseFactory
+{
+    /**
+     * Creates a new instance of a class using a string ID.
+     *
+     * @param string $classId The ID of the class.
+     * @return \Piwik\DataTable\Renderer
+     * @throws Exception if $classId is invalid.
+     */
+    public static function factory($classId)
+    {
+        $className = static::getClassNameFromClassId($classId);
+
+        if (!class_exists($className)) {
+            self::sendPlainHeader();
+            throw new Exception(static::getInvalidClassIdExceptionMessage($classId));
+        }
+
+        return new $className;
+    }
+
+    private static function sendPlainHeader()
+    {
+        Common::sendHeader('Content-Type: text/plain; charset=utf-8');
+    }
+
+    /**
+     * Should return a class name based on the class's associated string ID.
+     */
+    protected static function getClassNameFromClassId($id)
+    {
+        return $id;
+    }
+
+    /**
+     * Should return a message to use in an Exception when an invalid class ID is supplied to
+     * {@link factory()}.
+     */
+    protected static function getInvalidClassIdExceptionMessage($id)
+    {
+        return "Invalid class ID '$id' for " . get_called_class() . "::factory().";
+    }
+}

msd2/tracking/piwik/core/Cache.php

@@ -0,0 +1,117 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik;
+
+use Piwik\Cache\Backend;
+use Piwik\Container\StaticContainer;
+
+class Cache
+{
+
+    /**
+     * This can be considered as the default cache to use in case you don't know which one to pick. It does not support
+     * the caching of any objects though. Only boolean, numbers, strings and arrays are supported. Whenever you request
+     * an entry from the cache it will fetch the entry. Cache entries might be persisted but not necessarily. It
+     * depends on the configured backend.
+     *
+     * @return Cache\Lazy
+     */
+    public static function getLazyCache()
+    {
+        return StaticContainer::get('Piwik\Cache\Lazy');
+    }
+
+    /**
+     * This class is used to cache any data during one request. It won't be persisted between requests and it can
+     * cache all kind of data, even objects or resources. This cache is very fast.
+     *
+     * @return Cache\Transient
+     */
+    public static function getTransientCache()
+    {
+        return StaticContainer::get('Piwik\Cache\Transient');
+    }
+
+    /**
+     * This cache stores all its cache entries under one "cache" entry in a configurable backend.
+     *
+     * This comes handy for things that you need very often, nearly in every request. For example plugin metadata, the
+     * list of tracker plugins, the list of available languages, ...
+     * Instead of having to read eg. a hundred cache entries from files (or any other backend) it only loads one cache
+     * entry which contains the hundred keys. Should be used only for things that you need very often and only for
+     * cache entries that are not too large to keep loading and parsing the single cache entry fast.
+     * All cache entries it contains have the same life time. For fast performance it won't validate any cache ids.
+     * It is not possible to cache any objects using this cache.
+     *
+     * @return Cache\Eager
+     */
+    public static function getEagerCache()
+    {
+        return StaticContainer::get('Piwik\Cache\Eager');
+    }
+
+    public static function flushAll()
+    {
+        self::getLazyCache()->flushAll();
+        self::getTransientCache()->flushAll();
+        self::getEagerCache()->flushAll();
+    }
+
+    /**
+     * @param $type
+     * @return Cache\Backend
+     */
+    public static function buildBackend($type)
+    {
+        $factory = new Cache\Backend\Factory();
+        $options = self::getOptions($type);
+
+        $backend = $factory->buildBackend($type, $options);
+
+        return $backend;
+    }
+
+    private static function getOptions($type)
+    {
+        $options = self::getBackendOptions($type);
+
+        switch ($type) {
+            case 'file':
+
+                $options = array('directory' => StaticContainer::get('path.cache'));
+                break;
+
+            case 'chained':
+
+                foreach ($options['backends'] as $backend) {
+                    $options[$backend] = self::getOptions($backend);
+                }
+
+                break;
+
+            case 'redis':
+
+                if (!empty($options['timeout'])) {
+                    $options['timeout'] = (float)Common::forceDotAsSeparatorForDecimalPoint($options['timeout']);
+                }
+
+                break;
+        }
+
+        return $options;
+    }
+
+    private static function getBackendOptions($backend)
+    {
+        $key = ucfirst($backend) . 'Cache';
+        $options = Config::getInstance()->$key;
+
+        return $options;
+    }
+}

msd2/tracking/piwik/core/CacheId.php

@@ -0,0 +1,87 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik;
+
+use Piwik\Plugin\Manager;
+
+class CacheId
+{
+    public static function languageAware($cacheId)
+    {
+        return $cacheId . '-' . Translate::getLanguageLoaded();
+    }
+
+    public static function pluginAware($cacheId)
+    {
+        $pluginManager = Manager::getInstance();
+        $pluginNames   = $pluginManager->getLoadedPluginsName();
+        $cacheId       = $cacheId . '-' . md5(implode('', $pluginNames));
+        $cacheId       = self::languageAware($cacheId);
+
+        return $cacheId;
+    }
+
+    public static function siteAware($cacheId, array $idSites = null)
+    {
+        if ($idSites === null) {
+            $idSites = self::getIdSiteList('idSite');
+            $cacheId .= self::idSiteListCacheKey($idSites);
+
+            $idSites = self::getIdSiteList('idSites');
+            $cacheId .= self::idSiteListCacheKey($idSites);
+
+            $idSites = self::getIdSiteList('idsite'); // tracker param
+            $cacheId .= self::idSiteListCacheKey($idSites);
+        } else {
+            $cacheId .= self::idSiteListCacheKey($idSites);
+        }
+
+        return $cacheId;
+    }
+
+    private static function getIdSiteList($queryParamName)
+    {
+        if (empty($_GET[$queryParamName])
+            && empty($_POST[$queryParamName])
+        ) {
+            return [];
+        }
+
+        $idSiteGetParam = [];
+        if (!empty($_GET[$queryParamName])) {
+            $value = $_GET[$queryParamName];
+            $idSiteGetParam = is_array($value) ? $value : explode(',', $value);
+        }
+
+        $idSitePostParam = [];
+        if (!empty($_POST[$queryParamName])) {
+            $value = $_POST[$queryParamName];
+            $idSitePostParam = is_array($value) ? $value : explode(',', $value);
+        }
+
+        $idSiteList = array_merge($idSiteGetParam, $idSitePostParam);
+        $idSiteList = array_map('intval', $idSiteList);
+        $idSiteList = array_unique($idSiteList);
+        sort($idSiteList);
+        return $idSiteList;
+    }
+
+    private static function idSiteListCacheKey($idSites)
+    {
+        if (empty($idSites)) {
+            return '';
+        }
+
+        if (count($idSites) <= 5) {
+            return '-' . implode('_', $idSites); // we keep the cache key readable when possible
+        } else {
+            return '-' . md5(implode('_', $idSites)); // we need to shorten it
+        }
+    }
+}

msd2/tracking/piwik/core/Category/Category.php

@@ -0,0 +1,124 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+namespace Piwik\Category;
+use Piwik\Piwik;
+
+/**
+ * Base type for category. lets you change the name for a categoryId and specifiy a different order
+ * so the category appears eg at a different order in the reporting menu.
+ *
+ * This class is for now not exposed as public API until needed. Categories of plugins will be automatically
+ * displayed in the menu at the very right after all core categories.
+ */
+class Category
+{
+    /**
+     * The id of the category as specified eg in {@link Piwik\Widget\WidgetConfig::setCategoryId()`} or
+     * {@link Piwik\Report\getCategoryId()}. The id is used as the name in the menu and will be visible in the
+     * URL.
+     *
+     * @var string Should be a translation key, eg 'General_Vists'
+     */
+    protected $id = '';
+
+    /**
+     * @var Subcategory[]
+     */
+    protected $subcategories = array();
+
+    /**
+     * The order of the category. The lower the value the further left the category will appear in the menu.
+     * @var int
+     */
+    protected $order = 99;
+
+    /**
+     * The icon for this category, eg 'icon-user'
+     * @var int
+     */
+    protected $icon = '';
+
+    /**
+     * @param int $order
+     * @return static
+     */
+    public function setOrder($order)
+    {
+        $this->order = (int) $order;
+        return $this;
+    }
+
+    public function getOrder()
+    {
+        return $this->order;
+    }
+
+    public function setId($id)
+    {
+        $this->id = $id;
+        return $this;
+    }
+
+    public function getId()
+    {
+        return $this->id;
+    }
+
+    public function getDisplayName()
+    {
+        return Piwik::translate($this->getId());
+    }
+
+    public function addSubcategory(Subcategory $subcategory)
+    {
+        $subcategoryId = $subcategory->getId();
+
+        if ($this->hasSubcategory($subcategoryId)) {
+            throw new \Exception(sprintf('Subcategory %s already exists', $subcategoryId));
+        }
+
+        $this->subcategories[$subcategoryId] = $subcategory;
+    }
+
+    public function hasSubcategory($subcategoryId)
+    {
+        return isset($this->subcategories[$subcategoryId]);
+    }
+
+    public function getSubcategory($subcategoryId)
+    {
+        if ($this->hasSubcategory($subcategoryId)) {
+            return $this->subcategories[$subcategoryId];
+        }
+    }
+
+    /**
+     * @return Subcategory[]
+     */
+    public function getSubcategories()
+    {
+        return array_values($this->subcategories);
+    }
+
+    public function hasSubCategories()
+    {
+        return !empty($this->subcategories);
+    }
+
+    public function setIcon($icon)
+    {
+        $this->icon = $icon;
+        return $this;
+    }
+
+    public function getIcon()
+    {
+        return $this->icon;
+    }
+
+}

msd2/tracking/piwik/core/Category/CategoryList.php

@@ -0,0 +1,95 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+namespace Piwik\Category;
+
+use Piwik\Container\StaticContainer;
+use Piwik\Plugin;
+
+/**
+ * Base type for category. lets you change the name for a categoryId and specifiy a different order
+ * so the category appears eg at a different order in the reporting menu.
+ *
+ * This class is for now not exposed as public API until needed. Categories of plugins will be automatically
+ * displayed in the menu at the very right after all core categories.
+ */
+class CategoryList
+{
+    /**
+     * @var Category[] indexed by categoryId
+     */
+    private $categories = array();
+
+    public function addCategory(Category $category)
+    {
+        $categoryId = $category->getId();
+
+        if ($this->hasCategory($categoryId)) {
+            throw new \Exception(sprintf('Category %s already exists', $categoryId));
+        }
+
+        $this->categories[$categoryId] = $category;
+    }
+
+    public function getCategories()
+    {
+        return $this->categories;
+    }
+
+    public function hasCategory($categoryId)
+    {
+        return isset($this->categories[$categoryId]);
+    }
+
+    /**
+     * Get the category having the given id, if possible.
+     *
+     * @param string $categoryId
+     * @return Category|null
+     */
+    public function getCategory($categoryId)
+    {
+        if ($this->hasCategory($categoryId)) {
+            return $this->categories[$categoryId];
+        }
+    }
+
+    /**
+     * @return CategoryList
+     */
+    public static function get()
+    {
+        $list = new CategoryList();
+
+        $categories = StaticContainer::get('Piwik\Plugin\Categories');
+
+        foreach ($categories->getAllCategories() as $category) {
+            $list->addCategory($category);
+        }
+
+        // move subcategories into categories
+        foreach ($categories->getAllSubcategories() as $subcategory) {
+            $categoryId = $subcategory->getCategoryId();
+
+            if (!$categoryId) {
+                continue;
+            }
+
+            if ($list->hasCategory($categoryId)) {
+                $category = $list->getCategory($categoryId);
+            } else {
+                $category = new Category();
+                $category->setId($categoryId);
+                $list->addCategory($category);
+            }
+
+            $category->addSubcategory($subcategory);
+        }
+
+        return $list;
+    }
+}

msd2/tracking/piwik/core/Category/Subcategory.php

@@ -0,0 +1,146 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+namespace Piwik\Category;
+
+/**
+ * Base type for subcategories.
+ *
+ * All widgets within a subcategory will be rendered in the Piwik reporting UI under the same page. By default
+ * you do not have to specify any subcategory as they are created automatically. Only create a subcategory if you
+ * want to change the name for a specific subcategoryId or if you want to specifiy a different order so the subcategory
+ * appears eg at a different order in the reporting menu. It also affects the order of reports in
+ * `API.getReportMetadata` and wherever we display any reports.
+ *
+ * To define a subcategory just place a subclass within the `Categories` folder of your plugin.
+ *
+ * Subcategories can also be added through the {@hook Subcategory.addSubcategories} event.
+ *
+ * @api since Piwik 3.0.0
+ */
+class Subcategory
+{
+    /**
+     * The id of the subcategory, see eg {@link Piwik\Widget\WidgetConfig::setSubcategoryId()`} or
+     * {@link Piwik\Report\getSubcategoryId()}. The id will be used in the Piwik reporting URL and as the name
+     * in the Piwik reporting submenu. If you want to define a different URL and name, specify a {@link $name}.
+     * For example you might want to have the actual GoalId (eg '4') in the URL but the actual goal name in the
+     * submenu (eg 'Downloads'). In this case one should specify `$id=4;$name='Downloads'`.
+     *
+     * @var string eg 'General_Overview' or 'VisitTime_ByServerTimeWidgetName'.
+     */
+    protected $id = '';
+
+    /**
+     * The id of the category the subcategory belongs to, must be specified.
+     * See {@link Piwik\Widget\WidgetConfig::setCategoryId()`} or {@link Piwik\Report\getCategoryId()}.
+     *
+     * @var string A translation key eg 'General_Visits' or 'Goals_Goals'
+     */
+    protected $categoryId = '';
+
+    /**
+     * The name that shall be used in the menu etc, defaults to the specified {@link $id}. See {@link $id}.
+     * @var string
+     */
+    protected $name = '';
+
+    /**
+     * The order of the subcategory. The lower the value the earlier a widget or a report will be displayed.
+     * @var int
+     */
+    protected $order = 99;
+
+    /**
+     * Sets (overwrites) the id of the subcategory see {@link $id}.
+     *
+     * @param string $id A translation key eg 'General_Overview'.
+     * @return static
+     */
+    public function setId($id)
+    {
+        $this->id = $id;
+        return $this;
+    }
+
+    /**
+     * Get the id of the subcategory.
+     * @return string
+     */
+    public function getId()
+    {
+        return $this->id;
+    }
+
+    /**
+     * Get the specifed categoryId see {@link $categoryId}.
+     *
+     * @return string
+     */
+    public function getCategoryId()
+    {
+        return $this->categoryId;
+    }
+
+    /**
+     * Sets (overwrites) the categoryId see {@link $categoryId}.
+     *
+     * @param string $categoryId
+     * @return static
+     */
+    public function setCategoryId($categoryId)
+    {
+        $this->categoryId = $categoryId;
+        return $this;
+    }
+
+    /**
+     * Sets (overwrites) the name see {@link $name} and {@link $id}.
+     *
+     * @param string $name A translation key eg 'General_Overview'.
+     * @return static
+     */
+    public function setName($name)
+    {
+        $this->name = $name;
+        return $this;
+    }
+
+    /**
+     * Get the name of the subcategory.
+     * @return string
+     */
+    public function getName()
+    {
+        if (!empty($this->name)) {
+            return $this->name;
+        }
+
+        return $this->id;
+    }
+
+    /**
+     * Sets (overwrites) the order see {@link $order}.
+     *
+     * @param int $order
+     * @return static
+     */
+    public function setOrder($order)
+    {
+        $this->order = (int) $order;
+        return $this;
+    }
+
+    /**
+     * Get the order of the subcategory.
+     * @return int
+     */
+    public function getOrder()
+    {
+        return $this->order;
+    }
+}

msd2/tracking/piwik/core/CliMulti.php

@@ -0,0 +1,455 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+namespace Piwik;
+
+use Piwik\Archiver\Request;
+use Piwik\CliMulti\CliPhp;
+use Piwik\CliMulti\Output;
+use Piwik\CliMulti\Process;
+use Piwik\Container\StaticContainer;
+
+/**
+ * Class CliMulti.
+ */
+class CliMulti
+{
+    const BASE_WAIT_TIME = 250000; // 250 * 1000 = 250ms
+
+    /**
+     * If set to true or false it will overwrite whether async is supported or not.
+     *
+     * @var null|bool
+     */
+    public $supportsAsync = null;
+
+    /**
+     * @var Process[]
+     */
+    private $processes = array();
+
+    /**
+     * If set it will issue at most concurrentProcessesLimit requests
+     * @var int
+     */
+    private $concurrentProcessesLimit = null;
+
+    /**
+     * @var Output[]
+     */
+    private $outputs = array();
+
+    private $acceptInvalidSSLCertificate = false;
+
+    /**
+     * @var bool
+     */
+    private $runAsSuperUser = false;
+
+    /**
+     * Only used when doing synchronous curl requests.
+     *
+     * @var string
+     */
+    private $urlToPiwik = null;
+
+    private $phpCliOptions = '';
+
+    /**
+     * @var callable
+     */
+    private $onProcessFinish = null;
+
+    public function __construct()
+    {
+        $this->supportsAsync = $this->supportsAsync();
+    }
+
+    /**
+     * It will request all given URLs in parallel (async) using the CLI and wait until all requests are finished.
+     * If multi cli is not supported (eg windows) it will initiate an HTTP request instead (not async).
+     *
+     * @param string[]  $piwikUrls   An array of urls, for instance:
+     *
+     *                               `array('http://www.example.com/piwik?module=API...')`
+     *
+     *                               **Make sure query parameter values are properly encoded in the URLs.**
+     *
+     * @return array The response of each URL in the same order as the URLs. The array can contain null values in case
+     *               there was a problem with a request, for instance if the process died unexpected.
+     */
+    public function request(array $piwikUrls)
+    {
+        $chunks = array($piwikUrls);
+        if ($this->concurrentProcessesLimit) {
+            $chunks = array_chunk($piwikUrls, $this->concurrentProcessesLimit);
+        }
+
+        $results = array();
+        foreach ($chunks as $urlsChunk) {
+            $results = array_merge($results, $this->requestUrls($urlsChunk));
+        }
+
+        return $results;
+    }
+
+    /**
+     * Forwards the given configuration options to the PHP cli command.
+     * @param string $phpCliOptions  eg "-d memory_limit=8G -c=path/to/php.ini"
+     */
+    public function setPhpCliConfigurationOptions($phpCliOptions)
+    {
+        $this->phpCliOptions = (string) $phpCliOptions;
+    }
+
+    /**
+     * Ok, this sounds weird. Why should we care about ssl certificates when we are in CLI mode? It is needed for
+     * our simple fallback mode for Windows where we initiate HTTP requests instead of CLI.
+     * @param $acceptInvalidSSLCertificate
+     */
+    public function setAcceptInvalidSSLCertificate($acceptInvalidSSLCertificate)
+    {
+        $this->acceptInvalidSSLCertificate = $acceptInvalidSSLCertificate;
+    }
+
+    /**
+     * @param $limit int Maximum count of requests to issue in parallel
+     */
+    public function setConcurrentProcessesLimit($limit)
+    {
+        $this->concurrentProcessesLimit = $limit;
+    }
+
+    public function runAsSuperUser($runAsSuperUser = true)
+    {
+        $this->runAsSuperUser = $runAsSuperUser;
+    }
+
+    private function start($piwikUrls)
+    {
+        foreach ($piwikUrls as $index => $url) {
+            $shouldStart = null;
+            if ($url instanceof Request) {
+                $shouldStart = $url->start();
+            }
+
+            $cmdId = $this->generateCommandId($url) . $index;
+
+            if ($shouldStart === Request::ABORT) {
+                // output is needed to ensure same order of url to response
+                $output = new Output($cmdId);
+                $output->write(serialize(array('aborted' => '1')));
+                $this->outputs[] = $output;
+            } else {
+                $this->executeUrlCommand($cmdId, $url);
+            }
+        }
+    }
+
+    private function executeUrlCommand($cmdId, $url)
+    {
+        $output = new Output($cmdId);
+
+        if ($this->supportsAsync) {
+            $this->executeAsyncCli($url, $output, $cmdId);
+        } else {
+            $this->executeNotAsyncHttp($url, $output);
+        }
+
+        $this->outputs[] = $output;
+    }
+
+    private function buildCommand($hostname, $query, $outputFile, $doEsacpeArg = true)
+    {
+        $bin = $this->findPhpBinary();
+        $superuserCommand = $this->runAsSuperUser ? "--superuser" : "";
+
+        if ($doEsacpeArg) {
+            $hostname = escapeshellarg($hostname);
+            $query = escapeshellarg($query);
+        }
+
+        return sprintf('%s %s %s/console climulti:request -q --matomo-domain=%s %s %s > %s 2>&1 &',
+                       $bin, $this->phpCliOptions, PIWIK_INCLUDE_PATH, $hostname, $superuserCommand, $query, $outputFile);
+    }
+
+    private function getResponse()
+    {
+        $response = array();
+
+        foreach ($this->outputs as $output) {
+            $response[] = $output->get();
+        }
+
+        return $response;
+    }
+
+    private function hasFinished()
+    {
+        foreach ($this->processes as $index => $process) {
+            $hasStarted = $process->hasStarted();
+
+            if (!$hasStarted && 8 <= $process->getSecondsSinceCreation()) {
+                // if process was created more than 8 seconds ago but still not started there must be something wrong.
+                // ==> declare the process as finished
+                $process->finishProcess();
+                continue;
+            } elseif (!$hasStarted) {
+                return false;
+            }
+
+            if ($process->isRunning()) {
+                return false;
+            }
+
+            $pid = $process->getPid();
+            foreach ($this->outputs as $output) {
+                if ($output->getOutputId() === $pid && $output->isAbnormal()) {
+                    $process->finishProcess();
+                    return true;
+                }
+            }
+
+            if ($process->hasFinished()) {
+                // prevent from checking this process over and over again
+                unset($this->processes[$index]);
+
+                if ($this->onProcessFinish) {
+                    $onProcessFinish = $this->onProcessFinish;
+                    $onProcessFinish($pid);
+                }
+            }
+        }
+
+        return true;
+    }
+
+    private function generateCommandId($command)
+    {
+        return substr(Common::hash($command . microtime(true) . rand(0, 99999)), 0, 100);
+    }
+
+    /**
+     * What is missing under windows? Detection whether a process is still running in Process::isProcessStillRunning
+     * and how to send a process into background in start()
+     */
+    public function supportsAsync()
+    {
+        return Process::isSupported() && !Common::isPhpCgiType() && $this->findPhpBinary();
+    }
+
+    private function findPhpBinary()
+    {
+        $cliPhp = new CliPhp();
+        return $cliPhp->findPhpBinary();
+    }
+
+    private function cleanup()
+    {
+        foreach ($this->processes as $pid) {
+            $pid->finishProcess();
+        }
+
+        foreach ($this->outputs as $output) {
+            $output->destroy();
+        }
+
+        $this->processes = array();
+        $this->outputs   = array();
+    }
+
+    /**
+     * Remove files older than one week. They should be cleaned up automatically after each request but for whatever
+     * reason there can be always some files left.
+     */
+    public static function cleanupNotRemovedFiles()
+    {
+        $timeOneWeekAgo = strtotime('-1 week');
+
+        $files = _glob(self::getTmpPath() . '/*');
+        if (empty($files)) {
+            return;
+        }
+
+        foreach ($files as $file) {
+            if (file_exists($file)) {
+                $timeLastModified = filemtime($file);
+
+                if ($timeLastModified !== false && $timeOneWeekAgo > $timeLastModified) {
+                    unlink($file);
+                }
+            }
+        }
+    }
+
+    public static function getTmpPath()
+    {
+        return StaticContainer::get('path.tmp') . '/climulti';
+    }
+
+    public function isCommandAlreadyRunning($url)
+    {
+        if (defined('PIWIK_TEST_MODE')) {
+            return false; // skip check in tests as it might result in random failures
+        }
+
+        if (!$this->supportsAsync) {
+            // we cannot detect if web archive is still running
+            return false;
+        }
+
+        $query = UrlHelper::getQueryFromUrl($url, array('pid' => 'removeme'));
+        $hostname = Url::getHost($checkIfTrusted = false);
+        $commandToCheck = $this->buildCommand($hostname, $query, $output = '', $escape = false);
+
+        $currentlyRunningJobs = `ps aux`;
+
+        $posStart = strpos($commandToCheck, 'console climulti');
+        $posPid = strpos($commandToCheck, '&pid='); // the pid is random each time so we need to ignore it.
+        $shortendCommand = substr($commandToCheck, $posStart, $posPid - $posStart);
+        // equals eg console climulti:request -q --matomo-domain= --superuser module=API&method=API.get&idSite=1&period=month&date=2018-04-08,2018-04-30&format=php&trigger=archivephp
+        $shortendCommand      = preg_replace("/([&])date=.*?(&|$)/", "", $shortendCommand);
+        $currentlyRunningJobs = preg_replace("/([&])date=.*?(&|$)/", "", $currentlyRunningJobs);
+
+        if (strpos($currentlyRunningJobs, $shortendCommand) !== false) {
+            Log::debug($shortendCommand . ' is already running');
+            return true;
+        }
+
+        return false;
+    }
+
+    private function executeAsyncCli($url, Output $output, $cmdId)
+    {
+        $this->processes[] = new Process($cmdId);
+
+        $url = $this->appendTestmodeParamToUrlIfNeeded($url);
+        $query = UrlHelper::getQueryFromUrl($url, array('pid' => $cmdId, 'runid' => getmypid()));
+        $hostname = Url::getHost($checkIfTrusted = false);
+        $command = $this->buildCommand($hostname, $query, $output->getPathToFile());
+
+        Log::debug($command);
+        shell_exec($command);
+    }
+
+    private function executeNotAsyncHttp($url, Output $output)
+    {
+        $piwikUrl = $this->urlToPiwik ?: SettingsPiwik::getPiwikUrl();
+        if (empty($piwikUrl)) {
+            $piwikUrl = 'http://' . Url::getHost() . '/';
+        }
+
+        $url = $piwikUrl . $url;
+        if (Config::getInstance()->General['force_ssl'] == 1) {
+            $url = str_replace("http://", "https://", $url);
+        }
+
+        if ($this->runAsSuperUser) {
+            $tokenAuths = self::getSuperUserTokenAuths();
+            $tokenAuth = reset($tokenAuths);
+
+            if (strpos($url, '?') === false) {
+                $url .= '?';
+            } else {
+                $url .= '&';
+            }
+
+            $url .= 'token_auth=' . $tokenAuth;
+        }
+
+        try {
+            Log::debug("Execute HTTP API request: "  . $url);
+            $response = Http::sendHttpRequestBy('curl', $url, $timeout = 0, $userAgent = null, $destinationPath = null, $file = null, $followDepth = 0, $acceptLanguage = false, $this->acceptInvalidSSLCertificate);
+            $output->write($response);
+        } catch (\Exception $e) {
+            $message = "Got invalid response from API request: $url. ";
+
+            if (isset($response) && empty($response)) {
+                $message .= "The response was empty. This usually means a server error. This solution to this error is generally to increase the value of 'memory_limit' in your php.ini file. Please check your Web server Error Log file for more details.";
+            } else {
+                $message .= "Response was '" . $e->getMessage() . "'";
+            }
+
+            $output->write($message);
+
+            Log::debug($e);
+        }
+    }
+
+    private function appendTestmodeParamToUrlIfNeeded($url)
+    {
+        $isTestMode = defined('PIWIK_TEST_MODE');
+
+        if ($isTestMode && false === strpos($url, '?')) {
+            $url .= "?testmode=1";
+        } elseif ($isTestMode) {
+            $url .= "&testmode=1";
+        }
+
+        return $url;
+    }
+
+    /**
+     * @param array $piwikUrls
+     * @return array
+     */
+    private function requestUrls(array $piwikUrls)
+    {
+        $this->start($piwikUrls);
+
+        $startTime = time();
+        do {
+            $elapsed = time() - $startTime;
+            $timeToWait = $this->getTimeToWaitBeforeNextCheck($elapsed);
+
+            usleep($timeToWait);
+        } while (!$this->hasFinished());
+
+        $results = $this->getResponse();
+        $this->cleanup();
+
+        self::cleanupNotRemovedFiles();
+
+        return $results;
+    }
+
+    private static function getSuperUserTokenAuths()
+    {
+        $tokens = array();
+
+        /**
+         * Used to be in CronArchive, moved to CliMulti.
+         *
+         * @ignore
+         */
+        Piwik::postEvent('CronArchive.getTokenAuth', array(&$tokens));
+
+        return $tokens;
+    }
+
+    public function setUrlToPiwik($urlToPiwik)
+    {
+        $this->urlToPiwik = $urlToPiwik;
+    }
+
+    public function onProcessFinish(callable $callback)
+    {
+        $this->onProcessFinish = $callback;
+    }
+
+    // every minute that passes adds an extra 100ms to the wait time. so 5 minutes results in 500ms extra, 20mins results in 2s extra.
+    private function getTimeToWaitBeforeNextCheck($elapsed)
+    {
+        $minutes = floor($elapsed / 60);
+        return self::BASE_WAIT_TIME + $minutes * 100000; // 100 * 1000 = 100ms
+    }
+
+    public static function isCliMultiRequest()
+    {
+        return Common::getRequestVar('pid', false) !== false;
+    }
+}

msd2/tracking/piwik/core/CliMulti/CliPhp.php

@@ -0,0 +1,101 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+namespace Piwik\CliMulti;
+
+use Piwik\Common;
+
+class CliPhp
+{
+
+    public function findPhpBinary()
+    {
+        if (defined('PHP_BINARY')) {
+            if ($this->isHhvmBinary(PHP_BINARY)) {
+                return PHP_BINARY . ' --php';
+            }
+
+            if ($this->isValidPhpType(PHP_BINARY)) {
+                return PHP_BINARY . ' -q';
+            }
+        }
+
+        $bin = '';
+
+        if (!empty($_SERVER['_']) && Common::isPhpCliMode()) {
+            $bin = $this->getPhpCommandIfValid($_SERVER['_']);
+        }
+
+        if (empty($bin) && !empty($_SERVER['argv'][0]) && Common::isPhpCliMode()) {
+            $bin = $this->getPhpCommandIfValid($_SERVER['argv'][0]);
+        }
+
+        if (!$this->isValidPhpType($bin)) {
+            $bin = shell_exec('which php');
+        }
+
+        if (!$this->isValidPhpType($bin)) {
+            $bin = shell_exec('which php5');
+        }
+
+        if (!$this->isValidPhpType($bin)) {
+            return false;
+        }
+
+        $bin = trim($bin);
+
+        if (!$this->isValidPhpVersion($bin)) {
+            return false;
+        }
+
+        $bin .= ' -q';
+
+        return $bin;
+    }
+
+    private function isHhvmBinary($bin)
+    {
+        return false !== strpos($bin, 'hhvm');
+    }
+
+    private function isValidPhpVersion($bin)
+    {
+        global $piwik_minimumPHPVersion;
+        $cliVersion = $this->getPhpVersion($bin);
+        $isCliVersionValid = version_compare($piwik_minimumPHPVersion, $cliVersion) <= 0;
+        return $isCliVersionValid;
+    }
+
+    private function isValidPhpType($path)
+    {
+        return !empty($path)
+        && false === strpos($path, 'fpm')
+        && false === strpos($path, 'cgi')
+        && false === strpos($path, 'phpunit');
+    }
+
+    private function getPhpCommandIfValid($path)
+    {
+        if (!empty($path) && is_executable($path)) {
+            if (0 === strpos($path, PHP_BINDIR) && $this->isValidPhpType($path)) {
+                return $path;
+            }
+        }
+        return null;
+    }
+
+    /**
+     * @param string $bin PHP binary
+     * @return string
+     */
+    private function getPhpVersion($bin)
+    {
+        $command = sprintf("%s -r 'echo phpversion();'", $bin);
+        $version = shell_exec($command);
+        return $version;
+    }
+}

msd2/tracking/piwik/core/CliMulti/Output.php

@@ -0,0 +1,68 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+namespace Piwik\CliMulti;
+
+use Piwik\CliMulti;
+use Piwik\Filesystem;
+
+class Output
+{
+
+    private $tmpFile  = '';
+    private $outputId = null;
+
+    public function __construct($outputId)
+    {
+        if (!Filesystem::isValidFilename($outputId)) {
+            throw new \Exception('The given output id has an invalid format');
+        }
+
+        $dir = CliMulti::getTmpPath();
+        Filesystem::mkdir($dir);
+
+        $this->tmpFile  = $dir . '/' . $outputId . '.output';
+        $this->outputId = $outputId;
+    }
+
+    public function getOutputId()
+    {
+        return $this->outputId;
+    }
+
+    public function write($content)
+    {
+        file_put_contents($this->tmpFile, $content);
+    }
+
+    public function getPathToFile()
+    {
+        return $this->tmpFile;
+    }
+
+    public function isAbnormal()
+    {
+        $size = Filesystem::getFileSize($this->tmpFile, 'MB');
+
+        return $size !== null && $size >= 100;
+    }
+
+    public function exists()
+    {
+        return file_exists($this->tmpFile);
+    }
+
+    public function get()
+    {
+        return @file_get_contents($this->tmpFile);
+    }
+
+    public function destroy()
+    {
+        Filesystem::deleteFileIfExists($this->tmpFile);
+    }
+}

msd2/tracking/piwik/core/CliMulti/Process.php

@@ -0,0 +1,266 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+namespace Piwik\CliMulti;
+
+use Piwik\CliMulti;
+use Piwik\Filesystem;
+use Piwik\SettingsServer;
+
+/**
+ * There are three different states
+ * - PID file exists with empty content: Process is created but not started
+ * - PID file exists with the actual process PID as content: Process is running
+ * - PID file does not exist: Process is marked as finished
+ *
+ * Class Process
+ */
+class Process
+{
+    private $pidFile = '';
+    private $timeCreation = null;
+    private $isSupported = null;
+    private $pid = null;
+
+    public function __construct($pid)
+    {
+        if (!Filesystem::isValidFilename($pid)) {
+            throw new \Exception('The given pid has an invalid format');
+        }
+
+        $pidDir = CliMulti::getTmpPath();
+        Filesystem::mkdir($pidDir);
+
+        $this->isSupported  = self::isSupported();
+        $this->pidFile      = $pidDir . '/' . $pid . '.pid';
+        $this->timeCreation = time();
+        $this->pid = $pid;
+
+        $this->markAsNotStarted();
+    }
+
+    public function getPid()
+    {
+        return $this->pid;
+    }
+
+    private function markAsNotStarted()
+    {
+        $content = $this->getPidFileContent();
+
+        if ($this->doesPidFileExist($content)) {
+            return;
+        }
+
+        $this->writePidFileContent('');
+    }
+
+    public function hasStarted($content = null)
+    {
+        if (is_null($content)) {
+            $content = $this->getPidFileContent();
+        }
+
+        if (!$this->doesPidFileExist($content)) {
+            // process is finished, this means there was a start before
+            return true;
+        }
+
+        if ('' === trim($content)) {
+            // pid file is overwritten by startProcess()
+            return false;
+        }
+
+        // process is probably running or pid file was not removed
+        return true;
+    }
+
+    public function hasFinished()
+    {
+        $content = $this->getPidFileContent();
+
+        return !$this->doesPidFileExist($content);
+    }
+
+    public function getSecondsSinceCreation()
+    {
+        return time() - $this->timeCreation;
+    }
+
+    public function startProcess()
+    {
+        $this->writePidFileContent(getmypid());
+    }
+
+    public function isRunning()
+    {
+        $content = $this->getPidFileContent();
+
+        if (!$this->doesPidFileExist($content)) {
+            return false;
+        }
+
+        if (!$this->pidFileSizeIsNormal()) {
+            $this->finishProcess();
+            return false;
+        }
+
+        if ($this->isProcessStillRunning($content)) {
+            return true;
+        }
+
+        if ($this->hasStarted($content)) {
+            $this->finishProcess();
+        }
+
+        return false;
+    }
+
+    private function pidFileSizeIsNormal()
+    {
+        $size = Filesystem::getFileSize($this->pidFile);
+
+        return $size !== null && $size < 500;
+    }
+
+    public function finishProcess()
+    {
+        Filesystem::deleteFileIfExists($this->pidFile);
+    }
+
+    private function doesPidFileExist($content)
+    {
+        return false !== $content;
+    }
+
+    private function isProcessStillRunning($content)
+    {
+        if (!$this->isSupported) {
+            return true;
+        }
+
+        $lockedPID   = trim($content);
+        $runningPIDs = self::getRunningProcesses();
+
+        return !empty($lockedPID) && in_array($lockedPID, $runningPIDs);
+    }
+
+    private function getPidFileContent()
+    {
+        return @file_get_contents($this->pidFile);
+    }
+
+    private function writePidFileContent($content)
+    {
+        file_put_contents($this->pidFile, $content);
+    }
+
+    public static function isSupported()
+    {
+        if (SettingsServer::isWindows()) {
+            return false;
+        }
+
+        if (self::shellExecFunctionIsDisabled()) {
+            return false;
+        }
+
+        if (self::isSystemNotSupported()) {
+            return false;
+        }
+
+        if (!self::commandExists('ps') || !self::returnsSuccessCode('ps') || !self::commandExists('awk')) {
+            return false;
+        }
+
+        if (!in_array(getmypid(), self::getRunningProcesses())) {
+            return false;
+        }
+
+        if (!self::isProcFSMounted()) {
+            return false;
+        }
+
+        return true;
+    }
+
+    private static function isSystemNotSupported()
+    {
+        $uname = @shell_exec('uname -a 2> /dev/null');
+
+        if (empty($uname)) {
+            $uname = php_uname();
+        }
+
+        if (strpos($uname, 'synology') !== false) {
+            return true;
+        }
+        return false;
+    }
+
+    private static function shellExecFunctionIsDisabled()
+    {
+        $command = 'shell_exec';
+        $disabled = explode(',', ini_get('disable_functions'));
+        $disabled = array_map('trim', $disabled);
+        return in_array($command, $disabled)  || !function_exists($command);
+    }
+
+    private static function returnsSuccessCode($command)
+    {
+        $exec = $command . ' > /dev/null 2>&1; echo $?';
+        $returnCode = shell_exec($exec);
+        $returnCode = trim($returnCode);
+        return 0 == (int) $returnCode;
+    }
+
+    private static function commandExists($command)
+    {
+        $result = @shell_exec('which ' . escapeshellarg($command) . ' 2> /dev/null');
+
+        return !empty($result);
+    }
+
+    /**
+     * ps -e requires /proc
+     * @return bool
+     */
+    private static function isProcFSMounted()
+    {
+        if (is_resource(@fopen('/proc', 'r'))) {
+            return true;
+        }
+        // Testing if /proc is a resource with @fopen fails on systems with open_basedir set.
+        // by using stat we not only test the existence of /proc but also confirm it's a 'proc' filesystem
+        $type = @shell_exec('stat -f -c "%T" /proc 2>/dev/null');
+        return strpos($type, 'proc') === 0;
+    }
+
+    public static function getListOfRunningProcesses()
+    {
+        $processes = `ps ex 2>/dev/null`;
+        if (empty($processes)) {
+            return array();
+        }
+        return explode("\n", $processes);
+    }
+
+    /**
+     * @return int[] The ids of the currently running processes
+     */
+     public static function getRunningProcesses()
+     {
+         $ids = explode("\n", trim(`ps ex 2>/dev/null | awk '! /defunct/ {print $1}' 2>/dev/null`));
+
+         $ids = array_map('intval', $ids);
+         $ids = array_filter($ids, function ($id) {
+            return $id > 0;
+        });
+
+         return $ids;
+     }
+}

msd2/tracking/piwik/core/CliMulti/RequestCommand.php

@@ -0,0 +1,129 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+
+namespace Piwik\CliMulti;
+
+use Piwik\Application\Environment;
+use Piwik\Access;
+use Piwik\Container\StaticContainer;
+use Piwik\Db;
+use Piwik\Log;
+use Piwik\Option;
+use Piwik\Plugin\ConsoleCommand;
+use Piwik\Url;
+use Piwik\UrlHelper;
+use Symfony\Component\Console\Input\InputArgument;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
+use Symfony\Component\Console\Output\OutputInterface;
+
+/**
+ * RequestCommand
+ */
+class RequestCommand extends ConsoleCommand
+{
+    /**
+     * @var Environment
+     */
+    private $environment;
+
+    protected function configure()
+    {
+        $this->setName('climulti:request');
+        $this->setDescription('Parses and executes the given query. See Piwik\CliMulti. Intended only for system usage.');
+        $this->addArgument('url-query', InputArgument::REQUIRED, 'Matomo URL query string, for instance: "module=API&method=API.getPiwikVersion&token_auth=123456789"');
+        $this->addOption('superuser', null, InputOption::VALUE_NONE, 'If supplied, runs the code as superuser.');
+    }
+
+    protected function execute(InputInterface $input, OutputInterface $output)
+    {
+        $this->recreateContainerWithWebEnvironment();
+
+        $this->initHostAndQueryString($input);
+
+        if ($this->isTestModeEnabled()) {
+            $indexFile = '/tests/PHPUnit/proxy/';
+
+            $this->resetDatabase();
+        } else {
+            $indexFile = '/';
+        }
+
+        $indexFile .= 'index.php';
+
+        if (!empty($_GET['pid'])) {
+            $process = new Process($_GET['pid']);
+
+            if ($process->hasFinished()) {
+                return;
+            }
+
+            $process->startProcess();
+        }
+
+        if ($input->getOption('superuser')) {
+            StaticContainer::addDefinitions(array(
+                'observers.global' => \DI\add(array(
+                    array('Environment.bootstrapped', function () {
+                        Access::getInstance()->setSuperUserAccess(true);
+                    })
+                )),
+            ));
+        }
+
+        require_once PIWIK_INCLUDE_PATH . $indexFile;
+
+        if (!empty($process)) {
+            $process->finishProcess();
+        }
+    }
+
+    private function isTestModeEnabled()
+    {
+        return !empty($_GET['testmode']);
+    }
+
+    /**
+     * @param InputInterface $input
+     */
+    protected function initHostAndQueryString(InputInterface $input)
+    {
+        $_GET = array();
+
+        // @todo remove piwik-domain fallback in Matomo 4
+        $hostname = $input->getOption('matomo-domain') ?: $input->getOption('piwik-domain');
+        Url::setHost($hostname);
+
+        $query = $input->getArgument('url-query');
+        $query = UrlHelper::getArrayFromQueryString($query); // NOTE: this method can create the StaticContainer now
+        foreach ($query as $name => $value) {
+            $_GET[$name] = $value;
+        }
+    }
+
+    /**
+     * We will be simulating an HTTP request here (by including index.php).
+     *
+     * To avoid weird side-effects (e.g. the logging output messing up the HTTP response on the CLI output)
+     * we need to recreate the container with the default environment instead of the CLI environment.
+     */
+    private function recreateContainerWithWebEnvironment()
+    {
+        StaticContainer::clearContainer();
+        Log::unsetInstance();
+
+        $this->environment = new Environment(null);
+        $this->environment->init();
+    }
+
+    private function resetDatabase()
+    {
+        Option::clearCache();
+        Db::destroyDatabaseObject();
+    }
+}

msd2/tracking/piwik/core/Columns/ComputedMetricFactory.php

@@ -0,0 +1,57 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\Columns;
+
+use Piwik\Piwik;
+use Piwik\Plugin\ArchivedMetric;
+use Piwik\Plugin\ComputedMetric;
+use Piwik\Plugin\Report;
+
+/**
+ * A factory to create computed metrics.
+ *
+ * @api since Piwik 3.2.0
+ */
+class ComputedMetricFactory
+{
+    /**
+     * @var MetricsList
+     */
+    private $metricsList = null;
+
+    /**
+     * Generates a new report metric factory.
+     * @param MetricsList $list A report list instance
+     * @ignore
+     */
+    public function __construct(MetricsList $list)
+    {
+        $this->metricsList = $list;
+    }
+
+    /**
+     * @return \Piwik\Plugin\ComputedMetric
+     */
+    public function createComputedMetric($metricName1, $metricName2, $aggregation)
+    {
+        $metric1 = $this->metricsList->getMetric($metricName1);
+
+        if (!$metric1 instanceof ArchivedMetric || !$metric1->getDimension()) {
+            throw new \Exception('Only possible to create computed metric for an archived metric with a dimension');
+        }
+
+        $dimension1 = $metric1->getDimension();
+
+        $metric = new ComputedMetric($metricName1, $metricName2, $aggregation);
+        $metric->setCategory($dimension1->getCategoryId());
+
+        return $metric;
+    }
+
+}

msd2/tracking/piwik/core/Columns/Dimension.php

@@ -0,0 +1,907 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\Columns;
+use Piwik\Common;
+use Piwik\Db;
+use Piwik\Piwik;
+use Piwik\Plugin;
+use Piwik\Plugin\ArchivedMetric;
+use Piwik\Plugin\ComponentFactory;
+use Piwik\Plugin\Segment;
+use Exception;
+use Piwik\CacheId;
+use Piwik\Cache as PiwikCache;
+use Piwik\Plugin\Manager as PluginManager;
+use Piwik\Metrics\Formatter;
+
+/**
+ * @api
+ * @since 3.1.0
+ */
+abstract class Dimension
+{
+    const COMPONENT_SUBNAMESPACE = 'Columns';
+
+    /**
+     * Segment type 'dimension'. Can be used along with {@link setType()}.
+     * @api
+     */
+    const TYPE_DIMENSION = 'dimension';
+    const TYPE_BINARY = 'binary';
+    const TYPE_TEXT = 'text';
+    const TYPE_ENUM = 'enum';
+    const TYPE_MONEY = 'money';
+    const TYPE_BYTE = 'byte';
+    const TYPE_DURATION_MS = 'duration_ms';
+    const TYPE_DURATION_S = 'duration_s';
+    const TYPE_NUMBER = 'number';
+    const TYPE_FLOAT = 'float';
+    const TYPE_URL = 'url';
+    const TYPE_DATE = 'date';
+    const TYPE_TIME = 'time';
+    const TYPE_DATETIME = 'datetime';
+    const TYPE_TIMESTAMP = 'timestamp';
+    const TYPE_BOOL = 'bool';
+    const TYPE_PERCENT = 'percent';
+
+    /**
+     * This will be the name of the column in the database table if a $columnType is specified.
+     * @var string
+     * @api
+     */
+    protected $columnName = '';
+
+    /**
+     * If a columnType is defined, we will create a column in the MySQL table having this type. Please make sure
+     * MySQL understands this type. Once you change the column type the Piwik platform will notify the user to
+     * perform an update which can sometimes take a long time so be careful when choosing the correct column type.
+     * @var string
+     * @api
+     */
+    protected $columnType = '';
+
+    /**
+     * Holds an array of segment instances
+     * @var Segment[]
+     */
+    protected $segments = array();
+
+    /**
+     * Defines what kind of data type this dimension holds. By default the type is auto-detected based on
+     * `$columnType` but sometimes it may be needed to correct this value. Depending on this type, a dimension will be
+     * formatted differently for example.
+     * @var string
+     * @api since Piwik 3.2.0
+     */
+    protected $type = '';
+
+    /**
+     * Translation key for name singular
+     * @var string
+     */
+    protected $nameSingular = '';
+
+    /**
+     * Translation key for name plural
+     * @var string
+     * @api since Piwik 3.2.0
+     */
+    protected $namePlural = '';
+
+    /**
+     * Translation key for category
+     * @var string
+     */
+    protected $category = '';
+
+    /**
+     * By defining a segment name a user will be able to filter their visitors by this column. If you do not want to
+     * define a segment for this dimension, simply leave the name empty.
+     * @api since Piwik 3.2.0
+     */
+    protected $segmentName = '';
+
+    /**
+     * Sets a callback which will be executed when user will call for suggested values for segment.
+     *
+     * @var callable
+     * @api since Piwik 3.2.0
+     */
+    protected $suggestedValuesCallback;
+
+    /**
+     * Here you should explain which values are accepted/useful for your segment, for example:
+     * "1, 2, 3, etc." or "comcast.net, proxad.net, etc.". If the value needs any special encoding you should mention
+     * this as well. For example "Any URL including protocol. The URL must be URL encoded."
+     *
+     * @var string
+     * @api since Piwik 3.2.0
+     */
+    protected $acceptValues;
+
+    /**
+     * Defines to which column in the MySQL database the segment belongs (if one is conifugred). Defaults to
+     * `$this.dbTableName . '.'. $this.columnName` but you can customize it eg like `HOUR(log_visit.visit_last_action_time)`.
+     *
+     * @param string $sqlSegment
+     * @api since Piwik 3.2.0
+     */
+    protected $sqlSegment;
+
+    /**
+     * Interesting when specifying a segment. Sometimes you want users to set segment values that differ from the way
+     * they are actually stored. For instance if you want to allow to filter by any URL than you might have to resolve
+     * this URL to an action id. Or a country name maybe has to be mapped to a 2 letter country code. You can do this by
+     * specifing either a callable such as `array('Classname', 'methodName')` or by passing a closure.
+     * There will be four values passed to the given closure or callable: `string $valueToMatch`, `string $segment`
+     * (see {@link setSegment()}), `string $matchType` (eg SegmentExpression::MATCH_EQUAL or any other match constant
+     * of this class) and `$segmentName`.
+     *
+     * If the closure returns NULL, then Piwik assumes the segment sub-string will not match any visitor.
+     *
+     * @var string|\Closure
+     * @api since Piwik 3.2.0
+     */
+    protected $sqlFilter;
+
+    /**
+     * Similar to {@link $sqlFilter} you can map a given segment value to another value. For instance you could map
+     * "new" to 0, 'returning' to 1 and any other value to '2'. You can either define a callable or a closure. There
+     * will be only one value passed to the closure or callable which contains the value a user has set for this
+     * segment.
+     * @var string|array
+     * @api since Piwik 3.2.0
+     */
+    protected $sqlFilterValue;
+
+    /**
+     * Defines whether this dimension (and segment based on this dimension) is available to anonymous users.
+     * @var bool
+     * @api since Piwik 3.2.0
+     */
+    protected $allowAnonymous = true;
+
+    /**
+     * The name of the database table this dimension refers to
+     * @var string
+     * @api
+     */
+    protected $dbTableName = '';
+
+    /**
+     * By default the metricId is automatically generated based on the dimensionId. This might sometimes not be as
+     * readable and quite long. If you want more expressive metric names like `nb_visits` compared to
+     * `nb_corehomevisitid`, you can eg set a metricId `visit`.
+     *
+     * @var string
+     * @api since Piwik 3.2.0
+     */
+    protected $metricId = '';
+
+    /**
+     * To be implemented when a column references another column
+     * @return Join|null
+     * @api since Piwik 3.2.0
+     */
+    public function getDbColumnJoin()
+    {
+        return null;
+    }
+
+    /**
+     * @return Discriminator|null
+     * @api since Piwik 3.2.0
+     */
+    public function getDbDiscriminator()
+    {
+        return null;
+    }
+
+    /**
+     * To be implemented when a column represents an enum.
+     * @return array
+     * @api since Piwik 3.2.0
+     */
+    public function getEnumColumnValues()
+    {
+        return array();
+    }
+
+    /**
+     * Get the metricId which is used to generate metric names based on this dimension.
+     * @return string
+     */
+    public function getMetricId()
+    {
+        if (!empty($this->metricId)) {
+            return $this->metricId;
+        }
+
+        $id = $this->getId();
+
+        return str_replace(array('.', ' ', '-'), '_', strtolower($id));
+    }
+
+    /**
+     * Installs the action dimension in case it is not installed yet. The installation is already implemented based on
+     * the {@link $columnName} and {@link $columnType}. If you want to perform additional actions beside adding the
+     * column to the database - for instance adding an index - you can overwrite this method. We recommend to call
+     * this parent method to get the minimum required actions and then add further custom actions since this makes sure
+     * the column will be installed correctly. We also recommend to change the default install behavior only if really
+     * needed. FYI: We do not directly execute those alter table statements here as we group them together with several
+     * other alter table statements do execute those changes in one step which results in a faster installation. The
+     * column will be added to the `log_link_visit_action` MySQL table.
+     *
+     * Example:
+     * ```
+    public function install()
+    {
+    $changes = parent::install();
+    $changes['log_link_visit_action'][] = "ADD INDEX index_idsite_servertime ( idsite, server_time )";
+
+    return $changes;
+    }
+    ```
+     *
+     * @return array An array containing the table name as key and an array of MySQL alter table statements that should
+     *               be executed on the given table. Example:
+     * ```
+    array(
+    'log_link_visit_action' => array("ADD COLUMN `$this->columnName` $this->columnType", "ADD INDEX ...")
+    );
+    ```
+     * @api
+     */
+    public function install()
+    {
+        if (empty($this->columnName) || empty($this->columnType) || empty($this->dbTableName)) {
+            return array();
+        }
+
+        // TODO if table does not exist, create it with a primary key, but at this point we cannot really create it
+        // cause we need to show the query in the UI first and user needs to be able to create table manually.
+        // we cannot return something like "create table " here as it would be returned for each table etc.
+        // we need to do this in column updater etc!
+
+        return array(
+            $this->dbTableName => array("ADD COLUMN `$this->columnName` $this->columnType")
+        );
+    }
+
+    /**
+     * Updates the action dimension in case the {@link $columnType} has changed. The update is already implemented based
+     * on the {@link $columnName} and {@link $columnType}. This method is intended not to overwritten by plugin
+     * developers as it is only supposed to make sure the column has the correct type. Adding additional custom "alter
+     * table" actions would not really work since they would be executed with every {@link $columnType} change. So
+     * adding an index here would be executed whenever the columnType changes resulting in an error if the index already
+     * exists. If an index needs to be added after the first version is released a plugin update class should be
+     * created since this makes sure it is only executed once.
+     *
+     * @return array An array containing the table name as key and an array of MySQL alter table statements that should
+     *               be executed on the given table. Example:
+     * ```
+    array(
+    'log_link_visit_action' => array("MODIFY COLUMN `$this->columnName` $this->columnType", "DROP COLUMN ...")
+    );
+    ```
+     * @ignore
+     */
+    public function update()
+    {
+        if (empty($this->columnName) || empty($this->columnType) || empty($this->dbTableName)) {
+            return array();
+        }
+
+        return array(
+            $this->dbTableName => array("MODIFY COLUMN `$this->columnName` $this->columnType")
+        );
+    }
+
+    /**
+     * Uninstalls the dimension if a {@link $columnName} and {@link columnType} is set. In case you perform any custom
+     * actions during {@link install()} - for instance adding an index - you should make sure to undo those actions by
+     * overwriting this method. Make sure to call this parent method to make sure the uninstallation of the column
+     * will be done.
+     * @throws Exception
+     * @api
+     */
+    public function uninstall()
+    {
+        if (empty($this->columnName) || empty($this->columnType) || empty($this->dbTableName)) {
+            return;
+        }
+
+        try {
+            $sql = "ALTER TABLE `" . Common::prefixTable($this->dbTableName) . "` DROP COLUMN `$this->columnName`";
+            Db::exec($sql);
+        } catch (Exception $e) {
+            if (!Db::get()->isErrNo($e, '1091')) {
+                throw $e;
+            }
+        }
+    }
+
+    /**
+     * Returns the ID of the category (typically a translation key).
+     * @return string
+     */
+    public function getCategoryId()
+    {
+        return $this->category;
+    }
+
+    /**
+     * Returns the translated name of this dimension which is typically in singular.
+     *
+     * @return string
+     */
+    public function getName()
+    {
+        if (!empty($this->nameSingular)) {
+            return Piwik::translate($this->nameSingular);
+        }
+
+        return $this->nameSingular;
+    }
+
+    /**
+     * Returns a translated name in plural for this dimension.
+     * @return string
+     * @api since Piwik 3.2.0
+     */
+    public function getNamePlural()
+    {
+        if (!empty($this->namePlural)) {
+            return Piwik::translate($this->namePlural);
+        }
+
+        return $this->getName();
+    }
+
+    /**
+     * Defines whether an anonymous user is allowed to view this dimension
+     * @return bool
+     * @api since Piwik 3.2.0
+     */
+    public function isAnonymousAllowed()
+    {
+        return $this->allowAnonymous;
+    }
+
+    /**
+     * Sets (overwrites) the SQL segment
+     * @param $segment
+     * @api since Piwik 3.2.0
+     */
+    public function setSqlSegment($segment)
+    {
+        $this->sqlSegment = $segment;
+    }
+
+    /**
+     * Sets (overwrites the dimension type)
+     * @param $type
+     * @api since Piwik 3.2.0
+     */
+    public function setType($type)
+    {
+        $this->type = $type;
+    }
+
+    /**
+     * A dimension should group values by using this method. Otherwise the same row may appear several times.
+     *
+     * @param mixed $value
+     * @param int $idSite
+     * @return mixed
+     * @api since Piwik 3.2.0
+     */
+    public function groupValue($value, $idSite)
+    {
+        switch ($this->type) {
+            case Dimension::TYPE_URL:
+                return str_replace(array('http://', 'https://'), '', $value);
+            case Dimension::TYPE_BOOL:
+                return !empty($value) ? '1' : '0';
+            case Dimension::TYPE_DURATION_MS:
+                return number_format($value / 1000, 2); // because we divide we need to group them and cannot do this in formatting step
+        }
+        return $value;
+    }
+
+    /**
+     * Formats the dimension value. By default, the dimension is formatted based on the set dimension type.
+     *
+     * @param mixed $value
+     * @param int $idSite
+     * @param Formatter $formatter
+     * @return mixed
+     * @api since Piwik 3.2.0
+     */
+    public function formatValue($value, $idSite, Formatter $formatter)
+    {
+        switch ($this->type) {
+            case Dimension::TYPE_BOOL:
+                if (empty($value)) {
+                    return Piwik::translate('General_No');
+                }
+
+                return Piwik::translate('General_Yes');
+            case Dimension::TYPE_ENUM:
+                $values = $this->getEnumColumnValues();
+                if (isset($values[$value])) {
+                    return $values[$value];
+                }
+                break;
+            case Dimension::TYPE_MONEY:
+                return $formatter->getPrettyMoney($value, $idSite);
+            case Dimension::TYPE_FLOAT:
+                return $formatter->getPrettyNumber((float) $value, $precision = 2);
+            case Dimension::TYPE_NUMBER:
+                return $formatter->getPrettyNumber($value);
+            case Dimension::TYPE_DURATION_S:
+                return $formatter->getPrettyTimeFromSeconds($value, $displayAsSentence = false);
+            case Dimension::TYPE_DURATION_MS:
+                return $formatter->getPrettyTimeFromSeconds($value, $displayAsSentence = true);
+            case Dimension::TYPE_PERCENT:
+                return $formatter->getPrettyPercentFromQuotient($value);
+            case Dimension::TYPE_BYTE:
+                return $formatter->getPrettySizeFromBytes($value);
+        }
+
+        return $value;
+    }
+
+    /**
+     * Overwrite this method to configure segments. To do so just create an instance of a {@link \Piwik\Plugin\Segment}
+     * class, configure it and call the {@link addSegment()} method. You can add one or more segments for this
+     * dimension. Example:
+     *
+     * ```
+     * $segment = new Segment();
+     * $segment->setSegment('exitPageUrl');
+     * $segment->setName('Actions_ColumnExitPageURL');
+     * $segment->setCategory('General_Visit');
+     * $this->addSegment($segment);
+     * ```
+     */
+    protected function configureSegments()
+    {
+        if ($this->segmentName && $this->category
+            && ($this->sqlSegment || ($this->columnName && $this->dbTableName))
+            && $this->nameSingular) {
+            $segment = new Segment();
+            $this->addSegment($segment);
+        }
+    }
+
+    /**
+     * Configures metrics for this dimension.
+     *
+     * For certain dimension types, some metrics will be added automatically.
+     *
+     * @param MetricsList $metricsList
+     * @param DimensionMetricFactory $dimensionMetricFactory
+     */
+    public function configureMetrics(MetricsList $metricsList, DimensionMetricFactory $dimensionMetricFactory)
+    {
+        if ($this->getMetricId() && $this->dbTableName && $this->columnName && $this->getNamePlural()) {
+            if (in_array($this->getType(), array(self::TYPE_DATETIME, self::TYPE_DATE, self::TYPE_TIME, self::TYPE_TIMESTAMP))) {
+                // we do not generate any metrics from these types
+                return;
+            } elseif (in_array($this->getType(), array(self::TYPE_URL, self::TYPE_TEXT, self::TYPE_BINARY, self::TYPE_ENUM))) {
+                $metric = $dimensionMetricFactory->createMetric(ArchivedMetric::AGGREGATION_UNIQUE);
+                $metricsList->addMetric($metric);
+            } elseif (in_array($this->getType(), array(self::TYPE_BOOL))) {
+                $metric = $dimensionMetricFactory->createMetric(ArchivedMetric::AGGREGATION_SUM);
+                $metricsList->addMetric($metric);
+            } else {
+                $metric = $dimensionMetricFactory->createMetric(ArchivedMetric::AGGREGATION_SUM);
+                $metricsList->addMetric($metric);
+
+                $metric = $dimensionMetricFactory->createMetric(ArchivedMetric::AGGREGATION_MAX);
+                $metricsList->addMetric($metric);
+            }
+        }
+    }
+
+    /**
+     * Check whether a dimension has overwritten a specific method.
+     * @param $method
+     * @return bool
+     * @ignore
+     */
+    public function hasImplementedEvent($method)
+    {
+        $method = new \ReflectionMethod($this, $method);
+        $declaringClass = $method->getDeclaringClass();
+
+        return 0 === strpos($declaringClass->name, 'Piwik\Plugins');
+    }
+
+    /**
+     * Adds a new segment. It automatically sets the SQL segment depending on the column name in case none is set
+     * already.
+     * @see \Piwik\Columns\Dimension::addSegment()
+     * @param Segment $segment
+     * @api
+     */
+    protected function addSegment(Segment $segment)
+    {
+        if (!$segment->getSegment() && $this->segmentName) {
+            $segment->setSegment($this->segmentName);
+        }
+
+        if (!$segment->getType()) {
+            $metricTypes = array(self::TYPE_NUMBER, self::TYPE_FLOAT, self::TYPE_MONEY, self::TYPE_DURATION_S, self::TYPE_DURATION_MS);
+            if (in_array($this->getType(), $metricTypes, $strict = true)) {
+                $segment->setType(Segment::TYPE_METRIC);
+            } else {
+                $segment->setType(Segment::TYPE_DIMENSION);
+            }
+        }
+
+        if (!$segment->getCategoryId() && $this->category) {
+            $segment->setCategory($this->category);
+        }
+
+        if (!$segment->getName() && $this->nameSingular) {
+            $segment->setName($this->nameSingular);
+        }
+
+        $sqlSegment = $segment->getSqlSegment();
+
+        if (empty($sqlSegment) && !$segment->getUnionOfSegments()) {
+            if (!empty($this->sqlSegment)) {
+                $segment->setSqlSegment($this->sqlSegment);
+            } elseif ($this->dbTableName && $this->columnName) {
+                $segment->setSqlSegment($this->dbTableName . '.' . $this->columnName);
+            } else {
+                throw new Exception('Segment cannot be added because no sql segment is set');
+            }
+        }
+
+        if (!$this->suggestedValuesCallback) {
+            // we can generate effecient value callback for enums automatically
+            $enum = $this->getEnumColumnValues();
+            if (!empty($enum)) {
+                $this->suggestedValuesCallback = function ($idSite, $maxValuesToReturn) use ($enum) {
+                    $values = array_values($enum);
+                    return array_slice($values, 0, $maxValuesToReturn);
+                };
+            }
+        }
+
+        if (!$this->acceptValues) {
+            // we can generate accept values for enums automatically
+            $enum = $this->getEnumColumnValues();
+            if (!empty($enum)) {
+                $enumValues = array_values($enum);
+                $enumValues = array_slice($enumValues, 0, 20);
+                $this->acceptValues = 'Eg. ' . implode(', ', $enumValues);
+            };
+        }
+
+        if ($this->acceptValues && !$segment->getAcceptValues()) {
+            $segment->setAcceptedValues($this->acceptValues);
+        }
+
+        if (!$this->sqlFilterValue && !$segment->getSqlFilter() && !$segment->getSqlFilterValue()) {
+            // no sql filter configured, we try to configure automatically for enums
+            $enum = $this->getEnumColumnValues();
+            if (!empty($enum)) {
+                $this->sqlFilterValue = function ($value, $sqlSegmentName) use ($enum) {
+                    if (isset($enum[$value])) {
+                        return $value;
+                    }
+
+                    $id = array_search($value, $enum);
+
+                    if ($id === false) {
+                        $id = array_search(strtolower(trim(urldecode($value))), $enum);
+
+                        if ($id === false) {
+                            throw new \Exception("Invalid '$sqlSegmentName' segment value $value");
+                        }
+                    }
+
+                    return $id;
+                };
+            };
+        }
+
+        if ($this->suggestedValuesCallback && !$segment->getSuggestedValuesCallback()) {
+            $segment->setSuggestedValuesCallback($this->suggestedValuesCallback);
+        }
+
+        if ($this->sqlFilterValue && !$segment->getSqlFilterValue()) {
+            $segment->setSqlFilterValue($this->sqlFilterValue);
+        }
+
+        if ($this->sqlFilter && !$segment->getSqlFilter()) {
+            $segment->setSqlFilter($this->sqlFilter);
+        }
+
+        if (!$this->allowAnonymous) {
+            $segment->setRequiresAtLeastViewAccess(true);
+        }
+
+        $this->segments[] = $segment;
+    }
+
+    /**
+     * Get the list of configured segments.
+     * @return Segment[]
+     * @ignore
+     */
+    public function getSegments()
+    {
+        if (empty($this->segments)) {
+            $this->configureSegments();
+        }
+
+        return $this->segments;
+    }
+
+    /**
+     * Returns the name of the segment that this dimension defines
+     * @return string
+     * @api since Piwik 3.2.0
+     */
+    public function getSegmentName()
+    {
+        return $this->segmentName;
+    }
+
+    /**
+     * Get the name of the dimension column.
+     * @return string
+     * @ignore
+     */
+    public function getColumnName()
+    {
+        return $this->columnName;
+    }
+
+    /**
+     * Returns a sql segment expression for this dimension.
+     * @return string
+     * @api since Piwik 3.2.0
+     */
+    public function getSqlSegment()
+    {
+        if (!empty($this->sqlSegment)) {
+            return $this->sqlSegment;
+        }
+
+        if ($this->dbTableName && $this->columnName) {
+            return $this->dbTableName . '.' . $this->columnName;
+        }
+    }
+
+    /**
+     * Check whether the dimension has a column type configured
+     * @return bool
+     * @ignore
+     */
+    public function hasColumnType()
+    {
+        return !empty($this->columnType);
+    }
+
+    /**
+     * Returns the name of the database table this dimension belongs to.
+     * @return string
+     * @api since Piwik 3.2.0
+     */
+    public function getDbTableName()
+    {
+        return $this->dbTableName;
+    }
+
+    /**
+     * Returns a unique string ID for this dimension. The ID is built using the namespaced class name
+     * of the dimension, but is modified to be more human readable.
+     *
+     * @return string eg, `"Referrers.Keywords"`
+     * @throws Exception if the plugin and simple class name of this instance cannot be determined.
+     *                   This would only happen if the dimension is located in the wrong directory.
+     * @api
+     */
+    public function getId()
+    {
+        $className = get_class($this);
+
+        return $this->generateIdFromClass($className);
+    }
+
+    /**
+     * @param string $className
+     * @return string
+     * @throws Exception
+     * @ignore
+     */
+    protected function generateIdFromClass($className)
+    {
+        // parse plugin name & dimension name
+        $regex = "/Piwik\\\\Plugins\\\\([^\\\\]+)\\\\" . self::COMPONENT_SUBNAMESPACE . "\\\\([^\\\\]+)/";
+        if (!preg_match($regex, $className, $matches)) {
+            throw new Exception("'$className' is located in the wrong directory.");
+        }
+
+        $pluginName = $matches[1];
+        $dimensionName = $matches[2];
+
+        return $pluginName . '.' . $dimensionName;
+    }
+
+    /**
+     * Gets an instance of all available visit, action and conversion dimension.
+     * @return Dimension[]
+     */
+    public static function getAllDimensions()
+    {
+        $cacheId = CacheId::siteAware(CacheId::pluginAware('AllDimensions'));
+        $cache   = PiwikCache::getTransientCache();
+
+        if (!$cache->contains($cacheId)) {
+            $plugins   = PluginManager::getInstance()->getPluginsLoadedAndActivated();
+            $instances = array();
+
+            /**
+             * Triggered to add new dimensions that cannot be picked up automatically by the platform.
+             * This is useful if the plugin allows a user to create reports / dimensions dynamically. For example
+             * CustomDimensions or CustomVariables. There are a variable number of dimensions in this case and it
+             * wouldn't be really possible to create a report file for one of these dimensions as it is not known
+             * how many Custom Dimensions will exist.
+             *
+             * **Example**
+             *
+             *     public function addDimension(&$dimensions)
+             *     {
+             *         $dimensions[] = new MyCustomDimension();
+             *     }
+             *
+             * @param Dimension[] $reports An array of dimensions
+             */
+            Piwik::postEvent('Dimension.addDimensions', array(&$instances));
+
+            foreach ($plugins as $plugin) {
+                foreach (self::getDimensions($plugin) as $instance) {
+                    $instances[] = $instance;
+                }
+            }
+
+            /**
+             * Triggered to filter / restrict dimensions.
+             *
+             * **Example**
+             *
+             *     public function filterDimensions(&$dimensions)
+             *     {
+             *         foreach ($dimensions as $index => $dimension) {
+             *              if ($dimension->getName() === 'Page URL') {}
+             *                  unset($dimensions[$index]); // remove this dimension
+             *              }
+             *         }
+             *     }
+             *
+             * @param Dimension[] $dimensions An array of dimensions
+             */
+            Piwik::postEvent('Dimension.filterDimensions', array(&$instances));
+
+            $cache->save($cacheId, $instances);
+        }
+
+        return $cache->fetch($cacheId);
+    }
+
+    public static function getDimensions(Plugin $plugin)
+    {
+        $columns = $plugin->findMultipleComponents('Columns', '\\Piwik\\Columns\\Dimension');
+        $instances  = array();
+
+        foreach ($columns as $colum) {
+            $instances[] = new $colum();
+        }
+
+        return $instances;
+    }
+
+    /**
+     * Creates a Dimension instance from a string ID (see {@link getId()}).
+     *
+     * @param string $dimensionId See {@link getId()}.
+     * @return Dimension|null The created instance or null if there is no Dimension for
+     *                        $dimensionId or if the plugin that contains the Dimension is
+     *                        not loaded.
+     * @api
+     * @deprecated Please use DimensionsProvider::factory instead
+     */
+    public static function factory($dimensionId)
+    {
+        list($module, $dimension) = explode('.', $dimensionId);
+        return ComponentFactory::factory($module, $dimension, __CLASS__);
+    }
+
+    /**
+     * Returns the name of the plugin that contains this Dimension.
+     *
+     * @return string
+     * @throws Exception if the Dimension is not located within a Plugin module.
+     * @api
+     */
+    public function getModule()
+    {
+        $id = $this->getId();
+        if (empty($id)) {
+            throw new Exception("Invalid dimension ID: '$id'.");
+        }
+
+        $parts = explode('.', $id);
+        return reset($parts);
+    }
+
+    /**
+     * Returns the type of the dimension which defines what kind of value this dimension stores.
+     * @return string
+     * @api since Piwik 3.2.0
+     */
+    public function getType()
+    {
+        if (!empty($this->type)) {
+            return $this->type;
+        }
+
+        if ($this->getDbColumnJoin()) {
+            // best guess
+            return self::TYPE_TEXT;
+        }
+
+        if ($this->getEnumColumnValues()) {
+            // best guess
+            return self::TYPE_ENUM;
+        }
+
+        if (!empty($this->columnType)) {
+            // best guess
+            $type = strtolower($this->columnType);
+            if (strpos($type, 'datetime') !== false) {
+                return self::TYPE_DATETIME;
+            } elseif (strpos($type, 'timestamp') !== false) {
+                return self::TYPE_TIMESTAMP;
+            } elseif (strpos($type, 'date') !== false) {
+                return self::TYPE_DATE;
+            } elseif (strpos($type, 'time') !== false) {
+                return self::TYPE_TIME;
+            } elseif (strpos($type, 'float') !== false) {
+                return self::TYPE_FLOAT;
+            } elseif (strpos($type, 'decimal') !== false) {
+                return self::TYPE_FLOAT;
+            } elseif (strpos($type, 'int') !== false) {
+                return self::TYPE_NUMBER;
+            } elseif (strpos($type, 'binary') !== false) {
+                return self::TYPE_BINARY;
+            }
+        }
+
+        return self::TYPE_TEXT;
+    }
+
+    /**
+     * Get the version of the dimension which is used for update checks.
+     * @return string
+     * @ignore
+     */
+    public function getVersion()
+    {
+        return $this->columnType;
+    }
+
+}

msd2/tracking/piwik/core/Columns/DimensionMetricFactory.php

@@ -0,0 +1,122 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\Columns;
+
+use Piwik\Piwik;
+use Piwik\Plugin\ArchivedMetric;
+use Piwik\Plugin\ComputedMetric;
+use Piwik\Plugin\Report;
+
+
+/**
+ * A factory to create metrics from a dimension.
+ *
+ * @api since Piwik 3.2.0
+ */
+class DimensionMetricFactory
+{
+    /**
+     * @var Dimension
+     */
+    private $dimension = null;
+
+    /**
+     * Generates a new dimension metric factory.
+     * @param Dimension $dimension A dimension instance the created metrics should be based on.
+     */
+    public function __construct(Dimension $dimension)
+    {
+        $this->dimension = $dimension;
+    }
+
+    /**
+     * @return ArchivedMetric
+     */
+    public function createCustomMetric($metricName, $readableName, $aggregation, $documentation = '')
+    {
+        if (!$this->dimension->getDbTableName() || !$this->dimension->getColumnName()) {
+            throw new \Exception(sprintf('Cannot make metric from dimension %s because DB table or column missing', $this->dimension->getId()));
+        }
+
+        $metric = new ArchivedMetric($this->dimension, $aggregation);
+        $metric->setType($this->dimension->getType());
+        $metric->setName($metricName);
+        $metric->setTranslatedName($readableName);
+        $metric->setDocumentation($documentation);
+        $metric->setCategory($this->dimension->getCategoryId());
+
+        return $metric;
+    }
+
+    /**
+     * @return \Piwik\Plugin\ComputedMetric
+     */
+    public function createComputedMetric($metricName1, $metricName2, $aggregation)
+    {
+        // We cannot use reuse ComputedMetricFactory here as it would result in an endless loop since ComputedMetricFactory
+        // requires a MetricsList which is just being built here...
+        $metric = new ComputedMetric($metricName1, $metricName2, $aggregation);
+        $metric->setCategory($this->dimension->getCategoryId());
+        return $metric;
+    }
+
+    /**
+     * @return ArchivedMetric
+     */
+    public function createMetric($aggregation)
+    {
+        $dimension = $this->dimension;
+
+        if (!$dimension->getNamePlural()) {
+            throw new \Exception(sprintf('No metric can be created for this dimension %s automatically because no $namePlural is set.', $dimension->getId()));
+        }
+
+        $prefix = '';
+        $translatedName = $dimension->getNamePlural();
+
+        $documentation = '';
+
+        switch ($aggregation) {
+            case ArchivedMetric::AGGREGATION_COUNT;
+                $prefix = ArchivedMetric::AGGREGATION_COUNT_PREFIX;
+                $translatedName = $dimension->getNamePlural();
+                $documentation = Piwik::translate('General_ComputedMetricCountDocumentation', $dimension->getNamePlural());
+                break;
+            case ArchivedMetric::AGGREGATION_SUM;
+                $prefix = ArchivedMetric::AGGREGATION_SUM_PREFIX;
+                $translatedName = Piwik::translate('General_ComputedMetricSum', $dimension->getNamePlural());
+                $documentation  = Piwik::translate('General_ComputedMetricSumDocumentation', $dimension->getNamePlural());
+                break;
+            case ArchivedMetric::AGGREGATION_MAX;
+                $prefix = ArchivedMetric::AGGREGATION_MAX_PREFIX;
+                $translatedName = Piwik::translate('General_ComputedMetricMax', $dimension->getNamePlural());
+                $documentation  = Piwik::translate('General_ComputedMetricMaxDocumentation', $dimension->getNamePlural());
+                break;
+            case ArchivedMetric::AGGREGATION_MIN;
+                $prefix = ArchivedMetric::AGGREGATION_MIN_PREFIX;
+                $translatedName = Piwik::translate('General_ComputedMetricMin', $dimension->getNamePlural());
+                $documentation  = Piwik::translate('General_ComputedMetricMinDocumentation', $dimension->getNamePlural());
+                break;
+            case ArchivedMetric::AGGREGATION_UNIQUE;
+                $prefix = ArchivedMetric::AGGREGATION_UNIQUE_PREFIX;
+                $translatedName = Piwik::translate('General_ComputedMetricUniqueCount', $dimension->getNamePlural());
+                $documentation  = Piwik::translate('General_ComputedMetricUniqueCountDocumentation', $dimension->getNamePlural());
+                break;
+            case ArchivedMetric::AGGREGATION_COUNT_WITH_NUMERIC_VALUE;
+                $prefix = ArchivedMetric::AGGREGATION_COUNT_WITH_NUMERIC_VALUE_PREFIX;
+                $translatedName = Piwik::translate('General_ComputedMetricCountWithValue', $dimension->getName());
+                $documentation  = Piwik::translate('General_ComputedMetricCountWithValueDocumentation', $dimension->getName());
+                break;
+        }
+
+        $metricId = strtolower($dimension->getMetricId());
+
+        return $this->createCustomMetric($prefix . $metricId, $translatedName, $aggregation, $documentation);
+    }
+}

msd2/tracking/piwik/core/Columns/DimensionsProvider.php

@@ -0,0 +1,62 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\Columns;
+
+use Piwik\CacheId;
+use Piwik\Cache as PiwikCache;
+
+class DimensionsProvider
+{
+    /**
+     * @param $dimensionId
+     * @return Dimension
+     */
+    public function factory($dimensionId)
+    {
+        $listDimensions = self::getMapOfNameToDimension();
+
+        if (empty($listDimensions) || !is_array($listDimensions) || !$dimensionId || !array_key_exists($dimensionId, $listDimensions)) {
+            return null;
+        }
+
+        return $listDimensions[$dimensionId];
+    }
+
+    private static function getMapOfNameToDimension()
+    {
+        $cacheId = CacheId::siteAware(CacheId::pluginAware('DimensionFactoryMap'));
+
+        $cache = PiwikCache::getTransientCache();
+        if ($cache->contains($cacheId)) {
+            $mapIdToDimension = $cache->fetch($cacheId);
+        } else {
+            $dimensions = new static();
+            $dimensions = $dimensions->getAllDimensions();
+
+            $mapIdToDimension = array();
+            foreach ($dimensions as $dimension) {
+                $mapIdToDimension[$dimension->getId()] = $dimension;
+            }
+
+            $cache->save($cacheId, $mapIdToDimension);
+        }
+
+        return $mapIdToDimension;
+    }
+
+    /**
+     * Returns a list of all available dimensions.
+     * @return Dimension[]
+     */
+    public function getAllDimensions()
+    {
+        return Dimension::getAllDimensions();
+    }
+
+}

msd2/tracking/piwik/core/Columns/Discriminator.php

@@ -0,0 +1,75 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\Columns;
+
+use Exception;
+use Piwik\Plugins\Actions\Actions\ActionSiteSearch;
+
+/**
+ * @api
+ * @since 3.1.0
+ */
+class Discriminator
+{
+    private $table;
+    private $discriminatorColumn;
+    private $discriminatorValue;
+
+    /**
+     * Join constructor.
+     * @param string $table  unprefixed table name
+     * @param null|string $discriminatorColumn
+     * @param null|int $discriminatorValue should be only hard coded, safe values.
+     * @throws Exception
+     */
+    public function __construct($table, $discriminatorColumn = null, $discriminatorValue = null)
+    {
+        if (empty($discriminatorColumn) || !isset($discriminatorValue)) {
+            throw new Exception('Both discriminatorColumn and discriminatorValue need to be defined');
+        }
+        $this->table = $table;
+        $this->discriminatorColumn = $discriminatorColumn;
+        $this->discriminatorValue = $discriminatorValue;
+
+        if (!$this->isValid()) {
+            // if adding another string value please post an event instead to get a list of allowed values
+            throw new Exception('$discriminatorValue needs to be null or numeric');
+        }
+    }
+
+    public function isValid()
+    {
+        return isset($this->discriminatorColumn)
+            && (is_numeric($this->discriminatorValue) || $this->discriminatorValue == ActionSiteSearch::CVAR_KEY_SEARCH_CATEGORY);
+    }
+
+    /**
+     * @return string
+     */
+    public function getTable()
+    {
+        return $this->table;
+    }
+
+    /**
+     * @return string
+     */
+    public function getColumn()
+    {
+        return $this->discriminatorColumn;
+    }
+
+    /**
+     * @return int|null
+     */
+    public function getValue()
+    {
+        return $this->discriminatorValue;
+    }
+}

msd2/tracking/piwik/core/Columns/Join.php

@@ -0,0 +1,61 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\Columns;
+
+use Exception;
+
+/**
+ * @api
+ * @since 3.1.0
+ */
+class Join
+{
+    private $table;
+    private $column;
+    private $targetColumn;
+
+    /**
+     * Join constructor.
+     * @param $table
+     * @param $column
+     * @param $targetColumn
+     * @throws Exception
+     */
+    public function __construct($table, $column, $targetColumn)
+    {
+        $this->table = $table;
+        $this->column = $column;
+        $this->targetColumn = $targetColumn;
+    }
+
+    /**
+     * @return string
+     */
+    public function getTable()
+    {
+        return $this->table;
+    }
+
+    /**
+     * @return string
+     */
+    public function getColumn()
+    {
+        return $this->column;
+    }
+
+    /**
+     * @return string
+     */
+    public function getTargetColumn()
+    {
+        return $this->targetColumn;
+    }
+
+}

msd2/tracking/piwik/core/Columns/Join/ActionNameJoin.php

@@ -0,0 +1,24 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\Columns\Join;
+
+use Piwik\Columns;
+
+/**
+ * @api
+ * @since 3.1.0
+ */
+class ActionNameJoin extends Columns\Join
+{
+    public function __construct()
+    {
+        return parent::__construct('log_action', 'idaction', 'name');
+    }
+
+}

msd2/tracking/piwik/core/Columns/Join/GoalNameJoin.php

@@ -0,0 +1,24 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\Columns\Join;
+
+use Piwik\Columns;
+
+/**
+ * @api
+ * @since 3.1.0
+ */
+class GoalNameJoin extends Columns\Join
+{
+    public function __construct()
+    {
+        return parent::__construct('goal', 'idgoal', 'name');
+    }
+
+}

msd2/tracking/piwik/core/Columns/Join/SiteNameJoin.php

@@ -0,0 +1,24 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\Columns\Join;
+
+use Piwik\Columns;
+
+/**
+ * @api
+ * @since 3.1.0
+ */
+class SiteNameJoin extends Columns\Join
+{
+    public function __construct()
+    {
+        return parent::__construct('site', 'idsite', 'name');
+    }
+
+}

msd2/tracking/piwik/core/Columns/MetricsList.php

@@ -0,0 +1,191 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\Columns;
+
+use Piwik\Cache;
+use Piwik\CacheId;
+use Piwik\Piwik;
+use Piwik\Plugin\ArchivedMetric;
+use Piwik\Plugin\Metric;
+use Piwik\Plugin\ProcessedMetric;
+
+/**
+ * Manages the global list of metrics that can be used in reports.
+ *
+ * Metrics are added automatically by dimensions as well as through the {@hook Metric.addMetrics} and
+ * {@hook Metric.addComputedMetrics} and filtered through the {@hook Metric.filterMetrics} event.
+ * Observers for this event should call the {@link addMetric()} method to add metrics or use any of the other
+ * methods to remove metrics.
+ *
+ * @api since Piwik 3.2.0
+ */
+class MetricsList
+{
+    /**
+     * List of metrics
+     *
+     * @var Metric[]
+     */
+    private $metrics = array();
+
+    private $metricsByNameCache = array();
+
+    /**
+     * @param Metric $metric
+     */
+    public function addMetric(Metric $metric)
+    {
+        $this->metrics[] = $metric;
+        $this->metricsByNameCache = array();
+    }
+
+    /**
+     * Get all available metrics.
+     *
+     * @return Metric[]
+     */
+    public function getMetrics()
+    {
+        return $this->metrics;
+    }
+
+    /**
+     * Removes one or more metrics from the metrics list.
+     *
+     * @param string $metricCategory The metric category id. Can be a translation token eg 'General_Visits'
+     *                                 see {@link Metric::getCategory()}.
+     * @param string|false $metricName The name of the metric to remove eg 'nb_visits'.
+     *                                 If not supplied, all metrics within that category will be removed.
+     */
+    public function remove($metricCategory, $metricName = false)
+    {
+        foreach ($this->metrics as $index => $metric) {
+            if ($metric->getCategoryId() === $metricCategory) {
+                if (!$metricName || $metric->getName() === $metricName) {
+                    unset($this->metrics[$index]);
+                    $this->metricsByNameCache = array();
+                }
+            }
+        }
+    }
+
+    /**
+     * @param string $metricName
+     * @return Metric|ArchivedMetric|null
+     */
+    public function getMetric($metricName)
+    {
+        if (empty($this->metricsByNameCache)) {
+            // this method might be called quite often... eg when having heaps of goals... need to cache it
+            foreach ($this->metrics as $index => $metric) {
+                $this->metricsByNameCache[$metric->getName()] = $metric;
+            }
+        }
+
+        if (!empty($this->metricsByNameCache[$metricName])) {
+            return $this->metricsByNameCache[$metricName];
+        }
+
+        return null;
+    }
+
+    /**
+     * Get all metrics defined in the Piwik platform.
+     * @ignore
+     * @return static
+     */
+    public static function get()
+    {
+        $cache = Cache::getTransientCache();
+        $cacheKey = CacheId::siteAware('MetricsList');
+
+        if ($cache->contains($cacheKey)) {
+            return $cache->fetch($cacheKey);
+        }
+
+        $list = new static;
+
+        /**
+         * Triggered to add new metrics that cannot be picked up automatically by the platform.
+         * This is useful if the plugin allows a user to create metrics dynamically. For example
+         * CustomDimensions or CustomVariables.
+         *
+         * **Example**
+         *
+         *     public function addMetric(&$list)
+         *     {
+         *         $list->addMetric(new MyCustomMetric());
+         *     }
+         *
+         * @param MetricsList $list An instance of the MetricsList. You can add metrics to the list this way.
+         */
+        Piwik::postEvent('Metric.addMetrics', array($list));
+
+        $dimensions = Dimension::getAllDimensions();
+        foreach ($dimensions as $dimension) {
+            $factory = new DimensionMetricFactory($dimension);
+            $dimension->configureMetrics($list, $factory);
+        }
+
+        $computedFactory = new ComputedMetricFactory($list);
+
+        /**
+         * Triggered to add new metrics that cannot be picked up automatically by the platform.
+         * This is useful if the plugin allows a user to create metrics dynamically. For example
+         * CustomDimensions or CustomVariables.
+         *
+         * **Example**
+         *
+         *     public function addMetric(&$list)
+         *     {
+         *         $list->addMetric(new MyCustomMetric());
+         *     }
+         *
+         * @param MetricsList $list An instance of the MetricsList. You can add metrics to the list this way.
+         */
+        Piwik::postEvent('Metric.addComputedMetrics', array($list, $computedFactory));
+
+        /**
+         * Triggered to filter metrics.
+         *
+         * **Example**
+         *
+         *     public function removeMetrics(Piwik\Columns\MetricsList $list)
+         *     {
+         *         $list->remove($category='General_Visits'); // remove all metrics having this category
+         *     }
+         *
+         * @param MetricsList $list An instance of the MetricsList. You can change the list of metrics this way.
+         */
+        Piwik::postEvent('Metric.filterMetrics', array($list));
+
+        $availableMetrics = array();
+        foreach ($list->getMetrics() as $metric) {
+            $availableMetrics[] = $metric->getName();
+        }
+
+        foreach ($list->metrics as $index => $metric) {
+            if ($metric instanceof ProcessedMetric) {
+                $depMetrics = $metric->getDependentMetrics();
+                if (is_array($depMetrics)) {
+                    foreach ($depMetrics as $depMetric) {
+                        if (!in_array($depMetric, $availableMetrics, $strict = true)) {
+                            unset($list->metrics[$index]); // not resolvable metric
+                        }
+                    }
+                }
+            }
+        }
+
+        $cache->save($cacheKey, $list);
+
+        return $list;
+    }
+
+}

msd2/tracking/piwik/core/Columns/Updater.php

@@ -0,0 +1,380 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\Columns;
+
+use Piwik\Common;
+use Piwik\DbHelper;
+use Piwik\Plugin\Dimension\ActionDimension;
+use Piwik\Plugin\Dimension\VisitDimension;
+use Piwik\Plugin\Dimension\ConversionDimension;
+use Piwik\Db;
+use Piwik\Plugin\Manager;
+use Piwik\Updater as PiwikUpdater;
+use Piwik\Filesystem;
+use Piwik\Cache as PiwikCache;
+use Piwik\Updater\Migration;
+
+/**
+ * Class that handles dimension updates
+ */
+class Updater extends \Piwik\Updates
+{
+    private static $cacheId = 'AllDimensionModifyTime';
+
+    /**
+     * @var VisitDimension[]
+     */
+    public $visitDimensions;
+
+    /**
+     * @var ActionDimension[]
+     */
+    private $actionDimensions;
+
+    /**
+     * @var ConversionDimension[]
+     */
+    private $conversionDimensions;
+
+    /**
+     * @param VisitDimension[]|null $visitDimensions
+     * @param ActionDimension[]|null $actionDimensions
+     * @param ConversionDimension[]|null $conversionDimensions
+     */
+    public function __construct(array $visitDimensions = null, array $actionDimensions = null, array $conversionDimensions = null)
+    {
+        $this->visitDimensions = $visitDimensions;
+        $this->actionDimensions = $actionDimensions;
+        $this->conversionDimensions = $conversionDimensions;
+    }
+
+    /**
+     * @param PiwikUpdater $updater
+     * @return Migration\Db[]
+     */
+    public function getMigrationQueries(PiwikUpdater $updater)
+    {
+        $sqls = array();
+
+        $changingColumns = $this->getUpdates($updater);
+        $errorCodes = array(
+            Migration\Db\Sql::ERROR_CODE_COLUMN_NOT_EXISTS,
+            Migration\Db\Sql::ERROR_CODE_DUPLICATE_COLUMN
+        );
+
+        foreach ($changingColumns as $table => $columns) {
+            if (empty($columns) || !is_array($columns)) {
+                continue;
+            }
+
+            $sql = "ALTER TABLE `" . Common::prefixTable($table) . "` " . implode(', ', $columns);
+            $sqls[] = new Migration\Db\Sql($sql, $errorCodes);
+        }
+
+        return $sqls;
+    }
+
+    public function doUpdate(PiwikUpdater $updater)
+    {
+        $updater->executeMigrationQueries(__FILE__, $this->getMigrationQueries($updater));
+    }
+
+    private function getVisitDimensions()
+    {
+        // see eg https://github.com/piwik/piwik/issues/8399 we fetch them only on demand to improve performance
+        if (!isset($this->visitDimensions)) {
+            $this->visitDimensions = VisitDimension::getAllDimensions();
+        }
+
+        return $this->visitDimensions;
+    }
+
+    private function getActionDimensions()
+    {
+        // see eg https://github.com/piwik/piwik/issues/8399 we fetch them only on demand to improve performance
+        if (!isset($this->actionDimensions)) {
+            $this->actionDimensions = ActionDimension::getAllDimensions();
+        }
+
+        return $this->actionDimensions;
+    }
+
+    private function getConversionDimensions()
+    {
+        // see eg https://github.com/piwik/piwik/issues/8399 we fetch them only on demand to improve performance
+        if (!isset($this->conversionDimensions)) {
+            $this->conversionDimensions = ConversionDimension::getAllDimensions();
+        }
+
+        return $this->conversionDimensions;
+    }
+
+    private function getUpdates(PiwikUpdater $updater)
+    {
+        $visitColumns      = DbHelper::getTableColumns(Common::prefixTable('log_visit'));
+        $actionColumns     = DbHelper::getTableColumns(Common::prefixTable('log_link_visit_action'));
+        $conversionColumns = DbHelper::getTableColumns(Common::prefixTable('log_conversion'));
+
+        $allUpdatesToRun = array();
+
+        foreach ($this->getVisitDimensions() as $dimension) {
+            $updates         = $this->getUpdatesForDimension($updater, $dimension, 'log_visit.', $visitColumns);
+            $allUpdatesToRun = $this->mixinUpdates($allUpdatesToRun, $updates);
+        }
+
+        foreach ($this->getActionDimensions() as $dimension) {
+            $updates         = $this->getUpdatesForDimension($updater, $dimension, 'log_link_visit_action.', $actionColumns);
+            $allUpdatesToRun = $this->mixinUpdates($allUpdatesToRun, $updates);
+        }
+
+        foreach ($this->getConversionDimensions() as $dimension) {
+            $updates         = $this->getUpdatesForDimension($updater, $dimension, 'log_conversion.', $conversionColumns);
+            $allUpdatesToRun = $this->mixinUpdates($allUpdatesToRun, $updates);
+        }
+
+        return $allUpdatesToRun;
+    }
+
+    /**
+     * @param ActionDimension|ConversionDimension|VisitDimension $dimension
+     * @param string $componentPrefix
+     * @return array
+     */
+    private function getUpdatesForDimension(PiwikUpdater $updater, $dimension, $componentPrefix, $existingColumnsInDb)
+    {
+        $column = $dimension->getColumnName();
+        $componentName = $componentPrefix . $column;
+
+        if (!$updater->hasNewVersion($componentName)) {
+            return array();
+        }
+
+        if (array_key_exists($column, $existingColumnsInDb)) {
+            $sqlUpdates = $dimension->update();
+        } else {
+            $sqlUpdates = $dimension->install();
+        }
+
+        return $sqlUpdates;
+    }
+
+    private function mixinUpdates($allUpdatesToRun, $updatesFromDimension)
+    {
+        if (!empty($updatesFromDimension)) {
+            foreach ($updatesFromDimension as $table => $col) {
+                if (empty($allUpdatesToRun[$table])) {
+                    $allUpdatesToRun[$table] = $col;
+                } else {
+                    $allUpdatesToRun[$table] = array_merge($allUpdatesToRun[$table], $col);
+                }
+            }
+        }
+
+        return $allUpdatesToRun;
+    }
+
+    public function getAllVersions(PiwikUpdater $updater)
+    {
+        // to avoid having to load all dimensions on each request we check if there were any changes on the file system
+        // can easily save > 100ms for each request
+        $cachedTimes  = self::getCachedDimensionFileChanges();
+        $currentTimes = self::getCurrentDimensionFileChanges();
+        $diff         = array_diff_assoc($currentTimes, $cachedTimes);
+
+        if (empty($diff)) {
+            return array();
+        }
+
+        $versions = array();
+
+        $visitColumns      = DbHelper::getTableColumns(Common::prefixTable('log_visit'));
+        $actionColumns     = DbHelper::getTableColumns(Common::prefixTable('log_link_visit_action'));
+        $conversionColumns = DbHelper::getTableColumns(Common::prefixTable('log_conversion'));
+
+        foreach ($this->getVisitDimensions() as $dimension) {
+            $versions = $this->mixinVersions($updater, $dimension, VisitDimension::INSTALLER_PREFIX, $visitColumns, $versions);
+        }
+
+        foreach ($this->getActionDimensions() as $dimension) {
+            $versions = $this->mixinVersions($updater, $dimension, ActionDimension::INSTALLER_PREFIX, $actionColumns, $versions);
+        }
+
+        foreach ($this->getConversionDimensions() as $dimension) {
+            $versions = $this->mixinVersions($updater, $dimension, ConversionDimension::INSTALLER_PREFIX, $conversionColumns, $versions);
+        }
+
+        return $versions;
+    }
+
+    /**
+     * @param PiwikUpdater $updater
+     * @param Dimension $dimension
+     * @param string $componentPrefix
+     * @param array $columns
+     * @param array $versions
+     * @return array The modified versions array
+     */
+    private function mixinVersions(PiwikUpdater $updater, $dimension, $componentPrefix, $columns, $versions)
+    {
+        $columnName = $dimension->getColumnName();
+
+        // dimensions w/o columns do not need DB updates
+        if (!$columnName || !$dimension->hasColumnType()) {
+            return $versions;
+        }
+
+        $component = $componentPrefix . $columnName;
+        $version   = $dimension->getVersion();
+
+        // if the column exists in the table, but has no associated version, and was one of the core columns
+        // that was moved when the dimension refactor took place, then:
+        // - set the installed version in the DB to the current code version
+        // - and do not check for updates since we just set the version to the latest
+        if (array_key_exists($columnName, $columns)
+            && false === $updater->getCurrentComponentVersion($component)
+            && self::wasDimensionMovedFromCoreToPlugin($component, $version)
+        ) {
+            $updater->markComponentSuccessfullyUpdated($component, $version);
+            return $versions;
+        }
+
+        $versions[$component] = $version;
+
+        return $versions;
+    }
+
+    public static function isDimensionComponent($name)
+    {
+        return 0 === strpos($name, 'log_visit.')
+            || 0 === strpos($name, 'log_conversion.')
+            || 0 === strpos($name, 'log_conversion_item.')
+            || 0 === strpos($name, 'log_link_visit_action.');
+    }
+
+    public static function wasDimensionMovedFromCoreToPlugin($name, $version)
+    {
+        // maps names of core dimension columns that were part of the original dimension refactor with their
+        // initial "version" strings. The '1' that is sometimes appended to the end of the string (sometimes seen as
+        // NULL1) is from individual dimension "versioning" logic (eg, see VisitDimension::getVersion())
+        $initialCoreDimensionVersions = array(
+            'log_visit.config_resolution' => 'VARCHAR(9) NOT NULL',
+            'log_visit.config_device_brand' => 'VARCHAR( 100 ) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL',
+            'log_visit.config_device_model' => 'VARCHAR( 100 ) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL',
+            'log_visit.config_windowsmedia' => 'TINYINT(1) NOT NULL',
+            'log_visit.config_silverlight' => 'TINYINT(1) NOT NULL',
+            'log_visit.config_java' => 'TINYINT(1) NOT NULL',
+            'log_visit.config_gears' => 'TINYINT(1) NOT NULL',
+            'log_visit.config_pdf' => 'TINYINT(1) NOT NULL',
+            'log_visit.config_quicktime' => 'TINYINT(1) NOT NULL',
+            'log_visit.config_realplayer' => 'TINYINT(1) NOT NULL',
+            'log_visit.config_device_type' => 'TINYINT( 100 ) NULL DEFAULT NULL',
+            'log_visit.visitor_localtime' => 'TIME NOT NULL',
+            'log_visit.location_region' => 'char(2) DEFAULT NULL1',
+            'log_visit.visitor_days_since_last' => 'SMALLINT(5) UNSIGNED NOT NULL',
+            'log_visit.location_longitude' => 'float(10, 6) DEFAULT NULL1',
+            'log_visit.visit_total_events' => 'SMALLINT(5) UNSIGNED NOT NULL',
+            'log_visit.config_os_version' => 'VARCHAR( 100 ) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL',
+            'log_visit.location_city' => 'varchar(255) DEFAULT NULL1',
+            'log_visit.location_country' => 'CHAR(3) NOT NULL1',
+            'log_visit.location_latitude' => 'float(10, 6) DEFAULT NULL1',
+            'log_visit.config_flash' => 'TINYINT(1) NOT NULL',
+            'log_visit.config_director' => 'TINYINT(1) NOT NULL',
+            'log_visit.visit_total_time' => 'SMALLINT(5) UNSIGNED NOT NULL',
+            'log_visit.visitor_count_visits' => 'SMALLINT(5) UNSIGNED NOT NULL1',
+            'log_visit.visit_entry_idaction_name' => 'INTEGER(11) UNSIGNED NOT NULL',
+            'log_visit.visit_entry_idaction_url' => 'INTEGER(11) UNSIGNED NOT NULL',
+            'log_visit.visitor_returning' => 'TINYINT(1) NOT NULL1',
+            'log_visit.visitor_days_since_order' => 'SMALLINT(5) UNSIGNED NOT NULL1',
+            'log_visit.visit_goal_buyer' => 'TINYINT(1) NOT NULL',
+            'log_visit.visit_first_action_time' => 'DATETIME NOT NULL',
+            'log_visit.visit_goal_converted' => 'TINYINT(1) NOT NULL',
+            'log_visit.visitor_days_since_first' => 'SMALLINT(5) UNSIGNED NOT NULL1',
+            'log_visit.visit_exit_idaction_name' => 'INTEGER(11) UNSIGNED NOT NULL',
+            'log_visit.visit_exit_idaction_url' => 'INTEGER(11) UNSIGNED NULL DEFAULT 0',
+            'log_visit.config_browser_version' => 'VARCHAR(20) NOT NULL',
+            'log_visit.config_browser_name' => 'VARCHAR(10) NOT NULL',
+            'log_visit.config_browser_engine' => 'VARCHAR(10) NOT NULL',
+            'log_visit.location_browser_lang' => 'VARCHAR(20) NOT NULL',
+            'log_visit.config_os' => 'CHAR(3) NOT NULL',
+            'log_visit.config_cookie' => 'TINYINT(1) NOT NULL',
+            'log_visit.referer_url' => 'TEXT NOT NULL',
+            'log_visit.visit_total_searches' => 'SMALLINT(5) UNSIGNED NOT NULL',
+            'log_visit.visit_total_actions' => 'SMALLINT(5) UNSIGNED NOT NULL',
+            'log_visit.referer_keyword' => 'VARCHAR(255) NULL1',
+            'log_visit.referer_name' => 'VARCHAR(70) NULL1',
+            'log_visit.referer_type' => 'TINYINT(1) UNSIGNED NULL1',
+            'log_visit.user_id' => 'VARCHAR(200) NULL',
+            'log_link_visit_action.idaction_name' => 'INTEGER(10) UNSIGNED',
+            'log_link_visit_action.idaction_url' => 'INTEGER(10) UNSIGNED DEFAULT NULL',
+            'log_link_visit_action.server_time' => 'DATETIME NOT NULL',
+            'log_link_visit_action.time_spent_ref_action' => 'INTEGER(10) UNSIGNED NOT NULL',
+            'log_link_visit_action.idaction_event_action' => 'INTEGER(10) UNSIGNED DEFAULT NULL',
+            'log_link_visit_action.idaction_event_category' => 'INTEGER(10) UNSIGNED DEFAULT NULL',
+            'log_conversion.revenue_discount' => 'float default NULL',
+            'log_conversion.revenue' => 'float default NULL',
+            'log_conversion.revenue_shipping' => 'float default NULL',
+            'log_conversion.revenue_subtotal' => 'float default NULL',
+            'log_conversion.revenue_tax' => 'float default NULL',
+        );
+
+        if (!array_key_exists($name, $initialCoreDimensionVersions)) {
+            return false;
+        }
+
+        return strtolower($initialCoreDimensionVersions[$name]) === strtolower($version);
+    }
+
+    public function onNoUpdateAvailable($versionsThatWereChecked)
+    {
+        if (!empty($versionsThatWereChecked)) {
+            // invalidate cache only if there were actually file changes before, otherwise we write the cache on each
+            // request. There were versions checked only if there was a file change but no update, meaning we can
+            // set the cache and declare this state as "no update available".
+            self::cacheCurrentDimensionFileChanges();
+        }
+    }
+
+    private static function getCurrentDimensionFileChanges()
+    {
+        $times = array();
+        foreach (Manager::getPluginsDirectories() as $pluginsDir) {
+            $files = Filesystem::globr($pluginsDir . '*/Columns', '*.php');
+
+            foreach ($files as $file) {
+                $times[$file] = filemtime($file);
+            }
+        }
+
+        return $times;
+    }
+
+    private static function cacheCurrentDimensionFileChanges()
+    {
+        $changes = self::getCurrentDimensionFileChanges();
+
+        $cache = self::buildCache();
+        $cache->save(self::$cacheId, $changes);
+    }
+
+    private static function buildCache()
+    {
+        return PiwikCache::getEagerCache();
+    }
+
+    private static function getCachedDimensionFileChanges()
+    {
+        $cache = self::buildCache();
+
+        if ($cache->contains(self::$cacheId)) {
+            return $cache->fetch(self::$cacheId);
+        }
+
+        return array();
+    }
+}

msd2/tracking/piwik/core/Composer/ScriptHandler.php

@@ -0,0 +1,47 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+
+namespace Piwik\Composer;
+
+/**
+ * Scripts executed before/after Composer install and update.
+ *
+ * We use this PHP class because setting the bash scripts directly in composer.json breaks
+ * Composer on Windows systems.
+ */
+class ScriptHandler
+{
+    private static function isPhp7orLater()
+    {
+        return version_compare('7.0.0-dev', PHP_VERSION) < 1;
+    }
+
+    public static function cleanXhprof()
+    {
+        if (! is_dir('vendor/facebook/xhprof/extension')) {
+            return;
+        }
+
+        if (!self::isPhp7orLater()) {
+            // doesn't work with PHP 7 at the moment
+            passthru('misc/composer/clean-xhprof.sh');
+        }
+    }
+
+    public static function buildXhprof()
+    {
+        if (! is_dir('vendor/facebook/xhprof/extension')) {
+            return;
+        }
+
+
+        if (!self::isPhp7orLater()) {
+            passthru('misc/composer/clean-xhprof.sh');
+        }
+    }
+}

msd2/tracking/piwik/core/Concurrency/DistributedList.php

@@ -0,0 +1,171 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+namespace Piwik\Concurrency;
+
+use Piwik\Common;
+use Piwik\Container\StaticContainer;
+use Piwik\Option;
+use Psr\Log\LoggerInterface;
+
+/**
+ * Manages a simple distributed list stored in an Option. No locking occurs, so the list
+ * is not thread safe, and should only be used for use cases where atomicity is not
+ * important.
+ *
+ * The list of items is serialized and stored in an Option. Items are converted to string
+ * before being persisted, so it is not expected to unserialize objects.
+ */
+class DistributedList
+{
+    /**
+     * The name of the option to store the list in.
+     *
+     * @var string
+     */
+    private $optionName;
+
+    /**
+     * @var LoggerInterface
+     */
+    private $logger;
+
+    /**
+     * Constructor.
+     *
+     * @param string $optionName
+     */
+    public function __construct($optionName, LoggerInterface $logger = null)
+    {
+        $this->optionName = $optionName;
+        $this->logger = $logger ?: StaticContainer::get('Psr\Log\LoggerInterface');
+    }
+
+    /**
+     * Queries the option table and returns all items in this list.
+     *
+     * @return array
+     */
+    public function getAll()
+    {
+        $result = $this->getListOptionValue();
+
+        foreach ($result as $key => $item) {
+            // remove non-array items (unexpected state, though can happen when upgrading from an old Piwik)
+            if (is_array($item)) {
+                $this->logger->info("Found array item in DistributedList option value '{name}': {data}", array(
+                    'name' => $this->optionName,
+                    'data' => var_export($result, true)
+                ));
+
+                unset($result[$key]);
+            }
+        }
+
+        return $result;
+    }
+
+    /**
+     * Sets the contents of the list in the option table.
+     *
+     * @param string[] $items
+     */
+    public function setAll($items)
+    {
+        foreach ($items as $key => &$item) {
+            if (is_array($item)) {
+                throw new \InvalidArgumentException("Array item encountered in DistributedList::setAll() [ key = $key ].");
+            } else {
+                $item = (string)$item;
+            }
+        }
+
+        Option::set($this->optionName, serialize($items));
+    }
+
+    /**
+     * Adds one or more items to the list in the option table.
+     *
+     * @param string|array $item
+     */
+    public function add($item)
+    {
+        $allItems = $this->getAll();
+        if (is_array($item)) {
+            $allItems = array_merge($allItems, $item);
+        } else {
+            $allItems[] = $item;
+        }
+
+        $this->setAll($allItems);
+    }
+
+    /**
+     * Removes one or more items by value from the list in the option table.
+     *
+     * Does not preserve array keys.
+     *
+     * @param string|array $items
+     */
+    public function remove($items)
+    {
+        if (!is_array($items)) {
+            $items = array($items);
+        }
+
+        $allItems = $this->getAll();
+
+        foreach ($items as $item) {
+            $existingIndex = array_search($item, $allItems);
+            if ($existingIndex === false) {
+                return;
+            }
+
+            unset($allItems[$existingIndex]);
+        }
+
+        $this->setAll(array_values($allItems));
+    }
+
+    /**
+     * Removes one or more items by index from the list in the option table.
+     *
+     * Does not preserve array keys.
+     *
+     * @param int[]|int $indices
+     */
+    public function removeByIndex($indices)
+    {
+        if (!is_array($indices)) {
+            $indices = array($indices);
+        }
+
+        $indices = array_unique($indices);
+
+        $allItems = $this->getAll();
+        foreach ($indices as $index) {
+            unset($allItems[$index]);
+        }
+
+        $this->setAll(array_values($allItems));
+    }
+
+    protected function getListOptionValue()
+    {
+        Option::clearCachedOption($this->optionName);
+        $array = Option::get($this->optionName);
+
+        $result = array();
+        if ($array
+            && ($array = Common::safe_unserialize($array))
+            && count($array)
+        ) {
+            $result = $array;
+        }
+        return $result;
+    }
+}

msd2/tracking/piwik/core/Config.php

@@ -0,0 +1,481 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+
+namespace Piwik;
+
+use Exception;
+use Piwik\Application\Kernel\GlobalSettingsProvider;
+use Piwik\Container\StaticContainer;
+use Piwik\Exception\MissingFilePermissionException;
+use Piwik\ProfessionalServices\Advertising;
+
+/**
+ * Singleton that provides read & write access to Piwik's INI configuration.
+ *
+ * This class reads and writes to the `config/config.ini.php` file. If config
+ * options are missing from that file, this class will look for their default
+ * values in `config/global.ini.php`.
+ *
+ * ### Examples
+ *
+ * **Getting a value:**
+ *
+ *     // read the minimum_memory_limit option under the [General] section
+ *     $minValue = Config::getInstance()->General['minimum_memory_limit'];
+ *
+ * **Setting a value:**
+ *
+ *     // set the minimum_memory_limit option
+ *     Config::getInstance()->General['minimum_memory_limit'] = 256;
+ *     Config::getInstance()->forceSave();
+ *
+ * **Setting an entire section:**
+ *
+ *     Config::getInstance()->MySection = array('myoption' => 1);
+ *     Config::getInstance()->forceSave();
+ */
+class Config
+{
+    const DEFAULT_LOCAL_CONFIG_PATH = '/config/config.ini.php';
+    const DEFAULT_COMMON_CONFIG_PATH = '/config/common.config.ini.php';
+    const DEFAULT_GLOBAL_CONFIG_PATH = '/config/global.ini.php';
+
+    /**
+     * @var boolean
+     */
+    protected $doNotWriteConfigInTests = false;
+
+    /**
+     * @var GlobalSettingsProvider
+     */
+    protected $settings;
+
+    /**
+     * @return Config
+     */
+    public static function getInstance()
+    {
+        return StaticContainer::get('Piwik\Config');
+    }
+
+    public function __construct(GlobalSettingsProvider $settings)
+    {
+        $this->settings = $settings;
+    }
+
+    /**
+     * Returns the path to the local config file used by this instance.
+     *
+     * @return string
+     */
+    public function getLocalPath()
+    {
+        return $this->settings->getPathLocal();
+    }
+
+    /**
+     * Returns the path to the global config file used by this instance.
+     *
+     * @return string
+     */
+    public function getGlobalPath()
+    {
+        return $this->settings->getPathGlobal();
+    }
+
+    /**
+     * Returns the path to the common config file used by this instance.
+     *
+     * @return string
+     */
+    public function getCommonPath()
+    {
+        return $this->settings->getPathCommon();
+    }
+
+    /**
+     * Returns absolute path to the global configuration file
+     *
+     * @return string
+     */
+    public static function getGlobalConfigPath()
+    {
+        return PIWIK_USER_PATH . self::DEFAULT_GLOBAL_CONFIG_PATH;
+    }
+
+    /**
+     * Returns absolute path to the common configuration file.
+     *
+     * @return string
+     */
+    public static function getCommonConfigPath()
+    {
+        return PIWIK_USER_PATH . self::DEFAULT_COMMON_CONFIG_PATH;
+    }
+
+    /**
+     * Returns default absolute path to the local configuration file.
+     *
+     * @return string
+     */
+    public static function getDefaultLocalConfigPath()
+    {
+        return PIWIK_USER_PATH . self::DEFAULT_LOCAL_CONFIG_PATH;
+    }
+
+    /**
+     * Returns absolute path to the local configuration file
+     *
+     * @return string
+     */
+    public static function getLocalConfigPath()
+    {
+        $path = self::getByDomainConfigPath();
+        if ($path) {
+            return $path;
+        }
+        return self::getDefaultLocalConfigPath();
+    }
+
+    private static function getLocalConfigInfoForHostname($hostname)
+    {
+        if (!$hostname) {
+            return array();
+        }
+
+        // Remove any port number to get actual hostname
+        $hostname = Url::getHostSanitized($hostname);
+        $standardConfigName = 'config.ini.php';
+        $perHostFilename  = $hostname . '.' . $standardConfigName;
+        $pathDomainConfig = PIWIK_USER_PATH . '/config/' . $perHostFilename;
+        $pathDomainMiscUser = PIWIK_USER_PATH . '/misc/user/' . $hostname . '/' . $standardConfigName;
+
+        $locations = array(
+            array('file' => $perHostFilename, 'path' => $pathDomainConfig),
+            array('file' => $standardConfigName, 'path' => $pathDomainMiscUser)
+        );
+
+        return $locations;
+    }
+
+    public function getConfigHostnameIfSet()
+    {
+        if ($this->getByDomainConfigPath() === false) {
+            return false;
+        }
+        return $this->getHostname();
+    }
+
+    public function getClientSideOptions()
+    {
+        $general = $this->General;
+
+        return array(
+            'action_url_category_delimiter' => $general['action_url_category_delimiter'],
+            'action_title_category_delimiter' => $general['action_title_category_delimiter'],
+            'autocomplete_min_sites' => $general['autocomplete_min_sites'],
+            'datatable_export_range_as_day' => $general['datatable_export_range_as_day'],
+            'datatable_row_limits' => $this->getDatatableRowLimits(),
+            'are_ads_enabled' => Advertising::isAdsEnabledInConfig($general)
+        );
+    }
+
+    /**
+     * @param $general
+     * @return mixed
+     */
+    private function getDatatableRowLimits()
+    {
+        $limits = $this->General['datatable_row_limits'];
+        $limits = explode(",", $limits);
+        $limits = array_map('trim', $limits);
+        return $limits;
+    }
+
+    public static function getByDomainConfigPath()
+    {
+        $host       = self::getHostname();
+        $hostConfigs = self::getLocalConfigInfoForHostname($host);
+
+        foreach ($hostConfigs as $hostConfig) {
+            if (Filesystem::isValidFilename($hostConfig['file'])
+                && file_exists($hostConfig['path'])
+            ) {
+                return $hostConfig['path'];
+            }
+        }
+
+        return false;
+    }
+
+    /**
+     * Returns the hostname of the current request (without port number)
+     *
+     * @return string
+     */
+    public static function getHostname()
+    {
+        // Check trusted requires config file which is not ready yet
+        $host = Url::getHost($checkIfTrusted = false);
+
+        // Remove any port number to get actual hostname
+        $host = Url::getHostSanitized($host);
+
+        return $host;
+    }
+
+    /**
+     * If set, Piwik will use the hostname config no matter if it exists or not. Useful for instance if you want to
+     * create a new hostname config:
+     *
+     *     $config = Config::getInstance();
+     *     $config->forceUsageOfHostnameConfig('piwik.example.com');
+     *     $config->save();
+     *
+     * @param string $hostname eg piwik.example.com
+     * @param string $preferredPath If there are different paths for the config that can be used, eg /config/* and /misc/user/*,
+     *                              and a preferred path is given, then the config path must contain the preferred path.
+     * @return string
+     * @throws \Exception In case the domain contains not allowed characters
+     * @internal
+     */
+    public function forceUsageOfLocalHostnameConfig($hostname, $preferredPath = null)
+    {
+        $hostConfigs = self::getLocalConfigInfoForHostname($hostname);
+        $fileNames = '';
+
+        foreach ($hostConfigs as $hostConfig) {
+            if (count($hostConfigs) > 1
+                && $preferredPath
+                && strpos($hostConfig['path'], $preferredPath) === false) {
+                continue;
+            }
+
+            $filename = $hostConfig['file'];
+            $fileNames .= $filename . ' ';
+
+            if (Filesystem::isValidFilename($filename)) {
+                $pathLocal = $hostConfig['path'];
+
+                try {
+                    $this->reload($pathLocal);
+                } catch (Exception $ex) {
+                    // pass (not required for local file to exist at this point)
+                }
+
+                return $pathLocal;
+            }
+        }
+
+        throw new Exception('Matomo domain is not a valid looking hostname (' . trim($fileNames) . ').');
+    }
+
+    /**
+     * Returns `true` if the local configuration file is writable.
+     *
+     * @return bool
+     */
+    public function isFileWritable()
+    {
+        return is_writable($this->settings->getPathLocal());
+    }
+
+    /**
+     * Clear in-memory configuration so it can be reloaded
+     * @deprecated since v2.12.0
+     */
+    public function clear()
+    {
+        $this->reload();
+    }
+
+    /**
+     * Read configuration from files into memory
+     *
+     * @throws Exception if local config file is not readable; exits for other errors
+     * @deprecated since v2.12.0
+     */
+    public function init()
+    {
+        $this->reload();
+    }
+
+    /**
+     * Reloads config data from disk.
+     *
+     * @throws \Exception if the global config file is not found and this is a tracker request, or
+     *                    if the local config file is not found and this is NOT a tracker request.
+     */
+    protected function reload($pathLocal = null, $pathGlobal = null, $pathCommon = null)
+    {
+        $this->settings->reload($pathGlobal, $pathLocal, $pathCommon);
+    }
+
+    /**
+     * @deprecated
+     */
+    public function existsLocalConfig()
+    {
+        return is_readable($this->getLocalPath());
+    }
+
+    public function deleteLocalConfig()
+    {
+        $configLocal = $this->getLocalPath();
+        
+        if(file_exists($configLocal)){
+            @unlink($configLocal);
+        }
+    }
+
+    /**
+     * Returns a configuration value or section by name.
+     *
+     * @param string $name The value or section name.
+     * @return string|array The requested value requested. Returned by reference.
+     * @throws Exception If the value requested not found in either `config.ini.php` or
+     *                   `global.ini.php`.
+     * @api
+     */
+    public function &__get($name)
+    {
+        $section =& $this->settings->getIniFileChain()->get($name);
+        return $section;
+    }
+
+    /**
+     * @api
+     */
+    public function getFromGlobalConfig($name)
+    {
+        return $this->settings->getIniFileChain()->getFrom($this->getGlobalPath(), $name);
+    }
+
+    /**
+     * @api
+     */
+    public function getFromCommonConfig($name)
+    {
+        return $this->settings->getIniFileChain()->getFrom($this->getCommonPath(), $name);
+    }
+    
+    /**
+     * @api
+     */
+    public function getFromLocalConfig($name)
+    {
+        return $this->settings->getIniFileChain()->getFrom($this->getLocalPath(), $name);
+    }
+
+    /**
+     * Sets a configuration value or section.
+     *
+     * @param string $name This section name or value name to set.
+     * @param mixed $value
+     * @api
+     */
+    public function __set($name, $value)
+    {
+        $this->settings->getIniFileChain()->set($name, $value);
+    }
+
+    /**
+     * Dump config
+     *
+     * @return string|null
+     * @throws \Exception
+     */
+    public function dumpConfig()
+    {
+        $chain = $this->settings->getIniFileChain();
+
+        $header = "; <?php exit; ?> DO NOT REMOVE THIS LINE\n";
+        $header .= "; file automatically generated or modified by Matomo; you can manually override the default values in global.ini.php by redefining them in this file.\n";
+        return $chain->dumpChanges($header);
+    }
+
+    /**
+     * Write user configuration file
+     *
+     * @param array $configLocal
+     * @param array $configGlobal
+     * @param array $configCommon
+     * @param array $configCache
+     * @param string $pathLocal
+     * @param bool $clear
+     *
+     * @throws \Exception if config file not writable
+     */
+    protected function writeConfig($clear = true)
+    {
+        $output = $this->dumpConfig();
+        if ($output !== null
+            && $output !== false
+        ) {
+            $localPath = $this->getLocalPath();
+
+            if ($this->doNotWriteConfigInTests) {
+                // simulate whether it would be successful
+                $success = is_writable($localPath);
+            } else {
+                $success = @file_put_contents($localPath, $output, LOCK_EX);
+            }
+
+            if ($success === false) {
+                throw $this->getConfigNotWritableException();
+            }
+
+            /**
+             * Triggered when a INI config file is changed on disk.
+             *
+             * @param string $localPath Absolute path to the changed file on the server.
+             */
+            Piwik::postEvent('Core.configFileChanged', [$localPath]);
+        }
+
+        if ($clear) {
+            $this->reload();
+        }
+    }
+
+    /**
+     * Writes the current configuration to the **config.ini.php** file. Only writes options whose
+     * values are different from the default.
+     *
+     * @api
+     */
+    public function forceSave()
+    {
+        $this->writeConfig();
+    }
+
+    /**
+     * @throws \Exception
+     */
+    public function getConfigNotWritableException()
+    {
+        $path = "config/" . basename($this->getLocalPath());
+        return new MissingFilePermissionException(Piwik::translate('General_ConfigFileIsNotWritable', array("(" . $path . ")", "")));
+    }
+
+    /**
+     * Convenience method for setting settings in a single section. Will set them in a new array first
+     * to be compatible with certain PHP versions.
+     *
+     * @param string $sectionName Section name.
+     * @param string $name The setting name.
+     * @param mixed $value The setting value to set.
+     */
+    public static function setSetting($sectionName, $name, $value)
+    {
+        $section = self::getInstance()->$sectionName;
+        $section[$name] = $value;
+        self::getInstance()->$sectionName = $section;
+    }
+}

msd2/tracking/piwik/core/Config/ConfigNotFoundException.php

@@ -0,0 +1,16 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+
+namespace Piwik\Config;
+
+/**
+ * Exception thrown when the config file doesn't exist.
+ */
+class ConfigNotFoundException extends \Exception
+{
+}

msd2/tracking/piwik/core/Config/IniFileChain.php

@@ -0,0 +1,482 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+namespace Piwik\Config;
+
+use Piwik\Common;
+use Piwik\Ini\IniReader;
+use Piwik\Ini\IniReadingException;
+use Piwik\Ini\IniWriter;
+
+/**
+ * Manages a list of INI files where the settings in each INI file merge with or override the
+ * settings in the previous INI file.
+ *
+ * The IniFileChain class manages two types of INI files: multiple default setting files and one
+ * user settings file.
+ *
+ * The default setting files (for example, global.ini.php & common.ini.php) hold the default setting values.
+ * The settings in these files are merged recursively, however, array settings in one file will still
+ * overwrite settings in the previous file.
+ *
+ * Default settings files cannot be modified through the IniFileChain class.
+ *
+ * The user settings file (for example, config.ini.php) holds the actual setting values. Settings in the
+ * user settings files overwrite other settings. So array settings will not merge w/ previous values.
+ *
+ * HTML characters and dollar signs are stored as encoded HTML entities in INI files. This prevents
+ * several `parse_ini_file` issues, including one where parse_ini_file tries to insert a variable
+ * into a setting value if a string like `"$varname" is present.
+ */
+class IniFileChain
+{
+    /**
+     * Maps INI file names with their parsed contents. The order of the files signifies the order
+     * in the chain. Files with lower index are overwritten/merged with files w/ a higher index.
+     *
+     * @var array
+     */
+    protected $settingsChain = array();
+
+    /**
+     * The merged INI settings.
+     *
+     * @var array
+     */
+    protected $mergedSettings = array();
+
+    /**
+     * Constructor.
+     *
+     * @param string[] $defaultSettingsFiles The list of paths to INI files w/ the default setting values.
+     * @param string|null $userSettingsFile The path to the user settings file.
+     */
+    public function __construct(array $defaultSettingsFiles = array(), $userSettingsFile = null)
+    {
+        $this->reload($defaultSettingsFiles, $userSettingsFile);
+    }
+
+    /**
+     * Return setting section by reference.
+     *
+     * @param string $name
+     * @return mixed
+     */
+    public function &get($name)
+    {
+        if (!isset($this->mergedSettings[$name])) {
+            $this->mergedSettings[$name] = array();
+        }
+
+        $result =& $this->mergedSettings[$name];
+        return $result;
+    }
+
+    /**
+     * Return setting section from a specific file, rather than the current merged settings.
+     *
+     * @param string $file The path of the file. Should be the path used in construction or reload().
+     * @param string $name The name of the section to access.
+     */
+    public function getFrom($file, $name)
+    {
+        return @$this->settingsChain[$file][$name];
+    }
+
+    /**
+     * Sets a setting value.
+     *
+     * @param string $name
+     * @param mixed $value
+     */
+    public function set($name, $value)
+    {
+        $this->mergedSettings[$name] = $value;
+    }
+
+    /**
+     * Returns all settings. Changes made to the array result will be reflected in the
+     * IniFileChain instance.
+     *
+     * @return array
+     */
+    public function &getAll()
+    {
+        return $this->mergedSettings;
+    }
+
+    /**
+     * Dumps the current in-memory setting values to a string in INI format and returns it.
+     *
+     * @param string $header The header of the output INI file.
+     * @return string The dumped INI contents.
+     */
+    public function dump($header = '')
+    {
+        return $this->dumpSettings($this->mergedSettings, $header);
+    }
+
+    /**
+     * Writes the difference of the in-memory setting values and the on-disk user settings file setting
+     * values to a string in INI format, and returns it.
+     *
+     * If a config section is identical to the default settings section (as computed by merging
+     * all default setting files), it is not written to the user settings file.
+     *
+     * @param string $header The header of the INI output.
+     * @return string The dumped INI contents.
+     */
+    public function dumpChanges($header = '')
+    {
+        $userSettingsFile = $this->getUserSettingsFile();
+
+        $defaultSettings = $this->getMergedDefaultSettings();
+        $existingMutableSettings = $this->settingsChain[$userSettingsFile];
+
+        $dirty = false;
+
+        $configToWrite = array();
+        foreach ($this->mergedSettings as $sectionName => $changedSection) {
+            if(isset($existingMutableSettings[$sectionName])){
+                $existingMutableSection = $existingMutableSettings[$sectionName];
+            } else{
+                $existingMutableSection = array();
+            }
+
+            // remove default values from both (they should not get written to local)
+            if (isset($defaultSettings[$sectionName])) {
+                $changedSection = $this->arrayUnmerge($defaultSettings[$sectionName], $changedSection);
+                $existingMutableSection = $this->arrayUnmerge($defaultSettings[$sectionName], $existingMutableSection);
+            }
+
+            // if either local/config have non-default values and the other doesn't,
+            // OR both have values, but different values, we must write to config.ini.php
+            if (empty($changedSection) xor empty($existingMutableSection)
+                || (!empty($changedSection)
+                    && !empty($existingMutableSection)
+                    && self::compareElements($changedSection, $existingMutableSection))
+            ) {
+                $dirty = true;
+            }
+
+            $configToWrite[$sectionName] = $changedSection;
+        }
+
+        if ($dirty) {
+            // sort config sections by how early they appear in the file chain
+            $self = $this;
+            uksort($configToWrite, function ($sectionNameLhs, $sectionNameRhs) use ($self) {
+                $lhsIndex = $self->findIndexOfFirstFileWithSection($sectionNameLhs);
+                $rhsIndex = $self->findIndexOfFirstFileWithSection($sectionNameRhs);
+
+                if ($lhsIndex == $rhsIndex) {
+                    $lhsIndexInFile = $self->getIndexOfSectionInFile($lhsIndex, $sectionNameLhs);
+                    $rhsIndexInFile = $self->getIndexOfSectionInFile($rhsIndex, $sectionNameRhs);
+
+                    if ($lhsIndexInFile == $rhsIndexInFile) {
+                        return 0;
+                    } elseif ($lhsIndexInFile < $rhsIndexInFile) {
+                        return -1;
+                    } else {
+                        return 1;
+                    }
+                } elseif ($lhsIndex < $rhsIndex) {
+                    return -1;
+                } else {
+                    return 1;
+                }
+            });
+
+            return $this->dumpSettings($configToWrite, $header);
+        } else {
+            return null;
+        }
+    }
+
+    /**
+     * Reloads settings from disk.
+     */
+    public function reload($defaultSettingsFiles = array(), $userSettingsFile = null)
+    {
+        if (!empty($defaultSettingsFiles)
+            || !empty($userSettingsFile)
+        ) {
+            $this->resetSettingsChain($defaultSettingsFiles, $userSettingsFile);
+        }
+
+        $reader = new IniReader();
+        foreach ($this->settingsChain as $file => $ignore) {
+            if (is_readable($file)) {
+                try {
+                    $contents = $reader->readFile($file);
+                    $this->settingsChain[$file] = $this->decodeValues($contents);
+                } catch (IniReadingException $ex) {
+                    throw new IniReadingException('Unable to read INI file {' . $file . '}: ' . $ex->getMessage() . "\n Your host may have disabled parse_ini_file().");
+                }
+
+                $this->decodeValues($this->settingsChain[$file]);
+            }
+        }
+
+        $merged = $this->mergeFileSettings();
+        // remove reference to $this->settingsChain... otherwise dump() or compareElements() will never notice a difference
+        // on PHP 7+ as they would be always equal
+        $this->mergedSettings = $this->copy($merged);
+    }
+
+    private function copy($merged)
+    {
+        $copy = array();
+        foreach ($merged as $index => $value) {
+            if (is_array($value)) {
+                $copy[$index] = $this->copy($value);
+            } else {
+                $copy[$index] = $value;
+            }
+        }
+        return $copy;
+    }
+
+    private function resetSettingsChain($defaultSettingsFiles, $userSettingsFile)
+    {
+        $this->settingsChain = array();
+
+        if (!empty($defaultSettingsFiles)) {
+            foreach ($defaultSettingsFiles as $file) {
+                $this->settingsChain[$file] = null;
+            }
+        }
+
+        if (!empty($userSettingsFile)) {
+            $this->settingsChain[$userSettingsFile] = null;
+        }
+    }
+
+    protected function mergeFileSettings()
+    {
+        $mergedSettings = $this->getMergedDefaultSettings();
+
+        $userSettings = end($this->settingsChain) ?: array();
+        foreach ($userSettings as $sectionName => $section) {
+            if (!isset($mergedSettings[$sectionName])) {
+                $mergedSettings[$sectionName] = $section;
+            } else {
+                // the last user settings file completely overwrites INI sections. the other files in the chain
+                // can add to array options
+                $mergedSettings[$sectionName] = array_merge($mergedSettings[$sectionName], $section);
+            }
+        }
+
+        return $mergedSettings;
+    }
+
+    protected function getMergedDefaultSettings()
+    {
+        $userSettingsFile = $this->getUserSettingsFile();
+
+        $mergedSettings = array();
+        foreach ($this->settingsChain as $file => $settings) {
+            if ($file == $userSettingsFile
+                || empty($settings)
+            ) {
+                continue;
+            }
+
+            foreach ($settings as $sectionName => $section) {
+                if (!isset($mergedSettings[$sectionName])) {
+                    $mergedSettings[$sectionName] = $section;
+                } else {
+                    $mergedSettings[$sectionName] = $this->array_merge_recursive_distinct($mergedSettings[$sectionName], $section);
+                }
+            }
+        }
+        return $mergedSettings;
+    }
+
+    protected function getUserSettingsFile()
+    {
+        // the user settings file is the last key in $settingsChain
+        end($this->settingsChain);
+        return key($this->settingsChain);
+    }
+
+    /**
+     * Comparison function
+     *
+     * @param mixed $elem1
+     * @param mixed $elem2
+     * @return int;
+     */
+    public static function compareElements($elem1, $elem2)
+    {
+        if (is_array($elem1)) {
+            if (is_array($elem2)) {
+                return strcmp(serialize($elem1), serialize($elem2));
+            }
+
+            return 1;
+        }
+
+        if (is_array($elem2)) {
+            return -1;
+        }
+
+        if ((string)$elem1 === (string)$elem2) {
+            return 0;
+        }
+
+        return ((string)$elem1 > (string)$elem2) ? 1 : -1;
+    }
+
+    /**
+     * Compare arrays and return difference, such that:
+     *
+     *     $modified = array_merge($original, $difference);
+     *
+     * @param array $original original array
+     * @param array $modified modified array
+     * @return array differences between original and modified
+     */
+    public function arrayUnmerge($original, $modified)
+    {
+        // return key/value pairs for keys in $modified but not in $original
+        // return key/value pairs for keys in both $modified and $original, but values differ
+        // ignore keys that are in $original but not in $modified
+
+        if (empty($original) || !is_array($original)) {
+            $original = array();
+        }
+
+        if (empty($modified) || !is_array($modified)) {
+            $modified = array();
+        }
+
+        return array_udiff_assoc($modified, $original, array(__CLASS__, 'compareElements'));
+    }
+
+    /**
+     * array_merge_recursive does indeed merge arrays, but it converts values with duplicate
+     * keys to arrays rather than overwriting the value in the first array with the duplicate
+     * value in the second array, as array_merge does. I.e., with array_merge_recursive,
+     * this happens (documented behavior):
+     *
+     * array_merge_recursive(array('key' => 'org value'), array('key' => 'new value'));
+     *     => array('key' => array('org value', 'new value'));
+     *
+     * array_merge_recursive_distinct does not change the datatypes of the values in the arrays.
+     * Matching keys' values in the second array overwrite those in the first array, as is the
+     * case with array_merge, i.e.:
+     *
+     * array_merge_recursive_distinct(array('key' => 'org value'), array('key' => 'new value'));
+     *     => array('key' => array('new value'));
+     *
+     * Parameters are passed by reference, though only for performance reasons. They're not
+     * altered by this function.
+     *
+     * @param array $array1
+     * @param array $array2
+     * @return array
+     * @author Daniel <daniel (at) danielsmedegaardbuus (dot) dk>
+     * @author Gabriel Sobrinho <gabriel (dot) sobrinho (at) gmail (dot) com>
+     */
+    private function array_merge_recursive_distinct(array &$array1, array &$array2)
+    {
+        $merged = $array1;
+        foreach ($array2 as $key => &$value) {
+            if (is_array($value) && isset($merged [$key]) && is_array($merged [$key])) {
+                $merged [$key] = $this->array_merge_recursive_distinct($merged [$key], $value);
+            } else {
+                $merged [$key] = $value;
+            }
+        }
+        return $merged;
+    }
+
+    /**
+     * public for use in closure.
+     */
+    public function findIndexOfFirstFileWithSection($sectionName)
+    {
+        $count = 0;
+        foreach ($this->settingsChain as $file => $settings) {
+            if (isset($settings[$sectionName])) {
+                break;
+            }
+
+            ++$count;
+        }
+        return $count;
+    }
+
+    /**
+     * public for use in closure.
+     */
+    public function getIndexOfSectionInFile($fileIndex, $sectionName)
+    {
+        reset($this->settingsChain);
+        for ($i = 0; $i != $fileIndex; ++$i) {
+            next($this->settingsChain);
+        }
+
+        $settingsData = current($this->settingsChain);
+        if (empty($settingsData)) {
+            return -1;
+        }
+
+        $settingsDataSectionNames = array_keys($settingsData);
+
+        return array_search($sectionName, $settingsDataSectionNames);
+    }
+
+    /**
+     * Encode HTML entities
+     *
+     * @param mixed $values
+     * @return mixed
+     */
+    protected function encodeValues(&$values)
+    {
+        if (is_array($values)) {
+            foreach ($values as &$value) {
+                $value = $this->encodeValues($value);
+            }
+        } elseif (is_float($values)) {
+            $values = Common::forceDotAsSeparatorForDecimalPoint($values);
+        } elseif (is_string($values)) {
+            $values = htmlentities($values, ENT_COMPAT, 'UTF-8');
+            $values = str_replace('$', '&#36;', $values);
+        }
+        return $values;
+    }
+
+    /**
+     * Decode HTML entities
+     *
+     * @param mixed $values
+     * @return mixed
+     */
+    protected function decodeValues(&$values)
+    {
+        if (is_array($values)) {
+            foreach ($values as &$value) {
+                $value = $this->decodeValues($value);
+            }
+            return $values;
+        } elseif (is_string($values)) {
+            return html_entity_decode($values, ENT_COMPAT, 'UTF-8');
+        }
+        return $values;
+    }
+
+    private function dumpSettings($values, $header)
+    {
+        $values = $this->encodeValues($values);
+
+        $writer = new IniWriter();
+        return $writer->writeToString($values, $header);
+    }
+}

msd2/tracking/piwik/core/Console.php

@@ -0,0 +1,263 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik;
+
+use Piwik\Application\Environment;
+use Piwik\Config\ConfigNotFoundException;
+use Piwik\Container\StaticContainer;
+use Piwik\Plugin\Manager as PluginManager;
+use Piwik\Plugins\Monolog\Handler\FailureLogMessageDetector;
+use Piwik\Version;
+use Symfony\Bridge\Monolog\Handler\ConsoleHandler;
+use Symfony\Component\Console\Application;
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
+use Symfony\Component\Console\Output\OutputInterface;
+
+class Console extends Application
+{
+    /**
+     * @var Environment
+     */
+    private $environment;
+
+    public function __construct(Environment $environment = null)
+    {
+        $this->setServerArgsIfPhpCgi();
+
+        parent::__construct('Matomo', Version::VERSION);
+
+        $this->environment = $environment;
+
+        $option = new InputOption('matomo-domain',
+            null,
+            InputOption::VALUE_OPTIONAL,
+            'Matomo URL (protocol and domain) eg. "http://matomo.example.org"'
+        );
+
+        $this->getDefinition()->addOption($option);
+
+        // @todo  Remove this alias in Matomo 4.0
+        $option = new InputOption('piwik-domain',
+            null,
+            InputOption::VALUE_OPTIONAL,
+            '[DEPRECATED] Matomo URL (protocol and domain) eg. "http://matomo.example.org"'
+        );
+
+        $this->getDefinition()->addOption($option);
+
+        $option = new InputOption('xhprof',
+            null,
+            InputOption::VALUE_NONE,
+            'Enable profiling with XHProf'
+        );
+
+        $this->getDefinition()->addOption($option);
+    }
+
+    public function doRun(InputInterface $input, OutputInterface $output)
+    {
+        if ($input->hasParameterOption('--xhprof')) {
+            Profiler::setupProfilerXHProf(true, true);
+        }
+
+        $this->initMatomoHost($input);
+        $this->initEnvironment($output);
+        $this->initLoggerOutput($output);
+
+        try {
+            self::initPlugins();
+        } catch (ConfigNotFoundException $e) {
+            // Piwik not installed yet, no config file?
+            Log::warning($e->getMessage());
+        }
+
+        $commands = $this->getAvailableCommands();
+
+        foreach ($commands as $command) {
+            $this->addCommandIfExists($command);
+        }
+
+        $exitCode = null;
+
+        /**
+         * @ignore
+         */
+        Piwik::postEvent('Console.doRun', [&$exitCode, $input, $output]);
+
+        if ($exitCode === null) {
+            $self = $this;
+            $exitCode = Access::doAsSuperUser(function () use ($input, $output, $self) {
+                return call_user_func(array($self, 'Symfony\Component\Console\Application::doRun'), $input, $output);
+            });
+        }
+
+        $importantLogDetector = StaticContainer::get(FailureLogMessageDetector::class);
+        if ($exitCode === 0 && $importantLogDetector->hasEncounteredImportantLog()) {
+            $output->writeln("Error: error or warning logs detected, exit 1");
+            $exitCode = 1;
+        }
+
+        return $exitCode;
+    }
+
+    private function addCommandIfExists($command)
+    {
+        if (!class_exists($command)) {
+            Log::warning(sprintf('Cannot add command %s, class does not exist', $command));
+        } elseif (!is_subclass_of($command, 'Piwik\Plugin\ConsoleCommand')) {
+            Log::warning(sprintf('Cannot add command %s, class does not extend Piwik\Plugin\ConsoleCommand', $command));
+        } else {
+            /** @var Command $commandInstance */
+            $commandInstance = new $command;
+
+            // do not add the command if it already exists; this way we can add the command ourselves in tests
+            if (!$this->has($commandInstance->getName())) {
+                $this->add($commandInstance);
+            }
+        }
+    }
+
+    /**
+     * Returns a list of available command classnames.
+     *
+     * @return string[]
+     */
+    private function getAvailableCommands()
+    {
+        $commands = $this->getDefaultPiwikCommands();
+        $detected = PluginManager::getInstance()->findMultipleComponents('Commands', 'Piwik\\Plugin\\ConsoleCommand');
+
+        $commands = array_merge($commands, $detected);
+
+        /**
+         * Triggered to filter / restrict console commands. Plugins that want to restrict commands
+         * should subscribe to this event and remove commands from the existing list.
+         *
+         * **Example**
+         *
+         *     public function filterConsoleCommands(&$commands)
+         *     {
+         *         $key = array_search('Piwik\Plugins\MyPlugin\Commands\MyCommand', $commands);
+         *         if (false !== $key) {
+         *             unset($commands[$key]);
+         *         }
+         *     }
+         *
+         * @param array &$commands An array containing a list of command class names.
+         */
+        Piwik::postEvent('Console.filterCommands', array(&$commands));
+
+        $commands = array_values(array_unique($commands));
+
+        return $commands;
+    }
+
+    private function setServerArgsIfPhpCgi()
+    {
+        if (Common::isPhpCgiType()) {
+            $_SERVER['argv'] = array();
+            foreach ($_GET as $name => $value) {
+                $argument = $name;
+                if (!empty($value)) {
+                    $argument .= '=' . $value;
+                }
+
+                $_SERVER['argv'][] = $argument;
+            }
+
+            if (!defined('STDIN')) {
+                define('STDIN', fopen('php://stdin', 'r'));
+            }
+        }
+    }
+
+    public static function isSupported()
+    {
+        return Common::isPhpCliMode() && !Common::isPhpCgiType();
+    }
+
+    protected function initMatomoHost(InputInterface $input)
+    {
+        $matomoHostname = $input->getParameterOption('--matomo-domain');
+
+        if (empty($matomoHostname)) {
+            $matomoHostname = $input->getParameterOption('--piwik-domain');
+        }
+
+        if (empty($matomoHostname)) {
+            $matomoHostname = $input->getParameterOption('--url');
+        }
+
+        $matomoHostname = UrlHelper::getHostFromUrl($matomoHostname);
+        Url::setHost($matomoHostname);
+    }
+
+    protected function initEnvironment(OutputInterface $output)
+    {
+        try {
+            if ($this->environment === null) {
+                $this->environment = new Environment('cli');
+                $this->environment->init();
+            }
+
+            $config = Config::getInstance();
+            return $config;
+        } catch (\Exception $e) {
+            $output->writeln($e->getMessage() . "\n");
+        }
+    }
+
+    /**
+     * Register the console output into the logger.
+     *
+     * Ideally, this should be done automatically with events:
+     * @see http://symfony.com/fr/doc/current/components/console/events.html
+     * @see Symfony\Bridge\Monolog\Handler\ConsoleHandler::onCommand()
+     * But it would require to install Symfony's Event Dispatcher.
+     */
+    private function initLoggerOutput(OutputInterface $output)
+    {
+        /** @var ConsoleHandler $consoleLogHandler */
+        $consoleLogHandler = StaticContainer::get('Symfony\Bridge\Monolog\Handler\ConsoleHandler');
+        $consoleLogHandler->setOutput($output);
+    }
+
+    public static function initPlugins()
+    {
+        Plugin\Manager::getInstance()->loadActivatedPlugins();
+        Plugin\Manager::getInstance()->loadPluginTranslations();
+    }
+
+    private function getDefaultPiwikCommands()
+    {
+        $commands = array(
+            'Piwik\CliMulti\RequestCommand'
+        );
+
+        $commandsFromPluginsMarkedInConfig = $this->getCommandsFromPluginsMarkedInConfig();
+        $commands = array_merge($commands, $commandsFromPluginsMarkedInConfig);
+
+        return $commands;
+    }
+
+    private function getCommandsFromPluginsMarkedInConfig()
+    {
+        $plugins = Config::getInstance()->General['always_load_commands_from_plugin'];
+        $plugins = explode(',', $plugins);
+
+        $commands = array();
+        foreach($plugins as $plugin) {
+            $instance = new Plugin($plugin);
+            $commands = array_merge($commands, $instance->findMultipleComponents('Commands', 'Piwik\\Plugin\\ConsoleCommand'));
+        }
+        return $commands;
+    }
+}

msd2/tracking/piwik/core/Container/ContainerDoesNotExistException.php

@@ -0,0 +1,18 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+
+namespace Piwik\Container;
+
+use RuntimeException;
+
+/**
+ * Thrown if the root container has not been created and set in StaticContainer.
+ */
+class ContainerDoesNotExistException extends RuntimeException
+{
+}

msd2/tracking/piwik/core/Container/ContainerFactory.php

@@ -0,0 +1,152 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+
+namespace Piwik\Container;
+
+use DI\Container;
+use DI\ContainerBuilder;
+use Doctrine\Common\Cache\ArrayCache;
+use Piwik\Application\Kernel\GlobalSettingsProvider;
+use Piwik\Application\Kernel\PluginList;
+use Piwik\Plugin\Manager;
+
+/**
+ * Creates a configured DI container.
+ */
+class ContainerFactory
+{
+    /**
+     * @var PluginList
+     */
+    private $pluginList;
+
+    /**
+     * @var GlobalSettingsProvider
+     */
+    private $settings;
+
+    /**
+     * Optional environment configs to load.
+     *
+     * @var string[]
+     */
+    private $environments;
+
+    /**
+     * @var array[]
+     */
+    private $definitions;
+
+    /**
+     * @param PluginList $pluginList
+     * @param GlobalSettingsProvider $settings
+     * @param string[] $environment Optional environment configs to load.
+     * @param array[] $definitions
+     */
+    public function __construct(PluginList $pluginList, GlobalSettingsProvider $settings, array $environments = array(), array $definitions = array())
+    {
+        $this->pluginList = $pluginList;
+        $this->settings = $settings;
+        $this->environments = $environments;
+        $this->definitions = $definitions;
+    }
+
+    /**
+     * @link http://php-di.org/doc/container-configuration.html
+     * @throws \Exception
+     * @return Container
+     */
+    public function create()
+    {
+        $builder = new ContainerBuilder();
+
+        $builder->useAnnotations(false);
+        $builder->setDefinitionCache(new ArrayCache());
+
+        // INI config
+        $builder->addDefinitions(new IniConfigDefinitionSource($this->settings));
+
+        // Global config
+        $builder->addDefinitions(PIWIK_USER_PATH . '/config/global.php');
+
+        // Plugin configs
+        $this->addPluginConfigs($builder);
+
+        // Development config
+        if ($this->isDevelopmentModeEnabled()) {
+            $this->addEnvironmentConfig($builder, 'dev');
+        }
+
+        // Environment config
+        foreach ($this->environments as $environment) {
+            $this->addEnvironmentConfig($builder, $environment);
+        }
+
+        // User config
+        if (file_exists(PIWIK_USER_PATH . '/config/config.php')
+            && !in_array('test', $this->environments, true)) {
+            $builder->addDefinitions(PIWIK_USER_PATH . '/config/config.php');
+        }
+
+        if (!empty($this->definitions)) {
+            foreach ($this->definitions as $definitionArray) {
+                $builder->addDefinitions($definitionArray);
+            }
+        }
+
+        $container = $builder->build();
+        $container->set('Piwik\Application\Kernel\PluginList', $this->pluginList);
+        $container->set('Piwik\Application\Kernel\GlobalSettingsProvider', $this->settings);
+
+        return $container;
+    }
+
+    private function addEnvironmentConfig(ContainerBuilder $builder, $environment)
+    {
+        if (!$environment) {
+            return;
+        }
+
+        $file = sprintf('%s/config/environment/%s.php', PIWIK_USER_PATH, $environment);
+
+        if (file_exists($file)) {
+            $builder->addDefinitions($file);
+        }
+
+        // add plugin environment configs
+        $plugins = $this->pluginList->getActivatedPlugins();
+        foreach ($plugins as $plugin) {
+            $baseDir = Manager::getPluginDirectory($plugin);
+
+            $environmentFile = $baseDir . '/config/' . $environment . '.php';
+            if (file_exists($environmentFile)) {
+                $builder->addDefinitions($environmentFile);
+            }
+        }
+    }
+
+    private function addPluginConfigs(ContainerBuilder $builder)
+    {
+        $plugins = $this->pluginList->getActivatedPlugins();
+
+        foreach ($plugins as $plugin) {
+            $baseDir = Manager::getPluginDirectory($plugin);
+
+            $file = $baseDir . '/config/config.php';
+            if (file_exists($file)) {
+                $builder->addDefinitions($file);
+            }
+        }
+    }
+
+    private function isDevelopmentModeEnabled()
+    {
+        $section = $this->settings->getSection('Development');
+        return (bool) @$section['enabled']; // TODO: code redundancy w/ Development. hopefully ok for now.
+    }
+}

msd2/tracking/piwik/core/Container/IniConfigDefinitionSource.php

@@ -0,0 +1,95 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+
+namespace Piwik\Container;
+
+use DI\Definition\Exception\DefinitionException;
+use DI\Definition\Source\DefinitionSource;
+use DI\Definition\ValueDefinition;
+use Piwik\Application\Kernel\GlobalSettingsProvider;
+
+/**
+ * Expose the INI config into PHP-DI.
+ *
+ * The INI config can be used by prefixing `ini.` before the setting we want to get:
+ *
+ *     $maintenanceMode = $container->get('ini.General.maintenance_mode');
+ */
+class IniConfigDefinitionSource implements DefinitionSource
+{
+    /**
+     * @var GlobalSettingsProvider
+     */
+    private $config;
+
+    /**
+     * @var string
+     */
+    private $prefix;
+
+    /**
+     * @param GlobalSettingsProvider $config
+     * @param string $prefix Prefix for the container entries.
+     */
+    public function __construct(GlobalSettingsProvider $config, $prefix = 'ini.')
+    {
+        $this->config = $config;
+        $this->prefix = $prefix;
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function getDefinition($name)
+    {
+        if (strpos($name, $this->prefix) !== 0) {
+            return null;
+        }
+
+        list($sectionName, $configKey) = $this->parseEntryName($name);
+
+        $section = $this->getSection($sectionName);
+
+        if ($configKey === null) {
+            return new ValueDefinition($name, $section);
+        }
+
+        if (! array_key_exists($configKey, $section)) {
+            return null;
+        }
+
+        return new ValueDefinition($name, $section[$configKey]);
+    }
+
+    private function parseEntryName($name)
+    {
+        $parts = explode('.', $name, 3);
+
+        array_shift($parts);
+
+        if (! isset($parts[1])) {
+            $parts[1] = null;
+        }
+
+        return $parts;
+    }
+
+    private function getSection($sectionName)
+    {
+        $section = $this->config->getSection($sectionName);
+
+        if (!is_array($section)) {
+            throw new DefinitionException(sprintf(
+                'IniFileChain did not return an array for the config section %s',
+                $section
+            ));
+        }
+
+        return $section;
+    }
+}

msd2/tracking/piwik/core/Container/StaticContainer.php

@@ -0,0 +1,87 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+
+namespace Piwik\Container;
+
+use DI\Container;
+
+/**
+ * This class provides a static access to the container.
+ *
+ * @deprecated This class is introduced only to keep BC with the current static architecture. It will be removed in 3.0.
+ *     - it is global state (that class makes the container a global variable)
+ *     - using the container directly is the "service locator" anti-pattern (which is not dependency injection)
+ */
+class StaticContainer
+{
+    /**
+     * @var Container[]
+     */
+    private static $containerStack = array();
+
+    /**
+     * Definitions to register in the container.
+     *
+     * @var array[]
+     */
+    private static $definitions = array();
+
+    /**
+     * @return Container
+     */
+    public static function getContainer()
+    {
+        if (empty(self::$containerStack)) {
+            throw new ContainerDoesNotExistException("The root container has not been created yet.");
+        }
+
+        return end(self::$containerStack);
+    }
+
+    public static function clearContainer()
+    {
+        self::pop();
+    }
+
+    /**
+     * Only use this in tests.
+     *
+     * @param Container $container
+     */
+    public static function push(Container $container)
+    {
+        self::$containerStack[] = $container;
+    }
+
+    public static function pop()
+    {
+        array_pop(self::$containerStack);
+    }
+
+    public static function addDefinitions(array $definitions)
+    {
+        self::$definitions[] = $definitions;
+    }
+
+    /**
+     * Proxy to Container::get()
+     *
+     * @param string $name Container entry name.
+     * @return mixed
+     * @throws \DI\NotFoundException
+     */
+    public static function get($name)
+    {
+        return self::getContainer()->get($name);
+    }
+
+    public static function getDefinitions()
+    {
+        return self::$definitions;
+    }
+}

msd2/tracking/piwik/core/Context.php

@@ -0,0 +1,94 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+
+namespace Piwik;
+
+/**
+ * Methods related to changing the context Matomo code is running in.
+ */
+class Context
+{
+    public static function executeWithQueryParameters(array $parametersRequest, callable $callback)
+    {
+        // Temporarily sets the Request array to this API call context
+        $saveGET = $_GET;
+        $savePOST = $_POST;
+        $saveQUERY_STRING = @$_SERVER['QUERY_STRING'];
+        foreach ($parametersRequest as $param => $value) {
+            $_GET[$param] = $value;
+            $_POST[$param] = $value;
+        }
+
+        try {
+            return $callback();
+        } finally {
+            $_GET = $saveGET;
+            $_POST = $savePOST;
+            $_SERVER['QUERY_STRING'] = $saveQUERY_STRING;
+        }
+    }
+
+    /**
+     * Temporarily overwrites the idSite parameter so all code executed by `$callback()`
+     * will use that idSite.
+     *
+     * Useful when you need to change the idSite context for a chunk of code. For example,
+     * if we are archiving for more than one site in sequence, we don't want to use
+     * the same caches for both archiving executions.
+     *
+     * @param string|int $idSite
+     * @param callable $callback
+     * @return mixed returns result of $callback
+     */
+    public static function changeIdSite($idSite, $callback)
+    {
+        // temporarily set the idSite query parameter so archiving will end up using
+        // the correct site aware caches
+        $originalGetIdSite = isset($_GET['idSite']) ? $_GET['idSite'] : null;
+        $originalPostIdSite = isset($_POST['idSite']) ? $_POST['idSite'] : null;
+
+        $originalGetIdSites = isset($_GET['idSites']) ? $_GET['idSites'] : null;
+        $originalPostIdSites = isset($_POST['idSites']) ? $_POST['idSites'] : null;
+
+        $originalTrackerGetIdSite = isset($_GET['idsite']) ? $_GET['idsite'] : null;
+        $originalTrackerPostIdSite = isset($_POST['idsite']) ? $_POST['idsite'] : null;
+
+        try {
+            $_GET['idSite'] = $_POST['idSite'] = $idSite;
+
+            if (Tracker::$initTrackerMode) {
+                $_GET['idsite'] = $_POST['idsite'] = $idSite;
+            }
+
+            // idSites is a deprecated query param that is still in use. since it is deprecated and new
+            // supported code shouldn't rely on it, we can (more) safely unset it here, since we are just
+            // calling downstream matomo code. we unset it because we don't want it interfering w/
+            // code in $callback().
+            unset($_GET['idSites']);
+            unset($_POST['idSites']);
+
+            return $callback();
+        } finally {
+            self::resetIdSiteParam($_GET, 'idSite', $originalGetIdSite);
+            self::resetIdSiteParam($_POST, 'idSite', $originalPostIdSite);
+            self::resetIdSiteParam($_GET, 'idSites', $originalGetIdSites);
+            self::resetIdSiteParam($_POST, 'idSites', $originalPostIdSites);
+            self::resetIdSiteParam($_GET, 'idsite', $originalTrackerGetIdSite);
+            self::resetIdSiteParam($_POST, 'idsite', $originalTrackerPostIdSite);
+        }
+    }
+
+    private static function resetIdSiteParam(&$superGlobal, $paramName, $originalValue)
+    {
+        if ($originalValue !== null) {
+            $superGlobal[$paramName] = $originalValue;
+        } else {
+            unset($superGlobal[$paramName]);
+        }
+    }
+}

msd2/tracking/piwik/core/Cookie.php

@@ -0,0 +1,417 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik;
+
+/**
+ * Simple class to handle the cookies:
+ * - read a cookie values
+ * - edit an existing cookie and save it
+ * - create a new cookie, set values, expiration date, etc. and save it
+ *
+ */
+class Cookie
+{
+    /**
+     * Don't create a cookie bigger than 1k
+     */
+    const MAX_COOKIE_SIZE = 1024;
+
+    /**
+     * The name of the cookie
+     * @var string
+     */
+    protected $name = null;
+
+    /**
+     * The expire time for the cookie (expressed in UNIX Timestamp)
+     * @var int
+     */
+    protected $expire = null;
+
+    /**
+     * Restrict cookie path
+     * @var string
+     */
+    protected $path = '';
+
+    /**
+     * Restrict cookie to a domain (or subdomains)
+     * @var string
+     */
+    protected $domain = '';
+
+    /**
+     * If true, cookie should only be transmitted over secure HTTPS
+     * @var bool
+     */
+    protected $secure = false;
+
+    /**
+     * If true, cookie will only be made available via the HTTP protocol.
+     * Note: not well supported by browsers.
+     * @var bool
+     */
+    protected $httponly = false;
+
+    /**
+     * The content of the cookie
+     * @var array
+     */
+    protected $value = array();
+
+    /**
+     * The character used to separate the tuple name=value in the cookie
+     */
+    const VALUE_SEPARATOR = ':';
+
+    /**
+     * Instantiate a new Cookie object and tries to load the cookie content if the cookie
+     * exists already.
+     *
+     * @param string $cookieName cookie Name
+     * @param int $expire The timestamp after which the cookie will expire, eg time() + 86400;
+     *                                  use 0 (int zero) to expire cookie at end of browser session
+     * @param string $path The path on the server in which the cookie will be available on.
+     * @param bool|string $keyStore Will be used to store several bits of data (eg. one array per website)
+     */
+    public function __construct($cookieName, $expire = null, $path = null, $keyStore = false)
+    {
+        $this->name = $cookieName;
+        $this->path = $path;
+        $this->expire = $expire;
+        if (is_null($expire)
+            || !is_numeric($expire)
+            || $expire < 0
+        ) {
+            $this->expire = $this->getDefaultExpire();
+        }
+
+        $this->keyStore = $keyStore;
+        if ($this->isCookieFound()) {
+            $this->loadContentFromCookie();
+        }
+    }
+
+    /**
+     * Returns true if the visitor already has the cookie.
+     *
+     * @return bool
+     */
+    public function isCookieFound()
+    {
+        return self::isCookieInRequest($this->name);
+    }
+
+    /**
+     * Returns the default expiry time, 2 years
+     *
+     * @return int  Timestamp in 2 years
+     */
+    protected function getDefaultExpire()
+    {
+        return time() + 86400 * 365 * 2;
+    }
+
+    /**
+     * setcookie() replacement -- we don't use the built-in function because
+     * it is buggy for some PHP versions.
+     *
+     * @link http://php.net/setcookie
+     *
+     * @param string $Name Name of cookie
+     * @param string $Value Value of cookie
+     * @param int $Expires Time the cookie expires
+     * @param string $Path
+     * @param string $Domain
+     * @param bool $Secure
+     * @param bool $HTTPOnly
+     */
+    protected function setCookie($Name, $Value, $Expires, $Path = '', $Domain = '', $Secure = false, $HTTPOnly = false)
+    {
+        if (!empty($Domain)) {
+            // Fix the domain to accept domains with and without 'www.'.
+            if (!strncasecmp($Domain, 'www.', 4)) {
+                $Domain = substr($Domain, 4);
+            }
+            $Domain = '.' . $Domain;
+
+            // Remove port information.
+            $Port = strpos($Domain, ':');
+            if ($Port !== false) {
+                $Domain = substr($Domain, 0, $Port);
+            }
+        }
+
+        $header = 'Set-Cookie: ' . rawurlencode($Name) . '=' . rawurlencode($Value)
+            . (empty($Expires) ? '' : '; expires=' . gmdate('D, d-M-Y H:i:s', $Expires) . ' GMT')
+            . (empty($Path) ? '' : '; path=' . $Path)
+            . (empty($Domain) ? '' : '; domain=' . $Domain)
+            . (!$Secure ? '' : '; secure')
+            . (!$HTTPOnly ? '' : '; HttpOnly');
+
+        Common::sendHeader($header, false);
+    }
+
+    /**
+     * We set the privacy policy header
+     */
+    protected function setP3PHeader()
+    {
+        Common::sendHeader("P3P: CP='OTI DSP COR NID STP UNI OTPa OUR'");
+    }
+
+    /**
+     * Delete the cookie
+     */
+    public function delete()
+    {
+        $this->setP3PHeader();
+        $this->setCookie($this->name, 'deleted', time() - 31536001, $this->path, $this->domain);
+    }
+
+    /**
+     * Saves the cookie (set the Cookie header).
+     * You have to call this method before sending any text to the browser or you would get the
+     * "Header already sent" error.
+     */
+    public function save()
+    {
+        $cookieString = $this->generateContentString();
+        if (strlen($cookieString) > self::MAX_COOKIE_SIZE) {
+            // If the cookie was going to be too large, instead, delete existing cookie and start afresh
+            $this->delete();
+            return;
+        }
+
+        $this->setP3PHeader();
+        $this->setCookie($this->name, $cookieString, $this->expire, $this->path, $this->domain, $this->secure, $this->httponly);
+    }
+
+    /**
+     * Extract signed content from string: content VALUE_SEPARATOR '_=' signature
+     *
+     * @param string $content
+     * @return string|bool  Content or false if unsigned
+     */
+    private function extractSignedContent($content)
+    {
+        $signature = substr($content, -40);
+
+        if (substr($content, -43, 3) == self::VALUE_SEPARATOR . '_=' &&
+            $signature == sha1(substr($content, 0, -40) . SettingsPiwik::getSalt())
+        ) {
+            // strip trailing: VALUE_SEPARATOR '_=' signature"
+            return substr($content, 0, -43);
+        }
+
+        return false;
+    }
+
+    /**
+     * Load the cookie content into a php array.
+     * Parses the cookie string to extract the different variables.
+     * Unserialize the array when necessary.
+     * Decode the non numeric values that were base64 encoded.
+     */
+    protected function loadContentFromCookie()
+    {
+        $cookieStr = $this->extractSignedContent($_COOKIE[$this->name]);
+
+        if ($cookieStr === false) {
+            return;
+        }
+
+        $values = explode(self::VALUE_SEPARATOR, $cookieStr);
+        foreach ($values as $nameValue) {
+            $equalPos = strpos($nameValue, '=');
+            $varName = substr($nameValue, 0, $equalPos);
+            $varValue = substr($nameValue, $equalPos + 1);
+
+            // no numeric value are base64 encoded so we need to decode them
+            if (!is_numeric($varValue)) {
+                $tmpValue = base64_decode($varValue);
+                $varValue = safe_unserialize($tmpValue);
+
+                // discard entire cookie
+                // note: this assumes we never serialize a boolean
+                if ($varValue === false && $tmpValue !== 'b:0;') {
+                    $this->value = array();
+                    unset($_COOKIE[$this->name]);
+                    break;
+                }
+            }
+
+            $this->value[$varName] = $varValue;
+        }
+    }
+
+    /**
+     * Returns the string to save in the cookie from the $this->value array of values.
+     * It goes through the array and generates the cookie content string.
+     *
+     * @return string  Cookie content
+     */
+    public function generateContentString()
+    {
+        $cookieStr = '';
+
+        foreach ($this->value as $name => $value) {
+            if (!is_numeric($value)) {
+                $value = base64_encode(safe_serialize($value));
+            }
+
+            $cookieStr .= "$name=$value" . self::VALUE_SEPARATOR;
+        }
+
+        if (!empty($cookieStr)) {
+            $cookieStr .= '_=';
+
+            // sign cookie
+            $signature = sha1($cookieStr . SettingsPiwik::getSalt());
+            return $cookieStr . $signature;
+        }
+
+        return '';
+    }
+
+    /**
+     * Set cookie domain
+     *
+     * @param string $domain
+     */
+    public function setDomain($domain)
+    {
+        $this->domain = $domain;
+    }
+
+    /**
+     * Set secure flag
+     *
+     * @param bool $secure
+     */
+    public function setSecure($secure)
+    {
+        $this->secure = $secure;
+    }
+
+    /**
+     * Set HTTP only
+     *
+     * @param bool $httponly
+     */
+    public function setHttpOnly($httponly)
+    {
+        $this->httponly = $httponly;
+    }
+
+    /**
+     * Registers a new name => value association in the cookie.
+     *
+     * Registering new values is optimal if the value is a numeric value.
+     * If the value is a string, it will be saved as a base64 encoded string.
+     * If the value is an array, it will be saved as a serialized and base64 encoded
+     * string which is not very good in terms of bytes usage.
+     * You should save arrays only when you are sure about their maximum data size.
+     * A cookie has to stay small and its size shouldn't increase over time!
+     *
+     * @param string $name Name of the value to save; the name will be used to retrieve this value
+     * @param string|array|number $value Value to save. If null, entry will be deleted from cookie.
+     */
+    public function set($name, $value)
+    {
+        $name = self::escapeValue($name);
+
+        // Delete value if $value === null
+        if (is_null($value)) {
+            if ($this->keyStore === false) {
+                unset($this->value[$name]);
+                return;
+            }
+            unset($this->value[$this->keyStore][$name]);
+            return;
+        }
+
+        if ($this->keyStore === false) {
+            $this->value[$name] = $value;
+            return;
+        }
+
+        $this->value[$this->keyStore][$name] = $value;
+    }
+
+    /**
+     * Returns the value defined by $name from the cookie.
+     *
+     * @param string|integer Index name of the value to return
+     * @return mixed  The value if found, false if the value is not found
+     */
+    public function get($name)
+    {
+        $name = self::escapeValue($name);
+        if (false === $this->keyStore) {
+            if (isset($this->value[$name])) {
+                return self::escapeValue($this->value[$name]);
+            }
+
+            return false;
+        }
+
+        if (isset($this->value[$this->keyStore][$name])) {
+            return self::escapeValue($this->value[$this->keyStore][$name]);
+        }
+
+        return false;
+    }
+
+    /**
+     * Removes all values from the cookie.
+     */
+    public function clear()
+    {
+        $this->value = [];
+    }
+
+    /**
+     * Returns an easy to read cookie dump
+     *
+     * @return string  The cookie dump
+     */
+    public function __toString()
+    {
+        $str  = 'COOKIE ' . $this->name . ', rows count: ' . count($this->value) . ', cookie size = ' . strlen($this->generateContentString()) . " bytes, ";
+        $str .= 'path: ' . $this->path. ', expire: ' . $this->expire . "\n";
+        $str .= var_export($this->value, $return = true);
+
+        return $str;
+    }
+
+    /**
+     * Escape values from the cookie before sending them back to the client
+     * (when using the get() method).
+     *
+     * @param string $value Value to be escaped
+     * @return mixed  The value once cleaned.
+     */
+    protected static function escapeValue($value)
+    {
+        return Common::sanitizeInputValues($value);
+    }
+
+    /**
+     * Returns true if a cookie named '$name' is in the current HTTP request,
+     * false if otherwise.
+     *
+     * @param string $name the name of the cookie
+     * @return boolean
+     */
+    public static function isCookieInRequest($name)
+    {
+        return isset($_COOKIE[$name]);
+    }
+}

msd2/tracking/piwik/core/CronArchive/FixedSiteIds.php

@@ -0,0 +1,65 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\CronArchive;
+
+
+class FixedSiteIds
+{
+    private $siteIds = array();
+    private $index   = -1;
+
+    public function __construct($websiteIds)
+    {
+        if (!empty($websiteIds)) {
+            $this->siteIds = array_values($websiteIds);
+        }
+    }
+
+    public function getInitialSiteIds()
+    {
+        return $this->siteIds;
+    }
+
+    /**
+     * Get the number of total websites that needs to be processed.
+     *
+     * @return int
+     */
+    public function getNumSites()
+    {
+        return count($this->siteIds);
+    }
+
+    /**
+     * Get the number of already processed websites. All websites were processed by the current archiver.
+     *
+     * @return int
+     */
+    public function getNumProcessedWebsites()
+    {
+        $numProcessed = $this->index + 1;
+
+        if ($numProcessed > $this->getNumSites()) {
+            return $this->getNumSites();
+        }
+
+        return $numProcessed;
+    }
+
+    public function getNextSiteId()
+    {
+        $this->index++;
+
+        if (!empty($this->siteIds[$this->index])) {
+            return $this->siteIds[$this->index];
+        }
+
+        return null;
+    }
+}

msd2/tracking/piwik/core/CronArchive/Performance/Logger.php

@@ -0,0 +1,119 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+
+namespace Piwik\CronArchive\Performance;
+
+use Piwik\ArchiveProcessor;
+use Piwik\Common;
+use Piwik\Config;
+use Piwik\Option;
+use Piwik\Timer;
+use Piwik\Url;
+use Psr\Log\LoggerInterface;
+
+class Logger
+{
+    /**
+     * @var int
+     */
+    private $isEnabled;
+
+    /**
+     * @var LoggerInterface
+     */
+    private $logger;
+
+    /**
+     * @var int
+     */
+    private $archivingRunId;
+
+    public function __construct(Config $config, LoggerInterface $logger = null)
+    {
+        $this->isEnabled = $config->Debug['archiving_profile'] == 1;
+        $this->logger = $logger;
+
+        $this->archivingRunId = $this->getArchivingRunId();
+        if (empty($this->archivingRunId)) {
+            $this->isEnabled = false;
+        }
+    }
+
+    public function logMeasurement($category, $name, ArchiveProcessor\Parameters $activeArchivingParams, Timer $timer)
+    {
+        if (!$this->isEnabled || !$this->logger) {
+            return;
+        }
+
+        $measurement = new Measurement($category, $name, $activeArchivingParams->getSite()->getId(),
+            $activeArchivingParams->getPeriod()->getRangeString(), $activeArchivingParams->getPeriod()->getLabel(),
+            $activeArchivingParams->getSegment()->getString(), $timer->getTime(), $timer->getMemoryLeakValue(),
+            $timer->getPeakMemoryValue());
+
+        $params = array_merge($_GET);
+        unset($params['pid']);
+        unset($params['runid']);
+
+        $this->logger->info("[runid={runid},pid={pid}] {request}: {measurement}", [
+            'pid' => Common::getRequestVar('pid', false),
+            'runid' => $this->getArchivingRunId(),
+            'request' => Url::getQueryStringFromParameters($params),
+            'measurement' => $measurement,
+        ]);
+    }
+
+    public static function getMeasurementsFor($runId, $childPid)
+    {
+        $profilingLogFile = preg_replace('/[\'"]/', '', Config::getInstance()->Debug['archive_profiling_log']);
+        if (!is_readable($profilingLogFile)) {
+            return [];
+        }
+
+        $runId = self::cleanId($runId);
+        $childPid = self::cleanId($childPid);
+
+        $lineIdentifier = "[runid=$runId,pid=$childPid]";
+        $lines = `grep "$childPid" "$profilingLogFile"`;
+        $lines = explode("\n", $lines);
+        $lines = array_map(function ($line) use ($lineIdentifier) {
+            $index = strpos($line, $lineIdentifier);
+            if ($index === false) {
+                return null;
+            }
+            $line = substr($line, $index + strlen($lineIdentifier));
+            return trim($line);
+        }, $lines);
+        $lines = array_filter($lines);
+        $lines = array_map(function ($line) {
+            $parts = explode(":", $line, 2);
+            $parts = array_map('trim', $parts);
+            return $parts;
+        }, $lines);
+
+        $data = [];
+        foreach ($lines as $line) {
+            if (count($line) != 2) {
+                continue;
+            }
+
+            list($request, $measurement) = $line;
+            $data[$request][] = $measurement;
+        }
+        return $data;
+    }
+
+    private function getArchivingRunId()
+    {
+        return Common::getRequestVar('runid', false);
+    }
+
+    private static function cleanId($id)
+    {
+        return preg_replace('/[^a-zA-Z0-9_-]/', '', $id);
+    }
+}

msd2/tracking/piwik/core/CronArchive/Performance/Measurement.php

@@ -0,0 +1,165 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+
+namespace Piwik\CronArchive\Performance;
+
+class Measurement
+{
+    /**
+     * @var string
+     */
+    private $category;
+
+    /**
+     * @var string
+     */
+    private $measuredName;
+
+    /**
+     * @var string
+     */
+    private $idSite;
+
+    /**
+     * @var string
+     */
+    private $dateRange;
+
+    /**
+     * @var string
+     */
+    private $periodType;
+
+    /**
+     * @var string
+     */
+    private $segment;
+
+    /**
+     * @var float
+     */
+    private $time;
+
+    /**
+     * @var string
+     */
+    private $memory;
+
+    /**
+     * @var string
+     */
+    private $peakMemory;
+
+    public function __construct($category, $name, $idSite, $dateRange, $periodType, $segment, $time, $memory, $peakMemory)
+    {
+        $this->category = $category;
+        $this->measuredName = $name;
+        $this->idSite = $idSite;
+        $this->dateRange = $dateRange;
+        $this->periodType = $periodType;
+        $this->segment = trim($segment);
+        $this->time = $time;
+        $this->memory = $memory;
+        $this->peakMemory = $peakMemory;
+    }
+
+    public function __toString()
+    {
+        $parts = [
+            ucfirst($this->category) . ": {$this->measuredName}",
+            "idSite: {$this->idSite}",
+            "period: {$this->periodType} ({$this->dateRange})",
+            "segment: " . (!empty($this->segment) ? $this->segment : 'none'),
+            "duration: {$this->time}s",
+            "memory leak: {$this->memory}",
+            "peak memory usage: {$this->peakMemory}",
+        ];
+
+        return implode(', ', $parts);
+    }
+
+    /**
+     * @return string
+     */
+    public function getCategory()
+    {
+        return $this->category;
+    }
+
+    /**
+     * @param string $category
+     */
+    public function setCategory($category)
+    {
+        $this->category = $category;
+    }
+
+    /**
+     * @return string
+     */
+    public function getMeasuredName()
+    {
+        return $this->measuredName;
+    }
+
+    /**
+     * @param string $measuredName
+     */
+    public function setMeasuredName($measuredName)
+    {
+        $this->measuredName = $measuredName;
+    }
+
+    /**
+     * @return string
+     */
+    public function getIdSite()
+    {
+        return $this->idSite;
+    }
+
+    /**
+     * @param string $idSite
+     */
+    public function setIdSite($idSite)
+    {
+        $this->idSite = $idSite;
+    }
+
+    /**
+     * @return string
+     */
+    public function getDateRange()
+    {
+        return $this->dateRange;
+    }
+
+    /**
+     * @param string $dateRange
+     */
+    public function setDateRange($dateRange)
+    {
+        $this->dateRange = $dateRange;
+    }
+
+    /**
+     * @return string
+     */
+    public function getPeriodType()
+    {
+        return $this->periodType;
+    }
+
+    /**
+     * @param string $periodType
+     */
+    public function setPeriodType($periodType)
+    {
+        $this->periodType = $periodType;
+    }
+}

msd2/tracking/piwik/core/CronArchive/SegmentArchivingRequestUrlProvider.php

@@ -0,0 +1,208 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+namespace Piwik\CronArchive;
+
+use Piwik\Cache\Cache;
+use Piwik\Cache\Transient;
+use Piwik\Container\StaticContainer;
+use Piwik\Date;
+use Piwik\Period\Factory as PeriodFactory;
+use Piwik\Period\Range;
+use Piwik\Plugins\SegmentEditor\Model;
+use Psr\Log\LoggerInterface;
+
+/**
+ * Provides URLs that initiate archiving during cron archiving for segments.
+ *
+ * Handles the `[General] process_new_segments_from` INI option.
+ */
+class SegmentArchivingRequestUrlProvider
+{
+    const BEGINNING_OF_TIME = 'beginning_of_time';
+    const CREATION_TIME = 'segment_creation_time';
+    const LAST_EDIT_TIME = 'segment_last_edit_time';
+
+    /**
+     * @var Model
+     */
+    private $segmentEditorModel;
+
+    /**
+     * @var Cache
+     */
+    private $segmentListCache;
+
+    /**
+     * @var Date
+     */
+    private $now;
+
+    private $processNewSegmentsFrom;
+
+    /**
+     * @var LoggerInterface
+     */
+    private $logger;
+
+    public function __construct($processNewSegmentsFrom, Model $segmentEditorModel = null, Cache $segmentListCache = null,
+                                Date $now = null, LoggerInterface $logger = null)
+    {
+        $this->processNewSegmentsFrom = $processNewSegmentsFrom;
+        $this->segmentEditorModel = $segmentEditorModel ?: new Model();
+        $this->segmentListCache = $segmentListCache ?: new Transient();
+        $this->now = $now ?: Date::factory('now');
+        $this->logger = $logger ?: StaticContainer::get('Psr\Log\LoggerInterface');
+    }
+
+    public function getUrlParameterDateString($idSite, $period, $date, $segment)
+    {
+        $oldestDateToProcessForNewSegment = $this->getOldestDateToProcessForNewSegment($idSite, $segment);
+        if (empty($oldestDateToProcessForNewSegment)) {
+            return $date;
+        }
+
+        // if the start date for the archiving request is before the minimum date allowed for processing this segment,
+        // use the minimum allowed date as the start date
+        $periodObj = PeriodFactory::build($period, $date);
+        if ($periodObj->getDateStart()->getTimestamp() < $oldestDateToProcessForNewSegment->getTimestamp()) {
+            $this->logger->debug("Start date of archiving request period ({start}) is older than configured oldest date to process for the segment.", array(
+                'start' => $periodObj->getDateStart()
+            ));
+
+            $endDate = $periodObj->getDateEnd();
+
+            // if the creation time of a segment is older than the end date of the archiving request range, we cannot
+            // blindly rewrite the date string, since the resulting range would be incorrect. instead we make the
+            // start date equal to the end date, so less archiving occurs, and no fatal error occurs.
+            if ($oldestDateToProcessForNewSegment->getTimestamp() > $endDate->getTimestamp()) {
+                $this->logger->debug("Oldest date to process is greater than end date of archiving request period ({end}), so setting oldest date to end date.", array(
+                    'end' => $endDate
+                ));
+
+                $oldestDateToProcessForNewSegment = $endDate;
+            }
+
+            $date = $oldestDateToProcessForNewSegment->toString().','.$endDate;
+
+            $this->logger->debug("Archiving request date range changed to {date} w/ period {period}.", array('date' => $date, 'period' => $period));
+        }
+
+        return $date;
+    }
+
+    private function getOldestDateToProcessForNewSegment($idSite, $segment)
+    {
+        /**
+         * @var Date $segmentCreatedTime
+         * @var Date $segmentLastEditedTime
+         */
+        list($segmentCreatedTime, $segmentLastEditedTime) = $this->getCreatedTimeOfSegment($idSite, $segment);
+
+        if ($this->processNewSegmentsFrom == self::CREATION_TIME) {
+            $this->logger->debug("process_new_segments_from set to segment_creation_time, oldest date to process is {time}", array('time' => $segmentCreatedTime));
+
+            return $segmentCreatedTime;
+        } elseif ($this->processNewSegmentsFrom == self::LAST_EDIT_TIME) {
+            $this->logger->debug("process_new_segments_from set to segment_last_edit_time, segment last edit time is {time}",
+                array('time' => $segmentLastEditedTime));
+
+            if ($segmentLastEditedTime === null
+                || $segmentLastEditedTime->getTimestamp() < $segmentCreatedTime->getTimestamp()
+            ) {
+                $this->logger->debug("segment last edit time is older than created time, using created time instead");
+
+                $segmentLastEditedTime = $segmentCreatedTime;
+            }
+
+            return $segmentLastEditedTime;
+        } elseif (preg_match("/^last([0-9]+)$/", $this->processNewSegmentsFrom, $matches)) {
+            $lastN = $matches[1];
+
+            list($lastDate, $lastPeriod) = Range::getDateXPeriodsAgo($lastN, $segmentCreatedTime, 'day');
+            $result = Date::factory($lastDate);
+
+            $this->logger->debug("process_new_segments_from set to last{N}, oldest date to process is {time}", array('N' => $lastN, 'time' => $result));
+
+            return $result;
+        } else {
+            $this->logger->debug("process_new_segments_from set to beginning_of_time or cannot recognize value");
+
+            return null;
+        }
+    }
+
+    private function getCreatedTimeOfSegment($idSite, $segmentDefinition)
+    {
+        $segments = $this->getAllSegments();
+
+        /** @var Date $latestEditTime */
+        $latestEditTime = null;
+        $earliestCreatedTime = $this->now;
+        foreach ($segments as $segment) {
+            if (empty($segment['ts_created'])
+                || empty($segment['definition'])
+                || !isset($segment['enable_only_idsite'])
+            ) {
+                continue;
+            }
+
+            if ($this->isSegmentForSite($segment, $idSite)
+                && $segment['definition'] == $segmentDefinition
+            ) {
+                // check for an earlier ts_created timestamp
+                $createdTime = Date::factory($segment['ts_created']);
+                if ($createdTime->getTimestamp() < $earliestCreatedTime->getTimestamp()) {
+                    $earliestCreatedTime = $createdTime;
+                }
+
+                // if there is no ts_last_edit timestamp, initialize it to ts_created
+                if (empty($segment['ts_last_edit'])) {
+                    $segment['ts_last_edit'] = $segment['ts_created'];
+                }
+
+                // check for a later ts_last_edit timestamp
+                $lastEditTime = Date::factory($segment['ts_last_edit']);
+                if ($latestEditTime === null
+                    || $latestEditTime->getTimestamp() < $lastEditTime->getTimestamp()
+                ) {
+                    $latestEditTime = $lastEditTime;
+                }
+            }
+        }
+
+        $this->logger->debug(
+            "Earliest created time of segment '{segment}' w/ idSite = {idSite} is found to be {createdTime}. Latest " .
+            "edit time is found to be {latestEditTime}.",
+            array(
+                'segment' => $segmentDefinition,
+                'idSite' => $idSite,
+                'createdTime' => $earliestCreatedTime,
+                'latestEditTime' => $latestEditTime,
+            )
+        );
+
+        return array($earliestCreatedTime, $latestEditTime);
+    }
+
+    private function getAllSegments()
+    {
+        if (!$this->segmentListCache->contains('all')) {
+            $segments = $this->segmentEditorModel->getAllSegmentsAndIgnoreVisibility();
+
+            $this->segmentListCache->save('all', $segments);
+        }
+
+        return $this->segmentListCache->fetch('all');
+    }
+
+    private function isSegmentForSite($segment, $idSite)
+    {
+        return $segment['enable_only_idsite'] == 0
+            || $segment['enable_only_idsite'] == $idSite;
+    }
+}