PDF rausgenommen

msd2/tracking/piwik/core/API/ApiRenderer.php

@@ -0,0 +1,143 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\API;
+
+use Exception;
+use Piwik\Common;
+use Piwik\DataTable\Renderer;
+use Piwik\DataTable;
+use Piwik\Piwik;
+use Piwik\Plugin;
+
+/**
+ * API renderer
+ */
+abstract class ApiRenderer
+{
+    protected $request;
+
+    final public function __construct($request)
+    {
+        $this->request = $request;
+        $this->init();
+    }
+
+    protected function init()
+    {
+    }
+
+    abstract public function sendHeader();
+
+    public function renderSuccess($message)
+    {
+        return 'Success:' . $message;
+    }
+
+    /**
+     * @param $message
+     * @param Exception|\Throwable $exception
+     * @return mixed
+     */
+    public function renderException($message, $exception)
+    {
+        return $message;
+    }
+
+    public function renderScalar($scalar)
+    {
+        $dataTable = new DataTable\Simple();
+        $dataTable->addRowsFromArray(array($scalar));
+        return $this->renderDataTable($dataTable);
+    }
+
+    public function renderDataTable($dataTable)
+    {
+        $renderer = $this->buildDataTableRenderer($dataTable);
+        return $renderer->render();
+    }
+
+    public function renderArray($array)
+    {
+        $renderer = $this->buildDataTableRenderer($array);
+        return $renderer->render();
+    }
+
+    public function renderObject($object)
+    {
+        $exception = new Exception('The API cannot handle this data structure.');
+        return $this->renderException($exception->getMessage(), $exception);
+    }
+
+    public function renderResource($resource)
+    {
+        $exception = new Exception('The API cannot handle this data structure.');
+        return $this->renderException($exception->getMessage(), $exception);
+    }
+
+    /**
+     * @param $dataTable
+     * @return Renderer
+     */
+    protected function buildDataTableRenderer($dataTable)
+    {
+        $format   = self::getFormatFromClass(get_class($this));
+        if ($format == 'json2') {
+            $format = 'json';
+        }
+
+        $idSite = Common::getRequestVar('idSite', 0, 'int', $this->request);
+
+        if (empty($idSite)) {
+            $idSite = 'all';
+        }
+
+        $renderer = Renderer::factory($format);
+        $renderer->setTable($dataTable);
+        $renderer->setIdSite($idSite);
+        $renderer->setRenderSubTables(Common::getRequestVar('expanded', false, 'int', $this->request));
+        $renderer->setHideIdSubDatableFromResponse(Common::getRequestVar('hideIdSubDatable', false, 'int', $this->request));
+
+        return $renderer;
+    }
+
+    /**
+     * @param string $format
+     * @param array  $request
+     * @return ApiRenderer
+     * @throws Exception
+     */
+    public static function factory($format, $request)
+    {
+        $formatToCheck = '\\' . ucfirst(strtolower($format));
+
+        $rendererClassnames = Plugin\Manager::getInstance()->findMultipleComponents('Renderer', 'Piwik\\API\\ApiRenderer');
+
+        foreach ($rendererClassnames as $klassName) {
+            if (Common::stringEndsWith($klassName, $formatToCheck)) {
+                return new $klassName($request);
+            }
+        }
+
+        $availableRenderers = array();
+        foreach ($rendererClassnames as $rendererClassname) {
+            $availableRenderers[] = self::getFormatFromClass($rendererClassname);
+        }
+
+        $availableRenderers = implode(', ', $availableRenderers);
+        Common::sendHeader('Content-Type: text/plain; charset=utf-8');
+        throw new Exception(Piwik::translate('General_ExceptionInvalidRendererFormat', array($format, $availableRenderers)));
+    }
+
+    private static function getFormatFromClass($klassname)
+    {
+        $klass = explode('\\', $klassname);
+
+        return strtolower(end($klass));
+    }
+}

msd2/tracking/piwik/core/API/CORSHandler.php

@@ -0,0 +1,57 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\API;
+
+use Piwik\Common;
+use Piwik\Url;
+
+class CORSHandler
+{
+    /**
+     * @var array
+     */
+    protected $domains;
+
+    public function __construct()
+    {
+        $this->domains = Url::getCorsHostsFromConfig();
+    }
+
+    public function handle()
+    {
+        if (empty($this->domains)) {
+            return;
+        }
+        
+        Common::sendHeader('Vary: Origin');
+        
+        // allow Piwik to serve data to all domains
+        if (in_array("*", $this->domains)) {
+            
+            Common::sendHeader('Access-Control-Allow-Credentials: true');
+            
+            if (!empty($_SERVER['HTTP_ORIGIN'])) {
+                Common::sendHeader('Access-Control-Allow-Origin: ' . $_SERVER['HTTP_ORIGIN']);
+                return;
+            }
+            
+            Common::sendHeader('Access-Control-Allow-Origin: *');
+            return;
+        }
+
+        // specifically allow if it is one of the whitelisted CORS domains
+        if (!empty($_SERVER['HTTP_ORIGIN'])) {
+            $origin = $_SERVER['HTTP_ORIGIN'];
+            if (in_array($origin, $this->domains, true)) {
+                Common::sendHeader('Access-Control-Allow-Credentials: true');
+                Common::sendHeader('Access-Control-Allow-Origin: ' . $_SERVER['HTTP_ORIGIN']);
+            }
+        }
+    }
+}

msd2/tracking/piwik/core/API/DataTableGenericFilter.php

@@ -0,0 +1,245 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\API;
+
+use Exception;
+use Piwik\Common;
+use Piwik\DataTable;
+use Piwik\Plugin\ProcessedMetric;
+use Piwik\Plugin\Report;
+
+class DataTableGenericFilter
+{
+    /**
+     * List of filter names not to run.
+     *
+     * @var string[]
+     */
+    private $disabledFilters = array();
+
+    /**
+     * @var Report
+     */
+    private $report;
+
+    /**
+     * @var array
+     */
+    private $request;
+
+    /**
+     * Constructor
+     *
+     * @param $request
+     */
+    public function __construct($request, $report)
+    {
+        $this->request = $request;
+        $this->report  = $report;
+    }
+
+    /**
+     * Filters the given data table
+     *
+     * @param DataTable $table
+     */
+    public function filter($table)
+    {
+        $this->applyGenericFilters($table);
+    }
+
+    /**
+     * Makes sure a set of filters are not run.
+     *
+     * @param string[] $filterNames The name of each filter to disable.
+     */
+    public function disableFilters($filterNames)
+    {
+        $this->disabledFilters = array_unique(array_merge($this->disabledFilters, $filterNames));
+    }
+
+    /**
+     * Returns an array containing the information of the generic Filter
+     * to be applied automatically to the data resulting from the API calls.
+     *
+     * Order to apply the filters:
+     * 1 - Filter that remove filtered rows
+     * 2 - Filter that sort the remaining rows
+     * 3 - Filter that keep only a subset of the results
+     * 4 - Presentation filters
+     *
+     * @return array  See the code for spec
+     */
+    public static function getGenericFiltersInformation()
+    {
+        return array(
+            array('Pattern',
+                  array(
+                      'filter_column'  => array('string', 'label'),
+                      'filter_pattern' => array('string')
+                  )),
+            array('PatternRecursive',
+                  array(
+                      'filter_column_recursive'  => array('string', 'label'),
+                      'filter_pattern_recursive' => array('string'),
+                  )),
+            array('ExcludeLowPopulation',
+                  array(
+                      'filter_excludelowpop'       => array('string'),
+                      'filter_excludelowpop_value' => array('float', '0'),
+                  )),
+            array('Sort',
+                  array(
+                      'filter_sort_column' => array('string'),
+                      'filter_sort_order'  => array('string', 'desc'),
+                      $naturalSort = true,
+                      $recursiveSort = true,
+                      'filter_sort_column_secondary' => true
+                  )),
+            array('Truncate',
+                  array(
+                      'filter_truncate' => array('integer'),
+                  )),
+            array('Limit',
+                  array(
+                      'filter_offset'    => array('integer', '0'),
+                      'filter_limit'     => array('integer'),
+                      'keep_summary_row' => array('integer', '0'),
+                  ))
+        );
+    }
+
+    private function getGenericFiltersHavingDefaultValues()
+    {
+        $filters = self::getGenericFiltersInformation();
+
+        if ($this->report && $this->report->getDefaultSortColumn()) {
+            foreach ($filters as $index => $filter) {
+                if ($filter[0] === 'Sort') {
+                    $filters[$index][1]['filter_sort_column'] = array('string', $this->report->getDefaultSortColumn());
+                    $filters[$index][1]['filter_sort_order']  = array('string', $this->report->getDefaultSortOrder());
+
+                    $callback = $this->report->getSecondarySortColumnCallback();
+
+                    if (is_callable($callback)) {
+                        $filters[$index][1]['filter_sort_column_secondary'] = $callback;
+                    }
+
+                }
+            }
+        }
+
+        return $filters;
+    }
+
+    /**
+     * Apply generic filters to the DataTable object resulting from the API Call.
+     * Disable this feature by setting the parameter disable_generic_filters to 1 in the API call request.
+     *
+     * @param DataTable $datatable
+     * @return bool
+     */
+    protected function applyGenericFilters($datatable)
+    {
+        if ($datatable instanceof DataTable\Map) {
+            $tables = $datatable->getDataTables();
+            foreach ($tables as $table) {
+                $this->applyGenericFilters($table);
+            }
+            return;
+        }
+
+        $tableDisabledFilters = $datatable->getMetadata(DataTable::GENERIC_FILTERS_TO_DISABLE_METADATA_NAME) ?: [];
+        $genericFilters = $this->getGenericFiltersHavingDefaultValues();
+
+        $filterApplied = false;
+        foreach ($genericFilters as $filterMeta) {
+            $filterName = $filterMeta[0];
+            $filterParams = $filterMeta[1];
+            $filterParameters = array();
+            $exceptionRaised = false;
+
+            if (in_array($filterName, $this->disabledFilters)
+                || in_array($filterName, $tableDisabledFilters)
+            ) {
+                continue;
+            }
+
+            foreach ($filterParams as $name => $info) {
+                if (!is_array($info)) {
+                    // hard coded value that cannot be changed via API, see eg $naturalSort = true in 'Sort'
+                    $filterParameters[] = $info;
+                } else {
+                    // parameter type to cast to
+                    $type = $info[0];
+
+                    // default value if specified, when the parameter doesn't have a value
+                    $defaultValue = null;
+                    if (isset($info[1])) {
+                        $defaultValue = $info[1];
+                    }
+
+                    try {
+                        $value = Common::getRequestVar($name, $defaultValue, $type, $this->request);
+                        settype($value, $type);
+                        $filterParameters[] = $value;
+                    } catch (Exception $e) {
+                        $exceptionRaised = true;
+                        break;
+                    }
+                }
+            }
+
+            if (!$exceptionRaised) {
+                $datatable->filter($filterName, $filterParameters);
+                $filterApplied = true;
+            }
+        }
+
+        return $filterApplied;
+    }
+
+    public function areProcessedMetricsNeededFor($metrics)
+    {
+        $columnQueryParameters = array(
+            'filter_column',
+            'filter_column_recursive',
+            'filter_excludelowpop',
+            'filter_sort_column'
+        );
+
+        foreach ($columnQueryParameters as $queryParamName) {
+            $queryParamValue = Common::getRequestVar($queryParamName, false, $type = null, $this->request);
+            if (!empty($queryParamValue)
+                && $this->containsProcessedMetric($metrics, $queryParamValue)
+            ) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    /**
+     * @param ProcessedMetric[] $metrics
+     * @param string $name
+     * @return bool
+     */
+    private function containsProcessedMetric($metrics, $name)
+    {
+        foreach ($metrics as $metric) {
+            if ($metric instanceof ProcessedMetric
+                && $metric->getName() == $name
+            ) {
+                return true;
+            }
+        }
+        return false;
+    }
+}

msd2/tracking/piwik/core/API/DataTableManipulator.php

@@ -0,0 +1,207 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\API;
+
+use Exception;
+use Piwik\Archive\DataTableFactory;
+use Piwik\Container\StaticContainer;
+use Piwik\DataTable\Row;
+use Piwik\DataTable;
+use Piwik\Period\Range;
+use Piwik\Plugins\API\API;
+
+/**
+ * Base class for manipulating data tables.
+ * It provides generic mechanisms like iteration and loading subtables.
+ *
+ * The manipulators are used in ResponseBuilder and are triggered by
+ * API parameters. They are not filters because they don't work on the pre-
+ * fetched nested data tables. Instead, they load subtables using this base
+ * class. This way, they can only load the tables they really need instead
+ * of using expanded=1. Another difference between manipulators and filters
+ * is that filters keep the overall structure of the table intact while
+ * manipulators can change the entire thing.
+ */
+abstract class DataTableManipulator
+{
+    protected $apiModule;
+    protected $apiMethod;
+    protected $request;
+    protected $apiMethodForSubtable;
+
+    /**
+     * Constructor
+     *
+     * @param bool $apiModule
+     * @param bool $apiMethod
+     * @param array $request
+     */
+    public function __construct($apiModule = false, $apiMethod = false, $request = array())
+    {
+        $this->apiModule = $apiModule;
+        $this->apiMethod = $apiMethod;
+        $this->request = $request;
+    }
+
+    /**
+     * This method can be used by subclasses to iterate over data tables that might be
+     * data table maps. It calls back the template method self::doManipulate for each table.
+     * This way, data table arrays can be handled in a transparent fashion.
+     *
+     * @param DataTable\Map|DataTable $dataTable
+     * @throws Exception
+     * @return DataTable\Map|DataTable
+     */
+    protected function manipulate($dataTable)
+    {
+        if ($dataTable instanceof DataTable\Map) {
+            return $this->manipulateDataTableMap($dataTable);
+        } elseif ($dataTable instanceof DataTable) {
+            return $this->manipulateDataTable($dataTable);
+        } else {
+            return $dataTable;
+        }
+    }
+
+    /**
+     * Manipulates child DataTables of a DataTable\Map. See @manipulate for more info.
+     *
+     * @param DataTable\Map $dataTable
+     * @return DataTable\Map
+     */
+    protected function manipulateDataTableMap($dataTable)
+    {
+        $result = $dataTable->getEmptyClone();
+        foreach ($dataTable->getDataTables() as $tableLabel => $childTable) {
+            $newTable = $this->manipulate($childTable);
+            $result->addTable($newTable, $tableLabel);
+        }
+        return $result;
+    }
+
+    /**
+     * Manipulates a single DataTable instance. Derived classes must define
+     * this function.
+     */
+    abstract protected function manipulateDataTable($dataTable);
+
+    /**
+     * Load the subtable for a row.
+     * Returns null if none is found.
+     *
+     * @param DataTable $dataTable
+     * @param Row $row
+     *
+     * @return DataTable
+     */
+    protected function loadSubtable($dataTable, $row)
+    {
+        if (!($this->apiModule && $this->apiMethod && count($this->request))) {
+            return null;
+        }
+
+        $request = $this->request;
+
+        $idSubTable = $row->getIdSubDataTable();
+        if ($idSubTable === null) {
+            return null;
+        }
+
+        $request['idSubtable'] = $idSubTable;
+        if ($dataTable) {
+            $period = $dataTable->getMetadata(DataTableFactory::TABLE_METADATA_PERIOD_INDEX);
+            if ($period instanceof Range) {
+                $request['date'] = $period->getDateStart() . ',' . $period->getDateEnd();
+            } else {
+                $request['date'] = $period->getDateStart()->toString();
+            }
+        }
+
+        $method = $this->getApiMethodForSubtable($request);
+        return $this->callApiAndReturnDataTable($this->apiModule, $method, $request);
+    }
+
+    /**
+     * In this method, subclasses can clean up the request array for loading subtables
+     * in order to make ResponseBuilder behave correctly (e.g. not trigger the
+     * manipulator again).
+     *
+     * @param $request
+     * @return
+     */
+    abstract protected function manipulateSubtableRequest($request);
+
+    /**
+     * Extract the API method for loading subtables from the meta data
+     *
+     * @throws Exception
+     * @return string
+     */
+    protected function getApiMethodForSubtable($request)
+    {
+        if (!$this->apiMethodForSubtable) {
+            if (!empty($request['idSite'])) {
+                $idSite = $request['idSite'];
+            } else {
+                $idSite = 'all';
+            }
+
+            $apiParameters = array();
+            $entityNames = StaticContainer::get('entities.idNames');
+            foreach ($entityNames as $idName) {
+                if (!empty($request[$idName])) {
+                    $apiParameters[$idName] = $request[$idName];
+                }
+            }
+
+            $meta = API::getInstance()->getMetadata($idSite, $this->apiModule, $this->apiMethod, $apiParameters);
+
+            if (empty($meta) && array_key_exists('idGoal', $apiParameters)) {
+                unset($apiParameters['idGoal']);
+                $meta = API::getInstance()->getMetadata($idSite, $this->apiModule, $this->apiMethod, $apiParameters);
+            }
+
+            if (empty($meta)) {
+                throw new Exception(sprintf(
+                    "The DataTable cannot be manipulated: Metadata for report %s.%s could not be found. You can define the metadata in a hook, see example at: https://developer.matomo.org/api-reference/events#apigetreportmetadata",
+                    $this->apiModule, $this->apiMethod
+                ));
+            }
+
+            if (isset($meta[0]['actionToLoadSubTables'])) {
+                $this->apiMethodForSubtable = $meta[0]['actionToLoadSubTables'];
+            } else {
+                $this->apiMethodForSubtable = $this->apiMethod;
+            }
+        }
+        return $this->apiMethodForSubtable;
+    }
+
+    protected function callApiAndReturnDataTable($apiModule, $method, $request)
+    {
+        $class = Request::getClassNameAPI($apiModule);
+
+        $request = $this->manipulateSubtableRequest($request);
+        $request['serialize'] = 0;
+        $request['expanded'] = 0;
+        $request['format'] = 'original';
+        $request['format_metrics'] = 0;
+
+        // don't want to run recursive filters on the subtables as they are loaded,
+        // otherwise the result will be empty in places (or everywhere). instead we
+        // run it on the flattened table.
+        unset($request['filter_pattern_recursive']);
+
+        $dataTable = Proxy::getInstance()->call($class, $method, $request);
+        $response = new ResponseBuilder($format = 'original', $request);
+        $response->disableSendHeader();
+        $dataTable = $response->getResponse($dataTable, $apiModule, $method);
+        return $dataTable;
+    }
+}

msd2/tracking/piwik/core/API/DataTableManipulator/Flattener.php

@@ -0,0 +1,209 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\API\DataTableManipulator;
+
+use Piwik\API\DataTableManipulator;
+use Piwik\Common;
+use Piwik\DataTable;
+use Piwik\DataTable\Row;
+use Piwik\Plugin\ReportsProvider;
+
+/**
+ * This class is responsible for flattening data tables.
+ *
+ * It loads subtables and combines them into a single table by concatenating the labels.
+ * This manipulator is triggered by using flat=1 in the API request.
+ */
+class Flattener extends DataTableManipulator
+{
+
+    private $includeAggregateRows = false;
+
+    /**
+     * If the flattener is used after calling this method, aggregate rows will
+     * be included in the result. This can be useful when they contain data that
+     * the leafs don't have (e.g. conversion stats in some cases).
+     */
+    public function includeAggregateRows()
+    {
+        $this->includeAggregateRows = true;
+    }
+
+    /**
+     * Separator for building recursive labels (or paths)
+     * @var string
+     */
+    public $recursiveLabelSeparator = '';
+
+    /**
+     * @param  DataTable $dataTable
+     * @param string $recursiveLabelSeparator
+     * @return DataTable|DataTable\Map
+     */
+    public function flatten($dataTable, $recursiveLabelSeparator)
+    {
+        $this->recursiveLabelSeparator = $recursiveLabelSeparator;
+
+        return $this->manipulate($dataTable);
+    }
+
+    /**
+     * Template method called from self::manipulate.
+     * Flatten each data table.
+     *
+     * @param DataTable $dataTable
+     * @return DataTable
+     */
+    protected function manipulateDataTable($dataTable)
+    {
+        $newDataTable = $dataTable->getEmptyClone($keepFilters = true);
+        if ($dataTable->getTotalsRow()) {
+            $newDataTable->setTotalsRow($dataTable->getTotalsRow());
+        }
+
+        // this recursive filter will be applied to subtables
+        $dataTable->filter('ReplaceSummaryRowLabel');
+        $dataTable->filter('ReplaceColumnNames');
+
+        $report        = ReportsProvider::factory($this->apiModule, $this->apiMethod);
+        if (!empty($report)) {
+            $dimension = $report->getDimension();
+        }
+
+        $dimensionName = !empty($dimension) ? str_replace('.', '_', $dimension->getId()) : 'label1';
+
+        $this->flattenDataTableInto($dataTable, $newDataTable, $level = 1, $dimensionName);
+
+        return $newDataTable;
+    }
+
+    /**
+     * @param $dataTable DataTable
+     * @param $newDataTable
+     * @param $dimensionName
+     */
+    protected function flattenDataTableInto($dataTable, $newDataTable, $level, $dimensionName, $prefix = '', $logo = false)
+    {
+        foreach ($dataTable->getRows() as $rowId => $row) {
+            $this->flattenRow($row, $rowId, $newDataTable, $level, $dimensionName, $prefix, $logo);
+        }
+    }
+
+    /**
+     * @param Row $row
+     * @param DataTable $dataTable
+     * @param string $labelPrefix
+     * @param string $dimensionName
+     * @param bool $parentLogo
+     */
+    private function flattenRow
+    (Row $row, $rowId, DataTable $dataTable, $level, $dimensionName,
+                                $labelPrefix = '', $parentLogo = false)
+    {
+        $origLabel = $label = $row->getColumn('label');
+
+        if ($label !== false) {
+            $label = trim($label);
+
+            if ($this->recursiveLabelSeparator == '/') {
+                if (substr($label, 0, 1) == '/' && substr($labelPrefix, -1) == '/') {
+                    $label = substr($label, 1);
+                } elseif ($rowId === DataTable::ID_SUMMARY_ROW && $labelPrefix && $label != DataTable::LABEL_SUMMARY_ROW) {
+                    $label = ' - ' . $label;
+                }
+            }
+
+            $origLabel = $label;
+
+            $label = $labelPrefix . $label;
+            $row->setColumn('label', $label);
+
+            if ($row->getMetadata($dimensionName)) {
+                $origLabel = $row->getMetadata($dimensionName) . $this->recursiveLabelSeparator . $origLabel;
+            }
+
+            $row->setMetadata($dimensionName, $origLabel);
+        }
+
+        $logo = $row->getMetadata('logo');
+        if ($logo === false && $parentLogo !== false) {
+            $logo = $parentLogo;
+            $row->setMetadata('logo', $logo);
+        }
+
+        /** @var DataTable $subTable */
+        $subTable = $row->getSubtable();
+
+        if ($subTable) {
+            $subTable->applyQueuedFilters();
+            $row->deleteMetadata('idsubdatatable_in_db');
+        } else {
+            $subTable = $this->loadSubtable($dataTable, $row);
+        }
+
+        $row->removeSubtable();
+
+        if ($subTable === null) {
+            if ($this->includeAggregateRows) {
+                $row->setMetadata('is_aggregate', 0);
+            }
+            $dataTable->addRow($row);
+        } else {
+            if ($this->includeAggregateRows) {
+                $row->setMetadata('is_aggregate', 1);
+                $dataTable->addRow($row);
+            }
+            $prefix = $label . $this->recursiveLabelSeparator;
+
+            $report        = ReportsProvider::factory($this->apiModule, $this->apiMethod);
+            if (!empty($report)) {
+                $subDimension = $report->getSubtableDimension();
+            }
+
+            if ($level === 2) {
+                $subDimension = $report->getThirdLeveltableDimension();
+            }
+
+            if (empty($subDimension)) {
+                $report           = ReportsProvider::factory($this->apiModule, $this->getApiMethodForSubtable($this->request));
+                $subDimension     = $report->getDimension();
+            }
+
+            $subDimensionName = $subDimension ? str_replace('.', '_', $subDimension->getId()) : 'label' . (substr_count($prefix, $this->recursiveLabelSeparator) + 1);
+
+            if ($origLabel !== false) {
+                foreach ($subTable->getRows() as $subRow) {
+                    foreach ($row->getMetadata() as $name => $value) {
+                        if ($subRow->getMetadata($name) === false) {
+                            $subRow->setMetadata($name, $value);
+                        }
+                    }
+
+                    $subRow->setMetadata($dimensionName, $origLabel);
+                }
+            }
+
+            $this->flattenDataTableInto($subTable, $dataTable, $level + 1, $subDimensionName, $prefix, $logo);
+        }
+    }
+
+    /**
+     * Remove the flat parameter from the subtable request
+     *
+     * @param array $request
+     * @return array
+     */
+    protected function manipulateSubtableRequest($request)
+    {
+        unset($request['flat']);
+
+        return $request;
+    }
+
+}

msd2/tracking/piwik/core/API/DataTableManipulator/LabelFilter.php

@@ -0,0 +1,188 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\API\DataTableManipulator;
+
+use Piwik\API\DataTableManipulator;
+use Piwik\Common;
+use Piwik\DataTable;
+use Piwik\DataTable\Row;
+
+/**
+ * This class is responsible for handling the label parameter that can be
+ * added to every API call. If the parameter is set, only the row with the matching
+ * label is returned.
+ *
+ * The labels passed to this class should be urlencoded.
+ * Some reports use recursive labels (e.g. action reports). Use > to join them.
+ */
+class LabelFilter extends DataTableManipulator
+{
+    const SEPARATOR_RECURSIVE_LABEL = '>';
+    const TERMINAL_OPERATOR = '@';
+
+    private $labels;
+    private $addLabelIndex;
+    const FLAG_IS_ROW_EVOLUTION = 'label_index';
+
+    /**
+     * Filter a data table by label.
+     * The filtered table is returned, which might be a new instance.
+     *
+     * $apiModule, $apiMethod and $request are needed load sub-datatables
+     * for the recursive search. If the label is not recursive, these parameters
+     * are not needed.
+     *
+     * @param string $labels the labels to search for
+     * @param DataTable $dataTable the data table to be filtered
+     * @param bool $addLabelIndex Whether to add label_index metadata describing which
+     *                            label a row corresponds to.
+     * @return DataTable
+     */
+    public function filter($labels, $dataTable, $addLabelIndex = false)
+    {
+        if (!is_array($labels)) {
+            $labels = array($labels);
+        }
+
+        $this->labels = $labels;
+        $this->addLabelIndex = (bool)$addLabelIndex;
+        return $this->manipulate($dataTable);
+    }
+
+    /**
+     * Method for the recursive descend
+     *
+     * @param array $labelParts
+     * @param DataTable $dataTable
+     * @return Row|bool
+     */
+    private function doFilterRecursiveDescend($labelParts, $dataTable)
+    {
+        // we need to make sure to rebuild the index as some filters change the label column directly via
+        // $row->setColumn('label', '') which would not be noticed in the label index otherwise.
+        $dataTable->rebuildIndex();
+
+        // search for the first part of the tree search
+        $labelPart = array_shift($labelParts);
+
+        $row = false;
+        foreach ($this->getLabelVariations($labelPart) as $labelPart) {
+            $row = $dataTable->getRowFromLabel($labelPart);
+            if ($row !== false) {
+                break;
+            }
+        }
+
+        if ($row === false) {
+            // not found
+            return false;
+        }
+
+        // end of tree search reached
+        if (count($labelParts) == 0) {
+            return $row;
+        }
+
+        $subTable = $this->loadSubtable($dataTable, $row);
+        if ($subTable === null) {
+            // no more subtables but label parts left => no match found
+            return false;
+        }
+
+        return $this->doFilterRecursiveDescend($labelParts, $subTable);
+    }
+
+    /**
+     * Clean up request for ResponseBuilder to behave correctly
+     *
+     * @param $request
+     */
+    protected function manipulateSubtableRequest($request)
+    {
+        unset($request['label']);
+        unset($request['flat']);
+        $request['totals'] = 0;
+        $request['filter_sort_column'] = ''; // do not sort, we only want to find a matching column
+
+        return $request;
+    }
+
+    /**
+     * Use variations of the label to make it easier to specify the desired label
+     *
+     * Note: The HTML Encoded version must be tried first, since in ResponseBuilder the $label is unsanitized
+     * via Common::unsanitizeLabelParameter.
+     *
+     * @param string $originalLabel
+     * @return array
+     */
+    private function getLabelVariations($originalLabel)
+    {
+        static $pageTitleReports = array('getPageTitles', 'getEntryPageTitles', 'getExitPageTitles');
+
+        $originalLabel = trim($originalLabel);
+
+        $isTerminal = substr($originalLabel, 0, 1) == self::TERMINAL_OPERATOR;
+        if ($isTerminal) {
+            $originalLabel = substr($originalLabel, 1);
+        }
+
+        $variations = array();
+        $label = trim(urldecode($originalLabel));
+
+        $sanitizedLabel = Common::sanitizeInputValue($label);
+        $variations[] = $sanitizedLabel;
+
+        if ($this->apiModule == 'Actions'
+            && in_array($this->apiMethod, $pageTitleReports)
+        ) {
+            if ($isTerminal) {
+                array_unshift($variations, ' ' . $sanitizedLabel);
+                array_unshift($variations, ' ' . $label);
+            } else {
+                // special case: the Actions.getPageTitles report prefixes some labels with a blank.
+                // the blank might be passed by the user but is removed in Request::getRequestArrayFromString.
+                $variations[] = ' ' . $sanitizedLabel;
+                $variations[] = ' ' . $label;
+            }
+        }
+        $variations[] = $label;
+
+        $variations = array_unique($variations);
+
+        return $variations;
+    }
+
+    /**
+     * Filter a DataTable instance. See @filter for more info.
+     *
+     * @param DataTable\Simple|DataTable\Map $dataTable
+     * @return mixed
+     */
+    protected function manipulateDataTable($dataTable)
+    {
+        $result = $dataTable->getEmptyClone();
+        foreach ($this->labels as $labelIndex => $label) {
+            $row = null;
+            foreach ($this->getLabelVariations($label) as $labelVariation) {
+                $labelVariation = explode(self::SEPARATOR_RECURSIVE_LABEL, $labelVariation);
+
+                $row = $this->doFilterRecursiveDescend($labelVariation, $dataTable);
+                if ($row) {
+                    if ($this->addLabelIndex) {
+                        $row->setMetadata(self::FLAG_IS_ROW_EVOLUTION, $labelIndex);
+                    }
+                    $result->addRow($row);
+                    break;
+                }
+            }
+        }
+        return $result;
+    }
+}

msd2/tracking/piwik/core/API/DataTableManipulator/ReportTotalsCalculator.php

@@ -0,0 +1,249 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\API\DataTableManipulator;
+
+use Piwik\API\DataTableManipulator;
+use Piwik\API\DataTablePostProcessor;
+use Piwik\Common;
+use Piwik\DataTable;
+use Piwik\Metrics;
+use Piwik\Period;
+use Piwik\Piwik;
+use Piwik\Plugin\Report;
+use Piwik\Plugin\ReportsProvider;
+
+/**
+ * This class is responsible for setting the metadata property 'totals' on each dataTable if the report
+ * has a dimension. 'Totals' means it tries to calculate the total report value for each metric. For each
+ * the total number of visits, actions, ... for a given report / dataTable.
+ */
+class ReportTotalsCalculator extends DataTableManipulator
+{
+    /**
+     * @var Report
+     */
+    private $report;
+
+    /**
+     * Constructor
+     *
+     * @param bool $apiModule
+     * @param bool $apiMethod
+     * @param array $request
+     * @param Report $report
+     */
+    public function __construct($apiModule = false, $apiMethod = false, $request = array(), $report = null)
+    {
+        parent::__construct($apiModule, $apiMethod, $request);
+        $this->report = $report;
+    }
+
+    /**
+     * @param  DataTable $table
+     * @return \Piwik\DataTable|\Piwik\DataTable\Map
+     */
+    public function calculate($table)
+    {
+        // apiModule and/or apiMethod is empty for instance in case when flat=1 is called. Basically whenever a
+        // datamanipulator calls the API and wants the dataTable in return, see callApiAndReturnDataTable().
+        // it is also not set for some settings API request etc.
+        if (empty($this->apiModule) || empty($this->apiMethod)) {
+            return $table;
+        }
+
+        try {
+            return $this->manipulate($table);
+        } catch (\Exception $e) {
+            // eg. requests with idSubtable may trigger this exception
+            // (where idSubtable was removed in
+            // ?module=API&method=Events.getNameFromCategoryId&idSubtable=1&secondaryDimension=eventName&format=XML&idSite=1&period=day&date=yesterday&flat=0
+            return $table;
+        }
+    }
+
+    /**
+     * Adds ratio metrics if possible.
+     *
+     * @param  DataTable $dataTable
+     * @return DataTable
+     */
+    protected function manipulateDataTable($dataTable)
+    {
+        if (!empty($this->report) && !$this->report->getDimension() && !$this->isAllMetricsReport()) {
+            // we currently do not calculate the total value for reports having no dimension
+            return $dataTable;
+        }
+
+        if (1 != Common::getRequestVar('totals', 1, 'integer', $this->request)) {
+            return $dataTable;
+        }
+
+        $firstLevelTable = $this->makeSureToWorkOnFirstLevelDataTable($dataTable);
+
+        if (!$firstLevelTable->getRowsCount()
+            || $dataTable->getTotalsRow()
+            || $dataTable->getMetadata('totals')
+        ) {
+            return $dataTable;
+        }
+
+        // keeping queued filters would not only add various metadata but also break the totals calculator for some reports
+        // eg when needed metadata is missing to get site information (multisites.getall) etc
+        $clone = $firstLevelTable->getEmptyClone($keepFilters = false);
+        foreach ($firstLevelTable->getQueuedFilters() as $queuedFilter) {
+            if (is_array($queuedFilter) && 'ReplaceColumnNames' === $queuedFilter['className']) {
+                $clone->queueFilter($queuedFilter['className'], $queuedFilter['parameters']);
+            }
+        }
+        $tableMeta = $firstLevelTable->getMetadata(DataTable::COLUMN_AGGREGATION_OPS_METADATA_NAME);
+
+        /** @var DataTable\Row $totalRow */
+        $totalRow = null;
+        foreach ($firstLevelTable->getRows() as $row) {
+            if (!isset($totalRow)) {
+                $columns = $row->getColumns();
+                $columns['label'] = DataTable::LABEL_TOTALS_ROW;
+                $totalRow = new DataTable\Row(array(DataTable\Row::COLUMNS => $columns));
+            } else {
+                $totalRow->sumRow($row, $copyMetadata = false, $tableMeta);
+            }
+        }
+        $clone->addRow($totalRow);
+
+        if ($this->report
+            && $this->report->getProcessedMetrics()
+            && array_keys($this->report->getProcessedMetrics()) === array('nb_actions_per_visit', 'avg_time_on_site', 'bounce_rate', 'conversion_rate')) {
+            // hack for AllColumns table or default processed metrics
+            $clone->filter('AddColumnsProcessedMetrics', array($deleteRowsWithNoVisit = false));
+        }
+
+        $processor = new DataTablePostProcessor($this->apiModule, $this->apiMethod, $this->request);
+        $processor->applyComputeProcessedMetrics($clone);
+        $clone = $processor->applyQueuedFilters($clone);
+        $clone = $processor->applyMetricsFormatting($clone);
+
+        $totalRow = null;
+        foreach ($clone->getRows() as $row) {
+            /** * @var DataTable\Row $row */
+            if ($row->getColumn('label') === DataTable::LABEL_TOTALS_ROW) {
+                $totalRow = $row;
+                break;
+            }
+        }
+
+        if (!isset($totalRow) && $clone->getRowsCount() === 1) {
+            // if for some reason the processor renamed the totals row,
+            $totalRow = $clone->getFirstRow();
+        }
+
+        if (isset($totalRow)) {
+            $totals = $row->getColumns();
+            unset($totals['label']);
+            $dataTable->setMetadata('totals', $totals);
+
+            if (1 === Common::getRequestVar('keep_totals_row', 0, 'integer', $this->request)) {
+                $row->deleteMetadata(false);
+                $row->setColumn('label', Piwik::translate('General_Totals'));
+                $dataTable->setTotalsRow($row);
+            }
+        }
+
+        return $dataTable;
+    }
+
+    private function makeSureToWorkOnFirstLevelDataTable($table)
+    {
+        if (!array_key_exists('idSubtable', $this->request)) {
+            return $table;
+        }
+
+        $firstLevelReport = $this->findFirstLevelReport();
+
+        if (empty($firstLevelReport)) {
+            // it is not a subtable report
+            $module = $this->apiModule;
+            $action = $this->apiMethod;
+        } else {
+            $module = $firstLevelReport->getModule();
+            $action = $firstLevelReport->getAction();
+        }
+
+        $request = $this->request;
+        unset($request['idSubtable']); // to make sure we work on first level table
+
+        /** @var \Piwik\Period $period */
+        $period = $table->getMetadata('period');
+
+        if (!empty($period)) {
+            // we want a dataTable, not a dataTable\map
+            if (Period::isMultiplePeriod($request['date'], $request['period']) || 'range' == $period->getLabel()) {
+                $request['date']   = $period->getRangeString();
+                $request['period'] = 'range';
+            } else {
+                $request['date']   = $period->getDateStart()->toString();
+                $request['period'] = $period->getLabel();
+            }
+        }
+
+        $table = $this->callApiAndReturnDataTable($module, $action, $request);
+
+        if ($table instanceof DataTable\Map) {
+            $table = $table->mergeChildren();
+        }
+
+        return $table;
+    }
+
+    /**
+     * Make sure to get all rows of the first level table.
+     *
+     * @param array $request
+     * @return array
+     */
+    protected function manipulateSubtableRequest($request)
+    {
+        $request['totals']        = 0;
+        $request['expanded']      = 0;
+        $request['filter_limit']  = -1;
+        $request['filter_offset'] = 0;
+        $request['filter_sort_column'] = '';
+
+        $parametersToRemove = array('flat');
+
+        if (!array_key_exists('idSubtable', $this->request)) {
+            $parametersToRemove[] = 'idSubtable';
+        }
+
+        foreach ($parametersToRemove as $param) {
+            if (array_key_exists($param, $request)) {
+                unset($request[$param]);
+            }
+        }
+        return $request;
+    }
+
+    private function findFirstLevelReport()
+    {
+        $reports = new ReportsProvider();
+        foreach ($reports->getAllReports() as $report) {
+            $actionToLoadSubtables = $report->getActionToLoadSubTables();
+            if ($actionToLoadSubtables == $this->apiMethod
+                && $this->apiModule == $report->getModule()
+            ) {
+                return $report;
+            }
+        }
+        return null;
+    }
+
+    private function isAllMetricsReport()
+    {
+        return $this->report->getModule() == 'API' && $this->report->getAction() == 'get';
+    }
+}

msd2/tracking/piwik/core/API/DataTablePostProcessor.php

@@ -0,0 +1,459 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+
+namespace Piwik\API;
+
+use Exception;
+use Piwik\API\DataTableManipulator\Flattener;
+use Piwik\API\DataTableManipulator\LabelFilter;
+use Piwik\API\DataTableManipulator\ReportTotalsCalculator;
+use Piwik\Common;
+use Piwik\DataTable;
+use Piwik\DataTable\DataTableInterface;
+use Piwik\DataTable\Filter\PivotByDimension;
+use Piwik\Metrics\Formatter;
+use Piwik\Plugin\ProcessedMetric;
+use Piwik\Plugin\Report;
+use Piwik\Plugin\ReportsProvider;
+
+/**
+ * Processes DataTables that should be served through Piwik's APIs. This processing handles
+ * special query parameters and computes processed metrics. It does not included rendering to
+ * output formats (eg, 'xml').
+ */
+class DataTablePostProcessor
+{
+    const PROCESSED_METRICS_COMPUTED_FLAG = 'processed_metrics_computed';
+
+    /**
+     * @var null|Report
+     */
+    private $report;
+
+    /**
+     * @var string[]
+     */
+    private $request;
+
+    /**
+     * @var string
+     */
+    private $apiModule;
+
+    /**
+     * @var string
+     */
+    private $apiMethod;
+
+    /**
+     * @var Inconsistencies
+     */
+    private $apiInconsistencies;
+
+    /**
+     * @var Formatter
+     */
+    private $formatter;
+
+    private $callbackBeforeGenericFilters;
+    private $callbackAfterGenericFilters;
+
+    /**
+     * Constructor.
+     */
+    public function __construct($apiModule, $apiMethod, $request)
+    {
+        $this->apiModule = $apiModule;
+        $this->apiMethod = $apiMethod;
+        $this->setRequest($request);
+
+        $this->report = ReportsProvider::factory($apiModule, $apiMethod);
+        $this->apiInconsistencies = new Inconsistencies();
+        $this->setFormatter(new Formatter());
+    }
+
+    public function setFormatter(Formatter $formatter)
+    {
+        $this->formatter = $formatter;
+    }
+
+    public function setRequest($request)
+    {
+        $this->request = $request;
+    }
+
+    public function setCallbackBeforeGenericFilters($callbackBeforeGenericFilters)
+    {
+        $this->callbackBeforeGenericFilters = $callbackBeforeGenericFilters;
+    }
+
+    public function setCallbackAfterGenericFilters($callbackAfterGenericFilters)
+    {
+        $this->callbackAfterGenericFilters = $callbackAfterGenericFilters;
+    }
+
+    /**
+     * Apply post-processing logic to a DataTable of a report for an API request.
+     *
+     * @param DataTableInterface $dataTable The data table to process.
+     * @return DataTableInterface A new data table.
+     */
+    public function process(DataTableInterface $dataTable)
+    {
+        // TODO: when calculating metrics before hand, only calculate for needed metrics, not all. NOTE:
+        //       this is non-trivial since it will require, eg, to make sure processed metrics aren't added
+        //       after pivotBy is handled.
+        $dataTable = $this->applyPivotByFilter($dataTable);
+        $dataTable = $this->applyTotalsCalculator($dataTable);
+        $dataTable = $this->applyFlattener($dataTable);
+
+        if ($this->callbackBeforeGenericFilters) {
+            call_user_func($this->callbackBeforeGenericFilters, $dataTable);
+        }
+
+        $dataTable = $this->applyGenericFilters($dataTable);
+        $this->applyComputeProcessedMetrics($dataTable);
+
+        if ($this->callbackAfterGenericFilters) {
+            call_user_func($this->callbackAfterGenericFilters, $dataTable);
+        }
+
+        // we automatically safe decode all datatable labels (against xss)
+        $dataTable->queueFilter('SafeDecodeLabel');
+        $dataTable = $this->convertSegmentValueToSegment($dataTable);
+        $dataTable = $this->applyQueuedFilters($dataTable);
+        $dataTable = $this->applyRequestedColumnDeletion($dataTable);
+        $dataTable = $this->applyLabelFilter($dataTable);
+        $dataTable = $this->applyMetricsFormatting($dataTable);
+        return $dataTable;
+    }
+
+    private function convertSegmentValueToSegment(DataTableInterface $dataTable)
+    {
+        $dataTable->filter('AddSegmentBySegmentValue', array($this->report));
+        $dataTable->filter('ColumnCallbackDeleteMetadata', array('segmentValue'));
+
+        return $dataTable;
+    }
+
+    /**
+     * @param DataTableInterface $dataTable
+     * @return DataTableInterface
+     */
+    public function applyPivotByFilter(DataTableInterface $dataTable)
+    {
+        $pivotBy = Common::getRequestVar('pivotBy', false, 'string', $this->request);
+        if (!empty($pivotBy)) {
+            $this->applyComputeProcessedMetrics($dataTable);
+            $dataTable = $this->convertSegmentValueToSegment($dataTable);
+
+            $pivotByColumn = Common::getRequestVar('pivotByColumn', false, 'string', $this->request);
+            $pivotByColumnLimit = Common::getRequestVar('pivotByColumnLimit', false, 'int', $this->request);
+
+            $dataTable->filter('PivotByDimension', array($this->report, $pivotBy, $pivotByColumn, $pivotByColumnLimit,
+                PivotByDimension::isSegmentFetchingEnabledInConfig()));
+            $dataTable->filter('ColumnCallbackDeleteMetadata', array('segment'));
+        }
+        return $dataTable;
+    }
+
+    /**
+     * @param DataTableInterface $dataTable
+     * @return DataTable|DataTableInterface|DataTable\Map
+     */
+    public function applyFlattener($dataTable)
+    {
+        if (Common::getRequestVar('flat', '0', 'string', $this->request) == '1') {
+            // skip flattening if not supported by report and remove subtables only
+            if ($this->report && !$this->report->supportsFlatten()) {
+                $dataTable->filter('RemoveSubtables');
+                return $dataTable;
+            }
+
+            $flattener = new Flattener($this->apiModule, $this->apiMethod, $this->request);
+            if (Common::getRequestVar('include_aggregate_rows', '0', 'string', $this->request) == '1') {
+                $flattener->includeAggregateRows();
+            }
+
+            $recursiveLabelSeparator = ' - ';
+            if ($this->report) {
+                $recursiveLabelSeparator = $this->report->getRecursiveLabelSeparator();
+            }
+
+            $dataTable = $flattener->flatten($dataTable, $recursiveLabelSeparator);
+        }
+        return $dataTable;
+    }
+
+    /**
+     * @param DataTableInterface $dataTable
+     * @return DataTableInterface
+     */
+    public function applyTotalsCalculator($dataTable)
+    {
+        if (1 == Common::getRequestVar('totals', '1', 'integer', $this->request)) {
+            $calculator = new ReportTotalsCalculator($this->apiModule, $this->apiMethod, $this->request, $this->report);
+            $dataTable  = $calculator->calculate($dataTable);
+        }
+        return $dataTable;
+    }
+
+    /**
+     * @param DataTableInterface $dataTable
+     * @return DataTableInterface
+     */
+    public function applyGenericFilters($dataTable)
+    {
+        // if the flag disable_generic_filters is defined we skip the generic filters
+        if (0 == Common::getRequestVar('disable_generic_filters', '0', 'string', $this->request)) {
+            $this->applyProcessedMetricsGenericFilters($dataTable);
+
+            $genericFilter = new DataTableGenericFilter($this->request, $this->report);
+
+            $self = $this;
+            $report = $this->report;
+            $dataTable->filter(function (DataTable $table) use ($genericFilter, $report, $self) {
+                $processedMetrics = Report::getProcessedMetricsForTable($table, $report);
+                if ($genericFilter->areProcessedMetricsNeededFor($processedMetrics)) {
+                    $self->computeProcessedMetrics($table);
+                }
+            });
+
+            $label = self::getLabelFromRequest($this->request);
+            if (!empty($label)) {
+                $genericFilter->disableFilters(array('Limit', 'Truncate'));
+            }
+
+            $genericFilter->filter($dataTable);
+        }
+
+        return $dataTable;
+    }
+
+    /**
+     * @param DataTableInterface $dataTable
+     * @return DataTableInterface
+     */
+    public function applyProcessedMetricsGenericFilters($dataTable)
+    {
+        $addNormalProcessedMetrics = null;
+        try {
+            $addNormalProcessedMetrics = Common::getRequestVar(
+                'filter_add_columns_when_show_all_columns', null, 'integer', $this->request);
+        } catch (Exception $ex) {
+            // ignore
+        }
+
+        if ($addNormalProcessedMetrics !== null) {
+            $dataTable->filter('AddColumnsProcessedMetrics', array($addNormalProcessedMetrics));
+        }
+
+        $addGoalProcessedMetrics = null;
+        try {
+            $addGoalProcessedMetrics = Common::getRequestVar(
+                'filter_update_columns_when_show_all_goals', null, 'integer', $this->request);
+        } catch (Exception $ex) {
+            // ignore
+        }
+
+        if ($addGoalProcessedMetrics !== null) {
+            $idGoal = Common::getRequestVar(
+                'idGoal', DataTable\Filter\AddColumnsProcessedMetricsGoal::GOALS_OVERVIEW, 'string', $this->request);
+
+            $dataTable->filter('AddColumnsProcessedMetricsGoal', array($ignore = true, $idGoal));
+        }
+
+        return $dataTable;
+    }
+
+    /**
+     * @param DataTableInterface $dataTable
+     * @return DataTableInterface
+     */
+    public function applyQueuedFilters($dataTable)
+    {
+        // if the flag disable_queued_filters is defined we skip the filters that were queued
+        if (Common::getRequestVar('disable_queued_filters', 0, 'int', $this->request) == 0) {
+            $dataTable->applyQueuedFilters();
+        }
+        return $dataTable;
+    }
+
+    /**
+     * @param DataTableInterface $dataTable
+     * @return DataTableInterface
+     */
+    public function applyRequestedColumnDeletion($dataTable)
+    {
+        // use the ColumnDelete filter if hideColumns/showColumns is provided (must be done
+        // after queued filters are run so processed metrics can be removed, too)
+        $hideColumns = Common::getRequestVar('hideColumns', '', 'string', $this->request);
+        $showColumns = Common::getRequestVar('showColumns', '', 'string', $this->request);
+        $showRawMetrics = Common::getRequestVar('showRawMetrics', 0, 'int', $this->request);
+        if (!empty($hideColumns)
+            || !empty($showColumns)
+        ) {
+            $dataTable->filter('ColumnDelete', array($hideColumns, $showColumns));
+        } else if ($showRawMetrics !== 1) {
+            $this->removeTemporaryMetrics($dataTable);
+        }
+
+        return $dataTable;
+    }
+
+    /**
+     * @param DataTableInterface $dataTable
+     */
+    public function removeTemporaryMetrics(DataTableInterface $dataTable)
+    {
+        $allColumns = !empty($this->report) ? $this->report->getAllMetrics() : array();
+
+        $report = $this->report;
+        $dataTable->filter(function (DataTable $table) use ($report, $allColumns) {
+            $processedMetrics = Report::getProcessedMetricsForTable($table, $report);
+
+            $allTemporaryMetrics = array();
+            foreach ($processedMetrics as $metric) {
+                $allTemporaryMetrics = array_merge($allTemporaryMetrics, $metric->getTemporaryMetrics());
+            }
+
+            if (!empty($allTemporaryMetrics)) {
+                $table->filter('ColumnDelete', array($allTemporaryMetrics));
+            }
+        });
+    }
+
+    /**
+     * @param DataTableInterface $dataTable
+     * @return DataTableInterface
+     */
+    public function applyLabelFilter($dataTable)
+    {
+        $label = self::getLabelFromRequest($this->request);
+
+        // apply label filter: only return rows matching the label parameter (more than one if more than one label)
+        if (!empty($label)) {
+            $addLabelIndex = Common::getRequestVar('labelFilterAddLabelIndex', 0, 'int', $this->request) == 1;
+
+            $filter = new LabelFilter($this->apiModule, $this->apiMethod, $this->request);
+            $dataTable = $filter->filter($label, $dataTable, $addLabelIndex);
+        }
+        return $dataTable;
+    }
+
+    /**
+     * @param DataTableInterface $dataTable
+     * @return DataTableInterface
+     */
+    public function applyMetricsFormatting($dataTable)
+    {
+        $formatMetrics = Common::getRequestVar('format_metrics', 0, 'string', $this->request);
+        if ($formatMetrics == '0') {
+            return $dataTable;
+        }
+
+        // in Piwik 2.X & below, metrics are not formatted in API responses except for percents.
+        // this code implements this inconsistency
+        $onlyFormatPercents = $formatMetrics === 'bc';
+
+        $metricsToFormat = null;
+        if ($onlyFormatPercents) {
+            $metricsToFormat = $this->apiInconsistencies->getPercentMetricsToFormat();
+        }
+
+        // 'all' is a special value that indicates we should format non-processed metrics that are identified
+        // by string, like 'revenue'. this should be removed when all metrics are using the `Metric` class.
+        $formatAll = $formatMetrics === 'all';
+
+        $dataTable->filter(array($this->formatter, 'formatMetrics'), array($this->report, $metricsToFormat, $formatAll));
+        return $dataTable;
+    }
+
+    /**
+     * Returns the value for the label query parameter which can be either a string
+     * (ie, label=...) or array (ie, label[]=...).
+     *
+     * @param array $request
+     * @return array
+     */
+    public static function getLabelFromRequest($request)
+    {
+        $label = Common::getRequestVar('label', array(), 'array', $request);
+        if (empty($label)) {
+            $label = Common::getRequestVar('label', '', 'string', $request);
+            if (!empty($label)) {
+                $label = array($label);
+            }
+        }
+
+        $label = self::unsanitizeLabelParameter($label);
+        return $label;
+    }
+
+    public static function unsanitizeLabelParameter($label)
+    {
+        // this is needed because Proxy uses Common::getRequestVar which in turn
+        // uses Common::sanitizeInputValue. This causes the > that separates recursive labels
+        // to become &gt; and we need to undo that here.
+        $label = str_replace( htmlentities('>', ENT_COMPAT | ENT_HTML401, 'UTF-8'), '>', $label);
+        return $label;
+    }
+
+    public function computeProcessedMetrics(DataTable $dataTable)
+    {
+        if ($dataTable->getMetadata(self::PROCESSED_METRICS_COMPUTED_FLAG)) {
+            return;
+        }
+
+        /** @var ProcessedMetric[] $processedMetrics */
+        $processedMetrics = Report::getProcessedMetricsForTable($dataTable, $this->report);
+        if (empty($processedMetrics)) {
+            return;
+        }
+
+        $dataTable->setMetadata(self::PROCESSED_METRICS_COMPUTED_FLAG, true);
+
+        foreach ($processedMetrics as $name => $processedMetric) {
+            if (!$processedMetric->beforeCompute($this->report, $dataTable)) {
+                continue;
+            }
+
+            foreach ($dataTable->getRows() as $row) {
+                if ($row->getColumn($name) !== false) { // only compute the metric if it has not been computed already
+                    continue;
+                }
+
+                $computedValue = $processedMetric->compute($row);
+                if ($computedValue !== false) {
+                    $row->addColumn($name, $computedValue);
+                }
+            }
+        }
+
+        foreach ($dataTable->getRows() as $row) {
+            $subtable = $row->getSubtable();
+            if (!empty($subtable)) {
+                foreach ($processedMetrics as $name => $processedMetric) {
+                    $processedMetric->beforeComputeSubtable($row);
+                }
+
+                $this->computeProcessedMetrics($subtable);
+
+                foreach ($processedMetrics as $name => $processedMetric) {
+                    $processedMetric->afterComputeSubtable($row);
+                }
+            }
+        }
+    }
+
+    public function applyComputeProcessedMetrics(DataTableInterface $dataTable)
+    {
+        $dataTable->filter(array($this, 'computeProcessedMetrics'));
+    }
+}
+

msd2/tracking/piwik/core/API/DocumentationGenerator.php

@@ -0,0 +1,390 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\API;
+
+use Exception;
+use Piwik\Common;
+use Piwik\Container\StaticContainer;
+use Piwik\Piwik;
+use Piwik\Url;
+use ReflectionClass;
+
+/**
+ * Possible tags to use in APIs
+ *
+ * @hide -> Won't be shown in list of all APIs but is also not possible to be called via HTTP API
+ * @hideForAll Same as @hide
+ * @hideExceptForSuperUser Same as @hide but still shown and possible to be called by a user with super user access
+ * @internal -> Won't be shown in list of all APIs but is possible to be called via HTTP API
+ */
+class DocumentationGenerator
+{
+    protected $countPluginsLoaded = 0;
+
+    /**
+     * trigger loading all plugins with an API.php file in the Proxy
+     */
+    public function __construct()
+    {
+        $plugins = \Piwik\Plugin\Manager::getInstance()->getLoadedPluginsName();
+        foreach ($plugins as $plugin) {
+            try {
+                $className = Request::getClassNameAPI($plugin);
+                Proxy::getInstance()->registerClass($className);
+            } catch (Exception $e) {
+            }
+        }
+    }
+
+    /**
+     * Returns a HTML page containing help for all the successfully loaded APIs.
+     *
+     * @param bool $outputExampleUrls
+     * @return string
+     */
+    public function getApiDocumentationAsString($outputExampleUrls = true)
+    {
+        list($toc, $str) = $this->generateDocumentation($outputExampleUrls, $prefixUrls = '', $displayTitlesAsAngularDirective = true);
+
+        return "<div piwik-content-block content-title='Quick access to APIs' id='topApiRef' name='topApiRef'>
+				$toc</div>
+				$str";
+    }
+
+    /**
+     * Used on developer.piwik.org
+     *
+     * @param bool|true $outputExampleUrls
+     * @param string $prefixUrls
+     * @return string
+     */
+    public function getApiDocumentationAsStringForDeveloperReference($outputExampleUrls = true, $prefixUrls = '')
+    {
+        list($toc, $str) = $this->generateDocumentation($outputExampleUrls, $prefixUrls, $displayTitlesAsAngularDirective = false);
+
+        return "<h2 id='topApiRef' name='topApiRef'>Quick access to APIs</h2>
+				$toc
+				$str";
+    }
+
+    protected function prepareModuleToDisplay($moduleName)
+    {
+        return "<a href='#$moduleName'>$moduleName</a><br/>";
+    }
+
+    protected function prepareMethodToDisplay($moduleName, $info, $methods, $class, $outputExampleUrls, $prefixUrls, $displayTitlesAsAngularDirective)
+    {
+        $str = '';
+        $str .= "\n<a name='$moduleName' id='$moduleName'></a>";
+        if($displayTitlesAsAngularDirective) {
+            $str .= "<div piwik-content-block content-title='Module " . $moduleName . "'>";
+        } else {
+            $str .= "<h2>Module " . $moduleName . "</h2>";
+        }
+        $info['__documentation'] = $this->checkDocumentation($info['__documentation']);
+        $str .= "<div class='apiDescription'> " . $info['__documentation'] . " </div>";
+        foreach ($methods as $methodName) {
+            if (Proxy::getInstance()->isDeprecatedMethod($class, $methodName)) {
+                continue;
+            }
+
+            $params = $this->getParametersString($class, $methodName);
+
+            $str .= "\n <div class='apiMethod'>- <b>$moduleName.$methodName </b>" . $params . "";
+            $str .= '<small>';
+            if ($outputExampleUrls) {
+                $str .= $this->addExamples($class, $methodName, $prefixUrls);
+            }
+            $str .= '</small>';
+            $str .= "</div>\n";
+        }
+
+        if($displayTitlesAsAngularDirective) {
+            $str .= "</div>";
+        }
+
+        return $str;
+    }
+
+    protected function prepareModulesAndMethods($info, $moduleName)
+    {
+        $toDisplay = array();
+
+        foreach ($info as $methodName => $infoMethod) {
+            if ($methodName == '__documentation') {
+                continue;
+            }
+            $toDisplay[$moduleName][] = $methodName;
+        }
+
+        return $toDisplay;
+    }
+
+    protected function addExamples($class, $methodName, $prefixUrls)
+    {
+        $token_auth = "&token_auth=" . Piwik::getCurrentUserTokenAuth();
+        $parametersToSet = array(
+            'idSite' => Common::getRequestVar('idSite', 1, 'int'),
+            'period' => Common::getRequestVar('period', 'day', 'string'),
+            'date' => Common::getRequestVar('date', 'today', 'string')
+        );
+        $str = '';
+        $str .= "<span class=\"example\">";
+        $exampleUrl = $this->getExampleUrl($class, $methodName, $parametersToSet);
+        if ($exampleUrl !== false) {
+            $lastNUrls = '';
+            if (preg_match('/(&period)|(&date)/', $exampleUrl)) {
+                $exampleUrlRss = $prefixUrls . $this->getExampleUrl($class, $methodName, array('date' => 'last10', 'period' => 'day') + $parametersToSet);
+                $lastNUrls = ", RSS of the last <a target='_blank' href='$exampleUrlRss&format=rss$token_auth&translateColumnNames=1'>10 days</a>";
+            }
+            $exampleUrl = $prefixUrls . $exampleUrl;
+            $str .= " [ Example in
+                                                                    <a target='_blank' href='$exampleUrl&format=xml$token_auth'>XML</a>,
+                                                                    <a target='_blank' href='$exampleUrl&format=JSON$token_auth'>Json</a>,
+                                                                    <a target='_blank' href='$exampleUrl&format=Tsv$token_auth&translateColumnNames=1'>Tsv (Excel)</a>
+                                                                    $lastNUrls
+                                                                    ]";
+        } else {
+            $str .= " [ No example available ]";
+        }
+        $str .= "</span>";
+        return $str;
+    }
+
+    /**
+     * Check if Class contains @hide
+     *
+     * @param ReflectionClass $rClass instance of ReflectionMethod
+     * @return bool
+     */
+    public function checkIfClassCommentContainsHideAnnotation(ReflectionClass $rClass)
+    {
+        return false !== strstr($rClass->getDocComment(), '@hide');
+    }
+
+    /**
+     * Check if Class contains @internal
+     *
+     * @param ReflectionClass|\ReflectionMethod $rClass instance of ReflectionMethod
+     * @return bool
+     */
+    private function checkIfCommentContainsInternalAnnotation($rClass)
+    {
+        return false !== strstr($rClass->getDocComment(), '@internal');
+    }
+
+    /**
+     * Check if documentation contains @hide annotation and deletes it
+     *
+     * @param $moduleToCheck
+     * @return mixed
+     */
+    public function checkDocumentation($moduleToCheck)
+    {
+        if (strpos($moduleToCheck, '@hide') == true) {
+            $moduleToCheck = str_replace(strtok(strstr($moduleToCheck, '@hide'), "\n"), "", $moduleToCheck);
+        }
+        return $moduleToCheck;
+    }
+
+    /**
+     * Returns a string containing links to examples on how to call a given method on a given API
+     * It will export links to XML, CSV, HTML, JSON, PHP, etc.
+     * It will not export links for methods such as deleteSite or deleteUser
+     *
+     * @param string $class the class
+     * @param string $methodName the method
+     * @param array $parametersToSet parameters to set
+     * @return string|bool when not possible
+     */
+    public function getExampleUrl($class, $methodName, $parametersToSet = array())
+    {
+        $knowExampleDefaultParametersValues = array(
+            'access'         => 'view',
+            'userLogin'      => 'test',
+            'passwordMd5ied' => 'passwordExample',
+            'email'          => 'test@example.org',
+
+            'languageCode'   => 'fr',
+            'url'            => 'http://forum.piwik.org/',
+            'pageUrl'        => 'http://forum.piwik.org/',
+            'apiModule'      => 'UserCountry',
+            'apiAction'      => 'getCountry',
+            'lastMinutes'    => '30',
+            'abandonedCarts' => '0',
+            'segmentName'    => 'pageTitle',
+            'ip'             => '194.57.91.215',
+            'idSites'             => '1,2',
+            'idAlert'             => '1',
+            'seconds'        => '3600',
+//            'segmentName'    => 'browserCode',
+        );
+
+        foreach ($parametersToSet as $name => $value) {
+            $knowExampleDefaultParametersValues[$name] = $value;
+        }
+
+        // no links for these method names
+        $doNotPrintExampleForTheseMethods = array(
+            //Sites
+            'deleteSite',
+            'addSite',
+            'updateSite',
+            'addSiteAliasUrls',
+            //Users
+            'deleteUser',
+            'addUser',
+            'updateUser',
+            'setUserAccess',
+            //Goals
+            'addGoal',
+            'updateGoal',
+            'deleteGoal',
+            //Marketplace
+            'deleteLicenseKey'
+        );
+
+        if (in_array($methodName, $doNotPrintExampleForTheseMethods)) {
+            return false;
+        }
+
+        // we try to give an URL example to call the API
+        $aParameters = Proxy::getInstance()->getParametersList($class, $methodName);
+        $aParameters['format'] = false;
+        $aParameters['hideIdSubDatable'] = false;
+        $aParameters['serialize'] = false;
+        $aParameters['language'] = false;
+        $aParameters['translateColumnNames'] = false;
+        $aParameters['label'] = false;
+        $aParameters['flat'] = false;
+        $aParameters['include_aggregate_rows'] = false;
+        $aParameters['filter_offset'] = false; 
+        $aParameters['filter_limit'] = false; 
+        $aParameters['filter_sort_column'] = false; 
+        $aParameters['filter_sort_order'] = false; 
+        $aParameters['filter_excludelowpop'] = false; 
+        $aParameters['filter_excludelowpop_value'] = false; 
+        $aParameters['filter_column_recursive'] = false; 
+        $aParameters['filter_pattern'] = false; 
+        $aParameters['filter_pattern_recursive'] = false; 
+        $aParameters['filter_truncate'] = false;
+        $aParameters['hideColumns'] = false;
+        $aParameters['showColumns'] = false;
+        $aParameters['filter_pattern_recursive'] = false;
+        $aParameters['pivotBy'] = false;
+        $aParameters['pivotByColumn'] = false;
+        $aParameters['pivotByColumnLimit'] = false;
+        $aParameters['disable_queued_filters'] = false;
+        $aParameters['disable_generic_filters'] = false;
+        $aParameters['expanded'] = false;
+        $aParameters['idDimenson'] = false;
+        $aParameters['format_metrics'] = false;
+
+        $entityNames = StaticContainer::get('entities.idNames');
+        foreach ($entityNames as $entityName) {
+            if (isset($aParameters[$entityName])) {
+                continue;
+            }
+            $aParameters[$entityName] = false;
+        }
+
+        $moduleName = Proxy::getInstance()->getModuleNameFromClassName($class);
+        $aParameters = array_merge(array('module' => 'API', 'method' => $moduleName . '.' . $methodName), $aParameters);
+
+        foreach ($aParameters as $nameVariable => &$defaultValue) {
+            if (isset($knowExampleDefaultParametersValues[$nameVariable])) {
+                $defaultValue = $knowExampleDefaultParametersValues[$nameVariable];
+            } // if there isn't a default value for a given parameter,
+            // we need a 'know default value' or we can't generate the link
+            elseif ($defaultValue instanceof NoDefaultValue) {
+                return false;
+            }
+        }
+        return '?' . Url::getQueryStringFromParameters($aParameters);
+    }
+
+    /**
+     * Returns the methods $class.$name parameters (and default value if provided) as a string.
+     *
+     * @param string $class The class name
+     * @param string $name The method name
+     * @return string  For example "(idSite, period, date = 'today')"
+     */
+    protected function getParametersString($class, $name)
+    {
+        $aParameters = Proxy::getInstance()->getParametersList($class, $name);
+        $asParameters = array();
+        foreach ($aParameters as $nameVariable => $defaultValue) {
+            // Do not show API parameters starting with _
+            // They are supposed to be used only in internal API calls
+            if (strpos($nameVariable, '_') === 0) {
+                continue;
+            }
+            $str = $nameVariable;
+            if (!($defaultValue instanceof NoDefaultValue)) {
+                if (is_array($defaultValue)) {
+                    $str .= " = 'Array'";
+                } else {
+                    $str .= " = '$defaultValue'";
+                }
+            }
+            $asParameters[] = $str;
+        }
+        $sParameters = implode(", ", $asParameters);
+        return "($sParameters)";
+    }
+
+    /**
+     * @param $outputExampleUrls
+     * @param $prefixUrls
+     * @param $displayTitlesAsAngularDirective
+     * @return array
+     */
+    protected function generateDocumentation($outputExampleUrls, $prefixUrls, $displayTitlesAsAngularDirective)
+    {
+        $str = $toc = '';
+
+        foreach (Proxy::getInstance()->getMetadata() as $class => $info) {
+            $moduleName = Proxy::getInstance()->getModuleNameFromClassName($class);
+            $rClass = new ReflectionClass($class);
+
+            if (!Piwik::hasUserSuperUserAccess() && $this->checkIfClassCommentContainsHideAnnotation($rClass)) {
+                continue;
+            }
+
+            if ($this->checkIfCommentContainsInternalAnnotation($rClass)) {
+                continue;
+            }
+
+            $toDisplay = $this->prepareModulesAndMethods($info, $moduleName);
+
+            foreach ($toDisplay as $moduleName => $methods) {
+                foreach ($methods as $index => $method) {
+                    if (!method_exists($class, $method)) { // method is handled through API.Request.intercept event
+                        continue;
+                    }
+
+                    $reflectionMethod = new \ReflectionMethod($class, $method);
+                    if ($this->checkIfCommentContainsInternalAnnotation($reflectionMethod)) {
+                        unset($toDisplay[$moduleName][$index]);
+                    }
+                }
+                if (empty($toDisplay[$moduleName])) {
+                    unset($toDisplay[$moduleName]);
+                }
+            }
+
+            foreach ($toDisplay as $moduleName => $methods) {
+                $toc .= $this->prepareModuleToDisplay($moduleName);
+                $str .= $this->prepareMethodToDisplay($moduleName, $info, $methods, $class, $outputExampleUrls, $prefixUrls, $displayTitlesAsAngularDirective);
+            }
+        }
+        return array($toc, $str);
+    }
+}

msd2/tracking/piwik/core/API/Inconsistencies.php

@@ -0,0 +1,45 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+namespace Piwik\API;
+
+/**
+ * Contains logic to replicate inconsistencies in Piwik's API. This class exists
+ * to provide a way to clean up existing Piwik code and behavior without breaking
+ * backwards compatibility immediately.
+ *
+ * Code that handles the case when the 'format_metrics' query parameter value is
+ * 'bc' should be removed as well. This code is in API\Request and DataTablePostProcessor.
+ *
+ * Should be removed before releasing Piwik 3.0.
+ */
+class Inconsistencies
+{
+    /**
+     * In Piwik 2.X and below, the "raw" API would format percent values but no others.
+     * This method returns the list of percent metrics that were returned from the API
+     * formatted so we can maintain BC.
+     *
+     * Used by DataTablePostProcessor.
+     */
+    public function getPercentMetricsToFormat()
+    {
+        return array(
+            'bounce_rate',
+            'conversion_rate',
+            'abandoned_rate',
+            'interaction_rate',
+            'exit_rate',
+            'bounce_rate_returning',
+            'nb_visits_percentage',
+            '/.*_evolution/',
+            '/goal_.*_conversion_rate/',
+            '/form_.*_rate/',
+            '/field_.*_rate/',
+        );
+    }
+}

msd2/tracking/piwik/core/API/Proxy.php

@@ -0,0 +1,582 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+
+namespace Piwik\API;
+
+use Exception;
+use Piwik\Common;
+use Piwik\Container\StaticContainer;
+use Piwik\Context;
+use Piwik\Piwik;
+use Piwik\Plugin\Manager;
+use Piwik\Singleton;
+use ReflectionClass;
+use ReflectionMethod;
+
+/**
+ * Proxy is a singleton that has the knowledge of every method available, their parameters
+ * and default values.
+ * Proxy receives all the API calls requests via call() and forwards them to the right
+ * object, with the parameters in the right order.
+ *
+ * It will also log the performance of API calls (time spent, parameter values, etc.) if logger available
+ */
+class Proxy
+{
+    // array of already registered plugins names
+    protected $alreadyRegistered = array();
+
+    protected $metadataArray = array();
+    private $hideIgnoredFunctions = true;
+
+    // when a parameter doesn't have a default value we use this
+    private $noDefaultValue;
+
+    public function __construct()
+    {
+        $this->noDefaultValue = new NoDefaultValue();
+    }
+
+    public static function getInstance()
+    {
+        return StaticContainer::get(self::class);
+    }
+
+    /**
+     * Returns array containing reflection meta data for all the loaded classes
+     * eg. number of parameters, method names, etc.
+     *
+     * @return array
+     */
+    public function getMetadata()
+    {
+        ksort($this->metadataArray);
+        return $this->metadataArray;
+    }
+
+    /**
+     * Registers the API information of a given module.
+     *
+     * The module to be registered must be
+     * - a singleton (providing a getInstance() method)
+     * - the API file must be located in plugins/ModuleName/API.php
+     *   for example plugins/Referrers/API.php
+     *
+     * The method will introspect the methods, their parameters, etc.
+     *
+     * @param string $className ModuleName eg. "API"
+     */
+    public function registerClass($className)
+    {
+        if (isset($this->alreadyRegistered[$className])) {
+            return;
+        }
+        $this->includeApiFile($className);
+        $this->checkClassIsSingleton($className);
+
+        $rClass = new ReflectionClass($className);
+        if (!$this->shouldHideAPIMethod($rClass->getDocComment())) {
+            foreach ($rClass->getMethods() as $method) {
+                $this->loadMethodMetadata($className, $method);
+            }
+
+            $this->setDocumentation($rClass, $className);
+            $this->alreadyRegistered[$className] = true;
+        }
+    }
+
+    /**
+     * Will be displayed in the API page
+     *
+     * @param ReflectionClass $rClass Instance of ReflectionClass
+     * @param string $className Name of the class
+     */
+    private function setDocumentation($rClass, $className)
+    {
+        // Doc comment
+        $doc = $rClass->getDocComment();
+        $doc = str_replace(" * " . PHP_EOL, "<br>", $doc);
+
+        // boldify the first line only if there is more than one line, otherwise too much bold
+        if (substr_count($doc, '<br>') > 1) {
+            $firstLineBreak = strpos($doc, "<br>");
+            $doc = "<div class='apiFirstLine'>" . substr($doc, 0, $firstLineBreak) . "</div>" . substr($doc, $firstLineBreak + strlen("<br>"));
+        }
+        $doc = preg_replace("/(@package)[a-z _A-Z]*/", "", $doc);
+        $doc = preg_replace("/(@method).*/", "", $doc);
+        $doc = str_replace(array("\t", "\n", "/**", "*/", " * ", " *", "  ", "\t*", "  *  @package"), " ", $doc);
+
+        // replace 'foo' and `bar` and "foobar" with code blocks... much magic
+        $doc = preg_replace('/`(.*?)`/', '<code>$1</code>', $doc);
+        $this->metadataArray[$className]['__documentation'] = $doc;
+    }
+
+    /**
+     * Returns number of classes already loaded
+     * @return int
+     */
+    public function getCountRegisteredClasses()
+    {
+        return count($this->alreadyRegistered);
+    }
+
+    /**
+     * Will execute $className->$methodName($parametersValues)
+     * If any error is detected (wrong number of parameters, method not found, class not found, etc.)
+     * it will throw an exception
+     *
+     * It also logs the API calls, with the parameters values, the returned value, the performance, etc.
+     * You can enable logging in config/global.ini.php (log_api_call)
+     *
+     * @param string $className The class name (eg. API)
+     * @param string $methodName The method name
+     * @param array $parametersRequest The parameters pairs (name=>value)
+     *
+     * @return mixed|null
+     * @throws Exception|\Piwik\NoAccessException
+     */
+    public function call($className, $methodName, $parametersRequest)
+    {
+        // Temporarily sets the Request array to this API call context
+        return Context::executeWithQueryParameters($parametersRequest, function () use ($className, $methodName, $parametersRequest) {
+            $returnedValue = null;
+
+            $this->registerClass($className);
+
+            // instanciate the object
+            $object = $className::getInstance();
+
+            // check method exists
+            $this->checkMethodExists($className, $methodName);
+
+            // get the list of parameters required by the method
+            $parameterNamesDefaultValues = $this->getParametersList($className, $methodName);
+
+            // load parameters in the right order, etc.
+            $finalParameters = $this->getRequestParametersArray($parameterNamesDefaultValues, $parametersRequest);
+
+            // allow plugins to manipulate the value
+            $pluginName = $this->getModuleNameFromClassName($className);
+
+            $returnedValue = null;
+
+            /**
+             * Triggered before an API request is dispatched.
+             *
+             * This event can be used to modify the arguments passed to one or more API methods.
+             *
+             * **Example**
+             *
+             *     Piwik::addAction('API.Request.dispatch', function (&$parameters, $pluginName, $methodName) {
+             *         if ($pluginName == 'Actions') {
+             *             if ($methodName == 'getPageUrls') {
+             *                 // ... do something ...
+             *             } else {
+             *                 // ... do something else ...
+             *             }
+             *         }
+             *     });
+             *
+             * @param array &$finalParameters List of parameters that will be passed to the API method.
+             * @param string $pluginName The name of the plugin the API method belongs to.
+             * @param string $methodName The name of the API method that will be called.
+             */
+            Piwik::postEvent('API.Request.dispatch', array(&$finalParameters, $pluginName, $methodName));
+
+            /**
+             * Triggered before an API request is dispatched.
+             *
+             * This event exists for convenience and is triggered directly after the {@hook API.Request.dispatch}
+             * event is triggered. It can be used to modify the arguments passed to a **single** API method.
+             *
+             * _Note: This is can be accomplished with the {@hook API.Request.dispatch} event as well, however
+             * event handlers for that event will have to do more work._
+             *
+             * **Example**
+             *
+             *     Piwik::addAction('API.Actions.getPageUrls', function (&$parameters) {
+             *         // force use of a single website. for some reason.
+             *         $parameters['idSite'] = 1;
+             *     });
+             *
+             * @param array &$finalParameters List of parameters that will be passed to the API method.
+             */
+            Piwik::postEvent(sprintf('API.%s.%s', $pluginName, $methodName), array(&$finalParameters));
+
+            /**
+             * Triggered before an API request is dispatched.
+             *
+             * Use this event to intercept an API request and execute your own code instead. If you set
+             * `$returnedValue` in a handler for this event, the original API method will not be executed,
+             * and the result will be what you set in the event handler.
+             *
+             * @param mixed &$returnedValue Set this to set the result and preempt normal API invocation.
+             * @param array &$finalParameters List of parameters that will be passed to the API method.
+             * @param string $pluginName The name of the plugin the API method belongs to.
+             * @param string $methodName The name of the API method that will be called.
+             * @param array $parametersRequest The query parameters for this request.
+             */
+            Piwik::postEvent('API.Request.intercept', [&$returnedValue, $finalParameters, $pluginName, $methodName, $parametersRequest]);
+
+            $apiParametersInCorrectOrder = array();
+
+            foreach ($parameterNamesDefaultValues as $name => $defaultValue) {
+                if (isset($finalParameters[$name]) || array_key_exists($name, $finalParameters)) {
+                    $apiParametersInCorrectOrder[] = $finalParameters[$name];
+                }
+            }
+
+            // call the method if a hook hasn't already set an output variable
+            if ($returnedValue === null) {
+                $returnedValue = call_user_func_array(array($object, $methodName), $apiParametersInCorrectOrder);
+            }
+
+            $endHookParams = array(
+                &$returnedValue,
+                array('className'  => $className,
+                    'module'     => $pluginName,
+                    'action'     => $methodName,
+                    'parameters' => $finalParameters)
+            );
+
+            /**
+             * Triggered directly after an API request is dispatched.
+             *
+             * This event exists for convenience and is triggered immediately before the
+             * {@hook API.Request.dispatch.end} event. It can be used to modify the output of a **single**
+             * API method.
+             *
+             * _Note: This can be accomplished with the {@hook API.Request.dispatch.end} event as well,
+             * however event handlers for that event will have to do more work._
+             *
+             * **Example**
+             *
+             *     // append (0 hits) to the end of row labels whose row has 0 hits
+             *     Piwik::addAction('API.Actions.getPageUrls', function (&$returnValue, $info)) {
+             *         $returnValue->filter('ColumnCallbackReplace', 'label', function ($label, $hits) {
+             *             if ($hits === 0) {
+             *                 return $label . " (0 hits)";
+             *             } else {
+             *                 return $label;
+             *             }
+             *         }, null, array('nb_hits'));
+             *     }
+             *
+             * @param mixed &$returnedValue The API method's return value. Can be an object, such as a
+             *                              {@link Piwik\DataTable DataTable} instance.
+             *                              could be a {@link Piwik\DataTable DataTable}.
+             * @param array $extraInfo An array holding information regarding the API request. Will
+             *                         contain the following data:
+             *
+             *                         - **className**: The namespace-d class name of the API instance
+             *                                          that's being called.
+             *                         - **module**: The name of the plugin the API request was
+             *                                       dispatched to.
+             *                         - **action**: The name of the API method that was executed.
+             *                         - **parameters**: The array of parameters passed to the API
+             *                                           method.
+             */
+            Piwik::postEvent(sprintf('API.%s.%s.end', $pluginName, $methodName), $endHookParams);
+
+            /**
+             * Triggered directly after an API request is dispatched.
+             *
+             * This event can be used to modify the output of any API method.
+             *
+             * **Example**
+             *
+             *     // append (0 hits) to the end of row labels whose row has 0 hits for any report that has the 'nb_hits' metric
+             *     Piwik::addAction('API.Actions.getPageUrls.end', function (&$returnValue, $info)) {
+             *         // don't process non-DataTable reports and reports that don't have the nb_hits column
+             *         if (!($returnValue instanceof DataTableInterface)
+             *             || in_array('nb_hits', $returnValue->getColumns())
+             *         ) {
+             *             return;
+             *         }
+             *
+             *         $returnValue->filter('ColumnCallbackReplace', 'label', function ($label, $hits) {
+             *             if ($hits === 0) {
+             *                 return $label . " (0 hits)";
+             *             } else {
+             *                 return $label;
+             *             }
+             *         }, null, array('nb_hits'));
+             *     }
+             *
+             * @param mixed &$returnedValue The API method's return value. Can be an object, such as a
+             *                              {@link Piwik\DataTable DataTable} instance.
+             * @param array $extraInfo An array holding information regarding the API request. Will
+             *                         contain the following data:
+             *
+             *                         - **className**: The namespace-d class name of the API instance
+             *                                          that's being called.
+             *                         - **module**: The name of the plugin the API request was
+             *                                       dispatched to.
+             *                         - **action**: The name of the API method that was executed.
+             *                         - **parameters**: The array of parameters passed to the API
+             *                                           method.
+             */
+            Piwik::postEvent('API.Request.dispatch.end', $endHookParams);
+
+            return $returnedValue;
+        });
+    }
+
+    /**
+     * Returns the parameters names and default values for the method $name
+     * of the class $class
+     *
+     * @param string $class The class name
+     * @param string $name The method name
+     * @return array  Format array(
+     *                            'testParameter' => null, // no default value
+     *                            'life'          => 42, // default value = 42
+     *                            'date'          => 'yesterday',
+     *                       );
+     */
+    public function getParametersList($class, $name)
+    {
+        return $this->metadataArray[$class][$name]['parameters'];
+    }
+
+    /**
+     * Check if given method name is deprecated or not.
+     */
+    public function isDeprecatedMethod($class, $methodName)
+    {
+        return $this->metadataArray[$class][$methodName]['isDeprecated'];
+    }
+
+    /**
+     * Returns the 'moduleName' part of '\\Piwik\\Plugins\\moduleName\\API'
+     *
+     * @param string $className "API"
+     * @return string "Referrers"
+     */
+    public function getModuleNameFromClassName($className)
+    {
+        return str_replace(array('\\Piwik\\Plugins\\', '\\API'), '', $className);
+    }
+
+    public function isExistingApiAction($pluginName, $apiAction)
+    {
+        $namespacedApiClassName = "\\Piwik\\Plugins\\$pluginName\\API";
+        $api = $namespacedApiClassName::getInstance();
+
+        return method_exists($api, $apiAction);
+    }
+
+    public function buildApiActionName($pluginName, $apiAction)
+    {
+        return sprintf("%s.%s", $pluginName, $apiAction);
+    }
+
+    /**
+     * Sets whether to hide '@ignore'd functions from method metadata or not.
+     *
+     * @param bool $hideIgnoredFunctions
+     */
+    public function setHideIgnoredFunctions($hideIgnoredFunctions)
+    {
+        $this->hideIgnoredFunctions = $hideIgnoredFunctions;
+
+        // make sure metadata gets reloaded
+        $this->alreadyRegistered = array();
+        $this->metadataArray = array();
+    }
+
+    /**
+     * Returns an array containing the values of the parameters to pass to the method to call
+     *
+     * @param array $requiredParameters array of (parameter name, default value)
+     * @param array $parametersRequest
+     * @throws Exception
+     * @return array values to pass to the function call
+     */
+    private function getRequestParametersArray($requiredParameters, $parametersRequest)
+    {
+        $finalParameters = array();
+        foreach ($requiredParameters as $name => $defaultValue) {
+            try {
+                if ($defaultValue instanceof NoDefaultValue) {
+                    $requestValue = Common::getRequestVar($name, null, null, $parametersRequest);
+                } else {
+                    try {
+                        if ($name == 'segment' && !empty($parametersRequest['segment'])) {
+                            // segment parameter is an exception: we do not want to sanitize user input or it would break the segment encoding
+                            $requestValue = ($parametersRequest['segment']);
+                        } else {
+                            $requestValue = Common::getRequestVar($name, $defaultValue, null, $parametersRequest);
+                        }
+                    } catch (Exception $e) {
+                        // Special case: empty parameter in the URL, should return the empty string
+                        if (isset($parametersRequest[$name])
+                            && $parametersRequest[$name] === ''
+                        ) {
+                            $requestValue = '';
+                        } else {
+                            $requestValue = $defaultValue;
+                        }
+                    }
+                }
+            } catch (Exception $e) {
+                throw new Exception(Piwik::translate('General_PleaseSpecifyValue', array($name)));
+            }
+            $finalParameters[$name] = $requestValue;
+        }
+        return $finalParameters;
+    }
+
+    /**
+     * Includes the class API by looking up plugins/xxx/API.php
+     *
+     * @param string $fileName api class name eg. "API"
+     * @throws Exception
+     */
+    private function includeApiFile($fileName)
+    {
+        $module = self::getModuleNameFromClassName($fileName);
+        $path = Manager::getPluginDirectory($module) . '/API.php';
+
+        if (is_readable($path)) {
+            require_once $path; // prefixed by PIWIK_INCLUDE_PATH
+        } else {
+            throw new Exception("API module $module not found.");
+        }
+    }
+
+    /**
+     * @param string $class name of a class
+     * @param ReflectionMethod $method instance of ReflectionMethod
+     */
+    private function loadMethodMetadata($class, $method)
+    {
+        if (!$this->checkIfMethodIsAvailable($method)) {
+            return;
+        }
+        $name = $method->getName();
+        $parameters = $method->getParameters();
+        $docComment = $method->getDocComment();
+
+        $aParameters = array();
+        foreach ($parameters as $parameter) {
+            $nameVariable = $parameter->getName();
+
+            $defaultValue = $this->noDefaultValue;
+            if ($parameter->isDefaultValueAvailable()) {
+                $defaultValue = $parameter->getDefaultValue();
+            }
+
+            $aParameters[$nameVariable] = $defaultValue;
+        }
+        $this->metadataArray[$class][$name]['parameters'] = $aParameters;
+        $this->metadataArray[$class][$name]['numberOfRequiredParameters'] = $method->getNumberOfRequiredParameters();
+        $this->metadataArray[$class][$name]['isDeprecated'] = false !== strstr($docComment, '@deprecated');
+    }
+
+    /**
+     * Checks that the method exists in the class
+     *
+     * @param string $className The class name
+     * @param string $methodName The method name
+     * @throws Exception If the method is not found
+     */
+    private function checkMethodExists($className, $methodName)
+    {
+        if (!$this->isMethodAvailable($className, $methodName)) {
+            throw new Exception(Piwik::translate('General_ExceptionMethodNotFound', array($methodName, $className)));
+        }
+    }
+
+    /**
+     * @param $docComment
+     * @return bool
+     */
+    public function shouldHideAPIMethod($docComment)
+    {
+        $hideLine = strstr($docComment, '@hide');
+
+        if ($hideLine === false) {
+            return false;
+        }
+
+        $hideLine = trim($hideLine);
+        $hideLine .= ' ';
+
+        $token = trim(strtok($hideLine, " "), "\n");
+
+        $hide = false;
+
+        if (!empty($token)) {
+            /**
+             * This event exists for checking whether a Plugin API class or a Plugin API method tagged
+             * with a `@hideXYZ` should be hidden in the API listing.
+             *
+             * @param bool &$hide whether to hide APIs tagged with $token should be displayed.
+             */
+            Piwik::postEvent(sprintf('API.DocumentationGenerator.%s', $token), array(&$hide));
+        }
+
+        return $hide;
+    }
+
+    /**
+     * @param ReflectionMethod $method
+     * @return bool
+     */
+    protected function checkIfMethodIsAvailable(ReflectionMethod $method)
+    {
+        if (!$method->isPublic() || $method->isConstructor() || $method->getName() === 'getInstance') {
+            return false;
+        }
+
+        if ($this->hideIgnoredFunctions && false !== strstr($method->getDocComment(), '@ignore')) {
+            return false;
+        }
+
+        if ($this->shouldHideAPIMethod($method->getDocComment())) {
+            return false;
+        }
+
+        return true;
+    }
+
+    /**
+     * Returns true if the method is found in the API of the given class name.
+     *
+     * @param string $className The class name
+     * @param string $methodName The method name
+     * @return bool
+     */
+    private function isMethodAvailable($className, $methodName)
+    {
+        return isset($this->metadataArray[$className][$methodName]);
+    }
+
+    /**
+     * Checks that the class is a Singleton (presence of the getInstance() method)
+     *
+     * @param string $className The class name
+     * @throws Exception If the class is not a Singleton
+     */
+    private function checkClassIsSingleton($className)
+    {
+        if (!method_exists($className, "getInstance")) {
+            throw new Exception("$className that provide an API must be Singleton and have a 'public static function getInstance()' method.");
+        }
+    }
+}
+
+/**
+ * To differentiate between "no value" and default value of null
+ *
+ */
+class NoDefaultValue
+{
+}

msd2/tracking/piwik/core/API/Request.php

@@ -0,0 +1,643 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\API;
+
+use Exception;
+use Piwik\Access;
+use Piwik\Cache;
+use Piwik\Common;
+use Piwik\Container\StaticContainer;
+use Piwik\Context;
+use Piwik\DataTable;
+use Piwik\Exception\PluginDeactivatedException;
+use Piwik\IP;
+use Piwik\Log;
+use Piwik\Piwik;
+use Piwik\Plugin\Manager as PluginManager;
+use Piwik\Plugins\CoreHome\LoginWhitelist;
+use Piwik\SettingsServer;
+use Piwik\Url;
+use Piwik\UrlHelper;
+use Psr\Log\LoggerInterface;
+
+/**
+ * Dispatches API requests to the appropriate API method.
+ *
+ * The Request class is used throughout Piwik to call API methods. The difference
+ * between using Request and calling API methods directly is that Request
+ * will do more after calling the API including: applying generic filters, applying queued filters,
+ * and handling the **flat** and **label** query parameters.
+ *
+ * Additionally, the Request class will **forward current query parameters** to the request
+ * which is more convenient than calling {@link Piwik\Common::getRequestVar()} many times over.
+ *
+ * In most cases, using a Request object to query the API is the correct approach.
+ *
+ * ### Post-processing
+ *
+ * The return value of API methods undergo some extra processing before being returned by Request.
+ *
+ * ### Output Formats
+ *
+ * The value returned by Request will be serialized to a certain format before being returned.
+ *
+ * ### Examples
+ *
+ * **Basic Usage**
+ *
+ *     $request = new Request('method=UserLanguage.getLanguage&idSite=1&date=yesterday&period=week'
+ *                          . '&format=xml&filter_limit=5&filter_offset=0')
+ *     $result = $request->process();
+ *     echo $result;
+ *
+ * **Getting a unrendered DataTable**
+ *
+ *     // use the convenience method 'processRequest'
+ *     $dataTable = Request::processRequest('UserLanguage.getLanguage', array(
+ *         'idSite' => 1,
+ *         'date' => 'yesterday',
+ *         'period' => 'week',
+ *         'filter_limit' => 5,
+ *         'filter_offset' => 0
+ *
+ *         'format' => 'original', // this is the important bit
+ *     ));
+ *     echo "This DataTable has " . $dataTable->getRowsCount() . " rows.";
+ *
+ * @see http://piwik.org/docs/analytics-api
+ * @api
+ */
+class Request
+{
+    /**
+     * The count of nested API request invocations. Used to determine if the currently executing request is the root or not.
+     *
+     * @var int
+     */
+    private static $nestedApiInvocationCount = 0;
+
+    private $request = null;
+
+    /**
+     * Converts the supplied request string into an array of query paramater name/value
+     * mappings. The current query parameters (everything in `$_GET` and `$_POST`) are
+     * forwarded to request array before it is returned.
+     *
+     * @param string|array|null $request The base request string or array, eg,
+     *                                   `'module=UserLanguage&action=getLanguage'`.
+     * @param array $defaultRequest Default query parameters. If a query parameter is absent in `$request`, it will be loaded
+     *                              from this. Defaults to `$_GET + $_POST`.
+     * @return array
+     */
+    public static function getRequestArrayFromString($request, $defaultRequest = null)
+    {
+        if ($defaultRequest === null) {
+            $defaultRequest = self::getDefaultRequest();
+
+            $requestRaw = self::getRequestParametersGET();
+            if (!empty($requestRaw['segment'])) {
+                $defaultRequest['segment'] = $requestRaw['segment'];
+            }
+
+            if (!isset($defaultRequest['format_metrics'])) {
+                $defaultRequest['format_metrics'] = 'bc';
+            }
+        }
+
+        $requestArray = $defaultRequest;
+
+        if (!is_null($request)) {
+            if (is_array($request)) {
+                $requestParsed = $request;
+            } else {
+                $request = trim($request);
+                $request = str_replace(array("\n", "\t"), '', $request);
+
+                $requestParsed = UrlHelper::getArrayFromQueryString($request);
+            }
+
+            $requestArray = $requestParsed + $defaultRequest;
+        }
+
+        foreach ($requestArray as &$element) {
+            if (!is_array($element)) {
+                $element = trim($element);
+            }
+        }
+        return $requestArray;
+    }
+
+    /**
+     * Constructor.
+     *
+     * @param string|array $request Query string that defines the API call (must at least contain a **method** parameter),
+     *                              eg, `'method=UserLanguage.getLanguage&idSite=1&date=yesterday&period=week&format=xml'`
+     *                              If a request is not provided, then we use the values in the `$_GET` and `$_POST`
+     *                              superglobals.
+     * @param array $defaultRequest Default query parameters. If a query parameter is absent in `$request`, it will be loaded
+     *                              from this. Defaults to `$_GET + $_POST`.
+     */
+    public function __construct($request = null, $defaultRequest = null)
+    {
+        $this->request = self::getRequestArrayFromString($request, $defaultRequest);
+        $this->sanitizeRequest();
+        $this->renameModuleAndActionInRequest();
+    }
+
+    /**
+     * For backward compatibility: Piwik API still works if module=Referers,
+     * we rewrite to correct renamed plugin: Referrers
+     *
+     * @param $module
+     * @param $action
+     * @return array( $module, $action )
+     * @ignore
+     */
+    public static function getRenamedModuleAndAction($module, $action)
+    {
+        /**
+         * This event is posted in the Request dispatcher and can be used
+         * to overwrite the Module and Action to dispatch.
+         * This is useful when some Controller methods or API methods have been renamed or moved to another plugin.
+         *
+         * @param $module string
+         * @param $action string
+         */
+        Piwik::postEvent('Request.getRenamedModuleAndAction', array(&$module, &$action));
+
+        return array($module, $action);
+    }
+
+    /**
+     * Make sure that the request contains no logical errors
+     */
+    private function sanitizeRequest()
+    {
+        // The label filter does not work with expanded=1 because the data table IDs have a different meaning
+        // depending on whether the table has been loaded yet. expanded=1 causes all tables to be loaded, which
+        // is why the label filter can't descend when a recursive label has been requested.
+        // To fix this, we remove the expanded parameter if a label parameter is set.
+        if (isset($this->request['label']) && !empty($this->request['label'])
+            && isset($this->request['expanded']) && $this->request['expanded']
+        ) {
+            unset($this->request['expanded']);
+        }
+    }
+
+    /**
+     * Dispatches the API request to the appropriate API method and returns the result
+     * after post-processing.
+     *
+     * Post-processing includes:
+     *
+     * - flattening if **flat** is 0
+     * - running generic filters unless **disable_generic_filters** is set to 1
+     * - URL decoding label column values
+     * - running queued filters unless **disable_queued_filters** is set to 1
+     * - removing columns based on the values of the **hideColumns** and **showColumns** query parameters
+     * - filtering rows if the **label** query parameter is set
+     * - converting the result to the appropriate format (ie, XML, JSON, etc.)
+     *
+     * If `'original'` is supplied for the output format, the result is returned as a PHP
+     * object.
+     *
+     * @throws PluginDeactivatedException if the module plugin is not activated.
+     * @throws Exception if the requested API method cannot be called, if required parameters for the
+     *                   API method are missing or if the API method throws an exception and the **format**
+     *                   query parameter is **original**.
+     * @return DataTable|Map|string The data resulting from the API call.
+     */
+    public function process()
+    {
+        // read the format requested for the output data
+        $outputFormat = strtolower(Common::getRequestVar('format', 'xml', 'string', $this->request));
+
+        $disablePostProcessing = $this->shouldDisablePostProcessing();
+
+        // create the response
+        $response = new ResponseBuilder($outputFormat, $this->request);
+        if ($disablePostProcessing) {
+            $response->disableDataTablePostProcessor();
+        }
+
+        $corsHandler = new CORSHandler();
+        $corsHandler->handle();
+
+        $tokenAuth = Common::getRequestVar('token_auth', '', 'string', $this->request);
+        $shouldReloadAuth = false;
+
+        try {
+            ++self::$nestedApiInvocationCount;
+
+            // IP check is needed here as we cannot listen to API.Request.authenticate as it would then not return proper API format response.
+            // We can also not do it by listening to API.Request.dispatch as by then the user is already authenticated and we want to make sure
+            // to not expose any information in case the IP is not whitelisted.
+            $whitelist = new LoginWhitelist();
+            if ($whitelist->shouldCheckWhitelist() && $whitelist->shouldWhitelistApplyToAPI()) {
+                $ip = IP::getIpFromHeader();
+                $whitelist->checkIsWhitelisted($ip);
+            }
+
+            // read parameters
+            $moduleMethod = Common::getRequestVar('method', null, 'string', $this->request);
+
+            list($module, $method) = $this->extractModuleAndMethod($moduleMethod);
+            list($module, $method) = self::getRenamedModuleAndAction($module, $method);
+
+            PluginManager::getInstance()->checkIsPluginActivated($module);
+
+            $apiClassName = self::getClassNameAPI($module);
+
+            if ($shouldReloadAuth = self::shouldReloadAuthUsingTokenAuth($this->request)) {
+                $access = Access::getInstance();
+                $tokenAuthToRestore = $access->getTokenAuth();
+                $hadSuperUserAccess = $access->hasSuperUserAccess();
+                self::forceReloadAuthUsingTokenAuth($tokenAuth);
+            }
+
+            // call the method
+            $returnedValue = Proxy::getInstance()->call($apiClassName, $method, $this->request);
+
+            // get the response with the request query parameters loaded, since DataTablePost processor will use the Report
+            // class instance, which may inspect the query parameters. (eg, it may look for the idCustomReport parameters
+            // which may only exist in $this->request, if the request was called programatically)
+            $toReturn = Context::executeWithQueryParameters($this->request, function () use ($response, $returnedValue, $module, $method) {
+                return $response->getResponse($returnedValue, $module, $method);
+            });
+        } catch (Exception $e) {
+            StaticContainer::get(LoggerInterface::class)->error('Uncaught exception in API: {exception}', [
+                'exception' => $e,
+                'ignoreInScreenWriter' => true,
+            ]);
+
+            $toReturn = $response->getResponseException($e);
+        } finally {
+            --self::$nestedApiInvocationCount;
+        }
+
+        if ($shouldReloadAuth) {
+            $this->restoreAuthUsingTokenAuth($tokenAuthToRestore, $hadSuperUserAccess);
+        }
+
+        return $toReturn;
+    }
+
+    private function restoreAuthUsingTokenAuth($tokenToRestore, $hadSuperUserAccess)
+    {
+        // if we would not make sure to unset super user access, the tokenAuth would be not authenticated and any
+        // token would just keep super user access (eg if the token that was reloaded before had super user access)
+        Access::getInstance()->setSuperUserAccess(false);
+
+        // we need to restore by reloading the tokenAuth as some permissions could have been removed in the API
+        // request etc. Otherwise we could just store a clone of Access::getInstance() and restore here
+        self::forceReloadAuthUsingTokenAuth($tokenToRestore);
+
+        if ($hadSuperUserAccess && !Access::getInstance()->hasSuperUserAccess()) {
+            // we are in context of `doAsSuperUser()` and need to restore this behaviour
+            Access::getInstance()->setSuperUserAccess(true);
+        }
+    }
+
+    /**
+     * Returns the name of a plugin's API class by plugin name.
+     *
+     * @param string $plugin The plugin name, eg, `'Referrers'`.
+     * @return string The fully qualified API class name, eg, `'\Piwik\Plugins\Referrers\API'`.
+     */
+    public static function getClassNameAPI($plugin)
+    {
+        return sprintf('\Piwik\Plugins\%s\API', $plugin);
+    }
+
+    /**
+     * @ignore
+     * @internal
+     * @param string $currentApiMethod
+     */
+    public static function setIsRootRequestApiRequest($currentApiMethod)
+    {
+        Cache::getTransientCache()->save('API.setIsRootRequestApiRequest', $currentApiMethod);
+    }
+
+    /**
+     * @ignore
+     * @internal
+     * @return string current Api Method if it is an api request
+     */
+    public static function getRootApiRequestMethod()
+    {
+        return Cache::getTransientCache()->fetch('API.setIsRootRequestApiRequest');
+    }
+
+    /**
+     * Detect if the root request (the actual request) is an API request or not. To detect whether an API is currently
+     * request within any request, have a look at {@link isApiRequest()}.
+     *
+     * @return bool
+     * @throws Exception
+     */
+    public static function isRootRequestApiRequest()
+    {
+        $apiMethod = Cache::getTransientCache()->fetch('API.setIsRootRequestApiRequest');
+        return !empty($apiMethod);
+    }
+
+    /**
+     * Checks if the currently executing API request is the root API request or not.
+     *
+     * Note: the "root" API request is the first request made. Within that request, further API methods
+     * can be called programmatically. These requests are considered "child" API requests.
+     *
+     * @return bool
+     * @throws Exception
+     */
+    public static function isCurrentApiRequestTheRootApiRequest()
+    {
+        return self::$nestedApiInvocationCount == 1;
+    }
+
+    /**
+     * Detect if request is an API request. Meaning the module is 'API' and an API method having a valid format was
+     * specified. Note that this method will return true even if the actual request is for example a regular UI
+     * reporting page request but within this request we are currently processing an API request (eg a
+     * controller calls Request::processRequest('API.getMatomoVersion')). To find out if the root request is an API
+     * request or not, call {@link isRootRequestApiRequest()}
+     *
+     * @param array $request  eg array('module' => 'API', 'method' => 'Test.getMethod')
+     * @return bool
+     * @throws Exception
+     */
+    public static function isApiRequest($request)
+    {
+        $method = self::getMethodIfApiRequest($request);
+        return !empty($method);
+    }
+
+    /**
+     * Returns the current API method being executed, if the current request is an API request.
+     *
+     * @param array $request  eg array('module' => 'API', 'method' => 'Test.getMethod')
+     * @return string|null
+     * @throws Exception
+     */
+    public static function getMethodIfApiRequest($request)
+    {
+        $module = Common::getRequestVar('module', '', 'string', $request);
+        $method = Common::getRequestVar('method', '', 'string', $request);
+
+        $isApi = $module === 'API' && !empty($method) && (count(explode('.', $method)) === 2);
+        return $isApi ? $method : null;
+    }
+
+    /**
+     * If the token_auth is found in the $request parameter,
+     * the current session will be authenticated using this token_auth.
+     * It will overwrite the previous Auth object.
+     *
+     * @param array $request If null, uses the default request ($_GET)
+     * @return void
+     * @ignore
+     */
+    public static function reloadAuthUsingTokenAuth($request = null)
+    {
+        // if a token_auth is specified in the API request, we load the right permissions
+        $token_auth = Common::getRequestVar('token_auth', '', 'string', $request);
+
+        if (self::shouldReloadAuthUsingTokenAuth($request)) {
+            self::forceReloadAuthUsingTokenAuth($token_auth);
+        }
+    }
+
+    /**
+     * The current session will be authenticated using this token_auth.
+     * It will overwrite the previous Auth object.
+     *
+     * @param string $tokenAuth
+     * @return void
+     */
+    private static function forceReloadAuthUsingTokenAuth($tokenAuth)
+    {
+        /**
+         * Triggered when authenticating an API request, but only if the **token_auth**
+         * query parameter is found in the request.
+         *
+         * Plugins that provide authentication capabilities should subscribe to this event
+         * and make sure the global authentication object (the object returned by `StaticContainer::get('Piwik\Auth')`)
+         * is setup to use `$token_auth` when its `authenticate()` method is executed.
+         *
+         * @param string $token_auth The value of the **token_auth** query parameter.
+         */
+        Piwik::postEvent('API.Request.authenticate', array($tokenAuth));
+        if (!Access::getInstance()->reloadAccess() && $tokenAuth && $tokenAuth !== 'anonymous') {
+            /**
+             * @ignore
+             * @internal
+             */
+            Piwik::postEvent('API.Request.authenticate.failed');
+        }
+        SettingsServer::raiseMemoryLimitIfNecessary();
+    }
+
+    private static function shouldReloadAuthUsingTokenAuth($request)
+    {
+        if (is_null($request)) {
+            $request = self::getDefaultRequest();
+        }
+
+        if (!isset($request['token_auth'])) {
+            // no token is given so we just keep the current loaded user
+            return false;
+        }
+
+        // a token is specified, we need to reload auth in case it is different than the current one, even if it is empty
+        $tokenAuth = Common::getRequestVar('token_auth', '', 'string', $request);
+
+        // not using !== is on purpose as getTokenAuth() might return null whereas $tokenAuth is '' . In this case
+        // we do not need to reload.
+
+        return $tokenAuth != Access::getInstance()->getTokenAuth();
+    }
+
+    /**
+     * Returns array($class, $method) from the given string $class.$method
+     *
+     * @param string $parameter
+     * @throws Exception
+     * @return array
+     */
+    private function extractModuleAndMethod($parameter)
+    {
+        $a = explode('.', $parameter);
+        if (count($a) != 2) {
+            throw new Exception("The method name is invalid. Expected 'module.methodName'");
+        }
+        return $a;
+    }
+
+    /**
+     * Helper method that processes an API request in one line using the variables in `$_GET`
+     * and `$_POST`.
+     *
+     * @param string $method The API method to call, ie, `'Actions.getPageTitles'`.
+     * @param array $paramOverride The parameter name-value pairs to use instead of what's
+     *                             in `$_GET` & `$_POST`.
+     * @param array $defaultRequest Default query parameters. If a query parameter is absent in `$request`, it will be loaded
+     *                              from this. Defaults to `$_GET + $_POST`.
+     *
+     *                              To avoid using any parameters from $_GET or $_POST, set this to an empty `array()`.
+     * @return mixed The result of the API request. See {@link process()}.
+     */
+    public static function processRequest($method, $paramOverride = array(), $defaultRequest = null)
+    {
+        $params = array();
+        $params['format'] = 'original';
+        $params['serialize'] = '0';
+        $params['module'] = 'API';
+        $params['method'] = $method;
+        $params = $paramOverride + $params;
+
+        // process request
+        $request = new Request($params, $defaultRequest);
+        return $request->process();
+    }
+
+    /**
+     * Returns the original request parameters in the current query string as an array mapping
+     * query parameter names with values. The result of this function will not be affected
+     * by any modifications to `$_GET` and will not include parameters in `$_POST`.
+     *
+     * @return array
+     */
+    public static function getRequestParametersGET()
+    {
+        if (empty($_SERVER['QUERY_STRING'])) {
+            return array();
+        }
+        $GET = UrlHelper::getArrayFromQueryString($_SERVER['QUERY_STRING']);
+        return $GET;
+    }
+
+    /**
+     * Returns the URL for the current requested report w/o any filter parameters.
+     *
+     * @param string $module The API module.
+     * @param string $action The API action.
+     * @param array $queryParams Query parameter overrides.
+     * @return string
+     */
+    public static function getBaseReportUrl($module, $action, $queryParams = array())
+    {
+        $params = array_merge($queryParams, array('module' => $module, 'action' => $action));
+        return Request::getCurrentUrlWithoutGenericFilters($params);
+    }
+
+    /**
+     * Returns the current URL without generic filter query parameters.
+     *
+     * @param array $params Query parameter values to override in the new URL.
+     * @return string
+     */
+    public static function getCurrentUrlWithoutGenericFilters($params)
+    {
+        // unset all filter query params so the related report will show up in its default state,
+        // unless the filter param was in $queryParams
+        $genericFiltersInfo = DataTableGenericFilter::getGenericFiltersInformation();
+        foreach ($genericFiltersInfo as $filter) {
+            foreach ($filter[1] as $queryParamName => $queryParamInfo) {
+                if (!isset($params[$queryParamName])) {
+                    $params[$queryParamName] = null;
+                }
+            }
+        }
+
+        return Url::getCurrentQueryStringWithParametersModified($params);
+    }
+
+    /**
+     * Returns whether the DataTable result will have to be expanded for the
+     * current request before rendering.
+     *
+     * @return bool
+     * @ignore
+     */
+    public static function shouldLoadExpanded()
+    {
+        // if filter_column_recursive & filter_pattern_recursive are supplied, and flat isn't supplied
+        // we have to load all the child subtables.
+        return Common::getRequestVar('filter_column_recursive', false) !== false
+            && Common::getRequestVar('filter_pattern_recursive', false) !== false
+            && !self::shouldLoadFlatten();
+    }
+
+    /**
+     * @return bool
+     */
+    public static function shouldLoadFlatten()
+    {
+        return Common::getRequestVar('flat', false) == 1;
+    }
+
+    /**
+     * Returns the segment query parameter from the original request, without modifications.
+     *
+     * @return array|bool
+     */
+    public static function getRawSegmentFromRequest()
+    {
+        // we need the URL encoded segment parameter, we fetch it from _SERVER['QUERY_STRING'] instead of default URL decoded _GET
+        $segmentRaw = false;
+        $segment = Common::getRequestVar('segment', '', 'string');
+        if (!empty($segment)) {
+            $request = Request::getRequestParametersGET();
+            if (!empty($request['segment'])) {
+                $segmentRaw = $request['segment'];
+            }
+        }
+        return $segmentRaw;
+    }
+
+    private function renameModuleAndActionInRequest()
+    {
+        if (empty($this->request['apiModule'])) {
+            return;
+        }
+        if (empty($this->request['apiAction'])) {
+            $this->request['apiAction'] = null;
+        }
+        list($this->request['apiModule'], $this->request['apiAction']) = $this->getRenamedModuleAndAction($this->request['apiModule'], $this->request['apiAction']);
+    }
+
+    /**
+     * @return array
+     */
+    private static function getDefaultRequest()
+    {
+        return $_GET + $_POST;
+    }
+
+    private function shouldDisablePostProcessing()
+    {
+        $shouldDisable = false;
+
+        /**
+         * After an API method returns a value, the value is post processed (eg, rows are sorted
+         * based on the `filter_sort_column` query parameter, rows are truncated based on the
+         * `filter_limit`/`filter_offset` parameters, amongst other things).
+         *
+         * If you're creating a plugin that needs to disable post processing entirely for
+         * certain requests, use this event.
+         *
+         * @param bool &$shouldDisable Set this to true to disable datatable post processing for a request.
+         * @param array $request The request parameters.
+         */
+        Piwik::postEvent('Request.shouldDisablePostProcessing', [&$shouldDisable, $this->request]);
+
+        return $shouldDisable;
+    }
+}

msd2/tracking/piwik/core/API/ResponseBuilder.php

@@ -0,0 +1,233 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\API;
+
+use Exception;
+use Piwik\Common;
+use Piwik\DataTable;
+use Piwik\DataTable\Renderer;
+use Piwik\DataTable\DataTableInterface;
+use Piwik\DataTable\Filter\ColumnDelete;
+use Piwik\DataTable\Filter\Pattern;
+use Piwik\Plugins\Monolog\Processor\ExceptionToTextProcessor;
+
+/**
+ */
+class ResponseBuilder
+{
+    private $outputFormat = null;
+    private $apiRenderer  = null;
+    private $request      = null;
+    private $sendHeader   = true;
+    private $postProcessDataTable = true;
+
+    private $apiModule = false;
+    private $apiMethod = false;
+    private $shouldPrintBacktrace = false;
+
+    /**
+     * @param string $outputFormat
+     * @param array $request
+     */
+    public function __construct($outputFormat, $request = array(), $shouldPrintBacktrace = null)
+    {
+        $this->outputFormat = $outputFormat;
+        $this->request      = $request;
+        $this->apiRenderer  = ApiRenderer::factory($outputFormat, $request);
+        $this->shouldPrintBacktrace = $shouldPrintBacktrace === null ? \Piwik_ShouldPrintBackTraceWithMessage() : $shouldPrintBacktrace;
+    }
+
+    public function disableSendHeader()
+    {
+        $this->sendHeader = false;
+    }
+
+    public function disableDataTablePostProcessor()
+    {
+        $this->postProcessDataTable = false;
+    }
+
+    /**
+     * This method processes the data resulting from the API call.
+     *
+     * - If the data resulted from the API call is a DataTable then
+     *         - we apply the standard filters if the parameters have been found
+     *           in the URL. For example to offset,limit the Table you can add the following parameters to any API
+     *        call that returns a DataTable: filter_limit=10&filter_offset=20
+     *         - we apply the filters that have been previously queued on the DataTable
+     * @see DataTable::queueFilter()
+     *         - we apply the renderer that generate the DataTable in a given format (XML, PHP, HTML, JSON, etc.)
+     *           the format can be changed using the 'format' parameter in the request.
+     *        Example: format=xml
+     *
+     * - If there is nothing returned (void) we display a standard success message
+     *
+     * - If there is a PHP array returned, we try to convert it to a dataTable
+     *   It is then possible to convert this datatable to any requested format (xml/etc)
+     *
+     * - If a bool is returned we convert to a string (true is displayed as 'true' false as 'false')
+     *
+     * - If an integer / float is returned, we simply return it
+     *
+     * @param mixed $value The initial returned value, before post process. If set to null, success response is returned.
+     * @param bool|string $apiModule The API module that was called
+     * @param bool|string $apiMethod The API method that was called
+     * @return mixed  Usually a string, but can still be a PHP data structure if the format requested is 'original'
+     */
+    public function getResponse($value = null, $apiModule = false, $apiMethod = false)
+    {
+        $this->apiModule = $apiModule;
+        $this->apiMethod = $apiMethod;
+
+        $this->sendHeaderIfEnabled();
+
+        // when null or void is returned from the api call, we handle it as a successful operation
+        if (!isset($value)) {
+            if (ob_get_contents()) {
+                return null;
+            }
+
+            return $this->apiRenderer->renderSuccess('ok');
+        }
+
+        // If the returned value is an object DataTable we
+        // apply the set of generic filters if asked in the URL
+        // and we render the DataTable according to the format specified in the URL
+        if ($value instanceof DataTableInterface) {
+            return $this->handleDataTable($value);
+        }
+
+        // Case an array is returned from the API call, we convert it to the requested format
+        // - if calling from inside the application (format = original)
+        //    => the data stays unchanged (ie. a standard php array or whatever data structure)
+        // - if any other format is requested, we have to convert this data structure (which we assume
+        //   to be an array) to a DataTable in order to apply the requested DataTable_Renderer (for example XML)
+        if (is_array($value)) {
+            return $this->handleArray($value);
+        }
+
+        if (is_object($value)) {
+            return $this->apiRenderer->renderObject($value);
+        }
+
+        if (is_resource($value)) {
+            return $this->apiRenderer->renderResource($value);
+        }
+
+        return $this->apiRenderer->renderScalar($value);
+    }
+
+    /**
+     * Returns an error $message in the requested $format
+     *
+     * @param Exception|\Throwable $e
+     * @throws Exception
+     * @return string
+     */
+    public function getResponseException($e)
+    {
+        $e       = $this->decorateExceptionWithDebugTrace($e);
+        $message = $this->formatExceptionMessage($e);
+
+        $this->sendHeaderIfEnabled();
+
+        return $this->apiRenderer->renderException($message, $e);
+    }
+
+    /**
+     * @param Exception|\Throwable $e
+     * @return Exception
+     */
+    private function decorateExceptionWithDebugTrace($e)
+    {
+        // If we are in tests, show full backtrace
+        if (defined('PIWIK_PATH_TEST_TO_ROOT')) {
+            if ($this->shouldPrintBacktrace) {
+                $message = $e->getMessage() . " in \n " . $e->getFile() . ":" . $e->getLine() . " \n " . $e->getTraceAsString();
+            } else {
+                $message = $e->getMessage() . "\n \n --> To temporarily debug this error further, set const PIWIK_PRINT_ERROR_BACKTRACE=true; in index.php";
+            }
+
+            return new Exception($message);
+        }
+
+        return $e;
+    }
+
+    /**
+     * @param Exception|\Throwable $exception
+     * @return string
+     */
+    private function formatExceptionMessage($exception)
+    {
+        $message = ExceptionToTextProcessor::getWholeBacktrace($exception, $this->shouldPrintBacktrace);
+
+        return Renderer::formatValueXml($message);
+    }
+
+    private function handleDataTable(DataTableInterface $datatable)
+    {
+        if ($this->postProcessDataTable) {
+            $postProcessor = new DataTablePostProcessor($this->apiModule, $this->apiMethod, $this->request);
+            $datatable = $postProcessor->process($datatable);
+        }
+
+        return $this->apiRenderer->renderDataTable($datatable);
+    }
+
+    private function handleArray($array)
+    {
+        $firstArray = null;
+        $firstKey   = null;
+        if (!empty($array)) {
+            $firstArray = reset($array);
+            $firstKey   = key($array);
+        }
+
+        $isAssoc = !empty($firstArray) && is_numeric($firstKey) && is_array($firstArray) && count(array_filter(array_keys($firstArray), 'is_string'));
+
+        if (is_numeric($firstKey)) {
+            $columns = Common::getRequestVar('filter_column', false, 'array', $this->request);
+            $pattern = Common::getRequestVar('filter_pattern', '', 'string', $this->request);
+
+            if ($columns != array(false) && $pattern !== '') {
+                $pattern = new Pattern(new DataTable(), $columns, $pattern);
+                $array   = $pattern->filterArray($array);
+            }
+
+            $limit  = Common::getRequestVar('filter_limit', -1, 'integer', $this->request);
+            $offset = Common::getRequestVar('filter_offset', '0', 'integer', $this->request);
+
+            if ($limit >= 0 || $offset > 0) {
+                if ($limit < 0) {
+                    $limit = null; // make sure to return all results from offset
+                }
+                $array = array_slice($array, $offset, $limit, $preserveKeys = false);
+            }
+        }
+
+        if ($isAssoc) {
+            $hideColumns = Common::getRequestVar('hideColumns', '', 'string', $this->request);
+            $showColumns = Common::getRequestVar('showColumns', '', 'string', $this->request);
+            if ($hideColumns !== '' || $showColumns !== '') {
+                $columnDelete = new ColumnDelete(new DataTable(), $hideColumns, $showColumns);
+                $array = $columnDelete->filter($array);
+            }
+        }
+
+        return $this->apiRenderer->renderArray($array);
+    }
+
+    private function sendHeaderIfEnabled()
+    {
+        if ($this->sendHeader) {
+            $this->apiRenderer->sendHeader();
+        }
+    }
+}