PDF rausgenommen

msd2/tracking/piwik/core/ArchiveProcessor/Loader.php

@@ -0,0 +1,260 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\ArchiveProcessor;
+
+use Piwik\Archive;
+use Piwik\Cache;
+use Piwik\CacheId;
+use Piwik\Common;
+use Piwik\Config;
+use Piwik\Context;
+use Piwik\DataAccess\ArchiveSelector;
+use Piwik\Date;
+use Piwik\Period;
+use Piwik\Piwik;
+
+/**
+ * This class uses PluginsArchiver class to trigger data aggregation and create archives.
+ */
+class Loader
+{
+    /**
+     * Is the current archive temporary. ie.
+     * - today
+     * - current week / month / year
+     */
+    protected $temporaryArchive;
+
+    /**
+     * Idarchive in the DB for the requested archive
+     *
+     * @var int
+     */
+    protected $idArchive;
+
+    /**
+     * @var Parameters
+     */
+    protected $params;
+
+    public function __construct(Parameters $params)
+    {
+        $this->params = $params;
+    }
+
+    /**
+     * @return bool
+     */
+    protected function isThereSomeVisits($visits)
+    {
+        return $visits > 0;
+    }
+
+    /**
+     * @return bool
+     */
+    protected function mustProcessVisitCount($visits)
+    {
+        return $visits === false;
+    }
+
+    public function prepareArchive($pluginName)
+    {
+        return Context::changeIdSite($this->params->getSite()->getId(), function () use ($pluginName) {
+            return $this->prepareArchiveImpl($pluginName);
+        });
+    }
+
+    private function prepareArchiveImpl($pluginName)
+    {
+        $this->params->setRequestedPlugin($pluginName);
+
+        list($idArchive, $visits, $visitsConverted) = $this->loadExistingArchiveIdFromDb();
+        if (!empty($idArchive)) {
+            return $idArchive;
+        }
+
+        list($visits, $visitsConverted) = $this->prepareCoreMetricsArchive($visits, $visitsConverted);
+        list($idArchive, $visits) = $this->prepareAllPluginsArchive($visits, $visitsConverted);
+
+        if ($this->isThereSomeVisits($visits) || PluginsArchiver::doesAnyPluginArchiveWithoutVisits()) {
+            return $idArchive;
+        }
+        return false;
+    }
+
+    /**
+     * Prepares the core metrics if needed.
+     *
+     * @param $visits
+     * @return array
+     */
+    protected function prepareCoreMetricsArchive($visits, $visitsConverted)
+    {
+        $createSeparateArchiveForCoreMetrics = $this->mustProcessVisitCount($visits)
+                                && !$this->doesRequestedPluginIncludeVisitsSummary();
+
+        if ($createSeparateArchiveForCoreMetrics) {
+            $requestedPlugin = $this->params->getRequestedPlugin();
+
+            $this->params->setRequestedPlugin('VisitsSummary');
+
+            $pluginsArchiver = new PluginsArchiver($this->params, $this->isArchiveTemporary());
+            $metrics = $pluginsArchiver->callAggregateCoreMetrics();
+            $pluginsArchiver->finalizeArchive();
+
+            $this->params->setRequestedPlugin($requestedPlugin);
+
+            $visits = $metrics['nb_visits'];
+            $visitsConverted = $metrics['nb_visits_converted'];
+        }
+
+        return array($visits, $visitsConverted);
+    }
+
+    protected function prepareAllPluginsArchive($visits, $visitsConverted)
+    {
+        $pluginsArchiver = new PluginsArchiver($this->params, $this->isArchiveTemporary());
+
+        if ($this->mustProcessVisitCount($visits)
+            || $this->doesRequestedPluginIncludeVisitsSummary()
+        ) {
+            $metrics = $pluginsArchiver->callAggregateCoreMetrics();
+            $visits = $metrics['nb_visits'];
+            $visitsConverted = $metrics['nb_visits_converted'];
+        }
+
+        $forceArchivingWithoutVisits = !$this->isThereSomeVisits($visits) && $this->shouldArchiveForSiteEvenWhenNoVisits();
+        $pluginsArchiver->callAggregateAllPlugins($visits, $visitsConverted, $forceArchivingWithoutVisits);
+
+        $idArchive = $pluginsArchiver->finalizeArchive();
+
+        return array($idArchive, $visits);
+    }
+
+    protected function doesRequestedPluginIncludeVisitsSummary()
+    {
+        $processAllReportsIncludingVisitsSummary =
+                Rules::shouldProcessReportsAllPlugins($this->params->getIdSites(), $this->params->getSegment(), $this->params->getPeriod()->getLabel());
+        $doesRequestedPluginIncludeVisitsSummary = $processAllReportsIncludingVisitsSummary
+                                                        || $this->params->getRequestedPlugin() == 'VisitsSummary';
+        return $doesRequestedPluginIncludeVisitsSummary;
+    }
+
+    protected function isArchivingForcedToTrigger()
+    {
+        $period = $this->params->getPeriod()->getLabel();
+        $debugSetting = 'always_archive_data_period'; // default
+
+        if ($period == 'day') {
+            $debugSetting = 'always_archive_data_day';
+        } elseif ($period == 'range') {
+            $debugSetting = 'always_archive_data_range';
+        }
+
+        return (bool) Config::getInstance()->Debug[$debugSetting];
+    }
+
+    /**
+     * Returns the idArchive if the archive is available in the database for the requested plugin.
+     * Returns false if the archive needs to be processed.
+     *
+     * @return array
+     */
+    protected function loadExistingArchiveIdFromDb()
+    {
+        $noArchiveFound = array(false, false, false);
+
+        // see isArchiveTemporary()
+        $minDatetimeArchiveProcessedUTC = $this->getMinTimeArchiveProcessed();
+
+        if ($this->isArchivingForcedToTrigger()) {
+            return $noArchiveFound;
+        }
+
+        $idAndVisits = ArchiveSelector::getArchiveIdAndVisits($this->params, $minDatetimeArchiveProcessedUTC);
+
+        if (!$idAndVisits) {
+            return $noArchiveFound;
+        }
+
+        return $idAndVisits;
+    }
+
+    /**
+     * Returns the minimum archive processed datetime to look at. Only public for tests.
+     *
+     * @return int|bool  Datetime timestamp, or false if must look at any archive available
+     */
+    protected function getMinTimeArchiveProcessed()
+    {
+        $endDateTimestamp = self::determineIfArchivePermanent($this->params->getDateEnd());
+        $isArchiveTemporary = ($endDateTimestamp === false);
+        $this->temporaryArchive = $isArchiveTemporary;
+
+        if ($endDateTimestamp) {
+            // Permanent archive
+            return $endDateTimestamp;
+        }
+
+        $dateStart = $this->params->getDateStart();
+        $period    = $this->params->getPeriod();
+        $segment   = $this->params->getSegment();
+        $site      = $this->params->getSite();
+
+        // Temporary archive
+        return Rules::getMinTimeProcessedForTemporaryArchive($dateStart, $period, $segment, $site);
+    }
+
+    protected static function determineIfArchivePermanent(Date $dateEnd)
+    {
+        $now = time();
+        $endTimestampUTC = strtotime($dateEnd->getDateEndUTC());
+
+        if ($endTimestampUTC <= $now) {
+            // - if the period we are looking for is finished, we look for a ts_archived that
+            //   is greater than the last day of the archive
+            return $endTimestampUTC;
+        }
+
+        return false;
+    }
+
+    protected function isArchiveTemporary()
+    {
+        if (is_null($this->temporaryArchive)) {
+            throw new \Exception("getMinTimeArchiveProcessed() should be called prior to isArchiveTemporary()");
+        }
+
+        return $this->temporaryArchive;
+    }
+
+    private function shouldArchiveForSiteEvenWhenNoVisits()
+    {
+        $idSitesToArchive = $this->getIdSitesToArchiveWhenNoVisits();
+        return in_array($this->params->getSite()->getId(), $idSitesToArchive);
+    }
+
+    private function getIdSitesToArchiveWhenNoVisits()
+    {
+        $cache = Cache::getTransientCache();
+        $cacheKey = 'Archiving.getIdSitesToArchiveWhenNoVisits';
+
+        if (!$cache->contains($cacheKey)) {
+            $idSites = array();
+
+            // leaving undocumented unless decided otherwise
+            Piwik::postEvent('Archiving.getIdSitesToArchiveWhenNoVisits', array(&$idSites));
+
+            $cache->save($cacheKey, $idSites);
+        }
+
+        return $cache->fetch($cacheKey);
+    }
+}

msd2/tracking/piwik/core/ArchiveProcessor/Parameters.php

@@ -0,0 +1,264 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+
+namespace Piwik\ArchiveProcessor;
+
+use Piwik\Date;
+use Piwik\Log;
+use Piwik\Period;
+use Piwik\Piwik;
+use Piwik\Segment;
+use Piwik\Site;
+
+/**
+ * Contains the analytics parameters for the reports that are currently being archived. The analytics
+ * parameters include the **website** the reports describe, the **period** of time the reports describe
+ * and the **segment** used to limit the visit set.
+ */
+class Parameters
+{
+    /**
+     * @var Site
+     */
+    private $site = null;
+
+    /**
+     * @var Period
+     */
+    private $period = null;
+
+    /**
+     * @var Segment
+     */
+    private $segment = null;
+
+    /**
+     * @var string Plugin name which triggered this archive processor
+     */
+    private $requestedPlugin = false;
+
+    private $onlyArchiveRequestedPlugin = false;
+
+    /**
+     * @var bool
+     */
+    private $isRootArchiveRequest = true;
+
+    /**
+     * Constructor.
+     *
+     * @ignore
+     */
+    public function __construct(Site $site, Period $period, Segment $segment)
+    {
+        $this->site = $site;
+        $this->period = $period;
+        $this->segment = $segment;
+    }
+
+    /**
+     * @ignore
+     */
+    public function setRequestedPlugin($plugin)
+    {
+        $this->requestedPlugin = $plugin;
+    }
+
+    /**
+     * @ignore
+     */
+    public function onlyArchiveRequestedPlugin()
+    {
+        $this->onlyArchiveRequestedPlugin = true;
+    }
+
+    /**
+     * @ignore
+     */
+    public function shouldOnlyArchiveRequestedPlugin()
+    {
+        return $this->onlyArchiveRequestedPlugin;
+    }
+
+    /**
+     * @ignore
+     */
+    public function getRequestedPlugin()
+    {
+        return $this->requestedPlugin;
+    }
+
+    /**
+     * Returns the period we are computing statistics for.
+     *
+     * @return Period
+     * @api
+     */
+    public function getPeriod()
+    {
+        return $this->period;
+    }
+
+    /**
+     * Returns the array of Period which make up this archive.
+     *
+     * @return \Piwik\Period[]
+     * @ignore
+     */
+    public function getSubPeriods()
+    {
+        if ($this->getPeriod()->getLabel() == 'day') {
+            return array( $this->getPeriod() );
+        }
+        return $this->getPeriod()->getSubperiods();
+    }
+
+    /**
+     * @return array
+     * @ignore
+     */
+    public function getIdSites()
+    {
+        $idSite = $this->getSite()->getId();
+
+        $idSites = array($idSite);
+
+        Piwik::postEvent('ArchiveProcessor.Parameters.getIdSites', array(&$idSites, $this->getPeriod()));
+
+        return $idSites;
+    }
+
+    /**
+     * Returns the site we are computing statistics for.
+     *
+     * @return Site
+     * @api
+     */
+    public function getSite()
+    {
+        return $this->site;
+    }
+
+    /**
+     * The Segment used to limit the set of visits that are being aggregated.
+     *
+     * @return Segment
+     * @api
+     */
+    public function getSegment()
+    {
+        return $this->segment;
+    }
+
+    /**
+     * Returns the end day of the period in the site's timezone.
+     *
+     * @return Date
+     */
+    public function getDateEnd()
+    {
+        return $this->getPeriod()->getDateEnd()->setTimezone($this->getSite()->getTimezone());
+    }
+
+    /**
+     * Returns the start day of the period in the site's timezone.
+     *
+     * @return Date
+     */
+    public function getDateStart()
+    {
+        return $this->getPeriod()->getDateStart()->setTimezone($this->getSite()->getTimezone());
+    }
+
+    /**
+     * Returns the start day of the period in the site's timezone (includes the time of day).
+     *
+     * @return Date
+     */
+    public function getDateTimeStart()
+    {
+        return $this->getPeriod()->getDateTimeStart()->setTimezone($this->getSite()->getTimezone());
+    }
+
+    /**
+     * Returns the end day of the period in the site's timezone (includes the time of day).
+     *
+     * @return Date
+     */
+    public function getDateTimeEnd()
+    {
+        return $this->getPeriod()->getDateTimeEnd()->setTimezone($this->getSite()->getTimezone());
+    }
+
+    /**
+     * @return bool
+     */
+    public function isSingleSiteDayArchive()
+    {
+        return $this->isDayArchive() && $this->isSingleSite();
+    }
+
+    /**
+     * @return bool
+     */
+    public function isDayArchive()
+    {
+        $period = $this->getPeriod();
+        $secondsInPeriod = $period->getDateEnd()->getTimestampUTC() - $period->getDateStart()->getTimestampUTC();
+        $oneDay = $secondsInPeriod < Date::NUM_SECONDS_IN_DAY;
+
+        return $oneDay;
+    }
+
+    public function isSingleSite()
+    {
+        return count($this->getIdSites()) == 1;
+    }
+
+    public function logStatusDebug($isTemporary)
+    {
+        $temporary = 'definitive archive';
+        if ($isTemporary) {
+            $temporary = 'temporary archive';
+        }
+        Log::debug(
+            "%s archive, idSite = %d (%s), segment '%s', report = '%s', UTC datetime [%s -> %s]",
+            $this->getPeriod()->getLabel(),
+            $this->getSite()->getId(),
+            $temporary,
+            $this->getSegment()->getString(),
+            $this->getRequestedPlugin(),
+            $this->getDateStart()->getDateStartUTC(),
+            $this->getDateEnd()->getDateEndUTC()
+        );
+    }
+
+    /**
+     * Returns `true` if these parameters are part of an initial archiving request.
+     * Returns `false` if these parameters are for an archiving request that was initiated
+     * during archiving.
+     *
+     * @return bool
+     */
+    public function isRootArchiveRequest()
+    {
+        return $this->isRootArchiveRequest;
+    }
+
+    /**
+     * Sets whether these parameters are part of the initial archiving request or if they are
+     * for a request that was initiated during archiving.
+     *
+     * @param $isRootArchiveRequest
+     */
+    public function setIsRootArchiveRequest($isRootArchiveRequest)
+    {
+        $this->isRootArchiveRequest = $isRootArchiveRequest;
+    }
+}

msd2/tracking/piwik/core/ArchiveProcessor/PluginsArchiver.php

@@ -0,0 +1,318 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+
+namespace Piwik\ArchiveProcessor;
+
+use Piwik\ArchiveProcessor;
+use Piwik\Container\StaticContainer;
+use Piwik\CronArchive\Performance\Logger;
+use Piwik\DataAccess\ArchiveWriter;
+use Piwik\DataAccess\LogAggregator;
+use Piwik\DataTable\Manager;
+use Piwik\Metrics;
+use Piwik\Piwik;
+use Piwik\Plugin\Archiver;
+use Piwik\Log;
+use Piwik\Timer;
+use Exception;
+
+/**
+ * This class creates the Archiver objects found in plugins and will trigger aggregation,
+ * so each plugin can process their reports.
+ */
+class PluginsArchiver
+{
+    /**
+     * @param ArchiveProcessor $archiveProcessor
+     */
+    public $archiveProcessor;
+
+    /**
+     * @var Parameters
+     */
+    protected $params;
+
+    /**
+     * @var LogAggregator
+     */
+    private $logAggregator;
+
+    /**
+     * Public only for tests. Won't be necessary after DI changes are complete.
+     *
+     * @var Archiver[] $archivers
+     */
+    public static $archivers = array();
+
+    /**
+     * Defines if we should aggregate from raw data by using MySQL queries (when true) or aggregate archives (when false)
+     * @var bool
+     */
+    private $shouldAggregateFromRawData;
+
+    public function __construct(Parameters $params, $isTemporaryArchive, ArchiveWriter $archiveWriter = null)
+    {
+        $this->params = $params;
+        $this->isTemporaryArchive = $isTemporaryArchive;
+        $this->archiveWriter = $archiveWriter ?: new ArchiveWriter($this->params, $this->isTemporaryArchive);
+        $this->archiveWriter->initNewArchive();
+
+        $this->logAggregator = new LogAggregator($params);
+
+        $this->archiveProcessor = new ArchiveProcessor($this->params, $this->archiveWriter, $this->logAggregator);
+
+        $shouldAggregateFromRawData = $this->params->isSingleSiteDayArchive();
+
+        /**
+         * Triggered to detect if the archiver should aggregate from raw data by using MySQL queries (when true)
+         * or by aggregate archives (when false). Typically, data is aggregated from raw data for "day" period, and
+         * aggregregated from archives for all other periods.
+         *
+         * @param bool $shouldAggregateFromRawData  Set to true, to aggregate from raw data, or false to aggregate multiple reports.
+         * @param Parameters $params
+         * @ignore
+         * @deprecated
+         *
+         * In Matomo 4.0 we should maybe remove this event, and instead maybe always archive from raw data when it is daily archive,
+         * no matter if single site or not. We cannot do this in Matomo 3.X as some custom plugin archivers may not be able to handle multiple sites.
+         */
+        Piwik::postEvent('ArchiveProcessor.shouldAggregateFromRawData', array(&$shouldAggregateFromRawData, $this->params));
+
+        $this->shouldAggregateFromRawData = $shouldAggregateFromRawData;
+    }
+
+    /**
+     * If period is day, will get the core metrics (including visits) from the logs.
+     * If period is != day, will sum the core metrics from the existing archives.
+     * @return array Core metrics
+     */
+    public function callAggregateCoreMetrics()
+    {
+        $this->logAggregator->setQueryOriginHint('Core');
+
+        if ($this->shouldAggregateFromRawData) {
+            $metrics = $this->aggregateDayVisitsMetrics();
+        } else {
+            $metrics = $this->aggregateMultipleVisitsMetrics();
+        }
+
+        if (empty($metrics)) {
+            return array(
+                'nb_visits' => false,
+                'nb_visits_converted' => false
+            );
+        }
+        return array(
+            'nb_visits' => $metrics['nb_visits'],
+            'nb_visits_converted' => $metrics['nb_visits_converted']
+        );
+    }
+
+    /**
+     * Instantiates the Archiver class in each plugin that defines it,
+     * and triggers Aggregation processing on these plugins.
+     */
+    public function callAggregateAllPlugins($visits, $visitsConverted, $forceArchivingWithoutVisits = false)
+    {
+        Log::debug("PluginsArchiver::%s: Initializing archiving process for all plugins [visits = %s, visits converted = %s]",
+            __FUNCTION__, $visits, $visitsConverted);
+
+        /** @var Logger $performanceLogger */
+        $performanceLogger = StaticContainer::get(Logger::class);
+
+        $this->archiveProcessor->setNumberOfVisits($visits, $visitsConverted);
+
+        $archivers = static::getPluginArchivers();
+
+        foreach ($archivers as $pluginName => $archiverClass) {
+            // We clean up below all tables created during this function call (and recursive calls)
+            $latestUsedTableId = Manager::getInstance()->getMostRecentTableId();
+
+            /** @var Archiver $archiver */
+            $archiver = $this->makeNewArchiverObject($archiverClass, $pluginName);
+
+            if (!$archiver->isEnabled()) {
+                Log::debug("PluginsArchiver::%s: Skipping archiving for plugin '%s' (disabled).", __FUNCTION__, $pluginName);
+                continue;
+            }
+
+            if (!$forceArchivingWithoutVisits && !$visits && !$archiver->shouldRunEvenWhenNoVisits()) {
+                Log::debug("PluginsArchiver::%s: Skipping archiving for plugin '%s' (no visits).", __FUNCTION__, $pluginName);
+                continue;
+            }
+
+            if ($this->shouldProcessReportsForPlugin($pluginName)) {
+
+                $this->logAggregator->setQueryOriginHint($pluginName);
+
+                try {
+                    $timer = new Timer();
+                    if ($this->shouldAggregateFromRawData) {
+                        Log::debug("PluginsArchiver::%s: Archiving day reports for plugin '%s'.", __FUNCTION__, $pluginName);
+
+                        $archiver->callAggregateDayReport();
+                    } else {
+                        Log::debug("PluginsArchiver::%s: Archiving period reports for plugin '%s'.", __FUNCTION__, $pluginName);
+
+                        $archiver->callAggregateMultipleReports();
+                    }
+
+                    $this->logAggregator->setQueryOriginHint('');
+
+                    $performanceLogger->logMeasurement('plugin', $pluginName, $this->params, $timer);
+
+                    Log::debug("PluginsArchiver::%s: %s while archiving %s reports for plugin '%s' %s.",
+                        __FUNCTION__,
+                        $timer->getMemoryLeak(),
+                        $this->params->getPeriod()->getLabel(),
+                        $pluginName,
+                        $this->params->getSegment() ? sprintf("(for segment = '%s')", $this->params->getSegment()->getString()) : ''
+                    );
+                } catch (Exception $e) {
+                    throw new PluginsArchiverException($e->getMessage() . " - in plugin $pluginName", $e->getCode(), $e);
+                }
+            } else {
+                Log::debug("PluginsArchiver::%s: Not archiving reports for plugin '%s'.", __FUNCTION__, $pluginName);
+            }
+
+            Manager::getInstance()->deleteAll($latestUsedTableId);
+            unset($archiver);
+        }
+    }
+
+    public function finalizeArchive()
+    {
+        $this->params->logStatusDebug($this->archiveWriter->isArchiveTemporary);
+        $this->archiveWriter->finalizeArchive();
+        return $this->archiveWriter->getIdArchive();
+    }
+
+    /**
+     * Returns if any plugin archiver archives without visits
+     */
+    public static function doesAnyPluginArchiveWithoutVisits()
+    {
+        $archivers = static::getPluginArchivers();
+
+        foreach ($archivers as $pluginName => $archiverClass) {
+            if ($archiverClass::shouldRunEvenWhenNoVisits()) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    /**
+     * Loads Archiver class from any plugin that defines one.
+     *
+     * @return \Piwik\Plugin\Archiver[]
+     */
+    protected static function getPluginArchivers()
+    {
+        if (empty(static::$archivers)) {
+            $pluginNames = \Piwik\Plugin\Manager::getInstance()->getActivatedPlugins();
+            $archivers = array();
+            foreach ($pluginNames as $pluginName) {
+                $archivers[$pluginName] = self::getPluginArchiverClass($pluginName);
+            }
+            static::$archivers = array_filter($archivers);
+        }
+        return static::$archivers;
+    }
+
+    private static function getPluginArchiverClass($pluginName)
+    {
+        $klassName = 'Piwik\\Plugins\\' . $pluginName . '\\Archiver';
+        if (class_exists($klassName)
+            && is_subclass_of($klassName, 'Piwik\\Plugin\\Archiver')) {
+            return $klassName;
+        }
+        return false;
+    }
+
+    /**
+     * Whether the specified plugin's reports should be archived
+     * @param string $pluginName
+     * @return bool
+     */
+    protected function shouldProcessReportsForPlugin($pluginName)
+    {
+        if ($this->params->getRequestedPlugin() == $pluginName) {
+            return true;
+        }
+
+        if ($this->params->shouldOnlyArchiveRequestedPlugin()) {
+            return false;
+        }
+
+        if (Rules::shouldProcessReportsAllPlugins(
+            $this->params->getIdSites(),
+            $this->params->getSegment(),
+            $this->params->getPeriod()->getLabel())) {
+            return true;
+        }
+
+        if (!\Piwik\Plugin\Manager::getInstance()->isPluginLoaded($this->params->getRequestedPlugin())) {
+            return true;
+        }
+        return false;
+    }
+
+    protected function aggregateDayVisitsMetrics()
+    {
+        $query = $this->archiveProcessor->getLogAggregator()->queryVisitsByDimension();
+        $data = $query->fetch();
+
+        $metrics = $this->convertMetricsIdToName($data);
+        $this->archiveProcessor->insertNumericRecords($metrics);
+        return $metrics;
+    }
+
+    protected function convertMetricsIdToName($data)
+    {
+        $metrics = array();
+        foreach ($data as $metricId => $value) {
+            $readableMetric = Metrics::$mappingFromIdToName[$metricId];
+            $metrics[$readableMetric] = $value;
+        }
+        return $metrics;
+    }
+
+    protected function aggregateMultipleVisitsMetrics()
+    {
+        $toSum = Metrics::getVisitsMetricNames();
+        $metrics = $this->archiveProcessor->aggregateNumericMetrics($toSum);
+        return $metrics;
+    }
+
+
+    /**
+     * @param $archiverClass
+     * @return Archiver
+     */
+    private function makeNewArchiverObject($archiverClass, $pluginName)
+    {
+        $archiver = new $archiverClass($this->archiveProcessor);
+
+        /**
+         * Triggered right after a new **plugin archiver instance** is created.
+         * Subscribers to this event can configure the plugin archiver, for example prevent the archiving of a plugin's data
+         * by calling `$archiver->disable()` method.
+         *
+         * @param \Piwik\Plugin\Archiver &$archiver The newly created plugin archiver instance.
+         * @param string $pluginName The name of plugin of which archiver instance was created.
+         * @param array $this->params Array containing archive parameters (Site, Period, Date and Segment)
+         * @param bool $this->isTemporaryArchive Flag indicating whether the archive being processed is temporary (ie. the period isn't finished yet) or final (the period is already finished and in the past).
+         */
+        Piwik::postEvent('Archiving.makeNewArchiverObject', array($archiver, $pluginName, $this->params, $this->isTemporaryArchive));
+
+        return $archiver;
+    }
+}

msd2/tracking/piwik/core/ArchiveProcessor/PluginsArchiverException.php

@@ -0,0 +1,15 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+
+namespace Piwik\ArchiveProcessor;
+
+
+class PluginsArchiverException extends \Exception
+{
+}

msd2/tracking/piwik/core/ArchiveProcessor/Rules.php

@@ -0,0 +1,307 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\ArchiveProcessor;
+
+use Exception;
+use Piwik\Config;
+use Piwik\DataAccess\ArchiveWriter;
+use Piwik\Date;
+use Piwik\Log;
+use Piwik\Option;
+use Piwik\Piwik;
+use Piwik\Plugins\CoreAdminHome\Controller;
+use Piwik\Segment;
+use Piwik\SettingsPiwik;
+use Piwik\SettingsServer;
+use Piwik\Site;
+use Piwik\Tracker\Cache;
+
+/**
+ * This class contains Archiving rules/logic which are used when creating and processing Archives.
+ *
+ */
+class Rules
+{
+    const OPTION_TODAY_ARCHIVE_TTL = 'todayArchiveTimeToLive';
+
+    const OPTION_BROWSER_TRIGGER_ARCHIVING = 'enableBrowserTriggerArchiving';
+
+    const FLAG_TABLE_PURGED = 'lastPurge_';
+
+    /** Flag that will forcefully disable the archiving process (used in tests only) */
+    public static $archivingDisabledByTests = false;
+
+    /**
+     * Returns the name of the archive field used to tell the status of an archive, (ie,
+     * whether the archive was created successfully or not).
+     *
+     * @param array $idSites
+     * @param Segment $segment
+     * @param string $periodLabel
+     * @param string $plugin
+     * @return string
+     */
+    public static function getDoneStringFlagFor(array $idSites, $segment, $periodLabel, $plugin)
+    {
+        if (!self::shouldProcessReportsAllPlugins($idSites, $segment, $periodLabel)) {
+            return self::getDoneFlagArchiveContainsOnePlugin($segment, $plugin);
+        }
+        return self::getDoneFlagArchiveContainsAllPlugins($segment);
+    }
+
+    public static function shouldProcessReportsAllPlugins(array $idSites, Segment $segment, $periodLabel)
+    {
+        if ($segment->isEmpty() && $periodLabel != 'range') {
+            return true;
+        }
+
+        return self::isSegmentPreProcessed($idSites, $segment);
+    }
+
+    /**
+     * @param $idSites
+     * @return array
+     */
+    public static function getSegmentsToProcess($idSites)
+    {
+        $knownSegmentsToArchiveAllSites = SettingsPiwik::getKnownSegmentsToArchive();
+
+        $segmentsToProcess = $knownSegmentsToArchiveAllSites;
+        foreach ($idSites as $idSite) {
+            $segmentForThisWebsite = SettingsPiwik::getKnownSegmentsToArchiveForSite($idSite);
+            $segmentsToProcess = array_merge($segmentsToProcess, $segmentForThisWebsite);
+        }
+        $segmentsToProcess = array_unique($segmentsToProcess);
+        return $segmentsToProcess;
+    }
+
+    public static function getDoneFlagArchiveContainsOnePlugin(Segment $segment, $plugin)
+    {
+        return 'done' . $segment->getHash() . '.' . $plugin ;
+    }
+
+    public static function getDoneFlagArchiveContainsAllPlugins(Segment $segment)
+    {
+        return 'done' . $segment->getHash();
+    }
+
+    /**
+     * Return done flags used to tell how the archiving process for a specific archive was completed,
+     *
+     * @param array $plugins
+     * @param $segment
+     * @return array
+     */
+    public static function getDoneFlags(array $plugins, Segment $segment)
+    {
+        $doneFlags = array();
+        $doneAllPlugins = self::getDoneFlagArchiveContainsAllPlugins($segment);
+        $doneFlags[$doneAllPlugins] = $doneAllPlugins;
+
+        $plugins = array_unique($plugins);
+        foreach ($plugins as $plugin) {
+            $doneOnePlugin = self::getDoneFlagArchiveContainsOnePlugin($segment, $plugin);
+            $doneFlags[$plugin] = $doneOnePlugin;
+        }
+        return $doneFlags;
+    }
+
+    public static function getMinTimeProcessedForTemporaryArchive(
+        Date $dateStart, \Piwik\Period $period, Segment $segment, Site $site)
+    {
+        $todayArchiveTimeToLive = self::getPeriodArchiveTimeToLiveDefault($period->getLabel());
+
+        $now = time();
+        $minimumArchiveTime = $now - $todayArchiveTimeToLive;
+
+        $idSites = array($site->getId());
+        $isArchivingDisabled = Rules::isArchivingDisabledFor($idSites, $segment, $period->getLabel());
+        if ($isArchivingDisabled) {
+            if ($period->getNumberOfSubperiods() == 0
+                && $dateStart->getTimestamp() <= $now
+            ) {
+                // Today: accept any recent enough archive
+                $minimumArchiveTime = false;
+            } else {
+                // This week, this month, this year:
+                // accept any archive that was processed today after 00:00:01 this morning
+                $timezone = $site->getTimezone();
+                $minimumArchiveTime = Date::factory(Date::factory('now', $timezone)->getDateStartUTC())->setTimezone($timezone)->getTimestamp();
+            }
+        }
+        return $minimumArchiveTime;
+    }
+
+    public static function setTodayArchiveTimeToLive($timeToLiveSeconds)
+    {
+        $timeToLiveSeconds = (int)$timeToLiveSeconds;
+        if ($timeToLiveSeconds <= 0) {
+            throw new Exception(Piwik::translate('General_ExceptionInvalidArchiveTimeToLive'));
+        }
+        Option::set(self::OPTION_TODAY_ARCHIVE_TTL, $timeToLiveSeconds, $autoLoad = true);
+    }
+
+    public static function getTodayArchiveTimeToLive()
+    {
+        $uiSettingIsEnabled = Controller::isGeneralSettingsAdminEnabled();
+
+        if ($uiSettingIsEnabled) {
+            $timeToLive = Option::get(self::OPTION_TODAY_ARCHIVE_TTL);
+            if ($timeToLive !== false) {
+                return $timeToLive;
+            }
+        }
+        return self::getTodayArchiveTimeToLiveDefault();
+    }
+
+    public static function getPeriodArchiveTimeToLiveDefault($periodLabel)
+    {
+        if (empty($periodLabel) || strtolower($periodLabel) === 'day') {
+            return self::getTodayArchiveTimeToLive();
+        }
+
+        $config = Config::getInstance();
+        $general = $config->General;
+
+        $key = sprintf('time_before_%s_archive_considered_outdated', $periodLabel);
+        if (isset($general[$key]) && is_numeric($general[$key]) && $general[$key] > 0) {
+            return $general[$key];
+        }
+
+        return self::getTodayArchiveTimeToLive();
+    }
+
+    public static function getTodayArchiveTimeToLiveDefault()
+    {
+        return Config::getInstance()->General['time_before_today_archive_considered_outdated'];
+    }
+
+    public static function isBrowserArchivingAvailableForSegments()
+    {
+        $generalConfig = Config::getInstance()->General;
+        return !$generalConfig['browser_archiving_disabled_enforce'];
+    }
+
+    public static function isArchivingDisabledFor(array $idSites, Segment $segment, $periodLabel)
+    {
+        $generalConfig = Config::getInstance()->General;
+
+        if ($periodLabel == 'range') {
+            if (!isset($generalConfig['archiving_range_force_on_browser_request'])
+                || $generalConfig['archiving_range_force_on_browser_request'] != false
+            ) {
+                return false;
+            }
+
+            Log::debug("Not forcing archiving for range period.");
+            $processOneReportOnly = false;
+
+        } else {
+            $processOneReportOnly = !self::shouldProcessReportsAllPlugins($idSites, $segment, $periodLabel);
+        }
+
+        $isArchivingEnabled = self::isRequestAuthorizedToArchive() && !self::$archivingDisabledByTests;
+
+        if ($processOneReportOnly)  {
+            // When there is a segment, we disable archiving when browser_archiving_disabled_enforce applies
+            if (!$segment->isEmpty()
+                && !$isArchivingEnabled
+                && !self::isBrowserArchivingAvailableForSegments()
+                && !SettingsServer::isArchivePhpTriggered() // Only applies when we are not running core:archive command
+            ) {
+                Log::debug("Archiving is disabled because of config setting browser_archiving_disabled_enforce=1");
+                return true;
+            }
+
+            // Always allow processing one report
+            return false;
+        }
+
+        return !$isArchivingEnabled;
+    }
+
+    public static function isRequestAuthorizedToArchive()
+    {
+        return Rules::isBrowserTriggerEnabled() || SettingsServer::isArchivePhpTriggered();
+    }
+
+    public static function isBrowserTriggerEnabled()
+    {
+        $uiSettingIsEnabled = Controller::isGeneralSettingsAdminEnabled();
+
+        if ($uiSettingIsEnabled) {
+            $browserArchivingEnabled = Option::get(self::OPTION_BROWSER_TRIGGER_ARCHIVING);
+            if ($browserArchivingEnabled !== false) {
+                return (bool)$browserArchivingEnabled;
+            }
+        }
+        return (bool)Config::getInstance()->General['enable_browser_archiving_triggering'];
+    }
+
+    public static function setBrowserTriggerArchiving($enabled)
+    {
+        if (!is_bool($enabled)) {
+            throw new Exception('Browser trigger archiving must be set to true or false.');
+        }
+        Option::set(self::OPTION_BROWSER_TRIGGER_ARCHIVING, (int)$enabled, $autoLoad = true);
+        Cache::clearCacheGeneral();
+    }
+
+    /**
+     * Returns true if the archiving process should skip the calculation of unique visitors
+     * across several sites. The `[General] enable_processing_unique_visitors_multiple_sites`
+     * INI config option controls the value of this variable.
+     *
+     * @return bool
+     */
+    public static function shouldSkipUniqueVisitorsCalculationForMultipleSites()
+    {
+        return Config::getInstance()->General['enable_processing_unique_visitors_multiple_sites'] != 1;
+    }
+
+    /**
+     * @param array $idSites
+     * @param Segment $segment
+     * @return bool
+     */
+    public static function isSegmentPreProcessed(array $idSites, Segment $segment)
+    {
+        $segmentsToProcess = self::getSegmentsToProcess($idSites);
+
+        if (empty($segmentsToProcess)) {
+            return false;
+        }
+        // If the requested segment is one of the segments to pre-process
+        // we ensure that any call to the API will trigger archiving of all reports for this segment
+        $segment = $segment->getString();
+
+        // Turns out the getString() above returns the URL decoded segment string
+        $segmentsToProcessUrlDecoded = array_map('urldecode', $segmentsToProcess);
+
+        return in_array($segment, $segmentsToProcess)
+            || in_array($segment, $segmentsToProcessUrlDecoded);
+    }
+
+    /**
+     * Returns done flag values allowed to be selected
+     *
+     * @return string
+     */
+    public static function getSelectableDoneFlagValues()
+    {
+        $possibleValues = array(ArchiveWriter::DONE_OK, ArchiveWriter::DONE_OK_TEMPORARY);
+
+        if (!Rules::isRequestAuthorizedToArchive()) {
+            //If request is not authorized to archive then fetch also invalidated archives
+            $possibleValues[] = ArchiveWriter::DONE_INVALIDATED;
+        }
+
+        return $possibleValues;
+    }
+}