aschwarz/Leitgedanken
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

PDF rausgenommen

Browse Source
This commit is contained in:
aschwarz
2023-01-23 11:03:31 +01:00
parent 82d562a322
commit a6523903eb
28078 changed files with 4247552 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

362
msd2/tracking/piwik/core/Archive/ArchiveInvalidator.php Normal file
View File

@ -0,0 +1,362 @@
<?php
/**
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*
*/
namespace Piwik\Archive;
use Piwik\Archive\ArchiveInvalidator\InvalidationResult;
use Piwik\CronArchive\SitesToReprocessDistributedList;
use Piwik\DataAccess\ArchiveTableCreator;
use Piwik\DataAccess\Model;
use Piwik\Date;
use Piwik\Option;
use Piwik\Piwik;
use Piwik\Plugins\CoreAdminHome\Tasks\ArchivesToPurgeDistributedList;
use Piwik\Plugins\PrivacyManager\PrivacyManager;
use Piwik\Period;
use Piwik\Segment;
/**
* Service that can be used to invalidate archives or add archive references to a list so they will
* be invalidated later.
*
* Archives are put in an "invalidated" state by setting the done flag to `ArchiveWriter::DONE_INVALIDATED`.
* This class also adds the archive's associated site to the a distributed list and adding the archive's year month to another
* distributed list.
*
* CronArchive will reprocess the archive data for all sites in the first list, and a scheduled task
* will purge the old, invalidated data in archive tables identified by the second list.
*
* Until CronArchive, or browser triggered archiving, re-processes data for an invalidated archive, the invalidated
* archive data will still be displayed in the UI and API.
*
* ### Deferred Invalidation
*
* Invalidating archives means running queries on one or more archive tables. In some situations, like during
* tracking, this is not desired. In such cases, archive references can be added to a list via the
* rememberToInvalidateArchivedReportsLater method, which will add the reference to a distributed list
*
* Later, during Piwik's normal execution, the list will be read and every archive it references will
* be invalidated.
*/
class ArchiveInvalidator
{
private $rememberArchivedReportIdStart = 'report_to_invalidate_';
/**
* @var Model
*/
private $model;
public function __construct(Model $model)
{
$this->model = $model;
}
public function rememberToInvalidateArchivedReportsLater($idSite, Date $date)
{
// To support multiple transactions at once, look for any other process to have set (and committed)
// this report to be invalidated.
$key = $this->buildRememberArchivedReportIdForSiteAndDate($idSite, $date->toString());
// we do not really have to get the value first. we could simply always try to call set() and it would update or
// insert the record if needed but we do not want to lock the table (especially since there are still some
// MyISAM installations)
$value = Option::getLike($key . '%');
// In order to support multiple concurrent transactions, add our pid to the end of the key so that it will just insert
// rather than waiting on some other process to commit before proceeding.The issue is that with out this, more than
// one process is trying to add the exact same value to the table, which causes contention. With the pid suffixed to
// the value, each process can successfully enter its own row in the table. The net result will be the same. We could
// always just set this, but it would result in a lot of rows in the options table.. more than needed. With this
// change you'll have at most N rows per date/site, where N is the number of parallel requests on this same idsite/date
// that happen to run in overlapping transactions.
$mykey = $this->buildRememberArchivedReportIdProcessSafe($idSite, $date->toString());
// getLike() returns an empty array rather than 'false'
if (empty($value)) {
Option::set($mykey, '1');
}
}
public function getRememberedArchivedReportsThatShouldBeInvalidated()
{
$reports = Option::getLike($this->rememberArchivedReportIdStart . '%_%');
$sitesPerDay = array();
foreach ($reports as $report => $value) {
$report = str_replace($this->rememberArchivedReportIdStart, '', $report);
$report = explode('_', $report);
$siteId = (int) $report[0];
$date = $report[1];
if (empty($sitesPerDay[$date])) {
$sitesPerDay[$date] = array();
}
$sitesPerDay[$date][] = $siteId;
}
return $sitesPerDay;
}
private function buildRememberArchivedReportIdForSite($idSite)
{
return $this->rememberArchivedReportIdStart . (int) $idSite;
}
private function buildRememberArchivedReportIdForSiteAndDate($idSite, $date)
{
$id = $this->buildRememberArchivedReportIdForSite($idSite);
$id .= '_' . trim($date);
return $id;
}
// This version is multi process safe on the insert of a new date to invalidate.
private function buildRememberArchivedReportIdProcessSafe($idSite, $date)
{
$id = $this->buildRememberArchivedReportIdForSiteAndDate($idSite, $date);
$id .= '_' . getmypid();
return $id;
}
public function forgetRememberedArchivedReportsToInvalidateForSite($idSite)
{
$id = $this->buildRememberArchivedReportIdForSite($idSite) . '_%';
Option::deleteLike($id);
}
/**
* @internal
*/
public function forgetRememberedArchivedReportsToInvalidate($idSite, Date $date)
{
$id = $this->buildRememberArchivedReportIdForSiteAndDate($idSite, $date->toString());
// The process pid is added to the end of the entry in order to support multiple concurrent transactions.
// So this must be a deleteLike call to get all the entries, where there used to only be one.
Option::deleteLike($id . '%');
}
/**
* @param $idSites int[]
* @param $dates Date[]
* @param $period string
* @param $segment Segment
* @param bool $cascadeDown
* @return InvalidationResult
* @throws \Exception
*/
public function markArchivesAsInvalidated(array $idSites, array $dates, $period, Segment $segment = null, $cascadeDown = false)
{
$invalidationInfo = new InvalidationResult();
/**
* Triggered when a Matomo user requested the invalidation of some reporting archives. Using this event, plugin
* developers can automatically invalidate another site, when a site is being invalidated. A plugin may even
* remove an idSite from the list of sites that should be invalidated to prevent it from ever being
* invalidated.
*
* **Example**
*
* public function getIdSitesToMarkArchivesAsInvalidates(&$idSites)
* {
* if (in_array(1, $idSites)) {
* $idSites[] = 5; // when idSite 1 is being invalidated, also invalidate idSite 5
* }
* }
*
* @param array &$idSites An array containing a list of site IDs which are requested to be invalidated.
*/
Piwik::postEvent('Archiving.getIdSitesToMarkArchivesAsInvalidated', array(&$idSites));
// we trigger above event on purpose here and it is good that the segment was created like
// `new Segment($segmentString, $idSites)` because when a user adds a site via this event, the added idSite
// might not have this segment meaning we avoid a possible error. For the workflow to work, any added or removed
// idSite does not need to be added to $segment.
$datesToInvalidate = $this->removeDatesThatHaveBeenPurged($dates, $invalidationInfo);
if (empty($period)) {
// if the period is empty, we don't need to cascade in any way, since we'll remove all periods
$periodDates = $this->getDatesByYearMonthAndPeriodType($dates);
} else {
$periods = $this->getPeriodsToInvalidate($datesToInvalidate, $period, $cascadeDown);
$periodDates = $this->getPeriodDatesByYearMonthAndPeriodType($periods);
}
$periodDates = $this->getUniqueDates($periodDates);
$this->markArchivesInvalidated($idSites, $periodDates, $segment);
$yearMonths = array_keys($periodDates);
$this->markInvalidatedArchivesForReprocessAndPurge($idSites, $yearMonths);
foreach ($idSites as $idSite) {
foreach ($dates as $date) {
$this->forgetRememberedArchivedReportsToInvalidate($idSite, $date);
}
}
return $invalidationInfo;
}
/**
* @param string[][][] $periodDates
* @return string[][][]
*/
private function getUniqueDates($periodDates)
{
$result = array();
foreach ($periodDates as $yearMonth => $periodsByYearMonth) {
foreach ($periodsByYearMonth as $periodType => $periods) {
$result[$yearMonth][$periodType] = array_unique($periods);
}
}
return $result;
}
/**
* @param Date[] $dates
* @param string $periodType
* @param bool $cascadeDown
* @return Period[]
*/
private function getPeriodsToInvalidate($dates, $periodType, $cascadeDown)
{
$periodsToInvalidate = array();
foreach ($dates as $date) {
if ($periodType == 'range') {
$date = $date . ',' . $date;
}
$period = Period\Factory::build($periodType, $date);
$periodsToInvalidate[] = $period;
if ($cascadeDown) {
$periodsToInvalidate = array_merge($periodsToInvalidate, $period->getAllOverlappingChildPeriods());
}
if ($periodType != 'year'
&& $periodType != 'range'
) {
$periodsToInvalidate[] = Period\Factory::build('year', $date);
}
}
return $periodsToInvalidate;
}
/**
* @param Period[] $periods
* @return string[][][]
*/
private function getPeriodDatesByYearMonthAndPeriodType($periods)
{
$result = array();
foreach ($periods as $period) {
$date = $period->getDateStart();
$periodType = $period->getId();
$yearMonth = ArchiveTableCreator::getTableMonthFromDate($date);
$result[$yearMonth][$periodType][] = $date->toString();
}
return $result;
}
/**
* Called when deleting all periods.
*
* @param Date[] $dates
* @return string[][][]
*/
private function getDatesByYearMonthAndPeriodType($dates)
{
$result = array();
foreach ($dates as $date) {
$yearMonth = ArchiveTableCreator::getTableMonthFromDate($date);
$result[$yearMonth][null][] = $date->toString();
// since we're removing all periods, we must make sure to remove year periods as well.
// this means we have to make sure the january table is processed.
$janYearMonth = $date->toString('Y') . '_01';
$result[$janYearMonth][null][] = $date->toString();
}
return $result;
}
/**
* @param int[] $idSites
* @param string[][][] $dates
* @throws \Exception
*/
private function markArchivesInvalidated($idSites, $dates, Segment $segment = null)
{
$archiveNumericTables = ArchiveTableCreator::getTablesArchivesInstalled($type = ArchiveTableCreator::NUMERIC_TABLE);
foreach ($archiveNumericTables as $table) {
$tableDate = ArchiveTableCreator::getDateFromTableName($table);
if (empty($dates[$tableDate])) {
continue;
}
$this->model->updateArchiveAsInvalidated($table, $idSites, $dates[$tableDate], $segment);
}
}
/**
* @param Date[] $dates
* @param InvalidationResult $invalidationInfo
* @return \Piwik\Date[]
*/
private function removeDatesThatHaveBeenPurged($dates, InvalidationResult $invalidationInfo)
{
$this->findOlderDateWithLogs($invalidationInfo);
$result = array();
foreach ($dates as $date) {
// we should only delete reports for dates that are more recent than N days
if ($invalidationInfo->minimumDateWithLogs
&& $date->isEarlier($invalidationInfo->minimumDateWithLogs)
) {
$invalidationInfo->warningDates[] = $date->toString();
continue;
}
$result[] = $date;
$invalidationInfo->processedDates[] = $date->toString();
}
return $result;
}
private function findOlderDateWithLogs(InvalidationResult $info)
{
// If using the feature "Delete logs older than N days"...
$purgeDataSettings = PrivacyManager::getPurgeDataSettings();
$logsDeletedWhenOlderThanDays = (int)$purgeDataSettings['delete_logs_older_than'];
$logsDeleteEnabled = $purgeDataSettings['delete_logs_enable'];
if ($logsDeleteEnabled
&& $logsDeletedWhenOlderThanDays
) {
$info->minimumDateWithLogs = Date::factory('today')->subDay($logsDeletedWhenOlderThanDays);
}
}
/**
* @param array $idSites
* @param array $yearMonths
*/
private function markInvalidatedArchivesForReprocessAndPurge(array $idSites, $yearMonths)
{
$store = new SitesToReprocessDistributedList();
$store->add($idSites);
$archivesToPurge = new ArchivesToPurgeDistributedList();
$archivesToPurge->add($yearMonths);
}
}

56
msd2/tracking/piwik/core/Archive/ArchiveInvalidator/InvalidationResult.php Normal file
View File

@ -0,0 +1,56 @@
<?php
/**
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*/
namespace Piwik\Archive\ArchiveInvalidator;
use Piwik\Date;
/**
* Information about the result of an archive invalidation operation.
*/
class InvalidationResult
{
/**
* Dates that couldn't be invalidated because they are earlier than the configured log
* deletion limit.
*
* @var array
*/
public $warningDates = array();
/**
* Dates that were successfully invalidated.
*
* @var array
*/
public $processedDates = array();
/**
* The day of the oldest log entry.
*
* @var Date|bool
*/
public $minimumDateWithLogs = false;
/**
* @return string[]
*/
public function makeOutputLogs()
{
$output = array();
if ($this->warningDates) {
$output[] = 'Warning: the following Dates have not been invalidated, because they are earlier than your Log Deletion limit: ' .
implode(", ", $this->warningDates) .
"\n The last day with logs is " . $this->minimumDateWithLogs . ". " .
"\n Please disable 'Delete old Logs' or set it to a higher deletion threshold (eg. 180 days or 365 years).'.";
}
$output[] = "Success. The following dates were invalidated successfully: " . implode(", ", $this->processedDates);
return $output;
}
}

272
msd2/tracking/piwik/core/Archive/ArchivePurger.php Normal file
View File

@ -0,0 +1,272 @@
<?php
/**
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*
*/
namespace Piwik\Archive;
use Piwik\ArchiveProcessor\Rules;
use Piwik\Config;
use Piwik\Container\StaticContainer;
use Piwik\DataAccess\ArchiveTableCreator;
use Piwik\DataAccess\Model;
use Piwik\Date;
use Piwik\Piwik;
use Psr\Log\LoggerInterface;
use Psr\Log\LogLevel;
/**
* Service that purges temporary, error-ed, invalid and custom range archives from archive tables.
*
* Temporary archives are purged if they were archived before a specific time. The time is dependent
* on whether browser triggered archiving is enabled or not.
*
* Error-ed archives are purged w/o constraint.
*
* Invalid archives are purged if a new, valid, archive exists w/ the same site, date, period combination.
* Archives are marked as invalid via Piwik\Archive\ArchiveInvalidator.
*/
class ArchivePurger
{
/**
* @var Model
*/
private $model;
/**
* Date threshold for purging custom range archives. Archives that are older than this date
* are purged unconditionally from the requested archive table.
*
* @var Date
*/
private $purgeCustomRangesOlderThan;
/**
* Date to use for 'yesterday'. Exists so tests can override this value.
*
* @var Date
*/
private $yesterday;
/**
* Date to use for 'today'. Exists so tests can override this value.
*
* @var $today
*/
private $today;
/**
* Date to use for 'now'. Exists so tests can override this value.
*
* @var int
*/
private $now;
/**
* @var LoggerInterface
*/
private $logger;
public function __construct(Model $model = null, Date $purgeCustomRangesOlderThan = null, LoggerInterface $logger = null)
{
$this->model = $model ?: new Model();
$this->purgeCustomRangesOlderThan = $purgeCustomRangesOlderThan ?: self::getDefaultCustomRangeToPurgeAgeThreshold();
$this->yesterday = Date::factory('yesterday');
$this->today = Date::factory('today');
$this->now = time();
$this->logger = $logger ?: StaticContainer::get('Psr\Log\LoggerInterface');
}
/**
* Purge all invalidate archives for whom there are newer, valid archives from the archive
* table that stores data for `$date`.
*
* @param Date $date The date identifying the archive table.
* @return int The total number of archive rows deleted (from both the blog & numeric tables).
*/
public function purgeInvalidatedArchivesFrom(Date $date)
{
$numericTable = ArchiveTableCreator::getNumericTable($date);
// we don't want to do an INNER JOIN on every row in a archive table that can potentially have tens to hundreds of thousands of rows,
// so we first look for sites w/ invalidated archives, and use this as a constraint in getInvalidatedArchiveIdsSafeToDelete() below.
// the constraint will hit an INDEX and speed up the inner join that happens in getInvalidatedArchiveIdsSafeToDelete().
$idSites = $this->model->getSitesWithInvalidatedArchive($numericTable);
if (empty($idSites)) {
$this->logger->debug("No sites with invalidated archives found in {table}.", array('table' => $numericTable));
return 0;
}
$archiveIds = $this->model->getInvalidatedArchiveIdsSafeToDelete($numericTable, $idSites);
if (empty($archiveIds)) {
$this->logger->debug("No invalidated archives found in {table} with newer, valid archives.", array('table' => $numericTable));
return 0;
}
$this->logger->info("Found {countArchiveIds} invalidated archives safe to delete in {table}.", array(
'table' => $numericTable, 'countArchiveIds' => count($archiveIds)
));
$deletedRowCount = $this->deleteArchiveIds($date, $archiveIds);
$this->logger->debug("Deleted {count} rows in {table} and its associated blob table.", array(
'table' => $numericTable, 'count' => $deletedRowCount
));
return $deletedRowCount;
}
/**
* Removes the outdated archives for the given month.
* (meaning they are marked with a done flag of ArchiveWriter::DONE_OK_TEMPORARY or ArchiveWriter::DONE_ERROR)
*
* @param Date $dateStart Only the month will be used
* @return int Returns the total number of rows deleted.
*/
public function purgeOutdatedArchives(Date $dateStart)
{
$purgeArchivesOlderThan = $this->getOldestTemporaryArchiveToKeepThreshold();
$deletedRowCount = 0;
$idArchivesToDelete = $this->getOutdatedArchiveIds($dateStart, $purgeArchivesOlderThan);
if (!empty($idArchivesToDelete)) {
$deletedRowCount = $this->deleteArchiveIds($dateStart, $idArchivesToDelete);
$this->logger->info("Deleted {count} rows in archive tables (numeric + blob) for {date}.", array(
'count' => $deletedRowCount,
'date' => $dateStart
));
} else {
$this->logger->debug("No outdated archives found in archive numeric table for {date}.", array('date' => $dateStart));
}
$this->logger->debug("Purging temporary archives: done [ purged archives older than {date} in {yearMonth} ] [Deleted IDs: {deletedIds}]", array(
'date' => $purgeArchivesOlderThan,
'yearMonth' => $dateStart->toString('Y-m'),
'deletedIds' => implode(',', $idArchivesToDelete)
));
return $deletedRowCount;
}
protected function getOutdatedArchiveIds(Date $date, $purgeArchivesOlderThan)
{
$archiveTable = ArchiveTableCreator::getNumericTable($date);
$result = $this->model->getTemporaryArchivesOlderThan($archiveTable, $purgeArchivesOlderThan);
$idArchivesToDelete = array();
if (!empty($result)) {
foreach ($result as $row) {
$idArchivesToDelete[] = $row['idarchive'];
}
}
return $idArchivesToDelete;
}
/**
* Deleting "Custom Date Range" reports after 1 day, since they can be re-processed and would take up un-necessary space.
*
* @param $date Date
* @return int The total number of rows deleted from both the numeric & blob table.
*/
public function purgeArchivesWithPeriodRange(Date $date)
{
$numericTable = ArchiveTableCreator::getNumericTable($date);
$blobTable = ArchiveTableCreator::getBlobTable($date);
$deletedCount = $this->model->deleteArchivesWithPeriod(
$numericTable, $blobTable, Piwik::$idPeriods['range'], $this->purgeCustomRangesOlderThan);
$level = $deletedCount == 0 ? LogLevel::DEBUG : LogLevel::INFO;
$this->logger->log($level, "Purged {count} range archive rows from {numericTable} & {blobTable}.", array(
'count' => $deletedCount,
'numericTable' => $numericTable,
'blobTable' => $blobTable
));
$this->logger->debug(" [ purged archives older than {threshold} ]", array('threshold' => $this->purgeCustomRangesOlderThan));
return $deletedCount;
}
/**
* Deletes by batches Archive IDs in the specified month,
*
* @param Date $date
* @param $idArchivesToDelete
* @return int Number of rows deleted from both numeric + blob table.
*/
protected function deleteArchiveIds(Date $date, $idArchivesToDelete)
{
$batches = array_chunk($idArchivesToDelete, 1000);
$numericTable = ArchiveTableCreator::getNumericTable($date);
$blobTable = ArchiveTableCreator::getBlobTable($date);
$deletedCount = 0;
foreach ($batches as $idsToDelete) {
$deletedCount += $this->model->deleteArchiveIds($numericTable, $blobTable, $idsToDelete);
}
return $deletedCount;
}
/**
* Returns a timestamp indicating outdated archives older than this timestamp (processed before) can be purged.
*
* @return int|bool Outdated archives older than this timestamp should be purged
*/
protected function getOldestTemporaryArchiveToKeepThreshold()
{
$temporaryArchivingTimeout = Rules::getTodayArchiveTimeToLive();
if (Rules::isBrowserTriggerEnabled()) {
// If Browser Archiving is enabled, it is likely there are many more temporary archives
// We delete more often which is safe, since reports are re-processed on demand
return Date::factory($this->now - 2 * $temporaryArchivingTimeout)->getDateTime();
}
// If cron core:archive command is building the reports, we should keep all temporary reports from today
return $this->yesterday->getDateTime();
}
private static function getDefaultCustomRangeToPurgeAgeThreshold()
{
$daysRangesValid = Config::getInstance()->General['purge_date_range_archives_after_X_days'];
return Date::factory('today')->subDay($daysRangesValid)->getDateTime();
}
/**
* For tests.
*
* @param Date $yesterday
*/
public function setYesterdayDate(Date $yesterday)
{
$this->yesterday = $yesterday;
}
/**
* For tests.
*
* @param Date $today
*/
public function setTodayDate(Date $today)
{
$this->today = $today;
}
/**
* For tests.
*
* @param int $now
*/
public function setNow($now)
{
$this->now = $now;
}
}

49
msd2/tracking/piwik/core/Archive/ArchiveQuery.php Normal file
View File

@ -0,0 +1,49 @@
<?php
/**
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*
*/
namespace Piwik\Archive;
use Piwik\DataTable;
interface ArchiveQuery
{
/**
* @param string|string[] $names
* @return false|number|array
*/
public function getNumeric($names);
/**
* @param string|string[] $names
* @return DataTable|DataTable\Map
*/
public function getDataTableFromNumeric($names);
/**
* @param $names
* @return mixed
*/
public function getDataTableFromNumericAndMergeChildren($names);
/**
* @param string $name
* @param int|string|null $idSubtable
* @return DataTable|DataTable\Map
*/
public function getDataTable($name, $idSubtable = null);
/**
* @param string $name
* @param int|string|null $idSubtable
* @param int|null $depth
* @param bool $addMetadataSubtableId
* @return DataTable|DataTable\Map
*/
public function getDataTableExpanded($name, $idSubtable = null, $depth = null, $addMetadataSubtableId = true);
}

127
msd2/tracking/piwik/core/Archive/ArchiveQueryFactory.php Normal file
View File

@ -0,0 +1,127 @@
<?php
/**
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*
*/
namespace Piwik\Archive;
use Piwik\Archive;
use Piwik\Period;
use Piwik\Segment;
use Piwik\Site;
use Piwik\Period\Factory as PeriodFactory;
class ArchiveQueryFactory
{
public function __construct()
{
// empty
}
/**
* @see \Piwik\Archive::build()
*/
public function build($idSites, $strPeriod, $strDate, $strSegment = false, $_restrictSitesToLogin = false)
{
list($websiteIds, $timezone, $idSiteIsAll) = $this->getSiteInfoFromQueryParam($idSites, $_restrictSitesToLogin);
list($allPeriods, $isMultipleDate) = $this->getPeriodInfoFromQueryParam($strDate, $strPeriod, $timezone);
$segment = $this->getSegmentFromQueryParam($strSegment, $websiteIds);
return $this->factory($segment, $allPeriods, $websiteIds, $idSiteIsAll, $isMultipleDate);
}
/**
* @see \Piwik\Archive::factory()
*/
public function factory(Segment $segment, array $periods, array $idSites, $idSiteIsAll = false, $isMultipleDate = false)
{
$forceIndexedBySite = false;
$forceIndexedByDate = false;
if ($idSiteIsAll || count($idSites) > 1) {
$forceIndexedBySite = true;
}
if (count($periods) > 1 || $isMultipleDate) {
$forceIndexedByDate = true;
}
$params = new Parameters($idSites, $periods, $segment);
return $this->newInstance($params, $forceIndexedBySite, $forceIndexedByDate);
}
public function newInstance(Parameters $params, $forceIndexedBySite, $forceIndexedByDate)
{
return new Archive($params, $forceIndexedBySite, $forceIndexedByDate);
}
/**
* Parses the site ID string provided in the 'idSite' query parameter to a list of
* website IDs.
*
* @param string $idSites the value of the 'idSite' query parameter
* @param bool $_restrictSitesToLogin
* @return array an array containing three elements:
* - an array of website IDs
* - string timezone to use (or false to use no timezone) when creating periods.
* - true if the request was for all websites (this forces the archive result to
* be indexed by site, even if there is only one site in Piwik)
*/
protected function getSiteInfoFromQueryParam($idSites, $_restrictSitesToLogin)
{
$websiteIds = Site::getIdSitesFromIdSitesString($idSites, $_restrictSitesToLogin);
$timezone = false;
if (count($websiteIds) == 1) {
$timezone = Site::getTimezoneFor($websiteIds[0]);
}
$idSiteIsAll = $idSites == Archive::REQUEST_ALL_WEBSITES_FLAG;
return [$websiteIds, $timezone, $idSiteIsAll];
}
/**
* Parses the date & period query parameters into a list of periods.
*
* @param string $strDate the value of the 'date' query parameter
* @param string $strPeriod the value of the 'period' query parameter
* @param string $timezone the timezone to use when constructing periods.
* @return array an array containing two elements:
* - the list of period objects to query archive data for
* - true if the request was for multiple periods (ie, two months, two weeks, etc.), false if otherwise.
* (this forces the archive result to be indexed by period, even if the list of periods
* has only one period).
*/
protected function getPeriodInfoFromQueryParam($strDate, $strPeriod, $timezone)
{
if (Period::isMultiplePeriod($strDate, $strPeriod)) {
$oPeriod = PeriodFactory::build($strPeriod, $strDate, $timezone);
$allPeriods = $oPeriod->getSubperiods();
} else {
$oPeriod = PeriodFactory::makePeriodFromQueryParams($timezone, $strPeriod, $strDate);
$allPeriods = array($oPeriod);
}
$isMultipleDate = Period::isMultiplePeriod($strDate, $strPeriod);
return [$allPeriods, $isMultipleDate];
}
/**
* Parses the segment query parameter into a Segment object.
*
* @param string $strSegment the value of the 'segment' query parameter.
* @param int[] $websiteIds the list of sites being queried.
* @return Segment
*/
protected function getSegmentFromQueryParam($strSegment, $websiteIds)
{
return new Segment($strSegment, $websiteIds);
}
}

144
msd2/tracking/piwik/core/Archive/Chunk.php Normal file
View File

@ -0,0 +1,144 @@
<?php
/**
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*
*/
namespace Piwik\Archive;
use Piwik\DataTable;
/**
* This class is used to split blobs of DataTables into chunks. Each blob used to be stored under one blob in the
* archive table. For better efficiency we do now combine multiple DataTable into one blob entry.
*
* Chunks are identified by having the recordName $recordName_chunk_0_99, $recordName_chunk_100_199 (this chunk stores
* the subtable 100-199).
*/
class Chunk
{
const ARCHIVE_APPENDIX_SUBTABLES = 'chunk';
const NUM_TABLES_IN_CHUNK = 100;
/**
* Get's the record name to use for a given tableId/subtableId.
*
* @param string $recordName eg 'Actions_ActionsUrl'
* @param int $tableId eg '5' for tableId '5'
* @return string eg 'Actions_ActionsUrl_chunk_0_99' as the table should be stored under this blob id.
*/
public function getRecordNameForTableId($recordName, $tableId)
{
$chunk = (floor($tableId / self::NUM_TABLES_IN_CHUNK));
$start = $chunk * self::NUM_TABLES_IN_CHUNK;
$end = $start + self::NUM_TABLES_IN_CHUNK - 1;
return $recordName . $this->getAppendix() . $start . '_' . $end;
}
/**
* Moves the given blobs into chunks and assigns a proper record name containing the chunk number.
*
* @param string $recordName The original archive record name, eg 'Actions_ActionsUrl'
* @param array $blobs An array containg a mapping of tableIds to blobs. Eg array(0 => 'blob', 1 => 'subtableBlob', ...)
* @return array An array where each blob is moved into a chunk, indexed by recordNames.
* eg array('Actions_ActionsUrl_chunk_0_99' => array(0 => 'blob', 1 => 'subtableBlob', ...),
* 'Actions_ActionsUrl_chunk_100_199' => array(...))
*/
public function moveArchiveBlobsIntoChunks($recordName, $blobs)
{
$chunks = array();
foreach ($blobs as $tableId => $blob) {
$name = $this->getRecordNameForTableId($recordName, $tableId);
if (!array_key_exists($name, $chunks)) {
$chunks[$name] = array();
}
$chunks[$name][$tableId] = $blob;
}
return $chunks;
}
/**
* Detects whether a recordName like 'Actions_ActionUrls_chunk_0_99' or 'Actions_ActionUrls' belongs to a
* chunk or not.
*
* To be a valid recordName that belongs to a chunk it must end with '_chunk_NUMERIC_NUMERIC'.
*
* @param string $recordName
* @return bool
*/
public function isRecordNameAChunk($recordName)
{
$posAppendix = $this->getEndPosOfChunkAppendix($recordName);
if (false === $posAppendix) {
return false;
}
// will contain "0_99" of "chunk_0_99"
$blobId = substr($recordName, $posAppendix);
return $this->isChunkRange($blobId);
}
private function isChunkRange($blobId)
{
$blobId = explode('_', $blobId);
return 2 === count($blobId) && is_numeric($blobId[0]) && is_numeric($blobId[1]);
}
/**
* When having a record like 'Actions_ActionUrls_chunk_0_99" it will return the raw recordName 'Actions_ActionUrls'.
*
* @param string $recordName
* @return string
*/
public function getRecordNameWithoutChunkAppendix($recordName)
{
if (!$this->isRecordNameAChunk($recordName)) {
return $recordName;
}
$posAppendix = $this->getStartPosOfChunkAppendix($recordName);
if (false === $posAppendix) {
return $recordName;
}
return substr($recordName, 0, $posAppendix);
}
/**
* Returns the string that is appended to the original record name. This appendix identifes a record name is a
* chunk.
* @return string
*/
public function getAppendix()
{
return '_' . self::ARCHIVE_APPENDIX_SUBTABLES . '_';
}
private function getStartPosOfChunkAppendix($recordName)
{
return strpos($recordName, $this->getAppendix());
}
private function getEndPosOfChunkAppendix($recordName)
{
$pos = strpos($recordName, $this->getAppendix());
if ($pos === false) {
return false;
}
return $pos + strlen($this->getAppendix());
}
}

375
msd2/tracking/piwik/core/Archive/DataCollection.php Normal file
View File

@ -0,0 +1,375 @@
<?php
/**
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*
*/
namespace Piwik\Archive;
use Exception;
use Piwik\DataTable;
/**
* This class is used to hold and transform archive data for the Archive class.
*
* Archive data is loaded into an instance of this type, can be indexed by archive
* metadata (such as the site ID, period string, etc.), and can be transformed into
* DataTable and Map instances.
*/
class DataCollection
{
const METADATA_CONTAINER_ROW_KEY = '_metadata';
/**
* The archive data, indexed first by site ID and then by period date range. Eg,
*
* array(
* '0' => array(
* array(
* '2012-01-01,2012-01-01' => array(...),
* '2012-01-02,2012-01-02' => array(...),
* )
* ),
* '1' => array(
* array(
* '2012-01-01,2012-01-01' => array(...),
* )
* )
* )
*
* Archive data can be either a numeric value or a serialized string blob. Every
* piece of archive data is associated by it's archive name. For example,
* the array(...) above could look like:
*
* array(
* 'nb_visits' => 1,
* 'nb_actions' => 2
* )
*
* There is a special element '_metadata' in data rows that holds values treated
* as DataTable metadata.
*/
private $data = array();
/**
* The whole list of metric/record names that were used in the archive query.
*
* @var array
*/
private $dataNames;
/**
* The type of data that was queried for (ie, "blob" or "numeric").
*
* @var string
*/
private $dataType;
/**
* The default values to use for each metric/record name that's being queried
* for.
*
* @var array
*/
private $defaultRow;
/**
* The list of all site IDs that were queried for.
*
* @var array
*/
private $sitesId;
/**
* The list of all periods that were queried for. Each period is associated with
* the period's range string. Eg,
*
* array(
* '2012-01-01,2012-01-31' => new Period(...),
* '2012-02-01,2012-02-28' => new Period(...),
* )
*
* @var \Piwik\Period[]
*/
private $periods;
/**
* Constructor.
*
* @param array $dataNames @see $this->dataNames
* @param string $dataType @see $this->dataType
* @param array $sitesId @see $this->sitesId
* @param \Piwik\Period[] $periods @see $this->periods
* @param array $defaultRow @see $this->defaultRow
*/
public function __construct($dataNames, $dataType, $sitesId, $periods, $defaultRow = null)
{
$this->dataNames = $dataNames;
$this->dataType = $dataType;
if ($defaultRow === null) {
$defaultRow = array_fill_keys($dataNames, 0);
}
$this->sitesId = $sitesId;
foreach ($periods as $period) {
$this->periods[$period->getRangeString()] = $period;
}
$this->defaultRow = $defaultRow;
}
/**
* Returns a reference to the data for a specific site & period. If there is
* no data for the given site ID & period, it is set to the default row.
*
* @param int $idSite
* @param string $period eg, '2012-01-01,2012-01-31'
*/
public function &get($idSite, $period)
{
if (!isset($this->data[$idSite][$period])) {
$this->data[$idSite][$period] = $this->defaultRow;
}
return $this->data[$idSite][$period];
}
/**
* Set data for a specific site & period. If there is no data for the given site ID & period,
* it is set to the default row.
*
* @param int $idSite
* @param string $period eg, '2012-01-01,2012-01-31'
* @param string $name eg 'nb_visits'
* @param string $value eg 5
*/
public function set($idSite, $period, $name, $value)
{
$row = & $this->get($idSite, $period);
$row[$name] = $value;
}
/**
* Adds a new metadata to the data for specific site & period. If there is no
* data for the given site ID & period, it is set to the default row.
*
* Note: Site ID and period range string are two special types of metadata. Since
* the data stored in this class is indexed by site & period, this metadata is not
* stored in individual data rows.
*
* @param int $idSite
* @param string $period eg, '2012-01-01,2012-01-31'
* @param string $name The metadata name.
* @param mixed $value The metadata name.
*/
public function addMetadata($idSite, $period, $name, $value)
{
$row = & $this->get($idSite, $period);
$row[self::METADATA_CONTAINER_ROW_KEY][$name] = $value;
}
/**
* Returns archive data as an array indexed by metadata.
*
* @param array $resultIndices An array mapping metadata names to pretty labels
* for them. Each archive data row will be indexed
* by the metadata specified here.
*
* Eg, array('site' => 'idSite', 'period' => 'Date')
* @return array
*/
public function getIndexedArray($resultIndices)
{
$indexKeys = array_keys($resultIndices);
$result = $this->createOrderedIndex($indexKeys);
foreach ($this->data as $idSite => $rowsByPeriod) {
foreach ($rowsByPeriod as $period => $row) {
// FIXME: This hack works around a strange bug that occurs when getting
// archive IDs through ArchiveProcessing instances. When a table
// does not already exist, for some reason the archive ID for
// today (or from two days ago) will be added to the Archive
// instances list. The Archive instance will then select data
// for periods outside of the requested set.
// working around the bug here, but ideally, we need to figure
// out why incorrect idarchives are being selected.
if (empty($this->periods[$period])) {
continue;
}
$this->putRowInIndex($result, $indexKeys, $row, $idSite, $period);
}
}
return $result;
}
/**
* Returns archive data as a DataTable indexed by metadata. Indexed data will
* be represented by Map instances.
*
* @param array $resultIndices An array mapping metadata names to pretty labels
* for them. Each archive data row will be indexed
* by the metadata specified here.
*
* Eg, array('site' => 'idSite', 'period' => 'Date')
* @return DataTable|DataTable\Map
*/
public function getDataTable($resultIndices)
{
$dataTableFactory = new DataTableFactory(
$this->dataNames, $this->dataType, $this->sitesId, $this->periods, $this->defaultRow);
$index = $this->getIndexedArray($resultIndices);
return $dataTableFactory->make($index, $resultIndices);
}
/**
* See {@link DataTableFactory::makeMerged()}
*
* @param array $resultIndices
* @return DataTable|DataTable\Map
* @throws Exception
*/
public function getMergedDataTable($resultIndices)
{
$dataTableFactory = new DataTableFactory(
$this->dataNames, $this->dataType, $this->sitesId, $this->periods, $this->defaultRow);
$index = $this->getIndexedArray($resultIndices);
return $dataTableFactory->makeMerged($index, $resultIndices);
}
/**
* Returns archive data as a DataTable indexed by metadata. Indexed data will
* be represented by Map instances. Each DataTable will have
* its subtable IDs set.
*
* This function will only work if blob data was loaded and only one record
* was loaded (not including subtables of the record).
*
* @param array $resultIndices An array mapping metadata names to pretty labels
* for them. Each archive data row will be indexed
* by the metadata specified here.
*
* Eg, array('site' => 'idSite', 'period' => 'Date')
* @param int|null $idSubTable The subtable to return.
* @param int|null $depth max depth for subtables.
* @param bool $addMetadataSubTableId Whether to add the DB subtable ID as metadata
* to each datatable, or not.
* @throws Exception
* @return DataTable|DataTable\Map
*/
public function getExpandedDataTable($resultIndices, $idSubTable = null, $depth = null, $addMetadataSubTableId = false)
{
if ($this->dataType != 'blob') {
throw new Exception("DataCollection: cannot call getExpandedDataTable with "
. "{$this->dataType} data types. Only works with blob data.");
}
if (count($this->dataNames) !== 1) {
throw new Exception("DataCollection: cannot call getExpandedDataTable with "
. "more than one record.");
}
$dataTableFactory = new DataTableFactory(
$this->dataNames, 'blob', $this->sitesId, $this->periods, $this->defaultRow);
$dataTableFactory->expandDataTable($depth, $addMetadataSubTableId);
$dataTableFactory->useSubtable($idSubTable);
$index = $this->getIndexedArray($resultIndices);
return $dataTableFactory->make($index, $resultIndices);
}
/**
* Returns metadata for a data row.
*
* @param array $data The data row.
* @return array
*/
public static function getDataRowMetadata($data)
{
if (isset($data[self::METADATA_CONTAINER_ROW_KEY])) {
return $data[self::METADATA_CONTAINER_ROW_KEY];
} else {
return array();
}
}
/**
* Removes all table metadata from a data row.
*
* @param array $data The data row.
*/
public static function removeMetadataFromDataRow(&$data)
{
unset($data[self::METADATA_CONTAINER_ROW_KEY]);
}
/**
* Creates an empty index using a list of metadata names. If the 'site' and/or
* 'period' metadata names are supplied, empty rows are added for every site/period
* that was queried for.
*
* Using this function ensures consistent ordering in the indexed result.
*
* @param array $metadataNamesToIndexBy List of metadata names to index archive data by.
* @return array
*/
private function createOrderedIndex($metadataNamesToIndexBy)
{
$result = array();
if (!empty($metadataNamesToIndexBy)) {
$metadataName = array_shift($metadataNamesToIndexBy);
if ($metadataName == DataTableFactory::TABLE_METADATA_SITE_INDEX) {
$indexKeyValues = array_values($this->sitesId);
} elseif ($metadataName == DataTableFactory::TABLE_METADATA_PERIOD_INDEX) {
$indexKeyValues = array_keys($this->periods);
}
if (empty($metadataNamesToIndexBy)) {
$result = array_fill_keys($indexKeyValues, array());
} else {
foreach ($indexKeyValues as $key) {
$result[$key] = $this->createOrderedIndex($metadataNamesToIndexBy);
}
}
}
return $result;
}
/**
* Puts an archive data row in an index.
*/
private function putRowInIndex(&$index, $metadataNamesToIndexBy, $row, $idSite, $period)
{
$currentLevel = & $index;
foreach ($metadataNamesToIndexBy as $metadataName) {
if ($metadataName == DataTableFactory::TABLE_METADATA_SITE_INDEX) {
$key = $idSite;
} elseif ($metadataName == DataTableFactory::TABLE_METADATA_PERIOD_INDEX) {
$key = $period;
} else {
$key = $row[self::METADATA_CONTAINER_ROW_KEY][$metadataName];
}
if (!isset($currentLevel[$key])) {
$currentLevel[$key] = array();
}
$currentLevel = & $currentLevel[$key];
}
$currentLevel = $row;
}
}

552
msd2/tracking/piwik/core/Archive/DataTableFactory.php Normal file
View File

@ -0,0 +1,552 @@
<?php
/**
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*
*/
namespace Piwik\Archive;
use Piwik\DataTable;
use Piwik\DataTable\Row;
use Piwik\Period\Week;
use Piwik\Site;
/**
* Creates a DataTable or Set instance based on an array
* index created by DataCollection.
*
* This class is only used by DataCollection.
*/
class DataTableFactory
{
/**
* @see DataCollection::$dataNames.
*/
private $dataNames;
/**
* @see DataCollection::$dataType.
*/
private $dataType;
/**
* Whether to expand the DataTables that're created or not. Expanding a DataTable
* means creating DataTables using subtable blobs and correctly setting the subtable
* IDs of all DataTables.
*
* @var bool
*/
private $expandDataTable = false;
/**
* Whether to add the subtable ID used in the database to the in-memory DataTables
* as metadata or not.
*
* @var bool
*/
private $addMetadataSubtableId = false;
/**
* The maximum number of subtable levels to create when creating an expanded
* DataTable.
*
* @var int
*/
private $maxSubtableDepth = null;
/**
* @see DataCollection::$sitesId.
*/
private $sitesId;
/**
* @see DataCollection::$periods.
*/
private $periods;
/**
* The ID of the subtable to create a DataTable for. Only relevant for blob data.
*
* @var int|null
*/
private $idSubtable = null;
/**
* @see DataCollection::$defaultRow.
*/
private $defaultRow;
const TABLE_METADATA_SITE_INDEX = 'site';
const TABLE_METADATA_PERIOD_INDEX = 'period';
/**
* Constructor.
*/
public function __construct($dataNames, $dataType, $sitesId, $periods, $defaultRow)
{
$this->dataNames = $dataNames;
$this->dataType = $dataType;
$this->sitesId = $sitesId;
//here index period by string only
$this->periods = $periods;
$this->defaultRow = $defaultRow;
}
/**
* Returns the ID of the site a table is related to based on the 'site' metadata entry,
* or null if there is none.
*
* @param DataTable $table
* @return int|null
*/
public static function getSiteIdFromMetadata(DataTable $table)
{
$site = $table->getMetadata('site');
if (empty($site)) {
return null;
} else {
return $site->getId();
}
}
/**
* Tells the factory instance to expand the DataTables that are created by
* creating subtables and setting the subtable IDs of rows w/ subtables correctly.
*
* @param null|int $maxSubtableDepth max depth for subtables.
* @param bool $addMetadataSubtableId Whether to add the subtable ID used in the
* database to the in-memory DataTables as
* metadata or not.
*/
public function expandDataTable($maxSubtableDepth = null, $addMetadataSubtableId = false)
{
$this->expandDataTable = true;
$this->maxSubtableDepth = $maxSubtableDepth;
$this->addMetadataSubtableId = $addMetadataSubtableId;
}
/**
* Tells the factory instance to create a DataTable using a blob with the
* supplied subtable ID.
*
* @param int $idSubtable An in-database subtable ID.
* @throws \Exception
*/
public function useSubtable($idSubtable)
{
if (count($this->dataNames) !== 1) {
throw new \Exception("DataTableFactory: Getting subtables for multiple records in one"
. " archive query is not currently supported.");
}
$this->idSubtable = $idSubtable;
}
private function isNumericDataType()
{
return $this->dataType == 'numeric';
}
/**
* Creates a DataTable|Set instance using an index of
* archive data.
*
* @param array $index @see DataCollection
* @param array $resultIndices an array mapping metadata names with pretty metadata
* labels.
* @return DataTable|DataTable\Map
*/
public function make($index, $resultIndices)
{
$keyMetadata = $this->getDefaultMetadata();
if (empty($resultIndices)) {
// for numeric data, if there's no index (and thus only 1 site & period in the query),
// we want to display every queried metric name
if (empty($index)
&& $this->isNumericDataType()
) {
$index = $this->defaultRow;
}
$dataTable = $this->createDataTable($index, $keyMetadata);
} else {
$dataTable = $this->createDataTableMapFromIndex($index, $resultIndices, $keyMetadata);
}
return $dataTable;
}
/**
* Creates a merged DataTable|Map instance using an index of archive data similar to {@link make()}.
*
* Whereas {@link make()} creates a Map for each result index (period and|or site), this will only create a Map
* for a period result index and move all site related indices into one dataTable. This is the same as doing
* `$dataTableFactory->make()->mergeChildren()` just much faster. It is mainly useful for reports across many sites
* eg `MultiSites.getAll`. Was done as part of https://github.com/piwik/piwik/issues/6809
*
* @param array $index @see DataCollection
* @param array $resultIndices an array mapping metadata names with pretty metadata labels.
*
* @return DataTable|DataTable\Map
* @throws \Exception
*/
public function makeMerged($index, $resultIndices)
{
if (!$this->isNumericDataType()) {
throw new \Exception('This method is supposed to work with non-numeric data types but it is not tested. To use it, remove this exception and write tests to be sure it works.');
}
$hasSiteIndex = isset($resultIndices[self::TABLE_METADATA_SITE_INDEX]);
$hasPeriodIndex = isset($resultIndices[self::TABLE_METADATA_PERIOD_INDEX]);
$isNumeric = $this->isNumericDataType();
// to be backwards compatible use a Simple table if needed as it will be formatted differently
$useSimpleDataTable = !$hasSiteIndex && $isNumeric;
if (!$hasSiteIndex) {
$firstIdSite = reset($this->sitesId);
$index = array($firstIdSite => $index);
}
if ($hasPeriodIndex) {
$dataTable = $this->makeMergedTableWithPeriodAndSiteIndex($index, $resultIndices, $useSimpleDataTable, $isNumeric);
} else {
$dataTable = $this->makeMergedWithSiteIndex($index, $useSimpleDataTable, $isNumeric);
}
return $dataTable;
}
/**
* Creates a DataTable|Set instance using an array
* of blobs.
*
* If only one record is being queried, a single DataTable will
* be returned. Otherwise, a DataTable\Map is returned that indexes
* DataTables by record name.
*
* If expandDataTable was called, and only one record is being queried,
* the created DataTable's subtables will be expanded.
*
* @param array $blobRow
* @return DataTable|DataTable\Map
*/
private function makeFromBlobRow($blobRow, $keyMetadata)
{
if ($blobRow === false) {
return new DataTable();
}
if (count($this->dataNames) === 1) {
return $this->makeDataTableFromSingleBlob($blobRow, $keyMetadata);
} else {
return $this->makeIndexedByRecordNameDataTable($blobRow, $keyMetadata);
}
}
/**
* Creates a DataTable for one record from an archive data row.
*
* @see makeFromBlobRow
*
* @param array $blobRow
* @return DataTable
*/
private function makeDataTableFromSingleBlob($blobRow, $keyMetadata)
{
$recordName = reset($this->dataNames);
if ($this->idSubtable !== null) {
$recordName .= '_' . $this->idSubtable;
}
if (!empty($blobRow[$recordName])) {
$table = DataTable::fromSerializedArray($blobRow[$recordName]);
} else {
$table = new DataTable();
}
// set table metadata
$table->setAllTableMetadata(array_merge(DataCollection::getDataRowMetadata($blobRow), $keyMetadata));
if ($this->expandDataTable) {
$table->enableRecursiveFilters();
$this->setSubtables($table, $blobRow);
}
return $table;
}
/**
* Creates a DataTable for every record in an archive data row and puts them
* in a DataTable\Map instance.
*
* @param array $blobRow
* @return DataTable\Map
*/
private function makeIndexedByRecordNameDataTable($blobRow, $keyMetadata)
{
$table = new DataTable\Map();
$table->setKeyName('recordName');
$tableMetadata = array_merge(DataCollection::getDataRowMetadata($blobRow), $keyMetadata);
foreach ($blobRow as $name => $blob) {
$newTable = DataTable::fromSerializedArray($blob);
$newTable->setAllTableMetadata($tableMetadata);
$table->addTable($newTable, $name);
}
return $table;
}
/**
* Creates a Set from an array index.
*
* @param array $index @see DataCollection
* @param array $resultIndices @see make
* @param array $keyMetadata The metadata to add to the table when it's created.
* @return DataTable\Map
*/
private function createDataTableMapFromIndex($index, $resultIndices, $keyMetadata)
{
$result = new DataTable\Map();
$result->setKeyName(reset($resultIndices));
$resultIndex = key($resultIndices);
array_shift($resultIndices);
$hasIndices = !empty($resultIndices);
foreach ($index as $label => $value) {
$keyMetadata[$resultIndex] = $this->createTableIndexMetadata($resultIndex, $label);
if ($hasIndices) {
$newTable = $this->createDataTableMapFromIndex($value, $resultIndices, $keyMetadata);
} else {
$newTable = $this->createDataTable($value, $keyMetadata);
}
$result->addTable($newTable, $this->prettifyIndexLabel($resultIndex, $label));
}
return $result;
}
private function createTableIndexMetadata($resultIndex, $label)
{
if ($resultIndex === DataTableFactory::TABLE_METADATA_SITE_INDEX) {
return new Site($label);
} elseif ($resultIndex === DataTableFactory::TABLE_METADATA_PERIOD_INDEX) {
return $this->periods[$label];
}
}
/**
* Creates a DataTable instance from an index row.
*
* @param array $data An archive data row.
* @param array $keyMetadata The metadata to add to the table(s) when created.
* @return DataTable|DataTable\Map
*/
private function createDataTable($data, $keyMetadata)
{
if ($this->dataType == 'blob') {
$result = $this->makeFromBlobRow($data, $keyMetadata);
} else {
$result = $this->makeFromMetricsArray($data, $keyMetadata);
}
return $result;
}
/**
* Creates DataTables from $dataTable's subtable blobs (stored in $blobRow) and sets
* the subtable IDs of each DataTable row.
*
* @param DataTable $dataTable
* @param array $blobRow An array associating record names (w/ subtable if applicable)
* with blob values. This should hold every subtable blob for
* the loaded DataTable.
* @param int $treeLevel
*/
private function setSubtables($dataTable, $blobRow, $treeLevel = 0)
{
if ($this->maxSubtableDepth
&& $treeLevel >= $this->maxSubtableDepth
) {
// unset the subtables so DataTableManager doesn't throw
foreach ($dataTable->getRowsWithoutSummaryRow() as $row) {
$row->removeSubtable();
}
return;
}
$dataName = reset($this->dataNames);
foreach ($dataTable->getRowsWithoutSummaryRow() as $row) {
$sid = $row->getIdSubDataTable();
if ($sid === null) {
continue;
}
$blobName = $dataName . "_" . $sid;
if (isset($blobRow[$blobName])) {
$subtable = DataTable::fromSerializedArray($blobRow[$blobName]);
$this->setSubtables($subtable, $blobRow, $treeLevel + 1);
// we edit the subtable ID so that it matches the newly table created in memory
// NB: we don't overwrite the datatableid in the case we are displaying the table expanded.
if ($this->addMetadataSubtableId) {
// this will be written back to the column 'idsubdatatable' just before rendering,
// see Renderer/Php.php
$row->addMetadata('idsubdatatable_in_db', $row->getIdSubDataTable());
}
$row->setSubtable($subtable);
}
}
}
private function getDefaultMetadata()
{
return array(
DataTableFactory::TABLE_METADATA_SITE_INDEX => new Site(reset($this->sitesId)),
DataTableFactory::TABLE_METADATA_PERIOD_INDEX => reset($this->periods),
);
}
/**
* Returns the pretty version of an index label.
*
* @param string $labelType eg, 'site', 'period', etc.
* @param string $label eg, '0', '1', '2012-01-01,2012-01-31', etc.
* @return string
*/
private function prettifyIndexLabel($labelType, $label)
{
if ($labelType == self::TABLE_METADATA_PERIOD_INDEX) { // prettify period labels
$period = $this->periods[$label];
$label = $period->getLabel();
if ($label === 'week' || $label === 'range') {
return $period->getRangeString();
}
return $period->getPrettyString();
}
return $label;
}
/**
* @param $data
* @return DataTable\Simple
*/
private function makeFromMetricsArray($data, $keyMetadata)
{
$table = new DataTable\Simple();
if (!empty($data)) {
$table->setAllTableMetadata(array_merge(DataCollection::getDataRowMetadata($data), $keyMetadata));
DataCollection::removeMetadataFromDataRow($data);
$table->addRow(new Row(array(Row::COLUMNS => $data)));
} else {
// if we're querying numeric data, we couldn't find any, and we're only
// looking for one metric, add a row w/ one column w/ value 0. this is to
// ensure that the PHP renderer outputs 0 when only one column is queried.
// w/o this code, an empty array would be created, and other parts of Piwik
// would break.
if (count($this->dataNames) == 1
&& $this->isNumericDataType()
) {
$name = reset($this->dataNames);
$table->addRow(new Row(array(Row::COLUMNS => array($name => 0))));
}
$table->setAllTableMetadata($keyMetadata);
}
$result = $table;
return $result;
}
private function makeMergedTableWithPeriodAndSiteIndex($index, $resultIndices, $useSimpleDataTable, $isNumeric)
{
$map = new DataTable\Map();
$map->setKeyName($resultIndices[self::TABLE_METADATA_PERIOD_INDEX]);
// we save all tables of the map in this array to be able to add rows fast
$tables = array();
foreach ($this->periods as $range => $period) {
// as the resulting table is "merged", we do only set Period metedata and no metadata for site. Instead each
// row will have an idsite metadata entry.
$metadata = array(self::TABLE_METADATA_PERIOD_INDEX => $period);
if ($useSimpleDataTable) {
$table = new DataTable\Simple();
} else {
$table = new DataTable();
}
$table->setAllTableMetadata($metadata);
$map->addTable($table, $this->prettifyIndexLabel(self::TABLE_METADATA_PERIOD_INDEX, $range));
$tables[$range] = $table;
}
foreach ($index as $idsite => $table) {
$rowMeta = array('idsite' => $idsite);
foreach ($table as $range => $row) {
if (!empty($row)) {
$tables[$range]->addRow(new Row(array(
Row::COLUMNS => $row,
Row::METADATA => $rowMeta)
));
} elseif ($isNumeric) {
$tables[$range]->addRow(new Row(array(
Row::COLUMNS => $this->defaultRow,
Row::METADATA => $rowMeta)
));
}
}
}
return $map;
}
private function makeMergedWithSiteIndex($index, $useSimpleDataTable, $isNumeric)
{
if ($useSimpleDataTable) {
$table = new DataTable\Simple();
} else {
$table = new DataTable();
}
$table->setAllTableMetadata(array(DataTableFactory::TABLE_METADATA_PERIOD_INDEX => reset($this->periods)));
foreach ($index as $idsite => $row) {
if (!empty($row)) {
$table->addRow(new Row(array(
Row::COLUMNS => $row,
Row::METADATA => array('idsite' => $idsite))
));
} elseif ($isNumeric) {
$table->addRow(new Row(array(
Row::COLUMNS => $this->defaultRow,
Row::METADATA => array('idsite' => $idsite))
));
}
}
return $table;
}
}

59
msd2/tracking/piwik/core/Archive/Parameters.php Normal file
View File

@ -0,0 +1,59 @@
<?php
/**
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*
*/
namespace Piwik\Archive;
use Piwik\Period;
use Piwik\Segment;
class Parameters
{
/**
* The list of site IDs to query archive data for.
*
* @var array
*/
private $idSites = array();
/**
* The list of Period's to query archive data for.
*
* @var Period[]
*/
private $periods = array();
/**
* Segment applied to the visits set.
*
* @var Segment
*/
private $segment;
public function getSegment()
{
return $this->segment;
}
public function __construct($idSites, $periods, Segment $segment)
{
$this->idSites = $idSites;
$this->periods = $periods;
$this->segment = $segment;
}
public function getPeriods()
{
return $this->periods;
}
public function getIdSites()
{
return $this->idSites;
}
}