PDF rausgenommen

This commit is contained in:
aschwarz
2023-01-23 11:03:31 +01:00
parent 82d562a322
commit a6523903eb
28078 changed files with 4247552 additions and 2 deletions

View File

@ -0,0 +1,201 @@
<?php
/**
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*/
namespace Piwik\Plugins\CoreAdminHome\Commands;
use Piwik\Common;
use Piwik\Container\StaticContainer;
use Piwik\DataAccess\RawLogDao;
use Piwik\Date;
use Piwik\Db;
use Piwik\LogDeleter;
use Piwik\Plugin\ConsoleCommand;
use Piwik\Site;
use Piwik\Timer;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\Console\Question\ConfirmationQuestion;
/**
* Command to selectively delete visits.
*/
class DeleteLogsData extends ConsoleCommand
{
private static $logTables = array(
'log_visit',
'log_link_visit_action',
'log_conversion',
'log_conversion_item',
'log_action'
);
/**
* @var RawLogDao
*/
private $rawLogDao;
/**
* @var LogDeleter
*/
private $logDeleter;
public function __construct(LogDeleter $logDeleter = null, RawLogDao $rawLogDao = null)
{
parent::__construct();
$this->logDeleter = $logDeleter ?: StaticContainer::get('Piwik\LogDeleter');
$this->rawLogDao = $rawLogDao ?: StaticContainer::get('Piwik\DataAccess\RawLogDao');
}
protected function configure()
{
$this->setName('core:delete-logs-data');
$this->setDescription('Delete data from the user log tables: ' . implode(', ', self::$logTables) . '.');
$this->addOption('dates', null, InputOption::VALUE_REQUIRED, 'Delete log data with a date within this date range. Eg, 2012-01-01,2013-01-01');
$this->addOption('idsite', null, InputOption::VALUE_OPTIONAL,
'Delete log data belonging to the site with this ID. Comma separated list of website id. Eg, 1, 2, 3, etc. By default log data from all sites is purged.');
$this->addOption('limit', null, InputOption::VALUE_REQUIRED, "The number of rows to delete at a time. The larger the number, "
. "the more time is spent deleting logs, and the less progress will be printed to the screen.", 1000);
$this->addOption('optimize-tables', null, InputOption::VALUE_NONE,
"If supplied, the command will optimize log tables after deleting logs. Note: this can take a very long time.");
}
protected function execute(InputInterface $input, OutputInterface $output)
{
list($from, $to) = $this->getDateRangeToDeleteFrom($input);
$idSite = $this->getSiteToDeleteFrom($input);
$step = $this->getRowIterationStep($input);
$output->writeln( sprintf(
"<info>Preparing to delete all visits belonging to %s between $from and $to.</info>",
$idSite ? "website $idSite" : "ALL websites"
));
$confirm = $this->askForDeleteConfirmation($input, $output);
if (!$confirm) {
return;
}
$timer = new Timer();
try {
$logsDeleted = $this->logDeleter->deleteVisitsFor($from, $to, $idSite, $step, function () use ($output) {
$output->write('.');
});
} catch (\Exception $ex) {
$output->writeln("");
throw $ex;
}
$this->writeSuccessMessage($output, array(
"Successfully deleted $logsDeleted visits. <comment>" . $timer . "</comment>"));
if ($input->getOption('optimize-tables')) {
$this->optimizeTables($output);
}
}
/**
* @param InputInterface $input
* @return Date[]
*/
private function getDateRangeToDeleteFrom(InputInterface $input)
{
$dates = $input->getOption('dates');
if (empty($dates)) {
throw new \InvalidArgumentException("No date range supplied in --dates option. Deleting all logs by default is not allowed, you must specify a date range.");
}
$parts = explode(',', $dates);
$parts = array_map('trim', $parts);
if (count($parts) !== 2) {
throw new \InvalidArgumentException("Invalid date range supplied: $dates");
}
list($start, $end) = $parts;
try {
/** @var Date[] $dateObjects */
$dateObjects = array(Date::factory($start), Date::factory($end));
} catch (\Exception $ex) {
throw new \InvalidArgumentException("Invalid date range supplied: $dates (" . $ex->getMessage() . ")", $code = 0, $ex);
}
if ($dateObjects[0]->getTimestamp() > $dateObjects[1]->getTimestamp()) {
throw new \InvalidArgumentException("Invalid date range supplied: $dates (first date is older than the last date)");
}
$dateObjects = array($dateObjects[0]->getDatetime(), $dateObjects[1]->getDatetime());
return $dateObjects;
}
private function getSiteToDeleteFrom(InputInterface $input)
{
$idSite = $input->getOption('idsite');
if(is_null($idSite)) {
return $idSite;
}
// validate the site ID
try {
new Site($idSite);
} catch (\Exception $ex) {
throw new \InvalidArgumentException("Invalid site ID: $idSite", $code = 0, $ex);
}
return $idSite;
}
private function getRowIterationStep(InputInterface $input)
{
$step = (int) $input->getOption('limit');
if ($step <= 0) {
throw new \InvalidArgumentException("Invalid row limit supplied: $step. Must be a number greater than 0.");
}
return $step;
}
private function askForDeleteConfirmation(InputInterface $input, OutputInterface $output)
{
if ($input->getOption('no-interaction')) {
return true;
}
$helper = $this->getHelper('question');
$question = new ConfirmationQuestion('<comment>You are about to delete log data. This action cannot be undone, are you sure you want to continue? (Y/N)</comment> ', false);
return $helper->ask($input, $output, $question);
}
private function optimizeTables(OutputInterface $output)
{
foreach (self::$logTables as $table) {
$output->write("Optimizing table $table... ");
$timer = new Timer();
$prefixedTable = Common::prefixTable($table);
$done = Db::optimizeTables($prefixedTable);
if($done) {
$output->writeln("done. <comment>" . $timer . "</comment>");
} else {
$output->writeln("skipped! <comment>" . $timer . "</comment>");
}
}
$this->writeSuccessMessage($output, array("Table optimization finished."));
}
}

View File

@ -0,0 +1,201 @@
<?php
/**
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*
*/
namespace Piwik\Plugins\CoreAdminHome\Commands;
use Piwik\Common;
use Piwik\Container\StaticContainer;
use Piwik\DataAccess\Actions;
use Piwik\Archive\ArchiveInvalidator;
use Piwik\Date;
use Piwik\Plugin\ConsoleCommand;
use Piwik\Plugins\CoreAdminHome\Model\DuplicateActionRemover;
use Piwik\Timer;
use Psr\Log\LoggerInterface;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
/**
* Finds duplicate actions rows in log_action and removes them. Fixes references to duplicate
* actions in the log_link_visit_action table, log_conversion table, and log_conversion_item
* table.
*
* Prior to version 2.11, there was a race condition in the tracker where it was possible for
* two or more actions with the same name and type to be inserted simultaneously. This resulted
* in inaccurate data. A Piwik database with this problem can be fixed using this class.
*
* With version 2.11 and above, it is still possible for duplicate actions to be inserted, but
* ONLY if the tracker's PHP process fails suddenly right after inserting an action. This is
* very rare, and even if it does happen, report data will not be affected, but the extra
* actions can be deleted w/ this class.
*/
class FixDuplicateLogActions extends ConsoleCommand
{
/**
* Used to invalidate archives. Only used if $shouldInvalidateArchives is true.
*
* @var ArchiveInvalidator
*/
private $archiveInvalidator;
/**
* DAO used to find duplicate actions in log_action and fix references to them in other tables.
*
* @var DuplicateActionRemover
*/
private $duplicateActionRemover;
/**
* DAO used to remove actions from the log_action table.
*
* @var Actions
*/
private $actionsAccess;
/**
* @var LoggerInterface
*/
private $logger;
/**
* Constructor.
*
* @param ArchiveInvalidator $invalidator
* @param DuplicateActionRemover $duplicateActionRemover
* @param Actions $actionsAccess
* @param LoggerInterface $logger
*/
public function __construct(ArchiveInvalidator $invalidator = null, DuplicateActionRemover $duplicateActionRemover = null,
Actions $actionsAccess = null, LoggerInterface $logger = null)
{
parent::__construct();
$this->archiveInvalidator = $invalidator ?: StaticContainer::get('Piwik\Archive\ArchiveInvalidator');
$this->duplicateActionRemover = $duplicateActionRemover ?: new DuplicateActionRemover();
$this->actionsAccess = $actionsAccess ?: new Actions();
$this->logger = $logger ?: StaticContainer::get('Psr\Log\LoggerInterface');
}
protected function configure()
{
$this->setName('core:fix-duplicate-log-actions');
$this->addOption('invalidate-archives', null, InputOption::VALUE_NONE, "If supplied, archives for logs that use duplicate actions will be invalidated."
. " On the next cron archive run, the reports for those dates will be re-processed.");
$this->setDescription('Removes duplicates in the log action table and fixes references to the duplicates in '
. 'related tables. NOTE: This action can take a long time to run!');
}
protected function execute(InputInterface $input, OutputInterface $output)
{
$invalidateArchives = $input->getOption('invalidate-archives');
$timer = new Timer();
$duplicateActions = $this->duplicateActionRemover->getDuplicateIdActions();
if (empty($duplicateActions)) {
$output->writeln("Found no duplicate actions.");
return;
}
$output->writeln("<info>Found " . count($duplicateActions) . " actions with duplicates.</info>");
list($numberRemoved, $allArchivesAffected) = $this->fixDuplicateActionReferences($duplicateActions, $output);
$this->deleteDuplicatesFromLogAction($output, $duplicateActions);
if ($invalidateArchives) {
$this->invalidateArchivesUsingActionDuplicates($allArchivesAffected, $output);
} else {
$this->printAffectedArchives($allArchivesAffected, $output);
}
$logActionTable = Common::prefixTable('log_action');
$this->writeSuccessMessage($output, array(
"Found and deleted $numberRemoved duplicate action entries in the $logActionTable table.",
"References in log_link_visit_action, log_conversion and log_conversion_item were corrected.",
$timer->__toString()
));
}
private function invalidateArchivesUsingActionDuplicates($archivesAffected, OutputInterface $output)
{
$output->write("Invalidating archives affected by duplicates fixed...");
foreach ($archivesAffected as $archiveInfo) {
$dates = array(Date::factory($archiveInfo['server_time']));
$this->archiveInvalidator->markArchivesAsInvalidated(array($archiveInfo['idsite']), $dates, $period = false);
}
$output->writeln("Done.");
}
private function printAffectedArchives($allArchivesAffected, OutputInterface $output)
{
$output->writeln("The following archives used duplicate actions and should be invalidated if you want correct reports:");
foreach ($allArchivesAffected as $archiveInfo) {
$output->writeln("\t[ idSite = {$archiveInfo['idsite']}, date = {$archiveInfo['server_time']} ]");
}
}
private function fixDuplicateActionReferences($duplicateActions, OutputInterface $output)
{
$dupeCount = count($duplicateActions);
$numberRemoved = 0;
$allArchivesAffected = array();
foreach ($duplicateActions as $index => $dupeInfo) {
$name = $dupeInfo['name'];
$toIdAction = $dupeInfo['idaction'];
$fromIdActions = $dupeInfo['duplicateIdActions'];
$numberRemoved += count($fromIdActions);
$output->writeln("<info>[$index / $dupeCount]</info> Fixing duplicates for '$name'");
$this->logger->debug(" idaction = {idaction}, duplicate idactions = {duplicateIdActions}", array(
'idaction' => $toIdAction,
'duplicateIdActions' => $fromIdActions
));
foreach (DuplicateActionRemover::$tablesWithIdActionColumns as $table) {
$archivesAffected = $this->fixDuplicateActionsInTable($output, $table, $toIdAction, $fromIdActions);
$allArchivesAffected = array_merge($allArchivesAffected, $archivesAffected);
}
}
$allArchivesAffected = array_values(array_unique($allArchivesAffected, SORT_REGULAR));
return array($numberRemoved, $allArchivesAffected);
}
private function fixDuplicateActionsInTable(OutputInterface $output, $table, $toIdAction, $fromIdActions)
{
$timer = new Timer();
$archivesAffected = $this->duplicateActionRemover->getSitesAndDatesOfRowsUsingDuplicates($table, $fromIdActions);
$this->duplicateActionRemover->fixDuplicateActionsInTable($table, $toIdAction, $fromIdActions);
$output->writeln("\tFixed duplicates in " . Common::prefixTable($table) . ". <comment>" . $timer->__toString() . "</comment>.");
return $archivesAffected;
}
private function deleteDuplicatesFromLogAction(OutputInterface $output, $duplicateActions)
{
$logActionTable = Common::prefixTable('log_action');
$output->writeln("<info>Deleting duplicate actions from $logActionTable...</info>");
$idActions = array();
foreach ($duplicateActions as $dupeInfo) {
$idActions = array_merge($idActions, $dupeInfo['duplicateIdActions']);
}
$this->actionsAccess->delete($idActions);
}
}

View File

@ -0,0 +1,200 @@
<?php
/**
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*/
namespace Piwik\Plugins\CoreAdminHome\Commands;
use Piwik\Container\StaticContainer;
use Piwik\Date;
use Piwik\Period;
use Piwik\Period\Range;
use Piwik\Piwik;
use Piwik\Segment;
use Piwik\Plugin\ConsoleCommand;
use Piwik\Plugins\SitesManager\API as SitesManagerAPI;
use Piwik\Site;
use Piwik\Period\Factory as PeriodFactory;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
/**
* Provides a simple interface for invalidating report data by date ranges, site IDs and periods.
*/
class InvalidateReportData extends ConsoleCommand
{
const ALL_OPTION_VALUE = 'all';
protected function configure()
{
$this->setName('core:invalidate-report-data');
$this->setDescription('Invalidate archived report data by date range, site and period.');
$this->addOption('dates', null, InputOption::VALUE_IS_ARRAY | InputOption::VALUE_REQUIRED,
'List of dates or date ranges to invalidate report data for, eg, 2015-01-03 or 2015-01-05,2015-02-12.');
$this->addOption('sites', null, InputOption::VALUE_REQUIRED,
'List of site IDs to invalidate report data for, eg, "1,2,3,4" or "all" for all sites.',
self::ALL_OPTION_VALUE);
$this->addOption('periods', null, InputOption::VALUE_REQUIRED,
'List of period types to invalidate report data for. Can be one or more of the following values: day, '
. 'week, month, year or "all" for all of them.',
self::ALL_OPTION_VALUE);
$this->addOption('segment', null, InputOption::VALUE_IS_ARRAY | InputOption::VALUE_REQUIRED,
'List of segments to invalidate report data for.');
$this->addOption('cascade', null, InputOption::VALUE_NONE,
'If supplied, invalidation will cascade, invalidating child period types even if they aren\'t specified in'
. ' --periods. For example, if --periods=week, --cascade will cause the days within those weeks to be '
. 'invalidated as well. If --periods=month, then weeks and days will be invalidated. Note: if a period '
. 'falls partly outside of a date range, then --cascade will also invalidate data for child periods '
. 'outside the date range. For example, if --dates=2015-09-14,2015-09-15 & --periods=week, --cascade will'
. ' also invalidate all days within 2015-09-13,2015-09-19, even those outside the date range.');
$this->addOption('dry-run', null, InputOption::VALUE_NONE, 'For tests. Runs the command w/o actually '
. 'invalidating anything.');
$this->setHelp('Invalidate archived report data by date range, site and period. Invalidated archive data will '
. 'be re-archived during the next core:archive run. If your log data has changed for some reason, this '
. 'command can be used to make sure reports are generated using the new, changed log data.');
}
protected function execute(InputInterface $input, OutputInterface $output)
{
$invalidator = StaticContainer::get('Piwik\Archive\ArchiveInvalidator');
$cascade = $input->getOption('cascade');
$dryRun = $input->getOption('dry-run');
$sites = $this->getSitesToInvalidateFor($input);
$periodTypes = $this->getPeriodTypesToInvalidateFor($input);
$dateRanges = $this->getDateRangesToInvalidateFor($input);
$segments = $this->getSegmentsToInvalidateFor($input, $sites);
foreach ($periodTypes as $periodType) {
foreach ($dateRanges as $dateRange) {
foreach ($segments as $segment) {
$segmentStr = $segment ? $segment->getString() : '';
$output->writeln("Invalidating $periodType periods in $dateRange [segment = $segmentStr]...");
$dates = $this->getPeriodDates($periodType, $dateRange);
if ($dryRun) {
$output->writeln("[Dry-run] invalidating archives for site = [ " . implode(', ', $sites)
. " ], dates = [ " . implode(', ', $dates) . " ], period = [ $periodType ], segment = [ "
. "$segmentStr ], cascade = [ " . (int)$cascade . " ]");
} else {
$invalidationResult = $invalidator->markArchivesAsInvalidated($sites, $dates, $periodType, $segment, $cascade);
if ($output->getVerbosity() > OutputInterface::VERBOSITY_NORMAL) {
$output->writeln($invalidationResult->makeOutputLogs());
}
}
}
}
}
}
private function getSitesToInvalidateFor(InputInterface $input)
{
$sites = $input->getOption('sites');
$siteIds = Site::getIdSitesFromIdSitesString($sites);
if (empty($siteIds)) {
throw new \InvalidArgumentException("Invalid --sites value: '$sites'.");
}
$allSiteIds = SitesManagerAPI::getInstance()->getAllSitesId();
foreach ($siteIds as $idSite) {
if (!in_array($idSite, $allSiteIds)) {
throw new \InvalidArgumentException("Invalid --sites value: '$sites', there are no sites with IDs = $idSite");
}
}
return $siteIds;
}
private function getPeriodTypesToInvalidateFor(InputInterface $input)
{
$periods = $input->getOption('periods');
if (empty($periods)) {
throw new \InvalidArgumentException("The --periods argument is required.");
}
if ($periods == self::ALL_OPTION_VALUE) {
$result = array_keys(Piwik::$idPeriods);
unset($result[4]); // remove 'range' period
return $result;
}
$periods = explode(',', $periods);
$periods = array_map('trim', $periods);
foreach ($periods as $periodIdentifier) {
if ($periodIdentifier == 'range') {
throw new \InvalidArgumentException(
"Invalid period type: invalidating range periods is not currently supported.");
}
if (!isset(Piwik::$idPeriods[$periodIdentifier])) {
throw new \InvalidArgumentException("Invalid period type '$periodIdentifier' supplied in --periods.");
}
}
return $periods;
}
/**
* @param InputInterface $input
* @return Date[][]
*/
private function getDateRangesToInvalidateFor(InputInterface $input)
{
$dateRanges = $input->getOption('dates');
if (empty($dateRanges)) {
throw new \InvalidArgumentException("The --dates option is required.");
}
return $dateRanges;
}
private function getPeriodDates($periodType, $dateRange)
{
if (!isset(Piwik::$idPeriods[$periodType])) {
throw new \InvalidArgumentException("Invalid period type '$periodType'.");
}
try {
$period = PeriodFactory::build($periodType, $dateRange);
} catch (\Exception $ex) {
throw new \InvalidArgumentException("Invalid date or date range specifier '$dateRange'", $code = 0, $ex);
}
$result = array();
if ($period instanceof Range) {
foreach ($period->getSubperiods() as $subperiod) {
$result[] = $subperiod->getDateStart();
}
} else {
$result[] = $period->getDateStart();
}
return $result;
}
private function getSegmentsToInvalidateFor(InputInterface $input, $idSites)
{
$segments = $input->getOption('segment');
$segments = array_map('trim', $segments);
$segments = array_unique($segments);
if (empty($segments)) {
return array(null);
}
$result = array();
foreach ($segments as $segmentString) {
$result[] = new Segment($segmentString, $idSites);
}
return $result;
}
}

View File

@ -0,0 +1,124 @@
<?php
/**
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*/
namespace Piwik\Plugins\CoreAdminHome\Commands;
use Piwik\Common;
use Piwik\DataAccess\ArchiveTableCreator;
use Piwik\Date;
use Piwik\Db;
use Piwik\Plugin\ConsoleCommand;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
/**
* Administration command that optimizes archive tables (even if they use InnoDB).
*/
class OptimizeArchiveTables extends ConsoleCommand
{
const ALL_TABLES_STRING = 'all';
const CURRENT_MONTH_STRING = 'now';
protected function configure()
{
$this->setName('database:optimize-archive-tables');
$this->setDescription("Runs an OPTIMIZE TABLE query on the specified archive tables.");
$this->addArgument("dates", InputArgument::IS_ARRAY | InputArgument::REQUIRED,
"The months of the archive tables to optimize. Use '" . self::ALL_TABLES_STRING. "' for all dates or '" .
self::CURRENT_MONTH_STRING . "' to optimize the current month only.");
$this->addOption('dry-run', null, InputOption::VALUE_NONE, 'For testing purposes.');
$this->setHelp("This command can be used to ease or automate maintenance. Instead of manually running "
. "OPTIMIZE TABLE queries, the command can be used.\n\nYou should run the command if you find your "
. "archive tables grow and do not shrink after purging. Optimizing them will reclaim some space.");
}
protected function execute(InputInterface $input, OutputInterface $output)
{
$dryRun = $input->getOption('dry-run');
$tableMonths = $this->getTableMonthsToOptimize($input);
foreach ($tableMonths as $month) {
$this->optimizeTable($output, $dryRun, 'archive_numeric_' . $month);
$this->optimizeTable($output, $dryRun, 'archive_blob_' . $month);
}
}
private function optimizeTable(OutputInterface $output, $dryRun, $table)
{
$output->write("Optimizing table '$table'...");
if ($dryRun) {
$output->write("[dry-run, not optimising table]");
} else {
Db::optimizeTables(Common::prefixTable($table), $force = true);
}
$output->writeln("Done.");
}
private function getTableMonthsToOptimize(InputInterface $input)
{
$dateSpecifiers = $input->getArgument('dates');
if (count($dateSpecifiers) === 1) {
$dateSpecifier = reset($dateSpecifiers);
if ($dateSpecifier == self::ALL_TABLES_STRING) {
return $this->getAllArchiveTableMonths();
} else if ($dateSpecifier == self::CURRENT_MONTH_STRING) {
$now = Date::factory('now');
return array(ArchiveTableCreator::getTableMonthFromDate($now));
} else if (strpos($dateSpecifier, 'last') === 0) {
$lastN = substr($dateSpecifier, 4);
if (!ctype_digit($lastN)) {
throw new \Exception("Invalid lastN specifier '$lastN'. The end must be an integer, eg, last1 or last2.");
}
if ($lastN <= 0) {
throw new \Exception("Invalid lastN value '$lastN'.");
}
return $this->getLastNTableMonths((int)$lastN);
}
}
$tableMonths = array();
foreach ($dateSpecifiers as $date) {
$date = Date::factory($date);
$tableMonths[] = ArchiveTableCreator::getTableMonthFromDate($date);
}
return $tableMonths;
}
private function getAllArchiveTableMonths()
{
$tableMonths = array();
foreach (ArchiveTableCreator::getTablesArchivesInstalled() as $table) {
$tableMonths[] = ArchiveTableCreator::getDateFromTableName($table);
}
return $tableMonths;
}
/**
* @param int $lastN
* @return string[]
*/
private function getLastNTableMonths($lastN)
{
$now = Date::factory('now');
$result = array();
for ($i = 0; $i < $lastN; ++$i) {
$date = $now->subMonth($i + 1);
$result[] = ArchiveTableCreator::getTableMonthFromDate($date);
}
return $result;
}
}

View File

@ -0,0 +1,212 @@
<?php
/**
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*
*/
namespace Piwik\Plugins\CoreAdminHome\Commands;
use Piwik\Archive;
use Piwik\Archive\ArchivePurger;
use Piwik\DataAccess\ArchiveTableCreator;
use Piwik\Date;
use Piwik\Db;
use Piwik\Plugin\ConsoleCommand;
use Piwik\Timer;
use Psr\Log\NullLogger;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
/**
* Command that allows users to force purge old or invalid archive data. In the event of a failure
* in the archive purging scheduled task, this command can be used to manually delete old/invalid archives.
*/
class PurgeOldArchiveData extends ConsoleCommand
{
const ALL_DATES_STRING = 'all';
/**
* For tests.
*
* @var Date
*/
public static $todayOverride = null;
/**
* @var ArchivePurger
*/
private $archivePurger;
public function __construct(ArchivePurger $archivePurger = null)
{
parent::__construct();
$this->archivePurger = $archivePurger;
}
protected function configure()
{
$this->setName('core:purge-old-archive-data');
$this->setDescription('Purges out of date and invalid archive data from archive tables.');
$this->addArgument("dates", InputArgument::IS_ARRAY | InputArgument::OPTIONAL,
"The months of the archive tables to purge data from. By default, only deletes from the current month. Use '" . self::ALL_DATES_STRING. "' for all dates.",
array(self::getToday()->toString()));
$this->addOption('exclude-outdated', null, InputOption::VALUE_NONE, "Do not purge outdated archive data.");
$this->addOption('exclude-invalidated', null, InputOption::VALUE_NONE, "Do not purge invalidated archive data.");
$this->addOption('exclude-ranges', null, InputOption::VALUE_NONE, "Do not purge custom ranges.");
$this->addOption('skip-optimize-tables', null, InputOption::VALUE_NONE, "Do not run OPTIMIZE TABLES query on affected archive tables.");
$this->addOption('include-year-archives', null, InputOption::VALUE_NONE, "If supplied, the command will purge archive tables that contain year archives for every supplied date.");
$this->addOption('force-optimize-tables', null, InputOption::VALUE_NONE, "If supplied, forces optimize table SQL to be run, even on InnoDB tables.");
$this->setHelp("By default old and invalidated archives are purged. Custom ranges are also purged with outdated archives.\n\n"
. "Note: archive purging is done during scheduled task execution, so under normal circumstances, you should not need to "
. "run this command manually.");
}
protected function execute(InputInterface $input, OutputInterface $output)
{
// during normal command execution, we don't want the INFO level logs logged by the ArchivePurger service
// to display in the console, so we use a NullLogger for the service
$logger = null;
if ($output->getVerbosity() <= OutputInterface::VERBOSITY_NORMAL) {
$logger = new NullLogger();
}
$archivePurger = $this->archivePurger ?: new ArchivePurger($model = null, $purgeDatesOlderThan = null, $logger);
$dates = $this->getDatesToPurgeFor($input);
$excludeOutdated = $input->getOption('exclude-outdated');
if ($excludeOutdated) {
$output->writeln("Skipping purge outdated archive data.");
} else {
foreach ($dates as $date) {
$message = sprintf("Purging outdated archives for %s...", $date->toString('Y_m'));
$this->performTimedPurging($output, $message, function () use ($date, $archivePurger) {
$archivePurger->purgeOutdatedArchives($date);
});
}
}
$excludeInvalidated = $input->getOption('exclude-invalidated');
if ($excludeInvalidated) {
$output->writeln("Skipping purge invalidated archive data.");
} else {
foreach ($dates as $date) {
$message = sprintf("Purging invalidated archives for %s...", $date->toString('Y_m'));
$this->performTimedPurging($output, $message, function () use ($archivePurger, $date) {
$archivePurger->purgeInvalidatedArchivesFrom($date);
});
}
}
$excludeCustomRanges = $input->getOption('exclude-ranges');
if ($excludeCustomRanges) {
$output->writeln("Skipping purge custom range archives.");
} else {
foreach ($dates as $date) {
$message = sprintf("Purging custom range archives for %s...", $date->toString('Y_m'));
$this->performTimedPurging($output, $message, function () use ($date, $archivePurger) {
$archivePurger->purgeArchivesWithPeriodRange($date);
});
}
}
$skipOptimizeTables = $input->getOption('skip-optimize-tables');
if ($skipOptimizeTables) {
$output->writeln("Skipping OPTIMIZE TABLES.");
} else {
$this->optimizeArchiveTables($output, $dates, $input->getOption('force-optimize-tables'));
}
}
/**
* @param InputInterface $input
* @return Date[]
*/
private function getDatesToPurgeFor(InputInterface $input)
{
$dates = array();
$dateSpecifier = $input->getArgument('dates');
if (count($dateSpecifier) === 1
&& reset($dateSpecifier) == self::ALL_DATES_STRING
) {
foreach (ArchiveTableCreator::getTablesArchivesInstalled() as $table) {
$tableDate = ArchiveTableCreator::getDateFromTableName($table);
list($year, $month) = explode('_', $tableDate);
try {
$date = Date::factory($year . '-' . $month . '-' . '01');
$dates[] = $date;
} catch (\Exception $e) {
// this might occur if archive tables like piwik_archive_numeric_1875_09 exist
}
}
} else {
$includeYearArchives = $input->getOption('include-year-archives');
foreach ($dateSpecifier as $date) {
$dateObj = Date::factory($date);
$yearMonth = $dateObj->toString('Y-m');
$dates[$yearMonth] = $dateObj;
// if --include-year-archives is supplied, add a date for the january table for this date's year
// so year archives will be purged
if ($includeYearArchives) {
$janYearMonth = $dateObj->toString('Y') . '-01';
if (empty($dates[$janYearMonth])) {
$dates[$janYearMonth] = Date::factory($janYearMonth . '-01');
}
}
}
$dates = array_values($dates);
}
return $dates;
}
private function performTimedPurging(OutputInterface $output, $startMessage, $callback)
{
$timer = new Timer();
$output->write($startMessage);
$callback();
$output->writeln("Done. <comment>[" . $timer->__toString() . "]</comment>");
}
/**
* @param OutputInterface $output
* @param Date[] $dates
* @param bool $forceOptimzation
*/
private function optimizeArchiveTables(OutputInterface $output, $dates, $forceOptimzation = false)
{
$output->writeln("Optimizing archive tables...");
foreach ($dates as $date) {
$numericTable = ArchiveTableCreator::getNumericTable($date);
$this->performTimedPurging($output, "Optimizing table $numericTable...", function () use ($numericTable, $forceOptimzation) {
Db::optimizeTables($numericTable, $forceOptimzation);
});
$blobTable = ArchiveTableCreator::getBlobTable($date);
$this->performTimedPurging($output, "Optimizing table $blobTable...", function () use ($blobTable, $forceOptimzation) {
Db::optimizeTables($blobTable, $forceOptimzation);
});
}
}
private static function getToday()
{
return self::$todayOverride ?: Date::today();
}
}

View File

@ -0,0 +1,79 @@
<?php
/**
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*
*/
namespace Piwik\Plugins\CoreAdminHome\Commands;
use Piwik\Container\StaticContainer;
use Piwik\FrontController;
use Piwik\Plugin\ConsoleCommand;
use Piwik\Scheduler\Scheduler;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
class RunScheduledTasks extends ConsoleCommand
{
protected function configure()
{
$this->setName('scheduled-tasks:run');
$this->setAliases(array('core:run-scheduled-tasks'));
$this->setDescription('Will run all scheduled tasks due to run at this time.');
$this->addArgument('task', InputArgument::OPTIONAL, 'Optionally pass the name of a task to run (will run even if not scheduled to run now)');
$this->addOption('force', null, InputOption::VALUE_NONE, 'If set, it will execute all tasks even the ones not due to run at this time.');
}
/**
* Execute command like: ./console core:run-scheduled-tasks
*/
protected function execute(InputInterface $input, OutputInterface $output)
{
$this->forceRunAllTasksIfRequested($input);
FrontController::getInstance()->init();
// TODO use dependency injection
/** @var Scheduler $scheduler */
$scheduler = StaticContainer::get('Piwik\Scheduler\Scheduler');
$task = $input->getArgument('task');
if ($task) {
$this->runSingleTask($scheduler, $task, $output);
} else {
$scheduler->run();
}
$this->writeSuccessMessage($output, array('Scheduled Tasks executed'));
}
private function forceRunAllTasksIfRequested(InputInterface $input)
{
$force = $input->getOption('force');
if ($force && !defined('DEBUG_FORCE_SCHEDULED_TASKS')) {
define('DEBUG_FORCE_SCHEDULED_TASKS', true);
}
}
private function runSingleTask(Scheduler $scheduler, $task, OutputInterface $output)
{
try {
$message = $scheduler->runTaskNow($task);
} catch (\InvalidArgumentException $e) {
$message = $e->getMessage() . PHP_EOL
. 'Available tasks:' . PHP_EOL
. implode(PHP_EOL, $scheduler->getTaskList());
throw new \Exception($message);
}
$output->writeln($message);
}
}

View File

@ -0,0 +1,100 @@
<?php
/**
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*/
namespace Piwik\Plugins\CoreAdminHome\Commands;
use Piwik\Config;
use Piwik\Plugin\ConsoleCommand;
use Piwik\Plugins\CoreAdminHome\Commands\SetConfig\ConfigSettingManipulation;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
class SetConfig extends ConsoleCommand
{
protected function configure()
{
$this->setName('config:set');
$this->setDescription('Set one or more config settings in the file config/config.ini.php');
$this->addArgument('assignment', InputArgument::OPTIONAL | InputArgument::IS_ARRAY,
"List of config setting assignments, eg, Section.key=1 or Section.array_key[]=value");
$this->addOption('section', null, InputOption::VALUE_REQUIRED, 'The section the INI config setting belongs to.');
$this->addOption('key', null, InputOption::VALUE_REQUIRED, 'The name of the INI config setting.');
$this->addOption('value', null, InputOption::VALUE_REQUIRED, 'The value of the setting. (Not JSON encoded)');
$this->setHelp("This command can be used to set INI config settings on a Piwik instance.
You can set config values two ways, via --section, --key, --value or by command arguments.
To use --section, --key, --value, simply supply those options. You can only set one setting this way, and you cannot
append to arrays.
To use arguments, supply one or more arguments in the following format:
$ ./console config:set 'Section.config_setting_name=\"value\"'
'Section' is the name of the section,
'config_setting_name' the name of the setting and
'value' is the value.
NOTE: 'value' must be JSON encoded, so 'Section.config_setting_name=\"value\"' would work but 'Section.config_setting_name=value' would not.
To append to an array setting, supply an argument like this:
$ ./console config:set 'Section.config_setting_name[]=\"value to append\"'
To reset an array setting, supply an argument like this:
$ ./console config:set 'Section.config_setting_name=[]'
Resetting an array will not work if the array has default values in global.ini.php (such as, [log] log_writers).
In this case the values in global.ini.php will be used, since there is no way to explicitly set an
array setting to empty in INI config.
");
}
protected function execute(InputInterface $input, OutputInterface $output)
{
$section = $input->getOption('section');
$key = $input->getOption('key');
$value = $input->getOption('value');
$manipulations = $this->getAssignments($input);
$isSingleAssignment = !empty($section) && !empty($key) && $value !== false;
if ($isSingleAssignment) {
$manipulations[] = new ConfigSettingManipulation($section, $key, $value);
}
if (empty($manipulations)) {
throw new \InvalidArgumentException("Nothing to assign. Add assignments as arguments or use the "
. "--section, --key and --value options.");
}
$config = Config::getInstance();
foreach ($manipulations as $manipulation) {
$manipulation->manipulate($config);
$output->write("<info>Setting [{$manipulation->getSectionName()}] {$manipulation->getName()} = {$manipulation->getValueString()}...</info>");
$output->writeln("<info> done.</info>");
}
$config->forceSave();
}
/**
* @return ConfigSettingManipulation[]
*/
private function getAssignments(InputInterface $input)
{
$assignments = $input->getArgument('assignment');
$result = array();
foreach ($assignments as $assignment) {
$result[] = ConfigSettingManipulation::make($assignment);
}
return $result;
}
}

View File

@ -0,0 +1,176 @@
<?php
/**
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*/
namespace Piwik\Plugins\CoreAdminHome\Commands\SetConfig;
use Piwik\Config;
/**
* Representation of a INI config manipulation operation. Only supports two types
* of manipulations: appending to a config array and assigning a config value.
*/
class ConfigSettingManipulation
{
/**
* @var string
*/
private $sectionName;
/**
* @var string
*/
private $name;
/**
* @var string
*/
private $value;
/**
* @var bool
*/
private $isArrayAppend;
/**
* @param string $sectionName
* @param string $name
* @param string $value
* @param bool $isArrayAppend
*/
public function __construct($sectionName, $name, $value, $isArrayAppend = false)
{
$this->sectionName = $sectionName;
$this->name = $name;
$this->value = $value;
$this->isArrayAppend = $isArrayAppend;
}
/**
* Performs the INI config manipulation.
*
* @param Config $config
* @throws \Exception if trying to append to a non-array setting value or if trying to set an
* array value to a non-array setting
*/
public function manipulate(Config $config)
{
if ($this->isArrayAppend) {
$this->appendToArraySetting($config);
} else {
$this->setSingleConfigValue($config);
}
}
private function setSingleConfigValue(Config $config)
{
$sectionName = $this->sectionName;
$section = $config->$sectionName;
if (isset($section[$this->name])
&& is_array($section[$this->name])
&& !is_array($this->value)
) {
throw new \Exception("Trying to set non-array value to array setting " . $this->getSettingString() . ".");
}
$section[$this->name] = $this->value;
$config->$sectionName = $section;
}
private function appendToArraySetting(Config $config)
{
$sectionName = $this->sectionName;
$section = $config->$sectionName;
if (isset($section[$this->name])
&& !is_array($section[$this->name])
) {
throw new \Exception("Trying to append to non-array setting value " . $this->getSettingString() . ".");
}
$section[$this->name][] = $this->value;
$config->$sectionName = $section;
}
/**
* Creates a ConfigSettingManipulation instance from a string like:
*
* `SectionName.setting_name=value`
*
* or
*
* `SectionName.setting_name[]=value`
*
* The value must be JSON so `="string"` will work but `=string` will not.
*
* @param string $assignment
* @return self
*/
public static function make($assignment)
{
if (!preg_match('/^([a-zA-Z0-9_]+)\.([a-zA-Z0-9_]+)(\[\])?=(.*)/', $assignment, $matches)) {
throw new \InvalidArgumentException("Invalid assignment string '$assignment': expected section.name=value or section.name[]=value");
}
$section = $matches[1];
$name = $matches[2];
$isAppend = !empty($matches[3]);
$value = json_decode($matches[4], $isAssoc = true);
if ($value === null) {
throw new \InvalidArgumentException("Invalid assignment string '$assignment': could not parse value as JSON");
}
return new self($section, $name, $value, $isAppend);
}
private function getSettingString()
{
return "[{$this->sectionName}] {$this->name}";
}
/**
* @return string
*/
public function getSectionName()
{
return $this->sectionName;
}
/**
* @return string
*/
public function getName()
{
return $this->name;
}
/**
* @return string
*/
public function getValue()
{
return $this->value;
}
/**
* @return boolean
*/
public function isArrayAppend()
{
return $this->isArrayAppend;
}
/**
* @return string
*/
public function getValueString()
{
return json_encode($this->value);
}
}