site-accueil-insa/matomo/core/CronArchive.php

<?php
/**
 * Matomo - free/libre analytics platform
 *
 * @link https://matomo.org
 * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
 *
 */
namespace Piwik;

use Exception;
use Piwik\ArchiveProcessor\Loader;
use Piwik\ArchiveProcessor\Parameters;
use Piwik\ArchiveProcessor\Rules;
use Piwik\CliMulti\Process;
use Piwik\Container\StaticContainer;
use Piwik\CronArchive\ArchiveFilter;
use Piwik\CronArchive\FixedSiteIds;
use Piwik\CronArchive\Performance\Logger;
use Piwik\Archive\ArchiveInvalidator;
use Piwik\CliMulti\RequestParser;
use Piwik\CronArchive\QueueConsumer;
use Piwik\CronArchive\SharedSiteIds;
use Piwik\CronArchive\StopArchiverException;
use Piwik\DataAccess\ArchiveSelector;
use Piwik\DataAccess\Model;
use Piwik\DataAccess\RawLogDao;
use Piwik\Exception\UnexpectedWebsiteFoundException;
use Piwik\Metrics\Formatter;
use Piwik\Period\Factory as PeriodFactory;
use Piwik\CronArchive\SegmentArchiving;
use Piwik\Period\Range;
use Piwik\Plugins\CoreAdminHome\API as CoreAdminHomeAPI;
use Piwik\Plugins\Monolog\Processor\ExceptionToTextProcessor;
use Piwik\Plugins\SitesManager\API as APISitesManager;
use Piwik\Plugins\UsersManager\API as APIUsersManager;
use Piwik\Plugins\UsersManager\UserPreferences;
use Psr\Log\LoggerInterface;

/**
 * ./console core:archive runs as a cron and is a useful tool for general maintenance,
 * and pre-process reports for a Fast dashboard rendering.
 */
class CronArchive
{
    // the url can be set here before the init, and it will be used instead of --url=
    const CRON_INVALIDATION_TIME_OPTION_NAME = 'CronArchive.lastInvalidationTime';

    public static $url = false;

    const TABLES_WITH_INVALIDATED_ARCHIVES = 'CronArchive.getTablesWithInvalidatedArchives';
    const TABLES_WITH_INVALIDATED_ARCHIVES_TTL = 3600;

    // Max parallel requests for a same site's segments
    const MAX_CONCURRENT_API_REQUESTS = 3;

    // force-timeout-for-periods default (1 hour)
    const SECONDS_DELAY_BETWEEN_PERIOD_ARCHIVES = 3600;

    // Flag to know when the archive cron is calling the API
    const APPEND_TO_API_REQUEST = '&trigger=archivephp';

    // Flag used to record timestamp in Option::
    const OPTION_ARCHIVING_FINISHED_TS = "LastCompletedFullArchiving";

    // Name of option used to store starting timestamp
    const OPTION_ARCHIVING_STARTED_TS = "LastFullArchivingStartTime";

    // Show only first N characters from Piwik API output in case of errors
    const TRUNCATE_ERROR_MESSAGE_SUMMARY = 6000;

    // By default, we only process the current week/month/year at most once an hour
    private $todayArchiveTimeToLive;

    private $allWebsites = array();

    /**
     * @var FixedSiteIds|SharedSiteIds
     */
    private $websiteIdArchiveList;
    private $requests = 0;
    private $archiveAndRespectTTL = true;
    public $shouldArchiveAllSites = false;

    private $idSitesNotUsingTracker = [];

    /**
     * @var Model
     */
    private $model;

    private $lastSuccessRunTimestamp = false;
    private $errors = array();

    private $apiToInvalidateArchivedReport;

    const NO_ERROR = "no error";

    public $testmode = false;

    /**
     * The list of IDs for sites for whom archiving should be initiated. If supplied, only these
     * sites will be archived.
     *
     * @var int[]
     */
    public $shouldArchiveSpecifiedSites = array();

    public $shouldSkipSpecifiedSites = [];

    /**
     * If true, xhprof will be initiated for the archiving run. Only for development/testing.
     *
     * @var bool
     */
    public $shouldStartProfiler = false;

    /**
     * Given options will be forwarded to the PHP command if the archiver is executed via CLI.
     * @var string
     */
    public $phpCliConfigurationOptions = '';

    /**
     * If HTTP requests are used to initiate archiving, this controls whether invalid SSL certificates should
     * be accepted or not by each request.
     *
     * @var bool
     */
    public $acceptInvalidSSLCertificate = false;

    /**
     * If set to true, scheduled tasks will not be run.
     *
     * @var bool
     */
    public $disableScheduledTasks = false;

    /**
     * Forces CronArchive to invalidate data for the last [$dateLastForced] years when it notices a segment that
     * was recently created or updated. By default this is 7.
     *
     * @var int|false
     */
    public $dateLastForced = SegmentArchiving::DEFAULT_BEGINNING_OF_TIME_LAST_N_YEARS;

    /**
     * The number of concurrent requests to issue per website. Defaults to {@link MAX_CONCURRENT_API_REQUESTS}.
     *
     * Used when archiving a site's segments concurrently.
     *
     * @var int|false
     */
    public $concurrentRequestsPerWebsite = false;

    /**
     * The number of concurrent archivers to run at once max.
     *
     * @var int|false
     */
    public $maxConcurrentArchivers = false;

    /**
     * Maximum number of sites to process during a single execution of the archiver.
     *
     * @var int|null
     */
    public $maxSitesToProcess = null;

    /**
     * Maximum number of archives to process during a single execution of the archiver.
     *
     * Note that this is not a hard limit as the limit is only checked after all
     * archives for a site have been processed.
     *
     * @var int|null
     */
    public $maxArchivesToProcess = null;

    private $archivingStartingTime;

    private $formatter;

    private $lastDbReset = false;

    /**
     * @var SegmentArchiving
     */
    private $segmentArchiving;

    /**
     * @var LoggerInterface
     */
    private $logger;

    /**
     * Only used when archiving using HTTP requests.
     *
     * @var string
     */
    private $urlToPiwik = null;

    /**
     * @var ArchiveInvalidator
     */
    private $invalidator;

    /**
     * @var bool
     */
    private $isArchiveProfilingEnabled = false;

    /**
     * @var array
     */
    private $periodIdsToLabels;

    /**
     * @var ArchiveFilter
     */
    private $archiveFilter;

    /**
     * @var RequestParser
     */
    private $cliMultiRequestParser;

    /**
     * @var bool|mixed
     */
    private $supportsAsync;

    /**
     * Constructor.
     *
     * @param LoggerInterface|null $logger
     */
    public function __construct(LoggerInterface $logger = null)
    {
        $this->logger = $logger ?: StaticContainer::get('Psr\Log\LoggerInterface');
        $this->formatter = new Formatter();

        $this->invalidator = StaticContainer::get('Piwik\Archive\ArchiveInvalidator');

        $this->isArchiveProfilingEnabled = Config::getInstance()->Debug['archiving_profile'] == 1;

        $this->model = StaticContainer::get(Model::class);

        $this->periodIdsToLabels = array_flip(Piwik::$idPeriods);

        $this->rawLogDao = new RawLogDao();

        $this->supportsAsync = $this->makeCliMulti()->supportsAsync();
        $this->cliMultiRequestParser = new RequestParser($this->supportsAsync);

        $this->archiveFilter = new ArchiveFilter();
    }

    private function isMaintenanceModeEnabled()
    {
        return Config::getInstance()->General['maintenance_mode'] == 1;
    }

    /**
     * Initializes and runs the cron archiver.
     */
    public function main()
    {
        if ($this->isMaintenanceModeEnabled()) {
            $this->logger->info("Archiving won't run because maintenance mode is enabled");
            return;
        }

        $self = $this;
        Access::doAsSuperUser(function () use ($self) {
            try {
                $self->init();
                $self->run();
                $self->runScheduledTasks();
                $self->end();
            } catch (StopArchiverException $e) {
                $this->logger->info("Archiving stopped by stop archiver exception");
            }
        });
    }

    public function init()
    {
        $this->segmentArchiving = StaticContainer::get(SegmentArchiving::class);

        /**
         * This event is triggered during initializing archiving.
         *
         * @param CronArchive $this
         */
        Piwik::postEvent('CronArchive.init.start', array($this));

        SettingsServer::setMaxExecutionTime(0);

        $this->archivingStartingTime = time();

        // Note: the order of methods call matters here.
        $this->initStateFromParameters();

        $this->logInitInfo();
        $this->logArchiveTimeoutInfo();

        $idSitesNotUsingTracker = Loader::getSitesNotUsingTracker();
        if (!empty($idSitesNotUsingTracker)) {
            $this->logger->info("- The following websites do not use the tracker: " . implode(',', $this->idSitesNotUsingTracker));
        }

        // record archiving start time
        Option::set(self::OPTION_ARCHIVING_STARTED_TS, time());

        $allWebsites = APISitesManager::getInstance()->getAllSitesId();
        $websitesIds = $this->initWebsiteIds($allWebsites);
        $this->filterWebsiteIds($websitesIds, $allWebsites);
        $this->allWebsites = $websitesIds;
        $this->websiteIdArchiveList = $this->makeWebsiteIdArchiveList($websitesIds);

        if (method_exists($this->websiteIdArchiveList, 'isContinuingPreviousRun') &&
            $this->websiteIdArchiveList->isContinuingPreviousRun()
        ) {
            $this->logger->info("- Continuing ongoing archiving run by pulling from shared idSite queue.");
        }

        if ($this->archiveFilter) {
            $this->archiveFilter->logFilterInfo($this->logger);
        }

        /**
         * This event is triggered after a CronArchive instance is initialized.
         *
         * @param array $websiteIds The list of website IDs this CronArchive instance is processing.
         *                          This will be the entire list of IDs regardless of whether some have
         *                          already been processed.
         */
        Piwik::postEvent('CronArchive.init.finish', [$this->allWebsites]);
    }

    /**
     * Main function, runs archiving on all websites with new activity
     */
    public function run()
    {
        $pid = Common::getProcessId();

        $timer = new Timer;

        $this->logSection("START");
        $this->logger->info("Starting Matomo reports archiving...");

        $numArchivesFinished = 0;

        if ($this->hasReachedMaxConcurrentArchivers()) {
            $this->logger->info("Reached maximum concurrent archivers allowed ({$this->maxConcurrentArchivers}), aborting run.");
            return;
        }

        $this->logger->debug("Applying queued rearchiving...");
        \Piwik\Tracker\Cache::withDelegatedCacheClears(function () {
            $this->invalidator->applyScheduledReArchiving();
        });

        $failedJobs = $this->model->resetFailedArchivingJobs();
        if ($failedJobs) {
            $this->logger->info("Found {failed} failed jobs (ts_invalidated older than 1 day), resetting status to try them again.", [
                'failed' => $failedJobs,
            ]);
        }

        $countOfProcesses = $this->getMaxConcurrentApiRequests();

        $queueConsumer = new QueueConsumer($this->logger, $this->websiteIdArchiveList, $countOfProcesses, $pid,
            $this->model, $this->segmentArchiving, $this, $this->cliMultiRequestParser, $this->archiveFilter);

        $queueConsumer->setMaxSitesToProcess($this->maxSitesToProcess);

        while (true) {
            if ($this->isMaintenanceModeEnabled()) {
                $this->logger->info("Archiving will stop now because maintenance mode is enabled");
                return;
            }

            if (!Process::isMethodDisabled('getmypid') && !Process::isMethodDisabled('ignore_user_abort')) {
                // see https://github.com/matomo-org/wp-matomo/issues/163
                flush();
            }

            try {
                $archivesToProcess = $queueConsumer->getNextArchivesToProcess();
            } catch (UnexpectedWebsiteFoundException $ex) {
                $this->logger->debug("Site {$queueConsumer->getIdSite()} was deleted, skipping to next...");
                $queueConsumer->skipToNextSite();
                continue;
            }

            if ($archivesToProcess === null) {
                break;
            }

            if (empty($archivesToProcess)) {
                continue;
            }

            $successCount = $this->launchArchivingFor($archivesToProcess, $queueConsumer);
            $numArchivesFinished += $successCount;
            if ($this->maxArchivesToProcess && $numArchivesFinished >= $this->maxArchivesToProcess) {
                $this->logger->info("Maximum number of archives to process per execution has been reached.");
                break;
            }
        }

        $this->disconnectDb();

        $this->logger->info("Done archiving!");

        $this->logSection("SUMMARY");
        $this->logger->info("Processed $numArchivesFinished archives.");
        $this->logger->info("Total API requests: {$this->requests}");

        $this->logger->info("done: " .
            $this->requests . " req, " . round($timer->getTimeMs()) . " ms, " .
            (empty($this->errors)
                ? self::NO_ERROR
                : (count($this->errors) . " errors."))
        );

        $this->logger->info($timer->__toString());
    }

    private function launchArchivingFor($archives, QueueConsumer $queueConsumer)
    {
        $urls = [];
        $archivesBeingQueried = [];
        foreach ($archives as $index => $archive) {
            list($url, $segment, $plugin) = $this->generateUrlToArchiveFromArchiveInfo($archive);
            if (empty($url)) {
                // can happen if, for example, a segment was deleted after an archive was invalidated
                // in this case, we can just delete the archive entirely.
                $this->deleteInvalidatedArchives($archive);
                continue;
            }

            $idSite = $archive['idsite'];
            if (!$this->siteExists($idSite)) {
                $this->logger->debug("Site $idSite no longer exists, no longer launching archiving.");
                $this->deleteInvalidatedArchives($archive);
                continue;
            }

            $dateStr = $archive['period'] == Range::PERIOD_ID ? ($archive['date1'] . ',' . $archive['date2']) : $archive['date1'];
            $period = PeriodFactory::build($this->periodIdsToLabels[$archive['period']], $dateStr);
            $params = new Parameters(new Site($idSite), $period, new Segment($segment, [$idSite], $period->getDateStart(), $period->getDateEnd()));

            if (!empty($plugin)) {
                $params->setRequestedPlugin($plugin);
                $params->onlyArchiveRequestedPlugin();
            }

            $loader = new Loader($params);
            if ($loader->canSkipThisArchive()) {
                $this->logger->info("Found no visits for site ID = {idSite}, {period} ({date1},{date2}), site is using the tracker so skipping archiving...", [
                    'idSite' => $idSite,
                    'period' => $this->periodIdsToLabels[$archive['period']],
                    'date1' => $archive['date1'],
                    'date2' => $archive['date2'],
                ]);

                // site is using the tracker, but there are no visits for this period, so just delete the archive and move on
                $this->deleteInvalidatedArchives($archive);
                continue;
            }

            $this->logger->debug("Starting archiving for {url}", ['url' => $url]);

            $urls[] = $url;
            $archivesBeingQueried[] = $archive;
        }

        if (empty($urls)) {
            return 0; // all URLs had no visits and were using the tracker
        }

        $cliMulti = $this->makeCliMulti();
        $cliMulti->timeRequests();

        $responses = $cliMulti->request($urls);

        $this->disconnectDb();

        $timers = $cliMulti->getTimers();
        $successCount = 0;

        foreach ($urls as $index => $url) {
            $content = array_key_exists($index, $responses) ? $responses[$index] : null;
            $checkInvalid = $this->checkResponse($content, $url);

            $stats = json_decode($content, $assoc = true);
            if (!is_array($stats)) {
                $this->logger->info(var_export($content, true));

                $idinvalidation = $archivesBeingQueried[$index]['idinvalidation'];
                $this->model->releaseInProgressInvalidation($idinvalidation);

                $queueConsumer->ignoreIdInvalidation($idinvalidation);

                $this->logError("Error unserializing the following response from $url: '" . $content . "'");
                continue;
            }

            $visitsForPeriod = $this->getVisitsFromApiResponse($stats);


            $this->logArchiveJobFinished($url, $timers[$index], $visitsForPeriod,
              $archivesBeingQueried[$index]['plugin'], $archivesBeingQueried[$index]['report'], !$checkInvalid);


            $this->deleteInvalidatedArchives($archivesBeingQueried[$index]);

            $this->repairInvalidationsIfNeeded($archivesBeingQueried[$index]);

            ++$successCount;
        }

        $this->requests += count($urls);

        return $successCount;
    }

    private function deleteInvalidatedArchives($archive)
    {
        $this->model->deleteInvalidations([$archive]);
    }

    private function generateUrlToArchiveFromArchiveInfo($archive)
    {
        $plugin = $archive['plugin'];
        $report = $archive['report'];
        $period = $this->periodIdsToLabels[$archive['period']];

        if ($period == 'range') {
            $date = $archive['date1'] . ',' . $archive['date2'];
        } else {
            $date = $archive['date1'];
        }

        $idSite = $archive['idsite'];

        $segment = isset($archive['segment']) ? $archive['segment'] : '';

        $url = $this->getVisitsRequestUrl($idSite, $period, $date, $segment, $plugin);
        $url = $this->makeRequestUrl($url);

        if (!empty($segment)) {
            $shouldSkipToday = $this->archiveFilter->isSkipSegmentsForToday() && !$this->wasSegmentChangedRecently($segment,
                $this->segmentArchiving->getAllSegments());

            if ($shouldSkipToday) {
                $url .= '&skipArchiveSegmentToday=1';
            }
        }

        if (!empty($plugin)) {
            $url .= "&pluginOnly=1";
        }

        if (!empty($report)) {
            $url .= "&requestedReport=" . urlencode($report);
        }

        return [$url, $segment, $plugin];
    }

    private function logArchiveJobFinished($url, $timer, $visits, $plugin = null, $report = null, $wasSkipped = null)
    {
        $params = UrlHelper::getArrayFromQueryString($url);
        $visits = (int) $visits;

        $message = $wasSkipped ? "Skipped Archiving website" : "Archived website";

        $this->logger->info($message." id {$params['idSite']}, period = {$params['period']}, date = "
            . "{$params['date']}, segment = '" . (isset($params['segment']) ? urldecode(urldecode($params['segment'])) : '') . "', "
            . ($plugin ? "plugin = $plugin, " : "") . ($report ? "report = $report, " : "") . "$visits visits found. $timer");
    }

    public function getErrors()
    {
        return $this->errors;
    }

    /**
     * End of the script
     */
    public function end()
    {
        /**
         * This event is triggered after archiving.
         *
         * @param CronArchive $this
         */
        Piwik::postEvent('CronArchive.end', array($this));

        if (empty($this->errors)) {
            // No error -> Logs the successful script execution until completion
            Option::set(self::OPTION_ARCHIVING_FINISHED_TS, time());
            return;
        }

        $this->logSection("SUMMARY OF ERRORS");
        foreach ($this->errors as $error) {
            // do not logError since errors are already in stderr
            $this->logger->info("Error: " . $error);
        }

        $summary = count($this->errors) . " total errors during this script execution, please investigate and try and fix these errors.";
        $this->logFatalError($summary);
    }

    public function logFatalError($m)
    {
        $this->logError($m);

        throw new Exception($m);
    }

    public function runScheduledTasks()
    {
        $this->logSection("SCHEDULED TASKS");

        if ($this->disableScheduledTasks) {
            $this->logger->info("Scheduled tasks are disabled with --disable-scheduled-tasks");
            return;
        }

        $this->disconnectDb();

        Piwik::addAction('ScheduledTasks.execute.end', function () {
            // check if we need to reconnect after each task executes
            $this->disconnectDb();
        });

        // TODO: this is a HACK to get the purgeOutdatedArchives task to work when run below. without
        //       it, the task will not run because we no longer run the tasks through CliMulti.
        //       harder to implement alternatives include:
        //       - moving CronArchive logic to DI and setting a flag in the class when the whole process
        //         runs
        //       - setting a new DI environment for core:archive which CoreAdminHome can use to conditionally
        //         enable/disable the task
        Rules::$disablePureOutdatedArchive = true;

        CoreAdminHomeAPI::getInstance()->runScheduledTasks();

        $this->logSection("");
    }

    private function disconnectDb()
    {
        $twoHoursInSeconds = 60 * 60 * 2;

        if (time() > ($this->lastDbReset + $twoHoursInSeconds)) {
            // we aim to through DB connections away only after 2 hours
            $this->lastDbReset = time();
            Db::destroyDatabaseObject();
            Tracker::disconnectCachedDbConnection();
        }
    }

    /**
     * Returns base URL to process reports for the $idSite on a given $period
     *
     * @param string $idSite
     * @param string $period
     * @param string $date
     * @param bool|false $segment
     * @return string
     */
    private function getVisitsRequestUrl($idSite, $period, $date, $segment = false, $plugin = null)
    {
        $request = "?module=API&method=CoreAdminHome.archiveReports&idSite=$idSite&period=$period&date=" . $date . "&format=json";
        if ($segment) {
            $request .= '&segment=' . urlencode($segment);
        }
        if (!empty($plugin)) {
            $request .= "&plugin=" . $plugin;
        }
        return $request;
    }

    /**
     * Logs a section in the output
     *
     * @param string $title
     */
    private function logSection($title = "")
    {
        $this->logger->info("---------------------------");
        if (!empty($title)) {
            $this->logger->info($title);
        }
    }

    public function logError($m)
    {
        if (!defined('PIWIK_ARCHIVE_NO_TRUNCATE')) {
            $m = str_replace(array("\n", "\t"), " ", $m);
            if (mb_strlen($m) > self::TRUNCATE_ERROR_MESSAGE_SUMMARY) {
                $numCharactersKeepFromEnd = 100;
                $m = mb_substr($m, 0, self::TRUNCATE_ERROR_MESSAGE_SUMMARY - $numCharactersKeepFromEnd)
                     . ' ... ' .
                    mb_substr($m, -1 * $numCharactersKeepFromEnd);
            }
        }
        $this->errors[] = $m;
        $this->logger->error($m);
    }

    private function logNetworkError($url, $response)
    {

        if (preg_match("/Segment (.*?) is not a supported segment/i", $response, $match)) {
            $this->logger->info($match[0]);
            return false;
        }

        $message = "Got invalid response from API request: $url. ";
        if (empty($response)) {
            $message .= "The response was empty. This usually means a server error. A solution to this error is generally to increase the value of 'memory_limit' in your php.ini file. ";

            if($this->supportsAsync) {
                $message .= " For more information and the error message please check in your PHP CLI error log file. As this core:archive command triggers PHP processes over the CLI, you can find where PHP CLI logs are stored by running this command: php -i | grep error_log";
            } else {
                $message .= " For more information and the error message please check your web server's error Log file. As this core:archive command triggers PHP processes over HTTP, you can find the error message in your Matomo's web server error logs. ";
            }
        } else {
            $message .= "Response was '$response'";
        }

        $this->logError($message);
        return false;
    }

    private function checkResponse($response, $url)
    {
        if (empty($response)
            || stripos($response, 'error') !== false
        ) {
            return $this->logNetworkError($url, $response);
        }
        return true;
    }

    /**
     * Initializes the various parameters to the script, based on input parameters.
     *
     */
    private function initStateFromParameters()
    {
        $this->todayArchiveTimeToLive = Rules::getTodayArchiveTimeToLive();
        $this->lastSuccessRunTimestamp = $this->getLastSuccessRunTimestamp();
    }

    public function filterWebsiteIds(&$websiteIds, $allWebsites)
    {
        // Keep only the websites that do exist
        $websiteIds = array_intersect($websiteIds, $allWebsites);

        if (!empty($this->shouldSkipSpecifiedSites)) {
            $websiteIds = array_diff($websiteIds, $this->shouldSkipSpecifiedSites);
        }

        /**
         * Triggered by the **core:archive** console command so plugins can modify the priority of
         * websites that the archiving process will be launched for.
         *
         * Plugins can use this hook to add websites to archive, remove websites to archive, or change
         * the order in which websites will be archived.
         *
         * @param array $websiteIds The list of website IDs to launch the archiving process for.
         */
        Piwik::postEvent('CronArchive.filterWebsiteIds', array(&$websiteIds));
    }

    /**
     * @internal
     * @param $api
     */
    public function setApiToInvalidateArchivedReport($api)
    {
        $this->apiToInvalidateArchivedReport = $api;
    }

    private function getApiToInvalidateArchivedReport()
    {
        if ($this->apiToInvalidateArchivedReport) {
            return $this->apiToInvalidateArchivedReport;
        }

        return CoreAdminHomeAPI::getInstance();
    }

    public function invalidateArchivedReportsForSitesThatNeedToBeArchivedAgain($idSiteToInvalidate)
    {
        \Piwik\Tracker\Cache::withDelegatedCacheClears(function () use ($idSiteToInvalidate) {
            $this->invalidateArchivedReportsForSitesThatNeedToBeArchivedAgainImpl($idSiteToInvalidate);
        });
    }

    private function invalidateArchivedReportsForSitesThatNeedToBeArchivedAgainImpl($idSiteToInvalidate)
    {
        if (empty($this->segmentArchiving)) {
            // might not be initialised if init is not called
            $this->segmentArchiving = StaticContainer::get(SegmentArchiving::class);
        }

        $this->logger->debug("Checking for queued invalidations...");

        // invalidate remembered site/day pairs
        $sitesPerDays = $this->invalidator->getRememberedArchivedReportsThatShouldBeInvalidated();
        krsort($sitesPerDays); // for tests

        foreach ($sitesPerDays as $date => $siteIds) {
            //Concurrent transaction logic will end up with duplicates set.  Adding array_unique to the siteIds.
            $siteIds = array_unique($siteIds);

            $siteIdsToInvalidate = [];
            foreach ($siteIds as $idSite) {
                if ($idSite != $idSiteToInvalidate) {
                    continue;
                }

                $siteIdsToInvalidate[] = $idSite;
            }

            if (empty($siteIdsToInvalidate)) {
                continue;
            }

            $listSiteIds = implode(',', $siteIdsToInvalidate);

            try {
                $this->logger->debug('  Will invalidate archived reports for ' . $date . ' for following websites ids: ' . $listSiteIds);
                $this->invalidateWithSegments($siteIdsToInvalidate, $date, $period = 'day');
            } catch (Exception $e) {
                $message = ExceptionToTextProcessor::getMessageAndWholeBacktrace($e);
                $this->logger->info('  Failed to invalidate archived reports: ' . $message);
            }
        }

        // invalidate today if needed for all websites
        $this->invalidateRecentDate('today', $idSiteToInvalidate);

        // invalidate yesterday archive if the time of the latest valid archive is earlier than today
        // (means the day has changed and there might be more visits that weren't processed)
        $this->invalidateRecentDate('yesterday', $idSiteToInvalidate);

        // invalidate range archives
        $dates = $this->getCustomDateRangeToPreProcess($idSiteToInvalidate);

        foreach ($dates as $date) {
            try {
                PeriodFactory::build('range', $date);
            } catch (\Exception $ex) {
                $this->logger->debug("  Found invalid range date in [General] archiving_custom_ranges: {date}", ['date' => $date]);
                continue;
            }

            $this->logger->debug('  Invalidating custom date range ({date}) for site {idSite}', ['idSite' => $idSiteToInvalidate, 'date' => $date]);

            $this->invalidateWithSegments($idSiteToInvalidate, $date, 'range', $_forceInvalidateNonexistent = true);
        }

        $this->setInvalidationTime();

        $this->logger->debug("Done invalidating");
    }

    public function invalidateRecentDate($dateStr, $idSite)
    {
        $timezone = Site::getTimezoneFor($idSite);
        $date = Date::factoryInTimezone($dateStr, $timezone);
        $period = PeriodFactory::build('day', $date);

        $params = new Parameters(new Site($idSite), $period, new Segment('', [$idSite], $period->getDateStart(), $period->getDateEnd()));

        $loader = new Loader($params);
        if ($loader->canSkipThisArchive()) {
            $this->logger->debug("  " . ucfirst($dateStr) . " archive can be skipped due to no visits for idSite = $idSite, skipping invalidation...");
            return;
        }

        $this->logger->info("  Will invalidate archived reports for $dateStr in site ID = {idSite}'s timezone ({date}).", [
            'idSite' => $idSite,
            'date' => $date->getDatetime(),
        ]);

        // if we are invalidating yesterday here, we are only interested in checking if there is no archive for yesterday, or the day has changed since
        // the last archive was archived (in which there may have been more visits before midnight). so we disable the ttl check, since any archive
        // will be good enough, if the date hasn't changed.
        $isYesterday = $dateStr == 'yesterday';
        $this->invalidateWithSegments([$idSite], $date->toString(), 'day', false, $doNotIncludeTtlInExistingArchiveCheck = $isYesterday);
    }

    private function invalidateWithSegments($idSites, $date, $period, $_forceInvalidateNonexistent = false, $doNotIncludeTtlInExistingArchiveCheck = false)
    {
        if ($date instanceof Date) {
            $date = $date->toString();
        }

        $periodObj = PeriodFactory::build($period, $date);

        if ($period == 'range') {
            $date = [$date]; // so we don't split on the ',' in invalidateArchivedReports
        }

        if (!is_array($idSites)) {
            $idSites = [$idSites];
        }

        foreach ($idSites as $idSite) {
            $params = new Parameters(new Site($idSite), $periodObj, new Segment('', [$idSite], $periodObj->getDateStart(), $periodObj->getDateEnd()));
            if ($this->canWeSkipInvalidatingBecauseThereIsAUsablePeriod($params, $doNotIncludeTtlInExistingArchiveCheck)) {
                $this->logger->debug('  Found usable archive for {archive}, skipping invalidation.', ['archive' => $params]);
            } else {
                $this->getApiToInvalidateArchivedReport()->invalidateArchivedReports($idSite, $date, $period, $segment = false, $cascadeDown = false,
                    $_forceInvalidateNonexistent);
            }

            foreach ($this->segmentArchiving->getAllSegmentsToArchive($idSite) as $segmentDefinition) {

               // check if the segment is available
                if (!$this->isSegmentAvailable($segmentDefinition, [$idSite])) {
                    continue;
                }
                $params = new Parameters(new Site($idSite), $periodObj, new Segment($segmentDefinition, [$idSite], $periodObj->getDateStart(), $periodObj->getDateEnd()));
                if ($this->canWeSkipInvalidatingBecauseThereIsAUsablePeriod($params, $doNotIncludeTtlInExistingArchiveCheck)) {
                    $this->logger->debug('  Found usable archive for {archive}, skipping invalidation.', ['archive' => $params]);
                } else {
                    if (empty($this->segmentArchiving)) {
                        // might not be initialised if init is not called
                        $this->segmentArchiving = StaticContainer::get(SegmentArchiving::class);
                    }

                    $segmentInfo = $this->segmentArchiving->findSegmentForHash($params->getSegment()->getHash(), $idSite);

                    if ($segmentInfo) {
                        $segmentArchiveStartDate = $this->segmentArchiving->getReArchiveSegmentStartDate($segmentInfo);

                        if ($segmentArchiveStartDate !== null && $segmentArchiveStartDate->isLater($params->getPeriod()->getDateEnd()->getEndOfDay())) {
                            // the system is not allowed to invalidate reports for this period
                            // automatically, only a user can specifically invalidate
                            continue;
                        }
                    }

                    $this->getApiToInvalidateArchivedReport()->invalidateArchivedReports($idSite, $date, $period, $segmentDefinition,
                        $cascadeDown = false, $_forceInvalidateNonexistent);
                }
            }
        }
    }


    /**
     * check if segments that contain dimensions that don't exist anymore
     * @param $segmentDefinition
     * @param $idSites
     * @return bool
     */
    protected function isSegmentAvailable($segmentDefinition, $idSites)
    {
        try {
            new Segment($segmentDefinition, $idSites);
        } catch (\Exception $e) {
            $this->logger->info("Segment '".$segmentDefinition."' is not a supported segment");
            return false;
        }
        return true;
    }

    /**
     * Returns true if there is an existing valid period we can use, or false if there isn't and the invalidation should go through.
     *
     * Note: this method should only be used in the context of invalidation.
     *
     * @params Parameters $params The parameters for the archive we want to invalidate.
     */
    public function canWeSkipInvalidatingBecauseThereIsAUsablePeriod(Parameters $params, $doNotIncludeTtlInExistingArchiveCheck = false)
    {
        $today = Date::factoryInTimezone('today', Site::getTimezoneFor($params->getSite()->getId()));

        $isYesterday = $params->getPeriod()->getLabel() == 'day' && $params->getPeriod()->getDateStart()->toString() == Date::factory('yesterday')->toString();

        $isPeriodIncludesToday = $params->getPeriod()->isDateInPeriod($today);

        $minArchiveProcessedTime = $doNotIncludeTtlInExistingArchiveCheck ? null :
            Date::now()->subSeconds(Rules::getPeriodArchiveTimeToLiveDefault($params->getPeriod()->getLabel()));

        // empty plugins param since we only check for an 'all' archive
        list($idArchive, $visits, $visitsConverted, $ignore, $tsArchived) = ArchiveSelector::getArchiveIdAndVisits($params, $minArchiveProcessedTime, $includeInvalidated = $isPeriodIncludesToday);

        // day has changed since the archive was created, we need to reprocess it
        if ($isYesterday
            && !empty($idArchive)
            && Date::factory($tsArchived)->toString() != $today->toString()
        ) {
            return false;
        }

        return !empty($idArchive);
    }

    // public for tests
    public function repairInvalidationsIfNeeded($archiveToProcess)
    {
        $table = Common::prefixTable('archive_invalidations');

        $bind = [
            $archiveToProcess['idsite'],
            $archiveToProcess['name'],
            $archiveToProcess['period'],
            $archiveToProcess['date1'],
            $archiveToProcess['date2'],
        ];

        $reportClause = '';
        if (!empty($archiveToProcess['report'])) {
            $reportClause = " AND report = ?";
            $bind[] = $archiveToProcess['report'];
        }

        $sql = "SELECT DISTINCT period FROM `$table`
                 WHERE idsite = ? AND name = ? AND period > ? AND ? >= date1 AND date2 >= ? AND status = " . ArchiveInvalidator::INVALIDATION_STATUS_QUEUED . " $reportClause";

        $higherPeriods = Db::fetchAll($sql, $bind);
        $higherPeriods = array_column($higherPeriods, 'period');

        $invalidationsToInsert = [];
        foreach (Piwik::$idPeriods as $label => $id) {
            // lower period than the one we're processing or range, don't care
            if ($id <= $archiveToProcess['period'] || $label == 'range') {
                continue;
            }

            if (in_array($id, $higherPeriods)) { // period exists in table
                continue;
            }

            // period is disabled in API
            if (!PeriodFactory::isPeriodEnabledForAPI($label)
                || PeriodFactory::isAnyLowerPeriodDisabledForAPI($label)
            ) {
                continue;
            }

            // archive is for a week that is over two months, we don't need to care about the month
            if ($label == 'month'
                && Date::factory($archiveToProcess['date1'])->toString('m') != Date::factory($archiveToProcess['date2'])->toString('m')
            ) {
                continue;
            }

            // archive is for a week that is over two years, we don't need to care about the year
            if ($label == 'year'
                && Date::factory($archiveToProcess['date1'])->toString('y') != Date::factory($archiveToProcess['date2'])->toString('y')
            ) {
                continue;
            }

            $period = Period\Factory::build($label, $archiveToProcess['date1']);

            $invalidationToInsert = [
                'idarchive' => null,
                'name' => $archiveToProcess['name'],
                'report' => $archiveToProcess['report'],
                'idsite' => $archiveToProcess['idsite'],
                'date1' => $period->getDateStart()->getDatetime(),
                'date2' => $period->getDateEnd()->getDatetime(),
                'period' => $id,
                'ts_invalidated' => $archiveToProcess['ts_invalidated'],
            ];

            $this->logger->debug("Found dangling invalidation, inserting {invalidationToInsert}", [
                'invalidationToInsert' => json_encode($invalidationToInsert),
            ]);

            $invalidationsToInsert[] = $invalidationToInsert;
        }

        if (empty($invalidationsToInsert)) {
            return;
        }

        $fields = ['idarchive', 'name', 'report', 'idsite', 'date1', 'date2', 'period', 'ts_invalidated'];
        Db\BatchInsert::tableInsertBatch(Common::prefixTable('archive_invalidations'), $fields, $invalidationsToInsert);
    }

    private function setInvalidationTime()
    {
        $cache = Cache::getTransientCache();

        Option::set(self::CRON_INVALIDATION_TIME_OPTION_NAME, time());

        $cacheKey = 'CronArchive.getLastInvalidationTime';

        $cache->delete($cacheKey);
    }

    public static function getLastInvalidationTime()
    {
        $cache = Cache::getTransientCache();

        $cacheKey = 'CronArchive.getLastInvalidationTime';
        $result = $cache->fetch($cacheKey);
        if ($result !== false) {
            return $result;
        }

        Option::clearCachedOption(self::CRON_INVALIDATION_TIME_OPTION_NAME);
        $result = Option::get(self::CRON_INVALIDATION_TIME_OPTION_NAME);

        if (empty($result)) {
            Option::clearCachedOption(self::OPTION_ARCHIVING_FINISHED_TS);
            $result = Option::get(self::OPTION_ARCHIVING_FINISHED_TS);
        }

        $cache->save($cacheKey, $result, self::TABLES_WITH_INVALIDATED_ARCHIVES_TTL);

        return $result;
    }

    /**
     *  Returns the list of sites to loop over and archive.
     *  @return array
     */
    private function initWebsiteIds($allWebsites)
    {
        if (count($this->shouldArchiveSpecifiedSites) > 0) {
            $this->logger->info("- Will process " . count($this->shouldArchiveSpecifiedSites) . " websites (--force-idsites)");

            return $this->shouldArchiveSpecifiedSites;
        }

        return $allWebsites;
    }

    private function logInitInfo()
    {
        $this->logSection("INIT");
        $this->logger->info("Running Matomo " . Version::VERSION . " as Super User");
    }

    private function logArchiveTimeoutInfo()
    {
        $this->logSection("NOTES");

        // Recommend to disable browser archiving when using this script
        if (Rules::isBrowserTriggerEnabled()) {
            $this->logger->info("- If you execute this script at least once per hour (or more often) in a crontab, you may disable 'Browser trigger archiving' in Matomo UI > Settings > General Settings.");
            $this->logger->info("  See the doc at: https://matomo.org/docs/setup-auto-archiving/");
        }

        $cliMulti = new CliMulti($this->logger);
        $supportsAsync = $cliMulti->supportsAsync();
        $this->logger->info("- " . ($supportsAsync ? 'Async process archiving supported, using CliMulti.' : 'Async process archiving not supported, using curl requests.'));

        $this->logger->info("- Reports for today will be processed at most every " . $this->todayArchiveTimeToLive
            . " seconds. You can change this value in Matomo UI > Settings > General Settings.");

        foreach (array('week', 'month', 'year', 'range') as $period) {
            $ttl = Rules::getPeriodArchiveTimeToLiveDefault($period);

            if (!empty($ttl) && $ttl !== $this->todayArchiveTimeToLive) {
                $this->logger->info("- Reports for the current $period will be processed at most every " . $ttl
                    . " seconds. You can change this value in config/config.ini.php by editing 'time_before_" . $period . "_archive_considered_outdated' in the '[General]' section.");
            }
        }

        if ($this->maxSitesToProcess) {
            $this->logger->info("- Maximum {$this->maxSitesToProcess} websites will be processed.");
        }
        if ($this->maxArchivesToProcess) {
            $this->logger->info("- Maximum {$this->maxArchivesToProcess} archives will be processed (soft limit).");
        }

        // Try and not request older data we know is already archived
        if ($this->lastSuccessRunTimestamp !== false) {
            $dateLast = time() - $this->lastSuccessRunTimestamp;
            $this->logger->info("- Archiving was last executed without error "
                . $this->formatter->getPrettyTimeFromSeconds($dateLast, true) . " ago.");
        }
    }

    private function getVisitsFromApiResponse($stats)
    {
        if (empty($stats['nb_visits'])) {
            return 0;
        }

        return (int) $stats['nb_visits'];
    }

    /**
     * @return int
     */
    private function getMaxConcurrentApiRequests()
    {
        if (false !== $this->concurrentRequestsPerWebsite) {
            return $this->concurrentRequestsPerWebsite;
        }

        return self::MAX_CONCURRENT_API_REQUESTS;
    }

    /**
     * @return false|string
     */
    private function getLastSuccessRunTimestamp()
    {
        $timestamp = Option::get(self::OPTION_ARCHIVING_FINISHED_TS);
        return $this->sanitiseTimestamp($timestamp);
    }

    private function sanitiseTimestamp($timestamp)
    {
        $now = time();
        return ($timestamp < $now) ? $timestamp : $now;
    }

    /**
     * @param $idSite
     * @return array of date strings
     */
    private function getCustomDateRangeToPreProcess($idSite)
    {
        static $cache = null;
        if (is_null($cache)) {
            $cache = $this->loadCustomDateRangeToPreProcess();
        }

        if (empty($cache[$idSite])) {
            $cache[$idSite] = array();
        }

        $customRanges = array_filter(Config::getInstance()->General['archiving_custom_ranges']);

        if (!empty($customRanges)) {
            $cache[$idSite] = array_merge($cache[$idSite], $customRanges);
        }

        $dates = array_unique($cache[$idSite]);
        return $dates;
    }

    /**
     * @return array
     */
    private function loadCustomDateRangeToPreProcess()
    {
        $customDateRangesToProcessForSites = array();

        // For all users who have selected this website to load by default,
        // we load the default period/date that will be loaded for this user
        // and make sure it's pre-archived
        $allUsersPreferences = APIUsersManager::getInstance()->getAllUsersPreferences(array(
            APIUsersManager::PREFERENCE_DEFAULT_REPORT_DATE,
            APIUsersManager::PREFERENCE_DEFAULT_REPORT
        ));

        foreach ($allUsersPreferences as $userLogin => $userPreferences) {
            if (!isset($userPreferences[APIUsersManager::PREFERENCE_DEFAULT_REPORT_DATE])) {
                continue;
            }

            $defaultDate = $userPreferences[APIUsersManager::PREFERENCE_DEFAULT_REPORT_DATE];
            $preference = new UserPreferences();
            $period = $preference->getDefaultPeriod($defaultDate);
            if ($period != 'range') {
                continue;
            }

            if (isset($userPreferences[APIUsersManager::PREFERENCE_DEFAULT_REPORT])
                && is_numeric($userPreferences[APIUsersManager::PREFERENCE_DEFAULT_REPORT])) {
                // If user selected one particular website ID
                $idSites = array($userPreferences[APIUsersManager::PREFERENCE_DEFAULT_REPORT]);
            } else {
                // If user selected "All websites" or some other random value, we pre-process all websites that they have access to
                $idSites = APISitesManager::getInstance()->getSitesIdWithAtLeastViewAccess($userLogin);
            }

            foreach ($idSites as $idSite) {
                $customDateRangesToProcessForSites[$idSite][] = $defaultDate;
            }
        }

        return $customDateRangesToProcessForSites;
    }

    /**
     * @param $url
     * @return string
     */
    private function makeRequestUrl($url)
    {
        $url = $url . self::APPEND_TO_API_REQUEST;

        if ($this->shouldStartProfiler) {
            $url .= "&xhprof=2";
        }

        if ($this->testmode) {
            $url .= "&testmode=1";
        }

        /**
         * @ignore
         */
        Piwik::postEvent('CronArchive.alterArchivingRequestUrl', [&$url]);

        return $url;
    }

    /**
     * @return CliMulti
     */
    private function makeCliMulti()
    {
        /** @var CliMulti $cliMulti */
        $cliMulti = new CliMulti($this->logger);
        $cliMulti->setUrlToPiwik($this->urlToPiwik);
        $cliMulti->setPhpCliConfigurationOptions($this->phpCliConfigurationOptions);
        $cliMulti->setAcceptInvalidSSLCertificate($this->acceptInvalidSSLCertificate);
        $cliMulti->setConcurrentProcessesLimit($this->getMaxConcurrentApiRequests());
        $cliMulti->runAsSuperUser();
        $cliMulti->onProcessFinish(function ($pid) {
            $this->printPerformanceStatsForProcess($pid);
        });
        return $cliMulti;
    }

    public function setUrlToPiwik($url)
    {
        $this->urlToPiwik = $url;
    }

    private function printPerformanceStatsForProcess($childPid)
    {
        if (!$this->isArchiveProfilingEnabled) {
            return;
        }

        $data = Logger::getMeasurementsFor(getmypid(), $childPid);
        if (empty($data)) {
            return;
        }

        $message = "";
        foreach ($data as $request => $measurements) {
            $message .= "PERFORMANCE FOR " . $request . "\n  ";
            $message .= implode("\n  ", $measurements) . "\n";
        }
        $this->logger->info($message);
    }

    private function hasReachedMaxConcurrentArchivers()
    {
        $cliMulti = $this->makeCliMulti();
        if ($this->maxConcurrentArchivers && $cliMulti->supportsAsync()) {
            $numRunning = 0;
            $processes = Process::getListOfRunningProcesses();
            $instanceId = SettingsPiwik::getPiwikInstanceId();

            foreach ($processes as $process) {
                if (strpos($process, ' core:archive') !== false &&
                    strpos($process, 'console ') !== false &&
                    (!$instanceId
                        || strpos($process, '--matomo-domain=' . $instanceId) !== false
                        || strpos($process, '--matomo-domain="' . $instanceId . '"') !== false
                        || strpos($process, '--matomo-domain=\'' . $instanceId . "'") !== false
                        || strpos($process, '--piwik-domain=' . $instanceId) !== false
                        || strpos($process, '--piwik-domain="' . $instanceId . '"') !== false
                        || strpos($process, '--piwik-domain=\'' . $instanceId . "'") !== false)) {
                    $numRunning++;
                }
            }
            if ($this->maxConcurrentArchivers < $numRunning) {
                $this->logger->info(sprintf("Archiving will stop now because %s archivers are already running and max %s are supposed to run at once.", $numRunning, $this->maxConcurrentArchivers));
                return true;
            } else {
                $this->logger->info(sprintf("%s out of %s archivers running currently", $numRunning, $this->maxConcurrentArchivers));
            }
        }
        return false;
    }

    protected function wasSegmentChangedRecently($definition, $allSegments)
    {
        foreach ($allSegments as $segment) {
            if ($segment['definition'] === $definition) {
                $twentyFourHoursAgo = Date::now()->subHour(24);
                $segmentDate = $segment['ts_created'];
                if (!empty($segment['ts_last_edit'])) {
                    $segmentDate = $segment['ts_last_edit'];
                }
                return Date::factory($segmentDate)->isLater($twentyFourHoursAgo);
            }
        }

        return false;
    }

    /**
     * @param ArchiveFilter $archiveFilter
     */
    public function setArchiveFilter(ArchiveFilter $archiveFilter): void
    {
        $this->archiveFilter = $archiveFilter;
    }

    private function makeWebsiteIdArchiveList(array $websitesIds)
    {
        if ($this->shouldArchiveAllSites) {
            $this->logger->info("- Will process all " . count($websitesIds) . " websites");
            return new FixedSiteIds($websitesIds);
        }

        if (!empty($this->shouldArchiveSpecifiedSites)) {
            $this->logger->info("- Will process specified sites: " . implode(', ', $websitesIds));
            return new FixedSiteIds($websitesIds);
        }

        return new SharedSiteIds($websitesIds, SharedSiteIds::OPTION_ALL_WEBSITES);
    }

    private function siteExists($idSite)
    {
        try {
            new Site($idSite);
            return true;
        } catch (\UnexpectedValueException $ex) {
            return false;
        }
    }
}