diff options
Diffstat (limited to 'plugins/UserCountry/GeoIPAutoUpdater.php')
| -rw-r--r-- | plugins/UserCountry/GeoIPAutoUpdater.php | 731 |
1 files changed, 0 insertions, 731 deletions
diff --git a/plugins/UserCountry/GeoIPAutoUpdater.php b/plugins/UserCountry/GeoIPAutoUpdater.php deleted file mode 100644 index bee89ac8fd..0000000000 --- a/plugins/UserCountry/GeoIPAutoUpdater.php +++ /dev/null @@ -1,731 +0,0 @@ -<?php -/** - * Piwik - free/libre analytics platform - * - * @link https://matomo.org - * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later - * - */ -namespace Piwik\Plugins\UserCountry; - -require_once PIWIK_INCLUDE_PATH . "/core/ScheduledTask.php"; // for the tracker which doesn't include this file - -use Exception; -use Piwik\Common; -use Piwik\Container\StaticContainer; -use Piwik\Date; -use Piwik\Http; -use Piwik\Log; -use Piwik\Option; -use Piwik\Piwik; -use Piwik\Plugins\UserCountry\LocationProvider\GeoIp\Php; -use Piwik\Plugins\UserCountry\LocationProvider\GeoIp; -use Piwik\Plugins\UserCountry\LocationProvider; -use Piwik\Scheduler\Scheduler; -use Piwik\Scheduler\Task; -use Piwik\Scheduler\Timetable; -use Piwik\Scheduler\Schedule\Monthly; -use Piwik\Scheduler\Schedule\Weekly; -use Piwik\SettingsPiwik; -use Piwik\Unzip; -use Psr\Log\LoggerInterface; - -/** - * Used to automatically update installed GeoIP databases, and manages the updater's - * scheduled task. - */ -class GeoIPAutoUpdater extends Task -{ - const SCHEDULE_PERIOD_MONTHLY = 'month'; - const SCHEDULE_PERIOD_WEEKLY = 'week'; - - const SCHEDULE_PERIOD_OPTION_NAME = 'geoip.updater_period'; - const LOC_URL_OPTION_NAME = 'geoip.loc_db_url'; - const ISP_URL_OPTION_NAME = 'geoip.isp_db_url'; - const ORG_URL_OPTION_NAME = 'geoip.org_db_url'; - - const LAST_RUN_TIME_OPTION_NAME = 'geoip.updater_last_run_time'; - - private static $urlOptions = array( - 'loc' => self::LOC_URL_OPTION_NAME, - 'isp' => self::ISP_URL_OPTION_NAME, - 'org' => self::ORG_URL_OPTION_NAME, - ); - - /** - * PHP Error caught through a custom error handler while trying to use a downloaded - * GeoIP database. See catchGeoIPError for more info. - * - * @var array - */ - private static $unzipPhpError = null; - - /** - * Constructor. - */ - public function __construct() - { - if (!SettingsPiwik::isInternetEnabled()) { - // no automatic updates possible if no internet available - return; - } - - $schedulePeriodStr = self::getSchedulePeriod(); - - // created the scheduledtime instance, also, since GeoIP updates are done on tuesdays, - // get new DBs on Wednesday - switch ($schedulePeriodStr) { - case self::SCHEDULE_PERIOD_WEEKLY: - $schedulePeriod = new Weekly(); - $schedulePeriod->setDay(3); - break; - case self::SCHEDULE_PERIOD_MONTHLY: - default: - $schedulePeriod = new Monthly(); - $schedulePeriod->setDayOfWeek(3, 0); - break; - } - - parent::__construct($this, 'update', null, $schedulePeriod, Task::LOWEST_PRIORITY); - } - - /** - * Attempts to download new location, ISP & organization GeoIP databases and - * replace the existing ones w/ them. - */ - public function update() - { - try { - Option::set(self::LAST_RUN_TIME_OPTION_NAME, Date::factory('today')->getTimestamp()); - - $locUrl = Option::get(self::LOC_URL_OPTION_NAME); - if (!empty($locUrl)) { - $this->downloadFile('loc', $locUrl); - } - - $ispUrl = Option::get(self::ISP_URL_OPTION_NAME); - if (!empty($ispUrl)) { - $this->downloadFile('isp', $ispUrl); - } - - $orgUrl = Option::get(self::ORG_URL_OPTION_NAME); - if (!empty($orgUrl)) { - $this->downloadFile('org', $orgUrl); - } - } catch (Exception $ex) { - // message will already be prefixed w/ 'GeoIPAutoUpdater: ' - StaticContainer::get(LoggerInterface::class)->error('Auto-update failed: {exception}', [ - 'exception' => $ex, - 'ignoreInScreenWriter' => true, - ]); - $this->performRedundantDbChecks(); - throw $ex; - } - - $this->performRedundantDbChecks(); - } - - /** - * Downloads a GeoIP database archive, extracts the .dat file and overwrites the existing - * old database. - * - * If something happens that causes the download to fail, no exception is thrown, but - * an error is logged. - * - * @param string $dbType - * @param string $url URL to the database to download. The type of database is determined - * from this URL. - * @throws Exception - */ - protected function downloadFile($dbType, $url) - { - $url = trim($url); - - if (strpos($url, 'GeoLite')) { - Log::info('GeoLite databases have been discontinued. Skipping download of '.$url.'. Consider switching to GeoIP 2.'); - return; - } - - $ext = GeoIPAutoUpdater::getGeoIPUrlExtension($url); - - // NOTE: using the first item in $dbNames[$dbType] makes sure GeoLiteCity will be renamed to GeoIPCity - $zippedFilename = GeoIp::$dbNames[$dbType][0] . '.' . $ext; - - $zippedOutputPath = GeoIp::getPathForGeoIpDatabase($zippedFilename); - - $url = self::removeDateFromUrl($url); - - // download zipped file to misc dir - try { - $success = Http::sendHttpRequest($url, $timeout = 3600, $userAgent = null, $zippedOutputPath); - } catch (Exception $ex) { - throw new Exception("GeoIPAutoUpdater: failed to download '$url' to " - . "'$zippedOutputPath': " . $ex->getMessage()); - } - - if ($success !== true) { - throw new Exception("GeoIPAutoUpdater: failed to download '$url' to " - . "'$zippedOutputPath'! (Unknown error)"); - } - - Log::info("GeoIPAutoUpdater: successfully downloaded '%s'", $url); - - try { - self::unzipDownloadedFile($zippedOutputPath, $unlink = true); - } catch (Exception $ex) { - throw new Exception("GeoIPAutoUpdater: failed to unzip '$zippedOutputPath' after " - . "downloading " . "'$url': " . $ex->getMessage()); - } - - Log::info("GeoIPAutoUpdater: successfully updated GeoIP database '%s'", $url); - } - - /** - * Unzips a downloaded GeoIP database. Only unzips .gz & .tar.gz files. - * - * @param string $path Path to zipped file. - * @param bool $unlink Whether to unlink archive or not. - * @throws Exception - */ - public static function unzipDownloadedFile($path, $unlink = false) - { - $parts = explode('.', basename($path)); - $filenameStart = $parts[0]; - - $dbFilename = $filenameStart . '.dat'; - $tempFilename = $filenameStart . '.dat.new'; - $outputPath = GeoIp::getPathForGeoIpDatabase($tempFilename); - - // extract file - if (substr($path, -7, 7) == '.tar.gz') { - // find the .dat file in the tar archive - $unzip = Unzip::factory('tar.gz', $path); - $content = $unzip->listContent(); - - if (empty($content)) { - throw new Exception(Piwik::translate('UserCountry_CannotListContent', - array("'$path'", $unzip->errorInfo()))); - } - - $datFile = null; - foreach ($content as $info) { - $archivedPath = $info['filename']; - if (basename($archivedPath) === $dbFilename) { - $datFile = $archivedPath; - } - } - - if ($datFile === null) { - throw new Exception(Piwik::translate('UserCountry_CannotFindGeoIPDatabaseInArchive', - array($dbFilename, "'$path'"))); - } - - // extract JUST the .dat file - $unzipped = $unzip->extractInString($datFile); - - if (empty($unzipped)) { - throw new Exception(Piwik::translate('UserCountry_CannotUnzipDatFile', - array("'$path'", $unzip->errorInfo()))); - } - - // write unzipped to file - $fd = fopen($outputPath, 'wb'); - fwrite($fd, $unzipped); - fclose($fd); - } else if (substr($path, -3, 3) == '.gz') { - $unzip = Unzip::factory('gz', $path); - $success = $unzip->extract($outputPath); - - if ($success !== true) { - throw new Exception(Piwik::translate('UserCountry_CannotUnzipDatFile', - array("'$path'", $unzip->errorInfo()))); - } - } else { - $ext = end(explode(basename($path), '.', 2)); - throw new Exception(Piwik::translate('UserCountry_UnsupportedArchiveType', "'$ext'")); - } - - try { - // test that the new archive is a valid GeoIP database - $dbType = GeoIp::getGeoIPDatabaseTypeFromFilename($dbFilename); - if ($dbType === false) // sanity check - { - throw new Exception("Unexpected GeoIP archive file name '$path'."); - } - - $customDbNames = array( - 'loc' => array(), - 'isp' => array(), - 'org' => array() - ); - $customDbNames[$dbType] = array($tempFilename); - - $phpProvider = new Php($customDbNames); - - $location = self::getTestLocationCatchPhpErrors($phpProvider); - - if (empty($location) - || self::$unzipPhpError !== null - ) { - if (self::$unzipPhpError !== null) { - list($errno, $errstr, $errfile, $errline) = self::$unzipPhpError; - Log::info("GeoIPAutoUpdater: Encountered PHP error when testing newly downloaded" . - " GeoIP database: %s: %s on line %s of %s.", $errno, $errstr, $errline, $errfile); - } - - throw new Exception(Piwik::translate('UserCountry_ThisUrlIsNotAValidGeoIPDB')); - } - - // delete the existing GeoIP database (if any) and rename the downloaded file - $oldDbFile = GeoIp::getPathForGeoIpDatabase($dbFilename); - if (file_exists($oldDbFile)) { - unlink($oldDbFile); - } - - $tempFile = GeoIp::getPathForGeoIpDatabase($tempFilename); - if (@rename($tempFile, $oldDbFile) !== true) { - //In case the $tempfile cannot be renamed, we copy the file. - copy($tempFile, $oldDbFile); - unlink($tempFile); - } - - // delete original archive - if ($unlink) { - unlink($path); - } - } catch (Exception $ex) { - // remove downloaded files - if (file_exists($outputPath)) { - unlink($outputPath); - } - unlink($path); - - throw $ex; - } - } - - /** - * Sets the options used by this class based on query parameter values. - * - * See setUpdaterOptions for query params used. - */ - public static function setUpdaterOptionsFromUrl() - { - $options = array( - 'loc' => Common::getRequestVar('loc_db', false, 'string'), - 'isp' => Common::getRequestVar('isp_db', false, 'string'), - 'org' => Common::getRequestVar('org_db', false, 'string'), - 'period' => Common::getRequestVar('period', false, 'string'), - ); - - foreach (self::$urlOptions as $optionKey => $optionName) { - $options[$optionKey] = Common::unsanitizeInputValue($options[$optionKey]); // URLs should not be sanitized - } - - self::setUpdaterOptions($options); - } - - /** - * Sets the options used by this class based on the elements in $options. - * - * The following elements of $options are used: - * 'loc' - URL for location database. - * 'isp' - URL for ISP database. - * 'org' - URL for Organization database. - * 'period' - 'weekly' or 'monthly'. When to run the updates. - * - * @param array $options - * @throws Exception - */ - public static function setUpdaterOptions($options) - { - // set url options - foreach (self::$urlOptions as $optionKey => $optionName) { - if (!isset($options[$optionKey])) { - continue; - } - - $url = $options[$optionKey]; - $url = self::removeDateFromUrl($url); - - Option::set($optionName, $url); - } - - // set period option - if (!empty($options['period'])) { - $period = $options['period']; - - if ($period != self::SCHEDULE_PERIOD_MONTHLY - && $period != self::SCHEDULE_PERIOD_WEEKLY - ) { - throw new Exception(Piwik::translate( - 'UserCountry_InvalidGeoIPUpdatePeriod', - array("'$period'", "'" . self::SCHEDULE_PERIOD_MONTHLY . "', '" . self::SCHEDULE_PERIOD_WEEKLY . "'") - )); - } - - Option::set(self::SCHEDULE_PERIOD_OPTION_NAME, $period); - - /** @var Scheduler $scheduler */ - $scheduler = StaticContainer::getContainer()->get('Piwik\Scheduler\Scheduler'); - - $scheduler->rescheduleTaskAndRunTomorrow(new GeoIPAutoUpdater()); - } - } - - /** - * Removes all options to disable any configured automatic updates - */ - public static function clearOptions() - { - foreach (self::$urlOptions as $optionKey => $optionName) { - Option::delete($optionName); - } - Option::delete(self::SCHEDULE_PERIOD_OPTION_NAME); - } - - /** - * Returns true if the auto-updater is setup to update at least one type of - * database. False if otherwise. - * - * @return bool - */ - public static function isUpdaterSetup() - { - if (Option::get(self::LOC_URL_OPTION_NAME) !== false - || Option::get(self::ISP_URL_OPTION_NAME) !== false - || Option::get(self::ORG_URL_OPTION_NAME) !== false - ) { - return true; - } - - return false; - } - - /** - * Retrieves the URLs used to update various GeoIP database files. - * - * @return array - */ - public static function getConfiguredUrls() - { - $result = array(); - foreach (self::$urlOptions as $key => $optionName) { - $result[$key] = Option::get($optionName); - } - return $result; - } - - /** - * Returns the confiured URL (if any) for a type of database. - * - * @param string $key 'loc', 'isp' or 'org' - * @throws Exception - * @return string|false - */ - public static function getConfiguredUrl($key) - { - if (empty(self::$urlOptions[$key])) { - throw new Exception("Invalid key $key"); - } - $url = Option::get(self::$urlOptions[$key]); - return $url; - } - - /** - * Performs a GeoIP database update. - */ - public static function performUpdate() - { - $instance = new GeoIPAutoUpdater(); - $instance->update(); - } - - /** - * Returns the configured update period, either 'week' or 'month'. Defaults to - * 'month'. - * - * @return string - */ - public static function getSchedulePeriod() - { - $period = Option::get(self::SCHEDULE_PERIOD_OPTION_NAME); - if ($period === false) { - $period = self::SCHEDULE_PERIOD_MONTHLY; - } - return $period; - } - - /** - * Returns an array of strings for GeoIP databases that have update URLs configured, but - * are not present in the misc directory. Each string is a key describing the type of - * database (ie, 'loc', 'isp' or 'org'). - * - * @return array - */ - public static function getMissingDatabases() - { - $result = array(); - foreach (self::getConfiguredUrls() as $key => $url) { - if (!empty($url)) { - // if a database of the type does not exist, but there's a url to update, then - // a database is missing - $path = GeoIp::getPathToGeoIpDatabase( - GeoIp::$dbNames[$key]); - if ($path === false) { - $result[] = $key; - } - } - } - return $result; - } - - /** - * Returns the extension of a URL used to update a GeoIP database, if it can be found. - */ - public static function getGeoIPUrlExtension($url) - { - // check for &suffix= query param that is special to MaxMind URLs - if (preg_match('/suffix=([^&]+)/', $url, $matches)) { - $ext = $matches[1]; - } else { - // use basename of url - $filenameParts = explode('.', basename($url), 2); - if (count($filenameParts) > 1) { - $ext = end($filenameParts); - } else { - $ext = reset($filenameParts); - } - } - - self::checkForSupportedArchiveType($ext); - - return $ext; - } - - /** - * Avoid downloading archive types we don't support. No point in downloading it, - * if we can't unzip it... - * - * @param string $ext The URL file's extension. - * @throws \Exception - */ - private static function checkForSupportedArchiveType($ext) - { - if ($ext != 'tar.gz' - && $ext != 'gz' - && $ext != 'dat.gz' - && $ext != 'mmdb.gz' - ) { - throw new \Exception(Piwik::translate('UserCountry_UnsupportedArchiveType', "'$ext'")); - } - } - - /** - * Tests a location provider using a test IP address and catches PHP errors - * (ie, notices) if they occur. PHP error information is held in self::$unzipPhpError. - * - * @param LocationProvider $provider The provider to test. - * @return array|false $location The result of geolocation. False if no location - * can be found. - */ - private static function getTestLocationCatchPhpErrors($provider) - { - // note: in most cases where this will fail, the error will usually be a PHP fatal error/notice. - // in order to delete the files in such a case (which can be caused by a man-in-the-middle attack) - // we need to catch them, so we set a new error handler. - self::$unzipPhpError = null; - set_error_handler(array('Piwik\Plugins\UserCountry\GeoIPAutoUpdater', 'catchGeoIPError')); - - $location = $provider->getLocation(array('ip' => GeoIp::TEST_IP)); - - restore_error_handler(); - - return $location; - } - - /** - * Utility function that checks if geolocation works with each installed database, - * and if one or more doesn't, they are renamed to make sure tracking will work. - * This is a safety measure used to make sure tracking isn't affected if strange - * update errors occur. - * - * Databases are renamed to ${original}.broken . - * - * Note: method is protected for testability. - * - * @param $logErrors - only used to hide error logs during tests - */ - protected function performRedundantDbChecks($logErrors = true) - { - $databaseTypes = array_keys(GeoIp::$dbNames); - - foreach ($databaseTypes as $type) { - $customNames = array( - 'loc' => array(), - 'isp' => array(), - 'org' => array() - ); - $customNames[$type] = GeoIp::$dbNames[$type]; - - // create provider that only uses the DB type we're testing - $provider = new Php($customNames); - - // test the provider. on error, we rename the broken DB. - self::getTestLocationCatchPhpErrors($provider); - if (self::$unzipPhpError !== null) { - list($errno, $errstr, $errfile, $errline) = self::$unzipPhpError; - - if ($logErrors) { - StaticContainer::get(LoggerInterface::class)->error("GeoIPAutoUpdater: Encountered PHP error when performing redundant tests on GeoIP " - . "{type} database: {errno}: {errstr} on line {errline} of {errfile}.", [ - 'ignoreInScreenWriter' => true, - 'type' => $type, - 'errno' => $errno, - 'errstr' => $errstr, - 'errline' => $errline, - 'errfile' => $errfile, - ]); - } - - // get the current filename for the DB and an available new one to rename it to - list($oldPath, $newPath) = $this->getOldAndNewPathsForBrokenDb($customNames[$type]); - - // rename the DB so tracking will not fail - if ($oldPath !== false - && $newPath !== false - ) { - if (file_exists($newPath)) { - unlink($newPath); - } - - rename($oldPath, $newPath); - } - } - } - } - - /** - * Returns the path to a GeoIP database and a path to rename it to if it's broken. - * - * @param array $possibleDbNames The possible names of the database. - * @return array Array with two elements, the path to the existing database, and - * the path to rename it to if it is broken. The second will end - * with something like .broken . - */ - private function getOldAndNewPathsForBrokenDb($possibleDbNames) - { - $pathToDb = GeoIp::getPathToGeoIpDatabase($possibleDbNames); - $newPath = false; - - if ($pathToDb !== false) { - $newPath = $pathToDb . ".broken"; - } - - return array($pathToDb, $newPath); - } - - /** - * Custom PHP error handler used to catch any PHP errors that occur when - * testing a downloaded GeoIP file. - * - * If we download a file that is supposed to be a GeoIP database, we need to make - * sure it is one. This is done simply by attempting to use it. If this fails, it - * will most of the time fail as a PHP error, which we catch w/ this function - * after it is passed to set_error_handler. - * - * The PHP error is stored in self::$unzipPhpError. - * - * @param int $errno - * @param string $errstr - * @param string $errfile - * @param int $errline - */ - public static function catchGeoIPError($errno, $errstr, $errfile, $errline) - { - self::$unzipPhpError = array($errno, $errstr, $errfile, $errline); - } - - /** - * Returns the time the auto updater was last run. - * - * @return Date|false - */ - public static function getLastRunTime() - { - $timestamp = Option::get(self::LAST_RUN_TIME_OPTION_NAME); - return $timestamp === false ? false : Date::factory((int)$timestamp); - } - - /** - * Removes the &date=... query parameter if present in the URL. This query parameter - * is in MaxMind URLs by default and will force the download of an old database. - * - * @param string $url - * @return string - */ - private static function removeDateFromUrl($url) - { - return preg_replace("/&date=[^&#]*/", '', $url); - } - - /** - * Returns the next scheduled time for the auto updater. - * - * @return Date|false - */ - public static function getNextRunTime() - { - $task = new GeoIPAutoUpdater(); - - $timetable = new Timetable(); - return $timetable->getScheduledTaskTime($task->getName()); - } - - /** - * See {@link Piwik\Scheduler\Schedule\Schedule::getRescheduledTime()}. - */ - public function getRescheduledTime() - { - $nextScheduledTime = parent::getRescheduledTime(); - - // if a geoip database is out of date, run the updater as soon as possible - if ($this->isAtLeastOneGeoIpDbOutOfDate($nextScheduledTime)) { - return time(); - } - - return $nextScheduledTime; - } - - private function isAtLeastOneGeoIpDbOutOfDate($rescheduledTime) - { - $previousScheduledRuntime = $this->getPreviousScheduledTime($rescheduledTime)->setTime("00:00:00")->getTimestamp(); - - foreach (GeoIp::$dbNames as $type => $dbNames) { - $dbUrl = Option::get(self::$urlOptions[$type]); - $dbPath = GeoIp::getPathToGeoIpDatabase($dbNames); - - // if there is a URL for this DB type and the GeoIP DB file's last modified time is before - // the time the updater should have been previously run, then **the file is out of date** - if (!empty($dbUrl) - && filemtime($dbPath) < $previousScheduledRuntime - ) { - return true; - } - } - - return false; - } - - private function getPreviousScheduledTime($rescheduledTime) - { - $updaterPeriod = self::getSchedulePeriod(); - - if ($updaterPeriod == self::SCHEDULE_PERIOD_WEEKLY) { - return Date::factory($rescheduledTime)->subWeek(1); - } else if ($updaterPeriod == self::SCHEDULE_PERIOD_MONTHLY) { - return Date::factory($rescheduledTime)->subMonth(1); - } - throw new Exception("Unknown GeoIP updater period found in database: %s", $updaterPeriod); - } -} |