refs #5940 moved files into a subfolder framework, added autoloader to remove duplicated code to load autoload.php and to be able to register more autoloaders (eg for test files) on demand. This I got read of many includes that had to be updated all the time and that had to be updated all the time when moving iles

3 files changed, 828 insertions, 0 deletions

diff --git a/tests/PHPUnit/Framework/TestRequest/ApiTestConfig.php b/tests/PHPUnit/Framework/TestRequest/ApiTestConfig.php
new file mode 100644
index 0000000000..f8fa02a691
--- /dev/null
+++ b/tests/PHPUnit/Framework/TestRequest/ApiTestConfig.php
@@ -0,0 +1,222 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+
+namespace Piwik\Tests\Framework\TestRequest;
+
+use \Exception;
+
+/**
+ * Holds the specification for a set of API tests.
+ *
+ * An API test consists of calling Piwik's API and comparing the result with an expected
+ * result. The expected result is stored in a file, usually in an **expected** folder.
+ *
+ * The test specification describes how the API is called and how the API response is
+ * processed before comparison.
+ *
+ * Instances of this class are not created directly. Instead, an array mapping config
+ * property names w/ values is passed to SystemTestCase::runApiTests. For example,
+ *
+ *     $this->runApiTests("UserCountry", array(
+ *         'idSite' => 1,
+ *         'date' => '2012-01-01',
+ *         // ...
+ *     ));
+ */
+class ApiTestConfig
+{
+    /**
+     * The value of the idSite query parameter to send to Piwik's API. Can be a comma separated
+     * list of integers or `'all'`.
+     *
+     * This option is required.
+     *
+     * @var int|string
+     */
+    public $idSite;
+
+    /**
+     * The value of the date query parameter to send to Piwik's API.
+     *
+     * @var string
+     */
+    public $date;
+
+    /**
+     * One or more periods to test for. Multiple periods will result in multiple API calls and
+     * multiple comparisons.
+     *
+     * @var string[]
+     */
+    public $periods = array('day');
+
+    /**
+     * The desired output format of the API response. Used to test DataTable renderers.
+     *
+     * @var string
+     */
+    public $format = 'xml';
+
+    /**
+     * Controls whether to query for multiple periods or not. If set to true, the last 6 dates will be
+     * queried for. If set to an integer, then that number of periods will be queried for.
+     *
+     * @var bool|int
+     */
+    public $setDateLastN = false;
+
+    /**
+     * The language to retrieve results in. Defaults to 'en'.
+     *
+     * @var string|false
+     */
+    public $language = false;
+
+    /**
+     * An optional value to use for the segment query parameter.
+     *
+     * @var string|false
+     */
+    public $segment = false;
+
+    /**
+     * The value to use for the idGoal query parameter.
+     *
+     * @var int|bool
+     */
+    public $idGoal = false;
+
+    /**
+     * The value to use for the apiModule query parameter.
+     *
+     * @var string|false
+     */
+    public $apiModule = false;
+
+    /**
+     * The value to use for the apiAction query parameter.
+     *
+     * @var string|false
+     */
+    public $apiAction = false;
+
+    /**
+     * Associative array of query parameters to set in API requests. For example,
+     *
+     *     array('param1' => 'value1', 'param2' => 'value2')
+     *
+     * @var string[]
+     */
+    public $otherRequestParameters = array();
+
+    /**
+     * This property is used to test API methods that return subtables and should be set to the API method that
+     * returns the super table of the API method being tested. If set, TestRequest\Collection will look for the
+     * first valid idSubtable value to use in the test request. Since these values are assigned dynamically,
+     * there's no other way to set idSubtable.
+     * 
+     * @var string|bool eg, `"Referrers.getWebsites"`
+     */
+    public $supertableApi = false;
+
+    /**
+     * If supplied, this value will be used as the expected and processed file's extension **without**
+     * setting the 'format' query parameter.
+     *
+     * Used when testing scheduled reports.
+     *
+     * @var string|bool eg, `"html"`
+     */
+    public $fileExtension = false;
+
+    /**
+     * An array of API methods that shouldn't be called. If `'all'` is specified in SystemTestCase::runApiTests,
+     * the methods in this property will be ignored when calling all API methods.
+     *
+     * @var string[]|false eg, `array("Actions", "Referrers.getWebsites", ...)`
+     */
+    public $apiNotToCall = false;
+
+    /**
+     * If true, archiving will be disabled when the API is called.
+     *
+     * @var bool
+     */
+    public $disableArchiving = false;
+
+    /**
+     * An extra suffix to apply to the expected and processed output file names.
+     *
+     * @param string
+     */
+    public $testSuffix = '';
+
+    /**
+     * If supplied, tests will compare API responses with files using a different file prefix.
+     * Normally, the test name is used as the test prefix, so this will usually be set to the
+     * name of the system test. Either that or the value in the test's getOutputPrefix
+     * method.
+     *
+     * @param string|bool eg, `'OneVisitorTwoVisitsTest'`
+     */
+    public $compareAgainst = false;
+
+    /**
+     * An array of XML fields that should be removed from processed API response before
+     * comparing. These fields should be fields that change on every test execution and have
+     * to be removed in order to make tests pass.
+     *
+     * @param string[]|false
+     */
+    public $xmlFieldsToRemove = false;
+
+    /**
+     * If true, Date times XML fields that change on each request for Live API methods are retained.
+     * Normally, they are removed before comparing the API response w/ expected.
+     *
+     * @param bool
+     */
+    public $keepLiveDates = false;
+
+    /**
+     * If true, ID visitors/User ID/other IDs that change on each request for Live API methods are retained.
+     * Normally, they are removed before comparing the API response w/ expected.
+     *
+     * @param bool
+     */
+    public $keepLiveIds = false;
+
+    /**
+     * Constructor. Sets class properties using an associative array mapping property names w/ values.
+     *
+     * @param array $params eg, `array('idSite' => 1, 'date' => '2012-01-01', ...)`
+     * @throws Exception if a property name in `$params` is invalid
+     */
+    public function __construct($params)
+    {
+        foreach ($params as $key => $value) {
+            if ($key == 'period') {
+                $key = 'periods';
+            }
+
+            if (!property_exists($this, $key)) {
+                throw new Exception("Invalid API test property '$key'! Check your System tests.");
+            }
+
+            $this->$key = $value;
+        }
+
+        if (!is_array($this->periods)) {
+            $this->periods = array($this->periods);
+        }
+
+        if ($this->setDateLastN === true) {
+            $this->setDateLastN = 6;
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/PHPUnit/Framework/TestRequest/Collection.php b/tests/PHPUnit/Framework/TestRequest/Collection.php
new file mode 100644
index 0000000000..da1987181b
--- /dev/null
+++ b/tests/PHPUnit/Framework/TestRequest/Collection.php
@@ -0,0 +1,339 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+
+namespace Piwik\Tests\Framework\TestRequest;
+
+use Piwik\API\DocumentationGenerator;
+use Piwik\API\Proxy;
+use Piwik\API\Request;
+use Piwik\Tests\Framework\TestCase\SystemTestCase;
+use Piwik\UrlHelper;
+use \Exception;
+
+/**
+ * Utility class used to generate a set of API requests given API methods to call, API
+ * methods to exclude, and an ApiTestConfig instance.
+ */
+class Collection
+{
+    public $defaultApiNotToCall = array(
+        'LanguagesManager',
+        'DBStats',
+        'Dashboard',
+        'UsersManager',
+        'SitesManager',
+        'ExampleUI',
+        'Overlay',
+        'Live',
+        'SEO',
+        'ExampleAPI',
+        'ScheduledReports',
+        'MobileMessaging',
+        'Transitions',
+        'API',
+        'ImageGraph',
+        'Annotations',
+        'SegmentEditor',
+        'UserCountry.getLocationFromIP',
+        'Dashboard',
+        'ExamplePluginTemplate',
+        'CustomAlerts',
+        'Insights'
+    );
+
+    /**
+     * The list of generated API requests.
+     *
+     * @var array[]
+     */
+    private $requestUrls;
+
+    /**
+     * The config for this set of API requests.
+     *
+     * @var ApiTestConfig
+     */
+    private $testConfig;
+
+    /**
+     * The set of API methods to test. Each API method will have at least one request URL in
+     * $requestUrls.
+     *
+     * @var string[]|string Can be set to 'all' to test all available API methods.
+     */
+    private $apiToCall;
+
+    /**
+     * The set of API methods/modules that should not be called. These methods will be excluded
+     * from the generated request URLs.
+     *
+     * @var string[]|string
+     */
+    private $apiNotToCall;
+
+    /**
+     * Constructor.
+     */
+    public function __construct($api, ApiTestConfig $testConfig, $apiToCall)
+    {
+        $this->testConfig = $testConfig;
+        $this->setExplicitApiToCallAndNotCall($apiToCall);
+
+        $this->requestUrls = $this->_generateApiUrls();
+    }
+
+    public function getRequestUrls()
+    {
+        return $this->requestUrls;
+    }
+
+    /**
+     * Will return all api urls for the given data
+     *
+     * @return array
+     */
+    protected function _generateApiUrls()
+    {
+        $parametersToSet = array(
+            'idSite'         => $this->testConfig->idSite,
+            'date'           => ($this->testConfig->periods == array('range') || strpos($this->testConfig->date, ',') !== false) ?
+                                    $this->testConfig->date : date('Y-m-d', strtotime($this->testConfig->date)),
+            'expanded'       => '1',
+            'piwikUrl'       => 'http://example.org/piwik/',
+            // Used in getKeywordsForPageUrl
+            'url'            => 'http://example.org/store/purchase.htm',
+
+            // Used in Actions.getPageUrl, .getDownload, etc.
+            // tied to Main.test.php doTest_oneVisitorTwoVisits
+            // will need refactoring when these same API functions are tested in a new function
+            'downloadUrl'    => 'http://piwik.org/path/again/latest.zip?phpsessid=this is ignored when searching',
+            'outlinkUrl'     => 'http://dev.piwik.org/svn',
+            'pageUrl'        => 'http://example.org/index.htm?sessionid=this is also ignored by default',
+            'pageName'       => ' Checkout / Purchasing... ',
+
+            // do not show the millisec timer in response or tests would always fail as value is changing
+            'showTimer'      => 0,
+
+            'language'       => $this->testConfig->language ?: 'en',
+            'idSites'        => $this->testConfig->idSite,
+        );
+        $parametersToSet = array_merge($parametersToSet, $this->testConfig->otherRequestParameters);
+        if (!empty($this->testConfig->apiModule)) {
+            $parametersToSet['apiModule'] = $this->testConfig->apiModule;
+        }
+        if (!empty($this->testConfig->apiAction)) {
+            $parametersToSet['apiAction'] = $this->testConfig->apiAction;
+        }
+        if (!empty($this->testConfig->segment)) {
+            $parametersToSet['segment'] = urlencode($this->testConfig->segment);
+        }
+        if ($this->testConfig->idGoal !== false) {
+            $parametersToSet['idGoal'] = $this->testConfig->idGoal;
+        }
+
+        $requestUrls = $this->generateApiUrlPermutations($parametersToSet);
+
+        $this->checkEnoughUrlsAreTested($requestUrls);
+
+        return $requestUrls;
+    }
+
+    protected function checkEnoughUrlsAreTested($requestUrls)
+    {
+        $countUrls = count($requestUrls);
+        $approximateCountApiToCall = count($this->apiToCall);
+        if (empty($requestUrls)
+            || $approximateCountApiToCall > $countUrls
+        ) {
+            throw new Exception("Only generated $countUrls API calls to test but was expecting more for this test.\n" .
+                    "Want to test APIs: " . implode(", ", $this->apiToCall) . ")\n" .
+                    "But only generated these URLs: \n" . implode("\n", $requestUrls) . ")\n"
+            );
+        }
+    }
+
+    /**
+     * Given a list of default parameters to set, returns the URLs of APIs to call
+     * If any API was specified in $this->apiNotToCall we ensure only these are tested.
+     * If any API is set as excluded (see list below) then it will be ignored.
+     *
+     * @param array $parametersToSet Parameters to set in api call
+     * @param array $formats         Array of 'format' to fetch from API
+     * @param array $periods         Array of 'period' to query API
+     * @param bool  $supertableApi
+     * @param bool  $setDateLastN    If set to true, the 'date' parameter will be rewritten to query instead a range of dates, rather than one period only.
+     * @param bool|string $language        2 letter language code, defaults to default piwik language
+     * @param bool|string $fileExtension
+     *
+     * @throws Exception
+     *
+     * @return array of API URLs query strings
+     */
+    protected function generateApiUrlPermutations($parametersToSet)
+    {
+        $formats = array($this->testConfig->format);
+        $originalDate = $parametersToSet['date'];
+
+        $requestUrls = array();
+        $apiMetadata = new DocumentationGenerator;
+
+        // Get the URLs to query against the API for all functions starting with get*
+        foreach ($this->getAllApiMethods() as $apiMethodInfo) {
+            list($class, $moduleName, $methodName) = $apiMethodInfo;
+
+            $apiId = $moduleName . '.' . $methodName;
+
+            foreach ($this->testConfig->periods as $period) {
+                $parametersToSet['period'] = $period;
+
+                // If date must be a date range, we process this date range by adding 6 periods to it
+                if ($this->testConfig->setDateLastN) {
+                    if (!isset($parametersToSet['dateRewriteBackup'])) {
+                        $parametersToSet['dateRewriteBackup'] = $parametersToSet['date'];
+                    }
+
+                    $lastCount = $this->testConfig->setDateLastN;
+
+                    $secondDate = date('Y-m-d', strtotime("+$lastCount " . $period . "s", strtotime($originalDate)));
+                    $parametersToSet['date'] = $originalDate . ',' . $secondDate;
+                }
+
+                // Set response language
+                if ($this->testConfig->language !== false) {
+                    $parametersToSet['language'] = $this->testConfig->language;
+                }
+
+                // set idSubtable if subtable API is set
+                if ($this->testConfig->supertableApi !== false) {
+                    $request = new Request(array(
+                                                          'module'    => 'API',
+                                                          'method'    => $this->testConfig->supertableApi,
+                                                          'idSite'    => $parametersToSet['idSite'],
+                                                          'period'    => $parametersToSet['period'],
+                                                          'date'      => $parametersToSet['date'],
+                                                          'format'    => 'php',
+                                                          'serialize' => 0,
+                                                     ));
+
+                    $content = $request->process();
+                    SystemTestCase::assertApiResponseHasNoError($content);
+
+                    // find first row w/ subtable
+                    foreach ($content as $row) {
+                        if (isset($row['idsubdatatable'])) {
+                            $parametersToSet['idSubtable'] = $row['idsubdatatable'];
+                            break;
+                        }
+                    }
+
+                    // if no subtable found, throw
+                    if (!isset($parametersToSet['idSubtable'])) {
+                        throw new Exception(
+                            "Cannot find subtable to load for $apiId in {$this->testConfig->supertableApi}.");
+                    }
+                }
+
+                // Generate for each specified format
+                foreach ($formats as $format) {
+                    $parametersToSet['format'] = $format;
+                    $parametersToSet['hideIdSubDatable'] = 1;
+                    $parametersToSet['serialize'] = 1;
+
+                    $exampleUrl = $apiMetadata->getExampleUrl($class, $methodName, $parametersToSet);
+                    if ($exampleUrl === false) {
+                        continue;
+                    }
+
+                    // Remove the first ? in the query string
+                    $exampleUrl = substr($exampleUrl, 1);
+                    $apiRequestId = $apiId;
+                    if (strpos($exampleUrl, 'period=') !== false) {
+                        $apiRequestId .= '_' . $period;
+                    }
+
+                    $apiRequestId .= '.' . $format;
+
+                    if ($this->testConfig->fileExtension) {
+                        $apiRequestId .= '.' . $this->testConfig->fileExtension;
+                    }
+
+                    $requestUrls[$apiRequestId] = UrlHelper::getArrayFromQueryString($exampleUrl);
+                }
+            }
+        }
+        return $requestUrls;
+    }
+
+    private function getAllApiMethods()
+    {
+        $result = array();
+
+        foreach (Proxy::getInstance()->getMetadata() as $class => $info) {
+            $moduleName = Proxy::getInstance()->getModuleNameFromClassName($class);
+            foreach ($info as $methodName => $infoMethod) {
+                if ($this->shouldSkipApiMethod($moduleName, $methodName)) {
+                    continue;
+                }
+
+                $result[] = array($class, $moduleName, $methodName);
+            }
+        }
+
+        return $result;
+    }
+
+    private function shouldSkipApiMethod($moduleName, $methodName) {
+        $apiId = $moduleName . '.' . $methodName;
+
+        // If Api to test were set, we only test these
+        if (!empty($this->apiToCall)
+            && in_array($moduleName, $this->apiToCall) === false
+            && in_array($apiId, $this->apiToCall) === false
+        ) {
+            return true;
+        } elseif (
+            ((strpos($methodName, 'get') !== 0 && $methodName != 'generateReport')
+                || in_array($moduleName, $this->apiNotToCall) === true
+                || in_array($apiId, $this->apiNotToCall) === true
+                || $methodName == 'getLogoUrl'
+                || $methodName == 'getSVGLogoUrl'
+                || $methodName == 'hasSVGLogo'
+                || $methodName == 'getHeaderLogoUrl'
+            )
+        ) { // Excluded modules from test
+            return true;
+        }
+
+        return false;
+    }
+
+    private function setExplicitApiToCallAndNotCall($apiToCall)
+    {
+        if ($apiToCall == 'all') {
+            $this->apiToCall = array();
+            $this->apiNotToCall = $this->defaultApiNotToCall;
+        } else {
+            if (!is_array($apiToCall)) {
+                $apiToCall = array($apiToCall);
+            }
+
+            $this->apiToCall = $apiToCall;
+
+            if (!in_array('UserCountry.getLocationFromIP', $apiToCall)) {
+                $this->apiNotToCall = array('API.getPiwikVersion',
+                                            'UserCountry.getLocationFromIP');
+            } else {
+                $this->apiNotToCall = array();
+            }
+        }
+
+        if (!empty($this->testConfig->apiNotToCall)) {
+            $this->apiNotToCall = array_merge($this->apiNotToCall, $this->testConfig->apiNotToCall);
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/PHPUnit/Framework/TestRequest/Response.php b/tests/PHPUnit/Framework/TestRequest/Response.php
new file mode 100644
index 0000000000..76bf43c149
--- /dev/null
+++ b/tests/PHPUnit/Framework/TestRequest/Response.php
@@ -0,0 +1,267 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+
+namespace Piwik\Tests\Framework\TestRequest;
+
+use Piwik\API\Request;
+use PHPUnit_Framework_Assert as Asserts;
+use Exception;
+use Piwik\Tests\Framework\TestCase\SystemTestCase;
+
+/**
+ * Utility class used to obtain and process API responses for API tests.
+ */
+class Response
+{
+    private $processedResponseText;
+
+    private $params;
+
+    private $requestUrl;
+
+    public function __construct($apiResponse, $params, $requestUrl)
+    {
+        $this->params = $params;
+        $this->requestUrl = $requestUrl;
+
+        $apiResponse = (string) $apiResponse;
+        $this->processedResponseText = $this->normalizeApiResponse($apiResponse);
+    }
+
+    public function getResponseText()
+    {
+        return $this->processedResponseText;
+    }
+
+    public function save($path)
+    {
+        file_put_contents($path, $this->processedResponseText);
+    }
+
+    public static function loadFromFile($path, $params, $requestUrl)
+    {
+        $contents = @file_get_contents($path);
+
+        if (empty($contents)) {
+            throw new Exception("$path does not exist");
+        }
+
+        return new Response($contents, $params, $requestUrl);
+    }
+
+    public static function loadFromApi($params, $requestUrl)
+    {
+        $testRequest = new Request($requestUrl);
+
+        // Cast as string is important. For example when calling
+        // with format=original, objects or php arrays can be returned.
+        $response = (string) $testRequest->process();
+
+        return new Response($response, $params, $requestUrl);
+    }
+
+    public static function assertEquals(Response $expected, Response $actual, $message = false)
+    {
+        $expectedText = $expected->getResponseText();
+        $actualText = $actual->getResponseText();
+
+        if ($expected->requestUrl['format'] == 'xml') {
+            Asserts::assertXmlStringEqualsXmlString($expectedText, $actualText, $message);
+            return;
+        }
+
+        // check content size to get quick feedback and avoid lengthy diff
+        $checkSizeFirst = array('pdf', 'csv');
+        if(!empty($expected->requestUrl['reportFormat'])
+            && in_array($expected->requestUrl['reportFormat'], $checkSizeFirst)) {
+            Asserts::assertEquals(strlen($expectedText), strlen($actualText), $message);
+        }
+
+        Asserts::assertEquals($expectedText, $actualText, $message);
+    }
+
+    private function normalizeApiResponse($apiResponse)
+    {
+        if ($this->shouldDeleteLiveIds()) {
+            $apiResponse = $this->removeAllIdsFromXml($apiResponse);
+        }
+
+        if ($this->shouldDeleteLiveDates()) {
+            $apiResponse = $this->removeAllLiveDatesFromXml($apiResponse);
+        } else if ($this->requestHasNonDeterministicDate()) {
+            // If date=lastN the <prettyDate> element will change each day, we remove XML element before comparison
+
+            if ($this->requestUrl['method'] == 'API.getProcessedReport') {
+                $apiResponse = $this->removeXmlElement($apiResponse, 'prettyDate');
+            }
+
+            $apiResponse = $this->removeXmlElement($apiResponse, 'visitServerHour');
+
+            $regex = "/date=[-0-9,%Ca-z]+/"; // need to remove %2C which is encoded ,
+            $apiResponse = preg_replace($regex, 'date=', $apiResponse);
+        }
+
+        // if idSubtable is in request URL, make sure idSubtable values are not in any urls
+        if (!empty($this->requestUrl['idSubtable'])) {
+            $apiResponse = $this->removeIdSubtableParamFromUrlsInResponse($apiResponse);
+        }
+
+        $apiResponse = $this->normalizePdfContent($apiResponse);
+        $apiResponse = $this->removeXmlFields($apiResponse);
+        $apiResponse = $this->normalizeDecimalFields($apiResponse);
+        $apiResponse = $this->normalizeEncodingPhp533($apiResponse);
+        $apiResponse = $this->normalizeSpaces($apiResponse);
+
+        return $apiResponse;
+    }
+
+    private function normalizeEncodingPhp533($apiResponse)
+    {
+        if (!SystemTestCase::isPhpVersion53()
+            || strpos($apiResponse, '<result') === false) {
+            return $apiResponse;
+        }
+
+        return str_replace('&amp;#039;', "'", $apiResponse);
+    }
+
+    private function removeIdSubtableParamFromUrlsInResponse($apiResponse)
+    {
+        return preg_replace("/idSubtable=[0-9]+/", 'idSubtable=', $apiResponse);
+    }
+
+    private function removeAllIdsFromXml($apiResponse)
+    {
+        $toRemove = array(
+            'visitorId',
+            'nextVisitorId',
+            'previousVisitorId',
+        );
+
+        return $this->removeXmlFields($apiResponse, $toRemove);
+    }
+
+    private function removeAllLiveDatesFromXml($apiResponse)
+    {
+        $toRemove = array(
+            'serverDate',
+            'firstActionTimestamp',
+            'lastActionTimestamp',
+            'lastActionDateTime',
+            'serverTimestamp',
+            'serverTimePretty',
+            'serverDatePretty',
+            'serverDatePrettyFirstAction',
+            'serverTimePrettyFirstAction',
+            'goalTimePretty',
+            'serverTimePretty',
+            'visitServerHour',
+            'date',
+            'prettyDate',
+            'serverDateTimePrettyFirstAction'
+        );
+        return $this->removeXmlFields($apiResponse, $toRemove);
+    }
+
+    /**
+     * Removes content from PDF binary the content that changes with the datetime or other random Ids
+     */
+    private function normalizePdfContent($response)
+    {
+        // normalize date markups and document ID in pdf files :
+        // - /LastModified (D:20120820204023+00'00')
+        // - /CreationDate (D:20120820202226+00'00')
+        // - /ModDate (D:20120820202226+00'00')
+        // - /M (D:20120820202226+00'00')
+        // - /ID [ <0f5cc387dc28c0e13e682197f485fe65> <0f5cc387dc28c0e13e682197f485fe65> ]
+        $response = preg_replace('/\(D:[0-9]{14}/', '(D:19700101000000', $response);
+        $response = preg_replace('/\/ID \[ <.*> ]/', '', $response);
+        $response = preg_replace('/\/id:\[ <.*> ]/', '', $response);
+        $response = $this->removeXmlElement($response, "xmp:CreateDate");
+        $response = $this->removeXmlElement($response, "xmp:ModifyDate");
+        $response = $this->removeXmlElement($response, "xmp:MetadataDate");
+        $response = $this->removeXmlElement($response, "xmpMM:DocumentID");
+        $response = $this->removeXmlElement($response, "xmpMM:InstanceID");
+        return $response;
+    }
+
+    private function removeXmlFields($input, $fieldsToRemove = false)
+    {
+        if ($fieldsToRemove === false) {
+            $fieldsToRemove = @$this->params['xmlFieldsToRemove'];
+        }
+
+        $fieldsToRemove[] = 'idsubdatatable';
+
+        foreach ($fieldsToRemove as $xml) {
+            $input = $this->removeXmlElement($input, $xml);
+        }
+        return $input;
+    }
+
+    private function removeXmlElement($input, $xmlElement, $testNotSmallAfter = true)
+    {
+        // Only raise error if there was some data before
+        $testNotSmallAfter = strlen($input > 100) && $testNotSmallAfter;
+
+        $oldInput = $input;
+        $input = preg_replace('/(<' . $xmlElement . '>.+?<\/' . $xmlElement . '>)/', '', $input);
+
+        // check we didn't delete the whole string
+        if ($testNotSmallAfter && $input != $oldInput) {
+            $this->assertTrue(strlen($input) > 100);
+        }
+        return $input;
+    }
+
+    private function requestHasNonDeterministicDate()
+    {
+        if (empty($this->requestUrl['date'])) {
+            return false;
+        }
+
+        $dateTime = $this->requestUrl['date'];
+        return strpos($dateTime, 'last') !== false
+            || strpos($dateTime, 'today') !== false
+            || strpos($dateTime, 'now') !== false;
+    }
+
+    private function shouldDeleteLiveIds()
+    {
+        return empty($this->params['keepLiveIds']);
+    }
+
+    private function shouldDeleteLiveDates()
+    {
+        return empty($this->params['keepLiveDates'])
+            && ($this->requestUrl['method'] == 'Live.getLastVisits'
+                || $this->requestUrl['method'] == 'Live.getLastVisitsDetails'
+                || $this->requestUrl['method'] == 'Live.getVisitorProfile');
+    }
+
+    private function normalizeDecimalFields($response)
+    {
+        // Do not test for TRUNCATE(SUM()) returning .00 on mysqli since this is not working
+        // http://bugs.php.net/bug.php?id=54508
+        $response = str_replace('.000000</l', '</l', $response); //lat/long
+        $response = str_replace('.00</revenue>', '</revenue>', $response);
+        $response = str_replace('.1</revenue>', '</revenue>', $response);
+        $response = str_replace('.11</revenue>', '</revenue>', $response);
+
+        return $response;
+    }
+
+    private function normalizeSpaces($apiResponse)
+    {
+        if (strpos($this->requestUrl['format'], 'json') === 0) {
+            $apiResponse = str_replace('&nbsp;', '\u00a0', $apiResponse);
+        }
+
+        return $apiResponse;
+    }
+}
\ No newline at end of file