Welcome to mirror list, hosted at ThFree Co, Russian Federation.

github.com/matomo-org/matomo.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
path: root/misc
diff options
context:
space:
mode:
authorChristian Raue <christian.raue@gmail.com>2014-07-23 22:30:48 +0400
committerChristian Raue <christian.raue@gmail.com>2014-07-23 22:30:48 +0400
commit9e5cae9504c6552900c287ae9db53d82b7329332 (patch)
treefaf4b53f894acb45d24c9f2c1b8698d52d730cfa /misc
parentd9adcfe6169c6c10059a670f2ed984908eb4e105 (diff)
reduced multiple empty lines to exactly one
Diffstat (limited to 'misc')
-rw-r--r--misc/log-analytics/README.md7
-rwxr-xr-xmisc/log-analytics/import_logs.py47
-rw-r--r--misc/log-analytics/tests/tests.py4
-rw-r--r--misc/others/download-count.txt2
-rwxr-xr-xmisc/package/build.sh2
-rw-r--r--misc/proxy-hide-piwik-url/piwik.php1
6 files changed, 0 insertions, 63 deletions
diff --git a/misc/log-analytics/README.md b/misc/log-analytics/README.md
index fe7c963fa2..7842da798a 100644
--- a/misc/log-analytics/README.md
+++ b/misc/log-analytics/README.md
@@ -59,7 +59,6 @@ To improve performance,
you can disable server access logging for these requests.
Each Piwik webserver (Apache, Nginx, IIS) can also be tweaked a bit to handle more req/sec.
-
## Setup Apache CustomLog that directly imports in Piwik
Since apache CustomLog directives can send log data to a script, it is possible to import hits into piwik server-side in real-time rather than processing a logfile each day.
@@ -90,7 +89,6 @@ Useful options here are:
You can have as many CustomLog statements as you like. However, if you define any CustomLog directives within a <VirtualHost> block, all CustomLogs in the main config will be overridden. Therefore if you require custom logging for particular VirtualHosts, it is recommended to use mod_macro to make configuration more maintainable.
-
## Advanced Log Analytics use case: Apache vhost, custom logs, automatic website creation
As a rather extreme example of what you can do, here is an apache config with:
@@ -177,7 +175,6 @@ This log format can be specified for nginx access logs to capture multiple virtu
When executing import_logs.py specify the "common_complete" format.
-
## Import Page Speed Metric from logs
In Piwik> Actions> Page URLs and Page Title reports, Piwik reports the Avg. generation time, as an indicator of your website speed.
@@ -193,7 +190,6 @@ Apache LogFormat "%h %l %u %t \"%r\" %>s %b %D"
Note: the group <generation_time_milli> is also available if your server logs generation time in milliseconds rather than microseconds.
-
## Setup Nginx to directly imports in Piwik via syslog
With the syslog patch from http://wiki.nginx.org/3rdPartyModules which is compiled in dotdeb's release, you can log to syslog and imports them live to Piwik.
@@ -201,7 +197,6 @@ Path: Nginx -> syslog -> (syslog central server) -> this script -> piwik
You can use any log format that this script can handle, like Apache Combined, and Json format which needs less processing.
-
### Setup Nginx logs
```
@@ -254,7 +249,5 @@ exec python /path/to/misc/log-analytics/import_logs.py \
--log-format-name=nginx_json -
```
-
And that's all !
-
diff --git a/misc/log-analytics/import_logs.py b/misc/log-analytics/import_logs.py
index 90366304be..8b20cfe994 100755
--- a/misc/log-analytics/import_logs.py
+++ b/misc/log-analytics/import_logs.py
@@ -55,7 +55,6 @@ except ImportError:
except ImportError:
pass
-
##
## Constants.
##
@@ -64,7 +63,6 @@ STATIC_EXTENSIONS = set((
'gif jpg jpeg png bmp ico svg ttf eot woff class swf css js xml robots.txt'
).split())
-
DOWNLOAD_EXTENSIONS = set((
'7z aac arc arj asf asx avi bin csv deb dmg doc exe flv gz gzip hqx '
'jar mpg mp2 mp3 mp4 mpeg mov movie msi msp odb odf odg odp '
@@ -73,7 +71,6 @@ DOWNLOAD_EXTENSIONS = set((
'azw3 epub mobi'
).split())
-
# A good source is: http://phpbb-bots.blogspot.com/
EXCLUDED_USER_AGENTS = (
'adsbot-google',
@@ -104,7 +101,6 @@ EXCLUDED_USER_AGENTS = (
'yandex',
)
-
PIWIK_MAX_ATTEMPTS = 3
PIWIK_DELAY_AFTER_FAILURE = 2
@@ -112,8 +108,6 @@ PIWIK_EXPECTED_IMAGE = base64.b64decode(
'R0lGODlhAQABAIAAAAAAAAAAACH5BAEAAAAALAAAAAABAAEAAAICRAEAOw=='
)
-
-
##
## Formats.
##
@@ -134,7 +128,6 @@ class BaseFormat(object):
def check_format_line(self, line):
return False
-
class JsonFormat(BaseFormat):
def __init__(self, name):
super(JsonFormat, self).__init__(name)
@@ -174,8 +167,6 @@ class JsonFormat(BaseFormat):
def get_all(self,):
return self.json
-
-
class RegexFormat(BaseFormat):
def __init__(self, name, regex, date_format=None):
@@ -202,9 +193,6 @@ class RegexFormat(BaseFormat):
def get_all(self,):
return self.matched.groupdict()
-
-
-
class IisFormat(RegexFormat):
def __init__(self):
@@ -248,8 +236,6 @@ class IisFormat(RegexFormat):
file.seek(start_pos)
return self.check_format_line(nextline)
-
-
_HOST_PREFIX = '(?P<host>[\w\-\.]*)(?::\d+)? '
_COMMON_LOG_FORMAT = (
'(?P<ip>\S+) \S+ \S+ \[(?P<date>.*?) (?P<timezone>.*?)\] '
@@ -278,14 +264,10 @@ FORMATS = {
'nginx_json': JsonFormat('nginx_json'),
}
-
-
-
##
## Code.
##
-
class Configuration(object):
"""
Stores all the configuration options by reading sys.argv and parsing,
@@ -507,7 +489,6 @@ class Configuration(object):
)
return option_parser
-
def _parse_args(self, option_parser):
"""
Parse the command line args and create self.options and self.filenames.
@@ -575,11 +556,9 @@ class Configuration(object):
if self.options.recorders < 1:
self.options.recorders = 1
-
def __init__(self):
self._parse_args(self._create_parser())
-
def _get_token_auth(self):
"""
If the token auth is not specified in the options, get it from Piwik.
@@ -644,7 +623,6 @@ class Configuration(object):
except:
fatal_error("We couldn't detect PHP. You can run the importer using the --login and --password option to fix this issue")
-
command = [phpBinary, updatetokenfile]
if self.options.enable_testmode:
command.append('--testmode')
@@ -655,13 +633,11 @@ class Configuration(object):
if process.returncode != 0:
fatal_error("`" + command + "` failed with error: " + stderr + ".\nReponse code was: " + str(process.returncode) + ". You can alternatively run the importer using the --login and --password option")
-
filename = stdout
credentials = open(filename, 'r').readline()
credentials = credentials.split('\t')
return credentials[1]
-
def get_resolver(self):
if self.options.site_id:
logging.debug('Resolver: static')
@@ -670,8 +646,6 @@ class Configuration(object):
logging.debug('Resolver: dynamic')
return DynamicResolver()
-
-
class Statistics(object):
"""
Store statistics about parsed logs and recorded entries.
@@ -699,7 +673,6 @@ class Statistics(object):
def __str__(self):
return str(int(self.value))
-
def __init__(self):
self.time_start = None
self.time_stop = None
@@ -732,7 +705,6 @@ class Statistics(object):
self.dates_recorded = set()
self.monitor_stop = False
-
def set_time_start(self):
self.time_start = time.time()
@@ -848,7 +820,6 @@ Performance summary
)),
}
-
##
## The monitor is a thread that prints a short summary each second.
##
@@ -875,8 +846,6 @@ Performance summary
def stop_monitor(self):
self.monitor_stop = True
-
-
class Piwik(object):
"""
Make requests to Piwik.
@@ -950,7 +919,6 @@ class Piwik(object):
truncate_after = 4000
raise urllib2.URLError('Piwik returned an invalid response: ' + res[:truncate_after])
-
@staticmethod
def _call_wrapper(func, expected_response, on_failure, *args, **kwargs):
"""
@@ -994,7 +962,6 @@ class Piwik(object):
def call_api(cls, method, **kwargs):
return cls._call_wrapper(cls._call_api, None, None, method, **kwargs)
-
##
## Resolvers.
##
@@ -1033,7 +1000,6 @@ class StaticResolver(object):
def check_format(self, format):
pass
-
class DynamicResolver(object):
"""
Use Piwik API to determine the site ID.
@@ -1144,7 +1110,6 @@ class DynamicResolver(object):
else:
return self._resolve_by_host(hit)
-
def check_format(self, format):
if config.options.replay_tracking:
pass
@@ -1154,9 +1119,6 @@ class DynamicResolver(object):
"specify the Piwik site ID with the --idsite argument"
)
-
-
-
class Recorder(object):
"""
A Recorder fetches hits from the Queue and inserts them into Piwik using
@@ -1374,7 +1336,6 @@ class Recorder(object):
'$ /path/to/piwik/console core:archive --url=http://example/piwik/\n'
'\nReference: http://piwik.org/docs/setup-auto-archiving/ ')
-
class Hit(object):
"""
It's a simple container.
@@ -1387,7 +1348,6 @@ class Hit(object):
if config.options.force_lowercase_path:
self.full_path = self.full_path.lower()
-
class Parser(object):
"""
The Parser parses the lines in a specified file and inserts them into
@@ -1399,7 +1359,6 @@ class Parser(object):
in inspect.getmembers(self, predicate=inspect.ismethod)
if name.startswith('check_')]
-
## All check_* methods are called for each hit and must return True if the
## hit can be imported, False otherwise.
@@ -1741,9 +1700,6 @@ class Parser(object):
if len(hits) > 0:
Recorder.add_hits(hits)
-
-
-
def main():
"""
Start the importing process.
@@ -1774,8 +1730,6 @@ def main():
pass
stats.print_summary()
-
-
def fatal_error(error, filename=None, lineno=None):
print >> sys.stderr, 'Fatal error: %s' % error
if filename and lineno is not None:
@@ -1785,7 +1739,6 @@ def fatal_error(error, filename=None, lineno=None):
)
os._exit(1)
-
if __name__ == '__main__':
try:
piwik = Piwik()
diff --git a/misc/log-analytics/tests/tests.py b/misc/log-analytics/tests/tests.py
index 5d42e51797..163c100d3a 100644
--- a/misc/log-analytics/tests/tests.py
+++ b/misc/log-analytics/tests/tests.py
@@ -44,7 +44,6 @@ def test_format_detection():
f.description = 'Testing autodetection of format ' + format_name + ' w/ garbage at end of line'
yield f
-
class Options(object):
"""Mock config options necessary to run checkers from Parser class."""
debug = False
@@ -65,19 +64,16 @@ class Options(object):
included_paths = []
enable_http_errors = False
-
class Config(object):
"""Mock configuration."""
options = Options()
format = import_logs.FORMATS['ncsa_extended']
-
class Resolver(object):
"""Mock resolver which doesn't check connection to real piwik."""
def check_format(self, format_):
pass
-
class Recorder(object):
"""Mock recorder which collects hits but doesn't put their in database."""
recorders = []
diff --git a/misc/others/download-count.txt b/misc/others/download-count.txt
index 924d71ec38..3be6964b6f 100644
--- a/misc/others/download-count.txt
+++ b/misc/others/download-count.txt
@@ -2,10 +2,8 @@ Count the download for 'latest.zip' on the 20th March
# cat access.log | grep "20/Mar" | grep "latest.zip" | awk '{print $1}' | sort | uniq | wc -l
-
Value to be compared with the one given by Piwik in Actions > Downloads
-
Count the no of hits by referrers, excluding piwik.org as a referer
# cat /var/log/apache2/access.log | awk '{print $11}' | grep -vE "(^"-"$|/dev.piwik.org|/piwik.org)" | sort | uniq -c | sort -rn | head -n20
diff --git a/misc/package/build.sh b/misc/package/build.sh
index 35b230f47a..5e5c3a944e 100755
--- a/misc/package/build.sh
+++ b/misc/package/build.sh
@@ -92,7 +92,6 @@ if [ -z "$VERSION" ]; then
die "Expected a version number as a parameter"
fi
-
# Fail script, if Piwik has already been built, to prevent re-building a given release
destination=$HTTP_PATH/piwik-$VERSION.zip
if [ -f "$destination" ]
@@ -215,6 +214,5 @@ Piwik team" | mail -s"New Piwik Version $VERSION" "appgal@microsoft.com,hello@pi
echo "build finished! http://builds.piwik.org/piwik.zip"
fi
-
cleanupWorkspace
diff --git a/misc/proxy-hide-piwik-url/piwik.php b/misc/proxy-hide-piwik-url/piwik.php
index 7ec88affd6..846a82313b 100644
--- a/misc/proxy-hide-piwik-url/piwik.php
+++ b/misc/proxy-hide-piwik-url/piwik.php
@@ -24,7 +24,6 @@ $TOKEN_AUTH = 'xyz';
// Maximum time, in seconds, to wait for the Piwik server to return the 1*1 GIF
$timeout = 5;
-
// DO NOT MODIFY BELOW
// ---------------------------
// 1) PIWIK.JS PROXY: No _GET parameter, we serve the JS file