diff options
author | Christian Raue <christian.raue@gmail.com> | 2014-07-23 22:30:48 +0400 |
---|---|---|
committer | Christian Raue <christian.raue@gmail.com> | 2014-07-23 22:30:48 +0400 |
commit | 9e5cae9504c6552900c287ae9db53d82b7329332 (patch) | |
tree | faf4b53f894acb45d24c9f2c1b8698d52d730cfa /misc | |
parent | d9adcfe6169c6c10059a670f2ed984908eb4e105 (diff) |
reduced multiple empty lines to exactly one
Diffstat (limited to 'misc')
-rw-r--r-- | misc/log-analytics/README.md | 7 | ||||
-rwxr-xr-x | misc/log-analytics/import_logs.py | 47 | ||||
-rw-r--r-- | misc/log-analytics/tests/tests.py | 4 | ||||
-rw-r--r-- | misc/others/download-count.txt | 2 | ||||
-rwxr-xr-x | misc/package/build.sh | 2 | ||||
-rw-r--r-- | misc/proxy-hide-piwik-url/piwik.php | 1 |
6 files changed, 0 insertions, 63 deletions
diff --git a/misc/log-analytics/README.md b/misc/log-analytics/README.md index fe7c963fa2..7842da798a 100644 --- a/misc/log-analytics/README.md +++ b/misc/log-analytics/README.md @@ -59,7 +59,6 @@ To improve performance, you can disable server access logging for these requests. Each Piwik webserver (Apache, Nginx, IIS) can also be tweaked a bit to handle more req/sec. - ## Setup Apache CustomLog that directly imports in Piwik Since apache CustomLog directives can send log data to a script, it is possible to import hits into piwik server-side in real-time rather than processing a logfile each day. @@ -90,7 +89,6 @@ Useful options here are: You can have as many CustomLog statements as you like. However, if you define any CustomLog directives within a <VirtualHost> block, all CustomLogs in the main config will be overridden. Therefore if you require custom logging for particular VirtualHosts, it is recommended to use mod_macro to make configuration more maintainable. - ## Advanced Log Analytics use case: Apache vhost, custom logs, automatic website creation As a rather extreme example of what you can do, here is an apache config with: @@ -177,7 +175,6 @@ This log format can be specified for nginx access logs to capture multiple virtu When executing import_logs.py specify the "common_complete" format. - ## Import Page Speed Metric from logs In Piwik> Actions> Page URLs and Page Title reports, Piwik reports the Avg. generation time, as an indicator of your website speed. @@ -193,7 +190,6 @@ Apache LogFormat "%h %l %u %t \"%r\" %>s %b %D" Note: the group <generation_time_milli> is also available if your server logs generation time in milliseconds rather than microseconds. - ## Setup Nginx to directly imports in Piwik via syslog With the syslog patch from http://wiki.nginx.org/3rdPartyModules which is compiled in dotdeb's release, you can log to syslog and imports them live to Piwik. @@ -201,7 +197,6 @@ Path: Nginx -> syslog -> (syslog central server) -> this script -> piwik You can use any log format that this script can handle, like Apache Combined, and Json format which needs less processing. - ### Setup Nginx logs ``` @@ -254,7 +249,5 @@ exec python /path/to/misc/log-analytics/import_logs.py \ --log-format-name=nginx_json - ``` - And that's all ! - diff --git a/misc/log-analytics/import_logs.py b/misc/log-analytics/import_logs.py index 90366304be..8b20cfe994 100755 --- a/misc/log-analytics/import_logs.py +++ b/misc/log-analytics/import_logs.py @@ -55,7 +55,6 @@ except ImportError: except ImportError: pass - ## ## Constants. ## @@ -64,7 +63,6 @@ STATIC_EXTENSIONS = set(( 'gif jpg jpeg png bmp ico svg ttf eot woff class swf css js xml robots.txt' ).split()) - DOWNLOAD_EXTENSIONS = set(( '7z aac arc arj asf asx avi bin csv deb dmg doc exe flv gz gzip hqx ' 'jar mpg mp2 mp3 mp4 mpeg mov movie msi msp odb odf odg odp ' @@ -73,7 +71,6 @@ DOWNLOAD_EXTENSIONS = set(( 'azw3 epub mobi' ).split()) - # A good source is: http://phpbb-bots.blogspot.com/ EXCLUDED_USER_AGENTS = ( 'adsbot-google', @@ -104,7 +101,6 @@ EXCLUDED_USER_AGENTS = ( 'yandex', ) - PIWIK_MAX_ATTEMPTS = 3 PIWIK_DELAY_AFTER_FAILURE = 2 @@ -112,8 +108,6 @@ PIWIK_EXPECTED_IMAGE = base64.b64decode( 'R0lGODlhAQABAIAAAAAAAAAAACH5BAEAAAAALAAAAAABAAEAAAICRAEAOw==' ) - - ## ## Formats. ## @@ -134,7 +128,6 @@ class BaseFormat(object): def check_format_line(self, line): return False - class JsonFormat(BaseFormat): def __init__(self, name): super(JsonFormat, self).__init__(name) @@ -174,8 +167,6 @@ class JsonFormat(BaseFormat): def get_all(self,): return self.json - - class RegexFormat(BaseFormat): def __init__(self, name, regex, date_format=None): @@ -202,9 +193,6 @@ class RegexFormat(BaseFormat): def get_all(self,): return self.matched.groupdict() - - - class IisFormat(RegexFormat): def __init__(self): @@ -248,8 +236,6 @@ class IisFormat(RegexFormat): file.seek(start_pos) return self.check_format_line(nextline) - - _HOST_PREFIX = '(?P<host>[\w\-\.]*)(?::\d+)? ' _COMMON_LOG_FORMAT = ( '(?P<ip>\S+) \S+ \S+ \[(?P<date>.*?) (?P<timezone>.*?)\] ' @@ -278,14 +264,10 @@ FORMATS = { 'nginx_json': JsonFormat('nginx_json'), } - - - ## ## Code. ## - class Configuration(object): """ Stores all the configuration options by reading sys.argv and parsing, @@ -507,7 +489,6 @@ class Configuration(object): ) return option_parser - def _parse_args(self, option_parser): """ Parse the command line args and create self.options and self.filenames. @@ -575,11 +556,9 @@ class Configuration(object): if self.options.recorders < 1: self.options.recorders = 1 - def __init__(self): self._parse_args(self._create_parser()) - def _get_token_auth(self): """ If the token auth is not specified in the options, get it from Piwik. @@ -644,7 +623,6 @@ class Configuration(object): except: fatal_error("We couldn't detect PHP. You can run the importer using the --login and --password option to fix this issue") - command = [phpBinary, updatetokenfile] if self.options.enable_testmode: command.append('--testmode') @@ -655,13 +633,11 @@ class Configuration(object): if process.returncode != 0: fatal_error("`" + command + "` failed with error: " + stderr + ".\nReponse code was: " + str(process.returncode) + ". You can alternatively run the importer using the --login and --password option") - filename = stdout credentials = open(filename, 'r').readline() credentials = credentials.split('\t') return credentials[1] - def get_resolver(self): if self.options.site_id: logging.debug('Resolver: static') @@ -670,8 +646,6 @@ class Configuration(object): logging.debug('Resolver: dynamic') return DynamicResolver() - - class Statistics(object): """ Store statistics about parsed logs and recorded entries. @@ -699,7 +673,6 @@ class Statistics(object): def __str__(self): return str(int(self.value)) - def __init__(self): self.time_start = None self.time_stop = None @@ -732,7 +705,6 @@ class Statistics(object): self.dates_recorded = set() self.monitor_stop = False - def set_time_start(self): self.time_start = time.time() @@ -848,7 +820,6 @@ Performance summary )), } - ## ## The monitor is a thread that prints a short summary each second. ## @@ -875,8 +846,6 @@ Performance summary def stop_monitor(self): self.monitor_stop = True - - class Piwik(object): """ Make requests to Piwik. @@ -950,7 +919,6 @@ class Piwik(object): truncate_after = 4000 raise urllib2.URLError('Piwik returned an invalid response: ' + res[:truncate_after]) - @staticmethod def _call_wrapper(func, expected_response, on_failure, *args, **kwargs): """ @@ -994,7 +962,6 @@ class Piwik(object): def call_api(cls, method, **kwargs): return cls._call_wrapper(cls._call_api, None, None, method, **kwargs) - ## ## Resolvers. ## @@ -1033,7 +1000,6 @@ class StaticResolver(object): def check_format(self, format): pass - class DynamicResolver(object): """ Use Piwik API to determine the site ID. @@ -1144,7 +1110,6 @@ class DynamicResolver(object): else: return self._resolve_by_host(hit) - def check_format(self, format): if config.options.replay_tracking: pass @@ -1154,9 +1119,6 @@ class DynamicResolver(object): "specify the Piwik site ID with the --idsite argument" ) - - - class Recorder(object): """ A Recorder fetches hits from the Queue and inserts them into Piwik using @@ -1374,7 +1336,6 @@ class Recorder(object): '$ /path/to/piwik/console core:archive --url=http://example/piwik/\n' '\nReference: http://piwik.org/docs/setup-auto-archiving/ ') - class Hit(object): """ It's a simple container. @@ -1387,7 +1348,6 @@ class Hit(object): if config.options.force_lowercase_path: self.full_path = self.full_path.lower() - class Parser(object): """ The Parser parses the lines in a specified file and inserts them into @@ -1399,7 +1359,6 @@ class Parser(object): in inspect.getmembers(self, predicate=inspect.ismethod) if name.startswith('check_')] - ## All check_* methods are called for each hit and must return True if the ## hit can be imported, False otherwise. @@ -1741,9 +1700,6 @@ class Parser(object): if len(hits) > 0: Recorder.add_hits(hits) - - - def main(): """ Start the importing process. @@ -1774,8 +1730,6 @@ def main(): pass stats.print_summary() - - def fatal_error(error, filename=None, lineno=None): print >> sys.stderr, 'Fatal error: %s' % error if filename and lineno is not None: @@ -1785,7 +1739,6 @@ def fatal_error(error, filename=None, lineno=None): ) os._exit(1) - if __name__ == '__main__': try: piwik = Piwik() diff --git a/misc/log-analytics/tests/tests.py b/misc/log-analytics/tests/tests.py index 5d42e51797..163c100d3a 100644 --- a/misc/log-analytics/tests/tests.py +++ b/misc/log-analytics/tests/tests.py @@ -44,7 +44,6 @@ def test_format_detection(): f.description = 'Testing autodetection of format ' + format_name + ' w/ garbage at end of line' yield f - class Options(object): """Mock config options necessary to run checkers from Parser class.""" debug = False @@ -65,19 +64,16 @@ class Options(object): included_paths = [] enable_http_errors = False - class Config(object): """Mock configuration.""" options = Options() format = import_logs.FORMATS['ncsa_extended'] - class Resolver(object): """Mock resolver which doesn't check connection to real piwik.""" def check_format(self, format_): pass - class Recorder(object): """Mock recorder which collects hits but doesn't put their in database.""" recorders = [] diff --git a/misc/others/download-count.txt b/misc/others/download-count.txt index 924d71ec38..3be6964b6f 100644 --- a/misc/others/download-count.txt +++ b/misc/others/download-count.txt @@ -2,10 +2,8 @@ Count the download for 'latest.zip' on the 20th March # cat access.log | grep "20/Mar" | grep "latest.zip" | awk '{print $1}' | sort | uniq | wc -l - Value to be compared with the one given by Piwik in Actions > Downloads - Count the no of hits by referrers, excluding piwik.org as a referer # cat /var/log/apache2/access.log | awk '{print $11}' | grep -vE "(^"-"$|/dev.piwik.org|/piwik.org)" | sort | uniq -c | sort -rn | head -n20 diff --git a/misc/package/build.sh b/misc/package/build.sh index 35b230f47a..5e5c3a944e 100755 --- a/misc/package/build.sh +++ b/misc/package/build.sh @@ -92,7 +92,6 @@ if [ -z "$VERSION" ]; then die "Expected a version number as a parameter" fi - # Fail script, if Piwik has already been built, to prevent re-building a given release destination=$HTTP_PATH/piwik-$VERSION.zip if [ -f "$destination" ] @@ -215,6 +214,5 @@ Piwik team" | mail -s"New Piwik Version $VERSION" "appgal@microsoft.com,hello@pi echo "build finished! http://builds.piwik.org/piwik.zip" fi - cleanupWorkspace diff --git a/misc/proxy-hide-piwik-url/piwik.php b/misc/proxy-hide-piwik-url/piwik.php index 7ec88affd6..846a82313b 100644 --- a/misc/proxy-hide-piwik-url/piwik.php +++ b/misc/proxy-hide-piwik-url/piwik.php @@ -24,7 +24,6 @@ $TOKEN_AUTH = 'xyz'; // Maximum time, in seconds, to wait for the Piwik server to return the 1*1 GIF $timeout = 5; - // DO NOT MODIFY BELOW // --------------------------- // 1) PIWIK.JS PROXY: No _GET parameter, we serve the JS file |