diff options
author | Tomek Wójcik <t.wojcik@clearcode.cc> | 2013-02-20 18:22:21 +0400 |
---|---|---|
committer | Tomek Wójcik <t.wojcik@clearcode.cc> | 2013-02-20 18:22:21 +0400 |
commit | 17166e5dbb6a70029233df22cd65ab079f1d3aa9 (patch) | |
tree | 76e88a90313218c38cb2bdf114be70af02798aa3 /misc | |
parent | f10f5ea290ba1eeb65527bd3ca59279a0dd2a87f (diff) |
Reverted static methods to instance methods.
Diffstat (limited to 'misc')
-rwxr-xr-x | misc/log-analytics/import_logs.py | 44 | ||||
-rw-r--r-- | misc/log-analytics/tests/tests.py | 10 |
2 files changed, 27 insertions, 27 deletions
diff --git a/misc/log-analytics/import_logs.py b/misc/log-analytics/import_logs.py index acb7951bb5..265a068551 100755 --- a/misc/log-analytics/import_logs.py +++ b/misc/log-analytics/import_logs.py @@ -1176,13 +1176,17 @@ class Parser(object): The Parser parses the lines in a specified file and inserts them into a Queue. """ - check_methods = [] + + def __init__(self): + self.check_methods = [method for name, method + in inspect.getmembers(self, predicate=inspect.ismethod) + if name.startswith('check_')] + ## All check_* methods are called for each hit and must return True if the ## hit can be imported, False otherwise. - @staticmethod - def check_hostname(hit): + def check_hostname(self, hit): # Check against config.hostnames. if not hasattr(hit, 'host') or not config.options.hostnames: return True @@ -1196,8 +1200,7 @@ class Parser(object): stats.count_lines_hostname_skipped.increment() return result - @staticmethod - def check_static(hit): + def check_static(self, hit): extension = hit.path.rsplit('.')[-1].lower() if extension in STATIC_EXTENSIONS: if config.options.enable_static: @@ -1208,16 +1211,14 @@ class Parser(object): return False return True - @staticmethod - def check_download(hit): + def check_download(self, hit): extension = hit.path.rsplit('.')[-1].lower() if extension in DOWNLOAD_EXTENSIONS: stats.count_lines_downloads.increment() hit.is_download = True return True - @staticmethod - def check_user_agent(hit): + def check_user_agent(self, hit): user_agent = hit.user_agent.lower() for s in itertools.chain(EXCLUDED_USER_AGENTS, config.options.excluded_useragents): if s in user_agent: @@ -1229,8 +1230,7 @@ class Parser(object): return False return True - @staticmethod - def check_http_error(hit): + def check_http_error(self, hit): if hit.status[0] in ('4', '5'): if config.options.enable_http_errors: hit.is_error = True @@ -1240,8 +1240,7 @@ class Parser(object): return False return True - @staticmethod - def check_http_redirect(hit): + def check_http_redirect(self, hit): if hit.status[0] == '3' and hit.status != '304': if config.options.enable_http_redirects: hit.is_redirect = True @@ -1251,8 +1250,7 @@ class Parser(object): return False return True - @staticmethod - def check_path(hit): + def check_path(self, hit): for excluded_path in config.options.excluded_paths: if fnmatch.fnmatch(hit.path, excluded_path): return False @@ -1272,8 +1270,7 @@ class Parser(object): else: logging.debug('Format %s does not match', name) - @classmethod - def parse(cls, filename): + def parse(self, filename): """ Parse the specified filename and insert hits in the queue. """ @@ -1311,13 +1308,12 @@ class Parser(object): return file.seek(0) - format = cls.detect_format(file) + format = self.detect_format(file) if format is None: return fatal_error( 'Cannot guess the logs format. Please give one using ' 'either the --log-format-name or --log-format-regex option' ) - # Make sure the format is compatible with the resolver. resolver.check_format(format) @@ -1385,7 +1381,7 @@ class Parser(object): pass # Check if the hit must be excluded. - if not all((getattr(cls, name)(hit) for name in cls.check_methods)): + if not all((method(hit) for method in self.check_methods)): continue # Parse date. @@ -1411,7 +1407,7 @@ class Parser(object): hit.date -= datetime.timedelta(hours=timezone/100) # Check if the hit must be excluded. - if all((getattr(cls, name)(hit) for name in cls.check_methods)): + if all((method(hit) for method in self.check_methods)): hits.append(hit) if len(hits) >= config.options.recorder_max_payload_size * len(Recorder.recorders): @@ -1428,14 +1424,12 @@ class Parser(object): except UnicodeDecodeError: invalid_line(line, 'invalid encoding') continue - # add last chunk of hits if len(hits) > 0: Recorder.add_hits(hits) -for name, method in inspect.getmembers(Parser, predicate=callable): - if name.startswith('check_'): - Parser.check_methods.append(name) + + def main(): """ diff --git a/misc/log-analytics/tests/tests.py b/misc/log-analytics/tests/tests.py index 295229d41a..55b7194317 100644 --- a/misc/log-analytics/tests/tests.py +++ b/misc/log-analytics/tests/tests.py @@ -13,6 +13,7 @@ def test_format_detection(): f.description = 'Testing autodetection of format ' + format_name yield f + class Options(object): """Mock config options necessary to run checkers from Parser class.""" debug = False @@ -30,16 +31,19 @@ class Options(object): excluded_useragents = [] enable_bots = [] + class Config(object): """Mock configuration.""" options = Options() format = import_logs.FORMATS['ncsa_extended'] + class Resolver(object): """Mock resolver which doesn't check connection to real piwik.""" def check_format(self, format_): pass + class Recorder(object): """Mock recorder which collects hits but doesn't put their in database.""" recorders = [] @@ -48,15 +52,17 @@ class Recorder(object): def add_hits(cls, hits): cls.recorders.extend(hits) + def test_replay_tracking_arguments(): """Test data parsing from sample log file.""" + file_ = 'logs_to_tests.log' import_logs.stats = import_logs.Statistics() import_logs.config = Config() import_logs.resolver = Resolver() import_logs.Recorder = Recorder() + import_logs.parser = import_logs.Parser() + import_logs.parser.parse(file_) - file_ = 'logs_to_tests.log' - parser = import_logs.Parser.parse(file_) hits = [hit.args for hit in import_logs.Recorder.recorders] assert hits[0]['_idn'] == '0' |