Adding support for parsing amazon cloudfront log files in log importer & making W3cExtendedFormat treat time-taken field as seconds by default (since that is what the W3C format specifies). Also add support for event tracking and cloudfront RTMP logs (which provide event logs). Also allow multiple spaces in between W3C extended log files. Allow derived classes of W3cExtendedFormat to define new custom field regexes.

2 files changed, 16 insertions, 6 deletions

diff --git a/misc/log-analytics/import_logs.py b/misc/log-analytics/import_logs.py
index dfd8236d4a..889353377b 100755
--- a/misc/log-analytics/import_logs.py
+++ b/misc/log-analytics/import_logs.py
@@ -254,8 +254,6 @@ class W3cExtendedFormat(RegexFormat):
                 regex = '\S+'
             full_regex.append(regex)
         full_regex = '\s+'.join(full_regex)
-        logging.debug(full_regex)
-        logging.debug(first_line)
         self.regex = re.compile(full_regex)
 
         file.seek(0)
@@ -1392,8 +1390,15 @@ class Recorder(object):
 			)
 
         if hit.generation_time_milli > 0:
-            args['gt_ms'] = hit.generation_time_milli
-        logging.debug(args)
+            args['gt_ms'] = int(hit.generation_time_milli)
+
+        if hit.event_category and hit.event_action:
+            args['e_c'] = hit.event_category
+            args['e_a'] = hit.event_action
+
+            if hit.event_name:
+                args['e_n'] = hit.event_name
+
         return args
 
     def _record_hits(self, hits):
@@ -1583,7 +1588,7 @@ class Parser(object):
                 else:
                     match = candidate_format.check_format(lineOrFile)
             except Exception, e:
-                logging.debug(str(e))
+                logging.debug('Error in format checking: %s', str(e))
                 pass
 
             if match:
@@ -1790,9 +1795,14 @@ class Parser(object):
 
             # add event info
             try:
+                hit.event_category = hit.event_action = hit.event_name = None
+
                 hit.event_category = format.get('event_category')
                 hit.event_action = format.get('event_action')
+
                 hit.event_name = format.get('event_name')
+                if hit.event_name == '-':
+                    hit.event_name = None
             except:
                 pass
 
diff --git a/misc/log-analytics/tests/tests.py b/misc/log-analytics/tests/tests.py
index 6e0051c5ed..39e4bc7dbb 100644
--- a/misc/log-analytics/tests/tests.py
+++ b/misc/log-analytics/tests/tests.py
@@ -342,7 +342,7 @@ def test_iis_custom_format():
     import_logs.config.options.enable_http_redirects = True
     import_logs.config.options.enable_http_errors = True
     import_logs.config.options.replay_tracking = False
-    # import_logs.config.options.w3c_time_taken_in_millisecs = True test that even w/o this, we get the right format
+    # import_logs.config.options.w3c_time_taken_in_millisecs = True test that even w/o this, we get the right values
     import_logs.parser.parse(file_)
 
     hits = [hit.__dict__ for hit in Recorder.recorders]