Fixes #3805, reverted change in log importer that looked for end-of-line after format regex match and modified format autodetection logic to pick the format based on whether the format matches and the number of groups returned in the match.

Notes:
  * Added several more tests to log importer tests.py. Added tests for checking format of log files w/ extra junk info on log lines. Added individual tests for parsing regex format.
  * Modified log files used in ImportLogs test, added extra junk info to end of some lines.
  * Fixed failing test in tests.py for the S3 log file format.

2 files changed, 3 insertions, 3 deletions

diff --git a/tests/resources/fake_logs.log b/tests/resources/fake_logs.log
index 681104d57a..abd10d768d 100755
--- a/tests/resources/fake_logs.log
+++ b/tests/resources/fake_logs.log
@@ -2,10 +2,10 @@
 175.41.192.40 - - [09/Aug/2012:10:11:30 +0200] "GET /faq/ HTTP/1.1" 200 3574 "-" "Mozilla/5.0 (Linux; U; Android 2.3.5; en-us; HTC Vision Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1"
 175.41.192.40 - - [09/Aug/2012:10:11:56 +0200] "GET /blog/category/community/ HTTP/1.1" 200 3574 "-" "Mozilla/5.0 (X11; U; Linux x86_64; ca-ad) AppleWebKit/531.2+ (KHTML, like Gecko) Safari/531.2+ Epiphany/2.30.6"
 175.41.192.40 - - [09/Aug/2012:10:12:03 +0200] "GET /docs/manage-websites/ HTTP/1.1" 200 3574 "-" "Mozilla/5.0 (X11; Linux i686; rv:6.0) Gecko/20100101 Firefox/6.0"
-72.44.32.10 - - [09/Aug/2012:15:48:07 +0200] "GET / HTTP/1.1" 200 3574 "-" "Mozilla/5.0 (compatible; MSIE 10.6; Windows NT 6.1; Trident/5.0; InfoPath.2; SLCC1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 2.0.50727) 3gpp-gba UNTRUSTED/1.0"
+72.44.32.10 - - [09/Aug/2012:15:48:07 +0200] "GET / HTTP/1.1" 200 3574 "-" "Mozilla/5.0 (compatible; MSIE 10.6; Windows NT 6.1; Trident/5.0; InfoPath.2; SLCC1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 2.0.50727) 3gpp-gba UNTRUSTED/1.0" junk extra
 72.44.32.10 - - [09/Aug/2012:15:48:20 +0200] "GET /download/counter/ HTTP/1.1" 200 3574 "-" "Mozilla/5.0 (X11; U; Linux x86; en-US) AppleWebKit/534.7 (KHTML, like Gecko) Epiphany/2.30.6 Safari/534.7"
 72.44.32.10 - - [09/Aug/2012:15:49:48 +0200] "GET /translations/ HTTP/1.1" 200 3574 "-" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5"
-175.41.192.09 - - [09/Aug/2012:22:56:45 +0200] "GET /docs/ HTTP/1.1" 200 3574 "-" "Mozilla/5.0 (X11; Linux i686; rv:6.0) Gecko/20100101 Firefox/6.0"
+175.41.192.09 - - [09/Aug/2012:22:56:45 +0200] "GET /docs/ HTTP/1.1" 200 3574 "-" "Mozilla/5.0 (X11; Linux i686; rv:6.0) Gecko/20100101 Firefox/6.0" 456 789
 175.41.192.09 - - [09/Aug/2012:23:00:42 +0200] "GET /docs/manage-users/ HTTP/1.1" 200 3574 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3"
 79.125.00.21 - - [10/Aug/2012:20:03:40 +0200] "GET /newsletter/ HTTP/1.1" 200 3574 "-" "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/5.0)"
 175.41.192.34 - - [10/Aug/2012:21:59:50 +0200] "GET /faq/how-to/ HTTP/1.1" 200 3574 "-" "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/5.0)"
diff --git a/tests/resources/fake_logs_enable_all.log b/tests/resources/fake_logs_enable_all.log
index 4e120a4f8e..a8f2a58346 100755
--- a/tests/resources/fake_logs_enable_all.log
+++ b/tests/resources/fake_logs_enable_all.log
@@ -1,5 +1,5 @@
 1.2.3.4 - - [11/Aug/2012:18:46:03 +0100] "GET /19.pdf HTTP/1.0" 200 4324023 "-" "Apache-HttpClient/4.2.1 (java 1.5)"
-175.41.192.41 - - [11/Aug/2012:18:10:38 +0200] "GET /blog/category/meta/ HTTP/1.1" 200 3574 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_7) AppleWebKit/534.24 (KHTML, like Gecko) RockMelt/0.9.58.494 Chrome/11.0.696.71 Safari/534.24"
+175.41.192.41 - - [11/Aug/2012:18:10:38 +0200] "GET /blog/category/meta/ HTTP/1.1" 200 3574 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_7) AppleWebKit/534.24 (KHTML, like Gecko) RockMelt/0.9.58.494 Chrome/11.0.696.71 Safari/534.24" junk extra
 175.41.192.41 - - [11/Aug/2012:18:11:30 +0200] "GET /this/is/not/the/page/i/am/looking/for/ HTTP/1.1" 404 3574 "-" "Mozilla/5.0 (Linux; U; Android 2.3.5; en-us; HTC Vision Build/GRI40) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1"
 72.44.32.11 - - [11/Aug/2012:19:48:07 +0200] "GET /to-an-error HTTP/1.1" 500 3574 "-" "Mozilla/5.0 (compatible; MSIE 10.6; Windows NT 6.1; Trident/5.0; InfoPath.2; SLCC1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 2.0.50727) 3gpp-gba UNTRUSTED/1.0"
 72.44.32.11 - - [11/Aug/2012:19:48:08 +0200] "GET / HTTP/1.1" 200 3574 "-" "Googlebot/2.1 (+http://www.googlebot.com/bot.html)"