Add new --w3c-field-regex option to log importer which allows specifying regex for any w3c extended log field. Can combine this option w/ --regex-group-to-...-cvar options to track any field that the importer doesn't natively recognize.

4 files changed, 64 insertions, 13 deletions

diff --git a/tests/PHPUnit/Fixtures/ManySitesImportedLogs.php b/tests/PHPUnit/Fixtures/ManySitesImportedLogs.php
index 1dd043200f..c7e65c7499 100644
--- a/tests/PHPUnit/Fixtures/ManySitesImportedLogs.php
+++ b/tests/PHPUnit/Fixtures/ManySitesImportedLogs.php
@@ -272,8 +272,13 @@ class ManySitesImportedLogs extends Fixture
 
         if ($mapToCustom) {
             $opts['--regex-group-to-visit-cvar'] = 'userid=User Name';
-            $opts['--regex-group-to-page-cvar'] = 'generation_time_milli=Generation Time';
+            $opts['--regex-group-to-page-cvar'] = array(
+                'generation_time_milli=Generation Time',
+                'win32_status=Windows Status Code'
+            );
             $opts['--ignore-groups'] = 'userid';
+            $opts['--w3c-field-regex'] = 'sc-win32-status=(?P<win32_status>\S+)';
+            $opts['--w3c-time-taken-milli'] = false;
         }
 
         self::executeLogImporter($logFile, $opts);
diff --git a/tests/PHPUnit/System/expected/test_ImportLogs__CustomVariables.getCustomVariables_month.xml b/tests/PHPUnit/System/expected/test_ImportLogs__CustomVariables.getCustomVariables_month.xml
index 4a6377547d..a0aa7ab8ff 100644
--- a/tests/PHPUnit/System/expected/test_ImportLogs__CustomVariables.getCustomVariables_month.xml
+++ b/tests/PHPUnit/System/expected/test_ImportLogs__CustomVariables.getCustomVariables_month.xml
@@ -226,6 +226,36 @@
 		</subtable>
 	</row>
 	<row>
+		<label>Windows Status Code</label>
+		<nb_actions>4</nb_actions>
+		<subtable>
+			<row>
+				<label>24</label>
+				<nb_visits>1</nb_visits>
+				<nb_actions>1</nb_actions>
+				<sum_daily_nb_uniq_visitors>1</sum_daily_nb_uniq_visitors>
+			</row>
+			<row>
+				<label>32</label>
+				<nb_visits>1</nb_visits>
+				<nb_actions>1</nb_actions>
+				<sum_daily_nb_uniq_visitors>1</sum_daily_nb_uniq_visitors>
+			</row>
+			<row>
+				<label>42</label>
+				<nb_visits>1</nb_visits>
+				<nb_actions>1</nb_actions>
+				<sum_daily_nb_uniq_visitors>1</sum_daily_nb_uniq_visitors>
+			</row>
+			<row>
+				<label>96</label>
+				<nb_visits>1</nb_visits>
+				<nb_actions>1</nb_actions>
+				<sum_daily_nb_uniq_visitors>1</sum_daily_nb_uniq_visitors>
+			</row>
+		</subtable>
+	</row>
+	<row>
 		<label>Bot</label>
 		<nb_visits>1</nb_visits>
 		<nb_actions>1</nb_actions>
diff --git a/tests/PHPUnit/System/expected/test_ImportLogs__Live.getLastVisitsDetails_range.xml b/tests/PHPUnit/System/expected/test_ImportLogs__Live.getLastVisitsDetails_range.xml
index 544cac8cbb..3d2bb39bfc 100644
--- a/tests/PHPUnit/System/expected/test_ImportLogs__Live.getLastVisitsDetails_range.xml
+++ b/tests/PHPUnit/System/expected/test_ImportLogs__Live.getLastVisitsDetails_range.xml
@@ -2112,8 +2112,12 @@
 						<customVariablePageValue1>359</customVariablePageValue1>
 					</row>
 					<row>
-						<customVariablePageName2>HTTP-code</customVariablePageName2>
-						<customVariablePageValue2>200</customVariablePageValue2>
+						<customVariablePageName2>Windows Status Code</customVariablePageName2>
+						<customVariablePageValue2>96</customVariablePageValue2>
+					</row>
+					<row>
+						<customVariablePageName3>HTTP-code</customVariablePageName3>
+						<customVariablePageValue3>200</customVariablePageValue3>
 					</row>
 				</customVariables>
 				<generationTime>0.36s</generationTime>
@@ -2264,8 +2268,12 @@
 						<customVariablePageValue1>109</customVariablePageValue1>
 					</row>
 					<row>
-						<customVariablePageName2>HTTP-code</customVariablePageName2>
-						<customVariablePageValue2>200</customVariablePageValue2>
+						<customVariablePageName2>Windows Status Code</customVariablePageName2>
+						<customVariablePageValue2>32</customVariablePageValue2>
+					</row>
+					<row>
+						<customVariablePageName3>HTTP-code</customVariablePageName3>
+						<customVariablePageValue3>200</customVariablePageValue3>
 					</row>
 				</customVariables>
 				<generationTime>0.11s</generationTime>
@@ -2475,8 +2483,12 @@
 						<customVariablePageValue1>359</customVariablePageValue1>
 					</row>
 					<row>
-						<customVariablePageName2>HTTP-code</customVariablePageName2>
-						<customVariablePageValue2>404</customVariablePageValue2>
+						<customVariablePageName2>Windows Status Code</customVariablePageName2>
+						<customVariablePageValue2>24</customVariablePageValue2>
+					</row>
+					<row>
+						<customVariablePageName3>HTTP-code</customVariablePageName3>
+						<customVariablePageValue3>404</customVariablePageValue3>
 					</row>
 				</customVariables>
 				<generationTime>0.36s</generationTime>
@@ -2620,8 +2632,12 @@
 						<customVariablePageValue1>0</customVariablePageValue1>
 					</row>
 					<row>
-						<customVariablePageName2>HTTP-code</customVariablePageName2>
-						<customVariablePageValue2>301</customVariablePageValue2>
+						<customVariablePageName2>Windows Status Code</customVariablePageName2>
+						<customVariablePageValue2>42</customVariablePageValue2>
+					</row>
+					<row>
+						<customVariablePageName3>HTTP-code</customVariablePageName3>
+						<customVariablePageValue3>301</customVariablePageValue3>
 					</row>
 				</customVariables>
 				<icon />
diff --git a/tests/resources/access-logs/fake_logs_custom_iis.log b/tests/resources/access-logs/fake_logs_custom_iis.log
index d7ff476a6c..f2001c51dd 100644
--- a/tests/resources/access-logs/fake_logs_custom_iis.log
+++ b/tests/resources/access-logs/fake_logs_custom_iis.log
@@ -2,7 +2,7 @@
 #Version: 1.0
 #Start-Date: 2014-11-18 00:00:00.128
 #Fields:  date-local time-local s-ip cs-method cs-uri-stem cs-uri-query s-port cs-username c-ip cs(User-Agent) cs(Referer) cs(Host) sc-status sc-substatus sc-win32-status TimeTakenMS
-2012-08-15 17:00:00.363 10.10.28.140 GET /Products/theProduct - 80 user1 "70.95.0.0" "Mozilla/5.0 (Linux; Android 4.4.4; SM-G900V Build/KTU84P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.59 Mobile Safari/537.36" "http://example.com/Search/SearchResults.pg?informationRecipient.languageCode.c=en" "xzy.example.com" 200 0 0 109
-2012-08-15 17:00:00.660 10.10.28.140 GET /Topic/hw43061 - 80 user1 "70.95.32.0" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.1 Safari/537.36" - "example.hello.com" 301 0 0 0
-2012-08-15 17:00:00.675 10.10.28.140 GET /hello/world/6,681965 - 80 - "173.5.0.0" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.124 Safari/537.36" - "hello.example.com" 404 0 0 359
-2012-08-15 17:30:00.675 10.10.28.140 GET /hello/from/another/world/6,681965 - 80 user2 "173.5.0.0" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.124 Safari/537.36" - "hello.example.com" 200 0 0 359
+2012-08-15 17:00:00.363 10.10.28.140 GET /Products/theProduct - 80 user1 "70.95.0.0" "Mozilla/5.0 (Linux; Android 4.4.4; SM-G900V Build/KTU84P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.59 Mobile Safari/537.36" "http://example.com/Search/SearchResults.pg?informationRecipient.languageCode.c=en" "xzy.example.com" 200 0 32 109
+2012-08-15 17:00:00.660 10.10.28.140 GET /Topic/hw43061 - 80 user1 "70.95.32.0" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.1 Safari/537.36" - "example.hello.com" 301 0 42 0
+2012-08-15 17:00:00.675 10.10.28.140 GET /hello/world/6,681965 - 80 - "173.5.0.0" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.124 Safari/537.36" - "hello.example.com" 404 0 24 359
+2012-08-15 17:30:00.675 10.10.28.140 GET /hello/from/another/world/6,681965 - 80 user2 "173.5.0.0" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.124 Safari/537.36" - "hello.example.com" 200 0 96 359