From 91f8e734fee1f9e7a16573f7185c48f442e3bb5e Mon Sep 17 00:00:00 2001
From: Alessio Caiazza <acaiazza@gitlab.com>
Date: Mon, 25 Sep 2017 18:54:08 +0200
Subject: Add CI build trace sections extractor

---
 lib/gitlab/ci/ansi2html.rb            |  2 +-
 lib/gitlab/ci/trace.rb                |  6 +++
 lib/gitlab/ci/trace/section_parser.rb | 97 +++++++++++++++++++++++++++++++++++
 lib/gitlab/ci/trace/stream.rb         | 17 ++++++
 4 files changed, 121 insertions(+), 1 deletion(-)
 create mode 100644 lib/gitlab/ci/trace/section_parser.rb

(limited to 'lib/gitlab/ci')

diff --git a/lib/gitlab/ci/ansi2html.rb b/lib/gitlab/ci/ansi2html.rb
index 088adbdd267..72b75791bbb 100644
--- a/lib/gitlab/ci/ansi2html.rb
+++ b/lib/gitlab/ci/ansi2html.rb
@@ -155,7 +155,7 @@ module Gitlab
           stream.each_line do |line|
             s = StringScanner.new(line)
             until s.eos?
-              if s.scan(/section_((?:start)|(?:end)):(\d+):([^\r]+)\r\033\[0K/)
+              if s.scan(Gitlab::Regex.build_trace_section_regex)
                 handle_section(s)
               elsif s.scan(/\e([@-_])(.*?)([@-~])/)
                 handle_sequence(s)
diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb
index 5b835bb669a..baf55b1fa07 100644
--- a/lib/gitlab/ci/trace.rb
+++ b/lib/gitlab/ci/trace.rb
@@ -27,6 +27,12 @@ module Gitlab
         end
       end
 
+      def extract_sections
+        read do |stream|
+          stream.extract_sections
+        end
+      end
+
       def set(data)
         write do |stream|
           data = job.hide_secrets(data)
diff --git a/lib/gitlab/ci/trace/section_parser.rb b/lib/gitlab/ci/trace/section_parser.rb
new file mode 100644
index 00000000000..9bb0166c9e3
--- /dev/null
+++ b/lib/gitlab/ci/trace/section_parser.rb
@@ -0,0 +1,97 @@
+module Gitlab
+  module Ci
+    class Trace
+      class SectionParser
+        def initialize(lines)
+          @lines = lines
+        end
+
+        def parse!
+          @markers = {}
+
+          @lines.each do |line, pos|
+            parse_line(line, pos)
+          end
+        end
+
+        def sections
+          sanitize_markers.map do |name, markers|
+            start_, end_ = markers
+
+            {
+              name: name,
+              byte_start: start_[:marker],
+              byte_end: end_[:marker],
+              date_start: start_[:timestamp],
+              date_end: end_[:timestamp]
+            }
+          end
+        end
+
+        private
+
+        def parse_line(line, line_start_position)
+          s = StringScanner.new(line)
+          until s.eos?
+            find_next_marker(s) do |scanner|
+              marker_begins_at = line_start_position + scanner.pointer
+
+              if scanner.scan(Gitlab::Regex.build_trace_section_regex)
+                marker_ends_at = line_start_position + scanner.pointer
+                handle_line(scanner[1], scanner[2].to_i, scanner[3], marker_begins_at, marker_ends_at)
+                true
+              else
+                false
+              end
+            end
+          end
+        end
+
+        def sanitize_markers
+          @markers.select do |_, markers|
+            markers.size == 2 && markers[0][:action] == :start && markers[1][:action] == :end
+          end
+        end
+
+        def handle_line(action, time, name, marker_start, marker_end)
+          action = action.to_sym
+          timestamp = Time.at(time).utc
+          marker = if action == :start
+                     marker_end
+                   else
+                     marker_start
+                   end
+
+          @markers[name] ||= []
+          @markers[name] << {
+            name: name,
+            action: action,
+            timestamp: timestamp,
+            marker: marker
+          }
+        end
+
+        def beginning_of_section_regex
+          @beginning_of_section_regex ||= /section_/.freeze
+        end
+
+        def find_next_marker(s)
+          beginning_of_section_len = 8
+          maybe_marker = s.exist?(beginning_of_section_regex)
+
+          if maybe_marker.nil?
+            s.terminate
+          else
+            # repositioning at the beginning of the match
+            s.pos += maybe_marker - beginning_of_section_len
+            if block_given?
+              good_marker = yield(s)
+              # if not a good marker: Consuming the matched beginning_of_section_regex
+              s.pos += beginning_of_section_len unless good_marker
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb
index ab3408f48d6..d52194f688b 100644
--- a/lib/gitlab/ci/trace/stream.rb
+++ b/lib/gitlab/ci/trace/stream.rb
@@ -90,8 +90,25 @@ module Gitlab
           # so we just silently ignore error for now
         end
 
+        def extract_sections
+          return [] unless valid?
+
+          lines = to_enum(:each_line_with_pos)
+          parser = SectionParser.new(lines)
+
+          parser.parse!
+          parser.sections
+        end
+
         private
 
+        def each_line_with_pos
+          stream.seek(0, IO::SEEK_SET)
+          stream.each_line do |line|
+            yield [line, stream.pos - line.bytesize]
+          end
+        end
+
         def read_last_lines(limit)
           to_enum(:reverse_line).first(limit).reverse.join
         end
-- 
cgit v1.2.3