Merge branch 'master' into '18471-restrict-tag-pushes-protected-tags'

# Conflicts:
#   spec/lib/gitlab/import_export/all_models.yml

9 files changed, 324 insertions, 51 deletions

diff --git a/lib/gitlab/ci/cron_parser.rb b/lib/gitlab/ci/cron_parser.rb
new file mode 100644
index 00000000000..a3cc350ef22
--- /dev/null
+++ b/lib/gitlab/ci/cron_parser.rb
@@ -0,0 +1,34 @@
+module Gitlab
+  module Ci
+    class CronParser
+      VALID_SYNTAX_SAMPLE_TIME_ZONE = 'UTC'.freeze
+      VALID_SYNTAX_SAMPLE_CRON = '* * * * *'.freeze
+
+      def initialize(cron, cron_timezone = 'UTC')
+        @cron = cron
+        @cron_timezone = cron_timezone
+      end
+
+      def next_time_from(time)
+        @cron_line ||= try_parse_cron(@cron, @cron_timezone)
+        @cron_line.next_time(time).in_time_zone(Time.zone) if @cron_line.present?
+      end
+
+      def cron_valid?
+        try_parse_cron(@cron, VALID_SYNTAX_SAMPLE_TIME_ZONE).present?
+      end
+
+      def cron_timezone_valid?
+        try_parse_cron(VALID_SYNTAX_SAMPLE_CRON, @cron_timezone).present?
+      end
+
+      private
+
+      def try_parse_cron(cron, cron_timezone)
+        Rufus::Scheduler.parse("#{cron} #{cron_timezone}")
+      rescue
+        # noop
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb
new file mode 100644
index 00000000000..5b835bb669a
--- /dev/null
+++ b/lib/gitlab/ci/trace.rb
@@ -0,0 +1,136 @@
+module Gitlab
+  module Ci
+    class Trace
+      attr_reader :job
+
+      delegate :old_trace, to: :job
+
+      def initialize(job)
+        @job = job
+      end
+
+      def html(last_lines: nil)
+        read do |stream|
+          stream.html(last_lines: last_lines)
+        end
+      end
+
+      def raw(last_lines: nil)
+        read do |stream|
+          stream.raw(last_lines: last_lines)
+        end
+      end
+
+      def extract_coverage(regex)
+        read do |stream|
+          stream.extract_coverage(regex)
+        end
+      end
+
+      def set(data)
+        write do |stream|
+          data = job.hide_secrets(data)
+          stream.set(data)
+        end
+      end
+
+      def append(data, offset)
+        write do |stream|
+          current_length = stream.size
+          return -current_length unless current_length == offset
+
+          data = job.hide_secrets(data)
+          stream.append(data, offset)
+          stream.size
+        end
+      end
+
+      def exist?
+        current_path.present? || old_trace.present?
+      end
+
+      def read
+        stream = Gitlab::Ci::Trace::Stream.new do
+          if current_path
+            File.open(current_path, "rb")
+          elsif old_trace
+            StringIO.new(old_trace)
+          end
+        end
+
+        yield stream
+      ensure
+        stream&.close
+      end
+
+      def write
+        stream = Gitlab::Ci::Trace::Stream.new do
+          File.open(ensure_path, "a+b")
+        end
+
+        yield(stream).tap do
+          job.touch if job.needs_touch?
+        end
+      ensure
+        stream&.close
+      end
+
+      def erase!
+        paths.each do |trace_path|
+          FileUtils.rm(trace_path, force: true)
+        end
+
+        job.erase_old_trace!
+      end
+
+      private
+
+      def ensure_path
+        return current_path if current_path
+
+        ensure_directory
+        default_path
+      end
+
+      def ensure_directory
+        unless Dir.exist?(default_directory)
+          FileUtils.mkdir_p(default_directory)
+        end
+      end
+
+      def current_path
+        @current_path ||= paths.find do |trace_path|
+          File.exist?(trace_path)
+        end
+      end
+
+      def paths
+        [
+          default_path,
+          deprecated_path
+        ].compact
+      end
+
+      def default_directory
+        File.join(
+          Settings.gitlab_ci.builds_path,
+          job.created_at.utc.strftime("%Y_%m"),
+          job.project_id.to_s
+        )
+      end
+
+      def default_path
+        File.join(default_directory, "#{job.id}.log")
+      end
+
+      def deprecated_path
+        File.join(
+          Settings.gitlab_ci.builds_path,
+          job.created_at.utc.strftime("%Y_%m"),
+          job.project.ci_id.to_s,
+          "#{job.id}.log"
+        ) if job.project&.ci_id
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb
new file mode 100644
index 00000000000..2af94e2c60e
--- /dev/null
+++ b/lib/gitlab/ci/trace/stream.rb
@@ -0,0 +1,119 @@
+module Gitlab
+  module Ci
+    class Trace
+      # This was inspired from: http://stackoverflow.com/a/10219411/1520132
+      class Stream
+        BUFFER_SIZE = 4096
+        LIMIT_SIZE = 50.kilobytes
+
+        attr_reader :stream
+
+        delegate :close, :tell, :seek, :size, :path, :truncate, to: :stream, allow_nil: true
+
+        delegate :valid?, to: :stream, as: :present?, allow_nil: true
+
+        def initialize
+          @stream = yield
+        end
+
+        def valid?
+          self.stream.present?
+        end
+
+        def file?
+          self.path.present?
+        end
+
+        def limit(last_bytes = LIMIT_SIZE)
+          stream_size = size
+          if stream_size < last_bytes
+            last_bytes = stream_size
+          end
+          stream.seek(-last_bytes, IO::SEEK_END)
+        end
+
+        def append(data, offset)
+          stream.truncate(offset)
+          stream.seek(0, IO::SEEK_END)
+          stream.write(data)
+          stream.flush()
+        end
+
+        def set(data)
+          truncate(0)
+          stream.write(data)
+          stream.flush()
+        end
+
+        def raw(last_lines: nil)
+          return unless valid?
+
+          if last_lines.to_i > 0
+            read_last_lines(last_lines)
+          else
+            stream.read
+          end
+        end
+
+        def html_with_state(state = nil)
+          ::Ci::Ansi2html.convert(stream, state)
+        end
+
+        def html(last_lines: nil)
+          text = raw(last_lines: last_lines)
+          stream = StringIO.new(text)
+          ::Ci::Ansi2html.convert(stream).html
+        end
+
+        def extract_coverage(regex)
+          return unless valid?
+          return unless regex
+
+          regex = Regexp.new(regex)
+
+          match = ""
+
+          stream.each_line do |line|
+            matches = line.scan(regex)
+            next unless matches.is_a?(Array)
+
+            match = matches.flatten.last
+            coverage = match.gsub(/\d+(\.\d+)?/).first
+            return coverage.to_f if coverage.present?
+          end
+        rescue
+          # if bad regex or something goes wrong we dont want to interrupt transition
+          # so we just silentrly ignore error for now
+        end
+
+        private
+
+        def read_last_lines(last_lines)
+          chunks = []
+          pos = lines = 0
+          max = stream.size
+
+          # We want an extra line to make sure fist line has full contents
+          while lines <= last_lines && pos < max
+            pos += BUFFER_SIZE
+
+            buf =
+              if pos <= max
+                stream.seek(-pos, IO::SEEK_END)
+                stream.read(BUFFER_SIZE)
+              else # Reached the head, read only left
+                stream.seek(0)
+                stream.read(BUFFER_SIZE - (pos - max))
+              end
+
+            lines += buf.count("\n")
+            chunks.unshift(buf)
+          end
+
+          chunks.join.lines.last(last_lines).join
+            .force_encoding(Encoding.default_external)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/trace_reader.rb b/lib/gitlab/ci/trace_reader.rb
deleted file mode 100644
index 1d7ddeb3e0f..00000000000
--- a/lib/gitlab/ci/trace_reader.rb
+++ /dev/null
@@ -1,50 +0,0 @@
-module Gitlab
-  module Ci
-    # This was inspired from: http://stackoverflow.com/a/10219411/1520132
-    class TraceReader
-      BUFFER_SIZE = 4096
-
-      attr_accessor :path, :buffer_size
-
-      def initialize(new_path, buffer_size: BUFFER_SIZE)
-        self.path = new_path
-        self.buffer_size = Integer(buffer_size)
-      end
-
-      def read(last_lines: nil)
-        if last_lines
-          read_last_lines(last_lines)
-        else
-          File.read(path)
-        end
-      end
-
-      def read_last_lines(max_lines)
-        File.open(path) do |file|
-          chunks = []
-          pos = lines = 0
-          max = file.size
-
-          # We want an extra line to make sure fist line has full contents
-          while lines <= max_lines && pos < max
-            pos += buffer_size
-
-            buf = if pos <= max
-                    file.seek(-pos, IO::SEEK_END)
-                    file.read(buffer_size)
-                  else # Reached the head, read only left
-                    file.seek(0)
-                    file.read(buffer_size - (pos - max))
-                  end
-
-            lines += buf.count("\n")
-            chunks.unshift(buf)
-          end
-
-          chunks.join.lines.last(max_lines).join
-            .force_encoding(Encoding.default_external)
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index 2e4314932c8..9e338282e96 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -452,6 +452,21 @@ module Gitlab
         Gitlab::Git::DiffCollection.new(diff_patches(from, to, options, *paths), options)
       end
 
+      # Returns a RefName for a given SHA
+      def ref_name_for_sha(ref_path, sha)
+        Gitlab::GitalyClient.migrate(:find_ref_name) do |is_enabled|
+          if is_enabled
+            gitaly_ref_client.find_ref_name(sha, ref_path)
+          else
+            args = %W(#{Gitlab.config.git.bin_path} for-each-ref --count=1 #{ref_path} --contains #{sha})
+
+            # Not found -> ["", 0]
+            # Found -> ["b8d95eb4969eefacb0a58f6a28f6803f8070e7b9 commit\trefs/environments/production/77\n", 0]
+            Gitlab::Popen.popen(args, @path).first.split.last
+          end
+        end
+      end
+
       # Returns commits collection
       #
       # Ex.
diff --git a/lib/gitlab/gitaly_client/ref.rb b/lib/gitlab/gitaly_client/ref.rb
index bfc5fa573c7..fcdf452d567 100644
--- a/lib/gitlab/gitaly_client/ref.rb
+++ b/lib/gitlab/gitaly_client/ref.rb
@@ -23,6 +23,16 @@ module Gitlab
         consume_refs_response(stub.find_all_tag_names(request), prefix: 'refs/tags/')
       end
 
+      def find_ref_name(commit_id, ref_prefix)
+        request = Gitaly::FindRefNameRequest.new(
+          repository: @repository,
+          commit_id: commit_id,
+          prefix: ref_prefix
+        )
+
+        stub.find_ref_name(request).name
+      end
+
       private
 
       def consume_refs_response(response, prefix:)
diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml
index 745f9a1cfbd..899a6567768 100644
--- a/lib/gitlab/import_export/import_export.yml
+++ b/lib/gitlab/import_export/import_export.yml
@@ -39,7 +39,8 @@ project_tree:
       - :author
       - :events
     - :statuses
-  - :triggers
+  - triggers:
+    - :trigger_schedule
   - :deploy_keys
   - :services
   - :hooks
diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb
index 0545ca10862..71811be6f50 100644
--- a/lib/gitlab/import_export/relation_factory.rb
+++ b/lib/gitlab/import_export/relation_factory.rb
@@ -5,6 +5,7 @@ module Gitlab
                     pipelines: 'Ci::Pipeline',
                     statuses: 'commit_status',
                     triggers: 'Ci::Trigger',
+                    trigger_schedule: 'Ci::TriggerSchedule',
                     builds: 'Ci::Build',
                     hooks: 'ProjectHook',
                     merge_access_levels: 'ProtectedBranch::MergeAccessLevel',
diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb
index 5e5f5ff1589..e599dd4a656 100644
--- a/lib/gitlab/regex.rb
+++ b/lib/gitlab/regex.rb
@@ -121,6 +121,13 @@ module Gitlab
       git_reference_regex
     end
 
+    ##
+    # Docker Distribution Registry 2.4.1 repository name rules
+    #
+    def container_repository_name_regex
+      @container_repository_regex ||= %r{\A[a-z0-9]+(?:[-._/][a-z0-9]+)*\Z}
+    end
+
     def environment_name_regex
       @environment_name_regex ||= /\A[a-zA-Z0-9_\\\/\${}. -]+\z/.freeze
     end