Merge branch 'last-green-master' into 18471-restrict-tag-pushes-protected-tags

12 files changed, 279 insertions, 23 deletions

diff --git a/lib/api/deploy_keys.rb b/lib/api/deploy_keys.rb
index b888ede6fe8..8a54f7f3f05 100644
--- a/lib/api/deploy_keys.rb
+++ b/lib/api/deploy_keys.rb
@@ -47,6 +47,7 @@ module API
       params do
         requires :key, type: String, desc: 'The new deploy key'
         requires :title, type: String, desc: 'The name of the deploy key'
+        optional :can_push, type: Boolean, desc: "Can deploy key push to the project's repository"
       end
       post ":id/deploy_keys" do
         params[:key].strip!
diff --git a/lib/api/groups.rb b/lib/api/groups.rb
index 8f3799417e3..605769eddde 100644
--- a/lib/api/groups.rb
+++ b/lib/api/groups.rb
@@ -142,7 +142,7 @@ module API
       end
       get ":id/projects" do
         group = find_group!(params[:id])
-        projects = GroupProjectsFinder.new(group).execute(current_user)
+        projects = GroupProjectsFinder.new(group: group, current_user: current_user).execute
         projects = filter_projects(projects)
         entity = params[:simple] ? Entities::BasicProjectDetails : Entities::Project
         present paginate(projects), with: entity, current_user: current_user
diff --git a/lib/api/projects.rb b/lib/api/projects.rb
index 0fbe1669d45..766fbea53e6 100644
--- a/lib/api/projects.rb
+++ b/lib/api/projects.rb
@@ -84,7 +84,7 @@ module API
       end
       get do
         entity = current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails
-        present_projects ProjectsFinder.new.execute(current_user), with: entity, statistics: params[:statistics]
+        present_projects ProjectsFinder.new(current_user: current_user).execute, with: entity, statistics: params[:statistics]
       end
 
       desc 'Create new project' do
diff --git a/lib/api/users.rb b/lib/api/users.rb
index 530ca0b5235..992a751b37d 100644
--- a/lib/api/users.rb
+++ b/lib/api/users.rb
@@ -341,7 +341,7 @@ module API
         not_found!('User') unless user
 
         events = user.events.
-          merge(ProjectsFinder.new.execute(current_user)).
+          merge(ProjectsFinder.new(current_user: current_user).execute).
           references(:project).
           with_associations.
           recent
diff --git a/lib/api/v3/groups.rb b/lib/api/v3/groups.rb
index c5b37622d79..9b27411ae21 100644
--- a/lib/api/v3/groups.rb
+++ b/lib/api/v3/groups.rb
@@ -151,7 +151,7 @@ module API
         end
         get ":id/projects" do
           group = find_group!(params[:id])
-          projects = GroupProjectsFinder.new(group).execute(current_user)
+          projects = GroupProjectsFinder.new(group: group, current_user: current_user).execute
           projects = filter_projects(projects)
           entity = params[:simple] ? ::API::Entities::BasicProjectDetails : Entities::Project
           present paginate(projects), with: entity, current_user: current_user
diff --git a/lib/api/v3/projects.rb b/lib/api/v3/projects.rb
index b753dbab381..ba9748ada59 100644
--- a/lib/api/v3/projects.rb
+++ b/lib/api/v3/projects.rb
@@ -107,7 +107,7 @@ module API
         end
         get '/visible' do
           entity = current_user ? ::API::V3::Entities::ProjectWithAccess : ::API::Entities::BasicProjectDetails
-          present_projects ProjectsFinder.new.execute(current_user), with: entity
+          present_projects ProjectsFinder.new(current_user: current_user).execute, with: entity
         end
 
         desc 'Get a projects list for authenticated user' do
diff --git a/lib/api/v3/users.rb b/lib/api/v3/users.rb
index 5e18cecc431..f4cda3b2eba 100644
--- a/lib/api/v3/users.rb
+++ b/lib/api/v3/users.rb
@@ -138,7 +138,7 @@ module API
           not_found!('User') unless user
 
           events = user.events.
-            merge(ProjectsFinder.new.execute(current_user)).
+            merge(ProjectsFinder.new(current_user: current_user).execute).
             references(:project).
             with_associations.
             recent
diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb
index fc445ab9483..525aa920328 100644
--- a/lib/gitlab/database/migration_helpers.rb
+++ b/lib/gitlab/database/migration_helpers.rb
@@ -26,6 +26,30 @@ module Gitlab
         add_index(table_name, column_name, options)
       end
 
+      # Removes an existed index, concurrently when supported
+      #
+      # On PostgreSQL this method removes an index concurrently.
+      #
+      # Example:
+      #
+      #     remove_concurrent_index :users, :some_column
+      #
+      # See Rails' `remove_index` for more info on the available arguments.
+      def remove_concurrent_index(table_name, column_name, options = {})
+        if transaction_open?
+          raise 'remove_concurrent_index can not be run inside a transaction, ' \
+            'you can disable transactions by calling disable_ddl_transaction! ' \
+            'in the body of your migration class'
+        end
+
+        if Database.postgresql?
+          options = options.merge({ algorithm: :concurrently })
+          disable_statement_timeout
+        end
+
+        remove_index(table_name, options.merge({ column: column_name }))
+      end
+
       # Adds a foreign key with only minimal locking on the tables involved.
       #
       # This method only requires minimal locking when using PostgreSQL. When
diff --git a/lib/gitlab/etag_caching/middleware.rb b/lib/gitlab/etag_caching/middleware.rb
index cd4e318033d..630fe4fa849 100644
--- a/lib/gitlab/etag_caching/middleware.rb
+++ b/lib/gitlab/etag_caching/middleware.rb
@@ -2,26 +2,34 @@ module Gitlab
   module EtagCaching
     class Middleware
       RESERVED_WORDS = NamespaceValidator::WILDCARD_ROUTES.map { |word| "/#{word}/" }.join('|')
-      ROUTE_REGEXP = Regexp.union(
-        %r(^(?!.*(#{RESERVED_WORDS})).*/noteable/issue/\d+/notes\z),
-        %r(^(?!.*(#{RESERVED_WORDS})).*/issues/\d+/rendered_title\z)
-      )
+      ROUTES = [
+        {
+          regexp: %r(^(?!.*(#{RESERVED_WORDS})).*/noteable/issue/\d+/notes\z),
+          name: 'issue_notes'
+        },
+        {
+          regexp: %r(^(?!.*(#{RESERVED_WORDS})).*/issues/\d+/rendered_title\z),
+          name: 'issue_title'
+        }
+      ].freeze
 
       def initialize(app)
         @app = app
       end
 
       def call(env)
-        return @app.call(env) unless enabled_for_current_route?(env)
-        Gitlab::Metrics.add_event(:etag_caching_middleware_used)
+        route = match_current_route(env)
+        return @app.call(env) unless route
+
+        track_event(:etag_caching_middleware_used, route)
 
         etag, cached_value_present = get_etag(env)
         if_none_match = env['HTTP_IF_NONE_MATCH']
 
         if if_none_match == etag
-          handle_cache_hit(etag)
+          handle_cache_hit(etag, route)
         else
-          track_cache_miss(if_none_match, cached_value_present)
+          track_cache_miss(if_none_match, cached_value_present, route)
 
           status, headers, body = @app.call(env)
           headers['ETag'] = etag
@@ -31,8 +39,8 @@ module Gitlab
 
       private
 
-      def enabled_for_current_route?(env)
-        ROUTE_REGEXP.match(env['PATH_INFO'])
+      def match_current_route(env)
+        ROUTES.find { |route| route[:regexp].match(env['PATH_INFO']) }
       end
 
       def get_etag(env)
@@ -52,23 +60,27 @@ module Gitlab
         %Q{W/"#{value}"}
       end
 
-      def handle_cache_hit(etag)
-        Gitlab::Metrics.add_event(:etag_caching_cache_hit)
+      def handle_cache_hit(etag, route)
+        track_event(:etag_caching_cache_hit, route)
 
         status_code = Gitlab::PollingInterval.polling_enabled? ? 304 : 429
 
         [status_code, { 'ETag' => etag }, ['']]
       end
 
-      def track_cache_miss(if_none_match, cached_value_present)
+      def track_cache_miss(if_none_match, cached_value_present, route)
         if if_none_match.blank?
-          Gitlab::Metrics.add_event(:etag_caching_header_missing)
+          track_event(:etag_caching_header_missing, route)
         elsif !cached_value_present
-          Gitlab::Metrics.add_event(:etag_caching_key_not_found)
+          track_event(:etag_caching_key_not_found, route)
         else
-          Gitlab::Metrics.add_event(:etag_caching_resource_changed)
+          track_event(:etag_caching_resource_changed, route)
         end
       end
+
+      def track_event(name, route)
+        Gitlab::Metrics.add_event(name, endpoint: route[:name])
+      end
     end
   end
 end
diff --git a/lib/gitlab/sidekiq_status.rb b/lib/gitlab/sidekiq_status.rb
index 11e5f1b645c..ca8d3271541 100644
--- a/lib/gitlab/sidekiq_status.rb
+++ b/lib/gitlab/sidekiq_status.rb
@@ -72,6 +72,8 @@ module Gitlab
     # job_ids - The Sidekiq job IDs to check.
     #
     # Returns an array of true or false indicating job completion.
+    # true = job is still running
+    # false = job completed
     def self.job_status(job_ids)
       keys = job_ids.map { |jid| key_for(jid) }
 
@@ -82,6 +84,17 @@ module Gitlab
       end
     end
 
+    # Returns the JIDs that are completed
+    #
+    # job_ids - The Sidekiq job IDs to check.
+    #
+    # Returns an array of completed JIDs
+    def self.completed_jids(job_ids)
+      Sidekiq.redis do |redis|
+        job_ids.reject { |jid| redis.exists(key_for(jid)) }
+      end
+    end
+
     def self.key_for(jid)
       STATUS_KEY % jid
     end
diff --git a/lib/gitlab/sidekiq_status/client_middleware.rb b/lib/gitlab/sidekiq_status/client_middleware.rb
index d47609f490d..00983b3284a 100644
--- a/lib/gitlab/sidekiq_status/client_middleware.rb
+++ b/lib/gitlab/sidekiq_status/client_middleware.rb
@@ -2,7 +2,9 @@ module Gitlab
   module SidekiqStatus
     class ClientMiddleware
       def call(_, job, _, _)
-        Gitlab::SidekiqStatus.set(job['jid'])
+        status_expiration = job['status_expiration'] || Gitlab::SidekiqStatus::DEFAULT_EXPIRATION
+
+        Gitlab::SidekiqStatus.set(job['jid'], status_expiration)
         yield
       end
     end
diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake
new file mode 100644
index 00000000000..350afeb5c0b
--- /dev/null
+++ b/lib/tasks/import.rake
@@ -0,0 +1,204 @@
+require 'benchmark'
+require 'rainbow/ext/string'
+require_relative '../gitlab/shell_adapter'
+require_relative '../gitlab/github_import/importer'
+
+class NewImporter < ::Gitlab::GithubImport::Importer
+  def execute
+    # Same as ::Gitlab::GithubImport::Importer#execute, but showing some progress.
+    puts 'Importing repository...'.color(:aqua)
+    import_repository unless project.repository_exists?
+
+    puts 'Importing labels...'.color(:aqua)
+    import_labels
+
+    puts 'Importing milestones...'.color(:aqua)
+    import_milestones
+
+    puts 'Importing pull requests...'.color(:aqua)
+    import_pull_requests
+
+    puts 'Importing issues...'.color(:aqua)
+    import_issues
+
+    puts 'Importing issue comments...'.color(:aqua)
+    import_comments(:issues)
+
+    puts 'Importing pull request comments...'.color(:aqua)
+    import_comments(:pull_requests)
+
+    puts 'Importing wiki...'.color(:aqua)
+    import_wiki
+
+    # Gitea doesn't have a Release API yet
+    # See https://github.com/go-gitea/gitea/issues/330
+    unless project.gitea_import?
+      import_releases
+    end
+
+    handle_errors
+
+    project.repository.after_import
+    project.import_finish
+
+    true
+  end
+
+  def import_repository
+    begin
+      raise 'Blocked import URL.' if Gitlab::UrlBlocker.blocked_url?(project.import_url)
+
+      gitlab_shell.import_repository(project.repository_storage_path, project.path_with_namespace, project.import_url)
+    rescue => e
+      project.repository.before_import if project.repository_exists?
+
+      raise "Error importing repository #{project.import_url} into #{project.path_with_namespace} - #{e.message}"
+    end
+  end
+end
+
+class GithubImport
+  def self.run!(*args)
+    new(*args).run!
+  end
+
+  def initialize(token, gitlab_username, project_path, extras)
+    @token = token
+    @project_path = project_path
+    @current_user = User.find_by_username(gitlab_username)
+    @github_repo = extras.empty? ? nil : extras.first
+  end
+
+  def run!
+    @repo = GithubRepos.new(@token, @current_user, @github_repo).choose_one!
+
+    raise 'No repo found!' unless @repo
+
+    show_warning!
+
+    @project = Project.find_by_full_path(@project_path) || new_project
+
+    import!
+  end
+
+  private
+
+  def show_warning!
+    puts "This will import GH #{@repo.full_name.bright} into GL #{@project_path.bright} as #{@current_user.name}"
+    puts "Permission checks are ignored. Press any key to continue.".color(:red)
+
+    STDIN.getch
+
+    puts 'Starting the import...'.color(:green)
+  end
+
+  def import!
+    import_url = @project.import_url.gsub(/\:\/\/(.*@)?/, "://#{@token}@")
+    @project.update(import_url: import_url)
+
+    @project.import_start
+
+    timings = Benchmark.measure do
+      NewImporter.new(@project).execute
+    end
+
+    puts "Import finished. Timings: #{timings}".color(:green)
+  end
+
+  def new_project
+    Project.transaction do
+      namespace_path, _sep, name = @project_path.rpartition('/')
+      namespace = find_or_create_namespace(namespace_path)
+
+      Project.create!(
+        import_url: "https://#{@token}@github.com/#{@repo.full_name}.git",
+        name: name,
+        path: name,
+        description: @repo.description,
+        namespace: namespace,
+        visibility_level: visibility_level,
+        import_type: 'github',
+        import_source: @repo.full_name,
+        creator: @current_user
+      )
+    end
+  end
+
+  def find_or_create_namespace(names)
+    return @current_user.namespace if names == @current_user.namespace_path
+    return @current_user.namespace unless @current_user.can_create_group?
+
+    names = params[:target_namespace].presence || names
+    full_path_namespace = Namespace.find_by_full_path(names)
+
+    return full_path_namespace if full_path_namespace
+
+    names.split('/').inject(nil) do |parent, name|
+      begin
+        namespace = Group.create!(name: name,
+                                  path: name,
+                                  owner: @current_user,
+                                  parent: parent)
+        namespace.add_owner(@current_user)
+
+        namespace
+      rescue ActiveRecord::RecordNotUnique, ActiveRecord::RecordInvalid
+        Namespace.where(parent: parent).find_by_path_or_name(name)
+      end
+    end
+  end
+
+  def full_path_namespace(names)
+    @full_path_namespace ||= Namespace.find_by_full_path(names)
+  end
+
+  def visibility_level
+    @repo.private ? Gitlab::VisibilityLevel::PRIVATE : current_application_settings.default_project_visibility
+  end
+end
+
+class GithubRepos
+  def initialize(token, current_user, github_repo)
+    @token = token
+    @current_user = current_user
+    @github_repo = github_repo
+  end
+
+  def choose_one!
+    return found_github_repo if @github_repo
+
+    repos.each do |repo|
+      print "ID: #{repo[:id].to_s.bright} ".color(:green)
+      puts "- Name: #{repo[:full_name]}".color(:green)
+    end
+
+    print 'ID? '.bright
+
+    repos.find { |repo| repo[:id] == repo_id }
+  end
+
+  def found_github_repo
+    repos.find { |repo| repo[:full_name] == @github_repo }
+  end
+
+  def repo_id
+    @repo_id ||= STDIN.gets.chomp.to_i
+  end
+
+  def repos
+    @repos ||= client.repos
+  end
+
+  def client
+    @client ||= Gitlab::GithubImport::Client.new(@token, {})
+  end
+end
+
+namespace :import do
+  desc 'Import a GitHub project - Example: import:github[ToKeN,root,root/blah,my/github_repo] (optional my/github_repo)'
+  task :github, [:token, :gitlab_username, :project_path] => :environment do |_t, args|
+    abort 'Project path must be: namespace(s)/project_name'.color(:red) unless args.project_path.include?('/')
+
+    GithubImport.run!(args.token, args.gitlab_username, args.project_path, args.extras)
+  end
+end