diff options
26 files changed, 346 insertions, 25 deletions
diff --git a/app/controllers/projects/mirrors_controller.rb b/app/controllers/projects/mirrors_controller.rb index fb5aee2e702..5698ff4e706 100644 --- a/app/controllers/projects/mirrors_controller.rb +++ b/app/controllers/projects/mirrors_controller.rb @@ -2,8 +2,9 @@ class Projects::MirrorsController < Projects::ApplicationController include RepositorySettingsRedirect # Authorize - before_action :authorize_admin_mirror! before_action :remote_mirror, only: [:update] + before_action :check_mirror_available! + before_action :authorize_admin_project! layout "project_settings" @@ -45,6 +46,10 @@ class Projects::MirrorsController < Projects::ApplicationController @remote_mirror = project.remote_mirrors.first_or_initialize end + def check_mirror_available! + Gitlab::CurrentSettings.current_application_settings.mirror_available || current_user&.admin? + end + def mirror_params_attributes [ remote_mirrors_attributes: %i[ diff --git a/app/controllers/projects/settings/repository_controller.rb b/app/controllers/projects/settings/repository_controller.rb index a3bb60bb3b2..4697af4f26a 100644 --- a/app/controllers/projects/settings/repository_controller.rb +++ b/app/controllers/projects/settings/repository_controller.rb @@ -44,8 +44,6 @@ module Projects end def remote_mirror - return unless project.feature_available?(:repository_mirrors) - @remote_mirror = project.remote_mirrors.first_or_initialize end diff --git a/app/helpers/application_settings_helper.rb b/app/helpers/application_settings_helper.rb index 1bf98d550b0..b948e431882 100644 --- a/app/helpers/application_settings_helper.rb +++ b/app/helpers/application_settings_helper.rb @@ -250,7 +250,8 @@ module ApplicationSettingsHelper :version_check_enabled, :allow_local_requests_from_hooks_and_services, :enforce_terms, - :terms + :terms, + :mirror_available ] end end diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index a734cc7a26b..451e512aef7 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -334,7 +334,8 @@ class ApplicationSetting < ActiveRecord::Base gitaly_timeout_fast: 10, gitaly_timeout_medium: 30, gitaly_timeout_default: 55, - allow_local_requests_from_hooks_and_services: false + allow_local_requests_from_hooks_and_services: false, + mirror_available: true } end diff --git a/app/models/project.rb b/app/models/project.rb index 4021e4e85bf..3ad0d4f0c4c 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -65,7 +65,7 @@ class Project < ActiveRecord::Base add_authentication_token_field :runners_token - before_validation :mark_remote_mirrors_for_removal + before_validation :mark_remote_mirrors_for_removal, if: -> { ActiveRecord::Base.connection.table_exists?(:remote_mirrors) } before_save :ensure_runners_token diff --git a/app/models/remote_mirror.rb b/app/models/remote_mirror.rb index b7e45875830..3b9fd6d710f 100644 --- a/app/models/remote_mirror.rb +++ b/app/models/remote_mirror.rb @@ -86,9 +86,12 @@ class RemoteMirror < ActiveRecord::Base raw.update(options) end + def sync? + !enabled? + end + def sync - return unless enabled? - return if Gitlab::Geo.secondary? + return if sync? if recently_scheduled? RepositoryUpdateRemoteMirrorWorker.perform_in(backoff_delay, self.id, Time.now) diff --git a/app/models/repository.rb b/app/models/repository.rb index 2a0802482f5..b75c4aca982 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -854,7 +854,7 @@ class Repository add_remote(remote_name, url, mirror_refmap: refmap) fetch_remote(remote_name, forced: forced, prune: prune) ensure - remove_remote(remote_name) if tmp_remote_name + async_remove_remote(remote_name) if tmp_remote_name end def fetch_remote(remote, forced: false, ssh_auth: nil, no_tags: false, prune: true) diff --git a/app/policies/project_policy.rb b/app/policies/project_policy.rb index 3529d0aa60c..5759b1a376f 100644 --- a/app/policies/project_policy.rb +++ b/app/policies/project_policy.rb @@ -80,6 +80,11 @@ class ProjectPolicy < BasePolicy project.merge_requests_allowing_push_to_user(user).any? end + with_scope :global + condition(:mirror_available, score: 0) do + ::Gitlab::CurrentSettings.current_application_settings.mirror_available + end + # We aren't checking `:read_issue` or `:read_merge_request` in this case # because it could be possible for a user to see an issuable-iid # (`:read_issue_iid` or `:read_merge_request_iid`) but then wouldn't be @@ -246,6 +251,8 @@ class ProjectPolicy < BasePolicy enable :create_cluster end + rule { (mirror_available & can?(:admin_project)) | admin }.enable :admin_remote_mirror + rule { archived }.policy do prevent :push_code prevent :push_to_delete_protected_branch diff --git a/app/serializers/project_mirror_entity.rb b/app/serializers/project_mirror_entity.rb index 8051f6e7d35..a9c08ac021a 100644 --- a/app/serializers/project_mirror_entity.rb +++ b/app/serializers/project_mirror_entity.rb @@ -1,6 +1,4 @@ class ProjectMirrorEntity < Grape::Entity - prepend ::EE::ProjectMirrorEntity - expose :id expose :remote_mirrors_attributes do |project| diff --git a/app/services/concerns/exclusive_lease_guard.rb b/app/services/concerns/exclusive_lease_guard.rb new file mode 100644 index 00000000000..30be6accc32 --- /dev/null +++ b/app/services/concerns/exclusive_lease_guard.rb @@ -0,0 +1,52 @@ +# +# Concern that helps with getting an exclusive lease for running a block +# of code. +# +# `#try_obtain_lease` takes a block which will be run if it was able to +# obtain the lease. Implement `#lease_timeout` to configure the timeout +# for the exclusive lease. Optionally override `#lease_key` to set the +# lease key, it defaults to the class name with underscores. +# +module ExclusiveLeaseGuard + extend ActiveSupport::Concern + + def try_obtain_lease + lease = exclusive_lease.try_obtain + + unless lease + log_error('Cannot obtain an exclusive lease. There must be another instance already in execution.') + return + end + + begin + yield lease + ensure + release_lease(lease) + end + end + + def exclusive_lease + @lease ||= Gitlab::ExclusiveLease.new(lease_key, timeout: lease_timeout) + end + + def lease_key + @lease_key ||= self.class.name.underscore + end + + def lease_timeout + raise NotImplementedError, + "#{self.class.name} does not implement #{__method__}" + end + + def release_lease(uuid) + Gitlab::ExclusiveLease.cancel(lease_key, uuid) + end + + def renew_lease! + exclusive_lease.renew + end + + def log_error(message, extra_args = {}) + logger.error(message) + end +end diff --git a/app/views/admin/application_settings/_repository_mirrors_form.html.haml b/app/views/admin/application_settings/_repository_mirrors_form.html.haml new file mode 100644 index 00000000000..09183ec6260 --- /dev/null +++ b/app/views/admin/application_settings/_repository_mirrors_form.html.haml @@ -0,0 +1,16 @@ += form_for @application_setting, url: admin_application_settings_path, html: { class: 'form-horizontal fieldset-form' } do |f| + = form_errors(@application_setting) + + %fieldset + .form-group + = f.label :mirror_available, 'Enable mirror configuration', class: 'control-label col-sm-2' + .col-sm-10 + .checkbox + = f.label :mirror_available do + = f.check_box :mirror_available + Allow mirrors to be setup for projects + %span.help-block + If disabled, only admins will be able to setup mirrors in projects. + = link_to icon('question-circle'), help_page_path('workflow/repository_mirroring') + + = f.submit 'Save changes', class: "btn btn-success" diff --git a/app/views/admin/application_settings/show.html.haml b/app/views/admin/application_settings/show.html.haml index 3c00e3c8fc4..4d686d02046 100644 --- a/app/views/admin/application_settings/show.html.haml +++ b/app/views/admin/application_settings/show.html.haml @@ -313,3 +313,14 @@ = _('Allow requests to the local network from hooks and services.') .settings-content = render 'outbound' + +%section.settings.as-mirror.no-animate#js-mirror-settings{ class: ('expanded' if expanded) } + .settings-header + %h4 + = _('Repository mirror settings') + %button.btn.js-settings-toggle{ type: 'button' } + = expanded ? 'Collapse' : 'Expand' + %p + = _('Configure push and pull mirrors.') + .settings-content + = render partial: 'repository_mirrors_form' diff --git a/app/views/projects/mirrors/_instructions.html.haml b/app/views/projects/mirrors/_instructions.html.haml new file mode 100644 index 00000000000..64f0fde30cf --- /dev/null +++ b/app/views/projects/mirrors/_instructions.html.haml @@ -0,0 +1,10 @@ +.account-well.prepend-top-default.append-bottom-default + %ul + %li + The repository must be accessible over <code>http://</code>, <code>https://</code>, <code>ssh://</code> or <code>git://</code>. + %li + Include the username in the URL if required: <code>https://username@gitlab.company.com/group/project.git</code>. + %li + The update action will time out after 10 minutes. For big repositories, use a clone/push combination. + %li + The Git LFS objects will <strong>not</strong> be synced. diff --git a/app/views/projects/mirrors/_show.html.haml b/app/views/projects/mirrors/_show.html.haml index da789906aef..de77701a373 100644 --- a/app/views/projects/mirrors/_show.html.haml +++ b/app/views/projects/mirrors/_show.html.haml @@ -1,3 +1,3 @@ -- if can?(current_user, :admin_mirror, @project) - = render 'projects/mirrors/push' +- if can?(current_user, :admin_remote_mirror, @project) + = render 'projects/mirrors/push' diff --git a/app/views/projects/settings/repository/show.html.haml b/app/views/projects/settings/repository/show.html.haml index f57590a908f..5dda2ec28b4 100644 --- a/app/views/projects/settings/repository/show.html.haml +++ b/app/views/projects/settings/repository/show.html.haml @@ -2,6 +2,8 @@ - page_title "Repository" - @content_class = "limit-container-width" unless fluid_layout += render "projects/mirrors/show" + -# Protected branches & tags use a lot of nested partials. -# The shared parts of the views can be found in the `shared` directory. -# Those are used throughout the actual views. These `shared` views are then diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index d41bcd1abcb..1949522d6f0 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -106,6 +106,7 @@ - rebase - repository_fork - repository_import +- repository_remove_remote - storage_migrator - system_hook_push - update_merge_requests diff --git a/config/routes/project.rb b/config/routes/project.rb index f36341cdcaf..5a1be1a8b73 100644 --- a/config/routes/project.rb +++ b/config/routes/project.rb @@ -174,6 +174,12 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do end end + resource :mirror, only: [:show, :update] do + member do + post :update_now + end + end + resources :pipelines, only: [:index, :new, :create, :show] do collection do resource :pipelines_settings, path: 'settings', only: [:show, :update] diff --git a/db/migrate/20180503131624_create_remote_mirrors.rb b/db/migrate/20180503131624_create_remote_mirrors.rb new file mode 100644 index 00000000000..7800186455f --- /dev/null +++ b/db/migrate/20180503131624_create_remote_mirrors.rb @@ -0,0 +1,33 @@ +class CreateRemoteMirrors < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + return if table_exists?(:remote_mirrors) + + create_table :remote_mirrors do |t| + t.references :project, index: true, foreign_key: { on_delete: :cascade } + t.string :url + t.boolean :enabled, default: true + t.string :update_status + t.datetime :last_update_at + t.datetime :last_successful_update_at + t.datetime :last_update_started_at + t.string :last_error + t.boolean :only_protected_branches, default: false, null: false + t.string :remote_name + t.text :encrypted_credentials + t.string :encrypted_credentials_iv + t.string :encrypted_credentials_salt + + t.timestamps null: false + end + end + + def down + drop_table(:remote_mirrors) if table_exists?(:remote_mirrors) + end +end diff --git a/db/migrate/20180503141722_add_remote_mirror_available_overridden_to_projects.rb b/db/migrate/20180503141722_add_remote_mirror_available_overridden_to_projects.rb new file mode 100644 index 00000000000..841393971f4 --- /dev/null +++ b/db/migrate/20180503141722_add_remote_mirror_available_overridden_to_projects.rb @@ -0,0 +1,15 @@ +class AddRemoteMirrorAvailableOverriddenToProjects < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_column(:projects, :remote_mirror_available_overridden, :boolean) unless column_exists?(:projects, :remote_mirror_available_overridden) + end + + def down + remove_column(:projects, :remote_mirror_available_overridden) if column_exists?(:projects, :remote_mirror_available_overridden) + end +end diff --git a/db/migrate/20180503193542_add_indexes_to_remote_mirror.rb b/db/migrate/20180503193542_add_indexes_to_remote_mirror.rb new file mode 100644 index 00000000000..9a9decffdab --- /dev/null +++ b/db/migrate/20180503193542_add_indexes_to_remote_mirror.rb @@ -0,0 +1,15 @@ +class AddIndexesToRemoteMirror < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_concurrent_index :remote_mirrors, :last_successful_update_at unless index_exists?(:remote_mirrors, :last_successful_update_at) + end + + def down + remove_index :remote_mirrors, :last_successful_update_at if index_exists? :remote_mirrors, :last_successful_update_at + end +end diff --git a/db/migrate/20180503193953_add_mirror_available_to_application_settings.rb b/db/migrate/20180503193953_add_mirror_available_to_application_settings.rb new file mode 100644 index 00000000000..25b9905b1a9 --- /dev/null +++ b/db/migrate/20180503193953_add_mirror_available_to_application_settings.rb @@ -0,0 +1,15 @@ +class AddMirrorAvailableToApplicationSettings < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_column_with_default(:application_settings, :mirror_available, :boolean, default: true, allow_null: false) unless column_exists?(:application_settings, :mirror_available) + end + + def down + remove_column(:application_settings, :mirror_available) if column_exists?(:application_settings, :mirror_available) + end +end diff --git a/db/schema.rb b/db/schema.rb index f03ece6d75f..48f25eb1f1b 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -165,6 +165,7 @@ ActiveRecord::Schema.define(version: 20180503200320) do t.boolean "pages_domain_verification_enabled", default: true, null: false t.boolean "allow_local_requests_from_hooks_and_services", default: false, null: false t.boolean "enforce_terms", default: false + t.boolean "mirror_available", default: true, null: false end create_table "audit_events", force: :cascade do |t| @@ -314,10 +315,10 @@ ActiveRecord::Schema.define(version: 20180503200320) do t.integer "auto_canceled_by_id" t.boolean "retried" t.integer "stage_id" - t.boolean "protected" - t.integer "failure_reason" t.integer "artifacts_file_store" t.integer "artifacts_metadata_store" + t.boolean "protected" + t.integer "failure_reason" end add_index "ci_builds", ["artifacts_expire_at"], name: "index_ci_builds_on_artifacts_expire_at", where: "(artifacts_file <> ''::text)", using: :btree @@ -365,13 +366,13 @@ ActiveRecord::Schema.define(version: 20180503200320) do t.integer "project_id", null: false t.integer "job_id", null: false t.integer "file_type", null: false + t.integer "file_store" t.integer "size", limit: 8 t.datetime_with_timezone "created_at", null: false t.datetime_with_timezone "updated_at", null: false t.datetime_with_timezone "expire_at" t.string "file" t.binary "file_sha256" - t.integer "file_store" end add_index "ci_job_artifacts", ["expire_at", "job_id"], name: "index_ci_job_artifacts_on_expire_at_and_job_id", using: :btree @@ -1593,6 +1594,7 @@ ActiveRecord::Schema.define(version: 20180503200320) do t.boolean "merge_requests_rebase_enabled", default: false, null: false t.integer "jobs_cache_index" t.boolean "pages_https_only", default: true + t.boolean "remote_mirror_available_overridden" end add_index "projects", ["ci_id"], name: "index_projects_on_ci_id", using: :btree @@ -1698,6 +1700,27 @@ ActiveRecord::Schema.define(version: 20180503200320) do add_index "releases", ["project_id", "tag"], name: "index_releases_on_project_id_and_tag", using: :btree add_index "releases", ["project_id"], name: "index_releases_on_project_id", using: :btree + create_table "remote_mirrors", force: :cascade do |t| + t.integer "project_id" + t.string "url" + t.boolean "enabled", default: true + t.string "update_status" + t.datetime "last_update_at" + t.datetime "last_successful_update_at" + t.datetime "last_update_started_at" + t.string "last_error" + t.boolean "only_protected_branches", default: false, null: false + t.string "remote_name" + t.text "encrypted_credentials" + t.string "encrypted_credentials_iv" + t.string "encrypted_credentials_salt" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + end + + add_index "remote_mirrors", ["last_successful_update_at"], name: "index_remote_mirrors_on_last_successful_update_at", using: :btree + add_index "remote_mirrors", ["project_id"], name: "index_remote_mirrors_on_project_id", using: :btree + create_table "routes", force: :cascade do |t| t.integer "source_id", null: false t.string "source_type", null: false @@ -2233,6 +2256,7 @@ ActiveRecord::Schema.define(version: 20180503200320) do add_foreign_key "protected_tags", "projects", name: "fk_8e4af87648", on_delete: :cascade add_foreign_key "push_event_payloads", "events", name: "fk_36c74129da", on_delete: :cascade add_foreign_key "releases", "projects", name: "fk_47fe2a0596", on_delete: :cascade + add_foreign_key "remote_mirrors", "projects", on_delete: :cascade add_foreign_key "services", "projects", name: "fk_71cce407f9", on_delete: :cascade add_foreign_key "snippets", "projects", name: "fk_be41fd4bb7", on_delete: :cascade add_foreign_key "subscriptions", "projects", on_delete: :cascade diff --git a/doc/workflow/repository_mirroring.md b/doc/workflow/repository_mirroring.md new file mode 100644 index 00000000000..dbe63144e38 --- /dev/null +++ b/doc/workflow/repository_mirroring.md @@ -0,0 +1,111 @@ +# Repository mirroring + +Repository Mirroring is a way to mirror repositories from external sources. +It can be used to mirror all branches, tags, and commits that you have +in your repository. + +Your mirror at GitLab will be updated automatically. You can +also manually trigger an update at most once every 5 minutes. + +## Overview + +Repository mirroring is very useful when, for some reason, you must use a +project from another source. + +There are two kinds of repository mirroring features supported by GitLab: +**push** and **pull**, the latter being only available in GitLab Enterprise Edition. +The **push** method mirrors the repository in GitLab to another location. + +Once the mirror repository is updated, all new branches, +tags, and commits will be visible in the project's activity feed. +Users with at least [developer access][perms] to the project can also force an +immediate update with the click of a button. This button will not be available if +the mirror is already being updated or 5 minutes still haven't passed since its last update. + +A few things/limitations to consider: + +- The repository must be accessible over `http://`, `https://`, `ssh://` or `git://`. +- If your HTTP repository is not publicly accessible, add authentication + information to the URL, like: `https://username@gitlab.company.com/group/project.git`. + In some cases, you might need to use a personal access token instead of a + password, e.g., you want to mirror to GitHub and have 2FA enabled. +- The import will time out after 15 minutes. For repositories that take longer + use a clone/push combination. +- The Git LFS objects will not be synced. You'll need to push/pull them + manually. + +## Use-case + +- You have old projects in another source that you don't use actively anymore, + but don't want to remove for archiving purposes. In that case, you can create + a push mirror so that your active GitLab repository can push its changes to the + old location. + +## Pushing to a remote repository **[STARTER]** + +>[Introduced](https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/249) in +GitLab Enterprise Edition 8.7. [Moved to GitLab Community Edition][ce-18715] in 10.8. + +For an existing project, you can set up push mirror from your project's +**Settings ➔ Repository** and searching for the "Push to a remote repository" +section. Check the "Remote mirror repository" box and fill in the Git URL of +the repository to push to. Click **Save changes** for the changes to take +effect. + + + +When push mirroring is enabled, you are advised not to push commits directly +to the mirrored repository to prevent the mirror diverging. +All changes will end up in the mirrored repository whenever commits +are pushed to GitLab, or when a [forced update](#forcing-an-update) is +initiated. + +Pushes into GitLab are automatically pushed to the remote mirror at least once +every 5 minutes after they are received or once every minute if **push only +protected branches** is enabled. + +In case of a diverged branch, you will see an error indicated at the **Mirror +repository** settings. + + + +### Push only protected branches + +>[Introduced][ee-3350] in GitLab Enterprise Edition 10.3. [Moved to GitLab Community Edition][ce-18715] in 10.8. + +You can choose to only push your protected branches from GitLab to your remote repository. + +To use this option go to your project's repository settings page under push mirror. + +## Setting up a push mirror from GitLab to GitHub + +To set up a mirror from GitLab to GitHub, you need to follow these steps: + +1. Create a [GitHub personal access token](https://help.github.com/articles/creating-a-personal-access-token-for-the-command-line/) with the "public_repo" box checked: + +  + +1. Fill in the "Git repository URL" with the personal access token replacing the password `https://GitHubUsername:GitHubPersonalAccessToken@github.com/group/project.git`: + +  + +1. Save +1. And either wait or trigger the "Update Now" button: + +  + +## Forcing an update + +While mirrors are scheduled to update automatically, you can always force an update +by using the **Update now** button which is exposed in various places: + +- in the commits page +- in the branches page +- in the tags page +- in the **Mirror repository** settings page + +[ee-3350]: https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/3350 +[ce-18715]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/18715 +[perms]: ../user/permissions.md + diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index 8015c0926b6..41622fbbb6f 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -1852,7 +1852,7 @@ describe Project do it { expect(project.gitea_import?).to be true } end - describe '#has_remote_mirror?' do + describe '#has_remote_mirror?' do let(:project) { create(:project, :remote_mirror, :import_started) } subject { project.has_remote_mirror? } diff --git a/spec/models/remote_mirror_spec.rb b/spec/models/remote_mirror_spec.rb index d6181ca0783..a80800c6c92 100644 --- a/spec/models/remote_mirror_spec.rb +++ b/spec/models/remote_mirror_spec.rb @@ -164,14 +164,6 @@ describe RemoteMirror do end end - context 'as a Geo secondary' do - it 'returns nil' do - allow(Gitlab::Geo).to receive(:secondary?).and_return(true) - - expect(remote_mirror.sync).to be_nil - end - end - context 'with remote mirroring enabled' do context 'with only protected branches enabled' do context 'when it did not update in the last minute' do diff --git a/spec/models/repository_spec.rb b/spec/models/repository_spec.rb index 44d9ffd258d..4b736b02b7d 100644 --- a/spec/models/repository_spec.rb +++ b/spec/models/repository_spec.rb @@ -2370,6 +2370,11 @@ describe Repository do end end + def create_remote_branch(remote_name, branch_name, target) + rugged = repository.rugged + rugged.references.create("refs/remotes/#{remote_name}/#{branch_name}", target.id) + end + describe '#ancestor?' do let(:commit) { repository.commit } let(:ancestor) { commit.parents.first } |