diff options
author | Grzegorz Bizon <grzegorz@gitlab.com> | 2018-03-29 14:07:54 +0300 |
---|---|---|
committer | Grzegorz Bizon <grzegorz@gitlab.com> | 2018-03-29 14:07:54 +0300 |
commit | 5501dfd13ffda6abe8ff191a65964abbd4f3bb30 (patch) | |
tree | 3b25faf2d41893bc67e972fa56a89b6efd3ac0dc /lib/gitlab | |
parent | f4d81536ac26f75e0aad248ad95c31c9e1f2680b (diff) | |
parent | 8230b774b5a6375d9b84c71e72c8e41e3bfd7fad (diff) |
Merge branch 'master' into 'feature/gb/variables-expressions-in-only-except'
Conflicts:
app/models/ci/build.rb
Diffstat (limited to 'lib/gitlab')
-rw-r--r-- | lib/gitlab/background_migration/migrate_build_stage.rb | 1 | ||||
-rw-r--r-- | lib/gitlab/bare_repository_import/importer.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/checks/project_moved.rb | 20 | ||||
-rw-r--r-- | lib/gitlab/ci/build/step.rb | 4 | ||||
-rw-r--r-- | lib/gitlab/ci/config.rb | 3 | ||||
-rw-r--r-- | lib/gitlab/ci/trace/http_io.rb | 187 | ||||
-rw-r--r-- | lib/gitlab/ci/trace/stream.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/ci/yaml_processor.rb | 8 | ||||
-rw-r--r-- | lib/gitlab/database/migration_helpers.rb | 39 | ||||
-rw-r--r-- | lib/gitlab/git/repository.rb | 3 | ||||
-rw-r--r-- | lib/gitlab/git_access.rb | 18 | ||||
-rw-r--r-- | lib/gitlab/gitaly_client.rb | 4 | ||||
-rw-r--r-- | lib/gitlab/gitaly_client/remote_service.rb | 11 | ||||
-rw-r--r-- | lib/gitlab/github_import/importer/repository_importer.rb | 8 | ||||
-rw-r--r-- | lib/gitlab/verify/lfs_objects.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/verify/uploads.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/workhorse.rb | 37 |
17 files changed, 280 insertions, 71 deletions
diff --git a/lib/gitlab/background_migration/migrate_build_stage.rb b/lib/gitlab/background_migration/migrate_build_stage.rb index 8fe4f1a2289..242e3143e71 100644 --- a/lib/gitlab/background_migration/migrate_build_stage.rb +++ b/lib/gitlab/background_migration/migrate_build_stage.rb @@ -12,6 +12,7 @@ module Gitlab class Build < ActiveRecord::Base self.table_name = 'ci_builds' + self.inheritance_column = :_type_disabled def ensure_stage!(attempts: 2) find_stage || create_stage! diff --git a/lib/gitlab/bare_repository_import/importer.rb b/lib/gitlab/bare_repository_import/importer.rb index 884a3de8f62..1a25138e7d6 100644 --- a/lib/gitlab/bare_repository_import/importer.rb +++ b/lib/gitlab/bare_repository_import/importer.rb @@ -63,7 +63,7 @@ module Gitlab log " * Created #{project.name} (#{project_full_path})".color(:green) project.write_repository_config - project.repository.create_hooks + Gitlab::Git::Repository.create_hooks(project.repository.path_to_repo, Gitlab.config.gitlab_shell.hooks_path) ProjectCacheWorker.perform_async(project.id) else diff --git a/lib/gitlab/checks/project_moved.rb b/lib/gitlab/checks/project_moved.rb index 3263790a876..3a197078d08 100644 --- a/lib/gitlab/checks/project_moved.rb +++ b/lib/gitlab/checks/project_moved.rb @@ -9,20 +9,16 @@ module Gitlab super(project, user, protocol) end - def message(rejected: false) + def message <<~MESSAGE Project '#{redirected_path}' was moved to '#{project.full_path}'. Please update your Git remote: - #{remote_url_message(rejected)} + git remote set-url origin #{url_to_repo} MESSAGE end - def permanent_redirect? - RedirectRoute.permanent.exists?(path: redirected_path) - end - private attr_reader :redirected_path @@ -30,18 +26,6 @@ module Gitlab def self.message_key(user_id, project_id) "#{REDIRECT_NAMESPACE}:#{user_id}:#{project_id}" end - - def remote_url_message(rejected) - if rejected - "git remote set-url origin #{url_to_repo} and try again." - else - "git remote set-url origin #{url_to_repo}" - end - end - - def url - protocol == 'ssh' ? project.ssh_url_to_repo : project.http_url_to_repo - end end end end diff --git a/lib/gitlab/ci/build/step.rb b/lib/gitlab/ci/build/step.rb index 411f67f8ce7..0b1ebe4e048 100644 --- a/lib/gitlab/ci/build/step.rb +++ b/lib/gitlab/ci/build/step.rb @@ -14,7 +14,7 @@ module Gitlab self.new(:script).tap do |step| step.script = job.options[:before_script].to_a + job.options[:script].to_a step.script = job.commands.split("\n") if step.script.empty? - step.timeout = job.timeout + step.timeout = job.metadata_timeout step.when = WHEN_ON_SUCCESS end end @@ -25,7 +25,7 @@ module Gitlab self.new(:after_script).tap do |step| step.script = after_script - step.timeout = job.timeout + step.timeout = job.metadata_timeout step.when = WHEN_ALWAYS step.allow_failure = true end diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index f7ff7ea212e..66ac4a40616 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -4,7 +4,8 @@ module Gitlab # Base GitLab CI Configuration facade # class Config - def initialize(config) + # EE would override this and utilize opts argument + def initialize(config, opts = {}) @config = Loader.new(config).load! @global = Entry::Global.new(@config) diff --git a/lib/gitlab/ci/trace/http_io.rb b/lib/gitlab/ci/trace/http_io.rb new file mode 100644 index 00000000000..ac4308f4e2c --- /dev/null +++ b/lib/gitlab/ci/trace/http_io.rb @@ -0,0 +1,187 @@ +## +# This class is compatible with IO class (https://ruby-doc.org/core-2.3.1/IO.html) +# source: https://gitlab.com/snippets/1685610 +module Gitlab + module Ci + class Trace + class HttpIO + BUFFER_SIZE = 128.kilobytes + + InvalidURLError = Class.new(StandardError) + FailedToGetChunkError = Class.new(StandardError) + + attr_reader :uri, :size + attr_reader :tell + attr_reader :chunk, :chunk_range + + alias_method :pos, :tell + + def initialize(url, size) + raise InvalidURLError unless ::Gitlab::UrlSanitizer.valid?(url) + + @uri = URI(url) + @size = size + @tell = 0 + end + + def close + # no-op + end + + def binmode + # no-op + end + + def binmode? + true + end + + def path + nil + end + + def url + @uri.to_s + end + + def seek(pos, where = IO::SEEK_SET) + new_pos = + case where + when IO::SEEK_END + size + pos + when IO::SEEK_SET + pos + when IO::SEEK_CUR + tell + pos + else + -1 + end + + raise 'new position is outside of file' if new_pos < 0 || new_pos > size + + @tell = new_pos + end + + def eof? + tell == size + end + + def each_line + until eof? + line = readline + break if line.nil? + + yield(line) + end + end + + def read(length = nil) + out = "" + + until eof? || (length && out.length >= length) + data = get_chunk + break if data.empty? + + out << data + @tell += data.bytesize + end + + out = out[0, length] if length && out.length > length + + out + end + + def readline + out = "" + + until eof? + data = get_chunk + new_line = data.index("\n") + + if !new_line.nil? + out << data[0..new_line] + @tell += new_line + 1 + break + else + out << data + @tell += data.bytesize + end + end + + out + end + + def write(data) + raise NotImplementedError + end + + def truncate(offset) + raise NotImplementedError + end + + def flush + raise NotImplementedError + end + + def present? + true + end + + private + + ## + # The below methods are not implemented in IO class + # + def in_range? + @chunk_range&.include?(tell) + end + + def get_chunk + unless in_range? + response = Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == 'https') do |http| + http.request(request) + end + + raise FailedToGetChunkError unless response.code == '200' || response.code == '206' + + @chunk = response.body.force_encoding(Encoding::BINARY) + @chunk_range = response.content_range + + ## + # Note: If provider does not return content_range, then we set it as we requested + # Provider: minio + # - When the file size is larger than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206 + # - When the file size is smaller than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206 + # Provider: AWS + # - When the file size is larger than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206 + # - When the file size is smaller than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206 + # Provider: GCS + # - When the file size is larger than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206 + # - When the file size is smaller than requested Content-range, the Content-range is included in responces with Net::HTTPOK 200 + @chunk_range ||= (chunk_start...(chunk_start + @chunk.length)) + end + + @chunk[chunk_offset..BUFFER_SIZE] + end + + def request + Net::HTTP::Get.new(uri).tap do |request| + request.set_range(chunk_start, BUFFER_SIZE) + end + end + + def chunk_offset + tell % BUFFER_SIZE + end + + def chunk_start + (tell / BUFFER_SIZE) * BUFFER_SIZE + end + + def chunk_end + [chunk_start + BUFFER_SIZE, size].min + end + end + end + end +end diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb index d52194f688b..b3fe3ef1c4d 100644 --- a/lib/gitlab/ci/trace/stream.rb +++ b/lib/gitlab/ci/trace/stream.rb @@ -8,7 +8,7 @@ module Gitlab attr_reader :stream - delegate :close, :tell, :seek, :size, :path, :truncate, to: :stream, allow_nil: true + delegate :close, :tell, :seek, :size, :path, :url, :truncate, to: :stream, allow_nil: true delegate :valid?, to: :stream, as: :present?, allow_nil: true diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index bc2a6f98dae..e829f2a95f8 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -7,8 +7,8 @@ module Gitlab attr_reader :cache, :stages, :jobs - def initialize(config) - @ci_config = Gitlab::Ci::Config.new(config) + def initialize(config, opts = {}) + @ci_config = Gitlab::Ci::Config.new(config, opts) @config = @ci_config.to_hash unless @ci_config.valid? @@ -73,11 +73,11 @@ module Gitlab end end - def self.validation_message(content) + def self.validation_message(content, opts = {}) return 'Please provide content of .gitlab-ci.yml' if content.blank? begin - Gitlab::Ci::YamlProcessor.new(content) + Gitlab::Ci::YamlProcessor.new(content, opts) nil rescue ValidationError => e e.message diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 44ca434056f..1634fe4e9cb 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -900,11 +900,42 @@ into similar problems in the future (e.g. when new tables are created). end end - # Rails' index_exists? doesn't work when you only give it a table and index - # name. As such we have to use some extra code to check if an index exists for - # a given name. + # Fetches indexes on a column by name for postgres. + # + # This will include indexes using an expression on the column, for example: + # `CREATE INDEX CONCURRENTLY index_name ON table (LOWER(column));` + # + # For mysql, it falls back to the default ActiveRecord implementation that + # will not find custom indexes. But it will select by name without passing + # a column. + # + # We can remove this when upgrading to Rails 5 with an updated `index_exists?`: + # - https://github.com/rails/rails/commit/edc2b7718725016e988089b5fb6d6fb9d6e16882 + # + # Or this can be removed when we no longer support postgres < 9.5, so we + # can use `CREATE INDEX IF NOT EXISTS`. def index_exists_by_name?(table, index) - indexes(table).map(&:name).include?(index) + # We can't fall back to the normal `index_exists?` method because that + # does not find indexes without passing a column name. + if indexes(table).map(&:name).include?(index.to_s) + true + elsif Gitlab::Database.postgresql? + postgres_exists_by_name?(table, index) + else + false + end + end + + def postgres_exists_by_name?(table, name) + index_sql = <<~SQL + SELECT COUNT(*) + FROM pg_index + JOIN pg_class i ON (indexrelid=i.oid) + JOIN pg_class t ON (indrelid=t.oid) + WHERE i.relname = '#{name}' AND t.relname = '#{table}' + SQL + + connection.select_value(index_sql).to_i > 0 end end end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 20b0647fce9..2d16a81c888 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -8,6 +8,7 @@ module Gitlab class Repository include Gitlab::Git::RepositoryMirroring include Gitlab::Git::Popen + include Gitlab::EncodingHelper ALLOWED_OBJECT_DIRECTORIES_VARIABLES = %w[ GIT_OBJECT_DIRECTORY @@ -1479,7 +1480,7 @@ module Gitlab names.lines.each do |line| next unless line.start_with?(refs_prefix) - refs << line.rstrip[left_slice_count..-1] + refs << encode_utf8(line.rstrip[left_slice_count..-1]) end refs diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 6400089a22f..ed0644f6cf1 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -53,7 +53,7 @@ module Gitlab ensure_project_on_push!(cmd, changes) check_project_accessibility! - check_project_moved! + add_project_moved_message! check_repository_existence! case cmd @@ -99,8 +99,6 @@ module Gitlab end def check_active_user! - return if deploy_key? - if user && !user_access.allowed? raise UnauthorizedError, ERROR_MESSAGES[:account_blocked] end @@ -125,16 +123,12 @@ module Gitlab end end - def check_project_moved! + def add_project_moved_message! return if redirected_path.nil? project_moved = Checks::ProjectMoved.new(project, user, protocol, redirected_path) - if project_moved.permanent_redirect? - project_moved.add_message - else - raise ProjectMovedError, project_moved.message(rejected: true) - end + project_moved.add_message end def check_command_disabled!(cmd) @@ -219,7 +213,7 @@ module Gitlab raise UnauthorizedError, ERROR_MESSAGES[:read_only] end - if deploy_key + if deploy_key? unless deploy_key.can_push_to?(project) raise UnauthorizedError, ERROR_MESSAGES[:deploy_key_upload] end @@ -309,8 +303,10 @@ module Gitlab case actor when User actor + when DeployKey + nil when Key - actor.user unless actor.is_a?(DeployKey) + actor.user when :ci nil end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 8ca30ffc232..0abae70c443 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -83,6 +83,10 @@ module Gitlab end end + def self.random_storage + Gitlab.config.repositories.storages.keys.sample + end + def self.address(storage) params = Gitlab.config.repositories.storages[storage] raise "storage not found: #{storage.inspect}" if params.nil? diff --git a/lib/gitlab/gitaly_client/remote_service.rb b/lib/gitlab/gitaly_client/remote_service.rb index 58c356edfd1..f2d699d9dfb 100644 --- a/lib/gitlab/gitaly_client/remote_service.rb +++ b/lib/gitlab/gitaly_client/remote_service.rb @@ -3,6 +3,17 @@ module Gitlab class RemoteService MAX_MSG_SIZE = 128.kilobytes.freeze + def self.exists?(remote_url) + request = Gitaly::FindRemoteRepositoryRequest.new(remote: remote_url) + + response = GitalyClient.call(GitalyClient.random_storage, + :remote_service, + :find_remote_repository, request, + timeout: GitalyClient.medium_timeout) + + response.exists + end + def initialize(repository) @repository = repository @gitaly_repo = repository.gitaly_repository diff --git a/lib/gitlab/github_import/importer/repository_importer.rb b/lib/gitlab/github_import/importer/repository_importer.rb index ab0b751fe24..b1b283e98b5 100644 --- a/lib/gitlab/github_import/importer/repository_importer.rb +++ b/lib/gitlab/github_import/importer/repository_importer.rb @@ -16,7 +16,8 @@ module Gitlab # Returns true if we should import the wiki for the project. def import_wiki? client.repository(project.import_source)&.has_wiki && - !project.wiki_repository_exists? + !project.wiki_repository_exists? && + Gitlab::GitalyClient::RemoteService.exists?(wiki_url) end # Imports the repository data. @@ -55,7 +56,6 @@ module Gitlab def import_wiki_repository wiki_path = "#{project.disk_path}.wiki" - wiki_url = project.import_url.sub(/\.git\z/, '.wiki.git') storage_path = project.repository_storage_path gitlab_shell.import_repository(storage_path, wiki_path, wiki_url) @@ -70,6 +70,10 @@ module Gitlab end end + def wiki_url + project.import_url.sub(/\.git\z/, '.wiki.git') + end + def update_clone_time project.update_column(:last_repository_updated_at, Time.zone.now) end diff --git a/lib/gitlab/verify/lfs_objects.rb b/lib/gitlab/verify/lfs_objects.rb index fe51edbdeeb..970e2a7b718 100644 --- a/lib/gitlab/verify/lfs_objects.rb +++ b/lib/gitlab/verify/lfs_objects.rb @@ -12,7 +12,7 @@ module Gitlab private def relation - LfsObject.all + LfsObject.with_files_stored_locally end def expected_checksum(lfs_object) diff --git a/lib/gitlab/verify/uploads.rb b/lib/gitlab/verify/uploads.rb index 6972e517ea5..0ffa71a6d72 100644 --- a/lib/gitlab/verify/uploads.rb +++ b/lib/gitlab/verify/uploads.rb @@ -12,7 +12,7 @@ module Gitlab private def relation - Upload.all + Upload.with_files_stored_locally end def expected_checksum(upload) diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 0b0d667d4fd..b102812ec12 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -21,29 +21,18 @@ module Gitlab raise "Unsupported action: #{action}" unless ALLOWED_GIT_HTTP_ACTIONS.include?(action.to_s) project = repository.project - repo_path = repository.path_to_repo - params = { + + { GL_ID: Gitlab::GlId.gl_id(user), GL_REPOSITORY: Gitlab::GlRepository.gl_repository(project, is_wiki), GL_USERNAME: user&.username, - RepoPath: repo_path, - ShowAllRefs: show_all_refs - } - server = { - address: Gitlab::GitalyClient.address(project.repository_storage), - token: Gitlab::GitalyClient.token(project.repository_storage) - } - params[:Repository] = repository.gitaly_repository.to_h - params[:GitalyServer] = server - - params - end - - def lfs_upload_ok(oid, size) - { - StoreLFSPath: LfsObjectUploader.workhorse_upload_path, - LfsOid: oid, - LfsSize: size + ShowAllRefs: show_all_refs, + Repository: repository.gitaly_repository.to_h, + RepoPath: 'ignored but not allowed to be empty in gitlab-workhorse', + GitalyServer: { + address: Gitlab::GitalyClient.address(project.repository_storage), + token: Gitlab::GitalyClient.token(project.repository_storage) + } } end @@ -52,7 +41,7 @@ module Gitlab end def send_git_blob(repository, blob) - params = if Gitlab::GitalyClient.feature_enabled?(:workhorse_raw_show) + params = if Gitlab::GitalyClient.feature_enabled?(:workhorse_raw_show, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) { 'GitalyServer' => gitaly_server_hash(repository), 'GetBlobRequest' => { @@ -80,7 +69,7 @@ module Gitlab params = repository.archive_metadata(ref, Gitlab.config.gitlab.repository_downloads_path, format) raise "Repository or ref not found" if params.empty? - if Gitlab::GitalyClient.feature_enabled?(:workhorse_archive) + if Gitlab::GitalyClient.feature_enabled?(:workhorse_archive, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) params.merge!( 'GitalyServer' => gitaly_server_hash(repository), 'GitalyRepository' => repository.gitaly_repository.to_h @@ -97,7 +86,7 @@ module Gitlab end def send_git_diff(repository, diff_refs) - params = if Gitlab::GitalyClient.feature_enabled?(:workhorse_send_git_diff) + params = if Gitlab::GitalyClient.feature_enabled?(:workhorse_send_git_diff, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) { 'GitalyServer' => gitaly_server_hash(repository), 'RawDiffRequest' => Gitaly::RawDiffRequest.new( @@ -115,7 +104,7 @@ module Gitlab end def send_git_patch(repository, diff_refs) - params = if Gitlab::GitalyClient.feature_enabled?(:workhorse_send_git_patch) + params = if Gitlab::GitalyClient.feature_enabled?(:workhorse_send_git_patch, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) { 'GitalyServer' => gitaly_server_hash(repository), 'RawPatchRequest' => Gitaly::RawPatchRequest.new( |