Merge branch 'qmnguyen0711/publish-development-gem' into 'master'

Enhance developing workflow with Ruby gem in Gitlab Rails

See merge request https://gitlab.com/gitlab-org/gitaly/-/merge_requests/6593

Merged-by: Quang-Minh Nguyen <qmnguyen@gitlab.com>
Approved-by: Toon Claes <toon@gitlab.com>
Approved-by: Eric Ju <eju@gitlab.com>
Reviewed-by: Toon Claes <toon@gitlab.com>
Reviewed-by: Quang-Minh Nguyen <qmnguyen@gitlab.com>
Reviewed-by: Eric Ju <eju@gitlab.com>

3 files changed, 78 insertions, 8 deletions

diff --git a/Makefile b/Makefile
index fb5a9904a..b5a757c30 100644
--- a/Makefile
+++ b/Makefile
@@ -195,6 +195,8 @@ TEST_REPO_DIR     := ${BUILD_DIR}/testrepos
 BENCHMARK_REPO    := ${TEST_REPO_DIR}/benchmark.git
 ## Options to pass to the script which builds the Gitaly gem
 BUILD_GEM_OPTIONS ?=
+## Options to override the name of Gitaly gem
+BUILD_GEM_NAME ?= gitaly
 
 # All executables provided by Gitaly.
 GITALY_EXECUTABLES           = $(addprefix ${BUILD_DIR}/bin/,$(notdir $(shell find ${SOURCE_DIR}/cmd -mindepth 1 -maxdepth 1 -type d -print)))
@@ -465,9 +467,10 @@ lint-proto: ${PROTOC} ${PROTOLINT} ${PROTOC_GEN_GITALY_LINT}
 .PHONY: build-proto-gem
 ## Build the Ruby Gem that contains Gitaly's Protobuf definitons.
 build-proto-gem:
-	${Q}rm -rf "${BUILD_DIR}/gitaly.gem" && mkdir -p ${BUILD_DIR}
+	${Q}rm -rf "${BUILD_DIR}/${BUILD_GEM_NAME}.gem" && mkdir -p ${BUILD_DIR}
+	${Q}rm -rf "${BUILD_DIR}/${BUILD_GEM_NAME}-gem" && mkdir -p ${BUILD_DIR}/${BUILD_GEM_NAME}-gem
 	${Q}cd "${SOURCE_DIR}"/tools/protogem && bundle install
-	${Q}"${SOURCE_DIR}"/tools/protogem/build-proto-gem -o "${BUILD_DIR}/gitaly.gem" ${BUILD_GEM_OPTIONS}
+	${Q}"${SOURCE_DIR}"/tools/protogem/build-proto-gem -o "${BUILD_DIR}/${BUILD_GEM_NAME}.gem" --name ${BUILD_GEM_NAME} --working-dir ${BUILD_DIR}/${BUILD_GEM_NAME}-gem ${BUILD_GEM_OPTIONS}
 
 .PHONY: publish-proto-gem
 ## Build and publish the Ruby Gem that contains Gitaly's Protobuf definitons.
diff --git a/doc/PROCESS.md b/doc/PROCESS.md
index ba695920a..c266071b5 100644
--- a/doc/PROCESS.md
+++ b/doc/PROCESS.md
@@ -667,6 +667,51 @@ If the changes needed are not yet released, [create a release candidate](#creati
 - Run `make publish-proto-gem`, which automatically builds any publishes the
   Protobuf Gem.
 
+## Using an unpublished Gem in GitLab Rails
+
+Sometimes, we need to work on both Gitaly and GitLab Rails implementations in parallel. Ideally, we can define and
+publish all Protobuf changes beforehand. In practice, a complicated change might require modifying the Protobuf multiple
+times until reaching a stable state. It's more convenient to let GitLab Rails point the gem to the the developing branch
+in Gitaly.
+
+In the local environment, we can point it to the developing gem using following steps:
+
+- In Gitaly directory, run `BUILD_GEM_OPTIONS="--skip-verify-tag" make build-proto-gem`. This command
+  builds gem at `_built/gitaly.gem` and the gem contents to the `_build/gitaly-gem` directory.
+- Modify GitLab Rails' Gemfile:
+
+  ```ruby
+  gem 'gitaly', path: '/path/to/gitaly/_build/gitaly-gem'
+  ```
+
+- Re-run bundler to update the gem.
+
+Unfortunately, the Gitaly repository does not include the gemspec or any auto-generated Ruby code. All of it is
+generated on the fly while running `make build-proto-gem`. Thus, we cannot point the `Gemfile` to the developing branch
+directly on CI environment.
+
+As a workaround, we can build and publish the gem under a different name for development purpose only. Here's [an
+example](https://rubygems.org/gems/gitaly-qmnguyen).
+
+- Run `BUILD_GEM_OPTIONS="--skip-verify-tag" BUILD_GEM_NAME="gitaly-yourname" make build-proto-gem`. This command
+  builds the gem under a different name. The resulting gem is located at `_build/gitaly-yourname.gem`.
+- Run `gem push _build/gitaly-yourname.gem` to publish that custom gem to rubygems.org or similar package registry.
+- Modify `gitaly` gem in GitLab Rails' `Gemfile`:
+
+  ```ruby
+  gem 'gitaly-yourname'
+  ```
+
+  Or, we can specify a particular version.
+
+  ```ruby
+  gem 'gitaly-yourname', '~> 16.7.0.pre.0f0db5710'
+  ```
+
+After the development phase, please remember to remove this workaround and restore the `Gemfile` to the official Gitaly gem release.
+
+Note: Instead of setting `BUILD_GEM_OPTIONS` and `BUILD_GEM_NAME` in every command, we can also set them in `config.mak`.
+
 ## Publishing the go module
 
 If an [updated version](https://golang.org/doc/modules/release-workflow) of the go module is needed, it can be [published](https://golang.org/doc/modules/publishing)
diff --git a/tools/protogem/build-proto-gem b/tools/protogem/build-proto-gem
index aca451170..bfacea8f2 100755
--- a/tools/protogem/build-proto-gem
+++ b/tools/protogem/build-proto-gem
@@ -16,10 +16,18 @@ def parse_options
   option_parser = OptionParser.new do |opts|
     opts.banner = "Usage: build-proto-gem [options]"
 
+    opts.on("-n", "--name NAME", "Name of the gem. By default, the name is hard-coded to `gitaly`") do |name|
+      options[:gem_name] = name
+    end
+
     opts.on_tail("--skip-verify-tag", "Skip verification that this is run for a tagged Gitaly commit") do
       options[:skip_verify_tag] = true
     end
 
+    opts.on("-w", "--working-dir DIR", "Working dir of the gem. If not specified, a temporary dir is used") do |path|
+      options[:working_dir] = path
+    end
+
     opts.on("-o", "--output PATH", "output path for the gem") do |path|
       options[:output_path] = File.absolute_path(path)
     end
@@ -41,7 +49,15 @@ def main(options)
     abort "Version string #{version.inspect} does not look like a Gitaly Release tag (e.g. \"v1.0.2\"). Aborting."
   end
 
-  if !options[:skip_verify_tag]
+  if options[:skip_verify_tag]
+    matches = /^(\d+\.\d+\.\d+).*/.match(version)
+    if matches.nil?
+      abort "Invalid version number #{version}"
+    end
+
+    ref = capture!(%w[git rev-parse --short HEAD]).chomp
+    version = "#{matches[1]}-#{ref}"
+  else
     ref = capture!(%w[git describe --tag]).chomp
     if ref != "v#{version}"
       abort "Checkout tag v#{version} to publish.\n\t git checkout v#{version}"
@@ -54,9 +70,15 @@ def main(options)
   puts 'Testing for staged changes'
   run!(%w[git diff --quiet --cached --exit-code])
 
-  Dir.mktmpdir do |output_dir|
+  if options[:working_dir]
+    output_dir = File.absolute_path(options[:working_dir])
     generate_sources(output_dir, version)
-    build_gem(output_dir, options[:output_path])
+    build_gem(options, output_dir, options[:output_path])
+  else
+    Dir.mktmpdir do |output_dir|
+      generate_sources(output_dir, version)
+      build_gem(options, output_dir, options[:output_path])
+    end
   end
 end
 
@@ -77,7 +99,7 @@ def generate_sources(output_dir, version)
   write_ruby_requires(output_dir)
 end
 
-def build_gem(output_dir, output_path)
+def build_gem(options, output_dir, output_path)
   gemspec = <<~EOT
     # coding: utf-8
     prefix = 'ruby/proto'
@@ -85,7 +107,7 @@ def build_gem(output_dir, output_path)
     require 'gitaly/version'
 
     Gem::Specification.new do |spec|
-      spec.name          = "gitaly"
+      spec.name          = "#{options[:gem_name] || 'gitaly'}"
       spec.version       = Gitaly::VERSION
       spec.authors       = ["GitLab Engineering"]
       spec.email         = ["engineering@gitlab.com"]
@@ -108,7 +130,7 @@ def build_gem(output_dir, output_path)
     end
   EOT
 
-  gemspec_path = File.join(output_dir, 'gitaly.gemspec')
+  gemspec_path = File.absolute_path(File.join(output_dir, 'gitaly.gemspec'))
   open(gemspec_path, 'w') { |f| f.write(gemspec) }
 
   run!(['gem', 'build', gemspec_path, '--output', output_path], output_dir)