Merge branch 'zj-rakefile-gitaly' into 'master'

Move Gitaly config to Rake tasks

Closes #546

See merge request gitlab-org/gitlab-development-kit!717

11 files changed, 209 insertions, 75 deletions

diff --git a/.gitignore b/.gitignore
index 1bd4d5770f3..1e94ad3bc59 100644
--- a/.gitignore
+++ b/.gitignore
@@ -70,6 +70,7 @@ gitlab-development-kit/**
 /.gdk-install-root
 /builds
 /gitaly/
+praefect_enabled
 /gitlab-workhorse/config.toml
 /gitlab-workhorse/.cache
 /go-gitlab-shell
diff --git a/Makefile b/Makefile
index 31a6784820d..df9ee353b4a 100644
--- a/Makefile
+++ b/Makefile
@@ -92,14 +92,7 @@ auto_devops_registry_port: auto_devops_gitlab_port
 	expr ${auto_devops_gitlab_port} + 5000 > $@
 
 gitlab/config/gitlab.yml: support/templates/gitlab.yml.erb auto_devops_enabled auto_devops_gitlab_port auto_devops_registry_port
-	hostname=${hostname} port=${port} relative_url_root=${relative_url_root}\
-		https=${https}\
-		webpack_port=${webpack_port}\
-		registry_host=${registry_host} registry_external_port=${registry_external_port}\
-		registry_enabled=${registry_enabled} registry_port=${registry_port}\
-		object_store_enabled=${object_store_enabled} object_store_port=${object_store_port}\
-		gitlab_pages_port=${gitlab_pages_port}\
-		support/edit-gitlab-yml gitlab/config/gitlab.yml
+	rake gitlab/config/gitlab.yml
 
 gitlab/config/database.yml: database.yml.example
 	bin/safe-sed "$@" \
@@ -183,7 +176,7 @@ gitlab-shell/.gitlab_shell_secret:
 
 # Set up gitaly
 
-gitaly-setup: gitaly/bin/gitaly gitaly/config.toml ${gitaly_proto_clone_dir}/.git
+gitaly-setup: gitaly/bin/gitaly gitaly/gitaly.config.toml ${gitaly_proto_clone_dir}/.git
 
 ${gitaly_clone_dir}/.git:
 	git clone --quiet --branch "${gitaly_version}" ${git_depth_param} ${gitaly_repo} ${gitaly_clone_dir}
@@ -191,17 +184,8 @@ ${gitaly_clone_dir}/.git:
 ${gitaly_proto_clone_dir}/.git:
 	git clone ${git_depth_param} --quiet ${gitaly_proto_repo} ${gitaly_proto_clone_dir}
 
-gitaly/config.toml: $(gitaly_clone_dir)/config.toml.example
-	bin/safe-sed "$@" \
-		-e "s|/home/git|${gitlab_development_root}|g" \
-		-e "s|^socket_path.*|socket_path = \"${gitlab_development_root}/gitaly.socket\"|" \
-		-e "s|^bin_dir.*|bin_dir = \"${gitlab_development_root}/gitaly/bin\"|" \
-		-e "s|# prometheus_listen_addr|prometheus_listen_addr|" \
-		-e "s|# \[logging\]|\[logging\]|" \
-		-e "s|# level = \"warn\"|level = \"warn\"|" \
-		-e "s|^#[^[]*\[git\].*|\[git\]|" \
-		-e "s|^# catfile_cache_size.*|catfile_cache_size = 5|" \
-		"$<"
+gitaly/gitaly.config.toml: support/templates/gitaly.config.toml.erb
+	rake gitaly/gitaly.config.toml
 
 prom-setup:
 	if [ "$(uname -s)" = "Linux" ]; then \
@@ -627,14 +611,13 @@ clean-config:
 	.ruby-version \
 	Procfile \
 	gitlab-workhorse/config.toml \
-	gitaly/config.toml \
+	gitaly/gitaly.config.toml \
 	nginx/conf/nginx.conf \
 	registry/config.yml \
 	jaeger
 
 touch-examples:
 	touch \
-	$(gitaly_clone_dir)/config.toml.example \
 	Procfile.erb \
 	database.yml.example \
 	database_geo.yml.example \
@@ -650,6 +633,8 @@ touch-examples:
 	redis/redis.conf.example \
 	redis/resque.yml.example \
 	registry/config.yml.example \
+	support/templates/gitaly.toml.erb \
+	support/templates/praefect.toml.erb \
 	support/templates/gitlab.yml.erb
 
 unlock-dependency-installers:
diff --git a/Procfile.erb b/Procfile.erb
index f0920579a86..c31bd3783dc 100644
--- a/Procfile.erb
+++ b/Procfile.erb
@@ -19,7 +19,8 @@ postgresql: exec support/postgresql-signal-wrapper <%= config.postgresql.bin_dir
 
 # Backend services
 #
-gitaly: exec gitaly/bin/gitaly <%= config.gdk_root %>/gitaly/config.toml
+gitaly: exec gitaly/bin/gitaly <%= config.gdk_root %>/gitaly/gitaly.config.toml
+<%= '#' unless config.praefect? %>praefect: exec gitaly/bin/praefect -config <%= config.gdk_root %>/gitaly/praefect.config.toml
 gitlab-workhorse: exec /usr/bin/env PATH="<%= config.gdk_root %>/gitlab-workhorse/bin:$PATH" gitlab-workhorse -authSocket <%= config.gdk_root %>/gitlab.socket -listenAddr $host:<%= config.nginx? ? config.nginx.workhorse_port : '$port' %> -documentRoot <%= config.gdk_root %>/gitlab/public -developmentMode -secretPath <%= config.gdk_root %>/gitlab/.gitlab_workhorse_secret -config <%= config.gdk_root %>/gitlab-workhorse/config.toml
 rails-background-jobs: exec /usr/bin/env SIDEKIQ_LOG_ARGUMENTS=1 RAILS_ENV=development RAILS_RELATIVE_URL_ROOT=$relative_url_root support/exec-cd gitlab bin/background_jobs start_foreground
 
diff --git a/Rakefile b/Rakefile
index 16c27f57857..9fda7fe828a 100644
--- a/Rakefile
+++ b/Rakefile
@@ -4,7 +4,9 @@ $LOAD_PATH.unshift('.')
 
 require 'lib/gdk'
 require 'rake/clean'
-CLOBBER.include 'gdk.example.yml', 'Procfile', 'nginx/conf/nginx.conf'
+
+CONFIGS = FileList['Procfile', 'nginx/conf/nginx.conf', 'gitlab/config/gitlab.yml']
+CLOBBER.include *CONFIGS, 'gdk.example.yml'
 
 def config
   @config ||= GDK::Config.new
@@ -27,6 +29,12 @@ file 'gdk.example.yml' => 'clobber:gdk.example.yml' do |t|
   end
 end
 
+desc 'Regenerate all config files from scratch'
+task reconfigure: [:clobber, :all]
+
+desc 'Generate all config files'
+task all: CONFIGS
+
 task 'clobber:gdk.example.yml' do |t|
   Rake::Cleaner.cleanup_files([t.name])
 end
@@ -40,3 +48,34 @@ desc 'Generate nginx configuration'
 file 'nginx/conf/nginx.conf' => ['nginx/conf/nginx.conf.erb', GDK::Config::FILE] do |t|
   GDK::ErbRenderer.new(t.source, t.name).safe_render!
 end
+
+desc 'Generate the gitlab.yml config file'
+file 'gitlab/config/gitlab.yml' => ['support/templates/gitlab.yml.erb'] do |t|
+  GDK::ErbRenderer.new(t.source, t.name).render!
+end
+
+desc 'Generate Gitaly config toml'
+file 'gitaly/gitaly.config.toml' => ['support/templates/gitaly.config.toml.erb'] do |t|
+  GDK::ErbRenderer.new(t.source, t.name).render!
+end
+
+desc 'Generate Praefects config toml'
+file 'gitaly/praefect.config.toml' => ['support/templates/praefect.config.toml.erb'] do |t|
+  GDK::ErbRenderer.new(t.source, t.name).render!
+end
+
+namespace :praefect do
+  PRAEFECT_ENABLED_PATH = 'praefect_enabled'
+
+  desc 'Enable praefect and configure it to run'
+  task :enable => 'gitaly/praefect.config.toml' do
+    File.write(PRAEFECT_ENABLED_PATH, 'true')
+    Rake::Task[:reconfigure].invoke
+  end
+
+  desc 'Disable praefect and do not run it'
+  task :disable do
+    File.delete(PRAEFECT_ENABLED_PATH)
+    Rake::Task[:reconfigure].invoke
+  end
+end
diff --git a/gdk.example.yml b/gdk.example.yml
index 2ae4e388b33..511d5e7a4dc 100644
--- a/gdk.example.yml
+++ b/gdk.example.yml
@@ -42,6 +42,8 @@ postgresql:
   replica_dir: "/home/git/gdk/postgresql-replica"
   replication_user: gitlab_replication
 protocol: https
+praefect:
+  enabled: false
 registry:
   enabled: false
   external_port: 5000
diff --git a/lib/gdk/config.rb b/lib/gdk/config.rb
index 72190292374..9b83b18a5fe 100644
--- a/lib/gdk/config.rb
+++ b/lib/gdk/config.rb
@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require 'etc'
 require_relative 'config_settings'
 
 module GDK
@@ -16,7 +17,7 @@ module GDK
       r.gitlab_docs 'https://gitlab.com/gitlab-com/gitlab-docs.git'
     end
 
-    gdk_root { ENV['PWD'] }
+    gdk_root { Dir.pwd }
 
     hostname do
       next "#{config.auto_devops.gitlab.port}.qa-tunnel.gitlab.info" if config.auto_devops.enabled
@@ -38,7 +39,7 @@ module GDK
     protocol { config.https? ? 'https' : 'http' }
 
     relative_url_root { read!('relative_url_root') || nil }
-    username { ENV['USERNAME'] }
+    username { Etc.getlogin }
 
     webpack do |w|
       w.port { read!('webpack_port') || 3808 }
@@ -128,6 +129,17 @@ module GDK
 
     gitaly do |g|
       g.assembly_dir { "#{config.gdk_root}/gitaly/assembly" }
+      g.address do
+        if config.praefect?
+          File.join(config.gdk_root, 'praefect.socket')
+        else
+          File.join(config.gdk_root, 'gitaly.socket')
+        end
+      end
+    end
+
+    praefect do |p|
+      p.enabled { read!('praefect_enabled') || false }
     end
 
     sshd do |s|
diff --git a/lib/run.rb b/lib/run.rb
index d3638a85a87..7e8539ba332 100644
--- a/lib/run.rb
+++ b/lib/run.rb
@@ -5,7 +5,7 @@ def main(argv)
   when 'geo_db'
     foreman_exec(%w[postgresql-geo])
   when 'app'
-    svcs = %w[gitlab-workhorse nginx grafana sshd gitaly storage-check gitlab-pages]
+    svcs = %w[gitlab-workhorse nginx grafana sshd gitaly storage-check gitlab-pages praefect]
 
     foreman_exec(svcs + %w[rails-web rails-background-jobs])
   when 'grafana'
diff --git a/support/edit-gitlab-yml b/support/edit-gitlab-yml
deleted file mode 100755
index fa558a98e38..00000000000
--- a/support/edit-gitlab-yml
+++ /dev/null
@@ -1,22 +0,0 @@
-#!/usr/bin/env ruby
-require 'erb'
-require 'etc'
-
-def main(file_path, template)
-  File.open(file_path, 'w') { |f| f.puts template.result(binding) }
-end
-
-def env_or_default(key, default)
-  ENV[key] || default
-end
-
-raw_template = File.read(File.join(__dir__, 'templates', 'gitlab.yml.erb'))
-TEMPLATE = ERB.new(raw_template)
-GDK_ROOT = Dir.pwd
-GIT_BIN = `which git`
-
-file_name = ARGV.first
-
-abort "no file name as argument found" unless file_name
-
-main(ARGV.first, TEMPLATE)
diff --git a/support/templates/gitaly.config.toml.erb b/support/templates/gitaly.config.toml.erb
new file mode 100644
index 00000000000..a5998c3095b
--- /dev/null
+++ b/support/templates/gitaly.config.toml.erb
@@ -0,0 +1,80 @@
+# Example Gitaly configuration file
+
+socket_path = "<%= File.join(config.gdk_root, 'gitaly.socket') %>"
+
+# The directory where Gitaly's executables are stored
+bin_dir = "<%= File.join(config.gdk_root, 'gitaly', 'bin') %>"
+
+# # Optional: listen on a TCP socket. This is insecure (no authentication)
+# listen_addr = "localhost:9999"
+# tls_listen_addr = "localhost:8888
+
+# # Optional: export metrics via Prometheus
+# prometheus_listen_addr = "localhost:9236"
+
+# # Optional: authenticate Gitaly requests using a shared secret
+# [auth]
+# token = 'abc123secret'
+# transitioning = false # Set `transitioning` to true to temporarily allow unauthenticated while rolling out authentication.
+
+# [tls]
+# certificate_path = '/home/git/cert.cert'
+# key_path = '/home/git/key.pem'
+
+# # Git settings
+# [git]
+bin_path = "<%= config.git.bin %>"
+catfile_cache_size = 10
+
+[[storage]]
+name = "default"
+path = "<%= File.join(config.gdk_root, 'repositories') %>"
+
+# # You can optionally configure more storages for this Gitaly instance to serve up
+#
+# [[storage]]
+# name = "other_storage"
+# path = "/mnt/other_storage/repositories"
+#
+
+# # You can optionally configure Gitaly to output JSON-formatted log messages to stdout
+# [logging]
+# format = "json"
+# # Optional: Set log level to only log entries with that severity or above
+# # One of, in order: debug, info, warn, errror, fatal, panic
+# # Defaults to "info"
+level = "warn"
+
+# # Additionally exceptions from the Go server can be reported to Sentry
+# sentry_dsn = "https://<key>:<secret>@sentry.io/<project>"
+# # Exceptions from gitaly-ruby can also be reported to Sentry
+# ruby_sentry_dsn = "https://<key>:<secret>@sentry.io/<project>"
+
+# # You can optionally configure Gitaly to record histogram latencies on GRPC method calls
+# [prometheus]
+# grpc_latency_buckets = [0.001, 0.005, 0.025, 0.1, 0.5, 1.0, 10.0, 30.0, 60.0, 300.0, 1500.0]
+
+[gitaly-ruby]
+# The directory where gitaly-ruby is installed
+dir = "<%= File.join(config.gdk_root, 'gitaly', 'ruby') %>"
+
+# # Gitaly-ruby resident set size (RSS) that triggers a memory restart (bytes)
+# max_rss = 200000000
+#
+# # Grace period before a gitaly-ruby process is forcibly terminated after exceeding max_rss (seconds)
+# graceful_restart_timeout = "10m"
+#
+# # Time that gitaly-ruby memory must remain high before a restart (seconds)
+# restart_delay = "5m"
+#
+# # Number of gitaly-ruby worker processes
+# num_workers = 2
+
+[gitlab-shell]
+# The directory where gitlab-shell is installed
+dir = "<%= File.join(config.gdk_root, 'gitlab-shell') %>"
+
+# # You can adjust the concurrency of each RPC endpoint
+# [[concurrency]]
+# rpc = "/gitaly.RepositoryService/GarbageCollect"
+# max_per_repo = 1
diff --git a/support/templates/gitlab.yml.erb b/support/templates/gitlab.yml.erb
index 628654b3157..6f2de1d5815 100644
--- a/support/templates/gitlab.yml.erb
+++ b/support/templates/gitlab.yml.erb
@@ -29,11 +29,11 @@ production: &base
   ## GitLab settings
   gitlab:
     ## Web server settings (note: host is the FQDN, do not include http://)
-    host: <%= env_or_default('hostname', 'localhost') %>
-    port: <%= env_or_default('port', 3000).to_i %>
-    https: <%= env_or_default('https', false) %>
+    host: <%= config.hostname %>
+    port: <%= config.port %>
+    https: <%= config.https? %>
 
-    relative_url_root: "<%= env_or_default('relative_url_root', '') %>"
+    relative_url_root: "<%= config.relative_url_root %>"
 
     # Trusted Proxies
     # Customize if you have GitLab behind a reverse proxy which is running on a different machine.
@@ -44,7 +44,7 @@ production: &base
       #- 192.168.2.1
       #- 2001:0db8::/32
 
-    user: <%= Etc.getlogin %>
+    user: <%= config.username %>
 
     ## Date & Time settings
     # Uncomment and customize if you want to change the default time zone of GitLab application.
@@ -140,14 +140,14 @@ production: &base
   artifacts:
     enabled: true
     object_store:
-      enabled: <%= env_or_default('object_store_enabled', false) %>
+      enabled: <%= config.object_store? %>
       remote_directory: artifacts
       connection:
         provider: AWS
         aws_access_key_id: minio
         aws_secret_access_key: gdk-minio
         region: gdk
-        endpoint: 'http://127.0.0.1:<%= env_or_default('object_store_port', 9000) %>'
+        endpoint: 'http://127.0.0.1:<%= config.object_store.port %>'
         path_style: true
 
   ## Merge request external diff storage
@@ -172,7 +172,7 @@ production: &base
   lfs:
     enabled: true
     object_store:
-      enabled: <%= env_or_default('object_store_enabled', false) %>
+      enabled: <%= config.object_store? %>
       remote_directory: lfs-objects
       direct_upload: true
       connection:
@@ -180,7 +180,7 @@ production: &base
         aws_access_key_id: minio
         aws_secret_access_key: gdk-minio
         region: gdk
-        endpoint: 'http://127.0.0.1:<%= env_or_default('object_store_port', 9000) %>'
+        endpoint: 'http://127.0.0.1:<%= config.object_store.port %>'
         path_style: true
 
   ## Uploads (attachments, avatars, etc...)
@@ -189,7 +189,7 @@ production: &base
     # storage_path: public/
     # base_dir: uploads/-/system
     object_store:
-      enabled: <%= env_or_default('object_store_enabled', false) %>
+      enabled: <%= config.object_store? %>
       remote_directory: uploads
       direct_upload: true
       connection:
@@ -197,14 +197,14 @@ production: &base
         aws_access_key_id: minio
         aws_secret_access_key: gdk-minio
         region: gdk
-        endpoint: 'http://127.0.0.1:<%= env_or_default('object_store_port', 9000) %>'
+        endpoint: 'http://127.0.0.1:<%= config.object_store.port %>'
         path_style: true
 
   ## Packages (maven repository, npm registry, etc...)
   packages:
     enabled: true
     object_store:
-      enabled: <%= env_or_default('object_store_enabled', false) %>
+      enabled: <%= config.object_store? %>
       remote_directory: packages
       direct_upload: true
       connection:
@@ -212,7 +212,7 @@ production: &base
         aws_access_key_id: minio
         aws_secret_access_key: gdk-minio
         region: gdk
-        endpoint: 'http://127.0.0.1:<%= env_or_default('object_store_port', 9000) %>'
+        endpoint: 'http://127.0.0.1:<%= config.object_store.port %>'
         path_style: true
 
   ## GitLab Pages
@@ -220,13 +220,13 @@ production: &base
     enabled: true
     access_control: false
     host: 127.0.0.1.xip.io
-    port: <%= env_or_default('gitlab_pages_port', 3010) %>
+    port: <%= config.gitlab_pages.port %>
     https: false # Set to true if you serve the pages with HTTPS
     artifacts_server: true
     # external_http: ["1.1.1.1:80", "[2001::1]:80"] # If defined, enables custom domain support in GitLab Pages
     # external_https: ["1.1.1.1:443", "[2001::1]:443"] # If defined, enables custom domain and certificate support in GitLab Pages
     admin:
-      address: unix:<%= GDK_ROOT %>/gitlab/tmp/sockets/private/pages-admin.socket # TCP connections are supported too (e.g. tcp://host:port)
+      address: unix:<%= config.gdk_root %>/gitlab/tmp/sockets/private/pages-admin.socket # TCP connections are supported too (e.g. tcp://host:port)
 
   ## Mattermost
   ## For enabling Add to Mattermost button
@@ -284,10 +284,10 @@ production: &base
       cron: "15 * * * *"
 
   registry:
-    enabled: <%= env_or_default('registry_enabled', false) %>
-    host: <%= env_or_default('registry_host', '127.0.0.1') %>
-    port: <%= env_or_default('registry_external_port', 5000) %>
-    api_url: http://127.0.0.1:<%= env_or_default('registry_port', 5000) %>
+    enabled: <%= config.registry? %>
+    host: <%= config.registry.host %>
+    port: <%= config.registry.external_port %>
+    api_url: http://127.0.0.1:<%= config.registry.port %>
     key: ../localhost.key
     path: ../registry/storage/
     issuer: gitlab-issuer
@@ -634,7 +634,7 @@ production: &base
   # Gitaly settings
   gitaly:
     # Path to the directory containing Gitaly client executables.
-    client_path: <%= File.join(GDK_ROOT, "gitaly", "bin") %>
+    client_path: <%= File.join(config.gdk_root, "gitaly", "bin") %>
     # Default Gitaly authentication token. Can be overridden per storage. Can
     # be left blank when Gitaly is running locally on a Unix socket, which
     # is the normal way to deploy Gitaly.
@@ -652,8 +652,8 @@ production: &base
     # real path not the symlink.
     storages: # You must have at least a `default` storage path.
       default:
-        path: <%= File.join(GDK_ROOT, "repositories") %>
-        gitaly_address: unix:<%= File.join(GDK_ROOT, "gitaly.socket") %>
+        path: <%= File.join(config.gdk_root, "repositories") %>
+        gitaly_address: unix:<%= config.gitaly.address %>
 
   ## Backup settings
   backup:
@@ -684,8 +684,8 @@ production: &base
 
   ## GitLab Shell settings
   gitlab_shell:
-    path: <%= File.join(GDK_ROOT, "gitlab-shell") %>
-    authorized_keys_file: <%= File.join(GDK_ROOT, ".ssh", "authorized_keys") %>
+    path: <%= File.join(config.gdk_root, "gitlab-shell") %>
+    authorized_keys_file: <%= File.join(config.gdk_root, ".ssh", "authorized_keys") %>
 
     # Git over HTTP
     upload_pack: true
@@ -703,14 +703,14 @@ production: &base
   # CAUTION!
   # Use the default values unless you really know what you are doing
   git:
-    bin_path: <%= GIT_BIN %>
+    bin_path: <%= config.git.bin %>
 
   ## Webpack settings
   webpack:
     dev_server:
       enabled: true
       host: localhost
-      port: <%= env_or_default('webpack_port', 3808).to_i %>
+      port: <%= config.webpack.port %>
 
   ## Monitoring
   # Built in monitoring settings
diff --git a/support/templates/praefect.config.toml.erb b/support/templates/praefect.config.toml.erb
new file mode 100644
index 00000000000..a458d6e4e65
--- /dev/null
+++ b/support/templates/praefect.config.toml.erb
@@ -0,0 +1,36 @@
+# Example Praefect configuration file
+
+# # TCP address to listen on
+# listen_addr = "127.0.0.1:2305"
+
+# # Praefect can listen on a socket when placed on the same machine as all clients
+socket_path = "<%= File.join(config.gdk_root, 'praefect.socket') %>"
+# # Praefect will only replicate whitelisted repositories
+# whitelist = ["@hashed/3f/db/3fdba35f04dc8c462986c992bcf875546257113072a909c162f7e470e581e278.git"]
+# # Optional: export metrics via Prometheus
+# prometheus_listen_addr = "127.0.01:10101"
+
+# # You can optionally configure Praefect to output JSON-formatted log messages to stdout
+# [logging]
+#   format = "json"
+# # Optional: Set log level to only log entries with that severity or above
+# # One of, in order: debug, info, warn, errror, fatal, panic
+# # Defaults to "info"
+#   level = "warn"
+
+# # One or more Gitaly servers need to be configured to be managed. The names
+# of each server are used to link multiple nodes, or `gitaly_server`s together
+# as shard. listen_addr should be unique for all nodes.
+# Requires the protocol to be defined, e.g. tcp://host.tld:1234
+
+[primary_server]
+  name = "default"
+  listen_addr = "unix:<%= config.gitaly.address %>"
+
+# [[secondary_server]]
+#  name = "default"
+#  listen_addr = "tcp://gitaly-backup1.example.com"
+
+# [[secondary_server]]
+#   name = "backup"
+#  listen_addr = "tcp://gitaly-backup2.example.com"