diff options
author | Sean McGivern <sean@mcgivern.me.uk> | 2017-06-15 17:57:14 +0300 |
---|---|---|
committer | Sean McGivern <sean@mcgivern.me.uk> | 2017-06-15 17:57:14 +0300 |
commit | 0b408219ed5992a9c85dc86a452bf1ef5faf0fd2 (patch) | |
tree | dc62519a18dd7b274c2d283d56f49a0bb31906ab | |
parent | 051b8dc4e3d055f7ac848a80d26bf1e6e0af2e24 (diff) | |
parent | 58821ebbb4fa1bbd280749ca2f64d7592a852c98 (diff) |
Merge branch 'deprecate-gitaly-path' into 'master'
Stop using deprecated `path` field on Gitaly messages
See merge request !12112
-rw-r--r-- | doc/update/9.3-to-9.4.md | 317 | ||||
-rw-r--r-- | lib/gitlab/gitaly_client/util.rb | 1 | ||||
-rw-r--r-- | lib/tasks/gitlab/gitaly.rake | 20 | ||||
-rw-r--r-- | spec/lib/gitlab/gitaly_client/notifications_spec.rb | 5 | ||||
-rw-r--r-- | spec/lib/gitlab/gitaly_client/ref_spec.rb | 15 | ||||
-rw-r--r-- | spec/lib/gitlab/workhorse_spec.rb | 2 | ||||
-rw-r--r-- | spec/support/matchers/gitaly_matchers.rb | 9 | ||||
-rw-r--r-- | spec/tasks/gitlab/backup_rake_spec.rb | 4 | ||||
-rw-r--r-- | spec/tasks/gitlab/gitaly_rake_spec.rb | 11 |
9 files changed, 361 insertions, 23 deletions
diff --git a/doc/update/9.3-to-9.4.md b/doc/update/9.3-to-9.4.md new file mode 100644 index 00000000000..a712ce5a8b1 --- /dev/null +++ b/doc/update/9.3-to-9.4.md @@ -0,0 +1,317 @@ +# From 9.3 to 9.4 + +Make sure you view this update guide from the tag (version) of GitLab you would +like to install. In most cases this should be the highest numbered production +tag (without rc in it). You can select the tag in the version dropdown at the +top left corner of GitLab (below the menu bar). + +If the highest number stable branch is unclear please check the +[GitLab Blog](https://about.gitlab.com/blog/archives.html) for installation +guide links by version. + +### 1. Stop server + +```bash +sudo service gitlab stop +``` + +### 2. Backup + +```bash +cd /home/git/gitlab + +sudo -u git -H bundle exec rake gitlab:backup:create RAILS_ENV=production +``` + +### 3. Update Ruby + +NOTE: GitLab 9.0 and higher only support Ruby 2.3.x and dropped support for Ruby 2.1.x. Be +sure to upgrade your interpreter if necessary. + +You can check which version you are running with `ruby -v`. + +Download and compile Ruby: + +```bash +mkdir /tmp/ruby && cd /tmp/ruby +curl --remote-name --progress https://cache.ruby-lang.org/pub/ruby/2.3/ruby-2.3.3.tar.gz +echo '1014ee699071aa2ddd501907d18cbe15399c997d ruby-2.3.3.tar.gz' | shasum -c - && tar xzf ruby-2.3.3.tar.gz +cd ruby-2.3.3 +./configure --disable-install-rdoc +make +sudo make install +``` + +Install Bundler: + +```bash +sudo gem install bundler --no-ri --no-rdoc +``` + +### 4. Update Node + +GitLab now runs [webpack](http://webpack.js.org) to compile frontend assets and +it has a minimum requirement of node v4.3.0. + +You can check which version you are running with `node -v`. If you are running +a version older than `v4.3.0` you will need to update to a newer version. You +can find instructions to install from community maintained packages or compile +from source at the nodejs.org website. + +<https://nodejs.org/en/download/> + + +Since 8.17, GitLab requires the use of yarn `>= v0.17.0` to manage +JavaScript dependencies. + +```bash +curl --location https://yarnpkg.com/install.sh | bash - +``` + +More information can be found on the [yarn website](https://yarnpkg.com/en/docs/install). + +### 5. Update Go + +NOTE: GitLab 9.4 and higher only supports Go 1.8.3 and dropped support for Go 1.5.x through 1.7.x. Be +sure to upgrade your installation if necessary + +You can check which version you are running with `go version`. + +Download and install Go: + +```bash +# Remove former Go installation folder +sudo rm -rf /usr/local/go + +curl --remote-name --progress https://storage.googleapis.com/golang/go1.8.3.linux-amd64.tar.gz +echo '1862f4c3d3907e59b04a757cfda0ea7aa9ef39274af99a784f5be843c80c6772 go1.8.3.linux-amd64.tar.gz' | shasum -a256 -c - && \ + sudo tar -C /usr/local -xzf go1.8.3.linux-amd64.tar.gz +sudo ln -sf /usr/local/go/bin/{go,godoc,gofmt} /usr/local/bin/ +rm go1.8.3.linux-amd64.tar.gz +``` + +### 6. Get latest code + +```bash +cd /home/git/gitlab + +sudo -u git -H git fetch --all +sudo -u git -H git checkout -- db/schema.rb # local changes will be restored automatically +``` + +For GitLab Community Edition: + +```bash +cd /home/git/gitlab + +sudo -u git -H git checkout 9-4-stable +``` + +OR + +For GitLab Enterprise Edition: + +```bash +cd /home/git/gitlab + +sudo -u git -H git checkout 9-4-stable-ee +``` + +### 5. Update gitlab-shell + +```bash +cd /home/git/gitlab-shell + +sudo -u git -H git fetch --all --tags +sudo -u git -H git checkout v$(</home/git/gitlab/GITLAB_SHELL_VERSION) +sudo -u git -H bin/compile +``` + +### 6. Update gitlab-workhorse + +Install and compile gitlab-workhorse. This requires +[Go 1.8](https://golang.org/dl) which should already be on your system from +GitLab 8.1. GitLab-Workhorse uses [GNU Make](https://www.gnu.org/software/make/). +If you are not using Linux you may have to run `gmake` instead of +`make` below. + +```bash +cd /home/git/gitlab-workhorse + +sudo -u git -H git fetch --all --tags +sudo -u git -H git checkout v$(</home/git/gitlab/GITLAB_WORKHORSE_VERSION) +sudo -u git -H make +``` + +### 7. Update Gitaly + +If you have not yet set up Gitaly then follow [Gitaly section of the installation +guide](../install/installation.md#install-gitaly). + +#### Check Gitaly configuration + +Due to a bug in the `rake gitlab:gitaly:install` script your Gitaly +configuration file may contain syntax errors. The block name +`[[storages]]`, which may occur more than once in your `config.toml` +file, should be `[[storage]]` instead. + +```shell +cd /home/git/gitaly +sudo -u git -H editor config.toml +``` + +#### Compile Gitaly + +```shell +cd /home/git/gitaly +sudo -u git -H git fetch --all --tags +sudo -u git -H git checkout v$(</home/git/gitlab/GITALY_SERVER_VERSION) +sudo -u git -H make +``` + +### 10. Update configuration files + +#### New configuration options for `gitlab.yml` + +There might be configuration options available for [`gitlab.yml`][yaml]. View them with the command below and apply them manually to your current `gitlab.yml`: + +```sh +cd /home/git/gitlab + +git diff origin/9-3-stable:config/gitlab.yml.example origin/9-4-stable:config/gitlab.yml.example +``` + +#### Nginx configuration + +Ensure you're still up-to-date with the latest NGINX configuration changes: + +```sh +cd /home/git/gitlab + +# For HTTPS configurations +git diff origin/9-3-stable:lib/support/nginx/gitlab-ssl origin/9-4-stable:lib/support/nginx/gitlab-ssl + +# For HTTP configurations +git diff origin/9-3-stable:lib/support/nginx/gitlab origin/9-4-stable:lib/support/nginx/gitlab +``` + +If you are using Strict-Transport-Security in your installation to continue using it you must enable it in your Nginx +configuration as GitLab application no longer handles setting it. + +If you are using Apache instead of NGINX please see the updated [Apache templates]. +Also note that because Apache does not support upstreams behind Unix sockets you +will need to let gitlab-workhorse listen on a TCP port. You can do this +via [/etc/default/gitlab]. + +[Apache templates]: https://gitlab.com/gitlab-org/gitlab-recipes/tree/master/web-server/apache +[/etc/default/gitlab]: https://gitlab.com/gitlab-org/gitlab-ce/blob/9-4-stable/lib/support/init.d/gitlab.default.example#L38 + +#### SMTP configuration + +If you're installing from source and use SMTP to deliver mail, you will need to add the following line +to config/initializers/smtp_settings.rb: + +```ruby +ActionMailer::Base.delivery_method = :smtp +``` + +See [smtp_settings.rb.sample] as an example. + +[smtp_settings.rb.sample]: https://gitlab.com/gitlab-org/gitlab-ce/blob/9-3-stable/config/initializers/smtp_settings.rb.sample#L13 + +#### Init script + +There might be new configuration options available for [`gitlab.default.example`][gl-example]. View them with the command below and apply them manually to your current `/etc/default/gitlab`: + +```sh +cd /home/git/gitlab + +git diff origin/9-3-stable:lib/support/init.d/gitlab.default.example origin/9-4-stable:lib/support/init.d/gitlab.default.example +``` + +Ensure you're still up-to-date with the latest init script changes: + +```bash +cd /home/git/gitlab + +sudo cp lib/support/init.d/gitlab /etc/init.d/gitlab +``` + +For Ubuntu 16.04.1 LTS: + +```bash +sudo systemctl daemon-reload +``` + +### 11. Install libs, migrations, etc. + +```bash +cd /home/git/gitlab + +# MySQL installations (note: the line below states '--without postgres') +sudo -u git -H bundle install --without postgres development test --deployment + +# PostgreSQL installations (note: the line below states '--without mysql') +sudo -u git -H bundle install --without mysql development test --deployment + +# Optional: clean up old gems +sudo -u git -H bundle clean + +# Run database migrations +sudo -u git -H bundle exec rake db:migrate RAILS_ENV=production + +# Update node dependencies and recompile assets +sudo -u git -H bundle exec rake yarn:install gitlab:assets:clean gitlab:assets:compile RAILS_ENV=production NODE_ENV=production + +# Clean up cache +sudo -u git -H bundle exec rake cache:clear RAILS_ENV=production +``` + +**MySQL installations**: Run through the `MySQL strings limits` and `Tables and data conversion to utf8mb4` [tasks](../install/database_mysql.md). + +### 12. Start application + +```bash +sudo service gitlab start +sudo service nginx restart +``` + +### 13. Check application status + +Check if GitLab and its environment are configured correctly: + +```bash +cd /home/git/gitlab + +sudo -u git -H bundle exec rake gitlab:env:info RAILS_ENV=production +``` + +To make sure you didn't miss anything run a more thorough check: + +```bash +cd /home/git/gitlab + +sudo -u git -H bundle exec rake gitlab:check RAILS_ENV=production +``` + +If all items are green, then congratulations, the upgrade is complete! + +## Things went south? Revert to previous version (9.3) + +### 1. Revert the code to the previous version + +Follow the [upgrade guide from 9.2 to 9.3](9.2-to-9.3.md), except for the +database migration (the backup is already migrated to the previous version). + +### 2. Restore from the backup + +```bash +cd /home/git/gitlab + +sudo -u git -H bundle exec rake gitlab:backup:restore RAILS_ENV=production +``` + +If you have more than one backup `*.tar` file(s) please add `BACKUP=timestamp_of_backup` to the command above. + +[yaml]: https://gitlab.com/gitlab-org/gitlab-ce/blob/9-4-stable/config/gitlab.yml.example +[gl-example]: https://gitlab.com/gitlab-org/gitlab-ce/blob/9-4-stable/lib/support/init.d/gitlab.default.example diff --git a/lib/gitlab/gitaly_client/util.rb b/lib/gitlab/gitaly_client/util.rb index 86d055d3533..f5a4c5493ef 100644 --- a/lib/gitlab/gitaly_client/util.rb +++ b/lib/gitlab/gitaly_client/util.rb @@ -4,7 +4,6 @@ module Gitlab class << self def repository(repository_storage, relative_path) Gitaly::Repository.new( - path: File.join(Gitlab.config.repositories.storages[repository_storage]['path'], relative_path), storage_name: repository_storage, relative_path: relative_path ) diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake index 3c5bc0146a1..e88111c3725 100644 --- a/lib/tasks/gitlab/gitaly.rake +++ b/lib/tasks/gitlab/gitaly.rake @@ -30,11 +30,7 @@ namespace :gitlab do puts "# Gitaly storage configuration generated from #{Gitlab.config.source} on #{Time.current.to_s(:long)}" puts "# This is in TOML format suitable for use in Gitaly's config.toml file." - config = Gitlab.config.repositories.storages.map do |key, val| - { name: key, path: val['path'] } - end - - puts TOML.dump(storage: config) + puts gitaly_configuration_toml end private @@ -42,10 +38,10 @@ namespace :gitlab do # We cannot create config.toml files for all possible Gitaly configuations. # For instance, if Gitaly is running on another machine then it makes no # sense to write a config.toml file on the current machine. This method will - # only write a config.toml file in the most common and simplest case: the - # case where we have exactly one Gitaly process and we are sure it is - # running locally because it uses a Unix socket. - def create_gitaly_configuration + # only generate a configuration for the most common and simplest case: when + # we have exactly one Gitaly process and we are sure it is running locally + # because it uses a Unix socket. + def gitaly_configuration_toml storages = [] address = nil @@ -63,8 +59,12 @@ namespace :gitlab do storages << { name: key, path: val['path'] } end + TOML.dump(socket_path: address.sub(%r{\Aunix:}, ''), storage: storages) + end + + def create_gitaly_configuration File.open("config.toml", "w") do |f| - f.puts TOML.dump(socket_path: address.sub(%r{\Aunix:}, ''), storages: storages) + f.puts gitaly_configuration_toml end rescue ArgumentError => e puts "Skipping config.toml generation:" diff --git a/spec/lib/gitlab/gitaly_client/notifications_spec.rb b/spec/lib/gitlab/gitaly_client/notifications_spec.rb index b87dacb175b..e5c9e06a15e 100644 --- a/spec/lib/gitlab/gitaly_client/notifications_spec.rb +++ b/spec/lib/gitlab/gitaly_client/notifications_spec.rb @@ -3,12 +3,13 @@ require 'spec_helper' describe Gitlab::GitalyClient::Notifications do describe '#post_receive' do let(:project) { create(:empty_project) } - let(:repo_path) { project.repository.path_to_repo } + let(:storage_name) { project.repository_storage } + let(:relative_path) { project.path_with_namespace + '.git' } subject { described_class.new(project.repository) } it 'sends a post_receive message' do expect_any_instance_of(Gitaly::Notifications::Stub). - to receive(:post_receive).with(gitaly_request_with_repo_path(repo_path)) + to receive(:post_receive).with(gitaly_request_with_path(storage_name, relative_path)) subject.post_receive end diff --git a/spec/lib/gitlab/gitaly_client/ref_spec.rb b/spec/lib/gitlab/gitaly_client/ref_spec.rb index d8cd2dcbd2a..2ea44ef74b0 100644 --- a/spec/lib/gitlab/gitaly_client/ref_spec.rb +++ b/spec/lib/gitlab/gitaly_client/ref_spec.rb @@ -2,7 +2,8 @@ require 'spec_helper' describe Gitlab::GitalyClient::Ref do let(:project) { create(:empty_project) } - let(:repo_path) { project.repository.path_to_repo } + let(:storage_name) { project.repository_storage } + let(:relative_path) { project.path_with_namespace + '.git' } let(:client) { described_class.new(project.repository) } before do @@ -19,7 +20,8 @@ describe Gitlab::GitalyClient::Ref do describe '#branch_names' do it 'sends a find_all_branch_names message' do expect_any_instance_of(Gitaly::Ref::Stub). - to receive(:find_all_branch_names).with(gitaly_request_with_repo_path(repo_path)). + to receive(:find_all_branch_names). + with(gitaly_request_with_path(storage_name, relative_path)). and_return([]) client.branch_names @@ -29,7 +31,8 @@ describe Gitlab::GitalyClient::Ref do describe '#tag_names' do it 'sends a find_all_tag_names message' do expect_any_instance_of(Gitaly::Ref::Stub). - to receive(:find_all_tag_names).with(gitaly_request_with_repo_path(repo_path)). + to receive(:find_all_tag_names). + with(gitaly_request_with_path(storage_name, relative_path)). and_return([]) client.tag_names @@ -39,7 +42,8 @@ describe Gitlab::GitalyClient::Ref do describe '#default_branch_name' do it 'sends a find_default_branch_name message' do expect_any_instance_of(Gitaly::Ref::Stub). - to receive(:find_default_branch_name).with(gitaly_request_with_repo_path(repo_path)). + to receive(:find_default_branch_name). + with(gitaly_request_with_path(storage_name, relative_path)). and_return(double(name: 'foo')) client.default_branch_name @@ -49,7 +53,8 @@ describe Gitlab::GitalyClient::Ref do describe '#local_branches' do it 'sends a find_local_branches message' do expect_any_instance_of(Gitaly::Ref::Stub). - to receive(:find_local_branches).with(gitaly_request_with_repo_path(repo_path)). + to receive(:find_local_branches). + with(gitaly_request_with_path(storage_name, relative_path)). and_return([]) client.local_branches diff --git a/spec/lib/gitlab/workhorse_spec.rb b/spec/lib/gitlab/workhorse_spec.rb index b1999409170..ad19998dff4 100644 --- a/spec/lib/gitlab/workhorse_spec.rb +++ b/spec/lib/gitlab/workhorse_spec.rb @@ -212,7 +212,7 @@ describe Gitlab::Workhorse, lib: true do it 'includes a Repository param' do repo_param = { Repository: { - path: repo_path, + path: '', # deprecated field; grpc automatically creates it anyway storage_name: 'default', relative_path: project.full_path + '.git' } } diff --git a/spec/support/matchers/gitaly_matchers.rb b/spec/support/matchers/gitaly_matchers.rb index ed14bcec9f2..ebfabcd8f24 100644 --- a/spec/support/matchers/gitaly_matchers.rb +++ b/spec/support/matchers/gitaly_matchers.rb @@ -1,5 +1,10 @@ -RSpec::Matchers.define :gitaly_request_with_repo_path do |path| - match { |actual| actual.repository.path == path } +RSpec::Matchers.define :gitaly_request_with_path do |storage_name, relative_path| + match do |actual| + repository = actual.repository + + repository.storage_name == storage_name && + repository.relative_path == relative_path + end end RSpec::Matchers.define :gitaly_request_with_params do |params| diff --git a/spec/tasks/gitlab/backup_rake_spec.rb b/spec/tasks/gitlab/backup_rake_spec.rb index 0ff1a988a9e..1e5f55a738a 100644 --- a/spec/tasks/gitlab/backup_rake_spec.rb +++ b/spec/tasks/gitlab/backup_rake_spec.rb @@ -241,6 +241,10 @@ describe 'gitlab:app namespace rake task' do project_a project_b + # Avoid asking gitaly about the root ref (which will fail beacuse of the + # mocked storages) + allow_any_instance_of(Repository).to receive(:empty_repo?).and_return(false) + # We only need a backup of the repositories for this test ENV["SKIP"] = "db,uploads,builds,artifacts,lfs,registry" create_backup diff --git a/spec/tasks/gitlab/gitaly_rake_spec.rb b/spec/tasks/gitlab/gitaly_rake_spec.rb index 4a636decafd..cfa6c9ca8ce 100644 --- a/spec/tasks/gitlab/gitaly_rake_spec.rb +++ b/spec/tasks/gitlab/gitaly_rake_spec.rb @@ -79,8 +79,14 @@ describe 'gitlab:gitaly namespace rake task' do describe 'storage_config' do it 'prints storage configuration in a TOML format' do config = { - 'default' => { 'path' => '/path/to/default' }, - 'nfs_01' => { 'path' => '/path/to/nfs_01' } + 'default' => { + 'path' => '/path/to/default', + 'gitaly_address' => 'unix:/path/to/my.socket' + }, + 'nfs_01' => { + 'path' => '/path/to/nfs_01', + 'gitaly_address' => 'unix:/path/to/my.socket' + } } allow(Gitlab.config.repositories).to receive(:storages).and_return(config) @@ -89,6 +95,7 @@ describe 'gitlab:gitaly namespace rake task' do expected_output = <<~TOML # Gitaly storage configuration generated from #{Gitlab.config.source} on #{Time.current.to_s(:long)} # This is in TOML format suitable for use in Gitaly's config.toml file. + socket_path = "/path/to/my.socket" [[storage]] name = "default" path = "/path/to/default" |