Merge branch 'deduplicate-ssl-certs' into 'master'

de-duplicate certificates in gitaly remote repo ruby client

See merge request gitlab-org/gitaly!1005

6 files changed, 117 insertions, 35 deletions

diff --git a/changelogs/unreleased/deduplicate-ssl-certs.yml b/changelogs/unreleased/deduplicate-ssl-certs.yml
new file mode 100644
index 000000000..d626f81dc
--- /dev/null
+++ b/changelogs/unreleased/deduplicate-ssl-certs.yml
@@ -0,0 +1,5 @@
+---
+title: Deduplicate CA in gitaly tls
+merge_request: 1005
+author:
+type: fixed
diff --git a/ruby/lib/gitlab/git/gitaly_remote_repository.rb b/ruby/lib/gitlab/git/gitaly_remote_repository.rb
index 849172dd0..6fa6657d0 100644
--- a/ruby/lib/gitlab/git/gitaly_remote_repository.rb
+++ b/ruby/lib/gitlab/git/gitaly_remote_repository.rb
@@ -2,6 +2,7 @@ module Gitlab
   module Git
     class GitalyRemoteRepository < RemoteRepository
       CLIENT_NAME = 'gitaly-ruby'.freeze
+      PEM_REXP = /[-]+BEGIN CERTIFICATE[-]+.+?[-]+END CERTIFICATE[-]+/m
 
       attr_reader :gitaly_client
 
@@ -39,10 +40,18 @@ module Gitlab
         files = []
         files += Dir["#{ENV['SSL_CERT_DIR']}/*"] if ENV['SSL_CERT_DIR']
         files += [ENV['SSL_CERT_FILE']] if ENV['SSL_CERT_FILE']
-
-        @certs = files.map do |cert|
-          File.read(cert)
-        end.join("\n")
+        files.sort!
+
+        @certs = files.flat_map do |cert_file|
+          File.read(cert_file).scan(PEM_REXP).map do |cert|
+            begin
+              OpenSSL::X509::Certificate.new(cert).to_pem
+            rescue OpenSSL::OpenSSLError => e
+              Rails.logger.error "Could not load certificate #{cert_file} #{e}"
+              nil
+            end
+          end.compact
+        end.uniq.join("\n")
       end
 
       def credentials
diff --git a/ruby/spec/lib/gitlab/git/remote_repository_client_spec.rb b/ruby/spec/lib/gitlab/git/remote_repository_client_spec.rb
index 62891a2bd..e61acaa33 100644
--- a/ruby/spec/lib/gitlab/git/remote_repository_client_spec.rb
+++ b/ruby/spec/lib/gitlab/git/remote_repository_client_spec.rb
@@ -25,12 +25,22 @@ describe Gitlab::Git::GitalyRemoteRepository do
     end
 
     context 'when SSL_CERT_DIR is set' do
-      it 'Should return concatenation of gitalycert and gitalycert2' do
+      it 'Should return concatenation of gitalycert and gitalycert2 and gitalycert3 ommiting gitalycertdup.pem' do
         cert_pool_dir = File.join(File.dirname(__FILE__), "testdata/certs")
         allow(ENV).to receive(:[]).with('SSL_CERT_DIR').and_return(cert_pool_dir)
         allow(ENV).to receive(:[]).with('SSL_CERT_FILE').and_return(nil)
         certs = client.certs
-        expected_certs = [File.read(File.join(cert_pool_dir, "gitalycert2.pem")), File.read(File.join(cert_pool_dir, "gitalycert.pem"))].join "\n"
+
+        # gitalycertdup.pem must exist and must be a duplicate of gitalycert.pem
+        expect(File.exist?(File.join(cert_pool_dir, "gitalycertdup.pem"))).to be true
+        expect(File.read(File.join(cert_pool_dir, "gitalycertdup.pem")))
+          .to eq File.read(File.join(cert_pool_dir, "gitalycert.pem"))
+
+        # No gitalycertdup.pem because duplicates should be removed
+        expected_certs = [File.read(File.join(cert_pool_dir, "gitalycert.pem")),
+                          File.read(File.join(cert_pool_dir, "gitalycert2.pem")),
+                          File.read(File.join(cert_pool_dir, "gitalycert3.pem"))].join "\n"
+
         expect(certs).to eq expected_certs
       end
     end
@@ -41,7 +51,7 @@ describe Gitlab::Git::GitalyRemoteRepository do
         cert1_file = File.join(File.dirname(__FILE__), "testdata/gitalycert.pem")
         allow(ENV).to receive(:[]).with('SSL_CERT_DIR').and_return(cert_pool_dir)
         allow(ENV).to receive(:[]).with('SSL_CERT_FILE').and_return(cert1_file)
-        expected_certs_paths = [File.join(cert_pool_dir, "gitalycert2.pem"), File.join(cert_pool_dir, "gitalycert.pem"), cert1_file]
+        expected_certs_paths = [cert1_file, File.join(cert_pool_dir, "gitalycert2.pem"), File.join(cert_pool_dir, "gitalycert3.pem")]
 
         expected_certs = expected_certs_paths.map do |cert|
           File.read cert
diff --git a/ruby/spec/lib/gitlab/git/testdata/certs/gitalycert2.pem b/ruby/spec/lib/gitlab/git/testdata/certs/gitalycert2.pem
index 4708f8ec3..e794bf356 100755
--- a/ruby/spec/lib/gitlab/git/testdata/certs/gitalycert2.pem
+++ b/ruby/spec/lib/gitlab/git/testdata/certs/gitalycert2.pem
@@ -1,30 +1,31 @@
 -----BEGIN CERTIFICATE-----
-MIIFODCCAyACCQDpPfNtveVc8TANBgkqhkiG9w0BAQsFADBdMQswCQYDVQQGEwJV
-UzELMAkGA1UECAwCVVMxCzAJBgNVBAcMAlVTMQ8wDQYDVQQKDAZHaXRMYWIxDzAN
-BgNVBAsMBmdpdGFseTESMBAGA1UEAwwJbG9jYWxob3N0MCAXDTE4MTEwMjA5MDIx
-MloYDzIxMTgxMDA5MDkwMjEyWjBdMQswCQYDVQQGEwJVUzELMAkGA1UECAwCVVMx
-CzAJBgNVBAcMAlVTMQ8wDQYDVQQKDAZHaXRMYWIxDzANBgNVBAsMBmdpdGFseTES
-MBAGA1UEAwwJbG9jYWxob3N0MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKC
-AgEApJXJOWpUkV32v8gRXLWn6TEsQmy2WeilQXg96V6VOQjGZAGMEJLEjH9WHBNe
-Zi4V+W+j1FB8vWTNRGTcOcpSEmDFuewoBJVA8dFtNF4jj7QQymmnKeDuOW4fWLeU
-Ykkjkxyjlpkm2+DUg5CavT4bMZILqbsAavxJ8SKCdJpMtW3sxklnGuTHcAckHldab
-9ZxH/qYqLxc5Ek2BK4OibBxA84h1RUsqe2EdzZUOoet3xpwG3Vr8bGPqR7Psghs6
-TDdWU8hYYHlReCWezgZHiYDoRqY9HCZrHSpUZ1lbRo++2j4bvdFHOAUm4BEQ6fFc
-sgtW+xkNK8bxj9XTcpuDrEVscv3fyBlCMSvD+HpNbr2k1oZSOFhxISIwBLKWQBjq
-5muvMRbmrG5RgWqMWjXb+g0UmlyMa2YWAWsBgSuUSjJePgbUZWHuxp/dM8CQ4lHJ
-ADvfSI9ysJQM/trqjRu5BRhxiKWR72QSi1qpDPT0nKWlzQ58zs3RSuOJbWm8oOqr
-XL9G/XmvgzK1qwToI/WmXBeaqmfpkagYZm+TJW0GVnDqTC+EoXdFKW7aWIjlcb4p
-tYoiRA/2jjq5OqeV6iKnxz7mEJQR1xDebm6+AWgFy4zyB/QvzanaUTvNiLhyBy6Q
-YwXJHkNh+KrVszBlXxkARrGesXgqOznmDeErkOKDjxzQv+cCAwEAATANBgkqhkiG
-9w0BAQsFAAOCAgEAk83b9wY9iwRrx5Yep3DA3xZkVu3GJcKf0tTL8apP1MzVBSUK
-5tkvW2Z4D41jpZWgJDRF8/nT2lvVwvd5xQ8/oTUerFeG/ZZ+AiBagkBKl8piPHqD
-cefAO8N2SKoYHV4xBeoVU6InUuJ7xu7BLF6tY3xKvx0XsjGC7B621xmq+E56dPZg
-sQwekkxODbUw4NekqYFY21BT4xiWVrTRLIGY9AfV9Ry4gqQTxda7yst4ykWh1a9e
-O+426uz3jshzpQTjZwk8kCZquJKa8Qzqfdlevns0FQDP5jck4BH/YkMNsa/g9XCd
-ZHSB7gqAfNoNTB1rqNKIfPUF4mTu/RWMVwxb8f6h0TfywHZ4q/4R3Zfu3jUyeVVY
-ziJu2CJpcoR9SESKFbN4WFzk91nIhf2pCGo/qNO5f+n5ZPnS2jrrWL5h64e1rz2h
-rVKIYLfeM2M8lVzSL1V0aJ+POcruTRsmlrFT5f7na/5YFt5N+5Z5fzixCLr1MK2w
-4gFw+KhN7CAhKGzHq3NBdWpRFFMR53hyeYsb1vvwFu07JTRh+NbaePk/sk07WtCo
-u2w6pD7xlayTAWcR9WRBv7c3lDejN80U8DONb8fLwtI5oIrkSuwOqvmlDOeFpKiT
-MwTB6oC81Ar39P0R53247w7u9plhPUrmDn/A5KphW633UvgbkH6VmB4Isiw=
+MIIFZzCCBE+gAwIBAgISA8gRwdrBpJzyIeqjbHX1cpzVMA0GCSqGSIb3DQEBCwUA
+MEoxCzAJBgNVBAYTAlVTMRYwFAYDVQQKEw1MZXQncyBFbmNyeXB0MSMwIQYDVQQD
+ExpMZXQncyBFbmNyeXB0IEF1dGhvcml0eSBYMzAeFw0xODEyMTYxNTUwMTNaFw0x
+OTAzMTYxNTUwMTNaMBkxFzAVBgNVBAMTDm1hbnVzYW5rYXJjLmluMIIBIjANBgkq
+hkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA0tO2bYR23lwPdL/jNdIXU5lx+LhHrsTT
+6tB093w9b8gMHBXKLL8c9zHpCtt1JXHbDfa1KILpjd17zpuDNe6r+I7pDVI7T1w1
+xS8AwBlv6uUYvTkXpkQL+OEBPHsPef6GHTGP6qFn+zeWA9y77cfHuoTYTuLThWMA
+mMbitghgc5y2+HMY/5NXLKOcOHuLLkekiVMn2kkcP+JvaMc/7R+jmVHhHcHdIHGW
+GnI1R7zO4r1lbyqk9A7I56UbwxSkdIbzC2YTR+7LMzTwKSwKTh+a7Tli7q4TYTLi
+AUhaYvZ0BGXq4HVlwA8u1acy5hQrnEKkZ+/8jhYtBkQvqyZRS3OqxwIDAQABo4IC
+djCCAnIwDgYDVR0PAQH/BAQDAgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEF
+BQcDAjAMBgNVHRMBAf8EAjAAMB0GA1UdDgQWBBT1uOau0vgsenK8ldqAK6Yr0iLe
+pzAfBgNVHSMEGDAWgBSoSmpjBH3duubRObemRWXv86jsoTBvBggrBgEFBQcBAQRj
+MGEwLgYIKwYBBQUHMAGGImh0dHA6Ly9vY3NwLmludC14My5sZXRzZW5jcnlwdC5v
+cmcwLwYIKwYBBQUHMAKGI2h0dHA6Ly9jZXJ0LmludC14My5sZXRzZW5jcnlwdC5v
+cmcvMCsGA1UdEQQkMCKCECoubWFudXNhbmthcmMuaW6CDm1hbnVzYW5rYXJjLmlu
+MEwGA1UdIARFMEMwCAYGZ4EMAQIBMDcGCysGAQQBgt8TAQEBMCgwJgYIKwYBBQUH
+AgEWGmh0dHA6Ly9jcHMubGV0c2VuY3J5cHQub3JnMIIBBQYKKwYBBAHWeQIEAgSB
+9gSB8wDxAHcAdH7agzGtMxCRIZzOJU9CcMK//V5CIAjGNzV55hB7zFYAAAFnt+ze
+6wAABAMASDBGAiEAipzkToRSHF4c1V6+gcHtXAjujN+iEPWcRt7iP3C8IFsCIQC3
+giIZTBcEPZFmFr+3OfPSmY45ILHcwbzQUqXbESk1xgB2ACk8UZZUyDlluqpQ/FgH
+1Ldvv1h6KXLcpMMM9OVFR/R4AAABZ7fs31YAAAQDAEcwRQIhANYJjEcMRPn9vmHd
+ZxrKgmKGVz8x4kU7Rr0SQ86gNVLDAiA2yqiEI5hV77AtWitrxO/i5kswa3RE5WA6
+2AZX+eEhgTANBgkqhkiG9w0BAQsFAAOCAQEAYhp7fqPikgmyB0WyXzfOs45792IJ
+fu7ULs10mO/eG3MCyvUl9vciYpGZyeyGHzim0wVqi7kdIhxzO0yw9NQA8c52zYlL
+3qoQHMWm0wy6FKd8bYKi+MnlLAONE3Q5o+RMpztb2w2N/um9Apl8KTV00ecGtAYZ
++8tfrcp5VoAXJ1poTzhmhNWvAD89PM4snDhlm7UDRG6qjUedkct9NnW1LTUzyuY7
+IKOM1Zvnyh41jBGWPE6AATAmKEK8t5IO1WturXZe2FDDYSZ9ayQbFP2Yy4v42jd+
+vIGa7XKc9QAO3mdyRPdiva1J5reyXpkm7Z8suH+pGuIhZUeltOqUMN4KnQ==
 -----END CERTIFICATE-----
diff --git a/ruby/spec/lib/gitlab/git/testdata/certs/gitalycert3.pem b/ruby/spec/lib/gitlab/git/testdata/certs/gitalycert3.pem
new file mode 100755
index 000000000..0002462ce
--- /dev/null
+++ b/ruby/spec/lib/gitlab/git/testdata/certs/gitalycert3.pem
@@ -0,0 +1,27 @@
+-----BEGIN CERTIFICATE-----
+MIIEkjCCA3qgAwIBAgIQCgFBQgAAAVOFc2oLheynCDANBgkqhkiG9w0BAQsFADA/
+MSQwIgYDVQQKExtEaWdpdGFsIFNpZ25hdHVyZSBUcnVzdCBDby4xFzAVBgNVBAMT
+DkRTVCBSb290IENBIFgzMB4XDTE2MDMxNzE2NDA0NloXDTIxMDMxNzE2NDA0Nlow
+SjELMAkGA1UEBhMCVVMxFjAUBgNVBAoTDUxldCdzIEVuY3J5cHQxIzAhBgNVBAMT
+GkxldCdzIEVuY3J5cHQgQXV0aG9yaXR5IFgzMIIBIjANBgkqhkiG9w0BAQEFAAOC
+AQ8AMIIBCgKCAQEAnNMM8FrlLke3cl03g7NoYzDq1zUmGSXhvb418XCSL7e4S0EF
+q6meNQhY7LEqxGiHC6PjdeTm86dicbp5gWAf15Gan/PQeGdxyGkOlZHP/uaZ6WA8
+SMx+yk13EiSdRxta67nsHjcAHJyse6cF6s5K671B5TaYucv9bTyWaN8jKkKQDIZ0
+Z8h/pZq4UmEUEz9l6YKHy9v6Dlb2honzhT+Xhq+w3Brvaw2VFn3EK6BlspkENnWA
+a6xK8xuQSXgvopZPKiAlKQTGdMDQMc2PMTiVFrqoM7hD8bEfwzB/onkxEz0tNvjj
+/PIzark5McWvxI0NHWQWM6r6hCm21AvA2H3DkwIDAQABo4IBfTCCAXkwEgYDVR0T
+AQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwfwYIKwYBBQUHAQEEczBxMDIG
+CCsGAQUFBzABhiZodHRwOi8vaXNyZy50cnVzdGlkLm9jc3AuaWRlbnRydXN0LmNv
+bTA7BggrBgEFBQcwAoYvaHR0cDovL2FwcHMuaWRlbnRydXN0LmNvbS9yb290cy9k
+c3Ryb290Y2F4My5wN2MwHwYDVR0jBBgwFoAUxKexpHsscfrb4UuQdf/EFWCFiRAw
+VAYDVR0gBE0wSzAIBgZngQwBAgEwPwYLKwYBBAGC3xMBAQEwMDAuBggrBgEFBQcC
+ARYiaHR0cDovL2Nwcy5yb290LXgxLmxldHNlbmNyeXB0Lm9yZzA8BgNVHR8ENTAz
+MDGgL6AthitodHRwOi8vY3JsLmlkZW50cnVzdC5jb20vRFNUUk9PVENBWDNDUkwu
+Y3JsMB0GA1UdDgQWBBSoSmpjBH3duubRObemRWXv86jsoTANBgkqhkiG9w0BAQsF
+AAOCAQEA3TPXEfNjWDjdGBX7CVW+dla5cEilaUcne8IkCJLxWh9KEik3JHRRHGJo
+uM2VcGfl96S8TihRzZvoroed6ti6WqEBmtzw3Wodatg+VyOeph4EYpr/1wXKtx8/
+wApIvJSwtmVi4MFU5aMqrSDE6ea73Mj2tcMyo5jMd6jmeWUHK8so/joWUoHOUgwu
+X4Po1QYz+3dszkDqMp4fklxBwXRsW10KXzPMTZ+sOPAveyxindmjkW8lGy+QsRlG
+PfZ+G6Z6h7mjem0Y+iWlkYcV4PIWL1iwBi8saCbGS5jN2p8M+X+Q7UNKEkROb3N6
+KOqkqm57TH2H3eDJAkSnh6/DNFu0Qg==
+-----END CERTIFICATE-----
diff --git a/ruby/spec/lib/gitlab/git/testdata/certs/gitalycertdup.pem b/ruby/spec/lib/gitlab/git/testdata/certs/gitalycertdup.pem
new file mode 100755
index 000000000..8b1514548
--- /dev/null
+++ b/ruby/spec/lib/gitlab/git/testdata/certs/gitalycertdup.pem
@@ -0,0 +1,30 @@
+-----BEGIN CERTIFICATE-----
+MIIFODCCAyACCQDpPfNtveVc8TANBgkqhkiG9w0BAQsFADBdMQswCQYDVQQGEwJV
+UzELMAkGA1UECAwCVVMxCzAJBgNVBAcMAlVTMQ8wDQYDVQQKDAZHaXRMYWIxDzAN
+BgNVBAsMBmdpdGFseTESMBAGA1UEAwwJbG9jYWxob3N0MCAXDTE4MTEwMjA5MDIx
+MloYDzIxMTgxMDA5MDkwMjEyWjBdMQswCQYDVQQGEwJVUzELMAkGA1UECAwCVVMx
+CzAJBgNVBAcMAlVTMQ8wDQYDVQQKDAZHaXRMYWIxDzANBgNVBAsMBmdpdGFseTES
+MBAGA1UEAwwJbG9jYWxob3N0MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKC
+AgEApJXJOWpUkV32v8gRXLWn6TEsQmy2WeilQXg96V6VOQjGZAGMEJLEjH9WHBNe
+Zi4V+W+j1FB8vWTNRGTcOcpSEmDFuewoBJVA8dFtNF4jj7QQymmnKeDuOW4fWLeU
+YcyGxyjlpkm2+DUg5CavT4bMZILqbsAavxJ8SKCdJpMtW3sxklnGuTHcAckHldab
+9ZxH/qYqLxc5Ek2BK4OibBxA84h1RUsqe2EdzZUOoet3xpwG3Vr8bGPqR7Psghs6
+TDdWU8hYYHlReCWezgZHiYDoRqY9HCZrHSpUZ1lbRo++2j4bvdFHOAUm4BEQ6fFc
+sgtW+xkNK8bxj9XTcpuDrEVscv3fyBlCMSvD+HpNbr2k1oZSOFhxISIwBLKWQBjq
+5muvMRbmrG5RgWqMWjXb+g0UmlyMa2YWAWsBgSuUSjJePgbUZWHuxp/dM8CQ4lHJ
+ADvfSI9ysJQM/trqjRu5BRhxiKWR72QSi1qpDPT0nKWlzQ58zs3RSuOJbWm8oOqr
+XL9G/XmvgzK1qwToI/WmXBeaqmfpkagYZm+TJW0GVnDqTC+EoXdFKW7aWIjlcb4p
+tYoiRA/2jjq5OqeV6iKnxz7mEJQR1xDebm6+AWgFy4zyB/QvzanaUTvNiLhyBy6Q
+YwXJHkNh+KrVszBlXxkARrGesXgqOznmDeErkOKDjxzQv+cCAwEAATANBgkqhkiG
+9w0BAQsFAAOCAgEAk83b9wY9iwRrx5Yep3DA3xZkVu3GJcKf0tTL8apP1MzVBSUK
+5tkvW2Z4D41jpZWgJDRF8/nT2lvVwvd5xQ8/oTUerFeG/ZZ+AiBagkBKl8piPHqD
+cefAO8N2SKoYHV4xBeoVU6InUuJ7xu7BLF6tY3xKvx0XsjGC7B621xmq+E56dPZg
+sQwekkxODbUw4NekqYFY21BT4xiWVrTRLIGY9AfV9Ry4gqQTxda7yst4ykWh1a9e
+O+426uz3jshzpQTjZwk8kCZquJKa8Qzqfdlevns0FQDP5jck4BH/YkMNsa/g9XCd
+ZHSB7gqAfNoNTB1rqNKIfPUF4mTu/RWMVwxb8f6h0TfywHZ4q/4R3Zfu3jUyeVVY
+ziJu2CJpcoR9SESKFbN4WFzk91nIhf2pCGo/qNO5f+n5ZPnS2jrrWL5h64e1rz2h
+rVKIYLfeM2M8lVzSL1V0aJ+POcruTRsmlrFT5f7na/5YFt5N+5Z5fzixCLr1MK2w
+4gFw+KhN7CAhKGzHq3NBdWpRFFMR53hyeYsb1vvwFu07JTRh+NbaePk/sk07WtCo
+u2w6pD7xlayTAWcR9WRBv7c3lDejN80U8DONb8fLwtI5oIrkSuwOqvmlDOeFpKiT
+MwTB6oC81Ar39P0R53247w7u9plhPUrmDn/A5KphW633UvgbkH6VmB4Isiw=
+-----END CERTIFICATE-----