diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-03-18 23:02:30 +0300 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-03-18 23:02:30 +0300 |
| commit | 41fe97390ceddf945f3d967b8fdb3de4c66b7dea (patch) | |
| tree | 9c8d89a8624828992f06d892cd2f43818ff5dcc8 /workhorse | |
| parent | 0804d2dc31052fb45a1efecedc8e06ce9bc32862 (diff) | |
Add latest changes from gitlab-org/gitlab@14-9-stable-eev14.9.0-rc42
Diffstat (limited to 'workhorse')
58 files changed, 472 insertions, 431 deletions
diff --git a/workhorse/.tool-versions b/workhorse/.tool-versions index 29cc9a03144..108bdd0f6a5 100644 --- a/workhorse/.tool-versions +++ b/workhorse/.tool-versions @@ -1 +1 @@ -golang 1.17.6 +golang 1.17.7 diff --git a/workhorse/backend.go b/workhorse/backend.go index aef1214abc9..5aa246aa0e8 100644 --- a/workhorse/backend.go +++ b/workhorse/backend.go @@ -18,8 +18,8 @@ func parseAuthBackend(authBackend string) (*url.URL, error) { } } - if backendURL.Scheme != "http" { - return nil, fmt.Errorf("invalid scheme, only 'http' is allowed: %q", authBackend) + if backendURL.Scheme != "http" && backendURL.Scheme != "https" { + return nil, fmt.Errorf("invalid scheme, only 'http' and 'https' are allowed: %q", authBackend) } if backendURL.Host == "" { diff --git a/workhorse/backend_test.go b/workhorse/backend_test.go index c15947a75ad..8b62287fee4 100644 --- a/workhorse/backend_test.go +++ b/workhorse/backend_test.go @@ -10,7 +10,7 @@ func TestParseAuthBackendFailure(t *testing.T) { failures := []string{ "", "ftp://localhost", - "https://example.com", + "gopher://example.com", } for _, example := range failures { @@ -27,6 +27,7 @@ func TestParseAuthBackend(t *testing.T) { {"localhost:3000", "localhost:3000", "http"}, {"http://localhost", "localhost", "http"}, {"localhost", "localhost", "http"}, + {"https://localhost", "localhost", "https"}, } for _, example := range successes { diff --git a/workhorse/go.mod b/workhorse/go.mod index 9fb4c25166b..3a264d41dac 100644 --- a/workhorse/go.mod +++ b/workhorse/go.mod @@ -28,7 +28,7 @@ require ( github.com/sirupsen/logrus v1.8.1 github.com/smartystreets/goconvey v1.6.4 github.com/stretchr/testify v1.7.0 - gitlab.com/gitlab-org/gitaly/v14 v14.3.0-rc2.0.20211007055622-df7dadcc3f74 + gitlab.com/gitlab-org/gitaly/v14 v14.9.0-rc1 gitlab.com/gitlab-org/golang-archive-zip v0.1.1 gitlab.com/gitlab-org/labkit v1.6.0 gocloud.dev v0.23.0 diff --git a/workhorse/go.sum b/workhorse/go.sum index 76590474a00..a565e93bfcc 100644 --- a/workhorse/go.sum +++ b/workhorse/go.sum @@ -108,6 +108,7 @@ github.com/HdrHistogram/hdrhistogram-go v1.1.0/go.mod h1:yDgFjdqOqDEKOvasDdhWNXY github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= +github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= github.com/Microsoft/go-winio v0.4.19 h1:ZMZG0O5M8bhD0lgCURV8yu3hQ7TGvQ4L1ZW8+J0j9iE= @@ -157,6 +158,8 @@ github.com/aws/aws-sdk-go v1.38.35 h1:7AlAO0FC+8nFjxiGKEmq0QLpiA8/XFr6eIxgRTwkdT github.com/aws/aws-sdk-go v1.38.35/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= +github.com/beevik/ntp v0.3.0 h1:xzVrPrE4ziasFXgBVBZJDP0Wg/KpMwk2KHJ4Ba8GrDw= +github.com/beevik/ntp v0.3.0/go.mod h1:hIHWr+l3+/clUnF44zdK+CWW7fO8dR5cIylAQ76NRpg= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -187,6 +190,7 @@ github.com/cloudflare/tableflip v1.2.2/go.mod h1:P4gRehmV6Z2bY5ao5ml9Pd8u6kuEnlB github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= @@ -195,6 +199,8 @@ github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= github.com/coreos/go-systemd/v22 v22.3.1/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= @@ -264,7 +270,6 @@ github.com/getsentry/raven-go v0.2.0 h1:no+xWJRb5ZI7eE8TWgIq1jLulQiIoLG0IfYxv5JY github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= github.com/getsentry/sentry-go v0.5.1/go.mod h1:B8H7x8TYDPkeWPRzGpIiFO97LZP6rL8A3hEt8lUItMw= github.com/getsentry/sentry-go v0.7.0/go.mod h1:pLFpD2Y5RHIKF9Bw3KH6/68DeN2K/XBJd8awjdPnUwg= -github.com/getsentry/sentry-go v0.10.0 h1:6gwY+66NHKqyZrdi6O2jGdo7wGdo9b3B69E01NFgT5g= github.com/getsentry/sentry-go v0.10.0/go.mod h1:kELm/9iCblqUYh+ZRML7PNdCvEuw24wBvJPYyi86cws= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= @@ -293,6 +298,7 @@ github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3I github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= @@ -317,6 +323,7 @@ github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6Wezm github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -480,6 +487,44 @@ github.com/iris-contrib/i18n v0.0.0-20171121225848-987a633949d0/go.mod h1:pMCz62 github.com/iris-contrib/jade v1.1.3/go.mod h1:H/geBymxJhShH5kecoiOCSssPX7QWYH7UaeZTSWddIk= github.com/iris-contrib/pongo2 v0.0.1/go.mod h1:Ssh+00+3GAZqSQb30AvBRNxBx7rf0GqwkjqxNd0u65g= github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= +github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= +github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA= +github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE= +github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s= +github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= +github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= +github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgconn v1.10.1/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= +github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= +github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c= +github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= +github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.2.0/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= +github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= +github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= +github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= +github.com/jackc/pgtype v1.9.1/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= +github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= +github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= +github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= +github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= +github.com/jackc/pgx/v4 v4.14.1/go.mod h1:RgDuE4Z34o7XE92RpLsvFiOEfrAUT0Xt2KxvX73W06M= +github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.2.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jcmturner/gofork v0.0.0-20190328161633-dc7c13fece03/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= @@ -525,7 +570,6 @@ github.com/kataras/neffos v0.0.14/go.mod h1:8lqADm8PnbeFfL7CLXh1WHw53dG27MC3pgi2 github.com/kataras/pio v0.0.0-20190103105442-ea782b38602d/go.mod h1:NV88laa9UiiDuX9AhMbDPkGYSPugBOV6yTZB1l2K9Z0= github.com/kataras/pio v0.0.2/go.mod h1:hAoW0t9UmXi4R5Oyq5Z4irTbaTsOemSrDGUtaTl7Dro= github.com/kataras/sitemap v0.0.5/go.mod h1:KY2eugMKiPwsJgx7+U103YZehfvNGOXURubcGyk0Bz8= -github.com/kelseyhightower/envconfig v1.3.0 h1:IvRS4f2VcIQy6j4ORGIf9145T/AsUB+oY8LyvN8BXNM= github.com/kelseyhightower/envconfig v1.3.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= @@ -547,18 +591,22 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g= github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/lib/pq v1.10.1 h1:6VXZrLU0jHBYyAqrSPa+MgPfnSvTPuMgK+k0o5kVFWo= github.com/lib/pq v1.10.1/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/libgit2/git2go v0.0.0-20190104134018-ecaeb7a21d47/go.mod h1:4bKN42efkbNYMZlvDfxGDxzl066GhpvIircZDsm8Y+Y= github.com/libgit2/git2go/v31 v31.4.12/go.mod h1:c/rkJcBcUFx6wHaT++UwNpKvIsmPNqCeQ/vzO4DrEec= +github.com/libgit2/git2go/v33 v33.0.6/go.mod h1:KdpqkU+6+++4oHna/MIOgx4GCQ92IPCdpVRMRI80J+4= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20200305213919-a88bf8de3718/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20210210170715-a8dfcb80d3a7 h1:YjW+hUb8Fh2S58z4av4t/0cBMK/Q0aP48RocCFsC8yI= @@ -570,12 +618,14 @@ github.com/lightstep/lightstep-tracer-go v0.24.0/go.mod h1:RnONwHKg89zYPmF+Uig5P github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-ieproxy v0.0.1 h1:qiyop7gCflfhwCzGyeT0gro3sF9AIg9HU98JORTkqfI= github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= @@ -668,7 +718,6 @@ github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FI github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.8.1 h1:1Nf83orprkJyknT6h7zbuEGUEjcyVlCxSUGTENmNCRM= github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= github.com/philhofer/fwd v1.0.0 h1:UbZqGr5Y38ApvM/V/jEljVxwocdweyH+vmYvRPBnbqQ= @@ -721,7 +770,9 @@ github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.4.0/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rubenv/sql-migrate v0.0.0-20191213152630-06338513c237 h1:q6N3WgCVttyX9Fg3e4nrLohUXvAlTu44Ugc4m6qlezc= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= +github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= github.com/rubenv/sql-migrate v0.0.0-20191213152630-06338513c237/go.mod h1:rtQlpHw+eR6UrqaS3kX1VYeaCxzCVdimDS7g5Ln4pPc= github.com/rubyist/tracerx v0.0.0-20170927163412-787959303086/go.mod h1:YpdgDXpumPB/+EGmGTYHeiW/0QVFRzBYTNFaxWfPDk4= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= @@ -731,6 +782,7 @@ github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFo github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtmuhtR2uUrrJOpYzYRvbcPAid+g= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sebest/xff v0.0.0-20160910043805-6c115e0ffa35/go.mod h1:wozgYq9WEBQBaIJe4YZ0qTSFAMxmcwBhQH0fO0R34Z0= @@ -745,6 +797,8 @@ github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500/go.mod h1:+n github.com/shirou/gopsutil v2.20.1+incompatible h1:oIq9Cq4i84Hk8uQAUOG3eNdI/29hBawGrD5YRl6JRDY= github.com/shirou/gopsutil v2.20.1+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shogo82148/go-shuffle v0.0.0-20170808115208-59829097ff3b/go.mod h1:2htx6lmL0NGLHlO8ZCf+lQBGBHIbEujyywxJArf+2Yc= +github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -827,12 +881,13 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= gitlab.com/gitlab-org/gitaly v1.68.0 h1:VlcJs1+PrhW7lqJUU7Fh1q8FMJujmbbivdfde/cwB98= gitlab.com/gitlab-org/gitaly v1.68.0/go.mod h1:/pCsB918Zu5wFchZ9hLYin9WkJ2yQqdVNz0zlv5HbXg= gitlab.com/gitlab-org/gitaly/v14 v14.0.0-rc1/go.mod h1:4Cz8tOAyueSZX5o6gYum1F/unupaOclxqETPcg4ODvQ= -gitlab.com/gitlab-org/gitaly/v14 v14.3.0-rc2.0.20211007055622-df7dadcc3f74 h1:7R4FHfZ9lBH0BLtnGNjRmHvjz6epnXPodhiT0pR96PI= -gitlab.com/gitlab-org/gitaly/v14 v14.3.0-rc2.0.20211007055622-df7dadcc3f74/go.mod h1:2McjFiZrwflPGtXSquCAXWzewmxTPytoI/vamNz/QPM= +gitlab.com/gitlab-org/gitaly/v14 v14.9.0-rc1 h1:9vStRdXxcBQ8dHlVnpV28fwLOgyDkSFIpGnPqwzdTvw= +gitlab.com/gitlab-org/gitaly/v14 v14.9.0-rc1/go.mod h1:Xk5pn6IWsejg3z2X6BRczC5QaI97PRF3GU5OrJ5Amkg= gitlab.com/gitlab-org/gitlab-shell v1.9.8-0.20201117050822-3f9890ef73dc/go.mod h1:5QSTbpAHY2v0iIH5uHh2KA9w7sPUqPmnLjDApI/sv1U= gitlab.com/gitlab-org/gitlab-shell v1.9.8-0.20210720163109-50da611814d2/go.mod h1:QWDYBwuy24qGMandtCngLRPzFgnGPg6LSNoJWPKmJMc= gitlab.com/gitlab-org/golang-archive-zip v0.1.1 h1:35k9giivbxwF03+8A05Cm8YoxoakU8FBCj5gysjCTCE= @@ -871,6 +926,7 @@ go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+ go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= @@ -882,12 +938,14 @@ golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190621222207-cc06ce4a13d4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -897,8 +955,10 @@ golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= -golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf h1:B2n+Zi5QeYRDAEodEu72OS36gmTWjgpXr2+cWcBW90o= golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= +golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 h1:/UOmuWzQfxxo9UtlXMwuQU8CMgg1eZXqTRwkSQJWKOI= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1039,6 +1099,7 @@ golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190310054646-10058d7d4faa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1099,8 +1160,10 @@ golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210420205809-ac73e9fd8988/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210503080704-8803ae5d1324/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210503173754-0981d6026fa6 h1:cdsMqa2nXzqlgs183pHxtvoVwU7CyzaCTAUOg94af4c= golang.org/x/sys v0.0.0-20210503173754-0981d6026fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211102192858-4dd72447c267 h1:7zYaz3tjChtpayGDzu6H0hDAUM5zIGA2XW7kRNgQ0jc= +golang.org/x/sys v0.0.0-20211102192858-4dd72447c267/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1135,12 +1198,14 @@ golang.org/x/tools v0.0.0-20190327201419-c70d86f8b7cf/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190422233926-fe54fb35175b/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190829051458-42f498d34c4d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191004055002-72853e10c5a3/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -1185,6 +1250,8 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1343,8 +1410,8 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= -gopkg.in/gorp.v1 v1.7.2 h1:j3DWlAyGVv8whO7AcIWznQ2Yj7yJkn34B8s63GViAAw= gopkg.in/gorp.v1 v1.7.2/go.mod h1:Wo3h+DBQZIxATwftsglhdD/62zRFPhGhTiu5jUJmCaw= +gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= gopkg.in/ini.v1 v1.51.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo= gopkg.in/jcmturner/dnsutils.v1 v1.0.1/go.mod h1:m3v+5svpVOhtFAP/wSz+yzh4Mc0Fg7eRhxkJMWSIz9Q= diff --git a/workhorse/internal/staticpages/servefile.go b/workhorse/internal/staticpages/servefile.go index be314f181b7..18fcdadcbed 100644 --- a/workhorse/internal/staticpages/servefile.go +++ b/workhorse/internal/staticpages/servefile.go @@ -67,6 +67,9 @@ func (s *Static) ServeExisting(prefix urlprefix.Prefix, cache CacheMode, notFoun notFoundHandler.ServeHTTP(w, r) return } + + w.Header().Set("X-Content-Type-Options", "nosniff") + defer content.Close() switch cache { diff --git a/workhorse/internal/staticpages/servefile_test.go b/workhorse/internal/staticpages/servefile_test.go index f27bd0ccaeb..67675beccf8 100644 --- a/workhorse/internal/staticpages/servefile_test.go +++ b/workhorse/internal/staticpages/servefile_test.go @@ -78,6 +78,7 @@ func TestServingTheActualFile(t *testing.T) { w := httptest.NewRecorder() st := &Static{DocumentRoot: dir} st.ServeExisting("/", CacheDisabled, nil).ServeHTTP(w, httpRequest) + testhelper.RequireResponseHeader(t, w, "X-Content-Type-Options", "nosniff") require.Equal(t, 200, w.Code) if w.Body.String() != fileContent { t.Error("We should serve the file: ", w.Body.String()) @@ -109,6 +110,7 @@ func TestExcludedPaths(t *testing.T) { st.ServeExisting("/", CacheDisabled, nil).ServeHTTP(w, httpRequest) if tc.found { + testhelper.RequireResponseHeader(t, w, "X-Content-Type-Options", "nosniff") require.Equal(t, 200, w.Code) require.Equal(t, tc.contents, w.Body.String()) } else { @@ -144,6 +146,7 @@ func testServingThePregzippedFile(t *testing.T, enableGzip bool) { w := httptest.NewRecorder() st := &Static{DocumentRoot: dir} st.ServeExisting("/", CacheDisabled, nil).ServeHTTP(w, httpRequest) + testhelper.RequireResponseHeader(t, w, "X-Content-Type-Options", "nosniff") require.Equal(t, 200, w.Code) if enableGzip { testhelper.RequireResponseHeader(t, w, "Content-Encoding", "gzip") diff --git a/workhorse/internal/testhelper/testhelper.go b/workhorse/internal/testhelper/testhelper.go index dae8f9b3149..6bbdfddcd60 100644 --- a/workhorse/internal/testhelper/testhelper.go +++ b/workhorse/internal/testhelper/testhelper.go @@ -167,3 +167,16 @@ func Retry(t testing.TB, timeout time.Duration, fn func() error) { } t.Fatalf("test timeout after %v; last error: %v", timeout, err) } + +func SetupStaticFileHelper(t *testing.T, fpath, content, directory string) string { + cwd, err := os.Getwd() + require.NoError(t, err, "get working directory") + + absDocumentRoot := path.Join(cwd, directory) + require.NoError(t, os.MkdirAll(path.Join(absDocumentRoot, path.Dir(fpath)), 0755), "create document root") + + staticFile := path.Join(absDocumentRoot, fpath) + require.NoError(t, ioutil.WriteFile(staticFile, []byte(content), 0666), "write file content") + + return absDocumentRoot +} diff --git a/workhorse/internal/artifacts/artifacts_store_test.go b/workhorse/internal/upload/artifacts_store_test.go index f9fb28cf7ce..97e66fc37a4 100644 --- a/workhorse/internal/artifacts/artifacts_store_test.go +++ b/workhorse/internal/upload/artifacts_store_test.go @@ -1,4 +1,4 @@ -package artifacts +package upload import ( "archive/zip" @@ -17,8 +17,8 @@ import ( "github.com/stretchr/testify/require" "gitlab.com/gitlab-org/gitlab/workhorse/internal/api" - "gitlab.com/gitlab-org/gitlab/workhorse/internal/objectstore/test" "gitlab.com/gitlab-org/gitlab/workhorse/internal/testhelper" + "gitlab.com/gitlab-org/gitlab/workhorse/internal/upload/destination/objectstore/test" ) func createTestZipArchive(t *testing.T) (data []byte, md5Hash string) { diff --git a/workhorse/internal/artifacts/artifacts_upload_test.go b/workhorse/internal/upload/artifacts_upload_test.go index 3e8a52be1a1..0a9e4ef3869 100644 --- a/workhorse/internal/artifacts/artifacts_upload_test.go +++ b/workhorse/internal/upload/artifacts_upload_test.go @@ -1,4 +1,4 @@ -package artifacts +package upload import ( "archive/zip" @@ -16,12 +16,13 @@ import ( "github.com/golang-jwt/jwt/v4" + "gitlab.com/gitlab-org/labkit/log" + "gitlab.com/gitlab-org/gitlab/workhorse/internal/api" - "gitlab.com/gitlab-org/gitlab/workhorse/internal/filestore" "gitlab.com/gitlab-org/gitlab/workhorse/internal/helper" "gitlab.com/gitlab-org/gitlab/workhorse/internal/proxy" "gitlab.com/gitlab-org/gitlab/workhorse/internal/testhelper" - "gitlab.com/gitlab-org/gitlab/workhorse/internal/upload" + "gitlab.com/gitlab-org/gitlab/workhorse/internal/upload/destination" "gitlab.com/gitlab-org/gitlab/workhorse/internal/upstream/roundtripper" "gitlab.com/gitlab-org/gitlab/workhorse/internal/zipartifacts" @@ -35,6 +36,14 @@ const ( Path = "/url/path" ) +func TestMain(m *testing.M) { + if err := testhelper.BuildExecutables(); err != nil { + log.WithError(err).Fatal() + } + + os.Exit(m.Run()) +} + func testArtifactsUploadServer(t *testing.T, authResponse *api.Response, bodyProcessor func(w http.ResponseWriter, r *http.Request)) *httptest.Server { mux := http.NewServeMux() mux.HandleFunc(Path+"/authorize", func(w http.ResponseWriter, r *http.Request) { @@ -51,7 +60,7 @@ func testArtifactsUploadServer(t *testing.T, authResponse *api.Response, bodyPro w.Write(data) }) mux.HandleFunc(Path, func(w http.ResponseWriter, r *http.Request) { - opts, err := filestore.GetOpts(authResponse) + opts, err := destination.GetOpts(authResponse) require.NoError(t, err) if r.Method != "POST" { @@ -162,7 +171,7 @@ func testUploadArtifacts(t *testing.T, contentType, url string, body io.Reader) testhelper.ConfigureSecret() apiClient := api.NewAPI(parsedURL, "123", roundTripper) proxyClient := proxy.NewProxy(parsedURL, "123", roundTripper) - UploadArtifacts(apiClient, proxyClient, &upload.DefaultPreparer{}).ServeHTTP(response, httpRequest) + Artifacts(apiClient, proxyClient, &DefaultPreparer{}).ServeHTTP(response, httpRequest) return response } @@ -193,10 +202,10 @@ func TestUploadHandlerAddingMetadata(t *testing.T) { t.Run(tc.desc, func(t *testing.T) { s := setupWithTmpPath(t, "file", tc.includeFormat, tc.format, nil, func(w http.ResponseWriter, r *http.Request) { - token, err := jwt.ParseWithClaims(r.Header.Get(upload.RewrittenFieldsHeader), &upload.MultipartClaims{}, testhelper.ParseJWT) + token, err := jwt.ParseWithClaims(r.Header.Get(RewrittenFieldsHeader), &MultipartClaims{}, testhelper.ParseJWT) require.NoError(t, err) - rewrittenFields := token.Claims.(*upload.MultipartClaims).RewrittenFields + rewrittenFields := token.Claims.(*MultipartClaims).RewrittenFields require.Equal(t, 2, len(rewrittenFields)) require.Contains(t, rewrittenFields, "file") @@ -225,10 +234,10 @@ func TestUploadHandlerAddingMetadata(t *testing.T) { func TestUploadHandlerTarArtifact(t *testing.T) { s := setupWithTmpPath(t, "file", true, "tar", nil, func(w http.ResponseWriter, r *http.Request) { - token, err := jwt.ParseWithClaims(r.Header.Get(upload.RewrittenFieldsHeader), &upload.MultipartClaims{}, testhelper.ParseJWT) + token, err := jwt.ParseWithClaims(r.Header.Get(RewrittenFieldsHeader), &MultipartClaims{}, testhelper.ParseJWT) require.NoError(t, err) - rewrittenFields := token.Claims.(*upload.MultipartClaims).RewrittenFields + rewrittenFields := token.Claims.(*MultipartClaims).RewrittenFields require.Equal(t, 1, len(rewrittenFields)) require.Contains(t, rewrittenFields, "file") diff --git a/workhorse/internal/artifacts/artifacts_upload.go b/workhorse/internal/upload/artifacts_uploader.go index f1fd69082f8..2a91a05fe3d 100644 --- a/workhorse/internal/artifacts/artifacts_upload.go +++ b/workhorse/internal/upload/artifacts_uploader.go @@ -1,4 +1,4 @@ -package artifacts +package upload import ( "context" @@ -16,9 +16,8 @@ import ( "gitlab.com/gitlab-org/labkit/log" "gitlab.com/gitlab-org/gitlab/workhorse/internal/api" - "gitlab.com/gitlab-org/gitlab/workhorse/internal/filestore" "gitlab.com/gitlab-org/gitlab/workhorse/internal/helper" - "gitlab.com/gitlab-org/gitlab/workhorse/internal/upload" + "gitlab.com/gitlab-org/gitlab/workhorse/internal/upload/destination" "gitlab.com/gitlab-org/gitlab/workhorse/internal/zipartifacts" ) @@ -36,17 +35,33 @@ var zipSubcommandsErrorsCounter = promauto.NewCounterVec( }, []string{"error"}) type artifactsUploadProcessor struct { - opts *filestore.SaveFileOpts + opts *destination.UploadOpts format string - upload.SavedFileTracker + SavedFileTracker } -func (a *artifactsUploadProcessor) generateMetadataFromZip(ctx context.Context, file *filestore.FileHandler) (*filestore.FileHandler, error) { +// Artifacts is like a Multipart but specific for artifacts upload. +func Artifacts(myAPI *api.API, h http.Handler, p Preparer) http.Handler { + return myAPI.PreAuthorizeHandler(func(w http.ResponseWriter, r *http.Request, a *api.Response) { + opts, _, err := p.Prepare(a) + if err != nil { + helper.Fail500(w, r, fmt.Errorf("UploadArtifacts: error preparing file storage options")) + return + } + + format := r.URL.Query().Get(ArtifactFormatKey) + + mg := &artifactsUploadProcessor{opts: opts, format: format, SavedFileTracker: SavedFileTracker{Request: r}} + interceptMultipartFiles(w, r, h, a, mg, opts) + }, "/authorize") +} + +func (a *artifactsUploadProcessor) generateMetadataFromZip(ctx context.Context, file *destination.FileHandler) (*destination.FileHandler, error) { metaReader, metaWriter := io.Pipe() defer metaWriter.Close() - metaOpts := &filestore.SaveFileOpts{ + metaOpts := &destination.UploadOpts{ LocalTempPath: a.opts.LocalTempPath, TempFilePrefix: "metadata.gz", } @@ -71,12 +86,12 @@ func (a *artifactsUploadProcessor) generateMetadataFromZip(ctx context.Context, type saveResult struct { error - *filestore.FileHandler + *destination.FileHandler } done := make(chan saveResult) go func() { var result saveResult - result.FileHandler, result.error = filestore.SaveFileFromReader(ctx, metaReader, -1, metaOpts) + result.FileHandler, result.error = destination.Upload(ctx, metaReader, -1, metaOpts) done <- result }() @@ -104,7 +119,7 @@ func (a *artifactsUploadProcessor) generateMetadataFromZip(ctx context.Context, return result.FileHandler, result.error } -func (a *artifactsUploadProcessor) ProcessFile(ctx context.Context, formName string, file *filestore.FileHandler, writer *multipart.Writer) error { +func (a *artifactsUploadProcessor) ProcessFile(ctx context.Context, formName string, file *destination.FileHandler, writer *multipart.Writer) error { // ProcessFile for artifacts requires file form-data field name to eq `file` if formName != "file" { @@ -150,18 +165,3 @@ func (a *artifactsUploadProcessor) ProcessFile(ctx context.Context, formName str func (a *artifactsUploadProcessor) Name() string { return "artifacts" } - -func UploadArtifacts(myAPI *api.API, h http.Handler, p upload.Preparer) http.Handler { - return myAPI.PreAuthorizeHandler(func(w http.ResponseWriter, r *http.Request, a *api.Response) { - opts, _, err := p.Prepare(a) - if err != nil { - helper.Fail500(w, r, fmt.Errorf("UploadArtifacts: error preparing file storage options")) - return - } - - format := r.URL.Query().Get(ArtifactFormatKey) - - mg := &artifactsUploadProcessor{opts: opts, format: format, SavedFileTracker: upload.SavedFileTracker{Request: r}} - upload.InterceptMultipartFiles(w, r, h, a, mg, opts) - }, "/authorize") -} diff --git a/workhorse/internal/upload/body_uploader.go b/workhorse/internal/upload/body_uploader.go index 6c53bd9241b..d831f9f43a1 100644 --- a/workhorse/internal/upload/body_uploader.go +++ b/workhorse/internal/upload/body_uploader.go @@ -8,41 +8,10 @@ import ( "strings" "gitlab.com/gitlab-org/gitlab/workhorse/internal/api" - "gitlab.com/gitlab-org/gitlab/workhorse/internal/filestore" "gitlab.com/gitlab-org/gitlab/workhorse/internal/helper" + "gitlab.com/gitlab-org/gitlab/workhorse/internal/upload/destination" ) -type PreAuthorizer interface { - PreAuthorizeHandler(next api.HandleFunc, suffix string) http.Handler -} - -// Verifier is an optional pluggable behavior for upload paths. If -// Verify() returns an error, Workhorse will return an error response to -// the client instead of propagating the request to Rails. The motivating -// use case is Git LFS, where Workhorse checks the size and SHA256 -// checksum of the uploaded file. -type Verifier interface { - // Verify can abort the upload by returning an error - Verify(handler *filestore.FileHandler) error -} - -// Preparer is a pluggable behavior that interprets a Rails API response -// and either tells Workhorse how to handle the upload, via the -// SaveFileOpts and Verifier, or it rejects the request by returning a -// non-nil error. Its intended use is to make sure the upload gets stored -// in the right location: either a local directory, or one of several -// supported object storage backends. -type Preparer interface { - Prepare(a *api.Response) (*filestore.SaveFileOpts, Verifier, error) -} - -type DefaultPreparer struct{} - -func (s *DefaultPreparer) Prepare(a *api.Response) (*filestore.SaveFileOpts, Verifier, error) { - opts, err := filestore.GetOpts(a) - return opts, nil, err -} - // RequestBody is a request middleware. It will store the request body to // a location by determined an api.Response value. It then forwards the // request to gitlab-rails without the original request body. @@ -54,7 +23,7 @@ func RequestBody(rails PreAuthorizer, h http.Handler, p Preparer) http.Handler { return } - fh, err := filestore.SaveFileFromReader(r.Context(), r.Body, r.ContentLength, opts) + fh, err := destination.Upload(r.Context(), r.Body, r.ContentLength, opts) if err != nil { helper.Fail500(w, r, fmt.Errorf("RequestBody: upload failed: %v", err)) return diff --git a/workhorse/internal/upload/body_uploader_test.go b/workhorse/internal/upload/body_uploader_test.go index b3d561ac131..47490db8780 100644 --- a/workhorse/internal/upload/body_uploader_test.go +++ b/workhorse/internal/upload/body_uploader_test.go @@ -15,8 +15,8 @@ import ( "github.com/stretchr/testify/require" "gitlab.com/gitlab-org/gitlab/workhorse/internal/api" - "gitlab.com/gitlab-org/gitlab/workhorse/internal/filestore" "gitlab.com/gitlab-org/gitlab/workhorse/internal/testhelper" + "gitlab.com/gitlab-org/gitlab/workhorse/internal/upload/destination" ) const ( @@ -169,8 +169,8 @@ type alwaysLocalPreparer struct { prepareError error } -func (a *alwaysLocalPreparer) Prepare(_ *api.Response) (*filestore.SaveFileOpts, Verifier, error) { - opts, err := filestore.GetOpts(&api.Response{TempPath: os.TempDir()}) +func (a *alwaysLocalPreparer) Prepare(_ *api.Response) (*destination.UploadOpts, Verifier, error) { + opts, err := destination.GetOpts(&api.Response{TempPath: os.TempDir()}) if err != nil { return nil, nil, err } @@ -180,7 +180,7 @@ func (a *alwaysLocalPreparer) Prepare(_ *api.Response) (*filestore.SaveFileOpts, type alwaysFailsVerifier struct{} -func (alwaysFailsVerifier) Verify(handler *filestore.FileHandler) error { +func (alwaysFailsVerifier) Verify(handler *destination.FileHandler) error { return fmt.Errorf("Verification failed") } @@ -188,7 +188,7 @@ type mockVerifier struct { invoked bool } -func (m *mockVerifier) Verify(handler *filestore.FileHandler) error { +func (m *mockVerifier) Verify(handler *destination.FileHandler) error { m.invoked = true return nil diff --git a/workhorse/internal/filestore/file_handler.go b/workhorse/internal/upload/destination/destination.go index dac8d4d6247..7a030e59a64 100644 --- a/workhorse/internal/filestore/file_handler.go +++ b/workhorse/internal/upload/destination/destination.go @@ -1,4 +1,7 @@ -package filestore +// The destination package handles uploading to a specific destination (delegates +// to filestore or objectstore packages) based on options from the pre-authorization +// API and finalizing the upload. +package destination import ( "context" @@ -14,8 +17,9 @@ import ( "gitlab.com/gitlab-org/labkit/log" - "gitlab.com/gitlab-org/gitlab/workhorse/internal/objectstore" "gitlab.com/gitlab-org/gitlab/workhorse/internal/secret" + "gitlab.com/gitlab-org/gitlab/workhorse/internal/upload/destination/filestore" + "gitlab.com/gitlab-org/gitlab/workhorse/internal/upload/destination/objectstore" ) type SizeError error @@ -107,9 +111,9 @@ type consumer interface { Consume(context.Context, io.Reader, time.Time) (int64, error) } -// SaveFileFromReader persists the provided reader content to all the location specified in opts. A cleanup will be performed once ctx is Done +// Upload persists the provided reader content to all the location specified in opts. A cleanup will be performed once ctx is Done // Make sure the provided context will not expire before finalizing upload with GitLab Rails. -func SaveFileFromReader(ctx context.Context, reader io.Reader, size int64, opts *SaveFileOpts) (*FileHandler, error) { +func Upload(ctx context.Context, reader io.Reader, size int64, opts *UploadOpts) (*FileHandler, error) { fh := &FileHandler{ Name: opts.TempFilePrefix, RemoteID: opts.RemoteID, @@ -126,7 +130,7 @@ func SaveFileFromReader(ctx context.Context, reader io.Reader, size int64, opts switch { case opts.IsLocal(): clientMode = "local" - uploadDestination, err = fh.uploadLocalFile(ctx, opts) + uploadDestination, err = fh.newLocalFile(ctx, opts) case opts.UseWorkhorseClientEnabled() && opts.ObjectStorageConfig.IsGoCloud(): clientMode = fmt.Sprintf("go_cloud:%s", opts.ObjectStorageConfig.Provider) p := &objectstore.GoCloudObjectParams{ @@ -210,16 +214,16 @@ func SaveFileFromReader(ctx context.Context, reader io.Reader, size int64, opts return fh, nil } -func (fh *FileHandler) uploadLocalFile(ctx context.Context, opts *SaveFileOpts) (consumer, error) { +func (fh *FileHandler) newLocalFile(ctx context.Context, opts *UploadOpts) (consumer, error) { // make sure TempFolder exists err := os.MkdirAll(opts.LocalTempPath, 0700) if err != nil { - return nil, fmt.Errorf("uploadLocalFile: mkdir %q: %v", opts.LocalTempPath, err) + return nil, fmt.Errorf("newLocalFile: mkdir %q: %v", opts.LocalTempPath, err) } file, err := ioutil.TempFile(opts.LocalTempPath, opts.TempFilePrefix) if err != nil { - return nil, fmt.Errorf("uploadLocalFile: create file: %v", err) + return nil, fmt.Errorf("newLocalFile: create file: %v", err) } go func() { @@ -228,32 +232,5 @@ func (fh *FileHandler) uploadLocalFile(ctx context.Context, opts *SaveFileOpts) }() fh.LocalPath = file.Name() - return &localUpload{file}, nil -} - -type localUpload struct{ io.WriteCloser } - -func (loc *localUpload) Consume(_ context.Context, r io.Reader, _ time.Time) (int64, error) { - n, err := io.Copy(loc.WriteCloser, r) - errClose := loc.Close() - if err == nil { - err = errClose - } - return n, err -} - -// SaveFileFromDisk open the local file fileName and calls SaveFileFromReader -func SaveFileFromDisk(ctx context.Context, fileName string, opts *SaveFileOpts) (fh *FileHandler, err error) { - file, err := os.Open(fileName) - if err != nil { - return nil, err - } - defer file.Close() - - fi, err := file.Stat() - if err != nil { - return nil, err - } - - return SaveFileFromReader(ctx, file, fi.Size(), opts) + return &filestore.LocalFile{File: file}, nil } diff --git a/workhorse/internal/filestore/file_handler_test.go b/workhorse/internal/upload/destination/destination_test.go index 2fd034bb761..ddf0ea24d60 100644 --- a/workhorse/internal/filestore/file_handler_test.go +++ b/workhorse/internal/upload/destination/destination_test.go @@ -1,4 +1,4 @@ -package filestore_test +package destination_test import ( "context" @@ -17,13 +17,13 @@ import ( "gocloud.dev/blob" "gitlab.com/gitlab-org/gitlab/workhorse/internal/config" - "gitlab.com/gitlab-org/gitlab/workhorse/internal/filestore" - "gitlab.com/gitlab-org/gitlab/workhorse/internal/objectstore/test" "gitlab.com/gitlab-org/gitlab/workhorse/internal/testhelper" + "gitlab.com/gitlab-org/gitlab/workhorse/internal/upload/destination" + "gitlab.com/gitlab-org/gitlab/workhorse/internal/upload/destination/objectstore/test" ) func testDeadline() time.Time { - return time.Now().Add(filestore.DefaultObjectStoreTimeout) + return time.Now().Add(destination.DefaultObjectStoreTimeout) } func requireFileGetsRemovedAsync(t *testing.T, filePath string) { @@ -39,7 +39,7 @@ func requireObjectStoreDeletedAsync(t *testing.T, expectedDeletes int, osStub *t require.Eventually(t, func() bool { return osStub.DeletesCnt() == expectedDeletes }, time.Second, time.Millisecond, "Object not deleted") } -func TestSaveFileWrongSize(t *testing.T) { +func TestUploadWrongSize(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -47,15 +47,15 @@ func TestSaveFileWrongSize(t *testing.T) { require.NoError(t, err) defer os.RemoveAll(tmpFolder) - opts := &filestore.SaveFileOpts{LocalTempPath: tmpFolder, TempFilePrefix: "test-file"} - fh, err := filestore.SaveFileFromReader(ctx, strings.NewReader(test.ObjectContent), test.ObjectSize+1, opts) + opts := &destination.UploadOpts{LocalTempPath: tmpFolder, TempFilePrefix: "test-file"} + fh, err := destination.Upload(ctx, strings.NewReader(test.ObjectContent), test.ObjectSize+1, opts) require.Error(t, err) - _, isSizeError := err.(filestore.SizeError) + _, isSizeError := err.(destination.SizeError) require.True(t, isSizeError, "Should fail with SizeError") require.Nil(t, fh) } -func TestSaveFileWithKnownSizeExceedLimit(t *testing.T) { +func TestUploadWithKnownSizeExceedLimit(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -63,15 +63,15 @@ func TestSaveFileWithKnownSizeExceedLimit(t *testing.T) { require.NoError(t, err) defer os.RemoveAll(tmpFolder) - opts := &filestore.SaveFileOpts{LocalTempPath: tmpFolder, TempFilePrefix: "test-file", MaximumSize: test.ObjectSize - 1} - fh, err := filestore.SaveFileFromReader(ctx, strings.NewReader(test.ObjectContent), test.ObjectSize, opts) + opts := &destination.UploadOpts{LocalTempPath: tmpFolder, TempFilePrefix: "test-file", MaximumSize: test.ObjectSize - 1} + fh, err := destination.Upload(ctx, strings.NewReader(test.ObjectContent), test.ObjectSize, opts) require.Error(t, err) - _, isSizeError := err.(filestore.SizeError) + _, isSizeError := err.(destination.SizeError) require.True(t, isSizeError, "Should fail with SizeError") require.Nil(t, fh) } -func TestSaveFileWithUnknownSizeExceedLimit(t *testing.T) { +func TestUploadWithUnknownSizeExceedLimit(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -79,22 +79,13 @@ func TestSaveFileWithUnknownSizeExceedLimit(t *testing.T) { require.NoError(t, err) defer os.RemoveAll(tmpFolder) - opts := &filestore.SaveFileOpts{LocalTempPath: tmpFolder, TempFilePrefix: "test-file", MaximumSize: test.ObjectSize - 1} - fh, err := filestore.SaveFileFromReader(ctx, strings.NewReader(test.ObjectContent), -1, opts) - require.Equal(t, err, filestore.ErrEntityTooLarge) + opts := &destination.UploadOpts{LocalTempPath: tmpFolder, TempFilePrefix: "test-file", MaximumSize: test.ObjectSize - 1} + fh, err := destination.Upload(ctx, strings.NewReader(test.ObjectContent), -1, opts) + require.Equal(t, err, destination.ErrEntityTooLarge) require.Nil(t, fh) } -func TestSaveFromDiskNotExistingFile(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - fh, err := filestore.SaveFileFromDisk(ctx, "/I/do/not/exist", &filestore.SaveFileOpts{}) - require.Error(t, err, "SaveFileFromDisk should fail") - require.True(t, os.IsNotExist(err), "Provided file should not exists") - require.Nil(t, fh, "On error FileHandler should be nil") -} - -func TestSaveFileWrongETag(t *testing.T) { +func TestUploadWrongETag(t *testing.T) { tests := []struct { name string multipart bool @@ -110,7 +101,7 @@ func TestSaveFileWrongETag(t *testing.T) { objectURL := ts.URL + test.ObjectPath - opts := &filestore.SaveFileOpts{ + opts := &destination.UploadOpts{ RemoteID: "test-file", RemoteURL: objectURL, PresignedPut: objectURL + "?Signature=ASignature", @@ -126,7 +117,7 @@ func TestSaveFileWrongETag(t *testing.T) { osStub.InitiateMultipartUpload(test.ObjectPath) } ctx, cancel := context.WithCancel(context.Background()) - fh, err := filestore.SaveFileFromReader(ctx, strings.NewReader(test.ObjectContent), test.ObjectSize, opts) + fh, err := destination.Upload(ctx, strings.NewReader(test.ObjectContent), test.ObjectSize, opts) require.Nil(t, fh) require.Error(t, err) require.Equal(t, 1, osStub.PutsCnt(), "File not uploaded") @@ -138,32 +129,7 @@ func TestSaveFileWrongETag(t *testing.T) { } } -func TestSaveFileFromDiskToLocalPath(t *testing.T) { - f, err := ioutil.TempFile("", "workhorse-test") - require.NoError(t, err) - defer os.Remove(f.Name()) - - _, err = fmt.Fprint(f, test.ObjectContent) - require.NoError(t, err) - - tmpFolder, err := ioutil.TempDir("", "workhorse-test-tmp") - require.NoError(t, err) - defer os.RemoveAll(tmpFolder) - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - opts := &filestore.SaveFileOpts{LocalTempPath: tmpFolder} - fh, err := filestore.SaveFileFromDisk(ctx, f.Name(), opts) - require.NoError(t, err) - require.NotNil(t, fh) - - require.NotEmpty(t, fh.LocalPath, "File not persisted on disk") - _, err = os.Stat(fh.LocalPath) - require.NoError(t, err) -} - -func TestSaveFile(t *testing.T) { +func TestUpload(t *testing.T) { testhelper.ConfigureSecret() type remote int @@ -189,7 +155,7 @@ func TestSaveFile(t *testing.T) { for _, spec := range tests { t.Run(spec.name, func(t *testing.T) { - var opts filestore.SaveFileOpts + var opts destination.UploadOpts var expectedDeletes, expectedPuts int osStub, ts := test.StartObjectStore() @@ -231,7 +197,7 @@ func TestSaveFile(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - fh, err := filestore.SaveFileFromReader(ctx, strings.NewReader(test.ObjectContent), test.ObjectSize, &opts) + fh, err := destination.Upload(ctx, strings.NewReader(test.ObjectContent), test.ObjectSize, &opts) require.NoError(t, err) require.NotNil(t, fh) @@ -279,7 +245,7 @@ func TestSaveFile(t *testing.T) { } } -func TestSaveFileWithS3WorkhorseClient(t *testing.T) { +func TestUploadWithS3WorkhorseClient(t *testing.T) { tests := []struct { name string objectSize int64 @@ -298,7 +264,7 @@ func TestSaveFileWithS3WorkhorseClient(t *testing.T) { name: "unknown object size with limit", objectSize: -1, maxSize: test.ObjectSize - 1, - expectedErr: filestore.ErrEntityTooLarge, + expectedErr: destination.ErrEntityTooLarge, }, } @@ -312,12 +278,12 @@ func TestSaveFileWithS3WorkhorseClient(t *testing.T) { defer cancel() remoteObject := "tmp/test-file/1" - opts := filestore.SaveFileOpts{ + opts := destination.UploadOpts{ RemoteID: "test-file", Deadline: testDeadline(), UseWorkhorseClient: true, RemoteTempObjectID: remoteObject, - ObjectStorageConfig: filestore.ObjectStorageConfig{ + ObjectStorageConfig: destination.ObjectStorageConfig{ Provider: "AWS", S3Credentials: s3Creds, S3Config: s3Config, @@ -325,7 +291,7 @@ func TestSaveFileWithS3WorkhorseClient(t *testing.T) { MaximumSize: tc.maxSize, } - _, err := filestore.SaveFileFromReader(ctx, strings.NewReader(test.ObjectContent), tc.objectSize, &opts) + _, err := destination.Upload(ctx, strings.NewReader(test.ObjectContent), tc.objectSize, &opts) if tc.expectedErr == nil { require.NoError(t, err) @@ -338,7 +304,7 @@ func TestSaveFileWithS3WorkhorseClient(t *testing.T) { } } -func TestSaveFileWithAzureWorkhorseClient(t *testing.T) { +func TestUploadWithAzureWorkhorseClient(t *testing.T) { mux, bucketDir, cleanup := test.SetupGoCloudFileBucket(t, "azblob") defer cleanup() @@ -346,48 +312,48 @@ func TestSaveFileWithAzureWorkhorseClient(t *testing.T) { defer cancel() remoteObject := "tmp/test-file/1" - opts := filestore.SaveFileOpts{ + opts := destination.UploadOpts{ RemoteID: "test-file", Deadline: testDeadline(), UseWorkhorseClient: true, RemoteTempObjectID: remoteObject, - ObjectStorageConfig: filestore.ObjectStorageConfig{ + ObjectStorageConfig: destination.ObjectStorageConfig{ Provider: "AzureRM", URLMux: mux, GoCloudConfig: config.GoCloudConfig{URL: "azblob://test-container"}, }, } - _, err := filestore.SaveFileFromReader(ctx, strings.NewReader(test.ObjectContent), test.ObjectSize, &opts) + _, err := destination.Upload(ctx, strings.NewReader(test.ObjectContent), test.ObjectSize, &opts) require.NoError(t, err) test.GoCloudObjectExists(t, bucketDir, remoteObject) } -func TestSaveFileWithUnknownGoCloudScheme(t *testing.T) { +func TestUploadWithUnknownGoCloudScheme(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() mux := new(blob.URLMux) remoteObject := "tmp/test-file/1" - opts := filestore.SaveFileOpts{ + opts := destination.UploadOpts{ RemoteID: "test-file", Deadline: testDeadline(), UseWorkhorseClient: true, RemoteTempObjectID: remoteObject, - ObjectStorageConfig: filestore.ObjectStorageConfig{ + ObjectStorageConfig: destination.ObjectStorageConfig{ Provider: "SomeCloud", URLMux: mux, GoCloudConfig: config.GoCloudConfig{URL: "foo://test-container"}, }, } - _, err := filestore.SaveFileFromReader(ctx, strings.NewReader(test.ObjectContent), test.ObjectSize, &opts) + _, err := destination.Upload(ctx, strings.NewReader(test.ObjectContent), test.ObjectSize, &opts) require.Error(t, err) } -func TestSaveMultipartInBodyFailure(t *testing.T) { +func TestUploadMultipartInBodyFailure(t *testing.T) { osStub, ts := test.StartObjectStore() defer ts.Close() @@ -395,7 +361,7 @@ func TestSaveMultipartInBodyFailure(t *testing.T) { // this is the only way to get an in-body failure from our ObjectStoreStub objectPath := "/bucket-but-no-object-key" objectURL := ts.URL + objectPath - opts := filestore.SaveFileOpts{ + opts := destination.UploadOpts{ RemoteID: "test-file", RemoteURL: objectURL, PartSize: test.ObjectSize, @@ -409,13 +375,13 @@ func TestSaveMultipartInBodyFailure(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - fh, err := filestore.SaveFileFromReader(ctx, strings.NewReader(test.ObjectContent), test.ObjectSize, &opts) + fh, err := destination.Upload(ctx, strings.NewReader(test.ObjectContent), test.ObjectSize, &opts) require.Nil(t, fh) require.Error(t, err) require.EqualError(t, err, test.MultipartUploadInternalError().Error()) } -func TestSaveRemoteFileWithLimit(t *testing.T) { +func TestUploadRemoteFileWithLimit(t *testing.T) { testhelper.ConfigureSecret() type remote int @@ -449,20 +415,20 @@ func TestSaveRemoteFileWithLimit(t *testing.T) { testData: test.ObjectContent, objectSize: -1, maxSize: test.ObjectSize - 1, - expectedErr: filestore.ErrEntityTooLarge, + expectedErr: destination.ErrEntityTooLarge, }, { name: "large object with unknown size with limit", testData: string(make([]byte, 20000)), objectSize: -1, maxSize: 19000, - expectedErr: filestore.ErrEntityTooLarge, + expectedErr: destination.ErrEntityTooLarge, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - var opts filestore.SaveFileOpts + var opts destination.UploadOpts for _, remoteType := range remoteTypes { tmpFolder, err := ioutil.TempDir("", "workhorse-test-tmp") @@ -502,7 +468,7 @@ func TestSaveRemoteFileWithLimit(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - fh, err := filestore.SaveFileFromReader(ctx, strings.NewReader(tc.testData), tc.objectSize, &opts) + fh, err := destination.Upload(ctx, strings.NewReader(tc.testData), tc.objectSize, &opts) if tc.expectedErr == nil { require.NoError(t, err) @@ -516,7 +482,7 @@ func TestSaveRemoteFileWithLimit(t *testing.T) { } } -func checkFileHandlerWithFields(t *testing.T, fh *filestore.FileHandler, fields map[string]string, prefix string) { +func checkFileHandlerWithFields(t *testing.T, fh *destination.FileHandler, fields map[string]string, prefix string) { key := func(field string) string { if prefix == "" { return field diff --git a/workhorse/internal/upload/destination/filestore/filestore.go b/workhorse/internal/upload/destination/filestore/filestore.go new file mode 100644 index 00000000000..2d88874bf25 --- /dev/null +++ b/workhorse/internal/upload/destination/filestore/filestore.go @@ -0,0 +1,21 @@ +// The filestore package has a consumer specific to uploading to local disk storage. +package filestore + +import ( + "context" + "io" + "time" +) + +type LocalFile struct { + File io.WriteCloser +} + +func (lf *LocalFile) Consume(_ context.Context, r io.Reader, _ time.Time) (int64, error) { + n, err := io.Copy(lf.File, r) + errClose := lf.File.Close() + if err == nil { + err = errClose + } + return n, err +} diff --git a/workhorse/internal/upload/destination/filestore/filestore_test.go b/workhorse/internal/upload/destination/filestore/filestore_test.go new file mode 100644 index 00000000000..ec67eae96b9 --- /dev/null +++ b/workhorse/internal/upload/destination/filestore/filestore_test.go @@ -0,0 +1,38 @@ +package filestore + +import ( + "context" + "io/ioutil" + "os" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestConsume(t *testing.T) { + f, err := ioutil.TempFile("", "filestore-local-file") + if f != nil { + defer os.Remove(f.Name()) + } + require.NoError(t, err) + defer f.Close() + + localFile := &LocalFile{File: f} + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + content := "file content" + reader := strings.NewReader(content) + var deadline time.Time + + n, err := localFile.Consume(ctx, reader, deadline) + require.NoError(t, err) + require.Equal(t, int64(len(content)), n) + + consumedContent, err := ioutil.ReadFile(f.Name()) + require.NoError(t, err) + require.Equal(t, content, string(consumedContent)) +} diff --git a/workhorse/internal/filestore/multi_hash.go b/workhorse/internal/upload/destination/multi_hash.go index 40efd3a5c1f..7d4884af3dc 100644 --- a/workhorse/internal/filestore/multi_hash.go +++ b/workhorse/internal/upload/destination/multi_hash.go @@ -1,4 +1,4 @@ -package filestore +package destination import ( "crypto/md5" diff --git a/workhorse/internal/upload/destination/objectstore/doc.go b/workhorse/internal/upload/destination/objectstore/doc.go new file mode 100644 index 00000000000..00f98f230ec --- /dev/null +++ b/workhorse/internal/upload/destination/objectstore/doc.go @@ -0,0 +1,3 @@ +// The objectstore package consists of a number of consumers specific to uploading +// to object storage providers that are S3 compatible (e.g. AWS S3, GCP, Azure). +package objectstore diff --git a/workhorse/internal/objectstore/gocloud_object.go b/workhorse/internal/upload/destination/objectstore/gocloud_object.go index 38545086994..38545086994 100644 --- a/workhorse/internal/objectstore/gocloud_object.go +++ b/workhorse/internal/upload/destination/objectstore/gocloud_object.go diff --git a/workhorse/internal/objectstore/gocloud_object_test.go b/workhorse/internal/upload/destination/objectstore/gocloud_object_test.go index f320a65dbfb..57b3a35b41e 100644 --- a/workhorse/internal/objectstore/gocloud_object_test.go +++ b/workhorse/internal/upload/destination/objectstore/gocloud_object_test.go @@ -9,9 +9,9 @@ import ( "github.com/stretchr/testify/require" - "gitlab.com/gitlab-org/gitlab/workhorse/internal/objectstore" - "gitlab.com/gitlab-org/gitlab/workhorse/internal/objectstore/test" "gitlab.com/gitlab-org/gitlab/workhorse/internal/testhelper" + "gitlab.com/gitlab-org/gitlab/workhorse/internal/upload/destination/objectstore" + "gitlab.com/gitlab-org/gitlab/workhorse/internal/upload/destination/objectstore/test" ) func TestGoCloudObjectUpload(t *testing.T) { diff --git a/workhorse/internal/objectstore/multipart.go b/workhorse/internal/upload/destination/objectstore/multipart.go index 4c5b64b27ee..4c5b64b27ee 100644 --- a/workhorse/internal/objectstore/multipart.go +++ b/workhorse/internal/upload/destination/objectstore/multipart.go diff --git a/workhorse/internal/objectstore/multipart_test.go b/workhorse/internal/upload/destination/objectstore/multipart_test.go index 42ab5b4e535..4aff3467e30 100644 --- a/workhorse/internal/objectstore/multipart_test.go +++ b/workhorse/internal/upload/destination/objectstore/multipart_test.go @@ -11,8 +11,8 @@ import ( "github.com/stretchr/testify/require" - "gitlab.com/gitlab-org/gitlab/workhorse/internal/objectstore" - "gitlab.com/gitlab-org/gitlab/workhorse/internal/objectstore/test" + "gitlab.com/gitlab-org/gitlab/workhorse/internal/upload/destination/objectstore" + "gitlab.com/gitlab-org/gitlab/workhorse/internal/upload/destination/objectstore/test" ) func TestMultipartUploadWithUpcaseETags(t *testing.T) { diff --git a/workhorse/internal/objectstore/object.go b/workhorse/internal/upload/destination/objectstore/object.go index b7c4f12f009..b7c4f12f009 100644 --- a/workhorse/internal/objectstore/object.go +++ b/workhorse/internal/upload/destination/objectstore/object.go diff --git a/workhorse/internal/objectstore/object_test.go b/workhorse/internal/upload/destination/objectstore/object_test.go index b9c1fb2087b..24117891b6d 100644 --- a/workhorse/internal/objectstore/object_test.go +++ b/workhorse/internal/upload/destination/objectstore/object_test.go @@ -11,8 +11,8 @@ import ( "github.com/stretchr/testify/require" - "gitlab.com/gitlab-org/gitlab/workhorse/internal/objectstore" - "gitlab.com/gitlab-org/gitlab/workhorse/internal/objectstore/test" + "gitlab.com/gitlab-org/gitlab/workhorse/internal/upload/destination/objectstore" + "gitlab.com/gitlab-org/gitlab/workhorse/internal/upload/destination/objectstore/test" ) const testTimeout = 10 * time.Second diff --git a/workhorse/internal/objectstore/prometheus.go b/workhorse/internal/upload/destination/objectstore/prometheus.go index 20762fb52bc..20762fb52bc 100644 --- a/workhorse/internal/objectstore/prometheus.go +++ b/workhorse/internal/upload/destination/objectstore/prometheus.go diff --git a/workhorse/internal/objectstore/s3_complete_multipart_api.go b/workhorse/internal/upload/destination/objectstore/s3_complete_multipart_api.go index b84f5757f49..b84f5757f49 100644 --- a/workhorse/internal/objectstore/s3_complete_multipart_api.go +++ b/workhorse/internal/upload/destination/objectstore/s3_complete_multipart_api.go diff --git a/workhorse/internal/objectstore/s3_object.go b/workhorse/internal/upload/destination/objectstore/s3_object.go index ce0cccc7eb1..ce0cccc7eb1 100644 --- a/workhorse/internal/objectstore/s3_object.go +++ b/workhorse/internal/upload/destination/objectstore/s3_object.go diff --git a/workhorse/internal/objectstore/s3_object_test.go b/workhorse/internal/upload/destination/objectstore/s3_object_test.go index c7426e3843b..b81b0ae2024 100644 --- a/workhorse/internal/objectstore/s3_object_test.go +++ b/workhorse/internal/upload/destination/objectstore/s3_object_test.go @@ -18,9 +18,9 @@ import ( "github.com/stretchr/testify/require" "gitlab.com/gitlab-org/gitlab/workhorse/internal/config" - "gitlab.com/gitlab-org/gitlab/workhorse/internal/objectstore" - "gitlab.com/gitlab-org/gitlab/workhorse/internal/objectstore/test" "gitlab.com/gitlab-org/gitlab/workhorse/internal/testhelper" + "gitlab.com/gitlab-org/gitlab/workhorse/internal/upload/destination/objectstore" + "gitlab.com/gitlab-org/gitlab/workhorse/internal/upload/destination/objectstore/test" ) type failedReader struct { diff --git a/workhorse/internal/objectstore/s3_session.go b/workhorse/internal/upload/destination/objectstore/s3_session.go index a0c1f099145..a0c1f099145 100644 --- a/workhorse/internal/objectstore/s3_session.go +++ b/workhorse/internal/upload/destination/objectstore/s3_session.go diff --git a/workhorse/internal/objectstore/s3_session_test.go b/workhorse/internal/upload/destination/objectstore/s3_session_test.go index 5d57b4f9af8..5d57b4f9af8 100644 --- a/workhorse/internal/objectstore/s3_session_test.go +++ b/workhorse/internal/upload/destination/objectstore/s3_session_test.go diff --git a/workhorse/internal/objectstore/test/consts.go b/workhorse/internal/upload/destination/objectstore/test/consts.go index 7a1bcc28d45..7a1bcc28d45 100644 --- a/workhorse/internal/objectstore/test/consts.go +++ b/workhorse/internal/upload/destination/objectstore/test/consts.go diff --git a/workhorse/internal/objectstore/test/gocloud_stub.go b/workhorse/internal/upload/destination/objectstore/test/gocloud_stub.go index cf22075e407..cf22075e407 100644 --- a/workhorse/internal/objectstore/test/gocloud_stub.go +++ b/workhorse/internal/upload/destination/objectstore/test/gocloud_stub.go diff --git a/workhorse/internal/objectstore/test/objectstore_stub.go b/workhorse/internal/upload/destination/objectstore/test/objectstore_stub.go index ec5e271b759..d51a2de7456 100644 --- a/workhorse/internal/objectstore/test/objectstore_stub.go +++ b/workhorse/internal/upload/destination/objectstore/test/objectstore_stub.go @@ -13,7 +13,7 @@ import ( "strings" "sync" - "gitlab.com/gitlab-org/gitlab/workhorse/internal/objectstore" + "gitlab.com/gitlab-org/gitlab/workhorse/internal/upload/destination/objectstore" ) type partsEtagMap map[int]string diff --git a/workhorse/internal/objectstore/test/objectstore_stub_test.go b/workhorse/internal/upload/destination/objectstore/test/objectstore_stub_test.go index 8c0d52a2d79..8c0d52a2d79 100644 --- a/workhorse/internal/objectstore/test/objectstore_stub_test.go +++ b/workhorse/internal/upload/destination/objectstore/test/objectstore_stub_test.go diff --git a/workhorse/internal/objectstore/test/s3_stub.go b/workhorse/internal/upload/destination/objectstore/test/s3_stub.go index 6b83426b852..6b83426b852 100644 --- a/workhorse/internal/objectstore/test/s3_stub.go +++ b/workhorse/internal/upload/destination/objectstore/test/s3_stub.go diff --git a/workhorse/internal/objectstore/upload_strategy.go b/workhorse/internal/upload/destination/objectstore/upload_strategy.go index 5707ba5f24e..5707ba5f24e 100644 --- a/workhorse/internal/objectstore/upload_strategy.go +++ b/workhorse/internal/upload/destination/objectstore/upload_strategy.go diff --git a/workhorse/internal/objectstore/uploader.go b/workhorse/internal/upload/destination/objectstore/uploader.go index aedfbe55ead..aedfbe55ead 100644 --- a/workhorse/internal/objectstore/uploader.go +++ b/workhorse/internal/upload/destination/objectstore/uploader.go diff --git a/workhorse/internal/filestore/reader.go b/workhorse/internal/upload/destination/reader.go index b1045b991fc..925a9468e14 100644 --- a/workhorse/internal/filestore/reader.go +++ b/workhorse/internal/upload/destination/reader.go @@ -1,4 +1,4 @@ -package filestore +package destination import "io" diff --git a/workhorse/internal/filestore/reader_test.go b/workhorse/internal/upload/destination/reader_test.go index 424d921ecaf..a26f7746a13 100644 --- a/workhorse/internal/filestore/reader_test.go +++ b/workhorse/internal/upload/destination/reader_test.go @@ -1,4 +1,4 @@ -package filestore +package destination import ( "fmt" diff --git a/workhorse/internal/filestore/save_file_opts.go b/workhorse/internal/upload/destination/upload_opts.go index 544101d693a..750a79d7bc2 100644 --- a/workhorse/internal/filestore/save_file_opts.go +++ b/workhorse/internal/upload/destination/upload_opts.go @@ -1,4 +1,4 @@ -package filestore +package destination import ( "errors" @@ -27,8 +27,8 @@ type ObjectStorageConfig struct { GoCloudConfig config.GoCloudConfig } -// SaveFileOpts represents all the options available for saving a file to object store -type SaveFileOpts struct { +// UploadOpts represents all the options available for saving a file to object store +type UploadOpts struct { // TempFilePrefix is the prefix used to create temporary local file TempFilePrefix string // LocalTempPath is the directory where to write a local copy of the file @@ -66,28 +66,28 @@ type SaveFileOpts struct { } // UseWorkhorseClientEnabled checks if the options require direct access to object storage -func (s *SaveFileOpts) UseWorkhorseClientEnabled() bool { +func (s *UploadOpts) UseWorkhorseClientEnabled() bool { return s.UseWorkhorseClient && s.ObjectStorageConfig.IsValid() && s.RemoteTempObjectID != "" } // IsLocal checks if the options require the writing of the file on disk -func (s *SaveFileOpts) IsLocal() bool { +func (s *UploadOpts) IsLocal() bool { return s.LocalTempPath != "" } // IsMultipart checks if the options requires a Multipart upload -func (s *SaveFileOpts) IsMultipart() bool { +func (s *UploadOpts) IsMultipart() bool { return s.PartSize > 0 } -// GetOpts converts GitLab api.Response to a proper SaveFileOpts -func GetOpts(apiResponse *api.Response) (*SaveFileOpts, error) { +// GetOpts converts GitLab api.Response to a proper UploadOpts +func GetOpts(apiResponse *api.Response) (*UploadOpts, error) { timeout := time.Duration(apiResponse.RemoteObject.Timeout) * time.Second if timeout == 0 { timeout = DefaultObjectStoreTimeout } - opts := SaveFileOpts{ + opts := UploadOpts{ LocalTempPath: apiResponse.TempPath, RemoteID: apiResponse.RemoteObject.ID, RemoteURL: apiResponse.RemoteObject.GetURL, diff --git a/workhorse/internal/filestore/save_file_opts_test.go b/workhorse/internal/upload/destination/upload_opts_test.go index f389b2054e5..fde726c985d 100644 --- a/workhorse/internal/filestore/save_file_opts_test.go +++ b/workhorse/internal/upload/destination/upload_opts_test.go @@ -1,4 +1,4 @@ -package filestore_test +package destination_test import ( "testing" @@ -8,11 +8,11 @@ import ( "gitlab.com/gitlab-org/gitlab/workhorse/internal/api" "gitlab.com/gitlab-org/gitlab/workhorse/internal/config" - "gitlab.com/gitlab-org/gitlab/workhorse/internal/filestore" - "gitlab.com/gitlab-org/gitlab/workhorse/internal/objectstore/test" + "gitlab.com/gitlab-org/gitlab/workhorse/internal/upload/destination" + "gitlab.com/gitlab-org/gitlab/workhorse/internal/upload/destination/objectstore/test" ) -func TestSaveFileOptsLocalAndRemote(t *testing.T) { +func TestUploadOptsLocalAndRemote(t *testing.T) { tests := []struct { name string localTempPath string @@ -43,7 +43,7 @@ func TestSaveFileOptsLocalAndRemote(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - opts := filestore.SaveFileOpts{ + opts := destination.UploadOpts{ LocalTempPath: test.localTempPath, PresignedPut: test.presignedPut, PartSize: test.partSize, @@ -106,7 +106,7 @@ func TestGetOpts(t *testing.T) { }, } deadline := time.Now().Add(time.Duration(apiResponse.RemoteObject.Timeout) * time.Second) - opts, err := filestore.GetOpts(apiResponse) + opts, err := destination.GetOpts(apiResponse) require.NoError(t, err) require.Equal(t, apiResponse.TempPath, opts.LocalTempPath) @@ -155,22 +155,22 @@ func TestGetOptsFail(t *testing.T) { for _, tc := range testCases { t.Run(tc.desc, func(t *testing.T) { - _, err := filestore.GetOpts(tc.in) + _, err := destination.GetOpts(tc.in) require.Error(t, err, "expect input to be rejected") }) } } func TestGetOptsDefaultTimeout(t *testing.T) { - deadline := time.Now().Add(filestore.DefaultObjectStoreTimeout) - opts, err := filestore.GetOpts(&api.Response{TempPath: "/foo/bar"}) + deadline := time.Now().Add(destination.DefaultObjectStoreTimeout) + opts, err := destination.GetOpts(&api.Response{TempPath: "/foo/bar"}) require.NoError(t, err) require.WithinDuration(t, deadline, opts.Deadline, time.Minute) } func TestUseWorkhorseClientEnabled(t *testing.T) { - cfg := filestore.ObjectStorageConfig{ + cfg := destination.ObjectStorageConfig{ Provider: "AWS", S3Config: config.S3Config{ Bucket: "test-bucket", @@ -195,7 +195,7 @@ func TestUseWorkhorseClientEnabled(t *testing.T) { name string UseWorkhorseClient bool remoteTempObjectID string - objectStorageConfig filestore.ObjectStorageConfig + objectStorageConfig destination.ObjectStorageConfig expected bool }{ { @@ -243,7 +243,7 @@ func TestUseWorkhorseClientEnabled(t *testing.T) { name: "missing S3 bucket", UseWorkhorseClient: true, remoteTempObjectID: "test-object", - objectStorageConfig: filestore.ObjectStorageConfig{ + objectStorageConfig: destination.ObjectStorageConfig{ Provider: "AWS", S3Config: config.S3Config{}, }, @@ -269,7 +269,7 @@ func TestUseWorkhorseClientEnabled(t *testing.T) { }, } deadline := time.Now().Add(time.Duration(apiResponse.RemoteObject.Timeout) * time.Second) - opts, err := filestore.GetOpts(apiResponse) + opts, err := destination.GetOpts(apiResponse) require.NoError(t, err) opts.ObjectStorageConfig = test.objectStorageConfig @@ -323,7 +323,7 @@ func TestGoCloudConfig(t *testing.T) { }, } deadline := time.Now().Add(time.Duration(apiResponse.RemoteObject.Timeout) * time.Second) - opts, err := filestore.GetOpts(apiResponse) + opts, err := destination.GetOpts(apiResponse) require.NoError(t, err) opts.ObjectStorageConfig.URLMux = mux diff --git a/workhorse/internal/lfs/lfs.go b/workhorse/internal/upload/lfs_preparer.go index e26f59046ea..e7c5cf16a30 100644 --- a/workhorse/internal/lfs/lfs.go +++ b/workhorse/internal/upload/lfs_preparer.go @@ -1,16 +1,11 @@ -/* -In this file we handle git lfs objects downloads and uploads -*/ - -package lfs +package upload import ( "fmt" "gitlab.com/gitlab-org/gitlab/workhorse/internal/api" "gitlab.com/gitlab-org/gitlab/workhorse/internal/config" - "gitlab.com/gitlab-org/gitlab/workhorse/internal/filestore" - "gitlab.com/gitlab-org/gitlab/workhorse/internal/upload" + "gitlab.com/gitlab-org/gitlab/workhorse/internal/upload/destination" ) type object struct { @@ -18,7 +13,7 @@ type object struct { oid string } -func (l *object) Verify(fh *filestore.FileHandler) error { +func (l *object) Verify(fh *destination.FileHandler) error { if fh.Size != l.size { return fmt.Errorf("LFSObject: expected size %d, wrote %d", l.size, fh.Size) } @@ -31,14 +26,16 @@ func (l *object) Verify(fh *filestore.FileHandler) error { } type uploadPreparer struct { - objectPreparer upload.Preparer + objectPreparer Preparer } -func NewLfsUploadPreparer(c config.Config, objectPreparer upload.Preparer) upload.Preparer { +// NewLfs returns a new preparer instance which adds capability to a wrapped +// preparer to set options required for a LFS upload. +func NewLfsPreparer(c config.Config, objectPreparer Preparer) Preparer { return &uploadPreparer{objectPreparer: objectPreparer} } -func (l *uploadPreparer) Prepare(a *api.Response) (*filestore.SaveFileOpts, upload.Verifier, error) { +func (l *uploadPreparer) Prepare(a *api.Response) (*destination.UploadOpts, Verifier, error) { opts, _, err := l.objectPreparer.Prepare(a) if err != nil { return nil, nil, err diff --git a/workhorse/internal/lfs/lfs_test.go b/workhorse/internal/upload/lfs_preparer_test.go index 63b2628343e..6be4a7c2955 100644 --- a/workhorse/internal/lfs/lfs_test.go +++ b/workhorse/internal/upload/lfs_preparer_test.go @@ -1,17 +1,15 @@ -package lfs_test +package upload import ( "testing" "gitlab.com/gitlab-org/gitlab/workhorse/internal/api" "gitlab.com/gitlab-org/gitlab/workhorse/internal/config" - "gitlab.com/gitlab-org/gitlab/workhorse/internal/lfs" - "gitlab.com/gitlab-org/gitlab/workhorse/internal/upload" "github.com/stretchr/testify/require" ) -func TestLfsUploadPreparerWithConfig(t *testing.T) { +func TestLfsPreparerWithConfig(t *testing.T) { lfsOid := "abcd1234" creds := config.S3Credentials{ AwsAccessKeyID: "test-key", @@ -36,8 +34,8 @@ func TestLfsUploadPreparerWithConfig(t *testing.T) { }, } - uploadPreparer := upload.NewObjectStoragePreparer(c) - lfsPreparer := lfs.NewLfsUploadPreparer(c, uploadPreparer) + uploadPreparer := NewObjectStoragePreparer(c) + lfsPreparer := NewLfsPreparer(c, uploadPreparer) opts, verifier, err := lfsPreparer.Prepare(r) require.NoError(t, err) @@ -48,11 +46,11 @@ func TestLfsUploadPreparerWithConfig(t *testing.T) { require.NotNil(t, verifier) } -func TestLfsUploadPreparerWithNoConfig(t *testing.T) { +func TestLfsPreparerWithNoConfig(t *testing.T) { c := config.Config{} r := &api.Response{RemoteObject: api.RemoteObject{ID: "the upload ID"}} - uploadPreparer := upload.NewObjectStoragePreparer(c) - lfsPreparer := lfs.NewLfsUploadPreparer(c, uploadPreparer) + uploadPreparer := NewObjectStoragePreparer(c) + lfsPreparer := NewLfsPreparer(c, uploadPreparer) opts, verifier, err := lfsPreparer.Prepare(r) require.NoError(t, err) diff --git a/workhorse/internal/upload/accelerate.go b/workhorse/internal/upload/multipart_uploader.go index 28d3b3dee2e..d0097f9e153 100644 --- a/workhorse/internal/upload/accelerate.go +++ b/workhorse/internal/upload/multipart_uploader.go @@ -4,19 +4,10 @@ import ( "fmt" "net/http" - "github.com/golang-jwt/jwt/v4" - "gitlab.com/gitlab-org/gitlab/workhorse/internal/api" "gitlab.com/gitlab-org/gitlab/workhorse/internal/helper" ) -const RewrittenFieldsHeader = "Gitlab-Workhorse-Multipart-Fields" - -type MultipartClaims struct { - RewrittenFields map[string]string `json:"rewritten_fields"` - jwt.StandardClaims -} - // Multipart is a request middleware. If the request has a MIME multipart // request body, the middleware will iterate through the multipart parts. // When it finds a file part (filename != ""), the middleware will save @@ -32,6 +23,6 @@ func Multipart(rails PreAuthorizer, h http.Handler, p Preparer) http.Handler { return } - InterceptMultipartFiles(w, r, h, a, s, opts) + interceptMultipartFiles(w, r, h, a, s, opts) }, "/authorize") } diff --git a/workhorse/internal/upload/object_storage_preparer.go b/workhorse/internal/upload/object_storage_preparer.go index 241cfc2d9d2..f28f598c895 100644 --- a/workhorse/internal/upload/object_storage_preparer.go +++ b/workhorse/internal/upload/object_storage_preparer.go @@ -3,7 +3,7 @@ package upload import ( "gitlab.com/gitlab-org/gitlab/workhorse/internal/api" "gitlab.com/gitlab-org/gitlab/workhorse/internal/config" - "gitlab.com/gitlab-org/gitlab/workhorse/internal/filestore" + "gitlab.com/gitlab-org/gitlab/workhorse/internal/upload/destination" ) type ObjectStoragePreparer struct { @@ -11,12 +11,15 @@ type ObjectStoragePreparer struct { credentials config.ObjectStorageCredentials } +// NewObjectStoragePreparer returns a new preparer instance which is responsible for +// setting the object storage credentials and settings needed by an uploader +// to upload to object storage. func NewObjectStoragePreparer(c config.Config) Preparer { return &ObjectStoragePreparer{credentials: c.ObjectStorageCredentials, config: c.ObjectStorageConfig} } -func (p *ObjectStoragePreparer) Prepare(a *api.Response) (*filestore.SaveFileOpts, Verifier, error) { - opts, err := filestore.GetOpts(a) +func (p *ObjectStoragePreparer) Prepare(a *api.Response) (*destination.UploadOpts, Verifier, error) { + opts, err := destination.GetOpts(a) if err != nil { return nil, nil, err } diff --git a/workhorse/internal/upload/preparer.go b/workhorse/internal/upload/preparer.go new file mode 100644 index 00000000000..46a4cac01b5 --- /dev/null +++ b/workhorse/internal/upload/preparer.go @@ -0,0 +1,33 @@ +package upload + +import ( + "gitlab.com/gitlab-org/gitlab/workhorse/internal/api" + "gitlab.com/gitlab-org/gitlab/workhorse/internal/upload/destination" +) + +// Verifier is an optional pluggable behavior for upload paths. If +// Verify() returns an error, Workhorse will return an error response to +// the client instead of propagating the request to Rails. The motivating +// use case is Git LFS, where Workhorse checks the size and SHA256 +// checksum of the uploaded file. +type Verifier interface { + // Verify can abort the upload by returning an error + Verify(handler *destination.FileHandler) error +} + +// Preparer is a pluggable behavior that interprets a Rails API response +// and either tells Workhorse how to handle the upload, via the +// UploadOpts and Verifier, or it rejects the request by returning a +// non-nil error. Its intended use is to make sure the upload gets stored +// in the right location: either a local directory, or one of several +// supported object storage backends. +type Preparer interface { + Prepare(a *api.Response) (*destination.UploadOpts, Verifier, error) +} + +type DefaultPreparer struct{} + +func (s *DefaultPreparer) Prepare(a *api.Response) (*destination.UploadOpts, Verifier, error) { + opts, err := destination.GetOpts(a) + return opts, nil, err +} diff --git a/workhorse/internal/upload/rewrite.go b/workhorse/internal/upload/rewrite.go index bbabe840ef5..ff5190226af 100644 --- a/workhorse/internal/upload/rewrite.go +++ b/workhorse/internal/upload/rewrite.go @@ -21,8 +21,8 @@ import ( "golang.org/x/image/tiff" "gitlab.com/gitlab-org/gitlab/workhorse/internal/api" - "gitlab.com/gitlab-org/gitlab/workhorse/internal/filestore" "gitlab.com/gitlab-org/gitlab/workhorse/internal/lsif_transformer/parser" + "gitlab.com/gitlab-org/gitlab/workhorse/internal/upload/destination" "gitlab.com/gitlab-org/gitlab/workhorse/internal/upload/exif" ) @@ -68,7 +68,7 @@ type rewriter struct { finalizedFields map[string]bool } -func rewriteFormFilesFromMultipart(r *http.Request, writer *multipart.Writer, preauth *api.Response, filter MultipartFormProcessor, opts *filestore.SaveFileOpts) error { +func rewriteFormFilesFromMultipart(r *http.Request, writer *multipart.Writer, preauth *api.Response, filter MultipartFormProcessor, opts *destination.UploadOpts) error { // Create multipart reader reader, err := r.MultipartReader() if err != nil { @@ -128,7 +128,7 @@ func parseAndNormalizeContentDisposition(header textproto.MIMEHeader) (string, s return params["name"], params["filename"] } -func (rew *rewriter) handleFilePart(ctx context.Context, name string, p *multipart.Part, opts *filestore.SaveFileOpts) error { +func (rew *rewriter) handleFilePart(ctx context.Context, name string, p *multipart.Part, opts *destination.UploadOpts) error { if rew.filter.Count() >= maxFilesAllowed { return ErrTooManyFilesUploaded } @@ -164,10 +164,10 @@ func (rew *rewriter) handleFilePart(ctx context.Context, name string, p *multipa defer inputReader.Close() - fh, err := filestore.SaveFileFromReader(ctx, inputReader, -1, opts) + fh, err := destination.Upload(ctx, inputReader, -1, opts) if err != nil { switch err { - case filestore.ErrEntityTooLarge, exif.ErrRemovingExif: + case destination.ErrEntityTooLarge, exif.ErrRemovingExif: return err default: return fmt.Errorf("persisting multipart file: %v", err) diff --git a/workhorse/internal/upload/saved_file_tracker.go b/workhorse/internal/upload/saved_file_tracker.go index e6f9a8c9a88..b70a303a4a4 100644 --- a/workhorse/internal/upload/saved_file_tracker.go +++ b/workhorse/internal/upload/saved_file_tracker.go @@ -6,8 +6,8 @@ import ( "mime/multipart" "net/http" - "gitlab.com/gitlab-org/gitlab/workhorse/internal/filestore" "gitlab.com/gitlab-org/gitlab/workhorse/internal/secret" + "gitlab.com/gitlab-org/gitlab/workhorse/internal/upload/destination" ) type SavedFileTracker struct { @@ -26,7 +26,7 @@ func (s *SavedFileTracker) Count() int { return len(s.rewrittenFields) } -func (s *SavedFileTracker) ProcessFile(_ context.Context, fieldName string, file *filestore.FileHandler, _ *multipart.Writer) error { +func (s *SavedFileTracker) ProcessFile(_ context.Context, fieldName string, file *destination.FileHandler, _ *multipart.Writer) error { if _, ok := s.rewrittenFields[fieldName]; ok { return fmt.Errorf("the %v field has already been processed", fieldName) } diff --git a/workhorse/internal/upload/saved_file_tracker_test.go b/workhorse/internal/upload/saved_file_tracker_test.go index ba927db253e..4f323bf8888 100644 --- a/workhorse/internal/upload/saved_file_tracker_test.go +++ b/workhorse/internal/upload/saved_file_tracker_test.go @@ -10,8 +10,8 @@ import ( "github.com/stretchr/testify/require" - "gitlab.com/gitlab-org/gitlab/workhorse/internal/filestore" "gitlab.com/gitlab-org/gitlab/workhorse/internal/testhelper" + "gitlab.com/gitlab-org/gitlab/workhorse/internal/upload/destination" ) func TestSavedFileTracking(t *testing.T) { @@ -23,7 +23,7 @@ func TestSavedFileTracking(t *testing.T) { tracker := SavedFileTracker{Request: r} require.Equal(t, "accelerate", tracker.Name()) - file := &filestore.FileHandler{} + file := &destination.FileHandler{} ctx := context.Background() tracker.ProcessFile(ctx, "test", file, nil) require.Equal(t, 1, tracker.Count()) @@ -40,7 +40,7 @@ func TestSavedFileTracking(t *testing.T) { func TestDuplicatedFileProcessing(t *testing.T) { tracker := SavedFileTracker{} - file := &filestore.FileHandler{} + file := &destination.FileHandler{} require.NoError(t, tracker.ProcessFile(context.Background(), "file", file, nil)) diff --git a/workhorse/internal/upload/uploads.go b/workhorse/internal/upload/uploads.go index 1806e7563ce..8272a3d920d 100644 --- a/workhorse/internal/upload/uploads.go +++ b/workhorse/internal/upload/uploads.go @@ -8,27 +8,39 @@ import ( "mime/multipart" "net/http" + "github.com/golang-jwt/jwt/v4" + "gitlab.com/gitlab-org/gitlab/workhorse/internal/api" - "gitlab.com/gitlab-org/gitlab/workhorse/internal/filestore" "gitlab.com/gitlab-org/gitlab/workhorse/internal/helper" + "gitlab.com/gitlab-org/gitlab/workhorse/internal/upload/destination" "gitlab.com/gitlab-org/gitlab/workhorse/internal/upload/exif" "gitlab.com/gitlab-org/gitlab/workhorse/internal/zipartifacts" ) +const RewrittenFieldsHeader = "Gitlab-Workhorse-Multipart-Fields" + +type PreAuthorizer interface { + PreAuthorizeHandler(next api.HandleFunc, suffix string) http.Handler +} + +type MultipartClaims struct { + RewrittenFields map[string]string `json:"rewritten_fields"` + jwt.StandardClaims +} + // MultipartFormProcessor abstracts away implementation differences // between generic MIME multipart file uploads and CI artifact uploads. type MultipartFormProcessor interface { - ProcessFile(ctx context.Context, formName string, file *filestore.FileHandler, writer *multipart.Writer) error + ProcessFile(ctx context.Context, formName string, file *destination.FileHandler, writer *multipart.Writer) error ProcessField(ctx context.Context, formName string, writer *multipart.Writer) error Finalize(ctx context.Context) error Name() string Count() int } -// InterceptMultipartFiles is the core of the implementation of -// Multipart. Because it is also used for CI artifact uploads it is a -// public function. -func InterceptMultipartFiles(w http.ResponseWriter, r *http.Request, h http.Handler, preauth *api.Response, filter MultipartFormProcessor, opts *filestore.SaveFileOpts) { +// interceptMultipartFiles is the core of the implementation of +// Multipart. +func interceptMultipartFiles(w http.ResponseWriter, r *http.Request, h http.Handler, preauth *api.Response, filter MultipartFormProcessor, opts *destination.UploadOpts) { var body bytes.Buffer writer := multipart.NewWriter(&body) defer writer.Close() @@ -43,7 +55,7 @@ func InterceptMultipartFiles(w http.ResponseWriter, r *http.Request, h http.Hand helper.CaptureAndFail(w, r, err, err.Error(), http.StatusBadRequest) case http.ErrNotMultipart: h.ServeHTTP(w, r) - case filestore.ErrEntityTooLarge: + case destination.ErrEntityTooLarge: helper.RequestEntityTooLarge(w, r, err) case zipartifacts.ErrBadMetadata: helper.RequestEntityTooLarge(w, r, err) diff --git a/workhorse/internal/upload/uploads_test.go b/workhorse/internal/upload/uploads_test.go index f262bf94b08..9d787b10d1c 100644 --- a/workhorse/internal/upload/uploads_test.go +++ b/workhorse/internal/upload/uploads_test.go @@ -22,9 +22,9 @@ import ( "gitlab.com/gitlab-org/gitlab/workhorse/internal/api" "gitlab.com/gitlab-org/gitlab/workhorse/internal/helper" - "gitlab.com/gitlab-org/gitlab/workhorse/internal/objectstore/test" "gitlab.com/gitlab-org/gitlab/workhorse/internal/proxy" "gitlab.com/gitlab-org/gitlab/workhorse/internal/testhelper" + "gitlab.com/gitlab-org/gitlab/workhorse/internal/upload/destination/objectstore/test" "gitlab.com/gitlab-org/gitlab/workhorse/internal/upstream/roundtripper" ) @@ -78,7 +78,7 @@ func TestUploadHandlerForwardingRawData(t *testing.T) { opts, _, err := preparer.Prepare(apiResponse) require.NoError(t, err) - InterceptMultipartFiles(response, httpRequest, handler, apiResponse, nil, opts) + interceptMultipartFiles(response, httpRequest, handler, apiResponse, nil, opts) require.Equal(t, 202, response.Code) require.Equal(t, "RESPONSE", response.Body.String(), "response body") @@ -149,7 +149,7 @@ func TestUploadHandlerRewritingMultiPartData(t *testing.T) { opts, _, err := preparer.Prepare(apiResponse) require.NoError(t, err) - InterceptMultipartFiles(response, httpRequest, handler, apiResponse, &testFormProcessor{}, opts) + interceptMultipartFiles(response, httpRequest, handler, apiResponse, &testFormProcessor{}, opts) require.Equal(t, 202, response.Code) cancel() // this will trigger an async cleanup @@ -218,7 +218,7 @@ func TestUploadHandlerDetectingInjectedMultiPartData(t *testing.T) { opts, _, err := preparer.Prepare(apiResponse) require.NoError(t, err) - InterceptMultipartFiles(response, httpRequest, handler, apiResponse, &testFormProcessor{}, opts) + interceptMultipartFiles(response, httpRequest, handler, apiResponse, &testFormProcessor{}, opts) require.Equal(t, test.response, response.Code) cancel() // this will trigger an async cleanup @@ -248,7 +248,7 @@ func TestUploadProcessingField(t *testing.T) { opts, _, err := preparer.Prepare(apiResponse) require.NoError(t, err) - InterceptMultipartFiles(response, httpRequest, nilHandler, apiResponse, &testFormProcessor{}, opts) + interceptMultipartFiles(response, httpRequest, nilHandler, apiResponse, &testFormProcessor{}, opts) require.Equal(t, 500, response.Code) } @@ -279,7 +279,7 @@ func TestUploadingMultipleFiles(t *testing.T) { opts, _, err := preparer.Prepare(apiResponse) require.NoError(t, err) - InterceptMultipartFiles(response, httpRequest, nilHandler, apiResponse, &testFormProcessor{}, opts) + interceptMultipartFiles(response, httpRequest, nilHandler, apiResponse, &testFormProcessor{}, opts) require.Equal(t, 400, response.Code) require.Equal(t, "upload request contains more than 10 files\n", response.Body.String()) @@ -335,7 +335,7 @@ func TestUploadProcessingFile(t *testing.T) { opts, _, err := preparer.Prepare(apiResponse) require.NoError(t, err) - InterceptMultipartFiles(response, httpRequest, nilHandler, apiResponse, &testFormProcessor{}, opts) + interceptMultipartFiles(response, httpRequest, nilHandler, apiResponse, &testFormProcessor{}, opts) require.Equal(t, 200, response.Code) }) @@ -381,7 +381,7 @@ func TestInvalidFileNames(t *testing.T) { opts, _, err := preparer.Prepare(apiResponse) require.NoError(t, err) - InterceptMultipartFiles(response, httpRequest, nilHandler, apiResponse, &SavedFileTracker{Request: httpRequest}, opts) + interceptMultipartFiles(response, httpRequest, nilHandler, apiResponse, &SavedFileTracker{Request: httpRequest}, opts) require.Equal(t, testCase.code, response.Code) require.Equal(t, testCase.expectedPrefix, opts.TempFilePrefix) } @@ -447,7 +447,7 @@ func TestContentDispositionRewrite(t *testing.T) { opts, _, err := preparer.Prepare(apiResponse) require.NoError(t, err) - InterceptMultipartFiles(response, httpRequest, customHandler, apiResponse, &SavedFileTracker{Request: httpRequest}, opts) + interceptMultipartFiles(response, httpRequest, customHandler, apiResponse, &SavedFileTracker{Request: httpRequest}, opts) upstreamRequest, err := http.ReadRequest(bufio.NewReader(&upstreamRequestBuffer)) require.NoError(t, err) @@ -570,7 +570,7 @@ func runUploadTest(t *testing.T, image []byte, filename string, httpCode int, ts opts, _, err := preparer.Prepare(apiResponse) require.NoError(t, err) - InterceptMultipartFiles(response, httpRequest, handler, apiResponse, &testFormProcessor{}, opts) + interceptMultipartFiles(response, httpRequest, handler, apiResponse, &testFormProcessor{}, opts) require.Equal(t, httpCode, response.Code) } diff --git a/workhorse/internal/upstream/.gitignore b/workhorse/internal/upstream/.gitignore new file mode 100644 index 00000000000..d63cd8b2c40 --- /dev/null +++ b/workhorse/internal/upstream/.gitignore @@ -0,0 +1 @@ +testdata/public diff --git a/workhorse/internal/upstream/metrics.go b/workhorse/internal/upstream/metrics.go index 1a11bdc8b53..27cc6bb045b 100644 --- a/workhorse/internal/upstream/metrics.go +++ b/workhorse/internal/upstream/metrics.go @@ -6,6 +6,8 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/client_golang/prometheus/promhttp" + + "gitlab.com/gitlab-org/labkit/metrics" ) const ( @@ -13,95 +15,7 @@ const ( httpSubsystem = "http" ) -func secondsDurationBuckets() []float64 { - return []float64{ - 0.005, /* 5ms */ - 0.025, /* 25ms */ - 0.1, /* 100ms */ - 0.5, /* 500ms */ - 1.0, /* 1s */ - 10.0, /* 10s */ - 30.0, /* 30s */ - 60.0, /* 1m */ - 300.0, /* 10m */ - } -} - -func byteSizeBuckets() []float64 { - return []float64{ - 10, - 64, - 256, - 1024, /* 1kB */ - 64 * 1024, /* 64kB */ - 256 * 1024, /* 256kB */ - 1024 * 1024, /* 1mB */ - 64 * 1024 * 1024, /* 64mB */ - } -} - var ( - httpInFlightRequests = promauto.NewGauge(prometheus.GaugeOpts{ - Namespace: namespace, - Subsystem: httpSubsystem, - Name: "in_flight_requests", - Help: "A gauge of requests currently being served by workhorse.", - }) - - httpRequestsTotal = promauto.NewCounterVec( - prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: httpSubsystem, - Name: "requests_total", - Help: "A counter for requests to workhorse.", - }, - []string{"code", "method", "route"}, - ) - - httpRequestDurationSeconds = promauto.NewHistogramVec( - prometheus.HistogramOpts{ - Namespace: namespace, - Subsystem: httpSubsystem, - Name: "request_duration_seconds", - Help: "A histogram of latencies for requests to workhorse.", - Buckets: secondsDurationBuckets(), - }, - []string{"code", "method", "route"}, - ) - - httpRequestSizeBytes = promauto.NewHistogramVec( - prometheus.HistogramOpts{ - Namespace: namespace, - Subsystem: httpSubsystem, - Name: "request_size_bytes", - Help: "A histogram of sizes of requests to workhorse.", - Buckets: byteSizeBuckets(), - }, - []string{"code", "method", "route"}, - ) - - httpResponseSizeBytes = promauto.NewHistogramVec( - prometheus.HistogramOpts{ - Namespace: namespace, - Subsystem: httpSubsystem, - Name: "response_size_bytes", - Help: "A histogram of response sizes for requests to workhorse.", - Buckets: byteSizeBuckets(), - }, - []string{"code", "method", "route"}, - ) - - httpTimeToWriteHeaderSeconds = promauto.NewHistogramVec( - prometheus.HistogramOpts{ - Namespace: namespace, - Subsystem: httpSubsystem, - Name: "time_to_write_header_seconds", - Help: "A histogram of request durations until the response headers are written.", - Buckets: secondsDurationBuckets(), - }, - []string{"code", "method", "route"}, - ) - httpGeoProxiedRequestsTotal = promauto.NewCounterVec( prometheus.CounterOpts{ Namespace: namespace, @@ -111,19 +25,12 @@ var ( }, []string{"code", "method", "route"}, ) + + buildHandler = metrics.NewHandlerFactory(metrics.WithNamespace(namespace), metrics.WithLabels("route")) ) func instrumentRoute(next http.Handler, method string, regexpStr string) http.Handler { - handler := next - - handler = promhttp.InstrumentHandlerCounter(httpRequestsTotal.MustCurryWith(map[string]string{"route": regexpStr}), handler) - handler = promhttp.InstrumentHandlerDuration(httpRequestDurationSeconds.MustCurryWith(map[string]string{"route": regexpStr}), handler) - handler = promhttp.InstrumentHandlerInFlight(httpInFlightRequests, handler) - handler = promhttp.InstrumentHandlerRequestSize(httpRequestSizeBytes.MustCurryWith(map[string]string{"route": regexpStr}), handler) - handler = promhttp.InstrumentHandlerResponseSize(httpResponseSizeBytes.MustCurryWith(map[string]string{"route": regexpStr}), handler) - handler = promhttp.InstrumentHandlerTimeToWriteHeader(httpTimeToWriteHeaderSeconds.MustCurryWith(map[string]string{"route": regexpStr}), handler) - - return handler + return buildHandler(next, metrics.WithLabelValues(map[string]string{"route": regexpStr})) } func instrumentGeoProxyRoute(next http.Handler, method string, regexpStr string) http.Handler { diff --git a/workhorse/internal/upstream/routes.go b/workhorse/internal/upstream/routes.go index b8089865ffe..b1d76dfc1bd 100644 --- a/workhorse/internal/upstream/routes.go +++ b/workhorse/internal/upstream/routes.go @@ -20,7 +20,6 @@ import ( "gitlab.com/gitlab-org/gitlab/workhorse/internal/git" "gitlab.com/gitlab-org/gitlab/workhorse/internal/helper" "gitlab.com/gitlab-org/gitlab/workhorse/internal/imageresizer" - "gitlab.com/gitlab-org/gitlab/workhorse/internal/lfs" proxypkg "gitlab.com/gitlab-org/gitlab/workhorse/internal/proxy" "gitlab.com/gitlab-org/gitlab/workhorse/internal/queueing" "gitlab.com/gitlab-org/gitlab/workhorse/internal/redis" @@ -251,8 +250,8 @@ func configureRoutes(u *upstream) { u.route("PUT", gitProjectPattern+`gitlab-lfs/objects/([0-9a-f]{64})/([0-9]+)\z`, upload.RequestBody(api, signingProxy, preparers.lfs), withMatcher(isContentType("application/octet-stream"))), // CI Artifacts - u.route("POST", apiPattern+`v4/jobs/[0-9]+/artifacts\z`, contentEncodingHandler(artifacts.UploadArtifacts(api, signingProxy, preparers.artifacts))), - u.route("POST", ciAPIPattern+`v1/builds/[0-9]+/artifacts\z`, contentEncodingHandler(artifacts.UploadArtifacts(api, signingProxy, preparers.artifacts))), + u.route("POST", apiPattern+`v4/jobs/[0-9]+/artifacts\z`, contentEncodingHandler(upload.Artifacts(api, signingProxy, preparers.artifacts))), + u.route("POST", ciAPIPattern+`v1/builds/[0-9]+/artifacts\z`, contentEncodingHandler(upload.Artifacts(api, signingProxy, preparers.artifacts))), // ActionCable websocket u.wsRoute(`^/-/cable\z`, cableProxy), @@ -318,9 +317,12 @@ func configureRoutes(u *upstream) { // Group Import via UI upload acceleration u.route("POST", importPattern+`gitlab_group`, upload.Multipart(api, signingProxy, preparers.uploads)), - // Metric image upload + // Issuable Metric image upload u.route("POST", apiProjectPattern+`issues/[0-9]+/metric_images\z`, upload.Multipart(api, signingProxy, preparers.uploads)), + // Alert Metric image upload + u.route("POST", apiProjectPattern+`alert_management_alerts/[0-9]+/metric_images\z`, upload.Multipart(api, signingProxy, preparers.uploads)), + // Requirements Import via UI upload acceleration u.route("POST", projectPattern+`requirements_management/requirements/import_csv`, upload.Multipart(api, signingProxy, preparers.uploads)), @@ -383,11 +385,10 @@ func configureRoutes(u *upstream) { u.route("", "^/oauth/geo/(auth|callback|logout)$", defaultUpstream), // Admin Area > Geo routes - u.route("", "^/admin/geo$", defaultUpstream), - u.route("", "^/admin/geo/", defaultUpstream), + u.route("", "^/admin/geo/replication/projects", defaultUpstream), + u.route("", "^/admin/geo/replication/designs", defaultUpstream), // Geo API routes - u.route("", "^/api/v4/geo_nodes", defaultUpstream), u.route("", "^/api/v4/geo_replication", defaultUpstream), u.route("", "^/api/v4/geo/proxy_git_ssh", defaultUpstream), u.route("", "^/api/v4/geo/graphql", defaultUpstream), @@ -395,6 +396,16 @@ func configureRoutes(u *upstream) { // Internal API routes u.route("", "^/api/v4/internal", defaultUpstream), + u.route( + "", `^/assets/`, + static.ServeExisting( + u.URLPrefix, + staticpages.CacheExpireMax, + assetsNotFoundHandler, + ), + withoutTracing(), // Tracing on assets is very noisy + ), + // Don't define a catch-all route. If a route does not match, then we know // the request should be proxied. } @@ -405,7 +416,7 @@ func createUploadPreparers(cfg config.Config) uploadPreparers { return uploadPreparers{ artifacts: defaultPreparer, - lfs: lfs.NewLfsUploadPreparer(cfg, defaultPreparer), + lfs: upload.NewLfsPreparer(cfg, defaultPreparer), packages: defaultPreparer, uploads: defaultPreparer, } diff --git a/workhorse/internal/upstream/routes_test.go b/workhorse/internal/upstream/routes_test.go index f196433f5b4..8a032519bdf 100644 --- a/workhorse/internal/upstream/routes_test.go +++ b/workhorse/internal/upstream/routes_test.go @@ -2,8 +2,26 @@ package upstream import ( "testing" + + "gitlab.com/gitlab-org/gitlab/workhorse/internal/testhelper" ) +func TestAdminGeoPathsWithGeoProxy(t *testing.T) { + testCases := []testCase{ + {"Regular admin/geo", "/admin/geo", "Geo primary received request to path /admin/geo"}, + {"Specific object replication", "/admin/geo/replication/object_type", "Geo primary received request to path /admin/geo/replication/object_type"}, + {"Specific object replication per-site", "/admin/geo/sites/2/replication/object_type", "Geo primary received request to path /admin/geo/sites/2/replication/object_type"}, + {"Projects replication per-site", "/admin/geo/sites/2/replication/projects", "Geo primary received request to path /admin/geo/sites/2/replication/projects"}, + {"Designs replication per-site", "/admin/geo/sites/2/replication/designs", "Geo primary received request to path /admin/geo/sites/2/replication/designs"}, + {"Projects replication", "/admin/geo/replication/projects", "Local Rails server received request to path /admin/geo/replication/projects"}, + {"Projects replication subpaths", "/admin/geo/replication/projects/2", "Local Rails server received request to path /admin/geo/replication/projects/2"}, + {"Designs replication", "/admin/geo/replication/designs", "Local Rails server received request to path /admin/geo/replication/designs"}, + {"Designs replication subpaths", "/admin/geo/replication/designs/3", "Local Rails server received request to path /admin/geo/replication/designs/3"}, + } + + runTestCasesWithGeoProxyEnabled(t, testCases) +} + func TestProjectNotExistingGitHttpPullWithGeoProxy(t *testing.T) { testCases := []testCase{ {"secondary info/refs", "/group/project.git/info/refs", "Local Rails server received request to path /group/project.git/info/refs"}, @@ -45,3 +63,15 @@ func TestProjectNotExistingGitSSHPushWithGeoProxy(t *testing.T) { runTestCasesWithGeoProxyEnabled(t, testCases) } + +func TestAssetsServedLocallyWithGeoProxy(t *testing.T) { + path := "/assets/static.txt" + content := "local geo asset" + testhelper.SetupStaticFileHelper(t, path, content, testDocumentRoot) + + testCases := []testCase{ + {"assets path", "/assets/static.txt", "local geo asset"}, + } + + runTestCasesWithGeoProxyEnabled(t, testCases) +} diff --git a/workhorse/main_test.go b/workhorse/main_test.go index 349e2d78109..88db9e0103b 100644 --- a/workhorse/main_test.go +++ b/workhorse/main_test.go @@ -138,7 +138,7 @@ func TestDeniedXSendfileDownload(t *testing.T) { func TestAllowedStaticFile(t *testing.T) { content := "PUBLIC" - require.NoError(t, setupStaticFile("static file.txt", content)) + setupStaticFile(t, "static file.txt", content) proxied := false ts := testhelper.TestServerWithHandler(regexp.MustCompile(`.`), func(w http.ResponseWriter, r *http.Request) { @@ -164,7 +164,7 @@ func TestAllowedStaticFile(t *testing.T) { func TestStaticFileRelativeURL(t *testing.T) { content := "PUBLIC" - require.NoError(t, setupStaticFile("static.txt", content), "create public/static.txt") + setupStaticFile(t, "static.txt", content) ts := testhelper.TestServerWithHandler(regexp.MustCompile(`.`), http.HandlerFunc(http.NotFound)) defer ts.Close() @@ -182,7 +182,7 @@ func TestStaticFileRelativeURL(t *testing.T) { func TestAllowedPublicUploadsFile(t *testing.T) { content := "PRIVATE but allowed" - require.NoError(t, setupStaticFile("uploads/static file.txt", content), "create public/uploads/static file.txt") + setupStaticFile(t, "uploads/static file.txt", content) proxied := false ts := testhelper.TestServerWithHandler(regexp.MustCompile(`.`), func(w http.ResponseWriter, r *http.Request) { @@ -208,7 +208,7 @@ func TestAllowedPublicUploadsFile(t *testing.T) { func TestDeniedPublicUploadsFile(t *testing.T) { content := "PRIVATE" - require.NoError(t, setupStaticFile("uploads/static.txt", content), "create public/uploads/static.txt") + setupStaticFile(t, "uploads/static.txt", content) proxied := false ts := testhelper.TestServerWithHandler(regexp.MustCompile(`.`), func(w http.ResponseWriter, _ *http.Request) { @@ -241,7 +241,7 @@ This is a static error page for code 499 </body> </html> ` - require.NoError(t, setupStaticFile("499.html", errorPageBody)) + setupStaticFile(t, "499.html", errorPageBody) ts := testhelper.TestServerWithHandler(nil, func(w http.ResponseWriter, _ *http.Request) { upstreamError := "499" // This is the point of the test: the size of the upstream response body @@ -266,7 +266,7 @@ This is a static error page for code 499 func TestGzipAssets(t *testing.T) { path := "/assets/static.txt" content := "asset" - require.NoError(t, setupStaticFile(path, content)) + setupStaticFile(t, path, content) buf := &bytes.Buffer{} gzipWriter := gzip.NewWriter(buf) @@ -274,7 +274,7 @@ func TestGzipAssets(t *testing.T) { require.NoError(t, err) require.NoError(t, gzipWriter.Close()) contentGzip := buf.String() - require.NoError(t, setupStaticFile(path+".gz", contentGzip)) + setupStaticFile(t, path+".gz", contentGzip) proxied := false ts := testhelper.TestServerWithHandler(regexp.MustCompile(`.`), func(w http.ResponseWriter, r *http.Request) { @@ -319,7 +319,7 @@ func TestGzipAssets(t *testing.T) { func TestAltDocumentAssets(t *testing.T) { path := "/assets/static.txt" content := "asset" - require.NoError(t, setupAltStaticFile(path, content)) + setupAltStaticFile(t, path, content) buf := &bytes.Buffer{} gzipWriter := gzip.NewWriter(buf) @@ -327,7 +327,7 @@ func TestAltDocumentAssets(t *testing.T) { require.NoError(t, err) require.NoError(t, gzipWriter.Close()) contentGzip := buf.String() - require.NoError(t, setupAltStaticFile(path+".gz", contentGzip)) + setupAltStaticFile(t, path+".gz", contentGzip) proxied := false ts := testhelper.TestServerWithHandler(regexp.MustCompile(`.`), func(w http.ResponseWriter, r *http.Request) { @@ -712,25 +712,12 @@ func TestRejectUnknownMethod(t *testing.T) { require.Equal(t, http.StatusMethodNotAllowed, resp.StatusCode) } -func setupStaticFile(fpath, content string) error { - return setupStaticFileHelper(fpath, content, testDocumentRoot) +func setupStaticFile(t *testing.T, fpath, content string) { + absDocumentRoot = testhelper.SetupStaticFileHelper(t, fpath, content, testDocumentRoot) } -func setupAltStaticFile(fpath, content string) error { - return setupStaticFileHelper(fpath, content, testAltDocumentRoot) -} - -func setupStaticFileHelper(fpath, content, directory string) error { - cwd, err := os.Getwd() - if err != nil { - return err - } - absDocumentRoot = path.Join(cwd, directory) - if err := os.MkdirAll(path.Join(absDocumentRoot, path.Dir(fpath)), 0755); err != nil { - return err - } - staticFile := path.Join(absDocumentRoot, fpath) - return ioutil.WriteFile(staticFile, []byte(content), 0666) +func setupAltStaticFile(t *testing.T, fpath, content string) { + absDocumentRoot = testhelper.SetupStaticFileHelper(t, fpath, content, testAltDocumentRoot) } func prepareDownloadDir(t *testing.T) { @@ -896,7 +883,7 @@ This is a static error page for code 503 </body> </html> ` - require.NoError(t, setupStaticFile("503.html", errorPageBody)) + setupStaticFile(t, "503.html", errorPageBody) ts := testhelper.TestServerWithHandler(regexp.MustCompile(`.`), func(w http.ResponseWriter, _ *http.Request) { w.Header().Set("X-Gitlab-Custom-Error", "1") diff --git a/workhorse/upload_test.go b/workhorse/upload_test.go index 478cbdb1a44..180598ab260 100644 --- a/workhorse/upload_test.go +++ b/workhorse/upload_test.go @@ -141,6 +141,7 @@ func TestAcceleratedUpload(t *testing.T) { {"POST", `/api/v4/projects/group%2Fsubgroup%2Fproject/packages/pypi`, true}, {"POST", `/api/v4/projects/9001/issues/30/metric_images`, true}, {"POST", `/api/v4/projects/group%2Fproject/issues/30/metric_images`, true}, + {"POST", `/api/v4/projects/9001/alert_management_alerts/30/metric_images`, true}, {"POST", `/api/v4/projects/group%2Fsubgroup%2Fproject/issues/30/metric_images`, true}, {"POST", `/my/project/-/requirements_management/requirements/import_csv`, true}, {"POST", `/my/project/-/requirements_management/requirements/import_csv/`, true}, |