Welcome to mirror list, hosted at ThFree Co, Russian Federation.

gitlab.com/gitlab-org/gitlab-foss.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
path: root/workhorse/internal/config
diff options
context:
space:
mode:
Diffstat (limited to 'workhorse/internal/config')
-rw-r--r--workhorse/internal/config/config.go154
-rw-r--r--workhorse/internal/config/config_test.go111
-rw-r--r--workhorse/internal/config/url_openers.go51
-rw-r--r--workhorse/internal/config/url_openers_test.go117
4 files changed, 433 insertions, 0 deletions
diff --git a/workhorse/internal/config/config.go b/workhorse/internal/config/config.go
new file mode 100644
index 00000000000..84849c72744
--- /dev/null
+++ b/workhorse/internal/config/config.go
@@ -0,0 +1,154 @@
+package config
+
+import (
+ "math"
+ "net/url"
+ "runtime"
+ "strings"
+ "time"
+
+ "github.com/Azure/azure-storage-blob-go/azblob"
+ "github.com/BurntSushi/toml"
+ "gitlab.com/gitlab-org/labkit/log"
+ "gocloud.dev/blob"
+ "gocloud.dev/blob/azureblob"
+)
+
+type TomlURL struct {
+ url.URL
+}
+
+func (u *TomlURL) UnmarshalText(text []byte) error {
+ temp, err := url.Parse(string(text))
+ u.URL = *temp
+ return err
+}
+
+type TomlDuration struct {
+ time.Duration
+}
+
+func (d *TomlDuration) UnmarshalTest(text []byte) error {
+ temp, err := time.ParseDuration(string(text))
+ d.Duration = temp
+ return err
+}
+
+type ObjectStorageCredentials struct {
+ Provider string
+
+ S3Credentials S3Credentials `toml:"s3"`
+ AzureCredentials AzureCredentials `toml:"azurerm"`
+}
+
+type ObjectStorageConfig struct {
+ URLMux *blob.URLMux `toml:"-"`
+}
+
+type S3Credentials struct {
+ AwsAccessKeyID string `toml:"aws_access_key_id"`
+ AwsSecretAccessKey string `toml:"aws_secret_access_key"`
+}
+
+type S3Config struct {
+ Region string `toml:"-"`
+ Bucket string `toml:"-"`
+ PathStyle bool `toml:"-"`
+ Endpoint string `toml:"-"`
+ UseIamProfile bool `toml:"-"`
+ ServerSideEncryption string `toml:"-"` // Server-side encryption mode (e.g. AES256, aws:kms)
+ SSEKMSKeyID string `toml:"-"` // Server-side encryption key-management service key ID (e.g. arn:aws:xxx)
+}
+
+type GoCloudConfig struct {
+ URL string `toml:"-"`
+}
+
+type AzureCredentials struct {
+ AccountName string `toml:"azure_storage_account_name"`
+ AccountKey string `toml:"azure_storage_access_key"`
+}
+
+type RedisConfig struct {
+ URL TomlURL
+ Sentinel []TomlURL
+ SentinelMaster string
+ Password string
+ DB *int
+ ReadTimeout *TomlDuration
+ WriteTimeout *TomlDuration
+ KeepAlivePeriod *TomlDuration
+ MaxIdle *int
+ MaxActive *int
+}
+
+type ImageResizerConfig struct {
+ MaxScalerProcs uint32 `toml:"max_scaler_procs"`
+ MaxFilesize uint64 `toml:"max_filesize"`
+}
+
+type Config struct {
+ Redis *RedisConfig `toml:"redis"`
+ Backend *url.URL `toml:"-"`
+ CableBackend *url.URL `toml:"-"`
+ Version string `toml:"-"`
+ DocumentRoot string `toml:"-"`
+ DevelopmentMode bool `toml:"-"`
+ Socket string `toml:"-"`
+ CableSocket string `toml:"-"`
+ ProxyHeadersTimeout time.Duration `toml:"-"`
+ APILimit uint `toml:"-"`
+ APIQueueLimit uint `toml:"-"`
+ APIQueueTimeout time.Duration `toml:"-"`
+ APICILongPollingDuration time.Duration `toml:"-"`
+ ObjectStorageConfig ObjectStorageConfig `toml:"-"`
+ ObjectStorageCredentials ObjectStorageCredentials `toml:"object_storage"`
+ PropagateCorrelationID bool `toml:"-"`
+ ImageResizerConfig ImageResizerConfig `toml:"image_resizer"`
+ AltDocumentRoot string `toml:"alt_document_root"`
+}
+
+var DefaultImageResizerConfig = ImageResizerConfig{
+ MaxScalerProcs: uint32(math.Max(2, float64(runtime.NumCPU())/2)),
+ MaxFilesize: 250 * 1000, // 250kB,
+}
+
+func LoadConfig(data string) (*Config, error) {
+ cfg := &Config{ImageResizerConfig: DefaultImageResizerConfig}
+
+ if _, err := toml.Decode(data, cfg); err != nil {
+ return nil, err
+ }
+
+ return cfg, nil
+}
+
+func (c *Config) RegisterGoCloudURLOpeners() error {
+ c.ObjectStorageConfig.URLMux = new(blob.URLMux)
+
+ creds := c.ObjectStorageCredentials
+ if strings.EqualFold(creds.Provider, "AzureRM") && creds.AzureCredentials.AccountName != "" && creds.AzureCredentials.AccountKey != "" {
+ accountName := azureblob.AccountName(creds.AzureCredentials.AccountName)
+ accountKey := azureblob.AccountKey(creds.AzureCredentials.AccountKey)
+
+ credential, err := azureblob.NewCredential(accountName, accountKey)
+ if err != nil {
+ log.WithError(err).Error("error creating Azure credentials")
+ return err
+ }
+
+ pipeline := azureblob.NewPipeline(credential, azblob.PipelineOptions{})
+
+ azureURLOpener := &azureURLOpener{
+ &azureblob.URLOpener{
+ AccountName: accountName,
+ Pipeline: pipeline,
+ Options: azureblob.Options{Credential: credential},
+ },
+ }
+
+ c.ObjectStorageConfig.URLMux.RegisterBucket(azureblob.Scheme, azureURLOpener)
+ }
+
+ return nil
+}
diff --git a/workhorse/internal/config/config_test.go b/workhorse/internal/config/config_test.go
new file mode 100644
index 00000000000..102b29a0813
--- /dev/null
+++ b/workhorse/internal/config/config_test.go
@@ -0,0 +1,111 @@
+package config
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+const azureConfig = `
+[object_storage]
+provider = "AzureRM"
+
+[object_storage.azurerm]
+azure_storage_account_name = "azuretester"
+azure_storage_access_key = "deadbeef"
+`
+
+func TestLoadEmptyConfig(t *testing.T) {
+ config := ``
+
+ cfg, err := LoadConfig(config)
+ require.NoError(t, err)
+
+ require.Empty(t, cfg.AltDocumentRoot)
+ require.Equal(t, cfg.ImageResizerConfig.MaxFilesize, uint64(250000))
+ require.GreaterOrEqual(t, cfg.ImageResizerConfig.MaxScalerProcs, uint32(2))
+
+ require.Equal(t, ObjectStorageCredentials{}, cfg.ObjectStorageCredentials)
+ require.NoError(t, cfg.RegisterGoCloudURLOpeners())
+}
+
+func TestLoadObjectStorageConfig(t *testing.T) {
+ config := `
+[object_storage]
+provider = "AWS"
+
+[object_storage.s3]
+aws_access_key_id = "minio"
+aws_secret_access_key = "gdk-minio"
+`
+
+ cfg, err := LoadConfig(config)
+ require.NoError(t, err)
+
+ require.NotNil(t, cfg.ObjectStorageCredentials, "Expected object storage credentials")
+
+ expected := ObjectStorageCredentials{
+ Provider: "AWS",
+ S3Credentials: S3Credentials{
+ AwsAccessKeyID: "minio",
+ AwsSecretAccessKey: "gdk-minio",
+ },
+ }
+
+ require.Equal(t, expected, cfg.ObjectStorageCredentials)
+}
+
+func TestRegisterGoCloudURLOpeners(t *testing.T) {
+ cfg, err := LoadConfig(azureConfig)
+ require.NoError(t, err)
+
+ require.NotNil(t, cfg.ObjectStorageCredentials, "Expected object storage credentials")
+
+ expected := ObjectStorageCredentials{
+ Provider: "AzureRM",
+ AzureCredentials: AzureCredentials{
+ AccountName: "azuretester",
+ AccountKey: "deadbeef",
+ },
+ }
+
+ require.Equal(t, expected, cfg.ObjectStorageCredentials)
+ require.Nil(t, cfg.ObjectStorageConfig.URLMux)
+
+ require.NoError(t, cfg.RegisterGoCloudURLOpeners())
+ require.NotNil(t, cfg.ObjectStorageConfig.URLMux)
+
+ require.True(t, cfg.ObjectStorageConfig.URLMux.ValidBucketScheme("azblob"))
+ require.Equal(t, []string{"azblob"}, cfg.ObjectStorageConfig.URLMux.BucketSchemes())
+}
+
+func TestLoadImageResizerConfig(t *testing.T) {
+ config := `
+[image_resizer]
+max_scaler_procs = 200
+max_filesize = 350000
+`
+
+ cfg, err := LoadConfig(config)
+ require.NoError(t, err)
+
+ require.NotNil(t, cfg.ImageResizerConfig, "Expected image resizer config")
+
+ expected := ImageResizerConfig{
+ MaxScalerProcs: 200,
+ MaxFilesize: 350000,
+ }
+
+ require.Equal(t, expected, cfg.ImageResizerConfig)
+}
+
+func TestAltDocumentConfig(t *testing.T) {
+ config := `
+alt_document_root = "/path/to/documents"
+`
+
+ cfg, err := LoadConfig(config)
+ require.NoError(t, err)
+
+ require.Equal(t, "/path/to/documents", cfg.AltDocumentRoot)
+}
diff --git a/workhorse/internal/config/url_openers.go b/workhorse/internal/config/url_openers.go
new file mode 100644
index 00000000000..d3c96ee9eef
--- /dev/null
+++ b/workhorse/internal/config/url_openers.go
@@ -0,0 +1,51 @@
+package config
+
+import (
+ "context"
+ "fmt"
+ "net/url"
+
+ "gocloud.dev/blob"
+ "gocloud.dev/blob/azureblob"
+)
+
+// This code can be removed once https://github.com/google/go-cloud/pull/2851 is merged.
+
+// URLOpener opens Azure URLs like "azblob://mybucket".
+//
+// The URL host is used as the bucket name.
+//
+// The following query options are supported:
+// - domain: The domain name used to access the Azure Blob storage (e.g. blob.core.windows.net)
+type azureURLOpener struct {
+ *azureblob.URLOpener
+}
+
+func (o *azureURLOpener) OpenBucketURL(ctx context.Context, u *url.URL) (*blob.Bucket, error) {
+ opts := new(azureblob.Options)
+ *opts = o.Options
+
+ err := setOptionsFromURLParams(u.Query(), opts)
+ if err != nil {
+ return nil, err
+ }
+ return azureblob.OpenBucket(ctx, o.Pipeline, o.AccountName, u.Host, opts)
+}
+
+func setOptionsFromURLParams(q url.Values, opts *azureblob.Options) error {
+ for param, values := range q {
+ if len(values) > 1 {
+ return fmt.Errorf("multiple values of %v not allowed", param)
+ }
+
+ value := values[0]
+ switch param {
+ case "domain":
+ opts.StorageDomain = azureblob.StorageDomain(value)
+ default:
+ return fmt.Errorf("unknown query parameter %q", param)
+ }
+ }
+
+ return nil
+}
diff --git a/workhorse/internal/config/url_openers_test.go b/workhorse/internal/config/url_openers_test.go
new file mode 100644
index 00000000000..6a851cacbb8
--- /dev/null
+++ b/workhorse/internal/config/url_openers_test.go
@@ -0,0 +1,117 @@
+package config
+
+import (
+ "context"
+ "net/url"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+ "gocloud.dev/blob/azureblob"
+)
+
+func TestURLOpeners(t *testing.T) {
+ cfg, err := LoadConfig(azureConfig)
+ require.NoError(t, err)
+
+ require.NotNil(t, cfg.ObjectStorageCredentials, "Expected object storage credentials")
+
+ require.NoError(t, cfg.RegisterGoCloudURLOpeners())
+ require.NotNil(t, cfg.ObjectStorageConfig.URLMux)
+
+ tests := []struct {
+ url string
+ valid bool
+ }{
+
+ {
+ url: "azblob://container/object",
+ valid: true,
+ },
+ {
+ url: "azblob://container/object?domain=core.windows.net",
+ valid: true,
+ },
+ {
+ url: "azblob://container/object?domain=core.windows.net&domain=test",
+ valid: false,
+ },
+ {
+ url: "azblob://container/object?param=value",
+ valid: false,
+ },
+ {
+ url: "s3://bucket/object",
+ valid: false,
+ },
+ }
+
+ for _, test := range tests {
+ t.Run(test.url, func(t *testing.T) {
+ ctx := context.Background()
+ url, err := url.Parse(test.url)
+ require.NoError(t, err)
+
+ bucket, err := cfg.ObjectStorageConfig.URLMux.OpenBucketURL(ctx, url)
+ if bucket != nil {
+ defer bucket.Close()
+ }
+
+ if test.valid {
+ require.NotNil(t, bucket)
+ require.NoError(t, err)
+ } else {
+ require.Error(t, err)
+ }
+ })
+ }
+}
+
+func TestTestURLOpenersForParams(t *testing.T) {
+ tests := []struct {
+ name string
+ currOpts azureblob.Options
+ query url.Values
+ wantOpts azureblob.Options
+ wantErr bool
+ }{
+ {
+ name: "InvalidParam",
+ query: url.Values{
+ "foo": {"bar"},
+ },
+ wantErr: true,
+ },
+ {
+ name: "StorageDomain",
+ query: url.Values{
+ "domain": {"blob.core.usgovcloudapi.net"},
+ },
+ wantOpts: azureblob.Options{StorageDomain: "blob.core.usgovcloudapi.net"},
+ },
+ {
+ name: "duplicate StorageDomain",
+ query: url.Values{
+ "domain": {"blob.core.usgovcloudapi.net", "blob.core.windows.net"},
+ },
+ wantErr: true,
+ },
+ }
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ o := &azureURLOpener{
+ URLOpener: &azureblob.URLOpener{
+ Options: test.currOpts,
+ },
+ }
+ err := setOptionsFromURLParams(test.query, &o.Options)
+
+ if test.wantErr {
+ require.NotNil(t, err)
+ } else {
+ require.Nil(t, err)
+ require.Equal(t, test.wantOpts, o.Options)
+ }
+ })
+ }
+}
generated by cgit v1.2.3 (git 2.25.1) at 2024-08-13 13:14:34 +0300