Welcome to mirror list, hosted at ThFree Co, Russian Federation.

gitlab.com/gitlab-org/gitaly.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorQuang-Minh Nguyen <qmnguyen@gitlab.com>2023-07-07 09:12:36 +0300
committerQuang-Minh Nguyen <qmnguyen@gitlab.com>2023-07-11 09:25:46 +0300
commitcfad638107472e2fd4e831f9900cf23a3e0b1ebe (patch)
treed107a30b4eeb8893cccbe00373203ca90324f504
parent5258ae767e5c0821f20bb1a17a6ab2de562949c5 (diff)
Replace limithandler's stats by generic customfields
Limithandler has a dedicated utility to store gRPC log fields. This utility suite consists of a recorder, a FieldsProducer, and two interceptors to plug in gRPC server. customfields can provide the same service. It was built for general purposes. Hence, there is no reason to keep both of them. This commit removes limithandler's dedicated stats handler in favor of customfields.
-rw-r--r--internal/gitaly/server/server.go4
-rw-r--r--internal/grpc/middleware/limithandler/monitor.go23
-rw-r--r--internal/grpc/middleware/limithandler/monitor_test.go25
-rw-r--r--internal/grpc/middleware/limithandler/stats.go125
-rw-r--r--internal/grpc/middleware/limithandler/stats_interceptor_test.go171
-rw-r--r--internal/grpc/middleware/limithandler/stats_test.go30
6 files changed, 26 insertions, 352 deletions
diff --git a/internal/gitaly/server/server.go b/internal/gitaly/server/server.go
index f9caf4e08..79392c46f 100644
--- a/internal/gitaly/server/server.go
+++ b/internal/gitaly/server/server.go
@@ -16,7 +16,6 @@ import (
"gitlab.com/gitlab-org/gitaly/v16/internal/grpc/middleware/cache"
"gitlab.com/gitlab-org/gitaly/v16/internal/grpc/middleware/customfieldshandler"
"gitlab.com/gitlab-org/gitaly/v16/internal/grpc/middleware/featureflag"
- "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/middleware/limithandler"
"gitlab.com/gitlab-org/gitaly/v16/internal/grpc/middleware/metadatahandler"
"gitlab.com/gitlab-org/gitaly/v16/internal/grpc/middleware/panichandler"
"gitlab.com/gitlab-org/gitaly/v16/internal/grpc/middleware/sentryhandler"
@@ -111,7 +110,6 @@ func (s *GitalyServerFactory) New(secure bool, opts ...Option) (*grpc.Server, er
grpcstats.FieldsProducer,
featureflag.FieldsProducer,
structerr.FieldsProducer,
- limithandler.FieldsProducer,
),
)
@@ -121,7 +119,6 @@ func (s *GitalyServerFactory) New(secure bool, opts ...Option) (*grpc.Server, er
metadatahandler.StreamInterceptor,
grpcprometheus.StreamServerInterceptor,
customfieldshandler.StreamInterceptor,
- limithandler.StatsStreamInterceptor,
grpcmwlogrus.StreamServerInterceptor(s.logger,
grpcmwlogrus.WithTimestampFormat(gitalylog.LogTimestampFormat),
logMsgProducer,
@@ -138,7 +135,6 @@ func (s *GitalyServerFactory) New(secure bool, opts ...Option) (*grpc.Server, er
metadatahandler.UnaryInterceptor,
grpcprometheus.UnaryServerInterceptor,
customfieldshandler.UnaryInterceptor,
- limithandler.StatsUnaryInterceptor,
grpcmwlogrus.UnaryServerInterceptor(s.logger,
grpcmwlogrus.WithTimestampFormat(gitalylog.LogTimestampFormat),
logMsgProducer,
diff --git a/internal/grpc/middleware/limithandler/monitor.go b/internal/grpc/middleware/limithandler/monitor.go
index b251df6d4..e59c20eeb 100644
--- a/internal/grpc/middleware/limithandler/monitor.go
+++ b/internal/grpc/middleware/limithandler/monitor.go
@@ -6,6 +6,7 @@ import (
"time"
"github.com/prometheus/client_golang/prometheus"
+ "gitlab.com/gitlab-org/gitaly/v16/internal/log"
)
// ConcurrencyMonitor allows the concurrency monitor to be observed.
@@ -71,9 +72,10 @@ func newPerRPCPromMonitor(
// Queued is called when a request has been queued.
func (p *PromMonitor) Queued(ctx context.Context, key string, queueLength int) {
- if stats := limitStatsFromContext(ctx); stats != nil {
- stats.SetLimitingKey(p.limitingType, key)
- stats.SetConcurrencyQueueLength(queueLength)
+ if stats := log.CustomFieldsFromContext(ctx); stats != nil {
+ stats.RecordMetadata("limit.limiting_type", p.limitingType)
+ stats.RecordMetadata("limit.limiting_key", key)
+ stats.RecordMetadata("limit.concurrency_queue_length", queueLength)
}
p.queuedMetric.Inc()
}
@@ -88,8 +90,8 @@ func (p *PromMonitor) Enter(ctx context.Context, acquireTime time.Duration) {
p.inProgressMetric.Inc()
p.acquiringSecondsMetric.Observe(acquireTime.Seconds())
- if stats := limitStatsFromContext(ctx); stats != nil {
- stats.AddConcurrencyQueueMs(acquireTime.Milliseconds())
+ if stats := log.CustomFieldsFromContext(ctx); stats != nil {
+ stats.RecordMetadata("limit.concurrency_queue_ms", acquireTime.Milliseconds())
}
}
@@ -100,11 +102,12 @@ func (p *PromMonitor) Exit(ctx context.Context) {
// Dropped is called when a request is dropped.
func (p *PromMonitor) Dropped(ctx context.Context, key string, length int, acquireTime time.Duration, reason string) {
- if stats := limitStatsFromContext(ctx); stats != nil {
- stats.SetLimitingKey(p.limitingType, key)
- stats.SetConcurrencyQueueLength(length)
- stats.SetConcurrencyDroppedReason(reason)
- stats.AddConcurrencyQueueMs(acquireTime.Milliseconds())
+ if stats := log.CustomFieldsFromContext(ctx); stats != nil {
+ stats.RecordMetadata("limit.limiting_type", p.limitingType)
+ stats.RecordMetadata("limit.limiting_key", key)
+ stats.RecordMetadata("limit.concurrency_queue_length", length)
+ stats.RecordMetadata("limit.concurrency_dropped", reason)
+ stats.RecordMetadata("limit.concurrency_queue_ms", acquireTime.Milliseconds())
}
p.requestsDroppedMetric.WithLabelValues(reason).Inc()
}
diff --git a/internal/grpc/middleware/limithandler/monitor_test.go b/internal/grpc/middleware/limithandler/monitor_test.go
index 3b5d6163f..95814804d 100644
--- a/internal/grpc/middleware/limithandler/monitor_test.go
+++ b/internal/grpc/middleware/limithandler/monitor_test.go
@@ -10,6 +10,7 @@ import (
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/require"
promconfig "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config/prometheus"
+ "gitlab.com/gitlab-org/gitaly/v16/internal/log"
"gitlab.com/gitlab-org/gitaly/v16/internal/testhelper"
)
@@ -63,7 +64,7 @@ func TestNewPerRPCPromMonitor(t *testing.T) {
t.Run("request is dequeued successfully", func(t *testing.T) {
rpcMonitor := createNewMonitor()
- ctx := InitLimitStats(testhelper.Context(t))
+ ctx := log.InitContextCustomFields(testhelper.Context(t))
rpcMonitor.Queued(ctx, fullMethod, 5)
rpcMonitor.Enter(ctx, time.Second)
@@ -101,7 +102,7 @@ queued{grpc_method="unknown",grpc_service="unknown",system="gitaly"} 1
"acquiring_seconds",
))
- stats := limitStatsFromContext(ctx)
+ stats := log.CustomFieldsFromContext(ctx)
require.NotNil(t, stats)
require.Equal(t, logrus.Fields{
"limit.limiting_type": TypePerRPC,
@@ -132,7 +133,7 @@ queued{grpc_method="unknown",grpc_service="unknown",system="gitaly"} 0
t.Run("request is dropped after queueing", func(t *testing.T) {
rpcMonitor := createNewMonitor()
- ctx := InitLimitStats(testhelper.Context(t))
+ ctx := log.InitContextCustomFields(testhelper.Context(t))
rpcMonitor.Queued(ctx, fullMethod, 5)
rpcMonitor.Dropped(ctx, fullMethod, 5, time.Second, "load")
@@ -173,7 +174,7 @@ queued{grpc_method="unknown",grpc_service="unknown",system="gitaly"} 1
"acquiring_seconds",
))
- stats := limitStatsFromContext(ctx)
+ stats := log.CustomFieldsFromContext(ctx)
require.NotNil(t, stats)
require.Equal(t, logrus.Fields{
"limit.limiting_type": TypePerRPC,
@@ -186,7 +187,7 @@ queued{grpc_method="unknown",grpc_service="unknown",system="gitaly"} 1
t.Run("request is dropped before queueing", func(t *testing.T) {
rpcMonitor := createNewMonitor()
- ctx := InitLimitStats(testhelper.Context(t))
+ ctx := log.InitContextCustomFields(testhelper.Context(t))
rpcMonitor.Dropped(ctx, fullMethod, 5, time.Second, "load")
expectedMetrics := `# HELP acquiring_seconds seconds to acquire
@@ -225,7 +226,7 @@ queued{grpc_method="unknown",grpc_service="unknown",system="gitaly"} 0
"acquiring_seconds",
))
- stats := limitStatsFromContext(ctx)
+ stats := log.CustomFieldsFromContext(ctx)
require.NotNil(t, stats)
require.Equal(t, logrus.Fields{
"limit.limiting_type": TypePerRPC,
@@ -239,7 +240,7 @@ queued{grpc_method="unknown",grpc_service="unknown",system="gitaly"} 0
func TestNewPackObjectsConcurrencyMonitor(t *testing.T) {
t.Run("request is dequeued successfully", func(t *testing.T) {
- ctx := InitLimitStats(testhelper.Context(t))
+ ctx := log.InitContextCustomFields(testhelper.Context(t))
packObjectsConcurrencyMonitor := NewPackObjectsConcurrencyMonitor(
promconfig.DefaultConfig().GRPCLatencyBuckets,
)
@@ -280,7 +281,7 @@ gitaly_pack_objects_queued 1
"gitaly_pack_objects_dropped_total",
))
- stats := limitStatsFromContext(ctx)
+ stats := log.CustomFieldsFromContext(ctx)
require.NotNil(t, stats)
require.Equal(t, logrus.Fields{
"limit.limiting_type": TypePackObjects,
@@ -310,7 +311,7 @@ gitaly_pack_objects_queued 0
})
t.Run("request is dropped after queueing", func(t *testing.T) {
- ctx := InitLimitStats(testhelper.Context(t))
+ ctx := log.InitContextCustomFields(testhelper.Context(t))
packObjectsConcurrencyMonitor := NewPackObjectsConcurrencyMonitor(
promconfig.DefaultConfig().GRPCLatencyBuckets,
)
@@ -354,7 +355,7 @@ gitaly_pack_objects_queued 1
"gitaly_pack_objects_dropped_total",
))
- stats := limitStatsFromContext(ctx)
+ stats := log.CustomFieldsFromContext(ctx)
require.NotNil(t, stats)
require.Equal(t, logrus.Fields{
"limit.limiting_type": TypePackObjects,
@@ -366,7 +367,7 @@ gitaly_pack_objects_queued 1
})
t.Run("request is dropped before queueing", func(t *testing.T) {
- ctx := InitLimitStats(testhelper.Context(t))
+ ctx := log.InitContextCustomFields(testhelper.Context(t))
packObjectsConcurrencyMonitor := NewPackObjectsConcurrencyMonitor(
promconfig.DefaultConfig().GRPCLatencyBuckets,
)
@@ -409,7 +410,7 @@ gitaly_pack_objects_queued 0
"gitaly_pack_objects_dropped_total",
))
- stats := limitStatsFromContext(ctx)
+ stats := log.CustomFieldsFromContext(ctx)
require.NotNil(t, stats)
require.Equal(t, logrus.Fields{
"limit.limiting_type": TypePackObjects,
diff --git a/internal/grpc/middleware/limithandler/stats.go b/internal/grpc/middleware/limithandler/stats.go
deleted file mode 100644
index 30926695a..000000000
--- a/internal/grpc/middleware/limithandler/stats.go
+++ /dev/null
@@ -1,125 +0,0 @@
-package limithandler
-
-import (
- "context"
- "sync"
-
- grpcmw "github.com/grpc-ecosystem/go-grpc-middleware"
- "github.com/sirupsen/logrus"
- "google.golang.org/grpc"
-)
-
-type limitStatsKey struct{}
-
-// LimitStats contains info about the concurrency limiter.
-type LimitStats struct {
- sync.Mutex
- // limitingKey is the key used for limiting accounting
- limitingKey string
- // limitingType is the type of limiter
- limitingType string
- // concurrencyQueueLen is the combination of in-flight requests and in-queue requests. It tells
- // how busy the queue of the same limiting key is
- concurrencyQueueLength int
- // concurrencyQueueMs milliseconds waiting in concurrency limit queue.
- concurrencyQueueMs int64
- // concurrencyDropped stores the dropping reason of a request
- concurrencyDropped string
-}
-
-// InitLimitStats initializes context with a per-RPC stats struct.
-func InitLimitStats(ctx context.Context) context.Context {
- return context.WithValue(ctx, limitStatsKey{}, &LimitStats{})
-}
-
-// AddConcurrencyQueueMs adds queue time.
-func (s *LimitStats) AddConcurrencyQueueMs(queueMs int64) {
- s.Lock()
- defer s.Unlock()
- s.concurrencyQueueMs = queueMs
-}
-
-// SetLimitingKey set limiting key.
-func (s *LimitStats) SetLimitingKey(limitingType string, limitingKey string) {
- s.Lock()
- defer s.Unlock()
- s.limitingKey = limitingKey
- s.limitingType = limitingType
-}
-
-// SetConcurrencyQueueLength set concurrency queue length.
-func (s *LimitStats) SetConcurrencyQueueLength(queueLength int) {
- s.Lock()
- defer s.Unlock()
- s.concurrencyQueueLength = queueLength
-}
-
-// SetConcurrencyDroppedReason sets the reason why a call has been dropped from the queue.
-func (s *LimitStats) SetConcurrencyDroppedReason(reason string) {
- s.Lock()
- defer s.Unlock()
- s.concurrencyDropped = reason
-}
-
-// Fields returns logging info.
-func (s *LimitStats) Fields() logrus.Fields {
- s.Lock()
- defer s.Unlock()
-
- if s.limitingKey == "" {
- return nil
- }
- logs := logrus.Fields{
- "limit.limiting_type": s.limitingType,
- "limit.limiting_key": s.limitingKey,
- "limit.concurrency_queue_ms": s.concurrencyQueueMs,
- "limit.concurrency_queue_length": s.concurrencyQueueLength,
- }
- if s.concurrencyDropped != "" {
- logs["limit.concurrency_dropped"] = s.concurrencyDropped
- }
- return logs
-}
-
-// FieldsProducer extracts stats info from the context and returns it as a logging fields.
-func FieldsProducer(ctx context.Context, _ error) logrus.Fields {
- stats := limitStatsFromContext(ctx)
- if stats != nil {
- return stats.Fields()
- }
- return nil
-}
-
-func limitStatsFromContext(ctx context.Context) *LimitStats {
- v, ok := ctx.Value(limitStatsKey{}).(*LimitStats)
- if !ok {
- return nil
- }
- return v
-}
-
-// stats interceptors are separate from middleware and serve one main purpose:
-// initialize the stats object early, so that logrus can see it.
-// it must be placed before the limithandler middleware.
-
-// StatsUnaryInterceptor returns a Unary Interceptor that initializes the context.
-func StatsUnaryInterceptor(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
- ctx = InitLimitStats(ctx)
-
- res, err := handler(ctx, req)
-
- return res, err
-}
-
-// StatsStreamInterceptor returns a Stream Interceptor.
-func StatsStreamInterceptor(srv interface{}, stream grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
- ctx := stream.Context()
- ctx = InitLimitStats(ctx)
-
- wrapped := grpcmw.WrapServerStream(stream)
- wrapped.WrappedContext = ctx
-
- err := handler(srv, wrapped)
-
- return err
-}
diff --git a/internal/grpc/middleware/limithandler/stats_interceptor_test.go b/internal/grpc/middleware/limithandler/stats_interceptor_test.go
deleted file mode 100644
index 010e487c8..000000000
--- a/internal/grpc/middleware/limithandler/stats_interceptor_test.go
+++ /dev/null
@@ -1,171 +0,0 @@
-package limithandler_test
-
-import (
- "context"
- "io"
- "net"
- "testing"
-
- grpcmwlogrus "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus"
- "github.com/sirupsen/logrus"
- "github.com/sirupsen/logrus/hooks/test"
- "github.com/stretchr/testify/require"
- "gitlab.com/gitlab-org/gitaly/v16/internal/git/catfile"
- "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest"
- "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config"
- "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/service/ref"
- "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction"
- "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/backchannel"
- "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/middleware/limithandler"
- "gitlab.com/gitlab-org/gitaly/v16/internal/log"
- "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper"
- "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg"
- "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb"
- "google.golang.org/grpc"
- "google.golang.org/grpc/credentials/insecure"
- "google.golang.org/grpc/test/bufconn"
-)
-
-func createNewServer(t *testing.T, cfg config.Cfg, logger *logrus.Logger) *grpc.Server {
- t.Helper()
-
- logrusEntry := logrus.NewEntry(logger).WithField("test", t.Name())
-
- concurrencyLimitHandler := limithandler.New(
- cfg,
- func(ctx context.Context) string { return "@hashed/1234" },
- limithandler.WithConcurrencyLimiters,
- )
-
- opts := []grpc.ServerOption{
- grpc.ChainStreamInterceptor(
- limithandler.StatsStreamInterceptor,
- grpcmwlogrus.StreamServerInterceptor(logrusEntry,
- grpcmwlogrus.WithTimestampFormat(log.LogTimestampFormat),
- grpcmwlogrus.WithMessageProducer(log.MessageProducer(grpcmwlogrus.DefaultMessageProducer, limithandler.FieldsProducer))),
- concurrencyLimitHandler.StreamInterceptor(),
- ),
- grpc.ChainUnaryInterceptor(
- limithandler.StatsUnaryInterceptor,
- grpcmwlogrus.UnaryServerInterceptor(logrusEntry,
- grpcmwlogrus.WithTimestampFormat(log.LogTimestampFormat),
- grpcmwlogrus.WithMessageProducer(log.MessageProducer(grpcmwlogrus.DefaultMessageProducer, limithandler.FieldsProducer))),
- concurrencyLimitHandler.UnaryInterceptor(),
- ),
- }
-
- server := grpc.NewServer(opts...)
-
- gitCommandFactory := gittest.NewCommandFactory(t, cfg)
- catfileCache := catfile.NewCache(cfg)
- t.Cleanup(catfileCache.Stop)
-
- gitalypb.RegisterRefServiceServer(server, ref.NewServer(
- config.NewLocator(cfg),
- gitCommandFactory,
- transaction.NewManager(cfg, backchannel.NewRegistry()),
- catfileCache,
- ))
-
- return server
-}
-
-func getBufDialer(listener *bufconn.Listener) func(context.Context, string) (net.Conn, error) {
- return func(ctx context.Context, url string) (net.Conn, error) {
- return listener.Dial()
- }
-}
-
-func TestInterceptor(t *testing.T) {
- t.Parallel()
-
- ctx := testhelper.Context(t)
- cfg := testcfg.Build(t, testcfg.WithBase(config.Cfg{
- Concurrency: []config.Concurrency{
- {
- RPC: "/gitaly.RefService/RefExists",
- MaxPerRepo: 1,
- },
- {
- RPC: "/gitaly.RefService/ListRefs",
- MaxPerRepo: 1,
- },
- },
- }))
-
- repo, _ := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{
- SkipCreationViaService: true,
- })
-
- logger, hook := test.NewNullLogger()
-
- s := createNewServer(t, cfg, logger)
- defer s.Stop()
-
- bufferSize := 1024 * 1024
- listener := bufconn.Listen(bufferSize)
- go testhelper.MustServe(t, s, listener)
-
- tests := []struct {
- name string
- performRPC func(t *testing.T, ctx context.Context, client gitalypb.RefServiceClient)
- expectedLogData map[string]any
- }{
- {
- name: "Unary",
- performRPC: func(t *testing.T, ctx context.Context, client gitalypb.RefServiceClient) {
- req := &gitalypb.RefExistsRequest{Repository: repo, Ref: []byte("refs/foo")}
-
- _, err := client.RefExists(ctx, req)
- require.NoError(t, err)
- },
- expectedLogData: map[string]any{
- "limit.limiting_type": "per-rpc",
- "limit.limiting_key": "@hashed/1234",
- "limit.concurrency_queue_length": 0,
- },
- },
- {
- name: "Stream",
- performRPC: func(t *testing.T, ctx context.Context, client gitalypb.RefServiceClient) {
- req := &gitalypb.ListRefsRequest{Repository: repo, Patterns: [][]byte{[]byte("refs/heads/")}}
-
- stream, err := client.ListRefs(ctx, req)
- require.NoError(t, err)
-
- for {
- _, err := stream.Recv()
- if err == io.EOF {
- break
- }
- require.NoError(t, err)
- }
- },
- expectedLogData: map[string]any{
- "limit.limiting_type": "per-rpc",
- "limit.limiting_key": "@hashed/1234",
- "limit.concurrency_queue_length": 0,
- },
- },
- }
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- hook.Reset()
-
- conn, err := grpc.DialContext(ctx, "", grpc.WithContextDialer(getBufDialer(listener)), grpc.WithTransportCredentials(insecure.NewCredentials()))
- require.NoError(t, err)
- defer conn.Close()
-
- client := gitalypb.NewRefServiceClient(conn)
-
- tt.performRPC(t, ctx, client)
-
- logEntries := hook.AllEntries()
- require.Len(t, logEntries, 1)
- for expectedLogKey, expectedLogValue := range tt.expectedLogData {
- require.Equal(t, expectedLogValue, logEntries[0].Data[expectedLogKey])
- }
- require.GreaterOrEqual(t, logEntries[0].Data["limit.concurrency_queue_ms"], int64(0))
- })
- }
-}
diff --git a/internal/grpc/middleware/limithandler/stats_test.go b/internal/grpc/middleware/limithandler/stats_test.go
deleted file mode 100644
index 53ccccfd5..000000000
--- a/internal/grpc/middleware/limithandler/stats_test.go
+++ /dev/null
@@ -1,30 +0,0 @@
-package limithandler
-
-import (
- "testing"
-
- "github.com/sirupsen/logrus"
- "github.com/stretchr/testify/assert"
- "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper"
-)
-
-func TestLimitStats(t *testing.T) {
- t.Parallel()
-
- ctx := testhelper.Context(t)
- ctx = InitLimitStats(ctx)
-
- stats := limitStatsFromContext(ctx)
- stats.SetLimitingKey("test-limiter", "hello-world")
- stats.AddConcurrencyQueueMs(13)
- stats.SetConcurrencyQueueLength(99)
- stats.SetConcurrencyDroppedReason("max_time")
-
- assert.Equal(t, FieldsProducer(ctx, nil), logrus.Fields{
- "limit.limiting_type": "test-limiter",
- "limit.limiting_key": "hello-world",
- "limit.concurrency_queue_ms": int64(13),
- "limit.concurrency_queue_length": 99,
- "limit.concurrency_dropped": "max_time",
- })
-}