From 1bdf79827c623cc92504223a1085f366115bbb3d Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Wed, 2 Dec 2020 15:09:37 +0000 Subject: Add latest changes from gitlab-org/gitlab@master --- workhorse/internal/imageresizer/image_resizer.go | 448 +++++++++++++++++++++ .../internal/imageresizer/image_resizer_caching.go | 44 ++ .../internal/imageresizer/image_resizer_test.go | 236 +++++++++++ 3 files changed, 728 insertions(+) create mode 100644 workhorse/internal/imageresizer/image_resizer.go create mode 100644 workhorse/internal/imageresizer/image_resizer_caching.go create mode 100644 workhorse/internal/imageresizer/image_resizer_test.go (limited to 'workhorse/internal/imageresizer') diff --git a/workhorse/internal/imageresizer/image_resizer.go b/workhorse/internal/imageresizer/image_resizer.go new file mode 100644 index 00000000000..feefd9c6dee --- /dev/null +++ b/workhorse/internal/imageresizer/image_resizer.go @@ -0,0 +1,448 @@ +package imageresizer + +import ( + "bufio" + "context" + "fmt" + "io" + "math" + "net" + "net/http" + "os" + "os/exec" + "strconv" + "strings" + "sync/atomic" + "syscall" + "time" + + "github.com/prometheus/client_golang/prometheus" + + "gitlab.com/gitlab-org/labkit/correlation" + "gitlab.com/gitlab-org/labkit/log" + "gitlab.com/gitlab-org/labkit/mask" + "gitlab.com/gitlab-org/labkit/tracing" + + "gitlab.com/gitlab-org/gitlab-workhorse/internal/config" + "gitlab.com/gitlab-org/gitlab-workhorse/internal/helper" + "gitlab.com/gitlab-org/gitlab-workhorse/internal/senddata" +) + +type Resizer struct { + config.Config + senddata.Prefix + numScalerProcs processCounter +} + +type resizeParams struct { + Location string + ContentType string + Width uint +} + +type processCounter struct { + n int32 +} + +type resizeStatus = string + +type imageFile struct { + reader io.ReadCloser + contentLength int64 + lastModified time.Time +} + +// Carries information about how the scaler succeeded or failed. +type resizeOutcome struct { + bytesWritten int64 + originalFileSize int64 + status resizeStatus + err error +} + +const ( + statusSuccess = "success" // a rescaled image was served + statusClientCache = "success-client-cache" // scaling was skipped because client cache was fresh + statusServedOriginal = "served-original" // scaling failed but the original image was served + statusRequestFailure = "request-failed" // no image was served + statusUnknown = "unknown" // indicates an unhandled status case +) + +var envInjector = tracing.NewEnvInjector() + +// Images might be located remotely in object storage, in which case we need to stream +// it via http(s) +var httpTransport = tracing.NewRoundTripper(correlation.NewInstrumentedRoundTripper(&http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 10 * time.Second, + }).DialContext, + MaxIdleConns: 2, + IdleConnTimeout: 30 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 10 * time.Second, + ResponseHeaderTimeout: 30 * time.Second, +})) + +var httpClient = &http.Client{ + Transport: httpTransport, +} + +const ( + namespace = "gitlab_workhorse" + subsystem = "image_resize" + logSystem = "imageresizer" +) + +var ( + imageResizeConcurrencyLimitExceeds = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "concurrency_limit_exceeds_total", + Help: "Amount of image resizing requests that exceeded the maximum allowed scaler processes", + }, + ) + imageResizeProcesses = prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "processes", + Help: "Amount of image scaler processes working now", + }, + ) + imageResizeMaxProcesses = prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "max_processes", + Help: "The maximum amount of image scaler processes allowed to run concurrently", + }, + ) + imageResizeRequests = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "requests_total", + Help: "Image resizing operations requested", + }, + []string{"status"}, + ) + imageResizeDurations = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "duration_seconds", + Help: "Breakdown of total time spent serving successful image resizing requests (incl. data transfer)", + Buckets: []float64{ + 0.025, /* 25ms */ + 0.050, /* 50ms */ + 0.1, /* 100ms */ + 0.2, /* 200ms */ + 0.4, /* 400ms */ + 0.8, /* 800ms */ + }, + }, + []string{"content_type", "width"}, + ) +) + +const ( + jpegMagic = "\xff\xd8" // 2 bytes + pngMagic = "\x89PNG\r\n\x1a\n" // 8 bytes + maxMagicLen = 8 // 8 first bytes is enough to detect PNG or JPEG +) + +func init() { + prometheus.MustRegister(imageResizeConcurrencyLimitExceeds) + prometheus.MustRegister(imageResizeProcesses) + prometheus.MustRegister(imageResizeMaxProcesses) + prometheus.MustRegister(imageResizeRequests) + prometheus.MustRegister(imageResizeDurations) +} + +func NewResizer(cfg config.Config) *Resizer { + imageResizeMaxProcesses.Set(float64(cfg.ImageResizerConfig.MaxScalerProcs)) + + return &Resizer{Config: cfg, Prefix: "send-scaled-img:"} +} + +// Inject forks into a dedicated scaler process to resize an image identified by path or URL +// and streams the resized image back to the client +func (r *Resizer) Inject(w http.ResponseWriter, req *http.Request, paramsData string) { + var outcome = resizeOutcome{status: statusUnknown, originalFileSize: 0, bytesWritten: 0} + start := time.Now() + params, err := r.unpackParameters(paramsData) + + defer func() { + imageResizeRequests.WithLabelValues(outcome.status).Inc() + handleOutcome(w, req, start, params, &outcome) + }() + + if err != nil { + // This means the response header coming from Rails was malformed; there is no way + // to sensibly recover from this other than failing fast + outcome.error(fmt.Errorf("read image resize params: %v", err)) + return + } + + imageFile, err := openSourceImage(params.Location) + if err != nil { + // This means we cannot even read the input image; fail fast. + outcome.error(fmt.Errorf("open image data stream: %v", err)) + return + } + defer imageFile.reader.Close() + + outcome.originalFileSize = imageFile.contentLength + + setLastModified(w, imageFile.lastModified) + // If the original file has not changed, then any cached resized versions have not changed either. + if checkNotModified(req, imageFile.lastModified) { + writeNotModified(w) + outcome.ok(statusClientCache) + return + } + + // We first attempt to rescale the image; if this should fail for any reason, imageReader + // will point to the original image, i.e. we render it unchanged. + imageReader, resizeCmd, err := r.tryResizeImage(req, imageFile, params, r.Config.ImageResizerConfig) + if err != nil { + // Something failed, but we can still write out the original image, so don't return early. + // We need to log this separately since the subsequent steps might add other failures. + helper.LogErrorWithFields(req, err, *logFields(start, params, &outcome)) + } + defer helper.CleanUpProcessGroup(resizeCmd) + + w.Header().Del("Content-Length") + outcome.bytesWritten, err = serveImage(imageReader, w, resizeCmd) + + // We failed serving image data; this is a hard failure. + if err != nil { + outcome.error(err) + return + } + + // This means we served the original image because rescaling failed; this is a soft failure + if resizeCmd == nil { + outcome.ok(statusServedOriginal) + return + } + + widthLabelVal := strconv.Itoa(int(params.Width)) + imageResizeDurations.WithLabelValues(params.ContentType, widthLabelVal).Observe(time.Since(start).Seconds()) + + outcome.ok(statusSuccess) +} + +// Streams image data from the given reader to the given writer and returns the number of bytes written. +func serveImage(r io.Reader, w io.Writer, resizeCmd *exec.Cmd) (int64, error) { + bytesWritten, err := io.Copy(w, r) + if err != nil { + return bytesWritten, err + } + + if resizeCmd != nil { + // If a scaler process had been forked, wait for the command to finish. + if err = resizeCmd.Wait(); err != nil { + // err will be an ExitError; this is not useful beyond knowing the exit code since anything + // interesting has been written to stderr, so we turn that into an error we can return. + stdErr := resizeCmd.Stderr.(*strings.Builder) + return bytesWritten, fmt.Errorf(stdErr.String()) + } + } + + return bytesWritten, nil +} + +func (r *Resizer) unpackParameters(paramsData string) (*resizeParams, error) { + var params resizeParams + if err := r.Unpack(¶ms, paramsData); err != nil { + return nil, err + } + + if params.Location == "" { + return nil, fmt.Errorf("'Location' not set") + } + + if params.ContentType == "" { + return nil, fmt.Errorf("'ContentType' must be set") + } + + return ¶ms, nil +} + +// Attempts to rescale the given image data, or in case of errors, falls back to the original image. +func (r *Resizer) tryResizeImage(req *http.Request, f *imageFile, params *resizeParams, cfg config.ImageResizerConfig) (io.Reader, *exec.Cmd, error) { + if f.contentLength > int64(cfg.MaxFilesize) { + return f.reader, nil, fmt.Errorf("%d bytes exceeds maximum file size of %d bytes", f.contentLength, cfg.MaxFilesize) + } + + if !r.numScalerProcs.tryIncrement(int32(cfg.MaxScalerProcs)) { + return f.reader, nil, fmt.Errorf("too many running scaler processes (%d / %d)", r.numScalerProcs.n, cfg.MaxScalerProcs) + } + + ctx := req.Context() + go func() { + <-ctx.Done() + r.numScalerProcs.decrement() + }() + + // Prevents EOF if the file is smaller than 8 bytes + bufferSize := int(math.Min(maxMagicLen, float64(f.contentLength))) + buffered := bufio.NewReaderSize(f.reader, bufferSize) + + headerBytes, err := buffered.Peek(bufferSize) + if err != nil { + return buffered, nil, fmt.Errorf("peek stream: %v", err) + } + + // Check magic bytes to identify file type. + if string(headerBytes) != pngMagic && string(headerBytes[0:2]) != jpegMagic { + return buffered, nil, fmt.Errorf("unrecognized file signature: %v", headerBytes) + } + + resizeCmd, resizedImageReader, err := startResizeImageCommand(ctx, buffered, params) + if err != nil { + return buffered, nil, fmt.Errorf("fork into scaler process: %w", err) + } + return resizedImageReader, resizeCmd, nil +} + +func startResizeImageCommand(ctx context.Context, imageReader io.Reader, params *resizeParams) (*exec.Cmd, io.ReadCloser, error) { + cmd := exec.CommandContext(ctx, "gitlab-resize-image") + cmd.Stdin = imageReader + cmd.Stderr = &strings.Builder{} + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + cmd.Env = []string{ + "GL_RESIZE_IMAGE_WIDTH=" + strconv.Itoa(int(params.Width)), + } + cmd.Env = envInjector(ctx, cmd.Env) + + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, nil, err + } + + if err := cmd.Start(); err != nil { + return nil, nil, err + } + + return cmd, stdout, nil +} + +func isURL(location string) bool { + return strings.HasPrefix(location, "http://") || strings.HasPrefix(location, "https://") +} + +func openSourceImage(location string) (*imageFile, error) { + if isURL(location) { + return openFromURL(location) + } + + return openFromFile(location) +} + +func openFromURL(location string) (*imageFile, error) { + res, err := httpClient.Get(location) + if err != nil { + return nil, err + } + + switch res.StatusCode { + case http.StatusOK, http.StatusNotModified: + // Extract headers for conditional GETs from response. + lastModified, err := http.ParseTime(res.Header.Get("Last-Modified")) + if err != nil { + // This is unlikely to happen, coming from an object storage provider. + lastModified = time.Now().UTC() + } + return &imageFile{res.Body, res.ContentLength, lastModified}, nil + default: + res.Body.Close() + return nil, fmt.Errorf("stream data from %q: %d %s", location, res.StatusCode, res.Status) + } +} + +func openFromFile(location string) (*imageFile, error) { + file, err := os.Open(location) + if err != nil { + return nil, err + } + + fi, err := file.Stat() + if err != nil { + file.Close() + return nil, err + } + + return &imageFile{file, fi.Size(), fi.ModTime()}, nil +} + +// Only allow more scaling requests if we haven't yet reached the maximum +// allowed number of concurrent scaler processes +func (c *processCounter) tryIncrement(maxScalerProcs int32) bool { + if p := atomic.AddInt32(&c.n, 1); p > maxScalerProcs { + c.decrement() + imageResizeConcurrencyLimitExceeds.Inc() + + return false + } + + imageResizeProcesses.Set(float64(c.n)) + return true +} + +func (c *processCounter) decrement() { + atomic.AddInt32(&c.n, -1) + imageResizeProcesses.Set(float64(c.n)) +} + +func (o *resizeOutcome) ok(status resizeStatus) { + o.status = status + o.err = nil +} + +func (o *resizeOutcome) error(err error) { + o.status = statusRequestFailure + o.err = err +} + +func logFields(startTime time.Time, params *resizeParams, outcome *resizeOutcome) *log.Fields { + var targetWidth, contentType string + if params != nil { + targetWidth = fmt.Sprint(params.Width) + contentType = fmt.Sprint(params.ContentType) + } + return &log.Fields{ + "subsystem": logSystem, + "written_bytes": outcome.bytesWritten, + "duration_s": time.Since(startTime).Seconds(), + logSystem + ".status": outcome.status, + logSystem + ".target_width": targetWidth, + logSystem + ".content_type": contentType, + logSystem + ".original_filesize": outcome.originalFileSize, + } +} + +func handleOutcome(w http.ResponseWriter, req *http.Request, startTime time.Time, params *resizeParams, outcome *resizeOutcome) { + logger := log.ContextLogger(req.Context()) + fields := *logFields(startTime, params, outcome) + + switch outcome.status { + case statusRequestFailure: + if outcome.bytesWritten <= 0 { + helper.Fail500WithFields(w, req, outcome.err, fields) + } else { + helper.LogErrorWithFields(req, outcome.err, fields) + } + default: + logger.WithFields(fields).WithFields( + log.Fields{ + "method": req.Method, + "uri": mask.URL(req.RequestURI), + }, + ).Printf(outcome.status) + } +} diff --git a/workhorse/internal/imageresizer/image_resizer_caching.go b/workhorse/internal/imageresizer/image_resizer_caching.go new file mode 100644 index 00000000000..bbe0e3260d7 --- /dev/null +++ b/workhorse/internal/imageresizer/image_resizer_caching.go @@ -0,0 +1,44 @@ +// This file contains code derived from https://github.com/golang/go/blob/master/src/net/http/fs.go +// +// Copyright 2020 GitLab Inc. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. + +package imageresizer + +import ( + "net/http" + "time" +) + +func checkNotModified(r *http.Request, modtime time.Time) bool { + ims := r.Header.Get("If-Modified-Since") + if ims == "" || isZeroTime(modtime) { + // Treat bogus times as if there was no such header at all + return false + } + t, err := http.ParseTime(ims) + if err != nil { + return false + } + // The Last-Modified header truncates sub-second precision so + // the modtime needs to be truncated too. + return !modtime.Truncate(time.Second).After(t) +} + +// isZeroTime reports whether t is obviously unspecified (either zero or Unix epoch time). +func isZeroTime(t time.Time) bool { + return t.IsZero() || t.Equal(time.Unix(0, 0)) +} + +func setLastModified(w http.ResponseWriter, modtime time.Time) { + if !isZeroTime(modtime) { + w.Header().Set("Last-Modified", modtime.UTC().Format(http.TimeFormat)) + } +} + +func writeNotModified(w http.ResponseWriter) { + h := w.Header() + h.Del("Content-Type") + h.Del("Content-Length") + w.WriteHeader(http.StatusNotModified) +} diff --git a/workhorse/internal/imageresizer/image_resizer_test.go b/workhorse/internal/imageresizer/image_resizer_test.go new file mode 100644 index 00000000000..49cd88200aa --- /dev/null +++ b/workhorse/internal/imageresizer/image_resizer_test.go @@ -0,0 +1,236 @@ +package imageresizer + +import ( + "encoding/base64" + "encoding/json" + "image" + "image/png" + "io/ioutil" + "net/http" + "net/http/httptest" + "os" + "testing" + "time" + + "github.com/stretchr/testify/require" + "gitlab.com/gitlab-org/labkit/log" + + "gitlab.com/gitlab-org/gitlab-workhorse/internal/config" + "gitlab.com/gitlab-org/gitlab-workhorse/internal/testhelper" + + _ "image/jpeg" // need this for image.Decode with JPEG +) + +const imagePath = "../../testdata/image.png" + +func TestMain(m *testing.M) { + if err := testhelper.BuildExecutables(); err != nil { + log.WithError(err).Fatal() + } + + os.Exit(m.Run()) +} + +func requestScaledImage(t *testing.T, httpHeaders http.Header, params resizeParams, cfg config.ImageResizerConfig) *http.Response { + httpRequest := httptest.NewRequest("GET", "/image", nil) + if httpHeaders != nil { + httpRequest.Header = httpHeaders + } + responseWriter := httptest.NewRecorder() + paramsJSON := encodeParams(t, ¶ms) + + NewResizer(config.Config{ImageResizerConfig: cfg}).Inject(responseWriter, httpRequest, paramsJSON) + + return responseWriter.Result() +} + +func TestRequestScaledImageFromPath(t *testing.T) { + cfg := config.DefaultImageResizerConfig + + testCases := []struct { + desc string + imagePath string + contentType string + }{ + { + desc: "PNG", + imagePath: imagePath, + contentType: "image/png", + }, + { + desc: "JPEG", + imagePath: "../../testdata/image.jpg", + contentType: "image/jpeg", + }, + { + desc: "JPEG < 1kb", + imagePath: "../../testdata/image_single_pixel.jpg", + contentType: "image/jpeg", + }, + } + + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + params := resizeParams{Location: tc.imagePath, ContentType: tc.contentType, Width: 64} + + resp := requestScaledImage(t, nil, params, cfg) + require.Equal(t, http.StatusOK, resp.StatusCode) + + bounds := imageFromResponse(t, resp).Bounds() + require.Equal(t, int(params.Width), bounds.Size().X, "wrong width after resizing") + }) + } +} + +func TestRequestScaledImageWithConditionalGetAndImageNotChanged(t *testing.T) { + cfg := config.DefaultImageResizerConfig + params := resizeParams{Location: imagePath, ContentType: "image/png", Width: 64} + + clientTime := testImageLastModified(t) + header := http.Header{} + header.Set("If-Modified-Since", httpTimeStr(clientTime)) + + resp := requestScaledImage(t, header, params, cfg) + require.Equal(t, http.StatusNotModified, resp.StatusCode) + require.Equal(t, httpTimeStr(testImageLastModified(t)), resp.Header.Get("Last-Modified")) + require.Empty(t, resp.Header.Get("Content-Type")) + require.Empty(t, resp.Header.Get("Content-Length")) +} + +func TestRequestScaledImageWithConditionalGetAndImageChanged(t *testing.T) { + cfg := config.DefaultImageResizerConfig + params := resizeParams{Location: imagePath, ContentType: "image/png", Width: 64} + + clientTime := testImageLastModified(t).Add(-1 * time.Second) + header := http.Header{} + header.Set("If-Modified-Since", httpTimeStr(clientTime)) + + resp := requestScaledImage(t, header, params, cfg) + require.Equal(t, http.StatusOK, resp.StatusCode) + require.Equal(t, httpTimeStr(testImageLastModified(t)), resp.Header.Get("Last-Modified")) +} + +func TestRequestScaledImageWithConditionalGetAndInvalidClientTime(t *testing.T) { + cfg := config.DefaultImageResizerConfig + params := resizeParams{Location: imagePath, ContentType: "image/png", Width: 64} + + header := http.Header{} + header.Set("If-Modified-Since", "0") + + resp := requestScaledImage(t, header, params, cfg) + require.Equal(t, http.StatusOK, resp.StatusCode) + require.Equal(t, httpTimeStr(testImageLastModified(t)), resp.Header.Get("Last-Modified")) +} + +func TestServeOriginalImageWhenSourceImageTooLarge(t *testing.T) { + originalImage := testImage(t) + cfg := config.ImageResizerConfig{MaxScalerProcs: 1, MaxFilesize: 1} + params := resizeParams{Location: imagePath, ContentType: "image/png", Width: 64} + + resp := requestScaledImage(t, nil, params, cfg) + require.Equal(t, http.StatusOK, resp.StatusCode) + + img := imageFromResponse(t, resp) + require.Equal(t, originalImage.Bounds(), img.Bounds(), "expected original image size") +} + +func TestFailFastOnOpenStreamFailure(t *testing.T) { + cfg := config.DefaultImageResizerConfig + params := resizeParams{Location: "does_not_exist.png", ContentType: "image/png", Width: 64} + resp := requestScaledImage(t, nil, params, cfg) + + require.Equal(t, http.StatusInternalServerError, resp.StatusCode) +} + +func TestIgnoreContentTypeMismatchIfImageFormatIsAllowed(t *testing.T) { + cfg := config.DefaultImageResizerConfig + params := resizeParams{Location: imagePath, ContentType: "image/jpeg", Width: 64} + resp := requestScaledImage(t, nil, params, cfg) + require.Equal(t, http.StatusOK, resp.StatusCode) + + bounds := imageFromResponse(t, resp).Bounds() + require.Equal(t, int(params.Width), bounds.Size().X, "wrong width after resizing") +} + +func TestUnpackParametersReturnsParamsInstanceForValidInput(t *testing.T) { + r := Resizer{} + inParams := resizeParams{Location: imagePath, Width: 64, ContentType: "image/png"} + + outParams, err := r.unpackParameters(encodeParams(t, &inParams)) + + require.NoError(t, err, "unexpected error when unpacking params") + require.Equal(t, inParams, *outParams) +} + +func TestUnpackParametersReturnsErrorWhenLocationBlank(t *testing.T) { + r := Resizer{} + inParams := resizeParams{Location: "", Width: 64, ContentType: "image/jpg"} + + _, err := r.unpackParameters(encodeParams(t, &inParams)) + + require.Error(t, err, "expected error when Location is blank") +} + +func TestUnpackParametersReturnsErrorWhenContentTypeBlank(t *testing.T) { + r := Resizer{} + inParams := resizeParams{Location: imagePath, Width: 64, ContentType: ""} + + _, err := r.unpackParameters(encodeParams(t, &inParams)) + + require.Error(t, err, "expected error when ContentType is blank") +} + +func TestServeOriginalImageWhenSourceImageFormatIsNotAllowed(t *testing.T) { + cfg := config.DefaultImageResizerConfig + // SVG images are not allowed to be resized + svgImagePath := "../../testdata/image.svg" + svgImage, err := ioutil.ReadFile(svgImagePath) + require.NoError(t, err) + // ContentType is no longer used to perform the format validation. + // To make the test more strict, we'll use allowed, but incorrect ContentType. + params := resizeParams{Location: svgImagePath, ContentType: "image/png", Width: 64} + + resp := requestScaledImage(t, nil, params, cfg) + require.Equal(t, http.StatusOK, resp.StatusCode) + + responseData, err := ioutil.ReadAll(resp.Body) + require.NoError(t, err) + require.Equal(t, svgImage, responseData, "expected original image") +} + +// The Rails applications sends a Base64 encoded JSON string carrying +// these parameters in an HTTP response header +func encodeParams(t *testing.T, p *resizeParams) string { + json, err := json.Marshal(*p) + if err != nil { + require.NoError(t, err, "JSON encoder encountered unexpected error") + } + return base64.StdEncoding.EncodeToString(json) +} + +func testImage(t *testing.T) image.Image { + f, err := os.Open(imagePath) + require.NoError(t, err) + + image, err := png.Decode(f) + require.NoError(t, err, "decode original image") + + return image +} + +func testImageLastModified(t *testing.T) time.Time { + fi, err := os.Stat(imagePath) + require.NoError(t, err) + + return fi.ModTime() +} + +func imageFromResponse(t *testing.T, resp *http.Response) image.Image { + img, _, err := image.Decode(resp.Body) + require.NoError(t, err, "decode resized image") + return img +} + +func httpTimeStr(time time.Time) string { + return time.UTC().Format(http.TimeFormat) +} -- cgit v1.2.3