Welcome to mirror list, hosted at ThFree Co, Russian Federation.

gitlab.com/gitlab-org/gitlab-pages.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--internal/serving/disk/helpers.go16
-rw-r--r--internal/serving/disk/helpers_test.go32
-rw-r--r--internal/serving/disk/reader.go45
-rw-r--r--internal/serving/disk/serving.go3
-rw-r--r--internal/serving/disk/symlink/path_test.go19
-rw-r--r--internal/serving/disk/symlink/shims.go20
-rw-r--r--internal/serving/disk/symlink/symlink.go11
-rw-r--r--internal/vfs/local/local_test.go96
-rw-r--r--internal/vfs/local/testdata/file1
l---------internal/vfs/local/testdata/link1
-rw-r--r--internal/vfs/local/vfs.go19
-rw-r--r--internal/vfs/vfs.go55
-rw-r--r--metrics/metrics.go7
13 files changed, 243 insertions, 82 deletions
diff --git a/internal/serving/disk/helpers.go b/internal/serving/disk/helpers.go
index 271b63cc..e6d3f8ab 100644
--- a/internal/serving/disk/helpers.go
+++ b/internal/serving/disk/helpers.go
@@ -1,16 +1,14 @@
package disk
import (
+ "context"
"io"
"mime"
"net/http"
- "os"
"path/filepath"
"strconv"
"strings"
- "golang.org/x/sys/unix"
-
"gitlab.com/gitlab-org/gitlab-pages/internal/httputil"
)
@@ -22,20 +20,16 @@ func endsWithoutHTMLExtension(path string) bool {
return !strings.HasSuffix(path, ".html")
}
-func openNoFollow(path string) (*os.File, error) {
- return os.OpenFile(path, os.O_RDONLY|unix.O_NOFOLLOW, 0)
-}
-
// Detect file's content-type either by extension or mime-sniffing.
// Implementation is adapted from Golang's `http.serveContent()`
// See https://github.com/golang/go/blob/902fc114272978a40d2e65c2510a18e870077559/src/net/http/fs.go#L194
-func detectContentType(path string) (string, error) {
+func (reader *Reader) detectContentType(ctx context.Context, path string) (string, error) {
contentType := mime.TypeByExtension(filepath.Ext(path))
if contentType == "" {
var buf [512]byte
- file, err := os.Open(path)
+ file, err := reader.vfs.Open(ctx, path)
if err != nil {
return "", err
}
@@ -61,7 +55,7 @@ func acceptsGZip(r *http.Request) bool {
return acceptedEncoding == "gzip"
}
-func handleGZip(w http.ResponseWriter, r *http.Request, fullPath string) string {
+func (reader *Reader) handleGZip(ctx context.Context, w http.ResponseWriter, r *http.Request, fullPath string) string {
if !acceptsGZip(r) {
return fullPath
}
@@ -69,7 +63,7 @@ func handleGZip(w http.ResponseWriter, r *http.Request, fullPath string) string
gzipPath := fullPath + ".gz"
// Ensure the .gz file is not a symlink
- fi, err := os.Lstat(gzipPath)
+ fi, err := reader.vfs.Lstat(ctx, gzipPath)
if err != nil || !fi.Mode().IsRegular() {
return fullPath
}
diff --git a/internal/serving/disk/helpers_test.go b/internal/serving/disk/helpers_test.go
deleted file mode 100644
index eea51500..00000000
--- a/internal/serving/disk/helpers_test.go
+++ /dev/null
@@ -1,32 +0,0 @@
-package disk
-
-import (
- "io/ioutil"
- "os"
- "testing"
-
- "github.com/stretchr/testify/require"
-)
-
-func TestOpenNoFollow(t *testing.T) {
- tmpfile, err := ioutil.TempFile("", "link-test")
- require.NoError(t, err)
- defer tmpfile.Close()
-
- orig := tmpfile.Name()
- softLink := orig + ".link"
- defer os.Remove(orig)
-
- source, err := openNoFollow(orig)
- require.NoError(t, err)
- require.NotNil(t, source)
- defer source.Close()
-
- err = os.Symlink(orig, softLink)
- require.NoError(t, err)
- defer os.Remove(softLink)
-
- link, err := openNoFollow(softLink)
- require.Error(t, err)
- require.Nil(t, link)
-}
diff --git a/internal/serving/disk/reader.go b/internal/serving/disk/reader.go
index b8dce542..8c7ee3bf 100644
--- a/internal/serving/disk/reader.go
+++ b/internal/serving/disk/reader.go
@@ -1,10 +1,10 @@
package disk
import (
+ "context"
"fmt"
"io"
"net/http"
- "os"
"path/filepath"
"strconv"
"strings"
@@ -13,15 +13,19 @@ import (
"github.com/prometheus/client_golang/prometheus"
"gitlab.com/gitlab-org/gitlab-pages/internal/serving"
+ "gitlab.com/gitlab-org/gitlab-pages/internal/serving/disk/symlink"
+ "gitlab.com/gitlab-org/gitlab-pages/internal/vfs"
)
// Reader is a disk access driver
type Reader struct {
fileSizeMetric prometheus.Histogram
+ vfs vfs.VFS
}
func (reader *Reader) tryFile(h serving.Handler) error {
- fullPath, err := reader.resolvePath(h.LookupPath.Path, h.SubPath)
+ ctx := h.Request.Context()
+ fullPath, err := reader.resolvePath(ctx, h.LookupPath.Path, h.SubPath)
request := h.Request
host := request.Host
@@ -29,7 +33,7 @@ func (reader *Reader) tryFile(h serving.Handler) error {
if locationError, _ := err.(*locationDirectoryError); locationError != nil {
if endsWithSlash(urlPath) {
- fullPath, err = reader.resolvePath(h.LookupPath.Path, h.SubPath, "index.html")
+ fullPath, err = reader.resolvePath(ctx, h.LookupPath.Path, h.SubPath, "index.html")
} else {
// TODO why are we doing that? In tests it redirects to HTTPS. This seems wrong,
// issue about this: https://gitlab.com/gitlab-org/gitlab-pages/issues/273
@@ -46,23 +50,24 @@ func (reader *Reader) tryFile(h serving.Handler) error {
}
if locationError, _ := err.(*locationFileNoExtensionError); locationError != nil {
- fullPath, err = reader.resolvePath(h.LookupPath.Path, strings.TrimSuffix(h.SubPath, "/")+".html")
+ fullPath, err = reader.resolvePath(ctx, h.LookupPath.Path, strings.TrimSuffix(h.SubPath, "/")+".html")
}
if err != nil {
return err
}
- return reader.serveFile(h.Writer, h.Request, fullPath, h.LookupPath.HasAccessControl)
+ return reader.serveFile(ctx, h.Writer, h.Request, fullPath, h.LookupPath.HasAccessControl)
}
func (reader *Reader) tryNotFound(h serving.Handler) error {
- page404, err := reader.resolvePath(h.LookupPath.Path, "404.html")
+ ctx := h.Request.Context()
+ page404, err := reader.resolvePath(ctx, h.LookupPath.Path, "404.html")
if err != nil {
return err
}
- err = reader.serveCustomFile(h.Writer, h.Request, http.StatusNotFound, page404)
+ err = reader.serveCustomFile(ctx, h.Writer, h.Request, http.StatusNotFound, page404)
if err != nil {
return err
}
@@ -71,7 +76,7 @@ func (reader *Reader) tryNotFound(h serving.Handler) error {
// Resolve the HTTP request to a path on disk, converting requests for
// directories to requests for index.html inside the directory if appropriate.
-func (reader *Reader) resolvePath(publicPath string, subPath ...string) (string, error) {
+func (reader *Reader) resolvePath(ctx context.Context, publicPath string, subPath ...string) (string, error) {
// Ensure that publicPath always ends with "/"
publicPath = strings.TrimSuffix(publicPath, "/") + "/"
@@ -79,7 +84,7 @@ func (reader *Reader) resolvePath(publicPath string, subPath ...string) (string,
// where we want to traverse full path as supplied by user
// (including ..)
testPath := publicPath + strings.Join(subPath, "/")
- fullPath, err := filepath.EvalSymlinks(testPath)
+ fullPath, err := symlink.EvalSymlinks(ctx, reader.vfs, testPath)
if err != nil {
if endsWithoutHTMLExtension(testPath) {
@@ -96,7 +101,7 @@ func (reader *Reader) resolvePath(publicPath string, subPath ...string) (string,
return "", fmt.Errorf("%q should be in %q", fullPath, publicPath)
}
- fi, err := os.Lstat(fullPath)
+ fi, err := reader.vfs.Lstat(ctx, fullPath)
if err != nil {
return "", err
}
@@ -118,17 +123,17 @@ func (reader *Reader) resolvePath(publicPath string, subPath ...string) (string,
return fullPath, nil
}
-func (reader *Reader) serveFile(w http.ResponseWriter, r *http.Request, origPath string, accessControl bool) error {
- fullPath := handleGZip(w, r, origPath)
+func (reader *Reader) serveFile(ctx context.Context, w http.ResponseWriter, r *http.Request, origPath string, accessControl bool) error {
+ fullPath := reader.handleGZip(ctx, w, r, origPath)
- file, err := openNoFollow(fullPath)
+ file, err := reader.vfs.Open(ctx, fullPath)
if err != nil {
return err
}
defer file.Close()
- fi, err := file.Stat()
+ fi, err := reader.vfs.Lstat(ctx, fullPath)
if err != nil {
return err
}
@@ -139,7 +144,7 @@ func (reader *Reader) serveFile(w http.ResponseWriter, r *http.Request, origPath
w.Header().Set("Expires", time.Now().Add(10*time.Minute).Format(time.RFC1123))
}
- contentType, err := detectContentType(origPath)
+ contentType, err := reader.detectContentType(ctx, origPath)
if err != nil {
return err
}
@@ -152,22 +157,22 @@ func (reader *Reader) serveFile(w http.ResponseWriter, r *http.Request, origPath
return nil
}
-func (reader *Reader) serveCustomFile(w http.ResponseWriter, r *http.Request, code int, origPath string) error {
- fullPath := handleGZip(w, r, origPath)
+func (reader *Reader) serveCustomFile(ctx context.Context, w http.ResponseWriter, r *http.Request, code int, origPath string) error {
+ fullPath := reader.handleGZip(ctx, w, r, origPath)
// Open and serve content of file
- file, err := openNoFollow(fullPath)
+ file, err := reader.vfs.Open(ctx, fullPath)
if err != nil {
return err
}
defer file.Close()
- fi, err := file.Stat()
+ fi, err := reader.vfs.Lstat(ctx, fullPath)
if err != nil {
return err
}
- contentType, err := detectContentType(origPath)
+ contentType, err := reader.detectContentType(ctx, origPath)
if err != nil {
return err
}
diff --git a/internal/serving/disk/serving.go b/internal/serving/disk/serving.go
index f5ec0d78..b7501b80 100644
--- a/internal/serving/disk/serving.go
+++ b/internal/serving/disk/serving.go
@@ -3,12 +3,15 @@ package disk
import (
"gitlab.com/gitlab-org/gitlab-pages/internal/httperrors"
"gitlab.com/gitlab-org/gitlab-pages/internal/serving"
+ "gitlab.com/gitlab-org/gitlab-pages/internal/vfs"
+ "gitlab.com/gitlab-org/gitlab-pages/internal/vfs/local"
"gitlab.com/gitlab-org/gitlab-pages/metrics"
)
var disk = &Disk{
reader: Reader{
fileSizeMetric: metrics.DiskServingFileSize,
+ vfs: vfs.Instrumented(local.VFS{}, "disk"),
},
}
diff --git a/internal/serving/disk/symlink/path_test.go b/internal/serving/disk/symlink/path_test.go
index 077b0ad2..4d590db5 100644
--- a/internal/serving/disk/symlink/path_test.go
+++ b/internal/serving/disk/symlink/path_test.go
@@ -2,17 +2,22 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
-package filepath_test
+package symlink_test
import (
+ "context"
"io/ioutil"
"os"
+ "path/filepath"
"runtime"
"testing"
- filepath "gitlab.com/gitlab-org/gitlab-pages/internal/serving/disk/symlink"
+ "gitlab.com/gitlab-org/gitlab-pages/internal/serving/disk/symlink"
+ "gitlab.com/gitlab-org/gitlab-pages/internal/vfs/local"
)
+var fs = local.VFS{}
+
func chtmpdir(t *testing.T) (restore func()) {
oldwd, err := os.Getwd()
if err != nil {
@@ -81,7 +86,7 @@ func simpleJoin(dir, path string) string {
}
func testEvalSymlinks(t *testing.T, path, want string) {
- have, err := filepath.EvalSymlinks(path)
+ have, err := symlink.EvalSymlinks(context.Background(), fs, path)
if err != nil {
t.Errorf("EvalSymlinks(%q) error: %v", path, err)
return
@@ -108,7 +113,7 @@ func testEvalSymlinksAfterChdir(t *testing.T, wd, path, want string) {
t.Fatal(err)
}
- have, err := filepath.EvalSymlinks(path)
+ have, err := symlink.EvalSymlinks(context.Background(), fs, path)
if err != nil {
t.Errorf("EvalSymlinks(%q) in %q directory error: %v", path, wd, err)
return
@@ -183,7 +188,7 @@ func TestEvalSymlinks(t *testing.T) {
func TestEvalSymlinksIsNotExist(t *testing.T) {
defer chtmpdir(t)()
- _, err := filepath.EvalSymlinks("notexist")
+ _, err := symlink.EvalSymlinks(context.Background(), fs, "notexist")
if !os.IsNotExist(err) {
t.Errorf("expected the file is not found, got %v\n", err)
}
@@ -194,7 +199,7 @@ func TestEvalSymlinksIsNotExist(t *testing.T) {
}
defer os.Remove("link")
- _, err = filepath.EvalSymlinks("link")
+ _, err = symlink.EvalSymlinks(context.Background(), fs, "link")
if !os.IsNotExist(err) {
t.Errorf("expected the file is not found, got %v\n", err)
}
@@ -251,7 +256,7 @@ func TestIssue13582(t *testing.T) {
{link2, realFile},
}
for i, test := range tests {
- have, err := filepath.EvalSymlinks(test.path)
+ have, err := symlink.EvalSymlinks(context.Background(), fs, test.path)
if err != nil {
t.Fatal(err)
}
diff --git a/internal/serving/disk/symlink/shims.go b/internal/serving/disk/symlink/shims.go
index 4b344180..d383b96b 100644
--- a/internal/serving/disk/symlink/shims.go
+++ b/internal/serving/disk/symlink/shims.go
@@ -1,13 +1,17 @@
-package filepath
+package symlink
-import stdlib "path/filepath"
+import (
+ "context"
+ "path/filepath"
+
+ "gitlab.com/gitlab-org/gitlab-pages/internal/vfs"
+)
func volumeNameLen(s string) int { return 0 }
-func IsAbs(path string) bool { return stdlib.IsAbs(path) }
-func Clean(path string) string { return stdlib.Clean(path) }
-func Join(elem ...string) string { return stdlib.Join(elem...) }
-func VolumeName(path string) string { return stdlib.VolumeName(path) }
-func EvalSymlinks(path string) (string, error) { return walkSymlinks(path) }
+func IsAbs(path string) bool { return filepath.IsAbs(path) }
+func Clean(path string) string { return filepath.Clean(path) }
-const Separator = stdlib.Separator
+func EvalSymlinks(ctx context.Context, fs vfs.VFS, path string) (string, error) {
+ return walkSymlinks(ctx, fs, path)
+}
diff --git a/internal/serving/disk/symlink/symlink.go b/internal/serving/disk/symlink/symlink.go
index 335b315a..50714811 100644
--- a/internal/serving/disk/symlink/symlink.go
+++ b/internal/serving/disk/symlink/symlink.go
@@ -2,16 +2,19 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
-package filepath
+package symlink
import (
+ "context"
"errors"
"os"
"runtime"
"syscall"
+
+ "gitlab.com/gitlab-org/gitlab-pages/internal/vfs"
)
-func walkSymlinks(path string) (string, error) {
+func walkSymlinks(ctx context.Context, fs vfs.VFS, path string) (string, error) {
volLen := volumeNameLen(path)
pathSeparator := string(os.PathSeparator)
@@ -80,7 +83,7 @@ func walkSymlinks(path string) (string, error) {
// Resolve symlink.
- fi, err := os.Lstat(dest)
+ fi, err := fs.Lstat(ctx, dest)
if err != nil {
return "", err
}
@@ -99,7 +102,7 @@ func walkSymlinks(path string) (string, error) {
return "", errors.New("EvalSymlinks: too many links")
}
- link, err := os.Readlink(dest)
+ link, err := fs.Readlink(ctx, dest)
if err != nil {
return "", err
}
diff --git a/internal/vfs/local/local_test.go b/internal/vfs/local/local_test.go
new file mode 100644
index 00000000..c41f96bc
--- /dev/null
+++ b/internal/vfs/local/local_test.go
@@ -0,0 +1,96 @@
+package local
+
+import (
+ "context"
+ "io/ioutil"
+ "os"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestReadlink(t *testing.T) {
+ ctx := context.Background()
+ fs := VFS{}
+
+ target, err := fs.Readlink(ctx, "testdata/link")
+ require.NoError(t, err)
+ require.Equal(t, "file", target)
+}
+
+func TestReadlinkNotSymlink(t *testing.T) {
+ ctx := context.Background()
+ fs := VFS{}
+
+ for _, path := range []string{"testdata", "testdata/file"} {
+ t.Run(path, func(t *testing.T) {
+ _, err := os.Lstat(path)
+ require.NoError(t, err, "sanity check: input must actually exist")
+
+ _, err = fs.Readlink(ctx, path)
+ require.Error(t, err, "expect readlink to fail")
+ })
+ }
+}
+
+func TestLstat(t *testing.T) {
+ ctx := context.Background()
+ fs := VFS{}
+
+ testCases := []struct {
+ path string
+ modePerm os.FileMode
+ modeType os.FileMode
+ }{
+ {path: "testdata", modeType: os.ModeDir, modePerm: 0755},
+ {path: "testdata/file", modeType: os.FileMode(0), modePerm: 0644},
+ {path: "testdata/link", modeType: os.ModeSymlink}, // Permissions of symlinks are platform dependent
+ }
+
+ for _, tc := range testCases {
+ t.Run(tc.path, func(t *testing.T) {
+ if tc.modePerm > 0 {
+ require.NoError(t, os.Chmod(tc.path, tc.modePerm), "preparation: deterministic permissions")
+ }
+
+ fi, err := fs.Lstat(ctx, tc.path)
+ require.NoError(t, err, "lstat error")
+
+ require.Equal(t, tc.modeType, fi.Mode()&os.ModeType, "file mode: type")
+ if tc.modePerm > 0 {
+ require.Equal(t, tc.modePerm, fi.Mode()&os.ModePerm, "file mode: permissions")
+ }
+ })
+ }
+}
+
+func TestOpen(t *testing.T) {
+ ctx := context.Background()
+ fs := VFS{}
+
+ f, err := fs.Open(ctx, "testdata/file")
+ require.NoError(t, err, "open file")
+
+ data, err := ioutil.ReadAll(f)
+ require.NoError(t, err, "read from file")
+ require.Equal(t, "hello\n", string(data), "file contents")
+
+ require.NoError(t, f.Close(), "close file")
+}
+
+func TestOpenDenySymlink(t *testing.T) {
+ ctx := context.Background()
+ fs := VFS{}
+ const symlinkPath = "testdata/link"
+
+ fi, err := os.Stat(symlinkPath)
+ require.NoError(t, err, "stat link")
+ require.Equal(t, os.FileMode(0), fi.Mode()&os.ModeType, "sanity check: link target should be a regular file")
+
+ fi, err = os.Lstat(symlinkPath)
+ require.NoError(t, err, "lstat link")
+ require.Equal(t, os.ModeSymlink, fi.Mode()&os.ModeType, "sanity check: testdata/link should be a symlink")
+
+ _, err = fs.Open(ctx, symlinkPath)
+ require.Error(t, err, "opening symlink should fail (security mechanism)")
+}
diff --git a/internal/vfs/local/testdata/file b/internal/vfs/local/testdata/file
new file mode 100644
index 00000000..ce013625
--- /dev/null
+++ b/internal/vfs/local/testdata/file
@@ -0,0 +1 @@
+hello
diff --git a/internal/vfs/local/testdata/link b/internal/vfs/local/testdata/link
new file mode 120000
index 00000000..1a010b1c
--- /dev/null
+++ b/internal/vfs/local/testdata/link
@@ -0,0 +1 @@
+file \ No newline at end of file
diff --git a/internal/vfs/local/vfs.go b/internal/vfs/local/vfs.go
new file mode 100644
index 00000000..7c6f3ba6
--- /dev/null
+++ b/internal/vfs/local/vfs.go
@@ -0,0 +1,19 @@
+package local
+
+import (
+ "context"
+ "os"
+
+ "golang.org/x/sys/unix"
+
+ "gitlab.com/gitlab-org/gitlab-pages/internal/vfs"
+)
+
+type VFS struct{}
+
+func (fs VFS) Lstat(ctx context.Context, name string) (os.FileInfo, error) { return os.Lstat(name) }
+func (fs VFS) Readlink(ctx context.Context, name string) (string, error) { return os.Readlink(name) }
+
+func (fs VFS) Open(ctx context.Context, name string) (vfs.File, error) {
+ return os.OpenFile(name, os.O_RDONLY|unix.O_NOFOLLOW, 0)
+}
diff --git a/internal/vfs/vfs.go b/internal/vfs/vfs.go
new file mode 100644
index 00000000..07c99b77
--- /dev/null
+++ b/internal/vfs/vfs.go
@@ -0,0 +1,55 @@
+package vfs
+
+import (
+ "context"
+ "io"
+ "os"
+ "strconv"
+
+ "gitlab.com/gitlab-org/gitlab-pages/metrics"
+)
+
+// VFS abstracts the things Pages needs to serve a static site from disk.
+type VFS interface {
+ Lstat(ctx context.Context, name string) (os.FileInfo, error)
+ Readlink(ctx context.Context, name string) (string, error)
+ Open(ctx context.Context, name string) (File, error)
+}
+
+// File represents an open file, which will typically be the response body of a Pages request.
+type File interface {
+ io.Reader
+ io.Seeker
+ io.Closer
+}
+
+func Instrumented(fs VFS, name string) VFS {
+ return &InstrumentedVFS{fs: fs, name: name}
+}
+
+type InstrumentedVFS struct {
+ fs VFS
+ name string
+}
+
+func (i *InstrumentedVFS) increment(operation string, err error) {
+ metrics.VFSOperations.WithLabelValues(i.name, operation, strconv.FormatBool(err == nil)).Inc()
+}
+
+func (i *InstrumentedVFS) Lstat(ctx context.Context, name string) (os.FileInfo, error) {
+ fi, err := i.fs.Lstat(ctx, name)
+ i.increment("Lstat", err)
+ return fi, err
+}
+
+func (i *InstrumentedVFS) Readlink(ctx context.Context, name string) (string, error) {
+ target, err := i.fs.Readlink(ctx, name)
+ i.increment("Readlink", err)
+ return target, err
+}
+
+func (i *InstrumentedVFS) Open(ctx context.Context, name string) (File, error) {
+ f, err := i.fs.Open(ctx, name)
+ i.increment("Open", err)
+ return f, err
+}
diff --git a/metrics/metrics.go b/metrics/metrics.go
index 88636279..0792a41f 100644
--- a/metrics/metrics.go
+++ b/metrics/metrics.go
@@ -91,6 +91,12 @@ var (
Help: "The time (in seconds) taken to serve a file",
Buckets: []float64{0.1, 0.5, 1, 2.5, 5, 10, 60, 180},
})
+
+ // VFSOperations metric for VFS operations (lstat, readlink, open)
+ VFSOperations = prometheus.NewCounterVec(prometheus.CounterOpts{
+ Name: "gitlab_pages_vfs_operations_total",
+ Help: "The number of VFS operations",
+ }, []string{"vfs_name", "operation", "success"})
)
// MustRegister collectors with the Prometheus client
@@ -110,5 +116,6 @@ func MustRegister() {
ServerlessLatency,
DiskServingFileSize,
ServingTime,
+ VFSOperations,
)
}
generated by cgit v1.2.3 (git 2.25.1) at 2024-08-01 20:34:16 +0300