SearchFilesBy{Content,Name} Server Implementation

2 files changed, 412 insertions, 0 deletions

diff --git a/internal/service/repository/search_files.go b/internal/service/repository/search_files.go
new file mode 100644
index 000000000..479ccff78
--- /dev/null
+++ b/internal/service/repository/search_files.go
@@ -0,0 +1,113 @@
+package repository
+
+import (
+	"bytes"
+	"errors"
+
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+
+	pb "gitlab.com/gitlab-org/gitaly-proto/go"
+	"gitlab.com/gitlab-org/gitaly/internal/git"
+	"gitlab.com/gitlab-org/gitaly/internal/helper"
+	"gitlab.com/gitlab-org/gitaly/internal/helper/lines"
+)
+
+const surroundContext = "2"
+
+var contentDelimiter = []byte("--\n")
+
+func (s *server) SearchFilesByContent(req *pb.SearchFilesByContentRequest, stream pb.RepositoryService_SearchFilesByContentServer) error {
+	if err := validateSearchFilesRequest(req); err != nil {
+		return helper.DecorateError(codes.InvalidArgument, err)
+	}
+	repo := req.GetRepository()
+	if repo == nil {
+		return status.Errorf(codes.InvalidArgument, "SearchFilesByContent: empty Repository")
+	}
+
+	ctx := stream.Context()
+	cmd, err := git.Command(ctx, repo, "grep",
+		"--ignore-case",
+		"-I", // Don't match binary, there is no long-name for this one
+		"--line-number",
+		"--null",
+		"--before-context", surroundContext,
+		"--after-context", surroundContext,
+		"--extended-regexp",
+		"-e", // next arg is pattern, keep this last
+		req.GetQuery(),
+		string(req.GetRef()),
+	)
+	if err != nil {
+		return status.Errorf(codes.Internal, "SearchFilesByContent: cmd start failed: %v", err)
+	}
+
+	var (
+		buf     []byte
+		matches [][]byte
+	)
+	reader := func(objs [][]byte) error {
+		for _, obj := range objs {
+			obj = append(obj, '\n')
+			if bytes.Compare(obj, contentDelimiter) == 0 {
+				matches = append(matches, buf)
+				buf = nil
+			} else {
+				buf = append(buf, obj...)
+			}
+		}
+		if len(matches) > 1 {
+			err = stream.Send(&pb.SearchFilesByContentResponse{Matches: matches})
+			matches = nil
+			return err
+		}
+		return nil
+	}
+
+	err = lines.Send(cmd, reader, []byte{'\n'})
+	if err != nil {
+		return helper.DecorateError(codes.Internal, err)
+	}
+	if len(buf) > 1 {
+		return stream.Send(&pb.SearchFilesByContentResponse{Matches: [][]byte{buf}})
+	}
+	return nil
+}
+
+func (s *server) SearchFilesByName(req *pb.SearchFilesByNameRequest, stream pb.RepositoryService_SearchFilesByNameServer) error {
+	if err := validateSearchFilesRequest(req); err != nil {
+		return helper.DecorateError(codes.InvalidArgument, err)
+	}
+	repo := req.GetRepository()
+	if repo == nil {
+		return status.Errorf(codes.InvalidArgument, "SearchFilesByName: empty Repository")
+	}
+
+	ctx := stream.Context()
+	cmd, err := git.Command(ctx, repo, "ls-tree", "--full-tree", "--name-status", "-r", string(req.GetRef()), req.GetQuery())
+	if err != nil {
+		return status.Errorf(codes.Internal, "SearchFilesByName: cmd start failed: %v", err)
+	}
+
+	lr := func(objs [][]byte) error {
+		return stream.Send(&pb.SearchFilesByNameResponse{Files: objs})
+	}
+
+	return lines.Send(cmd, lr, []byte{'\n'})
+}
+
+type searchFilesRequest interface {
+	GetRef() []byte
+	GetQuery() string
+}
+
+func validateSearchFilesRequest(req searchFilesRequest) error {
+	if len(req.GetQuery()) == 0 {
+		return errors.New("no query given")
+	}
+	if len(req.GetRef()) == 0 {
+		return errors.New("no ref given")
+	}
+	return nil
+}
diff --git a/internal/service/repository/search_files_test.go b/internal/service/repository/search_files_test.go
new file mode 100644
index 000000000..042b929cb
--- /dev/null
+++ b/internal/service/repository/search_files_test.go
@@ -0,0 +1,299 @@
+package repository
+
+import (
+	"bytes"
+	"io"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	pb "gitlab.com/gitlab-org/gitaly-proto/go"
+	"gitlab.com/gitlab-org/gitaly/internal/testhelper"
+	"google.golang.org/grpc/codes"
+)
+
+var (
+	contentOutputLines = [][]byte{bytes.Join([][]byte{
+		[]byte("many_files:files/markdown/ruby-style-guide.md\x00128\x00    ```Ruby"),
+		[]byte("many_files:files/markdown/ruby-style-guide.md\x00129\x00    # bad"),
+		[]byte("many_files:files/markdown/ruby-style-guide.md\x00130\x00    puts 'foobar'; # superfluous semicolon"),
+		[]byte("many_files:files/markdown/ruby-style-guide.md\x00131\x00"),
+		[]byte("many_files:files/markdown/ruby-style-guide.md\x00132\x00    puts 'foo'; puts 'bar' # two expression on the same line"),
+		[]byte("many_files:files/markdown/ruby-style-guide.md\x00133\x00"),
+		[]byte("many_files:files/markdown/ruby-style-guide.md\x00134\x00    # good"),
+		[]byte("many_files:files/markdown/ruby-style-guide.md\x00135\x00    puts 'foobar'"),
+		[]byte("many_files:files/markdown/ruby-style-guide.md\x00136\x00"),
+		[]byte("many_files:files/markdown/ruby-style-guide.md\x00137\x00    puts 'foo'"),
+		[]byte(""),
+	}, []byte{'\n'})}
+	contentMultiLines = [][]byte{
+		bytes.Join([][]byte{
+			[]byte("many_files:CHANGELOG\x00306\x00  - Gitlab::Git set of objects to abstract from grit library"),
+			[]byte("many_files:CHANGELOG\x00307\x00  - Replace Unicorn web server with Puma"),
+			[]byte("many_files:CHANGELOG\x00308\x00  - Backup/Restore refactored. Backup dump project wiki too now"),
+			[]byte("many_files:CHANGELOG\x00309\x00  - Restyled Issues list. Show milestone version in issue row"),
+			[]byte("many_files:CHANGELOG\x00310\x00  - Restyled Merge Request list"),
+			[]byte("many_files:CHANGELOG\x00311\x00  - Backup now dump/restore uploads"),
+			[]byte("many_files:CHANGELOG\x00312\x00  - Improved performance of dashboard (Andrew Kumanyaev)"),
+			[]byte("many_files:CHANGELOG\x00313\x00  - File history now tracks renames (Akzhan Abdulin)"),
+			[]byte(""),
+		}, []byte{'\n'}),
+		bytes.Join([][]byte{
+			[]byte("many_files:CHANGELOG\x00377\x00  - fix routing issues"),
+			[]byte("many_files:CHANGELOG\x00378\x00  - cleanup rake tasks"),
+			[]byte("many_files:CHANGELOG\x00379\x00  - fix backup/restore"),
+			[]byte("many_files:CHANGELOG\x00380\x00  - scss cleanup"),
+			[]byte("many_files:CHANGELOG\x00381\x00  - show preview for note images"),
+			[]byte(""),
+		}, []byte{'\n'}),
+		bytes.Join([][]byte{
+			[]byte("many_files:CHANGELOG\x00393\x00  - Remove project code and path from API. Use id instead"),
+			[]byte("many_files:CHANGELOG\x00394\x00  - Return valid cloneable url to repo for web hook"),
+			[]byte("many_files:CHANGELOG\x00395\x00  - Fixed backup issue"),
+			[]byte("many_files:CHANGELOG\x00396\x00  - Reorganized settings"),
+			[]byte("many_files:CHANGELOG\x00397\x00  - Fixed commits compare"),
+			[]byte(""),
+		}, []byte{'\n'}),
+	}
+)
+
+func TestSearchFilesByContentSuccessful(t *testing.T) {
+	ctx, cancel := testhelper.Context()
+	defer cancel()
+
+	server, serverSocketPath := runRepoServer(t)
+	defer server.Stop()
+
+	client, conn := newRepositoryClient(t, serverSocketPath)
+	defer conn.Close()
+
+	testRepo, _, cleanupFn := testhelper.NewTestRepo(t)
+	defer cleanupFn()
+
+	testCases := []struct {
+		desc   string
+		query  string
+		ref    string
+		output [][]byte
+	}{
+		{
+			desc:   "single file in many_files",
+			query:  "foobar",
+			ref:    "many_files",
+			output: contentOutputLines,
+		},
+		{
+			desc:   "multi file in many_files",
+			query:  "backup",
+			ref:    "many_files",
+			output: contentMultiLines,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.desc, func(t *testing.T) {
+			stream, err := client.SearchFilesByContent(ctx, &pb.SearchFilesByContentRequest{
+				Repository: testRepo,
+				Query:      tc.query,
+				Ref:        []byte(tc.ref),
+			})
+			require.NoError(t, err)
+
+			resp, err := consumeFilenameByContent(stream)
+			require.NoError(t, err)
+
+			require.NotEmpty(t, resp)
+			require.Equal(t, len(tc.output), len(resp))
+			for i := 0; i < len(tc.output); i++ {
+				require.Equal(t, tc.output[i], resp[i])
+			}
+		})
+	}
+}
+
+func TestSearchFilesByContentFailure(t *testing.T) {
+	ctx, cancel := testhelper.Context()
+	defer cancel()
+
+	server, serverSocketPath := runRepoServer(t)
+	defer server.Stop()
+
+	client, conn := newRepositoryClient(t, serverSocketPath)
+	defer conn.Close()
+
+	testCases := []struct {
+		desc  string
+		repo  *pb.Repository
+		query string
+		ref   string
+		code  codes.Code
+		msg   string
+	}{
+		{
+			desc: "empty request",
+			code: codes.InvalidArgument,
+			msg:  "no query given",
+		},
+		{
+			desc:  "only query given",
+			query: "foo",
+			code:  codes.InvalidArgument,
+			msg:   "no ref given",
+		},
+		{
+			desc:  "no repo",
+			query: "foo",
+			ref:   "master",
+			code:  codes.InvalidArgument,
+			msg:   "empty Repo",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.desc, func(t *testing.T) {
+
+			stream, err := client.SearchFilesByContent(ctx, &pb.SearchFilesByContentRequest{
+				Repository: tc.repo,
+				Query:      tc.query,
+				Ref:        []byte(tc.ref),
+			})
+			require.NoError(t, err)
+
+			_, err = consumeFilenameByContent(stream)
+			testhelper.AssertGrpcError(t, err, tc.code, tc.msg)
+		})
+	}
+}
+
+func TestSearchFilesByNameSuccessful(t *testing.T) {
+	ctx, cancel := testhelper.Context()
+	defer cancel()
+
+	server, serverSocketPath := runRepoServer(t)
+	defer server.Stop()
+
+	client, conn := newRepositoryClient(t, serverSocketPath)
+	defer conn.Close()
+
+	testRepo, _, cleanupFn := testhelper.NewTestRepo(t)
+	defer cleanupFn()
+
+	testCases := []struct {
+		ref      []byte
+		query    string
+		numFiles int
+		testFile []byte
+	}{
+		{
+			ref:      []byte("many_files"),
+			query:    "files/images/logo-black.png",
+			numFiles: 1,
+			testFile: []byte("files/images/logo-black.png"),
+		},
+		{
+			ref:      []byte("many_files"),
+			query:    "many_files",
+			numFiles: 1001,
+			testFile: []byte("many_files/99"),
+		},
+	}
+
+	for _, tc := range testCases {
+		stream, err := client.SearchFilesByName(ctx, &pb.SearchFilesByNameRequest{
+			Repository: testRepo,
+			Ref:        tc.ref,
+			Query:      tc.query,
+		})
+		require.NoError(t, err)
+
+		var files [][]byte
+		files, err = consumeFilenameByName(stream)
+		require.NoError(t, err)
+
+		require.Equal(t, tc.numFiles, len(files))
+		require.Contains(t, files, tc.testFile)
+	}
+}
+
+func TestSearchFilesByNameFailure(t *testing.T) {
+	ctx, cancel := testhelper.Context()
+	defer cancel()
+
+	server, serverSocketPath := runRepoServer(t)
+	defer server.Stop()
+
+	client, conn := newRepositoryClient(t, serverSocketPath)
+	defer conn.Close()
+
+	testCases := []struct {
+		desc  string
+		repo  *pb.Repository
+		query string
+		ref   string
+		code  codes.Code
+		msg   string
+	}{
+		{
+			desc: "empty request",
+			code: codes.InvalidArgument,
+			msg:  "no query given",
+		},
+		{
+			desc:  "only query given",
+			query: "foo",
+			code:  codes.InvalidArgument,
+			msg:   "no ref given",
+		},
+		{
+			desc:  "no repo",
+			query: "foo",
+			ref:   "master",
+			code:  codes.InvalidArgument,
+			msg:   "empty Repo",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.desc, func(t *testing.T) {
+
+			stream, err := client.SearchFilesByName(ctx, &pb.SearchFilesByNameRequest{
+				Repository: tc.repo,
+				Query:      tc.query,
+				Ref:        []byte(tc.ref),
+			})
+			require.NoError(t, err)
+
+			_, err = consumeFilenameByName(stream)
+			testhelper.AssertGrpcError(t, err, tc.code, tc.msg)
+		})
+	}
+}
+
+func consumeFilenameByContent(stream pb.RepositoryService_SearchFilesByContentClient) ([][]byte, error) {
+	ret := make([][]byte, 0)
+	for done := false; !done; {
+		resp, err := stream.Recv()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return nil, err
+		}
+		ret = append(ret, resp.Matches...)
+	}
+	return ret, nil
+}
+
+func consumeFilenameByName(stream pb.RepositoryService_SearchFilesByNameClient) ([][]byte, error) {
+	ret := make([][]byte, 0)
+	for done := false; !done; {
+		resp, err := stream.Recv()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return nil, err
+		}
+		ret = append(ret, resp.Files...)
+	}
+	return ret, nil
+}