Merge branch '703-server-implementation-wikiservice-wikigetallpages' into 'master'

Server implementation WikiService::WikiGetAllPages

Closes #703

See merge request gitlab-org/gitaly!440

5 files changed, 230 insertions, 23 deletions

diff --git a/internal/service/wiki/find_page_test.go b/internal/service/wiki/find_page_test.go
index 2605d99a5..6318d1149 100644
--- a/internal/service/wiki/find_page_test.go
+++ b/internal/service/wiki/find_page_test.go
@@ -1,11 +1,9 @@
 package wiki
 
 import (
-	"bytes"
 	"io"
 	"testing"
 
-	gitlog "gitlab.com/gitlab-org/gitaly/internal/git/log"
 	"gitlab.com/gitlab-org/gitaly/internal/testhelper"
 
 	pb "gitlab.com/gitlab-org/gitaly-proto/go"
@@ -18,32 +16,18 @@ func TestSuccessfulWikiFindPageRequest(t *testing.T) {
 	wikiRepo, cleanupFunc := setupWikiRepo()
 	defer cleanupFunc()
 
-	ctx, cancel := testhelper.Context()
-	defer cancel()
-
 	server, serverSocketPath := runWikiServiceServer(t)
 	defer server.Stop()
 
 	client, conn := newWikiClient(t, serverSocketPath)
 	defer conn.Close()
 
-	content := bytes.Repeat([]byte("Mock wiki page content"), 10000)
-
 	page1Name := "Home Page"
 	page2Name := "Installing/Step 133-b"
 	page3Name := "Installing/Step 133-c"
-
-	writeWikiPage(t, client, wikiRepo, page1Name, content)
-	head1ID := testhelper.MustRunCommand(t, nil, "git", "-C", wikiRepoPath, "show", "--format=format:%H", "--no-patch", "HEAD")
-	page1Commit, err := gitlog.GetCommit(ctx, wikiRepo, string(head1ID), "")
-	require.NoError(t, err, "look up git commit after writing a wiki page")
-
-	writeWikiPage(t, client, wikiRepo, page2Name, content)
-
-	writeWikiPage(t, client, wikiRepo, page3Name, content)
-	head3ID := testhelper.MustRunCommand(t, nil, "git", "-C", wikiRepoPath, "show", "--format=format:%H", "--no-patch", "HEAD")
-	page3Commit, err := gitlog.GetCommit(ctx, wikiRepo, string(head3ID), "")
-	require.NoError(t, err, "look up git commit after writing a wiki page")
+	page1Commit := createTestWikiPage(t, client, wikiRepo, page1Name)
+	createTestWikiPage(t, client, wikiRepo, page2Name)
+	page3Commit := createTestWikiPage(t, client, wikiRepo, page3Name)
 
 	testCases := []struct {
 		desc         string
@@ -149,7 +133,7 @@ func TestSuccessfulWikiFindPageRequest(t *testing.T) {
 
 			require.Equal(t, expectedPage, receivedPage, "mismatched page attributes")
 			if expectedPage != nil {
-				require.Equal(t, content, receivedContent, "mismatched page content")
+				require.Equal(t, mockPageContent, receivedContent, "mismatched page content")
 			}
 		})
 	}
diff --git a/internal/service/wiki/get_all_pages.go b/internal/service/wiki/get_all_pages.go
index 524e05733..5aa430d20 100644
--- a/internal/service/wiki/get_all_pages.go
+++ b/internal/service/wiki/get_all_pages.go
@@ -1,10 +1,36 @@
 package wiki
 
 import (
+	"gitlab.com/gitlab-org/gitaly/internal/rubyserver"
+
 	pb "gitlab.com/gitlab-org/gitaly-proto/go"
-	"gitlab.com/gitlab-org/gitaly/internal/helper"
 )
 
 func (s *server) WikiGetAllPages(request *pb.WikiGetAllPagesRequest, stream pb.WikiService_WikiGetAllPagesServer) error {
-	return helper.Unimplemented
+	ctx := stream.Context()
+
+	client, err := s.WikiServiceClient(ctx)
+	if err != nil {
+		return err
+	}
+
+	clientCtx, err := rubyserver.SetHeaders(ctx, request.GetRepository())
+	if err != nil {
+		return err
+	}
+
+	rubyStream, err := client.WikiGetAllPages(clientCtx, request)
+	if err != nil {
+		return err
+	}
+
+	return rubyserver.Proxy(func() error {
+		resp, err := rubyStream.Recv()
+		if err != nil {
+			md := rubyStream.Trailer()
+			stream.SetTrailer(md)
+			return err
+		}
+		return stream.Send(resp)
+	})
 }
diff --git a/internal/service/wiki/get_all_pages_test.go b/internal/service/wiki/get_all_pages_test.go
new file mode 100644
index 000000000..f31ea38a1
--- /dev/null
+++ b/internal/service/wiki/get_all_pages_test.go
@@ -0,0 +1,122 @@
+package wiki
+
+import (
+	"io"
+	"testing"
+
+	"google.golang.org/grpc/codes"
+
+	"github.com/stretchr/testify/require"
+
+	pb "gitlab.com/gitlab-org/gitaly-proto/go"
+	"gitlab.com/gitlab-org/gitaly/internal/testhelper"
+)
+
+func TestSuccessfulWikiGetAllPagesRequest(t *testing.T) {
+	ctx, cancel := testhelper.Context()
+	defer cancel()
+
+	server, serverSocketPath := runWikiServiceServer(t)
+	defer server.Stop()
+
+	client, conn := newWikiClient(t, serverSocketPath)
+	defer conn.Close()
+
+	wikiRepo, cleanupFunc := setupWikiRepo()
+	defer cleanupFunc()
+
+	page1Name := "Page 1"
+	page2Name := "Page 2"
+	createTestWikiPage(t, client, wikiRepo, page1Name)
+	page2Commit := createTestWikiPage(t, client, wikiRepo, page2Name)
+	expectedPage1 := &pb.WikiPage{
+		Version:    &pb.WikiPageVersion{Commit: page2Commit, Format: "markdown"},
+		Title:      []byte(page1Name),
+		Format:     "markdown",
+		UrlPath:    "Page-1",
+		Path:       []byte("Page-1.md"),
+		Name:       []byte(page1Name),
+		RawData:    mockPageContent,
+		Historical: false,
+	}
+	expectedPage2 := &pb.WikiPage{
+		Version:    &pb.WikiPageVersion{Commit: page2Commit, Format: "markdown"},
+		Title:      []byte(page2Name),
+		Format:     "markdown",
+		UrlPath:    "Page-2",
+		Path:       []byte("Page-2.md"),
+		Name:       []byte(page2Name),
+		RawData:    mockPageContent,
+		Historical: false,
+	}
+
+	request := &pb.WikiGetAllPagesRequest{Repository: wikiRepo}
+	c, err := client.WikiGetAllPages(ctx, request)
+	require.NoError(t, err)
+
+	receivedPages := readWikiPagesFromWikiGetAllPagesClient(t, c)
+
+	require.Len(t, receivedPages, 2)
+	requireWikiPagesEqual(t, receivedPages[0], expectedPage1)
+	requireWikiPagesEqual(t, receivedPages[1], expectedPage2)
+}
+
+func TestFailedWikiGetAllPagesDueToValidation(t *testing.T) {
+	server, serverSocketPath := runWikiServiceServer(t)
+	defer server.Stop()
+
+	client, conn := newWikiClient(t, serverSocketPath)
+	defer conn.Close()
+
+	rpcRequests := []pb.WikiGetAllPagesRequest{
+		{Repository: &pb.Repository{StorageName: "fake", RelativePath: "path"}}, // Repository doesn't exist
+		{Repository: nil}, // Repository is nil
+	}
+
+	for _, rpcRequest := range rpcRequests {
+		ctx, cancel := testhelper.Context()
+		defer cancel()
+
+		c, err := client.WikiGetAllPages(ctx, &rpcRequest)
+		require.NoError(t, err)
+
+		err = drainWikiGetAllPagesResponse(c)
+		testhelper.AssertGrpcError(t, err, codes.InvalidArgument, "")
+	}
+}
+
+func readWikiPagesFromWikiGetAllPagesClient(t *testing.T, c pb.WikiService_WikiGetAllPagesClient) []*pb.WikiPage {
+	var wikiPage *pb.WikiPage
+	var wikiPages []*pb.WikiPage
+
+	for {
+		resp, err := c.Recv()
+		if err == io.EOF {
+			break
+		} else if err != nil {
+			t.Fatal(err)
+		}
+
+		if resp.EndOfPage {
+			wikiPages = append(wikiPages, wikiPage)
+			wikiPage = nil
+		} else if wikiPage == nil {
+			wikiPage = resp.GetPage()
+		} else {
+			wikiPage.RawData = append(wikiPage.RawData, resp.GetPage().GetRawData()...)
+		}
+	}
+
+	return wikiPages
+}
+
+func drainWikiGetAllPagesResponse(c pb.WikiService_WikiGetAllPagesClient) error {
+	for {
+		_, err := c.Recv()
+		if err == io.EOF {
+			return nil
+		} else if err != nil {
+			return err
+		}
+	}
+}
diff --git a/internal/service/wiki/testhelper_test.go b/internal/service/wiki/testhelper_test.go
index b79ec0c37..1d4952341 100644
--- a/internal/service/wiki/testhelper_test.go
+++ b/internal/service/wiki/testhelper_test.go
@@ -1,12 +1,14 @@
 package wiki
 
 import (
+	"bytes"
 	"net"
 	"os"
 	"path"
 	"testing"
 	"time"
 
+	gitlog "gitlab.com/gitlab-org/gitaly/internal/git/log"
 	"gitlab.com/gitlab-org/gitaly/internal/rubyserver"
 	"gitlab.com/gitlab-org/gitaly/internal/testhelper"
 
@@ -19,7 +21,8 @@ import (
 )
 
 var (
-	wikiRepoPath string
+	wikiRepoPath    string
+	mockPageContent = bytes.Repeat([]byte("Mock wiki page content"), 10000)
 )
 
 func TestMain(m *testing.M) {
@@ -159,3 +162,39 @@ func sendBytes(data []byte, chunkSize int, sender func([]byte) error) (int, erro
 
 	return i, nil
 }
+
+func createTestWikiPage(t *testing.T, client pb.WikiServiceClient, wikiRepo *pb.Repository, pageName string) *pb.GitCommit {
+	ctx, cancel := testhelper.Context()
+	defer cancel()
+
+	writeWikiPage(t, client, wikiRepo, pageName, mockPageContent)
+	head1ID := testhelper.MustRunCommand(t, nil, "git", "-C", wikiRepoPath, "show", "--format=format:%H", "--no-patch", "HEAD")
+	pageCommit, err := gitlog.GetCommit(ctx, wikiRepo, string(head1ID), "")
+	require.NoError(t, err, "look up git commit after writing a wiki page")
+
+	return pageCommit
+}
+
+func requireWikiPagesEqual(t *testing.T, expectedPage *pb.WikiPage, actualPage *pb.WikiPage) {
+	// require.Equal doesn't display a proper diff when either expected/actual has a field
+	// with large data (RawData in our case), so we compare file attributes and content separately.
+	expectedContent := expectedPage.GetRawData()
+	if expectedPage != nil {
+		expectedPage.RawData = nil
+		defer func() {
+			expectedPage.RawData = expectedContent
+		}()
+	}
+	actualContent := actualPage.GetRawData()
+	if actualPage != nil {
+		actualPage.RawData = nil
+		defer func() {
+			actualPage.RawData = actualContent
+		}()
+	}
+
+	require.Equal(t, expectedPage, actualPage, "mismatched page attributes")
+	if expectedPage != nil {
+		require.Equal(t, expectedContent, actualContent, "mismatched page content")
+	}
+}
diff --git a/ruby/lib/gitaly_server/wiki_service.rb b/ruby/lib/gitaly_server/wiki_service.rb
index 13353305e..3707e5056 100644
--- a/ruby/lib/gitaly_server/wiki_service.rb
+++ b/ruby/lib/gitaly_server/wiki_service.rb
@@ -88,6 +88,42 @@ module GitalyServer
       end
     end
 
+    def wiki_get_all_pages(request, call)
+      bridge_exceptions do
+        repo = Gitlab::Git::Repository.from_call(call)
+        wiki = Gitlab::Git::Wiki.new(repo)
+
+        Enumerator.new do |y|
+          wiki.pages.each do |page|
+            version = Gitaly::WikiPageVersion.new(
+              commit: gitaly_commit_from_rugged(page.version.commit.raw_commit),
+              format: page.version.format.to_s
+            )
+            gitaly_wiki_page = Gitaly::WikiPage.new(
+              version: version,
+              format: page.format.to_s,
+              title: page.title.b,
+              url_path: page.url_path.to_s,
+              path: page.path.b,
+              name: page.name.b,
+              historical: page.historical?
+            )
+
+            io = StringIO.new(page.text_data)
+            while chunk = io.read(Gitlab.config.git.write_buffer_size)
+              gitaly_wiki_page.raw_data = chunk
+
+              y.yield Gitaly::WikiGetAllPagesResponse.new(page: gitaly_wiki_page)
+
+              gitaly_wiki_page = Gitaly::WikiPage.new
+            end
+
+            y.yield Gitaly::WikiGetAllPagesResponse.new(end_of_page: true)
+          end
+        end
+      end
+    end
+
     def wiki_find_file(request, call)
       bridge_exceptions do
         repo = Gitlab::Git::Repository.from_call(call)