internalgitaly: WalkRepos to return last accessed timestamp

In order to make a decision on which repositories to clean up,
Praefect's repocleaner will match repository paths with what it sees in
the database. If it doesn't see a record in the database, that means
Praefect doesn't know about this repository.

However, it could be the case that a repository has just been created
and the record for it doesn't yet exist in the database.

In order for repocleaner to know, it needs the information to be
returned from Gitaly.

Add a field in the RPC response, and return the last modified date on the
refs/ dir so we can use it as a proxy for when the repository was
created.

5 files changed, 83 insertions, 37 deletions

diff --git a/internal/gitaly/service/internalgitaly/walkrepos.go b/internal/gitaly/service/internalgitaly/walkrepos.go
index 08979f92b..71a023fdd 100644
--- a/internal/gitaly/service/internalgitaly/walkrepos.go
+++ b/internal/gitaly/service/internalgitaly/walkrepos.go
@@ -9,6 +9,7 @@ import (
 	"gitlab.com/gitlab-org/gitaly/v14/proto/go/gitalypb"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
+	"google.golang.org/protobuf/types/known/timestamppb"
 )
 
 func (s *server) WalkRepos(req *gitalypb.WalkReposRequest, stream gitalypb.InternalGitaly_WalkReposServer) error {
@@ -55,8 +56,14 @@ func walkStorage(ctx context.Context, storagePath string, stream gitalypb.Intern
 				return err
 			}
 
+			gitDirInfo, err := os.Stat(path)
+			if err != nil {
+				return err
+			}
+
 			if err := stream.Send(&gitalypb.WalkReposResponse{
-				RelativePath: relPath,
+				RelativePath:     relPath,
+				ModificationTime: timestamppb.New(gitDirInfo.ModTime()),
 			}); err != nil {
 				return err
 			}
diff --git a/internal/gitaly/service/internalgitaly/walkrepos_test.go b/internal/gitaly/service/internalgitaly/walkrepos_test.go
index c4aad9fcd..23a3aaec4 100644
--- a/internal/gitaly/service/internalgitaly/walkrepos_test.go
+++ b/internal/gitaly/service/internalgitaly/walkrepos_test.go
@@ -6,6 +6,7 @@ import (
 	"path/filepath"
 	"sync"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/require"
 	"gitlab.com/gitlab-org/gitaly/v14/internal/git/gittest"
@@ -42,16 +43,26 @@ func TestWalkRepos(t *testing.T) {
 
 	// file walk happens lexicographically, so we delete repository in the middle
 	// of the seqeuence to ensure the walk proceeds normally
-	testRepo1, _ := gittest.CloneRepo(t, cfg, cfg.Storages[0], gittest.CloneRepoOpts{
+	testRepo1, testRepo1Path := gittest.CloneRepo(t, cfg, cfg.Storages[0], gittest.CloneRepoOpts{
 		RelativePath: "a",
 	})
 	deletedRepo, _ := gittest.CloneRepo(t, cfg, cfg.Storages[0], gittest.CloneRepoOpts{
 		RelativePath: "b",
 	})
-	testRepo2, _ := gittest.CloneRepo(t, cfg, cfg.Storages[0], gittest.CloneRepoOpts{
+	testRepo2, testRepo2Path := gittest.CloneRepo(t, cfg, cfg.Storages[0], gittest.CloneRepoOpts{
 		RelativePath: "c",
 	})
 
+	modifiedDate := time.Now().Add(-1 * time.Hour)
+	require.NoError(
+		t,
+		os.Chtimes(testRepo1Path, time.Now(), modifiedDate),
+	)
+	require.NoError(
+		t,
+		os.Chtimes(testRepo2Path, time.Now(), modifiedDate),
+	)
+
 	// to test a directory being deleted during a walk, we must delete a directory after
 	// the file walk has started. To achieve that, we wrap the server to pass down a wrapped
 	// stream that allows us to hook in to stream responses. We then delete 'b' when
@@ -93,14 +104,14 @@ func TestWalkRepos(t *testing.T) {
 	require.NoError(t, err)
 
 	actualRepos := consumeWalkReposStream(t, stream)
-	require.Equal(t, []string{
-		testRepo1.GetRelativePath(),
-		testRepo2.GetRelativePath(),
-	}, actualRepos)
+	require.Equal(t, testRepo1.GetRelativePath(), actualRepos[0].GetRelativePath())
+	require.Equal(t, modifiedDate.UTC(), actualRepos[0].GetModificationTime().AsTime())
+	require.Equal(t, testRepo2.GetRelativePath(), actualRepos[1].GetRelativePath())
+	require.Equal(t, modifiedDate.UTC(), actualRepos[1].GetModificationTime().AsTime())
 }
 
-func consumeWalkReposStream(t *testing.T, stream gitalypb.InternalGitaly_WalkReposClient) []string {
-	var repos []string
+func consumeWalkReposStream(t *testing.T, stream gitalypb.InternalGitaly_WalkReposClient) []*gitalypb.WalkReposResponse {
+	var repos []*gitalypb.WalkReposResponse
 	for {
 		resp, err := stream.Recv()
 		if err == io.EOF {
@@ -108,7 +119,7 @@ func consumeWalkReposStream(t *testing.T, stream gitalypb.InternalGitaly_WalkRep
 		} else {
 			require.NoError(t, err)
 		}
-		repos = append(repos, resp.RelativePath)
+		repos = append(repos, resp)
 	}
 	return repos
 }
diff --git a/proto/go/gitalypb/internal.pb.go b/proto/go/gitalypb/internal.pb.go
index b5ddcb5ef..d7b604b54 100644
--- a/proto/go/gitalypb/internal.pb.go
+++ b/proto/go/gitalypb/internal.pb.go
@@ -9,6 +9,7 @@ package gitalypb
 import (
 	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
 	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	timestamppb "google.golang.org/protobuf/types/known/timestamppb"
 	reflect "reflect"
 	sync "sync"
 )
@@ -73,6 +74,10 @@ type WalkReposResponse struct {
 	unknownFields protoimpl.UnknownFields
 
 	RelativePath string `protobuf:"bytes,1,opt,name=relative_path,json=relativePath,proto3" json:"relative_path,omitempty"`
+	// modification_time is the modification time of the repository directory.
+	// This can be used as a proxy for when the repository was last
+	// modified.
+	ModificationTime *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=modification_time,json=modificationTime,proto3" json:"modification_time,omitempty"`
 }
 
 func (x *WalkReposResponse) Reset() {
@@ -114,29 +119,43 @@ func (x *WalkReposResponse) GetRelativePath() string {
 	return ""
 }
 
+func (x *WalkReposResponse) GetModificationTime() *timestamppb.Timestamp {
+	if x != nil {
+		return x.ModificationTime
+	}
+	return nil
+}
+
 var File_internal_proto protoreflect.FileDescriptor
 
 var file_internal_proto_rawDesc = []byte{
 	0x0a, 0x0e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f,
 	0x12, 0x06, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x1a, 0x0a, 0x6c, 0x69, 0x6e, 0x74, 0x2e, 0x70,
-	0x72, 0x6f, 0x74, 0x6f, 0x22, 0x3b, 0x0a, 0x10, 0x57, 0x61, 0x6c, 0x6b, 0x52, 0x65, 0x70, 0x6f,
-	0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x27, 0x0a, 0x0c, 0x73, 0x74, 0x6f, 0x72,
-	0x61, 0x67, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x04,
-	0x88, 0xc6, 0x2c, 0x01, 0x52, 0x0b, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4e, 0x61, 0x6d,
-	0x65, 0x22, 0x38, 0x0a, 0x11, 0x57, 0x61, 0x6c, 0x6b, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x52, 0x65,
-	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x72, 0x65, 0x6c, 0x61, 0x74, 0x69,
-	0x76, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x72,
-	0x65, 0x6c, 0x61, 0x74, 0x69, 0x76, 0x65, 0x50, 0x61, 0x74, 0x68, 0x32, 0x5e, 0x0a, 0x0e, 0x49,
-	0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x47, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x12, 0x4c, 0x0a,
-	0x09, 0x57, 0x61, 0x6c, 0x6b, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x12, 0x18, 0x2e, 0x67, 0x69, 0x74,
-	0x61, 0x6c, 0x79, 0x2e, 0x57, 0x61, 0x6c, 0x6b, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x52, 0x65, 0x71,
-	0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x57, 0x61,
-	0x6c, 0x6b, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22,
-	0x08, 0xfa, 0x97, 0x28, 0x04, 0x08, 0x02, 0x10, 0x02, 0x30, 0x01, 0x42, 0x34, 0x5a, 0x32, 0x67,
-	0x69, 0x74, 0x6c, 0x61, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x69, 0x74, 0x6c, 0x61, 0x62,
-	0x2d, 0x6f, 0x72, 0x67, 0x2f, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2f, 0x76, 0x31, 0x34, 0x2f,
-	0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x67, 0x6f, 0x2f, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x70,
-	0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f,
+	0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e,
+	0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x3b, 0x0a, 0x10, 0x57, 0x61, 0x6c, 0x6b, 0x52, 0x65, 0x70,
+	0x6f, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x27, 0x0a, 0x0c, 0x73, 0x74, 0x6f,
+	0x72, 0x61, 0x67, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42,
+	0x04, 0x88, 0xc6, 0x2c, 0x01, 0x52, 0x0b, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4e, 0x61,
+	0x6d, 0x65, 0x22, 0x81, 0x01, 0x0a, 0x11, 0x57, 0x61, 0x6c, 0x6b, 0x52, 0x65, 0x70, 0x6f, 0x73,
+	0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x72, 0x65, 0x6c, 0x61,
+	0x74, 0x69, 0x76, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52,
+	0x0c, 0x72, 0x65, 0x6c, 0x61, 0x74, 0x69, 0x76, 0x65, 0x50, 0x61, 0x74, 0x68, 0x12, 0x47, 0x0a,
+	0x11, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x69,
+	0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c,
+	0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73,
+	0x74, 0x61, 0x6d, 0x70, 0x52, 0x10, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69,
+	0x6f, 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x32, 0x5e, 0x0a, 0x0e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e,
+	0x61, 0x6c, 0x47, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x12, 0x4c, 0x0a, 0x09, 0x57, 0x61, 0x6c, 0x6b,
+	0x52, 0x65, 0x70, 0x6f, 0x73, 0x12, 0x18, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x57,
+	0x61, 0x6c, 0x6b, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a,
+	0x19, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x57, 0x61, 0x6c, 0x6b, 0x52, 0x65, 0x70,
+	0x6f, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x08, 0xfa, 0x97, 0x28, 0x04,
+	0x08, 0x02, 0x10, 0x02, 0x30, 0x01, 0x42, 0x34, 0x5a, 0x32, 0x67, 0x69, 0x74, 0x6c, 0x61, 0x62,
+	0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x2d, 0x6f, 0x72, 0x67, 0x2f,
+	0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2f, 0x76, 0x31, 0x34, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f,
+	0x2f, 0x67, 0x6f, 0x2f, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72,
+	0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (
@@ -153,17 +172,19 @@ func file_internal_proto_rawDescGZIP() []byte {
 
 var file_internal_proto_msgTypes = make([]protoimpl.MessageInfo, 2)
 var file_internal_proto_goTypes = []interface{}{
-	(*WalkReposRequest)(nil),  // 0: gitaly.WalkReposRequest
-	(*WalkReposResponse)(nil), // 1: gitaly.WalkReposResponse
+	(*WalkReposRequest)(nil),      // 0: gitaly.WalkReposRequest
+	(*WalkReposResponse)(nil),     // 1: gitaly.WalkReposResponse
+	(*timestamppb.Timestamp)(nil), // 2: google.protobuf.Timestamp
 }
 var file_internal_proto_depIdxs = []int32{
-	0, // 0: gitaly.InternalGitaly.WalkRepos:input_type -> gitaly.WalkReposRequest
-	1, // 1: gitaly.InternalGitaly.WalkRepos:output_type -> gitaly.WalkReposResponse
-	1, // [1:2] is the sub-list for method output_type
-	0, // [0:1] is the sub-list for method input_type
-	0, // [0:0] is the sub-list for extension type_name
-	0, // [0:0] is the sub-list for extension extendee
-	0, // [0:0] is the sub-list for field type_name
+	2, // 0: gitaly.WalkReposResponse.modification_time:type_name -> google.protobuf.Timestamp
+	0, // 1: gitaly.InternalGitaly.WalkRepos:input_type -> gitaly.WalkReposRequest
+	1, // 2: gitaly.InternalGitaly.WalkRepos:output_type -> gitaly.WalkReposResponse
+	2, // [2:3] is the sub-list for method output_type
+	1, // [1:2] is the sub-list for method input_type
+	1, // [1:1] is the sub-list for extension type_name
+	1, // [1:1] is the sub-list for extension extendee
+	0, // [0:1] is the sub-list for field type_name
 }
 
 func init() { file_internal_proto_init() }
diff --git a/proto/internal.proto b/proto/internal.proto
index 33e1d96ae..1cacb3bc5 100644
--- a/proto/internal.proto
+++ b/proto/internal.proto
@@ -5,6 +5,7 @@ package gitaly;
 option go_package = "gitlab.com/gitlab-org/gitaly/v14/proto/go/gitalypb";
 
 import "lint.proto";
+import "google/protobuf/timestamp.proto";
 
 // InternalGitaly is a gRPC service meant to be served by a Gitaly node, but
 // only reachable by Praefect or other Gitalies
@@ -25,4 +26,8 @@ message WalkReposRequest {
 
 message WalkReposResponse {
   string relative_path = 1;
+  // modification_time is the modification time of the repository directory.
+  // This can be used as a proxy for when the repository was last
+  // modified.
+  google.protobuf.Timestamp modification_time = 2;
 }
diff --git a/ruby/proto/gitaly/internal_pb.rb b/ruby/proto/gitaly/internal_pb.rb
index 0b9206b10..ecf1659b4 100644
--- a/ruby/proto/gitaly/internal_pb.rb
+++ b/ruby/proto/gitaly/internal_pb.rb
@@ -2,6 +2,7 @@
 # source: internal.proto
 
 require 'lint_pb'
+require 'google/protobuf/timestamp_pb'
 require 'google/protobuf'
 
 Google::Protobuf::DescriptorPool.generated_pool.build do
@@ -11,6 +12,7 @@ Google::Protobuf::DescriptorPool.generated_pool.build do
     end
     add_message "gitaly.WalkReposResponse" do
       optional :relative_path, :string, 1
+      optional :modification_time, :message, 2, "google.protobuf.Timestamp"
     end
   end
 end