From 41529bbce407fbf1a925cfbc7c1aa30064f66ae7 Mon Sep 17 00:00:00 2001 From: David Barr Date: Sat, 5 Mar 2011 13:30:23 +1100 Subject: vcs-svn: set up channel to read fast-import cat-blob response Set up some plumbing: teach the svndump lib to pass a file descriptor number to the fast_export lib, representing where cat-blob/ls responses can be read from, and add a get_response_line helper function to the fast_export lib to read a line from that file. Unfortunately this means that svn-fe needs file descriptor 3 to be redirected from somewhere (preferrably the cat-blob stream of a fast-import backend); otherwise it will fail: $ svndump | svn-fe fatal: cannot read from file descriptor 3: Bad file descriptor For the moment, "svn-fe 3 Signed-off-by: David Barr Signed-off-by: Jonathan Nieder --- vcs-svn/fast_export.c | 28 ++++++++++++++++++++++++++++ vcs-svn/fast_export.h | 4 ++++ vcs-svn/svndump.c | 5 +++++ 3 files changed, 37 insertions(+) (limited to 'vcs-svn') diff --git a/vcs-svn/fast_export.c b/vcs-svn/fast_export.c index 260cf50e77..70bd9597e7 100644 --- a/vcs-svn/fast_export.c +++ b/vcs-svn/fast_export.c @@ -12,6 +12,24 @@ #define MAX_GITSVN_LINE_LEN 4096 static uint32_t first_commit_done; +static struct line_buffer report_buffer = LINE_BUFFER_INIT; + +void fast_export_init(int fd) +{ + if (buffer_fdinit(&report_buffer, fd)) + die_errno("cannot read from file descriptor %d", fd); +} + +void fast_export_deinit(void) +{ + if (buffer_deinit(&report_buffer)) + die_errno("error closing fast-import feedback stream"); +} + +void fast_export_reset(void) +{ + buffer_reset(&report_buffer); +} void fast_export_delete(uint32_t depth, uint32_t *path) { @@ -63,6 +81,16 @@ void fast_export_commit(uint32_t revision, uint32_t author, char *log, printf("progress Imported commit %"PRIu32".\n\n", revision); } +static const char *get_response_line(void) +{ + const char *line = buffer_read_line(&report_buffer); + if (line) + return line; + if (buffer_ferror(&report_buffer)) + die_errno("error reading from fast-import"); + die("unexpected end of fast-import feedback"); +} + void fast_export_blob(uint32_t mode, uint32_t mark, uint32_t len, struct line_buffer *input) { if (mode == REPO_MODE_LNK) { diff --git a/vcs-svn/fast_export.h b/vcs-svn/fast_export.h index 054e7d5eb1..fed30c14e6 100644 --- a/vcs-svn/fast_export.h +++ b/vcs-svn/fast_export.h @@ -3,6 +3,10 @@ #include "line_buffer.h" +void fast_export_init(int fd); +void fast_export_deinit(void); +void fast_export_reset(void); + void fast_export_delete(uint32_t depth, uint32_t *path); void fast_export_modify(uint32_t depth, uint32_t *path, uint32_t mode, uint32_t mark); diff --git a/vcs-svn/svndump.c b/vcs-svn/svndump.c index e6d84bada5..e05a99d51f 100644 --- a/vcs-svn/svndump.c +++ b/vcs-svn/svndump.c @@ -14,6 +14,8 @@ #include "obj_pool.h" #include "string_pool.h" +#define REPORT_FILENO 3 + #define NODEACT_REPLACE 4 #define NODEACT_DELETE 3 #define NODEACT_ADD 2 @@ -367,6 +369,7 @@ int svndump_init(const char *filename) if (buffer_init(&input, filename)) return error("cannot open %s: %s", filename, strerror(errno)); repo_init(); + fast_export_init(REPORT_FILENO); reset_dump_ctx(~0); reset_rev_ctx(0); reset_node_ctx(NULL); @@ -377,6 +380,7 @@ int svndump_init(const char *filename) void svndump_deinit(void) { log_reset(); + fast_export_deinit(); repo_reset(); reset_dump_ctx(~0); reset_rev_ctx(0); @@ -390,6 +394,7 @@ void svndump_deinit(void) void svndump_reset(void) { log_reset(); + fast_export_reset(); buffer_reset(&input); repo_reset(); reset_dump_ctx(~0); -- cgit v1.2.3 From d38f84484f21e7e509ff009d3a17167c9c09f893 Mon Sep 17 00:00:00 2001 From: Jonathan Nieder Date: Fri, 10 Dec 2010 04:21:35 -0600 Subject: vcs-svn: use higher mark numbers for blobs Prepare to use mark :5 for the commit corresponding to r5 (and so on). 1 billion seems sufficiently high for blob marks to avoid conflicting with rev marks, while still leaving room for 3 billion blobs. Such high mark numbers cause trouble with ancient fast-import versions, but this topic cannot support git fast-import versions before 1.7.4 (which introduces the cat-blob command) anyway. Signed-off-by: Jonathan Nieder --- vcs-svn/repo_tree.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'vcs-svn') diff --git a/vcs-svn/repo_tree.c b/vcs-svn/repo_tree.c index 14bcc192b6..036a6866b9 100644 --- a/vcs-svn/repo_tree.c +++ b/vcs-svn/repo_tree.c @@ -292,7 +292,7 @@ void repo_commit(uint32_t revision, uint32_t author, char *log, uint32_t uuid, static void mark_init(void) { uint32_t i; - mark = 0; + mark = 1024 * 1024 * 1024; for (i = 0; i < dent_pool.size; i++) if (!repo_dirent_is_dir(dent_pointer(i)) && dent_pointer(i)->content_offset > mark) -- cgit v1.2.3 From 78e1a3ff236af3afaf1ba9db92985df42141cb0e Mon Sep 17 00:00:00 2001 From: Jonathan Nieder Date: Thu, 9 Dec 2010 18:57:13 -0600 Subject: vcs-svn: save marks for imported commits This way, a person can use svnadmin dump $path | svn-fe | git fast-import --relative-marks --export-marks=svn-revs to get a list of what commit corresponds to each svn revision (plus some irrelevant blob names) in .git/info/fast-import/svn-revs. Signed-off-by: Jonathan Nieder Signed-off-by: David Barr Signed-off-by: Jonathan Nieder --- vcs-svn/fast_export.c | 1 + 1 file changed, 1 insertion(+) (limited to 'vcs-svn') diff --git a/vcs-svn/fast_export.c b/vcs-svn/fast_export.c index 70bd9597e7..0ad5382bfb 100644 --- a/vcs-svn/fast_export.c +++ b/vcs-svn/fast_export.c @@ -63,6 +63,7 @@ void fast_export_commit(uint32_t revision, uint32_t author, char *log, *gitsvnline = '\0'; } printf("commit refs/heads/master\n"); + printf("mark :%"PRIu32"\n", revision); printf("committer %s <%s@%s> %ld +0000\n", ~author ? pool_fetch(author) : "nobody", ~author ? pool_fetch(author) : "nobody", -- cgit v1.2.3 From 7e11902c995715836dec140eb55cfef1d24334bb Mon Sep 17 00:00:00 2001 From: Jonathan Nieder Date: Tue, 4 Jan 2011 21:53:33 -0600 Subject: vcs-svn: add a comment before each commit Current svn-fe produces output like this: blob mark :7382321 data 5 hello blob mark :7382322 data 5 Hello commit mark :3 [...] M 100644 :7382321 hello.c M 100644 :7382322 hello2.c This means svn-fe has to keep track of the paths modified in each commit and the corresponding marks, instead of dealing with each file as it arrives in input and then forgetting about it. A better strategy would be to use inline blobs: commit mark :3 [...] M 100644 inline hello.c data 5 hello [...] As a first step towards that, teach svn-fe to notice when the collection of blobs for each commit starts and write a comment ("# commit 3.") there. Signed-off-by: Jonathan Nieder --- vcs-svn/fast_export.c | 5 +++++ vcs-svn/fast_export.h | 1 + vcs-svn/svndump.c | 29 ++++++++++++++++++++++------- 3 files changed, 28 insertions(+), 7 deletions(-) (limited to 'vcs-svn') diff --git a/vcs-svn/fast_export.c b/vcs-svn/fast_export.c index 0ad5382bfb..8786ed234a 100644 --- a/vcs-svn/fast_export.c +++ b/vcs-svn/fast_export.c @@ -48,6 +48,11 @@ void fast_export_modify(uint32_t depth, uint32_t *path, uint32_t mode, putchar('\n'); } +void fast_export_begin_commit(uint32_t revision) +{ + printf("# commit %"PRIu32".\n", revision); +} + static char gitsvnline[MAX_GITSVN_LINE_LEN]; void fast_export_commit(uint32_t revision, uint32_t author, char *log, uint32_t uuid, uint32_t url, diff --git a/vcs-svn/fast_export.h b/vcs-svn/fast_export.h index fed30c14e6..09b2033772 100644 --- a/vcs-svn/fast_export.h +++ b/vcs-svn/fast_export.h @@ -10,6 +10,7 @@ void fast_export_reset(void); void fast_export_delete(uint32_t depth, uint32_t *path); void fast_export_modify(uint32_t depth, uint32_t *path, uint32_t mode, uint32_t mark); +void fast_export_begin_commit(uint32_t revision); void fast_export_commit(uint32_t revision, uint32_t author, char *log, uint32_t uuid, uint32_t url, unsigned long timestamp); void fast_export_blob(uint32_t mode, uint32_t mark, uint32_t len, diff --git a/vcs-svn/svndump.c b/vcs-svn/svndump.c index e05a99d51f..3cc4135892 100644 --- a/vcs-svn/svndump.c +++ b/vcs-svn/svndump.c @@ -22,9 +22,11 @@ #define NODEACT_CHANGE 1 #define NODEACT_UNKNOWN 0 -#define DUMP_CTX 0 -#define REV_CTX 1 -#define NODE_CTX 2 +/* States: */ +#define DUMP_CTX 0 /* dump metadata */ +#define REV_CTX 1 /* revision metadata */ +#define NODE_CTX 2 /* node metadata */ +#define INTERNODE_CTX 3 /* between nodes */ #define LENGTH_UNKNOWN (~0) #define DATE_RFC2822_LEN 31 @@ -269,7 +271,14 @@ static void handle_node(void) node_ctx.textLength, &input); } -static void handle_revision(void) +static void begin_revision(void) +{ + if (!rev_ctx.revision) /* revision 0 gets no git commit. */ + return; + fast_export_begin_commit(rev_ctx.revision); +} + +static void end_revision(void) { if (rev_ctx.revision) repo_commit(rev_ctx.revision, rev_ctx.author, rev_ctx.log, @@ -303,13 +312,17 @@ void svndump_read(const char *url) } else if (key == keys.revision_number) { if (active_ctx == NODE_CTX) handle_node(); + if (active_ctx == REV_CTX) + begin_revision(); if (active_ctx != DUMP_CTX) - handle_revision(); + end_revision(); active_ctx = REV_CTX; reset_rev_ctx(atoi(val)); } else if (key == keys.node_path) { if (active_ctx == NODE_CTX) handle_node(); + if (active_ctx == REV_CTX) + begin_revision(); active_ctx = NODE_CTX; reset_node_ctx(val); } else if (key == keys.node_kind) { @@ -351,7 +364,7 @@ void svndump_read(const char *url) read_props(); } else if (active_ctx == NODE_CTX) { handle_node(); - active_ctx = REV_CTX; + active_ctx = INTERNODE_CTX; } else { fprintf(stderr, "Unexpected content length header: %"PRIu32"\n", len); buffer_skip_bytes(&input, len); @@ -360,8 +373,10 @@ void svndump_read(const char *url) } if (active_ctx == NODE_CTX) handle_node(); + if (active_ctx == REV_CTX) + begin_revision(); if (active_ctx != DUMP_CTX) - handle_revision(); + end_revision(); } int svndump_init(const char *filename) -- cgit v1.2.3 From 723b7a2789d66c1365390cc9b9213e34ab8513d7 Mon Sep 17 00:00:00 2001 From: Jonathan Nieder Date: Fri, 10 Dec 2010 04:00:55 -0600 Subject: vcs-svn: eliminate repo_tree structure Rely on fast-import for information about previous revs. This requires always setting up backward flow of information, even for v2 dumps. On the plus side, it simplifies the code by quite a bit and opens the door to further simplifications. [db: adjusted to support final version of the cat-blob patch] [jn: avoiding hard-coding git's name for the empty tree for portability to other backends] Signed-off-by: Jonathan Nieder Signed-off-by: David Barr Signed-off-by: Jonathan Nieder --- vcs-svn/fast_export.c | 108 +++++++++++++--- vcs-svn/fast_export.h | 22 ++-- vcs-svn/repo_tree.c | 335 ++++++-------------------------------------------- vcs-svn/repo_tree.h | 2 +- vcs-svn/string_pool.c | 2 +- vcs-svn/string_pool.h | 2 +- vcs-svn/svndump.c | 53 +++++--- 7 files changed, 184 insertions(+), 340 deletions(-) (limited to 'vcs-svn') diff --git a/vcs-svn/fast_export.c b/vcs-svn/fast_export.c index 8786ed234a..a8ce5c64b2 100644 --- a/vcs-svn/fast_export.c +++ b/vcs-svn/fast_export.c @@ -8,6 +8,7 @@ #include "line_buffer.h" #include "repo_tree.h" #include "string_pool.h" +#include "strbuf.h" #define MAX_GITSVN_LINE_LEN 4096 @@ -31,7 +32,7 @@ void fast_export_reset(void) buffer_reset(&report_buffer); } -void fast_export_delete(uint32_t depth, uint32_t *path) +void fast_export_delete(uint32_t depth, const uint32_t *path) { putchar('D'); putchar(' '); @@ -39,22 +40,27 @@ void fast_export_delete(uint32_t depth, uint32_t *path) putchar('\n'); } -void fast_export_modify(uint32_t depth, uint32_t *path, uint32_t mode, - uint32_t mark) +static void fast_export_truncate(uint32_t depth, const uint32_t *path, uint32_t mode) { - /* Mode must be 100644, 100755, 120000, or 160000. */ - printf("M %06"PRIo32" :%"PRIu32" ", mode, mark); - pool_print_seq(depth, path, '/', stdout); - putchar('\n'); + fast_export_modify(depth, path, mode, "inline"); + printf("data 0\n\n"); } -void fast_export_begin_commit(uint32_t revision) +void fast_export_modify(uint32_t depth, const uint32_t *path, uint32_t mode, + const char *dataref) { - printf("# commit %"PRIu32".\n", revision); + /* Mode must be 100644, 100755, 120000, or 160000. */ + if (!dataref) { + fast_export_truncate(depth, path, mode); + return; + } + printf("M %06"PRIo32" %s ", mode, dataref); + pool_print_seq(depth, path, '/', stdout); + putchar('\n'); } static char gitsvnline[MAX_GITSVN_LINE_LEN]; -void fast_export_commit(uint32_t revision, uint32_t author, char *log, +void fast_export_begin_commit(uint32_t revision, uint32_t author, char *log, uint32_t uuid, uint32_t url, unsigned long timestamp) { @@ -81,12 +87,31 @@ void fast_export_commit(uint32_t revision, uint32_t author, char *log, printf("from refs/heads/master^0\n"); first_commit_done = 1; } - repo_diff(revision - 1, revision); - fputc('\n', stdout); +} +void fast_export_end_commit(uint32_t revision) +{ printf("progress Imported commit %"PRIu32".\n\n", revision); } +static void ls_from_rev(uint32_t rev, uint32_t depth, const uint32_t *path) +{ + /* ls :5 path/to/old/file */ + printf("ls :%"PRIu32" ", rev); + pool_print_seq(depth, path, '/', stdout); + putchar('\n'); + fflush(stdout); +} + +static void ls_from_active_commit(uint32_t depth, const uint32_t *path) +{ + /* ls "path/to/file" */ + printf("ls \""); + pool_print_seq(depth, path, '/', stdout); + printf("\"\n"); + fflush(stdout); +} + static const char *get_response_line(void) { const char *line = buffer_read_line(&report_buffer); @@ -97,14 +122,69 @@ static const char *get_response_line(void) die("unexpected end of fast-import feedback"); } -void fast_export_blob(uint32_t mode, uint32_t mark, uint32_t len, struct line_buffer *input) +void fast_export_data(uint32_t mode, uint32_t len, struct line_buffer *input) { if (mode == REPO_MODE_LNK) { /* svn symlink blobs start with "link " */ buffer_skip_bytes(input, 5); len -= 5; } - printf("blob\nmark :%"PRIu32"\ndata %"PRIu32"\n", mark, len); + printf("data %"PRIu32"\n", len); buffer_copy_bytes(input, len); fputc('\n', stdout); } + +static int parse_ls_response(const char *response, uint32_t *mode, + struct strbuf *dataref) +{ + const char *tab; + const char *response_end; + + assert(response); + response_end = response + strlen(response); + + if (*response == 'm') { /* Missing. */ + errno = ENOENT; + return -1; + } + + /* Mode. */ + if (response_end - response < strlen("100644") || + response[strlen("100644")] != ' ') + die("invalid ls response: missing mode: %s", response); + *mode = 0; + for (; *response != ' '; response++) { + char ch = *response; + if (ch < '0' || ch > '7') + die("invalid ls response: mode is not octal: %s", response); + *mode *= 8; + *mode += ch - '0'; + } + + /* ' blob ' or ' tree ' */ + if (response_end - response < strlen(" blob ") || + (response[1] != 'b' && response[1] != 't')) + die("unexpected ls response: not a tree or blob: %s", response); + response += strlen(" blob "); + + /* Dataref. */ + tab = memchr(response, '\t', response_end - response); + if (!tab) + die("invalid ls response: missing tab: %s", response); + strbuf_add(dataref, response, tab - response); + return 0; +} + +int fast_export_ls_rev(uint32_t rev, uint32_t depth, const uint32_t *path, + uint32_t *mode, struct strbuf *dataref) +{ + ls_from_rev(rev, depth, path); + return parse_ls_response(get_response_line(), mode, dataref); +} + +int fast_export_ls(uint32_t depth, const uint32_t *path, + uint32_t *mode, struct strbuf *dataref) +{ + ls_from_active_commit(depth, path); + return parse_ls_response(get_response_line(), mode, dataref); +} diff --git a/vcs-svn/fast_export.h b/vcs-svn/fast_export.h index 09b2033772..633d21944e 100644 --- a/vcs-svn/fast_export.h +++ b/vcs-svn/fast_export.h @@ -1,19 +1,25 @@ #ifndef FAST_EXPORT_H_ #define FAST_EXPORT_H_ -#include "line_buffer.h" +struct strbuf; +struct line_buffer; void fast_export_init(int fd); void fast_export_deinit(void); void fast_export_reset(void); -void fast_export_delete(uint32_t depth, uint32_t *path); -void fast_export_modify(uint32_t depth, uint32_t *path, uint32_t mode, - uint32_t mark); -void fast_export_begin_commit(uint32_t revision); -void fast_export_commit(uint32_t revision, uint32_t author, char *log, +void fast_export_delete(uint32_t depth, const uint32_t *path); +void fast_export_modify(uint32_t depth, const uint32_t *path, + uint32_t mode, const char *dataref); +void fast_export_begin_commit(uint32_t revision, uint32_t author, char *log, uint32_t uuid, uint32_t url, unsigned long timestamp); -void fast_export_blob(uint32_t mode, uint32_t mark, uint32_t len, - struct line_buffer *input); +void fast_export_end_commit(uint32_t revision); +void fast_export_data(uint32_t mode, uint32_t len, struct line_buffer *input); + +/* If there is no such file at that rev, returns -1, errno == ENOENT. */ +int fast_export_ls_rev(uint32_t rev, uint32_t depth, const uint32_t *path, + uint32_t *mode_out, struct strbuf *dataref_out); +int fast_export_ls(uint32_t depth, const uint32_t *path, + uint32_t *mode_out, struct strbuf *dataref_out); #endif diff --git a/vcs-svn/repo_tree.c b/vcs-svn/repo_tree.c index 036a6866b9..e75f58087c 100644 --- a/vcs-svn/repo_tree.c +++ b/vcs-svn/repo_tree.c @@ -4,322 +4,61 @@ */ #include "git-compat-util.h" - -#include "string_pool.h" +#include "strbuf.h" #include "repo_tree.h" -#include "obj_pool.h" #include "fast_export.h" -#include "trp.h" - -struct repo_dirent { - uint32_t name_offset; - struct trp_node children; - uint32_t mode; - uint32_t content_offset; -}; - -struct repo_dir { - struct trp_root entries; -}; - -struct repo_commit { - uint32_t root_dir_offset; -}; - -/* Memory pools for commit, dir and dirent */ -obj_pool_gen(commit, struct repo_commit, 4096) -obj_pool_gen(dir, struct repo_dir, 4096) -obj_pool_gen(dent, struct repo_dirent, 4096) - -static uint32_t active_commit; -static uint32_t mark; - -static int repo_dirent_name_cmp(const void *a, const void *b); - -/* Treap for directory entries */ -trp_gen(static, dent_, struct repo_dirent, children, dent, repo_dirent_name_cmp); - -uint32_t next_blob_mark(void) +const char *repo_read_path(const uint32_t *path) { - return mark++; -} + int err; + uint32_t dummy; + static struct strbuf buf = STRBUF_INIT; -static struct repo_dir *repo_commit_root_dir(struct repo_commit *commit) -{ - return dir_pointer(commit->root_dir_offset); -} - -static struct repo_dirent *repo_first_dirent(struct repo_dir *dir) -{ - return dent_first(&dir->entries); -} - -static int repo_dirent_name_cmp(const void *a, const void *b) -{ - const struct repo_dirent *dent1 = a, *dent2 = b; - uint32_t a_offset = dent1->name_offset; - uint32_t b_offset = dent2->name_offset; - return (a_offset > b_offset) - (a_offset < b_offset); -} - -static int repo_dirent_is_dir(struct repo_dirent *dent) -{ - return dent != NULL && dent->mode == REPO_MODE_DIR; -} - -static struct repo_dir *repo_dir_from_dirent(struct repo_dirent *dent) -{ - if (!repo_dirent_is_dir(dent)) + strbuf_reset(&buf); + err = fast_export_ls(REPO_MAX_PATH_DEPTH, path, &dummy, &buf); + if (err) { + if (errno != ENOENT) + die_errno("BUG: unexpected fast_export_ls error"); return NULL; - return dir_pointer(dent->content_offset); -} - -static struct repo_dir *repo_clone_dir(struct repo_dir *orig_dir) -{ - uint32_t orig_o, new_o; - orig_o = dir_offset(orig_dir); - if (orig_o >= dir_pool.committed) - return orig_dir; - new_o = dir_alloc(1); - orig_dir = dir_pointer(orig_o); - *dir_pointer(new_o) = *orig_dir; - return dir_pointer(new_o); -} - -static struct repo_dirent *repo_read_dirent(uint32_t revision, - const uint32_t *path) -{ - uint32_t name = 0; - struct repo_dirent *key = dent_pointer(dent_alloc(1)); - struct repo_dir *dir = NULL; - struct repo_dirent *dent = NULL; - dir = repo_commit_root_dir(commit_pointer(revision)); - while (~(name = *path++)) { - key->name_offset = name; - dent = dent_search(&dir->entries, key); - if (dent == NULL || !repo_dirent_is_dir(dent)) - break; - dir = repo_dir_from_dirent(dent); - } - dent_free(1); - return dent; -} - -static void repo_write_dirent(const uint32_t *path, uint32_t mode, - uint32_t content_offset, uint32_t del) -{ - uint32_t name, revision, dir_o = ~0, parent_dir_o = ~0; - struct repo_dir *dir; - struct repo_dirent *key; - struct repo_dirent *dent = NULL; - revision = active_commit; - dir = repo_commit_root_dir(commit_pointer(revision)); - dir = repo_clone_dir(dir); - commit_pointer(revision)->root_dir_offset = dir_offset(dir); - while (~(name = *path++)) { - parent_dir_o = dir_offset(dir); - - key = dent_pointer(dent_alloc(1)); - key->name_offset = name; - - dent = dent_search(&dir->entries, key); - if (dent == NULL) - dent = key; - else - dent_free(1); - - if (dent == key) { - dent->mode = REPO_MODE_DIR; - dent->content_offset = 0; - dent = dent_insert(&dir->entries, dent); - } - - if (dent_offset(dent) < dent_pool.committed) { - dir_o = repo_dirent_is_dir(dent) ? - dent->content_offset : ~0; - dent_remove(&dir->entries, dent); - dent = dent_pointer(dent_alloc(1)); - dent->name_offset = name; - dent->mode = REPO_MODE_DIR; - dent->content_offset = dir_o; - dent = dent_insert(&dir->entries, dent); - } - - dir = repo_dir_from_dirent(dent); - dir = repo_clone_dir(dir); - dent->content_offset = dir_offset(dir); } - if (dent == NULL) - return; - dent->mode = mode; - dent->content_offset = content_offset; - if (del && ~parent_dir_o) - dent_remove(&dir_pointer(parent_dir_o)->entries, dent); -} - -uint32_t repo_read_path(const uint32_t *path) -{ - uint32_t content_offset = 0; - struct repo_dirent *dent = repo_read_dirent(active_commit, path); - if (dent != NULL) - content_offset = dent->content_offset; - return content_offset; + return buf.buf; } uint32_t repo_read_mode(const uint32_t *path) { - struct repo_dirent *dent = repo_read_dirent(active_commit, path); - if (dent == NULL) - die("invalid dump: path to be modified is missing"); - return dent->mode; + int err; + uint32_t result; + static struct strbuf dummy = STRBUF_INIT; + + strbuf_reset(&dummy); + err = fast_export_ls(REPO_MAX_PATH_DEPTH, path, &result, &dummy); + if (err) { + if (errno != ENOENT) + die_errno("BUG: unexpected fast_export_ls error"); + /* Treat missing paths as directories. */ + return REPO_MODE_DIR; + } + return result; } void repo_copy(uint32_t revision, const uint32_t *src, const uint32_t *dst) { - uint32_t mode = 0, content_offset = 0; - struct repo_dirent *src_dent; - src_dent = repo_read_dirent(revision, src); - if (src_dent != NULL) { - mode = src_dent->mode; - content_offset = src_dent->content_offset; - repo_write_dirent(dst, mode, content_offset, 0); + int err; + uint32_t mode; + static struct strbuf data = STRBUF_INIT; + + strbuf_reset(&data); + err = fast_export_ls_rev(revision, REPO_MAX_PATH_DEPTH, src, &mode, &data); + if (err) { + if (errno != ENOENT) + die_errno("BUG: unexpected fast_export_ls_rev error"); + fast_export_delete(REPO_MAX_PATH_DEPTH, dst); + return; } -} - -void repo_add(uint32_t *path, uint32_t mode, uint32_t blob_mark) -{ - repo_write_dirent(path, mode, blob_mark, 0); + fast_export_modify(REPO_MAX_PATH_DEPTH, dst, mode, data.buf); } void repo_delete(uint32_t *path) { - repo_write_dirent(path, 0, 0, 1); -} - -static void repo_git_add_r(uint32_t depth, uint32_t *path, struct repo_dir *dir); - -static void repo_git_add(uint32_t depth, uint32_t *path, struct repo_dirent *dent) -{ - if (repo_dirent_is_dir(dent)) - repo_git_add_r(depth, path, repo_dir_from_dirent(dent)); - else - fast_export_modify(depth, path, - dent->mode, dent->content_offset); -} - -static void repo_git_add_r(uint32_t depth, uint32_t *path, struct repo_dir *dir) -{ - struct repo_dirent *de = repo_first_dirent(dir); - while (de) { - path[depth] = de->name_offset; - repo_git_add(depth + 1, path, de); - de = dent_next(&dir->entries, de); - } -} - -static void repo_diff_r(uint32_t depth, uint32_t *path, struct repo_dir *dir1, - struct repo_dir *dir2) -{ - struct repo_dirent *de1, *de2; - de1 = repo_first_dirent(dir1); - de2 = repo_first_dirent(dir2); - - while (de1 && de2) { - if (de1->name_offset < de2->name_offset) { - path[depth] = de1->name_offset; - fast_export_delete(depth + 1, path); - de1 = dent_next(&dir1->entries, de1); - continue; - } - if (de1->name_offset > de2->name_offset) { - path[depth] = de2->name_offset; - repo_git_add(depth + 1, path, de2); - de2 = dent_next(&dir2->entries, de2); - continue; - } - path[depth] = de1->name_offset; - - if (de1->mode == de2->mode && - de1->content_offset == de2->content_offset) { - ; /* No change. */ - } else if (repo_dirent_is_dir(de1) && repo_dirent_is_dir(de2)) { - repo_diff_r(depth + 1, path, - repo_dir_from_dirent(de1), - repo_dir_from_dirent(de2)); - } else if (!repo_dirent_is_dir(de1) && !repo_dirent_is_dir(de2)) { - repo_git_add(depth + 1, path, de2); - } else { - fast_export_delete(depth + 1, path); - repo_git_add(depth + 1, path, de2); - } - de1 = dent_next(&dir1->entries, de1); - de2 = dent_next(&dir2->entries, de2); - } - while (de1) { - path[depth] = de1->name_offset; - fast_export_delete(depth + 1, path); - de1 = dent_next(&dir1->entries, de1); - } - while (de2) { - path[depth] = de2->name_offset; - repo_git_add(depth + 1, path, de2); - de2 = dent_next(&dir2->entries, de2); - } -} - -static uint32_t path_stack[REPO_MAX_PATH_DEPTH]; - -void repo_diff(uint32_t r1, uint32_t r2) -{ - repo_diff_r(0, - path_stack, - repo_commit_root_dir(commit_pointer(r1)), - repo_commit_root_dir(commit_pointer(r2))); -} - -void repo_commit(uint32_t revision, uint32_t author, char *log, uint32_t uuid, - uint32_t url, unsigned long timestamp) -{ - fast_export_commit(revision, author, log, uuid, url, timestamp); - dent_commit(); - dir_commit(); - active_commit = commit_alloc(1); - commit_pointer(active_commit)->root_dir_offset = - commit_pointer(active_commit - 1)->root_dir_offset; -} - -static void mark_init(void) -{ - uint32_t i; - mark = 1024 * 1024 * 1024; - for (i = 0; i < dent_pool.size; i++) - if (!repo_dirent_is_dir(dent_pointer(i)) && - dent_pointer(i)->content_offset > mark) - mark = dent_pointer(i)->content_offset; - mark++; -} - -void repo_init(void) -{ - mark_init(); - if (commit_pool.size == 0) { - /* Create empty tree for commit 0. */ - commit_alloc(1); - commit_pointer(0)->root_dir_offset = dir_alloc(1); - dir_pointer(0)->entries.trp_root = ~0; - dir_commit(); - } - /* Preallocate next commit, ready for changes. */ - active_commit = commit_alloc(1); - commit_pointer(active_commit)->root_dir_offset = - commit_pointer(active_commit - 1)->root_dir_offset; -} - -void repo_reset(void) -{ - pool_reset(); - commit_reset(); - dir_reset(); - dent_reset(); + fast_export_delete(REPO_MAX_PATH_DEPTH, path); } diff --git a/vcs-svn/repo_tree.h b/vcs-svn/repo_tree.h index 11d48c2444..d690784fbb 100644 --- a/vcs-svn/repo_tree.h +++ b/vcs-svn/repo_tree.h @@ -14,7 +14,7 @@ uint32_t next_blob_mark(void); void repo_copy(uint32_t revision, const uint32_t *src, const uint32_t *dst); void repo_add(uint32_t *path, uint32_t mode, uint32_t blob_mark); -uint32_t repo_read_path(const uint32_t *path); +const char *repo_read_path(const uint32_t *path); uint32_t repo_read_mode(const uint32_t *path); void repo_delete(uint32_t *path); void repo_commit(uint32_t revision, uint32_t author, char *log, uint32_t uuid, diff --git a/vcs-svn/string_pool.c b/vcs-svn/string_pool.c index f5b1da836e..c08abac71d 100644 --- a/vcs-svn/string_pool.c +++ b/vcs-svn/string_pool.c @@ -65,7 +65,7 @@ uint32_t pool_tok_r(char *str, const char *delim, char **saveptr) return token ? pool_intern(token) : ~0; } -void pool_print_seq(uint32_t len, uint32_t *seq, char delim, FILE *stream) +void pool_print_seq(uint32_t len, const uint32_t *seq, char delim, FILE *stream) { uint32_t i; for (i = 0; i < len && ~seq[i]; i++) { diff --git a/vcs-svn/string_pool.h b/vcs-svn/string_pool.h index 222fb66e68..3720cf8164 100644 --- a/vcs-svn/string_pool.h +++ b/vcs-svn/string_pool.h @@ -4,7 +4,7 @@ uint32_t pool_intern(const char *key); const char *pool_fetch(uint32_t entry); uint32_t pool_tok_r(char *str, const char *delim, char **saveptr); -void pool_print_seq(uint32_t len, uint32_t *seq, char delim, FILE *stream); +void pool_print_seq(uint32_t len, const uint32_t *seq, char delim, FILE *stream); uint32_t pool_tok_seq(uint32_t sz, uint32_t *seq, const char *delim, char *str); void pool_reset(void); diff --git a/vcs-svn/svndump.c b/vcs-svn/svndump.c index 3cc4135892..7ecb227a6d 100644 --- a/vcs-svn/svndump.c +++ b/vcs-svn/svndump.c @@ -36,6 +36,8 @@ obj_pool_gen(log, char, 4096) static struct line_buffer input = LINE_BUFFER_INIT; +#define REPORT_FILENO 3 + static char *log_copy(uint32_t length, const char *log) { char *buffer; @@ -202,15 +204,21 @@ static void read_props(void) static void handle_node(void) { - uint32_t mark = 0; const uint32_t type = node_ctx.type; const int have_props = node_ctx.propLength != LENGTH_UNKNOWN; const int have_text = node_ctx.textLength != LENGTH_UNKNOWN; + /* + * Old text for this node: + * NULL - directory or bug + * empty_blob - empty + * "" - data retrievable from fast-import + */ + static const char *const empty_blob = "::empty::"; + const char *old_data = NULL; if (node_ctx.text_delta) die("text deltas not supported"); - if (have_text) - mark = next_blob_mark(); + if (node_ctx.action == NODEACT_DELETE) { if (have_text || have_props || node_ctx.srcRev) die("invalid dump: deletion node has " @@ -230,15 +238,15 @@ static void handle_node(void) die("invalid dump: directories cannot have text attached"); /* - * Decide on the new content (mark) and mode (node_ctx.type). + * Find old content (old_data) and decide on the new mode. */ if (node_ctx.action == NODEACT_CHANGE && !~*node_ctx.dst) { if (type != REPO_MODE_DIR) die("invalid dump: root of tree is not a regular file"); + old_data = NULL; } else if (node_ctx.action == NODEACT_CHANGE) { uint32_t mode; - if (!have_text) - mark = repo_read_path(node_ctx.dst); + old_data = repo_read_path(node_ctx.dst); mode = repo_read_mode(node_ctx.dst); if (mode == REPO_MODE_DIR && type != REPO_MODE_DIR) die("invalid dump: cannot modify a directory into a file"); @@ -246,7 +254,11 @@ static void handle_node(void) die("invalid dump: cannot modify a file into a directory"); node_ctx.type = mode; } else if (node_ctx.action == NODEACT_ADD) { - if (!have_text && type != REPO_MODE_DIR) + if (type == REPO_MODE_DIR) + old_data = NULL; + else if (have_text) + old_data = empty_blob; + else die("invalid dump: adds node without text"); } else { die("invalid dump: Node-path block lacks Node-action"); @@ -265,24 +277,34 @@ static void handle_node(void) /* * Save the result. */ - repo_add(node_ctx.dst, node_ctx.type, mark); - if (have_text) - fast_export_blob(node_ctx.type, mark, - node_ctx.textLength, &input); + if (type == REPO_MODE_DIR) /* directories are not tracked. */ + return; + assert(old_data); + if (old_data == empty_blob) + /* For the fast_export_* functions, NULL means empty. */ + old_data = NULL; + if (!have_text) { + fast_export_modify(REPO_MAX_PATH_DEPTH, node_ctx.dst, + node_ctx.type, old_data); + return; + } + fast_export_modify(REPO_MAX_PATH_DEPTH, node_ctx.dst, + node_ctx.type, "inline"); + fast_export_data(node_ctx.type, node_ctx.textLength, &input); } static void begin_revision(void) { if (!rev_ctx.revision) /* revision 0 gets no git commit. */ return; - fast_export_begin_commit(rev_ctx.revision); + fast_export_begin_commit(rev_ctx.revision, rev_ctx.author, rev_ctx.log, + dump_ctx.uuid, dump_ctx.url, rev_ctx.timestamp); } static void end_revision(void) { if (rev_ctx.revision) - repo_commit(rev_ctx.revision, rev_ctx.author, rev_ctx.log, - dump_ctx.uuid, dump_ctx.url, rev_ctx.timestamp); + fast_export_end_commit(rev_ctx.revision); } void svndump_read(const char *url) @@ -383,7 +405,6 @@ int svndump_init(const char *filename) { if (buffer_init(&input, filename)) return error("cannot open %s: %s", filename, strerror(errno)); - repo_init(); fast_export_init(REPORT_FILENO); reset_dump_ctx(~0); reset_rev_ctx(0); @@ -396,7 +417,6 @@ void svndump_deinit(void) { log_reset(); fast_export_deinit(); - repo_reset(); reset_dump_ctx(~0); reset_rev_ctx(0); reset_node_ctx(NULL); @@ -411,7 +431,6 @@ void svndump_reset(void) log_reset(); fast_export_reset(); buffer_reset(&input); - repo_reset(); reset_dump_ctx(~0); reset_rev_ctx(0); reset_node_ctx(NULL); -- cgit v1.2.3 From e43581120843f6f55f411af470faf806e052ad9d Mon Sep 17 00:00:00 2001 From: David Barr Date: Sun, 12 Dec 2010 03:59:31 +1100 Subject: vcs-svn: quote paths correctly for ls command This bug was found while importing rev 601865 of ASF. [jn: with test] Signed-off-by: David Barr Signed-off-by: Jonathan Nieder --- vcs-svn/fast_export.c | 2 +- vcs-svn/string_pool.c | 11 +++++++++++ vcs-svn/string_pool.h | 1 + 3 files changed, 13 insertions(+), 1 deletion(-) (limited to 'vcs-svn') diff --git a/vcs-svn/fast_export.c b/vcs-svn/fast_export.c index a8ce5c64b2..4d57efabd5 100644 --- a/vcs-svn/fast_export.c +++ b/vcs-svn/fast_export.c @@ -107,7 +107,7 @@ static void ls_from_active_commit(uint32_t depth, const uint32_t *path) { /* ls "path/to/file" */ printf("ls \""); - pool_print_seq(depth, path, '/', stdout); + pool_print_seq_q(depth, path, '/', stdout); printf("\"\n"); fflush(stdout); } diff --git a/vcs-svn/string_pool.c b/vcs-svn/string_pool.c index c08abac71d..be43598d5b 100644 --- a/vcs-svn/string_pool.c +++ b/vcs-svn/string_pool.c @@ -4,6 +4,7 @@ */ #include "git-compat-util.h" +#include "quote.h" #include "trp.h" #include "obj_pool.h" #include "string_pool.h" @@ -75,6 +76,16 @@ void pool_print_seq(uint32_t len, const uint32_t *seq, char delim, FILE *stream) } } +void pool_print_seq_q(uint32_t len, const uint32_t *seq, char delim, FILE *stream) +{ + uint32_t i; + for (i = 0; i < len && ~seq[i]; i++) { + quote_c_style(pool_fetch(seq[i]), NULL, stream, 1); + if (i < len - 1 && ~seq[i + 1]) + fputc(delim, stream); + } +} + uint32_t pool_tok_seq(uint32_t sz, uint32_t *seq, const char *delim, char *str) { char *context = NULL; diff --git a/vcs-svn/string_pool.h b/vcs-svn/string_pool.h index 3720cf8164..96e501dc53 100644 --- a/vcs-svn/string_pool.h +++ b/vcs-svn/string_pool.h @@ -5,6 +5,7 @@ uint32_t pool_intern(const char *key); const char *pool_fetch(uint32_t entry); uint32_t pool_tok_r(char *str, const char *delim, char **saveptr); void pool_print_seq(uint32_t len, const uint32_t *seq, char delim, FILE *stream); +void pool_print_seq_q(uint32_t len, const uint32_t *seq, char delim, FILE *stream); uint32_t pool_tok_seq(uint32_t sz, uint32_t *seq, const char *delim, char *str); void pool_reset(void); -- cgit v1.2.3 From 1ae469b06c50aade4781931ca1587453082f57eb Mon Sep 17 00:00:00 2001 From: Jonathan Nieder Date: Sat, 11 Dec 2010 17:08:51 -0600 Subject: vcs-svn: handle filenames with dq correctly Quote paths passed to fast-import so filenames with double quotes are not misinterpreted. One might imagine this could help with filenames with newlines, too, but svn does not allow those. Helped-by: David Barr Signed-off-by: Jonathan Nieder Signed-off-by: David Barr Signed-off-by: Jonathan Nieder --- vcs-svn/fast_export.c | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) (limited to 'vcs-svn') diff --git a/vcs-svn/fast_export.c b/vcs-svn/fast_export.c index 4d57efabd5..9c03f3e16d 100644 --- a/vcs-svn/fast_export.c +++ b/vcs-svn/fast_export.c @@ -34,10 +34,9 @@ void fast_export_reset(void) void fast_export_delete(uint32_t depth, const uint32_t *path) { - putchar('D'); - putchar(' '); - pool_print_seq(depth, path, '/', stdout); - putchar('\n'); + printf("D \""); + pool_print_seq_q(depth, path, '/', stdout); + printf("\"\n"); } static void fast_export_truncate(uint32_t depth, const uint32_t *path, uint32_t mode) @@ -54,9 +53,9 @@ void fast_export_modify(uint32_t depth, const uint32_t *path, uint32_t mode, fast_export_truncate(depth, path, mode); return; } - printf("M %06"PRIo32" %s ", mode, dataref); - pool_print_seq(depth, path, '/', stdout); - putchar('\n'); + printf("M %06"PRIo32" %s \"", mode, dataref); + pool_print_seq_q(depth, path, '/', stdout); + printf("\"\n"); } static char gitsvnline[MAX_GITSVN_LINE_LEN]; @@ -97,9 +96,9 @@ void fast_export_end_commit(uint32_t revision) static void ls_from_rev(uint32_t rev, uint32_t depth, const uint32_t *path) { /* ls :5 path/to/old/file */ - printf("ls :%"PRIu32" ", rev); - pool_print_seq(depth, path, '/', stdout); - putchar('\n'); + printf("ls :%"PRIu32" \"", rev); + pool_print_seq_q(depth, path, '/', stdout); + printf("\"\n"); fflush(stdout); } -- cgit v1.2.3 From dd3f42ad793b5334d506a451addcefd0054c27bb Mon Sep 17 00:00:00 2001 From: David Barr Date: Sun, 12 Dec 2010 13:41:38 +1100 Subject: vcs-svn: use mark from previous import for parent commit With this patch, overlapping incremental imports work. Signed-off-by: David Barr Signed-off-by: Jonathan Nieder --- vcs-svn/fast_export.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'vcs-svn') diff --git a/vcs-svn/fast_export.c b/vcs-svn/fast_export.c index 9c03f3e16d..f19db9ae82 100644 --- a/vcs-svn/fast_export.c +++ b/vcs-svn/fast_export.c @@ -83,7 +83,7 @@ void fast_export_begin_commit(uint32_t revision, uint32_t author, char *log, log, gitsvnline); if (!first_commit_done) { if (revision > 1) - printf("from refs/heads/master^0\n"); + printf("from :%"PRIu32"\n", revision - 1); first_commit_done = 1; } } -- cgit v1.2.3 From 43155cfe1415f5547791613a5de6399112ba3560 Mon Sep 17 00:00:00 2001 From: David Barr Date: Mon, 13 Dec 2010 17:09:31 +1100 Subject: vcs-svn: avoid using ls command twice Currently there are two functions to retrieve the mode and content at a path: const char *repo_read_path(const uint32_t *path); uint32_t repo_read_mode(const uint32_t *path) Replace them with a single function with two return values. This means we can use one round-trip to get the same information from fast-import that previously took two. Signed-off-by: David Barr Signed-off-by: Jonathan Nieder --- vcs-svn/repo_tree.c | 24 ++++-------------------- vcs-svn/repo_tree.h | 3 +-- vcs-svn/svndump.c | 3 +-- 3 files changed, 6 insertions(+), 24 deletions(-) (limited to 'vcs-svn') diff --git a/vcs-svn/repo_tree.c b/vcs-svn/repo_tree.c index e75f58087c..1681b654d1 100644 --- a/vcs-svn/repo_tree.c +++ b/vcs-svn/repo_tree.c @@ -8,39 +8,23 @@ #include "repo_tree.h" #include "fast_export.h" -const char *repo_read_path(const uint32_t *path) +const char *repo_read_path(const uint32_t *path, uint32_t *mode_out) { int err; - uint32_t dummy; static struct strbuf buf = STRBUF_INIT; strbuf_reset(&buf); - err = fast_export_ls(REPO_MAX_PATH_DEPTH, path, &dummy, &buf); + err = fast_export_ls(REPO_MAX_PATH_DEPTH, path, mode_out, &buf); if (err) { if (errno != ENOENT) die_errno("BUG: unexpected fast_export_ls error"); + /* Treat missing paths as directories. */ + *mode_out = REPO_MODE_DIR; return NULL; } return buf.buf; } -uint32_t repo_read_mode(const uint32_t *path) -{ - int err; - uint32_t result; - static struct strbuf dummy = STRBUF_INIT; - - strbuf_reset(&dummy); - err = fast_export_ls(REPO_MAX_PATH_DEPTH, path, &result, &dummy); - if (err) { - if (errno != ENOENT) - die_errno("BUG: unexpected fast_export_ls error"); - /* Treat missing paths as directories. */ - return REPO_MODE_DIR; - } - return result; -} - void repo_copy(uint32_t revision, const uint32_t *src, const uint32_t *dst) { int err; diff --git a/vcs-svn/repo_tree.h b/vcs-svn/repo_tree.h index d690784fbb..f506352dc2 100644 --- a/vcs-svn/repo_tree.h +++ b/vcs-svn/repo_tree.h @@ -14,8 +14,7 @@ uint32_t next_blob_mark(void); void repo_copy(uint32_t revision, const uint32_t *src, const uint32_t *dst); void repo_add(uint32_t *path, uint32_t mode, uint32_t blob_mark); -const char *repo_read_path(const uint32_t *path); -uint32_t repo_read_mode(const uint32_t *path); +const char *repo_read_path(const uint32_t *path, uint32_t *mode_out); void repo_delete(uint32_t *path); void repo_commit(uint32_t revision, uint32_t author, char *log, uint32_t uuid, uint32_t url, long unsigned timestamp); diff --git a/vcs-svn/svndump.c b/vcs-svn/svndump.c index 7ecb227a6d..99a5ba0d10 100644 --- a/vcs-svn/svndump.c +++ b/vcs-svn/svndump.c @@ -246,8 +246,7 @@ static void handle_node(void) old_data = NULL; } else if (node_ctx.action == NODEACT_CHANGE) { uint32_t mode; - old_data = repo_read_path(node_ctx.dst); - mode = repo_read_mode(node_ctx.dst); + old_data = repo_read_path(node_ctx.dst, &mode); if (mode == REPO_MODE_DIR && type != REPO_MODE_DIR) die("invalid dump: cannot modify a directory into a file"); if (mode != REPO_MODE_DIR && type == REPO_MODE_DIR) -- cgit v1.2.3