From eaa82f8e983199080369c44869ceac1e403f81fc Mon Sep 17 00:00:00 2001 From: Jiang Xin Date: Sun, 17 Dec 2023 22:41:36 +0800 Subject: test-pkt-line: add option parser for unpack-sideband We can use the test helper program "test-tool pkt-line" to test pkt-line related functions. E.g.: * Use "test-tool pkt-line send-split-sideband" to generate sideband messages. * Pipe these generated sideband messages to command "test-tool pkt-line unpack-sideband" to test packet_reader_read() function. In order to make a complete test of the packet_reader_read() function, add option parser for command "test-tool pkt-line unpack-sideband". * To remove newlines in sideband messages, we can use: $ test-tool pkt-line unpack-sideband --chomp-newline * To preserve newlines in sideband messages, we can use: $ test-tool pkt-line unpack-sideband --no-chomp-newline * To parse sideband messages using "demultiplex_sideband()" inside the function "packet_reader_read()", we can use: $ test-tool pkt-line unpack-sideband --reader-use-sideband We also add new example sideband packets in send_split_sideband() and add several new test cases in t0070. Among these test cases, we pipe output of the "send-split-sideband" subcommand to the "unpack-sideband" subcommand. We found two issues: 1. The two splitted sideband messages "Hello," and " world!\n" should be concatenated together. But when we turn on use_sideband field of reader to parse sideband messages, the first part of the splitted message ("Hello,") is lost. 2. The newline characters in sideband 2 (progress info) and sideband 3 (error message) should be preserved, but they are both trimmed. Will fix the above two issues in subsequent commits. Signed-off-by: Jiang Xin Signed-off-by: Junio C Hamano --- t/helper/test-pkt-line.c | 59 ++++++++++++++++++++++++++++++++++++++++++++---- t/t0070-fundamental.sh | 58 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 112 insertions(+), 5 deletions(-) diff --git a/t/helper/test-pkt-line.c b/t/helper/test-pkt-line.c index f4d134a145..6b306cf5ca 100644 --- a/t/helper/test-pkt-line.c +++ b/t/helper/test-pkt-line.c @@ -2,6 +2,7 @@ #include "test-tool.h" #include "pkt-line.h" #include "write-or-die.h" +#include "parse-options.h" static void pack_line(const char *line) { @@ -64,12 +65,33 @@ static void unpack(void) } } -static void unpack_sideband(void) +static void unpack_sideband(int argc, const char **argv) { struct packet_reader reader; - packet_reader_init(&reader, 0, NULL, 0, - PACKET_READ_GENTLE_ON_EOF | - PACKET_READ_CHOMP_NEWLINE); + int options = PACKET_READ_GENTLE_ON_EOF; + int chomp_newline = 1; + int reader_use_sideband = 0; + const char *const unpack_sideband_usage[] = { + "test_tool unpack_sideband [options...]", NULL + }; + struct option cmd_options[] = { + OPT_BOOL(0, "reader-use-sideband", &reader_use_sideband, + "set use_sideband bit for packet reader (Default: off)"), + OPT_BOOL(0, "chomp-newline", &chomp_newline, + "chomp newline in packet (Default: on)"), + OPT_END() + }; + + argc = parse_options(argc, argv, "", cmd_options, unpack_sideband_usage, + 0); + if (argc > 0) + usage_msg_opt(_("too many arguments"), unpack_sideband_usage, + cmd_options); + + if (chomp_newline) + options |= PACKET_READ_CHOMP_NEWLINE; + packet_reader_init(&reader, 0, NULL, 0, options); + reader.use_sideband = reader_use_sideband; while (packet_reader_read(&reader) != PACKET_READ_EOF) { int band; @@ -79,6 +101,17 @@ static void unpack_sideband(void) case PACKET_READ_EOF: break; case PACKET_READ_NORMAL: + /* + * When the "use_sideband" field of the reader is turned + * on, sideband packets other than the payload have been + * parsed and consumed in packet_reader_read(), and only + * the payload arrives here. + */ + if (reader.use_sideband) { + write_or_die(1, reader.line, reader.pktlen - 1); + break; + } + band = reader.line[0] & 0xff; if (band < 1 || band > 2) continue; /* skip non-sideband packets */ @@ -97,15 +130,31 @@ static void unpack_sideband(void) static int send_split_sideband(void) { + const char *foo = "Foo.\n"; + const char *bar = "Bar.\n"; const char *part1 = "Hello,"; const char *primary = "\001primary: regular output\n"; const char *part2 = " world!\n"; + /* Each sideband message has a trailing newline character. */ + send_sideband(1, 2, foo, strlen(foo), LARGE_PACKET_MAX); + send_sideband(1, 2, bar, strlen(bar), LARGE_PACKET_MAX); + + /* + * One sideband message is divided into part1 and part2 + * by the primary message. + */ send_sideband(1, 2, part1, strlen(part1), LARGE_PACKET_MAX); packet_write(1, primary, strlen(primary)); send_sideband(1, 2, part2, strlen(part2), LARGE_PACKET_MAX); packet_response_end(1); + /* + * We use unpack_sideband() to consume packets. A flush packet + * is required to end parsing. + */ + packet_flush(1); + return 0; } @@ -126,7 +175,7 @@ int cmd__pkt_line(int argc, const char **argv) else if (!strcmp(argv[1], "unpack")) unpack(); else if (!strcmp(argv[1], "unpack-sideband")) - unpack_sideband(); + unpack_sideband(argc - 1, argv + 1); else if (!strcmp(argv[1], "send-split-sideband")) send_split_sideband(); else if (!strcmp(argv[1], "receive-sideband")) diff --git a/t/t0070-fundamental.sh b/t/t0070-fundamental.sh index 574de34198..297a7f772e 100755 --- a/t/t0070-fundamental.sh +++ b/t/t0070-fundamental.sh @@ -53,4 +53,62 @@ test_expect_success 'missing sideband designator is reported' ' test_i18ngrep "missing sideband" err ' +test_expect_success 'unpack-sideband: --no-chomp-newline' ' + test_when_finished "rm -f expect-out expect-err" && + test-tool pkt-line send-split-sideband >split-sideband && + test-tool pkt-line unpack-sideband \ + --no-chomp-newline out 2>err && + cat >expect-out <<-EOF && + primary: regular output + EOF + cat >expect-err <<-EOF && + Foo. + Bar. + Hello, world! + EOF + test_cmp expect-out out && + test_cmp expect-err err +' + +test_expect_success 'unpack-sideband: --chomp-newline (default)' ' + test_when_finished "rm -f expect-out expect-err" && + test-tool pkt-line send-split-sideband >split-sideband && + test-tool pkt-line unpack-sideband \ + --chomp-newline out 2>err && + printf "primary: regular output" >expect-out && + printf "Foo.Bar.Hello, world!" >expect-err && + test_cmp expect-out out && + test_cmp expect-err err +' + +test_expect_failure 'unpack-sideband: packet_reader_read() consumes sideband, no chomp payload' ' + test_when_finished "rm -f expect-out expect-err" && + test-tool pkt-line send-split-sideband >split-sideband && + test-tool pkt-line unpack-sideband \ + --reader-use-sideband \ + --no-chomp-newline out 2>err && + cat >expect-out <<-EOF && + primary: regular output + EOF + printf "remote: Foo. \n" >expect-err && + printf "remote: Bar. \n" >>expect-err && + printf "remote: Hello, world! \n" >>expect-err && + test_cmp expect-out out && + test_cmp expect-err err +' + +test_expect_failure 'unpack-sideband: packet_reader_read() consumes sideband, chomp payload' ' + test_when_finished "rm -f expect-out expect-err" && + test-tool pkt-line send-split-sideband >split-sideband && + test-tool pkt-line unpack-sideband \ + --reader-use-sideband \ + --chomp-newline out 2>err && + printf "primary: regular output" >expect-out && + printf "remote: Foo. \n" >expect-err && + printf "remote: Bar. \n" >>expect-err && + printf "remote: Hello, world! \n" >>expect-err && + test_cmp expect-out out && + test_cmp expect-err err +' + test_done -- cgit v1.2.3 From 64220dc5f7450d8eca471fa484965b1dcddc2827 Mon Sep 17 00:00:00 2001 From: Jiang Xin Date: Sun, 17 Dec 2023 22:41:37 +0800 Subject: pkt-line: memorize sideband fragment in reader When we turn on the "use_sideband" field of the packet_reader, "packet_reader_read()" will call the function "demultiplex_sideband()" to parse and consume sideband messages. Sideband fragment which does not end with "\r" or "\n" will be saved in the sixth parameter "scratch" and it can be reused and be concatenated when parsing another sideband message. In "packet_reader_read()" function, the local variable "scratch" can only be reused by subsequent sideband messages. But if there is a payload message between two sideband fragments, the first fragment which is saved in the local variable "scratch" will be lost. To solve this problem, we can add a new field "scratch" in packet_reader to memorize the sideband fragment across different calls of "packet_reader_read()". Signed-off-by: Jiang Xin Signed-off-by: Junio C Hamano --- pkt-line.c | 5 ++--- pkt-line.h | 3 +++ t/t0070-fundamental.sh | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/pkt-line.c b/pkt-line.c index af83a19f4d..5943777a17 100644 --- a/pkt-line.c +++ b/pkt-line.c @@ -592,12 +592,11 @@ void packet_reader_init(struct packet_reader *reader, int fd, reader->options = options; reader->me = "git"; reader->hash_algo = &hash_algos[GIT_HASH_SHA1]; + strbuf_init(&reader->scratch, 0); } enum packet_read_status packet_reader_read(struct packet_reader *reader) { - struct strbuf scratch = STRBUF_INIT; - if (reader->line_peeked) { reader->line_peeked = 0; return reader->status; @@ -620,7 +619,7 @@ enum packet_read_status packet_reader_read(struct packet_reader *reader) break; if (demultiplex_sideband(reader->me, reader->status, reader->buffer, reader->pktlen, 1, - &scratch, &sideband_type)) + &reader->scratch, &sideband_type)) break; } diff --git a/pkt-line.h b/pkt-line.h index 954eec8719..be1010d34e 100644 --- a/pkt-line.h +++ b/pkt-line.h @@ -194,6 +194,9 @@ struct packet_reader { /* hash algorithm in use */ const struct git_hash_algo *hash_algo; + + /* hold temporary sideband message */ + struct strbuf scratch; }; /* diff --git a/t/t0070-fundamental.sh b/t/t0070-fundamental.sh index 297a7f772e..275edbf6e7 100755 --- a/t/t0070-fundamental.sh +++ b/t/t0070-fundamental.sh @@ -81,7 +81,7 @@ test_expect_success 'unpack-sideband: --chomp-newline (default)' ' test_cmp expect-err err ' -test_expect_failure 'unpack-sideband: packet_reader_read() consumes sideband, no chomp payload' ' +test_expect_success 'unpack-sideband: packet_reader_read() consumes sideband, no chomp payload' ' test_when_finished "rm -f expect-out expect-err" && test-tool pkt-line send-split-sideband >split-sideband && test-tool pkt-line unpack-sideband \ -- cgit v1.2.3 From 7033d5479b8a7b8e7c33892f23d106c33c938ff4 Mon Sep 17 00:00:00 2001 From: Jiang Xin Date: Sun, 17 Dec 2023 22:41:38 +0800 Subject: pkt-line: do not chomp newlines for sideband messages When calling "packet_read_with_status()" to parse pkt-line encoded packets, we can turn on the flag "PACKET_READ_CHOMP_NEWLINE" to chomp newline character for each packet for better line matching. But when receiving data and progress information using sideband, we should turn off the flag "PACKET_READ_CHOMP_NEWLINE" to prevent mangling newline characters from data and progress information. When both the server and the client support "sideband-all" capability, we have a dilemma that newline characters in negotiation packets should be removed, but the newline characters in the progress information should be left intact. Add new flag "PACKET_READ_USE_SIDEBAND" for "packet_read_with_status()" to prevent mangling newline characters in sideband messages. Helped-by: Jonathan Tan Helped-by: Oswald Buddenhagen Signed-off-by: Jiang Xin Signed-off-by: Junio C Hamano --- pkt-line.c | 31 +++++++++++++++++++++++++++++-- pkt-line.h | 1 + t/t0070-fundamental.sh | 2 +- 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/pkt-line.c b/pkt-line.c index 5943777a17..e9061e61a4 100644 --- a/pkt-line.c +++ b/pkt-line.c @@ -462,8 +462,32 @@ enum packet_read_status packet_read_with_status(int fd, char **src_buffer, } if ((options & PACKET_READ_CHOMP_NEWLINE) && - len && buffer[len-1] == '\n') - len--; + len && buffer[len-1] == '\n') { + if (options & PACKET_READ_USE_SIDEBAND) { + int band = *buffer & 0xff; + switch (band) { + case 1: + /* Chomp newline for payload */ + len--; + break; + case 2: + case 3: + /* + * Do not chomp newline for progress and error + * message. + */ + break; + default: + /* + * Bad sideband, let's leave it to + * demultiplex_sideband() to catch this error. + */ + break; + } + } else { + len--; + } + } buffer[len] = 0; if (options & PACKET_READ_REDACT_URI_PATH && @@ -602,6 +626,9 @@ enum packet_read_status packet_reader_read(struct packet_reader *reader) return reader->status; } + if (reader->use_sideband) + reader->options |= PACKET_READ_USE_SIDEBAND; + /* * Consume all progress packets until a primary payload packet is * received diff --git a/pkt-line.h b/pkt-line.h index be1010d34e..a7ff2e2f18 100644 --- a/pkt-line.h +++ b/pkt-line.h @@ -85,6 +85,7 @@ void packet_fflush(FILE *f); #define PACKET_READ_DIE_ON_ERR_PACKET (1u<<2) #define PACKET_READ_GENTLE_ON_READ_ERROR (1u<<3) #define PACKET_READ_REDACT_URI_PATH (1u<<4) +#define PACKET_READ_USE_SIDEBAND (1u<<5) int packet_read(int fd, char *buffer, unsigned size, int options); /* diff --git a/t/t0070-fundamental.sh b/t/t0070-fundamental.sh index 275edbf6e7..0d2b7d8d93 100755 --- a/t/t0070-fundamental.sh +++ b/t/t0070-fundamental.sh @@ -97,7 +97,7 @@ test_expect_success 'unpack-sideband: packet_reader_read() consumes sideband, no test_cmp expect-err err ' -test_expect_failure 'unpack-sideband: packet_reader_read() consumes sideband, chomp payload' ' +test_expect_success 'unpack-sideband: packet_reader_read() consumes sideband, chomp payload' ' test_when_finished "rm -f expect-out expect-err" && test-tool pkt-line send-split-sideband >split-sideband && test-tool pkt-line unpack-sideband \ -- cgit v1.2.3