diff options
90 files changed, 2254 insertions, 382 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cd98bcb18a..793243421c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -51,3 +51,13 @@ test: paths: - t/failed-test-artifacts when: on_failure + +static-analysis: + image: ubuntu:22.04 + variables: + jobname: StaticAnalysis + before_script: + - ./ci/install-docker-dependencies.sh + script: + - ./ci/run-static-analysis.sh + - ./ci/check-directional-formatting.bash diff --git a/Documentation/CodingGuidelines b/Documentation/CodingGuidelines index 8ed517a5ca..578587a471 100644 --- a/Documentation/CodingGuidelines +++ b/Documentation/CodingGuidelines @@ -450,7 +450,7 @@ For C programs: one of the approved headers that includes it first for you. (The approved headers currently include "builtin.h", "t/helper/test-tool.h", "xdiff/xinclude.h", or - "reftable/system.h"). You do not have to include more than one of + "reftable/system.h".) You do not have to include more than one of these. - A C file must directly include the header files that declare the @@ -578,7 +578,7 @@ Externally Visible Names . The variable name describes the effect of tweaking this knob. The section and variable names that consist of multiple words are - formed by concatenating the words without punctuations (e.g. `-`), + formed by concatenating the words without punctuation marks (e.g. `-`), and are broken using bumpyCaps in documentation as a hint to the reader. diff --git a/Documentation/RelNotes/2.44.0.txt b/Documentation/RelNotes/2.44.0.txt index e58095fc8d..2b9d2bd3c2 100644 --- a/Documentation/RelNotes/2.44.0.txt +++ b/Documentation/RelNotes/2.44.0.txt @@ -32,7 +32,11 @@ UI, Workflows & Features breaking change. "--ignore-other-worktrees" option is required to unbreak you, if you are used to the current behaviour that "-B" overrides the safety. - (merge b23285a921 jc/checkout-B-branch-in-use later to maint). + + * The builtin_objectmode attribute is populated for each path + without adding anything in .gitattributes files, which would be + useful in magic pathspec, e.g., ":(attr:builtin_objectmode=100755)" + to limit to executables. Performance, Internal Implementation, Development Support etc. @@ -64,6 +68,10 @@ Performance, Internal Implementation, Development Support etc. non-default hash function into a repository that uses the reftable backend. + * Streaming spans of packfile data used to be done only from a + single, primary, pack in a repository with multiple packfiles. It + has been extended to allow reuse from other packfiles, too. + Fixes since v2.43 ----------------- @@ -154,6 +162,17 @@ Fixes since v2.43 * "git sparse-checkout (add|set) --[no-]cone --end-of-options" did not handle "--end-of-options" correctly after a recent update. + * Unlike other environment variables that took the usual + true/false/yes/no as well as 0/1, GIT_FLUSH only understood 0/1, + which has been corrected. + (merge 556e68032f cp/git-flush-is-an-env-bool later to maint). + + * Clearing in-core repository (happens during e.g., "git fetch + --recurse-submodules" with commit graph enabled) made in-core + commit object in an inconsistent state by discarding the necessary + data from commit-graph too early, which has been corrected. + (merge d70f554cdf jk/commit-graph-slab-clear-fix later to maint). + * Other code cleanup, docfix, build fix, etc. (merge 50f1abcff6 js/packfile-h-typofix later to maint). (merge cbf498eb53 jb/reflog-expire-delete-dry-run-options later to maint). @@ -178,3 +197,10 @@ Fixes since v2.43 (merge 63956c553d ml/doc-merge-updates later to maint). (merge d57c671a51 en/header-cleanup later to maint). (merge 5b7eec4bc5 rs/fast-import-simplify-mempool-allocation later to maint). + (merge 291873e5d6 js/contributor-docs-updates later to maint). + (merge 54d8a2531b jk/t1006-cat-file-objectsize-disk later to maint). + (merge 7033d5479b jx/sideband-chomp-newline-fix later to maint). + (merge 9cd30af991 ms/rebase-insnformat-doc-fix later to maint). + (merge 03bcc93769 cp/sideband-array-index-comment-fix later to maint). + (merge 993d38a066 jk/index-pack-lsan-false-positive-fix later to maint). + (merge 25aec06326 ib/rebase-reschedule-doc later to maint). diff --git a/Documentation/SubmittingPatches b/Documentation/SubmittingPatches index bce7f97815..e734a3f0f1 100644 --- a/Documentation/SubmittingPatches +++ b/Documentation/SubmittingPatches @@ -355,9 +355,21 @@ If you like, you can put extra tags at the end: patch after a detailed analysis. . `Tested-by:` is used to indicate that the person applied the patch and found it to have the desired effect. - -You can also create your own tag or use one that's in common usage -such as "Thanks-to:", "Based-on-patch-by:", or "Mentored-by:". +. `Co-authored-by:` is used to indicate that people exchanged drafts + of a patch before submitting it. +. `Helped-by:` is used to credit someone who suggested ideas for + changes without providing the precise changes in patch form. +. `Mentored-by:` is used to credit someone with helping develop a + patch as part of a mentorship program (e.g., GSoC or Outreachy). +. `Suggested-by:` is used to credit someone with suggesting the idea + for a patch. + +While you can also create your own trailer if the situation warrants it, we +encourage you to instead use one of the common trailers in this project +highlighted above. + +Only capitalize the very first letter of tags, i.e. favor +"Signed-off-by" over "Signed-Off-By" and "Acked-by:" over "Acked-By". [[git-tools]] === Generate your patch using Git tools out of your commits. @@ -570,7 +582,7 @@ their trees themselves. master). * Read the Git mailing list, the maintainer regularly posts messages - entitled "What's cooking in git.git" and "What's in git.git" giving + entitled "What's cooking in git.git" giving the status of various proposed changes. == GitHub CI[[GHCI]] @@ -590,11 +602,12 @@ After the initial setup, CI will run whenever you push new changes to your fork of Git on GitHub. You can monitor the test state of all your branches here: `https://github.com/<Your GitHub handle>/git/actions/workflows/main.yml` -If a branch did not pass all test cases then it is marked with a red -cross. In that case you can click on the failing job and navigate to -"ci/run-build-and-tests.sh" and/or "ci/print-test-failures.sh". You -can also download "Artifacts" which are tarred (or zipped) archives -with test data relevant for debugging. +If a branch does not pass all test cases then it will be marked with a +red +x+, instead of a green check. In that case, you can click on the +failing job and navigate to "ci/run-build-and-tests.sh" and/or +"ci/print-test-failures.sh". You can also download "Artifacts" which +are zip archives containing tarred (or zipped) archives with test data +relevant for debugging. Then fix the problem and push your fix to your GitHub fork. This will trigger a new CI build to ensure all tests pass. @@ -686,7 +699,7 @@ message to an external program, and this is a handy way to drive `git am`. However, if the message is MIME encoded, what is piped into the program is the representation you see in your `*Article*` buffer after unwrapping MIME. This is often not what -you would want for two reasons. It tends to screw up non ASCII +you would want for two reasons. It tends to screw up non-ASCII characters (most notably in people's names), and also whitespaces (fatal in patches). Running "C-u g" to display the message in raw form before using "|" to run the pipe can work diff --git a/Documentation/config/extensions.txt b/Documentation/config/extensions.txt index bccaec7a96..66db0e15da 100644 --- a/Documentation/config/extensions.txt +++ b/Documentation/config/extensions.txt @@ -7,6 +7,17 @@ Note that this setting should only be set by linkgit:git-init[1] or linkgit:git-clone[1]. Trying to change it after initialization will not work and will produce hard-to-diagnose issues. +extensions.refStorage:: + Specify the ref storage format to use. The acceptable values are: ++ +include::../ref-storage-format.txt[] ++ +It is an error to specify this key unless `core.repositoryFormatVersion` is 1. ++ +Note that this setting should only be set by linkgit:git-init[1] or +linkgit:git-clone[1]. Trying to change it after initialization will not +work and will produce hard-to-diagnose issues. + extensions.worktreeConfig:: If enabled, then worktrees will load config settings from the `$GIT_DIR/config.worktree` file in addition to the diff --git a/Documentation/config/fetch.txt b/Documentation/config/fetch.txt index aea5b97477..d7dc461bd1 100644 --- a/Documentation/config/fetch.txt +++ b/Documentation/config/fetch.txt @@ -50,6 +50,12 @@ fetch.pruneTags:: refs. See also `remote.<name>.pruneTags` and the PRUNING section of linkgit:git-fetch[1]. +fetch.all:: + If true, fetch will attempt to update all available remotes. + This behavior can be overridden by passing `--no-all` or by + explicitly specifying one or more remote(s) to fetch from. + Defaults to false. + fetch.output:: Control how ref update status is printed. Valid values are `full` and `compact`. Default value is `full`. See the diff --git a/Documentation/config/pack.txt b/Documentation/config/pack.txt index f50df9dbce..9c630863e6 100644 --- a/Documentation/config/pack.txt +++ b/Documentation/config/pack.txt @@ -28,11 +28,17 @@ all existing objects. You can force recompression by passing the -F option to linkgit:git-repack[1]. pack.allowPackReuse:: - When true, and when reachability bitmaps are enabled, - pack-objects will try to send parts of the bitmapped packfile - verbatim. This can reduce memory and CPU usage to serve fetches, - but might result in sending a slightly larger pack. Defaults to - true. + When true or "single", and when reachability bitmaps are + enabled, pack-objects will try to send parts of the bitmapped + packfile verbatim. When "multi", and when a multi-pack + reachability bitmap is available, pack-objects will try to send + parts of all packs in the MIDX. ++ + If only a single pack bitmap is available, and + `pack.allowPackReuse` is set to "multi", reuse parts of just the + bitmapped packfile. This can reduce memory and CPU usage to + serve fetches, but might result in sending a slightly larger + pack. Defaults to true. pack.island:: An extended regular expression configuring a set of delta diff --git a/Documentation/config/rebase.txt b/Documentation/config/rebase.txt index d59576dbb2..c6187ab28b 100644 --- a/Documentation/config/rebase.txt +++ b/Documentation/config/rebase.txt @@ -40,7 +40,7 @@ rebase.missingCommitsCheck:: rebase.instructionFormat:: A format string, as specified in linkgit:git-log[1], to be used for the todo list during an interactive rebase. The format will - automatically have the long commit hash prepended to the format. + automatically have the commit hash prepended to the format. rebase.abbreviateCommands:: If set to true, `git rebase` will use abbreviated command names in the diff --git a/Documentation/fetch-options.txt b/Documentation/fetch-options.txt index a1d6633a4f..54ebb4452e 100644 --- a/Documentation/fetch-options.txt +++ b/Documentation/fetch-options.txt @@ -1,5 +1,6 @@ ---all:: - Fetch all remotes. +--[no-]all:: + Fetch all remotes. This overrides the configuration variable + `fetch.all`. -a:: --append:: diff --git a/Documentation/git-bisect.txt b/Documentation/git-bisect.txt index aa02e46224..8e01f1d618 100644 --- a/Documentation/git-bisect.txt +++ b/Documentation/git-bisect.txt @@ -17,7 +17,7 @@ The command takes various subcommands, and different options depending on the subcommand: git bisect start [--term-(new|bad)=<term-new> --term-(old|good)=<term-old>] - [--no-checkout] [--first-parent] [<bad> [<good>...]] [--] [<paths>...] + [--no-checkout] [--first-parent] [<bad> [<good>...]] [--] [<pathspec>...] git bisect (bad|new|<term-new>) [<rev>] git bisect (good|old|<term-old>) [<rev>...] git bisect terms [--term-good | --term-bad] @@ -299,7 +299,7 @@ Cutting down bisection by giving more parameters to bisect start You can further cut down the number of trials, if you know what part of the tree is involved in the problem you are tracking down, by specifying -path parameters when issuing the `bisect start` command: +pathspec parameters when issuing the `bisect start` command: ------------ $ git bisect start -- arch/i386 include/asm-i386 diff --git a/Documentation/git-branch.txt b/Documentation/git-branch.txt index 4395aa9354..0b08442932 100644 --- a/Documentation/git-branch.txt +++ b/Documentation/git-branch.txt @@ -312,7 +312,8 @@ superproject's "origin/main", but tracks the submodule's "origin/main". option is omitted, the current HEAD will be used instead. <oldbranch>:: - The name of an existing branch to rename. + The name of an existing branch. If this option is omitted, + the name of the current branch will be used instead. <newbranch>:: The new name for an existing branch. The same restrictions as for diff --git a/Documentation/git-clone.txt b/Documentation/git-clone.txt index c37c4a37f7..6e43eb9c20 100644 --- a/Documentation/git-clone.txt +++ b/Documentation/git-clone.txt @@ -311,6 +311,12 @@ or `--mirror` is given) The result is Git repository can be separated from working tree. +--ref-format=<ref-format:: + +Specify the given ref storage format for the repository. The valid values are: ++ +include::ref-storage-format.txt[] + -j <n>:: --jobs <n>:: The number of submodules fetched at the same time. diff --git a/Documentation/git-init.txt b/Documentation/git-init.txt index 6f0d2973bf..e8dc645bb5 100644 --- a/Documentation/git-init.txt +++ b/Documentation/git-init.txt @@ -11,6 +11,7 @@ SYNOPSIS [verse] 'git init' [-q | --quiet] [--bare] [--template=<template-directory>] [--separate-git-dir <git-dir>] [--object-format=<format>] + [--ref-format=<format>] [-b <branch-name> | --initial-branch=<branch-name>] [--shared[=<permissions>]] [<directory>] @@ -57,6 +58,12 @@ values are 'sha1' and (if enabled) 'sha256'. 'sha1' is the default. + include::object-format-disclaimer.txt[] +--ref-format=<format>:: + +Specify the given ref storage format for the repository. The valid values are: ++ +include::ref-storage-format.txt[] + --template=<template-directory>:: Specify the directory from which templates will be used. (See the "TEMPLATE diff --git a/Documentation/git-rebase.txt b/Documentation/git-rebase.txt index 1dd6555f66..8a8d32161b 100644 --- a/Documentation/git-rebase.txt +++ b/Documentation/git-rebase.txt @@ -523,7 +523,7 @@ See also INCOMPATIBLE OPTIONS below. + The commit list format can be changed by setting the configuration option rebase.instructionFormat. A customized instruction format will automatically -have the long commit hash prepended to the format. +have the commit hash prepended to the format. + See also INCOMPATIBLE OPTIONS below. @@ -626,13 +626,16 @@ See also INCOMPATIBLE OPTIONS below. Automatically reschedule `exec` commands that failed. This only makes sense in interactive mode (or when an `--exec` option was provided). + -Even though this option applies once a rebase is started, it's set for -the whole rebase at the start based on either the -`rebase.rescheduleFailedExec` configuration (see linkgit:git-config[1] -or "CONFIGURATION" below) or whether this option is -provided. Otherwise an explicit `--no-reschedule-failed-exec` at the -start would be overridden by the presence of -`rebase.rescheduleFailedExec=true` configuration. +This option applies once a rebase is started. It is preserved for the whole +rebase based on, in order, the command line option provided to the initial `git +rebase`, the `rebase.rescheduleFailedExec` configuration (see +linkgit:git-config[1] or "CONFIGURATION" below), or it defaults to false. ++ +Recording this option for the whole rebase is a convenience feature. Otherwise +an explicit `--no-reschedule-failed-exec` at the start would be overridden by +the presence of a `rebase.rescheduleFailedExec=true` configuration when `git +rebase --continue` is invoked. Currently, you cannot pass +`--[no-]reschedule-failed-exec` to `git rebase --continue`. --update-refs:: --no-update-refs:: diff --git a/Documentation/git-rev-parse.txt b/Documentation/git-rev-parse.txt index 912fab9f5e..546faf9017 100644 --- a/Documentation/git-rev-parse.txt +++ b/Documentation/git-rev-parse.txt @@ -307,6 +307,9 @@ The following options are unaffected by `--path-format`: input, multiple algorithms may be printed, space-separated. If not specified, the default is "storage". +--show-ref-format:: + Show the reference storage format used for the repository. + Other Options ~~~~~~~~~~~~~ diff --git a/Documentation/git.txt b/Documentation/git.txt index bf9e6af695..da4e8d1303 100644 --- a/Documentation/git.txt +++ b/Documentation/git.txt @@ -556,6 +556,11 @@ double-quotes and respecting backslash escapes. E.g., the value is always used. The default is "sha1". See `--object-format` in linkgit:git-init[1]. +`GIT_DEFAULT_REF_FORMAT`:: + If this variable is set, the default reference backend format for new + repositories will be set to this value. The default is "files". + See `--ref-format` in linkgit:git-init[1]. + Git Commits ~~~~~~~~~~~ `GIT_AUTHOR_NAME`:: @@ -724,13 +729,12 @@ for further details. waiting for someone with sufficient permissions to fix it. `GIT_FLUSH`:: -// NEEDSWORK: make it into a usual Boolean environment variable - If this environment variable is set to "1", then commands such + If this Boolean environment variable is set to true, then commands such as 'git blame' (in incremental mode), 'git rev-list', 'git log', 'git check-attr' and 'git check-ignore' will force a flush of the output stream after each record have been flushed. If this - variable is set to "0", the output of these commands will be done + variable is set to false, the output of these commands will be done using completely buffered I/O. If this environment variable is not set, Git will choose buffered or record-oriented flushing based on whether stdout appears to be redirected to a file or not. diff --git a/Documentation/gitattributes.txt b/Documentation/gitattributes.txt index 8c1793c148..201bdf5edb 100644 --- a/Documentation/gitattributes.txt +++ b/Documentation/gitattributes.txt @@ -100,6 +100,21 @@ for a path to `Unspecified` state. This can be done by listing the name of the attribute prefixed with an exclamation point `!`. +RESERVED BUILTIN_* ATTRIBUTES +----------------------------- + +builtin_* is a reserved namespace for builtin attribute values. Any +user defined attributes under this namespace will be ignored and +trigger a warning. + +`builtin_objectmode` +~~~~~~~~~~~~~~~~~~~~ +This attribute is for filtering files by their file bit modes (40000, +120000, 160000, 100755, 100644). e.g. ':(attr:builtin_objectmode=160000)'. +You may also check these values with `git check-attr builtin_objectmode -- <file>`. +If the object is not in the index `git check-attr --cached` will return unspecified. + + EFFECTS ------- diff --git a/Documentation/gitformat-pack.txt b/Documentation/gitformat-pack.txt index 9fcb29a9c8..d6ae229be5 100644 --- a/Documentation/gitformat-pack.txt +++ b/Documentation/gitformat-pack.txt @@ -396,6 +396,15 @@ CHUNK DATA: is padded at the end with between 0 and 3 NUL bytes to make the chunk size a multiple of 4 bytes. + Bitmapped Packfiles (ID: {'B', 'T', 'M', 'P'}) + Stores a table of two 4-byte unsigned integers in network order. + Each table entry corresponds to a single pack (in the order that + they appear above in the `PNAM` chunk). The values for each table + entry are as follows: + - The first bit position (in pseudo-pack order, see below) to + contain an object from that pack. + - The number of bits whose objects are selected from that pack. + OID Fanout (ID: {'O', 'I', 'D', 'F'}) The ith entry, F[i], stores the number of OIDs with first byte at most i. Thus F[255] stores the total @@ -509,6 +518,73 @@ packs arranged in MIDX order (with the preferred pack coming first). The MIDX's reverse index is stored in the optional 'RIDX' chunk within the MIDX itself. +=== `BTMP` chunk + +The Bitmapped Packfiles (`BTMP`) chunk encodes additional information +about the objects in the multi-pack index's reachability bitmap. Recall +that objects from the MIDX are arranged in "pseudo-pack" order (see +above) for reachability bitmaps. + +From the example above, suppose we have packs "a", "b", and "c", with +10, 15, and 20 objects, respectively. In pseudo-pack order, those would +be arranged as follows: + + |a,0|a,1|...|a,9|b,0|b,1|...|b,14|c,0|c,1|...|c,19| + +When working with single-pack bitmaps (or, equivalently, multi-pack +reachability bitmaps with a preferred pack), linkgit:git-pack-objects[1] +performs ``verbatim'' reuse, attempting to reuse chunks of the bitmapped +or preferred packfile instead of adding objects to the packing list. + +When a chunk of bytes is reused from an existing pack, any objects +contained therein do not need to be added to the packing list, saving +memory and CPU time. But a chunk from an existing packfile can only be +reused when the following conditions are met: + + - The chunk contains only objects which were requested by the caller + (i.e. does not contain any objects which the caller didn't ask for + explicitly or implicitly). + + - All objects stored in non-thin packs as offset- or reference-deltas + also include their base object in the resulting pack. + +The `BTMP` chunk encodes the necessary information in order to implement +multi-pack reuse over a set of packfiles as described above. +Specifically, the `BTMP` chunk encodes three pieces of information (all +32-bit unsigned integers in network byte-order) for each packfile `p` +that is stored in the MIDX, as follows: + +`bitmap_pos`:: The first bit position (in pseudo-pack order) in the + multi-pack index's reachability bitmap occupied by an object from `p`. + +`bitmap_nr`:: The number of bit positions (including the one at + `bitmap_pos`) that encode objects from that pack `p`. + +For example, the `BTMP` chunk corresponding to the above example (with +packs ``a'', ``b'', and ``c'') would look like: + +[cols="1,2,2"] +|=== +| |`bitmap_pos` |`bitmap_nr` + +|packfile ``a'' +|`0` +|`10` + +|packfile ``b'' +|`10` +|`15` + +|packfile ``c'' +|`25` +|`20` +|=== + +With this information in place, we can treat each packfile as +individually reusable in the same fashion as verbatim pack reuse is +performed on individual packs prior to the implementation of the `BTMP` +chunk. + == cruft packs The cruft packs feature offer an alternative to Git's traditional mechanism of diff --git a/Documentation/ref-storage-format.txt b/Documentation/ref-storage-format.txt new file mode 100644 index 0000000000..1a65cac468 --- /dev/null +++ b/Documentation/ref-storage-format.txt @@ -0,0 +1 @@ +* `files` for loose files with packed-refs. This is the default. diff --git a/Documentation/technical/repository-version.txt b/Documentation/technical/repository-version.txt index 045a76756f..27be3741e6 100644 --- a/Documentation/technical/repository-version.txt +++ b/Documentation/technical/repository-version.txt @@ -100,3 +100,8 @@ If set, by default "git config" reads from both "config" and multiple working directory mode, "config" file is shared while "config.worktree" is per-working directory (i.e., it's in GIT_COMMON_DIR/worktrees/<id>/config.worktree) + +==== `refStorage` + +Specifies the file format for the ref database. The only valid value +is `files` (loose references with a packed-refs file). @@ -17,6 +17,7 @@ #include "utf8.h" #include "quote.h" #include "read-cache-ll.h" +#include "refs.h" #include "revision.h" #include "object-store-ll.h" #include "setup.h" @@ -183,6 +184,15 @@ static void all_attrs_init(struct attr_hashmap *map, struct attr_check *check) } } +/* + * Atribute name cannot begin with "builtin_" which + * is a reserved namespace for built in attributes values. + */ +static int attr_name_reserved(const char *name) +{ + return starts_with(name, "builtin_"); +} + static int attr_name_valid(const char *name, size_t namelen) { /* @@ -315,7 +325,7 @@ static const char *parse_attr(const char *src, int lineno, const char *cp, cp++; len--; } - if (!attr_name_valid(cp, len)) { + if (!attr_name_valid(cp, len) || attr_name_reserved(cp)) { report_invalid_attr(cp, len, src, lineno); return NULL; } @@ -379,7 +389,7 @@ static struct match_attr *parse_attr_line(const char *line, const char *src, name += strlen(ATTRIBUTE_MACRO_PREFIX); name += strspn(name, blank); namelen = strcspn(name, blank); - if (!attr_name_valid(name, namelen)) { + if (!attr_name_valid(name, namelen) || attr_name_reserved(name)) { report_invalid_attr(name, namelen, src, lineno); goto fail_return; } @@ -1240,6 +1250,85 @@ static struct object_id *default_attr_source(void) return &attr_source; } +static const char *interned_mode_string(unsigned int mode) +{ + static struct { + unsigned int val; + char str[7]; + } mode_string[] = { + { .val = 0040000 }, + { .val = 0100644 }, + { .val = 0100755 }, + { .val = 0120000 }, + { .val = 0160000 }, + }; + int i; + + for (i = 0; i < ARRAY_SIZE(mode_string); i++) { + if (mode_string[i].val != mode) + continue; + if (!*mode_string[i].str) + snprintf(mode_string[i].str, sizeof(mode_string[i].str), + "%06o", mode); + return mode_string[i].str; + } + BUG("Unsupported mode 0%o", mode); +} + +static const char *builtin_object_mode_attr(struct index_state *istate, const char *path) +{ + unsigned int mode; + + if (direction == GIT_ATTR_CHECKIN) { + struct object_id oid; + struct stat st; + if (lstat(path, &st)) + die_errno(_("unable to stat '%s'"), path); + mode = canon_mode(st.st_mode); + if (S_ISDIR(mode)) { + /* + *`path` is either a directory or it is a submodule, + * in which case it is already indexed as submodule + * or it does not exist in the index yet and we need to + * check if we can resolve to a ref. + */ + int pos = index_name_pos(istate, path, strlen(path)); + if (pos >= 0) { + if (S_ISGITLINK(istate->cache[pos]->ce_mode)) + mode = istate->cache[pos]->ce_mode; + } else if (resolve_gitlink_ref(path, "HEAD", &oid) == 0) { + mode = S_IFGITLINK; + } + } + } else { + /* + * For GIT_ATTR_CHECKOUT and GIT_ATTR_INDEX we only check + * for mode in the index. + */ + int pos = index_name_pos(istate, path, strlen(path)); + if (pos >= 0) + mode = istate->cache[pos]->ce_mode; + else + return ATTR__UNSET; + } + + return interned_mode_string(mode); +} + + +static const char *compute_builtin_attr(struct index_state *istate, + const char *path, + const struct git_attr *attr) { + static const struct git_attr *object_mode_attr; + + if (!object_mode_attr) + object_mode_attr = git_attr("builtin_objectmode"); + + if (attr == object_mode_attr) + return builtin_object_mode_attr(istate, path); + return ATTR__UNSET; +} + void git_check_attr(struct index_state *istate, const char *path, struct attr_check *check) @@ -1253,7 +1342,7 @@ void git_check_attr(struct index_state *istate, unsigned int n = check->items[i].attr->attr_nr; const char *value = check->all_attrs[n].value; if (value == ATTR__UNKNOWN) - value = ATTR__UNSET; + value = compute_builtin_attr(istate, path, check->all_attrs[n].attr); check->items[i].value = value; } } diff --git a/builtin/clone.c b/builtin/clone.c index 0605fa79aa..bad1b70ce8 100644 --- a/builtin/clone.c +++ b/builtin/clone.c @@ -71,6 +71,7 @@ static char *remote_name = NULL; static char *option_branch = NULL; static struct string_list option_not = STRING_LIST_INIT_NODUP; static const char *real_git_dir; +static const char *ref_format; static char *option_upload_pack = "git-upload-pack"; static int option_verbosity; static int option_progress = -1; @@ -156,6 +157,8 @@ static struct option builtin_clone_options[] = { N_("any cloned submodules will be shallow")), OPT_STRING(0, "separate-git-dir", &real_git_dir, N_("gitdir"), N_("separate git dir from working tree")), + OPT_STRING(0, "ref-format", &ref_format, N_("format"), + N_("specify the reference format to use")), OPT_STRING_LIST('c', "config", &option_config, N_("key=value"), N_("set config inside the new repository")), OPT_STRING_LIST(0, "server-option", &server_options, @@ -931,6 +934,7 @@ int cmd_clone(int argc, const char **argv, const char *prefix) int submodule_progress; int filter_submodules = 0; int hash_algo; + unsigned int ref_storage_format = REF_STORAGE_FORMAT_UNKNOWN; const int do_not_override_repo_unix_permissions = -1; struct transport_ls_refs_options transport_ls_refs_options = @@ -956,6 +960,12 @@ int cmd_clone(int argc, const char **argv, const char *prefix) if (option_single_branch == -1) option_single_branch = deepen ? 1 : 0; + if (ref_format) { + ref_storage_format = ref_storage_format_by_name(ref_format); + if (ref_storage_format == REF_STORAGE_FORMAT_UNKNOWN) + die(_("unknown ref storage format '%s'"), ref_format); + } + if (option_mirror) option_bare = 1; @@ -1106,7 +1116,8 @@ int cmd_clone(int argc, const char **argv, const char *prefix) * repository, and reference backends may persist that information into * their on-disk data structures. */ - init_db(git_dir, real_git_dir, option_template, GIT_HASH_UNKNOWN, NULL, + init_db(git_dir, real_git_dir, option_template, GIT_HASH_UNKNOWN, + ref_storage_format, NULL, do_not_override_repo_unix_permissions, INIT_DB_QUIET | INIT_DB_SKIP_REFDB); if (real_git_dir) { @@ -1289,9 +1300,9 @@ int cmd_clone(int argc, const char **argv, const char *prefix) * ours to the same thing. */ hash_algo = hash_algo_by_ptr(transport_get_hash_algo(transport)); - initialize_repository_version(hash_algo, 1); + initialize_repository_version(hash_algo, the_repository->ref_storage_format, 1); repo_set_hash_algo(the_repository, hash_algo); - create_reference_database(NULL, 1); + create_reference_database(the_repository->ref_storage_format, NULL, 1); /* * Before fetching from the remote, download and install bundle diff --git a/builtin/fetch.c b/builtin/fetch.c index 119f1a72ac..3aedfd1bb6 100644 --- a/builtin/fetch.c +++ b/builtin/fetch.c @@ -100,6 +100,7 @@ static struct string_list negotiation_tip = STRING_LIST_INIT_NODUP; struct fetch_config { enum display_format display_format; + int all; int prune; int prune_tags; int show_forced_updates; @@ -113,6 +114,11 @@ static int git_fetch_config(const char *k, const char *v, { struct fetch_config *fetch_config = cb; + if (!strcmp(k, "fetch.all")) { + fetch_config->all = git_config_bool(k, v); + return 0; + } + if (!strcmp(k, "fetch.prune")) { fetch_config->prune = git_config_bool(k, v); return 0; @@ -2130,7 +2136,7 @@ int cmd_fetch(int argc, const char **argv, const char *prefix) const char *bundle_uri; struct string_list list = STRING_LIST_INIT_DUP; struct remote *remote = NULL; - int all = 0, multiple = 0; + int all = -1, multiple = 0; int result = 0; int prune_tags_ok = 1; int enable_auto_gc = 1; @@ -2335,11 +2341,20 @@ int cmd_fetch(int argc, const char **argv, const char *prefix) fetch_bundle_uri(the_repository, bundle_uri, NULL)) warning(_("failed to fetch bundles from '%s'"), bundle_uri); + if (all < 0) { + /* + * no --[no-]all given; + * only use config option if no remote was explicitly specified + */ + all = (!argc) ? config.all : 0; + } + if (all) { if (argc == 1) die(_("fetch --all does not take a repository argument")); else if (argc > 1) die(_("fetch --all does not make sense with refspecs")); + (void) for_each_remote(get_one_remote_for_fetch, &list); /* do not do fetch_multiple() of one */ diff --git a/builtin/index-pack.c b/builtin/index-pack.c index 0841b6940a..1ea87e01f2 100644 --- a/builtin/index-pack.c +++ b/builtin/index-pack.c @@ -1255,6 +1255,7 @@ static void resolve_deltas(void) base_cache_limit = delta_base_cache_limit * nr_threads; if (nr_threads > 1 || getenv("GIT_FORCE_THREADS")) { init_thread(); + work_lock(); for (i = 0; i < nr_threads; i++) { int ret = pthread_create(&thread_data[i].thread, NULL, threaded_second_pass, thread_data + i); @@ -1262,6 +1263,7 @@ static void resolve_deltas(void) die(_("unable to create thread: %s"), strerror(ret)); } + work_unlock(); for (i = 0; i < nr_threads; i++) pthread_join(thread_data[i].thread, NULL); cleanup_thread(); diff --git a/builtin/init-db.c b/builtin/init-db.c index b89814a6f8..0170469b84 100644 --- a/builtin/init-db.c +++ b/builtin/init-db.c @@ -10,6 +10,8 @@ #include "object-file.h" #include "parse-options.h" #include "path.h" +#include "refs.h" +#include "repository.h" #include "setup.h" #include "strbuf.h" @@ -56,6 +58,7 @@ static int shared_callback(const struct option *opt, const char *arg, int unset) static const char *const init_db_usage[] = { N_("git init [-q | --quiet] [--bare] [--template=<template-directory>]\n" " [--separate-git-dir <git-dir>] [--object-format=<format>]\n" + " [--ref-format=<format>]\n" " [-b <branch-name> | --initial-branch=<branch-name>]\n" " [--shared[=<permissions>]] [<directory>]"), NULL @@ -75,8 +78,10 @@ int cmd_init_db(int argc, const char **argv, const char *prefix) const char *template_dir = NULL; unsigned int flags = 0; const char *object_format = NULL; + const char *ref_format = NULL; const char *initial_branch = NULL; int hash_algo = GIT_HASH_UNKNOWN; + unsigned int ref_storage_format = REF_STORAGE_FORMAT_UNKNOWN; int init_shared_repository = -1; const struct option init_db_options[] = { OPT_STRING(0, "template", &template_dir, N_("template-directory"), @@ -94,6 +99,8 @@ int cmd_init_db(int argc, const char **argv, const char *prefix) N_("override the name of the initial branch")), OPT_STRING(0, "object-format", &object_format, N_("hash"), N_("specify the hash algorithm to use")), + OPT_STRING(0, "ref-format", &ref_format, N_("format"), + N_("specify the reference format to use")), OPT_END() }; @@ -157,6 +164,12 @@ int cmd_init_db(int argc, const char **argv, const char *prefix) die(_("unknown hash algorithm '%s'"), object_format); } + if (ref_format) { + ref_storage_format = ref_storage_format_by_name(ref_format); + if (ref_storage_format == REF_STORAGE_FORMAT_UNKNOWN) + die(_("unknown ref storage format '%s'"), ref_format); + } + if (init_shared_repository != -1) set_shared_repository(init_shared_repository); @@ -235,5 +248,6 @@ int cmd_init_db(int argc, const char **argv, const char *prefix) flags |= INIT_DB_EXIST_OK; return init_db(git_dir, real_git_dir, template_dir, hash_algo, - initial_branch, init_shared_repository, flags); + ref_storage_format, initial_branch, + init_shared_repository, flags); } diff --git a/builtin/pack-objects.c b/builtin/pack-objects.c index 5c8bfe1035..d8c2128a97 100644 --- a/builtin/pack-objects.c +++ b/builtin/pack-objects.c @@ -218,13 +218,19 @@ static int thin; static int num_preferred_base; static struct progress *progress_state; -static struct packed_git *reuse_packfile; +static struct bitmapped_pack *reuse_packfiles; +static size_t reuse_packfiles_nr; +static size_t reuse_packfiles_used_nr; static uint32_t reuse_packfile_objects; static struct bitmap *reuse_packfile_bitmap; static int use_bitmap_index_default = 1; static int use_bitmap_index = -1; -static int allow_pack_reuse = 1; +static enum { + NO_PACK_REUSE = 0, + SINGLE_PACK_REUSE, + MULTI_PACK_REUSE, +} allow_pack_reuse = SINGLE_PACK_REUSE; static enum { WRITE_BITMAP_FALSE = 0, WRITE_BITMAP_QUIET, @@ -1010,7 +1016,9 @@ static off_t find_reused_offset(off_t where) return reused_chunks[lo-1].difference; } -static void write_reused_pack_one(size_t pos, struct hashfile *out, +static void write_reused_pack_one(struct packed_git *reuse_packfile, + size_t pos, struct hashfile *out, + off_t pack_start, struct pack_window **w_curs) { off_t offset, next, cur; @@ -1020,7 +1028,8 @@ static void write_reused_pack_one(size_t pos, struct hashfile *out, offset = pack_pos_to_offset(reuse_packfile, pos); next = pack_pos_to_offset(reuse_packfile, pos + 1); - record_reused_object(offset, offset - hashfile_total(out)); + record_reused_object(offset, + offset - (hashfile_total(out) - pack_start)); cur = offset; type = unpack_object_header(reuse_packfile, w_curs, &cur, &size); @@ -1088,41 +1097,93 @@ static void write_reused_pack_one(size_t pos, struct hashfile *out, copy_pack_data(out, reuse_packfile, w_curs, offset, next - offset); } -static size_t write_reused_pack_verbatim(struct hashfile *out, +static size_t write_reused_pack_verbatim(struct bitmapped_pack *reuse_packfile, + struct hashfile *out, + off_t pack_start, struct pack_window **w_curs) { - size_t pos = 0; + size_t pos = reuse_packfile->bitmap_pos; + size_t end; + + if (pos % BITS_IN_EWORD) { + size_t word_pos = (pos / BITS_IN_EWORD); + size_t offset = pos % BITS_IN_EWORD; + size_t last; + eword_t word = reuse_packfile_bitmap->words[word_pos]; + + if (offset + reuse_packfile->bitmap_nr < BITS_IN_EWORD) + last = offset + reuse_packfile->bitmap_nr; + else + last = BITS_IN_EWORD; + + for (; offset < last; offset++) { + if (word >> offset == 0) + return word_pos; + if (!bitmap_get(reuse_packfile_bitmap, + word_pos * BITS_IN_EWORD + offset)) + return word_pos; + } - while (pos < reuse_packfile_bitmap->word_alloc && - reuse_packfile_bitmap->words[pos] == (eword_t)~0) - pos++; + pos += BITS_IN_EWORD - (pos % BITS_IN_EWORD); + } + + /* + * Now we're going to copy as many whole eword_t's as possible. + * "end" is the index of the last whole eword_t we copy, but + * there may be additional bits to process. Those are handled + * individually by write_reused_pack(). + * + * Begin by advancing to the first word boundary in range of the + * bit positions occupied by objects in "reuse_packfile". Then + * pick the last word boundary in the same range. If we have at + * least one word's worth of bits to process, continue on. + */ + end = reuse_packfile->bitmap_pos + reuse_packfile->bitmap_nr; + if (end % BITS_IN_EWORD) + end -= end % BITS_IN_EWORD; + if (pos >= end) + return reuse_packfile->bitmap_pos / BITS_IN_EWORD; - if (pos) { - off_t to_write; + while (pos < end && + reuse_packfile_bitmap->words[pos / BITS_IN_EWORD] == (eword_t)~0) + pos += BITS_IN_EWORD; - written = (pos * BITS_IN_EWORD); - to_write = pack_pos_to_offset(reuse_packfile, written) - - sizeof(struct pack_header); + if (pos > end) + pos = end; + + if (reuse_packfile->bitmap_pos < pos) { + off_t pack_start_off = pack_pos_to_offset(reuse_packfile->p, 0); + off_t pack_end_off = pack_pos_to_offset(reuse_packfile->p, + pos - reuse_packfile->bitmap_pos); + + written += pos - reuse_packfile->bitmap_pos; /* We're recording one chunk, not one object. */ - record_reused_object(sizeof(struct pack_header), 0); + record_reused_object(pack_start_off, + pack_start_off - (hashfile_total(out) - pack_start)); hashflush(out); - copy_pack_data(out, reuse_packfile, w_curs, - sizeof(struct pack_header), to_write); + copy_pack_data(out, reuse_packfile->p, w_curs, + pack_start_off, pack_end_off - pack_start_off); display_progress(progress_state, written); } - return pos; + if (pos % BITS_IN_EWORD) + BUG("attempted to jump past a word boundary to %"PRIuMAX, + (uintmax_t)pos); + return pos / BITS_IN_EWORD; } -static void write_reused_pack(struct hashfile *f) +static void write_reused_pack(struct bitmapped_pack *reuse_packfile, + struct hashfile *f) { - size_t i = 0; + size_t i = reuse_packfile->bitmap_pos / BITS_IN_EWORD; uint32_t offset; + off_t pack_start = hashfile_total(f) - sizeof(struct pack_header); struct pack_window *w_curs = NULL; if (allow_ofs_delta) - i = write_reused_pack_verbatim(f, &w_curs); + i = write_reused_pack_verbatim(reuse_packfile, f, pack_start, + &w_curs); for (; i < reuse_packfile_bitmap->word_alloc; ++i) { eword_t word = reuse_packfile_bitmap->words[i]; @@ -1133,16 +1194,23 @@ static void write_reused_pack(struct hashfile *f) break; offset += ewah_bit_ctz64(word >> offset); + if (pos + offset < reuse_packfile->bitmap_pos) + continue; + if (pos + offset >= reuse_packfile->bitmap_pos + reuse_packfile->bitmap_nr) + goto done; /* * Can use bit positions directly, even for MIDX * bitmaps. See comment in try_partial_reuse() * for why. */ - write_reused_pack_one(pos + offset, f, &w_curs); + write_reused_pack_one(reuse_packfile->p, + pos + offset - reuse_packfile->bitmap_pos, + f, pack_start, &w_curs); display_progress(progress_state, ++written); } } +done: unuse_pack(&w_curs); } @@ -1194,9 +1262,14 @@ static void write_pack_file(void) offset = write_pack_header(f, nr_remaining); - if (reuse_packfile) { + if (reuse_packfiles_nr) { assert(pack_to_stdout); - write_reused_pack(f); + for (j = 0; j < reuse_packfiles_nr; j++) { + reused_chunks_nr = 0; + write_reused_pack(&reuse_packfiles[j], f); + if (reused_chunks_nr) + reuse_packfiles_used_nr++; + } offset = hashfile_total(f); } @@ -3172,7 +3245,19 @@ static int git_pack_config(const char *k, const char *v, return 0; } if (!strcmp(k, "pack.allowpackreuse")) { - allow_pack_reuse = git_config_bool(k, v); + int res = git_parse_maybe_bool_text(v); + if (res < 0) { + if (!strcasecmp(v, "single")) + allow_pack_reuse = SINGLE_PACK_REUSE; + else if (!strcasecmp(v, "multi")) + allow_pack_reuse = MULTI_PACK_REUSE; + else + die(_("invalid pack.allowPackReuse value: '%s'"), v); + } else if (res) { + allow_pack_reuse = SINGLE_PACK_REUSE; + } else { + allow_pack_reuse = NO_PACK_REUSE; + } return 0; } if (!strcmp(k, "pack.threads")) { @@ -3931,7 +4016,7 @@ static void loosen_unused_packed_objects(void) */ static int pack_options_allow_reuse(void) { - return allow_pack_reuse && + return allow_pack_reuse != NO_PACK_REUSE && pack_to_stdout && !ignore_packed_keep_on_disk && !ignore_packed_keep_in_core && @@ -3944,13 +4029,18 @@ static int get_object_list_from_bitmap(struct rev_info *revs) if (!(bitmap_git = prepare_bitmap_walk(revs, 0))) return -1; - if (pack_options_allow_reuse() && - !reuse_partial_packfile_from_bitmap( - bitmap_git, - &reuse_packfile, - &reuse_packfile_objects, - &reuse_packfile_bitmap)) { - assert(reuse_packfile_objects); + if (pack_options_allow_reuse()) + reuse_partial_packfile_from_bitmap(bitmap_git, + &reuse_packfiles, + &reuse_packfiles_nr, + &reuse_packfile_bitmap, + allow_pack_reuse == MULTI_PACK_REUSE); + + if (reuse_packfiles) { + reuse_packfile_objects = bitmap_popcount(reuse_packfile_bitmap); + if (!reuse_packfile_objects) + BUG("expected non-empty reuse bitmap"); + nr_result += reuse_packfile_objects; nr_seen += reuse_packfile_objects; display_progress(progress_state, nr_seen); @@ -4518,11 +4608,20 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix) fprintf_ln(stderr, _("Total %"PRIu32" (delta %"PRIu32")," " reused %"PRIu32" (delta %"PRIu32")," - " pack-reused %"PRIu32), + " pack-reused %"PRIu32" (from %"PRIuMAX")"), written, written_delta, reused, reused_delta, - reuse_packfile_objects); + reuse_packfile_objects, + (uintmax_t)reuse_packfiles_used_nr); + + trace2_data_intmax("pack-objects", the_repository, "written", written); + trace2_data_intmax("pack-objects", the_repository, "written/delta", written_delta); + trace2_data_intmax("pack-objects", the_repository, "reused", reused); + trace2_data_intmax("pack-objects", the_repository, "reused/delta", reused_delta); + trace2_data_intmax("pack-objects", the_repository, "pack-reused", reuse_packfile_objects); + trace2_data_intmax("pack-objects", the_repository, "packs-reused", reuse_packfiles_used_nr); cleanup: + clear_packing_data(&to_pack); list_objects_filter_release(&filter_options); strvec_clear(&rp); diff --git a/builtin/rev-parse.c b/builtin/rev-parse.c index 917f122440..d08987646a 100644 --- a/builtin/rev-parse.c +++ b/builtin/rev-parse.c @@ -1062,6 +1062,10 @@ int cmd_rev_parse(int argc, const char **argv, const char *prefix) puts(the_hash_algo->name); continue; } + if (!strcmp(arg, "--show-ref-format")) { + puts(ref_storage_format_to_name(the_repository->ref_storage_format)); + continue; + } if (!strcmp(arg, "--end-of-options")) { seen_end_of_options = 1; if (filter & (DO_FLAGS | DO_REVS)) diff --git a/ci/install-docker-dependencies.sh b/ci/install-docker-dependencies.sh index 48c43f0f90..eb2c9e1eca 100755 --- a/ci/install-docker-dependencies.sh +++ b/ci/install-docker-dependencies.sh @@ -21,7 +21,7 @@ linux-musl) apache2 apache2-http2 apache2-proxy apache2-ssl apache2-webdav apr-util-dbd_sqlite3 \ bash cvs gnupg perl-cgi perl-dbd-sqlite >/dev/null ;; -linux-*) +linux-*|StaticAnalysis) # Required so that apt doesn't wait for user input on certain packages. export DEBIAN_FRONTEND=noninteractive @@ -31,6 +31,11 @@ linux-*) perl-modules liberror-perl libauthen-sasl-perl libemail-valid-perl \ libdbd-sqlite3-perl libio-socket-ssl-perl libnet-smtp-ssl-perl ${CC_PACKAGE:-${CC:-gcc}} \ apache2 cvs cvsps gnupg libcgi-pm-perl subversion + + if test "$jobname" = StaticAnalysis + then + apt install -q -y coccinelle + fi ;; pedantic) dnf -yq update >/dev/null && diff --git a/commit-graph.c b/commit-graph.c index bba316913c..f86c5e9f94 100644 --- a/commit-graph.c +++ b/commit-graph.c @@ -806,6 +806,9 @@ struct bloom_filter_settings *get_bloom_filter_settings(struct repository *r) void close_commit_graph(struct raw_object_store *o) { + if (!o->commit_graph) + return; + clear_commit_graph_data_slab(&commit_graph_data_slab); free_commit_graph(o->commit_graph); o->commit_graph = NULL; diff --git a/compat/mingw.c b/compat/mingw.c index 42053c1f65..238a84ddba 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2684,6 +2684,30 @@ static PSID get_current_user_sid(void) return result; } +static BOOL user_sid_to_user_name(PSID sid, LPSTR *str) +{ + SID_NAME_USE pe_use; + DWORD len_user = 0, len_domain = 0; + BOOL translate_sid_to_user; + + /* + * returns only FALSE, because the string pointers are NULL + */ + LookupAccountSidA(NULL, sid, NULL, &len_user, NULL, &len_domain, + &pe_use); + /* + * Alloc needed space of the strings + */ + ALLOC_ARRAY((*str), (size_t)len_domain + (size_t)len_user); + translate_sid_to_user = LookupAccountSidA(NULL, sid, + (*str) + len_domain, &len_user, *str, &len_domain, &pe_use); + if (!translate_sid_to_user) + FREE_AND_NULL(*str); + else + (*str)[len_domain] = '/'; + return translate_sid_to_user; +} + static int acls_supported(const char *path) { size_t offset = offset_1st_component(path); @@ -2765,27 +2789,47 @@ int is_path_owned_by_current_sid(const char *path, struct strbuf *report) strbuf_addf(report, "'%s' is on a file system that does " "not record ownership\n", path); } else if (report) { - LPSTR str1, str2, to_free1 = NULL, to_free2 = NULL; + LPSTR str1, str2, str3, str4, to_free1 = NULL, + to_free3 = NULL, to_local_free2 = NULL, + to_local_free4 = NULL; - if (ConvertSidToStringSidA(sid, &str1)) + if (user_sid_to_user_name(sid, &str1)) to_free1 = str1; else str1 = "(inconvertible)"; - - if (!current_user_sid) - str2 = "(none)"; - else if (!IsValidSid(current_user_sid)) - str2 = "(invalid)"; - else if (ConvertSidToStringSidA(current_user_sid, &str2)) - to_free2 = str2; + if (ConvertSidToStringSidA(sid, &str2)) + to_local_free2 = str2; else str2 = "(inconvertible)"; + + if (!current_user_sid) { + str3 = "(none)"; + str4 = "(none)"; + } + else if (!IsValidSid(current_user_sid)) { + str3 = "(invalid)"; + str4 = "(invalid)"; + } else { + if (user_sid_to_user_name(current_user_sid, + &str3)) + to_free3 = str3; + else + str3 = "(inconvertible)"; + if (ConvertSidToStringSidA(current_user_sid, + &str4)) + to_local_free4 = str4; + else + str4 = "(inconvertible)"; + } strbuf_addf(report, "'%s' is owned by:\n" - "\t'%s'\nbut the current user is:\n" - "\t'%s'\n", path, str1, str2); - LocalFree(to_free1); - LocalFree(to_free2); + "\t%s (%s)\nbut the current user is:\n" + "\t%s (%s)\n", + path, str1, str2, str3, str4); + free(to_free1); + LocalFree(to_local_free2); + free(to_free3); + LocalFree(to_local_free4); } } diff --git a/contrib/completion/git-prompt.sh b/contrib/completion/git-prompt.sh index 2c030050ae..71f179cba3 100644 --- a/contrib/completion/git-prompt.sh +++ b/contrib/completion/git-prompt.sh @@ -408,7 +408,7 @@ __git_ps1 () local repo_info rev_parse_exit_code repo_info="$(git rev-parse --git-dir --is-inside-git-dir \ - --is-bare-repository --is-inside-work-tree \ + --is-bare-repository --is-inside-work-tree --show-ref-format \ --short HEAD 2>/dev/null)" rev_parse_exit_code="$?" @@ -421,6 +421,8 @@ __git_ps1 () short_sha="${repo_info##*$'\n'}" repo_info="${repo_info%$'\n'*}" fi + local ref_format="${repo_info##*$'\n'}" + repo_info="${repo_info%$'\n'*}" local inside_worktree="${repo_info##*$'\n'}" repo_info="${repo_info%$'\n'*}" local bare_repo="${repo_info##*$'\n'}" @@ -479,12 +481,25 @@ __git_ps1 () b="$(git symbolic-ref HEAD 2>/dev/null)" else local head="" - if ! __git_eread "$g/HEAD" head; then - return $exit - fi - # is it a symbolic ref? - b="${head#ref: }" - if [ "$head" = "$b" ]; then + + case "$ref_format" in + files) + if ! __git_eread "$g/HEAD" head; then + return $exit + fi + + if [[ $head == "ref: "* ]]; then + head="${head#ref: }" + else + head="" + fi + ;; + *) + head="$(git symbolic-ref HEAD 2>/dev/null)" + ;; + esac + + if test -z "$head"; then detached=yes b="$( case "${GIT_PS1_DESCRIBE_STYLE-}" in @@ -502,6 +517,8 @@ __git_ps1 () b="$short_sha..." b="($b)" + else + b="$head" fi fi fi diff --git a/ewah/bitmap.c b/ewah/bitmap.c index 7b525b1ecd..ac7e0af622 100644 --- a/ewah/bitmap.c +++ b/ewah/bitmap.c @@ -169,6 +169,15 @@ size_t bitmap_popcount(struct bitmap *self) return count; } +int bitmap_is_empty(struct bitmap *self) +{ + size_t i; + for (i = 0; i < self->word_alloc; i++) + if (self->words[i]) + return 0; + return 1; +} + int bitmap_equals(struct bitmap *self, struct bitmap *other) { struct bitmap *big, *small; diff --git a/ewah/ewok.h b/ewah/ewok.h index 7eb8b9b630..c11d76c6f3 100644 --- a/ewah/ewok.h +++ b/ewah/ewok.h @@ -189,5 +189,6 @@ void bitmap_or_ewah(struct bitmap *self, struct ewah_bitmap *other); void bitmap_or(struct bitmap *self, const struct bitmap *other); size_t bitmap_popcount(struct bitmap *self); +int bitmap_is_empty(struct bitmap *self); #endif diff --git a/git-compat-util.h b/git-compat-util.h index 603c97e3b3..7c2a6538e5 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -1015,6 +1015,15 @@ static inline unsigned long cast_size_t_to_ulong(size_t a) return (unsigned long)a; } +static inline uint32_t cast_size_t_to_uint32_t(size_t a) +{ + if (a != (uint32_t)a) + die("object too large to read on this platform: %" + PRIuMAX" is cut off to %u", + (uintmax_t)a, (uint32_t)a); + return (uint32_t)a; +} + static inline int cast_size_t_to_int(size_t a) { if (a > INT_MAX) @@ -21,6 +21,7 @@ #include "refs.h" #include "revision.h" #include "list-objects.h" +#include "pack-revindex.h" #define MIDX_SIGNATURE 0x4d494458 /* "MIDX" */ #define MIDX_VERSION 1 @@ -33,6 +34,7 @@ #define MIDX_CHUNK_ALIGNMENT 4 #define MIDX_CHUNKID_PACKNAMES 0x504e414d /* "PNAM" */ +#define MIDX_CHUNKID_BITMAPPEDPACKS 0x42544d50 /* "BTMP" */ #define MIDX_CHUNKID_OIDFANOUT 0x4f494446 /* "OIDF" */ #define MIDX_CHUNKID_OIDLOOKUP 0x4f49444c /* "OIDL" */ #define MIDX_CHUNKID_OBJECTOFFSETS 0x4f4f4646 /* "OOFF" */ @@ -41,6 +43,7 @@ #define MIDX_CHUNK_FANOUT_SIZE (sizeof(uint32_t) * 256) #define MIDX_CHUNK_OFFSET_WIDTH (2 * sizeof(uint32_t)) #define MIDX_CHUNK_LARGE_OFFSET_WIDTH (sizeof(uint64_t)) +#define MIDX_CHUNK_BITMAPPED_PACKS_WIDTH (2 * sizeof(uint32_t)) #define MIDX_LARGE_OFFSET_NEEDED 0x80000000 #define PACK_EXPIRED UINT_MAX @@ -175,6 +178,8 @@ struct multi_pack_index *load_multi_pack_index(const char *object_dir, int local m->num_packs = get_be32(m->data + MIDX_BYTE_NUM_PACKS); + m->preferred_pack_idx = -1; + cf = init_chunkfile(NULL); if (read_table_of_contents(cf, m->data, midx_size, @@ -193,6 +198,9 @@ struct multi_pack_index *load_multi_pack_index(const char *object_dir, int local pair_chunk(cf, MIDX_CHUNKID_LARGEOFFSETS, &m->chunk_large_offsets, &m->chunk_large_offsets_len); + pair_chunk(cf, MIDX_CHUNKID_BITMAPPEDPACKS, + (const unsigned char **)&m->chunk_bitmapped_packs, + &m->chunk_bitmapped_packs_len); if (git_env_bool("GIT_TEST_MIDX_READ_RIDX", 1)) pair_chunk(cf, MIDX_CHUNKID_REVINDEX, &m->chunk_revindex, @@ -286,6 +294,26 @@ int prepare_midx_pack(struct repository *r, struct multi_pack_index *m, uint32_t return 0; } +int nth_bitmapped_pack(struct repository *r, struct multi_pack_index *m, + struct bitmapped_pack *bp, uint32_t pack_int_id) +{ + if (!m->chunk_bitmapped_packs) + return error(_("MIDX does not contain the BTMP chunk")); + + if (prepare_midx_pack(r, m, pack_int_id)) + return error(_("could not load bitmapped pack %"PRIu32), pack_int_id); + + bp->p = m->packs[pack_int_id]; + bp->bitmap_pos = get_be32((char *)m->chunk_bitmapped_packs + + MIDX_CHUNK_BITMAPPED_PACKS_WIDTH * pack_int_id); + bp->bitmap_nr = get_be32((char *)m->chunk_bitmapped_packs + + MIDX_CHUNK_BITMAPPED_PACKS_WIDTH * pack_int_id + + sizeof(uint32_t)); + bp->pack_int_id = pack_int_id; + + return 0; +} + int bsearch_midx(const struct object_id *oid, struct multi_pack_index *m, uint32_t *result) { return bsearch_hash(oid->hash, m->chunk_oid_fanout, m->chunk_oid_lookup, @@ -403,7 +431,8 @@ static int cmp_idx_or_pack_name(const char *idx_or_pack_name, return strcmp(idx_or_pack_name, idx_name); } -int midx_contains_pack(struct multi_pack_index *m, const char *idx_or_pack_name) +int midx_locate_pack(struct multi_pack_index *m, const char *idx_or_pack_name, + uint32_t *pos) { uint32_t first = 0, last = m->num_packs; @@ -414,8 +443,11 @@ int midx_contains_pack(struct multi_pack_index *m, const char *idx_or_pack_name) current = m->pack_names[mid]; cmp = cmp_idx_or_pack_name(idx_or_pack_name, current); - if (!cmp) + if (!cmp) { + if (pos) + *pos = mid; return 1; + } if (cmp > 0) { first = mid + 1; continue; @@ -426,6 +458,28 @@ int midx_contains_pack(struct multi_pack_index *m, const char *idx_or_pack_name) return 0; } +int midx_contains_pack(struct multi_pack_index *m, const char *idx_or_pack_name) +{ + return midx_locate_pack(m, idx_or_pack_name, NULL); +} + +int midx_preferred_pack(struct multi_pack_index *m, uint32_t *pack_int_id) +{ + if (m->preferred_pack_idx == -1) { + if (load_midx_revindex(m) < 0) { + m->preferred_pack_idx = -2; + return -1; + } + + m->preferred_pack_idx = + nth_midxed_pack_int_id(m, pack_pos_to_midx(m, 0)); + } else if (m->preferred_pack_idx == -2) + return -1; /* no revindex */ + + *pack_int_id = m->preferred_pack_idx; + return 0; +} + int prepare_multi_pack_index_one(struct repository *r, const char *object_dir, int local) { struct multi_pack_index *m; @@ -468,13 +522,31 @@ static size_t write_midx_header(struct hashfile *f, return MIDX_HEADER_SIZE; } +#define BITMAP_POS_UNKNOWN (~((uint32_t)0)) + struct pack_info { uint32_t orig_pack_int_id; char *pack_name; struct packed_git *p; + + uint32_t bitmap_pos; + uint32_t bitmap_nr; + unsigned expired : 1; }; +static void fill_pack_info(struct pack_info *info, + struct packed_git *p, const char *pack_name, + uint32_t orig_pack_int_id) +{ + memset(info, 0, sizeof(struct pack_info)); + + info->orig_pack_int_id = orig_pack_int_id; + info->pack_name = xstrdup(pack_name); + info->p = p; + info->bitmap_pos = BITMAP_POS_UNKNOWN; +} + static int pack_info_compare(const void *_a, const void *_b) { struct pack_info *a = (struct pack_info *)_a; @@ -515,6 +587,7 @@ static void add_pack_to_midx(const char *full_path, size_t full_path_len, const char *file_name, void *data) { struct write_midx_context *ctx = data; + struct packed_git *p; if (ends_with(file_name, ".idx")) { display_progress(ctx->progress, ++ctx->pack_paths_checked); @@ -541,27 +614,22 @@ static void add_pack_to_midx(const char *full_path, size_t full_path_len, ALLOC_GROW(ctx->info, ctx->nr + 1, ctx->alloc); - ctx->info[ctx->nr].p = add_packed_git(full_path, - full_path_len, - 0); - - if (!ctx->info[ctx->nr].p) { + p = add_packed_git(full_path, full_path_len, 0); + if (!p) { warning(_("failed to add packfile '%s'"), full_path); return; } - if (open_pack_index(ctx->info[ctx->nr].p)) { + if (open_pack_index(p)) { warning(_("failed to open pack-index '%s'"), full_path); - close_pack(ctx->info[ctx->nr].p); - FREE_AND_NULL(ctx->info[ctx->nr].p); + close_pack(p); + free(p); return; } - ctx->info[ctx->nr].pack_name = xstrdup(file_name); - ctx->info[ctx->nr].orig_pack_int_id = ctx->nr; - ctx->info[ctx->nr].expired = 0; + fill_pack_info(&ctx->info[ctx->nr], p, file_name, ctx->nr); ctx->nr++; } } @@ -817,6 +885,26 @@ static int write_midx_pack_names(struct hashfile *f, void *data) return 0; } +static int write_midx_bitmapped_packs(struct hashfile *f, void *data) +{ + struct write_midx_context *ctx = data; + size_t i; + + for (i = 0; i < ctx->nr; i++) { + struct pack_info *pack = &ctx->info[i]; + if (pack->expired) + continue; + + if (pack->bitmap_pos == BITMAP_POS_UNKNOWN && pack->bitmap_nr) + BUG("pack '%s' has no bitmap position, but has %d bitmapped object(s)", + pack->pack_name, pack->bitmap_nr); + + hashwrite_be32(f, pack->bitmap_pos); + hashwrite_be32(f, pack->bitmap_nr); + } + return 0; +} + static int write_midx_oid_fanout(struct hashfile *f, void *data) { @@ -984,8 +1072,19 @@ static uint32_t *midx_pack_order(struct write_midx_context *ctx) QSORT(data, ctx->entries_nr, midx_pack_order_cmp); ALLOC_ARRAY(pack_order, ctx->entries_nr); - for (i = 0; i < ctx->entries_nr; i++) + for (i = 0; i < ctx->entries_nr; i++) { + struct pack_midx_entry *e = &ctx->entries[data[i].nr]; + struct pack_info *pack = &ctx->info[ctx->pack_perm[e->pack_int_id]]; + if (pack->bitmap_pos == BITMAP_POS_UNKNOWN) + pack->bitmap_pos = i; + pack->bitmap_nr++; pack_order[i] = data[i].nr; + } + for (i = 0; i < ctx->nr; i++) { + struct pack_info *pack = &ctx->info[ctx->pack_perm[i]]; + if (pack->bitmap_pos == BITMAP_POS_UNKNOWN) + pack->bitmap_pos = 0; + } free(data); trace2_region_leave("midx", "midx_pack_order", the_repository); @@ -1286,6 +1385,7 @@ static int write_midx_internal(const char *object_dir, struct hashfile *f = NULL; struct lock_file lk; struct write_midx_context ctx = { 0 }; + int bitmapped_packs_concat_len = 0; int pack_name_concat_len = 0; int dropped_packs = 0; int result = 0; @@ -1321,11 +1421,6 @@ static int write_midx_internal(const char *object_dir, for (i = 0; i < ctx.m->num_packs; i++) { ALLOC_GROW(ctx.info, ctx.nr + 1, ctx.alloc); - ctx.info[ctx.nr].orig_pack_int_id = i; - ctx.info[ctx.nr].pack_name = xstrdup(ctx.m->pack_names[i]); - ctx.info[ctx.nr].p = ctx.m->packs[i]; - ctx.info[ctx.nr].expired = 0; - if (flags & MIDX_WRITE_REV_INDEX) { /* * If generating a reverse index, need to have @@ -1341,10 +1436,10 @@ static int write_midx_internal(const char *object_dir, if (open_pack_index(ctx.m->packs[i])) die(_("could not open index for %s"), ctx.m->packs[i]->pack_name); - ctx.info[ctx.nr].p = ctx.m->packs[i]; } - ctx.nr++; + fill_pack_info(&ctx.info[ctx.nr++], ctx.m->packs[i], + ctx.m->pack_names[i], i); } } @@ -1503,8 +1598,10 @@ static int write_midx_internal(const char *object_dir, } for (i = 0; i < ctx.nr; i++) { - if (!ctx.info[i].expired) - pack_name_concat_len += strlen(ctx.info[i].pack_name) + 1; + if (ctx.info[i].expired) + continue; + pack_name_concat_len += strlen(ctx.info[i].pack_name) + 1; + bitmapped_packs_concat_len += 2 * sizeof(uint32_t); } /* Check that the preferred pack wasn't expired (if given). */ @@ -1564,6 +1661,9 @@ static int write_midx_internal(const char *object_dir, add_chunk(cf, MIDX_CHUNKID_REVINDEX, st_mult(ctx.entries_nr, sizeof(uint32_t)), write_midx_revindex); + add_chunk(cf, MIDX_CHUNKID_BITMAPPEDPACKS, + bitmapped_packs_concat_len, + write_midx_bitmapped_packs); } write_midx_header(f, get_num_chunks(cf), ctx.nr - dropped_packs); @@ -1603,8 +1703,13 @@ static int write_midx_internal(const char *object_dir, flags) < 0) { error(_("could not write multi-pack bitmap")); result = 1; + clear_packing_data(&pdata); + free(commits); goto cleanup; } + + clear_packing_data(&pdata); + free(commits); } /* * NOTE: Do not use ctx.entries beyond this point, since it might @@ -6,6 +6,7 @@ struct object_id; struct pack_entry; struct repository; +struct bitmapped_pack; #define GIT_TEST_MULTI_PACK_INDEX "GIT_TEST_MULTI_PACK_INDEX" #define GIT_TEST_MULTI_PACK_INDEX_WRITE_BITMAP \ @@ -27,11 +28,14 @@ struct multi_pack_index { unsigned char num_chunks; uint32_t num_packs; uint32_t num_objects; + int preferred_pack_idx; int local; const unsigned char *chunk_pack_names; size_t chunk_pack_names_len; + const uint32_t *chunk_bitmapped_packs; + size_t chunk_bitmapped_packs_len; const uint32_t *chunk_oid_fanout; const unsigned char *chunk_oid_lookup; const unsigned char *chunk_object_offsets; @@ -57,6 +61,8 @@ void get_midx_rev_filename(struct strbuf *out, struct multi_pack_index *m); struct multi_pack_index *load_multi_pack_index(const char *object_dir, int local); int prepare_midx_pack(struct repository *r, struct multi_pack_index *m, uint32_t pack_int_id); +int nth_bitmapped_pack(struct repository *r, struct multi_pack_index *m, + struct bitmapped_pack *bp, uint32_t pack_int_id); int bsearch_midx(const struct object_id *oid, struct multi_pack_index *m, uint32_t *result); off_t nth_midxed_offset(struct multi_pack_index *m, uint32_t pos); uint32_t nth_midxed_pack_int_id(struct multi_pack_index *m, uint32_t pos); @@ -64,7 +70,11 @@ struct object_id *nth_midxed_object_oid(struct object_id *oid, struct multi_pack_index *m, uint32_t n); int fill_midx_entry(struct repository *r, const struct object_id *oid, struct pack_entry *e, struct multi_pack_index *m); -int midx_contains_pack(struct multi_pack_index *m, const char *idx_or_pack_name); +int midx_contains_pack(struct multi_pack_index *m, + const char *idx_or_pack_name); +int midx_locate_pack(struct multi_pack_index *m, const char *idx_or_pack_name, + uint32_t *pos); +int midx_preferred_pack(struct multi_pack_index *m, uint32_t *pack_int_id); int prepare_multi_pack_index_one(struct repository *r, const char *object_dir, int local); /* diff --git a/pack-bitmap-write.c b/pack-bitmap-write.c index be4733e3bd..990a9498d7 100644 --- a/pack-bitmap-write.c +++ b/pack-bitmap-write.c @@ -195,6 +195,13 @@ struct bb_commit { unsigned idx; /* within selected array */ }; +static void clear_bb_commit(struct bb_commit *commit) +{ + free_commit_list(commit->reverse_edges); + bitmap_free(commit->commit_mask); + bitmap_free(commit->bitmap); +} + define_commit_slab(bb_data, struct bb_commit); struct bitmap_builder { @@ -336,7 +343,7 @@ next: static void bitmap_builder_clear(struct bitmap_builder *bb) { - clear_bb_data(&bb->data); + deep_clear_bb_data(&bb->data, clear_bb_commit); free(bb->commits); bb->commits_nr = bb->commits_alloc = 0; } diff --git a/pack-bitmap.c b/pack-bitmap.c index 0260890341..229a11fb00 100644 --- a/pack-bitmap.c +++ b/pack-bitmap.c @@ -338,7 +338,7 @@ static int open_midx_bitmap_1(struct bitmap_index *bitmap_git, struct stat st; char *bitmap_name = midx_bitmap_filename(midx); int fd = git_open(bitmap_name); - uint32_t i; + uint32_t i, preferred_pack; struct packed_git *preferred; if (fd < 0) { @@ -393,7 +393,12 @@ static int open_midx_bitmap_1(struct bitmap_index *bitmap_git, } } - preferred = bitmap_git->midx->packs[midx_preferred_pack(bitmap_git)]; + if (midx_preferred_pack(bitmap_git->midx, &preferred_pack) < 0) { + warning(_("could not determine MIDX preferred pack")); + goto cleanup; + } + + preferred = bitmap_git->midx->packs[preferred_pack]; if (!is_pack_valid(preferred)) { warning(_("preferred pack (%s) is invalid"), preferred->pack_name); @@ -1280,6 +1285,8 @@ static struct bitmap *find_objects(struct bitmap_index *bitmap_git, base = fill_in_bitmap(bitmap_git, revs, base, seen); } + object_list_free(¬_mapped); + return base; } @@ -1834,8 +1841,10 @@ cleanup: * -1 means "stop trying further objects"; 0 means we may or may not have * reused, but you can keep feeding bits. */ -static int try_partial_reuse(struct packed_git *pack, - size_t pos, +static int try_partial_reuse(struct bitmap_index *bitmap_git, + struct bitmapped_pack *pack, + size_t bitmap_pos, + uint32_t pack_pos, struct bitmap *reuse, struct pack_window **w_curs) { @@ -1843,40 +1852,18 @@ static int try_partial_reuse(struct packed_git *pack, enum object_type type; unsigned long size; - /* - * try_partial_reuse() is called either on (a) objects in the - * bitmapped pack (in the case of a single-pack bitmap) or (b) - * objects in the preferred pack of a multi-pack bitmap. - * Importantly, the latter can pretend as if only a single pack - * exists because: - * - * - The first pack->num_objects bits of a MIDX bitmap are - * reserved for the preferred pack, and - * - * - Ties due to duplicate objects are always resolved in - * favor of the preferred pack. - * - * Therefore we do not need to ever ask the MIDX for its copy of - * an object by OID, since it will always select it from the - * preferred pack. Likewise, the selected copy of the base - * object for any deltas will reside in the same pack. - * - * This means that we can reuse pos when looking up the bit in - * the reuse bitmap, too, since bits corresponding to the - * preferred pack precede all bits from other packs. - */ - - if (pos >= pack->num_objects) - return -1; /* not actually in the pack or MIDX preferred pack */ + if (pack_pos >= pack->p->num_objects) + return -1; /* not actually in the pack */ - offset = delta_obj_offset = pack_pos_to_offset(pack, pos); - type = unpack_object_header(pack, w_curs, &offset, &size); + offset = delta_obj_offset = pack_pos_to_offset(pack->p, pack_pos); + type = unpack_object_header(pack->p, w_curs, &offset, &size); if (type < 0) return -1; /* broken packfile, punt */ if (type == OBJ_REF_DELTA || type == OBJ_OFS_DELTA) { off_t base_offset; uint32_t base_pos; + uint32_t base_bitmap_pos; /* * Find the position of the base object so we can look it up @@ -1886,24 +1873,48 @@ static int try_partial_reuse(struct packed_git *pack, * and the normal slow path will complain about it in * more detail. */ - base_offset = get_delta_base(pack, w_curs, &offset, type, + base_offset = get_delta_base(pack->p, w_curs, &offset, type, delta_obj_offset); if (!base_offset) return 0; - if (offset_to_pack_pos(pack, base_offset, &base_pos) < 0) - return 0; - /* - * We assume delta dependencies always point backwards. This - * lets us do a single pass, and is basically always true - * due to the way OFS_DELTAs work. You would not typically - * find REF_DELTA in a bitmapped pack, since we only bitmap - * packs we write fresh, and OFS_DELTA is the default). But - * let's double check to make sure the pack wasn't written with - * odd parameters. - */ - if (base_pos >= pos) - return 0; + offset_to_pack_pos(pack->p, base_offset, &base_pos); + + if (bitmap_is_midx(bitmap_git)) { + /* + * Cross-pack deltas are rejected for now, but could + * theoretically be supported in the future. + * + * We would need to ensure that we're sending both + * halves of the delta/base pair, regardless of whether + * or not the two cross a pack boundary. If they do, + * then we must convert the delta to an REF_DELTA to + * refer back to the base in the other pack. + * */ + if (midx_pair_to_pack_pos(bitmap_git->midx, + pack->pack_int_id, + base_offset, + &base_bitmap_pos) < 0) { + return 0; + } + } else { + if (offset_to_pack_pos(pack->p, base_offset, + &base_pos) < 0) + return 0; + /* + * We assume delta dependencies always point backwards. + * This lets us do a single pass, and is basically + * always true due to the way OFS_DELTAs work. You would + * not typically find REF_DELTA in a bitmapped pack, + * since we only bitmap packs we write fresh, and + * OFS_DELTA is the default). But let's double check to + * make sure the pack wasn't written with odd + * parameters. + */ + if (base_pos >= pack_pos) + return 0; + base_bitmap_pos = pack->bitmap_pos + base_pos; + } /* * And finally, if we're not sending the base as part of our @@ -1913,77 +1924,89 @@ static int try_partial_reuse(struct packed_git *pack, * to REF_DELTA on the fly. Better to just let the normal * object_entry code path handle it. */ - if (!bitmap_get(reuse, base_pos)) + if (!bitmap_get(reuse, base_bitmap_pos)) return 0; } /* * If we got here, then the object is OK to reuse. Mark it. */ - bitmap_set(reuse, pos); + bitmap_set(reuse, bitmap_pos); return 0; } -uint32_t midx_preferred_pack(struct bitmap_index *bitmap_git) +static void reuse_partial_packfile_from_bitmap_1(struct bitmap_index *bitmap_git, + struct bitmapped_pack *pack, + struct bitmap *reuse) { - struct multi_pack_index *m = bitmap_git->midx; - if (!m) - BUG("midx_preferred_pack: requires non-empty MIDX"); - return nth_midxed_pack_int_id(m, pack_pos_to_midx(bitmap_git->midx, 0)); -} - -int reuse_partial_packfile_from_bitmap(struct bitmap_index *bitmap_git, - struct packed_git **packfile_out, - uint32_t *entries, - struct bitmap **reuse_out) -{ - struct repository *r = the_repository; - struct packed_git *pack; struct bitmap *result = bitmap_git->result; - struct bitmap *reuse; struct pack_window *w_curs = NULL; - size_t i = 0; - uint32_t offset; - uint32_t objects_nr; + size_t pos = pack->bitmap_pos / BITS_IN_EWORD; - assert(result); + if (!pack->bitmap_pos) { + /* + * If we're processing the first (in the case of a MIDX, the + * preferred pack) or the only (in the case of single-pack + * bitmaps) pack, then we can reuse whole words at a time. + * + * This is because we know that any deltas in this range *must* + * have their bases chosen from the same pack, since: + * + * - In the single pack case, there is no other pack to choose + * them from. + * + * - In the MIDX case, the first pack is the preferred pack, so + * all ties are broken in favor of that pack (i.e. the one + * we're currently processing). So any duplicate bases will be + * resolved in favor of the pack we're processing. + */ + while (pos < result->word_alloc && + pos < pack->bitmap_nr / BITS_IN_EWORD && + result->words[pos] == (eword_t)~0) + pos++; + memset(reuse->words, 0xFF, pos * sizeof(eword_t)); + } - load_reverse_index(r, bitmap_git); + for (; pos < result->word_alloc; pos++) { + eword_t word = result->words[pos]; + size_t offset; - if (bitmap_is_midx(bitmap_git)) - pack = bitmap_git->midx->packs[midx_preferred_pack(bitmap_git)]; - else - pack = bitmap_git->pack; - objects_nr = pack->num_objects; + for (offset = 0; offset < BITS_IN_EWORD; offset++) { + size_t bit_pos; + uint32_t pack_pos; - while (i < result->word_alloc && result->words[i] == (eword_t)~0) - i++; + if (word >> offset == 0) + break; - /* - * Don't mark objects not in the packfile or preferred pack. This bitmap - * marks objects eligible for reuse, but the pack-reuse code only - * understands how to reuse a single pack. Since the preferred pack is - * guaranteed to have all bases for its deltas (in a multi-pack bitmap), - * we use it instead of another pack. In single-pack bitmaps, the choice - * is made for us. - */ - if (i > objects_nr / BITS_IN_EWORD) - i = objects_nr / BITS_IN_EWORD; + offset += ewah_bit_ctz64(word >> offset); - reuse = bitmap_word_alloc(i); - memset(reuse->words, 0xFF, i * sizeof(eword_t)); + bit_pos = pos * BITS_IN_EWORD + offset; + if (bit_pos < pack->bitmap_pos) + continue; + if (bit_pos >= pack->bitmap_pos + pack->bitmap_nr) + goto done; - for (; i < result->word_alloc; ++i) { - eword_t word = result->words[i]; - size_t pos = (i * BITS_IN_EWORD); + if (bitmap_is_midx(bitmap_git)) { + uint32_t midx_pos; + off_t ofs; - for (offset = 0; offset < BITS_IN_EWORD; ++offset) { - if ((word >> offset) == 0) - break; + midx_pos = pack_pos_to_midx(bitmap_git->midx, bit_pos); + ofs = nth_midxed_offset(bitmap_git->midx, midx_pos); - offset += ewah_bit_ctz64(word >> offset); - if (try_partial_reuse(pack, pos + offset, - reuse, &w_curs) < 0) { + if (offset_to_pack_pos(pack->p, ofs, &pack_pos) < 0) + BUG("could not find object in pack %s " + "at offset %"PRIuMAX" in MIDX", + pack_basename(pack->p), (uintmax_t)ofs); + } else { + pack_pos = cast_size_t_to_uint32_t(st_sub(bit_pos, pack->bitmap_pos)); + if (pack_pos >= pack->p->num_objects) + BUG("advanced beyond the end of pack %s (%"PRIuMAX" > %"PRIu32")", + pack_basename(pack->p), (uintmax_t)pack_pos, + pack->p->num_objects); + } + + if (try_partial_reuse(bitmap_git, pack, bit_pos, + pack_pos, reuse, &w_curs) < 0) { /* * try_partial_reuse indicated we couldn't reuse * any bits, so there is no point in trying more @@ -2000,11 +2023,97 @@ int reuse_partial_packfile_from_bitmap(struct bitmap_index *bitmap_git, done: unuse_pack(&w_curs); +} - *entries = bitmap_popcount(reuse); - if (!*entries) { - bitmap_free(reuse); +static int bitmapped_pack_cmp(const void *va, const void *vb) +{ + const struct bitmapped_pack *a = va; + const struct bitmapped_pack *b = vb; + + if (a->bitmap_pos < b->bitmap_pos) return -1; + if (a->bitmap_pos > b->bitmap_pos) + return 1; + return 0; +} + +void reuse_partial_packfile_from_bitmap(struct bitmap_index *bitmap_git, + struct bitmapped_pack **packs_out, + size_t *packs_nr_out, + struct bitmap **reuse_out, + int multi_pack_reuse) +{ + struct repository *r = the_repository; + struct bitmapped_pack *packs = NULL; + struct bitmap *result = bitmap_git->result; + struct bitmap *reuse; + size_t i; + size_t packs_nr = 0, packs_alloc = 0; + size_t word_alloc; + uint32_t objects_nr = 0; + + assert(result); + + load_reverse_index(r, bitmap_git); + + if (bitmap_is_midx(bitmap_git)) { + for (i = 0; i < bitmap_git->midx->num_packs; i++) { + struct bitmapped_pack pack; + if (nth_bitmapped_pack(r, bitmap_git->midx, &pack, i) < 0) { + warning(_("unable to load pack: '%s', disabling pack-reuse"), + bitmap_git->midx->pack_names[i]); + free(packs); + return; + } + + if (!pack.bitmap_nr) + continue; + + if (!multi_pack_reuse && pack.bitmap_pos) { + /* + * If we're only reusing a single pack, skip + * over any packs which are not positioned at + * the beginning of the MIDX bitmap. + * + * This is consistent with the existing + * single-pack reuse behavior, which only reuses + * parts of the MIDX's preferred pack. + */ + continue; + } + + ALLOC_GROW(packs, packs_nr + 1, packs_alloc); + memcpy(&packs[packs_nr++], &pack, sizeof(pack)); + + objects_nr += pack.p->num_objects; + + if (!multi_pack_reuse) + break; + } + + QSORT(packs, packs_nr, bitmapped_pack_cmp); + } else { + ALLOC_GROW(packs, packs_nr + 1, packs_alloc); + + packs[packs_nr].p = bitmap_git->pack; + packs[packs_nr].bitmap_nr = bitmap_git->pack->num_objects; + packs[packs_nr].bitmap_pos = 0; + + objects_nr = packs[packs_nr++].bitmap_nr; + } + + word_alloc = objects_nr / BITS_IN_EWORD; + if (objects_nr % BITS_IN_EWORD) + word_alloc++; + reuse = bitmap_word_alloc(word_alloc); + + for (i = 0; i < packs_nr; i++) + reuse_partial_packfile_from_bitmap_1(bitmap_git, &packs[i], reuse); + + if (bitmap_is_empty(reuse)) { + free(packs); + bitmap_free(reuse); + return; } /* @@ -2012,9 +2121,9 @@ done: * need to be handled separately. */ bitmap_and_not(result, reuse); - *packfile_out = pack; + *packs_out = packs; + *packs_nr_out = packs_nr; *reuse_out = reuse; - return 0; } int bitmap_walk_contains(struct bitmap_index *bitmap_git, diff --git a/pack-bitmap.h b/pack-bitmap.h index 5273a6a019..c7dea13217 100644 --- a/pack-bitmap.h +++ b/pack-bitmap.h @@ -52,6 +52,15 @@ typedef int (*show_reachable_fn)( struct bitmap_index; +struct bitmapped_pack { + struct packed_git *p; + + uint32_t bitmap_pos; + uint32_t bitmap_nr; + + uint32_t pack_int_id; /* MIDX only */ +}; + struct bitmap_index *prepare_bitmap_git(struct repository *r); struct bitmap_index *prepare_midx_bitmap_git(struct multi_pack_index *midx); void count_bitmap_commit_list(struct bitmap_index *, uint32_t *commits, @@ -68,11 +77,11 @@ int test_bitmap_hashes(struct repository *r); struct bitmap_index *prepare_bitmap_walk(struct rev_info *revs, int filter_provided_objects); -uint32_t midx_preferred_pack(struct bitmap_index *bitmap_git); -int reuse_partial_packfile_from_bitmap(struct bitmap_index *, - struct packed_git **packfile, - uint32_t *entries, - struct bitmap **reuse_out); +void reuse_partial_packfile_from_bitmap(struct bitmap_index *bitmap_git, + struct bitmapped_pack **packs_out, + size_t *packs_nr_out, + struct bitmap **reuse_out, + int multi_pack_reuse); int rebuild_existing_bitmaps(struct bitmap_index *, struct packing_data *mapping, kh_oid_map_t *reused_bitmaps, int show_progress); void free_bitmap_index(struct bitmap_index *); diff --git a/pack-objects.c b/pack-objects.c index f403ca6986..a9d9855063 100644 --- a/pack-objects.c +++ b/pack-objects.c @@ -151,6 +151,21 @@ void prepare_packing_data(struct repository *r, struct packing_data *pdata) init_recursive_mutex(&pdata->odb_lock); } +void clear_packing_data(struct packing_data *pdata) +{ + if (!pdata) + return; + + free(pdata->cruft_mtime); + free(pdata->in_pack); + free(pdata->in_pack_by_idx); + free(pdata->in_pack_pos); + free(pdata->index); + free(pdata->layer); + free(pdata->objects); + free(pdata->tree_depth); +} + struct object_entry *packlist_alloc(struct packing_data *pdata, const struct object_id *oid) { diff --git a/pack-objects.h b/pack-objects.h index 0d78db40cb..b9898a4e64 100644 --- a/pack-objects.h +++ b/pack-objects.h @@ -169,6 +169,7 @@ struct packing_data { }; void prepare_packing_data(struct repository *r, struct packing_data *pdata); +void clear_packing_data(struct packing_data *pdata); /* Protect access to object database */ static inline void packing_data_lock(struct packing_data *pdata) diff --git a/pack-revindex.c b/pack-revindex.c index acf1dd9786..a7624d8be8 100644 --- a/pack-revindex.c +++ b/pack-revindex.c @@ -520,19 +520,12 @@ static int midx_pack_order_cmp(const void *va, const void *vb) return 0; } -int midx_to_pack_pos(struct multi_pack_index *m, uint32_t at, uint32_t *pos) +static int midx_key_to_pack_pos(struct multi_pack_index *m, + struct midx_pack_key *key, + uint32_t *pos) { - struct midx_pack_key key; uint32_t *found; - if (!m->revindex_data) - BUG("midx_to_pack_pos: reverse index not yet loaded"); - if (m->num_objects <= at) - BUG("midx_to_pack_pos: out-of-bounds object at %"PRIu32, at); - - key.pack = nth_midxed_pack_int_id(m, at); - key.offset = nth_midxed_offset(m, at); - key.midx = m; /* * The preferred pack sorts first, so determine its identifier by * looking at the first object in pseudo-pack order. @@ -542,14 +535,43 @@ int midx_to_pack_pos(struct multi_pack_index *m, uint32_t at, uint32_t *pos) * implicitly is preferred (and includes all its objects, since ties are * broken first by pack identifier). */ - key.preferred_pack = nth_midxed_pack_int_id(m, pack_pos_to_midx(m, 0)); + if (midx_preferred_pack(key->midx, &key->preferred_pack) < 0) + return error(_("could not determine preferred pack")); - found = bsearch(&key, m->revindex_data, m->num_objects, - sizeof(*m->revindex_data), midx_pack_order_cmp); + found = bsearch(key, m->revindex_data, m->num_objects, + sizeof(*m->revindex_data), + midx_pack_order_cmp); if (!found) - return error("bad offset for revindex"); + return -1; *pos = found - m->revindex_data; return 0; } + +int midx_to_pack_pos(struct multi_pack_index *m, uint32_t at, uint32_t *pos) +{ + struct midx_pack_key key; + + if (!m->revindex_data) + BUG("midx_to_pack_pos: reverse index not yet loaded"); + if (m->num_objects <= at) + BUG("midx_to_pack_pos: out-of-bounds object at %"PRIu32, at); + + key.pack = nth_midxed_pack_int_id(m, at); + key.offset = nth_midxed_offset(m, at); + key.midx = m; + + return midx_key_to_pack_pos(m, &key, pos); +} + +int midx_pair_to_pack_pos(struct multi_pack_index *m, uint32_t pack_int_id, + off_t ofs, uint32_t *pos) +{ + struct midx_pack_key key = { + .pack = pack_int_id, + .offset = ofs, + .midx = m, + }; + return midx_key_to_pack_pos(m, &key, pos); +} diff --git a/pack-revindex.h b/pack-revindex.h index 6dd47efea1..422c2487ae 100644 --- a/pack-revindex.h +++ b/pack-revindex.h @@ -142,4 +142,7 @@ uint32_t pack_pos_to_midx(struct multi_pack_index *m, uint32_t pos); */ int midx_to_pack_pos(struct multi_pack_index *midx, uint32_t at, uint32_t *pos); +int midx_pair_to_pack_pos(struct multi_pack_index *midx, uint32_t pack_id, + off_t ofs, uint32_t *pos); + #endif diff --git a/pkt-line.c b/pkt-line.c index 236dd3a3ee..24479eae4d 100644 --- a/pkt-line.c +++ b/pkt-line.c @@ -463,8 +463,32 @@ enum packet_read_status packet_read_with_status(int fd, char **src_buffer, } if ((options & PACKET_READ_CHOMP_NEWLINE) && - len && buffer[len-1] == '\n') - len--; + len && buffer[len-1] == '\n') { + if (options & PACKET_READ_USE_SIDEBAND) { + int band = *buffer & 0xff; + switch (band) { + case 1: + /* Chomp newline for payload */ + len--; + break; + case 2: + case 3: + /* + * Do not chomp newline for progress and error + * message. + */ + break; + default: + /* + * Bad sideband, let's leave it to + * demultiplex_sideband() to catch this error. + */ + break; + } + } else { + len--; + } + } buffer[len] = 0; if (options & PACKET_READ_REDACT_URI_PATH && @@ -593,17 +617,19 @@ void packet_reader_init(struct packet_reader *reader, int fd, reader->options = options; reader->me = "git"; reader->hash_algo = &hash_algos[GIT_HASH_SHA1]; + strbuf_init(&reader->scratch, 0); } enum packet_read_status packet_reader_read(struct packet_reader *reader) { - struct strbuf scratch = STRBUF_INIT; - if (reader->line_peeked) { reader->line_peeked = 0; return reader->status; } + if (reader->use_sideband) + reader->options |= PACKET_READ_USE_SIDEBAND; + /* * Consume all progress packets until a primary payload packet is * received @@ -621,7 +647,7 @@ enum packet_read_status packet_reader_read(struct packet_reader *reader) break; if (demultiplex_sideband(reader->me, reader->status, reader->buffer, reader->pktlen, 1, - &scratch, &sideband_type)) + &reader->scratch, &sideband_type)) break; } diff --git a/pkt-line.h b/pkt-line.h index aedef56286..3b33cc64f3 100644 --- a/pkt-line.h +++ b/pkt-line.h @@ -84,6 +84,7 @@ void packet_fflush(FILE *f); #define PACKET_READ_DIE_ON_ERR_PACKET (1u<<2) #define PACKET_READ_GENTLE_ON_READ_ERROR (1u<<3) #define PACKET_READ_REDACT_URI_PATH (1u<<4) +#define PACKET_READ_USE_SIDEBAND (1u<<5) int packet_read(int fd, char *buffer, unsigned size, int options); /* @@ -193,6 +194,9 @@ struct packet_reader { /* hash algorithm in use */ const struct git_hash_algo *hash_algo; + + /* hold temporary sideband message */ + struct strbuf scratch; }; /* @@ -33,17 +33,33 @@ /* * List of all available backends */ -static struct ref_storage_be *refs_backends = &refs_be_files; +static const struct ref_storage_be *refs_backends[] = { + [REF_STORAGE_FORMAT_FILES] = &refs_be_files, +}; -static struct ref_storage_be *find_ref_storage_backend(const char *name) +static const struct ref_storage_be *find_ref_storage_backend(unsigned int ref_storage_format) { - struct ref_storage_be *be; - for (be = refs_backends; be; be = be->next) - if (!strcmp(be->name, name)) - return be; + if (ref_storage_format < ARRAY_SIZE(refs_backends)) + return refs_backends[ref_storage_format]; return NULL; } +unsigned int ref_storage_format_by_name(const char *name) +{ + for (unsigned int i = 0; i < ARRAY_SIZE(refs_backends); i++) + if (refs_backends[i] && !strcmp(refs_backends[i]->name, name)) + return i; + return REF_STORAGE_FORMAT_UNKNOWN; +} + +const char *ref_storage_format_to_name(unsigned int ref_storage_format) +{ + const struct ref_storage_be *be = find_ref_storage_backend(ref_storage_format); + if (!be) + return "unknown"; + return be->name; +} + /* * How to handle various characters in refnames: * 0: An acceptable character for refs @@ -2082,12 +2098,12 @@ static struct ref_store *ref_store_init(struct repository *repo, const char *gitdir, unsigned int flags) { - const char *be_name = "files"; - struct ref_storage_be *be = find_ref_storage_backend(be_name); + const struct ref_storage_be *be; struct ref_store *refs; + be = find_ref_storage_backend(repo->ref_storage_format); if (!be) - BUG("reference backend %s is unknown", be_name); + BUG("reference backend is unknown"); refs = be->init(repo, gitdir, flags); return refs; @@ -11,6 +11,9 @@ struct string_list; struct string_list_item; struct worktree; +unsigned int ref_storage_format_by_name(const char *name); +const char *ref_storage_format_to_name(unsigned int ref_storage_format); + /* * Resolve a reference, recursively following symbolic refererences. * diff --git a/refs/debug.c b/refs/debug.c index 83b7a0ba65..b9775f2c37 100644 --- a/refs/debug.c +++ b/refs/debug.c @@ -426,7 +426,6 @@ static int debug_reflog_expire(struct ref_store *ref_store, const char *refname, } struct ref_storage_be refs_be_debug = { - .next = NULL, .name = "debug", .init = NULL, .init_db = debug_init_db, diff --git a/refs/files-backend.c b/refs/files-backend.c index 6734f2a309..b288fc97db 100644 --- a/refs/files-backend.c +++ b/refs/files-backend.c @@ -3239,7 +3239,6 @@ static int files_init_db(struct ref_store *ref_store, struct strbuf *err UNUSED) } struct ref_storage_be refs_be_files = { - .next = NULL, .name = "files", .init = files_ref_store_create, .init_db = files_init_db, diff --git a/refs/packed-backend.c b/refs/packed-backend.c index e46906e612..bf04b93381 100644 --- a/refs/packed-backend.c +++ b/refs/packed-backend.c @@ -1704,7 +1704,6 @@ static struct ref_iterator *packed_reflog_iterator_begin(struct ref_store *ref_s } struct ref_storage_be refs_be_packed = { - .next = NULL, .name = "packed", .init = packed_ref_store_create, .init_db = packed_init_db, diff --git a/refs/refs-internal.h b/refs/refs-internal.h index 4af83bf9a5..8e9f04cc67 100644 --- a/refs/refs-internal.h +++ b/refs/refs-internal.h @@ -663,7 +663,6 @@ typedef int read_symbolic_ref_fn(struct ref_store *ref_store, const char *refnam struct strbuf *referent); struct ref_storage_be { - struct ref_storage_be *next; const char *name; ref_store_init_fn *init; ref_init_db_fn *init_db; diff --git a/reftable/block_test.c b/reftable/block_test.c index c00bbc8aed..dedb05c7d8 100644 --- a/reftable/block_test.c +++ b/reftable/block_test.c @@ -49,13 +49,11 @@ static void test_block_read_write(void) for (i = 0; i < N; i++) { char name[100]; - uint8_t hash[GIT_SHA1_RAWSZ]; snprintf(name, sizeof(name), "branch%02d", i); - memset(hash, i, sizeof(hash)); rec.u.ref.refname = name; rec.u.ref.value_type = REFTABLE_REF_VAL1; - rec.u.ref.value.val1 = hash; + memset(rec.u.ref.value.val1, i, GIT_SHA1_RAWSZ); names[i] = xstrdup(name); n = block_writer_add(&bw, &rec); diff --git a/reftable/merged.c b/reftable/merged.c index 574394092d..c258ce953e 100644 --- a/reftable/merged.c +++ b/reftable/merged.c @@ -123,12 +123,12 @@ static int merged_iter_next_entry(struct merged_iter *mi, reftable_record_release(&top.rec); } - reftable_record_copy_from(rec, &entry.rec, hash_size(mi->hash_id)); + reftable_record_release(rec); + *rec = entry.rec; done: - reftable_record_release(&entry.rec); - strbuf_release(&mi->entry_key); - strbuf_release(&mi->key); + if (err) + reftable_record_release(&entry.rec); return err; } diff --git a/reftable/merged_test.c b/reftable/merged_test.c index 0d6e0d4bf5..46908f738f 100644 --- a/reftable/merged_test.c +++ b/reftable/merged_test.c @@ -122,13 +122,11 @@ static void readers_destroy(struct reftable_reader **readers, size_t n) static void test_merged_between(void) { - uint8_t hash1[GIT_SHA1_RAWSZ] = { 1, 2, 3, 0 }; - struct reftable_ref_record r1[] = { { .refname = "b", .update_index = 1, .value_type = REFTABLE_REF_VAL1, - .value.val1 = hash1, + .value.val1 = { 1, 2, 3, 0 }, } }; struct reftable_ref_record r2[] = { { .refname = "a", @@ -164,26 +162,24 @@ static void test_merged_between(void) static void test_merged(void) { - uint8_t hash1[GIT_SHA1_RAWSZ] = { 1 }; - uint8_t hash2[GIT_SHA1_RAWSZ] = { 2 }; struct reftable_ref_record r1[] = { { .refname = "a", .update_index = 1, .value_type = REFTABLE_REF_VAL1, - .value.val1 = hash1, + .value.val1 = { 1 }, }, { .refname = "b", .update_index = 1, .value_type = REFTABLE_REF_VAL1, - .value.val1 = hash1, + .value.val1 = { 1 }, }, { .refname = "c", .update_index = 1, .value_type = REFTABLE_REF_VAL1, - .value.val1 = hash1, + .value.val1 = { 1 }, } }; struct reftable_ref_record r2[] = { { @@ -196,13 +192,13 @@ static void test_merged(void) .refname = "c", .update_index = 3, .value_type = REFTABLE_REF_VAL1, - .value.val1 = hash2, + .value.val1 = { 2 }, }, { .refname = "d", .update_index = 3, .value_type = REFTABLE_REF_VAL1, - .value.val1 = hash1, + .value.val1 = { 1 }, }, }; diff --git a/reftable/readwrite_test.c b/reftable/readwrite_test.c index 79cd4e4c28..b8a3224016 100644 --- a/reftable/readwrite_test.c +++ b/reftable/readwrite_test.c @@ -59,18 +59,15 @@ static void write_table(char ***names, struct strbuf *buf, int N, *names = reftable_calloc(sizeof(char *) * (N + 1)); reftable_writer_set_limits(w, update_index, update_index); for (i = 0; i < N; i++) { - uint8_t hash[GIT_SHA256_RAWSZ] = { 0 }; char name[100]; int n; - set_test_hash(hash, i); - snprintf(name, sizeof(name), "refs/heads/branch%02d", i); ref.refname = name; ref.update_index = update_index; ref.value_type = REFTABLE_REF_VAL1; - ref.value.val1 = hash; + set_test_hash(ref.value.val1, i); (*names)[i] = xstrdup(name); n = reftable_writer_add_ref(w, &ref); @@ -549,8 +546,6 @@ static void test_table_refs_for(int indexed) uint8_t hash[GIT_SHA1_RAWSZ]; char fill[51] = { 0 }; char name[100]; - uint8_t hash1[GIT_SHA1_RAWSZ]; - uint8_t hash2[GIT_SHA1_RAWSZ]; struct reftable_ref_record ref = { NULL }; memset(hash, i, sizeof(hash)); @@ -560,11 +555,9 @@ static void test_table_refs_for(int indexed) name[40] = 0; ref.refname = name; - set_test_hash(hash1, i / 4); - set_test_hash(hash2, 3 + i / 4); ref.value_type = REFTABLE_REF_VAL2; - ref.value.val2.value = hash1; - ref.value.val2.target_value = hash2; + set_test_hash(ref.value.val2.value, i / 4); + set_test_hash(ref.value.val2.target_value, 3 + i / 4); /* 80 bytes / entry, so 3 entries per block. Yields 17 */ @@ -572,8 +565,8 @@ static void test_table_refs_for(int indexed) n = reftable_writer_add_ref(w, &ref); EXPECT(n == 0); - if (!memcmp(hash1, want_hash, GIT_SHA1_RAWSZ) || - !memcmp(hash2, want_hash, GIT_SHA1_RAWSZ)) { + if (!memcmp(ref.value.val2.value, want_hash, GIT_SHA1_RAWSZ) || + !memcmp(ref.value.val2.target_value, want_hash, GIT_SHA1_RAWSZ)) { want_names[want_names_len++] = xstrdup(name); } } @@ -674,11 +667,10 @@ static void test_write_object_id_min_length(void) struct strbuf buf = STRBUF_INIT; struct reftable_writer *w = reftable_new_writer(&strbuf_add_void, &buf, &opts); - uint8_t hash[GIT_SHA1_RAWSZ] = {42}; struct reftable_ref_record ref = { .update_index = 1, .value_type = REFTABLE_REF_VAL1, - .value.val1 = hash, + .value.val1 = {42}, }; int err; int i; @@ -710,11 +702,10 @@ static void test_write_object_id_length(void) struct strbuf buf = STRBUF_INIT; struct reftable_writer *w = reftable_new_writer(&strbuf_add_void, &buf, &opts); - uint8_t hash[GIT_SHA1_RAWSZ] = {42}; struct reftable_ref_record ref = { .update_index = 1, .value_type = REFTABLE_REF_VAL1, - .value.val1 = hash, + .value.val1 = {42}, }; int err; int i; @@ -797,6 +788,84 @@ static void test_write_key_order(void) strbuf_release(&buf); } +static void test_write_multiple_indices(void) +{ + struct reftable_write_options opts = { + .block_size = 100, + }; + struct strbuf writer_buf = STRBUF_INIT, buf = STRBUF_INIT; + struct reftable_block_source source = { 0 }; + struct reftable_iterator it = { 0 }; + const struct reftable_stats *stats; + struct reftable_writer *writer; + struct reftable_reader *reader; + int err, i; + + writer = reftable_new_writer(&strbuf_add_void, &writer_buf, &opts); + reftable_writer_set_limits(writer, 1, 1); + for (i = 0; i < 100; i++) { + struct reftable_ref_record ref = { + .update_index = 1, + .value_type = REFTABLE_REF_VAL1, + .value.val1 = {i}, + }; + + strbuf_reset(&buf); + strbuf_addf(&buf, "refs/heads/%04d", i); + ref.refname = buf.buf, + + err = reftable_writer_add_ref(writer, &ref); + EXPECT_ERR(err); + } + + for (i = 0; i < 100; i++) { + unsigned char hash[GIT_SHA1_RAWSZ] = {i}; + struct reftable_log_record log = { + .update_index = 1, + .value_type = REFTABLE_LOG_UPDATE, + .value.update = { + .old_hash = hash, + .new_hash = hash, + }, + }; + + strbuf_reset(&buf); + strbuf_addf(&buf, "refs/heads/%04d", i); + log.refname = buf.buf, + + err = reftable_writer_add_log(writer, &log); + EXPECT_ERR(err); + } + + reftable_writer_close(writer); + + /* + * The written data should be sufficiently large to result in indices + * for each of the block types. + */ + stats = reftable_writer_stats(writer); + EXPECT(stats->ref_stats.index_offset > 0); + EXPECT(stats->obj_stats.index_offset > 0); + EXPECT(stats->log_stats.index_offset > 0); + + block_source_from_strbuf(&source, &writer_buf); + err = reftable_new_reader(&reader, &source, "filename"); + EXPECT_ERR(err); + + /* + * Seeking the log uses the log index now. In case there is any + * confusion regarding indices we would notice here. + */ + err = reftable_reader_seek_log(reader, &it, ""); + EXPECT_ERR(err); + + reftable_iterator_destroy(&it); + reftable_writer_free(writer); + reftable_reader_free(reader); + strbuf_release(&writer_buf); + strbuf_release(&buf); +} + static void test_corrupt_table_empty(void) { struct strbuf buf = STRBUF_INIT; @@ -846,5 +915,6 @@ int readwrite_test_main(int argc, const char *argv[]) RUN_TEST(test_log_overflow); RUN_TEST(test_write_object_id_length); RUN_TEST(test_write_object_id_min_length); + RUN_TEST(test_write_multiple_indices); return 0; } diff --git a/reftable/record.c b/reftable/record.c index fbaa1fbef5..5c3fbb7b2a 100644 --- a/reftable/record.c +++ b/reftable/record.c @@ -76,7 +76,7 @@ int reftable_is_block_type(uint8_t typ) return 0; } -uint8_t *reftable_ref_record_val1(const struct reftable_ref_record *rec) +const unsigned char *reftable_ref_record_val1(const struct reftable_ref_record *rec) { switch (rec->value_type) { case REFTABLE_REF_VAL1: @@ -88,7 +88,7 @@ uint8_t *reftable_ref_record_val1(const struct reftable_ref_record *rec) } } -uint8_t *reftable_ref_record_val2(const struct reftable_ref_record *rec) +const unsigned char *reftable_ref_record_val2(const struct reftable_ref_record *rec) { switch (rec->value_type) { case REFTABLE_REF_VAL2: @@ -219,13 +219,10 @@ static void reftable_ref_record_copy_from(void *rec, const void *src_rec, case REFTABLE_REF_DELETION: break; case REFTABLE_REF_VAL1: - ref->value.val1 = reftable_malloc(hash_size); memcpy(ref->value.val1, src->value.val1, hash_size); break; case REFTABLE_REF_VAL2: - ref->value.val2.value = reftable_malloc(hash_size); memcpy(ref->value.val2.value, src->value.val2.value, hash_size); - ref->value.val2.target_value = reftable_malloc(hash_size); memcpy(ref->value.val2.target_value, src->value.val2.target_value, hash_size); break; @@ -242,7 +239,7 @@ static char hexdigit(int c) return 'a' + (c - 10); } -static void hex_format(char *dest, uint8_t *src, int hash_size) +static void hex_format(char *dest, const unsigned char *src, int hash_size) { assert(hash_size > 0); if (src) { @@ -299,11 +296,8 @@ void reftable_ref_record_release(struct reftable_ref_record *ref) reftable_free(ref->value.symref); break; case REFTABLE_REF_VAL2: - reftable_free(ref->value.val2.target_value); - reftable_free(ref->value.val2.value); break; case REFTABLE_REF_VAL1: - reftable_free(ref->value.val1); break; case REFTABLE_REF_DELETION: break; @@ -394,7 +388,6 @@ static int reftable_ref_record_decode(void *rec, struct strbuf key, return -1; } - r->value.val1 = reftable_malloc(hash_size); memcpy(r->value.val1, in.buf, hash_size); string_view_consume(&in, hash_size); break; @@ -404,11 +397,9 @@ static int reftable_ref_record_decode(void *rec, struct strbuf key, return -1; } - r->value.val2.value = reftable_malloc(hash_size); memcpy(r->value.val2.value, in.buf, hash_size); string_view_consume(&in, hash_size); - r->value.val2.target_value = reftable_malloc(hash_size); memcpy(r->value.val2.target_value, in.buf, hash_size); string_view_consume(&in, hash_size); break; @@ -1164,7 +1155,7 @@ int reftable_record_equal(struct reftable_record *a, struct reftable_record *b, reftable_record_data(a), reftable_record_data(b), hash_size); } -static int hash_equal(uint8_t *a, uint8_t *b, int hash_size) +static int hash_equal(const unsigned char *a, const unsigned char *b, int hash_size) { if (a && b) return !memcmp(a, b, hash_size); diff --git a/reftable/record_test.c b/reftable/record_test.c index 70ae78feca..2876db7d27 100644 --- a/reftable/record_test.c +++ b/reftable/record_test.c @@ -119,15 +119,10 @@ static void test_reftable_ref_record_roundtrip(void) case REFTABLE_REF_DELETION: break; case REFTABLE_REF_VAL1: - in.u.ref.value.val1 = reftable_malloc(GIT_SHA1_RAWSZ); set_hash(in.u.ref.value.val1, 1); break; case REFTABLE_REF_VAL2: - in.u.ref.value.val2.value = - reftable_malloc(GIT_SHA1_RAWSZ); set_hash(in.u.ref.value.val2.value, 1); - in.u.ref.value.val2.target_value = - reftable_malloc(GIT_SHA1_RAWSZ); set_hash(in.u.ref.value.val2.target_value, 2); break; case REFTABLE_REF_SYMREF: diff --git a/reftable/reftable-record.h b/reftable/reftable-record.h index 67104f8fbf..bb6e99acd3 100644 --- a/reftable/reftable-record.h +++ b/reftable/reftable-record.h @@ -9,6 +9,7 @@ https://developers.google.com/open-source/licenses/bsd #ifndef REFTABLE_RECORD_H #define REFTABLE_RECORD_H +#include "hash-ll.h" #include <stdint.h> /* @@ -38,10 +39,10 @@ struct reftable_ref_record { #define REFTABLE_NR_REF_VALUETYPES 4 } value_type; union { - uint8_t *val1; /* malloced hash. */ + unsigned char val1[GIT_MAX_RAWSZ]; struct { - uint8_t *value; /* first value, malloced hash */ - uint8_t *target_value; /* second value, malloced hash */ + unsigned char value[GIT_MAX_RAWSZ]; /* first hash */ + unsigned char target_value[GIT_MAX_RAWSZ]; /* second hash */ } val2; char *symref; /* referent, malloced 0-terminated string */ } value; @@ -49,11 +50,11 @@ struct reftable_ref_record { /* Returns the first hash, or NULL if `rec` is not of type * REFTABLE_REF_VAL1 or REFTABLE_REF_VAL2. */ -uint8_t *reftable_ref_record_val1(const struct reftable_ref_record *rec); +const unsigned char *reftable_ref_record_val1(const struct reftable_ref_record *rec); /* Returns the second hash, or NULL if `rec` is not of type * REFTABLE_REF_VAL2. */ -uint8_t *reftable_ref_record_val2(const struct reftable_ref_record *rec); +const unsigned char *reftable_ref_record_val2(const struct reftable_ref_record *rec); /* returns whether 'ref' represents a deletion */ int reftable_ref_record_is_deletion(const struct reftable_ref_record *ref); diff --git a/reftable/stack.c b/reftable/stack.c index 16bab82063..7ffeb3ee10 100644 --- a/reftable/stack.c +++ b/reftable/stack.c @@ -425,9 +425,6 @@ int reftable_stack_add(struct reftable_stack *st, return err; } - if (!st->disable_auto_compact) - return reftable_stack_auto_compact(st); - return 0; } @@ -801,18 +798,16 @@ static int stack_write_compact(struct reftable_stack *st, err = 0; break; } - if (err < 0) { - break; - } + if (err < 0) + goto done; if (first == 0 && reftable_ref_record_is_deletion(&ref)) { continue; } err = reftable_writer_add_ref(wr, &ref); - if (err < 0) { - break; - } + if (err < 0) + goto done; entries++; } reftable_iterator_destroy(&it); @@ -827,9 +822,8 @@ static int stack_write_compact(struct reftable_stack *st, err = 0; break; } - if (err < 0) { - break; - } + if (err < 0) + goto done; if (first == 0 && reftable_log_record_is_deletion(&log)) { continue; } @@ -845,9 +839,8 @@ static int stack_write_compact(struct reftable_stack *st, } err = reftable_writer_add_log(wr, &log); - if (err < 0) { - break; - } + if (err < 0) + goto done; entries++; } diff --git a/reftable/stack_test.c b/reftable/stack_test.c index 82280c2fd5..289e902146 100644 --- a/reftable/stack_test.c +++ b/reftable/stack_test.c @@ -462,7 +462,6 @@ static void test_reftable_stack_add(void) refs[i].refname = xstrdup(buf); refs[i].update_index = i + 1; refs[i].value_type = REFTABLE_REF_VAL1; - refs[i].value.val1 = reftable_malloc(GIT_SHA1_RAWSZ); set_test_hash(refs[i].value.val1, i); logs[i].refname = xstrdup(buf); @@ -599,7 +598,6 @@ static void test_reftable_stack_tombstone(void) refs[i].update_index = i + 1; if (i % 2 == 0) { refs[i].value_type = REFTABLE_REF_VAL1; - refs[i].value.val1 = reftable_malloc(GIT_SHA1_RAWSZ); set_test_hash(refs[i].value.val1, i); } diff --git a/reftable/writer.c b/reftable/writer.c index 2e322a5683..ee4590e20f 100644 --- a/reftable/writer.c +++ b/reftable/writer.c @@ -432,12 +432,12 @@ static int writer_finish_section(struct reftable_writer *w) reftable_free(idx); } - writer_clear_index(w); - err = writer_flush_block(w); if (err < 0) return err; + writer_clear_index(w); + bstats = writer_reftable_block_stats(w, typ); bstats->index_blocks = w->stats.idx_stats.blocks - before_blocks; bstats->index_offset = index_start; diff --git a/repository.c b/repository.c index a7679ceeaa..d7d24d416a 100644 --- a/repository.c +++ b/repository.c @@ -104,6 +104,11 @@ void repo_set_hash_algo(struct repository *repo, int hash_algo) repo->hash_algo = &hash_algos[hash_algo]; } +void repo_set_ref_storage_format(struct repository *repo, unsigned int format) +{ + repo->ref_storage_format = format; +} + /* * Attempt to resolve and set the provided 'gitdir' for repository 'repo'. * Return 0 upon success and a non-zero value upon failure. @@ -184,6 +189,7 @@ int repo_init(struct repository *repo, goto error; repo_set_hash_algo(repo, format.hash_algo); + repo_set_ref_storage_format(repo, format.ref_storage_format); repo->repository_format_worktree_config = format.worktree_config; /* take ownership of format.partial_clone */ diff --git a/repository.h b/repository.h index 5f18486f64..f5269b3730 100644 --- a/repository.h +++ b/repository.h @@ -24,6 +24,9 @@ enum fetch_negotiation_setting { FETCH_NEGOTIATION_NOOP, }; +#define REF_STORAGE_FORMAT_UNKNOWN 0 +#define REF_STORAGE_FORMAT_FILES 1 + struct repo_settings { int initialized; @@ -160,6 +163,9 @@ struct repository { /* Repository's current hash algorithm, as serialized on disk. */ const struct git_hash_algo *hash_algo; + /* Repository's reference storage format, as serialized on disk. */ + unsigned int ref_storage_format; + /* A unique-id for tracing purposes. */ int trace2_repo_id; @@ -199,6 +205,7 @@ void repo_set_gitdir(struct repository *repo, const char *root, const struct set_gitdir_args *extra_args); void repo_set_worktree(struct repository *repo, const char *path); void repo_set_hash_algo(struct repository *repo, int algo); +void repo_set_ref_storage_format(struct repository *repo, unsigned int format); void initialize_the_repository(void); RESULT_MUST_BE_USED int repo_init(struct repository *r, const char *gitdir, const char *worktree); @@ -591,6 +591,17 @@ static enum extension_result handle_extension(const char *var, "extensions.objectformat", value); data->hash_algo = format; return EXTENSION_OK; + } else if (!strcmp(ext, "refstorage")) { + unsigned int format; + + if (!value) + return config_error_nonbool(var); + format = ref_storage_format_by_name(value); + if (format == REF_STORAGE_FORMAT_UNKNOWN) + return error(_("invalid value for '%s': '%s'"), + "extensions.refstorage", value); + data->ref_storage_format = format; + return EXTENSION_OK; } return EXTENSION_UNKNOWN; } @@ -1565,6 +1576,8 @@ const char *setup_git_directory_gently(int *nongit_ok) } if (startup_info->have_repository) { repo_set_hash_algo(the_repository, repo_fmt.hash_algo); + repo_set_ref_storage_format(the_repository, + repo_fmt.ref_storage_format); the_repository->repository_format_worktree_config = repo_fmt.worktree_config; /* take ownership of repo_fmt.partial_clone */ @@ -1658,6 +1671,8 @@ void check_repository_format(struct repository_format *fmt) check_repository_format_gently(get_git_dir(), fmt, NULL); startup_info->have_repository = 1; repo_set_hash_algo(the_repository, fmt->hash_algo); + repo_set_ref_storage_format(the_repository, + fmt->ref_storage_format); the_repository->repository_format_worktree_config = fmt->worktree_config; the_repository->repository_format_partial_clone = @@ -1866,12 +1881,15 @@ static int needs_work_tree_config(const char *git_dir, const char *work_tree) return 1; } -void initialize_repository_version(int hash_algo, int reinit) +void initialize_repository_version(int hash_algo, + unsigned int ref_storage_format, + int reinit) { char repo_version_string[10]; int repo_version = GIT_REPO_VERSION; - if (hash_algo != GIT_HASH_SHA1) + if (hash_algo != GIT_HASH_SHA1 || + ref_storage_format != REF_STORAGE_FORMAT_FILES) repo_version = GIT_REPO_VERSION_READ; /* This forces creation of new config file */ @@ -1884,6 +1902,10 @@ void initialize_repository_version(int hash_algo, int reinit) hash_algos[hash_algo].name); else if (reinit) git_config_set_gently("extensions.objectformat", NULL); + + if (ref_storage_format != REF_STORAGE_FORMAT_FILES) + git_config_set("extensions.refstorage", + ref_storage_format_to_name(ref_storage_format)); } static int is_reinit(void) @@ -1898,7 +1920,8 @@ static int is_reinit(void) return ret; } -void create_reference_database(const char *initial_branch, int quiet) +void create_reference_database(unsigned int ref_storage_format, + const char *initial_branch, int quiet) { struct strbuf err = STRBUF_INIT; int reinit = is_reinit(); @@ -1918,6 +1941,7 @@ void create_reference_database(const char *initial_branch, int quiet) safe_create_dir(git_path("refs"), 1); adjust_shared_perm(git_path("refs")); + repo_set_ref_storage_format(the_repository, ref_storage_format); if (refs_init_db(&err)) die("failed to set up refs db: %s", err.buf); @@ -2023,7 +2047,7 @@ static int create_default_files(const char *template_path, adjust_shared_perm(get_git_dir()); } - initialize_repository_version(fmt->hash_algo, 0); + initialize_repository_version(fmt->hash_algo, fmt->ref_storage_format, 0); /* Check filemode trustability */ path = git_path_buf(&buf, "config"); @@ -2136,8 +2160,29 @@ static void validate_hash_algorithm(struct repository_format *repo_fmt, int hash } } +static void validate_ref_storage_format(struct repository_format *repo_fmt, + unsigned int format) +{ + const char *name = getenv("GIT_DEFAULT_REF_FORMAT"); + + if (repo_fmt->version >= 0 && + format != REF_STORAGE_FORMAT_UNKNOWN && + format != repo_fmt->ref_storage_format) { + die(_("attempt to reinitialize repository with different reference storage format")); + } else if (format != REF_STORAGE_FORMAT_UNKNOWN) { + repo_fmt->ref_storage_format = format; + } else if (name) { + format = ref_storage_format_by_name(name); + if (format == REF_STORAGE_FORMAT_UNKNOWN) + die(_("unknown ref storage format '%s'"), name); + repo_fmt->ref_storage_format = format; + } +} + int init_db(const char *git_dir, const char *real_git_dir, - const char *template_dir, int hash, const char *initial_branch, + const char *template_dir, int hash, + unsigned int ref_storage_format, + const char *initial_branch, int init_shared_repository, unsigned int flags) { int reinit; @@ -2180,13 +2225,22 @@ int init_db(const char *git_dir, const char *real_git_dir, check_repository_format(&repo_fmt); validate_hash_algorithm(&repo_fmt, hash); + validate_ref_storage_format(&repo_fmt, ref_storage_format); reinit = create_default_files(template_dir, original_git_dir, &repo_fmt, prev_bare_repository, init_shared_repository); + /* + * Now that we have set up both the hash algorithm and the ref storage + * format we can update the repository's settings accordingly. + */ + repo_set_hash_algo(the_repository, repo_fmt.hash_algo); + repo_set_ref_storage_format(the_repository, repo_fmt.ref_storage_format); + if (!(flags & INIT_DB_SKIP_REFDB)) - create_reference_database(initial_branch, flags & INIT_DB_QUIET); + create_reference_database(repo_fmt.ref_storage_format, + initial_branch, flags & INIT_DB_QUIET); create_object_directory(); if (get_shared_repository()) { @@ -115,6 +115,7 @@ struct repository_format { int worktree_config; int is_bare; int hash_algo; + unsigned int ref_storage_format; int sparse_index; char *work_tree; struct string_list unknown_extensions; @@ -131,6 +132,7 @@ struct repository_format { .version = -1, \ .is_bare = -1, \ .hash_algo = GIT_HASH_SHA1, \ + .ref_storage_format = REF_STORAGE_FORMAT_FILES, \ .unknown_extensions = STRING_LIST_INIT_DUP, \ .v1_only_extensions = STRING_LIST_INIT_DUP, \ } @@ -175,10 +177,14 @@ void check_repository_format(struct repository_format *fmt); int init_db(const char *git_dir, const char *real_git_dir, const char *template_dir, int hash_algo, + unsigned int ref_storage_format, const char *initial_branch, int init_shared_repository, unsigned int flags); -void initialize_repository_version(int hash_algo, int reinit); -void create_reference_database(const char *initial_branch, int quiet); +void initialize_repository_version(int hash_algo, + unsigned int ref_storage_format, + int reinit); +void create_reference_database(unsigned int ref_storage_format, + const char *initial_branch, int quiet); /* * NOTE NOTE NOTE!! diff --git a/sideband.c b/sideband.c index 6cbfd391c4..266a67342b 100644 --- a/sideband.c +++ b/sideband.c @@ -69,7 +69,10 @@ void list_config_color_sideband_slots(struct string_list *list, const char *pref * of the line. This should be called for a single line only, which is * passed as the first N characters of the SRC array. * - * NEEDSWORK: use "size_t n" instead for clarity. + * It is fine to use "int n" here instead of "size_t n" as all calls to this + * function pass an 'int' parameter. Additionally, the buffer involved in + * storing these 'int' values takes input from a packet via the pkt-line + * interface, which is capable of transferring only 64kB at a time. */ static void maybe_colorize_sideband(struct strbuf *dest, const char *src, int n) { @@ -479,6 +479,9 @@ GIT_TEST_DEFAULT_HASH=<hash-algo> specifies which hash algorithm to use in the test scripts. Recognized values for <hash-algo> are "sha1" and "sha256". +GIT_TEST_DEFAULT_REF_FORMAT=<format> specifies which ref storage format +to use in the test scripts. Recognized values for <format> are "files". + GIT_TEST_NO_WRITE_REV_INDEX=<boolean>, when true disables the 'pack.writeReverseIndex' setting. diff --git a/t/helper/test-pkt-line.c b/t/helper/test-pkt-line.c index 77e99c37df..4daa82f00f 100644 --- a/t/helper/test-pkt-line.c +++ b/t/helper/test-pkt-line.c @@ -3,6 +3,7 @@ #include "pkt-line.h" #include "sideband.h" #include "write-or-die.h" +#include "parse-options.h" static void pack_line(const char *line) { @@ -65,12 +66,33 @@ static void unpack(void) } } -static void unpack_sideband(void) +static void unpack_sideband(int argc, const char **argv) { struct packet_reader reader; - packet_reader_init(&reader, 0, NULL, 0, - PACKET_READ_GENTLE_ON_EOF | - PACKET_READ_CHOMP_NEWLINE); + int options = PACKET_READ_GENTLE_ON_EOF; + int chomp_newline = 1; + int reader_use_sideband = 0; + const char *const unpack_sideband_usage[] = { + "test_tool unpack_sideband [options...]", NULL + }; + struct option cmd_options[] = { + OPT_BOOL(0, "reader-use-sideband", &reader_use_sideband, + "set use_sideband bit for packet reader (Default: off)"), + OPT_BOOL(0, "chomp-newline", &chomp_newline, + "chomp newline in packet (Default: on)"), + OPT_END() + }; + + argc = parse_options(argc, argv, "", cmd_options, unpack_sideband_usage, + 0); + if (argc > 0) + usage_msg_opt(_("too many arguments"), unpack_sideband_usage, + cmd_options); + + if (chomp_newline) + options |= PACKET_READ_CHOMP_NEWLINE; + packet_reader_init(&reader, 0, NULL, 0, options); + reader.use_sideband = reader_use_sideband; while (packet_reader_read(&reader) != PACKET_READ_EOF) { int band; @@ -80,6 +102,17 @@ static void unpack_sideband(void) case PACKET_READ_EOF: break; case PACKET_READ_NORMAL: + /* + * When the "use_sideband" field of the reader is turned + * on, sideband packets other than the payload have been + * parsed and consumed in packet_reader_read(), and only + * the payload arrives here. + */ + if (reader.use_sideband) { + write_or_die(1, reader.line, reader.pktlen - 1); + break; + } + band = reader.line[0] & 0xff; if (band < 1 || band > 2) continue; /* skip non-sideband packets */ @@ -98,15 +131,31 @@ static void unpack_sideband(void) static int send_split_sideband(void) { + const char *foo = "Foo.\n"; + const char *bar = "Bar.\n"; const char *part1 = "Hello,"; const char *primary = "\001primary: regular output\n"; const char *part2 = " world!\n"; + /* Each sideband message has a trailing newline character. */ + send_sideband(1, 2, foo, strlen(foo), LARGE_PACKET_MAX); + send_sideband(1, 2, bar, strlen(bar), LARGE_PACKET_MAX); + + /* + * One sideband message is divided into part1 and part2 + * by the primary message. + */ send_sideband(1, 2, part1, strlen(part1), LARGE_PACKET_MAX); packet_write(1, primary, strlen(primary)); send_sideband(1, 2, part2, strlen(part2), LARGE_PACKET_MAX); packet_response_end(1); + /* + * We use unpack_sideband() to consume packets. A flush packet + * is required to end parsing. + */ + packet_flush(1); + return 0; } @@ -127,7 +176,7 @@ int cmd__pkt_line(int argc, const char **argv) else if (!strcmp(argv[1], "unpack")) unpack(); else if (!strcmp(argv[1], "unpack-sideband")) - unpack_sideband(); + unpack_sideband(argc - 1, argv + 1); else if (!strcmp(argv[1], "send-split-sideband")) send_split_sideband(); else if (!strcmp(argv[1], "receive-sideband")) diff --git a/t/helper/test-read-midx.c b/t/helper/test-read-midx.c index e9a444ddba..4acae41bb9 100644 --- a/t/helper/test-read-midx.c +++ b/t/helper/test-read-midx.c @@ -6,6 +6,7 @@ #include "pack-bitmap.h" #include "packfile.h" #include "setup.h" +#include "gettext.h" static int read_midx_file(const char *object_dir, int show_objects) { @@ -79,7 +80,7 @@ static int read_midx_checksum(const char *object_dir) static int read_midx_preferred_pack(const char *object_dir) { struct multi_pack_index *midx = NULL; - struct bitmap_index *bitmap = NULL; + uint32_t preferred_pack; setup_git_directory(); @@ -87,23 +88,45 @@ static int read_midx_preferred_pack(const char *object_dir) if (!midx) return 1; - bitmap = prepare_bitmap_git(the_repository); - if (!bitmap) + if (midx_preferred_pack(midx, &preferred_pack) < 0) { + warning(_("could not determine MIDX preferred pack")); return 1; - if (!bitmap_is_midx(bitmap)) { - free_bitmap_index(bitmap); + } + + printf("%s\n", midx->pack_names[preferred_pack]); + return 0; +} + +static int read_midx_bitmapped_packs(const char *object_dir) +{ + struct multi_pack_index *midx = NULL; + struct bitmapped_pack pack; + uint32_t i; + + setup_git_directory(); + + midx = load_multi_pack_index(object_dir, 1); + if (!midx) return 1; + + for (i = 0; i < midx->num_packs; i++) { + if (nth_bitmapped_pack(the_repository, midx, &pack, i) < 0) + return 1; + + printf("%s\n", pack_basename(pack.p)); + printf(" bitmap_pos: %"PRIuMAX"\n", (uintmax_t)pack.bitmap_pos); + printf(" bitmap_nr: %"PRIuMAX"\n", (uintmax_t)pack.bitmap_nr); } - printf("%s\n", midx->pack_names[midx_preferred_pack(bitmap)]); - free_bitmap_index(bitmap); + close_midx(midx); + return 0; } int cmd__read_midx(int argc, const char **argv) { if (!(argc == 2 || argc == 3)) - usage("read-midx [--show-objects|--checksum|--preferred-pack] <object-dir>"); + usage("read-midx [--show-objects|--checksum|--preferred-pack|--bitmap] <object-dir>"); if (!strcmp(argv[1], "--show-objects")) return read_midx_file(argv[2], 1); @@ -111,5 +134,7 @@ int cmd__read_midx(int argc, const char **argv) return read_midx_checksum(argv[2]); else if (!strcmp(argv[1], "--preferred-pack")) return read_midx_preferred_pack(argv[2]); + else if (!strcmp(argv[1], "--bitmap")) + return read_midx_bitmapped_packs(argv[2]); return read_midx_file(argv[1], 0); } diff --git a/t/perf/p5332-multi-pack-reuse.sh b/t/perf/p5332-multi-pack-reuse.sh new file mode 100755 index 0000000000..5c6c575d62 --- /dev/null +++ b/t/perf/p5332-multi-pack-reuse.sh @@ -0,0 +1,81 @@ +#!/bin/sh + +test_description='tests pack performance with multi-pack reuse' + +. ./perf-lib.sh +. "${TEST_DIRECTORY}/perf/lib-pack.sh" + +packdir=.git/objects/pack + +test_perf_large_repo + +find_pack () { + for idx in $packdir/pack-*.idx + do + if git show-index <$idx | grep -q "$1" + then + basename $idx + fi || return 1 + done +} + +repack_into_n_chunks () { + git repack -adk && + + test "$1" -eq 1 && return || + + find $packdir -type f | sort >packs.before && + + # partition the repository into $1 chunks of consecutive commits, and + # then create $1 packs with the objects reachable from each chunk + # (excluding any objects reachable from the previous chunks) + sz="$(($(git rev-list --count --all) / $1))" + for rev in $(git rev-list --all | awk "NR % $sz == 0" | tac) + do + pack="$(echo "$rev" | git pack-objects --revs \ + --honor-pack-keep --delta-base-offset $packdir/pack)" && + touch $packdir/pack-$pack.keep || return 1 + done + + # grab any remaining objects not packed by the previous step(s) + git pack-objects --revs --all --honor-pack-keep --delta-base-offset \ + $packdir/pack && + + find $packdir -type f | sort >packs.after && + + # and install the whole thing + for f in $(comm -12 packs.before packs.after) + do + rm -f "$f" || return 1 + done + rm -fr $packdir/*.keep +} + +for nr_packs in 1 10 100 +do + test_expect_success "create $nr_packs-pack scenario" ' + repack_into_n_chunks $nr_packs + ' + + test_expect_success "setup bitmaps for $nr_packs-pack scenario" ' + find $packdir -type f -name "*.idx" | sed -e "s/.*\/\(.*\)$/+\1/g" | + git multi-pack-index write --stdin-packs --bitmap \ + --preferred-pack="$(find_pack $(git rev-parse HEAD))" + ' + + for reuse in single multi + do + test_perf "clone for $nr_packs-pack scenario ($reuse-pack reuse)" " + git for-each-ref --format='%(objectname)' refs/heads refs/tags >in && + git -c pack.allowPackReuse=$reuse pack-objects \ + --revs --delta-base-offset --use-bitmap-index \ + --stdout <in >result + " + + test_size "clone size for $nr_packs-pack scenario ($reuse-pack reuse)" ' + wc -c <result + ' + done +done + +test_done diff --git a/t/t0001-init.sh b/t/t0001-init.sh index 2b78e3be47..b131d665db 100755 --- a/t/t0001-init.sh +++ b/t/t0001-init.sh @@ -532,6 +532,76 @@ test_expect_success 'init rejects attempts to initialize with different hash' ' test_must_fail git -C sha256 init --object-format=sha1 ' +test_expect_success DEFAULT_REPO_FORMAT 'extensions.refStorage is not allowed with repo version 0' ' + test_when_finished "rm -rf refstorage" && + git init refstorage && + git -C refstorage config extensions.refStorage files && + test_must_fail git -C refstorage rev-parse 2>err && + grep "repo version is 0, but v1-only extension found" err +' + +test_expect_success DEFAULT_REPO_FORMAT 'extensions.refStorage with files backend' ' + test_when_finished "rm -rf refstorage" && + git init refstorage && + git -C refstorage config core.repositoryformatversion 1 && + git -C refstorage config extensions.refStorage files && + test_commit -C refstorage A && + git -C refstorage rev-parse --verify HEAD +' + +test_expect_success DEFAULT_REPO_FORMAT 'extensions.refStorage with unknown backend' ' + test_when_finished "rm -rf refstorage" && + git init refstorage && + git -C refstorage config core.repositoryformatversion 1 && + git -C refstorage config extensions.refStorage garbage && + test_must_fail git -C refstorage rev-parse 2>err && + grep "invalid value for ${SQ}extensions.refstorage${SQ}: ${SQ}garbage${SQ}" err +' + +test_expect_success DEFAULT_REPO_FORMAT 'init with GIT_DEFAULT_REF_FORMAT=files' ' + test_when_finished "rm -rf refformat" && + GIT_DEFAULT_REF_FORMAT=files git init refformat && + echo 0 >expect && + git -C refformat config core.repositoryformatversion >actual && + test_cmp expect actual && + test_must_fail git -C refformat config extensions.refstorage +' + +test_expect_success 'init with GIT_DEFAULT_REF_FORMAT=garbage' ' + test_when_finished "rm -rf refformat" && + cat >expect <<-EOF && + fatal: unknown ref storage format ${SQ}garbage${SQ} + EOF + test_must_fail env GIT_DEFAULT_REF_FORMAT=garbage git init refformat 2>err && + test_cmp expect err +' + +test_expect_success 'init with --ref-format=files' ' + test_when_finished "rm -rf refformat" && + git init --ref-format=files refformat && + echo files >expect && + git -C refformat rev-parse --show-ref-format >actual && + test_cmp expect actual +' + +test_expect_success 're-init with same format' ' + test_when_finished "rm -rf refformat" && + git init --ref-format=files refformat && + git init --ref-format=files refformat && + echo files >expect && + git -C refformat rev-parse --show-ref-format >actual && + test_cmp expect actual +' + +test_expect_success 'init with --ref-format=garbage' ' + test_when_finished "rm -rf refformat" && + cat >expect <<-EOF && + fatal: unknown ref storage format ${SQ}garbage${SQ} + EOF + test_must_fail git init --ref-format=garbage refformat 2>err && + test_cmp expect err +' + test_expect_success MINGW 'core.hidedotfiles = false' ' git config --global core.hidedotfiles false && rm -rf newdir && diff --git a/t/t0003-attributes.sh b/t/t0003-attributes.sh index aee2298f01..774b52c298 100755 --- a/t/t0003-attributes.sh +++ b/t/t0003-attributes.sh @@ -19,6 +19,20 @@ attr_check () { test_must_be_empty err } +attr_check_object_mode_basic () { + path="$1" && + expect="$2" && + check_opts="$3" && + git check-attr $check_opts builtin_objectmode -- "$path" >actual 2>err && + echo "$path: builtin_objectmode: $expect" >expect && + test_cmp expect actual +} + +attr_check_object_mode () { + attr_check_object_mode_basic "$@" && + test_must_be_empty err +} + attr_check_quote () { path="$1" quoted_path="$2" expect="$3" && @@ -558,4 +572,66 @@ test_expect_success EXPENSIVE 'large attributes file ignored in index' ' test_cmp expect err ' +test_expect_success 'builtin object mode attributes work (dir and regular paths)' ' + >normal && + attr_check_object_mode normal 100644 && + mkdir dir && + attr_check_object_mode dir 040000 +' + +test_expect_success POSIXPERM 'builtin object mode attributes work (executable)' ' + >exec && + chmod +x exec && + attr_check_object_mode exec 100755 +' + +test_expect_success SYMLINKS 'builtin object mode attributes work (symlinks)' ' + ln -s to_sym sym && + attr_check_object_mode sym 120000 +' + +test_expect_success 'native object mode attributes work with --cached' ' + >normal && + git add normal && + empty_blob=$(git rev-parse :normal) && + git update-index --index-info <<-EOF && + 100755 $empty_blob 0 exec + 120000 $empty_blob 0 symlink + EOF + attr_check_object_mode normal 100644 --cached && + attr_check_object_mode exec 100755 --cached && + attr_check_object_mode symlink 120000 --cached +' + +test_expect_success 'check object mode attributes work for submodules' ' + mkdir sub && + ( + cd sub && + git init && + mv .git .real && + echo "gitdir: .real" >.git && + test_commit first + ) && + attr_check_object_mode sub 160000 && + attr_check_object_mode sub unspecified --cached && + git add sub && + attr_check_object_mode sub 160000 --cached +' + +test_expect_success 'we do not allow user defined builtin_* attributes' ' + echo "foo* builtin_foo" >.gitattributes && + git add .gitattributes 2>actual && + echo "builtin_foo is not a valid attribute name: .gitattributes:1" >expect && + test_cmp expect actual +' + +test_expect_success 'user defined builtin_objectmode values are ignored' ' + echo "foo* builtin_objectmode=12345" >.gitattributes && + git add .gitattributes && + >foo_1 && + attr_check_object_mode_basic foo_1 100644 && + echo "builtin_objectmode is not a valid attribute name: .gitattributes:1" >expect && + test_cmp expect err +' + test_done diff --git a/t/t0070-fundamental.sh b/t/t0070-fundamental.sh index 487bc8d905..f18f9284a5 100755 --- a/t/t0070-fundamental.sh +++ b/t/t0070-fundamental.sh @@ -53,4 +53,62 @@ test_expect_success 'missing sideband designator is reported' ' test_grep "missing sideband" err ' +test_expect_success 'unpack-sideband: --no-chomp-newline' ' + test_when_finished "rm -f expect-out expect-err" && + test-tool pkt-line send-split-sideband >split-sideband && + test-tool pkt-line unpack-sideband \ + --no-chomp-newline <split-sideband >out 2>err && + cat >expect-out <<-EOF && + primary: regular output + EOF + cat >expect-err <<-EOF && + Foo. + Bar. + Hello, world! + EOF + test_cmp expect-out out && + test_cmp expect-err err +' + +test_expect_success 'unpack-sideband: --chomp-newline (default)' ' + test_when_finished "rm -f expect-out expect-err" && + test-tool pkt-line send-split-sideband >split-sideband && + test-tool pkt-line unpack-sideband \ + --chomp-newline <split-sideband >out 2>err && + printf "primary: regular output" >expect-out && + printf "Foo.Bar.Hello, world!" >expect-err && + test_cmp expect-out out && + test_cmp expect-err err +' + +test_expect_success 'unpack-sideband: packet_reader_read() consumes sideband, no chomp payload' ' + test_when_finished "rm -f expect-out expect-err" && + test-tool pkt-line send-split-sideband >split-sideband && + test-tool pkt-line unpack-sideband \ + --reader-use-sideband \ + --no-chomp-newline <split-sideband >out 2>err && + cat >expect-out <<-EOF && + primary: regular output + EOF + printf "remote: Foo. \n" >expect-err && + printf "remote: Bar. \n" >>expect-err && + printf "remote: Hello, world! \n" >>expect-err && + test_cmp expect-out out && + test_cmp expect-err err +' + +test_expect_success 'unpack-sideband: packet_reader_read() consumes sideband, chomp payload' ' + test_when_finished "rm -f expect-out expect-err" && + test-tool pkt-line send-split-sideband >split-sideband && + test-tool pkt-line unpack-sideband \ + --reader-use-sideband \ + --chomp-newline <split-sideband >out 2>err && + printf "primary: regular output" >expect-out && + printf "remote: Foo. \n" >expect-err && + printf "remote: Bar. \n" >>expect-err && + printf "remote: Hello, world! \n" >>expect-err && + test_cmp expect-out out && + test_cmp expect-err err +' + test_done diff --git a/t/t1006-cat-file.sh b/t/t1006-cat-file.sh index 271c5e4fd3..e0c6482797 100755 --- a/t/t1006-cat-file.sh +++ b/t/t1006-cat-file.sh @@ -1100,6 +1100,42 @@ test_expect_success 'cat-file --batch="batman" with --batch-all-objects will wor cmp expect actual ' +test_expect_success 'cat-file %(objectsize:disk) with --batch-all-objects' ' + # our state has both loose and packed objects, + # so find both for our expected output + { + find .git/objects/?? -type f | + awk -F/ "{ print \$0, \$3\$4 }" | + while read path oid + do + size=$(test_file_size "$path") && + echo "$oid $size" || + return 1 + done && + rawsz=$(test_oid rawsz) && + find .git/objects/pack -name "*.idx" | + while read idx + do + git show-index <"$idx" >idx.raw && + sort -nr <idx.raw >idx.sorted && + packsz=$(test_file_size "${idx%.idx}.pack") && + end=$((packsz - rawsz)) && + while read start oid rest + do + size=$((end - start)) && + end=$start && + echo "$oid $size" || + return 1 + done <idx.sorted || + return 1 + done + } >expect.raw && + sort <expect.raw >expect && + git cat-file --batch-all-objects \ + --batch-check="%(objectname) %(objectsize:disk)" >actual && + test_cmp expect actual +' + test_expect_success 'set up replacement object' ' orig=$(git rev-parse HEAD) && git cat-file commit $orig >orig && diff --git a/t/t1500-rev-parse.sh b/t/t1500-rev-parse.sh index 3f9e7f62e4..a669e592f1 100755 --- a/t/t1500-rev-parse.sh +++ b/t/t1500-rev-parse.sh @@ -208,6 +208,23 @@ test_expect_success 'rev-parse --show-object-format in repo' ' grep "unknown mode for --show-object-format: squeamish-ossifrage" err ' +test_expect_success 'rev-parse --show-ref-format' ' + test_detect_ref_format >expect && + git rev-parse --show-ref-format >actual && + test_cmp expect actual +' + +test_expect_success 'rev-parse --show-ref-format with invalid storage' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + git config extensions.refstorage broken && + test_must_fail git rev-parse --show-ref-format 2>err && + grep "error: invalid value for ${SQ}extensions.refstorage${SQ}: ${SQ}broken${SQ}" err + ) +' + test_expect_success '--show-toplevel from subdir of working tree' ' pwd >expect && git -C sub/dir rev-parse --show-toplevel >actual && diff --git a/t/t3200-branch.sh b/t/t3200-branch.sh index 6a316f081e..de7d3014e4 100755 --- a/t/t3200-branch.sh +++ b/t/t3200-branch.sh @@ -519,7 +519,7 @@ EOF mv .git/config .git/config-saved -test_expect_success SHA1 'git branch -m q q2 without config should succeed' ' +test_expect_success DEFAULT_REPO_FORMAT 'git branch -m q q2 without config should succeed' ' git branch -m q q2 && git branch -m q2 q ' diff --git a/t/t4129-apply-samemode.sh b/t/t4129-apply-samemode.sh index e7a7295f1b..2775bfadd8 100755 --- a/t/t4129-apply-samemode.sh +++ b/t/t4129-apply-samemode.sh @@ -41,7 +41,8 @@ test_expect_success FILEMODE 'same mode (index only)' ' chmod +x file && git add file && git apply --cached patch-0.txt && - git ls-files -s file | grep "^100755" + git ls-files -s file >ls-files-output && + test_grep "^100755" ls-files-output ' test_expect_success FILEMODE 'mode update (no index)' ' @@ -60,7 +61,8 @@ test_expect_success FILEMODE 'mode update (with index)' ' test_expect_success FILEMODE 'mode update (index only)' ' git reset --hard && git apply --cached patch-1.txt && - git ls-files -s file | grep "^100755" + git ls-files -s file >ls-files-output && + test_grep "^100755" ls-files-output ' test_expect_success FILEMODE 'empty mode is rejected' ' diff --git a/t/t5319-multi-pack-index.sh b/t/t5319-multi-pack-index.sh index c20aafe99a..dd09134db0 100755 --- a/t/t5319-multi-pack-index.sh +++ b/t/t5319-multi-pack-index.sh @@ -1171,4 +1171,39 @@ test_expect_success 'reader notices out-of-bounds fanout' ' test_cmp expect err ' +test_expect_success 'bitmapped packs are stored via the BTMP chunk' ' + test_when_finished "rm -fr repo" && + git init repo && + ( + cd repo && + + for i in 1 2 3 4 5 + do + test_commit "$i" && + git repack -d || return 1 + done && + + find $objdir/pack -type f -name "*.idx" | xargs -n 1 basename | + sort >packs && + + git multi-pack-index write --stdin-packs <packs && + test_must_fail test-tool read-midx --bitmap $objdir 2>err && + cat >expect <<-\EOF && + error: MIDX does not contain the BTMP chunk + EOF + test_cmp expect err && + + git multi-pack-index write --stdin-packs --bitmap \ + --preferred-pack="$(head -n1 <packs)" <packs && + test-tool read-midx --bitmap $objdir >actual && + for i in $(test_seq $(wc -l <packs)) + do + sed -ne "${i}s/\.idx$/\.pack/p" packs && + echo " bitmap_pos: $((($i - 1) * 3))" && + echo " bitmap_nr: 3" || return 1 + done >expect && + test_cmp expect actual + ) +' + test_done diff --git a/t/t5332-multi-pack-reuse.sh b/t/t5332-multi-pack-reuse.sh new file mode 100755 index 0000000000..2ba788b042 --- /dev/null +++ b/t/t5332-multi-pack-reuse.sh @@ -0,0 +1,203 @@ +#!/bin/sh + +test_description='pack-objects multi-pack reuse' + +. ./test-lib.sh +. "$TEST_DIRECTORY"/lib-bitmap.sh + +objdir=.git/objects +packdir=$objdir/pack + +test_pack_reused () { + test_trace2_data pack-objects pack-reused "$1" +} + +test_packs_reused () { + test_trace2_data pack-objects packs-reused "$1" +} + + +# pack_position <object> </path/to/pack.idx +pack_position () { + git show-index >objects && + grep "$1" objects | cut -d" " -f1 +} + +test_expect_success 'preferred pack is reused for single-pack reuse' ' + test_config pack.allowPackReuse single && + + for i in A B + do + test_commit "$i" && + git repack -d || return 1 + done && + + git multi-pack-index write --bitmap && + + : >trace2.txt && + GIT_TRACE2_EVENT="$PWD/trace2.txt" \ + git pack-objects --stdout --revs --all >/dev/null && + + test_pack_reused 3 <trace2.txt && + test_packs_reused 1 <trace2.txt +' + +test_expect_success 'enable multi-pack reuse' ' + git config pack.allowPackReuse multi +' + +test_expect_success 'reuse all objects from subset of bitmapped packs' ' + test_commit C && + git repack -d && + + git multi-pack-index write --bitmap && + + cat >in <<-EOF && + $(git rev-parse C) + ^$(git rev-parse A) + EOF + + : >trace2.txt && + GIT_TRACE2_EVENT="$PWD/trace2.txt" \ + git pack-objects --stdout --revs <in >/dev/null && + + test_pack_reused 6 <trace2.txt && + test_packs_reused 2 <trace2.txt +' + +test_expect_success 'reuse all objects from all packs' ' + : >trace2.txt && + GIT_TRACE2_EVENT="$PWD/trace2.txt" \ + git pack-objects --stdout --revs --all >/dev/null && + + test_pack_reused 9 <trace2.txt && + test_packs_reused 3 <trace2.txt +' + +test_expect_success 'reuse objects from first pack with middle gap' ' + for i in D E F + do + test_commit "$i" || return 1 + done && + + # Set "pack.window" to zero to ensure that we do not create any + # deltas, which could alter the amount of pack reuse we perform + # (if, for e.g., we are not sending one or more bases). + D="$(git -c pack.window=0 pack-objects --all --unpacked $packdir/pack)" && + + d_pos="$(pack_position $(git rev-parse D) <$packdir/pack-$D.idx)" && + e_pos="$(pack_position $(git rev-parse E) <$packdir/pack-$D.idx)" && + f_pos="$(pack_position $(git rev-parse F) <$packdir/pack-$D.idx)" && + + # commits F, E, and D, should appear in that order at the + # beginning of the pack + test $f_pos -lt $e_pos && + test $e_pos -lt $d_pos && + + # Ensure that the pack we are constructing sorts ahead of any + # other packs in lexical/bitmap order by choosing it as the + # preferred pack. + git multi-pack-index write --bitmap --preferred-pack="pack-$D.idx" && + + cat >in <<-EOF && + $(git rev-parse E) + ^$(git rev-parse D) + EOF + + : >trace2.txt && + GIT_TRACE2_EVENT="$PWD/trace2.txt" \ + git pack-objects --stdout --delta-base-offset --revs <in >/dev/null && + + test_pack_reused 3 <trace2.txt && + test_packs_reused 1 <trace2.txt +' + +test_expect_success 'reuse objects from middle pack with middle gap' ' + rm -fr $packdir/multi-pack-index* && + + # Ensure that the pack we are constructing sort into any + # position *but* the first one, by choosing a different pack as + # the preferred one. + git multi-pack-index write --bitmap --preferred-pack="pack-$A.idx" && + + cat >in <<-EOF && + $(git rev-parse E) + ^$(git rev-parse D) + EOF + + : >trace2.txt && + GIT_TRACE2_EVENT="$PWD/trace2.txt" \ + git pack-objects --stdout --delta-base-offset --revs <in >/dev/null && + + test_pack_reused 3 <trace2.txt && + test_packs_reused 1 <trace2.txt +' + +test_expect_success 'omit delta with uninteresting base (same pack)' ' + git repack -adk && + + test_seq 32 >f && + git add f && + test_tick && + git commit -m "delta" && + delta="$(git rev-parse HEAD)" && + + test_seq 64 >f && + test_tick && + git commit -a -m "base" && + base="$(git rev-parse HEAD)" && + + test_commit other && + + git repack -d && + + have_delta "$(git rev-parse $delta:f)" "$(git rev-parse $base:f)" && + + git multi-pack-index write --bitmap && + + cat >in <<-EOF && + $(git rev-parse other) + ^$base + EOF + + : >trace2.txt && + GIT_TRACE2_EVENT="$PWD/trace2.txt" \ + git pack-objects --stdout --delta-base-offset --revs <in >/dev/null && + + # We can only reuse the 3 objects corresponding to "other" from + # the latest pack. + # + # This is because even though we want "delta", we do not want + # "base", meaning that we have to inflate the delta/base-pair + # corresponding to the blob in commit "delta", which bypasses + # the pack-reuse mechanism. + # + # The remaining objects from the other pack are similarly not + # reused because their objects are on the uninteresting side of + # the query. + test_pack_reused 3 <trace2.txt && + test_packs_reused 1 <trace2.txt +' + +test_expect_success 'omit delta from uninteresting base (cross pack)' ' + cat >in <<-EOF && + $(git rev-parse $base) + ^$(git rev-parse $delta) + EOF + + P="$(git pack-objects --revs $packdir/pack <in)" && + + git multi-pack-index write --bitmap --preferred-pack="pack-$P.idx" && + + : >trace2.txt && + GIT_TRACE2_EVENT="$PWD/trace2.txt" \ + git pack-objects --stdout --delta-base-offset --all >/dev/null && + + packs_nr="$(find $packdir -type f -name "pack-*.pack" | wc -l)" && + objects_nr="$(git rev-list --count --all --objects)" && + + test_pack_reused $(($objects_nr - 1)) <trace2.txt && + test_packs_reused $packs_nr <trace2.txt +' + +test_done diff --git a/t/t5510-fetch.sh b/t/t5510-fetch.sh index 79592a3b0a..33d34d5ae9 100755 --- a/t/t5510-fetch.sh +++ b/t/t5510-fetch.sh @@ -803,7 +803,8 @@ test_expect_success 'fetch.writeCommitGraph with submodules' ' cd super-clone && rm -rf .git/objects/info && git -c fetch.writeCommitGraph=true fetch origin && - test_path_is_file .git/objects/info/commit-graphs/commit-graph-chain + test_path_is_file .git/objects/info/commit-graphs/commit-graph-chain && + git -c fetch.writeCommitGraph=true fetch --recurse-submodules origin ) ' diff --git a/t/t5514-fetch-multiple.sh b/t/t5514-fetch-multiple.sh index a95841dc36..25772c85c5 100755 --- a/t/t5514-fetch-multiple.sh +++ b/t/t5514-fetch-multiple.sh @@ -24,6 +24,15 @@ setup_repository () { ) } +setup_test_clone () { + test_dir="$1" && + git clone one "$test_dir" && + for r in one two three + do + git -C "$test_dir" remote add "$r" "../$r" || return 1 + done +} + test_expect_success setup ' setup_repository one && setup_repository two && @@ -209,4 +218,156 @@ test_expect_success 'git fetch --multiple --jobs=0 picks a default' ' git fetch --multiple --jobs=0) ' +create_fetch_all_expect () { + cat >expect <<-\EOF + one/main + one/side + origin/HEAD -> origin/main + origin/main + origin/side + three/another + three/main + three/side + two/another + two/main + two/side + EOF +} + +for fetch_all in true false +do + test_expect_success "git fetch --all (works with fetch.all = $fetch_all)" ' + test_dir="test_fetch_all_$fetch_all" && + setup_test_clone "$test_dir" && + ( + cd "$test_dir" && + git config fetch.all $fetch_all && + git fetch --all && + create_fetch_all_expect && + git branch -r >actual && + test_cmp expect actual + ) + ' +done + +test_expect_success 'git fetch (fetch all remotes with fetch.all = true)' ' + setup_test_clone test9 && + ( + cd test9 && + git config fetch.all true && + git fetch && + git branch -r >actual && + create_fetch_all_expect && + test_cmp expect actual + ) +' + +create_fetch_one_expect () { + cat >expect <<-\EOF + one/main + one/side + origin/HEAD -> origin/main + origin/main + origin/side + EOF +} + +test_expect_success 'git fetch one (explicit remote overrides fetch.all)' ' + setup_test_clone test10 && + ( + cd test10 && + git config fetch.all true && + git fetch one && + create_fetch_one_expect && + git branch -r >actual && + test_cmp expect actual + ) +' + +create_fetch_two_as_origin_expect () { + cat >expect <<-\EOF + origin/HEAD -> origin/main + origin/another + origin/main + origin/side + EOF +} + +test_expect_success 'git config fetch.all false (fetch only default remote)' ' + setup_test_clone test11 && + ( + cd test11 && + git config fetch.all false && + git remote set-url origin ../two && + git fetch && + create_fetch_two_as_origin_expect && + git branch -r >actual && + test_cmp expect actual + ) +' + +for fetch_all in true false +do + test_expect_success "git fetch --no-all (fetch only default remote with fetch.all = $fetch_all)" ' + test_dir="test_no_all_fetch_all_$fetch_all" && + setup_test_clone "$test_dir" && + ( + cd "$test_dir" && + git config fetch.all $fetch_all && + git remote set-url origin ../two && + git fetch --no-all && + create_fetch_two_as_origin_expect && + git branch -r >actual && + test_cmp expect actual + ) + ' +done + +test_expect_success 'git fetch --no-all (fetch only default remote without fetch.all)' ' + setup_test_clone test12 && + ( + cd test12 && + git config --unset-all fetch.all || true && + git remote set-url origin ../two && + git fetch --no-all && + create_fetch_two_as_origin_expect && + git branch -r >actual && + test_cmp expect actual + ) +' + +test_expect_success 'git fetch --all --no-all (fetch only default remote)' ' + setup_test_clone test13 && + ( + cd test13 && + git remote set-url origin ../two && + git fetch --all --no-all && + create_fetch_two_as_origin_expect && + git branch -r >actual && + test_cmp expect actual + ) +' + +test_expect_success 'git fetch --no-all one (fetch only explicit remote)' ' + setup_test_clone test14 && + ( + cd test14 && + git fetch --no-all one && + create_fetch_one_expect && + git branch -r >actual && + test_cmp expect actual + ) +' + +test_expect_success 'git fetch --no-all --all (fetch all remotes)' ' + setup_test_clone test15 && + ( + cd test15 && + git fetch --no-all --all && + create_fetch_all_expect && + git branch -r >actual && + test_cmp expect actual + ) +' + test_done diff --git a/t/t5601-clone.sh b/t/t5601-clone.sh index 47eae641f0..fb1b9c686d 100755 --- a/t/t5601-clone.sh +++ b/t/t5601-clone.sh @@ -157,6 +157,23 @@ test_expect_success 'clone --mirror does not repeat tags' ' ' +test_expect_success 'clone with files ref format' ' + test_when_finished "rm -rf ref-storage" && + git clone --ref-format=files --mirror src ref-storage && + echo files >expect && + git -C ref-storage rev-parse --show-ref-format >actual && + test_cmp expect actual +' + +test_expect_success 'clone with garbage ref format' ' + cat >expect <<-EOF && + fatal: unknown ref storage format ${SQ}garbage${SQ} + EOF + test_must_fail git clone --ref-format=garbage --mirror src ref-storage 2>err && + test_cmp expect err && + test_path_is_missing ref-storage +' + test_expect_success 'clone to destination with trailing /' ' git clone src target-1/ && diff --git a/t/t6113-rev-list-bitmap-filters.sh b/t/t6113-rev-list-bitmap-filters.sh index 86c70521f1..459f0d7412 100755 --- a/t/t6113-rev-list-bitmap-filters.sh +++ b/t/t6113-rev-list-bitmap-filters.sh @@ -4,6 +4,8 @@ test_description='rev-list combining bitmaps and filters' . ./test-lib.sh . "$TEST_DIRECTORY"/lib-bitmap.sh +TEST_PASSES_SANITIZE_LEAK=true + test_expect_success 'set up bitmapped repo' ' # one commit will have bitmaps, the other will not test_commit one && diff --git a/t/t6135-pathspec-with-attrs.sh b/t/t6135-pathspec-with-attrs.sh index 00b84ecba6..120dcd74a5 100755 --- a/t/t6135-pathspec-with-attrs.sh +++ b/t/t6135-pathspec-with-attrs.sh @@ -393,4 +393,31 @@ test_expect_success 'reading from .gitattributes in a subdirectory (3)' ' test_cmp expect actual ' +test_expect_success POSIXPERM 'pathspec with builtin_objectmode attr can be used' ' + >mode_exec_file_1 && + + git status -s ":(attr:builtin_objectmode=100644)mode_exec_*" >actual && + echo ?? mode_exec_file_1 >expect && + test_cmp expect actual && + + git add mode_exec_file_1 && + chmod +x mode_exec_file_1 && + git status -s ":(attr:builtin_objectmode=100755)mode_exec_*" >actual && + echo AM mode_exec_file_1 >expect && + test_cmp expect actual +' + +test_expect_success POSIXPERM 'builtin_objectmode attr can be excluded' ' + >mode_1_regular && + >mode_1_exec && + chmod +x mode_1_exec && + git status -s ":(exclude,attr:builtin_objectmode=100644)" "mode_1_*" >actual && + echo ?? mode_1_exec >expect && + test_cmp expect actual && + + git status -s ":(exclude,attr:builtin_objectmode=100755)" "mode_1_*" >actual && + echo ?? mode_1_regular >expect && + test_cmp expect actual +' + test_done diff --git a/t/t9500-gitweb-standalone-no-errors.sh b/t/t9500-gitweb-standalone-no-errors.sh index 0333065d4d..7679780fb8 100755 --- a/t/t9500-gitweb-standalone-no-errors.sh +++ b/t/t9500-gitweb-standalone-no-errors.sh @@ -627,6 +627,7 @@ test_expect_success \ test_expect_success 'setup' ' version=$(git config core.repositoryformatversion) && algo=$(test_might_fail git config extensions.objectformat) && + refstorage=$(test_might_fail git config extensions.refstorage) && cat >.git/config <<-\EOF && # testing noval and alternate separator [gitweb] @@ -637,6 +638,10 @@ test_expect_success 'setup' ' if test -n "$algo" then git config extensions.objectformat "$algo" + fi && + if test -n "$refstorage" + then + git config extensions.refstorage "$refstorage" fi ' diff --git a/t/test-lib-functions.sh b/t/test-lib-functions.sh index 5eb57914ab..b5eaf7fdc1 100644 --- a/t/test-lib-functions.sh +++ b/t/test-lib-functions.sh @@ -1659,6 +1659,11 @@ test_detect_hash () { test_hash_algo="${GIT_TEST_DEFAULT_HASH:-sha1}" } +# Detect the hash algorithm in use. +test_detect_ref_format () { + echo "${GIT_TEST_DEFAULT_REF_FORMAT:-files}" +} + # Load common hash metadata and common placeholder object IDs for use with # test_oid. test_oid_init () { @@ -1874,6 +1879,20 @@ test_region () { return 0 } +# Check that the given data fragment was included as part of the +# trace2-format trace on stdin. +# +# test_trace2_data <category> <key> <value> +# +# For example, to look for trace2_data_intmax("pack-objects", repo, +# "reused", N) in an invocation of "git pack-objects", run: +# +# GIT_TRACE2_EVENT="$(pwd)/trace.txt" git pack-objects ... && +# test_trace2_data pack-objects reused N <trace2.txt +test_trace2_data () { + grep -e '"category":"'"$1"'","key":"'"$2"'","value":"'"$3"'"' +} + # Given a GIT_TRACE2_EVENT log over stdin, writes to stdout a list of URLs # sent to git-remote-https child processes. test_remote_https_urls() { diff --git a/t/test-lib.sh b/t/test-lib.sh index 876b99562a..fc93aa57e6 100644 --- a/t/test-lib.sh +++ b/t/test-lib.sh @@ -542,6 +542,8 @@ export EDITOR GIT_DEFAULT_HASH="${GIT_TEST_DEFAULT_HASH:-sha1}" export GIT_DEFAULT_HASH +GIT_DEFAULT_REF_FORMAT="${GIT_TEST_DEFAULT_REF_FORMAT:-files}" +export GIT_DEFAULT_REF_FORMAT GIT_TEST_MERGE_ALGORITHM="${GIT_TEST_MERGE_ALGORITHM:-ort}" export GIT_TEST_MERGE_ALGORITHM @@ -1745,7 +1747,14 @@ parisc* | hppa*) ;; esac -test_set_prereq REFFILES +case "$GIT_DEFAULT_REF_FORMAT" in +files) + test_set_prereq REFFILES;; +*) + echo 2>&1 "error: unknown ref format $GIT_DEFAULT_REF_FORMAT" + exit 1 + ;; +esac ( COLUMNS=1 && test $COLUMNS = 1 ) && test_set_prereq COLUMNS_CAN_BE_1 test -z "$NO_CURL" && test_set_prereq LIBCURL @@ -1936,6 +1945,10 @@ test_lazy_prereq SHA1 ' esac ' +test_lazy_prereq DEFAULT_REPO_FORMAT ' + test_have_prereq SHA1,REFFILES +' + # Ensure that no test accidentally triggers a Git command # that runs the actual maintenance scheduler, affecting a user's # system permanently. diff --git a/worktree.c b/worktree.c index 1399d452ac..dbc670d369 100644 --- a/worktree.c +++ b/worktree.c @@ -51,7 +51,7 @@ static void add_head_info(struct worktree *wt) /** * get the main worktree */ -static struct worktree *get_main_worktree(void) +static struct worktree *get_main_worktree(int skip_reading_head) { struct worktree *worktree = NULL; struct strbuf worktree_path = STRBUF_INIT; @@ -70,11 +70,13 @@ static struct worktree *get_main_worktree(void) */ worktree->is_bare = (is_bare_repository_cfg == 1) || is_bare_repository(); - add_head_info(worktree); + if (!skip_reading_head) + add_head_info(worktree); return worktree; } -static struct worktree *get_linked_worktree(const char *id) +static struct worktree *get_linked_worktree(const char *id, + int skip_reading_head) { struct worktree *worktree = NULL; struct strbuf path = STRBUF_INIT; @@ -93,7 +95,8 @@ static struct worktree *get_linked_worktree(const char *id) CALLOC_ARRAY(worktree, 1); worktree->path = strbuf_detach(&worktree_path, NULL); worktree->id = xstrdup(id); - add_head_info(worktree); + if (!skip_reading_head) + add_head_info(worktree); done: strbuf_release(&path); @@ -118,7 +121,14 @@ static void mark_current_worktree(struct worktree **worktrees) free(git_dir); } -struct worktree **get_worktrees(void) +/* + * NEEDSWORK: This function exists so that we can look up metadata of a + * worktree without trying to access any of its internals like the refdb. It + * would be preferable to instead have a corruption-tolerant function for + * retrieving worktree metadata that could be used when the worktree is known + * to not be in a healthy state, e.g. when creating or repairing it. + */ +static struct worktree **get_worktrees_internal(int skip_reading_head) { struct worktree **list = NULL; struct strbuf path = STRBUF_INIT; @@ -128,7 +138,7 @@ struct worktree **get_worktrees(void) ALLOC_ARRAY(list, alloc); - list[counter++] = get_main_worktree(); + list[counter++] = get_main_worktree(skip_reading_head); strbuf_addf(&path, "%s/worktrees", get_git_common_dir()); dir = opendir(path.buf); @@ -137,7 +147,7 @@ struct worktree **get_worktrees(void) while ((d = readdir_skip_dot_and_dotdot(dir)) != NULL) { struct worktree *linked = NULL; - if ((linked = get_linked_worktree(d->d_name))) { + if ((linked = get_linked_worktree(d->d_name, skip_reading_head))) { ALLOC_GROW(list, counter + 1, alloc); list[counter++] = linked; } @@ -151,6 +161,11 @@ struct worktree **get_worktrees(void) return list; } +struct worktree **get_worktrees(void) +{ + return get_worktrees_internal(0); +} + const char *get_worktree_git_dir(const struct worktree *wt) { if (!wt) @@ -591,7 +606,7 @@ static void repair_noop(int iserr UNUSED, void repair_worktrees(worktree_repair_fn fn, void *cb_data) { - struct worktree **worktrees = get_worktrees(); + struct worktree **worktrees = get_worktrees_internal(1); struct worktree **wt = worktrees + 1; /* +1 skips main worktree */ if (!fn) diff --git a/write-or-die.c b/write-or-die.c index 42a2dc73cd..3942152865 100644 --- a/write-or-die.c +++ b/write-or-die.c @@ -19,20 +19,17 @@ void maybe_flush_or_die(FILE *f, const char *desc) { static int skip_stdout_flush = -1; - struct stat st; - char *cp; if (f == stdout) { if (skip_stdout_flush < 0) { - /* NEEDSWORK: make this a normal Boolean */ - cp = getenv("GIT_FLUSH"); - if (cp) - skip_stdout_flush = (atoi(cp) == 0); - else if ((fstat(fileno(stdout), &st) == 0) && - S_ISREG(st.st_mode)) - skip_stdout_flush = 1; - else - skip_stdout_flush = 0; + skip_stdout_flush = git_env_bool("GIT_FLUSH", -1); + if (skip_stdout_flush < 0) { + struct stat st; + if (fstat(fileno(stdout), &st)) + skip_stdout_flush = 0; + else + skip_stdout_flush = S_ISREG(st.st_mode); + } } if (skip_stdout_flush && !ferror(f)) return; |