diff options
290 files changed, 8120 insertions, 2948 deletions
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9fdbd54028..4d97da57ec 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -309,6 +309,17 @@ jobs: with: name: failed-tests-${{matrix.vector.jobname}} path: ${{env.FAILED_TEST_ARTIFACTS}} + fuzz-smoke-test: + name: fuzz smoke test + needs: ci-config + if: needs.ci-config.outputs.enabled == 'yes' + env: + CC: clang + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - run: ci/install-dependencies.sh + - run: ci/run-build-and-minimal-fuzzers.sh dockerized: name: ${{matrix.vector.jobname}} (${{matrix.vector.image}}) needs: ci-config diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cd98bcb18a..43bfbd8834 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,7 +7,7 @@ workflow: - if: $CI_COMMIT_TAG - if: $CI_COMMIT_REF_PROTECTED == "true" -test: +test:linux: image: $image before_script: - ./ci/install-docker-dependencies.sh @@ -51,3 +51,45 @@ test: paths: - t/failed-test-artifacts when: on_failure + +test:osx: + image: $image + tags: + - saas-macos-medium-m1 + variables: + TEST_OUTPUT_DIRECTORY: "/Volumes/RAMDisk" + before_script: + # Create a 4GB RAM disk that we use to store test output on. This small hack + # significantly speeds up tests by more than a factor of 2 because the + # macOS runners use network-attached storage as disks, which is _really_ + # slow with the many small writes that our tests do. + - sudo diskutil apfs create $(hdiutil attach -nomount ram://8192000) RAMDisk + - ./ci/install-dependencies.sh + script: + - ./ci/run-build-and-tests.sh + after_script: + - | + if test "$CI_JOB_STATUS" != 'success' + then + ./ci/print-test-failures.sh + mv "$TEST_OUTPUT_DIRECTORY"/failed-test-artifacts t/ + fi + parallel: + matrix: + - jobname: osx-clang + image: macos-13-xcode-14 + CC: clang + artifacts: + paths: + - t/failed-test-artifacts + when: on_failure + +static-analysis: + image: ubuntu:22.04 + variables: + jobname: StaticAnalysis + before_script: + - ./ci/install-docker-dependencies.sh + script: + - ./ci/run-static-analysis.sh + - ./ci/check-directional-formatting.bash diff --git a/Documentation/CodingGuidelines b/Documentation/CodingGuidelines index 8ed517a5ca..578587a471 100644 --- a/Documentation/CodingGuidelines +++ b/Documentation/CodingGuidelines @@ -450,7 +450,7 @@ For C programs: one of the approved headers that includes it first for you. (The approved headers currently include "builtin.h", "t/helper/test-tool.h", "xdiff/xinclude.h", or - "reftable/system.h"). You do not have to include more than one of + "reftable/system.h".) You do not have to include more than one of these. - A C file must directly include the header files that declare the @@ -578,7 +578,7 @@ Externally Visible Names . The variable name describes the effect of tweaking this knob. The section and variable names that consist of multiple words are - formed by concatenating the words without punctuations (e.g. `-`), + formed by concatenating the words without punctuation marks (e.g. `-`), and are broken using bumpyCaps in documentation as a hint to the reader. diff --git a/Documentation/MyFirstContribution.txt b/Documentation/MyFirstContribution.txt index 279f6a3e7c..f06563e981 100644 --- a/Documentation/MyFirstContribution.txt +++ b/Documentation/MyFirstContribution.txt @@ -35,8 +35,9 @@ announcements, design discussions, and more take place. Those interested in contributing are welcome to post questions here. The Git list requires plain-text-only emails and prefers inline and bottom-posting when replying to mail; you will be CC'd in all replies to you. Optionally, you can subscribe to -the list by sending an email to majordomo@vger.kernel.org with "subscribe git" -in the body. The https://lore.kernel.org/git[archive] of this mailing list is +the list by sending an email to <git+subscribe@vger.kernel.org> +(see https://subspace.kernel.org/subscribing.html for details). +The https://lore.kernel.org/git[archive] of this mailing list is available to view in a browser. ==== https://groups.google.com/forum/#!forum/git-mentoring[git-mentoring@googlegroups.com] diff --git a/Documentation/RelNotes/2.44.0.txt b/Documentation/RelNotes/2.44.0.txt index e58095fc8d..4dda977fc3 100644 --- a/Documentation/RelNotes/2.44.0.txt +++ b/Documentation/RelNotes/2.44.0.txt @@ -32,7 +32,19 @@ UI, Workflows & Features breaking change. "--ignore-other-worktrees" option is required to unbreak you, if you are used to the current behaviour that "-B" overrides the safety. - (merge b23285a921 jc/checkout-B-branch-in-use later to maint). + + * The builtin_objectmode attribute is populated for each path + without adding anything in .gitattributes files, which would be + useful in magic pathspec, e.g., ":(attr:builtin_objectmode=100755)" + to limit to executables. + + * "git fetch" learned to pay attention to "fetch.all" configuration + variable, which pretends as if "--all" was passed from the command + line when no remote parameter was given. + + * In addition to (rather cryptic) Security Identifiers, show username + and domain in the error message when we barf on mismatch between + the Git directory and the current user on Windows. Performance, Internal Implementation, Development Support etc. @@ -64,6 +76,10 @@ Performance, Internal Implementation, Development Support etc. non-default hash function into a repository that uses the reftable backend. + * Streaming spans of packfile data used to be done only from a + single, primary, pack in a repository with multiple packfiles. It + has been extended to allow reuse from other packfiles, too. + Fixes since v2.43 ----------------- @@ -154,6 +170,17 @@ Fixes since v2.43 * "git sparse-checkout (add|set) --[no-]cone --end-of-options" did not handle "--end-of-options" correctly after a recent update. + * Unlike other environment variables that took the usual + true/false/yes/no as well as 0/1, GIT_FLUSH only understood 0/1, + which has been corrected. + (merge 556e68032f cp/git-flush-is-an-env-bool later to maint). + + * Clearing in-core repository (happens during e.g., "git fetch + --recurse-submodules" with commit graph enabled) made in-core + commit object in an inconsistent state by discarding the necessary + data from commit-graph too early, which has been corrected. + (merge d70f554cdf jk/commit-graph-slab-clear-fix later to maint). + * Other code cleanup, docfix, build fix, etc. (merge 50f1abcff6 js/packfile-h-typofix later to maint). (merge cbf498eb53 jb/reflog-expire-delete-dry-run-options later to maint). @@ -178,3 +205,12 @@ Fixes since v2.43 (merge 63956c553d ml/doc-merge-updates later to maint). (merge d57c671a51 en/header-cleanup later to maint). (merge 5b7eec4bc5 rs/fast-import-simplify-mempool-allocation later to maint). + (merge 291873e5d6 js/contributor-docs-updates later to maint). + (merge 54d8a2531b jk/t1006-cat-file-objectsize-disk later to maint). + (merge 7033d5479b jx/sideband-chomp-newline-fix later to maint). + (merge 9cd30af991 ms/rebase-insnformat-doc-fix later to maint). + (merge 03bcc93769 cp/sideband-array-index-comment-fix later to maint). + (merge 993d38a066 jk/index-pack-lsan-false-positive-fix later to maint). + (merge 25aec06326 ib/rebase-reschedule-doc later to maint). + (merge 5aea3955bc rj/clarify-branch-doc-m later to maint). + (merge 9cce3be2df bk/bisect-doc-fix later to maint). diff --git a/Documentation/SubmittingPatches b/Documentation/SubmittingPatches index bce7f97815..e734a3f0f1 100644 --- a/Documentation/SubmittingPatches +++ b/Documentation/SubmittingPatches @@ -355,9 +355,21 @@ If you like, you can put extra tags at the end: patch after a detailed analysis. . `Tested-by:` is used to indicate that the person applied the patch and found it to have the desired effect. - -You can also create your own tag or use one that's in common usage -such as "Thanks-to:", "Based-on-patch-by:", or "Mentored-by:". +. `Co-authored-by:` is used to indicate that people exchanged drafts + of a patch before submitting it. +. `Helped-by:` is used to credit someone who suggested ideas for + changes without providing the precise changes in patch form. +. `Mentored-by:` is used to credit someone with helping develop a + patch as part of a mentorship program (e.g., GSoC or Outreachy). +. `Suggested-by:` is used to credit someone with suggesting the idea + for a patch. + +While you can also create your own trailer if the situation warrants it, we +encourage you to instead use one of the common trailers in this project +highlighted above. + +Only capitalize the very first letter of tags, i.e. favor +"Signed-off-by" over "Signed-Off-By" and "Acked-by:" over "Acked-By". [[git-tools]] === Generate your patch using Git tools out of your commits. @@ -570,7 +582,7 @@ their trees themselves. master). * Read the Git mailing list, the maintainer regularly posts messages - entitled "What's cooking in git.git" and "What's in git.git" giving + entitled "What's cooking in git.git" giving the status of various proposed changes. == GitHub CI[[GHCI]] @@ -590,11 +602,12 @@ After the initial setup, CI will run whenever you push new changes to your fork of Git on GitHub. You can monitor the test state of all your branches here: `https://github.com/<Your GitHub handle>/git/actions/workflows/main.yml` -If a branch did not pass all test cases then it is marked with a red -cross. In that case you can click on the failing job and navigate to -"ci/run-build-and-tests.sh" and/or "ci/print-test-failures.sh". You -can also download "Artifacts" which are tarred (or zipped) archives -with test data relevant for debugging. +If a branch does not pass all test cases then it will be marked with a +red +x+, instead of a green check. In that case, you can click on the +failing job and navigate to "ci/run-build-and-tests.sh" and/or +"ci/print-test-failures.sh". You can also download "Artifacts" which +are zip archives containing tarred (or zipped) archives with test data +relevant for debugging. Then fix the problem and push your fix to your GitHub fork. This will trigger a new CI build to ensure all tests pass. @@ -686,7 +699,7 @@ message to an external program, and this is a handy way to drive `git am`. However, if the message is MIME encoded, what is piped into the program is the representation you see in your `*Article*` buffer after unwrapping MIME. This is often not what -you would want for two reasons. It tends to screw up non ASCII +you would want for two reasons. It tends to screw up non-ASCII characters (most notably in people's names), and also whitespaces (fatal in patches). Running "C-u g" to display the message in raw form before using "|" to run the pipe can work diff --git a/Documentation/config/advice.txt b/Documentation/config/advice.txt index 4d7e5d8759..c7ea70f2e2 100644 --- a/Documentation/config/advice.txt +++ b/Documentation/config/advice.txt @@ -1,30 +1,63 @@ advice.*:: These variables control various optional help messages designed to - aid new users. All 'advice.*' variables default to 'true', and you - can tell Git that you do not need help by setting these to 'false': + aid new users. When left unconfigured, Git will give the message + alongside instructions on how to squelch it. You can tell Git + that you do not need the help message by setting these to 'false': + -- + addEmbeddedRepo:: + Advice on what to do when you've accidentally added one + git repo inside of another. + addEmptyPathspec:: + Advice shown if a user runs the add command without providing + the pathspec parameter. + addIgnoredFile:: + Advice shown if a user attempts to add an ignored file to + the index. + amWorkDir:: + Advice that shows the location of the patch file when + linkgit:git-am[1] fails to apply it. ambiguousFetchRefspec:: Advice shown when a fetch refspec for multiple remotes maps to the same remote-tracking branch namespace and causes branch tracking set-up to fail. + checkoutAmbiguousRemoteBranchName:: + Advice shown when the argument to + linkgit:git-checkout[1] and linkgit:git-switch[1] + ambiguously resolves to a + remote tracking branch on more than one remote in + situations where an unambiguous argument would have + otherwise caused a remote-tracking branch to be + checked out. See the `checkout.defaultRemote` + configuration variable for how to set a given remote + to be used by default in some situations where this + advice would be printed. + commitBeforeMerge:: + Advice shown when linkgit:git-merge[1] refuses to + merge to avoid overwriting local changes. + detachedHead:: + Advice shown when you used + linkgit:git-switch[1] or linkgit:git-checkout[1] + to move to the detached HEAD state, to instruct how to + create a local branch after the fact. + diverging:: + Advice shown when a fast-forward is not possible. fetchShowForcedUpdates:: Advice shown when linkgit:git-fetch[1] takes a long time to calculate forced updates after ref updates, or to warn that the check is disabled. - pushUpdateRejected:: - Set this variable to 'false' if you want to disable - 'pushNonFFCurrent', 'pushNonFFMatching', 'pushAlreadyExists', - 'pushFetchFirst', 'pushNeedsForce', and 'pushRefNeedsUpdate' - simultaneously. - pushNonFFCurrent:: - Advice shown when linkgit:git-push[1] fails due to a - non-fast-forward update to the current branch. - pushNonFFMatching:: - Advice shown when you ran linkgit:git-push[1] and pushed - 'matching refs' explicitly (i.e. you used ':', or - specified a refspec that isn't your current branch) and - it resulted in a non-fast-forward error. + forceDeleteBranch:: + Advice shown when a user tries to delete a not fully merged + branch without the force option set. + ignoredHook:: + Advice shown if a hook is ignored because the hook is not + set as executable. + implicitIdentity:: + Advice on how to set your identity configuration when + your information is guessed from the system username and + domain name. + nestedTag:: + Advice shown if a user attempts to recursively tag a tag object. pushAlreadyExists:: Shown when linkgit:git-push[1] rejects an update that does not qualify for fast-forwarding (e.g., a tag.) @@ -37,6 +70,18 @@ advice.*:: tries to overwrite a remote ref that points at an object that is not a commit-ish, or make the remote ref point at an object that is not a commit-ish. + pushNonFFCurrent:: + Advice shown when linkgit:git-push[1] fails due to a + non-fast-forward update to the current branch. + pushNonFFMatching:: + Advice shown when you ran linkgit:git-push[1] and pushed + 'matching refs' explicitly (i.e. you used ':', or + specified a refspec that isn't your current branch) and + it resulted in a non-fast-forward error. + pushRefNeedsUpdate:: + Shown when linkgit:git-push[1] rejects a forced update of + a branch when its remote-tracking ref has updates that we + do not have locally. pushUnqualifiedRefname:: Shown when linkgit:git-push[1] gives up trying to guess based on the source and destination refs what @@ -44,10 +89,23 @@ advice.*:: we can still suggest that the user push to either refs/heads/* or refs/tags/* based on the type of the source object. - pushRefNeedsUpdate:: - Shown when linkgit:git-push[1] rejects a forced update of - a branch when its remote-tracking ref has updates that we - do not have locally. + pushUpdateRejected:: + Set this variable to 'false' if you want to disable + 'pushNonFFCurrent', 'pushNonFFMatching', 'pushAlreadyExists', + 'pushFetchFirst', 'pushNeedsForce', and 'pushRefNeedsUpdate' + simultaneously. + resetNoRefresh:: + Advice to consider using the `--no-refresh` option to + linkgit:git-reset[1] when the command takes more than 2 seconds + to refresh the index after reset. + resolveConflict:: + Advice shown by various commands when conflicts + prevent the operation from being performed. + rmHints:: + In case of failure in the output of linkgit:git-rm[1], + show directions on how to proceed from the current state. + sequencerInUse:: + Advice shown when a sequencer command is already in progress. skippedCherryPicks:: Shown when linkgit:git-rebase[1] skips a commit that has already been cherry-picked onto the upstream branch. @@ -68,76 +126,22 @@ advice.*:: Advise to consider using the `-u` option to linkgit:git-status[1] when the command takes more than 2 seconds to enumerate untracked files. - commitBeforeMerge:: - Advice shown when linkgit:git-merge[1] refuses to - merge to avoid overwriting local changes. - resetNoRefresh:: - Advice to consider using the `--no-refresh` option to - linkgit:git-reset[1] when the command takes more than 2 seconds - to refresh the index after reset. - resolveConflict:: - Advice shown by various commands when conflicts - prevent the operation from being performed. - sequencerInUse:: - Advice shown when a sequencer command is already in progress. - implicitIdentity:: - Advice on how to set your identity configuration when - your information is guessed from the system username and - domain name. - detachedHead:: - Advice shown when you used - linkgit:git-switch[1] or linkgit:git-checkout[1] - to move to the detached HEAD state, to instruct how to - create a local branch after the fact. - suggestDetachingHead:: - Advice shown when linkgit:git-switch[1] refuses to detach HEAD - without the explicit `--detach` option. - checkoutAmbiguousRemoteBranchName:: - Advice shown when the argument to - linkgit:git-checkout[1] and linkgit:git-switch[1] - ambiguously resolves to a - remote tracking branch on more than one remote in - situations where an unambiguous argument would have - otherwise caused a remote-tracking branch to be - checked out. See the `checkout.defaultRemote` - configuration variable for how to set a given remote - to be used by default in some situations where this - advice would be printed. - amWorkDir:: - Advice that shows the location of the patch file when - linkgit:git-am[1] fails to apply it. - rmHints:: - In case of failure in the output of linkgit:git-rm[1], - show directions on how to proceed from the current state. - addEmbeddedRepo:: - Advice on what to do when you've accidentally added one - git repo inside of another. - ignoredHook:: - Advice shown if a hook is ignored because the hook is not - set as executable. - waitingForEditor:: - Print a message to the terminal whenever Git is waiting for - editor input from the user. - nestedTag:: - Advice shown if a user attempts to recursively tag a tag object. submoduleAlternateErrorStrategyDie:: Advice shown when a submodule.alternateErrorStrategy option configured to "die" causes a fatal error. submodulesNotUpdated:: Advice shown when a user runs a submodule command that fails because `git submodule update --init` was not run. - addIgnoredFile:: - Advice shown if a user attempts to add an ignored file to - the index. - addEmptyPathspec:: - Advice shown if a user runs the add command without providing - the pathspec parameter. + suggestDetachingHead:: + Advice shown when linkgit:git-switch[1] refuses to detach HEAD + without the explicit `--detach` option. updateSparsePath:: Advice shown when either linkgit:git-add[1] or linkgit:git-rm[1] is asked to update index entries outside the current sparse checkout. - diverging:: - Advice shown when a fast-forward is not possible. + waitingForEditor:: + Print a message to the terminal whenever Git is waiting for + editor input from the user. worktreeAddOrphan:: Advice shown when a user tries to create a worktree from an invalid reference, to instruct how to create a new unborn diff --git a/Documentation/config/color.txt b/Documentation/config/color.txt index 2f2275ac69..cf8b03add4 100644 --- a/Documentation/config/color.txt +++ b/Documentation/config/color.txt @@ -74,10 +74,34 @@ color.diff.<slot>:: `oldBold`, and `newBold` (see linkgit:git-range-diff[1] for details). color.decorate.<slot>:: - Use customized color for 'git log --decorate' output. `<slot>` is one - of `branch`, `remoteBranch`, `tag`, `stash` or `HEAD` for local - branches, remote-tracking branches, tags, stash and HEAD, respectively - and `grafted` for grafted commits. + Use customized color for the output of 'git log --decorate' as well as + the `%d`, `%D` and `%(decorate)` placeholders in custom log formats, + whereby `<slot>` specifies which decoration elements the color applies + to: ++ +-- +`HEAD`;; + the current HEAD +`branch`;; + local branches +`remoteBranch`;; + remote-tracking branches +`tag`;; + lightweight and annotated tags +`stash`;; + the stash ref +`ref`;; + any other refs (not shown by default) +`pseudoref`;; + pseudorefs such as ORIG_HEAD or MERGE_HEAD (not shown by default) +`grafted`;; + grafted and replaced commits +`symbol`;; + punctuation symbols surrounding the other elements +-- ++ +(Variable `log.initialDecorationSet` or linkgit:git-log[1] option +`--clear-decorations` can be used to show all refs and pseudorefs.) color.grep:: When set to `always`, always highlight matches. When `false` (or diff --git a/Documentation/config/commitgraph.txt b/Documentation/config/commitgraph.txt index 30604e4a4c..7f8c9d6638 100644 --- a/Documentation/config/commitgraph.txt +++ b/Documentation/config/commitgraph.txt @@ -9,6 +9,29 @@ commitGraph.maxNewFilters:: commit-graph write` (c.f., linkgit:git-commit-graph[1]). commitGraph.readChangedPaths:: - If true, then git will use the changed-path Bloom filters in the - commit-graph file (if it exists, and they are present). Defaults to - true. See linkgit:git-commit-graph[1] for more information. + Deprecated. Equivalent to commitGraph.changedPathsVersion=-1 if true, and + commitGraph.changedPathsVersion=0 if false. (If commitGraph.changedPathVersion + is also set, commitGraph.changedPathsVersion takes precedence.) + +commitGraph.changedPathsVersion:: + Specifies the version of the changed-path Bloom filters that Git will read and + write. May be -1, 0, 1, or 2. Note that values greater than 1 may be + incompatible with older versions of Git which do not yet understand + those versions. Use caution when operating in a mixed-version + environment. ++ +Defaults to -1. ++ +If -1, Git will use the version of the changed-path Bloom filters in the +repository, defaulting to 1 if there are none. ++ +If 0, Git will not read any Bloom filters, and will write version 1 Bloom +filters when instructed to write. ++ +If 1, Git will only read version 1 Bloom filters, and will write version 1 +Bloom filters. ++ +If 2, Git will only read version 2 Bloom filters, and will write version 2 +Bloom filters. ++ +See linkgit:git-commit-graph[1] for more information. diff --git a/Documentation/config/extensions.txt b/Documentation/config/extensions.txt index bccaec7a96..38dce3df35 100644 --- a/Documentation/config/extensions.txt +++ b/Documentation/config/extensions.txt @@ -7,6 +7,29 @@ Note that this setting should only be set by linkgit:git-init[1] or linkgit:git-clone[1]. Trying to change it after initialization will not work and will produce hard-to-diagnose issues. +extensions.compatObjectFormat:: + + Specify a compatitbility hash algorithm to use. The acceptable values + are `sha1` and `sha256`. The value specified must be different from the + value of extensions.objectFormat. This allows client level + interoperability between git repositories whose objectFormat matches + this compatObjectFormat. In particular when fully implemented the + pushes and pulls from a repository in whose objectFormat matches + compatObjectFormat. As well as being able to use oids encoded in + compatObjectFormat in addition to oids encoded with objectFormat to + locally specify objects. + +extensions.refStorage:: + Specify the ref storage format to use. The acceptable values are: ++ +include::../ref-storage-format.txt[] ++ +It is an error to specify this key unless `core.repositoryFormatVersion` is 1. ++ +Note that this setting should only be set by linkgit:git-init[1] or +linkgit:git-clone[1]. Trying to change it after initialization will not +work and will produce hard-to-diagnose issues. + extensions.worktreeConfig:: If enabled, then worktrees will load config settings from the `$GIT_DIR/config.worktree` file in addition to the diff --git a/Documentation/config/fetch.txt b/Documentation/config/fetch.txt index aea5b97477..d7dc461bd1 100644 --- a/Documentation/config/fetch.txt +++ b/Documentation/config/fetch.txt @@ -50,6 +50,12 @@ fetch.pruneTags:: refs. See also `remote.<name>.pruneTags` and the PRUNING section of linkgit:git-fetch[1]. +fetch.all:: + If true, fetch will attempt to update all available remotes. + This behavior can be overridden by passing `--no-all` or by + explicitly specifying one or more remote(s) to fetch from. + Defaults to false. + fetch.output:: Control how ref update status is printed. Valid values are `full` and `compact`. Default value is `full`. See the diff --git a/Documentation/config/pack.txt b/Documentation/config/pack.txt index f50df9dbce..9c630863e6 100644 --- a/Documentation/config/pack.txt +++ b/Documentation/config/pack.txt @@ -28,11 +28,17 @@ all existing objects. You can force recompression by passing the -F option to linkgit:git-repack[1]. pack.allowPackReuse:: - When true, and when reachability bitmaps are enabled, - pack-objects will try to send parts of the bitmapped packfile - verbatim. This can reduce memory and CPU usage to serve fetches, - but might result in sending a slightly larger pack. Defaults to - true. + When true or "single", and when reachability bitmaps are + enabled, pack-objects will try to send parts of the bitmapped + packfile verbatim. When "multi", and when a multi-pack + reachability bitmap is available, pack-objects will try to send + parts of all packs in the MIDX. ++ + If only a single pack bitmap is available, and + `pack.allowPackReuse` is set to "multi", reuse parts of just the + bitmapped packfile. This can reduce memory and CPU usage to + serve fetches, but might result in sending a slightly larger + pack. Defaults to true. pack.island:: An extended regular expression configuring a set of delta diff --git a/Documentation/config/rebase.txt b/Documentation/config/rebase.txt index d59576dbb2..c6187ab28b 100644 --- a/Documentation/config/rebase.txt +++ b/Documentation/config/rebase.txt @@ -40,7 +40,7 @@ rebase.missingCommitsCheck:: rebase.instructionFormat:: A format string, as specified in linkgit:git-log[1], to be used for the todo list during an interactive rebase. The format will - automatically have the long commit hash prepended to the format. + automatically have the commit hash prepended to the format. rebase.abbreviateCommands:: If set to true, `git rebase` will use abbreviated command names in the diff --git a/Documentation/diff-options.txt b/Documentation/diff-options.txt index 53ec3c9a34..aaaff0d46f 100644 --- a/Documentation/diff-options.txt +++ b/Documentation/diff-options.txt @@ -299,7 +299,7 @@ and accumulating child directory counts in the parent directories: Synonym for --dirstat=cumulative --dirstat-by-file[=<param1,param2>...]:: - Synonym for --dirstat=files,param1,param2... + Synonym for --dirstat=files,<param1>,<param2>... --summary:: Output a condensed summary of extended header information diff --git a/Documentation/fetch-options.txt b/Documentation/fetch-options.txt index a1d6633a4f..54ebb4452e 100644 --- a/Documentation/fetch-options.txt +++ b/Documentation/fetch-options.txt @@ -1,5 +1,6 @@ ---all:: - Fetch all remotes. +--[no-]all:: + Fetch all remotes. This overrides the configuration variable + `fetch.all`. -a:: --append:: diff --git a/Documentation/git-bisect.txt b/Documentation/git-bisect.txt index aa02e46224..fcb42ca34d 100644 --- a/Documentation/git-bisect.txt +++ b/Documentation/git-bisect.txt @@ -17,10 +17,10 @@ The command takes various subcommands, and different options depending on the subcommand: git bisect start [--term-(new|bad)=<term-new> --term-(old|good)=<term-old>] - [--no-checkout] [--first-parent] [<bad> [<good>...]] [--] [<paths>...] + [--no-checkout] [--first-parent] [<bad> [<good>...]] [--] [<pathspec>...] git bisect (bad|new|<term-new>) [<rev>] git bisect (good|old|<term-old>) [<rev>...] - git bisect terms [--term-good | --term-bad] + git bisect terms [--term-(good|old) | --term-(bad|new)] git bisect skip [(<rev>|<range>)...] git bisect reset [<commit>] git bisect (visualize|view) @@ -165,8 +165,10 @@ To get a reminder of the currently used terms, use git bisect terms ------------------------------------------------ -You can get just the old (respectively new) term with `git bisect terms ---term-old` or `git bisect terms --term-good`. +You can get just the old term with `git bisect terms --term-old` +or `git bisect terms --term-good`; `git bisect terms --term-new` +and `git bisect terms --term-bad` can be used to learn how to call +the commits more recent than the sought change. If you would like to use your own terms instead of "bad"/"good" or "new"/"old", you can choose any names you like (except existing bisect @@ -299,7 +301,7 @@ Cutting down bisection by giving more parameters to bisect start You can further cut down the number of trials, if you know what part of the tree is involved in the problem you are tracking down, by specifying -path parameters when issuing the `bisect start` command: +pathspec parameters when issuing the `bisect start` command: ------------ $ git bisect start -- arch/i386 include/asm-i386 diff --git a/Documentation/git-blame.txt b/Documentation/git-blame.txt index 5720d04ffe..b1d7fb539d 100644 --- a/Documentation/git-blame.txt +++ b/Documentation/git-blame.txt @@ -210,7 +210,7 @@ annotated. . Each blame entry always starts with a line of: - <40-byte hex sha1> <sourceline> <resultline> <num_lines> + <40-byte-hex-sha1> <sourceline> <resultline> <num-lines> + Line numbers count from 1. diff --git a/Documentation/git-branch.txt b/Documentation/git-branch.txt index 4395aa9354..0b08442932 100644 --- a/Documentation/git-branch.txt +++ b/Documentation/git-branch.txt @@ -312,7 +312,8 @@ superproject's "origin/main", but tracks the submodule's "origin/main". option is omitted, the current HEAD will be used instead. <oldbranch>:: - The name of an existing branch to rename. + The name of an existing branch. If this option is omitted, + the name of the current branch will be used instead. <newbranch>:: The new name for an existing branch. The same restrictions as for diff --git a/Documentation/git-bugreport.txt b/Documentation/git-bugreport.txt index 392d9eb6ae..ca626f7fc6 100644 --- a/Documentation/git-bugreport.txt +++ b/Documentation/git-bugreport.txt @@ -52,7 +52,7 @@ OPTIONS -s <format>:: --suffix <format>:: Specify an alternate suffix for the bugreport name, to create a file - named 'git-bugreport-<formatted suffix>'. This should take the form of a + named 'git-bugreport-<formatted-suffix>'. This should take the form of a strftime(3) format string; the current local time will be used. --no-diagnose:: @@ -60,7 +60,7 @@ OPTIONS Create a zip archive of supplemental information about the user's machine, Git client, and repository state. The archive is written to the same output directory as the bug report and is named - 'git-diagnostics-<formatted suffix>'. + 'git-diagnostics-<formatted-suffix>'. + Without `mode` specified, the diagnostic archive will contain the default set of statistics reported by `git diagnose`. An optional `mode` value may be specified diff --git a/Documentation/git-clone.txt b/Documentation/git-clone.txt index c37c4a37f7..6e43eb9c20 100644 --- a/Documentation/git-clone.txt +++ b/Documentation/git-clone.txt @@ -311,6 +311,12 @@ or `--mirror` is given) The result is Git repository can be separated from working tree. +--ref-format=<ref-format:: + +Specify the given ref storage format for the repository. The valid values are: ++ +include::ref-storage-format.txt[] + -j <n>:: --jobs <n>:: The number of submodules fetched at the same time. diff --git a/Documentation/git-commit-graph.txt b/Documentation/git-commit-graph.txt index c8dbceba01..903b16830e 100644 --- a/Documentation/git-commit-graph.txt +++ b/Documentation/git-commit-graph.txt @@ -13,7 +13,7 @@ SYNOPSIS 'git commit-graph write' [--object-dir <dir>] [--append] [--split[=<strategy>]] [--reachable | --stdin-packs | --stdin-commits] [--changed-paths] [--[no-]max-new-filters <n>] [--[no-]progress] - <split options> + <split-options> DESCRIPTION diff --git a/Documentation/git-config.txt b/Documentation/git-config.txt index b1caac887a..dff39093b5 100644 --- a/Documentation/git-config.txt +++ b/Documentation/git-config.txt @@ -103,11 +103,11 @@ OPTIONS names are not. --get-urlmatch <name> <URL>:: - When given a two-part name section.key, the value for - section.<URL>.key whose <URL> part matches the best to the + When given a two-part <name> as <section>.<key>, the value for + <section>.<URL>.<key> whose <URL> part matches the best to the given URL is returned (if no such key exists, the value for - section.key is used as a fallback). When given just the - section as name, do so for all the keys in the section and + <section>.<key> is used as a fallback). When given just the + <section> as name, do so for all the keys in the section and list them. Returns error code 1 if no value is found. --global:: diff --git a/Documentation/git-cvsserver.txt b/Documentation/git-cvsserver.txt index cf4a5a283e..4c475efeab 100644 --- a/Documentation/git-cvsserver.txt +++ b/Documentation/git-cvsserver.txt @@ -197,7 +197,7 @@ allowing access over SSH. 5. Clients should now be able to check out the project. Use the CVS 'module' name to indicate what Git 'head' you want to check out. This also sets the name of your newly checked-out directory, unless you tell it otherwise with - `-d <dir_name>`. For example, this checks out 'master' branch to the + `-d <dir-name>`. For example, this checks out 'master' branch to the `project-master` directory: + ------ @@ -224,7 +224,7 @@ the database to work reliably (otherwise you need to make sure that the database is up to date any time 'git-cvsserver' is executed). By default it uses SQLite databases in the Git directory, named -`gitcvs.<module_name>.sqlite`. Note that the SQLite backend creates +`gitcvs.<module-name>.sqlite`. Note that the SQLite backend creates temporary files in the same directory as the database file on write so it might not be enough to grant the users using 'git-cvsserver' write access to the database file without granting diff --git a/Documentation/git-daemon.txt b/Documentation/git-daemon.txt index e064f91c9e..ede7b935d6 100644 --- a/Documentation/git-daemon.txt +++ b/Documentation/git-daemon.txt @@ -18,7 +18,7 @@ SYNOPSIS [--allow-override=<service>] [--forbid-override=<service>] [--access-hook=<path>] [--[no-]informative-errors] [--inetd | - [--listen=<host_or_ipaddr>] [--port=<n>] + [--listen=<host-or-ipaddr>] [--port=<n>] [--user=<user> [--group=<group>]]] [--log-destination=(stderr|syslog|none)] [<directory>...] @@ -86,10 +86,10 @@ OPTIONS Incompatible with --detach, --port, --listen, --user and --group options. ---listen=<host_or_ipaddr>:: +--listen=<host-or-ipaddr>:: Listen on a specific IP address or hostname. IP addresses can be either an IPv4 address or an IPv6 address if supported. If IPv6 - is not supported, then --listen=hostname is also not supported and + is not supported, then --listen=<hostname> is also not supported and --listen must be given an IPv4 address. Can be given more than once. Incompatible with `--inetd` option. @@ -141,8 +141,8 @@ otherwise `stderr`. specified with no parameter, a request to git://host/{tilde}alice/foo is taken as a request to access 'foo' repository in the home directory of user `alice`. - If `--user-path=path` is specified, the same request is - taken as a request to access `path/foo` repository in + If `--user-path=<path>` is specified, the same request is + taken as a request to access `<path>/foo` repository in the home directory of user `alice`. --verbose:: diff --git a/Documentation/git-diagnose.txt b/Documentation/git-diagnose.txt index 3ec8cc7ad7..0711959e6f 100644 --- a/Documentation/git-diagnose.txt +++ b/Documentation/git-diagnose.txt @@ -45,7 +45,7 @@ OPTIONS -s <format>:: --suffix <format>:: Specify an alternate suffix for the diagnostics archive name, to create - a file named 'git-diagnostics-<formatted suffix>'. This should take the + a file named 'git-diagnostics-<formatted-suffix>'. This should take the form of a strftime(3) format string; the current local time will be used. diff --git a/Documentation/git-difftool.txt b/Documentation/git-difftool.txt index 50cb080085..c05f97aca9 100644 --- a/Documentation/git-difftool.txt +++ b/Documentation/git-difftool.txt @@ -90,7 +90,7 @@ instead. `--no-symlinks` is the default on Windows. --extcmd=<command>:: Specify a custom command for viewing diffs. 'git-difftool' ignores the configured defaults and runs - `$command $LOCAL $REMOTE` when this option is specified. + `<command> $LOCAL $REMOTE` when this option is specified. Additionally, `$BASE` is set in the environment. -g:: diff --git a/Documentation/git-fast-import.txt b/Documentation/git-fast-import.txt index bd7b1e0a2e..b2607366b9 100644 --- a/Documentation/git-fast-import.txt +++ b/Documentation/git-fast-import.txt @@ -745,11 +745,11 @@ paths for a commit are encouraged to do so. `notemodify` ^^^^^^^^^^^^ -Included in a `commit` `<notes_ref>` command to add a new note +Included in a `commit` `<notes-ref>` command to add a new note annotating a `<commit-ish>` or change this annotation contents. Internally it is similar to filemodify 100644 on `<commit-ish>` path (maybe split into subdirectories). It's not advised to -use any other commands to write to the `<notes_ref>` tree except +use any other commands to write to the `<notes-ref>` tree except `filedeleteall` to delete all existing notes in this tree. This command has two different means of specifying the content of the note. diff --git a/Documentation/git-fetch.txt b/Documentation/git-fetch.txt index f123139c58..50900a50da 100644 --- a/Documentation/git-fetch.txt +++ b/Documentation/git-fetch.txt @@ -186,8 +186,8 @@ origin: ------------------------------------------------ $ git fetch origin --prune --prune-tags $ git fetch origin --prune 'refs/tags/*:refs/tags/*' -$ git fetch <url of origin> --prune --prune-tags -$ git fetch <url of origin> --prune 'refs/tags/*:refs/tags/*' +$ git fetch <url-of-origin> --prune --prune-tags +$ git fetch <url-of-origin> --prune 'refs/tags/*:refs/tags/*' ------------------------------------------------ OUTPUT diff --git a/Documentation/git-filter-branch.txt b/Documentation/git-filter-branch.txt index 62e482a95e..5a4f853785 100644 --- a/Documentation/git-filter-branch.txt +++ b/Documentation/git-filter-branch.txt @@ -14,7 +14,7 @@ SYNOPSIS [--msg-filter <command>] [--commit-filter <command>] [--tag-name-filter <command>] [--prune-empty] [--original <namespace>] [-d <directory>] [-f | --force] - [--state-branch <branch>] [--] [<rev-list options>...] + [--state-branch <branch>] [--] [<rev-list-options>...] WARNING ------- @@ -32,7 +32,7 @@ listed there as reasonably possible. DESCRIPTION ----------- Lets you rewrite Git revision history by rewriting the branches mentioned -in the <rev-list options>, applying custom filters on each revision. +in the <rev-list-options>, applying custom filters on each revision. Those filters can modify each tree (e.g. removing a file or running a perl rewrite on all files) or information about each commit. Otherwise, all information (including original commit times or merge @@ -624,7 +624,7 @@ with: real backup; it dereferences tags first.) ** Running git-filter-branch with either --tags or --all in your - <rev-list options>. In order to retain annotated tags as + <rev-list-options>. In order to retain annotated tags as annotated, you must use --tag-name-filter (and must not have restored from refs/original/ in a previously botched rewrite). diff --git a/Documentation/git-for-each-ref.txt b/Documentation/git-for-each-ref.txt index be9543f684..b1cb482bf5 100644 --- a/Documentation/git-for-each-ref.txt +++ b/Documentation/git-for-each-ref.txt @@ -32,7 +32,8 @@ OPTIONS If one or more patterns are given, only refs are shown that match against at least one pattern, either using fnmatch(3) or literally, in the latter case matching completely or from the - beginning up to a slash. + beginning up to a slash. If an empty string is provided all refs + are printed, including HEAD and pseudorefs. --stdin:: If `--stdin` is supplied, then the list of patterns is read from diff --git a/Documentation/git-format-patch.txt b/Documentation/git-format-patch.txt index 414da6b73e..728bb3821c 100644 --- a/Documentation/git-format-patch.txt +++ b/Documentation/git-format-patch.txt @@ -17,10 +17,10 @@ SYNOPSIS [--signature-file=<file>] [-n | --numbered | -N | --no-numbered] [--start-number <n>] [--numbered-files] - [--in-reply-to=<message id>] [--suffix=.<sfx>] + [--in-reply-to=<message-id>] [--suffix=.<sfx>] [--ignore-if-in-upstream] [--always] [--cover-from-description=<mode>] - [--rfc] [--subject-prefix=<subject prefix>] + [--rfc] [--subject-prefix=<subject-prefix>] [(--reroll-count|-v) <n>] [--to=<email>] [--cc=<email>] [--[no-]cover-letter] [--quiet] @@ -30,8 +30,8 @@ SYNOPSIS [--range-diff=<previous> [--creation-factor=<percent>]] [--filename-max-length=<n>] [--progress] - [<common diff options>] - [ <since> | <revision range> ] + [<common-diff-options>] + [ <since> | <revision-range> ] DESCRIPTION ----------- @@ -64,7 +64,7 @@ There are two ways to specify which commits to operate on. to the tip of the current branch that are not in the history that leads to the <since> to be output. -2. Generic <revision range> expression (see "SPECIFYING +2. Generic <revision-range> expression (see "SPECIFYING REVISIONS" section in linkgit:gitrevisions[7]) means the commits in the specified range. @@ -179,9 +179,9 @@ Beware that the default for 'git send-email' is to thread emails itself. If you want `git format-patch` to take care of threading, you will want to ensure that threading is disabled for `git send-email`. ---in-reply-to=<message id>:: +--in-reply-to=<message-id>:: Make the first mail (or all the mails with `--no-thread`) appear as a - reply to the given <message id>, which avoids breaking threads to + reply to the given <message-id>, which avoids breaking threads to provide a new patch series. --ignore-if-in-upstream:: @@ -219,9 +219,9 @@ populated with placeholder text. Use the contents of <file> instead of the branch's description for generating the cover letter. ---subject-prefix=<subject prefix>:: +--subject-prefix=<subject-prefix>:: Instead of the standard '[PATCH]' prefix in the subject - line, instead use '[<subject prefix>]'. This can be used + line, instead use '[<subject-prefix>]'. This can be used to name a patch series, and can be combined with the `--numbered` option. + @@ -403,7 +403,7 @@ you can use `--suffix=-patch` to get `0001-description-of-my-change-patch`. `format.useAutoBase` configuration. --root:: - Treat the revision argument as a <revision range>, even if it + Treat the revision argument as a <revision-range>, even if it is just a single commit (that would normally be treated as a <since>). Note that root commits included in the specified range are always formatted as creation patches, independently diff --git a/Documentation/git-init.txt b/Documentation/git-init.txt index 6f0d2973bf..e8dc645bb5 100644 --- a/Documentation/git-init.txt +++ b/Documentation/git-init.txt @@ -11,6 +11,7 @@ SYNOPSIS [verse] 'git init' [-q | --quiet] [--bare] [--template=<template-directory>] [--separate-git-dir <git-dir>] [--object-format=<format>] + [--ref-format=<format>] [-b <branch-name> | --initial-branch=<branch-name>] [--shared[=<permissions>]] [<directory>] @@ -57,6 +58,12 @@ values are 'sha1' and (if enabled) 'sha256'. 'sha1' is the default. + include::object-format-disclaimer.txt[] +--ref-format=<format>:: + +Specify the given ref storage format for the repository. The valid values are: ++ +include::ref-storage-format.txt[] + --template=<template-directory>:: Specify the directory from which templates will be used. (See the "TEMPLATE diff --git a/Documentation/git-mv.txt b/Documentation/git-mv.txt index 7f991a3380..dc1bf61534 100644 --- a/Documentation/git-mv.txt +++ b/Documentation/git-mv.txt @@ -16,7 +16,7 @@ DESCRIPTION Move or rename a file, directory, or symlink. git mv [-v] [-f] [-n] [-k] <source> <destination> - git mv [-v] [-f] [-n] [-k] <source> ... <destination directory> + git mv [-v] [-f] [-n] [-k] <source> ... <destination-directory> In the first form, it renames <source>, which must exist and be either a file, symlink or directory, to <destination>. diff --git a/Documentation/git-notes.txt b/Documentation/git-notes.txt index f8310e56a8..c9221a68cc 100644 --- a/Documentation/git-notes.txt +++ b/Documentation/git-notes.txt @@ -56,7 +56,7 @@ SUBCOMMANDS list:: List the notes object for a given object. If no object is given, show a list of all note objects and the objects they - annotate (in the format "<note object> <annotated object>"). + annotate (in the format "<note-object> <annotated-object>"). This is the default subcommand if no subcommand is given. add:: diff --git a/Documentation/git-rebase.txt b/Documentation/git-rebase.txt index 1dd6555f66..06206521fc 100644 --- a/Documentation/git-rebase.txt +++ b/Documentation/git-rebase.txt @@ -523,7 +523,7 @@ See also INCOMPATIBLE OPTIONS below. + The commit list format can be changed by setting the configuration option rebase.instructionFormat. A customized instruction format will automatically -have the long commit hash prepended to the format. +have the commit hash prepended to the format. + See also INCOMPATIBLE OPTIONS below. @@ -626,13 +626,16 @@ See also INCOMPATIBLE OPTIONS below. Automatically reschedule `exec` commands that failed. This only makes sense in interactive mode (or when an `--exec` option was provided). + -Even though this option applies once a rebase is started, it's set for -the whole rebase at the start based on either the -`rebase.rescheduleFailedExec` configuration (see linkgit:git-config[1] -or "CONFIGURATION" below) or whether this option is -provided. Otherwise an explicit `--no-reschedule-failed-exec` at the -start would be overridden by the presence of -`rebase.rescheduleFailedExec=true` configuration. +This option applies once a rebase is started. It is preserved for the whole +rebase based on, in order, the command line option provided to the initial `git +rebase`, the `rebase.rescheduleFailedExec` configuration (see +linkgit:git-config[1] or "CONFIGURATION" below), or it defaults to false. ++ +Recording this option for the whole rebase is a convenience feature. Otherwise +an explicit `--no-reschedule-failed-exec` at the start would be overridden by +the presence of a `rebase.rescheduleFailedExec=true` configuration when `git +rebase --continue` is invoked. Currently, you cannot pass +`--[no-]reschedule-failed-exec` to `git rebase --continue`. --update-refs:: --no-update-refs:: @@ -963,10 +966,9 @@ The interactive rebase will stop when a command fails (i.e. exits with non-0 status) to give you an opportunity to fix the problem. You can continue with `git rebase --continue`. -The "exec" command launches the command in a shell (the one specified -in `$SHELL`, or the default shell if `$SHELL` is not set), so you can -use shell features (like "cd", ">", ";" ...). The command is run from -the root of the working tree. +The "exec" command launches the command in a shell (the default one, usually +/bin/sh), so you can use shell features (like "cd", ">", ";" ...). The command +is run from the root of the working tree. ---------------------------------- $ git rebase -i --exec "make test" diff --git a/Documentation/git-replace.txt b/Documentation/git-replace.txt index 4f257126e3..0a65460adb 100644 --- a/Documentation/git-replace.txt +++ b/Documentation/git-replace.txt @@ -114,11 +114,11 @@ FORMATS The following formats are available: * 'short': - <replaced sha1> + <replaced-sha1> * 'medium': - <replaced sha1> -> <replacement sha1> + <replaced-sha1> -> <replacement-sha1> * 'long': - <replaced sha1> (<replaced type>) -> <replacement sha1> (<replacement type>) + <replaced-sha1> (<replaced-type>) -> <replacement-sha1> (<replacement-type>) CREATING REPLACEMENT OBJECTS ---------------------------- diff --git a/Documentation/git-rev-parse.txt b/Documentation/git-rev-parse.txt index 912fab9f5e..59c5ac3ca9 100644 --- a/Documentation/git-rev-parse.txt +++ b/Documentation/git-rev-parse.txt @@ -159,6 +159,18 @@ for another option. unfortunately named tag "master"), and shows them as full refnames (e.g. "refs/heads/master"). +--output-object-format=(sha1|sha256|storage):: + + Allow oids to be input from any object format that the current + repository supports. + + Specifying "sha1" translates if necessary and returns a sha1 oid. + + Specifying "sha256" translates if necessary and returns a sha256 oid. + + Specifying "storage" translates if necessary and returns an oid in + encoded in the storage hash algorithm. + Options for Objects ~~~~~~~~~~~~~~~~~~~ @@ -307,6 +319,9 @@ The following options are unaffected by `--path-format`: input, multiple algorithms may be printed, space-separated. If not specified, the default is "storage". +--show-ref-format:: + Show the reference storage format used for the repository. + Other Options ~~~~~~~~~~~~~ diff --git a/Documentation/git-revert.txt b/Documentation/git-revert.txt index cbe0208834..568925db53 100644 --- a/Documentation/git-revert.txt +++ b/Documentation/git-revert.txt @@ -116,7 +116,7 @@ include::rerere-options.txt[] --reference:: Instead of starting the body of the log message with "This - reverts <full object name of the commit being reverted>.", + reverts <full-object-name-of-the-commit-being-reverted>.", refer to the commit using "--pretty=reference" format (cf. linkgit:git-log[1]). The `revert.reference` configuration variable can be used to enable this option by @@ -149,7 +149,7 @@ While git creates a basic commit message automatically, it is _strongly_ recommended to explain why the original commit is being reverted. In addition, repeatedly reverting reverts will result in increasingly -unwieldy subject lines, for example 'Reapply "Reapply "<original subject>""'. +unwieldy subject lines, for example 'Reapply "Reapply "<original-subject>""'. Please consider rewording these to be shorter and more unique. CONFIGURATION diff --git a/Documentation/git-send-email.txt b/Documentation/git-send-email.txt index 30deb7fe2a..d1ef6a204e 100644 --- a/Documentation/git-send-email.txt +++ b/Documentation/git-send-email.txt @@ -10,7 +10,7 @@ SYNOPSIS -------- [verse] 'git send-email' [<options>] <file|directory>... -'git send-email' [<options>] <format-patch options> +'git send-email' [<options>] <format-patch-options> 'git send-email' --dump-aliases diff --git a/Documentation/git-status.txt b/Documentation/git-status.txt index 10fecc51a7..4dbb88373b 100644 --- a/Documentation/git-status.txt +++ b/Documentation/git-status.txt @@ -309,7 +309,7 @@ Line Notes ------------------------------------------------------------ # branch.oid <commit> | (initial) Current commit. # branch.head <branch> | (detached) Current branch. -# branch.upstream <upstream_branch> If upstream is set. +# branch.upstream <upstream-branch> If upstream is set. # branch.ab +<ahead> -<behind> If upstream is set and the commit is present. ------------------------------------------------------------ @@ -502,7 +502,7 @@ results, so it could be faster on subsequent runs. usually worth the additional size. * `core.untrackedCache=true` and `core.fsmonitor=true` or - `core.fsmonitor=<hook_command_pathname>` (see + `core.fsmonitor=<hook-command-pathname>` (see linkgit:git-update-index[1]): enable both the untracked cache and FSMonitor features and only search directories that have been modified since the previous `git status` command. This diff --git a/Documentation/git-submodule.txt b/Documentation/git-submodule.txt index 695730609a..ca0347a37b 100644 --- a/Documentation/git-submodule.txt +++ b/Documentation/git-submodule.txt @@ -136,7 +136,7 @@ If you really want to remove a submodule from the repository and commit that use linkgit:git-rm[1] instead. See linkgit:gitsubmodules[7] for removal options. -update [--init] [--remote] [-N|--no-fetch] [--[no-]recommend-shallow] [-f|--force] [--checkout|--rebase|--merge] [--reference <repository>] [--depth <depth>] [--recursive] [--jobs <n>] [--[no-]single-branch] [--filter <filter spec>] [--] [<path>...]:: +update [--init] [--remote] [-N|--no-fetch] [--[no-]recommend-shallow] [-f|--force] [--checkout|--rebase|--merge] [--reference <repository>] [--depth <depth>] [--recursive] [--jobs <n>] [--[no-]single-branch] [--filter <filter-spec>] [--] [<path>...]:: + -- Update the registered submodules to match what the superproject @@ -185,7 +185,7 @@ submodule with the `--init` option. If `--recursive` is specified, this command will recurse into the registered submodules, and update any nested submodules within. -If `--filter <filter spec>` is specified, the given partial clone filter will be +If `--filter <filter-spec>` is specified, the given partial clone filter will be applied to the submodule. See linkgit:git-rev-list[1] for details on filter specifications. -- diff --git a/Documentation/git-svn.txt b/Documentation/git-svn.txt index 4e92308e85..43c68c2ec4 100644 --- a/Documentation/git-svn.txt +++ b/Documentation/git-svn.txt @@ -37,12 +37,12 @@ COMMANDS argument. Normally this command initializes the current directory. --T<trunk_subdir>;; ---trunk=<trunk_subdir>;; --t<tags_subdir>;; ---tags=<tags_subdir>;; --b<branches_subdir>;; ---branches=<branches_subdir>;; +-T<trunk-subdir>;; +--trunk=<trunk-subdir>;; +-t<tags-subdir>;; +--tags=<tags-subdir>;; +-b<branches-subdir>;; +--branches=<branches-subdir>;; -s;; --stdlayout;; These are optional command-line options for init. Each of @@ -726,9 +726,9 @@ ADVANCED OPTIONS when tracking a single URL. The 'log' and 'dcommit' commands no longer require this switch as an argument. --R<remote name>:: ---svn-remote <remote name>:: - Specify the [svn-remote "<remote name>"] section to use, +-R<remote-name>:: +--svn-remote <remote-name>:: + Specify the [svn-remote "<remote-name>"] section to use, this allows SVN multiple repositories to be tracked. Default: "svn" diff --git a/Documentation/git-tag.txt b/Documentation/git-tag.txt index d42efb3112..5fe519c31e 100644 --- a/Documentation/git-tag.txt +++ b/Documentation/git-tag.txt @@ -224,7 +224,7 @@ it in the repository configuration as follows: ------------------------------------- [user] - signingKey = <gpg-key_id> + signingKey = <gpg-key-id> ------------------------------------- `pager.tag` is only respected when listing tags, i.e., when `-l` is diff --git a/Documentation/git.txt b/Documentation/git.txt index bf9e6af695..0d25224c96 100644 --- a/Documentation/git.txt +++ b/Documentation/git.txt @@ -202,7 +202,7 @@ If you just want to run git as if it was started in `<path>` then use Do not perform optional operations that require locks. This is equivalent to setting the `GIT_OPTIONAL_LOCKS` to `0`. ---list-cmds=group[,group...]:: +--list-cmds=<group>[,<group>...]:: List commands by group. This is an internal/experimental option and may change or be removed in the future. Supported groups are: builtins, parseopt (builtin commands that use @@ -556,6 +556,11 @@ double-quotes and respecting backslash escapes. E.g., the value is always used. The default is "sha1". See `--object-format` in linkgit:git-init[1]. +`GIT_DEFAULT_REF_FORMAT`:: + If this variable is set, the default reference backend format for new + repositories will be set to this value. The default is "files". + See `--ref-format` in linkgit:git-init[1]. + Git Commits ~~~~~~~~~~~ `GIT_AUTHOR_NAME`:: @@ -724,13 +729,12 @@ for further details. waiting for someone with sufficient permissions to fix it. `GIT_FLUSH`:: -// NEEDSWORK: make it into a usual Boolean environment variable - If this environment variable is set to "1", then commands such + If this Boolean environment variable is set to true, then commands such as 'git blame' (in incremental mode), 'git rev-list', 'git log', 'git check-attr' and 'git check-ignore' will force a flush of the output stream after each record have been flushed. If this - variable is set to "0", the output of these commands will be done + variable is set to false, the output of these commands will be done using completely buffered I/O. If this environment variable is not set, Git will choose buffered or record-oriented flushing based on whether stdout appears to be redirected to a file or not. @@ -838,7 +842,7 @@ of the SID and an optional counter (to avoid filename collisions). + In addition, if the variable is set to -`af_unix:[<socket_type>:]<absolute-pathname>`, Git will try +`af_unix:[<socket-type>:]<absolute-pathname>`, Git will try to open the path as a Unix Domain Socket. The socket type can be either `stream` or `dgram`. + diff --git a/Documentation/gitattributes.txt b/Documentation/gitattributes.txt index 8c1793c148..201bdf5edb 100644 --- a/Documentation/gitattributes.txt +++ b/Documentation/gitattributes.txt @@ -100,6 +100,21 @@ for a path to `Unspecified` state. This can be done by listing the name of the attribute prefixed with an exclamation point `!`. +RESERVED BUILTIN_* ATTRIBUTES +----------------------------- + +builtin_* is a reserved namespace for builtin attribute values. Any +user defined attributes under this namespace will be ignored and +trigger a warning. + +`builtin_objectmode` +~~~~~~~~~~~~~~~~~~~~ +This attribute is for filtering files by their file bit modes (40000, +120000, 160000, 100755, 100644). e.g. ':(attr:builtin_objectmode=160000)'. +You may also check these values with `git check-attr builtin_objectmode -- <file>`. +If the object is not in the index `git check-attr --cached` will return unspecified. + + EFFECTS ------- diff --git a/Documentation/gitdiffcore.txt b/Documentation/gitdiffcore.txt index 3cda2e07c2..642c51227b 100644 --- a/Documentation/gitdiffcore.txt +++ b/Documentation/gitdiffcore.txt @@ -245,20 +245,20 @@ diffcore-pickaxe: For Detecting Addition/Deletion of Specified String This transformation limits the set of filepairs to those that change specified strings between the preimage and the postimage in a certain -way. -S<block of text> and -G<regular expression> options are used to +way. -S<block-of-text> and -G<regular-expression> options are used to specify different ways these strings are sought. -"-S<block of text>" detects filepairs whose preimage and postimage +"-S<block-of-text>" detects filepairs whose preimage and postimage have different number of occurrences of the specified block of text. By definition, it will not detect in-file moves. Also, when a changeset moves a file wholesale without affecting the interesting string, diffcore-rename kicks in as usual, and `-S` omits the filepair (since the number of occurrences of that string didn't change in that rename-detected filepair). When used with `--pickaxe-regex`, treat -the <block of text> as an extended POSIX regular expression to match, +the <block-of-text> as an extended POSIX regular expression to match, instead of a literal string. -"-G<regular expression>" (mnemonic: grep) detects filepairs whose +"-G<regular-expression>" (mnemonic: grep) detects filepairs whose textual diff has an added or a deleted line that matches the given regular expression. This means that it will detect in-file (or what rename-detection considers the same file) moves, which is noise. The diff --git a/Documentation/gitformat-commit-graph.txt b/Documentation/gitformat-commit-graph.txt index 31cad585e2..3e906e8030 100644 --- a/Documentation/gitformat-commit-graph.txt +++ b/Documentation/gitformat-commit-graph.txt @@ -142,13 +142,16 @@ All multi-byte numbers are in network byte order. ==== Bloom Filter Data (ID: {'B', 'D', 'A', 'T'}) [Optional] * It starts with header consisting of three unsigned 32-bit integers: - - Version of the hash algorithm being used. We currently only support - value 1 which corresponds to the 32-bit version of the murmur3 hash + - Version of the hash algorithm being used. We currently support + value 2 which corresponds to the 32-bit version of the murmur3 hash implemented exactly as described in https://en.wikipedia.org/wiki/MurmurHash#Algorithm and the double hashing technique using seed values 0x293ae76f and 0x7e646e2 as described in https://doi.org/10.1007/978-3-540-30494-4_26 "Bloom Filters - in Probabilistic Verification" + in Probabilistic Verification". Version 1 Bloom filters have a bug that appears + when char is signed and the repository has path names that have characters >= + 0x80; Git supports reading and writing them, but this ability will be removed + in a future version of Git. - The number of times a path is hashed and hence the number of bit positions that cumulatively determine whether a file is present in the commit. - The minimum number of bits 'b' per entry in the Bloom filter. If the filter diff --git a/Documentation/gitformat-index.txt b/Documentation/gitformat-index.txt index 0773e5c380..145cace1fe 100644 --- a/Documentation/gitformat-index.txt +++ b/Documentation/gitformat-index.txt @@ -386,8 +386,8 @@ The remaining data of each directory block is grouped by type: long, "REUC" extension that is M-bytes long, followed by "EOIE", then the hash would be: - Hash("TREE" + <binary representation of N> + - "REUC" + <binary representation of M>) + Hash("TREE" + <binary-representation-of-N> + + "REUC" + <binary-representation-of-M>) == Index Entry Offset Table diff --git a/Documentation/gitformat-pack.txt b/Documentation/gitformat-pack.txt index 9fcb29a9c8..d6ae229be5 100644 --- a/Documentation/gitformat-pack.txt +++ b/Documentation/gitformat-pack.txt @@ -396,6 +396,15 @@ CHUNK DATA: is padded at the end with between 0 and 3 NUL bytes to make the chunk size a multiple of 4 bytes. + Bitmapped Packfiles (ID: {'B', 'T', 'M', 'P'}) + Stores a table of two 4-byte unsigned integers in network order. + Each table entry corresponds to a single pack (in the order that + they appear above in the `PNAM` chunk). The values for each table + entry are as follows: + - The first bit position (in pseudo-pack order, see below) to + contain an object from that pack. + - The number of bits whose objects are selected from that pack. + OID Fanout (ID: {'O', 'I', 'D', 'F'}) The ith entry, F[i], stores the number of OIDs with first byte at most i. Thus F[255] stores the total @@ -509,6 +518,73 @@ packs arranged in MIDX order (with the preferred pack coming first). The MIDX's reverse index is stored in the optional 'RIDX' chunk within the MIDX itself. +=== `BTMP` chunk + +The Bitmapped Packfiles (`BTMP`) chunk encodes additional information +about the objects in the multi-pack index's reachability bitmap. Recall +that objects from the MIDX are arranged in "pseudo-pack" order (see +above) for reachability bitmaps. + +From the example above, suppose we have packs "a", "b", and "c", with +10, 15, and 20 objects, respectively. In pseudo-pack order, those would +be arranged as follows: + + |a,0|a,1|...|a,9|b,0|b,1|...|b,14|c,0|c,1|...|c,19| + +When working with single-pack bitmaps (or, equivalently, multi-pack +reachability bitmaps with a preferred pack), linkgit:git-pack-objects[1] +performs ``verbatim'' reuse, attempting to reuse chunks of the bitmapped +or preferred packfile instead of adding objects to the packing list. + +When a chunk of bytes is reused from an existing pack, any objects +contained therein do not need to be added to the packing list, saving +memory and CPU time. But a chunk from an existing packfile can only be +reused when the following conditions are met: + + - The chunk contains only objects which were requested by the caller + (i.e. does not contain any objects which the caller didn't ask for + explicitly or implicitly). + + - All objects stored in non-thin packs as offset- or reference-deltas + also include their base object in the resulting pack. + +The `BTMP` chunk encodes the necessary information in order to implement +multi-pack reuse over a set of packfiles as described above. +Specifically, the `BTMP` chunk encodes three pieces of information (all +32-bit unsigned integers in network byte-order) for each packfile `p` +that is stored in the MIDX, as follows: + +`bitmap_pos`:: The first bit position (in pseudo-pack order) in the + multi-pack index's reachability bitmap occupied by an object from `p`. + +`bitmap_nr`:: The number of bit positions (including the one at + `bitmap_pos`) that encode objects from that pack `p`. + +For example, the `BTMP` chunk corresponding to the above example (with +packs ``a'', ``b'', and ``c'') would look like: + +[cols="1,2,2"] +|=== +| |`bitmap_pos` |`bitmap_nr` + +|packfile ``a'' +|`0` +|`10` + +|packfile ``b'' +|`10` +|`15` + +|packfile ``c'' +|`25` +|`20` +|=== + +With this information in place, we can treat each packfile as +individually reusable in the same fashion as verbatim pack reuse is +performed on individual packs prior to the implementation of the `BTMP` +chunk. + == cruft packs The cruft packs feature offer an alternative to Git's traditional mechanism of diff --git a/Documentation/githooks.txt b/Documentation/githooks.txt index 883982e7a0..37f91d5b50 100644 --- a/Documentation/githooks.txt +++ b/Documentation/githooks.txt @@ -243,7 +243,7 @@ named remote is not being used both values will be the same. Information about what is to be pushed is provided on the hook's standard input with lines of the form: - <local ref> SP <local object name> SP <remote ref> SP <remote object name> LF + <local-ref> SP <local-object-name> SP <remote-ref> SP <remote-object-name> LF For instance, if the command +git push origin master:foreign+ were run the hook would receive a line like the following: @@ -251,9 +251,9 @@ hook would receive a line like the following: refs/heads/master 67890 refs/heads/foreign 12345 although the full object name would be supplied. If the foreign ref does not -yet exist the `<remote object name>` will be the all-zeroes object name. If a -ref is to be deleted, the `<local ref>` will be supplied as `(delete)` and the -`<local object name>` will be the all-zeroes object name. If the local commit +yet exist the `<remote-object-name>` will be the all-zeroes object name. If a +ref is to be deleted, the `<local-ref>` will be supplied as `(delete)` and the +`<local-object-name>` will be the all-zeroes object name. If the local commit was specified by something other than a name which could be expanded (such as `HEAD~`, or an object name) it will be supplied as it was originally given. diff --git a/Documentation/gitk.txt b/Documentation/gitk.txt index c2213bb77b..35b3996029 100644 --- a/Documentation/gitk.txt +++ b/Documentation/gitk.txt @@ -8,7 +8,7 @@ gitk - The Git repository browser SYNOPSIS -------- [verse] -'gitk' [<options>] [<revision range>] [--] [<path>...] +'gitk' [<options>] [<revision-range>] [--] [<path>...] DESCRIPTION ----------- @@ -124,7 +124,7 @@ gitk-specific options range to show. The command is expected to print on its standard output a list of additional revisions to be shown, one per line. Use this instead of explicitly specifying a - '<revision range>' if the set of commits to show may vary + '<revision-range>' if the set of commits to show may vary between refreshes. --select-commit=<ref>:: diff --git a/Documentation/gitprotocol-capabilities.txt b/Documentation/gitprotocol-capabilities.txt index d6c6effc21..2cf7735be4 100644 --- a/Documentation/gitprotocol-capabilities.txt +++ b/Documentation/gitprotocol-capabilities.txt @@ -378,7 +378,7 @@ fetch-pack may send "filter" commands to request a partial clone or partial fetch and request that the server omit various objects from the packfile. -session-id=<session id> +session-id=<session-id> ----------------------- The server may advertise a session ID that can be used to identify this process diff --git a/Documentation/gitprotocol-http.txt b/Documentation/gitprotocol-http.txt index 836b3490cc..ec40a550cc 100644 --- a/Documentation/gitprotocol-http.txt +++ b/Documentation/gitprotocol-http.txt @@ -391,14 +391,14 @@ C: Start a queue, `c_pending`, ordered by commit time (popping newest C: Send one `$GIT_URL/git-upload-pack` request: - C: 0032want <want #1>............................... - C: 0032want <want #2>............................... + C: 0032want <want-#1>............................... + C: 0032want <want-#2>............................... .... - C: 0032have <common #1>............................. - C: 0032have <common #2>............................. + C: 0032have <common-#1>............................. + C: 0032have <common-#2>............................. .... - C: 0032have <have #1>............................... - C: 0032have <have #2>............................... + C: 0032have <have-#1>............................... + C: 0032have <have-#2>............................... .... C: 0000 @@ -512,7 +512,7 @@ Within the command portion of the request body clients SHOULD send the id obtained through ref discovery as old_id. update_request = command_list - "PACK" <binary data> + "PACK" <binary-data> command_list = PKT-LINE(command NUL cap_list LF) *(command_pkt) diff --git a/Documentation/gitprotocol-v2.txt b/Documentation/gitprotocol-v2.txt index 8c1e7c61ea..0b800abd56 100644 --- a/Documentation/gitprotocol-v2.txt +++ b/Documentation/gitprotocol-v2.txt @@ -199,7 +199,7 @@ which can be used to limit the refs sent from the server. Additional features not supported in the base command will be advertised as the value of the command in the capability advertisement in the form -of a space separated list of features: "<command>=<feature 1> <feature 2>" +of a space separated list of features: "<command>=<feature-1> <feature-2>" ls-refs takes in the following arguments: @@ -245,7 +245,7 @@ addition of future extensions. Additional features not supported in the base command will be advertised as the value of the command in the capability advertisement in the form -of a space separated list of features: "<command>=<feature 1> <feature 2>" +of a space separated list of features: "<command>=<feature-1> <feature-2>" A `fetch` request can take the following arguments: @@ -363,7 +363,7 @@ can be included in the client's request as well as the potential addition of the 'packfile-uris' section in the server's response as explained below. - packfile-uris <comma-separated list of protocols> + packfile-uris <comma-separated-list-of-protocols> Indicates to the server that the client is willing to receive URIs of any of the given protocols in place of objects in the sent packfile. Before performing the connectivity check, the @@ -534,7 +534,7 @@ with objects using hash algorithm X. If not specified, the server is assumed to only handle SHA-1. If the client would like to use a hash algorithm other than SHA-1, it should specify its object-format string. -session-id=<session id> +session-id=<session-id> ~~~~~~~~~~~~~~~~~~~~~~~ The server may advertise a session ID that can be used to identify this process diff --git a/Documentation/gitsubmodules.txt b/Documentation/gitsubmodules.txt index 8400d591da..f7b5a25a0c 100644 --- a/Documentation/gitsubmodules.txt +++ b/Documentation/gitsubmodules.txt @@ -151,7 +151,7 @@ the superproject's `$GIT_DIR/config` file, so the superproject's history is not affected. This can be undone using `git submodule init`. * Deleted submodule: A submodule can be deleted by running -`git rm <submodule path> && git commit`. This can be undone +`git rm <submodule-path> && git commit`. This can be undone using `git revert`. + The deletion removes the superproject's tracking data, which are @@ -229,7 +229,7 @@ Workflow for a third party library git submodule add <URL> <path> # Occasionally update the submodule to a new version: - git -C <path> checkout <new version> + git -C <path> checkout <new-version> git add <path> git commit -m "update submodule to new version" diff --git a/Documentation/gitweb.conf.txt b/Documentation/gitweb.conf.txt index 59fc1d2741..85983587fc 100644 --- a/Documentation/gitweb.conf.txt +++ b/Documentation/gitweb.conf.txt @@ -343,7 +343,7 @@ $home_link_str:: Label for the "home link" at the top of all pages, leading to `$home_link` (usually the main gitweb page, which contains the projects list). It is used as the first component of gitweb's "breadcrumb trail": - `<home link> / <project> / <action>`. Can be set at build time using + `<home-link> / <project> / <action>`. Can be set at build time using the `GITWEB_HOME_LINK_STR` variable. By default it is set to "projects", as this link leads to the list of projects. Another popular choice is to set it to the name of site. Note that it is treated as raw HTML so it @@ -604,9 +604,9 @@ Many gitweb features can be enabled (or disabled) and configured using the Each `%feature` hash element is a hash reference and has the following structure: ---------------------------------------------------------------------- -"<feature_name>" => { - "sub" => <feature-sub (subroutine)>, - "override" => <allow-override (boolean)>, +"<feature-name>" => { + "sub" => <feature-sub-(subroutine)>, + "override" => <allow-override-(boolean)>, "default" => [ <options>... ] }, ---------------------------------------------------------------------- @@ -614,7 +614,7 @@ Some features cannot be overridden per project. For those features the structure of appropriate `%feature` hash element has a simpler form: ---------------------------------------------------------------------- -"<feature_name>" => { +"<feature-name>" => { "override" => 0, "default" => [ <options>... ] }, diff --git a/Documentation/gitweb.txt b/Documentation/gitweb.txt index ddd4a0fc70..56d24a30a3 100644 --- a/Documentation/gitweb.txt +++ b/Documentation/gitweb.txt @@ -305,7 +305,7 @@ pathnames. In most general form such path_info (component) based gitweb URL looks like this: ----------------------------------------------------------------------- -.../gitweb.cgi/<repo>/<action>/<revision_from>:/<path_from>..<revision_to>:/<path_to>?<arguments> +.../gitweb.cgi/<repo>/<action>/<revision-from>:/<path-from>..<revision-to>:/<path-to>?<arguments> ----------------------------------------------------------------------- diff --git a/Documentation/glossary-content.txt b/Documentation/glossary-content.txt index f7d98c11e3..d71b199955 100644 --- a/Documentation/glossary-content.txt +++ b/Documentation/glossary-content.txt @@ -638,6 +638,20 @@ The most notable example is `HEAD`. An <<def_object,object>> used to temporarily store the contents of a <<def_dirty,dirty>> working directory and the index for future reuse. +[[def_special_ref]]special ref:: + A ref that has different semantics than normal refs. These refs can be + accessed via normal Git commands but may not behave the same as a + normal ref in some cases. ++ +The following special refs are known to Git: + + - "`FETCH_HEAD`" is written by linkgit:git-fetch[1] or linkgit:git-pull[1]. It + may refer to multiple object IDs. Each object ID is annotated with metadata + indicating where it was fetched from and its fetch status. + + - "`MERGE_HEAD`" is written by linkgit:git-merge[1] when resolving merge + conflicts. It contains all commit IDs which are being merged. + [[def_submodule]]submodule:: A <<def_repository,repository>> that holds the history of a separate project inside another repository (the latter of diff --git a/Documentation/ref-storage-format.txt b/Documentation/ref-storage-format.txt new file mode 100644 index 0000000000..1a65cac468 --- /dev/null +++ b/Documentation/ref-storage-format.txt @@ -0,0 +1 @@ +* `files` for loose files with packed-refs. This is the default. diff --git a/Documentation/rev-list-options.txt b/Documentation/rev-list-options.txt index 2bf239ff03..a583b52c61 100644 --- a/Documentation/rev-list-options.txt +++ b/Documentation/rev-list-options.txt @@ -947,10 +947,10 @@ ifdef::git-rev-list[] + The form '--filter=blob:none' omits all blobs. + -The form '--filter=blob:limit=<n>[kmg]' omits blobs larger than n bytes -or units. n may be zero. The suffixes k, m, and g can be used to name -units in KiB, MiB, or GiB. For example, 'blob:limit=1k' is the same -as 'blob:limit=1024'. +The form '--filter=blob:limit=<n>[kmg]' omits blobs of size at least n +bytes or units. n may be zero. The suffixes k, m, and g can be used +to name units in KiB, MiB, or GiB. For example, 'blob:limit=1k' +is the same as 'blob:limit=1024'. + The form '--filter=object:type=(tag|commit|tree|blob)' omits all objects which are not of the requested type. diff --git a/Documentation/technical/repository-version.txt b/Documentation/technical/repository-version.txt index 045a76756f..27be3741e6 100644 --- a/Documentation/technical/repository-version.txt +++ b/Documentation/technical/repository-version.txt @@ -100,3 +100,8 @@ If set, by default "git config" reads from both "config" and multiple working directory mode, "config" file is shared while "config.worktree" is per-working directory (i.e., it's in GIT_COMMON_DIR/worktrees/<id>/config.worktree) + +==== `refStorage` + +Specifies the file format for the ref database. The only valid value +is `files` (loose references with a packed-refs file). diff --git a/Documentation/trace2-target-values.txt b/Documentation/trace2-target-values.txt index 3985b6d3c2..06f1953313 100644 --- a/Documentation/trace2-target-values.txt +++ b/Documentation/trace2-target-values.txt @@ -5,7 +5,7 @@ * `<absolute-pathname>` - Writes to the file in append mode. If the target already exists and is a directory, the traces will be written to files (one per process) underneath the given directory. -* `af_unix:[<socket_type>:]<absolute-pathname>` - Write to a +* `af_unix:[<socket-type>:]<absolute-pathname>` - Write to a Unix DomainSocket (on platforms that support them). Socket type can be either `stream` or `dgram`; if omitted Git will try both. diff --git a/Documentation/urls.txt b/Documentation/urls.txt index 4e79c1589e..ce671f812d 100644 --- a/Documentation/urls.txt +++ b/Documentation/urls.txt @@ -73,8 +73,8 @@ use will be rewritten into URLs that work), you can create a configuration section of the form: ------------ - [url "<actual url base>"] - insteadOf = <other url base> + [url "<actual-url-base>"] + insteadOf = <other-url-base> ------------ For example, with this: @@ -92,8 +92,8 @@ If you want to rewrite URLs for push only, you can create a configuration section of the form: ------------ - [url "<actual url base>"] - pushInsteadOf = <other url base> + [url "<actual-url-base>"] + pushInsteadOf = <other-url-base> ------------ For example, with this: diff --git a/Documentation/user-manual.txt b/Documentation/user-manual.txt index 5d32ff2384..6433903491 100644 --- a/Documentation/user-manual.txt +++ b/Documentation/user-manual.txt @@ -4100,8 +4100,8 @@ independently of the contents or the type of the object: all objects can be validated by verifying that (a) their hashes match the content of the file and (b) the object successfully inflates to a stream of bytes that forms a sequence of -`<ascii type without space> + <space> + <ascii decimal size> + -<byte\0> + <binary object data>`. +`<ascii-type-without-space> + <space> + <ascii-decimal-size> + +<byte\0> + <binary-object-data>`. The structured objects can further have their structure and connectivity to other objects verified. This is generally done with @@ -752,6 +752,10 @@ SCRIPTS = $(SCRIPT_SH_GEN) \ ETAGS_TARGET = TAGS +# If you add a new fuzzer, please also make sure to run it in +# ci/run-build-and-minimal-fuzzers.sh so that we make sure it still links and +# runs in the future. +FUZZ_OBJS += oss-fuzz/dummy-cmd-main.o FUZZ_OBJS += oss-fuzz/fuzz-commit-graph.o FUZZ_OBJS += oss-fuzz/fuzz-date.o FUZZ_OBJS += oss-fuzz/fuzz-pack-headers.o @@ -762,7 +766,7 @@ fuzz-objs: $(FUZZ_OBJS) # Always build fuzz objects even if not testing, to prevent bit-rot. all:: $(FUZZ_OBJS) -FUZZ_PROGRAMS += $(patsubst %.o,%,$(FUZZ_OBJS)) +FUZZ_PROGRAMS += $(patsubst %.o,%,$(filter-out %dummy-cmd-main.o,$(FUZZ_OBJS))) # Empty... EXTRA_PROGRAMS = @@ -792,8 +796,8 @@ TEST_BUILTINS_OBJS += test-chmtime.o TEST_BUILTINS_OBJS += test-config.o TEST_BUILTINS_OBJS += test-crontab.o TEST_BUILTINS_OBJS += test-csprng.o -TEST_BUILTINS_OBJS += test-ctype.o TEST_BUILTINS_OBJS += test-date.o +TEST_BUILTINS_OBJS += test-delete-gpgsig.o TEST_BUILTINS_OBJS += test-delta.o TEST_BUILTINS_OBJS += test-dir-iterator.o TEST_BUILTINS_OBJS += test-drop-caches.o @@ -828,7 +832,6 @@ TEST_BUILTINS_OBJS += test-partial-clone.o TEST_BUILTINS_OBJS += test-path-utils.o TEST_BUILTINS_OBJS += test-pcre2-config.o TEST_BUILTINS_OBJS += test-pkt-line.o -TEST_BUILTINS_OBJS += test-prio-queue.o TEST_BUILTINS_OBJS += test-proc-receive.o TEST_BUILTINS_OBJS += test-progress.o TEST_BUILTINS_OBJS += test-reach.o @@ -1058,6 +1061,7 @@ LIB_OBJS += list-objects-filter.o LIB_OBJS += list-objects.o LIB_OBJS += lockfile.o LIB_OBJS += log-tree.o +LIB_OBJS += loose.o LIB_OBJS += ls-refs.o LIB_OBJS += mailinfo.o LIB_OBJS += mailmap.o @@ -1078,6 +1082,7 @@ LIB_OBJS += notes-cache.o LIB_OBJS += notes-merge.o LIB_OBJS += notes-utils.o LIB_OBJS += notes.o +LIB_OBJS += object-file-convert.o LIB_OBJS += object-file.o LIB_OBJS += object-name.o LIB_OBJS += object.o @@ -1342,6 +1347,8 @@ THIRD_PARTY_SOURCES += sha1dc/% UNIT_TEST_PROGRAMS += t-basic UNIT_TEST_PROGRAMS += t-mem-pool UNIT_TEST_PROGRAMS += t-strbuf +UNIT_TEST_PROGRAMS += t-ctype +UNIT_TEST_PROGRAMS += t-prio-queue UNIT_TEST_PROGS = $(patsubst %,$(UNIT_TEST_BIN)/%$X,$(UNIT_TEST_PROGRAMS)) UNIT_TEST_OBJS = $(patsubst %,$(UNIT_TEST_DIR)/%.o,$(UNIT_TEST_PROGRAMS)) UNIT_TEST_OBJS += $(UNIT_TEST_DIR)/test-lib.o @@ -3850,16 +3857,17 @@ cover_db_html: cover_db # # make CC=clang CXX=clang++ \ # CFLAGS="-fsanitize=fuzzer-no-link,address" \ -# LIB_FUZZING_ENGINE="-fsanitize=fuzzer" \ +# LIB_FUZZING_ENGINE="-fsanitize=fuzzer,address" \ # fuzz-all # -FUZZ_CXXFLAGS ?= $(CFLAGS) +FUZZ_CXXFLAGS ?= $(ALL_CFLAGS) .PHONY: fuzz-all -$(FUZZ_PROGRAMS): all - $(QUIET_LINK)$(CXX) $(FUZZ_CXXFLAGS) $(LIB_OBJS) $(BUILTIN_OBJS) \ - $(XDIFF_OBJS) $(EXTLIBS) git.o $@.o $(LIB_FUZZING_ENGINE) -o $@ +$(FUZZ_PROGRAMS): %: %.o oss-fuzz/dummy-cmd-main.o $(GITLIBS) GIT-LDFLAGS + $(QUIET_LINK)$(CXX) $(FUZZ_CXXFLAGS) -o $@ $(ALL_LDFLAGS) \ + -Wl,--allow-multiple-definition \ + $(filter %.o,$^) $(filter %.a,$^) $(LIBS) $(LIB_FUZZING_ENGINE) fuzz-all: $(FUZZ_PROGRAMS) @@ -39,8 +39,8 @@ Those wishing to help with error message, usage and informational message string translations (localization l10) should see [po/README.md][] (a `po` file is a Portable Object file that holds the translations). -To subscribe to the list, send an email with just "subscribe git" in -the body to majordomo@vger.kernel.org (not the Git list). The mailing +To subscribe to the list, send an email to <git+subscribe@vger.kernel.org> +(see https://subspace.kernel.org/subscribing.html for details). The mailing list archives are available at <https://lore.kernel.org/git/>, <https://marc.info/?l=git> and other archival sites. @@ -33,52 +33,56 @@ static const char *advise_get_color(enum color_advice ix) return ""; } +enum advice_level { + ADVICE_LEVEL_NONE = 0, + ADVICE_LEVEL_DISABLED, + ADVICE_LEVEL_ENABLED, +}; + static struct { const char *key; - int enabled; + enum advice_level level; } advice_setting[] = { - [ADVICE_ADD_EMBEDDED_REPO] = { "addEmbeddedRepo", 1 }, - [ADVICE_ADD_EMPTY_PATHSPEC] = { "addEmptyPathspec", 1 }, - [ADVICE_ADD_IGNORED_FILE] = { "addIgnoredFile", 1 }, - [ADVICE_AM_WORK_DIR] = { "amWorkDir", 1 }, - [ADVICE_AMBIGUOUS_FETCH_REFSPEC] = { "ambiguousFetchRefspec", 1 }, - [ADVICE_CHECKOUT_AMBIGUOUS_REMOTE_BRANCH_NAME] = { "checkoutAmbiguousRemoteBranchName", 1 }, - [ADVICE_COMMIT_BEFORE_MERGE] = { "commitBeforeMerge", 1 }, - [ADVICE_DETACHED_HEAD] = { "detachedHead", 1 }, - [ADVICE_SUGGEST_DETACHING_HEAD] = { "suggestDetachingHead", 1 }, - [ADVICE_DIVERGING] = { "diverging", 1 }, - [ADVICE_FETCH_SHOW_FORCED_UPDATES] = { "fetchShowForcedUpdates", 1 }, - [ADVICE_GRAFT_FILE_DEPRECATED] = { "graftFileDeprecated", 1 }, - [ADVICE_IGNORED_HOOK] = { "ignoredHook", 1 }, - [ADVICE_IMPLICIT_IDENTITY] = { "implicitIdentity", 1 }, - [ADVICE_NESTED_TAG] = { "nestedTag", 1 }, - [ADVICE_OBJECT_NAME_WARNING] = { "objectNameWarning", 1 }, - [ADVICE_PUSH_ALREADY_EXISTS] = { "pushAlreadyExists", 1 }, - [ADVICE_PUSH_FETCH_FIRST] = { "pushFetchFirst", 1 }, - [ADVICE_PUSH_NEEDS_FORCE] = { "pushNeedsForce", 1 }, - [ADVICE_PUSH_REF_NEEDS_UPDATE] = { "pushRefNeedsUpdate", 1 }, - - /* make this an alias for backward compatibility */ - [ADVICE_PUSH_UPDATE_REJECTED_ALIAS] = { "pushNonFastForward", 1 }, - - [ADVICE_PUSH_NON_FF_CURRENT] = { "pushNonFFCurrent", 1 }, - [ADVICE_PUSH_NON_FF_MATCHING] = { "pushNonFFMatching", 1 }, - [ADVICE_PUSH_UNQUALIFIED_REF_NAME] = { "pushUnqualifiedRefName", 1 }, - [ADVICE_PUSH_UPDATE_REJECTED] = { "pushUpdateRejected", 1 }, - [ADVICE_RESET_NO_REFRESH_WARNING] = { "resetNoRefresh", 1 }, - [ADVICE_RESOLVE_CONFLICT] = { "resolveConflict", 1 }, - [ADVICE_RM_HINTS] = { "rmHints", 1 }, - [ADVICE_SEQUENCER_IN_USE] = { "sequencerInUse", 1 }, - [ADVICE_SET_UPSTREAM_FAILURE] = { "setUpstreamFailure", 1 }, - [ADVICE_SKIPPED_CHERRY_PICKS] = { "skippedCherryPicks", 1 }, - [ADVICE_STATUS_AHEAD_BEHIND_WARNING] = { "statusAheadBehindWarning", 1 }, - [ADVICE_STATUS_HINTS] = { "statusHints", 1 }, - [ADVICE_STATUS_U_OPTION] = { "statusUoption", 1 }, - [ADVICE_SUBMODULE_ALTERNATE_ERROR_STRATEGY_DIE] = { "submoduleAlternateErrorStrategyDie", 1 }, - [ADVICE_SUBMODULES_NOT_UPDATED] = { "submodulesNotUpdated", 1 }, - [ADVICE_UPDATE_SPARSE_PATH] = { "updateSparsePath", 1 }, - [ADVICE_WAITING_FOR_EDITOR] = { "waitingForEditor", 1 }, - [ADVICE_WORKTREE_ADD_ORPHAN] = { "worktreeAddOrphan", 1 }, + [ADVICE_ADD_EMBEDDED_REPO] = { "addEmbeddedRepo" }, + [ADVICE_ADD_EMPTY_PATHSPEC] = { "addEmptyPathspec" }, + [ADVICE_ADD_IGNORED_FILE] = { "addIgnoredFile" }, + [ADVICE_AMBIGUOUS_FETCH_REFSPEC] = { "ambiguousFetchRefspec" }, + [ADVICE_AM_WORK_DIR] = { "amWorkDir" }, + [ADVICE_CHECKOUT_AMBIGUOUS_REMOTE_BRANCH_NAME] = { "checkoutAmbiguousRemoteBranchName" }, + [ADVICE_COMMIT_BEFORE_MERGE] = { "commitBeforeMerge" }, + [ADVICE_DETACHED_HEAD] = { "detachedHead" }, + [ADVICE_DIVERGING] = { "diverging" }, + [ADVICE_FETCH_SHOW_FORCED_UPDATES] = { "fetchShowForcedUpdates" }, + [ADVICE_FORCE_DELETE_BRANCH] = { "forceDeleteBranch" }, + [ADVICE_GRAFT_FILE_DEPRECATED] = { "graftFileDeprecated" }, + [ADVICE_IGNORED_HOOK] = { "ignoredHook" }, + [ADVICE_IMPLICIT_IDENTITY] = { "implicitIdentity" }, + [ADVICE_NESTED_TAG] = { "nestedTag" }, + [ADVICE_OBJECT_NAME_WARNING] = { "objectNameWarning" }, + [ADVICE_PUSH_ALREADY_EXISTS] = { "pushAlreadyExists" }, + [ADVICE_PUSH_FETCH_FIRST] = { "pushFetchFirst" }, + [ADVICE_PUSH_NEEDS_FORCE] = { "pushNeedsForce" }, + [ADVICE_PUSH_NON_FF_CURRENT] = { "pushNonFFCurrent" }, + [ADVICE_PUSH_NON_FF_MATCHING] = { "pushNonFFMatching" }, + [ADVICE_PUSH_REF_NEEDS_UPDATE] = { "pushRefNeedsUpdate" }, + [ADVICE_PUSH_UNQUALIFIED_REF_NAME] = { "pushUnqualifiedRefName" }, + [ADVICE_PUSH_UPDATE_REJECTED] = { "pushUpdateRejected" }, + [ADVICE_PUSH_UPDATE_REJECTED_ALIAS] = { "pushNonFastForward" }, /* backwards compatibility */ + [ADVICE_RESET_NO_REFRESH_WARNING] = { "resetNoRefresh" }, + [ADVICE_RESOLVE_CONFLICT] = { "resolveConflict" }, + [ADVICE_RM_HINTS] = { "rmHints" }, + [ADVICE_SEQUENCER_IN_USE] = { "sequencerInUse" }, + [ADVICE_SET_UPSTREAM_FAILURE] = { "setUpstreamFailure" }, + [ADVICE_SKIPPED_CHERRY_PICKS] = { "skippedCherryPicks" }, + [ADVICE_STATUS_AHEAD_BEHIND_WARNING] = { "statusAheadBehindWarning" }, + [ADVICE_STATUS_HINTS] = { "statusHints" }, + [ADVICE_STATUS_U_OPTION] = { "statusUoption" }, + [ADVICE_SUBMODULES_NOT_UPDATED] = { "submodulesNotUpdated" }, + [ADVICE_SUBMODULE_ALTERNATE_ERROR_STRATEGY_DIE] = { "submoduleAlternateErrorStrategyDie" }, + [ADVICE_SUGGEST_DETACHING_HEAD] = { "suggestDetachingHead" }, + [ADVICE_UPDATE_SPARSE_PATH] = { "updateSparsePath" }, + [ADVICE_WAITING_FOR_EDITOR] = { "waitingForEditor" }, + [ADVICE_WORKTREE_ADD_ORPHAN] = { "worktreeAddOrphan" }, }; static const char turn_off_instructions[] = @@ -118,13 +122,13 @@ void advise(const char *advice, ...) int advice_enabled(enum advice_type type) { - switch(type) { - case ADVICE_PUSH_UPDATE_REJECTED: - return advice_setting[ADVICE_PUSH_UPDATE_REJECTED].enabled && - advice_setting[ADVICE_PUSH_UPDATE_REJECTED_ALIAS].enabled; - default: - return advice_setting[type].enabled; - } + int enabled = advice_setting[type].level != ADVICE_LEVEL_DISABLED; + + if (type == ADVICE_PUSH_UPDATE_REJECTED) + return enabled && + advice_enabled(ADVICE_PUSH_UPDATE_REJECTED_ALIAS); + + return enabled; } void advise_if_enabled(enum advice_type type, const char *advice, ...) @@ -135,7 +139,8 @@ void advise_if_enabled(enum advice_type type, const char *advice, ...) return; va_start(params, advice); - vadvise(advice, 1, advice_setting[type].key, params); + vadvise(advice, !advice_setting[type].level, advice_setting[type].key, + params); va_end(params); } @@ -164,7 +169,9 @@ int git_default_advice_config(const char *var, const char *value) for (i = 0; i < ARRAY_SIZE(advice_setting); i++) { if (strcasecmp(k, advice_setting[i].key)) continue; - advice_setting[i].enabled = git_config_bool(var, value); + advice_setting[i].level = git_config_bool(var, value) + ? ADVICE_LEVEL_ENABLED + : ADVICE_LEVEL_DISABLED; return 0; } @@ -10,18 +10,18 @@ struct string_list; * Add the new config variable to Documentation/config/advice.txt. * Call advise_if_enabled to print your advice. */ - enum advice_type { +enum advice_type { ADVICE_ADD_EMBEDDED_REPO, ADVICE_ADD_EMPTY_PATHSPEC, ADVICE_ADD_IGNORED_FILE, - ADVICE_AM_WORK_DIR, ADVICE_AMBIGUOUS_FETCH_REFSPEC, + ADVICE_AM_WORK_DIR, ADVICE_CHECKOUT_AMBIGUOUS_REMOTE_BRANCH_NAME, ADVICE_COMMIT_BEFORE_MERGE, ADVICE_DETACHED_HEAD, ADVICE_DIVERGING, - ADVICE_SUGGEST_DETACHING_HEAD, ADVICE_FETCH_SHOW_FORCED_UPDATES, + ADVICE_FORCE_DELETE_BRANCH, ADVICE_GRAFT_FILE_DEPRECATED, ADVICE_IGNORED_HOOK, ADVICE_IMPLICIT_IDENTITY, @@ -32,23 +32,24 @@ struct string_list; ADVICE_PUSH_NEEDS_FORCE, ADVICE_PUSH_NON_FF_CURRENT, ADVICE_PUSH_NON_FF_MATCHING, + ADVICE_PUSH_REF_NEEDS_UPDATE, ADVICE_PUSH_UNQUALIFIED_REF_NAME, - ADVICE_PUSH_UPDATE_REJECTED_ALIAS, ADVICE_PUSH_UPDATE_REJECTED, - ADVICE_PUSH_REF_NEEDS_UPDATE, + ADVICE_PUSH_UPDATE_REJECTED_ALIAS, ADVICE_RESET_NO_REFRESH_WARNING, ADVICE_RESOLVE_CONFLICT, ADVICE_RM_HINTS, ADVICE_SEQUENCER_IN_USE, ADVICE_SET_UPSTREAM_FAILURE, + ADVICE_SKIPPED_CHERRY_PICKS, ADVICE_STATUS_AHEAD_BEHIND_WARNING, ADVICE_STATUS_HINTS, ADVICE_STATUS_U_OPTION, - ADVICE_SUBMODULE_ALTERNATE_ERROR_STRATEGY_DIE, ADVICE_SUBMODULES_NOT_UPDATED, + ADVICE_SUBMODULE_ALTERNATE_ERROR_STRATEGY_DIE, + ADVICE_SUGGEST_DETACHING_HEAD, ADVICE_UPDATE_SPARSE_PATH, ADVICE_WAITING_FOR_EDITOR, - ADVICE_SKIPPED_CHERRY_PICKS, ADVICE_WORKTREE_ADD_ORPHAN, }; @@ -2219,7 +2219,8 @@ static void reverse_patches(struct patch *p) struct fragment *frag = p->fragments; SWAP(p->new_name, p->old_name); - SWAP(p->new_mode, p->old_mode); + if (p->new_mode) + SWAP(p->new_mode, p->old_mode); SWAP(p->is_new, p->is_delete); SWAP(p->lines_added, p->lines_deleted); SWAP(p->old_oid_prefix, p->new_oid_prefix); @@ -3777,8 +3778,17 @@ static int check_preimage(struct apply_state *state, return error_errno("%s", old_name); } - if (!state->cached && !previous) - st_mode = ce_mode_from_stat(*ce, st->st_mode); + if (!state->cached && !previous) { + if (*ce && !(*ce)->ce_mode) + BUG("ce_mode == 0 for path '%s'", old_name); + + if (trust_executable_bit) + st_mode = ce_mode_from_stat(*ce, st->st_mode); + else if (*ce) + st_mode = (*ce)->ce_mode; + else + st_mode = patch->old_mode; + } if (patch->is_new < 0) patch->is_new = 0; @@ -339,7 +339,8 @@ int write_archive_entries(struct archiver_args *args, opts.src_index = args->repo->index; opts.dst_index = args->repo->index; opts.fn = oneway_merge; - init_tree_desc(&t, args->tree->buffer, args->tree->size); + init_tree_desc(&t, &args->tree->object.oid, + args->tree->buffer, args->tree->size); if (unpack_trees(1, &t, &opts)) return -1; git_attr_set_direction(GIT_ATTR_INDEX); @@ -17,6 +17,7 @@ #include "utf8.h" #include "quote.h" #include "read-cache-ll.h" +#include "refs.h" #include "revision.h" #include "object-store-ll.h" #include "setup.h" @@ -183,6 +184,15 @@ static void all_attrs_init(struct attr_hashmap *map, struct attr_check *check) } } +/* + * Atribute name cannot begin with "builtin_" which + * is a reserved namespace for built in attributes values. + */ +static int attr_name_reserved(const char *name) +{ + return starts_with(name, "builtin_"); +} + static int attr_name_valid(const char *name, size_t namelen) { /* @@ -315,7 +325,7 @@ static const char *parse_attr(const char *src, int lineno, const char *cp, cp++; len--; } - if (!attr_name_valid(cp, len)) { + if (!attr_name_valid(cp, len) || attr_name_reserved(cp)) { report_invalid_attr(cp, len, src, lineno); return NULL; } @@ -379,7 +389,7 @@ static struct match_attr *parse_attr_line(const char *line, const char *src, name += strlen(ATTRIBUTE_MACRO_PREFIX); name += strspn(name, blank); namelen = strcspn(name, blank); - if (!attr_name_valid(name, namelen)) { + if (!attr_name_valid(name, namelen) || attr_name_reserved(name)) { report_invalid_attr(name, namelen, src, lineno); goto fail_return; } @@ -1240,6 +1250,85 @@ static struct object_id *default_attr_source(void) return &attr_source; } +static const char *interned_mode_string(unsigned int mode) +{ + static struct { + unsigned int val; + char str[7]; + } mode_string[] = { + { .val = 0040000 }, + { .val = 0100644 }, + { .val = 0100755 }, + { .val = 0120000 }, + { .val = 0160000 }, + }; + int i; + + for (i = 0; i < ARRAY_SIZE(mode_string); i++) { + if (mode_string[i].val != mode) + continue; + if (!*mode_string[i].str) + snprintf(mode_string[i].str, sizeof(mode_string[i].str), + "%06o", mode); + return mode_string[i].str; + } + BUG("Unsupported mode 0%o", mode); +} + +static const char *builtin_object_mode_attr(struct index_state *istate, const char *path) +{ + unsigned int mode; + + if (direction == GIT_ATTR_CHECKIN) { + struct object_id oid; + struct stat st; + if (lstat(path, &st)) + die_errno(_("unable to stat '%s'"), path); + mode = canon_mode(st.st_mode); + if (S_ISDIR(mode)) { + /* + *`path` is either a directory or it is a submodule, + * in which case it is already indexed as submodule + * or it does not exist in the index yet and we need to + * check if we can resolve to a ref. + */ + int pos = index_name_pos(istate, path, strlen(path)); + if (pos >= 0) { + if (S_ISGITLINK(istate->cache[pos]->ce_mode)) + mode = istate->cache[pos]->ce_mode; + } else if (resolve_gitlink_ref(path, "HEAD", &oid) == 0) { + mode = S_IFGITLINK; + } + } + } else { + /* + * For GIT_ATTR_CHECKOUT and GIT_ATTR_INDEX we only check + * for mode in the index. + */ + int pos = index_name_pos(istate, path, strlen(path)); + if (pos >= 0) + mode = istate->cache[pos]->ce_mode; + else + return ATTR__UNSET; + } + + return interned_mode_string(mode); +} + + +static const char *compute_builtin_attr(struct index_state *istate, + const char *path, + const struct git_attr *attr) { + static const struct git_attr *object_mode_attr; + + if (!object_mode_attr) + object_mode_attr = git_attr("builtin_objectmode"); + + if (attr == object_mode_attr) + return builtin_object_mode_attr(istate, path); + return ATTR__UNSET; +} + void git_check_attr(struct index_state *istate, const char *path, struct attr_check *check) @@ -1253,7 +1342,7 @@ void git_check_attr(struct index_state *istate, unsigned int n = check->items[i].attr->attr_nr; const char *value = check->all_attrs[n].value; if (value == ATTR__UNKNOWN) - value = ATTR__UNSET; + value = compute_builtin_attr(istate, path, check->all_attrs[n].attr); check->items[i].value = value; } } @@ -6,6 +6,9 @@ #include "commit-graph.h" #include "commit.h" #include "commit-slab.h" +#include "tree.h" +#include "tree-walk.h" +#include "config.h" define_commit_slab(bloom_filter_slab, struct bloom_filter); @@ -48,9 +51,9 @@ static int check_bloom_offset(struct commit_graph *g, uint32_t pos, return -1; } -static int load_bloom_filter_from_graph(struct commit_graph *g, - struct bloom_filter *filter, - uint32_t graph_pos) +int load_bloom_filter_from_graph(struct commit_graph *g, + struct bloom_filter *filter, + uint32_t graph_pos) { uint32_t lex_pos, start_index, end_index; @@ -88,6 +91,8 @@ static int load_bloom_filter_from_graph(struct commit_graph *g, filter->data = (unsigned char *)(g->chunk_bloom_data + sizeof(unsigned char) * start_index + BLOOMDATA_CHUNK_HEADER_SIZE); + filter->version = g->bloom_filter_settings->hash_version; + filter->to_free = NULL; return 1; } @@ -99,7 +104,64 @@ static int load_bloom_filter_from_graph(struct commit_graph *g, * Not considered to be cryptographically secure. * Implemented as described in https://en.wikipedia.org/wiki/MurmurHash#Algorithm */ -uint32_t murmur3_seeded(uint32_t seed, const char *data, size_t len) +uint32_t murmur3_seeded_v2(uint32_t seed, const char *data, size_t len) +{ + const uint32_t c1 = 0xcc9e2d51; + const uint32_t c2 = 0x1b873593; + const uint32_t r1 = 15; + const uint32_t r2 = 13; + const uint32_t m = 5; + const uint32_t n = 0xe6546b64; + int i; + uint32_t k1 = 0; + const char *tail; + + int len4 = len / sizeof(uint32_t); + + uint32_t k; + for (i = 0; i < len4; i++) { + uint32_t byte1 = (uint32_t)(unsigned char)data[4*i]; + uint32_t byte2 = ((uint32_t)(unsigned char)data[4*i + 1]) << 8; + uint32_t byte3 = ((uint32_t)(unsigned char)data[4*i + 2]) << 16; + uint32_t byte4 = ((uint32_t)(unsigned char)data[4*i + 3]) << 24; + k = byte1 | byte2 | byte3 | byte4; + k *= c1; + k = rotate_left(k, r1); + k *= c2; + + seed ^= k; + seed = rotate_left(seed, r2) * m + n; + } + + tail = (data + len4 * sizeof(uint32_t)); + + switch (len & (sizeof(uint32_t) - 1)) { + case 3: + k1 ^= ((uint32_t)(unsigned char)tail[2]) << 16; + /*-fallthrough*/ + case 2: + k1 ^= ((uint32_t)(unsigned char)tail[1]) << 8; + /*-fallthrough*/ + case 1: + k1 ^= ((uint32_t)(unsigned char)tail[0]) << 0; + k1 *= c1; + k1 = rotate_left(k1, r1); + k1 *= c2; + seed ^= k1; + break; + } + + seed ^= (uint32_t)len; + seed ^= (seed >> 16); + seed *= 0x85ebca6b; + seed ^= (seed >> 13); + seed *= 0xc2b2ae35; + seed ^= (seed >> 16); + + return seed; +} + +static uint32_t murmur3_seeded_v1(uint32_t seed, const char *data, size_t len) { const uint32_t c1 = 0xcc9e2d51; const uint32_t c2 = 0x1b873593; @@ -164,8 +226,14 @@ void fill_bloom_key(const char *data, int i; const uint32_t seed0 = 0x293ae76f; const uint32_t seed1 = 0x7e646e2c; - const uint32_t hash0 = murmur3_seeded(seed0, data, len); - const uint32_t hash1 = murmur3_seeded(seed1, data, len); + uint32_t hash0, hash1; + if (settings->hash_version == 2) { + hash0 = murmur3_seeded_v2(seed0, data, len); + hash1 = murmur3_seeded_v2(seed1, data, len); + } else { + hash0 = murmur3_seeded_v1(seed0, data, len); + hash1 = murmur3_seeded_v1(seed1, data, len); + } key->hashes = (uint32_t *)xcalloc(settings->num_hashes, sizeof(uint32_t)); for (i = 0; i < settings->num_hashes; i++) @@ -197,6 +265,18 @@ void init_bloom_filters(void) init_bloom_filter_slab(&bloom_filters); } +static void free_one_bloom_filter(struct bloom_filter *filter) +{ + if (!filter) + return; + free(filter->to_free); +} + +void deinit_bloom_filters(void) +{ + deep_clear_bloom_filter_slab(&bloom_filters, free_one_bloom_filter); +} + static int pathmap_cmp(const void *hashmap_cmp_fn_data UNUSED, const struct hashmap_entry *eptr, const struct hashmap_entry *entry_or_key, @@ -210,11 +290,97 @@ static int pathmap_cmp(const void *hashmap_cmp_fn_data UNUSED, return strcmp(e1->path, e2->path); } -static void init_truncated_large_filter(struct bloom_filter *filter) +static void init_truncated_large_filter(struct bloom_filter *filter, + int version) { - filter->data = xmalloc(1); + filter->data = filter->to_free = xmalloc(1); filter->data[0] = 0xFF; filter->len = 1; + filter->version = version; +} + +#define VISITED (1u<<21) +#define HIGH_BITS (1u<<22) + +static int has_entries_with_high_bit(struct repository *r, struct tree *t) +{ + if (parse_tree(t)) + return 1; + + if (!(t->object.flags & VISITED)) { + struct tree_desc desc; + struct name_entry entry; + + init_tree_desc(&desc, &t->object.oid, t->buffer, t->size); + while (tree_entry(&desc, &entry)) { + size_t i; + for (i = 0; i < entry.pathlen; i++) { + if (entry.path[i] & 0x80) { + t->object.flags |= HIGH_BITS; + goto done; + } + } + + if (S_ISDIR(entry.mode)) { + struct tree *sub = lookup_tree(r, &entry.oid); + if (sub && has_entries_with_high_bit(r, sub)) { + t->object.flags |= HIGH_BITS; + goto done; + } + } + + } + +done: + t->object.flags |= VISITED; + } + + return !!(t->object.flags & HIGH_BITS); +} + +static int commit_tree_has_high_bit_paths(struct repository *r, + struct commit *c) +{ + struct tree *t; + if (repo_parse_commit(r, c)) + return 1; + t = repo_get_commit_tree(r, c); + if (!t) + return 1; + return has_entries_with_high_bit(r, t); +} + +static struct bloom_filter *upgrade_filter(struct repository *r, struct commit *c, + struct bloom_filter *filter, + int hash_version) +{ + struct commit_list *p = c->parents; + if (commit_tree_has_high_bit_paths(r, c)) + return NULL; + + if (p && commit_tree_has_high_bit_paths(r, p->item)) + return NULL; + + filter->version = hash_version; + + return filter; +} + +struct bloom_filter *get_bloom_filter(struct repository *r, struct commit *c) +{ + struct bloom_filter *filter; + int hash_version; + + filter = get_or_compute_bloom_filter(r, c, 0, NULL, NULL); + if (!filter) + return NULL; + + prepare_repo_settings(r); + hash_version = r->settings.commit_graph_changed_paths_version; + + if (!(hash_version == -1 || hash_version == filter->version)) + return NULL; /* unusable filter */ + return filter; } struct bloom_filter *get_or_compute_bloom_filter(struct repository *r, @@ -242,8 +408,23 @@ struct bloom_filter *get_or_compute_bloom_filter(struct repository *r, filter, graph_pos); } - if (filter->data && filter->len) - return filter; + if (filter->data && filter->len) { + struct bloom_filter *upgrade; + if (!settings || settings->hash_version == filter->version) + return filter; + + /* version mismatch, see if we can upgrade */ + if (compute_if_not_present && + git_env_bool("GIT_TEST_UPGRADE_BLOOM_FILTERS", 1)) { + upgrade = upgrade_filter(r, c, filter, + settings->hash_version); + if (upgrade) { + if (computed) + *computed |= BLOOM_UPGRADED; + return upgrade; + } + } + } if (!compute_if_not_present) return NULL; @@ -299,19 +480,22 @@ struct bloom_filter *get_or_compute_bloom_filter(struct repository *r, } if (hashmap_get_size(&pathmap) > settings->max_changed_paths) { - init_truncated_large_filter(filter); + init_truncated_large_filter(filter, + settings->hash_version); if (computed) *computed |= BLOOM_TRUNC_LARGE; goto cleanup; } filter->len = (hashmap_get_size(&pathmap) * settings->bits_per_entry + BITS_PER_WORD - 1) / BITS_PER_WORD; + filter->version = settings->hash_version; if (!filter->len) { if (computed) *computed |= BLOOM_TRUNC_EMPTY; filter->len = 1; } CALLOC_ARRAY(filter->data, filter->len); + filter->to_free = filter->data; hashmap_for_each_entry(&pathmap, &iter, e, entry) { struct bloom_key key; @@ -325,7 +509,7 @@ struct bloom_filter *get_or_compute_bloom_filter(struct repository *r, } else { for (i = 0; i < diff_queued_diff.nr; i++) diff_free_filepair(diff_queued_diff.queue[i]); - init_truncated_large_filter(filter); + init_truncated_large_filter(filter, settings->hash_version); if (computed) *computed |= BLOOM_TRUNC_LARGE; @@ -3,13 +3,16 @@ struct commit; struct repository; +struct commit_graph; struct bloom_filter_settings { /* * The version of the hashing technique being used. - * We currently only support version = 1 which is + * The newest version is 2, which is * the seeded murmur3 hashing technique implemented - * in bloom.c. + * in bloom.c. Bloom filters of version 1 were created + * with prior versions of Git, which had a bug in the + * implementation of the hash function. */ uint32_t hash_version; @@ -52,6 +55,9 @@ struct bloom_filter_settings { struct bloom_filter { unsigned char *data; size_t len; + int version; + + void *to_free; }; /* @@ -68,6 +74,10 @@ struct bloom_key { uint32_t *hashes; }; +int load_bloom_filter_from_graph(struct commit_graph *g, + struct bloom_filter *filter, + uint32_t graph_pos); + /* * Calculate the murmur3 32-bit hash value for the given data * using the given seed. @@ -75,7 +85,7 @@ struct bloom_key { * Not considered to be cryptographically secure. * Implemented as described in https://en.wikipedia.org/wiki/MurmurHash#Algorithm */ -uint32_t murmur3_seeded(uint32_t seed, const char *data, size_t len); +uint32_t murmur3_seeded_v2(uint32_t seed, const char *data, size_t len); void fill_bloom_key(const char *data, size_t len, @@ -88,12 +98,14 @@ void add_key_to_filter(const struct bloom_key *key, const struct bloom_filter_settings *settings); void init_bloom_filters(void); +void deinit_bloom_filters(void); enum bloom_filter_computed { BLOOM_NOT_COMPUTED = (1 << 0), BLOOM_COMPUTED = (1 << 1), BLOOM_TRUNC_LARGE = (1 << 2), BLOOM_TRUNC_EMPTY = (1 << 3), + BLOOM_UPGRADED = (1 << 4), }; struct bloom_filter *get_or_compute_bloom_filter(struct repository *r, @@ -102,8 +114,24 @@ struct bloom_filter *get_or_compute_bloom_filter(struct repository *r, const struct bloom_filter_settings *settings, enum bloom_filter_computed *computed); -#define get_bloom_filter(r, c) get_or_compute_bloom_filter( \ - (r), (c), 0, NULL, NULL) +/* + * Find the Bloom filter associated with the given commit "c". + * + * If any of the following are true + * + * - the repository does not have a commit-graph, or + * - the repository disables reading from the commit-graph, or + * - the given commit does not have a Bloom filter computed, or + * - there is a Bloom filter for commit "c", but it cannot be read + * because the filter uses an incompatible version of murmur3 + * + * , then `get_bloom_filter()` will return NULL. Otherwise, the corresponding + * Bloom filter will be returned. + * + * For callers who wish to inspect Bloom filters with incompatible hash + * versions, use get_or_compute_bloom_filter(). + */ +struct bloom_filter *get_bloom_filter(struct repository *r, struct commit *c); int bloom_filter_contains(const struct bloom_filter *filter, const struct bloom_key *key, @@ -817,8 +817,9 @@ void remove_merge_branch_state(struct repository *r) unlink(git_path_merge_rr(r)); unlink(git_path_merge_msg(r)); unlink(git_path_merge_mode(r)); - unlink(git_path_auto_merge(r)); - save_autostash(git_path_merge_autostash(r)); + refs_delete_ref(get_main_ref_store(r), "", "AUTO_MERGE", + NULL, REF_NO_DEREF); + save_autostash_ref(r, "MERGE_AUTOSTASH"); } void remove_branch_state(struct repository *r, int verbose) diff --git a/builtin/am.c b/builtin/am.c index d1990d7edc..e8fb27a8ef 100644 --- a/builtin/am.c +++ b/builtin/am.c @@ -1994,8 +1994,8 @@ static int fast_forward_to(struct tree *head, struct tree *remote, int reset) opts.reset = reset ? UNPACK_RESET_PROTECT_UNTRACKED : 0; opts.preserve_ignored = 0; /* FIXME: !overwrite_ignore */ opts.fn = twoway_merge; - init_tree_desc(&t[0], head->buffer, head->size); - init_tree_desc(&t[1], remote->buffer, remote->size); + init_tree_desc(&t[0], &head->object.oid, head->buffer, head->size); + init_tree_desc(&t[1], &remote->object.oid, remote->buffer, remote->size); if (unpack_trees(2, t, &opts)) { rollback_lock_file(&lock_file); @@ -2029,7 +2029,7 @@ static int merge_tree(struct tree *tree) opts.dst_index = &the_index; opts.merge = 1; opts.fn = oneway_merge; - init_tree_desc(&t[0], tree->buffer, tree->size); + init_tree_desc(&t[0], &tree->object.oid, tree->buffer, tree->size); if (unpack_trees(1, t, &opts)) { rollback_lock_file(&lock_file); diff --git a/builtin/branch.c b/builtin/branch.c index 0a32d1b6c8..cfb63cce5f 100644 --- a/builtin/branch.c +++ b/builtin/branch.c @@ -24,6 +24,7 @@ #include "ref-filter.h" #include "worktree.h" #include "help.h" +#include "advice.h" #include "commit-reach.h" static const char * const builtin_branch_usage[] = { @@ -190,9 +191,10 @@ static int check_branch_commit(const char *branchname, const char *refname, return -1; } if (!force && !branch_merged(kinds, branchname, rev, head_rev)) { - error(_("the branch '%s' is not fully merged.\n" - "If you are sure you want to delete it, " - "run 'git branch -D %s'"), branchname, branchname); + error(_("the branch '%s' is not fully merged"), branchname); + advise_if_enabled(ADVICE_FORCE_DELETE_BRANCH, + _("If you are sure you want to delete it, " + "run 'git branch -D %s'"), branchname); return -1; } return 0; diff --git a/builtin/cat-file.c b/builtin/cat-file.c index 7d4899348a..1607adb43f 100644 --- a/builtin/cat-file.c +++ b/builtin/cat-file.c @@ -106,7 +106,10 @@ static int cat_one_file(int opt, const char *exp_type, const char *obj_name, struct object_info oi = OBJECT_INFO_INIT; struct strbuf sb = STRBUF_INIT; unsigned flags = OBJECT_INFO_LOOKUP_REPLACE; - unsigned get_oid_flags = GET_OID_RECORD_PATH | GET_OID_ONLY_TO_DIE; + unsigned get_oid_flags = + GET_OID_RECORD_PATH | + GET_OID_ONLY_TO_DIE | + GET_OID_HASH_ANY; const char *path = force_path; const int opt_cw = (opt == 'c' || opt == 'w'); if (!path && opt_cw) @@ -222,7 +225,8 @@ static int cat_one_file(int opt, const char *exp_type, const char *obj_name, &size); const char *target; if (!skip_prefix(buffer, "object ", &target) || - get_oid_hex(target, &blob_oid)) + get_oid_hex_algop(target, &blob_oid, + &hash_algos[oid.algo])) die("%s not a valid tag", oid_to_hex(&oid)); free(buffer); } else @@ -511,7 +515,9 @@ static void batch_one_object(const char *obj_name, struct expand_data *data) { struct object_context ctx; - int flags = opt->follow_symlinks ? GET_OID_FOLLOW_SYMLINKS : 0; + int flags = + GET_OID_HASH_ANY | + (opt->follow_symlinks ? GET_OID_FOLLOW_SYMLINKS : 0); enum get_oid_result result; result = get_oid_with_context(the_repository, obj_name, diff --git a/builtin/checkout.c b/builtin/checkout.c index a6e30931b5..78e7c1f796 100644 --- a/builtin/checkout.c +++ b/builtin/checkout.c @@ -705,7 +705,7 @@ static int reset_tree(struct tree *tree, const struct checkout_opts *o, info->commit ? &info->commit->object.oid : null_oid(), NULL); parse_tree(tree); - init_tree_desc(&tree_desc, tree->buffer, tree->size); + init_tree_desc(&tree_desc, &tree->object.oid, tree->buffer, tree->size); switch (unpack_trees(1, &tree_desc, &opts)) { case -2: *writeout_error = 1; @@ -819,10 +819,12 @@ static int merge_working_tree(const struct checkout_opts *opts, die(_("unable to parse commit %s"), oid_to_hex(old_commit_oid)); - init_tree_desc(&trees[0], tree->buffer, tree->size); + init_tree_desc(&trees[0], &tree->object.oid, + tree->buffer, tree->size); parse_tree(new_tree); tree = new_tree; - init_tree_desc(&trees[1], tree->buffer, tree->size); + init_tree_desc(&trees[1], &tree->object.oid, + tree->buffer, tree->size); ret = unpack_trees(2, trees, &topts); clear_unpack_trees_porcelain(&topts); diff --git a/builtin/clone.c b/builtin/clone.c index 0605fa79aa..e68b5925ef 100644 --- a/builtin/clone.c +++ b/builtin/clone.c @@ -71,6 +71,7 @@ static char *remote_name = NULL; static char *option_branch = NULL; static struct string_list option_not = STRING_LIST_INIT_NODUP; static const char *real_git_dir; +static const char *ref_format; static char *option_upload_pack = "git-upload-pack"; static int option_verbosity; static int option_progress = -1; @@ -156,6 +157,8 @@ static struct option builtin_clone_options[] = { N_("any cloned submodules will be shallow")), OPT_STRING(0, "separate-git-dir", &real_git_dir, N_("gitdir"), N_("separate git dir from working tree")), + OPT_STRING(0, "ref-format", &ref_format, N_("format"), + N_("specify the reference format to use")), OPT_STRING_LIST('c', "config", &option_config, N_("key=value"), N_("set config inside the new repository")), OPT_STRING_LIST(0, "server-option", &server_options, @@ -736,7 +739,7 @@ static int checkout(int submodule_progress, int filter_submodules) if (!tree) die(_("unable to parse commit %s"), oid_to_hex(&oid)); parse_tree(tree); - init_tree_desc(&t, tree->buffer, tree->size); + init_tree_desc(&t, &tree->object.oid, tree->buffer, tree->size); if (unpack_trees(1, &t, &opts) < 0) die(_("unable to checkout working tree")); @@ -931,6 +934,7 @@ int cmd_clone(int argc, const char **argv, const char *prefix) int submodule_progress; int filter_submodules = 0; int hash_algo; + unsigned int ref_storage_format = REF_STORAGE_FORMAT_UNKNOWN; const int do_not_override_repo_unix_permissions = -1; struct transport_ls_refs_options transport_ls_refs_options = @@ -956,6 +960,12 @@ int cmd_clone(int argc, const char **argv, const char *prefix) if (option_single_branch == -1) option_single_branch = deepen ? 1 : 0; + if (ref_format) { + ref_storage_format = ref_storage_format_by_name(ref_format); + if (ref_storage_format == REF_STORAGE_FORMAT_UNKNOWN) + die(_("unknown ref storage format '%s'"), ref_format); + } + if (option_mirror) option_bare = 1; @@ -1106,7 +1116,8 @@ int cmd_clone(int argc, const char **argv, const char *prefix) * repository, and reference backends may persist that information into * their on-disk data structures. */ - init_db(git_dir, real_git_dir, option_template, GIT_HASH_UNKNOWN, NULL, + init_db(git_dir, real_git_dir, option_template, GIT_HASH_UNKNOWN, + ref_storage_format, NULL, do_not_override_repo_unix_permissions, INIT_DB_QUIET | INIT_DB_SKIP_REFDB); if (real_git_dir) { @@ -1289,9 +1300,9 @@ int cmd_clone(int argc, const char **argv, const char *prefix) * ours to the same thing. */ hash_algo = hash_algo_by_ptr(transport_get_hash_algo(transport)); - initialize_repository_version(hash_algo, 1); + initialize_repository_version(hash_algo, the_repository->ref_storage_format, 1); repo_set_hash_algo(the_repository, hash_algo); - create_reference_database(NULL, 1); + create_reference_database(the_repository->ref_storage_format, NULL, 1); /* * Before fetching from the remote, download and install bundle diff --git a/builtin/commit-graph.c b/builtin/commit-graph.c index 666ad574a4..7102ee90a0 100644 --- a/builtin/commit-graph.c +++ b/builtin/commit-graph.c @@ -21,7 +21,7 @@ N_("git commit-graph write [--object-dir <dir>] [--append]\n" \ " [--split[=<strategy>]] [--reachable | --stdin-packs | --stdin-commits]\n" \ " [--changed-paths] [--[no-]max-new-filters <n>] [--[no-]progress]\n" \ - " <split options>") + " <split-options>") static const char * builtin_commit_graph_verify_usage[] = { BUILTIN_COMMIT_GRAPH_VERIFY_USAGE, diff --git a/builtin/commit.c b/builtin/commit.c index 65196a2827..0cbf6815ea 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -332,7 +332,7 @@ static void create_base_index(const struct commit *current_head) if (!tree) die(_("failed to unpack HEAD tree object")); parse_tree(tree); - init_tree_desc(&t, tree->buffer, tree->size); + init_tree_desc(&t, &tree->object.oid, tree->buffer, tree->size); if (unpack_trees(1, &t, &opts)) exit(128); /* We've already reported the error, finish dying */ } @@ -1877,7 +1877,7 @@ int cmd_commit(int argc, const char **argv, const char *prefix) &oid, flags); } - apply_autostash(git_path_merge_autostash(the_repository)); + apply_autostash_ref(the_repository, "MERGE_AUTOSTASH"); cleanup: strbuf_release(&author_ident); diff --git a/builtin/config.c b/builtin/config.c index 11a4d4ef14..08fe36d499 100644 --- a/builtin/config.c +++ b/builtin/config.c @@ -708,30 +708,11 @@ int cmd_config(int argc, const char **argv, const char *prefix) } if (use_global_config) { - char *user_config, *xdg_config; - - git_global_config(&user_config, &xdg_config); - if (!user_config) - /* - * It is unknown if HOME/.gitconfig exists, so - * we do not know if we should write to XDG - * location; error out even if XDG_CONFIG_HOME - * is set and points at a sane location. - */ + given_config_source.file = git_global_config(); + if (!given_config_source.file) die(_("$HOME not set")); - given_config_source.scope = CONFIG_SCOPE_GLOBAL; - - if (access_or_warn(user_config, R_OK, 0) && - xdg_config && !access_or_warn(xdg_config, R_OK, 0)) { - given_config_source.file = xdg_config; - free(user_config); - } else { - given_config_source.file = user_config; - free(xdg_config); - } - } - else if (use_system_config) { + } else if (use_system_config) { given_config_source.file = git_system_config(); given_config_source.scope = CONFIG_SCOPE_SYSTEM; } else if (use_local_config) { @@ -760,7 +741,6 @@ int cmd_config(int argc, const char **argv, const char *prefix) given_config_source.scope = CONFIG_SCOPE_COMMAND; } - if (respect_includes_opt == -1) config_options.respect_includes = !given_config_source.file; else diff --git a/builtin/fast-import.c b/builtin/fast-import.c index 92eda20683..70abe942cd 100644 --- a/builtin/fast-import.c +++ b/builtin/fast-import.c @@ -1236,20 +1236,6 @@ static void *gfi_unpack_entry( return unpack_entry(the_repository, p, oe->idx.offset, &type, sizep); } -static const char *get_mode(const char *str, uint16_t *modep) -{ - unsigned char c; - uint16_t mode = 0; - - while ((c = *str++) != ' ') { - if (c < '0' || c > '7') - return NULL; - mode = (mode << 3) + (c - '0'); - } - *modep = mode; - return str; -} - static void load_tree(struct tree_entry *root) { struct object_id *oid = &root->versions[1].oid; @@ -1287,7 +1273,7 @@ static void load_tree(struct tree_entry *root) t->entries[t->entry_count++] = e; e->tree = NULL; - c = get_mode(c, &e->versions[1].mode); + c = parse_mode(c, &e->versions[1].mode); if (!c) die("Corrupt mode in %s", oid_to_hex(oid)); e->versions[0].mode = e->versions[1].mode; @@ -2276,7 +2262,7 @@ static void file_change_m(const char *p, struct branch *b) struct object_id oid; uint16_t mode, inline_data = 0; - p = get_mode(p, &mode); + p = parse_mode(p, &mode); if (!p) die("Corrupt mode: %s", command_buf.buf); switch (mode) { diff --git a/builtin/fetch.c b/builtin/fetch.c index 119f1a72ac..3aedfd1bb6 100644 --- a/builtin/fetch.c +++ b/builtin/fetch.c @@ -100,6 +100,7 @@ static struct string_list negotiation_tip = STRING_LIST_INIT_NODUP; struct fetch_config { enum display_format display_format; + int all; int prune; int prune_tags; int show_forced_updates; @@ -113,6 +114,11 @@ static int git_fetch_config(const char *k, const char *v, { struct fetch_config *fetch_config = cb; + if (!strcmp(k, "fetch.all")) { + fetch_config->all = git_config_bool(k, v); + return 0; + } + if (!strcmp(k, "fetch.prune")) { fetch_config->prune = git_config_bool(k, v); return 0; @@ -2130,7 +2136,7 @@ int cmd_fetch(int argc, const char **argv, const char *prefix) const char *bundle_uri; struct string_list list = STRING_LIST_INIT_DUP; struct remote *remote = NULL; - int all = 0, multiple = 0; + int all = -1, multiple = 0; int result = 0; int prune_tags_ok = 1; int enable_auto_gc = 1; @@ -2335,11 +2341,20 @@ int cmd_fetch(int argc, const char **argv, const char *prefix) fetch_bundle_uri(the_repository, bundle_uri, NULL)) warning(_("failed to fetch bundles from '%s'"), bundle_uri); + if (all < 0) { + /* + * no --[no-]all given; + * only use config option if no remote was explicitly specified + */ + all = (!argc) ? config.all : 0; + } + if (all) { if (argc == 1) die(_("fetch --all does not take a repository argument")); else if (argc > 1) die(_("fetch --all does not make sense with refspecs")); + (void) for_each_remote(get_one_remote_for_fetch, &list); /* do not do fetch_multiple() of one */ diff --git a/builtin/for-each-ref.c b/builtin/for-each-ref.c index 3885a9c28e..5aa879e8be 100644 --- a/builtin/for-each-ref.c +++ b/builtin/for-each-ref.c @@ -25,6 +25,7 @@ int cmd_for_each_ref(int argc, const char **argv, const char *prefix) struct ref_format format = REF_FORMAT_INIT; int from_stdin = 0; struct strvec vec = STRVEC_INIT; + unsigned int flags = FILTER_REFS_ALL; struct option opts[] = { OPT_BIT('s', "shell", &format.quote_style, @@ -93,11 +94,29 @@ int cmd_for_each_ref(int argc, const char **argv, const char *prefix) /* vec.v is NULL-terminated, just like 'argv'. */ filter.name_patterns = vec.v; } else { + size_t i; + filter.name_patterns = argv; + + /* + * Search for any empty string pattern, if it exists then we + * print all refs without any filtering. + */ + i = 0; + while (argv[i]) { + if (!argv[i][0]) { + flags = FILTER_REFS_NO_FILTER; + /* doing this removes any pattern from being matched */ + filter.name_patterns[0] = NULL; + break; + } + + i++; + } } filter.match_as_path = 1; - filter_and_format_refs(&filter, FILTER_REFS_ALL, sorting, &format); + filter_and_format_refs(&filter, flags, sorting, &format); ref_filter_clear(&filter); ref_sorting_release(sorting); diff --git a/builtin/gc.c b/builtin/gc.c index 7c11d5ebef..cb80ced6cb 100644 --- a/builtin/gc.c +++ b/builtin/gc.c @@ -1543,19 +1543,18 @@ static int maintenance_register(int argc, const char **argv, const char *prefix) if (!found) { int rc; - char *user_config = NULL, *xdg_config = NULL; + char *global_config_file = NULL; if (!config_file) { - git_global_config(&user_config, &xdg_config); - config_file = user_config; - if (!user_config) - die(_("$HOME not set")); + global_config_file = git_global_config(); + config_file = global_config_file; } + if (!config_file) + die(_("$HOME not set")); rc = git_config_set_multivar_in_file_gently( config_file, "maintenance.repo", maintpath, CONFIG_REGEX_NONE, 0); - free(user_config); - free(xdg_config); + free(global_config_file); if (rc) die(_("unable to add '%s' value of '%s'"), @@ -1612,18 +1611,18 @@ static int maintenance_unregister(int argc, const char **argv, const char *prefi if (found) { int rc; - char *user_config = NULL, *xdg_config = NULL; + char *global_config_file = NULL; + if (!config_file) { - git_global_config(&user_config, &xdg_config); - config_file = user_config; - if (!user_config) - die(_("$HOME not set")); + global_config_file = git_global_config(); + config_file = global_config_file; } + if (!config_file) + die(_("$HOME not set")); rc = git_config_set_multivar_in_file_gently( config_file, key, NULL, maintpath, CONFIG_FLAGS_MULTI_REPLACE | CONFIG_FLAGS_FIXED_VALUE); - free(user_config); - free(xdg_config); + free(global_config_file); if (rc && (!force || rc == CONFIG_NOTHING_SET)) diff --git a/builtin/grep.c b/builtin/grep.c index c8e33f9775..93191ccb47 100644 --- a/builtin/grep.c +++ b/builtin/grep.c @@ -527,7 +527,7 @@ static int grep_submodule(struct grep_opt *opt, strbuf_addstr(&base, filename); strbuf_addch(&base, '/'); - init_tree_desc(&tree, data, size); + init_tree_desc(&tree, oid, data, size); hit = grep_tree(&subopt, pathspec, &tree, &base, base.len, object_type == OBJ_COMMIT); strbuf_release(&base); @@ -571,7 +571,7 @@ static int grep_cache(struct grep_opt *opt, data = repo_read_object_file(the_repository, &ce->oid, &type, &size); - init_tree_desc(&tree, data, size); + init_tree_desc(&tree, &ce->oid, data, size); hit |= grep_tree(opt, pathspec, &tree, &name, 0, 0); strbuf_setlen(&name, name_base_len); @@ -667,7 +667,7 @@ static int grep_tree(struct grep_opt *opt, const struct pathspec *pathspec, oid_to_hex(&entry.oid)); strbuf_addch(base, '/'); - init_tree_desc(&sub, data, size); + init_tree_desc(&sub, &entry.oid, data, size); hit |= grep_tree(opt, pathspec, &sub, base, tn_len, check_attr); free(data); @@ -711,7 +711,7 @@ static int grep_object(struct grep_opt *opt, const struct pathspec *pathspec, strbuf_add(&base, name, len); strbuf_addch(&base, ':'); } - init_tree_desc(&tree, data, size); + init_tree_desc(&tree, &obj->oid, data, size); hit = grep_tree(opt, pathspec, &tree, &base, base.len, obj->type == OBJ_COMMIT); strbuf_release(&base); diff --git a/builtin/index-pack.c b/builtin/index-pack.c index 0841b6940a..1ea87e01f2 100644 --- a/builtin/index-pack.c +++ b/builtin/index-pack.c @@ -1255,6 +1255,7 @@ static void resolve_deltas(void) base_cache_limit = delta_base_cache_limit * nr_threads; if (nr_threads > 1 || getenv("GIT_FORCE_THREADS")) { init_thread(); + work_lock(); for (i = 0; i < nr_threads; i++) { int ret = pthread_create(&thread_data[i].thread, NULL, threaded_second_pass, thread_data + i); @@ -1262,6 +1263,7 @@ static void resolve_deltas(void) die(_("unable to create thread: %s"), strerror(ret)); } + work_unlock(); for (i = 0; i < nr_threads; i++) pthread_join(thread_data[i].thread, NULL); cleanup_thread(); diff --git a/builtin/init-db.c b/builtin/init-db.c index b89814a6f8..0170469b84 100644 --- a/builtin/init-db.c +++ b/builtin/init-db.c @@ -10,6 +10,8 @@ #include "object-file.h" #include "parse-options.h" #include "path.h" +#include "refs.h" +#include "repository.h" #include "setup.h" #include "strbuf.h" @@ -56,6 +58,7 @@ static int shared_callback(const struct option *opt, const char *arg, int unset) static const char *const init_db_usage[] = { N_("git init [-q | --quiet] [--bare] [--template=<template-directory>]\n" " [--separate-git-dir <git-dir>] [--object-format=<format>]\n" + " [--ref-format=<format>]\n" " [-b <branch-name> | --initial-branch=<branch-name>]\n" " [--shared[=<permissions>]] [<directory>]"), NULL @@ -75,8 +78,10 @@ int cmd_init_db(int argc, const char **argv, const char *prefix) const char *template_dir = NULL; unsigned int flags = 0; const char *object_format = NULL; + const char *ref_format = NULL; const char *initial_branch = NULL; int hash_algo = GIT_HASH_UNKNOWN; + unsigned int ref_storage_format = REF_STORAGE_FORMAT_UNKNOWN; int init_shared_repository = -1; const struct option init_db_options[] = { OPT_STRING(0, "template", &template_dir, N_("template-directory"), @@ -94,6 +99,8 @@ int cmd_init_db(int argc, const char **argv, const char *prefix) N_("override the name of the initial branch")), OPT_STRING(0, "object-format", &object_format, N_("hash"), N_("specify the hash algorithm to use")), + OPT_STRING(0, "ref-format", &ref_format, N_("format"), + N_("specify the reference format to use")), OPT_END() }; @@ -157,6 +164,12 @@ int cmd_init_db(int argc, const char **argv, const char *prefix) die(_("unknown hash algorithm '%s'"), object_format); } + if (ref_format) { + ref_storage_format = ref_storage_format_by_name(ref_format); + if (ref_storage_format == REF_STORAGE_FORMAT_UNKNOWN) + die(_("unknown ref storage format '%s'"), ref_format); + } + if (init_shared_repository != -1) set_shared_repository(init_shared_repository); @@ -235,5 +248,6 @@ int cmd_init_db(int argc, const char **argv, const char *prefix) flags |= INIT_DB_EXIST_OK; return init_db(git_dir, real_git_dir, template_dir, hash_algo, - initial_branch, init_shared_repository, flags); + ref_storage_format, initial_branch, + init_shared_repository, flags); } diff --git a/builtin/interpret-trailers.c b/builtin/interpret-trailers.c index 033bd1556c..4da4eac3b4 100644 --- a/builtin/interpret-trailers.c +++ b/builtin/interpret-trailers.c @@ -9,6 +9,7 @@ #include "gettext.h" #include "parse-options.h" #include "string-list.h" +#include "tempfile.h" #include "trailer.h" #include "config.h" @@ -44,23 +45,17 @@ static int option_parse_if_missing(const struct option *opt, return trailer_set_if_missing(opt->value, arg); } -static void new_trailers_clear(struct list_head *trailers) -{ - struct list_head *pos, *tmp; - struct new_trailer_item *item; - - list_for_each_safe(pos, tmp, trailers) { - item = list_entry(pos, struct new_trailer_item, list); - list_del(pos); - free(item); - } -} +static char *cl_separators; static int option_parse_trailer(const struct option *opt, const char *arg, int unset) { struct list_head *trailers = opt->value; - struct new_trailer_item *item; + struct strbuf tok = STRBUF_INIT; + struct strbuf val = STRBUF_INIT; + const struct trailer_conf *conf; + struct trailer_conf *conf_current = new_trailer_conf(); + ssize_t separator_pos; if (unset) { new_trailers_clear(trailers); @@ -70,12 +65,31 @@ static int option_parse_trailer(const struct option *opt, if (!arg) return -1; - item = xmalloc(sizeof(*item)); - item->text = arg; - item->where = where; - item->if_exists = if_exists; - item->if_missing = if_missing; - list_add_tail(&item->list, trailers); + separator_pos = find_separator(arg, cl_separators); + if (separator_pos) { + parse_trailer(arg, separator_pos, &tok, &val, &conf); + duplicate_trailer_conf(conf_current, conf); + + /* + * Override conf_current with settings specified via CLI flags. + */ + trailer_conf_set(where, if_exists, if_missing, conf_current); + + add_arg_item(strbuf_detach(&tok, NULL), + strbuf_detach(&val, NULL), + conf_current, + trailers); + } else { + struct strbuf sb = STRBUF_INIT; + strbuf_addstr(&sb, arg); + strbuf_trim(&sb); + error(_("empty trailer token in trailer '%.*s'"), + (int) sb.len, sb.buf); + strbuf_release(&sb); + } + + free(conf_current); + return 0; } @@ -91,10 +105,102 @@ static int parse_opt_parse(const struct option *opt, const char *arg, return 0; } +static struct tempfile *trailers_tempfile; + +static FILE *create_in_place_tempfile(const char *file) +{ + struct stat st; + struct strbuf filename_template = STRBUF_INIT; + const char *tail; + FILE *outfile; + + if (stat(file, &st)) + die_errno(_("could not stat %s"), file); + if (!S_ISREG(st.st_mode)) + die(_("file %s is not a regular file"), file); + if (!(st.st_mode & S_IWUSR)) + die(_("file %s is not writable by user"), file); + + /* Create temporary file in the same directory as the original */ + tail = strrchr(file, '/'); + if (tail) + strbuf_add(&filename_template, file, tail - file + 1); + strbuf_addstr(&filename_template, "git-interpret-trailers-XXXXXX"); + + trailers_tempfile = xmks_tempfile_m(filename_template.buf, st.st_mode); + strbuf_release(&filename_template); + outfile = fdopen_tempfile(trailers_tempfile, "w"); + if (!outfile) + die_errno(_("could not open temporary file")); + + return outfile; +} + +static void read_input_file(struct strbuf *sb, const char *file) +{ + if (file) { + if (strbuf_read_file(sb, file, 0) < 0) + die_errno(_("could not read input file '%s'"), file); + } else { + if (strbuf_read(sb, fileno(stdin), 0) < 0) + die_errno(_("could not read from stdin")); + } +} + +static void interpret_trailers(const char *file, + const struct process_trailer_options *opts, + struct list_head *arg_trailers) +{ + LIST_HEAD(head); + struct strbuf sb = STRBUF_INIT; + struct strbuf tb = STRBUF_INIT; + struct trailer_block *trailer_block; + FILE *outfile = stdout; + + read_input_file(&sb, file); + + if (opts->in_place) + outfile = create_in_place_tempfile(file); + + trailer_block = parse_trailers(sb.buf, opts, &head); + + /* Print the lines before the trailer block */ + if (!opts->only_trailers) + fwrite(sb.buf, 1, trailer_block_start(trailer_block), outfile); + + if (!opts->only_trailers && !blank_line_before_trailer_block(trailer_block)) + fprintf(outfile, "\n"); + + + if (!opts->only_input) { + process_trailers_lists(&head, arg_trailers); + } + + /* Print trailer block. */ + format_trailers(&head, opts, &tb); + fwrite(tb.buf, 1, tb.len, outfile); + strbuf_release(&tb); + + free_trailers(&head); + + /* Print the lines after the trailer block as is */ + if (!opts->only_trailers) + fwrite(sb.buf + trailer_block_end(trailer_block), + 1, sb.len - trailer_block_end(trailer_block), outfile); + trailer_block_release(trailer_block); + + if (opts->in_place) + if (rename_tempfile(&trailers_tempfile, file)) + die_errno(_("could not rename temporary file to %s"), file); + + strbuf_release(&sb); +} + int cmd_interpret_trailers(int argc, const char **argv, const char *prefix) { struct process_trailer_options opts = PROCESS_TRAILER_OPTIONS_INIT; - LIST_HEAD(trailers); + LIST_HEAD(configured_trailers); + LIST_HEAD(arg_trailers); struct option options[] = { OPT_BOOL(0, "in-place", &opts.in_place, N_("edit files in place")), @@ -113,33 +219,48 @@ int cmd_interpret_trailers(int argc, const char **argv, const char *prefix) OPT_CALLBACK_F(0, "parse", &opts, NULL, N_("alias for --only-trailers --only-input --unfold"), PARSE_OPT_NOARG | PARSE_OPT_NONEG, parse_opt_parse), OPT_BOOL(0, "no-divider", &opts.no_divider, N_("do not treat \"---\" as the end of input")), - OPT_CALLBACK(0, "trailer", &trailers, N_("trailer"), + OPT_CALLBACK(0, "trailer", &arg_trailers, N_("trailer"), N_("trailer(s) to add"), option_parse_trailer), OPT_END() }; git_config(git_default_config, NULL); + trailer_config_init(); + + if (!opts.only_input) { + parse_trailers_from_config(&configured_trailers); + } + + /* + * In command-line arguments, '=' is accepted (in addition to the + * separators that are defined). + */ + cl_separators = xstrfmt("=%s", default_separators()); argc = parse_options(argc, argv, prefix, options, git_interpret_trailers_usage, 0); - if (opts.only_input && !list_empty(&trailers)) + free(cl_separators); + + if (opts.only_input && !list_empty(&arg_trailers)) usage_msg_opt( _("--trailer with --only-input does not make sense"), git_interpret_trailers_usage, options); + list_splice(&configured_trailers, &arg_trailers); + if (argc) { int i; for (i = 0; i < argc; i++) - process_trailers(argv[i], &opts, &trailers); + interpret_trailers(argv[i], &opts, &arg_trailers); } else { if (opts.in_place) die(_("no input file given for in-place editing")); - process_trailers(NULL, &opts, &trailers); + interpret_trailers(NULL, &opts, &arg_trailers); } - new_trailers_clear(&trailers); + new_trailers_clear(&arg_trailers); return 0; } diff --git a/builtin/ls-tree.c b/builtin/ls-tree.c index e4a891337c..61a2965c8a 100644 --- a/builtin/ls-tree.c +++ b/builtin/ls-tree.c @@ -375,6 +375,7 @@ int cmd_ls_tree(int argc, const char **argv, const char *prefix) OPT_END() }; struct ls_tree_cmdmode_to_fmt *m2f = ls_tree_cmdmode_format; + struct object_context obj_context; int ret; git_config(git_default_config, NULL); @@ -406,7 +407,9 @@ int cmd_ls_tree(int argc, const char **argv, const char *prefix) ls_tree_usage, ls_tree_options); if (argc < 1) usage_with_options(ls_tree_usage, ls_tree_options); - if (repo_get_oid(the_repository, argv[0], &oid)) + if (get_oid_with_context(the_repository, argv[0], + GET_OID_HASH_ANY, &oid, + &obj_context)) die("Not a valid object name %s", argv[0]); /* diff --git a/builtin/merge.c b/builtin/merge.c index ebbe05033e..a5237d9cc2 100644 --- a/builtin/merge.c +++ b/builtin/merge.c @@ -476,7 +476,7 @@ static void finish(struct commit *head_commit, run_hooks_l("post-merge", squash ? "1" : "0", NULL); if (new_head) - apply_autostash(git_path_merge_autostash(the_repository)); + apply_autostash_ref(the_repository, "MERGE_AUTOSTASH"); strbuf_release(&reflog_message); } @@ -678,7 +678,8 @@ static int read_tree_trivial(struct object_id *common, struct object_id *head, cache_tree_free(&the_index.cache_tree); for (i = 0; i < nr_trees; i++) { parse_tree(trees[i]); - init_tree_desc(t+i, trees[i]->buffer, trees[i]->size); + init_tree_desc(t+i, &trees[i]->object.oid, + trees[i]->buffer, trees[i]->size); } if (unpack_trees(nr_trees, t, &opts)) return -1; @@ -1315,7 +1316,8 @@ int cmd_merge(int argc, const char **argv, const char *prefix) if (abort_current_merge) { int nargc = 2; const char *nargv[] = {"reset", "--merge", NULL}; - struct strbuf stash_oid = STRBUF_INIT; + char stash_oid_hex[GIT_MAX_HEXSZ + 1]; + struct object_id stash_oid = {0}; if (orig_argc != 2) usage_msg_opt(_("--abort expects no arguments"), @@ -1324,17 +1326,17 @@ int cmd_merge(int argc, const char **argv, const char *prefix) if (!file_exists(git_path_merge_head(the_repository))) die(_("There is no merge to abort (MERGE_HEAD missing).")); - if (read_oneliner(&stash_oid, git_path_merge_autostash(the_repository), - READ_ONELINER_SKIP_IF_EMPTY)) - unlink(git_path_merge_autostash(the_repository)); + if (!read_ref("MERGE_AUTOSTASH", &stash_oid)) + delete_ref("", "MERGE_AUTOSTASH", &stash_oid, REF_NO_DEREF); /* Invoke 'git reset --merge' */ ret = cmd_reset(nargc, nargv, prefix); - if (stash_oid.len) - apply_autostash_oid(stash_oid.buf); + if (!is_null_oid(&stash_oid)) { + oid_to_hex_r(stash_oid_hex, &stash_oid); + apply_autostash_oid(stash_oid_hex); + } - strbuf_release(&stash_oid); goto done; } @@ -1563,13 +1565,12 @@ int cmd_merge(int argc, const char **argv, const char *prefix) } if (autostash) - create_autostash(the_repository, - git_path_merge_autostash(the_repository)); + create_autostash_ref(the_repository, "MERGE_AUTOSTASH"); if (checkout_fast_forward(the_repository, &head_commit->object.oid, &commit->object.oid, overwrite_ignore)) { - apply_autostash(git_path_merge_autostash(the_repository)); + apply_autostash_ref(the_repository, "MERGE_AUTOSTASH"); ret = 1; goto done; } @@ -1655,8 +1656,7 @@ int cmd_merge(int argc, const char **argv, const char *prefix) die_ff_impossible(); if (autostash) - create_autostash(the_repository, - git_path_merge_autostash(the_repository)); + create_autostash_ref(the_repository, "MERGE_AUTOSTASH"); /* We are going to make a new commit. */ git_committer_info(IDENT_STRICT); @@ -1741,7 +1741,7 @@ int cmd_merge(int argc, const char **argv, const char *prefix) else fprintf(stderr, _("Merge with strategy %s failed.\n"), use_strategies[0]->name); - apply_autostash(git_path_merge_autostash(the_repository)); + apply_autostash_ref(the_repository, "MERGE_AUTOSTASH"); ret = 2; goto done; } else if (best_strategy == wt_strategy) diff --git a/builtin/pack-objects.c b/builtin/pack-objects.c index 5c8bfe1035..b98ef7c749 100644 --- a/builtin/pack-objects.c +++ b/builtin/pack-objects.c @@ -218,13 +218,19 @@ static int thin; static int num_preferred_base; static struct progress *progress_state; -static struct packed_git *reuse_packfile; +static struct bitmapped_pack *reuse_packfiles; +static size_t reuse_packfiles_nr; +static size_t reuse_packfiles_used_nr; static uint32_t reuse_packfile_objects; static struct bitmap *reuse_packfile_bitmap; static int use_bitmap_index_default = 1; static int use_bitmap_index = -1; -static int allow_pack_reuse = 1; +static enum { + NO_PACK_REUSE = 0, + SINGLE_PACK_REUSE, + MULTI_PACK_REUSE, +} allow_pack_reuse = SINGLE_PACK_REUSE; static enum { WRITE_BITMAP_FALSE = 0, WRITE_BITMAP_QUIET, @@ -1010,7 +1016,9 @@ static off_t find_reused_offset(off_t where) return reused_chunks[lo-1].difference; } -static void write_reused_pack_one(size_t pos, struct hashfile *out, +static void write_reused_pack_one(struct packed_git *reuse_packfile, + size_t pos, struct hashfile *out, + off_t pack_start, struct pack_window **w_curs) { off_t offset, next, cur; @@ -1020,7 +1028,8 @@ static void write_reused_pack_one(size_t pos, struct hashfile *out, offset = pack_pos_to_offset(reuse_packfile, pos); next = pack_pos_to_offset(reuse_packfile, pos + 1); - record_reused_object(offset, offset - hashfile_total(out)); + record_reused_object(offset, + offset - (hashfile_total(out) - pack_start)); cur = offset; type = unpack_object_header(reuse_packfile, w_curs, &cur, &size); @@ -1088,41 +1097,93 @@ static void write_reused_pack_one(size_t pos, struct hashfile *out, copy_pack_data(out, reuse_packfile, w_curs, offset, next - offset); } -static size_t write_reused_pack_verbatim(struct hashfile *out, +static size_t write_reused_pack_verbatim(struct bitmapped_pack *reuse_packfile, + struct hashfile *out, + off_t pack_start, struct pack_window **w_curs) { - size_t pos = 0; + size_t pos = reuse_packfile->bitmap_pos; + size_t end; + + if (pos % BITS_IN_EWORD) { + size_t word_pos = (pos / BITS_IN_EWORD); + size_t offset = pos % BITS_IN_EWORD; + size_t last; + eword_t word = reuse_packfile_bitmap->words[word_pos]; + + if (offset + reuse_packfile->bitmap_nr < BITS_IN_EWORD) + last = offset + reuse_packfile->bitmap_nr; + else + last = BITS_IN_EWORD; + + for (; offset < last; offset++) { + if (word >> offset == 0) + return word_pos; + if (!bitmap_get(reuse_packfile_bitmap, + word_pos * BITS_IN_EWORD + offset)) + return word_pos; + } - while (pos < reuse_packfile_bitmap->word_alloc && - reuse_packfile_bitmap->words[pos] == (eword_t)~0) - pos++; + pos += BITS_IN_EWORD - (pos % BITS_IN_EWORD); + } + + /* + * Now we're going to copy as many whole eword_t's as possible. + * "end" is the index of the last whole eword_t we copy, but + * there may be additional bits to process. Those are handled + * individually by write_reused_pack(). + * + * Begin by advancing to the first word boundary in range of the + * bit positions occupied by objects in "reuse_packfile". Then + * pick the last word boundary in the same range. If we have at + * least one word's worth of bits to process, continue on. + */ + end = reuse_packfile->bitmap_pos + reuse_packfile->bitmap_nr; + if (end % BITS_IN_EWORD) + end -= end % BITS_IN_EWORD; + if (pos >= end) + return reuse_packfile->bitmap_pos / BITS_IN_EWORD; - if (pos) { - off_t to_write; + while (pos < end && + reuse_packfile_bitmap->words[pos / BITS_IN_EWORD] == (eword_t)~0) + pos += BITS_IN_EWORD; - written = (pos * BITS_IN_EWORD); - to_write = pack_pos_to_offset(reuse_packfile, written) - - sizeof(struct pack_header); + if (pos > end) + pos = end; + + if (reuse_packfile->bitmap_pos < pos) { + off_t pack_start_off = pack_pos_to_offset(reuse_packfile->p, 0); + off_t pack_end_off = pack_pos_to_offset(reuse_packfile->p, + pos - reuse_packfile->bitmap_pos); + + written += pos - reuse_packfile->bitmap_pos; /* We're recording one chunk, not one object. */ - record_reused_object(sizeof(struct pack_header), 0); + record_reused_object(pack_start_off, + pack_start_off - (hashfile_total(out) - pack_start)); hashflush(out); - copy_pack_data(out, reuse_packfile, w_curs, - sizeof(struct pack_header), to_write); + copy_pack_data(out, reuse_packfile->p, w_curs, + pack_start_off, pack_end_off - pack_start_off); display_progress(progress_state, written); } - return pos; + if (pos % BITS_IN_EWORD) + BUG("attempted to jump past a word boundary to %"PRIuMAX, + (uintmax_t)pos); + return pos / BITS_IN_EWORD; } -static void write_reused_pack(struct hashfile *f) +static void write_reused_pack(struct bitmapped_pack *reuse_packfile, + struct hashfile *f) { - size_t i = 0; + size_t i = reuse_packfile->bitmap_pos / BITS_IN_EWORD; uint32_t offset; + off_t pack_start = hashfile_total(f) - sizeof(struct pack_header); struct pack_window *w_curs = NULL; if (allow_ofs_delta) - i = write_reused_pack_verbatim(f, &w_curs); + i = write_reused_pack_verbatim(reuse_packfile, f, pack_start, + &w_curs); for (; i < reuse_packfile_bitmap->word_alloc; ++i) { eword_t word = reuse_packfile_bitmap->words[i]; @@ -1133,16 +1194,23 @@ static void write_reused_pack(struct hashfile *f) break; offset += ewah_bit_ctz64(word >> offset); + if (pos + offset < reuse_packfile->bitmap_pos) + continue; + if (pos + offset >= reuse_packfile->bitmap_pos + reuse_packfile->bitmap_nr) + goto done; /* * Can use bit positions directly, even for MIDX * bitmaps. See comment in try_partial_reuse() * for why. */ - write_reused_pack_one(pos + offset, f, &w_curs); + write_reused_pack_one(reuse_packfile->p, + pos + offset - reuse_packfile->bitmap_pos, + f, pack_start, &w_curs); display_progress(progress_state, ++written); } } +done: unuse_pack(&w_curs); } @@ -1194,9 +1262,14 @@ static void write_pack_file(void) offset = write_pack_header(f, nr_remaining); - if (reuse_packfile) { + if (reuse_packfiles_nr) { assert(pack_to_stdout); - write_reused_pack(f); + for (j = 0; j < reuse_packfiles_nr; j++) { + reused_chunks_nr = 0; + write_reused_pack(&reuse_packfiles[j], f); + if (reused_chunks_nr) + reuse_packfiles_used_nr++; + } offset = hashfile_total(f); } @@ -1753,7 +1826,8 @@ static void add_pbase_object(struct tree_desc *tree, tree = pbase_tree_get(&entry.oid); if (!tree) return; - init_tree_desc(&sub, tree->tree_data, tree->tree_size); + init_tree_desc(&sub, &tree->oid, + tree->tree_data, tree->tree_size); add_pbase_object(&sub, down, downlen, fullname); pbase_tree_put(tree); @@ -1813,7 +1887,8 @@ static void add_preferred_base_object(const char *name) } else { struct tree_desc tree; - init_tree_desc(&tree, it->pcache.tree_data, it->pcache.tree_size); + init_tree_desc(&tree, &it->pcache.oid, + it->pcache.tree_data, it->pcache.tree_size); add_pbase_object(&tree, name, cmplen, name); } } @@ -3172,7 +3247,19 @@ static int git_pack_config(const char *k, const char *v, return 0; } if (!strcmp(k, "pack.allowpackreuse")) { - allow_pack_reuse = git_config_bool(k, v); + int res = git_parse_maybe_bool_text(v); + if (res < 0) { + if (!strcasecmp(v, "single")) + allow_pack_reuse = SINGLE_PACK_REUSE; + else if (!strcasecmp(v, "multi")) + allow_pack_reuse = MULTI_PACK_REUSE; + else + die(_("invalid pack.allowPackReuse value: '%s'"), v); + } else if (res) { + allow_pack_reuse = SINGLE_PACK_REUSE; + } else { + allow_pack_reuse = NO_PACK_REUSE; + } return 0; } if (!strcmp(k, "pack.threads")) { @@ -3931,7 +4018,7 @@ static void loosen_unused_packed_objects(void) */ static int pack_options_allow_reuse(void) { - return allow_pack_reuse && + return allow_pack_reuse != NO_PACK_REUSE && pack_to_stdout && !ignore_packed_keep_on_disk && !ignore_packed_keep_in_core && @@ -3944,13 +4031,18 @@ static int get_object_list_from_bitmap(struct rev_info *revs) if (!(bitmap_git = prepare_bitmap_walk(revs, 0))) return -1; - if (pack_options_allow_reuse() && - !reuse_partial_packfile_from_bitmap( - bitmap_git, - &reuse_packfile, - &reuse_packfile_objects, - &reuse_packfile_bitmap)) { - assert(reuse_packfile_objects); + if (pack_options_allow_reuse()) + reuse_partial_packfile_from_bitmap(bitmap_git, + &reuse_packfiles, + &reuse_packfiles_nr, + &reuse_packfile_bitmap, + allow_pack_reuse == MULTI_PACK_REUSE); + + if (reuse_packfiles) { + reuse_packfile_objects = bitmap_popcount(reuse_packfile_bitmap); + if (!reuse_packfile_objects) + BUG("expected non-empty reuse bitmap"); + nr_result += reuse_packfile_objects; nr_seen += reuse_packfile_objects; display_progress(progress_state, nr_seen); @@ -4518,11 +4610,20 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix) fprintf_ln(stderr, _("Total %"PRIu32" (delta %"PRIu32")," " reused %"PRIu32" (delta %"PRIu32")," - " pack-reused %"PRIu32), + " pack-reused %"PRIu32" (from %"PRIuMAX")"), written, written_delta, reused, reused_delta, - reuse_packfile_objects); + reuse_packfile_objects, + (uintmax_t)reuse_packfiles_used_nr); + + trace2_data_intmax("pack-objects", the_repository, "written", written); + trace2_data_intmax("pack-objects", the_repository, "written/delta", written_delta); + trace2_data_intmax("pack-objects", the_repository, "reused", reused); + trace2_data_intmax("pack-objects", the_repository, "reused/delta", reused_delta); + trace2_data_intmax("pack-objects", the_repository, "pack-reused", reuse_packfile_objects); + trace2_data_intmax("pack-objects", the_repository, "packs-reused", reuse_packfiles_used_nr); cleanup: + clear_packing_data(&to_pack); list_objects_filter_release(&filter_options); strvec_clear(&rp); diff --git a/builtin/read-tree.c b/builtin/read-tree.c index 20e7db1973..c1275ed5b6 100644 --- a/builtin/read-tree.c +++ b/builtin/read-tree.c @@ -262,7 +262,7 @@ int cmd_read_tree(int argc, const char **argv, const char *cmd_prefix) for (i = 0; i < nr_trees; i++) { struct tree *tree = trees[i]; parse_tree(tree); - init_tree_desc(t+i, tree->buffer, tree->size); + init_tree_desc(t+i, &tree->object.oid, tree->buffer, tree->size); } if (unpack_trees(nr_trees, t, &opts)) return 128; diff --git a/builtin/rebase.c b/builtin/rebase.c index 995818c28d..5b086f651a 100644 --- a/builtin/rebase.c +++ b/builtin/rebase.c @@ -515,7 +515,7 @@ static int finish_rebase(struct rebase_options *opts) int ret = 0; delete_ref(NULL, "REBASE_HEAD", NULL, REF_NO_DEREF); - unlink(git_path_auto_merge(the_repository)); + delete_ref(NULL, "AUTO_MERGE", NULL, REF_NO_DEREF); apply_autostash(state_dir_path("autostash", opts)); /* * We ignore errors in 'git maintenance run --auto', since the diff --git a/builtin/rev-parse.c b/builtin/rev-parse.c index 917f122440..73c9afadb1 100644 --- a/builtin/rev-parse.c +++ b/builtin/rev-parse.c @@ -25,6 +25,7 @@ #include "submodule.h" #include "commit-reach.h" #include "shallow.h" +#include "object-file-convert.h" #define DO_REVS 1 #define DO_NOREV 2 @@ -675,6 +676,8 @@ static void print_path(const char *path, const char *prefix, enum format_type fo int cmd_rev_parse(int argc, const char **argv, const char *prefix) { int i, as_is = 0, verify = 0, quiet = 0, revs_count = 0, type = 0; + const struct git_hash_algo *output_algo = NULL; + const struct git_hash_algo *compat = NULL; int did_repo_setup = 0; int has_dashdash = 0; int output_prefix = 0; @@ -746,6 +749,7 @@ int cmd_rev_parse(int argc, const char **argv, const char *prefix) prepare_repo_settings(the_repository); the_repository->settings.command_requires_full_index = 0; + compat = the_repository->compat_hash_algo; } if (!strcmp(arg, "--")) { @@ -833,6 +837,22 @@ int cmd_rev_parse(int argc, const char **argv, const char *prefix) flags |= GET_OID_QUIETLY; continue; } + if (opt_with_value(arg, "--output-object-format", &arg)) { + if (!arg) + die(_("no object format specified")); + if (!strcmp(arg, the_hash_algo->name) || + !strcmp(arg, "storage")) { + flags |= GET_OID_HASH_ANY; + output_algo = the_hash_algo; + continue; + } + else if (compat && !strcmp(arg, compat->name)) { + flags |= GET_OID_HASH_ANY; + output_algo = compat; + continue; + } + else die(_("unsupported object format: %s"), arg); + } if (opt_with_value(arg, "--short", &arg)) { filter &= ~(DO_FLAGS|DO_NOREV); verify = 1; @@ -882,7 +902,7 @@ int cmd_rev_parse(int argc, const char **argv, const char *prefix) continue; } if (skip_prefix(arg, "--disambiguate=", &arg)) { - repo_for_each_abbrev(the_repository, arg, + repo_for_each_abbrev(the_repository, arg, the_hash_algo, show_abbrev, NULL); continue; } @@ -1062,6 +1082,10 @@ int cmd_rev_parse(int argc, const char **argv, const char *prefix) puts(the_hash_algo->name); continue; } + if (!strcmp(arg, "--show-ref-format")) { + puts(ref_storage_format_to_name(the_repository->ref_storage_format)); + continue; + } if (!strcmp(arg, "--end-of-options")) { seen_end_of_options = 1; if (filter & (DO_FLAGS | DO_REVS)) @@ -1086,6 +1110,9 @@ int cmd_rev_parse(int argc, const char **argv, const char *prefix) } if (!get_oid_with_context(the_repository, name, flags, &oid, &unused)) { + if (output_algo) + repo_oid_to_algop(the_repository, &oid, + output_algo, &oid); if (verify) revs_count++; else diff --git a/builtin/send-pack.c b/builtin/send-pack.c index b7183be970..3df9eaad09 100644 --- a/builtin/send-pack.c +++ b/builtin/send-pack.c @@ -333,6 +333,7 @@ int cmd_send_pack(int argc, const char **argv, const char *prefix) } if (!ret && !transport_refs_pushed(remote_refs)) + /* stable plumbing output; do not modify or localize */ fprintf(stderr, "Everything up-to-date\n"); return ret; diff --git a/builtin/shortlog.c b/builtin/shortlog.c index 1307ed2b88..dc8fd5a553 100644 --- a/builtin/shortlog.c +++ b/builtin/shortlog.c @@ -172,7 +172,7 @@ static void insert_records_from_trailers(struct shortlog *log, const char *oneline) { struct trailer_iterator iter; - const char *commit_buffer, *body; + const char *commit_buffer, *body, *value; struct strbuf ident = STRBUF_INIT; if (!log->trailers.nr) @@ -190,7 +190,10 @@ static void insert_records_from_trailers(struct shortlog *log, trailer_iterator_init(&iter, body); while (trailer_iterator_advance(&iter)) { - const char *value = iter.val.buf; + if (!iter.is_trailer) + continue; + + value = iter.val.buf; if (!string_list_has_string(&log->trailers, iter.key.buf)) continue; diff --git a/builtin/show-ref.c b/builtin/show-ref.c index aaa2c39b2f..79955c2856 100644 --- a/builtin/show-ref.c +++ b/builtin/show-ref.c @@ -238,7 +238,7 @@ static int cmd_show_ref__exists(const char **refs) if (refs_read_raw_ref(get_main_ref_store(the_repository), ref, &unused_oid, &unused_referent, &unused_type, &failure_errno)) { - if (failure_errno == ENOENT) { + if (failure_errno == ENOENT || failure_errno == EISDIR) { error(_("reference does not exist")); ret = 2; } else { diff --git a/builtin/stash.c b/builtin/stash.c index b2813c614c..01766d4dda 100644 --- a/builtin/stash.c +++ b/builtin/stash.c @@ -284,7 +284,7 @@ static int reset_tree(struct object_id *i_tree, int update, int reset) if (parse_tree(tree)) return -1; - init_tree_desc(t, tree->buffer, tree->size); + init_tree_desc(t, &tree->object.oid, tree->buffer, tree->size); opts.head_idx = 1; opts.src_index = &the_index; @@ -870,7 +870,8 @@ static void diff_include_untracked(const struct stash_info *info, struct diff_op tree[i] = parse_tree_indirect(oid[i]); if (parse_tree(tree[i]) < 0) die(_("failed to parse tree")); - init_tree_desc(&tree_desc[i], tree[i]->buffer, tree[i]->size); + init_tree_desc(&tree_desc[i], &tree[i]->object.oid, + tree[i]->buffer, tree[i]->size); } unpack_tree_opt.head_idx = -1; diff --git a/builtin/tag.c b/builtin/tag.c index f036cf32f5..264817a628 100644 --- a/builtin/tag.c +++ b/builtin/tag.c @@ -27,6 +27,7 @@ #include "ref-filter.h" #include "date.h" #include "write-or-die.h" +#include "object-file-convert.h" static const char * const git_tag_usage[] = { N_("git tag [-a | -s | -u <key-id>] [-f] [-m <msg> | -F <file>] [-e]\n" @@ -151,9 +152,43 @@ static int verify_tag(const char *name, const char *ref UNUSED, return 0; } -static int do_sign(struct strbuf *buffer) +static int do_sign(struct strbuf *buffer, struct object_id **compat_oid, + struct object_id *compat_oid_buf) { - return sign_buffer(buffer, buffer, get_signing_key()); + const struct git_hash_algo *compat = the_repository->compat_hash_algo; + struct strbuf sig = STRBUF_INIT, compat_sig = STRBUF_INIT; + struct strbuf compat_buf = STRBUF_INIT; + const char *keyid = get_signing_key(); + int ret = -1; + + if (sign_buffer(buffer, &sig, keyid)) + return -1; + + if (compat) { + const struct git_hash_algo *algo = the_repository->hash_algo; + + if (convert_object_file(&compat_buf, algo, compat, + buffer->buf, buffer->len, OBJ_TAG, 1)) + goto out; + if (sign_buffer(&compat_buf, &compat_sig, keyid)) + goto out; + add_header_signature(&compat_buf, &sig, algo); + strbuf_addbuf(&compat_buf, &compat_sig); + hash_object_file(compat, compat_buf.buf, compat_buf.len, + OBJ_TAG, compat_oid_buf); + *compat_oid = compat_oid_buf; + } + + if (compat_sig.len) + add_header_signature(buffer, &compat_sig, compat); + + strbuf_addbuf(buffer, &sig); + ret = 0; +out: + strbuf_release(&sig); + strbuf_release(&compat_sig); + strbuf_release(&compat_buf); + return ret; } static const char tag_template[] = @@ -226,9 +261,11 @@ static void write_tag_body(int fd, const struct object_id *oid) static int build_tag_object(struct strbuf *buf, int sign, struct object_id *result) { - if (sign && do_sign(buf) < 0) + struct object_id *compat_oid = NULL, compat_oid_buf; + if (sign && do_sign(buf, &compat_oid, &compat_oid_buf) < 0) return error(_("unable to sign the tag")); - if (write_object_file(buf->buf, buf->len, OBJ_TAG, result) < 0) + if (write_object_file_flags(buf->buf, buf->len, OBJ_TAG, result, + compat_oid, 0) < 0) return error(_("unable to write tag file")); return 0; } diff --git a/builtin/var.c b/builtin/var.c index 8cf7dd9e2e..cf5567208a 100644 --- a/builtin/var.c +++ b/builtin/var.c @@ -90,7 +90,7 @@ static char *git_config_val_global(int ident_flag UNUSED) char *user, *xdg; size_t unused; - git_global_config(&user, &xdg); + git_global_config_paths(&user, &xdg); if (xdg && *xdg) { normalize_path_copy(xdg, xdg); strbuf_addf(&buf, "%s\n", xdg); diff --git a/builtin/worktree.c b/builtin/worktree.c index cac83a9419..6d7da11746 100644 --- a/builtin/worktree.c +++ b/builtin/worktree.c @@ -416,7 +416,6 @@ static int add_worktree(const char *path, const char *refname, struct strbuf sb_git = STRBUF_INIT, sb_repo = STRBUF_INIT; struct strbuf sb = STRBUF_INIT, realpath = STRBUF_INIT; const char *name; - struct child_process cp = CHILD_PROCESS_INIT; struct strvec child_env = STRVEC_INIT; unsigned int counter = 0; int len, ret; @@ -424,7 +423,8 @@ static int add_worktree(const char *path, const char *refname, struct commit *commit = NULL; int is_branch = 0; struct strbuf sb_name = STRBUF_INIT; - struct worktree **worktrees; + struct worktree **worktrees, *wt = NULL; + struct ref_store *wt_refs; worktrees = get_worktrees(); check_candidate_path(path, opts->force, worktrees, "add"); @@ -495,21 +495,33 @@ static int add_worktree(const char *path, const char *refname, strbuf_realpath(&realpath, get_git_common_dir(), 1); write_file(sb_git.buf, "gitdir: %s/worktrees/%s", realpath.buf, name); - /* - * This is to keep resolve_ref() happy. We need a valid HEAD - * or is_git_directory() will reject the directory. Any value which - * looks like an object ID will do since it will be immediately - * replaced by the symbolic-ref or update-ref invocation in the new - * worktree. - */ - strbuf_reset(&sb); - strbuf_addf(&sb, "%s/HEAD", sb_repo.buf); - write_file(sb.buf, "%s", oid_to_hex(null_oid())); strbuf_reset(&sb); strbuf_addf(&sb, "%s/commondir", sb_repo.buf); write_file(sb.buf, "../.."); /* + * Set up the ref store of the worktree and create the HEAD reference. + */ + wt = get_linked_worktree(name, 1); + if (!wt) { + ret = error(_("could not find created worktree '%s'"), name); + goto done; + } + wt_refs = get_worktree_ref_store(wt); + + ret = refs_init_db(wt_refs, REFS_INIT_DB_IS_WORKTREE, &sb); + if (ret) + goto done; + + if (!is_branch && commit) + ret = refs_update_ref(wt_refs, NULL, "HEAD", &commit->object.oid, + NULL, 0, UPDATE_REFS_MSG_ON_ERR); + else + ret = refs_create_symref(wt_refs, "HEAD", symref.buf, NULL); + if (ret) + goto done; + + /* * If the current worktree has sparse-checkout enabled, then copy * the sparse-checkout patterns from the current worktree. */ @@ -526,22 +538,6 @@ static int add_worktree(const char *path, const char *refname, strvec_pushf(&child_env, "%s=%s", GIT_DIR_ENVIRONMENT, sb_git.buf); strvec_pushf(&child_env, "%s=%s", GIT_WORK_TREE_ENVIRONMENT, path); - cp.git_cmd = 1; - - if (!is_branch && commit) { - strvec_pushl(&cp.args, "update-ref", "HEAD", - oid_to_hex(&commit->object.oid), NULL); - } else { - strvec_pushl(&cp.args, "symbolic-ref", "HEAD", - symref.buf, NULL); - if (opts->quiet) - strvec_push(&cp.args, "--quiet"); - } - - strvec_pushv(&cp.env, child_env.v); - ret = run_command(&cp); - if (ret) - goto done; if (opts->orphan && (ret = make_worktree_orphan(refname, opts, &child_env))) @@ -587,6 +583,7 @@ done: strbuf_release(&sb_git); strbuf_release(&sb_name); strbuf_release(&realpath); + free_worktree(wt); return ret; } diff --git a/cache-tree.c b/cache-tree.c index 64678fe199..06886af90c 100644 --- a/cache-tree.c +++ b/cache-tree.c @@ -447,7 +447,7 @@ static int update_one(struct cache_tree *it, hash_object_file(the_hash_algo, buffer.buf, buffer.len, OBJ_TREE, &it->oid); } else if (write_object_file_flags(buffer.buf, buffer.len, OBJ_TREE, - &it->oid, flags & WRITE_TREE_SILENT + &it->oid, NULL, flags & WRITE_TREE_SILENT ? HASH_SILENT : 0)) { strbuf_release(&buffer); return -1; @@ -769,7 +769,7 @@ static void prime_cache_tree_rec(struct repository *r, oidcpy(&it->oid, &tree->object.oid); - init_tree_desc(&desc, tree->buffer, tree->size); + init_tree_desc(&desc, &tree->object.oid, tree->buffer, tree->size); cnt = 0; while (tree_entry(&desc, &entry)) { if (!S_ISDIR(entry.mode)) diff --git a/chunk-format.c b/chunk-format.c index cdc7f39b70..be078dcca8 100644 --- a/chunk-format.c +++ b/chunk-format.c @@ -163,6 +163,10 @@ int read_table_of_contents(struct chunkfile *cf, struct pair_chunk_data { const unsigned char **p; size_t *size; + + /* for pair_chunk_expect() only */ + size_t record_size; + size_t record_nr; }; static int pair_chunk_fn(const unsigned char *chunk_start, @@ -175,6 +179,17 @@ static int pair_chunk_fn(const unsigned char *chunk_start, return 0; } +static int pair_chunk_expect_fn(const unsigned char *chunk_start, + size_t chunk_size, + void *data) +{ + struct pair_chunk_data *pcd = data; + if (chunk_size / pcd->record_size != pcd->record_nr) + return -1; + *pcd->p = chunk_start; + return 0; +} + int pair_chunk(struct chunkfile *cf, uint32_t chunk_id, const unsigned char **p, @@ -184,6 +199,20 @@ int pair_chunk(struct chunkfile *cf, return read_chunk(cf, chunk_id, pair_chunk_fn, &pcd); } +int pair_chunk_expect(struct chunkfile *cf, + uint32_t chunk_id, + const unsigned char **p, + size_t record_size, + size_t record_nr) +{ + struct pair_chunk_data pcd = { + .p = p, + .record_size = record_size, + .record_nr = record_nr, + }; + return read_chunk(cf, chunk_id, pair_chunk_expect_fn, &pcd); +} + int read_chunk(struct chunkfile *cf, uint32_t chunk_id, chunk_read_fn fn, diff --git a/chunk-format.h b/chunk-format.h index 14b76180ef..10806d7a9a 100644 --- a/chunk-format.h +++ b/chunk-format.h @@ -17,7 +17,8 @@ struct chunkfile; * * If reading a file, use a NULL 'struct hashfile *' and then call * read_table_of_contents(). Supply the memory-mapped data to the - * pair_chunk() or read_chunk() methods, as appropriate. + * pair_chunk(), pair_chunk_expect(), or read_chunk() methods, as + * appropriate. * * DO NOT MIX THESE MODES. Use different 'struct chunkfile' instances * for reading and writing. @@ -54,6 +55,16 @@ int pair_chunk(struct chunkfile *cf, const unsigned char **p, size_t *size); +/* + * Similar to 'pair_chunk', but used for callers who are reading a chunk + * with a known number of fixed-width records. + */ +int pair_chunk_expect(struct chunkfile *cf, + uint32_t chunk_id, + const unsigned char **p, + size_t record_size, + size_t record_nr); + typedef int (*chunk_read_fn)(const unsigned char *chunk_start, size_t chunk_size, void *data); /* diff --git a/ci/install-dependencies.sh b/ci/install-dependencies.sh index 4f407530d3..b4e22de3cb 100755 --- a/ci/install-dependencies.sh +++ b/ci/install-dependencies.sh @@ -37,15 +37,13 @@ macos-*) test -z "$BREW_INSTALL_PACKAGES" || brew install $BREW_INSTALL_PACKAGES brew link --force gettext - mkdir -p $HOME/bin - ( - cd $HOME/bin + + mkdir -p "$P4_PATH" + pushd "$P4_PATH" wget -q "$P4WHENCE/bin.macosx1015x86_64/helix-core-server.tgz" && tar -xf helix-core-server.tgz && sudo xattr -d com.apple.quarantine p4 p4d 2>/dev/null || true - ) - PATH="$PATH:${HOME}/bin" - export PATH + popd if test -n "$CC_PACKAGE" then diff --git a/ci/install-docker-dependencies.sh b/ci/install-docker-dependencies.sh index 48c43f0f90..eb2c9e1eca 100755 --- a/ci/install-docker-dependencies.sh +++ b/ci/install-docker-dependencies.sh @@ -21,7 +21,7 @@ linux-musl) apache2 apache2-http2 apache2-proxy apache2-ssl apache2-webdav apr-util-dbd_sqlite3 \ bash cvs gnupg perl-cgi perl-dbd-sqlite >/dev/null ;; -linux-*) +linux-*|StaticAnalysis) # Required so that apt doesn't wait for user input on certain packages. export DEBIAN_FRONTEND=noninteractive @@ -31,6 +31,11 @@ linux-*) perl-modules liberror-perl libauthen-sasl-perl libemail-valid-perl \ libdbd-sqlite3-perl libio-socket-ssl-perl libnet-smtp-ssl-perl ${CC_PACKAGE:-${CC:-gcc}} \ apache2 cvs cvsps gnupg libcgi-pm-perl subversion + + if test "$jobname" = StaticAnalysis + then + apt install -q -y coccinelle + fi ;; pedantic) dnf -yq update >/dev/null && @@ -252,7 +252,14 @@ then CI_COMMIT="$CI_COMMIT_SHA" case "$CI_JOB_IMAGE" in macos-*) - CI_OS_NAME=osx;; + # GitLab CI has Python installed via multiple package managers, + # most notably via asdf and Homebrew. Ensure that our builds + # pick up the Homebrew one by prepending it to our PATH as the + # asdf one breaks tests. + export PATH="$(brew --prefix)/bin:$PATH" + + CI_OS_NAME=osx + ;; alpine:*|fedora:*|ubuntu:*) CI_OS_NAME=linux;; *) @@ -344,6 +351,9 @@ macos-*) then MAKEFLAGS="$MAKEFLAGS APPLE_COMMON_CRYPTO_SHA1=Yes" fi + + P4_PATH="$HOME/custom/p4" + export PATH="$P4_PATH:$PATH" ;; esac diff --git a/ci/print-test-failures.sh b/ci/print-test-failures.sh index c33ad4e3a2..b1f80aeac3 100755 --- a/ci/print-test-failures.sh +++ b/ci/print-test-failures.sh @@ -8,7 +8,7 @@ # Tracing executed commands would produce too much noise in the loop below. set +x -cd t/ +cd "${TEST_OUTPUT_DIRECTORY:-t/}" if ! ls test-results/*.exit >/dev/null 2>/dev/null then diff --git a/ci/run-build-and-minimal-fuzzers.sh b/ci/run-build-and-minimal-fuzzers.sh new file mode 100755 index 0000000000..8ba486f659 --- /dev/null +++ b/ci/run-build-and-minimal-fuzzers.sh @@ -0,0 +1,19 @@ +#!/bin/sh +# +# Build and test Git's fuzzers +# + +. ${0%/*}/lib.sh + +group "Build fuzzers" make \ + CC=clang \ + CXX=clang++ \ + CFLAGS="-fsanitize=fuzzer-no-link,address" \ + LIB_FUZZING_ENGINE="-fsanitize=fuzzer,address" \ + fuzz-all + +for fuzzer in commit-graph date pack-headers pack-idx ; do + begin_group "fuzz-$fuzzer" + ./oss-fuzz/fuzz-$fuzzer -verbosity=0 -runs=1 || exit 1 + end_group "fuzz-$fuzzer" +done diff --git a/commit-graph.c b/commit-graph.c index bba316913c..32583991f8 100644 --- a/commit-graph.c +++ b/commit-graph.c @@ -298,48 +298,6 @@ static int graph_read_oid_fanout(const unsigned char *chunk_start, return 0; } -static int graph_read_oid_lookup(const unsigned char *chunk_start, - size_t chunk_size, void *data) -{ - struct commit_graph *g = data; - g->chunk_oid_lookup = chunk_start; - if (chunk_size / g->hash_len != g->num_commits) - return error(_("commit-graph OID lookup chunk is the wrong size")); - return 0; -} - -static int graph_read_commit_data(const unsigned char *chunk_start, - size_t chunk_size, void *data) -{ - struct commit_graph *g = data; - if (chunk_size / GRAPH_DATA_WIDTH != g->num_commits) - return error(_("commit-graph commit data chunk is wrong size")); - g->chunk_commit_data = chunk_start; - return 0; -} - -static int graph_read_generation_data(const unsigned char *chunk_start, - size_t chunk_size, void *data) -{ - struct commit_graph *g = data; - if (chunk_size / sizeof(uint32_t) != g->num_commits) - return error(_("commit-graph generations chunk is wrong size")); - g->chunk_generation_data = chunk_start; - return 0; -} - -static int graph_read_bloom_index(const unsigned char *chunk_start, - size_t chunk_size, void *data) -{ - struct commit_graph *g = data; - if (chunk_size / 4 != g->num_commits) { - warning(_("commit-graph changed-path index chunk is too small")); - return -1; - } - g->chunk_bloom_indexes = chunk_start; - return 0; -} - static int graph_read_bloom_data(const unsigned char *chunk_start, size_t chunk_size, void *data) { @@ -354,13 +312,10 @@ static int graph_read_bloom_data(const unsigned char *chunk_start, return -1; } - g->chunk_bloom_data = chunk_start; - g->chunk_bloom_data_size = chunk_size; hash_version = get_be32(chunk_start); - if (hash_version != 1) - return 0; - + g->chunk_bloom_data = chunk_start; + g->chunk_bloom_data_size = chunk_size; g->bloom_filter_settings = xmalloc(sizeof(struct bloom_filter_settings)); g->bloom_filter_settings->hash_version = hash_version; g->bloom_filter_settings->num_hashes = get_be32(chunk_start + 4); @@ -434,12 +389,15 @@ struct commit_graph *parse_commit_graph(struct repo_settings *s, error(_("commit-graph required OID fanout chunk missing or corrupted")); goto free_and_return; } - if (read_chunk(cf, GRAPH_CHUNKID_OIDLOOKUP, graph_read_oid_lookup, graph)) { - error(_("commit-graph required OID lookup chunk missing or corrupted")); + if (pair_chunk_expect(cf, GRAPH_CHUNKID_OIDLOOKUP, + &graph->chunk_oid_lookup, graph->hash_len, + graph->num_commits)) { + error(_("commit-graph OID lookup chunk is the wrong size")); goto free_and_return; } - if (read_chunk(cf, GRAPH_CHUNKID_DATA, graph_read_commit_data, graph)) { - error(_("commit-graph required commit data chunk missing or corrupted")); + if (pair_chunk_expect(cf, GRAPH_CHUNKID_DATA, &graph->chunk_commit_data, + GRAPH_DATA_WIDTH, graph->num_commits)) { + error(_("commit-graph commit data chunk is wrong size")); goto free_and_return; } @@ -449,8 +407,11 @@ struct commit_graph *parse_commit_graph(struct repo_settings *s, &graph->chunk_base_graphs_size); if (s->commit_graph_generation_version >= 2) { - read_chunk(cf, GRAPH_CHUNKID_GENERATION_DATA, - graph_read_generation_data, graph); + if (pair_chunk_expect(cf, GRAPH_CHUNKID_GENERATION_DATA, + &graph->chunk_generation_data, + sizeof(uint32_t), + graph->num_commits)) + error(_("commit-graph generations chunk is wrong size")); pair_chunk(cf, GRAPH_CHUNKID_GENERATION_DATA_OVERFLOW, &graph->chunk_generation_data_overflow, &graph->chunk_generation_data_overflow_size); @@ -459,9 +420,12 @@ struct commit_graph *parse_commit_graph(struct repo_settings *s, graph->read_generation_data = 1; } - if (s->commit_graph_read_changed_paths) { - read_chunk(cf, GRAPH_CHUNKID_BLOOMINDEXES, - graph_read_bloom_index, graph); + if (s->commit_graph_changed_paths_version) { + int res = pair_chunk_expect(cf, GRAPH_CHUNKID_BLOOMINDEXES, + &graph->chunk_bloom_indexes, + sizeof(uint32_t), graph->num_commits); + if (res && res != CHUNK_NOT_FOUND) + warning(_("commit-graph changed-path index chunk is too small")); read_chunk(cf, GRAPH_CHUNKID_BLOOMDATA, graph_read_bloom_data, graph); } @@ -543,6 +507,31 @@ static int validate_mixed_generation_chain(struct commit_graph *g) return 0; } +static void validate_mixed_bloom_settings(struct commit_graph *g) +{ + struct bloom_filter_settings *settings = NULL; + for (; g; g = g->base_graph) { + if (!g->bloom_filter_settings) + continue; + if (!settings) { + settings = g->bloom_filter_settings; + continue; + } + + if (g->bloom_filter_settings->bits_per_entry != settings->bits_per_entry || + g->bloom_filter_settings->num_hashes != settings->num_hashes || + g->bloom_filter_settings->hash_version != settings->hash_version) { + g->chunk_bloom_indexes = NULL; + g->chunk_bloom_data = NULL; + FREE_AND_NULL(g->bloom_filter_settings); + + warning(_("disabling Bloom filters for commit-graph " + "layer '%s' due to incompatible settings"), + oid_to_hex(&g->oid)); + } + } +} + static int add_graph_to_chain(struct commit_graph *g, struct commit_graph *chain, struct object_id *oids, @@ -666,6 +655,7 @@ struct commit_graph *load_commit_graph_chain_fd_st(struct repository *r, } validate_mixed_generation_chain(graph_chain); + validate_mixed_bloom_settings(graph_chain); free(oids); fclose(fp); @@ -806,7 +796,11 @@ struct bloom_filter_settings *get_bloom_filter_settings(struct repository *r) void close_commit_graph(struct raw_object_store *o) { + if (!o->commit_graph) + return; + clear_commit_graph_data_slab(&commit_graph_data_slab); + deinit_bloom_filters(); free_commit_graph(o->commit_graph); o->commit_graph = NULL; } @@ -1144,6 +1138,7 @@ struct write_commit_graph_context { int count_bloom_filter_not_computed; int count_bloom_filter_trunc_empty; int count_bloom_filter_trunc_large; + int count_bloom_filter_upgraded; }; static int write_graph_chunk_fanout(struct hashfile *f, @@ -1751,6 +1746,8 @@ static void trace2_bloom_filter_write_statistics(struct write_commit_graph_conte ctx->count_bloom_filter_trunc_empty); trace2_data_intmax("commit-graph", ctx->r, "filter-trunc-large", ctx->count_bloom_filter_trunc_large); + trace2_data_intmax("commit-graph", ctx->r, "filter-upgraded", + ctx->count_bloom_filter_upgraded); } static void compute_bloom_filters(struct write_commit_graph_context *ctx) @@ -1792,6 +1789,8 @@ static void compute_bloom_filters(struct write_commit_graph_context *ctx) ctx->count_bloom_filter_trunc_empty++; if (computed & BLOOM_TRUNC_LARGE) ctx->count_bloom_filter_trunc_large++; + } else if (computed & BLOOM_UPGRADED) { + ctx->count_bloom_filter_upgraded++; } else if (computed & BLOOM_NOT_COMPUTED) ctx->count_bloom_filter_not_computed++; ctx->total_bloom_filter_data_size += filter @@ -2475,6 +2474,13 @@ int write_commit_graph(struct object_directory *odb, } if (!commit_graph_compatible(r)) return 0; + if (r->settings.commit_graph_changed_paths_version < -1 + || r->settings.commit_graph_changed_paths_version > 2) { + warning(_("attempting to write a commit-graph, but " + "'commitgraph.changedPathsVersion' (%d) is not supported"), + r->settings.commit_graph_changed_paths_version); + return 0; + } CALLOC_ARRAY(ctx, 1); ctx->r = r; @@ -2487,6 +2493,7 @@ int write_commit_graph(struct object_directory *odb, ctx->write_generation_data = (get_configured_generation_version(r) == 2); ctx->num_generation_data_overflows = 0; + bloom_settings.hash_version = r->settings.commit_graph_changed_paths_version; bloom_settings.bits_per_entry = git_env_ulong("GIT_TEST_BLOOM_SETTINGS_BITS_PER_ENTRY", bloom_settings.bits_per_entry); bloom_settings.num_hashes = git_env_ulong("GIT_TEST_BLOOM_SETTINGS_NUM_HASHES", @@ -2516,12 +2523,20 @@ int write_commit_graph(struct object_directory *odb, g = ctx->r->objects->commit_graph; /* We have changed-paths already. Keep them in the next graph */ - if (g && g->chunk_bloom_data) { + if (g && g->bloom_filter_settings) { ctx->changed_paths = 1; - ctx->bloom_settings = g->bloom_filter_settings; + + /* don't propagate the hash_version unless unspecified */ + if (bloom_settings.hash_version == -1) + bloom_settings.hash_version = g->bloom_filter_settings->hash_version; + bloom_settings.bits_per_entry = g->bloom_filter_settings->bits_per_entry; + bloom_settings.num_hashes = g->bloom_filter_settings->num_hashes; + bloom_settings.max_changed_paths = g->bloom_filter_settings->max_changed_paths; } } + bloom_settings.hash_version = bloom_settings.hash_version == 2 ? 2 : 1; + if (ctx->split) { struct commit_graph *g = ctx->r->objects->commit_graph; @@ -2604,6 +2619,9 @@ int write_commit_graph(struct object_directory *odb, res = write_commit_graph_file(ctx); + if (ctx->changed_paths) + deinit_bloom_filters(); + if (ctx->split) mark_commit_graphs(ctx); @@ -2616,19 +2634,16 @@ cleanup: oid_array_clear(&ctx->oids); clear_topo_level_slab(&topo_levels); - if (ctx->commit_graph_filenames_after) { - for (i = 0; i < ctx->num_commit_graphs_after; i++) { - free(ctx->commit_graph_filenames_after[i]); - free(ctx->commit_graph_hash_after[i]); - } - - for (i = 0; i < ctx->num_commit_graphs_before; i++) - free(ctx->commit_graph_filenames_before[i]); + for (i = 0; i < ctx->num_commit_graphs_before; i++) + free(ctx->commit_graph_filenames_before[i]); + free(ctx->commit_graph_filenames_before); - free(ctx->commit_graph_filenames_after); - free(ctx->commit_graph_filenames_before); - free(ctx->commit_graph_hash_after); + for (i = 0; i < ctx->num_commit_graphs_after; i++) { + free(ctx->commit_graph_filenames_after[i]); + free(ctx->commit_graph_hash_after[i]); } + free(ctx->commit_graph_filenames_after); + free(ctx->commit_graph_hash_after); free(ctx); @@ -27,6 +27,7 @@ #include "tree.h" #include "hook.h" #include "parse.h" +#include "object-file-convert.h" static struct commit_extra_header *read_commit_extra_header_lines(const char *buf, size_t len, const char **); @@ -1112,12 +1113,11 @@ static const char *gpg_sig_headers[] = { "gpgsig-sha256", }; -int sign_with_header(struct strbuf *buf, const char *keyid) +int add_header_signature(struct strbuf *buf, struct strbuf *sig, const struct git_hash_algo *algo) { - struct strbuf sig = STRBUF_INIT; int inspos, copypos; const char *eoh; - const char *gpg_sig_header = gpg_sig_headers[hash_algo_by_ptr(the_hash_algo)]; + const char *gpg_sig_header = gpg_sig_headers[hash_algo_by_ptr(algo)]; int gpg_sig_header_len = strlen(gpg_sig_header); /* find the end of the header */ @@ -1127,15 +1127,8 @@ int sign_with_header(struct strbuf *buf, const char *keyid) else inspos = eoh - buf->buf + 1; - if (!keyid || !*keyid) - keyid = get_signing_key(); - if (sign_buffer(buf, &sig, keyid)) { - strbuf_release(&sig); - return -1; - } - - for (copypos = 0; sig.buf[copypos]; ) { - const char *bol = sig.buf + copypos; + for (copypos = 0; sig->buf[copypos]; ) { + const char *bol = sig->buf + copypos; const char *eol = strchrnul(bol, '\n'); int len = (eol - bol) + !!*eol; @@ -1148,11 +1141,17 @@ int sign_with_header(struct strbuf *buf, const char *keyid) inspos += len; copypos += len; } - strbuf_release(&sig); return 0; } - +static int sign_commit_to_strbuf(struct strbuf *sig, struct strbuf *buf, const char *keyid) +{ + if (!keyid || !*keyid) + keyid = get_signing_key(); + if (sign_buffer(buf, sig, keyid)) + return -1; + return 0; +} int parse_signed_commit(const struct commit *commit, struct strbuf *payload, struct strbuf *signature, @@ -1368,6 +1367,39 @@ void append_merge_tag_headers(struct commit_list *parents, } } +static int convert_commit_extra_headers(struct commit_extra_header *orig, + struct commit_extra_header **result) +{ + const struct git_hash_algo *compat = the_repository->compat_hash_algo; + const struct git_hash_algo *algo = the_repository->hash_algo; + struct commit_extra_header *extra = NULL, **tail = &extra; + struct strbuf out = STRBUF_INIT; + while (orig) { + struct commit_extra_header *new; + CALLOC_ARRAY(new, 1); + if (!strcmp(orig->key, "mergetag")) { + if (convert_object_file(&out, algo, compat, + orig->value, orig->len, + OBJ_TAG, 1)) { + free(new); + free_commit_extra_headers(extra); + return -1; + } + new->key = xstrdup("mergetag"); + new->value = strbuf_detach(&out, &new->len); + } else { + new->key = xstrdup(orig->key); + new->len = orig->len; + new->value = xmemdupz(orig->value, orig->len); + } + *tail = new; + tail = &new->next; + orig = orig->next; + } + *result = extra; + return 0; +} + static void add_extra_header(struct strbuf *buffer, struct commit_extra_header *extra) { @@ -1611,70 +1643,169 @@ N_("Warning: commit message did not conform to UTF-8.\n" "You may want to amend it after fixing the message, or set the config\n" "variable i18n.commitEncoding to the encoding your project uses.\n"); -int commit_tree_extended(const char *msg, size_t msg_len, - const struct object_id *tree, - struct commit_list *parents, struct object_id *ret, - const char *author, const char *committer, - const char *sign_commit, - struct commit_extra_header *extra) +static void write_commit_tree(struct strbuf *buffer, const char *msg, size_t msg_len, + const struct object_id *tree, + const struct object_id *parents, size_t parents_len, + const char *author, const char *committer, + struct commit_extra_header *extra) { - int result; int encoding_is_utf8; - struct strbuf buffer; - - assert_oid_type(tree, OBJ_TREE); - - if (memchr(msg, '\0', msg_len)) - return error("a NUL byte in commit log message not allowed."); + size_t i; /* Not having i18n.commitencoding is the same as having utf-8 */ encoding_is_utf8 = is_encoding_utf8(git_commit_encoding); - strbuf_init(&buffer, 8192); /* should avoid reallocs for the headers */ - strbuf_addf(&buffer, "tree %s\n", oid_to_hex(tree)); + strbuf_grow(buffer, 8192); /* should avoid reallocs for the headers */ + strbuf_addf(buffer, "tree %s\n", oid_to_hex(tree)); /* * NOTE! This ordering means that the same exact tree merged with a * different order of parents will be a _different_ changeset even * if everything else stays the same. */ - while (parents) { - struct commit *parent = pop_commit(&parents); - strbuf_addf(&buffer, "parent %s\n", - oid_to_hex(&parent->object.oid)); - } + for (i = 0; i < parents_len; i++) + strbuf_addf(buffer, "parent %s\n", oid_to_hex(&parents[i])); /* Person/date information */ if (!author) author = git_author_info(IDENT_STRICT); - strbuf_addf(&buffer, "author %s\n", author); + strbuf_addf(buffer, "author %s\n", author); if (!committer) committer = git_committer_info(IDENT_STRICT); - strbuf_addf(&buffer, "committer %s\n", committer); + strbuf_addf(buffer, "committer %s\n", committer); if (!encoding_is_utf8) - strbuf_addf(&buffer, "encoding %s\n", git_commit_encoding); + strbuf_addf(buffer, "encoding %s\n", git_commit_encoding); while (extra) { - add_extra_header(&buffer, extra); + add_extra_header(buffer, extra); extra = extra->next; } - strbuf_addch(&buffer, '\n'); + strbuf_addch(buffer, '\n'); /* And add the comment */ - strbuf_add(&buffer, msg, msg_len); + strbuf_add(buffer, msg, msg_len); +} - /* And check the encoding */ - if (encoding_is_utf8 && !verify_utf8(&buffer)) - fprintf(stderr, _(commit_utf8_warn)); +int commit_tree_extended(const char *msg, size_t msg_len, + const struct object_id *tree, + struct commit_list *parents, struct object_id *ret, + const char *author, const char *committer, + const char *sign_commit, + struct commit_extra_header *extra) +{ + struct repository *r = the_repository; + int result = 0; + int encoding_is_utf8; + struct strbuf buffer = STRBUF_INIT, compat_buffer = STRBUF_INIT; + struct strbuf sig = STRBUF_INIT, compat_sig = STRBUF_INIT; + struct object_id *parent_buf = NULL, *compat_oid = NULL; + struct object_id compat_oid_buf; + size_t i, nparents; + + /* Not having i18n.commitencoding is the same as having utf-8 */ + encoding_is_utf8 = is_encoding_utf8(git_commit_encoding); + + assert_oid_type(tree, OBJ_TREE); - if (sign_commit && sign_with_header(&buffer, sign_commit)) { + if (memchr(msg, '\0', msg_len)) + return error("a NUL byte in commit log message not allowed."); + + nparents = commit_list_count(parents); + CALLOC_ARRAY(parent_buf, nparents); + i = 0; + while (parents) { + struct commit *parent = pop_commit(&parents); + oidcpy(&parent_buf[i++], &parent->object.oid); + } + + write_commit_tree(&buffer, msg, msg_len, tree, parent_buf, nparents, author, committer, extra); + if (sign_commit && sign_commit_to_strbuf(&sig, &buffer, sign_commit)) { result = -1; goto out; } + if (r->compat_hash_algo) { + struct commit_extra_header *compat_extra = NULL; + struct object_id mapped_tree; + struct object_id *mapped_parents; + + CALLOC_ARRAY(mapped_parents, nparents); - result = write_object_file(buffer.buf, buffer.len, OBJ_COMMIT, ret); + if (repo_oid_to_algop(r, tree, r->compat_hash_algo, &mapped_tree)) { + result = -1; + free(mapped_parents); + goto out; + } + for (i = 0; i < nparents; i++) + if (repo_oid_to_algop(r, &parent_buf[i], r->compat_hash_algo, &mapped_parents[i])) { + result = -1; + free(mapped_parents); + goto out; + } + if (convert_commit_extra_headers(extra, &compat_extra)) { + result = -1; + free(mapped_parents); + goto out; + } + write_commit_tree(&compat_buffer, msg, msg_len, &mapped_tree, + mapped_parents, nparents, author, committer, compat_extra); + free_commit_extra_headers(compat_extra); + free(mapped_parents); + + if (sign_commit && sign_commit_to_strbuf(&compat_sig, &compat_buffer, sign_commit)) { + result = -1; + goto out; + } + } + + if (sign_commit) { + struct sig_pairs { + struct strbuf *sig; + const struct git_hash_algo *algo; + } bufs [2] = { + { &compat_sig, r->compat_hash_algo }, + { &sig, r->hash_algo }, + }; + int i; + + /* + * We write algorithms in the order they were implemented in + * Git to produce a stable hash when multiple algorithms are + * used. + */ + if (r->compat_hash_algo && hash_algo_by_ptr(bufs[0].algo) > hash_algo_by_ptr(bufs[1].algo)) + SWAP(bufs[0], bufs[1]); + + /* + * We traverse each algorithm in order, and apply the signature + * to each buffer. + */ + for (i = 0; i < ARRAY_SIZE(bufs); i++) { + if (!bufs[i].algo) + continue; + add_header_signature(&buffer, bufs[i].sig, bufs[i].algo); + if (r->compat_hash_algo) + add_header_signature(&compat_buffer, bufs[i].sig, bufs[i].algo); + } + } + + /* And check the encoding. */ + if (encoding_is_utf8 && (!verify_utf8(&buffer) || !verify_utf8(&compat_buffer))) + fprintf(stderr, _(commit_utf8_warn)); + + if (r->compat_hash_algo) { + hash_object_file(r->compat_hash_algo, compat_buffer.buf, compat_buffer.len, + OBJ_COMMIT, &compat_oid_buf); + compat_oid = &compat_oid_buf; + } + + result = write_object_file_flags(buffer.buf, buffer.len, OBJ_COMMIT, + ret, compat_oid, 0); out: + free(parent_buf); strbuf_release(&buffer); + strbuf_release(&compat_buffer); + strbuf_release(&sig); + strbuf_release(&compat_sig); return result; } @@ -54,8 +54,11 @@ enum decoration_type { DECORATION_REF_REMOTE, DECORATION_REF_TAG, DECORATION_REF_STASH, + DECORATION_REF, DECORATION_REF_HEAD, + DECORATION_REF_PSEUDO, DECORATION_GRAFTED, + DECORATION_SYMBOL, }; void add_name_decoration(enum decoration_type type, const char *name, struct object *obj); @@ -370,5 +373,6 @@ int parse_buffer_signed_by_header(const char *buffer, struct strbuf *payload, struct strbuf *signature, const struct git_hash_algo *algop); +int add_header_signature(struct strbuf *buf, struct strbuf *sig, const struct git_hash_algo *algo); #endif /* COMMIT_H */ diff --git a/compat/mingw.c b/compat/mingw.c index 42053c1f65..238a84ddba 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2684,6 +2684,30 @@ static PSID get_current_user_sid(void) return result; } +static BOOL user_sid_to_user_name(PSID sid, LPSTR *str) +{ + SID_NAME_USE pe_use; + DWORD len_user = 0, len_domain = 0; + BOOL translate_sid_to_user; + + /* + * returns only FALSE, because the string pointers are NULL + */ + LookupAccountSidA(NULL, sid, NULL, &len_user, NULL, &len_domain, + &pe_use); + /* + * Alloc needed space of the strings + */ + ALLOC_ARRAY((*str), (size_t)len_domain + (size_t)len_user); + translate_sid_to_user = LookupAccountSidA(NULL, sid, + (*str) + len_domain, &len_user, *str, &len_domain, &pe_use); + if (!translate_sid_to_user) + FREE_AND_NULL(*str); + else + (*str)[len_domain] = '/'; + return translate_sid_to_user; +} + static int acls_supported(const char *path) { size_t offset = offset_1st_component(path); @@ -2765,27 +2789,47 @@ int is_path_owned_by_current_sid(const char *path, struct strbuf *report) strbuf_addf(report, "'%s' is on a file system that does " "not record ownership\n", path); } else if (report) { - LPSTR str1, str2, to_free1 = NULL, to_free2 = NULL; + LPSTR str1, str2, str3, str4, to_free1 = NULL, + to_free3 = NULL, to_local_free2 = NULL, + to_local_free4 = NULL; - if (ConvertSidToStringSidA(sid, &str1)) + if (user_sid_to_user_name(sid, &str1)) to_free1 = str1; else str1 = "(inconvertible)"; - - if (!current_user_sid) - str2 = "(none)"; - else if (!IsValidSid(current_user_sid)) - str2 = "(invalid)"; - else if (ConvertSidToStringSidA(current_user_sid, &str2)) - to_free2 = str2; + if (ConvertSidToStringSidA(sid, &str2)) + to_local_free2 = str2; else str2 = "(inconvertible)"; + + if (!current_user_sid) { + str3 = "(none)"; + str4 = "(none)"; + } + else if (!IsValidSid(current_user_sid)) { + str3 = "(invalid)"; + str4 = "(invalid)"; + } else { + if (user_sid_to_user_name(current_user_sid, + &str3)) + to_free3 = str3; + else + str3 = "(inconvertible)"; + if (ConvertSidToStringSidA(current_user_sid, + &str4)) + to_local_free4 = str4; + else + str4 = "(inconvertible)"; + } strbuf_addf(report, "'%s' is owned by:\n" - "\t'%s'\nbut the current user is:\n" - "\t'%s'\n", path, str1, str2); - LocalFree(to_free1); - LocalFree(to_free2); + "\t%s (%s)\nbut the current user is:\n" + "\t%s (%s)\n", + path, str1, str2, str3, str4); + free(to_free1); + LocalFree(to_local_free2); + free(to_free3); + LocalFree(to_local_free4); } } @@ -95,7 +95,6 @@ static long config_file_ftell(struct config_source *conf) return ftell(conf->u.file); } - static int config_buf_fgetc(struct config_source *conf) { if (conf->u.buf.pos < conf->u.buf.len) @@ -1988,7 +1987,27 @@ char *git_system_config(void) return system_config; } -void git_global_config(char **user_out, char **xdg_out) +char *git_global_config(void) +{ + char *user_config, *xdg_config; + + git_global_config_paths(&user_config, &xdg_config); + if (!user_config) { + free(xdg_config); + return NULL; + } + + if (access_or_warn(user_config, R_OK, 0) && xdg_config && + !access_or_warn(xdg_config, R_OK, 0)) { + free(user_config); + return xdg_config; + } else { + free(xdg_config); + return user_config; + } +} + +void git_global_config_paths(char **user_out, char **xdg_out) { char *user_config = xstrdup_or_null(getenv("GIT_CONFIG_GLOBAL")); char *xdg_config = NULL; @@ -2041,7 +2060,7 @@ static int do_git_config_sequence(const struct config_options *opts, data, CONFIG_SCOPE_SYSTEM, NULL); - git_global_config(&user_config, &xdg_config); + git_global_config_paths(&user_config, &xdg_config); if (xdg_config && !access_or_die(xdg_config, R_OK, ACCESS_EACCES_OK)) ret += git_config_from_file_with_options(fn, xdg_config, data, @@ -3418,7 +3437,6 @@ out_free: write_err_out: ret = write_error(get_lock_file_path(&lock)); goto out_free; - } void git_config_set_multivar_in_file(const char *config_filename, @@ -382,7 +382,8 @@ int config_error_nonbool(const char *); #endif char *git_system_config(void); -void git_global_config(char **user, char **xdg); +char *git_global_config(void); +void git_global_config_paths(char **user, char **xdg); int git_config_parse_parameter(const char *, config_fn_t fn, void *data); diff --git a/config.mak.uname b/config.mak.uname index 3bb03f423a..dacc95172d 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -158,6 +158,19 @@ ifeq ($(uname_S),Darwin) ifeq ($(shell test -x /usr/local/opt/gettext/bin/msgfmt && echo y),y) MSGFMT = /usr/local/opt/gettext/bin/msgfmt endif + # On newer ARM-based machines the default installation path has changed to + # /opt/homebrew. Include it in our search paths so that the user does not + # have to configure this manually. + # + # Note that we do not employ the same workaround as above where we manually + # add gettext. The issue was fixed more than three years ago by now, and at + # that point there haven't been any ARM-based Macs yet. + else ifeq ($(shell test -d /opt/homebrew/ && echo y),y) + BASIC_CFLAGS += -I/opt/homebrew/include + BASIC_LDFLAGS += -L/opt/homebrew/lib + ifeq ($(shell test -x /opt/homebrew/bin/msgfmt && echo y),y) + MSGFMT = /opt/homebrew/bin/msgfmt + endif endif # The builtin FSMonitor on MacOS builds upon Simple-IPC. Both require diff --git a/contrib/completion/git-completion.bash b/contrib/completion/git-completion.bash index 8c40ade494..7c7284f17f 100644 --- a/contrib/completion/git-completion.bash +++ b/contrib/completion/git-completion.bash @@ -137,6 +137,9 @@ __git_eread () __git_pseudoref_exists () { local ref=$1 + local head + + __git_find_repo_path # If the reftable is in use, we have to shell out to 'git rev-parse' # to determine whether the ref exists instead of looking directly in @@ -144,9 +147,8 @@ __git_pseudoref_exists () # Bash builtins since executing Git commands are expensive on some # platforms. if __git_eread "$__git_repo_path/HEAD" head; then - b="${head#ref: }" - if [ "$b" == "refs/heads/.invalid" ]; then - __git -C "$__git_repo_path" rev-parse --verify --quiet "$ref" 2>/dev/null + if [ "$head" == "ref: refs/heads/.invalid" ]; then + __git show-ref --exists "$ref" return $? fi fi @@ -1477,16 +1479,181 @@ _git_archive () __git_complete_file } +# Options that go well for log, shortlog and gitk +__git_log_common_options=" + --not --all + --branches --tags --remotes + --first-parent --merges --no-merges + --max-count= + --max-age= --since= --after= + --min-age= --until= --before= + --min-parents= --max-parents= + --no-min-parents --no-max-parents + --alternate-refs --ancestry-path + --author-date-order --basic-regexp + --bisect --boundary --exclude-first-parent-only + --exclude-hidden --extended-regexp + --fixed-strings --grep-reflog + --ignore-missing --left-only --perl-regexp + --reflog --regexp-ignore-case --remove-empty + --right-only --show-linear-break + --show-notes-by-default --show-pulls + --since-as-filter --single-worktree +" +# Options that go well for log and gitk (not shortlog) +__git_log_gitk_options=" + --dense --sparse --full-history + --simplify-merges --simplify-by-decoration + --left-right --notes --no-notes +" +# Options that go well for log and shortlog (not gitk) +__git_log_shortlog_options=" + --author= --committer= --grep= + --all-match --invert-grep +" +# Options accepted by log and show +__git_log_show_options=" + --diff-merges --diff-merges= --no-diff-merges --dd --remerge-diff + --encoding= +" + +__git_diff_merges_opts="off none on first-parent 1 separate m combined c dense-combined cc remerge r" + +__git_log_pretty_formats="oneline short medium full fuller reference email raw format: tformat: mboxrd" +__git_log_date_formats="relative iso8601 iso8601-strict rfc2822 short local default human raw unix auto: format:" + +# Check for only porcelain (i.e. not git-rev-list) option (not argument) +# and selected option argument completions for git-log options and if any +# are found put them in COMPREPLY. COMPREPLY must be empty at the start, +# and will be empty on return if no candidates are found. +__git_complete_log_opts () +{ + [ -z "$COMPREPLY" ] || return 1 # Precondition + + local merge="" + if __git_pseudoref_exists MERGE_HEAD; then + merge="--merge" + fi + case "$prev,$cur" in + -L,:*:*) + return # fall back to Bash filename completion + ;; + -L,:*) + __git_complete_symbol --cur="${cur#:}" --sfx=":" + return + ;; + -G,*|-S,*) + __git_complete_symbol + return + ;; + esac + case "$cur" in + --pretty=*|--format=*) + __gitcomp "$__git_log_pretty_formats $(__git_pretty_aliases) + " "" "${cur#*=}" + return + ;; + --date=*) + __gitcomp "$__git_log_date_formats" "" "${cur##--date=}" + return + ;; + --decorate=*) + __gitcomp "full short no" "" "${cur##--decorate=}" + return + ;; + --diff-algorithm=*) + __gitcomp "$__git_diff_algorithms" "" "${cur##--diff-algorithm=}" + return + ;; + --submodule=*) + __gitcomp "$__git_diff_submodule_formats" "" "${cur##--submodule=}" + return + ;; + --ws-error-highlight=*) + __gitcomp "$__git_ws_error_highlight_opts" "" "${cur##--ws-error-highlight=}" + return + ;; + --no-walk=*) + __gitcomp "sorted unsorted" "" "${cur##--no-walk=}" + return + ;; + --diff-merges=*) + __gitcomp "$__git_diff_merges_opts" "" "${cur##--diff-merges=}" + return + ;; + --*) + __gitcomp " + $__git_log_common_options + $__git_log_shortlog_options + $__git_log_gitk_options + $__git_log_show_options + --root --topo-order --date-order --reverse + --follow --full-diff + --abbrev-commit --no-abbrev-commit --abbrev= + --relative-date --date= + --pretty= --format= --oneline + --show-signature + --cherry-mark + --cherry-pick + --graph + --decorate --decorate= --no-decorate + --walk-reflogs + --no-walk --no-walk= --do-walk + --parents --children + --expand-tabs --expand-tabs= --no-expand-tabs + --clear-decorations --decorate-refs= + --decorate-refs-exclude= + $merge + $__git_diff_common_options + " + return + ;; + -L:*:*) + return # fall back to Bash filename completion + ;; + -L:*) + __git_complete_symbol --cur="${cur#-L:}" --sfx=":" + return + ;; + -G*) + __git_complete_symbol --pfx="-G" --cur="${cur#-G}" + return + ;; + -S*) + __git_complete_symbol --pfx="-S" --cur="${cur#-S}" + return + ;; + esac +} + _git_bisect () { __git_has_doubledash && return - local subcommands="start bad good skip reset visualize replay log run" - local subcommand="$(__git_find_on_cmdline "$subcommands")" + __git_find_repo_path + + local term_bad term_good + if [ -f "$__git_repo_path"/BISECT_START ]; then + term_bad=`__git bisect terms --term-bad` + term_good=`__git bisect terms --term-good` + fi + + # We will complete any custom terms, but still always complete the + # more usual bad/new/good/old because git bisect gives a good error + # message if these are given when not in use and that's better than + # silent refusal to complete if the user is confused. + # + # We want to recognize 'view' but not complete it, because it overlaps + # with 'visualize' too much and is just an alias for it. + # + local completable_subcommands="start bad new $term_bad good old $term_good terms skip reset visualize replay log run help" + local all_subcommands="$completable_subcommands view" + + local subcommand="$(__git_find_on_cmdline "$all_subcommands")" + if [ -z "$subcommand" ]; then - __git_find_repo_path if [ -f "$__git_repo_path"/BISECT_START ]; then - __gitcomp "$subcommands" + __gitcomp "$completable_subcommands" else __gitcomp "replay start" fi @@ -1494,7 +1661,30 @@ _git_bisect () fi case "$subcommand" in - bad|good|reset|skip|start) + start) + case "$cur" in + --*) + __gitcomp "--term-new --term-bad --term-old --term-good --first-parent --no-checkout" + return + ;; + *) + ;; + esac + ;; + visualize|view) + case "$cur" in + -*) + __git_complete_log_opts + return + ;; + *) + ;; + esac + ;; + esac + + case "$subcommand" in + bad|new|"$term_bad"|good|old|"$term_good"|reset|skip|start) __git_complete_refs ;; *) @@ -1656,7 +1846,6 @@ __git_cherry_pick_inprogress_options=$__git_sequencer_inprogress_options _git_cherry_pick () { - __git_find_repo_path if __git_pseudoref_exists CHERRY_PICK_HEAD; then __gitcomp "$__git_cherry_pick_inprogress_options" return @@ -1807,7 +1996,7 @@ __git_diff_common_options="--stat --numstat --shortstat --summary --output= --output-indicator-context= --output-indicator-new= --output-indicator-old= --ws-error-highlight= - --pickaxe-all --pickaxe-regex + --pickaxe-all --pickaxe-regex --patch-with-raw " # Options for diff/difftool @@ -2061,135 +2250,14 @@ _git_ls_tree () __git_complete_file } -# Options that go well for log, shortlog and gitk -__git_log_common_options=" - --not --all - --branches --tags --remotes - --first-parent --merges --no-merges - --max-count= - --max-age= --since= --after= - --min-age= --until= --before= - --min-parents= --max-parents= - --no-min-parents --no-max-parents -" -# Options that go well for log and gitk (not shortlog) -__git_log_gitk_options=" - --dense --sparse --full-history - --simplify-merges --simplify-by-decoration - --left-right --notes --no-notes -" -# Options that go well for log and shortlog (not gitk) -__git_log_shortlog_options=" - --author= --committer= --grep= - --all-match --invert-grep -" -# Options accepted by log and show -__git_log_show_options=" - --diff-merges --diff-merges= --no-diff-merges --dd --remerge-diff -" - -__git_diff_merges_opts="off none on first-parent 1 separate m combined c dense-combined cc remerge r" - -__git_log_pretty_formats="oneline short medium full fuller reference email raw format: tformat: mboxrd" -__git_log_date_formats="relative iso8601 iso8601-strict rfc2822 short local default human raw unix auto: format:" - _git_log () { __git_has_doubledash && return __git_find_repo_path - local merge="" - if __git_pseudoref_exists MERGE_HEAD; then - merge="--merge" - fi - case "$prev,$cur" in - -L,:*:*) - return # fall back to Bash filename completion - ;; - -L,:*) - __git_complete_symbol --cur="${cur#:}" --sfx=":" - return - ;; - -G,*|-S,*) - __git_complete_symbol - return - ;; - esac - case "$cur" in - --pretty=*|--format=*) - __gitcomp "$__git_log_pretty_formats $(__git_pretty_aliases) - " "" "${cur#*=}" - return - ;; - --date=*) - __gitcomp "$__git_log_date_formats" "" "${cur##--date=}" - return - ;; - --decorate=*) - __gitcomp "full short no" "" "${cur##--decorate=}" - return - ;; - --diff-algorithm=*) - __gitcomp "$__git_diff_algorithms" "" "${cur##--diff-algorithm=}" - return - ;; - --submodule=*) - __gitcomp "$__git_diff_submodule_formats" "" "${cur##--submodule=}" - return - ;; - --ws-error-highlight=*) - __gitcomp "$__git_ws_error_highlight_opts" "" "${cur##--ws-error-highlight=}" - return - ;; - --no-walk=*) - __gitcomp "sorted unsorted" "" "${cur##--no-walk=}" - return - ;; - --diff-merges=*) - __gitcomp "$__git_diff_merges_opts" "" "${cur##--diff-merges=}" - return - ;; - --*) - __gitcomp " - $__git_log_common_options - $__git_log_shortlog_options - $__git_log_gitk_options - $__git_log_show_options - --root --topo-order --date-order --reverse - --follow --full-diff - --abbrev-commit --no-abbrev-commit --abbrev= - --relative-date --date= - --pretty= --format= --oneline - --show-signature - --cherry-mark - --cherry-pick - --graph - --decorate --decorate= --no-decorate - --walk-reflogs - --no-walk --no-walk= --do-walk - --parents --children - --expand-tabs --expand-tabs= --no-expand-tabs - $merge - $__git_diff_common_options - " - return - ;; - -L:*:*) - return # fall back to Bash filename completion - ;; - -L:*) - __git_complete_symbol --cur="${cur#-L:}" --sfx=":" - return - ;; - -G*) - __git_complete_symbol --pfx="-G" --cur="${cur#-G}" - return - ;; - -S*) - __git_complete_symbol --pfx="-S" --cur="${cur#-S}" - return - ;; - esac + __git_complete_log_opts + [ -z "$COMPREPLY" ] || return + __git_complete_revlist } @@ -2966,7 +3034,6 @@ _git_reset () _git_restore () { - __git_find_repo_path case "$prev" in -s) __git_complete_refs @@ -2995,7 +3062,6 @@ __git_revert_inprogress_options=$__git_sequencer_inprogress_options _git_revert () { - __git_find_repo_path if __git_pseudoref_exists REVERT_HEAD; then __gitcomp "$__git_revert_inprogress_options" return diff --git a/contrib/completion/git-prompt.sh b/contrib/completion/git-prompt.sh index 2c030050ae..71f179cba3 100644 --- a/contrib/completion/git-prompt.sh +++ b/contrib/completion/git-prompt.sh @@ -408,7 +408,7 @@ __git_ps1 () local repo_info rev_parse_exit_code repo_info="$(git rev-parse --git-dir --is-inside-git-dir \ - --is-bare-repository --is-inside-work-tree \ + --is-bare-repository --is-inside-work-tree --show-ref-format \ --short HEAD 2>/dev/null)" rev_parse_exit_code="$?" @@ -421,6 +421,8 @@ __git_ps1 () short_sha="${repo_info##*$'\n'}" repo_info="${repo_info%$'\n'*}" fi + local ref_format="${repo_info##*$'\n'}" + repo_info="${repo_info%$'\n'*}" local inside_worktree="${repo_info##*$'\n'}" repo_info="${repo_info%$'\n'*}" local bare_repo="${repo_info##*$'\n'}" @@ -479,12 +481,25 @@ __git_ps1 () b="$(git symbolic-ref HEAD 2>/dev/null)" else local head="" - if ! __git_eread "$g/HEAD" head; then - return $exit - fi - # is it a symbolic ref? - b="${head#ref: }" - if [ "$head" = "$b" ]; then + + case "$ref_format" in + files) + if ! __git_eread "$g/HEAD" head; then + return $exit + fi + + if [[ $head == "ref: "* ]]; then + head="${head#ref: }" + else + head="" + fi + ;; + *) + head="$(git symbolic-ref HEAD 2>/dev/null)" + ;; + esac + + if test -z "$head"; then detached=yes b="$( case "${GIT_PS1_DESCRIBE_STYLE-}" in @@ -502,6 +517,8 @@ __git_ps1 () b="$short_sha..." b="($b)" + else + b="$head" fi fi fi diff --git a/delta-islands.c b/delta-islands.c index ee2318d45a..f7e079425f 100644 --- a/delta-islands.c +++ b/delta-islands.c @@ -284,7 +284,7 @@ void resolve_tree_islands(struct repository *r, if (!tree || parse_tree(tree) < 0) die(_("bad tree object %s"), oid_to_hex(&ent->idx.oid)); - init_tree_desc(&desc, tree->buffer, tree->size); + init_tree_desc(&desc, &tree->object.oid, tree->buffer, tree->size); while (tree_entry(&desc, &entry)) { struct object *obj; diff --git a/diff-lib.c b/diff-lib.c index 6c8df04273..afc55761fa 100644 --- a/diff-lib.c +++ b/diff-lib.c @@ -562,7 +562,7 @@ static int diff_cache(struct rev_info *revs, opts.pathspec = &revs->diffopt.pathspec; opts.pathspec->recursive = 1; - init_tree_desc(&t, tree->buffer, tree->size); + init_tree_desc(&t, &tree->object.oid, tree->buffer, tree->size); return unpack_trees(1, &t, &opts); } diff --git a/diffcore-delta.c b/diffcore-delta.c index 4927ab8fb0..ba6cbee76b 100644 --- a/diffcore-delta.c +++ b/diffcore-delta.c @@ -158,6 +158,10 @@ static struct spanhash_top *hash_chars(struct repository *r, n = 0; accum1 = accum2 = 0; } + if (n > 0) { + hashval = (accum1 + accum2 * 0x61) % HASHBASE; + hash = add_spanhash(hash, hashval, n); + } QSORT(hash->data, (size_t)1ul << hash->alloc_log2, spanhash_cmp); return hash; } diff --git a/ewah/bitmap.c b/ewah/bitmap.c index 7b525b1ecd..ac7e0af622 100644 --- a/ewah/bitmap.c +++ b/ewah/bitmap.c @@ -169,6 +169,15 @@ size_t bitmap_popcount(struct bitmap *self) return count; } +int bitmap_is_empty(struct bitmap *self) +{ + size_t i; + for (i = 0; i < self->word_alloc; i++) + if (self->words[i]) + return 0; + return 1; +} + int bitmap_equals(struct bitmap *self, struct bitmap *other) { struct bitmap *big, *small; diff --git a/ewah/ewok.h b/ewah/ewok.h index 7eb8b9b630..c11d76c6f3 100644 --- a/ewah/ewok.h +++ b/ewah/ewok.h @@ -189,5 +189,6 @@ void bitmap_or_ewah(struct bitmap *self, struct ewah_bitmap *other); void bitmap_or(struct bitmap *self, const struct bitmap *other); size_t bitmap_popcount(struct bitmap *self); +int bitmap_is_empty(struct bitmap *self); #endif diff --git a/fetch-pack.c b/fetch-pack.c index 5b8aa0adc7..091f9a80a9 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -2216,7 +2216,7 @@ void negotiate_using_fetch(const struct oid_array *negotiation_tips, the_repository, "%d", negotiation_round); } - trace2_region_enter("fetch-pack", "negotiate_using_fetch", the_repository); + trace2_region_leave("fetch-pack", "negotiate_using_fetch", the_repository); trace2_data_intmax("negotiate_using_fetch", the_repository, "total_rounds", negotiation_round); clear_common_flag(acked_commits); @@ -20,7 +20,6 @@ #include "packfile.h" #include "submodule-config.h" #include "config.h" -#include "credential.h" #include "help.h" static ssize_t max_tree_entry_len = 4096; @@ -328,7 +327,8 @@ static int fsck_walk_tree(struct tree *tree, void *data, struct fsck_options *op return -1; name = fsck_get_object_name(options, &tree->object.oid); - if (init_tree_desc_gently(&desc, tree->buffer, tree->size, 0)) + if (init_tree_desc_gently(&desc, &tree->object.oid, + tree->buffer, tree->size, 0)) return -1; while (tree_entry_gently(&desc, &entry)) { struct object *obj; @@ -599,7 +599,8 @@ static int fsck_tree(const struct object_id *tree_oid, const char *o_name; struct name_stack df_dup_candidates = { NULL }; - if (init_tree_desc_gently(&desc, buffer, size, TREE_DESC_RAW_MODES)) { + if (init_tree_desc_gently(&desc, tree_oid, buffer, size, + TREE_DESC_RAW_MODES)) { retval += report(options, tree_oid, OBJ_TREE, FSCK_MSG_BAD_TREE, "cannot be parsed as a tree"); @@ -1047,138 +1048,6 @@ done: return ret; } -static int starts_with_dot_slash(const char *const path) -{ - return path_match_flags(path, PATH_MATCH_STARTS_WITH_DOT_SLASH | - PATH_MATCH_XPLATFORM); -} - -static int starts_with_dot_dot_slash(const char *const path) -{ - return path_match_flags(path, PATH_MATCH_STARTS_WITH_DOT_DOT_SLASH | - PATH_MATCH_XPLATFORM); -} - -static int submodule_url_is_relative(const char *url) -{ - return starts_with_dot_slash(url) || starts_with_dot_dot_slash(url); -} - -/* - * Count directory components that a relative submodule URL should chop - * from the remote_url it is to be resolved against. - * - * In other words, this counts "../" components at the start of a - * submodule URL. - * - * Returns the number of directory components to chop and writes a - * pointer to the next character of url after all leading "./" and - * "../" components to out. - */ -static int count_leading_dotdots(const char *url, const char **out) -{ - int result = 0; - while (1) { - if (starts_with_dot_dot_slash(url)) { - result++; - url += strlen("../"); - continue; - } - if (starts_with_dot_slash(url)) { - url += strlen("./"); - continue; - } - *out = url; - return result; - } -} -/* - * Check whether a transport is implemented by git-remote-curl. - * - * If it is, returns 1 and writes the URL that would be passed to - * git-remote-curl to the "out" parameter. - * - * Otherwise, returns 0 and leaves "out" untouched. - * - * Examples: - * http::https://example.com/repo.git -> 1, https://example.com/repo.git - * https://example.com/repo.git -> 1, https://example.com/repo.git - * git://example.com/repo.git -> 0 - * - * This is for use in checking for previously exploitable bugs that - * required a submodule URL to be passed to git-remote-curl. - */ -static int url_to_curl_url(const char *url, const char **out) -{ - /* - * We don't need to check for case-aliases, "http.exe", and so - * on because in the default configuration, is_transport_allowed - * prevents URLs with those schemes from being cloned - * automatically. - */ - if (skip_prefix(url, "http::", out) || - skip_prefix(url, "https::", out) || - skip_prefix(url, "ftp::", out) || - skip_prefix(url, "ftps::", out)) - return 1; - if (starts_with(url, "http://") || - starts_with(url, "https://") || - starts_with(url, "ftp://") || - starts_with(url, "ftps://")) { - *out = url; - return 1; - } - return 0; -} - -static int check_submodule_url(const char *url) -{ - const char *curl_url; - - if (looks_like_command_line_option(url)) - return -1; - - if (submodule_url_is_relative(url) || starts_with(url, "git://")) { - char *decoded; - const char *next; - int has_nl; - - /* - * This could be appended to an http URL and url-decoded; - * check for malicious characters. - */ - decoded = url_decode(url); - has_nl = !!strchr(decoded, '\n'); - - free(decoded); - if (has_nl) - return -1; - - /* - * URLs which escape their root via "../" can overwrite - * the host field and previous components, resolving to - * URLs like https::example.com/submodule.git and - * https:///example.com/submodule.git that were - * susceptible to CVE-2020-11008. - */ - if (count_leading_dotdots(url, &next) > 0 && - (*next == ':' || *next == '/')) - return -1; - } - - else if (url_to_curl_url(url, &curl_url)) { - struct credential c = CREDENTIAL_INIT; - int ret = 0; - if (credential_from_url_gently(&c, curl_url, 1) || - !*c.host) - ret = -1; - credential_clear(&c); - return ret; - } - - return 0; -} - struct fsck_gitmodules_data { const struct object_id *oid; struct fsck_options *options; diff --git a/git-compat-util.h b/git-compat-util.h index 603c97e3b3..7c2a6538e5 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -1015,6 +1015,15 @@ static inline unsigned long cast_size_t_to_ulong(size_t a) return (unsigned long)a; } +static inline uint32_t cast_size_t_to_uint32_t(size_t a) +{ + if (a != (uint32_t)a) + die("object too large to read on this platform: %" + PRIuMAX" is cut off to %u", + (uintmax_t)a, (uint32_t)a); + return (uint32_t)a; +} + static inline int cast_size_t_to_int(size_t a) { if (a > INT_MAX) @@ -4251,7 +4251,8 @@ class P4Sync(Command, P4UserMap): if self.tempBranches != []: for branch in self.tempBranches: read_pipe(["git", "update-ref", "-d", branch]) - os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation)) + if len(read_pipe(["git", "for-each-ref", self.tempBranchLocation])) > 0: + die("There are unexpected temporary branches") # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow # a convenient shortcut refname "p4". diff --git a/gitweb/gitweb.perl b/gitweb/gitweb.perl index fc6d5dd522..ccd14e0e30 100755 --- a/gitweb/gitweb.perl +++ b/gitweb/gitweb.perl @@ -728,9 +728,11 @@ our $per_request_config = 1; sub read_config_file { my $filename = shift; return unless defined $filename; - # die if there are errors parsing config file if (-e $filename) { do $filename; + # die if there is a problem accessing the file + die $! if $!; + # die if there are errors parsing config file die $@ if $@; return 1; } @@ -145,6 +145,7 @@ struct object_id { #define GET_OID_RECORD_PATH 0200 #define GET_OID_ONLY_TO_DIE 04000 #define GET_OID_REQUIRE_PATH 010000 +#define GET_OID_HASH_ANY 020000 #define GET_OID_DISAMBIGUATORS \ (GET_OID_COMMIT | GET_OID_COMMITTISH | \ @@ -73,10 +73,15 @@ static inline void oidclr(struct object_id *oid) oid->algo = hash_algo_by_ptr(the_hash_algo); } +static inline void oidread_algop(struct object_id *oid, const unsigned char *hash, const struct git_hash_algo *algop) +{ + memcpy(oid->hash, hash, algop->rawsz); + oid->algo = hash_algo_by_ptr(algop); +} + static inline void oidread(struct object_id *oid, const unsigned char *hash) { - memcpy(oid->hash, hash, the_hash_algo->rawsz); - oid->algo = hash_algo_by_ptr(the_hash_algo); + oidread_algop(oid, hash, the_hash_algo); } static inline int is_empty_blob_sha1(const unsigned char *sha1) diff --git a/http-backend.c b/http-backend.c index ff07b87e64..1ed1e29d07 100644 --- a/http-backend.c +++ b/http-backend.c @@ -38,6 +38,7 @@ struct rpc_service { static struct rpc_service rpc_service[] = { { "upload-pack", "uploadpack", 1, 1 }, { "receive-pack", "receivepack", 0, -1 }, + { "upload-archive", "uploadarchive", 0, -1 }, }; static struct string_list *get_parameters(void) @@ -639,10 +640,15 @@ static void check_content_type(struct strbuf *hdr, const char *accepted_type) static void service_rpc(struct strbuf *hdr, char *service_name) { - const char *argv[] = {NULL, "--stateless-rpc", ".", NULL}; + struct strvec argv = STRVEC_INIT; struct rpc_service *svc = select_service(hdr, service_name); struct strbuf buf = STRBUF_INIT; + strvec_push(&argv, svc->name); + if (strcmp(service_name, "git-upload-archive")) + strvec_push(&argv, "--stateless-rpc"); + strvec_push(&argv, "."); + strbuf_reset(&buf); strbuf_addf(&buf, "application/x-git-%s-request", svc->name); check_content_type(hdr, buf.buf); @@ -655,9 +661,9 @@ static void service_rpc(struct strbuf *hdr, char *service_name) end_headers(hdr); - argv[0] = svc->name; - run_service(argv, svc->buffer_input); + run_service(argv.v, svc->buffer_input); strbuf_release(&buf); + strvec_clear(&argv); } static int dead; @@ -723,6 +729,7 @@ static struct service_cmd { {"GET", "/objects/pack/pack-[0-9a-f]{64}\\.idx$", get_idx_file}, {"POST", "/git-upload-pack$", service_rpc}, + {"POST", "/git-upload-archive$", service_rpc}, {"POST", "/git-receive-pack$", service_rpc} }; diff --git a/http-push.c b/http-push.c index b4d0b2a6aa..7890e823c4 100644 --- a/http-push.c +++ b/http-push.c @@ -1307,7 +1307,7 @@ static struct object_list **process_tree(struct tree *tree, obj->flags |= SEEN; p = add_one_object(obj, p); - init_tree_desc(&desc, tree->buffer, tree->size); + init_tree_desc(&desc, &tree->object.oid, tree->buffer, tree->size); while (tree_entry(&desc, &entry)) switch (object_type(entry.mode)) { @@ -1851,6 +1851,7 @@ int cmd_main(int argc, const char **argv) if (oideq(&ref->old_oid, &ref->peer_ref->new_oid)) { if (push_verbosely) + /* stable plumbing output; do not modify or localize */ fprintf(stderr, "'%s': up-to-date\n", ref->name); if (helper_status) printf("ok %s up to date\n", ref->name); @@ -1871,6 +1872,7 @@ int cmd_main(int argc, const char **argv) * commits at the remote end and likely * we were not up to date to begin with. */ + /* stable plumbing output; do not modify or localize */ error("remote '%s' is not an ancestor of\n" "local '%s'.\n" "Maybe you are not up-to-date and " diff --git a/list-objects.c b/list-objects.c index f39b68faf5..11ad8be411 100644 --- a/list-objects.c +++ b/list-objects.c @@ -102,7 +102,7 @@ static void process_tree_contents(struct traversal_context *ctx, enum interesting match = ctx->revs->diffopt.pathspec.nr == 0 ? all_entries_interesting : entry_not_interesting; - init_tree_desc(&desc, tree->buffer, tree->size); + init_tree_desc(&desc, &tree->object.oid, tree->buffer, tree->size); while (tree_entry(&desc, &entry)) { if (match != all_entries_interesting) { diff --git a/log-tree.c b/log-tree.c index 337b9334cd..5f79e98adb 100644 --- a/log-tree.c +++ b/log-tree.c @@ -35,13 +35,16 @@ static int decoration_loaded; static int decoration_flags; static char decoration_colors[][COLOR_MAXLEN] = { - GIT_COLOR_RESET, - GIT_COLOR_BOLD_GREEN, /* REF_LOCAL */ - GIT_COLOR_BOLD_RED, /* REF_REMOTE */ - GIT_COLOR_BOLD_YELLOW, /* REF_TAG */ - GIT_COLOR_BOLD_MAGENTA, /* REF_STASH */ - GIT_COLOR_BOLD_CYAN, /* REF_HEAD */ - GIT_COLOR_BOLD_BLUE, /* GRAFTED */ + [DECORATION_NONE] = GIT_COLOR_RESET, + [DECORATION_REF_LOCAL] = GIT_COLOR_BOLD_GREEN, + [DECORATION_REF_REMOTE] = GIT_COLOR_BOLD_RED, + [DECORATION_REF_TAG] = GIT_COLOR_BOLD_YELLOW, + [DECORATION_REF_STASH] = GIT_COLOR_BOLD_MAGENTA, + [DECORATION_REF] = GIT_COLOR_BOLD_MAGENTA, + [DECORATION_REF_HEAD] = GIT_COLOR_BOLD_CYAN, + [DECORATION_REF_PSEUDO] = GIT_COLOR_BOLD_CYAN, + [DECORATION_GRAFTED] = GIT_COLOR_BOLD_BLUE, + [DECORATION_SYMBOL] = GIT_COLOR_NIL, }; static const char *color_decorate_slots[] = { @@ -49,8 +52,11 @@ static const char *color_decorate_slots[] = { [DECORATION_REF_REMOTE] = "remoteBranch", [DECORATION_REF_TAG] = "tag", [DECORATION_REF_STASH] = "stash", + [DECORATION_REF] = "ref", [DECORATION_REF_HEAD] = "HEAD", + [DECORATION_REF_PSEUDO] = "pseudoref", [DECORATION_GRAFTED] = "grafted", + [DECORATION_SYMBOL] = "symbol", }; static const char *decorate_get_color(int decorate_use_color, enum decoration_type ix) @@ -150,7 +156,7 @@ static int add_ref_decoration(const char *refname, const struct object_id *oid, int i; struct object *obj; enum object_type objtype; - enum decoration_type deco_type = DECORATION_NONE; + enum decoration_type deco_type = DECORATION_REF; struct decoration_filter *filter = (struct decoration_filter *)cb_data; const char *git_replace_ref_base = ref_namespace[NAMESPACE_REPLACE].ref; @@ -205,6 +211,27 @@ static int add_ref_decoration(const char *refname, const struct object_id *oid, return 0; } +static int add_pseudoref_decoration(const char *refname, + const struct object_id *oid, + int flags UNUSED, + void *cb_data) +{ + struct object *obj; + enum object_type objtype; + struct decoration_filter *filter = (struct decoration_filter *)cb_data; + + if (filter && !ref_filter_match(refname, filter)) + return 0; + + objtype = oid_object_info(the_repository, oid, NULL); + if (objtype < 0) + return 0; + + obj = lookup_object_by_type(the_repository, oid, objtype); + add_name_decoration(DECORATION_REF_PSEUDO, refname, obj); + return 0; +} + static int add_graft_decoration(const struct commit_graft *graft, void *cb_data UNUSED) { @@ -233,6 +260,7 @@ void load_ref_decorations(struct decoration_filter *filter, int flags) decoration_loaded = 1; decoration_flags = flags; for_each_ref(add_ref_decoration, filter); + for_each_pseudoref(add_pseudoref_decoration, filter); head_ref(add_ref_decoration, filter); for_each_commit_graft(add_graft_decoration, filter); } @@ -313,7 +341,7 @@ void format_decorations(struct strbuf *sb, { const struct name_decoration *decoration; const struct name_decoration *current_and_HEAD; - const char *color_commit, *color_reset; + const char *color_symbol, *color_reset; const char *prefix = " ("; const char *suffix = ")"; @@ -338,7 +366,10 @@ void format_decorations(struct strbuf *sb, tag = opts->tag; } - color_commit = diff_get_color(use_color, DIFF_COMMIT); + color_symbol = decorate_get_color(use_color, DECORATION_SYMBOL); + if (color_is_nil(color_symbol)) + color_symbol = diff_get_color(use_color, DIFF_COMMIT); + color_reset = decorate_get_color(use_color, DECORATION_NONE); current_and_HEAD = current_pointed_by_HEAD(decoration); @@ -353,7 +384,7 @@ void format_decorations(struct strbuf *sb, decorate_get_color(use_color, decoration->type); if (*prefix) { - strbuf_addstr(sb, color_commit); + strbuf_addstr(sb, color_symbol); strbuf_addstr(sb, prefix); strbuf_addstr(sb, color_reset); } @@ -370,7 +401,7 @@ void format_decorations(struct strbuf *sb, if (current_and_HEAD && decoration->type == DECORATION_REF_HEAD) { - strbuf_addstr(sb, color_commit); + strbuf_addstr(sb, color_symbol); strbuf_addstr(sb, pointer); strbuf_addstr(sb, color_reset); strbuf_addstr(sb, decorate_get_color(use_color, current_and_HEAD->type)); @@ -383,7 +414,7 @@ void format_decorations(struct strbuf *sb, decoration = decoration->next; } if (*suffix) { - strbuf_addstr(sb, color_commit); + strbuf_addstr(sb, color_symbol); strbuf_addstr(sb, suffix); strbuf_addstr(sb, color_reset); } diff --git a/loose.c b/loose.c new file mode 100644 index 0000000000..f6faa6216a --- /dev/null +++ b/loose.c @@ -0,0 +1,259 @@ +#include "git-compat-util.h" +#include "hash.h" +#include "path.h" +#include "object-store.h" +#include "hex.h" +#include "wrapper.h" +#include "gettext.h" +#include "loose.h" +#include "lockfile.h" +#include "oidtree.h" + +static const char *loose_object_header = "# loose-object-idx\n"; + +static inline int should_use_loose_object_map(struct repository *repo) +{ + return repo->compat_hash_algo && repo->gitdir; +} + +void loose_object_map_init(struct loose_object_map **map) +{ + struct loose_object_map *m; + m = xmalloc(sizeof(**map)); + m->to_compat = kh_init_oid_map(); + m->to_storage = kh_init_oid_map(); + *map = m; +} + +static int insert_oid_pair(kh_oid_map_t *map, const struct object_id *key, const struct object_id *value) +{ + khiter_t pos; + int ret; + struct object_id *stored; + + pos = kh_put_oid_map(map, *key, &ret); + + /* This item already exists in the map. */ + if (ret == 0) + return 0; + + stored = xmalloc(sizeof(*stored)); + oidcpy(stored, value); + kh_value(map, pos) = stored; + return 1; +} + +static int insert_loose_map(struct object_directory *odb, + const struct object_id *oid, + const struct object_id *compat_oid) +{ + struct loose_object_map *map = odb->loose_map; + int inserted = 0; + + inserted |= insert_oid_pair(map->to_compat, oid, compat_oid); + inserted |= insert_oid_pair(map->to_storage, compat_oid, oid); + if (inserted) + oidtree_insert(odb->loose_objects_cache, compat_oid); + + return inserted; +} + +static int load_one_loose_object_map(struct repository *repo, struct object_directory *dir) +{ + struct strbuf buf = STRBUF_INIT, path = STRBUF_INIT; + FILE *fp; + + if (!dir->loose_map) + loose_object_map_init(&dir->loose_map); + if (!dir->loose_objects_cache) { + ALLOC_ARRAY(dir->loose_objects_cache, 1); + oidtree_init(dir->loose_objects_cache); + } + + insert_loose_map(dir, repo->hash_algo->empty_tree, repo->compat_hash_algo->empty_tree); + insert_loose_map(dir, repo->hash_algo->empty_blob, repo->compat_hash_algo->empty_blob); + insert_loose_map(dir, repo->hash_algo->null_oid, repo->compat_hash_algo->null_oid); + + strbuf_git_common_path(&path, repo, "objects/loose-object-idx"); + fp = fopen(path.buf, "rb"); + if (!fp) { + strbuf_release(&path); + return 0; + } + + errno = 0; + if (strbuf_getwholeline(&buf, fp, '\n') || strcmp(buf.buf, loose_object_header)) + goto err; + while (!strbuf_getline_lf(&buf, fp)) { + const char *p; + struct object_id oid, compat_oid; + if (parse_oid_hex_algop(buf.buf, &oid, &p, repo->hash_algo) || + *p++ != ' ' || + parse_oid_hex_algop(p, &compat_oid, &p, repo->compat_hash_algo) || + p != buf.buf + buf.len) + goto err; + insert_loose_map(dir, &oid, &compat_oid); + } + + strbuf_release(&buf); + strbuf_release(&path); + return errno ? -1 : 0; +err: + strbuf_release(&buf); + strbuf_release(&path); + return -1; +} + +int repo_read_loose_object_map(struct repository *repo) +{ + struct object_directory *dir; + + if (!should_use_loose_object_map(repo)) + return 0; + + prepare_alt_odb(repo); + + for (dir = repo->objects->odb; dir; dir = dir->next) { + if (load_one_loose_object_map(repo, dir) < 0) { + return -1; + } + } + return 0; +} + +int repo_write_loose_object_map(struct repository *repo) +{ + kh_oid_map_t *map = repo->objects->odb->loose_map->to_compat; + struct lock_file lock; + int fd; + khiter_t iter; + struct strbuf buf = STRBUF_INIT, path = STRBUF_INIT; + + if (!should_use_loose_object_map(repo)) + return 0; + + strbuf_git_common_path(&path, repo, "objects/loose-object-idx"); + fd = hold_lock_file_for_update_timeout(&lock, path.buf, LOCK_DIE_ON_ERROR, -1); + iter = kh_begin(map); + if (write_in_full(fd, loose_object_header, strlen(loose_object_header)) < 0) + goto errout; + + for (; iter != kh_end(map); iter++) { + if (kh_exist(map, iter)) { + if (oideq(&kh_key(map, iter), the_hash_algo->empty_tree) || + oideq(&kh_key(map, iter), the_hash_algo->empty_blob)) + continue; + strbuf_addf(&buf, "%s %s\n", oid_to_hex(&kh_key(map, iter)), oid_to_hex(kh_value(map, iter))); + if (write_in_full(fd, buf.buf, buf.len) < 0) + goto errout; + strbuf_reset(&buf); + } + } + strbuf_release(&buf); + if (commit_lock_file(&lock) < 0) { + error_errno(_("could not write loose object index %s"), path.buf); + strbuf_release(&path); + return -1; + } + strbuf_release(&path); + return 0; +errout: + rollback_lock_file(&lock); + strbuf_release(&buf); + error_errno(_("failed to write loose object index %s\n"), path.buf); + strbuf_release(&path); + return -1; +} + +static int write_one_object(struct repository *repo, const struct object_id *oid, + const struct object_id *compat_oid) +{ + struct lock_file lock; + int fd; + struct stat st; + struct strbuf buf = STRBUF_INIT, path = STRBUF_INIT; + + strbuf_git_common_path(&path, repo, "objects/loose-object-idx"); + hold_lock_file_for_update_timeout(&lock, path.buf, LOCK_DIE_ON_ERROR, -1); + + fd = open(path.buf, O_WRONLY | O_CREAT | O_APPEND, 0666); + if (fd < 0) + goto errout; + if (fstat(fd, &st) < 0) + goto errout; + if (!st.st_size && write_in_full(fd, loose_object_header, strlen(loose_object_header)) < 0) + goto errout; + + strbuf_addf(&buf, "%s %s\n", oid_to_hex(oid), oid_to_hex(compat_oid)); + if (write_in_full(fd, buf.buf, buf.len) < 0) + goto errout; + if (close(fd)) + goto errout; + adjust_shared_perm(path.buf); + rollback_lock_file(&lock); + strbuf_release(&buf); + strbuf_release(&path); + return 0; +errout: + error_errno(_("failed to write loose object index %s\n"), path.buf); + close(fd); + rollback_lock_file(&lock); + strbuf_release(&buf); + strbuf_release(&path); + return -1; +} + +int repo_add_loose_object_map(struct repository *repo, const struct object_id *oid, + const struct object_id *compat_oid) +{ + int inserted = 0; + + if (!should_use_loose_object_map(repo)) + return 0; + + inserted = insert_loose_map(repo->objects->odb, oid, compat_oid); + if (inserted) + return write_one_object(repo, oid, compat_oid); + return 0; +} + +int repo_loose_object_map_oid(struct repository *repo, + const struct object_id *src, + const struct git_hash_algo *to, + struct object_id *dest) +{ + struct object_directory *dir; + kh_oid_map_t *map; + khiter_t pos; + + for (dir = repo->objects->odb; dir; dir = dir->next) { + struct loose_object_map *loose_map = dir->loose_map; + if (!loose_map) + continue; + map = (to == repo->compat_hash_algo) ? + loose_map->to_compat : + loose_map->to_storage; + pos = kh_get_oid_map(map, *src); + if (pos < kh_end(map)) { + oidcpy(dest, kh_value(map, pos)); + return 0; + } + } + return -1; +} + +void loose_object_map_clear(struct loose_object_map **map) +{ + struct loose_object_map *m = *map; + struct object_id *oid; + + if (!m) + return; + + kh_foreach_value(m->to_compat, oid, free(oid)); + kh_foreach_value(m->to_storage, oid, free(oid)); + kh_destroy_oid_map(m->to_compat); + kh_destroy_oid_map(m->to_storage); + free(m); + *map = NULL; +} diff --git a/loose.h b/loose.h new file mode 100644 index 0000000000..2c2957072c --- /dev/null +++ b/loose.h @@ -0,0 +1,22 @@ +#ifndef LOOSE_H +#define LOOSE_H + +#include "khash.h" + +struct loose_object_map { + kh_oid_map_t *to_compat; + kh_oid_map_t *to_storage; +}; + +void loose_object_map_init(struct loose_object_map **map); +void loose_object_map_clear(struct loose_object_map **map); +int repo_loose_object_map_oid(struct repository *repo, + const struct object_id *src, + const struct git_hash_algo *dest_algo, + struct object_id *dest); +int repo_add_loose_object_map(struct repository *repo, const struct object_id *oid, + const struct object_id *compat_oid); +int repo_read_loose_object_map(struct repository *repo); +int repo_write_loose_object_map(struct repository *repo); + +#endif diff --git a/match-trees.c b/match-trees.c index 0885ac681c..3412b6a140 100644 --- a/match-trees.c +++ b/match-trees.c @@ -63,7 +63,7 @@ static void *fill_tree_desc_strict(struct tree_desc *desc, die("unable to read tree (%s)", oid_to_hex(hash)); if (type != OBJ_TREE) die("%s is not a tree", oid_to_hex(hash)); - init_tree_desc(desc, buffer, size); + init_tree_desc(desc, hash, buffer, size); return buffer; } @@ -194,7 +194,7 @@ static int splice_tree(const struct object_id *oid1, const char *prefix, buf = repo_read_object_file(the_repository, oid1, &type, &sz); if (!buf) die("cannot read tree %s", oid_to_hex(oid1)); - init_tree_desc(&desc, buf, sz); + init_tree_desc(&desc, oid1, buf, sz); rewrite_here = NULL; while (desc.size) { diff --git a/merge-ort.c b/merge-ort.c index 77ba7f3020..3dcd5e56f4 100644 --- a/merge-ort.c +++ b/merge-ort.c @@ -38,6 +38,7 @@ #include "path.h" #include "promisor-remote.h" #include "read-cache-ll.h" +#include "refs.h" #include "revision.h" #include "sparse-index.h" #include "strmap.h" @@ -1659,9 +1660,10 @@ static int collect_merge_info(struct merge_options *opt, parse_tree(merge_base); parse_tree(side1); parse_tree(side2); - init_tree_desc(t + 0, merge_base->buffer, merge_base->size); - init_tree_desc(t + 1, side1->buffer, side1->size); - init_tree_desc(t + 2, side2->buffer, side2->size); + init_tree_desc(t + 0, &merge_base->object.oid, + merge_base->buffer, merge_base->size); + init_tree_desc(t + 1, &side1->object.oid, side1->buffer, side1->size); + init_tree_desc(t + 2, &side2->object.oid, side2->buffer, side2->size); trace2_region_enter("merge", "traverse_trees", opt->repo); ret = traverse_trees(NULL, 3, t, &info); @@ -4376,9 +4378,9 @@ static int checkout(struct merge_options *opt, unpack_opts.fn = twoway_merge; unpack_opts.preserve_ignored = 0; /* FIXME: !opts->overwrite_ignore */ parse_tree(prev); - init_tree_desc(&trees[0], prev->buffer, prev->size); + init_tree_desc(&trees[0], &prev->object.oid, prev->buffer, prev->size); parse_tree(next); - init_tree_desc(&trees[1], next->buffer, next->size); + init_tree_desc(&trees[1], &next->object.oid, next->buffer, next->size); ret = unpack_trees(2, trees, &unpack_opts); clear_unpack_trees_porcelain(&unpack_opts); @@ -4659,9 +4661,6 @@ void merge_switch_to_result(struct merge_options *opt, { assert(opt->priv == NULL); if (result->clean >= 0 && update_worktree_and_index) { - const char *filename; - FILE *fp; - trace2_region_enter("merge", "checkout", opt->repo); if (checkout(opt, head, result->tree)) { /* failure to function */ @@ -4687,10 +4686,17 @@ void merge_switch_to_result(struct merge_options *opt, trace2_region_leave("merge", "record_conflicted", opt->repo); trace2_region_enter("merge", "write_auto_merge", opt->repo); - filename = git_path_auto_merge(opt->repo); - fp = xfopen(filename, "w"); - fprintf(fp, "%s\n", oid_to_hex(&result->tree->object.oid)); - fclose(fp); + if (refs_update_ref(get_main_ref_store(opt->repo), "", "AUTO_MERGE", + &result->tree->object.oid, NULL, REF_NO_DEREF, + UPDATE_REFS_MSG_ON_ERR)) { + /* failure to function */ + opt->priv = NULL; + result->clean = -1; + merge_finalize(opt, result); + trace2_region_leave("merge", "write_auto_merge", + opt->repo); + return; + } trace2_region_leave("merge", "write_auto_merge", opt->repo); } if (display_update_msgs) diff --git a/merge-recursive.c b/merge-recursive.c index a0c3e7a2d9..be468f0c72 100644 --- a/merge-recursive.c +++ b/merge-recursive.c @@ -406,7 +406,7 @@ static inline int merge_detect_rename(struct merge_options *opt) static void init_tree_desc_from_tree(struct tree_desc *desc, struct tree *tree) { parse_tree(tree); - init_tree_desc(desc, tree->buffer, tree->size); + init_tree_desc(desc, &tree->object.oid, tree->buffer, tree->size); } static int unpack_trees_start(struct merge_options *opt, @@ -78,7 +78,8 @@ int checkout_fast_forward(struct repository *r, } for (i = 0; i < nr_trees; i++) { parse_tree(trees[i]); - init_tree_desc(t+i, trees[i]->buffer, trees[i]->size); + init_tree_desc(t+i, &trees[i]->object.oid, + trees[i]->buffer, trees[i]->size); } memset(&opts, 0, sizeof(opts)); @@ -21,6 +21,7 @@ #include "refs.h" #include "revision.h" #include "list-objects.h" +#include "pack-revindex.h" #define MIDX_SIGNATURE 0x4d494458 /* "MIDX" */ #define MIDX_VERSION 1 @@ -33,6 +34,7 @@ #define MIDX_CHUNK_ALIGNMENT 4 #define MIDX_CHUNKID_PACKNAMES 0x504e414d /* "PNAM" */ +#define MIDX_CHUNKID_BITMAPPEDPACKS 0x42544d50 /* "BTMP" */ #define MIDX_CHUNKID_OIDFANOUT 0x4f494446 /* "OIDF" */ #define MIDX_CHUNKID_OIDLOOKUP 0x4f49444c /* "OIDL" */ #define MIDX_CHUNKID_OBJECTOFFSETS 0x4f4f4646 /* "OOFF" */ @@ -41,6 +43,7 @@ #define MIDX_CHUNK_FANOUT_SIZE (sizeof(uint32_t) * 256) #define MIDX_CHUNK_OFFSET_WIDTH (2 * sizeof(uint32_t)) #define MIDX_CHUNK_LARGE_OFFSET_WIDTH (sizeof(uint64_t)) +#define MIDX_CHUNK_BITMAPPED_PACKS_WIDTH (2 * sizeof(uint32_t)) #define MIDX_LARGE_OFFSET_NEEDED 0x80000000 #define PACK_EXPIRED UINT_MAX @@ -86,32 +89,6 @@ static int midx_read_oid_fanout(const unsigned char *chunk_start, return 0; } -static int midx_read_oid_lookup(const unsigned char *chunk_start, - size_t chunk_size, void *data) -{ - struct multi_pack_index *m = data; - m->chunk_oid_lookup = chunk_start; - - if (chunk_size != st_mult(m->hash_len, m->num_objects)) { - error(_("multi-pack-index OID lookup chunk is the wrong size")); - return 1; - } - return 0; -} - -static int midx_read_object_offsets(const unsigned char *chunk_start, - size_t chunk_size, void *data) -{ - struct multi_pack_index *m = data; - m->chunk_object_offsets = chunk_start; - - if (chunk_size != st_mult(m->num_objects, MIDX_CHUNK_OFFSET_WIDTH)) { - error(_("multi-pack-index object offset chunk is the wrong size")); - return 1; - } - return 0; -} - struct multi_pack_index *load_multi_pack_index(const char *object_dir, int local) { struct multi_pack_index *m = NULL; @@ -175,6 +152,8 @@ struct multi_pack_index *load_multi_pack_index(const char *object_dir, int local m->num_packs = get_be32(m->data + MIDX_BYTE_NUM_PACKS); + m->preferred_pack_idx = -1; + cf = init_chunkfile(NULL); if (read_table_of_contents(cf, m->data, midx_size, @@ -186,13 +165,19 @@ struct multi_pack_index *load_multi_pack_index(const char *object_dir, int local die(_("multi-pack-index required pack-name chunk missing or corrupted")); if (read_chunk(cf, MIDX_CHUNKID_OIDFANOUT, midx_read_oid_fanout, m)) die(_("multi-pack-index required OID fanout chunk missing or corrupted")); - if (read_chunk(cf, MIDX_CHUNKID_OIDLOOKUP, midx_read_oid_lookup, m)) + if (pair_chunk_expect(cf, MIDX_CHUNKID_OIDLOOKUP, &m->chunk_oid_lookup, + m->hash_len, m->num_objects)) die(_("multi-pack-index required OID lookup chunk missing or corrupted")); - if (read_chunk(cf, MIDX_CHUNKID_OBJECTOFFSETS, midx_read_object_offsets, m)) + if (pair_chunk_expect(cf, MIDX_CHUNKID_OBJECTOFFSETS, + &m->chunk_object_offsets, MIDX_CHUNK_OFFSET_WIDTH, + m->num_objects)) die(_("multi-pack-index required object offsets chunk missing or corrupted")); pair_chunk(cf, MIDX_CHUNKID_LARGEOFFSETS, &m->chunk_large_offsets, &m->chunk_large_offsets_len); + pair_chunk(cf, MIDX_CHUNKID_BITMAPPEDPACKS, + (const unsigned char **)&m->chunk_bitmapped_packs, + &m->chunk_bitmapped_packs_len); if (git_env_bool("GIT_TEST_MIDX_READ_RIDX", 1)) pair_chunk(cf, MIDX_CHUNKID_REVINDEX, &m->chunk_revindex, @@ -286,6 +271,26 @@ int prepare_midx_pack(struct repository *r, struct multi_pack_index *m, uint32_t return 0; } +int nth_bitmapped_pack(struct repository *r, struct multi_pack_index *m, + struct bitmapped_pack *bp, uint32_t pack_int_id) +{ + if (!m->chunk_bitmapped_packs) + return error(_("MIDX does not contain the BTMP chunk")); + + if (prepare_midx_pack(r, m, pack_int_id)) + return error(_("could not load bitmapped pack %"PRIu32), pack_int_id); + + bp->p = m->packs[pack_int_id]; + bp->bitmap_pos = get_be32((char *)m->chunk_bitmapped_packs + + MIDX_CHUNK_BITMAPPED_PACKS_WIDTH * pack_int_id); + bp->bitmap_nr = get_be32((char *)m->chunk_bitmapped_packs + + MIDX_CHUNK_BITMAPPED_PACKS_WIDTH * pack_int_id + + sizeof(uint32_t)); + bp->pack_int_id = pack_int_id; + + return 0; +} + int bsearch_midx(const struct object_id *oid, struct multi_pack_index *m, uint32_t *result) { return bsearch_hash(oid->hash, m->chunk_oid_fanout, m->chunk_oid_lookup, @@ -403,7 +408,8 @@ static int cmp_idx_or_pack_name(const char *idx_or_pack_name, return strcmp(idx_or_pack_name, idx_name); } -int midx_contains_pack(struct multi_pack_index *m, const char *idx_or_pack_name) +int midx_locate_pack(struct multi_pack_index *m, const char *idx_or_pack_name, + uint32_t *pos) { uint32_t first = 0, last = m->num_packs; @@ -414,8 +420,11 @@ int midx_contains_pack(struct multi_pack_index *m, const char *idx_or_pack_name) current = m->pack_names[mid]; cmp = cmp_idx_or_pack_name(idx_or_pack_name, current); - if (!cmp) + if (!cmp) { + if (pos) + *pos = mid; return 1; + } if (cmp > 0) { first = mid + 1; continue; @@ -426,6 +435,28 @@ int midx_contains_pack(struct multi_pack_index *m, const char *idx_or_pack_name) return 0; } +int midx_contains_pack(struct multi_pack_index *m, const char *idx_or_pack_name) +{ + return midx_locate_pack(m, idx_or_pack_name, NULL); +} + +int midx_preferred_pack(struct multi_pack_index *m, uint32_t *pack_int_id) +{ + if (m->preferred_pack_idx == -1) { + if (load_midx_revindex(m) < 0) { + m->preferred_pack_idx = -2; + return -1; + } + + m->preferred_pack_idx = + nth_midxed_pack_int_id(m, pack_pos_to_midx(m, 0)); + } else if (m->preferred_pack_idx == -2) + return -1; /* no revindex */ + + *pack_int_id = m->preferred_pack_idx; + return 0; +} + int prepare_multi_pack_index_one(struct repository *r, const char *object_dir, int local) { struct multi_pack_index *m; @@ -468,13 +499,31 @@ static size_t write_midx_header(struct hashfile *f, return MIDX_HEADER_SIZE; } +#define BITMAP_POS_UNKNOWN (~((uint32_t)0)) + struct pack_info { uint32_t orig_pack_int_id; char *pack_name; struct packed_git *p; + + uint32_t bitmap_pos; + uint32_t bitmap_nr; + unsigned expired : 1; }; +static void fill_pack_info(struct pack_info *info, + struct packed_git *p, const char *pack_name, + uint32_t orig_pack_int_id) +{ + memset(info, 0, sizeof(struct pack_info)); + + info->orig_pack_int_id = orig_pack_int_id; + info->pack_name = xstrdup(pack_name); + info->p = p; + info->bitmap_pos = BITMAP_POS_UNKNOWN; +} + static int pack_info_compare(const void *_a, const void *_b) { struct pack_info *a = (struct pack_info *)_a; @@ -515,6 +564,7 @@ static void add_pack_to_midx(const char *full_path, size_t full_path_len, const char *file_name, void *data) { struct write_midx_context *ctx = data; + struct packed_git *p; if (ends_with(file_name, ".idx")) { display_progress(ctx->progress, ++ctx->pack_paths_checked); @@ -541,27 +591,22 @@ static void add_pack_to_midx(const char *full_path, size_t full_path_len, ALLOC_GROW(ctx->info, ctx->nr + 1, ctx->alloc); - ctx->info[ctx->nr].p = add_packed_git(full_path, - full_path_len, - 0); - - if (!ctx->info[ctx->nr].p) { + p = add_packed_git(full_path, full_path_len, 0); + if (!p) { warning(_("failed to add packfile '%s'"), full_path); return; } - if (open_pack_index(ctx->info[ctx->nr].p)) { + if (open_pack_index(p)) { warning(_("failed to open pack-index '%s'"), full_path); - close_pack(ctx->info[ctx->nr].p); - FREE_AND_NULL(ctx->info[ctx->nr].p); + close_pack(p); + free(p); return; } - ctx->info[ctx->nr].pack_name = xstrdup(file_name); - ctx->info[ctx->nr].orig_pack_int_id = ctx->nr; - ctx->info[ctx->nr].expired = 0; + fill_pack_info(&ctx->info[ctx->nr], p, file_name, ctx->nr); ctx->nr++; } } @@ -817,6 +862,26 @@ static int write_midx_pack_names(struct hashfile *f, void *data) return 0; } +static int write_midx_bitmapped_packs(struct hashfile *f, void *data) +{ + struct write_midx_context *ctx = data; + size_t i; + + for (i = 0; i < ctx->nr; i++) { + struct pack_info *pack = &ctx->info[i]; + if (pack->expired) + continue; + + if (pack->bitmap_pos == BITMAP_POS_UNKNOWN && pack->bitmap_nr) + BUG("pack '%s' has no bitmap position, but has %d bitmapped object(s)", + pack->pack_name, pack->bitmap_nr); + + hashwrite_be32(f, pack->bitmap_pos); + hashwrite_be32(f, pack->bitmap_nr); + } + return 0; +} + static int write_midx_oid_fanout(struct hashfile *f, void *data) { @@ -984,8 +1049,19 @@ static uint32_t *midx_pack_order(struct write_midx_context *ctx) QSORT(data, ctx->entries_nr, midx_pack_order_cmp); ALLOC_ARRAY(pack_order, ctx->entries_nr); - for (i = 0; i < ctx->entries_nr; i++) + for (i = 0; i < ctx->entries_nr; i++) { + struct pack_midx_entry *e = &ctx->entries[data[i].nr]; + struct pack_info *pack = &ctx->info[ctx->pack_perm[e->pack_int_id]]; + if (pack->bitmap_pos == BITMAP_POS_UNKNOWN) + pack->bitmap_pos = i; + pack->bitmap_nr++; pack_order[i] = data[i].nr; + } + for (i = 0; i < ctx->nr; i++) { + struct pack_info *pack = &ctx->info[ctx->pack_perm[i]]; + if (pack->bitmap_pos == BITMAP_POS_UNKNOWN) + pack->bitmap_pos = 0; + } free(data); trace2_region_leave("midx", "midx_pack_order", the_repository); @@ -1286,6 +1362,7 @@ static int write_midx_internal(const char *object_dir, struct hashfile *f = NULL; struct lock_file lk; struct write_midx_context ctx = { 0 }; + int bitmapped_packs_concat_len = 0; int pack_name_concat_len = 0; int dropped_packs = 0; int result = 0; @@ -1321,11 +1398,6 @@ static int write_midx_internal(const char *object_dir, for (i = 0; i < ctx.m->num_packs; i++) { ALLOC_GROW(ctx.info, ctx.nr + 1, ctx.alloc); - ctx.info[ctx.nr].orig_pack_int_id = i; - ctx.info[ctx.nr].pack_name = xstrdup(ctx.m->pack_names[i]); - ctx.info[ctx.nr].p = ctx.m->packs[i]; - ctx.info[ctx.nr].expired = 0; - if (flags & MIDX_WRITE_REV_INDEX) { /* * If generating a reverse index, need to have @@ -1341,10 +1413,10 @@ static int write_midx_internal(const char *object_dir, if (open_pack_index(ctx.m->packs[i])) die(_("could not open index for %s"), ctx.m->packs[i]->pack_name); - ctx.info[ctx.nr].p = ctx.m->packs[i]; } - ctx.nr++; + fill_pack_info(&ctx.info[ctx.nr++], ctx.m->packs[i], + ctx.m->pack_names[i], i); } } @@ -1503,8 +1575,10 @@ static int write_midx_internal(const char *object_dir, } for (i = 0; i < ctx.nr; i++) { - if (!ctx.info[i].expired) - pack_name_concat_len += strlen(ctx.info[i].pack_name) + 1; + if (ctx.info[i].expired) + continue; + pack_name_concat_len += strlen(ctx.info[i].pack_name) + 1; + bitmapped_packs_concat_len += 2 * sizeof(uint32_t); } /* Check that the preferred pack wasn't expired (if given). */ @@ -1564,6 +1638,9 @@ static int write_midx_internal(const char *object_dir, add_chunk(cf, MIDX_CHUNKID_REVINDEX, st_mult(ctx.entries_nr, sizeof(uint32_t)), write_midx_revindex); + add_chunk(cf, MIDX_CHUNKID_BITMAPPEDPACKS, + bitmapped_packs_concat_len, + write_midx_bitmapped_packs); } write_midx_header(f, get_num_chunks(cf), ctx.nr - dropped_packs); @@ -1603,8 +1680,13 @@ static int write_midx_internal(const char *object_dir, flags) < 0) { error(_("could not write multi-pack bitmap")); result = 1; + clear_packing_data(&pdata); + free(commits); goto cleanup; } + + clear_packing_data(&pdata); + free(commits); } /* * NOTE: Do not use ctx.entries beyond this point, since it might @@ -6,6 +6,7 @@ struct object_id; struct pack_entry; struct repository; +struct bitmapped_pack; #define GIT_TEST_MULTI_PACK_INDEX "GIT_TEST_MULTI_PACK_INDEX" #define GIT_TEST_MULTI_PACK_INDEX_WRITE_BITMAP \ @@ -27,11 +28,14 @@ struct multi_pack_index { unsigned char num_chunks; uint32_t num_packs; uint32_t num_objects; + int preferred_pack_idx; int local; const unsigned char *chunk_pack_names; size_t chunk_pack_names_len; + const uint32_t *chunk_bitmapped_packs; + size_t chunk_bitmapped_packs_len; const uint32_t *chunk_oid_fanout; const unsigned char *chunk_oid_lookup; const unsigned char *chunk_object_offsets; @@ -57,6 +61,8 @@ void get_midx_rev_filename(struct strbuf *out, struct multi_pack_index *m); struct multi_pack_index *load_multi_pack_index(const char *object_dir, int local); int prepare_midx_pack(struct repository *r, struct multi_pack_index *m, uint32_t pack_int_id); +int nth_bitmapped_pack(struct repository *r, struct multi_pack_index *m, + struct bitmapped_pack *bp, uint32_t pack_int_id); int bsearch_midx(const struct object_id *oid, struct multi_pack_index *m, uint32_t *result); off_t nth_midxed_offset(struct multi_pack_index *m, uint32_t pos); uint32_t nth_midxed_pack_int_id(struct multi_pack_index *m, uint32_t pos); @@ -64,7 +70,11 @@ struct object_id *nth_midxed_object_oid(struct object_id *oid, struct multi_pack_index *m, uint32_t n); int fill_midx_entry(struct repository *r, const struct object_id *oid, struct pack_entry *e, struct multi_pack_index *m); -int midx_contains_pack(struct multi_pack_index *m, const char *idx_or_pack_name); +int midx_contains_pack(struct multi_pack_index *m, + const char *idx_or_pack_name); +int midx_locate_pack(struct multi_pack_index *m, const char *idx_or_pack_name, + uint32_t *pos); +int midx_preferred_pack(struct multi_pack_index *m, uint32_t *pack_int_id); int prepare_multi_pack_index_one(struct repository *r, const char *object_dir, int local); /* diff --git a/object-file-convert.c b/object-file-convert.c new file mode 100644 index 0000000000..4f6189095b --- /dev/null +++ b/object-file-convert.c @@ -0,0 +1,277 @@ +#include "git-compat-util.h" +#include "gettext.h" +#include "strbuf.h" +#include "hex.h" +#include "repository.h" +#include "hash-ll.h" +#include "hash.h" +#include "object.h" +#include "loose.h" +#include "commit.h" +#include "gpg-interface.h" +#include "object-file-convert.h" + +int repo_oid_to_algop(struct repository *repo, const struct object_id *src, + const struct git_hash_algo *to, struct object_id *dest) +{ + /* + * If the source algorithm is not set, then we're using the + * default hash algorithm for that object. + */ + const struct git_hash_algo *from = + src->algo ? &hash_algos[src->algo] : repo->hash_algo; + + if (from == to) { + if (src != dest) + oidcpy(dest, src); + return 0; + } + if (repo_loose_object_map_oid(repo, src, to, dest)) { + /* + * We may have loaded the object map at repo initialization but + * another process (perhaps upstream of a pipe from us) may have + * written a new object into the map. If the object is missing, + * let's reload the map to see if the object has appeared. + */ + repo_read_loose_object_map(repo); + if (repo_loose_object_map_oid(repo, src, to, dest)) + return -1; + } + return 0; +} + +static int decode_tree_entry_raw(struct object_id *oid, const char **path, + size_t *len, const struct git_hash_algo *algo, + const char *buf, unsigned long size) +{ + uint16_t mode; + const unsigned hashsz = algo->rawsz; + + if (size < hashsz + 3 || buf[size - (hashsz + 1)]) { + return -1; + } + + *path = parse_mode(buf, &mode); + if (!*path || !**path) + return -1; + *len = strlen(*path) + 1; + + oidread_algop(oid, (const unsigned char *)*path + *len, algo); + return 0; +} + +static int convert_tree_object(struct strbuf *out, + const struct git_hash_algo *from, + const struct git_hash_algo *to, + const char *buffer, size_t size) +{ + const char *p = buffer, *end = buffer + size; + + while (p < end) { + struct object_id entry_oid, mapped_oid; + const char *path = NULL; + size_t pathlen; + + if (decode_tree_entry_raw(&entry_oid, &path, &pathlen, from, p, + end - p)) + return error(_("failed to decode tree entry")); + if (repo_oid_to_algop(the_repository, &entry_oid, to, &mapped_oid)) + return error(_("failed to map tree entry for %s"), oid_to_hex(&entry_oid)); + strbuf_add(out, p, path - p); + strbuf_add(out, path, pathlen); + strbuf_add(out, mapped_oid.hash, to->rawsz); + p = path + pathlen + from->rawsz; + } + return 0; +} + +static int convert_tag_object(struct strbuf *out, + const struct git_hash_algo *from, + const struct git_hash_algo *to, + const char *buffer, size_t size) +{ + struct strbuf payload = STRBUF_INIT, oursig = STRBUF_INIT, othersig = STRBUF_INIT; + const int entry_len = from->hexsz + 7; + size_t payload_size; + struct object_id oid, mapped_oid; + const char *p; + + /* Consume the object line */ + if ((entry_len >= size) || + memcmp(buffer, "object ", 7) || buffer[entry_len] != '\n') + return error("bogus tag object"); + if (parse_oid_hex_algop(buffer + 7, &oid, &p, from) < 0) + return error("bad tag object ID"); + if (repo_oid_to_algop(the_repository, &oid, to, &mapped_oid)) + return error("unable to map tree %s in tag object", + oid_to_hex(&oid)); + size -= ((p + 1) - buffer); + buffer = p + 1; + + /* Is there a signature for our algorithm? */ + payload_size = parse_signed_buffer(buffer, size); + if (payload_size != size) { + /* Yes, there is. */ + strbuf_add(&oursig, buffer + payload_size, size - payload_size); + } + + /* Now, is there a signature for the other algorithm? */ + parse_buffer_signed_by_header(buffer, payload_size, &payload, &othersig, to); + /* + * Our payload is now in payload and we may have up to two signatrures + * in oursig and othersig. + */ + + /* Add some slop for longer signature header in the new algorithm. */ + strbuf_grow(out, (7 + to->hexsz + 1) + size + 7); + strbuf_addf(out, "object %s\n", oid_to_hex(&mapped_oid)); + strbuf_addbuf(out, &payload); + if (oursig.len) + add_header_signature(out, &oursig, from); + strbuf_addbuf(out, &othersig); + + strbuf_release(&payload); + strbuf_release(&othersig); + strbuf_release(&oursig); + return 0; +} + +static int convert_commit_object(struct strbuf *out, + const struct git_hash_algo *from, + const struct git_hash_algo *to, + const char *buffer, size_t size) +{ + const char *tail = buffer; + const char *bufptr = buffer; + const int tree_entry_len = from->hexsz + 5; + const int parent_entry_len = from->hexsz + 7; + struct object_id oid, mapped_oid; + const char *p, *eol; + + tail += size; + + while ((bufptr < tail) && (*bufptr != '\n')) { + eol = memchr(bufptr, '\n', tail - bufptr); + if (!eol) + return error(_("bad %s in commit"), "line"); + + if (((bufptr + 5) < eol) && !memcmp(bufptr, "tree ", 5)) + { + if (((bufptr + tree_entry_len) != eol) || + parse_oid_hex_algop(bufptr + 5, &oid, &p, from) || + (p != eol)) + return error(_("bad %s in commit"), "tree"); + + if (repo_oid_to_algop(the_repository, &oid, to, &mapped_oid)) + return error(_("unable to map %s %s in commit object"), + "tree", oid_to_hex(&oid)); + strbuf_addf(out, "tree %s\n", oid_to_hex(&mapped_oid)); + } + else if (((bufptr + 7) < eol) && !memcmp(bufptr, "parent ", 7)) + { + if (((bufptr + parent_entry_len) != eol) || + parse_oid_hex_algop(bufptr + 7, &oid, &p, from) || + (p != eol)) + return error(_("bad %s in commit"), "parent"); + + if (repo_oid_to_algop(the_repository, &oid, to, &mapped_oid)) + return error(_("unable to map %s %s in commit object"), + "parent", oid_to_hex(&oid)); + + strbuf_addf(out, "parent %s\n", oid_to_hex(&mapped_oid)); + } + else if (((bufptr + 9) < eol) && !memcmp(bufptr, "mergetag ", 9)) + { + struct strbuf tag = STRBUF_INIT, new_tag = STRBUF_INIT; + + /* Recover the tag object from the mergetag */ + strbuf_add(&tag, bufptr + 9, (eol - (bufptr + 9)) + 1); + + bufptr = eol + 1; + while ((bufptr < tail) && (*bufptr == ' ')) { + eol = memchr(bufptr, '\n', tail - bufptr); + if (!eol) { + strbuf_release(&tag); + return error(_("bad %s in commit"), "mergetag continuation"); + } + strbuf_add(&tag, bufptr + 1, (eol - (bufptr + 1)) + 1); + bufptr = eol + 1; + } + + /* Compute the new tag object */ + if (convert_tag_object(&new_tag, from, to, tag.buf, tag.len)) { + strbuf_release(&tag); + strbuf_release(&new_tag); + return -1; + } + + /* Write the new mergetag */ + strbuf_addstr(out, "mergetag"); + strbuf_add_lines(out, " ", new_tag.buf, new_tag.len); + strbuf_release(&tag); + strbuf_release(&new_tag); + } + else if (((bufptr + 7) < tail) && !memcmp(bufptr, "author ", 7)) + strbuf_add(out, bufptr, (eol - bufptr) + 1); + else if (((bufptr + 10) < tail) && !memcmp(bufptr, "committer ", 10)) + strbuf_add(out, bufptr, (eol - bufptr) + 1); + else if (((bufptr + 9) < tail) && !memcmp(bufptr, "encoding ", 9)) + strbuf_add(out, bufptr, (eol - bufptr) + 1); + else if (((bufptr + 6) < tail) && !memcmp(bufptr, "gpgsig", 6)) + strbuf_add(out, bufptr, (eol - bufptr) + 1); + else { + /* Unknown line fail it might embed an oid */ + return -1; + } + /* Consume any trailing continuation lines */ + bufptr = eol + 1; + while ((bufptr < tail) && (*bufptr == ' ')) { + eol = memchr(bufptr, '\n', tail - bufptr); + if (!eol) + return error(_("bad %s in commit"), "continuation"); + strbuf_add(out, bufptr, (eol - bufptr) + 1); + bufptr = eol + 1; + } + } + if (bufptr < tail) + strbuf_add(out, bufptr, tail - bufptr); + return 0; +} + +int convert_object_file(struct strbuf *outbuf, + const struct git_hash_algo *from, + const struct git_hash_algo *to, + const void *buf, size_t len, + enum object_type type, + int gentle) +{ + int ret; + + /* Don't call this function when no conversion is necessary */ + if ((from == to) || (type == OBJ_BLOB)) + BUG("Refusing noop object file conversion"); + + switch (type) { + case OBJ_COMMIT: + ret = convert_commit_object(outbuf, from, to, buf, len); + break; + case OBJ_TREE: + ret = convert_tree_object(outbuf, from, to, buf, len); + break; + case OBJ_TAG: + ret = convert_tag_object(outbuf, from, to, buf, len); + break; + default: + /* Not implemented yet, so fail. */ + ret = -1; + break; + } + if (!ret) + return 0; + if (gentle) { + strbuf_release(outbuf); + return ret; + } + die(_("Failed to convert object from %s to %s"), + from->name, to->name); +} diff --git a/object-file-convert.h b/object-file-convert.h new file mode 100644 index 0000000000..a4f802aa8e --- /dev/null +++ b/object-file-convert.h @@ -0,0 +1,24 @@ +#ifndef OBJECT_CONVERT_H +#define OBJECT_CONVERT_H + +struct repository; +struct object_id; +struct git_hash_algo; +struct strbuf; +#include "object.h" + +int repo_oid_to_algop(struct repository *repo, const struct object_id *src, + const struct git_hash_algo *to, struct object_id *dest); + +/* + * Convert an object file from one hash algorithm to another algorithm. + * Return -1 on failure, 0 on success. + */ +int convert_object_file(struct strbuf *outbuf, + const struct git_hash_algo *from, + const struct git_hash_algo *to, + const void *buf, size_t len, + enum object_type type, + int gentle); + +#endif /* OBJECT_CONVERT_H */ diff --git a/object-file.c b/object-file.c index 619f039ebc..610b1f465c 100644 --- a/object-file.c +++ b/object-file.c @@ -35,6 +35,8 @@ #include "setup.h" #include "submodule.h" #include "fsck.h" +#include "loose.h" +#include "object-file-convert.h" /* The maximum size for an object header. */ #define MAX_HEADER_LEN 32 @@ -1084,9 +1086,11 @@ int check_object_signature(struct repository *r, const struct object_id *oid, void *buf, unsigned long size, enum object_type type) { + const struct git_hash_algo *algo = + oid->algo ? &hash_algos[oid->algo] : r->hash_algo; struct object_id real_oid; - hash_object_file(r->hash_algo, buf, size, type, &real_oid); + hash_object_file(algo, buf, size, type, &real_oid); return !oideq(oid, &real_oid) ? -1 : 0; } @@ -1652,10 +1656,101 @@ static int do_oid_object_info_extended(struct repository *r, return 0; } +static int oid_object_info_convert(struct repository *r, + const struct object_id *input_oid, + struct object_info *input_oi, unsigned flags) +{ + const struct git_hash_algo *input_algo = &hash_algos[input_oid->algo]; + int do_die = flags & OBJECT_INFO_DIE_IF_CORRUPT; + struct strbuf type_name = STRBUF_INIT; + struct object_id oid, delta_base_oid; + struct object_info new_oi, *oi; + unsigned long size; + void *content; + int ret; + + if (repo_oid_to_algop(r, input_oid, the_hash_algo, &oid)) { + if (do_die) + die(_("missing mapping of %s to %s"), + oid_to_hex(input_oid), the_hash_algo->name); + return -1; + } + + /* Is new_oi needed? */ + oi = input_oi; + if (input_oi && (input_oi->delta_base_oid || input_oi->sizep || + input_oi->contentp)) { + new_oi = *input_oi; + /* Does delta_base_oid need to be converted? */ + if (input_oi->delta_base_oid) + new_oi.delta_base_oid = &delta_base_oid; + /* Will the attributes differ when converted? */ + if (input_oi->sizep || input_oi->contentp) { + new_oi.contentp = &content; + new_oi.sizep = &size; + new_oi.type_name = &type_name; + } + oi = &new_oi; + } + + ret = oid_object_info_extended(r, &oid, oi, flags); + if (ret) + return -1; + if (oi == input_oi) + return ret; + + if (new_oi.contentp) { + struct strbuf outbuf = STRBUF_INIT; + enum object_type type; + + type = type_from_string_gently(type_name.buf, type_name.len, + !do_die); + if (type == -1) + return -1; + if (type != OBJ_BLOB) { + ret = convert_object_file(&outbuf, + the_hash_algo, input_algo, + content, size, type, !do_die); + if (ret == -1) + return -1; + free(content); + size = outbuf.len; + content = strbuf_detach(&outbuf, NULL); + } + if (input_oi->sizep) + *input_oi->sizep = size; + if (input_oi->contentp) + *input_oi->contentp = content; + else + free(content); + if (input_oi->type_name) + *input_oi->type_name = type_name; + else + strbuf_release(&type_name); + } + if (new_oi.delta_base_oid == &delta_base_oid) { + if (repo_oid_to_algop(r, &delta_base_oid, input_algo, + input_oi->delta_base_oid)) { + if (do_die) + die(_("missing mapping of %s to %s"), + oid_to_hex(&delta_base_oid), + input_algo->name); + return -1; + } + } + input_oi->whence = new_oi.whence; + input_oi->u = new_oi.u; + return ret; +} + int oid_object_info_extended(struct repository *r, const struct object_id *oid, struct object_info *oi, unsigned flags) { int ret; + + if (oid->algo && (hash_algo_by_ptr(r->hash_algo) != oid->algo)) + return oid_object_info_convert(r, oid, oi, flags); + obj_read_lock(); ret = do_oid_object_info_extended(r, oid, oi, flags); obj_read_unlock(); @@ -1944,9 +2039,12 @@ static int start_loose_object_common(struct strbuf *tmp_file, const char *filename, unsigned flags, git_zstream *stream, unsigned char *buf, size_t buflen, - git_hash_ctx *c, + git_hash_ctx *c, git_hash_ctx *compat_c, char *hdr, int hdrlen) { + struct repository *repo = the_repository; + const struct git_hash_algo *algo = repo->hash_algo; + const struct git_hash_algo *compat = repo->compat_hash_algo; int fd; fd = create_tmpfile(tmp_file, filename); @@ -1966,14 +2064,18 @@ static int start_loose_object_common(struct strbuf *tmp_file, git_deflate_init(stream, zlib_compression_level); stream->next_out = buf; stream->avail_out = buflen; - the_hash_algo->init_fn(c); + algo->init_fn(c); + if (compat && compat_c) + compat->init_fn(compat_c); /* Start to feed header to zlib stream */ stream->next_in = (unsigned char *)hdr; stream->avail_in = hdrlen; while (git_deflate(stream, 0) == Z_OK) ; /* nothing */ - the_hash_algo->update_fn(c, hdr, hdrlen); + algo->update_fn(c, hdr, hdrlen); + if (compat && compat_c) + compat->update_fn(compat_c, hdr, hdrlen); return fd; } @@ -1982,16 +2084,21 @@ static int start_loose_object_common(struct strbuf *tmp_file, * Common steps for the inner git_deflate() loop for writing loose * objects. Returns what git_deflate() returns. */ -static int write_loose_object_common(git_hash_ctx *c, +static int write_loose_object_common(git_hash_ctx *c, git_hash_ctx *compat_c, git_zstream *stream, const int flush, unsigned char *in0, const int fd, unsigned char *compressed, const size_t compressed_len) { + struct repository *repo = the_repository; + const struct git_hash_algo *algo = repo->hash_algo; + const struct git_hash_algo *compat = repo->compat_hash_algo; int ret; ret = git_deflate(stream, flush ? Z_FINISH : 0); - the_hash_algo->update_fn(c, in0, stream->next_in - in0); + algo->update_fn(c, in0, stream->next_in - in0); + if (compat && compat_c) + compat->update_fn(compat_c, in0, stream->next_in - in0); if (write_in_full(fd, compressed, stream->next_out - compressed) < 0) die_errno(_("unable to write loose object file")); stream->next_out = compressed; @@ -2006,15 +2113,21 @@ static int write_loose_object_common(git_hash_ctx *c, * - End the compression of zlib stream. * - Get the calculated oid to "oid". */ -static int end_loose_object_common(git_hash_ctx *c, git_zstream *stream, - struct object_id *oid) +static int end_loose_object_common(git_hash_ctx *c, git_hash_ctx *compat_c, + git_zstream *stream, struct object_id *oid, + struct object_id *compat_oid) { + struct repository *repo = the_repository; + const struct git_hash_algo *algo = repo->hash_algo; + const struct git_hash_algo *compat = repo->compat_hash_algo; int ret; ret = git_deflate_end_gently(stream); if (ret != Z_OK) return ret; - the_hash_algo->final_oid_fn(oid, c); + algo->final_oid_fn(oid, c); + if (compat && compat_c) + compat->final_oid_fn(compat_oid, compat_c); return Z_OK; } @@ -2038,7 +2151,7 @@ static int write_loose_object(const struct object_id *oid, char *hdr, fd = start_loose_object_common(&tmp_file, filename.buf, flags, &stream, compressed, sizeof(compressed), - &c, hdr, hdrlen); + &c, NULL, hdr, hdrlen); if (fd < 0) return -1; @@ -2048,14 +2161,14 @@ static int write_loose_object(const struct object_id *oid, char *hdr, do { unsigned char *in0 = stream.next_in; - ret = write_loose_object_common(&c, &stream, 1, in0, fd, + ret = write_loose_object_common(&c, NULL, &stream, 1, in0, fd, compressed, sizeof(compressed)); } while (ret == Z_OK); if (ret != Z_STREAM_END) die(_("unable to deflate new object %s (%d)"), oid_to_hex(oid), ret); - ret = end_loose_object_common(&c, &stream, ¶no_oid); + ret = end_loose_object_common(&c, NULL, &stream, ¶no_oid, NULL); if (ret != Z_OK) die(_("deflateEnd on object %s failed (%d)"), oid_to_hex(oid), ret); @@ -2100,10 +2213,12 @@ static int freshen_packed_object(const struct object_id *oid) int stream_loose_object(struct input_stream *in_stream, size_t len, struct object_id *oid) { + const struct git_hash_algo *compat = the_repository->compat_hash_algo; + struct object_id compat_oid; int fd, ret, err = 0, flush = 0; unsigned char compressed[4096]; git_zstream stream; - git_hash_ctx c; + git_hash_ctx c, compat_c; struct strbuf tmp_file = STRBUF_INIT; struct strbuf filename = STRBUF_INIT; int dirlen; @@ -2127,7 +2242,7 @@ int stream_loose_object(struct input_stream *in_stream, size_t len, */ fd = start_loose_object_common(&tmp_file, filename.buf, 0, &stream, compressed, sizeof(compressed), - &c, hdr, hdrlen); + &c, &compat_c, hdr, hdrlen); if (fd < 0) { err = -1; goto cleanup; @@ -2145,7 +2260,7 @@ int stream_loose_object(struct input_stream *in_stream, size_t len, if (in_stream->is_finished) flush = 1; } - ret = write_loose_object_common(&c, &stream, flush, in0, fd, + ret = write_loose_object_common(&c, &compat_c, &stream, flush, in0, fd, compressed, sizeof(compressed)); /* * Unlike write_loose_object(), we do not have the entire @@ -2168,7 +2283,7 @@ int stream_loose_object(struct input_stream *in_stream, size_t len, */ if (ret != Z_STREAM_END) die(_("unable to stream deflate new object (%d)"), ret); - ret = end_loose_object_common(&c, &stream, oid); + ret = end_loose_object_common(&c, &compat_c, &stream, oid, &compat_oid); if (ret != Z_OK) die(_("deflateEnd on stream object failed (%d)"), ret); close_loose_object(fd, tmp_file.buf); @@ -2195,6 +2310,8 @@ int stream_loose_object(struct input_stream *in_stream, size_t len, } err = finalize_object_file(tmp_file.buf, filename.buf); + if (!err && compat) + err = repo_add_loose_object_map(the_repository, oid, &compat_oid); cleanup: strbuf_release(&tmp_file); strbuf_release(&filename); @@ -2203,19 +2320,42 @@ cleanup: int write_object_file_flags(const void *buf, unsigned long len, enum object_type type, struct object_id *oid, - unsigned flags) + struct object_id *compat_oid_in, unsigned flags) { + struct repository *repo = the_repository; + const struct git_hash_algo *algo = repo->hash_algo; + const struct git_hash_algo *compat = repo->compat_hash_algo; + struct object_id compat_oid; char hdr[MAX_HEADER_LEN]; int hdrlen = sizeof(hdr); + /* Generate compat_oid */ + if (compat) { + if (compat_oid_in) + oidcpy(&compat_oid, compat_oid_in); + else if (type == OBJ_BLOB) + hash_object_file(compat, buf, len, type, &compat_oid); + else { + struct strbuf converted = STRBUF_INIT; + convert_object_file(&converted, algo, compat, + buf, len, type, 0); + hash_object_file(compat, converted.buf, converted.len, + type, &compat_oid); + strbuf_release(&converted); + } + } + /* Normally if we have it in the pack then we do not bother writing * it out into .git/objects/??/?{38} file. */ - write_object_file_prepare(the_hash_algo, buf, len, type, oid, hdr, - &hdrlen); + write_object_file_prepare(algo, buf, len, type, oid, hdr, &hdrlen); if (freshen_packed_object(oid) || freshen_loose_object(oid)) return 0; - return write_loose_object(oid, hdr, hdrlen, buf, len, 0, flags); + if (write_loose_object(oid, hdr, hdrlen, buf, len, 0, flags)) + return -1; + if (compat) + return repo_add_loose_object_map(repo, oid, &compat_oid); + return 0; } int write_object_file_literally(const void *buf, unsigned long len, @@ -2223,7 +2363,27 @@ int write_object_file_literally(const void *buf, unsigned long len, unsigned flags) { char *header; + struct repository *repo = the_repository; + const struct git_hash_algo *algo = repo->hash_algo; + const struct git_hash_algo *compat = repo->compat_hash_algo; + struct object_id compat_oid; int hdrlen, status = 0; + int compat_type = -1; + + if (compat) { + compat_type = type_from_string_gently(type, -1, 1); + if (compat_type == OBJ_BLOB) + hash_object_file(compat, buf, len, compat_type, + &compat_oid); + else if (compat_type != -1) { + struct strbuf converted = STRBUF_INIT; + convert_object_file(&converted, algo, compat, + buf, len, compat_type, 0); + hash_object_file(compat, converted.buf, converted.len, + compat_type, &compat_oid); + strbuf_release(&converted); + } + } /* type string, SP, %lu of the length plus NUL must fit this */ hdrlen = strlen(type) + MAX_HEADER_LEN; @@ -2236,6 +2396,8 @@ int write_object_file_literally(const void *buf, unsigned long len, if (freshen_packed_object(oid) || freshen_loose_object(oid)) goto cleanup; status = write_loose_object(oid, header, hdrlen, buf, len, 0, 0); + if (compat_type != -1) + return repo_add_loose_object_map(repo, oid, &compat_oid); cleanup: free(header); @@ -2244,9 +2406,12 @@ cleanup: int force_object_loose(const struct object_id *oid, time_t mtime) { + struct repository *repo = the_repository; + const struct git_hash_algo *compat = repo->compat_hash_algo; void *buf; unsigned long len; struct object_info oi = OBJECT_INFO_INIT; + struct object_id compat_oid; enum object_type type; char hdr[MAX_HEADER_LEN]; int hdrlen; @@ -2259,8 +2424,15 @@ int force_object_loose(const struct object_id *oid, time_t mtime) oi.contentp = &buf; if (oid_object_info_extended(the_repository, oid, &oi, 0)) return error(_("cannot read object for %s"), oid_to_hex(oid)); + if (compat) { + if (repo_oid_to_algop(repo, oid, compat, &compat_oid)) + return error(_("cannot map object %s to %s"), + oid_to_hex(oid), compat->name); + } hdrlen = format_object_header(hdr, sizeof(hdr), type, len); ret = write_loose_object(oid, hdr, hdrlen, buf, len, mtime, 0); + if (!ret && compat) + ret = repo_add_loose_object_map(the_repository, oid, &compat_oid); free(buf); return ret; diff --git a/object-name.c b/object-name.c index 3a2ef5d680..89df4a85b8 100644 --- a/object-name.c +++ b/object-name.c @@ -23,6 +23,7 @@ #include "midx.h" #include "commit-reach.h" #include "date.h" +#include "object-file-convert.h" static int get_oid_oneline(struct repository *r, const char *, struct object_id *, struct commit_list *); @@ -47,6 +48,7 @@ struct disambiguate_state { static void update_candidates(struct disambiguate_state *ds, const struct object_id *current) { + /* The hash algorithm of current has already been filtered */ if (ds->always_call_fn) { ds->ambiguous = ds->fn(ds->repo, current, ds->cb_data) ? 1 : 0; return; @@ -132,6 +134,8 @@ static void unique_in_midx(struct multi_pack_index *m, { uint32_t num, i, first = 0; const struct object_id *current = NULL; + int len = ds->len > ds->repo->hash_algo->hexsz ? + ds->repo->hash_algo->hexsz : ds->len; num = m->num_objects; if (!num) @@ -147,7 +151,7 @@ static void unique_in_midx(struct multi_pack_index *m, for (i = first; i < num && !ds->ambiguous; i++) { struct object_id oid; current = nth_midxed_object_oid(&oid, m, i); - if (!match_hash(ds->len, ds->bin_pfx.hash, current->hash)) + if (!match_hash(len, ds->bin_pfx.hash, current->hash)) break; update_candidates(ds, current); } @@ -157,6 +161,8 @@ static void unique_in_pack(struct packed_git *p, struct disambiguate_state *ds) { uint32_t num, i, first = 0; + int len = ds->len > ds->repo->hash_algo->hexsz ? + ds->repo->hash_algo->hexsz : ds->len; if (p->multi_pack_index) return; @@ -175,7 +181,7 @@ static void unique_in_pack(struct packed_git *p, for (i = first; i < num && !ds->ambiguous; i++) { struct object_id oid; nth_packed_object_id(&oid, p, i); - if (!match_hash(ds->len, ds->bin_pfx.hash, oid.hash)) + if (!match_hash(len, ds->bin_pfx.hash, oid.hash)) break; update_candidates(ds, &oid); } @@ -186,6 +192,10 @@ static void find_short_packed_object(struct disambiguate_state *ds) struct multi_pack_index *m; struct packed_git *p; + /* Skip, unless oids from the storage hash algorithm are wanted */ + if (ds->bin_pfx.algo && (&hash_algos[ds->bin_pfx.algo] != ds->repo->hash_algo)) + return; + for (m = get_multi_pack_index(ds->repo); m && !ds->ambiguous; m = m->next) unique_in_midx(m, ds); @@ -324,11 +334,12 @@ int set_disambiguate_hint_config(const char *var, const char *value) static int init_object_disambiguation(struct repository *r, const char *name, int len, + const struct git_hash_algo *algo, struct disambiguate_state *ds) { int i; - if (len < MINIMUM_ABBREV || len > the_hash_algo->hexsz) + if (len < MINIMUM_ABBREV || len > GIT_MAX_HEXSZ) return -1; memset(ds, 0, sizeof(*ds)); @@ -355,6 +366,7 @@ static int init_object_disambiguation(struct repository *r, ds->len = len; ds->hex_pfx[len] = '\0'; ds->repo = r; + ds->bin_pfx.algo = algo ? hash_algo_by_ptr(algo) : GIT_HASH_UNKNOWN; prepare_alt_odb(r); return 0; } @@ -489,9 +501,10 @@ static int repo_collect_ambiguous(struct repository *r UNUSED, return collect_ambiguous(oid, data); } -static int sort_ambiguous(const void *a, const void *b, void *ctx) +static int sort_ambiguous(const void *va, const void *vb, void *ctx) { struct repository *sort_ambiguous_repo = ctx; + const struct object_id *a = va, *b = vb; int a_type = oid_object_info(sort_ambiguous_repo, a, NULL); int b_type = oid_object_info(sort_ambiguous_repo, b, NULL); int a_type_sort; @@ -501,8 +514,12 @@ static int sort_ambiguous(const void *a, const void *b, void *ctx) * Sorts by hash within the same object type, just as * oid_array_for_each_unique() would do. */ - if (a_type == b_type) - return oidcmp(a, b); + if (a_type == b_type) { + if (a->algo == b->algo) + return oidcmp(a, b); + else + return a->algo > b->algo ? 1 : -1; + } /* * Between object types show tags, then commits, and finally @@ -531,8 +548,12 @@ static enum get_oid_result get_short_oid(struct repository *r, int status; struct disambiguate_state ds; int quietly = !!(flags & GET_OID_QUIETLY); + const struct git_hash_algo *algo = r->hash_algo; + + if (flags & GET_OID_HASH_ANY) + algo = NULL; - if (init_object_disambiguation(r, name, len, &ds) < 0) + if (init_object_disambiguation(r, name, len, algo, &ds) < 0) return -1; if (HAS_MULTI_BITS(flags & GET_OID_DISAMBIGUATORS)) @@ -586,7 +607,7 @@ static enum get_oid_result get_short_oid(struct repository *r, if (!ds.ambiguous) ds.fn = NULL; - repo_for_each_abbrev(r, ds.hex_pfx, collect_ambiguous, &collect); + repo_for_each_abbrev(r, ds.hex_pfx, algo, collect_ambiguous, &collect); sort_ambiguous_oid_array(r, &collect); if (oid_array_for_each(&collect, show_ambiguous_object, &out)) @@ -608,13 +629,14 @@ static enum get_oid_result get_short_oid(struct repository *r, } int repo_for_each_abbrev(struct repository *r, const char *prefix, + const struct git_hash_algo *algo, each_abbrev_fn fn, void *cb_data) { struct oid_array collect = OID_ARRAY_INIT; struct disambiguate_state ds; int ret; - if (init_object_disambiguation(r, prefix, strlen(prefix), &ds) < 0) + if (init_object_disambiguation(r, prefix, strlen(prefix), algo, &ds) < 0) return -1; ds.always_call_fn = 1; @@ -785,10 +807,12 @@ void strbuf_add_unique_abbrev(struct strbuf *sb, const struct object_id *oid, int repo_find_unique_abbrev_r(struct repository *r, char *hex, const struct object_id *oid, int len) { + const struct git_hash_algo *algo = + oid->algo ? &hash_algos[oid->algo] : r->hash_algo; struct disambiguate_state ds; struct min_abbrev_data mad; struct object_id oid_ret; - const unsigned hexsz = r->hash_algo->hexsz; + const unsigned hexsz = algo->hexsz; if (len < 0) { unsigned long count = repo_approximate_object_count(r); @@ -824,7 +848,7 @@ int repo_find_unique_abbrev_r(struct repository *r, char *hex, find_abbrev_len_packed(&mad); - if (init_object_disambiguation(r, hex, mad.cur_len, &ds) < 0) + if (init_object_disambiguation(r, hex, mad.cur_len, algo, &ds) < 0) return -1; ds.fn = repo_extend_abbrev_len; diff --git a/object-name.h b/object-name.h index 9ae5223071..064ddc97d1 100644 --- a/object-name.h +++ b/object-name.h @@ -67,7 +67,8 @@ enum get_oid_result get_oid_with_context(struct repository *repo, const char *st typedef int each_abbrev_fn(const struct object_id *oid, void *); -int repo_for_each_abbrev(struct repository *r, const char *prefix, each_abbrev_fn, void *); +int repo_for_each_abbrev(struct repository *r, const char *prefix, + const struct git_hash_algo *algo, each_abbrev_fn, void *); int set_disambiguate_hint_config(const char *var, const char *value); diff --git a/object-store-ll.h b/object-store-ll.h index 26a3895c82..c5f2bb2fc2 100644 --- a/object-store-ll.h +++ b/object-store-ll.h @@ -26,6 +26,9 @@ struct object_directory { uint32_t loose_objects_subdir_seen[8]; /* 256 bits */ struct oidtree *loose_objects_cache; + /* Map between object IDs for loose objects. */ + struct loose_object_map *loose_map; + /* * This is a temporary object store created by the tmp_objdir * facility. Disable ref updates since the objects in the store @@ -252,11 +255,11 @@ void hash_object_file(const struct git_hash_algo *algo, const void *buf, int write_object_file_flags(const void *buf, unsigned long len, enum object_type type, struct object_id *oid, - unsigned flags); + struct object_id *comapt_oid_in, unsigned flags); static inline int write_object_file(const void *buf, unsigned long len, enum object_type type, struct object_id *oid) { - return write_object_file_flags(buf, len, type, oid, 0); + return write_object_file_flags(buf, len, type, oid, NULL, 0); } int write_object_file_literally(const void *buf, unsigned long len, @@ -13,6 +13,7 @@ #include "alloc.h" #include "packfile.h" #include "commit-graph.h" +#include "loose.h" unsigned int get_max_object_index(void) { @@ -540,6 +541,7 @@ void free_object_directory(struct object_directory *odb) { free(odb->path); odb_clear_loose_cache(odb); + loose_object_map_clear(&odb->loose_map); free(odb); } @@ -62,7 +62,7 @@ void object_array_init(struct object_array *array); /* * object flag allocation: - * revision.h: 0---------10 15 23------27 + * revision.h: 0---------10 15 23------27 * fetch-pack.c: 01 67 * negotiator/default.c: 2--5 * walker.c: 0-2 @@ -75,6 +75,7 @@ void object_array_init(struct object_array *array); * commit-reach.c: 16-----19 * sha1-name.c: 20 * list-objects-filter.c: 21 + * bloom.c: 2122 * builtin/fsck.c: 0--3 * builtin/gc.c: 0 * builtin/index-pack.c: 2021 @@ -190,6 +191,24 @@ void *create_object(struct repository *r, const struct object_id *oid, void *obj void *object_as_type(struct object *obj, enum object_type type, int quiet); + +static inline const char *parse_mode(const char *str, uint16_t *modep) +{ + unsigned char c; + unsigned int mode = 0; + + if (*str == ' ') + return NULL; + + while ((c = *str++) != ' ') { + if (c < '0' || c > '7') + return NULL; + mode = (mode << 3) + (c - '0'); + } + *modep = mode; + return str; +} + /* * Returns the object, having parsed it to find out what it is. * diff --git a/oid-array.c b/oid-array.c index 8e4717746c..1f36651754 100644 --- a/oid-array.c +++ b/oid-array.c @@ -6,12 +6,20 @@ void oid_array_append(struct oid_array *array, const struct object_id *oid) { ALLOC_GROW(array->oid, array->nr + 1, array->alloc); oidcpy(&array->oid[array->nr++], oid); + if (!oid->algo) + oid_set_algo(&array->oid[array->nr - 1], the_hash_algo); array->sorted = 0; } -static int void_hashcmp(const void *a, const void *b) +static int void_hashcmp(const void *va, const void *vb) { - return oidcmp(a, b); + const struct object_id *a = va, *b = vb; + int ret; + if (a->algo == b->algo) + ret = oidcmp(a, b); + else + ret = a->algo > b->algo ? 1 : -1; + return ret; } void oid_array_sort(struct oid_array *array) diff --git a/oss-fuzz/dummy-cmd-main.c b/oss-fuzz/dummy-cmd-main.c new file mode 100644 index 0000000000..071cb231ba --- /dev/null +++ b/oss-fuzz/dummy-cmd-main.c @@ -0,0 +1,14 @@ +#include "git-compat-util.h" + +/* + * When linking the fuzzers, we link against common-main.o to pick up some + * symbols. However, even though we ignore common-main:main(), we still need to + * provide all the symbols it references. In the fuzzers' case, we need to + * provide a dummy cmd_main() for the linker to be happy. It will never be + * executed. + */ + +int cmd_main(int argc, const char **argv) { + BUG("We should not execute cmd_main() from a fuzz target"); + return 1; +} diff --git a/oss-fuzz/fuzz-commit-graph.c b/oss-fuzz/fuzz-commit-graph.c index 2992079dd9..325c0b991a 100644 --- a/oss-fuzz/fuzz-commit-graph.c +++ b/oss-fuzz/fuzz-commit-graph.c @@ -19,7 +19,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) * possible. */ the_repository->settings.commit_graph_generation_version = 2; - the_repository->settings.commit_graph_read_changed_paths = 1; + the_repository->settings.commit_graph_changed_paths_version = 1; g = parse_commit_graph(&the_repository->settings, (void *)data, size); repo_clear(the_repository); free_commit_graph(g); diff --git a/pack-bitmap-write.c b/pack-bitmap-write.c index be4733e3bd..c6c8f94cc5 100644 --- a/pack-bitmap-write.c +++ b/pack-bitmap-write.c @@ -195,6 +195,13 @@ struct bb_commit { unsigned idx; /* within selected array */ }; +static void clear_bb_commit(struct bb_commit *commit) +{ + free_commit_list(commit->reverse_edges); + bitmap_free(commit->commit_mask); + bitmap_free(commit->bitmap); +} + define_commit_slab(bb_data, struct bb_commit); struct bitmap_builder { @@ -336,7 +343,7 @@ next: static void bitmap_builder_clear(struct bitmap_builder *bb) { - clear_bb_data(&bb->data); + deep_clear_bb_data(&bb->data, clear_bb_commit); free(bb->commits); bb->commits_nr = bb->commits_alloc = 0; } @@ -363,7 +370,7 @@ static int fill_bitmap_tree(struct bitmap *bitmap, if (parse_tree(tree) < 0) die("unable to load tree object %s", oid_to_hex(&tree->object.oid)); - init_tree_desc(&desc, tree->buffer, tree->size); + init_tree_desc(&desc, &tree->object.oid, tree->buffer, tree->size); while (tree_entry(&desc, &entry)) { switch (object_type(entry.mode)) { diff --git a/pack-bitmap.c b/pack-bitmap.c index 0260890341..229a11fb00 100644 --- a/pack-bitmap.c +++ b/pack-bitmap.c @@ -338,7 +338,7 @@ static int open_midx_bitmap_1(struct bitmap_index *bitmap_git, struct stat st; char *bitmap_name = midx_bitmap_filename(midx); int fd = git_open(bitmap_name); - uint32_t i; + uint32_t i, preferred_pack; struct packed_git *preferred; if (fd < 0) { @@ -393,7 +393,12 @@ static int open_midx_bitmap_1(struct bitmap_index *bitmap_git, } } - preferred = bitmap_git->midx->packs[midx_preferred_pack(bitmap_git)]; + if (midx_preferred_pack(bitmap_git->midx, &preferred_pack) < 0) { + warning(_("could not determine MIDX preferred pack")); + goto cleanup; + } + + preferred = bitmap_git->midx->packs[preferred_pack]; if (!is_pack_valid(preferred)) { warning(_("preferred pack (%s) is invalid"), preferred->pack_name); @@ -1280,6 +1285,8 @@ static struct bitmap *find_objects(struct bitmap_index *bitmap_git, base = fill_in_bitmap(bitmap_git, revs, base, seen); } + object_list_free(¬_mapped); + return base; } @@ -1834,8 +1841,10 @@ cleanup: * -1 means "stop trying further objects"; 0 means we may or may not have * reused, but you can keep feeding bits. */ -static int try_partial_reuse(struct packed_git *pack, - size_t pos, +static int try_partial_reuse(struct bitmap_index *bitmap_git, + struct bitmapped_pack *pack, + size_t bitmap_pos, + uint32_t pack_pos, struct bitmap *reuse, struct pack_window **w_curs) { @@ -1843,40 +1852,18 @@ static int try_partial_reuse(struct packed_git *pack, enum object_type type; unsigned long size; - /* - * try_partial_reuse() is called either on (a) objects in the - * bitmapped pack (in the case of a single-pack bitmap) or (b) - * objects in the preferred pack of a multi-pack bitmap. - * Importantly, the latter can pretend as if only a single pack - * exists because: - * - * - The first pack->num_objects bits of a MIDX bitmap are - * reserved for the preferred pack, and - * - * - Ties due to duplicate objects are always resolved in - * favor of the preferred pack. - * - * Therefore we do not need to ever ask the MIDX for its copy of - * an object by OID, since it will always select it from the - * preferred pack. Likewise, the selected copy of the base - * object for any deltas will reside in the same pack. - * - * This means that we can reuse pos when looking up the bit in - * the reuse bitmap, too, since bits corresponding to the - * preferred pack precede all bits from other packs. - */ - - if (pos >= pack->num_objects) - return -1; /* not actually in the pack or MIDX preferred pack */ + if (pack_pos >= pack->p->num_objects) + return -1; /* not actually in the pack */ - offset = delta_obj_offset = pack_pos_to_offset(pack, pos); - type = unpack_object_header(pack, w_curs, &offset, &size); + offset = delta_obj_offset = pack_pos_to_offset(pack->p, pack_pos); + type = unpack_object_header(pack->p, w_curs, &offset, &size); if (type < 0) return -1; /* broken packfile, punt */ if (type == OBJ_REF_DELTA || type == OBJ_OFS_DELTA) { off_t base_offset; uint32_t base_pos; + uint32_t base_bitmap_pos; /* * Find the position of the base object so we can look it up @@ -1886,24 +1873,48 @@ static int try_partial_reuse(struct packed_git *pack, * and the normal slow path will complain about it in * more detail. */ - base_offset = get_delta_base(pack, w_curs, &offset, type, + base_offset = get_delta_base(pack->p, w_curs, &offset, type, delta_obj_offset); if (!base_offset) return 0; - if (offset_to_pack_pos(pack, base_offset, &base_pos) < 0) - return 0; - /* - * We assume delta dependencies always point backwards. This - * lets us do a single pass, and is basically always true - * due to the way OFS_DELTAs work. You would not typically - * find REF_DELTA in a bitmapped pack, since we only bitmap - * packs we write fresh, and OFS_DELTA is the default). But - * let's double check to make sure the pack wasn't written with - * odd parameters. - */ - if (base_pos >= pos) - return 0; + offset_to_pack_pos(pack->p, base_offset, &base_pos); + + if (bitmap_is_midx(bitmap_git)) { + /* + * Cross-pack deltas are rejected for now, but could + * theoretically be supported in the future. + * + * We would need to ensure that we're sending both + * halves of the delta/base pair, regardless of whether + * or not the two cross a pack boundary. If they do, + * then we must convert the delta to an REF_DELTA to + * refer back to the base in the other pack. + * */ + if (midx_pair_to_pack_pos(bitmap_git->midx, + pack->pack_int_id, + base_offset, + &base_bitmap_pos) < 0) { + return 0; + } + } else { + if (offset_to_pack_pos(pack->p, base_offset, + &base_pos) < 0) + return 0; + /* + * We assume delta dependencies always point backwards. + * This lets us do a single pass, and is basically + * always true due to the way OFS_DELTAs work. You would + * not typically find REF_DELTA in a bitmapped pack, + * since we only bitmap packs we write fresh, and + * OFS_DELTA is the default). But let's double check to + * make sure the pack wasn't written with odd + * parameters. + */ + if (base_pos >= pack_pos) + return 0; + base_bitmap_pos = pack->bitmap_pos + base_pos; + } /* * And finally, if we're not sending the base as part of our @@ -1913,77 +1924,89 @@ static int try_partial_reuse(struct packed_git *pack, * to REF_DELTA on the fly. Better to just let the normal * object_entry code path handle it. */ - if (!bitmap_get(reuse, base_pos)) + if (!bitmap_get(reuse, base_bitmap_pos)) return 0; } /* * If we got here, then the object is OK to reuse. Mark it. */ - bitmap_set(reuse, pos); + bitmap_set(reuse, bitmap_pos); return 0; } -uint32_t midx_preferred_pack(struct bitmap_index *bitmap_git) +static void reuse_partial_packfile_from_bitmap_1(struct bitmap_index *bitmap_git, + struct bitmapped_pack *pack, + struct bitmap *reuse) { - struct multi_pack_index *m = bitmap_git->midx; - if (!m) - BUG("midx_preferred_pack: requires non-empty MIDX"); - return nth_midxed_pack_int_id(m, pack_pos_to_midx(bitmap_git->midx, 0)); -} - -int reuse_partial_packfile_from_bitmap(struct bitmap_index *bitmap_git, - struct packed_git **packfile_out, - uint32_t *entries, - struct bitmap **reuse_out) -{ - struct repository *r = the_repository; - struct packed_git *pack; struct bitmap *result = bitmap_git->result; - struct bitmap *reuse; struct pack_window *w_curs = NULL; - size_t i = 0; - uint32_t offset; - uint32_t objects_nr; + size_t pos = pack->bitmap_pos / BITS_IN_EWORD; - assert(result); + if (!pack->bitmap_pos) { + /* + * If we're processing the first (in the case of a MIDX, the + * preferred pack) or the only (in the case of single-pack + * bitmaps) pack, then we can reuse whole words at a time. + * + * This is because we know that any deltas in this range *must* + * have their bases chosen from the same pack, since: + * + * - In the single pack case, there is no other pack to choose + * them from. + * + * - In the MIDX case, the first pack is the preferred pack, so + * all ties are broken in favor of that pack (i.e. the one + * we're currently processing). So any duplicate bases will be + * resolved in favor of the pack we're processing. + */ + while (pos < result->word_alloc && + pos < pack->bitmap_nr / BITS_IN_EWORD && + result->words[pos] == (eword_t)~0) + pos++; + memset(reuse->words, 0xFF, pos * sizeof(eword_t)); + } - load_reverse_index(r, bitmap_git); + for (; pos < result->word_alloc; pos++) { + eword_t word = result->words[pos]; + size_t offset; - if (bitmap_is_midx(bitmap_git)) - pack = bitmap_git->midx->packs[midx_preferred_pack(bitmap_git)]; - else - pack = bitmap_git->pack; - objects_nr = pack->num_objects; + for (offset = 0; offset < BITS_IN_EWORD; offset++) { + size_t bit_pos; + uint32_t pack_pos; - while (i < result->word_alloc && result->words[i] == (eword_t)~0) - i++; + if (word >> offset == 0) + break; - /* - * Don't mark objects not in the packfile or preferred pack. This bitmap - * marks objects eligible for reuse, but the pack-reuse code only - * understands how to reuse a single pack. Since the preferred pack is - * guaranteed to have all bases for its deltas (in a multi-pack bitmap), - * we use it instead of another pack. In single-pack bitmaps, the choice - * is made for us. - */ - if (i > objects_nr / BITS_IN_EWORD) - i = objects_nr / BITS_IN_EWORD; + offset += ewah_bit_ctz64(word >> offset); - reuse = bitmap_word_alloc(i); - memset(reuse->words, 0xFF, i * sizeof(eword_t)); + bit_pos = pos * BITS_IN_EWORD + offset; + if (bit_pos < pack->bitmap_pos) + continue; + if (bit_pos >= pack->bitmap_pos + pack->bitmap_nr) + goto done; - for (; i < result->word_alloc; ++i) { - eword_t word = result->words[i]; - size_t pos = (i * BITS_IN_EWORD); + if (bitmap_is_midx(bitmap_git)) { + uint32_t midx_pos; + off_t ofs; - for (offset = 0; offset < BITS_IN_EWORD; ++offset) { - if ((word >> offset) == 0) - break; + midx_pos = pack_pos_to_midx(bitmap_git->midx, bit_pos); + ofs = nth_midxed_offset(bitmap_git->midx, midx_pos); - offset += ewah_bit_ctz64(word >> offset); - if (try_partial_reuse(pack, pos + offset, - reuse, &w_curs) < 0) { + if (offset_to_pack_pos(pack->p, ofs, &pack_pos) < 0) + BUG("could not find object in pack %s " + "at offset %"PRIuMAX" in MIDX", + pack_basename(pack->p), (uintmax_t)ofs); + } else { + pack_pos = cast_size_t_to_uint32_t(st_sub(bit_pos, pack->bitmap_pos)); + if (pack_pos >= pack->p->num_objects) + BUG("advanced beyond the end of pack %s (%"PRIuMAX" > %"PRIu32")", + pack_basename(pack->p), (uintmax_t)pack_pos, + pack->p->num_objects); + } + + if (try_partial_reuse(bitmap_git, pack, bit_pos, + pack_pos, reuse, &w_curs) < 0) { /* * try_partial_reuse indicated we couldn't reuse * any bits, so there is no point in trying more @@ -2000,11 +2023,97 @@ int reuse_partial_packfile_from_bitmap(struct bitmap_index *bitmap_git, done: unuse_pack(&w_curs); +} - *entries = bitmap_popcount(reuse); - if (!*entries) { - bitmap_free(reuse); +static int bitmapped_pack_cmp(const void *va, const void *vb) +{ + const struct bitmapped_pack *a = va; + const struct bitmapped_pack *b = vb; + + if (a->bitmap_pos < b->bitmap_pos) return -1; + if (a->bitmap_pos > b->bitmap_pos) + return 1; + return 0; +} + +void reuse_partial_packfile_from_bitmap(struct bitmap_index *bitmap_git, + struct bitmapped_pack **packs_out, + size_t *packs_nr_out, + struct bitmap **reuse_out, + int multi_pack_reuse) +{ + struct repository *r = the_repository; + struct bitmapped_pack *packs = NULL; + struct bitmap *result = bitmap_git->result; + struct bitmap *reuse; + size_t i; + size_t packs_nr = 0, packs_alloc = 0; + size_t word_alloc; + uint32_t objects_nr = 0; + + assert(result); + + load_reverse_index(r, bitmap_git); + + if (bitmap_is_midx(bitmap_git)) { + for (i = 0; i < bitmap_git->midx->num_packs; i++) { + struct bitmapped_pack pack; + if (nth_bitmapped_pack(r, bitmap_git->midx, &pack, i) < 0) { + warning(_("unable to load pack: '%s', disabling pack-reuse"), + bitmap_git->midx->pack_names[i]); + free(packs); + return; + } + + if (!pack.bitmap_nr) + continue; + + if (!multi_pack_reuse && pack.bitmap_pos) { + /* + * If we're only reusing a single pack, skip + * over any packs which are not positioned at + * the beginning of the MIDX bitmap. + * + * This is consistent with the existing + * single-pack reuse behavior, which only reuses + * parts of the MIDX's preferred pack. + */ + continue; + } + + ALLOC_GROW(packs, packs_nr + 1, packs_alloc); + memcpy(&packs[packs_nr++], &pack, sizeof(pack)); + + objects_nr += pack.p->num_objects; + + if (!multi_pack_reuse) + break; + } + + QSORT(packs, packs_nr, bitmapped_pack_cmp); + } else { + ALLOC_GROW(packs, packs_nr + 1, packs_alloc); + + packs[packs_nr].p = bitmap_git->pack; + packs[packs_nr].bitmap_nr = bitmap_git->pack->num_objects; + packs[packs_nr].bitmap_pos = 0; + + objects_nr = packs[packs_nr++].bitmap_nr; + } + + word_alloc = objects_nr / BITS_IN_EWORD; + if (objects_nr % BITS_IN_EWORD) + word_alloc++; + reuse = bitmap_word_alloc(word_alloc); + + for (i = 0; i < packs_nr; i++) + reuse_partial_packfile_from_bitmap_1(bitmap_git, &packs[i], reuse); + + if (bitmap_is_empty(reuse)) { + free(packs); + bitmap_free(reuse); + return; } /* @@ -2012,9 +2121,9 @@ done: * need to be handled separately. */ bitmap_and_not(result, reuse); - *packfile_out = pack; + *packs_out = packs; + *packs_nr_out = packs_nr; *reuse_out = reuse; - return 0; } int bitmap_walk_contains(struct bitmap_index *bitmap_git, diff --git a/pack-bitmap.h b/pack-bitmap.h index 5273a6a019..c7dea13217 100644 --- a/pack-bitmap.h +++ b/pack-bitmap.h @@ -52,6 +52,15 @@ typedef int (*show_reachable_fn)( struct bitmap_index; +struct bitmapped_pack { + struct packed_git *p; + + uint32_t bitmap_pos; + uint32_t bitmap_nr; + + uint32_t pack_int_id; /* MIDX only */ +}; + struct bitmap_index *prepare_bitmap_git(struct repository *r); struct bitmap_index *prepare_midx_bitmap_git(struct multi_pack_index *midx); void count_bitmap_commit_list(struct bitmap_index *, uint32_t *commits, @@ -68,11 +77,11 @@ int test_bitmap_hashes(struct repository *r); struct bitmap_index *prepare_bitmap_walk(struct rev_info *revs, int filter_provided_objects); -uint32_t midx_preferred_pack(struct bitmap_index *bitmap_git); -int reuse_partial_packfile_from_bitmap(struct bitmap_index *, - struct packed_git **packfile, - uint32_t *entries, - struct bitmap **reuse_out); +void reuse_partial_packfile_from_bitmap(struct bitmap_index *bitmap_git, + struct bitmapped_pack **packs_out, + size_t *packs_nr_out, + struct bitmap **reuse_out, + int multi_pack_reuse); int rebuild_existing_bitmaps(struct bitmap_index *, struct packing_data *mapping, kh_oid_map_t *reused_bitmaps, int show_progress); void free_bitmap_index(struct bitmap_index *); diff --git a/pack-objects.c b/pack-objects.c index f403ca6986..a9d9855063 100644 --- a/pack-objects.c +++ b/pack-objects.c @@ -151,6 +151,21 @@ void prepare_packing_data(struct repository *r, struct packing_data *pdata) init_recursive_mutex(&pdata->odb_lock); } +void clear_packing_data(struct packing_data *pdata) +{ + if (!pdata) + return; + + free(pdata->cruft_mtime); + free(pdata->in_pack); + free(pdata->in_pack_by_idx); + free(pdata->in_pack_pos); + free(pdata->index); + free(pdata->layer); + free(pdata->objects); + free(pdata->tree_depth); +} + struct object_entry *packlist_alloc(struct packing_data *pdata, const struct object_id *oid) { diff --git a/pack-objects.h b/pack-objects.h index 0d78db40cb..b9898a4e64 100644 --- a/pack-objects.h +++ b/pack-objects.h @@ -169,6 +169,7 @@ struct packing_data { }; void prepare_packing_data(struct repository *r, struct packing_data *pdata); +void clear_packing_data(struct packing_data *pdata); /* Protect access to object database */ static inline void packing_data_lock(struct packing_data *pdata) diff --git a/pack-revindex.c b/pack-revindex.c index acf1dd9786..a7624d8be8 100644 --- a/pack-revindex.c +++ b/pack-revindex.c @@ -520,19 +520,12 @@ static int midx_pack_order_cmp(const void *va, const void *vb) return 0; } -int midx_to_pack_pos(struct multi_pack_index *m, uint32_t at, uint32_t *pos) +static int midx_key_to_pack_pos(struct multi_pack_index *m, + struct midx_pack_key *key, + uint32_t *pos) { - struct midx_pack_key key; uint32_t *found; - if (!m->revindex_data) - BUG("midx_to_pack_pos: reverse index not yet loaded"); - if (m->num_objects <= at) - BUG("midx_to_pack_pos: out-of-bounds object at %"PRIu32, at); - - key.pack = nth_midxed_pack_int_id(m, at); - key.offset = nth_midxed_offset(m, at); - key.midx = m; /* * The preferred pack sorts first, so determine its identifier by * looking at the first object in pseudo-pack order. @@ -542,14 +535,43 @@ int midx_to_pack_pos(struct multi_pack_index *m, uint32_t at, uint32_t *pos) * implicitly is preferred (and includes all its objects, since ties are * broken first by pack identifier). */ - key.preferred_pack = nth_midxed_pack_int_id(m, pack_pos_to_midx(m, 0)); + if (midx_preferred_pack(key->midx, &key->preferred_pack) < 0) + return error(_("could not determine preferred pack")); - found = bsearch(&key, m->revindex_data, m->num_objects, - sizeof(*m->revindex_data), midx_pack_order_cmp); + found = bsearch(key, m->revindex_data, m->num_objects, + sizeof(*m->revindex_data), + midx_pack_order_cmp); if (!found) - return error("bad offset for revindex"); + return -1; *pos = found - m->revindex_data; return 0; } + +int midx_to_pack_pos(struct multi_pack_index *m, uint32_t at, uint32_t *pos) +{ + struct midx_pack_key key; + + if (!m->revindex_data) + BUG("midx_to_pack_pos: reverse index not yet loaded"); + if (m->num_objects <= at) + BUG("midx_to_pack_pos: out-of-bounds object at %"PRIu32, at); + + key.pack = nth_midxed_pack_int_id(m, at); + key.offset = nth_midxed_offset(m, at); + key.midx = m; + + return midx_key_to_pack_pos(m, &key, pos); +} + +int midx_pair_to_pack_pos(struct multi_pack_index *m, uint32_t pack_int_id, + off_t ofs, uint32_t *pos) +{ + struct midx_pack_key key = { + .pack = pack_int_id, + .offset = ofs, + .midx = m, + }; + return midx_key_to_pack_pos(m, &key, pos); +} diff --git a/pack-revindex.h b/pack-revindex.h index 6dd47efea1..422c2487ae 100644 --- a/pack-revindex.h +++ b/pack-revindex.h @@ -142,4 +142,7 @@ uint32_t pack_pos_to_midx(struct multi_pack_index *m, uint32_t pos); */ int midx_to_pack_pos(struct multi_pack_index *midx, uint32_t at, uint32_t *pos); +int midx_pair_to_pack_pos(struct multi_pack_index *midx, uint32_t pack_id, + off_t ofs, uint32_t *pos); + #endif diff --git a/packfile.c b/packfile.c index 84a005674d..d4df7fdeea 100644 --- a/packfile.c +++ b/packfile.c @@ -2249,7 +2249,8 @@ static int add_promisor_object(const struct object_id *oid, struct tree *tree = (struct tree *)obj; struct tree_desc desc; struct name_entry entry; - if (init_tree_desc_gently(&desc, tree->buffer, tree->size, 0)) + if (init_tree_desc_gently(&desc, &tree->object.oid, + tree->buffer, tree->size, 0)) /* * Error messages are given when packs are * verified, so do not print any here. diff --git a/parse-options.c b/parse-options.c index 4ce2b7ca16..63a99dea6e 100644 --- a/parse-options.c +++ b/parse-options.c @@ -357,6 +357,7 @@ static enum parse_opt_result parse_long_opt( const char *arg_end = strchrnul(arg, '='); const struct option *abbrev_option = NULL, *ambiguous_option = NULL; enum opt_parsed abbrev_flags = OPT_LONG, ambiguous_flags = OPT_LONG; + int allow_abbrev = !(p->flags & PARSE_OPT_KEEP_UNKNOWN_OPT); for (; options->type != OPTION_END; options++) { const char *rest, *long_name = options->long_name; @@ -367,12 +368,16 @@ static enum parse_opt_result parse_long_opt( if (!long_name) continue; -again: + if (!starts_with(arg, "no-") && + !(options->flags & PARSE_OPT_NONEG) && + skip_prefix(long_name, "no-", &long_name)) + opt_flags |= OPT_UNSET; + if (!skip_prefix(arg, long_name, &rest)) rest = NULL; if (!rest) { /* abbreviated? */ - if (!(p->flags & PARSE_OPT_KEEP_UNKNOWN_OPT) && + if (allow_abbrev && !strncmp(long_name, arg, arg_end - arg)) { is_abbreviated: if (abbrev_option && @@ -396,22 +401,18 @@ is_abbreviated: if (options->flags & PARSE_OPT_NONEG) continue; /* negated and abbreviated very much? */ - if (starts_with("no-", arg)) { + if (allow_abbrev && starts_with("no-", arg)) { flags |= OPT_UNSET; goto is_abbreviated; } /* negated? */ - if (!starts_with(arg, "no-")) { - if (skip_prefix(long_name, "no-", &long_name)) { - opt_flags |= OPT_UNSET; - goto again; - } + if (!starts_with(arg, "no-")) continue; - } flags |= OPT_UNSET; if (!skip_prefix(arg + 3, long_name, &rest)) { /* abbreviated and negated? */ - if (starts_with(long_name, arg + 3)) + if (allow_abbrev && + starts_with(long_name, arg + 3)) goto is_abbreviated; else continue; @@ -1588,7 +1588,5 @@ REPO_GIT_PATH_FUNC(merge_msg, "MERGE_MSG") REPO_GIT_PATH_FUNC(merge_rr, "MERGE_RR") REPO_GIT_PATH_FUNC(merge_mode, "MERGE_MODE") REPO_GIT_PATH_FUNC(merge_head, "MERGE_HEAD") -REPO_GIT_PATH_FUNC(merge_autostash, "MERGE_AUTOSTASH") -REPO_GIT_PATH_FUNC(auto_merge, "AUTO_MERGE") REPO_GIT_PATH_FUNC(fetch_head, "FETCH_HEAD") REPO_GIT_PATH_FUNC(shallow, "shallow") @@ -175,8 +175,6 @@ const char *git_path_merge_msg(struct repository *r); const char *git_path_merge_rr(struct repository *r); const char *git_path_merge_mode(struct repository *r); const char *git_path_merge_head(struct repository *r); -const char *git_path_merge_autostash(struct repository *r); -const char *git_path_auto_merge(struct repository *r); const char *git_path_fetch_head(struct repository *r); const char *git_path_shallow(struct repository *r); diff --git a/pkt-line.c b/pkt-line.c index 236dd3a3ee..24479eae4d 100644 --- a/pkt-line.c +++ b/pkt-line.c @@ -463,8 +463,32 @@ enum packet_read_status packet_read_with_status(int fd, char **src_buffer, } if ((options & PACKET_READ_CHOMP_NEWLINE) && - len && buffer[len-1] == '\n') - len--; + len && buffer[len-1] == '\n') { + if (options & PACKET_READ_USE_SIDEBAND) { + int band = *buffer & 0xff; + switch (band) { + case 1: + /* Chomp newline for payload */ + len--; + break; + case 2: + case 3: + /* + * Do not chomp newline for progress and error + * message. + */ + break; + default: + /* + * Bad sideband, let's leave it to + * demultiplex_sideband() to catch this error. + */ + break; + } + } else { + len--; + } + } buffer[len] = 0; if (options & PACKET_READ_REDACT_URI_PATH && @@ -593,17 +617,19 @@ void packet_reader_init(struct packet_reader *reader, int fd, reader->options = options; reader->me = "git"; reader->hash_algo = &hash_algos[GIT_HASH_SHA1]; + strbuf_init(&reader->scratch, 0); } enum packet_read_status packet_reader_read(struct packet_reader *reader) { - struct strbuf scratch = STRBUF_INIT; - if (reader->line_peeked) { reader->line_peeked = 0; return reader->status; } + if (reader->use_sideband) + reader->options |= PACKET_READ_USE_SIDEBAND; + /* * Consume all progress packets until a primary payload packet is * received @@ -621,7 +647,7 @@ enum packet_read_status packet_reader_read(struct packet_reader *reader) break; if (demultiplex_sideband(reader->me, reader->status, reader->buffer, reader->pktlen, 1, - &scratch, &sideband_type)) + &reader->scratch, &sideband_type)) break; } diff --git a/pkt-line.h b/pkt-line.h index aedef56286..3b33cc64f3 100644 --- a/pkt-line.h +++ b/pkt-line.h @@ -84,6 +84,7 @@ void packet_fflush(FILE *f); #define PACKET_READ_DIE_ON_ERR_PACKET (1u<<2) #define PACKET_READ_GENTLE_ON_READ_ERROR (1u<<3) #define PACKET_READ_REDACT_URI_PATH (1u<<4) +#define PACKET_READ_USE_SIDEBAND (1u<<5) int packet_read(int fd, char *buffer, unsigned size, int options); /* @@ -193,6 +194,9 @@ struct packet_reader { /* hash algorithm in use */ const struct git_hash_algo *hash_algo; + + /* hold temporary sideband message */ + struct strbuf scratch; }; /* @@ -1759,7 +1759,7 @@ static size_t format_commit_one(struct strbuf *sb, /* in UTF-8 */ goto trailer_out; } if (*arg == ')') { - format_trailers_from_commit(sb, msg + c->subject_off, &opts); + format_trailers_from_commit(msg + c->subject_off, &opts, sb); ret = arg - placeholder + 1; } trailer_out: diff --git a/ref-filter.c b/ref-filter.c index 35b989e1df..abfc7fdd72 100644 --- a/ref-filter.c +++ b/ref-filter.c @@ -1985,7 +1985,7 @@ static void grab_sub_body_contents(struct atom_value *val, int deref, struct exp struct strbuf s = STRBUF_INIT; /* Format the trailer info according to the trailer_opts given */ - format_trailers_from_commit(&s, subpos, &atom->u.contents.trailer_opts); + format_trailers_from_commit(subpos, &atom->u.contents.trailer_opts, &s); v->s = strbuf_detach(&s, NULL); } else if (atom->u.contents.option == C_BARE) @@ -2622,6 +2622,11 @@ static int for_each_fullref_in_pattern(struct ref_filter *filter, each_ref_fn cb, void *cb_data) { + if (filter->kind & FILTER_REFS_NO_FILTER) { + return refs_for_each_all_refs( + get_main_ref_store(the_repository), cb, cb_data); + } + if (!filter->match_as_path) { /* * in this case, the patterns are applied after @@ -2775,8 +2780,12 @@ static struct ref_array_item *apply_ref_filter(const char *refname, const struct /* Obtain the current ref kind from filter_ref_kind() and ignore unwanted refs. */ kind = filter_ref_kind(filter, refname); - if (!(kind & filter->kind)) + if (filter->kind & FILTER_REFS_NO_FILTER) { + if (kind == FILTER_REFS_DETACHED_HEAD) + kind = FILTER_REFS_OTHERS; + } else if (!(kind & filter->kind)) { return NULL; + } if (!filter_pattern_match(filter, refname)) return NULL; @@ -3041,7 +3050,7 @@ static int do_filter_refs(struct ref_filter *filter, unsigned int type, each_ref ret = for_each_fullref_in("refs/remotes/", fn, cb_data); else if (filter->kind == FILTER_REFS_TAGS) ret = for_each_fullref_in("refs/tags/", fn, cb_data); - else if (filter->kind & FILTER_REFS_ALL) + else if (filter->kind & FILTER_REFS_ALL || filter->kind & FILTER_REFS_NO_FILTER) ret = for_each_fullref_in_pattern(filter, fn, cb_data); if (!ret && (filter->kind & FILTER_REFS_DETACHED_HEAD)) head_ref(fn, cb_data); diff --git a/ref-filter.h b/ref-filter.h index 07cd6f6da3..1eab325ce0 100644 --- a/ref-filter.h +++ b/ref-filter.h @@ -22,7 +22,9 @@ #define FILTER_REFS_ALL (FILTER_REFS_TAGS | FILTER_REFS_BRANCHES | \ FILTER_REFS_REMOTES | FILTER_REFS_OTHERS) #define FILTER_REFS_DETACHED_HEAD 0x0020 -#define FILTER_REFS_KIND_MASK (FILTER_REFS_ALL | FILTER_REFS_DETACHED_HEAD) +#define FILTER_REFS_NO_FILTER 0x0040 +#define FILTER_REFS_KIND_MASK (FILTER_REFS_ALL | FILTER_REFS_DETACHED_HEAD | \ + FILTER_REFS_NO_FILTER) struct atom_value; struct ref_sorting; @@ -39,7 +39,7 @@ static int tree_is_complete(const struct object_id *oid) tree->buffer = data; tree->size = size; } - init_tree_desc(&desc, tree->buffer, tree->size); + init_tree_desc(&desc, &tree->object.oid, tree->buffer, tree->size); complete = 1; while (tree_entry(&desc, &entry)) { if (!repo_has_object_file(the_repository, &entry.oid) || @@ -33,17 +33,33 @@ /* * List of all available backends */ -static struct ref_storage_be *refs_backends = &refs_be_files; +static const struct ref_storage_be *refs_backends[] = { + [REF_STORAGE_FORMAT_FILES] = &refs_be_files, +}; -static struct ref_storage_be *find_ref_storage_backend(const char *name) +static const struct ref_storage_be *find_ref_storage_backend(unsigned int ref_storage_format) { - struct ref_storage_be *be; - for (be = refs_backends; be; be = be->next) - if (!strcmp(be->name, name)) - return be; + if (ref_storage_format < ARRAY_SIZE(refs_backends)) + return refs_backends[ref_storage_format]; return NULL; } +unsigned int ref_storage_format_by_name(const char *name) +{ + for (unsigned int i = 0; i < ARRAY_SIZE(refs_backends); i++) + if (refs_backends[i] && !strcmp(refs_backends[i]->name, name)) + return i; + return REF_STORAGE_FORMAT_UNKNOWN; +} + +const char *ref_storage_format_to_name(unsigned int ref_storage_format) +{ + const struct ref_storage_be *be = find_ref_storage_backend(ref_storage_format); + if (!be) + return "unknown"; + return be->name; +} + /* * How to handle various characters in refnames: * 0: An acceptable character for refs @@ -65,6 +81,21 @@ static unsigned char refname_disposition[256] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4, 4 }; +/* + * List of documented pseudorefs. This needs to be kept in sync with the list + * in Documentation/revisions.txt. + */ +static const char *const pseudorefs[] = { + "FETCH_HEAD", + "ORIG_HEAD", + "MERGE_HEAD", + "REBASE_HEAD", + "CHERRY_PICK_HEAD", + "REVERT_HEAD", + "BISECT_HEAD", + "AUTO_MERGE", +}; + struct ref_namespace_info ref_namespace[] = { [NAMESPACE_HEAD] = { .ref = "HEAD", @@ -550,13 +581,16 @@ void normalize_glob_ref(struct string_list_item *item, const char *prefix, if (prefix) strbuf_addstr(&normalized_pattern, prefix); - else if (!starts_with(pattern, "refs/") && - strcmp(pattern, "HEAD")) - strbuf_addstr(&normalized_pattern, "refs/"); - /* - * NEEDSWORK: Special case other symrefs such as REBASE_HEAD, - * MERGE_HEAD, etc. - */ + else if (!starts_with(pattern, "refs/") && strcmp(pattern, "HEAD")) { + int i; + + for (i = 0; i < ARRAY_SIZE(pseudorefs); i++) + if (!strcmp(pattern, pseudorefs[i])) + break; + + if (i == ARRAY_SIZE(pseudorefs)) + strbuf_addstr(&normalized_pattern, "refs/"); + } strbuf_addstr(&normalized_pattern, pattern); strbuf_strip_suffix(&normalized_pattern, "/"); @@ -827,8 +861,16 @@ int is_per_worktree_ref(const char *refname) starts_with(refname, "refs/rewritten/"); } -static int is_pseudoref_syntax(const char *refname) +int is_pseudoref_syntax(const char *refname) { + /* TODO: move these pseudorefs to have _HEAD suffix */ + static const char *const irregular_pseudorefs[] = { + "BISECT_EXPECTED_REV", + "NOTES_MERGE_PARTIAL", + "NOTES_MERGE_REF", + "AUTO_MERGE" + }; + size_t i; const char *c; for (c = refname; *c; c++) { @@ -837,10 +879,17 @@ static int is_pseudoref_syntax(const char *refname) } /* - * HEAD is not a pseudoref, but it certainly uses the - * pseudoref syntax. + * Most pseudorefs end with _HEAD. HEAD itself is not a + * pseudoref, but it certainly uses the pseudoref syntax. */ - return 1; + if (ends_with(refname, "HEAD")) + return 1; + + for (i = 0; i < ARRAY_SIZE(irregular_pseudorefs); i++) + if (!strcmp(refname, irregular_pseudorefs[i])) + return 1; + + return 0; } static int is_current_worktree_ref(const char *ref) { @@ -1549,6 +1598,33 @@ int head_ref(each_ref_fn fn, void *cb_data) return refs_head_ref(get_main_ref_store(the_repository), fn, cb_data); } +int refs_for_each_pseudoref(struct ref_store *refs, + each_ref_fn fn, void *cb_data) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(pseudorefs); i++) { + struct object_id oid; + int flag; + + if (refs_resolve_ref_unsafe(refs, pseudorefs[i], + RESOLVE_REF_READING, &oid, &flag)) { + int ret = fn(pseudorefs[i], &oid, flag, cb_data); + + if (ret) + return ret; + } + } + + return 0; +} + +int for_each_pseudoref(each_ref_fn fn, void *cb_data) +{ + return refs_for_each_pseudoref(get_main_ref_store(the_repository), + fn, cb_data); +} + struct ref_iterator *refs_ref_iterator_begin( struct ref_store *refs, const char *prefix, @@ -1707,6 +1783,13 @@ int for_each_rawref(each_ref_fn fn, void *cb_data) return refs_for_each_rawref(get_main_ref_store(the_repository), fn, cb_data); } +int refs_for_each_all_refs(struct ref_store *refs, each_ref_fn fn, + void *cb_data) +{ + return do_for_each_ref(refs, "", NULL, fn, 0, + DO_FOR_EACH_INCLUDE_ALL_REFS, cb_data); +} + static int qsort_strcmp(const void *va, const void *vb) { const char *a = *(const char **)va; @@ -1823,13 +1906,10 @@ done: static int is_special_ref(const char *refname) { /* - * Special references get written and read directly via the filesystem - * by the subsystems that create them. Thus, they must not go through - * the reference backend but must instead be read directly. It is - * arguable whether this behaviour is sensible, or whether it's simply - * a leaky abstraction enabled by us only having a single reference - * backend implementation. But at least for a subset of references it - * indeed does make sense to treat them specially: + * Special references are refs that have different semantics compared + * to "normal" refs. These refs can thus not be stored in the ref + * backend, but must always be accessed via the filesystem. The + * following refs are special: * * - FETCH_HEAD may contain multiple object IDs, and each one of them * carries additional metadata like where it came from. @@ -1837,30 +1917,12 @@ static int is_special_ref(const char *refname) * - MERGE_HEAD may contain multiple object IDs when merging multiple * heads. * - * There are some exceptions that you might expect to see on this list - * but which are handled exclusively via the reference backend: - * - * - BISECT_EXPECTED_REV - * - * - CHERRY_PICK_HEAD - * - * - HEAD - * - * - ORIG_HEAD - * - * - "rebase-apply/" and "rebase-merge/" contain all of the state for - * rebases, including some reference-like files. These are - * exclusively read and written via the filesystem and never go - * through the refdb. - * - * Writing or deleting references must consistently go either through - * the filesystem (special refs) or through the reference backend - * (normal ones). + * Reading, writing or deleting references must consistently go either + * through the filesystem (special refs) or through the reference + * backend (normal ones). */ static const char * const special_refs[] = { - "AUTO_MERGE", "FETCH_HEAD", - "MERGE_AUTOSTASH", "MERGE_HEAD", }; size_t i; @@ -1981,11 +2043,9 @@ const char *refs_resolve_ref_unsafe(struct ref_store *refs, } /* backend functions */ -int refs_init_db(struct strbuf *err) +int refs_init_db(struct ref_store *refs, int flags, struct strbuf *err) { - struct ref_store *refs = get_main_ref_store(the_repository); - - return refs->be->init_db(refs, err); + return refs->be->init_db(refs, flags, err); } const char *resolve_ref_unsafe(const char *refname, int resolve_flags, @@ -2082,12 +2142,12 @@ static struct ref_store *ref_store_init(struct repository *repo, const char *gitdir, unsigned int flags) { - const char *be_name = "files"; - struct ref_storage_be *be = find_ref_storage_backend(be_name); + const struct ref_storage_be *be; struct ref_store *refs; + be = find_ref_storage_backend(repo->ref_storage_format); if (!be) - BUG("reference backend %s is unknown", be_name); + BUG("reference backend is unknown"); refs = be->init(repo, gitdir, flags); return refs; @@ -11,6 +11,9 @@ struct string_list; struct string_list_item; struct worktree; +unsigned int ref_storage_format_by_name(const char *name); +const char *ref_storage_format_to_name(unsigned int ref_storage_format); + /* * Resolve a reference, recursively following symbolic refererences. * @@ -123,7 +126,9 @@ int should_autocreate_reflog(const char *refname); int is_branch(const char *refname); -int refs_init_db(struct strbuf *err); +#define REFS_INIT_DB_IS_WORKTREE (1 << 0) + +int refs_init_db(struct ref_store *refs, int flags, struct strbuf *err); /* * Return the peeled value of the oid currently being iterated via @@ -320,6 +325,8 @@ typedef int each_repo_ref_fn(struct repository *r, */ int refs_head_ref(struct ref_store *refs, each_ref_fn fn, void *cb_data); +int refs_for_each_pseudoref(struct ref_store *refs, + each_ref_fn fn, void *cb_data); int refs_for_each_ref(struct ref_store *refs, each_ref_fn fn, void *cb_data); int refs_for_each_ref_in(struct ref_store *refs, const char *prefix, @@ -334,6 +341,9 @@ int refs_for_each_remote_ref(struct ref_store *refs, /* just iterates the head ref. */ int head_ref(each_ref_fn fn, void *cb_data); +/* iterates pseudorefs. */ +int for_each_pseudoref(each_ref_fn fn, void *cb_data); + /* iterates all refs. */ int for_each_ref(each_ref_fn fn, void *cb_data); @@ -394,6 +404,12 @@ int refs_for_each_rawref(struct ref_store *refs, each_ref_fn fn, void *cb_data); int for_each_rawref(each_ref_fn fn, void *cb_data); /* + * Iterates over all ref types, regular, pseudorefs and HEAD. + */ +int refs_for_each_all_refs(struct ref_store *refs, each_ref_fn fn, + void *cb_data); + +/* * Normalizes partial refs to their fully qualified form. * Will prepend <prefix> to the <pattern> if it doesn't start with 'refs/'. * <prefix> will default to 'refs/' if NULL. @@ -846,6 +862,12 @@ const char **hidden_refs_to_excludes(const struct strvec *hide_refs); /* Is this a per-worktree ref living in the refs/ namespace? */ int is_per_worktree_ref(const char *refname); +/* + * Check whether a refname matches the pseudoref syntax. This is a surface + * level check and can present false positives. + */ +int is_pseudoref_syntax(const char *refname); + /* Describes how a refname relates to worktrees */ enum ref_worktree_type { REF_WORKTREE_CURRENT, /* implicitly per worktree, eg. HEAD or diff --git a/refs/debug.c b/refs/debug.c index 83b7a0ba65..634681ca44 100644 --- a/refs/debug.c +++ b/refs/debug.c @@ -33,10 +33,10 @@ struct ref_store *maybe_debug_wrap_ref_store(const char *gitdir, struct ref_stor return (struct ref_store *)res; } -static int debug_init_db(struct ref_store *refs, struct strbuf *err) +static int debug_init_db(struct ref_store *refs, int flags, struct strbuf *err) { struct debug_ref_store *drefs = (struct debug_ref_store *)refs; - int res = drefs->refs->be->init_db(drefs->refs, err); + int res = drefs->refs->be->init_db(drefs->refs, flags, err); trace_printf_key(&trace_refs, "init_db: %d\n", res); return res; } @@ -426,7 +426,6 @@ static int debug_reflog_expire(struct ref_store *ref_store, const char *refname, } struct ref_storage_be refs_be_debug = { - .next = NULL, .name = "debug", .init = NULL, .init_db = debug_init_db, diff --git a/refs/files-backend.c b/refs/files-backend.c index 6734f2a309..940beac1f9 100644 --- a/refs/files-backend.c +++ b/refs/files-backend.c @@ -229,6 +229,38 @@ static void add_per_worktree_entries_to_dir(struct ref_dir *dir, const char *dir } } +static void loose_fill_ref_dir_regular_file(struct files_ref_store *refs, + const char *refname, + struct ref_dir *dir) +{ + struct object_id oid; + int flag; + + if (!refs_resolve_ref_unsafe(&refs->base, refname, RESOLVE_REF_READING, + &oid, &flag)) { + oidclr(&oid); + flag |= REF_ISBROKEN; + } else if (is_null_oid(&oid)) { + /* + * It is so astronomically unlikely + * that null_oid is the OID of an + * actual object that we consider its + * appearance in a loose reference + * file to be repo corruption + * (probably due to a software bug). + */ + flag |= REF_ISBROKEN; + } + + if (check_refname_format(refname, REFNAME_ALLOW_ONELEVEL)) { + if (!refname_is_safe(refname)) + die("loose refname is dangerous: %s", refname); + oidclr(&oid); + flag |= REF_BAD_NAME | REF_ISBROKEN; + } + add_entry_to_dir(dir, create_ref_entry(refname, &oid, flag)); +} + /* * Read the loose references from the namespace dirname into dir * (without recursing). dirname must end with '/'. dir must be the @@ -257,8 +289,6 @@ static void loose_fill_ref_dir(struct ref_store *ref_store, strbuf_add(&refname, dirname, dirnamelen); while ((de = readdir(d)) != NULL) { - struct object_id oid; - int flag; unsigned char dtype; if (de->d_name[0] == '.') @@ -274,33 +304,7 @@ static void loose_fill_ref_dir(struct ref_store *ref_store, create_dir_entry(dir->cache, refname.buf, refname.len)); } else if (dtype == DT_REG) { - if (!refs_resolve_ref_unsafe(&refs->base, - refname.buf, - RESOLVE_REF_READING, - &oid, &flag)) { - oidclr(&oid); - flag |= REF_ISBROKEN; - } else if (is_null_oid(&oid)) { - /* - * It is so astronomically unlikely - * that null_oid is the OID of an - * actual object that we consider its - * appearance in a loose reference - * file to be repo corruption - * (probably due to a software bug). - */ - flag |= REF_ISBROKEN; - } - - if (check_refname_format(refname.buf, - REFNAME_ALLOW_ONELEVEL)) { - if (!refname_is_safe(refname.buf)) - die("loose refname is dangerous: %s", refname.buf); - oidclr(&oid); - flag |= REF_BAD_NAME | REF_ISBROKEN; - } - add_entry_to_dir(dir, - create_ref_entry(refname.buf, &oid, flag)); + loose_fill_ref_dir_regular_file(refs, refname.buf, dir); } strbuf_setlen(&refname, dirnamelen); } @@ -311,9 +315,58 @@ static void loose_fill_ref_dir(struct ref_store *ref_store, add_per_worktree_entries_to_dir(dir, dirname); } -static struct ref_cache *get_loose_ref_cache(struct files_ref_store *refs) +/* + * Add pseudorefs and HEAD to the ref dir by parsing the directory + * for any files which follow the pseudoref syntax. + */ +static void add_pseudoref_like_entries(struct ref_store *ref_store, + struct ref_dir *dir, + const char *dirname) +{ + struct files_ref_store *refs = + files_downcast(ref_store, REF_STORE_READ, "fill_ref_dir"); + struct strbuf path = STRBUF_INIT, refname = STRBUF_INIT; + struct dirent *de; + size_t dirnamelen; + DIR *d; + + files_ref_path(refs, &path, dirname); + + d = opendir(path.buf); + if (!d) { + strbuf_release(&path); + return; + } + + strbuf_addstr(&refname, dirname); + dirnamelen = refname.len; + + while ((de = readdir(d)) != NULL) { + unsigned char dtype; + + if (de->d_name[0] == '.') + continue; + if (ends_with(de->d_name, ".lock")) + continue; + strbuf_addstr(&refname, de->d_name); + + dtype = get_dtype(de, &path, 1); + if (dtype == DT_REG && is_pseudoref_syntax(de->d_name)) + loose_fill_ref_dir_regular_file(refs, refname.buf, dir); + + strbuf_setlen(&refname, dirnamelen); + } + strbuf_release(&refname); + strbuf_release(&path); + closedir(d); +} + +static struct ref_cache *get_loose_ref_cache(struct files_ref_store *refs, + unsigned int flags) { if (!refs->loose) { + struct ref_dir *dir; + /* * Mark the top-level directory complete because we * are about to read the only subdirectory that can @@ -324,12 +377,17 @@ static struct ref_cache *get_loose_ref_cache(struct files_ref_store *refs) /* We're going to fill the top level ourselves: */ refs->loose->root->flag &= ~REF_INCOMPLETE; + dir = get_ref_dir(refs->loose->root); + + if (flags & DO_FOR_EACH_INCLUDE_ALL_REFS) + add_pseudoref_like_entries(dir->cache->ref_store, dir, + refs->loose->root->name); + /* * Add an incomplete entry for "refs/" (to be filled * lazily): */ - add_entry_to_dir(get_ref_dir(refs->loose->root), - create_dir_entry(refs->loose, "refs/", 5)); + add_entry_to_dir(dir, create_dir_entry(refs->loose, "refs/", 5)); } return refs->loose; } @@ -857,7 +915,7 @@ static struct ref_iterator *files_ref_iterator_begin( * disk, and re-reads it if not. */ - loose_iter = cache_ref_iterator_begin(get_loose_ref_cache(refs), + loose_iter = cache_ref_iterator_begin(get_loose_ref_cache(refs, flags), prefix, ref_store->repo, 1); /* @@ -1218,7 +1276,7 @@ static int files_pack_refs(struct ref_store *ref_store, packed_refs_lock(refs->packed_ref_store, LOCK_DIE_ON_ERROR, &err); - iter = cache_ref_iterator_begin(get_loose_ref_cache(refs), NULL, + iter = cache_ref_iterator_begin(get_loose_ref_cache(refs, 0), NULL, the_repository, 0); while ((ok = ref_iterator_advance(iter)) == ITER_OK) { /* @@ -3218,28 +3276,52 @@ static int files_reflog_expire(struct ref_store *ref_store, return -1; } -static int files_init_db(struct ref_store *ref_store, struct strbuf *err UNUSED) +static int files_init_db(struct ref_store *ref_store, + int flags, + struct strbuf *err UNUSED) { struct files_ref_store *refs = files_downcast(ref_store, REF_STORE_WRITE, "init_db"); struct strbuf sb = STRBUF_INIT; /* - * Create .git/refs/{heads,tags} + * We need to create a "refs" dir in any case so that older versions of + * Git can tell that this is a repository. This serves two main purposes: + * + * - Clients will know to stop walking the parent-directory chain when + * detecting the Git repository. Otherwise they may end up detecting + * a Git repository in a parent directory instead. + * + * - Instead of failing to detect a repository with unknown reference + * format altogether, old clients will print an error saying that + * they do not understand the reference format extension. */ - files_ref_path(refs, &sb, "refs/heads"); + strbuf_addf(&sb, "%s/refs", ref_store->gitdir); safe_create_dir(sb.buf, 1); + adjust_shared_perm(sb.buf); - strbuf_reset(&sb); - files_ref_path(refs, &sb, "refs/tags"); - safe_create_dir(sb.buf, 1); + /* + * There is no need to create directories for common refs when creating + * a worktree ref store. + */ + if (!(flags & REFS_INIT_DB_IS_WORKTREE)) { + /* + * Create .git/refs/{heads,tags} + */ + strbuf_reset(&sb); + files_ref_path(refs, &sb, "refs/heads"); + safe_create_dir(sb.buf, 1); + + strbuf_reset(&sb); + files_ref_path(refs, &sb, "refs/tags"); + safe_create_dir(sb.buf, 1); + } strbuf_release(&sb); return 0; } struct ref_storage_be refs_be_files = { - .next = NULL, .name = "files", .init = files_ref_store_create, .init_db = files_init_db, diff --git a/refs/packed-backend.c b/refs/packed-backend.c index e46906e612..a499a91c7e 100644 --- a/refs/packed-backend.c +++ b/refs/packed-backend.c @@ -1245,6 +1245,7 @@ static const char PACKED_REFS_HEADER[] = "# pack-refs with: peeled fully-peeled sorted \n"; static int packed_init_db(struct ref_store *ref_store UNUSED, + int flags UNUSED, struct strbuf *err UNUSED) { /* Nothing to do. */ @@ -1704,7 +1705,6 @@ static struct ref_iterator *packed_reflog_iterator_begin(struct ref_store *ref_s } struct ref_storage_be refs_be_packed = { - .next = NULL, .name = "packed", .init = packed_ref_store_create, .init_db = packed_init_db, diff --git a/refs/refs-internal.h b/refs/refs-internal.h index 4af83bf9a5..be15c19c02 100644 --- a/refs/refs-internal.h +++ b/refs/refs-internal.h @@ -260,6 +260,13 @@ enum do_for_each_ref_flags { * INCLUDE_BROKEN, since they are otherwise not included at all. */ DO_FOR_EACH_OMIT_DANGLING_SYMREFS = (1 << 2), + + /* + * Include all refs in the $GIT_DIR in contrast to generally only listing + * references having the "refs/" prefix. In the files-backend this is + * limited to regular refs, pseudorefs and HEAD. + */ + DO_FOR_EACH_INCLUDE_ALL_REFS = (1 << 3), }; /* @@ -529,7 +536,9 @@ typedef struct ref_store *ref_store_init_fn(struct repository *repo, const char *gitdir, unsigned int flags); -typedef int ref_init_db_fn(struct ref_store *refs, struct strbuf *err); +typedef int ref_init_db_fn(struct ref_store *refs, + int flags, + struct strbuf *err); typedef int ref_transaction_prepare_fn(struct ref_store *refs, struct ref_transaction *transaction, @@ -663,7 +672,6 @@ typedef int read_symbolic_ref_fn(struct ref_store *ref_store, const char *refnam struct strbuf *referent); struct ref_storage_be { - struct ref_storage_be *next; const char *name; ref_store_init_fn *init; ref_init_db_fn *init_db; diff --git a/reftable/block_test.c b/reftable/block_test.c index c00bbc8aed..dedb05c7d8 100644 --- a/reftable/block_test.c +++ b/reftable/block_test.c @@ -49,13 +49,11 @@ static void test_block_read_write(void) for (i = 0; i < N; i++) { char name[100]; - uint8_t hash[GIT_SHA1_RAWSZ]; snprintf(name, sizeof(name), "branch%02d", i); - memset(hash, i, sizeof(hash)); rec.u.ref.refname = name; rec.u.ref.value_type = REFTABLE_REF_VAL1; - rec.u.ref.value.val1 = hash; + memset(rec.u.ref.value.val1, i, GIT_SHA1_RAWSZ); names[i] = xstrdup(name); n = block_writer_add(&bw, &rec); diff --git a/reftable/blocksource.c b/reftable/blocksource.c index a1ea304429..8c41e3c70f 100644 --- a/reftable/blocksource.c +++ b/reftable/blocksource.c @@ -76,8 +76,8 @@ struct reftable_block_source malloc_block_source(void) } struct file_block_source { - int fd; uint64_t size; + unsigned char *data; }; static uint64_t file_size(void *b) @@ -87,19 +87,12 @@ static uint64_t file_size(void *b) static void file_return_block(void *b, struct reftable_block *dest) { - if (dest->len) - memset(dest->data, 0xff, dest->len); - reftable_free(dest->data); } -static void file_close(void *b) +static void file_close(void *v) { - int fd = ((struct file_block_source *)b)->fd; - if (fd > 0) { - close(fd); - ((struct file_block_source *)b)->fd = 0; - } - + struct file_block_source *b = v; + munmap(b->data, b->size); reftable_free(b); } @@ -108,9 +101,7 @@ static int file_read_block(void *v, struct reftable_block *dest, uint64_t off, { struct file_block_source *b = v; assert(off + size <= b->size); - dest->data = reftable_malloc(size); - if (pread_in_full(b->fd, dest->data, size, off) != size) - return -1; + dest->data = b->data + off; dest->len = size; return size; } @@ -125,26 +116,26 @@ static struct reftable_block_source_vtable file_vtable = { int reftable_block_source_from_file(struct reftable_block_source *bs, const char *name) { - struct stat st = { 0 }; - int err = 0; - int fd = open(name, O_RDONLY); - struct file_block_source *p = NULL; + struct file_block_source *p; + struct stat st; + int fd; + + fd = open(name, O_RDONLY); if (fd < 0) { - if (errno == ENOENT) { + if (errno == ENOENT) return REFTABLE_NOT_EXIST_ERROR; - } return -1; } - err = fstat(fd, &st); - if (err < 0) { + if (fstat(fd, &st) < 0) { close(fd); return REFTABLE_IO_ERROR; } - p = reftable_calloc(sizeof(struct file_block_source)); + p = reftable_calloc(sizeof(*p)); p->size = st.st_size; - p->fd = fd; + p->data = xmmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0); + close(fd); assert(!bs->ops); bs->ops = &file_vtable; diff --git a/reftable/merged.c b/reftable/merged.c index 574394092d..c258ce953e 100644 --- a/reftable/merged.c +++ b/reftable/merged.c @@ -123,12 +123,12 @@ static int merged_iter_next_entry(struct merged_iter *mi, reftable_record_release(&top.rec); } - reftable_record_copy_from(rec, &entry.rec, hash_size(mi->hash_id)); + reftable_record_release(rec); + *rec = entry.rec; done: - reftable_record_release(&entry.rec); - strbuf_release(&mi->entry_key); - strbuf_release(&mi->key); + if (err) + reftable_record_release(&entry.rec); return err; } diff --git a/reftable/merged_test.c b/reftable/merged_test.c index 0d6e0d4bf5..bf090b474e 100644 --- a/reftable/merged_test.c +++ b/reftable/merged_test.c @@ -42,7 +42,7 @@ static void write_test_table(struct strbuf *buf, } } - w = reftable_new_writer(&strbuf_add_void, buf, &opts); + w = reftable_new_writer(&strbuf_add_void, &noop_flush, buf, &opts); reftable_writer_set_limits(w, min, max); for (i = 0; i < n; i++) { @@ -70,7 +70,7 @@ static void write_test_log_table(struct strbuf *buf, .exact_log_message = 1, }; struct reftable_writer *w = NULL; - w = reftable_new_writer(&strbuf_add_void, buf, &opts); + w = reftable_new_writer(&strbuf_add_void, &noop_flush, buf, &opts); reftable_writer_set_limits(w, update_index, update_index); for (i = 0; i < n; i++) { @@ -122,13 +122,11 @@ static void readers_destroy(struct reftable_reader **readers, size_t n) static void test_merged_between(void) { - uint8_t hash1[GIT_SHA1_RAWSZ] = { 1, 2, 3, 0 }; - struct reftable_ref_record r1[] = { { .refname = "b", .update_index = 1, .value_type = REFTABLE_REF_VAL1, - .value.val1 = hash1, + .value.val1 = { 1, 2, 3, 0 }, } }; struct reftable_ref_record r2[] = { { .refname = "a", @@ -164,26 +162,24 @@ static void test_merged_between(void) static void test_merged(void) { - uint8_t hash1[GIT_SHA1_RAWSZ] = { 1 }; - uint8_t hash2[GIT_SHA1_RAWSZ] = { 2 }; struct reftable_ref_record r1[] = { { .refname = "a", .update_index = 1, .value_type = REFTABLE_REF_VAL1, - .value.val1 = hash1, + .value.val1 = { 1 }, }, { .refname = "b", .update_index = 1, .value_type = REFTABLE_REF_VAL1, - .value.val1 = hash1, + .value.val1 = { 1 }, }, { .refname = "c", .update_index = 1, .value_type = REFTABLE_REF_VAL1, - .value.val1 = hash1, + .value.val1 = { 1 }, } }; struct reftable_ref_record r2[] = { { @@ -196,13 +192,13 @@ static void test_merged(void) .refname = "c", .update_index = 3, .value_type = REFTABLE_REF_VAL1, - .value.val1 = hash2, + .value.val1 = { 2 }, }, { .refname = "d", .update_index = 3, .value_type = REFTABLE_REF_VAL1, - .value.val1 = hash1, + .value.val1 = { 1 }, }, }; @@ -416,7 +412,7 @@ static void test_default_write_opts(void) struct reftable_write_options opts = { 0 }; struct strbuf buf = STRBUF_INIT; struct reftable_writer *w = - reftable_new_writer(&strbuf_add_void, &buf, &opts); + reftable_new_writer(&strbuf_add_void, &noop_flush, &buf, &opts); struct reftable_ref_record rec = { .refname = "master", diff --git a/reftable/readwrite_test.c b/reftable/readwrite_test.c index 79cd4e4c28..6b99daeaf2 100644 --- a/reftable/readwrite_test.c +++ b/reftable/readwrite_test.c @@ -51,7 +51,7 @@ static void write_table(char ***names, struct strbuf *buf, int N, .hash_id = hash_id, }; struct reftable_writer *w = - reftable_new_writer(&strbuf_add_void, buf, &opts); + reftable_new_writer(&strbuf_add_void, &noop_flush, buf, &opts); struct reftable_ref_record ref = { NULL }; int i = 0, n; struct reftable_log_record log = { NULL }; @@ -59,18 +59,15 @@ static void write_table(char ***names, struct strbuf *buf, int N, *names = reftable_calloc(sizeof(char *) * (N + 1)); reftable_writer_set_limits(w, update_index, update_index); for (i = 0; i < N; i++) { - uint8_t hash[GIT_SHA256_RAWSZ] = { 0 }; char name[100]; int n; - set_test_hash(hash, i); - snprintf(name, sizeof(name), "refs/heads/branch%02d", i); ref.refname = name; ref.update_index = update_index; ref.value_type = REFTABLE_REF_VAL1; - ref.value.val1 = hash; + set_test_hash(ref.value.val1, i); (*names)[i] = xstrdup(name); n = reftable_writer_add_ref(w, &ref); @@ -133,7 +130,7 @@ static void test_log_buffer_size(void) .message = "commit: 9\n", } } }; struct reftable_writer *w = - reftable_new_writer(&strbuf_add_void, &buf, &opts); + reftable_new_writer(&strbuf_add_void, &noop_flush, &buf, &opts); /* This tests buffer extension for log compression. Must use a random hash, to ensure that the compressed part is larger than the original. @@ -174,7 +171,7 @@ static void test_log_overflow(void) .message = msg, } } }; struct reftable_writer *w = - reftable_new_writer(&strbuf_add_void, &buf, &opts); + reftable_new_writer(&strbuf_add_void, &noop_flush, &buf, &opts); uint8_t hash1[GIT_SHA1_RAWSZ] = {1}, hash2[GIT_SHA1_RAWSZ] = { 2 }; @@ -205,7 +202,7 @@ static void test_log_write_read(void) struct reftable_block_source source = { NULL }; struct strbuf buf = STRBUF_INIT; struct reftable_writer *w = - reftable_new_writer(&strbuf_add_void, &buf, &opts); + reftable_new_writer(&strbuf_add_void, &noop_flush, &buf, &opts); const struct reftable_stats *stats = NULL; reftable_writer_set_limits(w, 0, N); for (i = 0; i < N; i++) { @@ -297,7 +294,7 @@ static void test_log_zlib_corruption(void) struct reftable_block_source source = { 0 }; struct strbuf buf = STRBUF_INIT; struct reftable_writer *w = - reftable_new_writer(&strbuf_add_void, &buf, &opts); + reftable_new_writer(&strbuf_add_void, &noop_flush, &buf, &opts); const struct reftable_stats *stats = NULL; uint8_t hash1[GIT_SHA1_RAWSZ] = { 1 }; uint8_t hash2[GIT_SHA1_RAWSZ] = { 2 }; @@ -538,7 +535,7 @@ static void test_table_refs_for(int indexed) struct strbuf buf = STRBUF_INIT; struct reftable_writer *w = - reftable_new_writer(&strbuf_add_void, &buf, &opts); + reftable_new_writer(&strbuf_add_void, &noop_flush, &buf, &opts); struct reftable_iterator it = { NULL }; int j; @@ -549,8 +546,6 @@ static void test_table_refs_for(int indexed) uint8_t hash[GIT_SHA1_RAWSZ]; char fill[51] = { 0 }; char name[100]; - uint8_t hash1[GIT_SHA1_RAWSZ]; - uint8_t hash2[GIT_SHA1_RAWSZ]; struct reftable_ref_record ref = { NULL }; memset(hash, i, sizeof(hash)); @@ -560,11 +555,9 @@ static void test_table_refs_for(int indexed) name[40] = 0; ref.refname = name; - set_test_hash(hash1, i / 4); - set_test_hash(hash2, 3 + i / 4); ref.value_type = REFTABLE_REF_VAL2; - ref.value.val2.value = hash1; - ref.value.val2.target_value = hash2; + set_test_hash(ref.value.val2.value, i / 4); + set_test_hash(ref.value.val2.target_value, 3 + i / 4); /* 80 bytes / entry, so 3 entries per block. Yields 17 */ @@ -572,8 +565,8 @@ static void test_table_refs_for(int indexed) n = reftable_writer_add_ref(w, &ref); EXPECT(n == 0); - if (!memcmp(hash1, want_hash, GIT_SHA1_RAWSZ) || - !memcmp(hash2, want_hash, GIT_SHA1_RAWSZ)) { + if (!memcmp(ref.value.val2.value, want_hash, GIT_SHA1_RAWSZ) || + !memcmp(ref.value.val2.target_value, want_hash, GIT_SHA1_RAWSZ)) { want_names[want_names_len++] = xstrdup(name); } } @@ -635,7 +628,7 @@ static void test_write_empty_table(void) struct reftable_write_options opts = { 0 }; struct strbuf buf = STRBUF_INIT; struct reftable_writer *w = - reftable_new_writer(&strbuf_add_void, &buf, &opts); + reftable_new_writer(&strbuf_add_void, &noop_flush, &buf, &opts); struct reftable_block_source source = { NULL }; struct reftable_reader *rd = NULL; struct reftable_ref_record rec = { NULL }; @@ -673,12 +666,11 @@ static void test_write_object_id_min_length(void) }; struct strbuf buf = STRBUF_INIT; struct reftable_writer *w = - reftable_new_writer(&strbuf_add_void, &buf, &opts); - uint8_t hash[GIT_SHA1_RAWSZ] = {42}; + reftable_new_writer(&strbuf_add_void, &noop_flush, &buf, &opts); struct reftable_ref_record ref = { .update_index = 1, .value_type = REFTABLE_REF_VAL1, - .value.val1 = hash, + .value.val1 = {42}, }; int err; int i; @@ -709,12 +701,11 @@ static void test_write_object_id_length(void) }; struct strbuf buf = STRBUF_INIT; struct reftable_writer *w = - reftable_new_writer(&strbuf_add_void, &buf, &opts); - uint8_t hash[GIT_SHA1_RAWSZ] = {42}; + reftable_new_writer(&strbuf_add_void, &noop_flush, &buf, &opts); struct reftable_ref_record ref = { .update_index = 1, .value_type = REFTABLE_REF_VAL1, - .value.val1 = hash, + .value.val1 = {42}, }; int err; int i; @@ -744,7 +735,7 @@ static void test_write_empty_key(void) struct reftable_write_options opts = { 0 }; struct strbuf buf = STRBUF_INIT; struct reftable_writer *w = - reftable_new_writer(&strbuf_add_void, &buf, &opts); + reftable_new_writer(&strbuf_add_void, &noop_flush, &buf, &opts); struct reftable_ref_record ref = { .refname = "", .update_index = 1, @@ -767,7 +758,7 @@ static void test_write_key_order(void) struct reftable_write_options opts = { 0 }; struct strbuf buf = STRBUF_INIT; struct reftable_writer *w = - reftable_new_writer(&strbuf_add_void, &buf, &opts); + reftable_new_writer(&strbuf_add_void, &noop_flush, &buf, &opts); struct reftable_ref_record refs[2] = { { .refname = "b", @@ -797,6 +788,84 @@ static void test_write_key_order(void) strbuf_release(&buf); } +static void test_write_multiple_indices(void) +{ + struct reftable_write_options opts = { + .block_size = 100, + }; + struct strbuf writer_buf = STRBUF_INIT, buf = STRBUF_INIT; + struct reftable_block_source source = { 0 }; + struct reftable_iterator it = { 0 }; + const struct reftable_stats *stats; + struct reftable_writer *writer; + struct reftable_reader *reader; + int err, i; + + writer = reftable_new_writer(&strbuf_add_void, &noop_flush, &writer_buf, &opts); + reftable_writer_set_limits(writer, 1, 1); + for (i = 0; i < 100; i++) { + struct reftable_ref_record ref = { + .update_index = 1, + .value_type = REFTABLE_REF_VAL1, + .value.val1 = {i}, + }; + + strbuf_reset(&buf); + strbuf_addf(&buf, "refs/heads/%04d", i); + ref.refname = buf.buf, + + err = reftable_writer_add_ref(writer, &ref); + EXPECT_ERR(err); + } + + for (i = 0; i < 100; i++) { + unsigned char hash[GIT_SHA1_RAWSZ] = {i}; + struct reftable_log_record log = { + .update_index = 1, + .value_type = REFTABLE_LOG_UPDATE, + .value.update = { + .old_hash = hash, + .new_hash = hash, + }, + }; + + strbuf_reset(&buf); + strbuf_addf(&buf, "refs/heads/%04d", i); + log.refname = buf.buf, + + err = reftable_writer_add_log(writer, &log); + EXPECT_ERR(err); + } + + reftable_writer_close(writer); + + /* + * The written data should be sufficiently large to result in indices + * for each of the block types. + */ + stats = reftable_writer_stats(writer); + EXPECT(stats->ref_stats.index_offset > 0); + EXPECT(stats->obj_stats.index_offset > 0); + EXPECT(stats->log_stats.index_offset > 0); + + block_source_from_strbuf(&source, &writer_buf); + err = reftable_new_reader(&reader, &source, "filename"); + EXPECT_ERR(err); + + /* + * Seeking the log uses the log index now. In case there is any + * confusion regarding indices we would notice here. + */ + err = reftable_reader_seek_log(reader, &it, ""); + EXPECT_ERR(err); + + reftable_iterator_destroy(&it); + reftable_writer_free(writer); + reftable_reader_free(reader); + strbuf_release(&writer_buf); + strbuf_release(&buf); +} + static void test_corrupt_table_empty(void) { struct strbuf buf = STRBUF_INIT; @@ -846,5 +915,6 @@ int readwrite_test_main(int argc, const char *argv[]) RUN_TEST(test_log_overflow); RUN_TEST(test_write_object_id_length); RUN_TEST(test_write_object_id_min_length); + RUN_TEST(test_write_multiple_indices); return 0; } diff --git a/reftable/record.c b/reftable/record.c index fbaa1fbef5..5c3fbb7b2a 100644 --- a/reftable/record.c +++ b/reftable/record.c @@ -76,7 +76,7 @@ int reftable_is_block_type(uint8_t typ) return 0; } -uint8_t *reftable_ref_record_val1(const struct reftable_ref_record *rec) +const unsigned char *reftable_ref_record_val1(const struct reftable_ref_record *rec) { switch (rec->value_type) { case REFTABLE_REF_VAL1: @@ -88,7 +88,7 @@ uint8_t *reftable_ref_record_val1(const struct reftable_ref_record *rec) } } -uint8_t *reftable_ref_record_val2(const struct reftable_ref_record *rec) +const unsigned char *reftable_ref_record_val2(const struct reftable_ref_record *rec) { switch (rec->value_type) { case REFTABLE_REF_VAL2: @@ -219,13 +219,10 @@ static void reftable_ref_record_copy_from(void *rec, const void *src_rec, case REFTABLE_REF_DELETION: break; case REFTABLE_REF_VAL1: - ref->value.val1 = reftable_malloc(hash_size); memcpy(ref->value.val1, src->value.val1, hash_size); break; case REFTABLE_REF_VAL2: - ref->value.val2.value = reftable_malloc(hash_size); memcpy(ref->value.val2.value, src->value.val2.value, hash_size); - ref->value.val2.target_value = reftable_malloc(hash_size); memcpy(ref->value.val2.target_value, src->value.val2.target_value, hash_size); break; @@ -242,7 +239,7 @@ static char hexdigit(int c) return 'a' + (c - 10); } -static void hex_format(char *dest, uint8_t *src, int hash_size) +static void hex_format(char *dest, const unsigned char *src, int hash_size) { assert(hash_size > 0); if (src) { @@ -299,11 +296,8 @@ void reftable_ref_record_release(struct reftable_ref_record *ref) reftable_free(ref->value.symref); break; case REFTABLE_REF_VAL2: - reftable_free(ref->value.val2.target_value); - reftable_free(ref->value.val2.value); break; case REFTABLE_REF_VAL1: - reftable_free(ref->value.val1); break; case REFTABLE_REF_DELETION: break; @@ -394,7 +388,6 @@ static int reftable_ref_record_decode(void *rec, struct strbuf key, return -1; } - r->value.val1 = reftable_malloc(hash_size); memcpy(r->value.val1, in.buf, hash_size); string_view_consume(&in, hash_size); break; @@ -404,11 +397,9 @@ static int reftable_ref_record_decode(void *rec, struct strbuf key, return -1; } - r->value.val2.value = reftable_malloc(hash_size); memcpy(r->value.val2.value, in.buf, hash_size); string_view_consume(&in, hash_size); - r->value.val2.target_value = reftable_malloc(hash_size); memcpy(r->value.val2.target_value, in.buf, hash_size); string_view_consume(&in, hash_size); break; @@ -1164,7 +1155,7 @@ int reftable_record_equal(struct reftable_record *a, struct reftable_record *b, reftable_record_data(a), reftable_record_data(b), hash_size); } -static int hash_equal(uint8_t *a, uint8_t *b, int hash_size) +static int hash_equal(const unsigned char *a, const unsigned char *b, int hash_size) { if (a && b) return !memcmp(a, b, hash_size); diff --git a/reftable/record_test.c b/reftable/record_test.c index 70ae78feca..2876db7d27 100644 --- a/reftable/record_test.c +++ b/reftable/record_test.c @@ -119,15 +119,10 @@ static void test_reftable_ref_record_roundtrip(void) case REFTABLE_REF_DELETION: break; case REFTABLE_REF_VAL1: - in.u.ref.value.val1 = reftable_malloc(GIT_SHA1_RAWSZ); set_hash(in.u.ref.value.val1, 1); break; case REFTABLE_REF_VAL2: - in.u.ref.value.val2.value = - reftable_malloc(GIT_SHA1_RAWSZ); set_hash(in.u.ref.value.val2.value, 1); - in.u.ref.value.val2.target_value = - reftable_malloc(GIT_SHA1_RAWSZ); set_hash(in.u.ref.value.val2.target_value, 2); break; case REFTABLE_REF_SYMREF: diff --git a/reftable/refname_test.c b/reftable/refname_test.c index 699e1aea41..b9cc62554e 100644 --- a/reftable/refname_test.c +++ b/reftable/refname_test.c @@ -30,7 +30,7 @@ static void test_conflict(void) struct reftable_write_options opts = { 0 }; struct strbuf buf = STRBUF_INIT; struct reftable_writer *w = - reftable_new_writer(&strbuf_add_void, &buf, &opts); + reftable_new_writer(&strbuf_add_void, &noop_flush, &buf, &opts); struct reftable_ref_record rec = { .refname = "a/b", .value_type = REFTABLE_REF_SYMREF, diff --git a/reftable/reftable-record.h b/reftable/reftable-record.h index 67104f8fbf..bb6e99acd3 100644 --- a/reftable/reftable-record.h +++ b/reftable/reftable-record.h @@ -9,6 +9,7 @@ https://developers.google.com/open-source/licenses/bsd #ifndef REFTABLE_RECORD_H #define REFTABLE_RECORD_H +#include "hash-ll.h" #include <stdint.h> /* @@ -38,10 +39,10 @@ struct reftable_ref_record { #define REFTABLE_NR_REF_VALUETYPES 4 } value_type; union { - uint8_t *val1; /* malloced hash. */ + unsigned char val1[GIT_MAX_RAWSZ]; struct { - uint8_t *value; /* first value, malloced hash */ - uint8_t *target_value; /* second value, malloced hash */ + unsigned char value[GIT_MAX_RAWSZ]; /* first hash */ + unsigned char target_value[GIT_MAX_RAWSZ]; /* second hash */ } val2; char *symref; /* referent, malloced 0-terminated string */ } value; @@ -49,11 +50,11 @@ struct reftable_ref_record { /* Returns the first hash, or NULL if `rec` is not of type * REFTABLE_REF_VAL1 or REFTABLE_REF_VAL2. */ -uint8_t *reftable_ref_record_val1(const struct reftable_ref_record *rec); +const unsigned char *reftable_ref_record_val1(const struct reftable_ref_record *rec); /* Returns the second hash, or NULL if `rec` is not of type * REFTABLE_REF_VAL2. */ -uint8_t *reftable_ref_record_val2(const struct reftable_ref_record *rec); +const unsigned char *reftable_ref_record_val2(const struct reftable_ref_record *rec); /* returns whether 'ref' represents a deletion */ int reftable_ref_record_is_deletion(const struct reftable_ref_record *ref); diff --git a/reftable/reftable-writer.h b/reftable/reftable-writer.h index db8de197f6..7c7cae5f99 100644 --- a/reftable/reftable-writer.h +++ b/reftable/reftable-writer.h @@ -88,6 +88,7 @@ struct reftable_stats { /* reftable_new_writer creates a new writer */ struct reftable_writer * reftable_new_writer(ssize_t (*writer_func)(void *, const void *, size_t), + int (*flush_func)(void *), void *writer_arg, struct reftable_write_options *opts); /* Set the range of update indices for the records we will add. When writing a diff --git a/reftable/stack.c b/reftable/stack.c index 16bab82063..833259fe5c 100644 --- a/reftable/stack.c +++ b/reftable/stack.c @@ -8,6 +8,7 @@ https://developers.google.com/open-source/licenses/bsd #include "stack.h" +#include "../write-or-die.h" #include "system.h" #include "merged.h" #include "reader.h" @@ -16,7 +17,6 @@ https://developers.google.com/open-source/licenses/bsd #include "reftable-record.h" #include "reftable-merged.h" #include "writer.h" - #include "tempfile.h" static int stack_try_add(struct reftable_stack *st, @@ -47,6 +47,13 @@ static ssize_t reftable_fd_write(void *arg, const void *data, size_t sz) return write_in_full(*fdp, data, sz); } +static int reftable_fd_flush(void *arg) +{ + int *fdp = (int *)arg; + + return fsync_component(FSYNC_COMPONENT_REFERENCE, *fdp); +} + int reftable_new_stack(struct reftable_stack **dest, const char *dir, struct reftable_write_options config) { @@ -66,6 +73,7 @@ int reftable_new_stack(struct reftable_stack **dest, const char *dir, strbuf_addstr(&list_file_name, "/tables.list"); p->list_file = strbuf_detach(&list_file_name, NULL); + p->list_fd = -1; p->reftable_dir = xstrdup(dir); p->config = config; @@ -175,6 +183,12 @@ void reftable_stack_destroy(struct reftable_stack *st) st->readers_len = 0; FREE_AND_NULL(st->readers); } + + if (st->list_fd >= 0) { + close(st->list_fd); + st->list_fd = -1; + } + FREE_AND_NULL(st->list_file); FREE_AND_NULL(st->reftable_dir); reftable_free(st); @@ -304,69 +318,134 @@ static int tv_cmp(struct timeval *a, struct timeval *b) static int reftable_stack_reload_maybe_reuse(struct reftable_stack *st, int reuse_open) { - struct timeval deadline = { 0 }; - int err = gettimeofday(&deadline, NULL); + char **names = NULL, **names_after = NULL; + struct timeval deadline; int64_t delay = 0; - int tries = 0; - if (err < 0) - return err; + int tries = 0, err; + int fd = -1; + err = gettimeofday(&deadline, NULL); + if (err < 0) + goto out; deadline.tv_sec += 3; + while (1) { - char **names = NULL; - char **names_after = NULL; - struct timeval now = { 0 }; - int err = gettimeofday(&now, NULL); - int err2 = 0; - if (err < 0) { - return err; - } + struct timeval now; - /* Only look at deadlines after the first few times. This - simplifies debugging in GDB */ + err = gettimeofday(&now, NULL); + if (err < 0) + goto out; + + /* + * Only look at deadlines after the first few times. This + * simplifies debugging in GDB. + */ tries++; - if (tries > 3 && tv_cmp(&now, &deadline) >= 0) { - break; - } + if (tries > 3 && tv_cmp(&now, &deadline) >= 0) + goto out; - err = read_lines(st->list_file, &names); - if (err < 0) { - free_names(names); - return err; - } - err = reftable_stack_reload_once(st, names, reuse_open); - if (err == 0) { - free_names(names); - break; - } - if (err != REFTABLE_NOT_EXIST_ERROR) { - free_names(names); - return err; - } + fd = open(st->list_file, O_RDONLY); + if (fd < 0) { + if (errno != ENOENT) { + err = REFTABLE_IO_ERROR; + goto out; + } - /* err == REFTABLE_NOT_EXIST_ERROR can be caused by a concurrent - writer. Check if there was one by checking if the name list - changed. - */ - err2 = read_lines(st->list_file, &names_after); - if (err2 < 0) { - free_names(names); - return err2; + names = reftable_calloc(sizeof(char *)); + } else { + err = fd_read_lines(fd, &names); + if (err < 0) + goto out; } + err = reftable_stack_reload_once(st, names, reuse_open); + if (!err) + break; + if (err != REFTABLE_NOT_EXIST_ERROR) + goto out; + + /* + * REFTABLE_NOT_EXIST_ERROR can be caused by a concurrent + * writer. Check if there was one by checking if the name list + * changed. + */ + err = read_lines(st->list_file, &names_after); + if (err < 0) + goto out; if (names_equal(names_after, names)) { - free_names(names); - free_names(names_after); - return err; + err = REFTABLE_NOT_EXIST_ERROR; + goto out; } + free_names(names); + names = NULL; free_names(names_after); + names_after = NULL; + close(fd); + fd = -1; delay = delay + (delay * rand()) / RAND_MAX + 1; sleep_millisec(delay); } - return 0; +out: + /* + * Invalidate the stat cache. It is sufficient to only close the file + * descriptor and keep the cached stat info because we never use the + * latter when the former is negative. + */ + if (st->list_fd >= 0) { + close(st->list_fd); + st->list_fd = -1; + } + + /* + * Cache stat information in case it provides a useful signal to us. + * According to POSIX, "The st_ino and st_dev fields taken together + * uniquely identify the file within the system." That being said, + * Windows is not POSIX compliant and we do not have these fields + * available. So the information we have there is insufficient to + * determine whether two file descriptors point to the same file. + * + * While we could fall back to using other signals like the file's + * mtime, those are not sufficient to avoid races. We thus refrain from + * using the stat cache on such systems and fall back to the secondary + * caching mechanism, which is to check whether contents of the file + * have changed. + * + * On other systems which are POSIX compliant we must keep the file + * descriptor open. This is to avoid a race condition where two + * processes access the reftable stack at the same point in time: + * + * 1. A reads the reftable stack and caches its stat info. + * + * 2. B updates the stack, appending a new table to "tables.list". + * This will both use a new inode and result in a different file + * size, thus invalidating A's cache in theory. + * + * 3. B decides to auto-compact the stack and merges two tables. The + * file size now matches what A has cached again. Furthermore, the + * filesystem may decide to recycle the inode number of the file + * we have replaced in (2) because it is not in use anymore. + * + * 4. A reloads the reftable stack. Neither the inode number nor the + * file size changed. If the timestamps did not change either then + * we think the cached copy of our stack is up-to-date. + * + * By keeping the file descriptor open the inode number cannot be + * recycled, mitigating the race. + */ + if (!err && fd >= 0 && !fstat(fd, &st->list_st) && + st->list_st.st_dev && st->list_st.st_ino) { + st->list_fd = fd; + fd = -1; + } + + if (fd >= 0) + close(fd); + free_names(names); + free_names(names_after); + return err; } /* -1 = error @@ -375,8 +454,44 @@ static int reftable_stack_reload_maybe_reuse(struct reftable_stack *st, static int stack_uptodate(struct reftable_stack *st) { char **names = NULL; - int err = read_lines(st->list_file, &names); + int err; int i = 0; + + /* + * When we have cached stat information available then we use it to + * verify whether the file has been rewritten. + * + * Note that we explicitly do not want to use `stat_validity_check()` + * and friends here because they may end up not comparing the `st_dev` + * and `st_ino` fields. These functions thus cannot guarantee that we + * indeed still have the same file. + */ + if (st->list_fd >= 0) { + struct stat list_st; + + if (stat(st->list_file, &list_st) < 0) { + /* + * It's fine for "tables.list" to not exist. In that + * case, we have to refresh when the loaded stack has + * any readers. + */ + if (errno == ENOENT) + return !!st->readers_len; + return REFTABLE_IO_ERROR; + } + + /* + * When "tables.list" refers to the same file we can assume + * that it didn't change. This is because we always use + * rename(3P) to update the file and never write to it + * directly. + */ + if (st->list_st.st_dev == list_st.st_dev && + st->list_st.st_ino == list_st.st_ino) + return 0; + } + + err = read_lines(st->list_file, &names); if (err < 0) return err; @@ -425,9 +540,6 @@ int reftable_stack_add(struct reftable_stack *st, return err; } - if (!st->disable_auto_compact) - return reftable_stack_auto_compact(st); - return 0; } @@ -548,6 +660,9 @@ int reftable_addition_commit(struct reftable_addition *add) goto done; } + fsync_component_or_die(FSYNC_COMPONENT_REFERENCE, lock_file_fd, + get_tempfile_path(add->lock_file)); + err = rename_tempfile(&add->lock_file, add->stack->list_file); if (err < 0) { err = REFTABLE_IO_ERROR; @@ -562,7 +677,7 @@ int reftable_addition_commit(struct reftable_addition *add) add->new_tables = NULL; add->new_tables_len = 0; - err = reftable_stack_reload(add->stack); + err = reftable_stack_reload_maybe_reuse(add->stack, 1); if (err) goto done; @@ -642,7 +757,7 @@ int reftable_addition_add(struct reftable_addition *add, goto done; } } - wr = reftable_new_writer(reftable_fd_write, &tab_fd, + wr = reftable_new_writer(reftable_fd_write, reftable_fd_flush, &tab_fd, &add->stack->config); err = write_table(wr, arg); if (err < 0) @@ -734,7 +849,7 @@ static int stack_compact_locked(struct reftable_stack *st, int first, int last, strbuf_addstr(temp_tab, ".temp.XXXXXX"); tab_fd = mkstemp(temp_tab->buf); - wr = reftable_new_writer(reftable_fd_write, &tab_fd, &st->config); + wr = reftable_new_writer(reftable_fd_write, reftable_fd_flush, &tab_fd, &st->config); err = stack_write_compact(st, wr, first, last, config); if (err < 0) @@ -801,18 +916,16 @@ static int stack_write_compact(struct reftable_stack *st, err = 0; break; } - if (err < 0) { - break; - } + if (err < 0) + goto done; if (first == 0 && reftable_ref_record_is_deletion(&ref)) { continue; } err = reftable_writer_add_ref(wr, &ref); - if (err < 0) { - break; - } + if (err < 0) + goto done; entries++; } reftable_iterator_destroy(&it); @@ -827,9 +940,8 @@ static int stack_write_compact(struct reftable_stack *st, err = 0; break; } - if (err < 0) { - break; - } + if (err < 0) + goto done; if (first == 0 && reftable_log_record_is_deletion(&log)) { continue; } @@ -845,9 +957,8 @@ static int stack_write_compact(struct reftable_stack *st, } err = reftable_writer_add_log(wr, &log); - if (err < 0) { - break; - } + if (err < 0) + goto done; entries++; } diff --git a/reftable/stack.h b/reftable/stack.h index f57005846e..c1e3efa899 100644 --- a/reftable/stack.h +++ b/reftable/stack.h @@ -14,7 +14,10 @@ https://developers.google.com/open-source/licenses/bsd #include "reftable-stack.h" struct reftable_stack { + struct stat list_st; char *list_file; + int list_fd; + char *reftable_dir; int disable_auto_compact; diff --git a/reftable/stack_test.c b/reftable/stack_test.c index 82280c2fd5..289e902146 100644 --- a/reftable/stack_test.c +++ b/reftable/stack_test.c @@ -462,7 +462,6 @@ static void test_reftable_stack_add(void) refs[i].refname = xstrdup(buf); refs[i].update_index = i + 1; refs[i].value_type = REFTABLE_REF_VAL1; - refs[i].value.val1 = reftable_malloc(GIT_SHA1_RAWSZ); set_test_hash(refs[i].value.val1, i); logs[i].refname = xstrdup(buf); @@ -599,7 +598,6 @@ static void test_reftable_stack_tombstone(void) refs[i].update_index = i + 1; if (i % 2 == 0) { refs[i].value_type = REFTABLE_REF_VAL1; - refs[i].value.val1 = reftable_malloc(GIT_SHA1_RAWSZ); set_test_hash(refs[i].value.val1, i); } diff --git a/reftable/test_framework.c b/reftable/test_framework.c index 04044fc1a0..4066924eee 100644 --- a/reftable/test_framework.c +++ b/reftable/test_framework.c @@ -20,3 +20,8 @@ ssize_t strbuf_add_void(void *b, const void *data, size_t sz) strbuf_add(b, data, sz); return sz; } + +int noop_flush(void *arg) +{ + return 0; +} diff --git a/reftable/test_framework.h b/reftable/test_framework.h index ee44f735ae..687390f9c2 100644 --- a/reftable/test_framework.h +++ b/reftable/test_framework.h @@ -56,4 +56,6 @@ void set_test_hash(uint8_t *p, int i); */ ssize_t strbuf_add_void(void *b, const void *data, size_t sz); +int noop_flush(void *); + #endif diff --git a/reftable/writer.c b/reftable/writer.c index 2e322a5683..92935baa70 100644 --- a/reftable/writer.c +++ b/reftable/writer.c @@ -121,6 +121,7 @@ static struct strbuf reftable_empty_strbuf = STRBUF_INIT; struct reftable_writer * reftable_new_writer(ssize_t (*writer_func)(void *, const void *, size_t), + int (*flush_func)(void *), void *writer_arg, struct reftable_write_options *opts) { struct reftable_writer *wp = @@ -136,6 +137,7 @@ reftable_new_writer(ssize_t (*writer_func)(void *, const void *, size_t), wp->write = writer_func; wp->write_arg = writer_arg; wp->opts = *opts; + wp->flush = flush_func; writer_reinit_block_writer(wp, BLOCK_TYPE_REF); return wp; @@ -432,12 +434,12 @@ static int writer_finish_section(struct reftable_writer *w) reftable_free(idx); } - writer_clear_index(w); - err = writer_flush_block(w); if (err < 0) return err; + writer_clear_index(w); + bstats = writer_reftable_block_stats(w, typ); bstats->index_blocks = w->stats.idx_stats.blocks - before_blocks; bstats->index_offset = index_start; @@ -603,6 +605,12 @@ int reftable_writer_close(struct reftable_writer *w) put_be32(p, crc32(0, footer, p - footer)); p += 4; + err = w->flush(w->write_arg); + if (err < 0) { + err = REFTABLE_IO_ERROR; + goto done; + } + err = padded_write(w, footer, footer_size(writer_version(w)), 0); if (err < 0) goto done; diff --git a/reftable/writer.h b/reftable/writer.h index 09b88673d9..8d0df9cc52 100644 --- a/reftable/writer.h +++ b/reftable/writer.h @@ -16,6 +16,7 @@ https://developers.google.com/open-source/licenses/bsd struct reftable_writer { ssize_t (*write)(void *, const void *, size_t); + int (*flush)(void *); void *write_arg; int pending_padding; struct strbuf last_key; diff --git a/remote-curl.c b/remote-curl.c index cb0182b582..1161dc7fed 100644 --- a/remote-curl.c +++ b/remote-curl.c @@ -1446,8 +1446,14 @@ static int stateless_connect(const char *service_name) * establish a stateless connection, otherwise we need to tell the * client to fallback to using other transport helper functions to * complete their request. + * + * The "git-upload-archive" service is a read-only operation. Fallback + * to use "git-upload-pack" service to discover protocol version. */ - discover = discover_refs(service_name, 0); + if (!strcmp(service_name, "git-upload-archive")) + discover = discover_refs("git-upload-pack", 0); + else + discover = discover_refs(service_name, 0); if (discover->version != protocol_v2) { printf("fallback\n"); fflush(stdout); @@ -1485,9 +1491,11 @@ static int stateless_connect(const char *service_name) /* * Dump the capability listing that we got from the server earlier - * during the info/refs request. + * during the info/refs request. This does not work with the + * "git-upload-archive" service. */ - write_or_die(rpc.in, discover->buf, discover->len); + if (strcmp(service_name, "git-upload-archive")) + write_or_die(rpc.in, discover->buf, discover->len); /* Until we see EOF keep sending POSTs */ while (1) { diff --git a/repo-settings.c b/repo-settings.c index 30cd478762..c821583fe5 100644 --- a/repo-settings.c +++ b/repo-settings.c @@ -23,6 +23,7 @@ void prepare_repo_settings(struct repository *r) int value; const char *strval; int manyfiles; + int read_changed_paths; if (!r->gitdir) BUG("Cannot add settings for uninitialized repository"); @@ -53,7 +54,10 @@ void prepare_repo_settings(struct repository *r) /* Commit graph config or default, does not cascade (simple) */ repo_cfg_bool(r, "core.commitgraph", &r->settings.core_commit_graph, 1); repo_cfg_int(r, "commitgraph.generationversion", &r->settings.commit_graph_generation_version, 2); - repo_cfg_bool(r, "commitgraph.readchangedpaths", &r->settings.commit_graph_read_changed_paths, 1); + repo_cfg_bool(r, "commitgraph.readchangedpaths", &read_changed_paths, 1); + repo_cfg_int(r, "commitgraph.changedpathsversion", + &r->settings.commit_graph_changed_paths_version, + read_changed_paths ? -1 : 0); repo_cfg_bool(r, "gc.writecommitgraph", &r->settings.gc_write_commit_graph, 1); repo_cfg_bool(r, "fetch.writecommitgraph", &r->settings.fetch_write_commit_graph, 0); diff --git a/repository.c b/repository.c index a7679ceeaa..e15b416944 100644 --- a/repository.c +++ b/repository.c @@ -14,6 +14,7 @@ #include "read-cache-ll.h" #include "remote.h" #include "setup.h" +#include "loose.h" #include "submodule-config.h" #include "sparse-index.h" #include "trace2.h" @@ -104,6 +105,20 @@ void repo_set_hash_algo(struct repository *repo, int hash_algo) repo->hash_algo = &hash_algos[hash_algo]; } +void repo_set_compat_hash_algo(struct repository *repo, int algo) +{ + if (hash_algo_by_ptr(repo->hash_algo) == algo) + BUG("hash_algo and compat_hash_algo match"); + repo->compat_hash_algo = algo ? &hash_algos[algo] : NULL; + if (repo->compat_hash_algo) + repo_read_loose_object_map(repo); +} + +void repo_set_ref_storage_format(struct repository *repo, unsigned int format) +{ + repo->ref_storage_format = format; +} + /* * Attempt to resolve and set the provided 'gitdir' for repository 'repo'. * Return 0 upon success and a non-zero value upon failure. @@ -184,6 +199,8 @@ int repo_init(struct repository *repo, goto error; repo_set_hash_algo(repo, format.hash_algo); + repo_set_compat_hash_algo(repo, format.compat_hash_algo); + repo_set_ref_storage_format(repo, format.ref_storage_format); repo->repository_format_worktree_config = format.worktree_config; /* take ownership of format.partial_clone */ @@ -193,6 +210,9 @@ int repo_init(struct repository *repo, if (worktree) repo_set_worktree(repo, worktree); + if (repo->compat_hash_algo) + repo_read_loose_object_map(repo); + clear_repository_format(&format); return 0; @@ -256,8 +276,6 @@ static void repo_clear_path_cache(struct repo_path_cache *cache) FREE_AND_NULL(cache->merge_rr); FREE_AND_NULL(cache->merge_mode); FREE_AND_NULL(cache->merge_head); - FREE_AND_NULL(cache->merge_autostash); - FREE_AND_NULL(cache->auto_merge); FREE_AND_NULL(cache->fetch_head); FREE_AND_NULL(cache->shallow); } diff --git a/repository.h b/repository.h index 5f18486f64..d11d073eb7 100644 --- a/repository.h +++ b/repository.h @@ -24,12 +24,15 @@ enum fetch_negotiation_setting { FETCH_NEGOTIATION_NOOP, }; +#define REF_STORAGE_FORMAT_UNKNOWN 0 +#define REF_STORAGE_FORMAT_FILES 1 + struct repo_settings { int initialized; int core_commit_graph; int commit_graph_generation_version; - int commit_graph_read_changed_paths; + int commit_graph_changed_paths_version; int gc_write_commit_graph; int fetch_write_commit_graph; int command_requires_full_index; @@ -64,8 +67,6 @@ struct repo_path_cache { char *merge_rr; char *merge_mode; char *merge_head; - char *merge_autostash; - char *auto_merge; char *fetch_head; char *shallow; }; @@ -160,6 +161,12 @@ struct repository { /* Repository's current hash algorithm, as serialized on disk. */ const struct git_hash_algo *hash_algo; + /* Repository's compatibility hash algorithm. */ + const struct git_hash_algo *compat_hash_algo; + + /* Repository's reference storage format, as serialized on disk. */ + unsigned int ref_storage_format; + /* A unique-id for tracing purposes. */ int trace2_repo_id; @@ -199,6 +206,8 @@ void repo_set_gitdir(struct repository *repo, const char *root, const struct set_gitdir_args *extra_args); void repo_set_worktree(struct repository *repo, const char *path); void repo_set_hash_algo(struct repository *repo, int algo); +void repo_set_compat_hash_algo(struct repository *repo, int compat_algo); +void repo_set_ref_storage_format(struct repository *repo, unsigned int format); void initialize_the_repository(void); RESULT_MUST_BE_USED int repo_init(struct repository *r, const char *gitdir, const char *worktree); @@ -388,12 +388,8 @@ static int handle_conflict(struct strbuf *out, struct rerere_io *io, strbuf_addbuf(out, &two); rerere_strbuf_putconflict(out, '>', marker_size); if (ctx) { - the_hash_algo->update_fn(ctx, one.buf ? - one.buf : "", - one.len + 1); - the_hash_algo->update_fn(ctx, two.buf ? - two.buf : "", - two.len + 1); + the_hash_algo->update_fn(ctx, one.buf, one.len + 1); + the_hash_algo->update_fn(ctx, two.buf, two.len + 1); } break; } else if (hunk == RR_SIDE_1) @@ -452,7 +448,7 @@ static int handle_path(unsigned char *hash, struct rerere_io *io, int marker_siz /* * Scan the path for conflicts, do the "handle_path()" thing above, and - * return the number of conflict hunks found. + * return true iff conflict hunks were found. */ static int handle_file(struct index_state *istate, const char *path, unsigned char *hash, const char *output) @@ -497,9 +493,11 @@ static int handle_file(struct index_state *istate, } /* - * Look at a cache entry at "i" and see if it is not conflicting, - * conflicting and we are willing to handle, or conflicting and - * we are unable to handle, and return the determination in *type. + * Look at a cache entry at "i" and see if it is not conflicting + * (RESOLVED), conflicting and we are willing to handle (THREE_STAGED), + * or conflicting and we are unable to handle (PUNTED), and return the + * determination in *type. + * * Return the cache index to be looked at next, by skipping the * stages we have already looked at in this invocation of this * function. @@ -507,6 +505,7 @@ static int handle_file(struct index_state *istate, static int check_one_conflict(struct index_state *istate, int i, int *type) { const struct cache_entry *e = istate->cache[i]; + unsigned int seen_stages = 0; if (!ce_stage(e)) { *type = RESOLVED; @@ -514,24 +513,17 @@ static int check_one_conflict(struct index_state *istate, int i, int *type) } *type = PUNTED; - while (i < istate->cache_nr && ce_stage(istate->cache[i]) == 1) - i++; - - /* Only handle regular files with both stages #2 and #3 */ - if (i + 1 < istate->cache_nr) { - const struct cache_entry *e2 = istate->cache[i]; - const struct cache_entry *e3 = istate->cache[i + 1]; - if (ce_stage(e2) == 2 && - ce_stage(e3) == 3 && - ce_same_name(e, e3) && - S_ISREG(e2->ce_mode) && - S_ISREG(e3->ce_mode)) - *type = THREE_STAGED; + for (; i < istate->cache_nr; i++) { + const struct cache_entry *n = istate->cache[i]; + if (!ce_same_name(n, e)) + break; + if (S_ISREG(n->ce_mode)) + seen_stages |= 1u << (ce_stage(n) - 1); } - /* Skip the entries with the same name */ - while (i < istate->cache_nr && ce_same_name(e, istate->cache[i])) - i++; + if ((seen_stages & 6) == 6) + *type = THREE_STAGED; /* has both stages #2 and #3 */ + return i; } @@ -619,7 +611,7 @@ static int try_merge(struct index_state *istate, if (read_mmfile(&base, rerere_path(id, "preimage")) || read_mmfile(&other, rerere_path(id, "postimage"))) { - ret = LL_MERGE_CONFLICT; + ret = LL_MERGE_ERROR; } else { /* * A three-way merge. Note that this honors user-customizable diff --git a/revision.c b/revision.c index 2424c9bd67..4e9afd8474 100644 --- a/revision.c +++ b/revision.c @@ -81,7 +81,7 @@ static void mark_tree_contents_uninteresting(struct repository *r, if (parse_tree_gently(tree, 1) < 0) return; - init_tree_desc(&desc, tree->buffer, tree->size); + init_tree_desc(&desc, &tree->object.oid, tree->buffer, tree->size); while (tree_entry(&desc, &entry)) { switch (object_type(entry.mode)) { case OBJ_TREE: @@ -188,7 +188,7 @@ static void add_children_by_path(struct repository *r, if (parse_tree_gently(tree, 1) < 0) return; - init_tree_desc(&desc, tree->buffer, tree->size); + init_tree_desc(&desc, &tree->object.oid, tree->buffer, tree->size); while (tree_entry(&desc, &entry)) { switch (object_type(entry.mode)) { case OBJ_TREE: @@ -833,17 +833,28 @@ static int rev_compare_tree(struct rev_info *revs, return tree_difference; } -static int rev_same_tree_as_empty(struct rev_info *revs, struct commit *commit) +static int rev_same_tree_as_empty(struct rev_info *revs, struct commit *commit, + int nth_parent) { struct tree *t1 = repo_get_commit_tree(the_repository, commit); + int bloom_ret = -1; if (!t1) return 0; + if (!nth_parent && revs->bloom_keys_nr) { + bloom_ret = check_maybe_different_in_bloom_filter(revs, commit); + if (!bloom_ret) + return 1; + } + tree_difference = REV_TREE_SAME; revs->pruning.flags.has_changes = 0; diff_tree_oid(NULL, &t1->object.oid, "", &revs->pruning); + if (bloom_ret == 1 && tree_difference == REV_TREE_SAME) + count_bloom_filter_false_positive++; + return tree_difference == REV_TREE_SAME; } @@ -881,7 +892,7 @@ static int compact_treesame(struct rev_info *revs, struct commit *commit, unsign if (nth_parent != 0) die("compact_treesame %u", nth_parent); old_same = !!(commit->object.flags & TREESAME); - if (rev_same_tree_as_empty(revs, commit)) + if (rev_same_tree_as_empty(revs, commit, nth_parent)) commit->object.flags |= TREESAME; else commit->object.flags &= ~TREESAME; @@ -977,7 +988,14 @@ static void try_to_simplify_commit(struct rev_info *revs, struct commit *commit) return; if (!commit->parents) { - if (rev_same_tree_as_empty(revs, commit)) + /* + * Pretend as if we are comparing ourselves to the + * (non-existent) first parent of this commit object. Even + * though no such parent exists, its changed-path Bloom filter + * (if one exists) is relative to the empty tree, using Bloom + * filters is allowed here. + */ + if (rev_same_tree_as_empty(revs, commit, 0)) commit->object.flags |= TREESAME; return; } @@ -1058,7 +1076,7 @@ static void try_to_simplify_commit(struct rev_info *revs, struct commit *commit) case REV_TREE_NEW: if (revs->remove_empty_trees && - rev_same_tree_as_empty(revs, p)) { + rev_same_tree_as_empty(revs, p, nth_parent)) { /* We are adding all the specified * paths from this parent, so the * history beyond this parent is not @@ -1961,11 +1979,31 @@ static void add_pending_commit_list(struct rev_info *revs, } } +static const char *lookup_other_head(struct object_id *oid) +{ + int i; + static const char *const other_head[] = { + "MERGE_HEAD", "REBASE_HEAD", "CHERRY_PICK_HEAD", "REVERT_HEAD" + }; + + for (i = 0; i < ARRAY_SIZE(other_head); i++) + if (!read_ref_full(other_head[i], + RESOLVE_REF_READING | RESOLVE_REF_NO_RECURSE, + oid, NULL)) { + if (is_null_oid(oid)) + die("%s is a symbolic ref???", other_head[i]); + return other_head[i]; + } + + die("--merge without MERGE_HEAD, REBASE_HEAD, CHERRY_PICK_HEAD or REVERT_HEAD?"); +} + static void prepare_show_merge(struct rev_info *revs) { struct commit_list *bases; struct commit *head, *other; struct object_id oid; + const char *other_name; const char **prune = NULL; int i, prune_num = 1; /* counting terminating NULL */ struct index_state *istate = revs->repo->index; @@ -1973,11 +2011,10 @@ static void prepare_show_merge(struct rev_info *revs) if (repo_get_oid(the_repository, "HEAD", &oid)) die("--merge without HEAD?"); head = lookup_commit_or_die(&oid, "HEAD"); - if (repo_get_oid(the_repository, "MERGE_HEAD", &oid)) - die("--merge without MERGE_HEAD?"); - other = lookup_commit_or_die(&oid, "MERGE_HEAD"); + other_name = lookup_other_head(&oid); + other = lookup_commit_or_die(&oid, other_name); add_pending_object(revs, &head->object, "HEAD"); - add_pending_object(revs, &other->object, "MERGE_HEAD"); + add_pending_object(revs, &other->object, other_name); bases = repo_get_merge_bases(the_repository, head, other); add_rev_cmdline_list(revs, bases, REV_CMD_MERGE_BASE, UNINTERESTING | BOTTOM); add_pending_commit_list(revs, bases, UNINTERESTING | BOTTOM); diff --git a/sequencer.c b/sequencer.c index 3cc88d8a80..4df3e6f59b 100644 --- a/sequencer.c +++ b/sequencer.c @@ -319,37 +319,32 @@ static const char *get_todo_path(const struct replay_opts *opts) static int has_conforming_footer(struct strbuf *sb, struct strbuf *sob, size_t ignore_footer) { - struct process_trailer_options opts = PROCESS_TRAILER_OPTIONS_INIT; - struct trailer_info info; - size_t i; - int found_sob = 0, found_sob_last = 0; - char saved_char; - - opts.no_divider = 1; + struct trailer_iterator iter; + size_t i = 0, found_sob = 0; + char saved_char = sb->buf[sb->len - ignore_footer]; if (ignore_footer) { - saved_char = sb->buf[sb->len - ignore_footer]; sb->buf[sb->len - ignore_footer] = '\0'; } - trailer_info_get(&info, sb->buf, &opts); + trailer_iterator_init(&iter, sb->buf); + while (trailer_iterator_advance(&iter)) { + i++; + if (sob && + iter.is_trailer && + !strncmp(iter.raw.buf, sob->buf, sob->len)) { + found_sob = i; + } + } + trailer_iterator_release(&iter); if (ignore_footer) sb->buf[sb->len - ignore_footer] = saved_char; - if (info.trailer_block_start == info.trailer_block_end) + if (!i) return 0; - for (i = 0; i < info.trailer_nr; i++) - if (sob && !strncmp(info.trailers[i], sob->buf, sob->len)) { - found_sob = 1; - if (i == info.trailer_nr - 1) - found_sob_last = 1; - } - - trailer_info_release(&info); - - if (found_sob_last) + if (found_sob == i) return 3; if (found_sob) return 2; @@ -474,7 +469,7 @@ static void print_advice(struct repository *r, int show_hint, * of the commit itself so remove CHERRY_PICK_HEAD */ refs_delete_ref(get_main_ref_store(r), "", "CHERRY_PICK_HEAD", - NULL, 0); + NULL, REF_NO_DEREF); return; } @@ -1667,7 +1662,7 @@ static int do_commit(struct repository *r, strbuf_release(&sb); if (!res) { refs_delete_ref(get_main_ref_store(r), "", - "CHERRY_PICK_HEAD", NULL, 0); + "CHERRY_PICK_HEAD", NULL, REF_NO_DEREF); unlink(git_path_merge_msg(r)); if (!is_rebase_i(opts)) print_commit_summary(r, NULL, &oid, @@ -2406,9 +2401,10 @@ static int do_pick_commit(struct repository *r, } else if (allow == 2) { drop_commit = 1; refs_delete_ref(get_main_ref_store(r), "", "CHERRY_PICK_HEAD", - NULL, 0); + NULL, REF_NO_DEREF); unlink(git_path_merge_msg(r)); - unlink(git_path_auto_merge(r)); + refs_delete_ref(get_main_ref_store(r), "", "AUTO_MERGE", + NULL, REF_NO_DEREF); fprintf(stderr, _("dropping %s %s -- patch contents already upstream\n"), oid_to_hex(&commit->object.oid), msg.subject); @@ -2802,7 +2798,7 @@ void sequencer_post_commit_cleanup(struct repository *r, int verbose) if (refs_ref_exists(get_main_ref_store(r), "CHERRY_PICK_HEAD")) { if (!refs_delete_ref(get_main_ref_store(r), "", - "CHERRY_PICK_HEAD", NULL, 0) && + "CHERRY_PICK_HEAD", NULL, REF_NO_DEREF) && verbose) warning(_("cancelling a cherry picking in progress")); opts.action = REPLAY_PICK; @@ -2811,14 +2807,15 @@ void sequencer_post_commit_cleanup(struct repository *r, int verbose) if (refs_ref_exists(get_main_ref_store(r), "REVERT_HEAD")) { if (!refs_delete_ref(get_main_ref_store(r), "", "REVERT_HEAD", - NULL, 0) && + NULL, REF_NO_DEREF) && verbose) warning(_("cancelling a revert in progress")); opts.action = REPLAY_REVERT; need_cleanup = 1; } - unlink(git_path_auto_merge(r)); + refs_delete_ref(get_main_ref_store(r), "", "AUTO_MERGE", + NULL, REF_NO_DEREF); if (!need_cleanup) return; @@ -4116,7 +4113,7 @@ static int do_merge(struct repository *r, strbuf_release(&ref_name); refs_delete_ref(get_main_ref_store(r), "", "CHERRY_PICK_HEAD", - NULL, 0); + NULL, REF_NO_DEREF); rollback_lock_file(&lock); ret = run_command(&cmd); @@ -4461,12 +4458,17 @@ static enum todo_command peek_command(struct todo_list *todo_list, int offset) return -1; } -void create_autostash(struct repository *r, const char *path) +static void create_autostash_internal(struct repository *r, + const char *path, + const char *refname) { struct strbuf buf = STRBUF_INIT; struct lock_file lock_file = LOCK_INIT; int fd; + if (path && refname) + BUG("can only pass path or refname"); + fd = repo_hold_locked_index(r, &lock_file, 0); refresh_index(r->index, REFRESH_QUIET, NULL, NULL, NULL); if (0 <= fd) @@ -4493,10 +4495,16 @@ void create_autostash(struct repository *r, const char *path) strbuf_reset(&buf); strbuf_add_unique_abbrev(&buf, &oid, DEFAULT_ABBREV); - if (safe_create_leading_directories_const(path)) - die(_("Could not create directory for '%s'"), - path); - write_file(path, "%s", oid_to_hex(&oid)); + if (path) { + if (safe_create_leading_directories_const(path)) + die(_("Could not create directory for '%s'"), + path); + write_file(path, "%s", oid_to_hex(&oid)); + } else { + refs_update_ref(get_main_ref_store(r), "", refname, + &oid, null_oid(), 0, UPDATE_REFS_DIE_ON_ERR); + } + printf(_("Created autostash: %s\n"), buf.buf); if (reset_head(r, &ropts) < 0) die(_("could not reset --hard")); @@ -4507,6 +4515,16 @@ void create_autostash(struct repository *r, const char *path) strbuf_release(&buf); } +void create_autostash(struct repository *r, const char *path) +{ + create_autostash_internal(r, path, NULL); +} + +void create_autostash_ref(struct repository *r, const char *refname) +{ + create_autostash_internal(r, NULL, refname); +} + static int apply_save_autostash_oid(const char *stash_oid, int attempt_apply) { struct child_process child = CHILD_PROCESS_INIT; @@ -4584,6 +4602,41 @@ int apply_autostash_oid(const char *stash_oid) return apply_save_autostash_oid(stash_oid, 1); } +static int apply_save_autostash_ref(struct repository *r, const char *refname, + int attempt_apply) +{ + struct object_id stash_oid; + char stash_oid_hex[GIT_MAX_HEXSZ + 1]; + int flag, ret; + + if (!refs_ref_exists(get_main_ref_store(r), refname)) + return 0; + + if (!refs_resolve_ref_unsafe(get_main_ref_store(r), refname, + RESOLVE_REF_READING, &stash_oid, &flag)) + return -1; + if (flag & REF_ISSYMREF) + return error(_("autostash reference is a symref")); + + oid_to_hex_r(stash_oid_hex, &stash_oid); + ret = apply_save_autostash_oid(stash_oid_hex, attempt_apply); + + refs_delete_ref(get_main_ref_store(r), "", refname, + &stash_oid, REF_NO_DEREF); + + return ret; +} + +int save_autostash_ref(struct repository *r, const char *refname) +{ + return apply_save_autostash_ref(r, refname, 0); +} + +int apply_autostash_ref(struct repository *r, const char *refname) +{ + return apply_save_autostash_ref(r, refname, 1); +} + static int checkout_onto(struct repository *r, struct replay_opts *opts, const char *onto_name, const struct object_id *onto, const struct object_id *orig_head) @@ -4766,8 +4819,10 @@ static int pick_commits(struct repository *r, } unlink(rebase_path_author_script()); unlink(git_path_merge_head(r)); - unlink(git_path_auto_merge(r)); - delete_ref(NULL, "REBASE_HEAD", NULL, REF_NO_DEREF); + refs_delete_ref(get_main_ref_store(r), "", "AUTO_MERGE", + NULL, REF_NO_DEREF); + refs_delete_ref(get_main_ref_store(r), "", "REBASE_HEAD", + NULL, REF_NO_DEREF); if (item->command == TODO_BREAK) { if (!opts->verbose) @@ -5108,7 +5163,7 @@ static int commit_staged_changes(struct repository *r, if (refs_ref_exists(get_main_ref_store(r), "CHERRY_PICK_HEAD") && refs_delete_ref(get_main_ref_store(r), "", - "CHERRY_PICK_HEAD", NULL, 0)) + "CHERRY_PICK_HEAD", NULL, REF_NO_DEREF)) return error(_("could not remove CHERRY_PICK_HEAD")); if (unlink(git_path_merge_msg(r)) && errno != ENOENT) return error_errno(_("could not remove '%s'"), @@ -5122,7 +5177,8 @@ static int commit_staged_changes(struct repository *r, return error(_("could not commit staged changes.")); unlink(rebase_path_amend()); unlink(git_path_merge_head(r)); - unlink(git_path_auto_merge(r)); + refs_delete_ref(get_main_ref_store(r), "", "AUTO_MERGE", + NULL, REF_NO_DEREF); if (final_fixup) { unlink(rebase_path_fixup_msg()); unlink(rebase_path_squash_msg()); diff --git a/sequencer.h b/sequencer.h index 913a0f652d..dcef7bb99c 100644 --- a/sequencer.h +++ b/sequencer.h @@ -225,9 +225,12 @@ void commit_post_rewrite(struct repository *r, const struct object_id *new_head); void create_autostash(struct repository *r, const char *path); +void create_autostash_ref(struct repository *r, const char *refname); int save_autostash(const char *path); +int save_autostash_ref(struct repository *r, const char *refname); int apply_autostash(const char *path); int apply_autostash_oid(const char *stash_oid); +int apply_autostash_ref(struct repository *r, const char *refname); #define SUMMARY_INITIAL_COMMIT (1 << 0) #define SUMMARY_SHOW_AUTHOR_DATE (1 << 1) @@ -591,6 +591,36 @@ static enum extension_result handle_extension(const char *var, "extensions.objectformat", value); data->hash_algo = format; return EXTENSION_OK; + } else if (!strcmp(ext, "compatobjectformat")) { + struct string_list_item *item; + int format; + + if (!value) + return config_error_nonbool(var); + format = hash_algo_by_name(value); + if (format == GIT_HASH_UNKNOWN) + return error(_("invalid value for '%s': '%s'"), + "extensions.compatobjectformat", value); + /* For now only support compatObjectFormat being specified once. */ + for_each_string_list_item(item, &data->v1_only_extensions) { + if (!strcmp(item->string, "compatobjectformat")) + return error(_("'%s' already specified as '%s'"), + "extensions.compatobjectformat", + hash_algos[data->compat_hash_algo].name); + } + data->compat_hash_algo = format; + return EXTENSION_OK; + } else if (!strcmp(ext, "refstorage")) { + unsigned int format; + + if (!value) + return config_error_nonbool(var); + format = ref_storage_format_by_name(value); + if (format == REF_STORAGE_FORMAT_UNKNOWN) + return error(_("invalid value for '%s': '%s'"), + "extensions.refstorage", value); + data->ref_storage_format = format; + return EXTENSION_OK; } return EXTENSION_UNKNOWN; } @@ -1565,6 +1595,10 @@ const char *setup_git_directory_gently(int *nongit_ok) } if (startup_info->have_repository) { repo_set_hash_algo(the_repository, repo_fmt.hash_algo); + repo_set_compat_hash_algo(the_repository, + repo_fmt.compat_hash_algo); + repo_set_ref_storage_format(the_repository, + repo_fmt.ref_storage_format); the_repository->repository_format_worktree_config = repo_fmt.worktree_config; /* take ownership of repo_fmt.partial_clone */ @@ -1658,6 +1692,9 @@ void check_repository_format(struct repository_format *fmt) check_repository_format_gently(get_git_dir(), fmt, NULL); startup_info->have_repository = 1; repo_set_hash_algo(the_repository, fmt->hash_algo); + repo_set_compat_hash_algo(the_repository, fmt->compat_hash_algo); + repo_set_ref_storage_format(the_repository, + fmt->ref_storage_format); the_repository->repository_format_worktree_config = fmt->worktree_config; the_repository->repository_format_partial_clone = @@ -1866,12 +1903,15 @@ static int needs_work_tree_config(const char *git_dir, const char *work_tree) return 1; } -void initialize_repository_version(int hash_algo, int reinit) +void initialize_repository_version(int hash_algo, + unsigned int ref_storage_format, + int reinit) { char repo_version_string[10]; int repo_version = GIT_REPO_VERSION; - if (hash_algo != GIT_HASH_SHA1) + if (hash_algo != GIT_HASH_SHA1 || + ref_storage_format != REF_STORAGE_FORMAT_FILES) repo_version = GIT_REPO_VERSION_READ; /* This forces creation of new config file */ @@ -1884,6 +1924,10 @@ void initialize_repository_version(int hash_algo, int reinit) hash_algos[hash_algo].name); else if (reinit) git_config_set_gently("extensions.objectformat", NULL); + + if (ref_storage_format != REF_STORAGE_FORMAT_FILES) + git_config_set("extensions.refstorage", + ref_storage_format_to_name(ref_storage_format)); } static int is_reinit(void) @@ -1898,27 +1942,14 @@ static int is_reinit(void) return ret; } -void create_reference_database(const char *initial_branch, int quiet) +void create_reference_database(unsigned int ref_storage_format, + const char *initial_branch, int quiet) { struct strbuf err = STRBUF_INIT; int reinit = is_reinit(); - /* - * We need to create a "refs" dir in any case so that older versions of - * Git can tell that this is a repository. This serves two main purposes: - * - * - Clients will know to stop walking the parent-directory chain when - * detecting the Git repository. Otherwise they may end up detecting - * a Git repository in a parent directory instead. - * - * - Instead of failing to detect a repository with unknown reference - * format altogether, old clients will print an error saying that - * they do not understand the reference format extension. - */ - safe_create_dir(git_path("refs"), 1); - adjust_shared_perm(git_path("refs")); - - if (refs_init_db(&err)) + repo_set_ref_storage_format(the_repository, ref_storage_format); + if (refs_init_db(get_main_ref_store(the_repository), 0, &err)) die("failed to set up refs db: %s", err.buf); /* @@ -2023,7 +2054,7 @@ static int create_default_files(const char *template_path, adjust_shared_perm(get_git_dir()); } - initialize_repository_version(fmt->hash_algo, 0); + initialize_repository_version(fmt->hash_algo, fmt->ref_storage_format, 0); /* Check filemode trustability */ path = git_path_buf(&buf, "config"); @@ -2136,8 +2167,29 @@ static void validate_hash_algorithm(struct repository_format *repo_fmt, int hash } } +static void validate_ref_storage_format(struct repository_format *repo_fmt, + unsigned int format) +{ + const char *name = getenv("GIT_DEFAULT_REF_FORMAT"); + + if (repo_fmt->version >= 0 && + format != REF_STORAGE_FORMAT_UNKNOWN && + format != repo_fmt->ref_storage_format) { + die(_("attempt to reinitialize repository with different reference storage format")); + } else if (format != REF_STORAGE_FORMAT_UNKNOWN) { + repo_fmt->ref_storage_format = format; + } else if (name) { + format = ref_storage_format_by_name(name); + if (format == REF_STORAGE_FORMAT_UNKNOWN) + die(_("unknown ref storage format '%s'"), name); + repo_fmt->ref_storage_format = format; + } +} + int init_db(const char *git_dir, const char *real_git_dir, - const char *template_dir, int hash, const char *initial_branch, + const char *template_dir, int hash, + unsigned int ref_storage_format, + const char *initial_branch, int init_shared_repository, unsigned int flags) { int reinit; @@ -2180,13 +2232,22 @@ int init_db(const char *git_dir, const char *real_git_dir, check_repository_format(&repo_fmt); validate_hash_algorithm(&repo_fmt, hash); + validate_ref_storage_format(&repo_fmt, ref_storage_format); reinit = create_default_files(template_dir, original_git_dir, &repo_fmt, prev_bare_repository, init_shared_repository); + /* + * Now that we have set up both the hash algorithm and the ref storage + * format we can update the repository's settings accordingly. + */ + repo_set_hash_algo(the_repository, repo_fmt.hash_algo); + repo_set_ref_storage_format(the_repository, repo_fmt.ref_storage_format); + if (!(flags & INIT_DB_SKIP_REFDB)) - create_reference_database(initial_branch, flags & INIT_DB_QUIET); + create_reference_database(repo_fmt.ref_storage_format, + initial_branch, flags & INIT_DB_QUIET); create_object_directory(); if (get_shared_repository()) { @@ -115,6 +115,8 @@ struct repository_format { int worktree_config; int is_bare; int hash_algo; + int compat_hash_algo; + unsigned int ref_storage_format; int sparse_index; char *work_tree; struct string_list unknown_extensions; @@ -131,6 +133,7 @@ struct repository_format { .version = -1, \ .is_bare = -1, \ .hash_algo = GIT_HASH_SHA1, \ + .ref_storage_format = REF_STORAGE_FORMAT_FILES, \ .unknown_extensions = STRING_LIST_INIT_DUP, \ .v1_only_extensions = STRING_LIST_INIT_DUP, \ } @@ -175,10 +178,14 @@ void check_repository_format(struct repository_format *fmt); int init_db(const char *git_dir, const char *real_git_dir, const char *template_dir, int hash_algo, + unsigned int ref_storage_format, const char *initial_branch, int init_shared_repository, unsigned int flags); -void initialize_repository_version(int hash_algo, int reinit); -void create_reference_database(const char *initial_branch, int quiet); +void initialize_repository_version(int hash_algo, + unsigned int ref_storage_format, + int reinit); +void create_reference_database(unsigned int ref_storage_format, + const char *initial_branch, int quiet); /* * NOTE NOTE NOTE!! diff --git a/sideband.c b/sideband.c index 6cbfd391c4..266a67342b 100644 --- a/sideband.c +++ b/sideband.c @@ -69,7 +69,10 @@ void list_config_color_sideband_slots(struct string_list *list, const char *pref * of the line. This should be called for a single line only, which is * passed as the first N characters of the SRC array. * - * NEEDSWORK: use "size_t n" instead for clarity. + * It is fine to use "int n" here instead of "size_t n" as all calls to this + * function pass an 'int' parameter. Additionally, the buffer involved in + * storing these 'int' values takes input from a packet via the pkt-line + * interface, which is capable of transferring only 64kB at a time. */ static void maybe_colorize_sideband(struct strbuf *dest, const char *src, int n) { @@ -4,8 +4,8 @@ /** * The strvec API allows one to dynamically build and store * NULL-terminated arrays of strings. A strvec maintains the invariant that the - * `items` member always points to a non-NULL array, and that the array is - * always NULL-terminated at the element pointed to by `items[nr]`. This + * `v` member always points to a non-NULL array, and that the array is + * always NULL-terminated at the element pointed to by `v[nr]`. This * makes the result suitable for passing to functions expecting to receive * argv from main(). * @@ -22,7 +22,7 @@ extern const char *empty_strvec[]; /** * A single array. This should be initialized by assignment from - * `STRVEC_INIT`, or by calling `strvec_init`. The `items` + * `STRVEC_INIT`, or by calling `strvec_init`. The `v` * member contains the actual array; the `nr` member contains the * number of elements in the array, not including the terminating * NULL. @@ -80,7 +80,7 @@ void strvec_split(struct strvec *, const char *); void strvec_clear(struct strvec *); /** - * Disconnect the `items` member from the `strvec` struct and + * Disconnect the `v` member from the `strvec` struct and * return it. The caller is responsible for freeing the memory used * by the array, and by the strings it references. After detaching, * the `strvec` is in a reinitialized state and can be pushed diff --git a/submodule-config.c b/submodule-config.c index f4dd482abc..54130f6a38 100644 --- a/submodule-config.c +++ b/submodule-config.c @@ -14,6 +14,8 @@ #include "parse-options.h" #include "thread-utils.h" #include "tree-walk.h" +#include "url.h" +#include "urlmatch.h" /* * submodule cache lookup structure @@ -228,6 +230,144 @@ in_component: return 0; } +static int starts_with_dot_slash(const char *const path) +{ + return path_match_flags(path, PATH_MATCH_STARTS_WITH_DOT_SLASH | + PATH_MATCH_XPLATFORM); +} + +static int starts_with_dot_dot_slash(const char *const path) +{ + return path_match_flags(path, PATH_MATCH_STARTS_WITH_DOT_DOT_SLASH | + PATH_MATCH_XPLATFORM); +} + +static int submodule_url_is_relative(const char *url) +{ + return starts_with_dot_slash(url) || starts_with_dot_dot_slash(url); +} + +/* + * Count directory components that a relative submodule URL should chop + * from the remote_url it is to be resolved against. + * + * In other words, this counts "../" components at the start of a + * submodule URL. + * + * Returns the number of directory components to chop and writes a + * pointer to the next character of url after all leading "./" and + * "../" components to out. + */ +static int count_leading_dotdots(const char *url, const char **out) +{ + int result = 0; + while (1) { + if (starts_with_dot_dot_slash(url)) { + result++; + url += strlen("../"); + continue; + } + if (starts_with_dot_slash(url)) { + url += strlen("./"); + continue; + } + *out = url; + return result; + } +} +/* + * Check whether a transport is implemented by git-remote-curl. + * + * If it is, returns 1 and writes the URL that would be passed to + * git-remote-curl to the "out" parameter. + * + * Otherwise, returns 0 and leaves "out" untouched. + * + * Examples: + * http::https://example.com/repo.git -> 1, https://example.com/repo.git + * https://example.com/repo.git -> 1, https://example.com/repo.git + * git://example.com/repo.git -> 0 + * + * This is for use in checking for previously exploitable bugs that + * required a submodule URL to be passed to git-remote-curl. + */ +static int url_to_curl_url(const char *url, const char **out) +{ + /* + * We don't need to check for case-aliases, "http.exe", and so + * on because in the default configuration, is_transport_allowed + * prevents URLs with those schemes from being cloned + * automatically. + */ + if (skip_prefix(url, "http::", out) || + skip_prefix(url, "https::", out) || + skip_prefix(url, "ftp::", out) || + skip_prefix(url, "ftps::", out)) + return 1; + if (starts_with(url, "http://") || + starts_with(url, "https://") || + starts_with(url, "ftp://") || + starts_with(url, "ftps://")) { + *out = url; + return 1; + } + return 0; +} + +int check_submodule_url(const char *url) +{ + const char *curl_url; + + if (looks_like_command_line_option(url)) + return -1; + + if (submodule_url_is_relative(url) || starts_with(url, "git://")) { + char *decoded; + const char *next; + int has_nl; + + /* + * This could be appended to an http URL and url-decoded; + * check for malicious characters. + */ + decoded = url_decode(url); + has_nl = !!strchr(decoded, '\n'); + + free(decoded); + if (has_nl) + return -1; + + /* + * URLs which escape their root via "../" can overwrite + * the host field and previous components, resolving to + * URLs like https::example.com/submodule.git and + * https:///example.com/submodule.git that were + * susceptible to CVE-2020-11008. + */ + if (count_leading_dotdots(url, &next) > 0 && + (*next == ':' || *next == '/')) + return -1; + } + + else if (url_to_curl_url(url, &curl_url)) { + int ret = 0; + char *normalized = url_normalize(curl_url, NULL); + if (normalized) { + char *decoded = url_decode(normalized); + if (strchr(decoded, '\n')) + ret = -1; + free(normalized); + free(decoded); + } else { + ret = -1; + } + + return ret; + } + + return 0; +} + static int name_and_item_from_var(const char *var, struct strbuf *name, struct strbuf *item) { diff --git a/submodule-config.h b/submodule-config.h index 958f320ac6..b6133af71b 100644 --- a/submodule-config.h +++ b/submodule-config.h @@ -89,6 +89,9 @@ int config_set_in_gitmodules_file_gently(const char *key, const char *value); */ int check_submodule_name(const char *name); +/* Returns 0 if the URL valid per RFC3986 and -1 otherwise. */ +int check_submodule_url(const char *url); + /* * Note: these helper functions exist solely to maintain backward * compatibility with 'fetch' and 'update_clone' storing configuration in @@ -479,6 +479,9 @@ GIT_TEST_DEFAULT_HASH=<hash-algo> specifies which hash algorithm to use in the test scripts. Recognized values for <hash-algo> are "sha1" and "sha256". +GIT_TEST_DEFAULT_REF_FORMAT=<format> specifies which ref storage format +to use in the test scripts. Recognized values for <format> are "files". + GIT_TEST_NO_WRITE_REV_INDEX=<boolean>, when true disables the 'pack.writeReverseIndex' setting. diff --git a/t/helper/test-bloom.c b/t/helper/test-bloom.c index 1281e66876..eefc1668c7 100644 --- a/t/helper/test-bloom.c +++ b/t/helper/test-bloom.c @@ -49,6 +49,7 @@ static void get_bloom_filter_for_commit(const struct object_id *commit_oid) static const char *bloom_usage = "\n" " test-tool bloom get_murmur3 <string>\n" +" test-tool bloom get_murmur3_seven_highbit\n" " test-tool bloom generate_filter <string> [<string>...]\n" " test-tool bloom get_filter_for_commit <commit-hex>\n"; @@ -63,7 +64,13 @@ int cmd__bloom(int argc, const char **argv) uint32_t hashed; if (argc < 3) usage(bloom_usage); - hashed = murmur3_seeded(0, argv[2], strlen(argv[2])); + hashed = murmur3_seeded_v2(0, argv[2], strlen(argv[2])); + printf("Murmur3 Hash with seed=0:0x%08x\n", hashed); + } + + if (!strcmp(argv[1], "get_murmur3_seven_highbit")) { + uint32_t hashed; + hashed = murmur3_seeded_v2(0, "\x99\xaa\xbb\xcc\xdd\xee\xff", 7); printf("Murmur3 Hash with seed=0:0x%08x\n", hashed); } diff --git a/t/helper/test-ctype.c b/t/helper/test-ctype.c deleted file mode 100644 index e5659df40b..0000000000 --- a/t/helper/test-ctype.c +++ /dev/null @@ -1,70 +0,0 @@ -#include "test-tool.h" - -static int rc; - -static void report_error(const char *class, int ch) -{ - printf("%s classifies char %d (0x%02x) wrongly\n", class, ch, ch); - rc = 1; -} - -static int is_in(const char *s, int ch) -{ - /* - * We can't find NUL using strchr. Accept it as the first - * character in the spec -- there are no empty classes. - */ - if (ch == '\0') - return ch == *s; - if (*s == '\0') - s++; - return !!strchr(s, ch); -} - -#define TEST_CLASS(t,s) { \ - int i; \ - for (i = 0; i < 256; i++) { \ - if (is_in(s, i) != t(i)) \ - report_error(#t, i); \ - } \ - if (t(EOF)) \ - report_error(#t, EOF); \ -} - -#define DIGIT "0123456789" -#define LOWER "abcdefghijklmnopqrstuvwxyz" -#define UPPER "ABCDEFGHIJKLMNOPQRSTUVWXYZ" -#define PUNCT "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~" -#define ASCII \ - "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" \ - "\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f" \ - "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f" \ - "\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f" \ - "\x40\x41\x42\x43\x44\x45\x46\x47\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f" \ - "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f" \ - "\x60\x61\x62\x63\x64\x65\x66\x67\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f" \ - "\x70\x71\x72\x73\x74\x75\x76\x77\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f" -#define CNTRL \ - "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" \ - "\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f" \ - "\x7f" - -int cmd__ctype(int argc UNUSED, const char **argv UNUSED) -{ - TEST_CLASS(isdigit, DIGIT); - TEST_CLASS(isspace, " \n\r\t"); - TEST_CLASS(isalpha, LOWER UPPER); - TEST_CLASS(isalnum, LOWER UPPER DIGIT); - TEST_CLASS(is_glob_special, "*?[\\"); - TEST_CLASS(is_regex_special, "$()*+.?[\\^{|"); - TEST_CLASS(is_pathspec_magic, "!\"#%&',-/:;<=>@_`~"); - TEST_CLASS(isascii, ASCII); - TEST_CLASS(islower, LOWER); - TEST_CLASS(isupper, UPPER); - TEST_CLASS(iscntrl, CNTRL); - TEST_CLASS(ispunct, PUNCT); - TEST_CLASS(isxdigit, DIGIT "abcdefABCDEF"); - TEST_CLASS(isprint, LOWER UPPER DIGIT PUNCT " "); - - return rc; -} diff --git a/t/helper/test-delete-gpgsig.c b/t/helper/test-delete-gpgsig.c new file mode 100644 index 0000000000..e36831af03 --- /dev/null +++ b/t/helper/test-delete-gpgsig.c @@ -0,0 +1,62 @@ +#include "test-tool.h" +#include "gpg-interface.h" +#include "strbuf.h" + + +int cmd__delete_gpgsig(int argc, const char **argv) +{ + struct strbuf buf = STRBUF_INIT; + const char *pattern = "gpgsig"; + const char *bufptr, *tail, *eol; + int deleting = 0; + size_t plen; + + if (argc >= 2) { + pattern = argv[1]; + argv++; + argc--; + } + + plen = strlen(pattern); + strbuf_read(&buf, 0, 0); + + if (!strcmp(pattern, "trailer")) { + size_t payload_size = parse_signed_buffer(buf.buf, buf.len); + fwrite(buf.buf, 1, payload_size, stdout); + fflush(stdout); + return 0; + } + + bufptr = buf.buf; + tail = bufptr + buf.len; + + while (bufptr < tail) { + /* Find the end of the line */ + eol = memchr(bufptr, '\n', tail - bufptr); + if (!eol) + eol = tail; + + /* Drop continuation lines */ + if (deleting && (bufptr < eol) && (bufptr[0] == ' ')) { + bufptr = eol + 1; + continue; + } + deleting = 0; + + /* Does the line match the prefix? */ + if (((bufptr + plen) < eol) && + !memcmp(bufptr, pattern, plen) && + (bufptr[plen] == ' ')) { + deleting = 1; + bufptr = eol + 1; + continue; + } + + /* Print all other lines */ + fwrite(bufptr, 1, (eol - bufptr) + 1, stdout); + bufptr = eol + 1; + } + fflush(stdout); + + return 0; +} diff --git a/t/helper/test-pkt-line.c b/t/helper/test-pkt-line.c index 77e99c37df..4daa82f00f 100644 --- a/t/helper/test-pkt-line.c +++ b/t/helper/test-pkt-line.c @@ -3,6 +3,7 @@ #include "pkt-line.h" #include "sideband.h" #include "write-or-die.h" +#include "parse-options.h" static void pack_line(const char *line) { @@ -65,12 +66,33 @@ static void unpack(void) } } -static void unpack_sideband(void) +static void unpack_sideband(int argc, const char **argv) { struct packet_reader reader; - packet_reader_init(&reader, 0, NULL, 0, - PACKET_READ_GENTLE_ON_EOF | - PACKET_READ_CHOMP_NEWLINE); + int options = PACKET_READ_GENTLE_ON_EOF; + int chomp_newline = 1; + int reader_use_sideband = 0; + const char *const unpack_sideband_usage[] = { + "test_tool unpack_sideband [options...]", NULL + }; + struct option cmd_options[] = { + OPT_BOOL(0, "reader-use-sideband", &reader_use_sideband, + "set use_sideband bit for packet reader (Default: off)"), + OPT_BOOL(0, "chomp-newline", &chomp_newline, + "chomp newline in packet (Default: on)"), + OPT_END() + }; + + argc = parse_options(argc, argv, "", cmd_options, unpack_sideband_usage, + 0); + if (argc > 0) + usage_msg_opt(_("too many arguments"), unpack_sideband_usage, + cmd_options); + + if (chomp_newline) + options |= PACKET_READ_CHOMP_NEWLINE; + packet_reader_init(&reader, 0, NULL, 0, options); + reader.use_sideband = reader_use_sideband; while (packet_reader_read(&reader) != PACKET_READ_EOF) { int band; @@ -80,6 +102,17 @@ static void unpack_sideband(void) case PACKET_READ_EOF: break; case PACKET_READ_NORMAL: + /* + * When the "use_sideband" field of the reader is turned + * on, sideband packets other than the payload have been + * parsed and consumed in packet_reader_read(), and only + * the payload arrives here. + */ + if (reader.use_sideband) { + write_or_die(1, reader.line, reader.pktlen - 1); + break; + } + band = reader.line[0] & 0xff; if (band < 1 || band > 2) continue; /* skip non-sideband packets */ @@ -98,15 +131,31 @@ static void unpack_sideband(void) static int send_split_sideband(void) { + const char *foo = "Foo.\n"; + const char *bar = "Bar.\n"; const char *part1 = "Hello,"; const char *primary = "\001primary: regular output\n"; const char *part2 = " world!\n"; + /* Each sideband message has a trailing newline character. */ + send_sideband(1, 2, foo, strlen(foo), LARGE_PACKET_MAX); + send_sideband(1, 2, bar, strlen(bar), LARGE_PACKET_MAX); + + /* + * One sideband message is divided into part1 and part2 + * by the primary message. + */ send_sideband(1, 2, part1, strlen(part1), LARGE_PACKET_MAX); packet_write(1, primary, strlen(primary)); send_sideband(1, 2, part2, strlen(part2), LARGE_PACKET_MAX); packet_response_end(1); + /* + * We use unpack_sideband() to consume packets. A flush packet + * is required to end parsing. + */ + packet_flush(1); + return 0; } @@ -127,7 +176,7 @@ int cmd__pkt_line(int argc, const char **argv) else if (!strcmp(argv[1], "unpack")) unpack(); else if (!strcmp(argv[1], "unpack-sideband")) - unpack_sideband(); + unpack_sideband(argc - 1, argv + 1); else if (!strcmp(argv[1], "send-split-sideband")) send_split_sideband(); else if (!strcmp(argv[1], "receive-sideband")) diff --git a/t/helper/test-prio-queue.c b/t/helper/test-prio-queue.c deleted file mode 100644 index f0bf255f5f..0000000000 --- a/t/helper/test-prio-queue.c +++ /dev/null @@ -1,51 +0,0 @@ -#include "test-tool.h" -#include "prio-queue.h" - -static int intcmp(const void *va, const void *vb, void *data UNUSED) -{ - const int *a = va, *b = vb; - return *a - *b; -} - -static void show(int *v) -{ - if (!v) - printf("NULL\n"); - else - printf("%d\n", *v); - free(v); -} - -int cmd__prio_queue(int argc UNUSED, const char **argv) -{ - struct prio_queue pq = { intcmp }; - - while (*++argv) { - if (!strcmp(*argv, "get")) { - void *peek = prio_queue_peek(&pq); - void *get = prio_queue_get(&pq); - if (peek != get) - BUG("peek and get results do not match"); - show(get); - } else if (!strcmp(*argv, "dump")) { - void *peek; - void *get; - while ((peek = prio_queue_peek(&pq))) { - get = prio_queue_get(&pq); - if (peek != get) - BUG("peek and get results do not match"); - show(get); - } - } else if (!strcmp(*argv, "stack")) { - pq.compare = NULL; - } else { - int *v = xmalloc(sizeof(*v)); - *v = atoi(*argv); - prio_queue_put(&pq, v); - } - } - - clear_prio_queue(&pq); - - return 0; -} diff --git a/t/helper/test-read-graph.c b/t/helper/test-read-graph.c index 8c7a83f578..899b5f41cc 100644 --- a/t/helper/test-read-graph.c +++ b/t/helper/test-read-graph.c @@ -5,20 +5,8 @@ #include "bloom.h" #include "setup.h" -int cmd__read_graph(int argc UNUSED, const char **argv UNUSED) +static void dump_graph_info(struct commit_graph *graph) { - struct commit_graph *graph = NULL; - struct object_directory *odb; - - setup_git_directory(); - odb = the_repository->objects->odb; - - prepare_repo_settings(the_repository); - - graph = read_commit_graph_one(the_repository, odb); - if (!graph) - return 1; - printf("header: %08x %d %d %d %d\n", ntohl(*(uint32_t*)graph->data), *(unsigned char*)(graph->data + 4), @@ -57,8 +45,57 @@ int cmd__read_graph(int argc UNUSED, const char **argv UNUSED) if (graph->topo_levels) printf(" topo_levels"); printf("\n"); +} + +static void dump_graph_bloom_filters(struct commit_graph *graph) +{ + uint32_t i; + + for (i = 0; i < graph->num_commits + graph->num_commits_in_base; i++) { + struct bloom_filter filter = { 0 }; + size_t j; + + if (load_bloom_filter_from_graph(graph, &filter, i) < 0) { + fprintf(stderr, "missing Bloom filter for graph " + "position %"PRIu32"\n", i); + continue; + } + + for (j = 0; j < filter.len; j++) + printf("%02x", filter.data[j]); + if (filter.len) + printf("\n"); + } +} + +int cmd__read_graph(int argc, const char **argv) +{ + struct commit_graph *graph = NULL; + struct object_directory *odb; + int ret = 0; + + setup_git_directory(); + odb = the_repository->objects->odb; + + prepare_repo_settings(the_repository); + + graph = read_commit_graph_one(the_repository, odb); + if (!graph) { + ret = 1; + goto done; + } + + if (argc <= 1) + dump_graph_info(graph); + else if (!strcmp(argv[1], "bloom-filters")) + dump_graph_bloom_filters(graph); + else { + fprintf(stderr, "unknown sub-command: '%s'\n", argv[1]); + ret = 1; + } +done: UNLEAK(graph); - return 0; + return ret; } diff --git a/t/helper/test-read-midx.c b/t/helper/test-read-midx.c index e9a444ddba..4acae41bb9 100644 --- a/t/helper/test-read-midx.c +++ b/t/helper/test-read-midx.c @@ -6,6 +6,7 @@ #include "pack-bitmap.h" #include "packfile.h" #include "setup.h" +#include "gettext.h" static int read_midx_file(const char *object_dir, int show_objects) { @@ -79,7 +80,7 @@ static int read_midx_checksum(const char *object_dir) static int read_midx_preferred_pack(const char *object_dir) { struct multi_pack_index *midx = NULL; - struct bitmap_index *bitmap = NULL; + uint32_t preferred_pack; setup_git_directory(); @@ -87,23 +88,45 @@ static int read_midx_preferred_pack(const char *object_dir) if (!midx) return 1; - bitmap = prepare_bitmap_git(the_repository); - if (!bitmap) + if (midx_preferred_pack(midx, &preferred_pack) < 0) { + warning(_("could not determine MIDX preferred pack")); return 1; - if (!bitmap_is_midx(bitmap)) { - free_bitmap_index(bitmap); + } + + printf("%s\n", midx->pack_names[preferred_pack]); + return 0; +} + +static int read_midx_bitmapped_packs(const char *object_dir) +{ + struct multi_pack_index *midx = NULL; + struct bitmapped_pack pack; + uint32_t i; + + setup_git_directory(); + + midx = load_multi_pack_index(object_dir, 1); + if (!midx) return 1; + + for (i = 0; i < midx->num_packs; i++) { + if (nth_bitmapped_pack(the_repository, midx, &pack, i) < 0) + return 1; + + printf("%s\n", pack_basename(pack.p)); + printf(" bitmap_pos: %"PRIuMAX"\n", (uintmax_t)pack.bitmap_pos); + printf(" bitmap_nr: %"PRIuMAX"\n", (uintmax_t)pack.bitmap_nr); } - printf("%s\n", midx->pack_names[midx_preferred_pack(bitmap)]); - free_bitmap_index(bitmap); + close_midx(midx); + return 0; } int cmd__read_midx(int argc, const char **argv) { if (!(argc == 2 || argc == 3)) - usage("read-midx [--show-objects|--checksum|--preferred-pack] <object-dir>"); + usage("read-midx [--show-objects|--checksum|--preferred-pack|--bitmap] <object-dir>"); if (!strcmp(argv[1], "--show-objects")) return read_midx_file(argv[2], 1); @@ -111,5 +134,7 @@ int cmd__read_midx(int argc, const char **argv) return read_midx_checksum(argv[2]); else if (!strcmp(argv[1], "--preferred-pack")) return read_midx_preferred_pack(argv[2]); + else if (!strcmp(argv[1], "--bitmap")) + return read_midx_bitmapped_packs(argv[2]); return read_midx_file(argv[1], 0); } diff --git a/t/helper/test-submodule.c b/t/helper/test-submodule.c index 50c154d037..7197969a08 100644 --- a/t/helper/test-submodule.c +++ b/t/helper/test-submodule.c @@ -9,12 +9,19 @@ #include "submodule.h" #define TEST_TOOL_CHECK_NAME_USAGE \ - "test-tool submodule check-name <name>" + "test-tool submodule check-name" static const char *submodule_check_name_usage[] = { TEST_TOOL_CHECK_NAME_USAGE, NULL }; +#define TEST_TOOL_CHECK_URL_USAGE \ + "test-tool submodule check-url" +static const char *submodule_check_url_usage[] = { + TEST_TOOL_CHECK_URL_USAGE, + NULL +}; + #define TEST_TOOL_IS_ACTIVE_USAGE \ "test-tool submodule is-active <name>" static const char *submodule_is_active_usage[] = { @@ -31,31 +38,26 @@ static const char *submodule_resolve_relative_url_usage[] = { static const char *submodule_usage[] = { TEST_TOOL_CHECK_NAME_USAGE, + TEST_TOOL_CHECK_URL_USAGE, TEST_TOOL_IS_ACTIVE_USAGE, TEST_TOOL_RESOLVE_RELATIVE_URL_USAGE, NULL }; +typedef int (*check_fn_t)(const char *); + /* - * Exit non-zero if any of the submodule names given on the command line is - * invalid. If no names are given, filter stdin to print only valid names - * (which is primarily intended for testing). + * Apply 'check_fn' to each line of stdin, printing values that pass the check + * to stdout. */ -static int check_name(int argc, const char **argv) +static int check_submodule(check_fn_t check_fn) { - if (argc > 1) { - while (*++argv) { - if (check_submodule_name(*argv) < 0) - return 1; - } - } else { - struct strbuf buf = STRBUF_INIT; - while (strbuf_getline(&buf, stdin) != EOF) { - if (!check_submodule_name(buf.buf)) - printf("%s\n", buf.buf); - } - strbuf_release(&buf); + struct strbuf buf = STRBUF_INIT; + while (strbuf_getline(&buf, stdin) != EOF) { + if (!check_fn(buf.buf)) + printf("%s\n", buf.buf); } + strbuf_release(&buf); return 0; } @@ -69,7 +71,20 @@ static int cmd__submodule_check_name(int argc, const char **argv) if (argc) usage_with_options(submodule_check_name_usage, options); - return check_name(argc, argv); + return check_submodule(check_submodule_name); +} + +static int cmd__submodule_check_url(int argc, const char **argv) +{ + struct option options[] = { + OPT_END() + }; + argc = parse_options(argc, argv, "test-tools", options, + submodule_check_url_usage, 0); + if (argc) + usage_with_options(submodule_check_url_usage, options); + + return check_submodule(check_submodule_url); } static int cmd__submodule_is_active(int argc, const char **argv) @@ -195,6 +210,7 @@ static int cmd__submodule_config_writeable(int argc, const char **argv UNUSED) static struct test_cmd cmds[] = { { "check-name", cmd__submodule_check_name }, + { "check-url", cmd__submodule_check_url }, { "is-active", cmd__submodule_is_active }, { "resolve-relative-url", cmd__submodule_resolve_relative_url}, { "config-list", cmd__submodule_config_list }, diff --git a/t/helper/test-tool.c b/t/helper/test-tool.c index 37ba996539..80a946b847 100644 --- a/t/helper/test-tool.c +++ b/t/helper/test-tool.c @@ -19,8 +19,8 @@ static struct test_cmd cmds[] = { { "config", cmd__config }, { "crontab", cmd__crontab }, { "csprng", cmd__csprng }, - { "ctype", cmd__ctype }, { "date", cmd__date }, + { "delete-gpgsig", cmd__delete_gpgsig }, { "delta", cmd__delta }, { "dir-iterator", cmd__dir_iterator }, { "drop-caches", cmd__drop_caches }, @@ -56,7 +56,6 @@ static struct test_cmd cmds[] = { { "path-utils", cmd__path_utils }, { "pcre2-config", cmd__pcre2_config }, { "pkt-line", cmd__pkt_line }, - { "prio-queue", cmd__prio_queue }, { "proc-receive", cmd__proc_receive }, { "progress", cmd__progress }, { "reach", cmd__reach }, diff --git a/t/helper/test-tool.h b/t/helper/test-tool.h index 8a1a7c63da..2808b92419 100644 --- a/t/helper/test-tool.h +++ b/t/helper/test-tool.h @@ -12,9 +12,9 @@ int cmd__chmtime(int argc, const char **argv); int cmd__config(int argc, const char **argv); int cmd__crontab(int argc, const char **argv); int cmd__csprng(int argc, const char **argv); -int cmd__ctype(int argc, const char **argv); int cmd__date(int argc, const char **argv); int cmd__delta(int argc, const char **argv); +int cmd__delete_gpgsig(int argc, const char **argv); int cmd__dir_iterator(int argc, const char **argv); int cmd__drop_caches(int argc, const char **argv); int cmd__dump_cache_tree(int argc, const char **argv); @@ -49,7 +49,6 @@ int cmd__partial_clone(int argc, const char **argv); int cmd__path_utils(int argc, const char **argv); int cmd__pcre2_config(int argc, const char **argv); int cmd__pkt_line(int argc, const char **argv); -int cmd__prio_queue(int argc, const char **argv); int cmd__proc_receive(int argc, const char **argv); int cmd__progress(int argc, const char **argv); int cmd__reach(int argc, const char **argv); diff --git a/t/perf/p5332-multi-pack-reuse.sh b/t/perf/p5332-multi-pack-reuse.sh new file mode 100755 index 0000000000..5c6c575d62 --- /dev/null +++ b/t/perf/p5332-multi-pack-reuse.sh @@ -0,0 +1,81 @@ +#!/bin/sh + +test_description='tests pack performance with multi-pack reuse' + +. ./perf-lib.sh +. "${TEST_DIRECTORY}/perf/lib-pack.sh" + +packdir=.git/objects/pack + +test_perf_large_repo + +find_pack () { + for idx in $packdir/pack-*.idx + do + if git show-index <$idx | grep -q "$1" + then + basename $idx + fi || return 1 + done +} + +repack_into_n_chunks () { + git repack -adk && + + test "$1" -eq 1 && return || + + find $packdir -type f | sort >packs.before && + + # partition the repository into $1 chunks of consecutive commits, and + # then create $1 packs with the objects reachable from each chunk + # (excluding any objects reachable from the previous chunks) + sz="$(($(git rev-list --count --all) / $1))" + for rev in $(git rev-list --all | awk "NR % $sz == 0" | tac) + do + pack="$(echo "$rev" | git pack-objects --revs \ + --honor-pack-keep --delta-base-offset $packdir/pack)" && + touch $packdir/pack-$pack.keep || return 1 + done + + # grab any remaining objects not packed by the previous step(s) + git pack-objects --revs --all --honor-pack-keep --delta-base-offset \ + $packdir/pack && + + find $packdir -type f | sort >packs.after && + + # and install the whole thing + for f in $(comm -12 packs.before packs.after) + do + rm -f "$f" || return 1 + done + rm -fr $packdir/*.keep +} + +for nr_packs in 1 10 100 +do + test_expect_success "create $nr_packs-pack scenario" ' + repack_into_n_chunks $nr_packs + ' + + test_expect_success "setup bitmaps for $nr_packs-pack scenario" ' + find $packdir -type f -name "*.idx" | sed -e "s/.*\/\(.*\)$/+\1/g" | + git multi-pack-index write --stdin-packs --bitmap \ + --preferred-pack="$(find_pack $(git rev-parse HEAD))" + ' + + for reuse in single multi + do + test_perf "clone for $nr_packs-pack scenario ($reuse-pack reuse)" " + git for-each-ref --format='%(objectname)' refs/heads refs/tags >in && + git -c pack.allowPackReuse=$reuse pack-objects \ + --revs --delta-base-offset --use-bitmap-index \ + --stdout <in >result + " + + test_size "clone size for $nr_packs-pack scenario ($reuse-pack reuse)" ' + wc -c <result + ' + done +done + +test_done diff --git a/t/t0001-init.sh b/t/t0001-init.sh index 2b78e3be47..b131d665db 100755 --- a/t/t0001-init.sh +++ b/t/t0001-init.sh @@ -532,6 +532,76 @@ test_expect_success 'init rejects attempts to initialize with different hash' ' test_must_fail git -C sha256 init --object-format=sha1 ' +test_expect_success DEFAULT_REPO_FORMAT 'extensions.refStorage is not allowed with repo version 0' ' + test_when_finished "rm -rf refstorage" && + git init refstorage && + git -C refstorage config extensions.refStorage files && + test_must_fail git -C refstorage rev-parse 2>err && + grep "repo version is 0, but v1-only extension found" err +' + +test_expect_success DEFAULT_REPO_FORMAT 'extensions.refStorage with files backend' ' + test_when_finished "rm -rf refstorage" && + git init refstorage && + git -C refstorage config core.repositoryformatversion 1 && + git -C refstorage config extensions.refStorage files && + test_commit -C refstorage A && + git -C refstorage rev-parse --verify HEAD +' + +test_expect_success DEFAULT_REPO_FORMAT 'extensions.refStorage with unknown backend' ' + test_when_finished "rm -rf refstorage" && + git init refstorage && + git -C refstorage config core.repositoryformatversion 1 && + git -C refstorage config extensions.refStorage garbage && + test_must_fail git -C refstorage rev-parse 2>err && + grep "invalid value for ${SQ}extensions.refstorage${SQ}: ${SQ}garbage${SQ}" err +' + +test_expect_success DEFAULT_REPO_FORMAT 'init with GIT_DEFAULT_REF_FORMAT=files' ' + test_when_finished "rm -rf refformat" && + GIT_DEFAULT_REF_FORMAT=files git init refformat && + echo 0 >expect && + git -C refformat config core.repositoryformatversion >actual && + test_cmp expect actual && + test_must_fail git -C refformat config extensions.refstorage +' + +test_expect_success 'init with GIT_DEFAULT_REF_FORMAT=garbage' ' + test_when_finished "rm -rf refformat" && + cat >expect <<-EOF && + fatal: unknown ref storage format ${SQ}garbage${SQ} + EOF + test_must_fail env GIT_DEFAULT_REF_FORMAT=garbage git init refformat 2>err && + test_cmp expect err +' + +test_expect_success 'init with --ref-format=files' ' + test_when_finished "rm -rf refformat" && + git init --ref-format=files refformat && + echo files >expect && + git -C refformat rev-parse --show-ref-format >actual && + test_cmp expect actual +' + +test_expect_success 're-init with same format' ' + test_when_finished "rm -rf refformat" && + git init --ref-format=files refformat && + git init --ref-format=files refformat && + echo files >expect && + git -C refformat rev-parse --show-ref-format >actual && + test_cmp expect actual +' + +test_expect_success 'init with --ref-format=garbage' ' + test_when_finished "rm -rf refformat" && + cat >expect <<-EOF && + fatal: unknown ref storage format ${SQ}garbage${SQ} + EOF + test_must_fail git init --ref-format=garbage refformat 2>err && + test_cmp expect err +' + test_expect_success MINGW 'core.hidedotfiles = false' ' git config --global core.hidedotfiles false && rm -rf newdir && diff --git a/t/t0003-attributes.sh b/t/t0003-attributes.sh index aee2298f01..774b52c298 100755 --- a/t/t0003-attributes.sh +++ b/t/t0003-attributes.sh @@ -19,6 +19,20 @@ attr_check () { test_must_be_empty err } +attr_check_object_mode_basic () { + path="$1" && + expect="$2" && + check_opts="$3" && + git check-attr $check_opts builtin_objectmode -- "$path" >actual 2>err && + echo "$path: builtin_objectmode: $expect" >expect && + test_cmp expect actual +} + +attr_check_object_mode () { + attr_check_object_mode_basic "$@" && + test_must_be_empty err +} + attr_check_quote () { path="$1" quoted_path="$2" expect="$3" && @@ -558,4 +572,66 @@ test_expect_success EXPENSIVE 'large attributes file ignored in index' ' test_cmp expect err ' +test_expect_success 'builtin object mode attributes work (dir and regular paths)' ' + >normal && + attr_check_object_mode normal 100644 && + mkdir dir && + attr_check_object_mode dir 040000 +' + +test_expect_success POSIXPERM 'builtin object mode attributes work (executable)' ' + >exec && + chmod +x exec && + attr_check_object_mode exec 100755 +' + +test_expect_success SYMLINKS 'builtin object mode attributes work (symlinks)' ' + ln -s to_sym sym && + attr_check_object_mode sym 120000 +' + +test_expect_success 'native object mode attributes work with --cached' ' + >normal && + git add normal && + empty_blob=$(git rev-parse :normal) && + git update-index --index-info <<-EOF && + 100755 $empty_blob 0 exec + 120000 $empty_blob 0 symlink + EOF + attr_check_object_mode normal 100644 --cached && + attr_check_object_mode exec 100755 --cached && + attr_check_object_mode symlink 120000 --cached +' + +test_expect_success 'check object mode attributes work for submodules' ' + mkdir sub && + ( + cd sub && + git init && + mv .git .real && + echo "gitdir: .real" >.git && + test_commit first + ) && + attr_check_object_mode sub 160000 && + attr_check_object_mode sub unspecified --cached && + git add sub && + attr_check_object_mode sub 160000 --cached +' + +test_expect_success 'we do not allow user defined builtin_* attributes' ' + echo "foo* builtin_foo" >.gitattributes && + git add .gitattributes 2>actual && + echo "builtin_foo is not a valid attribute name: .gitattributes:1" >expect && + test_cmp expect actual +' + +test_expect_success 'user defined builtin_objectmode values are ignored' ' + echo "foo* builtin_objectmode=12345" >.gitattributes && + git add .gitattributes && + >foo_1 && + attr_check_object_mode_basic foo_1 100644 && + echo "builtin_objectmode is not a valid attribute name: .gitattributes:1" >expect && + test_cmp expect err +' + test_done diff --git a/t/t0009-prio-queue.sh b/t/t0009-prio-queue.sh deleted file mode 100755 index eea99107a4..0000000000 --- a/t/t0009-prio-queue.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/bin/sh - -test_description='basic tests for priority queue implementation' - -TEST_PASSES_SANITIZE_LEAK=true -. ./test-lib.sh - -cat >expect <<'EOF' -1 -2 -3 -4 -5 -5 -6 -7 -8 -9 -10 -EOF -test_expect_success 'basic ordering' ' - test-tool prio-queue 2 6 3 10 9 5 7 4 5 8 1 dump >actual && - test_cmp expect actual -' - -cat >expect <<'EOF' -2 -3 -4 -1 -5 -6 -EOF -test_expect_success 'mixed put and get' ' - test-tool prio-queue 6 2 4 get 5 3 get get 1 dump >actual && - test_cmp expect actual -' - -cat >expect <<'EOF' -1 -2 -NULL -1 -2 -NULL -EOF -test_expect_success 'notice empty queue' ' - test-tool prio-queue 1 2 get get get 1 2 get get get >actual && - test_cmp expect actual -' - -cat >expect <<'EOF' -3 -2 -6 -4 -5 -1 -8 -EOF -test_expect_success 'stack order' ' - test-tool prio-queue stack 8 1 5 4 6 2 3 dump >actual && - test_cmp expect actual -' - -test_done diff --git a/t/t0018-advice.sh b/t/t0018-advice.sh index c13057a4ca..0dcfb760a2 100755 --- a/t/t0018-advice.sh +++ b/t/t0018-advice.sh @@ -17,7 +17,6 @@ test_expect_success 'advice should be printed when config variable is unset' ' test_expect_success 'advice should be printed when config variable is set to true' ' cat >expect <<-\EOF && hint: This is a piece of advice - hint: Disable this message with "git config advice.nestedTag false" EOF test_config advice.nestedTag true && test-tool advise "This is a piece of advice" 2>actual && diff --git a/t/t0024-crlf-archive.sh b/t/t0024-crlf-archive.sh index a34de56420..a7f4de4a43 100755 --- a/t/t0024-crlf-archive.sh +++ b/t/t0024-crlf-archive.sh @@ -9,7 +9,7 @@ test_expect_success setup ' git config core.autocrlf true && - printf "CRLF line ending\r\nAnd another\r\n" > sample && + printf "CRLF line ending\r\nAnd another\r\n" >sample && git add sample && test_tick && @@ -19,8 +19,9 @@ test_expect_success setup ' test_expect_success 'tar archive' ' - git archive --format=tar HEAD | - ( mkdir untarred && cd untarred && "$TAR" -xf - ) && + git archive --format=tar HEAD >test.tar && + mkdir untarred && + "$TAR" xf test.tar -C untarred && test_cmp sample untarred/sample @@ -30,7 +31,11 @@ test_expect_success UNZIP 'zip archive' ' git archive --format=zip HEAD >test.zip && - ( mkdir unzipped && cd unzipped && "$GIT_UNZIP" ../test.zip ) && + mkdir unzipped && + ( + cd unzipped && + "$GIT_UNZIP" ../test.zip + ) && test_cmp sample unzipped/sample diff --git a/t/t0070-fundamental.sh b/t/t0070-fundamental.sh index 487bc8d905..0ecec2ba71 100755 --- a/t/t0070-fundamental.sh +++ b/t/t0070-fundamental.sh @@ -9,10 +9,6 @@ Verify wrappers and compatibility functions. TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh -test_expect_success 'character classes (isspace, isalpha etc.)' ' - test-tool ctype -' - test_expect_success 'mktemp to nonexistent directory prints filename' ' test_must_fail test-tool mktemp doesnotexist/testXXXXXX 2>err && grep "doesnotexist/test" err @@ -53,4 +49,62 @@ test_expect_success 'missing sideband designator is reported' ' test_grep "missing sideband" err ' +test_expect_success 'unpack-sideband: --no-chomp-newline' ' + test_when_finished "rm -f expect-out expect-err" && + test-tool pkt-line send-split-sideband >split-sideband && + test-tool pkt-line unpack-sideband \ + --no-chomp-newline <split-sideband >out 2>err && + cat >expect-out <<-EOF && + primary: regular output + EOF + cat >expect-err <<-EOF && + Foo. + Bar. + Hello, world! + EOF + test_cmp expect-out out && + test_cmp expect-err err +' + +test_expect_success 'unpack-sideband: --chomp-newline (default)' ' + test_when_finished "rm -f expect-out expect-err" && + test-tool pkt-line send-split-sideband >split-sideband && + test-tool pkt-line unpack-sideband \ + --chomp-newline <split-sideband >out 2>err && + printf "primary: regular output" >expect-out && + printf "Foo.Bar.Hello, world!" >expect-err && + test_cmp expect-out out && + test_cmp expect-err err +' + +test_expect_success 'unpack-sideband: packet_reader_read() consumes sideband, no chomp payload' ' + test_when_finished "rm -f expect-out expect-err" && + test-tool pkt-line send-split-sideband >split-sideband && + test-tool pkt-line unpack-sideband \ + --reader-use-sideband \ + --no-chomp-newline <split-sideband >out 2>err && + cat >expect-out <<-EOF && + primary: regular output + EOF + printf "remote: Foo. \n" >expect-err && + printf "remote: Bar. \n" >>expect-err && + printf "remote: Hello, world! \n" >>expect-err && + test_cmp expect-out out && + test_cmp expect-err err +' + +test_expect_success 'unpack-sideband: packet_reader_read() consumes sideband, chomp payload' ' + test_when_finished "rm -f expect-out expect-err" && + test-tool pkt-line send-split-sideband >split-sideband && + test-tool pkt-line unpack-sideband \ + --reader-use-sideband \ + --chomp-newline <split-sideband >out 2>err && + printf "primary: regular output" >expect-out && + printf "remote: Foo. \n" >expect-err && + printf "remote: Bar. \n" >>expect-err && + printf "remote: Hello, world! \n" >>expect-err && + test_cmp expect-out out && + test_cmp expect-err err +' + test_done diff --git a/t/t0095-bloom.sh b/t/t0095-bloom.sh index b567383eb8..c8d84ab606 100755 --- a/t/t0095-bloom.sh +++ b/t/t0095-bloom.sh @@ -29,6 +29,14 @@ test_expect_success 'compute unseeded murmur3 hash for test string 2' ' test_cmp expect actual ' +test_expect_success 'compute unseeded murmur3 hash for test string 3' ' + cat >expect <<-\EOF && + Murmur3 Hash with seed=0:0xa183ccfd + EOF + test-tool bloom get_murmur3_seven_highbit >actual && + test_cmp expect actual +' + test_expect_success 'compute bloom key for empty string' ' cat >expect <<-\EOF && Hashes:0x5615800c|0x5b966560|0x61174ab4|0x66983008|0x6c19155c|0x7199fab0|0x771ae004| diff --git a/t/t0600-reffiles-backend.sh b/t/t0600-reffiles-backend.sh new file mode 100755 index 0000000000..260c54e9d6 --- /dev/null +++ b/t/t0600-reffiles-backend.sh @@ -0,0 +1,384 @@ +#!/bin/sh + +test_description='Test reffiles backend' + +GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main +export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME + +TEST_PASSES_SANITIZE_LEAK=true +. ./test-lib.sh + +if ! test_have_prereq REFFILES +then + skip_all='skipping reffiles specific tests' + test_done +fi + +test_expect_success 'setup' ' + git commit --allow-empty -m Initial && + C=$(git rev-parse HEAD) && + git commit --allow-empty -m Second && + D=$(git rev-parse HEAD) && + git commit --allow-empty -m Third && + E=$(git rev-parse HEAD) +' + +test_expect_success 'empty directory should not fool rev-parse' ' + prefix=refs/e-rev-parse && + git update-ref $prefix/foo $C && + git pack-refs --all && + mkdir -p .git/$prefix/foo/bar/baz && + echo "$C" >expected && + git rev-parse $prefix/foo >actual && + test_cmp expected actual +' + +test_expect_success 'empty directory should not fool for-each-ref' ' + prefix=refs/e-for-each-ref && + git update-ref $prefix/foo $C && + git for-each-ref $prefix >expected && + git pack-refs --all && + mkdir -p .git/$prefix/foo/bar/baz && + git for-each-ref $prefix >actual && + test_cmp expected actual +' + +test_expect_success 'empty directory should not fool create' ' + prefix=refs/e-create && + mkdir -p .git/$prefix/foo/bar/baz && + printf "create %s $C\n" $prefix/foo | + git update-ref --stdin +' + +test_expect_success 'empty directory should not fool verify' ' + prefix=refs/e-verify && + git update-ref $prefix/foo $C && + git pack-refs --all && + mkdir -p .git/$prefix/foo/bar/baz && + printf "verify %s $C\n" $prefix/foo | + git update-ref --stdin +' + +test_expect_success 'empty directory should not fool 1-arg update' ' + prefix=refs/e-update-1 && + git update-ref $prefix/foo $C && + git pack-refs --all && + mkdir -p .git/$prefix/foo/bar/baz && + printf "update %s $D\n" $prefix/foo | + git update-ref --stdin +' + +test_expect_success 'empty directory should not fool 2-arg update' ' + prefix=refs/e-update-2 && + git update-ref $prefix/foo $C && + git pack-refs --all && + mkdir -p .git/$prefix/foo/bar/baz && + printf "update %s $D $C\n" $prefix/foo | + git update-ref --stdin +' + +test_expect_success 'empty directory should not fool 0-arg delete' ' + prefix=refs/e-delete-0 && + git update-ref $prefix/foo $C && + git pack-refs --all && + mkdir -p .git/$prefix/foo/bar/baz && + printf "delete %s\n" $prefix/foo | + git update-ref --stdin +' + +test_expect_success 'empty directory should not fool 1-arg delete' ' + prefix=refs/e-delete-1 && + git update-ref $prefix/foo $C && + git pack-refs --all && + mkdir -p .git/$prefix/foo/bar/baz && + printf "delete %s $C\n" $prefix/foo | + git update-ref --stdin +' + +test_expect_success 'non-empty directory blocks create' ' + prefix=refs/ne-create && + mkdir -p .git/$prefix/foo/bar && + : >.git/$prefix/foo/bar/baz.lock && + test_when_finished "rm -f .git/$prefix/foo/bar/baz.lock" && + cat >expected <<-EOF && + fatal: cannot lock ref $SQ$prefix/foo$SQ: there is a non-empty directory $SQ.git/$prefix/foo$SQ blocking reference $SQ$prefix/foo$SQ + EOF + printf "%s\n" "update $prefix/foo $C" | + test_must_fail git update-ref --stdin 2>output.err && + test_cmp expected output.err && + cat >expected <<-EOF && + fatal: cannot lock ref $SQ$prefix/foo$SQ: unable to resolve reference $SQ$prefix/foo$SQ + EOF + printf "%s\n" "update $prefix/foo $D $C" | + test_must_fail git update-ref --stdin 2>output.err && + test_cmp expected output.err +' + +test_expect_success 'broken reference blocks create' ' + prefix=refs/broken-create && + mkdir -p .git/$prefix && + echo "gobbledigook" >.git/$prefix/foo && + test_when_finished "rm -f .git/$prefix/foo" && + cat >expected <<-EOF && + fatal: cannot lock ref $SQ$prefix/foo$SQ: unable to resolve reference $SQ$prefix/foo$SQ: reference broken + EOF + printf "%s\n" "update $prefix/foo $C" | + test_must_fail git update-ref --stdin 2>output.err && + test_cmp expected output.err && + cat >expected <<-EOF && + fatal: cannot lock ref $SQ$prefix/foo$SQ: unable to resolve reference $SQ$prefix/foo$SQ: reference broken + EOF + printf "%s\n" "update $prefix/foo $D $C" | + test_must_fail git update-ref --stdin 2>output.err && + test_cmp expected output.err +' + +test_expect_success 'non-empty directory blocks indirect create' ' + prefix=refs/ne-indirect-create && + git symbolic-ref $prefix/symref $prefix/foo && + mkdir -p .git/$prefix/foo/bar && + : >.git/$prefix/foo/bar/baz.lock && + test_when_finished "rm -f .git/$prefix/foo/bar/baz.lock" && + cat >expected <<-EOF && + fatal: cannot lock ref $SQ$prefix/symref$SQ: there is a non-empty directory $SQ.git/$prefix/foo$SQ blocking reference $SQ$prefix/foo$SQ + EOF + printf "%s\n" "update $prefix/symref $C" | + test_must_fail git update-ref --stdin 2>output.err && + test_cmp expected output.err && + cat >expected <<-EOF && + fatal: cannot lock ref $SQ$prefix/symref$SQ: unable to resolve reference $SQ$prefix/foo$SQ + EOF + printf "%s\n" "update $prefix/symref $D $C" | + test_must_fail git update-ref --stdin 2>output.err && + test_cmp expected output.err +' + +test_expect_success 'broken reference blocks indirect create' ' + prefix=refs/broken-indirect-create && + git symbolic-ref $prefix/symref $prefix/foo && + echo "gobbledigook" >.git/$prefix/foo && + test_when_finished "rm -f .git/$prefix/foo" && + cat >expected <<-EOF && + fatal: cannot lock ref $SQ$prefix/symref$SQ: unable to resolve reference $SQ$prefix/foo$SQ: reference broken + EOF + printf "%s\n" "update $prefix/symref $C" | + test_must_fail git update-ref --stdin 2>output.err && + test_cmp expected output.err && + cat >expected <<-EOF && + fatal: cannot lock ref $SQ$prefix/symref$SQ: unable to resolve reference $SQ$prefix/foo$SQ: reference broken + EOF + printf "%s\n" "update $prefix/symref $D $C" | + test_must_fail git update-ref --stdin 2>output.err && + test_cmp expected output.err +' + +test_expect_success 'no bogus intermediate values during delete' ' + prefix=refs/slow-transaction && + # Set up a reference with differing loose and packed versions: + git update-ref $prefix/foo $C && + git pack-refs --all && + git update-ref $prefix/foo $D && + # Now try to update the reference, but hold the `packed-refs` lock + # for a while to see what happens while the process is blocked: + : >.git/packed-refs.lock && + test_when_finished "rm -f .git/packed-refs.lock" && + { + # Note: the following command is intentionally run in the + # background. We increase the timeout so that `update-ref` + # attempts to acquire the `packed-refs` lock for much longer + # than it takes for us to do the check then delete it: + git -c core.packedrefstimeout=30000 update-ref -d $prefix/foo & + } && + pid2=$! && + # Give update-ref plenty of time to get to the point where it tries + # to lock packed-refs: + sleep 1 && + # Make sure that update-ref did not complete despite the lock: + kill -0 $pid2 && + # Verify that the reference still has its old value: + sha1=$(git rev-parse --verify --quiet $prefix/foo || echo undefined) && + case "$sha1" in + $D) + # This is what we hope for; it means that nothing + # user-visible has changed yet. + : ;; + undefined) + # This is not correct; it means the deletion has happened + # already even though update-ref should not have been + # able to acquire the lock yet. + echo "$prefix/foo deleted prematurely" && + break + ;; + $C) + # This value should never be seen. Probably the loose + # reference has been deleted but the packed reference + # is still there: + echo "$prefix/foo incorrectly observed to be C" && + break + ;; + *) + # WTF? + echo "unexpected value observed for $prefix/foo: $sha1" && + break + ;; + esac >out && + rm -f .git/packed-refs.lock && + wait $pid2 && + test_must_be_empty out && + test_must_fail git rev-parse --verify --quiet $prefix/foo +' + +test_expect_success 'delete fails cleanly if packed-refs file is locked' ' + prefix=refs/locked-packed-refs && + # Set up a reference with differing loose and packed versions: + git update-ref $prefix/foo $C && + git pack-refs --all && + git update-ref $prefix/foo $D && + git for-each-ref $prefix >unchanged && + # Now try to delete it while the `packed-refs` lock is held: + : >.git/packed-refs.lock && + test_when_finished "rm -f .git/packed-refs.lock" && + test_must_fail git update-ref -d $prefix/foo >out 2>err && + git for-each-ref $prefix >actual && + test_grep "Unable to create $SQ.*packed-refs.lock$SQ: " err && + test_cmp unchanged actual +' + +test_expect_success 'delete fails cleanly if packed-refs.new write fails' ' + # Setup and expectations are similar to the test above. + prefix=refs/failed-packed-refs && + git update-ref $prefix/foo $C && + git pack-refs --all && + git update-ref $prefix/foo $D && + git for-each-ref $prefix >unchanged && + # This should not happen in practice, but it is an easy way to get a + # reliable error (we open with create_tempfile(), which uses O_EXCL). + : >.git/packed-refs.new && + test_when_finished "rm -f .git/packed-refs.new" && + test_must_fail git update-ref -d $prefix/foo && + git for-each-ref $prefix >actual && + test_cmp unchanged actual +' + +RWT="test-tool ref-store worktree:wt" +RMAIN="test-tool ref-store worktree:main" + +test_expect_success 'setup worktree' ' + test_commit first && + git worktree add -b wt-main wt && + ( + cd wt && + test_commit second + ) +' + +# Some refs (refs/bisect/*, pseudorefs) are kept per worktree, so they should +# only appear in the for-each-reflog output if it is called from the correct +# worktree, which is exercised in this test. This test is poorly written for +# mulitple reasons: 1) it creates invalidly formatted log entres. 2) it uses +# direct FS access for creating the reflogs. 3) PSEUDO-WT and refs/bisect/random +# do not create reflogs by default, so it is not testing a realistic scenario. +test_expect_success 'for_each_reflog()' ' + echo $ZERO_OID >.git/logs/PSEUDO_MAIN_HEAD && + mkdir -p .git/logs/refs/bisect && + echo $ZERO_OID >.git/logs/refs/bisect/random && + + echo $ZERO_OID >.git/worktrees/wt/logs/PSEUDO_WT_HEAD && + mkdir -p .git/worktrees/wt/logs/refs/bisect && + echo $ZERO_OID >.git/worktrees/wt/logs/refs/bisect/wt-random && + + $RWT for-each-reflog | cut -d" " -f 2- | sort >actual && + cat >expected <<-\EOF && + HEAD 0x1 + PSEUDO_WT_HEAD 0x0 + refs/bisect/wt-random 0x0 + refs/heads/main 0x0 + refs/heads/wt-main 0x0 + EOF + test_cmp expected actual && + + $RMAIN for-each-reflog | cut -d" " -f 2- | sort >actual && + cat >expected <<-\EOF && + HEAD 0x1 + PSEUDO_MAIN_HEAD 0x0 + refs/bisect/random 0x0 + refs/heads/main 0x0 + refs/heads/wt-main 0x0 + EOF + test_cmp expected actual +' + +# Triggering the bug detected by this test requires a newline to fall +# exactly BUFSIZ-1 bytes from the end of the file. We don't know +# what that value is, since it's platform dependent. However, if +# we choose some value N, we also catch any D which divides N evenly +# (since we will read backwards in chunks of D). So we choose 8K, +# which catches glibc (with an 8K BUFSIZ) and *BSD (1K). +# +# Each line is 114 characters, so we need 75 to still have a few before the +# last 8K. The 89-character padding on the final entry lines up our +# newline exactly. +test_expect_success SHA1 'parsing reverse reflogs at BUFSIZ boundaries' ' + git checkout -b reflogskip && + zf=$(test_oid zero_2) && + ident="abc <xyz> 0000000001 +0000" && + for i in $(test_seq 1 75); do + printf "$zf%02d $zf%02d %s\t" $i $(($i+1)) "$ident" && + if test $i = 75; then + for j in $(test_seq 1 89); do + printf X || return 1 + done + else + printf X + fi && + printf "\n" || return 1 + done >.git/logs/refs/heads/reflogskip && + git rev-parse reflogskip@{73} >actual && + echo ${zf}03 >expect && + test_cmp expect actual +' + +# This test takes a lock on an individual ref; this is not supported in +# reftable. +test_expect_success 'reflog expire operates on symref not referrent' ' + git branch --create-reflog the_symref && + git branch --create-reflog referrent && + git update-ref referrent HEAD && + git symbolic-ref refs/heads/the_symref refs/heads/referrent && + test_when_finished "rm -f .git/refs/heads/referrent.lock" && + touch .git/refs/heads/referrent.lock && + git reflog expire --expire=all the_symref +' + +test_expect_success 'empty reflog' ' + test_when_finished "rm -rf empty" && + git init empty && + test_commit -C empty A && + >empty/.git/logs/refs/heads/foo && + git -C empty reflog expire --all 2>err && + test_must_be_empty err +' + +test_expect_success SYMLINKS 'ref resolution not confused by broken symlinks' ' + ln -s does-not-exist .git/refs/heads/broken && + test_must_fail git rev-parse --verify broken +' + +test_expect_success 'log diagnoses bogus HEAD hash' ' + git init empty && + test_when_finished "rm -rf empty" && + echo 1234abcd >empty/.git/refs/heads/main && + test_must_fail git -C empty log 2>stderr && + test_grep broken stderr +' + +test_expect_success 'log diagnoses bogus HEAD symref' ' + git init empty && + test-tool -C empty ref-store main create-symref HEAD refs/heads/invalid.lock && + test_must_fail git -C empty log 2>stderr && + test_grep broken stderr && + test_must_fail git -C empty log --default totally-bogus 2>stderr && + test_grep broken stderr +' + +test_done diff --git a/t/t3210-pack-refs.sh b/t/t0601-reffiles-pack-refs.sh index c0f1f9cfb7..c309d2bae8 100755 --- a/t/t3210-pack-refs.sh +++ b/t/t0601-reffiles-pack-refs.sh @@ -15,10 +15,10 @@ export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh -if test_have_prereq !REFFILES +if ! test_have_prereq REFFILES then - skip_all='skipping files-backend specific pack-refs tests' - test_done + skip_all='skipping reffiles specific tests' + test_done fi test_expect_success 'enable reflogs' ' @@ -32,6 +32,14 @@ test_expect_success 'prepare a trivial repository' ' HEAD=$(git rev-parse --verify HEAD) ' +test_expect_success 'pack_refs(PACK_REFS_ALL | PACK_REFS_PRUNE)' ' + N=`find .git/refs -type f | wc -l` && + test "$N" != 0 && + test-tool ref-store main pack-refs PACK_REFS_PRUNE,PACK_REFS_ALL && + N=`find .git/refs -type f` && + test -z "$N" +' + SHA1= test_expect_success 'see if git show-ref works as expected' ' @@ -300,4 +308,54 @@ test_expect_success SYMLINKS 'pack symlinked packed-refs' ' test "$(test_readlink .git/packed-refs)" = "my-deviant-packed-refs" ' +# The 'packed-refs' file is stored directly in .git/. This means it is global +# to the repository, and can only contain refs that are shared across all +# worktrees. +test_expect_success 'refs/worktree must not be packed' ' + test_commit initial && + test_commit wt1 && + test_commit wt2 && + git worktree add wt1 wt1 && + git worktree add wt2 wt2 && + git checkout initial && + git update-ref refs/worktree/foo HEAD && + git -C wt1 update-ref refs/worktree/foo HEAD && + git -C wt2 update-ref refs/worktree/foo HEAD && + git pack-refs --all && + test_path_is_missing .git/refs/tags/wt1 && + test_path_is_file .git/refs/worktree/foo && + test_path_is_file .git/worktrees/wt1/refs/worktree/foo && + test_path_is_file .git/worktrees/wt2/refs/worktree/foo +' + +# we do not want to count on running pack-refs to +# actually pack it, as it is perfectly reasonable to +# skip processing a broken ref +test_expect_success 'create packed-refs file with broken ref' ' + test_tick && git commit --allow-empty -m one && + recoverable=$(git rev-parse HEAD) && + test_tick && git commit --allow-empty -m two && + missing=$(git rev-parse HEAD) && + rm -f .git/refs/heads/main && + cat >.git/packed-refs <<-EOF && + $missing refs/heads/main + $recoverable refs/heads/other + EOF + echo $missing >expect && + git rev-parse refs/heads/main >actual && + test_cmp expect actual +' + +test_expect_success 'pack-refs does not silently delete broken packed ref' ' + git pack-refs --all --prune && + git rev-parse refs/heads/main >actual && + test_cmp expect actual +' + +test_expect_success 'pack-refs does not drop broken refs during deletion' ' + git update-ref -d refs/heads/other && + git rev-parse refs/heads/main >actual && + test_cmp expect actual +' + test_done diff --git a/t/t1006-cat-file.sh b/t/t1006-cat-file.sh index 271c5e4fd3..e12b221972 100755 --- a/t/t1006-cat-file.sh +++ b/t/t1006-cat-file.sh @@ -112,65 +112,65 @@ strlen () { run_tests () { type=$1 - sha1=$2 + oid=$2 size=$3 content=$4 pretty_content=$5 - batch_output="$sha1 $type $size + batch_output="$oid $type $size $content" test_expect_success "$type exists" ' - git cat-file -e $sha1 + git cat-file -e $oid ' test_expect_success "Type of $type is correct" ' echo $type >expect && - git cat-file -t $sha1 >actual && + git cat-file -t $oid >actual && test_cmp expect actual ' test_expect_success "Size of $type is correct" ' echo $size >expect && - git cat-file -s $sha1 >actual && + git cat-file -s $oid >actual && test_cmp expect actual ' test_expect_success "Type of $type is correct using --allow-unknown-type" ' echo $type >expect && - git cat-file -t --allow-unknown-type $sha1 >actual && + git cat-file -t --allow-unknown-type $oid >actual && test_cmp expect actual ' test_expect_success "Size of $type is correct using --allow-unknown-type" ' echo $size >expect && - git cat-file -s --allow-unknown-type $sha1 >actual && + git cat-file -s --allow-unknown-type $oid >actual && test_cmp expect actual ' test -z "$content" || test_expect_success "Content of $type is correct" ' echo_without_newline "$content" >expect && - git cat-file $type $sha1 >actual && + git cat-file $type $oid >actual && test_cmp expect actual ' test_expect_success "Pretty content of $type is correct" ' echo_without_newline "$pretty_content" >expect && - git cat-file -p $sha1 >actual && + git cat-file -p $oid >actual && test_cmp expect actual ' test -z "$content" || test_expect_success "--batch output of $type is correct" ' echo "$batch_output" >expect && - echo $sha1 | git cat-file --batch >actual && + echo $oid | git cat-file --batch >actual && test_cmp expect actual ' test_expect_success "--batch-check output of $type is correct" ' - echo "$sha1 $type $size" >expect && - echo_without_newline $sha1 | git cat-file --batch-check >actual && + echo "$oid $type $size" >expect && + echo_without_newline $oid | git cat-file --batch-check >actual && test_cmp expect actual ' @@ -179,33 +179,33 @@ $content" test -z "$content" || test_expect_success "--batch-command $opt output of $type content is correct" ' echo "$batch_output" >expect && - test_write_lines "contents $sha1" | git cat-file --batch-command $opt >actual && + test_write_lines "contents $oid" | git cat-file --batch-command $opt >actual && test_cmp expect actual ' test_expect_success "--batch-command $opt output of $type info is correct" ' - echo "$sha1 $type $size" >expect && - test_write_lines "info $sha1" | + echo "$oid $type $size" >expect && + test_write_lines "info $oid" | git cat-file --batch-command $opt >actual && test_cmp expect actual ' done test_expect_success "custom --batch-check format" ' - echo "$type $sha1" >expect && - echo $sha1 | git cat-file --batch-check="%(objecttype) %(objectname)" >actual && + echo "$type $oid" >expect && + echo $oid | git cat-file --batch-check="%(objecttype) %(objectname)" >actual && test_cmp expect actual ' test_expect_success "custom --batch-command format" ' - echo "$type $sha1" >expect && - echo "info $sha1" | git cat-file --batch-command="%(objecttype) %(objectname)" >actual && + echo "$type $oid" >expect && + echo "info $oid" | git cat-file --batch-command="%(objecttype) %(objectname)" >actual && test_cmp expect actual ' test_expect_success '--batch-check with %(rest)' ' echo "$type this is some extra content" >expect && - echo "$sha1 this is some extra content" | + echo "$oid this is some extra content" | git cat-file --batch-check="%(objecttype) %(rest)" >actual && test_cmp expect actual ' @@ -216,7 +216,7 @@ $content" echo "$size" && echo "$content" } >expect && - echo $sha1 | git cat-file --batch="%(objectsize)" >actual && + echo $oid | git cat-file --batch="%(objectsize)" >actual && test_cmp expect actual ' @@ -226,114 +226,154 @@ $content" echo "$type" && echo "$content" } >expect && - echo $sha1 | git cat-file --batch="%(objecttype)" >actual && + echo $oid | git cat-file --batch="%(objecttype)" >actual && test_cmp expect actual ' } hello_content="Hello World" hello_size=$(strlen "$hello_content") -hello_sha1=$(echo_without_newline "$hello_content" | git hash-object --stdin) +hello_oid=$(echo_without_newline "$hello_content" | git hash-object --stdin) test_expect_success "setup" ' + git config core.repositoryformatversion 1 && + git config extensions.objectformat $test_hash_algo && + git config extensions.compatobjectformat $test_compat_hash_algo && echo_without_newline "$hello_content" > hello && git update-index --add hello ' -run_tests 'blob' $hello_sha1 $hello_size "$hello_content" "$hello_content" +run_blob_tests () { + oid=$1 -test_expect_success '--batch-command --buffer with flush for blob info' ' - echo "$hello_sha1 blob $hello_size" >expect && - test_write_lines "info $hello_sha1" "flush" | + run_tests 'blob' $oid $hello_size "$hello_content" "$hello_content" + + test_expect_success '--batch-command --buffer with flush for blob info' ' + echo "$oid blob $hello_size" >expect && + test_write_lines "info $oid" "flush" | GIT_TEST_CAT_FILE_NO_FLUSH_ON_EXIT=1 \ git cat-file --batch-command --buffer >actual && test_cmp expect actual -' + ' -test_expect_success '--batch-command --buffer without flush for blob info' ' + test_expect_success '--batch-command --buffer without flush for blob info' ' touch output && - test_write_lines "info $hello_sha1" | + test_write_lines "info $oid" | GIT_TEST_CAT_FILE_NO_FLUSH_ON_EXIT=1 \ git cat-file --batch-command --buffer >>output && test_must_be_empty output -' + ' +} + +hello_compat_oid=$(git rev-parse --output-object-format=$test_compat_hash_algo $hello_oid) +run_blob_tests $hello_oid +run_blob_tests $hello_compat_oid test_expect_success '--batch-check without %(rest) considers whole line' ' - echo "$hello_sha1 blob $hello_size" >expect && - git update-index --add --cacheinfo 100644 $hello_sha1 "white space" && + echo "$hello_oid blob $hello_size" >expect && + git update-index --add --cacheinfo 100644 $hello_oid "white space" && test_when_finished "git update-index --remove \"white space\"" && echo ":white space" | git cat-file --batch-check >actual && test_cmp expect actual ' -tree_sha1=$(git write-tree) +tree_oid=$(git write-tree) +tree_compat_oid=$(git rev-parse --output-object-format=$test_compat_hash_algo $tree_oid) tree_size=$(($(test_oid rawsz) + 13)) -tree_pretty_content="100644 blob $hello_sha1 hello${LF}" +tree_compat_size=$(($(test_oid --hash=compat rawsz) + 13)) +tree_pretty_content="100644 blob $hello_oid hello${LF}" +tree_compat_pretty_content="100644 blob $hello_compat_oid hello${LF}" -run_tests 'tree' $tree_sha1 $tree_size "" "$tree_pretty_content" +run_tests 'tree' $tree_oid $tree_size "" "$tree_pretty_content" +run_tests 'tree' $tree_compat_oid $tree_compat_size "" "$tree_compat_pretty_content" commit_message="Initial commit" -commit_sha1=$(echo_without_newline "$commit_message" | git commit-tree $tree_sha1) +commit_oid=$(echo_without_newline "$commit_message" | git commit-tree $tree_oid) +commit_compat_oid=$(git rev-parse --output-object-format=$test_compat_hash_algo $commit_oid) commit_size=$(($(test_oid hexsz) + 137)) -commit_content="tree $tree_sha1 +commit_compat_size=$(($(test_oid --hash=compat hexsz) + 137)) +commit_content="tree $tree_oid +author $GIT_AUTHOR_NAME <$GIT_AUTHOR_EMAIL> $GIT_AUTHOR_DATE +committer $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE + +$commit_message" + +commit_compat_content="tree $tree_compat_oid author $GIT_AUTHOR_NAME <$GIT_AUTHOR_EMAIL> $GIT_AUTHOR_DATE committer $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE $commit_message" -run_tests 'commit' $commit_sha1 $commit_size "$commit_content" "$commit_content" +run_tests 'commit' $commit_oid $commit_size "$commit_content" "$commit_content" +run_tests 'commit' $commit_compat_oid $commit_compat_size "$commit_compat_content" "$commit_compat_content" -tag_header_without_timestamp="object $hello_sha1 -type blob +tag_header_without_oid="type blob tag hellotag tagger $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL>" +tag_header_without_timestamp="object $hello_oid +$tag_header_without_oid" +tag_compat_header_without_timestamp="object $hello_compat_oid +$tag_header_without_oid" tag_description="This is a tag" tag_content="$tag_header_without_timestamp 0 +0000 $tag_description" +tag_compat_content="$tag_compat_header_without_timestamp 0 +0000 -tag_sha1=$(echo_without_newline "$tag_content" | git hash-object -t tag --stdin -w) +$tag_description" + +tag_oid=$(echo_without_newline "$tag_content" | git hash-object -t tag --stdin -w) tag_size=$(strlen "$tag_content") -run_tests 'tag' $tag_sha1 $tag_size "$tag_content" "$tag_content" +tag_compat_oid=$(git rev-parse --output-object-format=$test_compat_hash_algo $tag_oid) +tag_compat_size=$(strlen "$tag_compat_content") + +run_tests 'tag' $tag_oid $tag_size "$tag_content" "$tag_content" +run_tests 'tag' $tag_compat_oid $tag_compat_size "$tag_compat_content" "$tag_compat_content" test_expect_success "Reach a blob from a tag pointing to it" ' echo_without_newline "$hello_content" >expect && - git cat-file blob $tag_sha1 >actual && + git cat-file blob $tag_oid >actual && test_cmp expect actual ' -for batch in batch batch-check batch-command +for oid in $hello_oid $hello_compat_oid do - for opt in t s e p + for batch in batch batch-check batch-command do + for opt in t s e p + do test_expect_success "Passing -$opt with --$batch fails" ' - test_must_fail git cat-file --$batch -$opt $hello_sha1 + test_must_fail git cat-file --$batch -$opt $oid ' test_expect_success "Passing --$batch with -$opt fails" ' - test_must_fail git cat-file -$opt --$batch $hello_sha1 + test_must_fail git cat-file -$opt --$batch $oid ' - done + done - test_expect_success "Passing <type> with --$batch fails" ' - test_must_fail git cat-file --$batch blob $hello_sha1 - ' + test_expect_success "Passing <type> with --$batch fails" ' + test_must_fail git cat-file --$batch blob $oid + ' - test_expect_success "Passing --$batch with <type> fails" ' - test_must_fail git cat-file blob --$batch $hello_sha1 - ' + test_expect_success "Passing --$batch with <type> fails" ' + test_must_fail git cat-file blob --$batch $oid + ' - test_expect_success "Passing sha1 with --$batch fails" ' - test_must_fail git cat-file --$batch $hello_sha1 - ' + test_expect_success "Passing oid with --$batch fails" ' + test_must_fail git cat-file --$batch $oid + ' + done done -for opt in t s e p +for oid in $hello_oid $hello_compat_oid do - test_expect_success "Passing -$opt with --follow-symlinks fails" ' - test_must_fail git cat-file --follow-symlinks -$opt $hello_sha1 + for opt in t s e p + do + test_expect_success "Passing -$opt with --follow-symlinks fails" ' + test_must_fail git cat-file --follow-symlinks -$opt $oid ' + done done test_expect_success "--batch-check for a non-existent named object" ' @@ -360,12 +400,12 @@ test_expect_success "--batch-check for a non-existent hash" ' test_expect_success "--batch for an existent and a non-existent hash" ' cat >expect <<-EOF && - $tag_sha1 tag $tag_size + $tag_oid tag $tag_size $tag_content 0000000000000000000000000000000000000000 missing EOF - printf "$tag_sha1\n0000000000000000000000000000000000000000" >in && + printf "$tag_oid\n0000000000000000000000000000000000000000" >in && git cat-file --batch <in >actual && test_cmp expect actual ' @@ -386,112 +426,102 @@ test_expect_success 'empty --batch-check notices missing object' ' test_cmp expect actual ' -batch_input="$hello_sha1 -$commit_sha1 -$tag_sha1 +batch_tests () { + boid=$1 + loid=$2 + lsize=$3 + coid=$4 + csize=$5 + ccontent=$6 + toid=$7 + tsize=$8 + tcontent=$9 + + batch_input="$boid +$coid +$toid deadbeef " -printf "%s\0" \ - "$hello_sha1 blob $hello_size" \ + printf "%s\0" \ + "$boid blob $hello_size" \ "$hello_content" \ - "$commit_sha1 commit $commit_size" \ - "$commit_content" \ - "$tag_sha1 tag $tag_size" \ - "$tag_content" \ + "$coid commit $csize" \ + "$ccontent" \ + "$toid tag $tsize" \ + "$tcontent" \ "deadbeef missing" \ " missing" >batch_output -test_expect_success '--batch with multiple sha1s gives correct format' ' + test_expect_success '--batch with multiple oids gives correct format' ' tr "\0" "\n" <batch_output >expect && echo_without_newline "$batch_input" >in && git cat-file --batch <in >actual && test_cmp expect actual -' + ' -test_expect_success '--batch, -z with multiple sha1s gives correct format' ' + test_expect_success '--batch, -z with multiple oids gives correct format' ' echo_without_newline_nul "$batch_input" >in && tr "\0" "\n" <batch_output >expect && git cat-file --batch -z <in >actual && test_cmp expect actual -' + ' -test_expect_success '--batch, -Z with multiple sha1s gives correct format' ' + test_expect_success '--batch, -Z with multiple oids gives correct format' ' echo_without_newline_nul "$batch_input" >in && git cat-file --batch -Z <in >actual && test_cmp batch_output actual -' + ' -batch_check_input="$hello_sha1 -$tree_sha1 -$commit_sha1 -$tag_sha1 +batch_check_input="$boid +$loid +$coid +$toid deadbeef " -printf "%s\0" \ - "$hello_sha1 blob $hello_size" \ - "$tree_sha1 tree $tree_size" \ - "$commit_sha1 commit $commit_size" \ - "$tag_sha1 tag $tag_size" \ + printf "%s\0" \ + "$boid blob $hello_size" \ + "$loid tree $lsize" \ + "$coid commit $csize" \ + "$toid tag $tsize" \ "deadbeef missing" \ " missing" >batch_check_output -test_expect_success "--batch-check with multiple sha1s gives correct format" ' + test_expect_success "--batch-check with multiple oids gives correct format" ' tr "\0" "\n" <batch_check_output >expect && echo_without_newline "$batch_check_input" >in && git cat-file --batch-check <in >actual && test_cmp expect actual -' + ' -test_expect_success "--batch-check, -z with multiple sha1s gives correct format" ' + test_expect_success "--batch-check, -z with multiple oids gives correct format" ' tr "\0" "\n" <batch_check_output >expect && echo_without_newline_nul "$batch_check_input" >in && git cat-file --batch-check -z <in >actual && test_cmp expect actual -' + ' -test_expect_success "--batch-check, -Z with multiple sha1s gives correct format" ' + test_expect_success "--batch-check, -Z with multiple oids gives correct format" ' echo_without_newline_nul "$batch_check_input" >in && git cat-file --batch-check -Z <in >actual && test_cmp batch_check_output actual -' - -test_expect_success FUNNYNAMES 'setup with newline in input' ' - touch -- "newline${LF}embedded" && - git add -- "newline${LF}embedded" && - git commit -m "file with newline embedded" && - test_tick && - - printf "HEAD:newline${LF}embedded" >in -' - -test_expect_success FUNNYNAMES '--batch-check, -z with newline in input' ' - git cat-file --batch-check -z <in >actual && - echo "$(git rev-parse "HEAD:newline${LF}embedded") blob 0" >expect && - test_cmp expect actual -' - -test_expect_success FUNNYNAMES '--batch-check, -Z with newline in input' ' - git cat-file --batch-check -Z <in >actual && - printf "%s\0" "$(git rev-parse "HEAD:newline${LF}embedded") blob 0" >expect && - test_cmp expect actual -' + ' -batch_command_multiple_info="info $hello_sha1 -info $tree_sha1 -info $commit_sha1 -info $tag_sha1 +batch_command_multiple_info="info $boid +info $loid +info $coid +info $toid info deadbeef" -test_expect_success '--batch-command with multiple info calls gives correct format' ' + test_expect_success '--batch-command with multiple info calls gives correct format' ' cat >expect <<-EOF && - $hello_sha1 blob $hello_size - $tree_sha1 tree $tree_size - $commit_sha1 commit $commit_size - $tag_sha1 tag $tag_size + $boid blob $hello_size + $loid tree $lsize + $coid commit $csize + $toid tag $tsize deadbeef missing EOF @@ -510,22 +540,22 @@ test_expect_success '--batch-command with multiple info calls gives correct form git cat-file --batch-command --buffer -Z <in >actual && test_cmp expect_nul actual -' + ' -batch_command_multiple_contents="contents $hello_sha1 -contents $commit_sha1 -contents $tag_sha1 +batch_command_multiple_contents="contents $boid +contents $coid +contents $toid contents deadbeef flush" -test_expect_success '--batch-command with multiple command calls gives correct format' ' + test_expect_success '--batch-command with multiple command calls gives correct format' ' printf "%s\0" \ - "$hello_sha1 blob $hello_size" \ + "$boid blob $hello_size" \ "$hello_content" \ - "$commit_sha1 commit $commit_size" \ - "$commit_content" \ - "$tag_sha1 tag $tag_size" \ - "$tag_content" \ + "$coid commit $csize" \ + "$ccontent" \ + "$toid tag $tsize" \ + "$tcontent" \ "deadbeef missing" >expect_nul && tr "\0" "\n" <expect_nul >expect && @@ -543,6 +573,33 @@ test_expect_success '--batch-command with multiple command calls gives correct f git cat-file --batch-command --buffer -Z <in >actual && test_cmp expect_nul actual + ' + +} + +batch_tests $hello_oid $tree_oid $tree_size $commit_oid $commit_size "$commit_content" $tag_oid $tag_size "$tag_content" +batch_tests $hello_compat_oid $tree_compat_oid $tree_compat_size $commit_compat_oid $commit_compat_size "$commit_compat_content" $tag_compat_oid $tag_compat_size "$tag_compat_content" + + +test_expect_success FUNNYNAMES 'setup with newline in input' ' + touch -- "newline${LF}embedded" && + git add -- "newline${LF}embedded" && + git commit -m "file with newline embedded" && + test_tick && + + printf "HEAD:newline${LF}embedded" >in +' + +test_expect_success FUNNYNAMES '--batch-check, -z with newline in input' ' + git cat-file --batch-check -z <in >actual && + echo "$(git rev-parse "HEAD:newline${LF}embedded") blob 0" >expect && + test_cmp expect actual +' + +test_expect_success FUNNYNAMES '--batch-check, -Z with newline in input' ' + git cat-file --batch-check -Z <in >actual && + printf "%s\0" "$(git rev-parse "HEAD:newline${LF}embedded") blob 0" >expect && + test_cmp expect actual ' test_expect_success 'setup blobs which are likely to delta' ' @@ -569,7 +626,7 @@ test_expect_success 'confirm that neither loose blob is a delta' ' # we will check only that one of the two objects is a delta # against the other, but not the order. We can do so by just # asking for the base of both, and checking whether either -# sha1 appears in the output. +# oid appears in the output. test_expect_success '%(deltabase) reports packed delta bases' ' git repack -ad && git cat-file --batch-check="%(deltabase)" <blobs >actual && @@ -583,12 +640,12 @@ test_expect_success 'setup bogus data' ' bogus_short_type="bogus" && bogus_short_content="bogus" && bogus_short_size=$(strlen "$bogus_short_content") && - bogus_short_sha1=$(echo_without_newline "$bogus_short_content" | git hash-object -t $bogus_short_type --literally -w --stdin) && + bogus_short_oid=$(echo_without_newline "$bogus_short_content" | git hash-object -t $bogus_short_type --literally -w --stdin) && bogus_long_type="abcdefghijklmnopqrstuvwxyz1234679" && bogus_long_content="bogus" && bogus_long_size=$(strlen "$bogus_long_content") && - bogus_long_sha1=$(echo_without_newline "$bogus_long_content" | git hash-object -t $bogus_long_type --literally -w --stdin) + bogus_long_oid=$(echo_without_newline "$bogus_long_content" | git hash-object -t $bogus_long_type --literally -w --stdin) ' for arg1 in '' --allow-unknown-type @@ -608,9 +665,9 @@ do if test "$arg1" = "--allow-unknown-type" then - git cat-file $arg1 $arg2 $bogus_short_sha1 + git cat-file $arg1 $arg2 $bogus_short_oid else - test_must_fail git cat-file $arg1 $arg2 $bogus_short_sha1 >out 2>actual && + test_must_fail git cat-file $arg1 $arg2 $bogus_short_oid >out 2>actual && test_must_be_empty out && test_cmp expect actual fi @@ -620,21 +677,21 @@ do if test "$arg2" = "-p" then cat >expect <<-EOF - error: header for $bogus_long_sha1 too long, exceeds 32 bytes - fatal: Not a valid object name $bogus_long_sha1 + error: header for $bogus_long_oid too long, exceeds 32 bytes + fatal: Not a valid object name $bogus_long_oid EOF else cat >expect <<-EOF - error: header for $bogus_long_sha1 too long, exceeds 32 bytes + error: header for $bogus_long_oid too long, exceeds 32 bytes fatal: git cat-file: could not get object info EOF fi && if test "$arg1" = "--allow-unknown-type" then - git cat-file $arg1 $arg2 $bogus_short_sha1 + git cat-file $arg1 $arg2 $bogus_short_oid else - test_must_fail git cat-file $arg1 $arg2 $bogus_long_sha1 >out 2>actual && + test_must_fail git cat-file $arg1 $arg2 $bogus_long_oid >out 2>actual && test_must_be_empty out && test_cmp expect actual fi @@ -668,28 +725,28 @@ do done test_expect_success '-e is OK with a broken object without --allow-unknown-type' ' - git cat-file -e $bogus_short_sha1 + git cat-file -e $bogus_short_oid ' test_expect_success '-e can not be combined with --allow-unknown-type' ' - test_expect_code 128 git cat-file -e --allow-unknown-type $bogus_short_sha1 + test_expect_code 128 git cat-file -e --allow-unknown-type $bogus_short_oid ' test_expect_success '-p cannot print a broken object even with --allow-unknown-type' ' - test_must_fail git cat-file -p $bogus_short_sha1 && - test_expect_code 128 git cat-file -p --allow-unknown-type $bogus_short_sha1 + test_must_fail git cat-file -p $bogus_short_oid && + test_expect_code 128 git cat-file -p --allow-unknown-type $bogus_short_oid ' test_expect_success '<type> <hash> does not work with objects of broken types' ' cat >err.expect <<-\EOF && fatal: invalid object type "bogus" EOF - test_must_fail git cat-file $bogus_short_type $bogus_short_sha1 2>err.actual && + test_must_fail git cat-file $bogus_short_type $bogus_short_oid 2>err.actual && test_cmp err.expect err.actual ' test_expect_success 'broken types combined with --batch and --batch-check' ' - echo $bogus_short_sha1 >bogus-oid && + echo $bogus_short_oid >bogus-oid && cat >err.expect <<-\EOF && fatal: invalid object type @@ -711,52 +768,52 @@ test_expect_success 'the --allow-unknown-type option does not consider replaceme cat >expect <<-EOF && $bogus_short_type EOF - git cat-file -t --allow-unknown-type $bogus_short_sha1 >actual && + git cat-file -t --allow-unknown-type $bogus_short_oid >actual && test_cmp expect actual && # Create it manually, as "git replace" will die on bogus # types. head=$(git rev-parse --verify HEAD) && - test_when_finished "test-tool ref-store main delete-refs 0 msg refs/replace/$bogus_short_sha1" && - test-tool ref-store main update-ref msg "refs/replace/$bogus_short_sha1" $head $ZERO_OID REF_SKIP_OID_VERIFICATION && + test_when_finished "test-tool ref-store main delete-refs 0 msg refs/replace/$bogus_short_oid" && + test-tool ref-store main update-ref msg "refs/replace/$bogus_short_oid" $head $ZERO_OID REF_SKIP_OID_VERIFICATION && cat >expect <<-EOF && commit EOF - git cat-file -t --allow-unknown-type $bogus_short_sha1 >actual && + git cat-file -t --allow-unknown-type $bogus_short_oid >actual && test_cmp expect actual ' test_expect_success "Type of broken object is correct" ' echo $bogus_short_type >expect && - git cat-file -t --allow-unknown-type $bogus_short_sha1 >actual && + git cat-file -t --allow-unknown-type $bogus_short_oid >actual && test_cmp expect actual ' test_expect_success "Size of broken object is correct" ' echo $bogus_short_size >expect && - git cat-file -s --allow-unknown-type $bogus_short_sha1 >actual && + git cat-file -s --allow-unknown-type $bogus_short_oid >actual && test_cmp expect actual ' test_expect_success 'clean up broken object' ' - rm .git/objects/$(test_oid_to_path $bogus_short_sha1) + rm .git/objects/$(test_oid_to_path $bogus_short_oid) ' test_expect_success "Type of broken object is correct when type is large" ' echo $bogus_long_type >expect && - git cat-file -t --allow-unknown-type $bogus_long_sha1 >actual && + git cat-file -t --allow-unknown-type $bogus_long_oid >actual && test_cmp expect actual ' test_expect_success "Size of large broken object is correct when type is large" ' echo $bogus_long_size >expect && - git cat-file -s --allow-unknown-type $bogus_long_sha1 >actual && + git cat-file -s --allow-unknown-type $bogus_long_oid >actual && test_cmp expect actual ' test_expect_success 'clean up broken object' ' - rm .git/objects/$(test_oid_to_path $bogus_long_sha1) + rm .git/objects/$(test_oid_to_path $bogus_long_oid) ' test_expect_success 'cat-file -t and -s on corrupt loose object' ' @@ -853,7 +910,7 @@ test_expect_success 'prep for symlink tests' ' test_ln_s_add loop2 loop1 && git add morx dir/subdir/ind2 dir/ind1 && git commit -am "test" && - echo $hello_sha1 blob $hello_size >found + echo $hello_oid blob $hello_size >found ' test_expect_success 'git cat-file --batch-check --follow-symlinks works for non-links' ' @@ -941,7 +998,7 @@ test_expect_success 'git cat-file --batch-check --follow-symlinks works for dir/ echo HEAD:dirlink/morx >>expect && echo HEAD:dirlink/morx | git cat-file --batch-check --follow-symlinks >actual && test_cmp expect actual && - echo $hello_sha1 blob $hello_size >expect && + echo $hello_oid blob $hello_size >expect && echo HEAD:dirlink/ind1 | git cat-file --batch-check --follow-symlinks >actual && test_cmp expect actual ' @@ -1100,6 +1157,42 @@ test_expect_success 'cat-file --batch="batman" with --batch-all-objects will wor cmp expect actual ' +test_expect_success 'cat-file %(objectsize:disk) with --batch-all-objects' ' + # our state has both loose and packed objects, + # so find both for our expected output + { + find .git/objects/?? -type f | + awk -F/ "{ print \$0, \$3\$4 }" | + while read path oid + do + size=$(test_file_size "$path") && + echo "$oid $size" || + return 1 + done && + rawsz=$(test_oid rawsz) && + find .git/objects/pack -name "*.idx" | + while read idx + do + git show-index <"$idx" >idx.raw && + sort -nr <idx.raw >idx.sorted && + packsz=$(test_file_size "${idx%.idx}.pack") && + end=$((packsz - rawsz)) && + while read start oid rest + do + size=$((end - start)) && + end=$start && + echo "$oid $size" || + return 1 + done <idx.sorted || + return 1 + done + } >expect.raw && + sort <expect.raw >expect && + git cat-file --batch-all-objects \ + --batch-check="%(objectname) %(objectsize:disk)" >actual && + test_cmp expect actual +' + test_expect_success 'set up replacement object' ' orig=$(git rev-parse HEAD) && git cat-file commit $orig >orig && diff --git a/t/t1016-compatObjectFormat.sh b/t/t1016-compatObjectFormat.sh new file mode 100755 index 0000000000..8132cd37b8 --- /dev/null +++ b/t/t1016-compatObjectFormat.sh @@ -0,0 +1,281 @@ +#!/bin/sh +# +# Copyright (c) 2023 Eric Biederman +# + +test_description='Test how well compatObjectFormat works' + +TEST_PASSES_SANITIZE_LEAK=true +. ./test-lib.sh +. "$TEST_DIRECTORY"/lib-gpg.sh + +# All of the follow variables must be defined in the environment: +# GIT_AUTHOR_NAME +# GIT_AUTHOR_EMAIL +# GIT_AUTHOR_DATE +# GIT_COMMITTER_NAME +# GIT_COMMITTER_EMAIL +# GIT_COMMITTER_DATE +# +# The test relies on these variables being set so that the two +# different commits in two different repositories encoded with two +# different hash functions result in the same content in the commits. +# This means that when the commit is translated between hash functions +# the commit is identical to the commit in the other repository. + +compat_hash () { + case "$1" in + "sha1") + echo "sha256" + ;; + "sha256") + echo "sha1" + ;; + esac +} + +hello_oid () { + case "$1" in + "sha1") + echo "$hello_sha1_oid" + ;; + "sha256") + echo "$hello_sha256_oid" + ;; + esac +} + +tree_oid () { + case "$1" in + "sha1") + echo "$tree_sha1_oid" + ;; + "sha256") + echo "$tree_sha256_oid" + ;; + esac +} + +commit_oid () { + case "$1" in + "sha1") + echo "$commit_sha1_oid" + ;; + "sha256") + echo "$commit_sha256_oid" + ;; + esac +} + +commit2_oid () { + case "$1" in + "sha1") + echo "$commit2_sha1_oid" + ;; + "sha256") + echo "$commit2_sha256_oid" + ;; + esac +} + +del_sigcommit () { + local delete=$1 + + if test "$delete" = "sha256" ; then + local pattern="gpgsig-sha256" + else + local pattern="gpgsig" + fi + test-tool delete-gpgsig "$pattern" +} + + +del_sigtag () { + local storage=$1 + local delete=$2 + + if test "$storage" = "$delete" ; then + local pattern="trailer" + elif test "$storage" = "sha256" ; then + local pattern="gpgsig" + else + local pattern="gpgsig-sha256" + fi + test-tool delete-gpgsig "$pattern" +} + +base=$(pwd) +for hash in sha1 sha256 +do + cd "$base" + mkdir -p repo-$hash + cd repo-$hash + + test_expect_success "setup $hash repository" ' + git init --object-format=$hash && + git config core.repositoryformatversion 1 && + git config extensions.objectformat $hash && + git config extensions.compatobjectformat $(compat_hash $hash) && + git config gpg.program $TEST_DIRECTORY/t1016/gpg && + echo "Hellow World!" > hello && + eval hello_${hash}_oid=$(git hash-object hello) && + git update-index --add hello && + git commit -m "Initial commit" && + eval commit_${hash}_oid=$(git rev-parse HEAD) && + eval tree_${hash}_oid=$(git rev-parse HEAD^{tree}) + ' + test_expect_success "create a $hash tagged blob" ' + git tag --no-sign -m "This is a tag" hellotag $(hello_oid $hash) && + eval hellotag_${hash}_oid=$(git rev-parse hellotag) + ' + test_expect_success "create a $hash tagged tree" ' + git tag --no-sign -m "This is a tag" treetag $(tree_oid $hash) && + eval treetag_${hash}_oid=$(git rev-parse treetag) + ' + test_expect_success "create a $hash tagged commit" ' + git tag --no-sign -m "This is a tag" committag $(commit_oid $hash) && + eval committag_${hash}_oid=$(git rev-parse committag) + ' + test_expect_success GPG2 "create a $hash signed commit" ' + git commit --gpg-sign --allow-empty -m "This is a signed commit" && + eval signedcommit_${hash}_oid=$(git rev-parse HEAD) + ' + test_expect_success GPG2 "create a $hash signed tag" ' + git tag -s -m "This is a signed tag" signedtag HEAD && + eval signedtag_${hash}_oid=$(git rev-parse signedtag) + ' + test_expect_success "create a $hash branch" ' + git checkout -b branch $(commit_oid $hash) && + echo "More more more give me more!" > more && + eval more_${hash}_oid=$(git hash-object more) && + echo "Another and another and another" > another && + eval another_${hash}_oid=$(git hash-object another) && + git update-index --add more another && + git commit -m "Add more files!" && + eval commit2_${hash}_oid=$(git rev-parse HEAD) && + eval tree2_${hash}_oid=$(git rev-parse HEAD^{tree}) + ' + test_expect_success GPG2 "create another $hash signed tag" ' + git tag -s -m "This is another signed tag" signedtag2 $(commit2_oid $hash) && + eval signedtag2_${hash}_oid=$(git rev-parse signedtag2) + ' + test_expect_success GPG2 "merge the $hash branches together" ' + git merge -S -m "merge some signed tags together" signedtag signedtag2 && + eval signedcommit2_${hash}_oid=$(git rev-parse HEAD) + ' + test_expect_success GPG2 "create additional $hash signed commits" ' + git commit --gpg-sign --allow-empty -m "This is an additional signed commit" && + git cat-file commit HEAD | del_sigcommit sha256 > "../${hash}_signedcommit3" && + git cat-file commit HEAD | del_sigcommit sha1 > "../${hash}_signedcommit4" && + eval signedcommit3_${hash}_oid=$(git hash-object -t commit -w ../${hash}_signedcommit3) && + eval signedcommit4_${hash}_oid=$(git hash-object -t commit -w ../${hash}_signedcommit4) + ' + test_expect_success GPG2 "create additional $hash signed tags" ' + git tag -s -m "This is an additional signed tag" signedtag34 HEAD && + git cat-file tag signedtag34 | del_sigtag "${hash}" sha256 > ../${hash}_signedtag3 && + git cat-file tag signedtag34 | del_sigtag "${hash}" sha1 > ../${hash}_signedtag4 && + eval signedtag3_${hash}_oid=$(git hash-object -t tag -w ../${hash}_signedtag3) && + eval signedtag4_${hash}_oid=$(git hash-object -t tag -w ../${hash}_signedtag4) + ' +done +cd "$base" + +compare_oids () { + test "$#" = 5 && { local PREREQ=$1; shift; } || PREREQ= + local type="$1" + local name="$2" + local sha1_oid="$3" + local sha256_oid="$4" + + echo ${sha1_oid} > ${name}_sha1_expected + echo ${sha256_oid} > ${name}_sha256_expected + echo ${type} > ${name}_type_expected + + git --git-dir=repo-sha1/.git rev-parse --output-object-format=sha256 ${sha1_oid} > ${name}_sha1_sha256_found + git --git-dir=repo-sha256/.git rev-parse --output-object-format=sha1 ${sha256_oid} > ${name}_sha256_sha1_found + local sha1_sha256_oid=$(cat ${name}_sha1_sha256_found) + local sha256_sha1_oid=$(cat ${name}_sha256_sha1_found) + + test_expect_success $PREREQ "Verify ${type} ${name}'s sha1 oid" ' + git --git-dir=repo-sha256/.git rev-parse --output-object-format=sha1 ${sha256_oid} > ${name}_sha1 && + test_cmp ${name}_sha1 ${name}_sha1_expected +' + + test_expect_success $PREREQ "Verify ${type} ${name}'s sha256 oid" ' + git --git-dir=repo-sha1/.git rev-parse --output-object-format=sha256 ${sha1_oid} > ${name}_sha256 && + test_cmp ${name}_sha256 ${name}_sha256_expected +' + + test_expect_success $PREREQ "Verify ${name}'s sha1 type" ' + git --git-dir=repo-sha1/.git cat-file -t ${sha1_oid} > ${name}_type1 && + git --git-dir=repo-sha256/.git cat-file -t ${sha256_sha1_oid} > ${name}_type2 && + test_cmp ${name}_type1 ${name}_type2 && + test_cmp ${name}_type1 ${name}_type_expected +' + + test_expect_success $PREREQ "Verify ${name}'s sha256 type" ' + git --git-dir=repo-sha256/.git cat-file -t ${sha256_oid} > ${name}_type3 && + git --git-dir=repo-sha1/.git cat-file -t ${sha1_sha256_oid} > ${name}_type4 && + test_cmp ${name}_type3 ${name}_type4 && + test_cmp ${name}_type3 ${name}_type_expected +' + + test_expect_success $PREREQ "Verify ${name}'s sha1 size" ' + git --git-dir=repo-sha1/.git cat-file -s ${sha1_oid} > ${name}_size1 && + git --git-dir=repo-sha256/.git cat-file -s ${sha256_sha1_oid} > ${name}_size2 && + test_cmp ${name}_size1 ${name}_size2 +' + + test_expect_success $PREREQ "Verify ${name}'s sha256 size" ' + git --git-dir=repo-sha256/.git cat-file -s ${sha256_oid} > ${name}_size3 && + git --git-dir=repo-sha1/.git cat-file -s ${sha1_sha256_oid} > ${name}_size4 && + test_cmp ${name}_size3 ${name}_size4 +' + + test_expect_success $PREREQ "Verify ${name}'s sha1 pretty content" ' + git --git-dir=repo-sha1/.git cat-file -p ${sha1_oid} > ${name}_content1 && + git --git-dir=repo-sha256/.git cat-file -p ${sha256_sha1_oid} > ${name}_content2 && + test_cmp ${name}_content1 ${name}_content2 +' + + test_expect_success $PREREQ "Verify ${name}'s sha256 pretty content" ' + git --git-dir=repo-sha256/.git cat-file -p ${sha256_oid} > ${name}_content3 && + git --git-dir=repo-sha1/.git cat-file -p ${sha1_sha256_oid} > ${name}_content4 && + test_cmp ${name}_content3 ${name}_content4 +' + + test_expect_success $PREREQ "Verify ${name}'s sha1 content" ' + git --git-dir=repo-sha1/.git cat-file ${type} ${sha1_oid} > ${name}_content5 && + git --git-dir=repo-sha256/.git cat-file ${type} ${sha256_sha1_oid} > ${name}_content6 && + test_cmp ${name}_content5 ${name}_content6 +' + + test_expect_success $PREREQ "Verify ${name}'s sha256 content" ' + git --git-dir=repo-sha256/.git cat-file ${type} ${sha256_oid} > ${name}_content7 && + git --git-dir=repo-sha1/.git cat-file ${type} ${sha1_sha256_oid} > ${name}_content8 && + test_cmp ${name}_content7 ${name}_content8 +' + +} + +compare_oids 'blob' hello "$hello_sha1_oid" "$hello_sha256_oid" +compare_oids 'tree' tree "$tree_sha1_oid" "$tree_sha256_oid" +compare_oids 'commit' commit "$commit_sha1_oid" "$commit_sha256_oid" +compare_oids GPG2 'commit' signedcommit "$signedcommit_sha1_oid" "$signedcommit_sha256_oid" +compare_oids 'tag' hellotag "$hellotag_sha1_oid" "$hellotag_sha256_oid" +compare_oids 'tag' treetag "$treetag_sha1_oid" "$treetag_sha256_oid" +compare_oids 'tag' committag "$committag_sha1_oid" "$committag_sha256_oid" +compare_oids GPG2 'tag' signedtag "$signedtag_sha1_oid" "$signedtag_sha256_oid" + +compare_oids 'blob' more "$more_sha1_oid" "$more_sha256_oid" +compare_oids 'blob' another "$another_sha1_oid" "$another_sha256_oid" +compare_oids 'tree' tree2 "$tree2_sha1_oid" "$tree2_sha256_oid" +compare_oids 'commit' commit2 "$commit2_sha1_oid" "$commit2_sha256_oid" +compare_oids GPG2 'tag' signedtag2 "$signedtag2_sha1_oid" "$signedtag2_sha256_oid" +compare_oids GPG2 'commit' signedcommit2 "$signedcommit2_sha1_oid" "$signedcommit2_sha256_oid" +compare_oids GPG2 'commit' signedcommit3 "$signedcommit3_sha1_oid" "$signedcommit3_sha256_oid" +compare_oids GPG2 'commit' signedcommit4 "$signedcommit4_sha1_oid" "$signedcommit4_sha256_oid" +compare_oids GPG2 'tag' signedtag3 "$signedtag3_sha1_oid" "$signedtag3_sha256_oid" +compare_oids GPG2 'tag' signedtag4 "$signedtag4_sha1_oid" "$signedtag4_sha256_oid" + +test_done diff --git a/t/t1016/gpg b/t/t1016/gpg new file mode 100755 index 0000000000..2601cb18a5 --- /dev/null +++ b/t/t1016/gpg @@ -0,0 +1,2 @@ +#!/bin/sh +exec gpg --faked-system-time "20230918T154812" "$@" diff --git a/t/t1401-symbolic-ref.sh b/t/t1401-symbolic-ref.sh index 3241d35917..5c60d6f812 100755 --- a/t/t1401-symbolic-ref.sh +++ b/t/t1401-symbolic-ref.sh @@ -106,9 +106,8 @@ test_expect_success LONG_REF 'we can parse long symbolic ref' ' ' test_expect_success 'symbolic-ref reports failure in exit code' ' - test_when_finished "rm -f .git/HEAD.lock" && - >.git/HEAD.lock && - test_must_fail git symbolic-ref HEAD refs/heads/whatever + # Create d/f conflict to simulate failure. + test_must_fail git symbolic-ref refs/heads refs/heads/foo ' test_expect_success 'symbolic-ref writes reflog entry' ' diff --git a/t/t1403-show-ref.sh b/t/t1403-show-ref.sh index ec1957b709..d0a8f7b121 100755 --- a/t/t1403-show-ref.sh +++ b/t/t1403-show-ref.sh @@ -262,9 +262,9 @@ test_expect_success '--exists with non-commit object' ' test_expect_success '--exists with directory fails with generic error' ' cat >expect <<-EOF && - error: failed to look up reference: Is a directory + error: reference does not exist EOF - test_expect_code 1 git show-ref --exists refs/heads 2>err && + test_expect_code 2 git show-ref --exists refs/heads 2>err && test_cmp expect err ' diff --git a/t/t1404-update-ref-errors.sh b/t/t1404-update-ref-errors.sh index 0369beea33..00b7013705 100755 --- a/t/t1404-update-ref-errors.sh +++ b/t/t1404-update-ref-errors.sh @@ -191,78 +191,6 @@ test_expect_success 'one new ref is a simple prefix of another' ' ' -test_expect_success REFFILES 'empty directory should not fool rev-parse' ' - prefix=refs/e-rev-parse && - git update-ref $prefix/foo $C && - git pack-refs --all && - mkdir -p .git/$prefix/foo/bar/baz && - echo "$C" >expected && - git rev-parse $prefix/foo >actual && - test_cmp expected actual -' - -test_expect_success REFFILES 'empty directory should not fool for-each-ref' ' - prefix=refs/e-for-each-ref && - git update-ref $prefix/foo $C && - git for-each-ref $prefix >expected && - git pack-refs --all && - mkdir -p .git/$prefix/foo/bar/baz && - git for-each-ref $prefix >actual && - test_cmp expected actual -' - -test_expect_success REFFILES 'empty directory should not fool create' ' - prefix=refs/e-create && - mkdir -p .git/$prefix/foo/bar/baz && - printf "create %s $C\n" $prefix/foo | - git update-ref --stdin -' - -test_expect_success REFFILES 'empty directory should not fool verify' ' - prefix=refs/e-verify && - git update-ref $prefix/foo $C && - git pack-refs --all && - mkdir -p .git/$prefix/foo/bar/baz && - printf "verify %s $C\n" $prefix/foo | - git update-ref --stdin -' - -test_expect_success REFFILES 'empty directory should not fool 1-arg update' ' - prefix=refs/e-update-1 && - git update-ref $prefix/foo $C && - git pack-refs --all && - mkdir -p .git/$prefix/foo/bar/baz && - printf "update %s $D\n" $prefix/foo | - git update-ref --stdin -' - -test_expect_success REFFILES 'empty directory should not fool 2-arg update' ' - prefix=refs/e-update-2 && - git update-ref $prefix/foo $C && - git pack-refs --all && - mkdir -p .git/$prefix/foo/bar/baz && - printf "update %s $D $C\n" $prefix/foo | - git update-ref --stdin -' - -test_expect_success REFFILES 'empty directory should not fool 0-arg delete' ' - prefix=refs/e-delete-0 && - git update-ref $prefix/foo $C && - git pack-refs --all && - mkdir -p .git/$prefix/foo/bar/baz && - printf "delete %s\n" $prefix/foo | - git update-ref --stdin -' - -test_expect_success REFFILES 'empty directory should not fool 1-arg delete' ' - prefix=refs/e-delete-1 && - git update-ref $prefix/foo $C && - git pack-refs --all && - mkdir -p .git/$prefix/foo/bar/baz && - printf "delete %s $C\n" $prefix/foo | - git update-ref --stdin -' - test_expect_success REFFILES 'D/F conflict prevents add long + delete short' ' df_test refs/df-al-ds --add-del foo/bar foo ' @@ -468,169 +396,4 @@ test_expect_success 'incorrect old value blocks indirect no-deref delete' ' test_cmp expected output.err ' -test_expect_success REFFILES 'non-empty directory blocks create' ' - prefix=refs/ne-create && - mkdir -p .git/$prefix/foo/bar && - : >.git/$prefix/foo/bar/baz.lock && - test_when_finished "rm -f .git/$prefix/foo/bar/baz.lock" && - cat >expected <<-EOF && - fatal: cannot lock ref $SQ$prefix/foo$SQ: there is a non-empty directory $SQ.git/$prefix/foo$SQ blocking reference $SQ$prefix/foo$SQ - EOF - printf "%s\n" "update $prefix/foo $C" | - test_must_fail git update-ref --stdin 2>output.err && - test_cmp expected output.err && - cat >expected <<-EOF && - fatal: cannot lock ref $SQ$prefix/foo$SQ: unable to resolve reference $SQ$prefix/foo$SQ - EOF - printf "%s\n" "update $prefix/foo $D $C" | - test_must_fail git update-ref --stdin 2>output.err && - test_cmp expected output.err -' - -test_expect_success REFFILES 'broken reference blocks create' ' - prefix=refs/broken-create && - mkdir -p .git/$prefix && - echo "gobbledigook" >.git/$prefix/foo && - test_when_finished "rm -f .git/$prefix/foo" && - cat >expected <<-EOF && - fatal: cannot lock ref $SQ$prefix/foo$SQ: unable to resolve reference $SQ$prefix/foo$SQ: reference broken - EOF - printf "%s\n" "update $prefix/foo $C" | - test_must_fail git update-ref --stdin 2>output.err && - test_cmp expected output.err && - cat >expected <<-EOF && - fatal: cannot lock ref $SQ$prefix/foo$SQ: unable to resolve reference $SQ$prefix/foo$SQ: reference broken - EOF - printf "%s\n" "update $prefix/foo $D $C" | - test_must_fail git update-ref --stdin 2>output.err && - test_cmp expected output.err -' - -test_expect_success REFFILES 'non-empty directory blocks indirect create' ' - prefix=refs/ne-indirect-create && - git symbolic-ref $prefix/symref $prefix/foo && - mkdir -p .git/$prefix/foo/bar && - : >.git/$prefix/foo/bar/baz.lock && - test_when_finished "rm -f .git/$prefix/foo/bar/baz.lock" && - cat >expected <<-EOF && - fatal: cannot lock ref $SQ$prefix/symref$SQ: there is a non-empty directory $SQ.git/$prefix/foo$SQ blocking reference $SQ$prefix/foo$SQ - EOF - printf "%s\n" "update $prefix/symref $C" | - test_must_fail git update-ref --stdin 2>output.err && - test_cmp expected output.err && - cat >expected <<-EOF && - fatal: cannot lock ref $SQ$prefix/symref$SQ: unable to resolve reference $SQ$prefix/foo$SQ - EOF - printf "%s\n" "update $prefix/symref $D $C" | - test_must_fail git update-ref --stdin 2>output.err && - test_cmp expected output.err -' - -test_expect_success REFFILES 'broken reference blocks indirect create' ' - prefix=refs/broken-indirect-create && - git symbolic-ref $prefix/symref $prefix/foo && - echo "gobbledigook" >.git/$prefix/foo && - test_when_finished "rm -f .git/$prefix/foo" && - cat >expected <<-EOF && - fatal: cannot lock ref $SQ$prefix/symref$SQ: unable to resolve reference $SQ$prefix/foo$SQ: reference broken - EOF - printf "%s\n" "update $prefix/symref $C" | - test_must_fail git update-ref --stdin 2>output.err && - test_cmp expected output.err && - cat >expected <<-EOF && - fatal: cannot lock ref $SQ$prefix/symref$SQ: unable to resolve reference $SQ$prefix/foo$SQ: reference broken - EOF - printf "%s\n" "update $prefix/symref $D $C" | - test_must_fail git update-ref --stdin 2>output.err && - test_cmp expected output.err -' - -test_expect_success REFFILES 'no bogus intermediate values during delete' ' - prefix=refs/slow-transaction && - # Set up a reference with differing loose and packed versions: - git update-ref $prefix/foo $C && - git pack-refs --all && - git update-ref $prefix/foo $D && - # Now try to update the reference, but hold the `packed-refs` lock - # for a while to see what happens while the process is blocked: - : >.git/packed-refs.lock && - test_when_finished "rm -f .git/packed-refs.lock" && - { - # Note: the following command is intentionally run in the - # background. We increase the timeout so that `update-ref` - # attempts to acquire the `packed-refs` lock for much longer - # than it takes for us to do the check then delete it: - git -c core.packedrefstimeout=30000 update-ref -d $prefix/foo & - } && - pid2=$! && - # Give update-ref plenty of time to get to the point where it tries - # to lock packed-refs: - sleep 1 && - # Make sure that update-ref did not complete despite the lock: - kill -0 $pid2 && - # Verify that the reference still has its old value: - sha1=$(git rev-parse --verify --quiet $prefix/foo || echo undefined) && - case "$sha1" in - $D) - # This is what we hope for; it means that nothing - # user-visible has changed yet. - : ;; - undefined) - # This is not correct; it means the deletion has happened - # already even though update-ref should not have been - # able to acquire the lock yet. - echo "$prefix/foo deleted prematurely" && - break - ;; - $C) - # This value should never be seen. Probably the loose - # reference has been deleted but the packed reference - # is still there: - echo "$prefix/foo incorrectly observed to be C" && - break - ;; - *) - # WTF? - echo "unexpected value observed for $prefix/foo: $sha1" && - break - ;; - esac >out && - rm -f .git/packed-refs.lock && - wait $pid2 && - test_must_be_empty out && - test_must_fail git rev-parse --verify --quiet $prefix/foo -' - -test_expect_success REFFILES 'delete fails cleanly if packed-refs file is locked' ' - prefix=refs/locked-packed-refs && - # Set up a reference with differing loose and packed versions: - git update-ref $prefix/foo $C && - git pack-refs --all && - git update-ref $prefix/foo $D && - git for-each-ref $prefix >unchanged && - # Now try to delete it while the `packed-refs` lock is held: - : >.git/packed-refs.lock && - test_when_finished "rm -f .git/packed-refs.lock" && - test_must_fail git update-ref -d $prefix/foo >out 2>err && - git for-each-ref $prefix >actual && - test_grep "Unable to create $SQ.*packed-refs.lock$SQ: " err && - test_cmp unchanged actual -' - -test_expect_success REFFILES 'delete fails cleanly if packed-refs.new write fails' ' - # Setup and expectations are similar to the test above. - prefix=refs/failed-packed-refs && - git update-ref $prefix/foo $C && - git pack-refs --all && - git update-ref $prefix/foo $D && - git for-each-ref $prefix >unchanged && - # This should not happen in practice, but it is an easy way to get a - # reliable error (we open with create_tempfile(), which uses O_EXCL). - : >.git/packed-refs.new && - test_when_finished "rm -f .git/packed-refs.new" && - test_must_fail git update-ref -d $prefix/foo && - git for-each-ref $prefix >actual && - test_cmp unchanged actual -' - test_done diff --git a/t/t1405-main-ref-store.sh b/t/t1405-main-ref-store.sh index e4627cf1b6..976bd71efb 100755 --- a/t/t1405-main-ref-store.sh +++ b/t/t1405-main-ref-store.sh @@ -15,14 +15,6 @@ test_expect_success 'setup' ' test_commit one ' -test_expect_success REFFILES 'pack_refs(PACK_REFS_ALL | PACK_REFS_PRUNE)' ' - N=`find .git/refs -type f | wc -l` && - test "$N" != 0 && - $RUN pack-refs PACK_REFS_PRUNE,PACK_REFS_ALL && - N=`find .git/refs -type f` && - test -z "$N" -' - test_expect_success 'create_symref(FOO, refs/heads/main)' ' $RUN create-symref FOO refs/heads/main nothing && echo refs/heads/main >expected && @@ -112,7 +104,7 @@ test_expect_success 'delete_reflog(HEAD)' ' test_must_fail git reflog exists HEAD ' -test_expect_success REFFILES 'create-reflog(HEAD)' ' +test_expect_success 'create-reflog(HEAD)' ' $RUN create-reflog HEAD && git reflog exists HEAD ' diff --git a/t/t1407-worktree-ref-store.sh b/t/t1407-worktree-ref-store.sh index 05b1881c59..48b1c92a41 100755 --- a/t/t1407-worktree-ref-store.sh +++ b/t/t1407-worktree-ref-store.sh @@ -53,41 +53,4 @@ test_expect_success 'create_symref(FOO, refs/heads/main)' ' test_cmp expected actual ' -# Some refs (refs/bisect/*, pseudorefs) are kept per worktree, so they should -# only appear in the for-each-reflog output if it is called from the correct -# worktree, which is exercised in this test. This test is poorly written (and -# therefore marked REFFILES) for mulitple reasons: 1) it creates invalidly -# formatted log entres. 2) it uses direct FS access for creating the reflogs. 3) -# PSEUDO-WT and refs/bisect/random do not create reflogs by default, so it is -# not testing a realistic scenario. -test_expect_success REFFILES 'for_each_reflog()' ' - echo $ZERO_OID > .git/logs/PSEUDO-MAIN && - mkdir -p .git/logs/refs/bisect && - echo $ZERO_OID > .git/logs/refs/bisect/random && - - echo $ZERO_OID > .git/worktrees/wt/logs/PSEUDO-WT && - mkdir -p .git/worktrees/wt/logs/refs/bisect && - echo $ZERO_OID > .git/worktrees/wt/logs/refs/bisect/wt-random && - - $RWT for-each-reflog | cut -d" " -f 2- | sort >actual && - cat >expected <<-\EOF && - HEAD 0x1 - PSEUDO-WT 0x0 - refs/bisect/wt-random 0x0 - refs/heads/main 0x0 - refs/heads/wt-main 0x0 - EOF - test_cmp expected actual && - - $RMAIN for-each-reflog | cut -d" " -f 2- | sort >actual && - cat >expected <<-\EOF && - HEAD 0x1 - PSEUDO-MAIN 0x0 - refs/bisect/random 0x0 - refs/heads/main 0x0 - refs/heads/wt-main 0x0 - EOF - test_cmp expected actual -' - test_done diff --git a/t/t1410-reflog.sh b/t/t1410-reflog.sh index a0ff8d51f0..d2f5f42e67 100755 --- a/t/t1410-reflog.sh +++ b/t/t1410-reflog.sh @@ -354,36 +354,6 @@ test_expect_success 'stale dirs do not cause d/f conflicts (reflogs off)' ' test_must_be_empty actual ' -# Triggering the bug detected by this test requires a newline to fall -# exactly BUFSIZ-1 bytes from the end of the file. We don't know -# what that value is, since it's platform dependent. However, if -# we choose some value N, we also catch any D which divides N evenly -# (since we will read backwards in chunks of D). So we choose 8K, -# which catches glibc (with an 8K BUFSIZ) and *BSD (1K). -# -# Each line is 114 characters, so we need 75 to still have a few before the -# last 8K. The 89-character padding on the final entry lines up our -# newline exactly. -test_expect_success REFFILES,SHA1 'parsing reverse reflogs at BUFSIZ boundaries' ' - git checkout -b reflogskip && - zf=$(test_oid zero_2) && - ident="abc <xyz> 0000000001 +0000" && - for i in $(test_seq 1 75); do - printf "$zf%02d $zf%02d %s\t" $i $(($i+1)) "$ident" && - if test $i = 75; then - for j in $(test_seq 1 89); do - printf X || return 1 - done - else - printf X - fi && - printf "\n" || return 1 - done >.git/logs/refs/heads/reflogskip && - git rev-parse reflogskip@{73} >actual && - echo ${zf}03 >expect && - test_cmp expect actual -' - test_expect_success 'no segfaults for reflog containing non-commit sha1s' ' git update-ref --create-reflog -m "Creating ref" \ refs/tests/tree-in-reflog HEAD && @@ -397,18 +367,6 @@ test_expect_failure 'reflog with non-commit entries displays all entries' ' test_line_count = 3 actual ' -# This test takes a lock on an individual ref; this is not supported in -# reftable. -test_expect_success REFFILES 'reflog expire operates on symref not referrent' ' - git branch --create-reflog the_symref && - git branch --create-reflog referrent && - git update-ref referrent HEAD && - git symbolic-ref refs/heads/the_symref refs/heads/referrent && - test_when_finished "rm -f .git/refs/heads/referrent.lock" && - touch .git/refs/heads/referrent.lock && - git reflog expire --expire=all the_symref -' - test_expect_success 'continue walking past root commits' ' git init orphanage && ( diff --git a/t/t1414-reflog-walk.sh b/t/t1414-reflog-walk.sh index ea64cecf47..be6c3f472c 100755 --- a/t/t1414-reflog-walk.sh +++ b/t/t1414-reflog-walk.sh @@ -121,13 +121,12 @@ test_expect_success 'min/max age uses entry date to limit' ' # Create a situation where the reflog and ref database disagree about the latest # state of HEAD. -test_expect_success REFFILES 'walk prefers reflog to ref tip' ' +test_expect_success 'walk prefers reflog to ref tip' ' + test_commit A && + test_commit B && + git reflog delete HEAD@{0} && head=$(git rev-parse HEAD) && - one=$(git rev-parse one) && - ident="$GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE" && - echo "$head $one $ident broken reflog entry" >>.git/logs/HEAD && - - echo $one >expect && + git rev-parse A >expect && git log -g --format=%H -1 >actual && test_cmp expect actual ' diff --git a/t/t1415-worktree-refs.sh b/t/t1415-worktree-refs.sh index 3b531842dd..eb4eec8bec 100755 --- a/t/t1415-worktree-refs.sh +++ b/t/t1415-worktree-refs.sh @@ -17,17 +17,6 @@ test_expect_success 'setup' ' git -C wt2 update-ref refs/worktree/foo HEAD ' -# The 'packed-refs' file is stored directly in .git/. This means it is global -# to the repository, and can only contain refs that are shared across all -# worktrees. -test_expect_success REFFILES 'refs/worktree must not be packed' ' - git pack-refs --all && - test_path_is_missing .git/refs/tags/wt1 && - test_path_is_file .git/refs/worktree/foo && - test_path_is_file .git/worktrees/wt1/refs/worktree/foo && - test_path_is_file .git/worktrees/wt2/refs/worktree/foo -' - test_expect_success 'refs/worktree are per-worktree' ' test_cmp_rev worktree/foo initial && ( cd wt1 && test_cmp_rev worktree/foo wt1 ) && diff --git a/t/t1500-rev-parse.sh b/t/t1500-rev-parse.sh index 3f9e7f62e4..a669e592f1 100755 --- a/t/t1500-rev-parse.sh +++ b/t/t1500-rev-parse.sh @@ -208,6 +208,23 @@ test_expect_success 'rev-parse --show-object-format in repo' ' grep "unknown mode for --show-object-format: squeamish-ossifrage" err ' +test_expect_success 'rev-parse --show-ref-format' ' + test_detect_ref_format >expect && + git rev-parse --show-ref-format >actual && + test_cmp expect actual +' + +test_expect_success 'rev-parse --show-ref-format with invalid storage' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + git config extensions.refstorage broken && + test_must_fail git rev-parse --show-ref-format 2>err && + grep "error: invalid value for ${SQ}extensions.refstorage${SQ}: ${SQ}broken${SQ}" err + ) +' + test_expect_success '--show-toplevel from subdir of working tree' ' pwd >expect && git -C sub/dir rev-parse --show-toplevel >actual && diff --git a/t/t1503-rev-parse-verify.sh b/t/t1503-rev-parse-verify.sh index bc136833c1..79df65ec7f 100755 --- a/t/t1503-rev-parse-verify.sh +++ b/t/t1503-rev-parse-verify.sh @@ -144,11 +144,6 @@ test_expect_success 'main@{n} for various n' ' test_must_fail git rev-parse --verify main@{$Np1} ' -test_expect_success SYMLINKS,REFFILES 'ref resolution not confused by broken symlinks' ' - ln -s does-not-exist .git/refs/heads/broken && - test_must_fail git rev-parse --verify broken -' - test_expect_success 'options can appear after --verify' ' git rev-parse --verify HEAD >expect && git rev-parse --verify -q HEAD >actual && diff --git a/t/t2017-checkout-orphan.sh b/t/t2017-checkout-orphan.sh index 947d1587ac..a5c7358eea 100755 --- a/t/t2017-checkout-orphan.sh +++ b/t/t2017-checkout-orphan.sh @@ -86,7 +86,7 @@ test_expect_success '--orphan makes reflog by default' ' git rev-parse --verify delta@{0} ' -test_expect_success REFFILES '--orphan does not make reflog when core.logAllRefUpdates = false' ' +test_expect_success '--orphan does not make reflog when core.logAllRefUpdates = false' ' git checkout main && git config core.logAllRefUpdates false && git checkout --orphan epsilon && diff --git a/t/t2400-worktree-add.sh b/t/t2400-worktree-add.sh index 3742971105..c28c04133c 100755 --- a/t/t2400-worktree-add.sh +++ b/t/t2400-worktree-add.sh @@ -490,7 +490,8 @@ test_expect_success 'put a worktree under rebase' ' cd under-rebase && set_fake_editor && FAKE_LINES="edit 1" git rebase -i HEAD^ && - git worktree list | grep "under-rebase.*detached HEAD" + git worktree list >actual && + grep "under-rebase.*detached HEAD" actual ) ' @@ -531,7 +532,8 @@ test_expect_success 'checkout a branch under bisect' ' git bisect start && git bisect bad && git bisect good HEAD~2 && - git worktree list | grep "under-bisect.*detached HEAD" && + git worktree list >actual && + grep "under-bisect.*detached HEAD" actual && test_must_fail git worktree add new-bisect under-bisect && ! test -d new-bisect ) diff --git a/t/t3200-branch.sh b/t/t3200-branch.sh index 6a316f081e..de7d3014e4 100755 --- a/t/t3200-branch.sh +++ b/t/t3200-branch.sh @@ -519,7 +519,7 @@ EOF mv .git/config .git/config-saved -test_expect_success SHA1 'git branch -m q q2 without config should succeed' ' +test_expect_success DEFAULT_REPO_FORMAT 'git branch -m q q2 without config should succeed' ' git branch -m q q2 && git branch -m q2 q ' diff --git a/t/t3903-stash.sh b/t/t3903-stash.sh index 34faeac3f1..3319240515 100755 --- a/t/t3903-stash.sh +++ b/t/t3903-stash.sh @@ -200,7 +200,7 @@ test_expect_success 'drop stash reflog updates refs/stash' ' test_cmp expect actual ' -test_expect_success REFFILES 'drop stash reflog updates refs/stash with rewrite' ' +test_expect_success 'drop stash reflog updates refs/stash with rewrite' ' git init repo && ( cd repo && @@ -213,16 +213,16 @@ test_expect_success REFFILES 'drop stash reflog updates refs/stash with rewrite' new_oid="$(git -C repo rev-parse stash@{0})" && cat >expect <<-EOF && - $(test_oid zero) $old_oid - $old_oid $new_oid + $new_oid + $old_oid EOF - cut -d" " -f1-2 repo/.git/logs/refs/stash >actual && + git -C repo reflog show refs/stash --format=%H >actual && test_cmp expect actual && git -C repo stash drop stash@{1} && - cut -d" " -f1-2 repo/.git/logs/refs/stash >actual && + git -C repo reflog show refs/stash --format=%H >actual && cat >expect <<-EOF && - $(test_oid zero) $new_oid + $new_oid EOF test_cmp expect actual ' diff --git a/t/t4001-diff-rename.sh b/t/t4001-diff-rename.sh index 85be1367de..49c042a38a 100755 --- a/t/t4001-diff-rename.sh +++ b/t/t4001-diff-rename.sh @@ -286,4 +286,28 @@ test_expect_success 'basename similarity vs best similarity' ' test_cmp expected actual ' +test_expect_success 'last line matters too' ' + { + test_write_lines a 0 1 2 3 4 5 6 7 8 9 && + printf "git ignores final up to 63 characters if not newline terminated" + } >no-final-lf && + git add no-final-lf && + git commit -m "original version of file with no final newline" && + + # Change ONLY the first character of the whole file + { + test_write_lines b 0 1 2 3 4 5 6 7 8 9 && + printf "git ignores final up to 63 characters if not newline terminated" + } >no-final-lf && + git add no-final-lf && + git mv no-final-lf still-absent-final-lf && + git commit -a -m "rename no-final-lf -> still-absent-final-lf" && + git diff-tree -r -M --name-status HEAD^ HEAD >actual && + sed -e "s/^R[0-9]* /R /" actual >actual.munged && + cat >expected <<-\EOF && + R no-final-lf still-absent-final-lf + EOF + test_cmp expected actual.munged +' + test_done diff --git a/t/t4013-diff-various.sh b/t/t4013-diff-various.sh index cb094241ec..1e3b2dbea4 100755 --- a/t/t4013-diff-various.sh +++ b/t/t4013-diff-various.sh @@ -663,4 +663,10 @@ test_expect_success 'diff --default-prefix overrides diff.mnemonicprefix' ' check_prefix actual a/file0 b/file0 ' +test_expect_success 'diff --no-renames cannot be abbreviated' ' + test_expect_code 129 git diff --no-rename >actual 2>error && + test_must_be_empty actual && + grep "invalid option: --no-rename" error +' + test_done diff --git a/t/t4013/diff.log_--decorate=full_--clear-decorations_--all b/t/t4013/diff.log_--decorate=full_--clear-decorations_--all index 1c030a6554..7d16978e7f 100644 --- a/t/t4013/diff.log_--decorate=full_--clear-decorations_--all +++ b/t/t4013/diff.log_--decorate=full_--clear-decorations_--all @@ -33,13 +33,13 @@ Date: Mon Jun 26 00:04:00 2006 +0000 Merge branch 'side' -commit c7a2ab9e8eac7b117442a607d5a9b3950ae34d5a (refs/heads/side) +commit c7a2ab9e8eac7b117442a607d5a9b3950ae34d5a (FETCH_HEAD, refs/heads/side) Author: A U Thor <author@example.com> Date: Mon Jun 26 00:03:00 2006 +0000 Side -commit 9a6d4949b6b76956d9d5e26f2791ec2ceff5fdc0 +commit 9a6d4949b6b76956d9d5e26f2791ec2ceff5fdc0 (ORIG_HEAD) Author: A U Thor <author@example.com> Date: Mon Jun 26 00:02:00 2006 +0000 diff --git a/t/t4013/diff.log_--decorate_--clear-decorations_--all b/t/t4013/diff.log_--decorate_--clear-decorations_--all index 88be82cce3..4f9be50ce0 100644 --- a/t/t4013/diff.log_--decorate_--clear-decorations_--all +++ b/t/t4013/diff.log_--decorate_--clear-decorations_--all @@ -33,13 +33,13 @@ Date: Mon Jun 26 00:04:00 2006 +0000 Merge branch 'side' -commit c7a2ab9e8eac7b117442a607d5a9b3950ae34d5a (side) +commit c7a2ab9e8eac7b117442a607d5a9b3950ae34d5a (FETCH_HEAD, side) Author: A U Thor <author@example.com> Date: Mon Jun 26 00:03:00 2006 +0000 Side -commit 9a6d4949b6b76956d9d5e26f2791ec2ceff5fdc0 +commit 9a6d4949b6b76956d9d5e26f2791ec2ceff5fdc0 (ORIG_HEAD) Author: A U Thor <author@example.com> Date: Mon Jun 26 00:02:00 2006 +0000 diff --git a/t/t4129-apply-samemode.sh b/t/t4129-apply-samemode.sh index e7a7295f1b..4eb8444029 100755 --- a/t/t4129-apply-samemode.sh +++ b/t/t4129-apply-samemode.sh @@ -41,7 +41,8 @@ test_expect_success FILEMODE 'same mode (index only)' ' chmod +x file && git add file && git apply --cached patch-0.txt && - git ls-files -s file | grep "^100755" + git ls-files -s file >ls-files-output && + test_grep "^100755" ls-files-output ' test_expect_success FILEMODE 'mode update (no index)' ' @@ -60,7 +61,8 @@ test_expect_success FILEMODE 'mode update (with index)' ' test_expect_success FILEMODE 'mode update (index only)' ' git reset --hard && git apply --cached patch-1.txt && - git ls-files -s file | grep "^100755" + git ls-files -s file >ls-files-output && + test_grep "^100755" ls-files-output ' test_expect_success FILEMODE 'empty mode is rejected' ' @@ -101,4 +103,31 @@ test_expect_success POSIXPERM 'do not use core.sharedRepository for working tree ) ' +test_expect_success 'git apply respects core.fileMode' ' + test_config core.fileMode false && + echo true >script.sh && + git add --chmod=+x script.sh && + git ls-files -s script.sh >ls-files-output && + test_grep "^100755" ls-files-output && + test_tick && git commit -m "Add script" && + git ls-tree -r HEAD script.sh >ls-tree-output && + test_grep "^100755" ls-tree-output && + + echo true >>script.sh && + test_tick && git commit -m "Modify script" script.sh && + git format-patch -1 --stdout >patch && + test_grep "^index.*100755$" patch && + + git switch -c branch HEAD^ && + git apply --index patch 2>err && + test_grep ! "has type 100644, expected 100755" err && + git reset --hard && + + git apply patch 2>err && + test_grep ! "has type 100644, expected 100755" err && + + git apply --cached patch 2>err && + test_grep ! "has type 100644, expected 100755" err +' + test_done diff --git a/t/t4202-log.sh b/t/t4202-log.sh index ddd205f98a..7137b25c81 100755 --- a/t/t4202-log.sh +++ b/t/t4202-log.sh @@ -927,7 +927,7 @@ test_expect_success 'multiple decorate-refs' ' test_expect_success 'decorate-refs-exclude with glob' ' cat >expect.decorate <<-\EOF && Merge-tag-reach (HEAD -> main) - Merge-tags-octopus-a-and-octopus-b + Merge-tags-octopus-a-and-octopus-b (ORIG_HEAD) seventh (tag: seventh) octopus-b (tag: octopus-b) octopus-a (tag: octopus-a) @@ -944,7 +944,7 @@ test_expect_success 'decorate-refs-exclude with glob' ' test_expect_success 'decorate-refs-exclude without globs' ' cat >expect.decorate <<-\EOF && Merge-tag-reach (HEAD -> main) - Merge-tags-octopus-a-and-octopus-b + Merge-tags-octopus-a-and-octopus-b (ORIG_HEAD) seventh (tag: seventh) octopus-b (tag: octopus-b, octopus-b) octopus-a (tag: octopus-a, octopus-a) @@ -961,7 +961,7 @@ test_expect_success 'decorate-refs-exclude without globs' ' test_expect_success 'multiple decorate-refs-exclude' ' cat >expect.decorate <<-\EOF && Merge-tag-reach (HEAD -> main) - Merge-tags-octopus-a-and-octopus-b + Merge-tags-octopus-a-and-octopus-b (ORIG_HEAD) seventh (tag: seventh) octopus-b (tag: octopus-b) octopus-a (tag: octopus-a) @@ -1022,10 +1022,12 @@ test_expect_success 'decorate-refs-exclude and simplify-by-decoration' ' EOF git log -n6 --decorate=short --pretty="tformat:%f%d" \ --decorate-refs-exclude="*octopus*" \ + --decorate-refs-exclude="ORIG_HEAD" \ --simplify-by-decoration >actual && test_cmp expect.decorate actual && - git -c log.excludeDecoration="*octopus*" log \ - -n6 --decorate=short --pretty="tformat:%f%d" \ + git -c log.excludeDecoration="*octopus*" \ + -c log.excludeDecoration="ORIG_HEAD" \ + log -n6 --decorate=short --pretty="tformat:%f%d" \ --simplify-by-decoration >actual && test_cmp expect.decorate actual ' @@ -1067,9 +1069,10 @@ test_expect_success 'decorate-refs and simplify-by-decoration without output' ' test_cmp expect actual ' -test_expect_success 'decorate-refs-exclude HEAD' ' +test_expect_success 'decorate-refs-exclude HEAD ORIG_HEAD' ' git log --decorate=full --oneline \ - --decorate-refs-exclude="HEAD" >actual && + --decorate-refs-exclude="HEAD" \ + --decorate-refs-exclude="ORIG_HEAD" >actual && ! grep HEAD actual ' @@ -1107,7 +1110,7 @@ test_expect_success '--clear-decorations overrides defaults' ' cat >expect.all <<-\EOF && Merge-tag-reach (HEAD -> refs/heads/main) - Merge-tags-octopus-a-and-octopus-b + Merge-tags-octopus-a-and-octopus-b (ORIG_HEAD) seventh (tag: refs/tags/seventh) octopus-b (tag: refs/tags/octopus-b, refs/heads/octopus-b) octopus-a (tag: refs/tags/octopus-a, refs/heads/octopus-a) @@ -1139,7 +1142,7 @@ test_expect_success '--clear-decorations clears previous exclusions' ' cat >expect.all <<-\EOF && Merge-tag-reach (HEAD -> refs/heads/main) reach (tag: refs/tags/reach, refs/heads/reach) - Merge-tags-octopus-a-and-octopus-b + Merge-tags-octopus-a-and-octopus-b (ORIG_HEAD) octopus-b (tag: refs/tags/octopus-b, refs/heads/octopus-b) octopus-a (tag: refs/tags/octopus-a, refs/heads/octopus-a) seventh (tag: refs/tags/seventh) @@ -2255,23 +2258,6 @@ test_expect_success 'log on empty repo fails' ' test_grep does.not.have.any.commits stderr ' -test_expect_success REFFILES 'log diagnoses bogus HEAD hash' ' - git init empty && - test_when_finished "rm -rf empty" && - echo 1234abcd >empty/.git/refs/heads/main && - test_must_fail git -C empty log 2>stderr && - test_grep broken stderr -' - -test_expect_success REFFILES 'log diagnoses bogus HEAD symref' ' - git init empty && - test-tool -C empty ref-store main create-symref HEAD refs/heads/invalid.lock && - test_must_fail git -C empty log 2>stderr && - test_grep broken stderr && - test_must_fail git -C empty log --default totally-bogus 2>stderr && - test_grep broken stderr -' - test_expect_success 'log does not default to HEAD when rev input is given' ' git log --branches=does-not-exist >actual && test_must_be_empty actual diff --git a/t/t4207-log-decoration-colors.sh b/t/t4207-log-decoration-colors.sh index 73ea9e5155..9be3c523d4 100755 --- a/t/t4207-log-decoration-colors.sh +++ b/t/t4207-log-decoration-colors.sh @@ -17,7 +17,10 @@ test_expect_success setup ' git config color.decorate.remoteBranch red && git config color.decorate.tag "reverse bold yellow" && git config color.decorate.stash magenta && + git config color.decorate.ref blue && + git config color.decorate.pseudoref "bold cyan" && git config color.decorate.grafted black && + git config color.decorate.symbol white && git config color.decorate.HEAD cyan && c_reset="<RESET>" && @@ -27,10 +30,14 @@ test_expect_success setup ' c_remoteBranch="<RED>" && c_tag="<BOLD;REVERSE;YELLOW>" && c_stash="<MAGENTA>" && + c_ref="<BLUE>" && + c_pseudoref="<BOLD;CYAN>" && c_HEAD="<CYAN>" && c_grafted="<BLACK>" && + c_symbol="<WHITE>" && test_commit A && + git update-ref refs/foo A && git clone . other && ( cd other && @@ -41,7 +48,10 @@ test_expect_success setup ' test_commit B && git tag v1.0 && echo >>A.t && - git stash save Changes to A.t + git stash save Changes to A.t && + git reset other/main && + git reset ORIG_HEAD && + git revert --no-commit @~ ' cmp_filtered_decorations () { @@ -53,20 +63,24 @@ cmp_filtered_decorations () { # to this test since it does not contain any decoration, hence --first-parent test_expect_success 'commit decorations colored correctly' ' cat >expect <<-EOF && - ${c_commit}COMMIT_ID${c_reset}${c_commit} (${c_reset}${c_HEAD}HEAD${c_reset}\ -${c_commit} -> ${c_reset}${c_branch}main${c_reset}${c_commit}, \ -${c_reset}${c_tag}tag: ${c_reset}${c_tag}v1.0${c_reset}${c_commit}, \ -${c_reset}${c_tag}tag: ${c_reset}${c_tag}B${c_reset}${c_commit})${c_reset} B -${c_commit}COMMIT_ID${c_reset}${c_commit} (${c_reset}\ -${c_tag}tag: ${c_reset}${c_tag}A1${c_reset}${c_commit}, \ -${c_reset}${c_remoteBranch}other/main${c_reset}${c_commit})${c_reset} A1 - ${c_commit}COMMIT_ID${c_reset}${c_commit} (${c_reset}\ -${c_stash}refs/stash${c_reset}${c_commit})${c_reset} On main: Changes to A.t - ${c_commit}COMMIT_ID${c_reset}${c_commit} (${c_reset}\ -${c_tag}tag: ${c_reset}${c_tag}A${c_reset}${c_commit})${c_reset} A + ${c_commit}COMMIT_ID${c_reset}${c_symbol} (${c_reset}${c_HEAD}HEAD${c_reset}\ +${c_symbol} -> ${c_reset}${c_branch}main${c_reset}${c_symbol}, ${c_reset}\ +${c_tag}tag: ${c_reset}${c_tag}v1.0${c_reset}${c_symbol}, ${c_reset}\ +${c_tag}tag: ${c_reset}${c_tag}B${c_reset}${c_symbol})${c_reset} B +${c_commit}COMMIT_ID${c_reset}${c_symbol} (${c_reset}\ +${c_pseudoref}ORIG_HEAD${c_reset}${c_symbol}, ${c_reset}\ +${c_tag}tag: ${c_reset}${c_tag}A1${c_reset}${c_symbol}, ${c_reset}\ +${c_remoteBranch}other/main${c_reset}${c_symbol})${c_reset} A1 + ${c_commit}COMMIT_ID${c_reset}${c_symbol} (${c_reset}\ +${c_stash}refs/stash${c_reset}${c_symbol})${c_reset} On main: Changes to A.t + ${c_commit}COMMIT_ID${c_reset}${c_symbol} (${c_reset}\ +${c_pseudoref}REVERT_HEAD${c_reset}${c_symbol}, ${c_reset}\ +${c_tag}tag: ${c_reset}${c_tag}A${c_reset}${c_symbol}, ${c_reset}\ +${c_ref}refs/foo${c_reset}${c_symbol})${c_reset} A EOF - git log --first-parent --no-abbrev --decorate --oneline --color=always --all >actual && + git log --first-parent --no-abbrev --decorate --color=always \ + --decorate-refs-exclude=FETCH_HEAD --oneline --all >actual && cmp_filtered_decorations ' @@ -84,14 +98,14 @@ test_expect_success 'test coloring with replace-objects' ' git replace HEAD~1 HEAD~2 && cat >expect <<-EOF && - ${c_commit}COMMIT_ID${c_reset}${c_commit} (${c_reset}${c_HEAD}HEAD${c_reset}\ -${c_commit} -> ${c_reset}${c_branch}main${c_reset}${c_commit}, \ -${c_reset}${c_tag}tag: ${c_reset}${c_tag}D${c_reset}${c_commit})${c_reset} D - ${c_commit}COMMIT_ID${c_reset}${c_commit} (${c_reset}\ -${c_tag}tag: ${c_reset}${c_tag}C${c_reset}${c_commit}, \ -${c_reset}${c_grafted}replaced${c_reset}${c_commit})${c_reset} B - ${c_commit}COMMIT_ID${c_reset}${c_commit} (${c_reset}\ -${c_tag}tag: ${c_reset}${c_tag}A${c_reset}${c_commit})${c_reset} A + ${c_commit}COMMIT_ID${c_reset}${c_symbol} (${c_reset}${c_HEAD}HEAD${c_reset}\ +${c_symbol} -> ${c_reset}${c_branch}main${c_reset}${c_symbol}, ${c_reset}\ +${c_tag}tag: ${c_reset}${c_tag}D${c_reset}${c_symbol})${c_reset} D + ${c_commit}COMMIT_ID${c_reset}${c_symbol} (${c_reset}\ +${c_tag}tag: ${c_reset}${c_tag}C${c_reset}${c_symbol}, ${c_reset}\ +${c_grafted}replaced${c_reset}${c_symbol})${c_reset} B + ${c_commit}COMMIT_ID${c_reset}${c_symbol} (${c_reset}\ +${c_tag}tag: ${c_reset}${c_tag}A${c_reset}${c_symbol})${c_reset} A EOF git log --first-parent --no-abbrev --decorate --oneline --color=always HEAD >actual && @@ -110,15 +124,15 @@ test_expect_success 'test coloring with grafted commit' ' git replace --graft HEAD HEAD~2 && cat >expect <<-EOF && - ${c_commit}COMMIT_ID${c_reset}${c_commit} (${c_reset}${c_HEAD}HEAD${c_reset}\ -${c_commit} -> ${c_reset}${c_branch}main${c_reset}${c_commit}, \ -${c_reset}${c_tag}tag: ${c_reset}${c_tag}D${c_reset}${c_commit}, \ -${c_reset}${c_grafted}replaced${c_reset}${c_commit})${c_reset} D - ${c_commit}COMMIT_ID${c_reset}${c_commit} (${c_reset}\ -${c_tag}tag: ${c_reset}${c_tag}v1.0${c_reset}${c_commit}, \ -${c_reset}${c_tag}tag: ${c_reset}${c_tag}B${c_reset}${c_commit})${c_reset} B - ${c_commit}COMMIT_ID${c_reset}${c_commit} (${c_reset}\ -${c_tag}tag: ${c_reset}${c_tag}A${c_reset}${c_commit})${c_reset} A + ${c_commit}COMMIT_ID${c_reset}${c_symbol} (${c_reset}${c_HEAD}HEAD${c_reset}\ +${c_symbol} -> ${c_reset}${c_branch}main${c_reset}${c_symbol}, ${c_reset}\ +${c_tag}tag: ${c_reset}${c_tag}D${c_reset}${c_symbol}, ${c_reset}\ +${c_grafted}replaced${c_reset}${c_symbol})${c_reset} D + ${c_commit}COMMIT_ID${c_reset}${c_symbol} (${c_reset}\ +${c_tag}tag: ${c_reset}${c_tag}v1.0${c_reset}${c_symbol}, ${c_reset}\ +${c_tag}tag: ${c_reset}${c_tag}B${c_reset}${c_symbol})${c_reset} B + ${c_commit}COMMIT_ID${c_reset}${c_symbol} (${c_reset}\ +${c_tag}tag: ${c_reset}${c_tag}A${c_reset}${c_symbol})${c_reset} A EOF git log --first-parent --no-abbrev --decorate --oneline --color=always HEAD >actual && diff --git a/t/t4216-log-bloom.sh b/t/t4216-log-bloom.sh index 2ba0324a69..823d1cf773 100755 --- a/t/t4216-log-bloom.sh +++ b/t/t4216-log-bloom.sh @@ -82,7 +82,23 @@ test_bloom_filters_used () { test_bloom_filters_not_used () { log_args=$1 setup "$log_args" && - ! grep -q "statistics:{\"filter_not_present\":" "$TRASH_DIRECTORY/trace.perf" && + + if grep -q "statistics:{\"filter_not_present\":" "$TRASH_DIRECTORY/trace.perf" + then + # if the Bloom filter system is initialized, ensure that no + # filters were used + data="statistics:{" + # unusable filters (e.g., those computed with a + # different value of commitGraph.changedPathsVersion) + # are counted in the filter_not_present bucket, so any + # value is OK there. + data="$data\"filter_not_present\":[0-9][0-9]*," + data="$data\"maybe\":0," + data="$data\"definitely_not\":0," + data="$data\"false_positive\":0}" + + grep -q "$data" "$TRASH_DIRECTORY/trace.perf" + fi && test_cmp log_wo_bloom log_w_bloom } @@ -163,7 +179,7 @@ test_expect_success 'setup - add commit-graph to the chain with Bloom filters' ' test_bloom_filters_used_when_some_filters_are_missing () { log_args=$1 - bloom_trace_prefix="statistics:{\"filter_not_present\":3,\"maybe\":6,\"definitely_not\":9" + bloom_trace_prefix="statistics:{\"filter_not_present\":3,\"maybe\":6,\"definitely_not\":10" setup "$log_args" && grep -q "$bloom_trace_prefix" "$TRASH_DIRECTORY/trace.perf" && test_cmp log_wo_bloom log_w_bloom @@ -206,6 +222,10 @@ test_filter_trunc_large () { grep "\"key\":\"filter-trunc-large\",\"value\":\"$1\"" $2 } +test_filter_upgraded () { + grep "\"key\":\"filter-upgraded\",\"value\":\"$1\"" $2 +} + test_expect_success 'correctly report changes over limit' ' git init limits && ( @@ -405,8 +425,292 @@ test_expect_success 'Bloom generation backfills empty commits' ' ) ' +graph=.git/objects/info/commit-graph +graphdir=.git/objects/info/commit-graphs +chain=$graphdir/commit-graph-chain + +test_expect_success 'setup for mixed Bloom setting tests' ' + repo=mixed-bloom-settings && + + git init $repo && + for i in one two three + do + test_commit -C $repo $i file || return 1 + done +' + +test_expect_success 'ensure Bloom filters with incompatible settings are ignored' ' + # Compute Bloom filters with "unusual" settings. + git -C $repo rev-parse one >in && + GIT_TEST_BLOOM_SETTINGS_NUM_HASHES=3 git -C $repo commit-graph write \ + --stdin-commits --changed-paths --split <in && + layer=$(head -n 1 $repo/$chain) && + + # A commit-graph layer without Bloom filters "hides" the layers + # below ... + git -C $repo rev-parse two >in && + git -C $repo commit-graph write --stdin-commits --no-changed-paths \ + --split=no-merge <in && + + # Another commit-graph layer that has Bloom filters, but with + # standard settings, and is thus incompatible with the base + # layer written above. + git -C $repo rev-parse HEAD >in && + git -C $repo commit-graph write --stdin-commits --changed-paths \ + --split=no-merge <in && + + test_line_count = 3 $repo/$chain && + + # Ensure that incompatible Bloom filters are ignored. + git -C $repo -c core.commitGraph=false log --oneline --no-decorate -- file \ + >expect 2>err && + git -C $repo log --oneline --no-decorate -- file >actual 2>err && + test_cmp expect actual && + grep "disabling Bloom filters for commit-graph layer .$layer." err +' + +test_expect_success 'merge graph layers with incompatible Bloom settings' ' + # Ensure that incompatible Bloom filters are ignored when + # merging existing layers. + git -C $repo commit-graph write --reachable --changed-paths 2>err && + grep "disabling Bloom filters for commit-graph layer .$layer." err && + + test_path_is_file $repo/$graph && + test_dir_is_empty $repo/$graphdir && + + git -C $repo -c core.commitGraph=false log --oneline --no-decorate -- \ + file >expect && + trace_out="$(pwd)/trace.perf" && + GIT_TRACE2_PERF="$trace_out" \ + git -C $repo log --oneline --no-decorate -- file >actual 2>err && + + test_cmp expect actual && + grep "statistics:{\"filter_not_present\":0," trace.perf && + test_must_be_empty err +' + +test_expect_success 'ensure Bloom filter with incompatible versions are ignored' ' + rm "$repo/$graph" && + + git -C $repo log --oneline --no-decorate -- $CENT >expect && + + # Compute v1 Bloom filters for commits at the bottom. + git -C $repo rev-parse HEAD^ >in && + git -C $repo commit-graph write --stdin-commits --changed-paths \ + --split <in && + + # Compute v2 Bloomfilters for the rest of the commits at the top. + git -C $repo rev-parse HEAD >in && + git -C $repo -c commitGraph.changedPathsVersion=2 commit-graph write \ + --stdin-commits --changed-paths --split=no-merge <in && + + test_line_count = 2 $repo/$chain && + + git -C $repo log --oneline --no-decorate -- $CENT >actual 2>err && + test_cmp expect actual && + + layer="$(head -n 1 $repo/$chain)" && + cat >expect.err <<-EOF && + warning: disabling Bloom filters for commit-graph layer $SQ$layer$SQ due to incompatible settings + EOF + test_cmp expect.err err +' + +get_first_changed_path_filter () { + test-tool read-graph bloom-filters >filters.dat && + head -n 1 filters.dat +} + +# chosen to be the same under all Unicode normalization forms +CENT=$(printf "\302\242") + +test_expect_success 'set up repo with high bit path, version 1 changed-path' ' + git init highbit1 && + test_commit -C highbit1 c1 "$CENT" && + git -C highbit1 commit-graph write --reachable --changed-paths +' + +test_expect_success 'setup check value of version 1 changed-path' ' + ( + cd highbit1 && + echo "52a9" >expect && + get_first_changed_path_filter >actual + ) +' + +# expect will not match actual if char is unsigned by default. Write the test +# in this way, so that a user running this test script can still see if the two +# files match. (It will appear as an ordinary success if they match, and a skip +# if not.) +if test_cmp highbit1/expect highbit1/actual +then + test_set_prereq SIGNED_CHAR_BY_DEFAULT +fi +test_expect_success SIGNED_CHAR_BY_DEFAULT 'check value of version 1 changed-path' ' + # Only the prereq matters for this test. + true +' + +test_expect_success 'setup make another commit' ' + # "git log" does not use Bloom filters for root commits - see how, in + # revision.c, rev_compare_tree() (the only code path that eventually calls + # get_bloom_filter()) is only called by try_to_simplify_commit() when the commit + # has one parent. Therefore, make another commit so that we perform the tests on + # a non-root commit. + test_commit -C highbit1 anotherc1 "another$CENT" +' + +test_expect_success 'version 1 changed-path used when version 1 requested' ' + ( + cd highbit1 && + test_bloom_filters_used "-- another$CENT" + ) +' + +test_expect_success 'version 1 changed-path not used when version 2 requested' ' + ( + cd highbit1 && + git config --add commitgraph.changedPathsVersion 2 && + test_bloom_filters_not_used "-- another$CENT" + ) +' + +test_expect_success 'version 1 changed-path used when autodetect requested' ' + ( + cd highbit1 && + git config --add commitgraph.changedPathsVersion -1 && + test_bloom_filters_used "-- another$CENT" + ) +' + +test_expect_success 'when writing another commit graph, preserve existing version 1 of changed-path' ' + test_commit -C highbit1 c1double "$CENT$CENT" && + git -C highbit1 commit-graph write --reachable --changed-paths && + ( + cd highbit1 && + git config --add commitgraph.changedPathsVersion -1 && + echo "options: bloom(1,10,7) read_generation_data" >expect && + test-tool read-graph >full && + grep options full >actual && + test_cmp expect actual + ) +' + +test_expect_success 'set up repo with high bit path, version 2 changed-path' ' + git init highbit2 && + git -C highbit2 config --add commitgraph.changedPathsVersion 2 && + test_commit -C highbit2 c2 "$CENT" && + git -C highbit2 commit-graph write --reachable --changed-paths +' + +test_expect_success 'check value of version 2 changed-path' ' + ( + cd highbit2 && + echo "c01f" >expect && + get_first_changed_path_filter >actual && + test_cmp expect actual + ) +' + +test_expect_success 'setup make another commit' ' + # "git log" does not use Bloom filters for root commits - see how, in + # revision.c, rev_compare_tree() (the only code path that eventually calls + # get_bloom_filter()) is only called by try_to_simplify_commit() when the commit + # has one parent. Therefore, make another commit so that we perform the tests on + # a non-root commit. + test_commit -C highbit2 anotherc2 "another$CENT" +' + +test_expect_success 'version 2 changed-path used when version 2 requested' ' + ( + cd highbit2 && + test_bloom_filters_used "-- another$CENT" + ) +' + +test_expect_success 'version 2 changed-path not used when version 1 requested' ' + ( + cd highbit2 && + git config --add commitgraph.changedPathsVersion 1 && + test_bloom_filters_not_used "-- another$CENT" + ) +' + +test_expect_success 'version 2 changed-path used when autodetect requested' ' + ( + cd highbit2 && + git config --add commitgraph.changedPathsVersion -1 && + test_bloom_filters_used "-- another$CENT" + ) +' + +test_expect_success 'when writing another commit graph, preserve existing version 2 of changed-path' ' + test_commit -C highbit2 c2double "$CENT$CENT" && + git -C highbit2 commit-graph write --reachable --changed-paths && + ( + cd highbit2 && + git config --add commitgraph.changedPathsVersion -1 && + echo "options: bloom(2,10,7) read_generation_data" >expect && + test-tool read-graph >full && + grep options full >actual && + test_cmp expect actual + ) +' + +test_expect_success 'when writing commit graph, do not reuse changed-path of another version' ' + git init doublewrite && + test_commit -C doublewrite c "$CENT" && + + git -C doublewrite config --add commitgraph.changedPathsVersion 1 && + GIT_TRACE2_EVENT="$(pwd)/trace2.txt" \ + git -C doublewrite commit-graph write --reachable --changed-paths && + test_filter_computed 1 trace2.txt && + test_filter_upgraded 0 trace2.txt && + + git -C doublewrite commit-graph write --reachable --changed-paths && + for v in -2 3 + do + git -C doublewrite config --add commitgraph.changedPathsVersion $v && + git -C doublewrite commit-graph write --reachable --changed-paths 2>err && + cat >expect <<-EOF && + warning: attempting to write a commit-graph, but ${SQ}commitgraph.changedPathsVersion${SQ} ($v) is not supported + EOF + test_cmp expect err || return 1 + done && + + git -C doublewrite config --add commitgraph.changedPathsVersion 2 && + GIT_TRACE2_EVENT="$(pwd)/trace2.txt" \ + git -C doublewrite commit-graph write --reachable --changed-paths && + test_filter_computed 1 trace2.txt && + test_filter_upgraded 0 trace2.txt && + + ( + cd doublewrite && + echo "c01f" >expect && + get_first_changed_path_filter >actual && + test_cmp expect actual + ) +' + +test_expect_success 'when writing commit graph, reuse changed-path of another version where possible' ' + git init upgrade && + + test_commit -C upgrade base no-high-bits && + + git -C upgrade config --add commitgraph.changedPathsVersion 1 && + GIT_TRACE2_EVENT="$(pwd)/trace2.txt" \ + git -C upgrade commit-graph write --reachable --changed-paths && + test_filter_computed 1 trace2.txt && + test_filter_upgraded 0 trace2.txt && + + git -C upgrade config --add commitgraph.changedPathsVersion 2 && + GIT_TRACE2_EVENT="$(pwd)/trace2.txt" \ + git -C upgrade commit-graph write --reachable --changed-paths && + test_filter_computed 0 trace2.txt && + test_filter_upgraded 1 trace2.txt +' + corrupt_graph () { - graph=.git/objects/info/commit-graph && test_when_finished "rm -rf $graph" && git commit-graph write --reachable --changed-paths && corrupt_chunk_file $graph "$@" diff --git a/t/t5003-archive-zip.sh b/t/t5003-archive-zip.sh index fc499cdff0..961c6aac25 100755 --- a/t/t5003-archive-zip.sh +++ b/t/t5003-archive-zip.sh @@ -239,4 +239,38 @@ check_zip with_untracked2 check_added with_untracked2 untracked one/untracked check_added with_untracked2 untracked two/untracked +# Test remote archive over HTTP protocol. +# +# Note: this should be the last part of this test suite, because +# by including lib-httpd.sh, the test may end early if httpd tests +# should not be run. +# +. "$TEST_DIRECTORY"/lib-httpd.sh +start_httpd + +test_expect_success "setup for HTTP protocol" ' + cp -R bare.git "$HTTPD_DOCUMENT_ROOT_PATH/bare.git" && + git -C "$HTTPD_DOCUMENT_ROOT_PATH/bare.git" \ + config http.uploadpack true && + set_askpass user@host pass@host +' + +setup_askpass_helper + +test_expect_success 'remote archive does not work with protocol v1' ' + test_must_fail git -c protocol.version=1 archive \ + --remote="$HTTPD_URL/auth/smart/bare.git" \ + --output=remote-http.zip HEAD >actual 2>&1 && + cat >expect <<-EOF && + fatal: can${SQ}t connect to subservice git-upload-archive + EOF + test_cmp expect actual +' + +test_expect_success 'archive remote http repository' ' + git archive --remote="$HTTPD_URL/auth/smart/bare.git" \ + --output=remote-http.zip HEAD && + test_cmp_bin d.zip remote-http.zip +' + test_done diff --git a/t/t5312-prune-corruption.sh b/t/t5312-prune-corruption.sh index 230cb38712..d8d2e30468 100755 --- a/t/t5312-prune-corruption.sh +++ b/t/t5312-prune-corruption.sh @@ -111,30 +111,4 @@ test_expect_success 'pack-refs does not silently delete broken loose ref' ' test_cmp expect actual ' -# we do not want to count on running pack-refs to -# actually pack it, as it is perfectly reasonable to -# skip processing a broken ref -test_expect_success REFFILES 'create packed-refs file with broken ref' ' - rm -f .git/refs/heads/main && - cat >.git/packed-refs <<-EOF && - $missing refs/heads/main - $recoverable refs/heads/other - EOF - echo $missing >expect && - git rev-parse refs/heads/main >actual && - test_cmp expect actual -' - -test_expect_success REFFILES 'pack-refs does not silently delete broken packed ref' ' - git pack-refs --all --prune && - git rev-parse refs/heads/main >actual && - test_cmp expect actual -' - -test_expect_success REFFILES 'pack-refs does not drop broken refs during deletion' ' - git update-ref -d refs/heads/other && - git rev-parse refs/heads/main >actual && - test_cmp expect actual -' - test_done diff --git a/t/t5318-commit-graph.sh b/t/t5318-commit-graph.sh index a2b4442660..73b43051c8 100755 --- a/t/t5318-commit-graph.sh +++ b/t/t5318-commit-graph.sh @@ -545,12 +545,12 @@ test_expect_success 'detect missing OID fanout chunk' ' test_expect_success 'detect missing OID lookup chunk' ' corrupt_graph_and_verify $GRAPH_BYTE_OID_LOOKUP_ID "\0" \ - "commit-graph required OID lookup chunk missing or corrupted" + "commit-graph OID lookup chunk is the wrong size" ' test_expect_success 'detect missing commit data chunk' ' corrupt_graph_and_verify $GRAPH_BYTE_COMMIT_DATA_ID "\0" \ - "commit-graph required commit data chunk missing or corrupted" + "commit-graph commit data chunk is wrong size" ' test_expect_success 'detect incorrect fanout' ' @@ -851,7 +851,6 @@ test_expect_success 'reader notices fanout/lookup table mismatch' ' check_corrupt_chunk OIDF 1020 "FFFFFFFF" && cat >expect.err <<-\EOF && error: commit-graph OID lookup chunk is the wrong size - error: commit-graph required OID lookup chunk missing or corrupted EOF test_cmp expect.err err ' @@ -876,7 +875,6 @@ test_expect_success 'reader notices too-small commit data chunk' ' check_corrupt_chunk CDAT clear 00000000 && cat >expect.err <<-\EOF && error: commit-graph commit data chunk is wrong size - error: commit-graph required commit data chunk missing or corrupted EOF test_cmp expect.err err ' diff --git a/t/t5319-multi-pack-index.sh b/t/t5319-multi-pack-index.sh index c20aafe99a..295ecca966 100755 --- a/t/t5319-multi-pack-index.sh +++ b/t/t5319-multi-pack-index.sh @@ -1077,7 +1077,6 @@ test_expect_success 'reader notices too-small oid lookup chunk' ' corrupt_chunk OIDL clear 00000000 && test_must_fail git log 2>err && cat >expect <<-\EOF && - error: multi-pack-index OID lookup chunk is the wrong size fatal: multi-pack-index required OID lookup chunk missing or corrupted EOF test_cmp expect err @@ -1112,7 +1111,6 @@ test_expect_success 'reader notices too-small object offset chunk' ' corrupt_chunk OOFF clear 00000000 && test_must_fail git log 2>err && cat >expect <<-\EOF && - error: multi-pack-index object offset chunk is the wrong size fatal: multi-pack-index required object offsets chunk missing or corrupted EOF test_cmp expect err @@ -1171,4 +1169,39 @@ test_expect_success 'reader notices out-of-bounds fanout' ' test_cmp expect err ' +test_expect_success 'bitmapped packs are stored via the BTMP chunk' ' + test_when_finished "rm -fr repo" && + git init repo && + ( + cd repo && + + for i in 1 2 3 4 5 + do + test_commit "$i" && + git repack -d || return 1 + done && + + find $objdir/pack -type f -name "*.idx" | xargs -n 1 basename | + sort >packs && + + git multi-pack-index write --stdin-packs <packs && + test_must_fail test-tool read-midx --bitmap $objdir 2>err && + cat >expect <<-\EOF && + error: MIDX does not contain the BTMP chunk + EOF + test_cmp expect err && + + git multi-pack-index write --stdin-packs --bitmap \ + --preferred-pack="$(head -n1 <packs)" <packs && + test-tool read-midx --bitmap $objdir >actual && + for i in $(test_seq $(wc -l <packs)) + do + sed -ne "${i}s/\.idx$/\.pack/p" packs && + echo " bitmap_pos: $((($i - 1) * 3))" && + echo " bitmap_nr: 3" || return 1 + done >expect && + test_cmp expect actual + ) +' + test_done diff --git a/t/t5332-multi-pack-reuse.sh b/t/t5332-multi-pack-reuse.sh new file mode 100755 index 0000000000..2ba788b042 --- /dev/null +++ b/t/t5332-multi-pack-reuse.sh @@ -0,0 +1,203 @@ +#!/bin/sh + +test_description='pack-objects multi-pack reuse' + +. ./test-lib.sh +. "$TEST_DIRECTORY"/lib-bitmap.sh + +objdir=.git/objects +packdir=$objdir/pack + +test_pack_reused () { + test_trace2_data pack-objects pack-reused "$1" +} + +test_packs_reused () { + test_trace2_data pack-objects packs-reused "$1" +} + + +# pack_position <object> </path/to/pack.idx +pack_position () { + git show-index >objects && + grep "$1" objects | cut -d" " -f1 +} + +test_expect_success 'preferred pack is reused for single-pack reuse' ' + test_config pack.allowPackReuse single && + + for i in A B + do + test_commit "$i" && + git repack -d || return 1 + done && + + git multi-pack-index write --bitmap && + + : >trace2.txt && + GIT_TRACE2_EVENT="$PWD/trace2.txt" \ + git pack-objects --stdout --revs --all >/dev/null && + + test_pack_reused 3 <trace2.txt && + test_packs_reused 1 <trace2.txt +' + +test_expect_success 'enable multi-pack reuse' ' + git config pack.allowPackReuse multi +' + +test_expect_success 'reuse all objects from subset of bitmapped packs' ' + test_commit C && + git repack -d && + + git multi-pack-index write --bitmap && + + cat >in <<-EOF && + $(git rev-parse C) + ^$(git rev-parse A) + EOF + + : >trace2.txt && + GIT_TRACE2_EVENT="$PWD/trace2.txt" \ + git pack-objects --stdout --revs <in >/dev/null && + + test_pack_reused 6 <trace2.txt && + test_packs_reused 2 <trace2.txt +' + +test_expect_success 'reuse all objects from all packs' ' + : >trace2.txt && + GIT_TRACE2_EVENT="$PWD/trace2.txt" \ + git pack-objects --stdout --revs --all >/dev/null && + + test_pack_reused 9 <trace2.txt && + test_packs_reused 3 <trace2.txt +' + +test_expect_success 'reuse objects from first pack with middle gap' ' + for i in D E F + do + test_commit "$i" || return 1 + done && + + # Set "pack.window" to zero to ensure that we do not create any + # deltas, which could alter the amount of pack reuse we perform + # (if, for e.g., we are not sending one or more bases). + D="$(git -c pack.window=0 pack-objects --all --unpacked $packdir/pack)" && + + d_pos="$(pack_position $(git rev-parse D) <$packdir/pack-$D.idx)" && + e_pos="$(pack_position $(git rev-parse E) <$packdir/pack-$D.idx)" && + f_pos="$(pack_position $(git rev-parse F) <$packdir/pack-$D.idx)" && + + # commits F, E, and D, should appear in that order at the + # beginning of the pack + test $f_pos -lt $e_pos && + test $e_pos -lt $d_pos && + + # Ensure that the pack we are constructing sorts ahead of any + # other packs in lexical/bitmap order by choosing it as the + # preferred pack. + git multi-pack-index write --bitmap --preferred-pack="pack-$D.idx" && + + cat >in <<-EOF && + $(git rev-parse E) + ^$(git rev-parse D) + EOF + + : >trace2.txt && + GIT_TRACE2_EVENT="$PWD/trace2.txt" \ + git pack-objects --stdout --delta-base-offset --revs <in >/dev/null && + + test_pack_reused 3 <trace2.txt && + test_packs_reused 1 <trace2.txt +' + +test_expect_success 'reuse objects from middle pack with middle gap' ' + rm -fr $packdir/multi-pack-index* && + + # Ensure that the pack we are constructing sort into any + # position *but* the first one, by choosing a different pack as + # the preferred one. + git multi-pack-index write --bitmap --preferred-pack="pack-$A.idx" && + + cat >in <<-EOF && + $(git rev-parse E) + ^$(git rev-parse D) + EOF + + : >trace2.txt && + GIT_TRACE2_EVENT="$PWD/trace2.txt" \ + git pack-objects --stdout --delta-base-offset --revs <in >/dev/null && + + test_pack_reused 3 <trace2.txt && + test_packs_reused 1 <trace2.txt +' + +test_expect_success 'omit delta with uninteresting base (same pack)' ' + git repack -adk && + + test_seq 32 >f && + git add f && + test_tick && + git commit -m "delta" && + delta="$(git rev-parse HEAD)" && + + test_seq 64 >f && + test_tick && + git commit -a -m "base" && + base="$(git rev-parse HEAD)" && + + test_commit other && + + git repack -d && + + have_delta "$(git rev-parse $delta:f)" "$(git rev-parse $base:f)" && + + git multi-pack-index write --bitmap && + + cat >in <<-EOF && + $(git rev-parse other) + ^$base + EOF + + : >trace2.txt && + GIT_TRACE2_EVENT="$PWD/trace2.txt" \ + git pack-objects --stdout --delta-base-offset --revs <in >/dev/null && + + # We can only reuse the 3 objects corresponding to "other" from + # the latest pack. + # + # This is because even though we want "delta", we do not want + # "base", meaning that we have to inflate the delta/base-pair + # corresponding to the blob in commit "delta", which bypasses + # the pack-reuse mechanism. + # + # The remaining objects from the other pack are similarly not + # reused because their objects are on the uninteresting side of + # the query. + test_pack_reused 3 <trace2.txt && + test_packs_reused 1 <trace2.txt +' + +test_expect_success 'omit delta from uninteresting base (cross pack)' ' + cat >in <<-EOF && + $(git rev-parse $base) + ^$(git rev-parse $delta) + EOF + + P="$(git pack-objects --revs $packdir/pack <in)" && + + git multi-pack-index write --bitmap --preferred-pack="pack-$P.idx" && + + : >trace2.txt && + GIT_TRACE2_EVENT="$PWD/trace2.txt" \ + git pack-objects --stdout --delta-base-offset --all >/dev/null && + + packs_nr="$(find $packdir -type f -name "pack-*.pack" | wc -l)" && + objects_nr="$(git rev-list --count --all --objects)" && + + test_pack_reused $(($objects_nr - 1)) <trace2.txt && + test_packs_reused $packs_nr <trace2.txt +' + +test_done diff --git a/t/t5510-fetch.sh b/t/t5510-fetch.sh index 79592a3b0a..33d34d5ae9 100755 --- a/t/t5510-fetch.sh +++ b/t/t5510-fetch.sh @@ -803,7 +803,8 @@ test_expect_success 'fetch.writeCommitGraph with submodules' ' cd super-clone && rm -rf .git/objects/info && git -c fetch.writeCommitGraph=true fetch origin && - test_path_is_file .git/objects/info/commit-graphs/commit-graph-chain + test_path_is_file .git/objects/info/commit-graphs/commit-graph-chain && + git -c fetch.writeCommitGraph=true fetch --recurse-submodules origin ) ' diff --git a/t/t5514-fetch-multiple.sh b/t/t5514-fetch-multiple.sh index a95841dc36..25772c85c5 100755 --- a/t/t5514-fetch-multiple.sh +++ b/t/t5514-fetch-multiple.sh @@ -24,6 +24,15 @@ setup_repository () { ) } +setup_test_clone () { + test_dir="$1" && + git clone one "$test_dir" && + for r in one two three + do + git -C "$test_dir" remote add "$r" "../$r" || return 1 + done +} + test_expect_success setup ' setup_repository one && setup_repository two && @@ -209,4 +218,156 @@ test_expect_success 'git fetch --multiple --jobs=0 picks a default' ' git fetch --multiple --jobs=0) ' +create_fetch_all_expect () { + cat >expect <<-\EOF + one/main + one/side + origin/HEAD -> origin/main + origin/main + origin/side + three/another + three/main + three/side + two/another + two/main + two/side + EOF +} + +for fetch_all in true false +do + test_expect_success "git fetch --all (works with fetch.all = $fetch_all)" ' + test_dir="test_fetch_all_$fetch_all" && + setup_test_clone "$test_dir" && + ( + cd "$test_dir" && + git config fetch.all $fetch_all && + git fetch --all && + create_fetch_all_expect && + git branch -r >actual && + test_cmp expect actual + ) + ' +done + +test_expect_success 'git fetch (fetch all remotes with fetch.all = true)' ' + setup_test_clone test9 && + ( + cd test9 && + git config fetch.all true && + git fetch && + git branch -r >actual && + create_fetch_all_expect && + test_cmp expect actual + ) +' + +create_fetch_one_expect () { + cat >expect <<-\EOF + one/main + one/side + origin/HEAD -> origin/main + origin/main + origin/side + EOF +} + +test_expect_success 'git fetch one (explicit remote overrides fetch.all)' ' + setup_test_clone test10 && + ( + cd test10 && + git config fetch.all true && + git fetch one && + create_fetch_one_expect && + git branch -r >actual && + test_cmp expect actual + ) +' + +create_fetch_two_as_origin_expect () { + cat >expect <<-\EOF + origin/HEAD -> origin/main + origin/another + origin/main + origin/side + EOF +} + +test_expect_success 'git config fetch.all false (fetch only default remote)' ' + setup_test_clone test11 && + ( + cd test11 && + git config fetch.all false && + git remote set-url origin ../two && + git fetch && + create_fetch_two_as_origin_expect && + git branch -r >actual && + test_cmp expect actual + ) +' + +for fetch_all in true false +do + test_expect_success "git fetch --no-all (fetch only default remote with fetch.all = $fetch_all)" ' + test_dir="test_no_all_fetch_all_$fetch_all" && + setup_test_clone "$test_dir" && + ( + cd "$test_dir" && + git config fetch.all $fetch_all && + git remote set-url origin ../two && + git fetch --no-all && + create_fetch_two_as_origin_expect && + git branch -r >actual && + test_cmp expect actual + ) + ' +done + +test_expect_success 'git fetch --no-all (fetch only default remote without fetch.all)' ' + setup_test_clone test12 && + ( + cd test12 && + git config --unset-all fetch.all || true && + git remote set-url origin ../two && + git fetch --no-all && + create_fetch_two_as_origin_expect && + git branch -r >actual && + test_cmp expect actual + ) +' + +test_expect_success 'git fetch --all --no-all (fetch only default remote)' ' + setup_test_clone test13 && + ( + cd test13 && + git remote set-url origin ../two && + git fetch --all --no-all && + create_fetch_two_as_origin_expect && + git branch -r >actual && + test_cmp expect actual + ) +' + +test_expect_success 'git fetch --no-all one (fetch only explicit remote)' ' + setup_test_clone test14 && + ( + cd test14 && + git fetch --no-all one && + create_fetch_one_expect && + git branch -r >actual && + test_cmp expect actual + ) +' + +test_expect_success 'git fetch --no-all --all (fetch all remotes)' ' + setup_test_clone test15 && + ( + cd test15 && + git fetch --no-all --all && + create_fetch_all_expect && + git branch -r >actual && + test_cmp expect actual + ) +' + test_done diff --git a/t/t5541-http-push-smart.sh b/t/t5541-http-push-smart.sh index df758e187d..71428f3d5c 100755 --- a/t/t5541-http-push-smart.sh +++ b/t/t5541-http-push-smart.sh @@ -232,8 +232,9 @@ test_expect_success 'push --atomic fails on server-side errors' ' test_config -C "$d" http.receivepack true && up="$HTTPD_URL"/smart/atomic-branches.git && - # break ref updates for other on the remote site - mkdir "$d/refs/heads/other.lock" && + # Create d/f conflict to break ref updates for other on the remote site. + git -C "$d" update-ref -d refs/heads/other && + git -C "$d" update-ref refs/heads/other/conflict HEAD && # add the new commit to other git branch -f other collateral && @@ -241,18 +242,9 @@ test_expect_success 'push --atomic fails on server-side errors' ' # --atomic should cause entire push to be rejected test_must_fail git push --atomic "$up" atomic other 2>output && - # the new branch should not have been created upstream - test_must_fail git -C "$d" show-ref --verify refs/heads/atomic && - - # upstream should still reflect atomic2, the last thing we pushed - # successfully - git rev-parse atomic2 >expected && - # ...to other. - git -C "$d" rev-parse refs/heads/other >actual && - test_cmp expected actual && - - # the new branch should not have been created upstream + # The atomic and other branches should not be created upstream. test_must_fail git -C "$d" show-ref --verify refs/heads/atomic && + test_must_fail git -C "$d" show-ref --verify refs/heads/other && # the failed refs should be indicated to the user grep "^ ! .*rejected.* other -> other .*atomic transaction failed" output && diff --git a/t/t5601-clone.sh b/t/t5601-clone.sh index 47eae641f0..fb1b9c686d 100755 --- a/t/t5601-clone.sh +++ b/t/t5601-clone.sh @@ -157,6 +157,23 @@ test_expect_success 'clone --mirror does not repeat tags' ' ' +test_expect_success 'clone with files ref format' ' + test_when_finished "rm -rf ref-storage" && + git clone --ref-format=files --mirror src ref-storage && + echo files >expect && + git -C ref-storage rev-parse --show-ref-format >actual && + test_cmp expect actual +' + +test_expect_success 'clone with garbage ref format' ' + cat >expect <<-EOF && + fatal: unknown ref storage format ${SQ}garbage${SQ} + EOF + test_must_fail git clone --ref-format=garbage --mirror src ref-storage 2>err && + test_cmp expect err && + test_path_is_missing ref-storage +' + test_expect_success 'clone to destination with trailing /' ' git clone src target-1/ && diff --git a/t/t6113-rev-list-bitmap-filters.sh b/t/t6113-rev-list-bitmap-filters.sh index 86c70521f1..459f0d7412 100755 --- a/t/t6113-rev-list-bitmap-filters.sh +++ b/t/t6113-rev-list-bitmap-filters.sh @@ -4,6 +4,8 @@ test_description='rev-list combining bitmaps and filters' . ./test-lib.sh . "$TEST_DIRECTORY"/lib-bitmap.sh +TEST_PASSES_SANITIZE_LEAK=true + test_expect_success 'set up bitmapped repo' ' # one commit will have bitmaps, the other will not test_commit one && diff --git a/t/t6135-pathspec-with-attrs.sh b/t/t6135-pathspec-with-attrs.sh index 00b84ecba6..120dcd74a5 100755 --- a/t/t6135-pathspec-with-attrs.sh +++ b/t/t6135-pathspec-with-attrs.sh @@ -393,4 +393,31 @@ test_expect_success 'reading from .gitattributes in a subdirectory (3)' ' test_cmp expect actual ' +test_expect_success POSIXPERM 'pathspec with builtin_objectmode attr can be used' ' + >mode_exec_file_1 && + + git status -s ":(attr:builtin_objectmode=100644)mode_exec_*" >actual && + echo ?? mode_exec_file_1 >expect && + test_cmp expect actual && + + git add mode_exec_file_1 && + chmod +x mode_exec_file_1 && + git status -s ":(attr:builtin_objectmode=100755)mode_exec_*" >actual && + echo AM mode_exec_file_1 >expect && + test_cmp expect actual +' + +test_expect_success POSIXPERM 'builtin_objectmode attr can be excluded' ' + >mode_1_regular && + >mode_1_exec && + chmod +x mode_1_exec && + git status -s ":(exclude,attr:builtin_objectmode=100644)" "mode_1_*" >actual && + echo ?? mode_1_exec >expect && + test_cmp expect actual && + + git status -s ":(exclude,attr:builtin_objectmode=100755)" "mode_1_*" >actual && + echo ?? mode_1_regular >expect && + test_cmp expect actual +' + test_done diff --git a/t/t6302-for-each-ref-filter.sh b/t/t6302-for-each-ref-filter.sh index 82f3d1ea0f..3922326cab 100755 --- a/t/t6302-for-each-ref-filter.sh +++ b/t/t6302-for-each-ref-filter.sh @@ -31,6 +31,40 @@ test_expect_success 'setup some history and refs' ' git update-ref refs/odd/spot main ' +cat >expect <<-\EOF + HEAD + ORIG_HEAD + refs/heads/main + refs/heads/side + refs/odd/spot + refs/tags/annotated-tag + refs/tags/doubly-annotated-tag + refs/tags/doubly-signed-tag + refs/tags/four + refs/tags/one + refs/tags/signed-tag + refs/tags/three + refs/tags/two +EOF + +test_expect_success 'empty pattern prints pseudorefs' ' + git update-ref ORIG_HEAD main && + git for-each-ref --format="%(refname)" "" >actual && + test_cmp expect actual +' + +test_expect_success 'empty pattern with other patterns' ' + git update-ref ORIG_HEAD main && + git for-each-ref --format="%(refname)" "" "refs/" >actual && + test_cmp expect actual +' + +test_expect_success 'empty pattern towards the end' ' + git update-ref ORIG_HEAD main && + git for-each-ref --format="%(refname)" "refs/" "" >actual && + test_cmp expect actual +' + test_expect_success 'filtering with --points-at' ' cat >expect <<-\EOF && refs/heads/main diff --git a/t/t7450-bad-git-dotfiles.sh b/t/t7450-bad-git-dotfiles.sh index 35a31acd4d..46d4fb0354 100755 --- a/t/t7450-bad-git-dotfiles.sh +++ b/t/t7450-bad-git-dotfiles.sh @@ -45,6 +45,32 @@ test_expect_success 'check names' ' test_cmp expect actual ' +test_expect_success 'check urls' ' + cat >expect <<-\EOF && + ./bar/baz/foo.git + https://example.com/foo.git + http://example.com:80/deeper/foo.git + EOF + + test-tool submodule check-url >actual <<-\EOF && + ./bar/baz/foo.git + https://example.com/foo.git + http://example.com:80/deeper/foo.git + -a./foo + ../../..//test/foo.git + ../../../../../:localhost:8080/foo.git + ..\../.\../:example.com/foo.git + ./%0ahost=example.com/foo.git + https://one.example.com/evil?%0ahost=two.example.com + https:///example.com/foo.git + http://example.com:test/foo.git + https::example.com/foo.git + http:::example.com/foo.git + EOF + + test_cmp expect actual +' + test_expect_success 'create innocent subrepo' ' git init innocent && git -C innocent commit --allow-empty -m foo diff --git a/t/t7501-commit-basic-functionality.sh b/t/t7501-commit-basic-functionality.sh index 3d8500a52e..bced44a0fc 100755 --- a/t/t7501-commit-basic-functionality.sh +++ b/t/t7501-commit-basic-functionality.sh @@ -3,8 +3,7 @@ # Copyright (c) 2007 Kristian Høgsberg <krh@redhat.com> # -# FIXME: Test the various index usages, -i and -o, test reflog, -# signoff +# FIXME: Test the various index usages, test reflog test_description='git commit' GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main @@ -92,6 +91,34 @@ test_expect_success '--long fails with nothing to commit' ' test_must_fail git commit -m initial --long ' +test_expect_success 'fail to commit untracked file (even with --include/--only)' ' + echo content >baz && + error="error: pathspec .baz. did not match any file(s) known to git" && + + test_must_fail git commit -m "baz" baz 2>err && + test_grep -e "$error" err && + + test_must_fail git commit --only -m "baz" baz 2>err && + test_grep -e "$error" err && + + # TODO: as for --include, the below command will fail because + # nothing is staged. If something was staged, it would not fail + # even though the provided pathspec does not match any tracked + # path. (However, the untracked paths that match the pathspec are + # not committed and only the staged changes get committed.) + # In either cases, no error is returned to stderr like in (--only + # and without --only/--include) cases. In a similar manner, + # "git add -u baz" also does not error out. + # + # Therefore, the below test is just to document the current behavior + # and is not an endorsement to the current behavior, and we may + # want to fix this. And when that happens, this test should be + # updated accordingly. + + test_must_fail git commit --include -m "baz" baz 2>err && + test_must_be_empty err +' + test_expect_success 'setup: non-initial commit' ' echo bongo bongo bongo >file && git commit -m next -a @@ -117,6 +144,51 @@ test_expect_success '--long with stuff to commit returns ok' ' git commit -m next -a --long ' +for opt in "" "-o" "--only" +do + test_expect_success 'exclude additional staged changes when given pathspec' ' + echo content >>file && + echo content >>baz && + git add baz && + git commit $opt -m "file" file && + + git diff --name-only >actual && + test_must_be_empty actual && + + test_write_lines baz >expect && + git diff --name-only --cached >actual && + test_cmp expect actual && + + test_write_lines file >expect && + git diff --name-only HEAD^ HEAD >actual && + test_cmp expect actual + ' +done + +test_expect_success '-i/--include includes staged changes' ' + echo content >>file && + echo content >>baz && + git add file && + + # baz is in the index, therefore, it will be committed + git commit --include -m "file and baz" baz && + + git diff --name-only HEAD >remaining && + test_must_be_empty remaining && + + test_write_lines baz file >expect && + git diff --name-only HEAD^ HEAD >actual && + test_cmp expect actual +' + +test_expect_success '--include and --only do not mix' ' + test_when_finished "git reset --hard" && + echo content >>file && + echo content >>baz && + test_must_fail git commit --include --only -m "file baz" file baz 2>actual && + test_grep -e "fatal: options .-i/--include. and .-o/--only. cannot be used together" actual +' + test_expect_success 'commit message from non-existing file' ' echo more bongo: bongo bongo bongo bongo >file && test_must_fail git commit -F gah -a @@ -389,6 +461,28 @@ test_expect_success 'amend commit to fix date' ' ' +test_expect_success 'amend commit to add signoff' ' + + test_commit "msg" file content && + git commit --amend --signoff && + test_commit_message HEAD <<-EOF + msg + + Signed-off-by: $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> + EOF +' + +test_expect_success 'amend does not add signoff if it already exists' ' + + test_commit --signoff "tenor" file newcontent && + git commit --amend --signoff && + test_commit_message HEAD <<-EOF + tenor + + Signed-off-by: $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> + EOF +' + test_expect_success 'commit mentions forced date in output' ' git commit --amend --date=2010-01-02T03:04:05 >output && grep "Date: *Sat Jan 2 03:04:05 2010" output diff --git a/t/t7527-builtin-fsmonitor.sh b/t/t7527-builtin-fsmonitor.sh index 78503158fd..363f9dc0e4 100755 --- a/t/t7527-builtin-fsmonitor.sh +++ b/t/t7527-builtin-fsmonitor.sh @@ -978,7 +978,7 @@ test_expect_success !UNICODE_COMPOSITION_SENSITIVE 'Unicode nfc/nfd' ' mkdir test_unicode/nfd && mkdir test_unicode/nfd/d_${utf8_nfd} && - git -C test_unicode fsmonitor--daemon stop && + test-tool -C test_unicode fsmonitor-client query --token 0 && if test_have_prereq UNICODE_NFC_PRESERVED then diff --git a/t/t7900-maintenance.sh b/t/t7900-maintenance.sh index 00d29871e6..0943dfa18a 100755 --- a/t/t7900-maintenance.sh +++ b/t/t7900-maintenance.sh @@ -67,6 +67,51 @@ test_expect_success 'maintenance.auto config option' ' test_subcommand ! git maintenance run --auto --quiet <false ' +test_expect_success 'register uses XDG_CONFIG_HOME config if it exists' ' + test_when_finished rm -r .config/git/config && + ( + XDG_CONFIG_HOME=.config && + export XDG_CONFIG_HOME && + mkdir -p $XDG_CONFIG_HOME/git && + >$XDG_CONFIG_HOME/git/config && + git maintenance register && + git config --file=$XDG_CONFIG_HOME/git/config --get maintenance.repo >actual && + pwd >expect && + test_cmp expect actual + ) +' + +test_expect_success 'register does not need XDG_CONFIG_HOME config to exist' ' + test_when_finished git maintenance unregister && + test_path_is_missing $XDG_CONFIG_HOME/git/config && + git maintenance register && + git config --global --get maintenance.repo >actual && + pwd >expect && + test_cmp expect actual +' + +test_expect_success 'unregister uses XDG_CONFIG_HOME config if it exists' ' + test_when_finished rm -r .config/git/config && + ( + XDG_CONFIG_HOME=.config && + export XDG_CONFIG_HOME && + mkdir -p $XDG_CONFIG_HOME/git && + >$XDG_CONFIG_HOME/git/config && + git maintenance register && + git maintenance unregister && + test_must_fail git config --file=$XDG_CONFIG_HOME/git/config --get maintenance.repo >actual && + test_must_be_empty actual + ) +' + +test_expect_success 'unregister does not need XDG_CONFIG_HOME config to exist' ' + test_path_is_missing $XDG_CONFIG_HOME/git/config && + git maintenance register && + git maintenance unregister && + test_must_fail git config --global --get maintenance.repo >actual && + test_must_be_empty actual +' + test_expect_success 'maintenance.<task>.enabled' ' git config maintenance.gc.enabled false && git config maintenance.commit-graph.enabled true && diff --git a/t/t9500-gitweb-standalone-no-errors.sh b/t/t9500-gitweb-standalone-no-errors.sh index 0333065d4d..7679780fb8 100755 --- a/t/t9500-gitweb-standalone-no-errors.sh +++ b/t/t9500-gitweb-standalone-no-errors.sh @@ -627,6 +627,7 @@ test_expect_success \ test_expect_success 'setup' ' version=$(git config core.repositoryformatversion) && algo=$(test_might_fail git config extensions.objectformat) && + refstorage=$(test_might_fail git config extensions.refstorage) && cat >.git/config <<-\EOF && # testing noval and alternate separator [gitweb] @@ -637,6 +638,10 @@ test_expect_success 'setup' ' if test -n "$algo" then git config extensions.objectformat "$algo" + fi && + if test -n "$refstorage" + then + git config extensions.refstorage "$refstorage" fi ' diff --git a/t/t9902-completion.sh b/t/t9902-completion.sh index aa9a614de3..35eb534fdd 100755 --- a/t/t9902-completion.sh +++ b/t/t9902-completion.sh @@ -5,6 +5,12 @@ test_description='test bash completion' +# The Bash completion scripts must not print anything to either stdout or +# stderr, which we try to verify. When tracing is enabled without support for +# BASH_XTRACEFD this assertion will fail, so we have to mark the test as +# untraceable with such ancient Bash versions. +test_untraceable=UnfortunatelyYes + . ./lib-bash.sh complete () @@ -87,9 +93,11 @@ test_completion () else sed -e 's/Z$//' |sort >expected fi && - run_completion "$1" && + run_completion "$1" >"$TRASH_DIRECTORY"/bash-completion-output 2>&1 && sort out >out_sorted && - test_cmp expected out_sorted + test_cmp expected out_sorted && + test_must_be_empty "$TRASH_DIRECTORY"/bash-completion-output && + rm "$TRASH_DIRECTORY"/bash-completion-output } # Test __gitcomp. @@ -1925,6 +1933,14 @@ test_expect_success 'git checkout - --orphan with branch already provided comple EOF ' +test_expect_success 'git restore completes modified files' ' + test_commit A a.file && + echo B >a.file && + test_completion "git restore a." <<-\EOF + a.file + EOF +' + test_expect_success 'teardown after ref completion' ' git branch -d matching-branch && git tag -d matching-tag && @@ -2720,4 +2736,31 @@ test_expect_success '__git_complete' ' test_must_fail __git_complete ga missing ' +test_expect_success '__git_pseudoref_exists' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + sane_unset __git_repo_path && + + # HEAD should exist, even if it points to an unborn branch. + __git_pseudoref_exists HEAD >output 2>&1 && + test_must_be_empty output && + + # HEAD points to an existing branch, so it should exist. + test_commit A && + __git_pseudoref_exists HEAD >output 2>&1 && + test_must_be_empty output && + + # CHERRY_PICK_HEAD does not exist, so the existence check should fail. + ! __git_pseudoref_exists CHERRY_PICK_HEAD >output 2>&1 && + test_must_be_empty output && + + # CHERRY_PICK_HEAD points to a commit, so it should exist. + git update-ref CHERRY_PICK_HEAD A && + __git_pseudoref_exists CHERRY_PICK_HEAD >output 2>&1 && + test_must_be_empty output + ) +' + test_done diff --git a/t/test-lib-functions.sh b/t/test-lib-functions.sh index 5eb57914ab..192a1ada2d 100644 --- a/t/test-lib-functions.sh +++ b/t/test-lib-functions.sh @@ -1656,7 +1656,21 @@ test_set_hash () { # Detect the hash algorithm in use. test_detect_hash () { - test_hash_algo="${GIT_TEST_DEFAULT_HASH:-sha1}" + case "$GIT_TEST_DEFAULT_HASH" in + "sha256") + test_hash_algo=sha256 + test_compat_hash_algo=sha1 + ;; + *) + test_hash_algo=sha1 + test_compat_hash_algo=sha256 + ;; + esac +} + +# Detect the hash algorithm in use. +test_detect_ref_format () { + echo "${GIT_TEST_DEFAULT_REF_FORMAT:-files}" } # Load common hash metadata and common placeholder object IDs for use with @@ -1708,6 +1722,12 @@ test_oid () { local algo="${test_hash_algo}" && case "$1" in + --hash=storage) + algo="$test_hash_algo" && + shift;; + --hash=compat) + algo="$test_compat_hash_algo" && + shift;; --hash=*) algo="${1#--hash=}" && shift;; @@ -1874,6 +1894,20 @@ test_region () { return 0 } +# Check that the given data fragment was included as part of the +# trace2-format trace on stdin. +# +# test_trace2_data <category> <key> <value> +# +# For example, to look for trace2_data_intmax("pack-objects", repo, +# "reused", N) in an invocation of "git pack-objects", run: +# +# GIT_TRACE2_EVENT="$(pwd)/trace.txt" git pack-objects ... && +# test_trace2_data pack-objects reused N <trace2.txt +test_trace2_data () { + grep -e '"category":"'"$1"'","key":"'"$2"'","value":"'"$3"'"' +} + # Given a GIT_TRACE2_EVENT log over stdin, writes to stdout a list of URLs # sent to git-remote-https child processes. test_remote_https_urls() { diff --git a/t/test-lib-github-workflow-markup.sh b/t/test-lib-github-workflow-markup.sh index 970c6538cb..33405c90d7 100644 --- a/t/test-lib-github-workflow-markup.sh +++ b/t/test-lib-github-workflow-markup.sh @@ -42,8 +42,8 @@ finalize_test_case_output () { fixed) echo >>$github_markup_output "::notice::fixed: $this_test.$test_count $1" ;; - ok) - # Exit without printing the "ok" tests + ok|broken) + # Exit without printing the "ok" or ""broken" tests return ;; esac diff --git a/t/test-lib.sh b/t/test-lib.sh index 876b99562a..fc93aa57e6 100644 --- a/t/test-lib.sh +++ b/t/test-lib.sh @@ -542,6 +542,8 @@ export EDITOR GIT_DEFAULT_HASH="${GIT_TEST_DEFAULT_HASH:-sha1}" export GIT_DEFAULT_HASH +GIT_DEFAULT_REF_FORMAT="${GIT_TEST_DEFAULT_REF_FORMAT:-files}" +export GIT_DEFAULT_REF_FORMAT GIT_TEST_MERGE_ALGORITHM="${GIT_TEST_MERGE_ALGORITHM:-ort}" export GIT_TEST_MERGE_ALGORITHM @@ -1745,7 +1747,14 @@ parisc* | hppa*) ;; esac -test_set_prereq REFFILES +case "$GIT_DEFAULT_REF_FORMAT" in +files) + test_set_prereq REFFILES;; +*) + echo 2>&1 "error: unknown ref format $GIT_DEFAULT_REF_FORMAT" + exit 1 + ;; +esac ( COLUMNS=1 && test $COLUMNS = 1 ) && test_set_prereq COLUMNS_CAN_BE_1 test -z "$NO_CURL" && test_set_prereq LIBCURL @@ -1936,6 +1945,10 @@ test_lazy_prereq SHA1 ' esac ' +test_lazy_prereq DEFAULT_REPO_FORMAT ' + test_have_prereq SHA1,REFFILES +' + # Ensure that no test accidentally triggers a Git command # that runs the actual maintenance scheduler, affecting a user's # system permanently. diff --git a/t/unit-tests/t-ctype.c b/t/unit-tests/t-ctype.c new file mode 100644 index 0000000000..f315489984 --- /dev/null +++ b/t/unit-tests/t-ctype.c @@ -0,0 +1,80 @@ +#include "test-lib.h" + +static int is_in(const char *s, int ch) +{ + /* + * We can't find NUL using strchr. Accept it as the first + * character in the spec -- there are no empty classes. + */ + if (ch == '\0') + return ch == *s; + if (*s == '\0') + s++; + return !!strchr(s, ch); +} + +/* Macro to test a character type */ +#define TEST_CTYPE_FUNC(func, string) \ +static void test_ctype_##func(void) { \ + for (int i = 0; i < 256; i++) { \ + if (!check_int(func(i), ==, is_in(string, i))) \ + test_msg(" i: 0x%02x", i); \ + } \ + if (!check(!func(EOF))) \ + test_msg(" i: 0x%02x (EOF)", EOF); \ +} + +#define TEST_CHAR_CLASS(class) TEST(test_ctype_##class(), #class " works") + +#define DIGIT "0123456789" +#define LOWER "abcdefghijklmnopqrstuvwxyz" +#define UPPER "ABCDEFGHIJKLMNOPQRSTUVWXYZ" +#define PUNCT "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~" +#define ASCII \ + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" \ + "\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f" \ + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f" \ + "\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f" \ + "\x40\x41\x42\x43\x44\x45\x46\x47\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f" \ + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f" \ + "\x60\x61\x62\x63\x64\x65\x66\x67\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f" \ + "\x70\x71\x72\x73\x74\x75\x76\x77\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f" +#define CNTRL \ + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" \ + "\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f" \ + "\x7f" + +TEST_CTYPE_FUNC(isdigit, DIGIT) +TEST_CTYPE_FUNC(isspace, " \n\r\t") +TEST_CTYPE_FUNC(isalpha, LOWER UPPER) +TEST_CTYPE_FUNC(isalnum, LOWER UPPER DIGIT) +TEST_CTYPE_FUNC(is_glob_special, "*?[\\") +TEST_CTYPE_FUNC(is_regex_special, "$()*+.?[\\^{|") +TEST_CTYPE_FUNC(is_pathspec_magic, "!\"#%&',-/:;<=>@_`~") +TEST_CTYPE_FUNC(isascii, ASCII) +TEST_CTYPE_FUNC(islower, LOWER) +TEST_CTYPE_FUNC(isupper, UPPER) +TEST_CTYPE_FUNC(iscntrl, CNTRL) +TEST_CTYPE_FUNC(ispunct, PUNCT) +TEST_CTYPE_FUNC(isxdigit, DIGIT "abcdefABCDEF") +TEST_CTYPE_FUNC(isprint, LOWER UPPER DIGIT PUNCT " ") + +int cmd_main(int argc, const char **argv) { + /* Run all character type tests */ + TEST_CHAR_CLASS(isspace); + TEST_CHAR_CLASS(isdigit); + TEST_CHAR_CLASS(isalpha); + TEST_CHAR_CLASS(isalnum); + TEST_CHAR_CLASS(is_glob_special); + TEST_CHAR_CLASS(is_regex_special); + TEST_CHAR_CLASS(is_pathspec_magic); + TEST_CHAR_CLASS(isascii); + TEST_CHAR_CLASS(islower); + TEST_CHAR_CLASS(isupper); + TEST_CHAR_CLASS(iscntrl); + TEST_CHAR_CLASS(ispunct); + TEST_CHAR_CLASS(isxdigit); + TEST_CHAR_CLASS(isprint); + + return test_done(); +} diff --git a/t/unit-tests/t-prio-queue.c b/t/unit-tests/t-prio-queue.c new file mode 100644 index 0000000000..d78b002f9e --- /dev/null +++ b/t/unit-tests/t-prio-queue.c @@ -0,0 +1,98 @@ +#include "test-lib.h" +#include "prio-queue.h" + +static int intcmp(const void *va, const void *vb, void *data UNUSED) +{ + const int *a = va, *b = vb; + return *a - *b; +} + + +#define MISSING -1 +#define DUMP -2 +#define STACK -3 +#define GET -4 +#define REVERSE -5 + +static int show(int *v) +{ + return v ? *v : MISSING; +} + +static void test_prio_queue(int *input, int *result, size_t input_size) +{ + struct prio_queue pq = { intcmp }; + + for (int i = 0, j = 0; i < input_size; i++) { + void *peek, *get; + switch(input[i]) { + case GET: + peek = prio_queue_peek(&pq); + get = prio_queue_get(&pq); + if (!check(peek == get)) + return; + if(!check_int(result[j++], ==, show(get))) + test_msg("failed at result[] index %d", j-1); + break; + case DUMP: + while ((peek = prio_queue_peek(&pq))) { + get = prio_queue_get(&pq); + if (!check(peek == get)) + return; + if(!check_int(result[j++], ==, show(get))) + test_msg("failed at result[] index %d", j-1); + } + break; + case STACK: + pq.compare = NULL; + break; + case REVERSE: + prio_queue_reverse(&pq); + break; + default: + prio_queue_put(&pq, &input[i]); + break; + } + } + clear_prio_queue(&pq); +} + +#define BASIC_INPUT 2, 6, 3, 10, 9, 5, 7, 4, 5, 8, 1, DUMP +#define BASIC_RESULT 1, 2, 3, 4, 5, 5, 6, 7, 8, 9, 10 + +#define MIXED_PUT_GET_INPUT 6, 2, 4, GET, 5, 3, GET, GET, 1, DUMP +#define MIXED_PUT_GET_RESULT 2, 3, 4, 1, 5, 6 + +#define EMPTY_QUEUE_INPUT 1, 2, GET, GET, GET, 1, 2, GET, GET, GET +#define EMPTY_QUEUE_RESULT 1, 2, MISSING, 1, 2, MISSING + +#define STACK_INPUT STACK, 8, 1, 5, 4, 6, 2, 3, DUMP +#define STACK_RESULT 3, 2, 6, 4, 5, 1, 8 + +#define REVERSE_STACK_INPUT STACK, 1, 2, 3, 4, 5, 6, REVERSE, DUMP +#define REVERSE_STACK_RESULT 1, 2, 3, 4, 5, 6 + +#define TEST_INPUT(INPUT, RESULT, name) \ + static void test_##name(void) \ +{ \ + int input[] = {INPUT}; \ + int result[] = {RESULT}; \ + test_prio_queue(input, result, ARRAY_SIZE(input)); \ +} + +TEST_INPUT(BASIC_INPUT, BASIC_RESULT, basic) +TEST_INPUT(MIXED_PUT_GET_INPUT, MIXED_PUT_GET_RESULT, mixed) +TEST_INPUT(EMPTY_QUEUE_INPUT, EMPTY_QUEUE_RESULT, empty) +TEST_INPUT(STACK_INPUT, STACK_RESULT, stack) +TEST_INPUT(REVERSE_STACK_INPUT, REVERSE_STACK_RESULT, reverse) + +int cmd_main(int argc, const char **argv) +{ + TEST(test_basic(), "prio-queue works for basic input"); + TEST(test_mixed(), "prio-queue works for mixed put & get commands"); + TEST(test_empty(), "prio-queue works when queue is empty"); + TEST(test_stack(), "prio-queue works when used as a LIFO stack"); + TEST(test_reverse(), "prio-queue works when LIFO stack is reversed"); + + return test_done(); +} @@ -5,14 +5,35 @@ #include "string-list.h" #include "run-command.h" #include "commit.h" -#include "tempfile.h" #include "trailer.h" #include "list.h" /* * Copyright (c) 2013, 2014 Christian Couder <chriscool@tuxfamily.org> */ -struct conf_info { +struct trailer_block { + /* + * True if there is a blank line before the location pointed to by + * "start". + */ + int blank_line_before_trailer; + + /* + * The locations of the start and end positions of the trailer block + * found, as offsets from the beginning of the source text from which + * this trailer block was parsed. If no trailer block is found, these + * are both set to 0. + */ + size_t start, end; + + /* + * Array of trailers found. + */ + char **trailers; + size_t trailer_nr; +}; + +struct trailer_conf { char *name; char *key; char *command; @@ -22,7 +43,7 @@ struct conf_info { enum trailer_if_missing if_missing; }; -static struct conf_info default_conf_info; +static struct trailer_conf default_trailer_conf; struct trailer_item { struct list_head list; @@ -38,13 +59,18 @@ struct arg_item { struct list_head list; char *token; char *value; - struct conf_info conf; + struct trailer_conf conf; }; static LIST_HEAD(conf_head); static char *separators = ":"; +const char *default_separators(void) +{ + return separators; +} + static int configured; #define TRAILER_ARG_STRING "$ARG" @@ -145,37 +171,6 @@ static char last_non_space_char(const char *s) return '\0'; } -static void print_tok_val(FILE *outfile, const char *tok, const char *val) -{ - char c; - - if (!tok) { - fprintf(outfile, "%s\n", val); - return; - } - - c = last_non_space_char(tok); - if (!c) - return; - if (strchr(separators, c)) - fprintf(outfile, "%s%s\n", tok, val); - else - fprintf(outfile, "%s%c %s\n", tok, separators[0], val); -} - -static void print_all(FILE *outfile, struct list_head *head, - const struct process_trailer_options *opts) -{ - struct list_head *pos; - struct trailer_item *item; - list_for_each(pos, head) { - item = list_entry(pos, struct trailer_item, list); - if ((!opts->trim_empty || strlen(item->value) > 0) && - (!opts->only_trailers || item->token)) - print_tok_val(outfile, item->token, item->value); - } -} - static struct trailer_item *trailer_from_arg(struct arg_item *arg_tok) { struct trailer_item *new_item = xcalloc(1, sizeof(*new_item)); @@ -220,7 +215,7 @@ static int check_if_different(struct trailer_item *in_tok, return 1; } -static char *apply_command(struct conf_info *conf, const char *arg) +static char *apply_command(struct trailer_conf *conf, const char *arg) { struct strbuf cmd = STRBUF_INIT; struct strbuf buf = STRBUF_INIT; @@ -366,8 +361,8 @@ static int find_same_and_apply_arg(struct list_head *head, return 0; } -static void process_trailers_lists(struct list_head *head, - struct list_head *arg_head) +void process_trailers_lists(struct list_head *head, + struct list_head *arg_head) { struct list_head *pos, *p; struct arg_item *arg_tok; @@ -434,7 +429,27 @@ int trailer_set_if_missing(enum trailer_if_missing *item, const char *value) return 0; } -static void duplicate_conf(struct conf_info *dst, const struct conf_info *src) +void trailer_conf_set(enum trailer_where where, + enum trailer_if_exists if_exists, + enum trailer_if_missing if_missing, + struct trailer_conf *conf) +{ + if (where != WHERE_DEFAULT) + conf->where = where; + if (if_exists != EXISTS_DEFAULT) + conf->if_exists = if_exists; + if (if_missing != MISSING_DEFAULT) + conf->if_missing = if_missing; +} + +struct trailer_conf *new_trailer_conf(void) +{ + struct trailer_conf *new = xcalloc(1, sizeof(*new)); + return new; +} + +void duplicate_trailer_conf(struct trailer_conf *dst, + const struct trailer_conf *src) { *dst = *src; dst->name = xstrdup_or_null(src->name); @@ -457,7 +472,7 @@ static struct arg_item *get_conf_item(const char *name) /* Item does not already exists, create it */ CALLOC_ARRAY(item, 1); - duplicate_conf(&item->conf, &default_conf_info); + duplicate_trailer_conf(&item->conf, &default_trailer_conf); item->conf.name = xstrdup(name); list_add_tail(&item->list, &conf_head); @@ -492,17 +507,17 @@ static int git_trailer_default_config(const char *conf_key, const char *value, variable_name = strrchr(trailer_item, '.'); if (!variable_name) { if (!strcmp(trailer_item, "where")) { - if (trailer_set_where(&default_conf_info.where, + if (trailer_set_where(&default_trailer_conf.where, value) < 0) warning(_("unknown value '%s' for key '%s'"), value, conf_key); } else if (!strcmp(trailer_item, "ifexists")) { - if (trailer_set_if_exists(&default_conf_info.if_exists, + if (trailer_set_if_exists(&default_trailer_conf.if_exists, value) < 0) warning(_("unknown value '%s' for key '%s'"), value, conf_key); } else if (!strcmp(trailer_item, "ifmissing")) { - if (trailer_set_if_missing(&default_conf_info.if_missing, + if (trailer_set_if_missing(&default_trailer_conf.if_missing, value) < 0) warning(_("unknown value '%s' for key '%s'"), value, conf_key); @@ -521,7 +536,7 @@ static int git_trailer_config(const char *conf_key, const char *value, { const char *trailer_item, *variable_name; struct arg_item *item; - struct conf_info *conf; + struct trailer_conf *conf; char *name = NULL; enum trailer_info_type type; int i; @@ -589,15 +604,15 @@ static int git_trailer_config(const char *conf_key, const char *value, return 0; } -static void ensure_configured(void) +void trailer_config_init(void) { if (configured) return; /* Default config must be setup first */ - default_conf_info.where = WHERE_END; - default_conf_info.if_exists = EXISTS_ADD_IF_DIFFERENT_NEIGHBOR; - default_conf_info.if_missing = MISSING_ADD; + default_trailer_conf.where = WHERE_END; + default_trailer_conf.if_exists = EXISTS_ADD_IF_DIFFERENT_NEIGHBOR; + default_trailer_conf.if_missing = MISSING_ADD; git_config(git_trailer_default_config, NULL); git_config(git_trailer_config, NULL); configured = 1; @@ -630,7 +645,7 @@ static int token_matches_item(const char *tok, struct arg_item *item, size_t tok * distinguished from the non-well-formed-line case (in which this function * returns -1) because some callers of this function need such a distinction. */ -static ssize_t find_separator(const char *line, const char *separators) +ssize_t find_separator(const char *line, const char *separators) { int whitespace_found = 0; const char *c; @@ -651,32 +666,35 @@ static ssize_t find_separator(const char *line, const char *separators) /* * Obtain the token, value, and conf from the given trailer. * + * The conf needs special handling. We first read hardcoded defaults, and + * override them if we find a matching trailer configuration in the config. + * * separator_pos must not be 0, since the token cannot be an empty string. * * If separator_pos is -1, interpret the whole trailer as a token. */ -static void parse_trailer(struct strbuf *tok, struct strbuf *val, - const struct conf_info **conf, const char *trailer, - ssize_t separator_pos) +void parse_trailer(const char *line, ssize_t separator_pos, + struct strbuf *tok, struct strbuf *val, + const struct trailer_conf **conf) { struct arg_item *item; size_t tok_len; struct list_head *pos; if (separator_pos != -1) { - strbuf_add(tok, trailer, separator_pos); + strbuf_add(tok, line, separator_pos); strbuf_trim(tok); - strbuf_addstr(val, trailer + separator_pos + 1); + strbuf_addstr(val, line + separator_pos + 1); strbuf_trim(val); } else { - strbuf_addstr(tok, trailer); + strbuf_addstr(tok, line); strbuf_trim(tok); } /* Lookup if the token matches something in the config */ tok_len = token_len_without_separator(tok->buf, tok->len); if (conf) - *conf = &default_conf_info; + *conf = &default_trailer_conf; list_for_each(pos, &conf_head) { item = list_entry(pos, struct arg_item, list); if (token_matches_item(tok->buf, item, tok_len)) { @@ -700,26 +718,18 @@ static struct trailer_item *add_trailer_item(struct list_head *head, char *tok, return new_item; } -static void add_arg_item(struct list_head *arg_head, char *tok, char *val, - const struct conf_info *conf, - const struct new_trailer_item *new_trailer_item) +void add_arg_item(char *tok, char *val, const struct trailer_conf *conf, + struct list_head *arg_head) + { struct arg_item *new_item = xcalloc(1, sizeof(*new_item)); new_item->token = tok; new_item->value = val; - duplicate_conf(&new_item->conf, conf); - if (new_trailer_item) { - if (new_trailer_item->where != WHERE_DEFAULT) - new_item->conf.where = new_trailer_item->where; - if (new_trailer_item->if_exists != EXISTS_DEFAULT) - new_item->conf.if_exists = new_trailer_item->if_exists; - if (new_trailer_item->if_missing != MISSING_DEFAULT) - new_item->conf.if_missing = new_trailer_item->if_missing; - } + duplicate_trailer_conf(&new_item->conf, conf); list_add_tail(&new_item->list, arg_head); } -static void parse_trailers_from_config(struct list_head *config_head) +void parse_trailers_from_config(struct list_head *config_head) { struct arg_item *item; struct list_head *pos; @@ -728,61 +738,10 @@ static void parse_trailers_from_config(struct list_head *config_head) list_for_each(pos, &conf_head) { item = list_entry(pos, struct arg_item, list); if (item->conf.command) - add_arg_item(config_head, - xstrdup(token_from_item(item, NULL)), + add_arg_item(xstrdup(token_from_item(item, NULL)), xstrdup(""), - &item->conf, NULL); - } -} - -static void parse_trailers_from_command_line_args(struct list_head *arg_head, - struct list_head *new_trailer_head) -{ - struct strbuf tok = STRBUF_INIT; - struct strbuf val = STRBUF_INIT; - const struct conf_info *conf; - struct list_head *pos; - - /* - * In command-line arguments, '=' is accepted (in addition to the - * separators that are defined). - */ - char *cl_separators = xstrfmt("=%s", separators); - - /* Add an arg item for each trailer on the command line */ - list_for_each(pos, new_trailer_head) { - struct new_trailer_item *tr = - list_entry(pos, struct new_trailer_item, list); - ssize_t separator_pos = find_separator(tr->text, cl_separators); - - if (separator_pos == 0) { - struct strbuf sb = STRBUF_INIT; - strbuf_addstr(&sb, tr->text); - strbuf_trim(&sb); - error(_("empty trailer token in trailer '%.*s'"), - (int) sb.len, sb.buf); - strbuf_release(&sb); - } else { - parse_trailer(&tok, &val, &conf, tr->text, - separator_pos); - add_arg_item(arg_head, - strbuf_detach(&tok, NULL), - strbuf_detach(&val, NULL), - conf, tr); - } - } - - free(cl_separators); -} - -static void read_input_file(struct strbuf *sb, const char *file) -{ - if (file) { - if (strbuf_read_file(sb, file, 0) < 0) - die_errno(_("could not read input file '%s'"), file); - } else { - if (strbuf_read(sb, fileno(stdin), 0) < 0) - die_errno(_("could not read from stdin")); + &item->conf, + config_head); } } @@ -996,146 +955,95 @@ static void unfold_value(struct strbuf *val) strbuf_release(&out); } -/* - * Parse trailers in "str", populating the trailer info and "head" - * linked list structure. - */ -static void parse_trailers(struct trailer_info *info, - const char *str, - struct list_head *head, - const struct process_trailer_options *opts) -{ - struct strbuf tok = STRBUF_INIT; - struct strbuf val = STRBUF_INIT; - size_t i; - - trailer_info_get(info, str, opts); - - for (i = 0; i < info->trailer_nr; i++) { - int separator_pos; - char *trailer = info->trailers[i]; - if (trailer[0] == comment_line_char) - continue; - separator_pos = find_separator(trailer, separators); - if (separator_pos >= 1) { - parse_trailer(&tok, &val, NULL, trailer, - separator_pos); - if (opts->unfold) - unfold_value(&val); - add_trailer_item(head, - strbuf_detach(&tok, NULL), - strbuf_detach(&val, NULL)); - } else if (!opts->only_trailers) { - strbuf_addstr(&val, trailer); - strbuf_strip_suffix(&val, "\n"); - add_trailer_item(head, - NULL, - strbuf_detach(&val, NULL)); - } - } -} - -static void free_all(struct list_head *head) -{ - struct list_head *pos, *p; - list_for_each_safe(pos, p, head) { - list_del(pos); - free_trailer_item(list_entry(pos, struct trailer_item, list)); - } -} - -static struct tempfile *trailers_tempfile; - -static FILE *create_in_place_tempfile(const char *file) +void format_trailers(struct list_head *head, + const struct process_trailer_options *opts, + struct strbuf *out) { - struct stat st; - struct strbuf filename_template = STRBUF_INIT; - const char *tail; - FILE *outfile; - - if (stat(file, &st)) - die_errno(_("could not stat %s"), file); - if (!S_ISREG(st.st_mode)) - die(_("file %s is not a regular file"), file); - if (!(st.st_mode & S_IWUSR)) - die(_("file %s is not writable by user"), file); - - /* Create temporary file in the same directory as the original */ - tail = strrchr(file, '/'); - if (tail) - strbuf_add(&filename_template, file, tail - file + 1); - strbuf_addstr(&filename_template, "git-interpret-trailers-XXXXXX"); - - trailers_tempfile = xmks_tempfile_m(filename_template.buf, st.st_mode); - strbuf_release(&filename_template); - outfile = fdopen_tempfile(trailers_tempfile, "w"); - if (!outfile) - die_errno(_("could not open temporary file")); - - return outfile; -} - -void process_trailers(const char *file, - const struct process_trailer_options *opts, - struct list_head *new_trailer_head) -{ - LIST_HEAD(head); - struct strbuf sb = STRBUF_INIT; - struct trailer_info info; - FILE *outfile = stdout; + struct list_head *pos; + struct trailer_item *item; + int need_separator = 0; - ensure_configured(); + list_for_each(pos, head) { + item = list_entry(pos, struct trailer_item, list); + if (item->token) { + char c; - read_input_file(&sb, file); + struct strbuf tok = STRBUF_INIT; + struct strbuf val = STRBUF_INIT; + strbuf_addstr(&tok, item->token); + strbuf_addstr(&val, item->value); + + /* + * Skip key/value pairs where the value was empty. This + * can happen from trailers specified without a + * separator, like `--trailer "Reviewed-by"` (no + * corresponding value). + */ + if (opts->trim_empty && !strlen(item->value)) + continue; - if (opts->in_place) - outfile = create_in_place_tempfile(file); + if (!opts->filter || opts->filter(&tok, opts->filter_data)) { + if (opts->unfold) + unfold_value(&val); - parse_trailers(&info, sb.buf, &head, opts); + if (opts->separator && need_separator) + strbuf_addbuf(out, opts->separator); + if (!opts->value_only) + strbuf_addbuf(out, &tok); + if (!opts->key_only && !opts->value_only) { + if (opts->key_value_separator) + strbuf_addbuf(out, opts->key_value_separator); + else { + c = last_non_space_char(tok.buf); + if (c) { + if (!strchr(separators, c)) + strbuf_addf(out, "%c ", separators[0]); + } + } + } + if (!opts->key_only) + strbuf_addbuf(out, &val); + if (!opts->separator) + strbuf_addch(out, '\n'); - /* Print the lines before the trailers */ - if (!opts->only_trailers) - fwrite(sb.buf, 1, info.trailer_block_start, outfile); + need_separator = 1; + } - if (!opts->only_trailers && !info.blank_line_before_trailer) - fprintf(outfile, "\n"); + strbuf_release(&tok); + strbuf_release(&val); + } else if (!opts->only_trailers) { + if (opts->separator && need_separator) { + strbuf_addbuf(out, opts->separator); + } + strbuf_addstr(out, item->value); + if (opts->separator) + strbuf_rtrim(out); + else + strbuf_addch(out, '\n'); + need_separator = 1; + } - if (!opts->only_input) { - LIST_HEAD(config_head); - LIST_HEAD(arg_head); - parse_trailers_from_config(&config_head); - parse_trailers_from_command_line_args(&arg_head, new_trailer_head); - list_splice(&config_head, &arg_head); - process_trailers_lists(&head, &arg_head); } +} - print_all(outfile, &head, opts); - - free_all(&head); - trailer_info_release(&info); - - /* Print the lines after the trailers as is */ - if (!opts->only_trailers) - fwrite(sb.buf + info.trailer_block_end, 1, sb.len - info.trailer_block_end, outfile); - - if (opts->in_place) - if (rename_tempfile(&trailers_tempfile, file)) - die_errno(_("could not rename temporary file to %s"), file); - - strbuf_release(&sb); +static struct trailer_block *trailer_block_new(void) +{ + struct trailer_block *trailer_block = xcalloc(1, sizeof(*trailer_block)); + return trailer_block; } -void trailer_info_get(struct trailer_info *info, const char *str, - const struct process_trailer_options *opts) +static struct trailer_block *trailer_block_get(const char *str, + const struct process_trailer_options *opts) { + struct trailer_block *trailer_block = trailer_block_new(); size_t end_of_log_message = 0, trailer_block_start = 0; struct strbuf **trailer_lines, **ptr; char **trailer_strings = NULL; size_t nr = 0, alloc = 0; char **last = NULL; - ensure_configured(); + trailer_config_init(); end_of_log_message = find_end_of_log_message(str, opts->no_divider); trailer_block_start = find_trailer_block_start(str, end_of_log_message); @@ -1161,91 +1069,117 @@ void trailer_info_get(struct trailer_info *info, const char *str, } strbuf_list_free(trailer_lines); - info->blank_line_before_trailer = ends_with_blank_line(str, - trailer_block_start); - info->trailer_block_start = trailer_block_start; - info->trailer_block_end = end_of_log_message; - info->trailers = trailer_strings; - info->trailer_nr = nr; + trailer_block->blank_line_before_trailer = ends_with_blank_line(str, + trailer_block_start); + trailer_block->start = trailer_block_start; + trailer_block->end = end_of_log_message; + trailer_block->trailers = trailer_strings; + trailer_block->trailer_nr = nr; + + return trailer_block; } -void trailer_info_release(struct trailer_info *info) +/* + * Parse trailers in "str", populating the trailer_block info and "head" linked + * list structure. + */ +struct trailer_block *parse_trailers(const char *str, + const struct process_trailer_options *opts, + struct list_head *head) { + struct trailer_block *trailer_block; + struct strbuf tok = STRBUF_INIT; + struct strbuf val = STRBUF_INIT; size_t i; - for (i = 0; i < info->trailer_nr; i++) - free(info->trailers[i]); - free(info->trailers); + + trailer_block = trailer_block_get(str, opts); + + for (i = 0; i < trailer_block->trailer_nr; i++) { + int separator_pos; + char *line = trailer_block->trailers[i]; + if (line[0] == comment_line_char) + continue; + separator_pos = find_separator(line, separators); + if (separator_pos >= 1) { + parse_trailer(line, separator_pos, &tok, &val, NULL); + if (opts->unfold) + unfold_value(&val); + add_trailer_item(head, + strbuf_detach(&tok, NULL), + strbuf_detach(&val, NULL)); + } else if (!opts->only_trailers) { + strbuf_addstr(&val, line); + strbuf_strip_suffix(&val, "\n"); + add_trailer_item(head, + NULL, + strbuf_detach(&val, NULL)); + } + } + + return trailer_block; } -static void format_trailer_info(struct strbuf *out, - const struct trailer_info *info, - const char *msg, - const struct process_trailer_options *opts) +void free_trailers(struct list_head *head) { - size_t origlen = out->len; - size_t i; - - /* If we want the whole block untouched, we can take the fast path. */ - if (!opts->only_trailers && !opts->unfold && !opts->filter && - !opts->separator && !opts->key_only && !opts->value_only && - !opts->key_value_separator) { - strbuf_add(out, msg + info->trailer_block_start, - info->trailer_block_end - info->trailer_block_start); - return; + struct list_head *pos, *p; + list_for_each_safe(pos, p, head) { + list_del(pos); + free_trailer_item(list_entry(pos, struct trailer_item, list)); } +} - for (i = 0; i < info->trailer_nr; i++) { - char *trailer = info->trailers[i]; - ssize_t separator_pos = find_separator(trailer, separators); +void new_trailers_clear(struct list_head *trailers) +{ + struct list_head *pos, *p; - if (separator_pos >= 1) { - struct strbuf tok = STRBUF_INIT; - struct strbuf val = STRBUF_INIT; + list_for_each_safe(pos, p, trailers) { + list_del(pos); + free_arg_item(list_entry(pos, struct arg_item, list)); + } +} - parse_trailer(&tok, &val, NULL, trailer, separator_pos); - if (!opts->filter || opts->filter(&tok, opts->filter_data)) { - if (opts->unfold) - unfold_value(&val); +size_t trailer_block_start(struct trailer_block *trailer_block) +{ + return trailer_block->start; +} - if (opts->separator && out->len != origlen) - strbuf_addbuf(out, opts->separator); - if (!opts->value_only) - strbuf_addbuf(out, &tok); - if (!opts->key_only && !opts->value_only) { - if (opts->key_value_separator) - strbuf_addbuf(out, opts->key_value_separator); - else - strbuf_addstr(out, ": "); - } - if (!opts->key_only) - strbuf_addbuf(out, &val); - if (!opts->separator) - strbuf_addch(out, '\n'); - } - strbuf_release(&tok); - strbuf_release(&val); +size_t trailer_block_end(struct trailer_block *trailer_block) +{ + return trailer_block->end; +} - } else if (!opts->only_trailers) { - if (opts->separator && out->len != origlen) { - strbuf_addbuf(out, opts->separator); - } - strbuf_addstr(out, trailer); - if (opts->separator) { - strbuf_rtrim(out); - } - } - } +int blank_line_before_trailer_block(struct trailer_block *trailer_block) +{ + return trailer_block->blank_line_before_trailer; +} +void trailer_block_release(struct trailer_block *trailer_block) +{ + size_t i; + for (i = 0; i < trailer_block->trailer_nr; i++) + free(trailer_block->trailers[i]); + free(trailer_block->trailers); + free(trailer_block); } -void format_trailers_from_commit(struct strbuf *out, const char *msg, - const struct process_trailer_options *opts) +void format_trailers_from_commit(const char *msg, + const struct process_trailer_options *opts, + struct strbuf *out) { - struct trailer_info info; + LIST_HEAD(head); + struct trailer_block *trailer_block = parse_trailers(msg, opts, &head); - trailer_info_get(&info, msg, opts); - format_trailer_info(out, &info, msg, opts); - trailer_info_release(&info); + /* If we want the whole block untouched, we can take the fast path. */ + if (!opts->only_trailers && !opts->unfold && !opts->filter && + !opts->separator && !opts->key_only && !opts->value_only && + !opts->key_value_separator) { + strbuf_add(out, msg + trailer_block->start, + trailer_block->end - trailer_block->start); + } else + format_trailers(&head, opts, out); + + free_trailers(&head); + trailer_block_release(trailer_block); } void trailer_iterator_init(struct trailer_iterator *iter, const char *msg) @@ -1253,24 +1187,26 @@ void trailer_iterator_init(struct trailer_iterator *iter, const char *msg) struct process_trailer_options opts = PROCESS_TRAILER_OPTIONS_INIT; strbuf_init(&iter->key, 0); strbuf_init(&iter->val, 0); + strbuf_init(&iter->raw, 0); opts.no_divider = 1; - trailer_info_get(&iter->internal.info, msg, &opts); + iter->internal.trailer_block = trailer_block_get(msg, &opts); iter->internal.cur = 0; } int trailer_iterator_advance(struct trailer_iterator *iter) { - while (iter->internal.cur < iter->internal.info.trailer_nr) { - char *trailer = iter->internal.info.trailers[iter->internal.cur++]; - int separator_pos = find_separator(trailer, separators); - - if (separator_pos < 1) - continue; /* not a real trailer */ - + char *line; + int separator_pos; + if (iter->internal.cur < iter->internal.trailer_block->trailer_nr) { + line = iter->internal.trailer_block->trailers[iter->internal.cur++]; + separator_pos = find_separator(line, separators); + iter->is_trailer = (separator_pos > 0); + + strbuf_reset(&iter->raw); + strbuf_addstr(&iter->raw, line); strbuf_reset(&iter->key); strbuf_reset(&iter->val); - parse_trailer(&iter->key, &iter->val, NULL, - trailer, separator_pos); + parse_trailer(line, separator_pos, &iter->key, &iter->val, NULL); unfold_value(&iter->val); return 1; } @@ -1279,7 +1215,8 @@ int trailer_iterator_advance(struct trailer_iterator *iter) void trailer_iterator_release(struct trailer_iterator *iter) { - trailer_info_release(&iter->internal.info); + trailer_block_release(iter->internal.trailer_block); strbuf_release(&iter->val); strbuf_release(&iter->key); + strbuf_release(&iter->raw); } @@ -4,6 +4,9 @@ #include "list.h" #include "strbuf.h" +struct trailer_block; +struct trailer_conf; + enum trailer_where { WHERE_DEFAULT, WHERE_END, @@ -29,40 +32,19 @@ int trailer_set_where(enum trailer_where *item, const char *value); int trailer_set_if_exists(enum trailer_if_exists *item, const char *value); int trailer_set_if_missing(enum trailer_if_missing *item, const char *value); -struct trailer_info { - /* - * True if there is a blank line before the location pointed to by - * trailer_block_start. - */ - int blank_line_before_trailer; +void trailer_conf_set(enum trailer_where where, + enum trailer_if_exists if_exists, + enum trailer_if_missing if_missing, + struct trailer_conf *conf); - /* - * Offsets to the trailer block start and end positions in the input - * string. If no trailer block is found, these are both set to the - * "true" end of the input (find_end_of_log_message()). - */ - size_t trailer_block_start, trailer_block_end; +struct trailer_conf *new_trailer_conf(void); +void duplicate_trailer_conf(struct trailer_conf *dst, + const struct trailer_conf *src); - /* - * Array of trailers found. - */ - char **trailers; - size_t trailer_nr; -}; - -/* - * A list that represents newly-added trailers, such as those provided - * with the --trailer command line option of git-interpret-trailers. - */ -struct new_trailer_item { - struct list_head list; - - const char *text; +const char *default_separators(void); - enum trailer_where where; - enum trailer_if_exists if_exists; - enum trailer_if_missing if_missing; -}; +void add_arg_item(char *tok, char *val, const struct trailer_conf *conf, + struct list_head *arg_head); struct process_trailer_options { int in_place; @@ -81,28 +63,41 @@ struct process_trailer_options { #define PROCESS_TRAILER_OPTIONS_INIT {0} -void process_trailers(const char *file, - const struct process_trailer_options *opts, - struct list_head *new_trailer_head); +void parse_trailers_from_config(struct list_head *config_head); -void trailer_info_get(struct trailer_info *info, const char *str, - const struct process_trailer_options *opts); +void process_trailers_lists(struct list_head *head, + struct list_head *arg_head); -void trailer_info_release(struct trailer_info *info); +ssize_t find_separator(const char *line, const char *separators); +void parse_trailer(const char *line, ssize_t separator_pos, + struct strbuf *tok, struct strbuf *val, + const struct trailer_conf **conf); + +struct trailer_block *parse_trailers(const char *str, + const struct process_trailer_options *opts, + struct list_head *head); + +size_t trailer_block_start(struct trailer_block *trailer_block); +size_t trailer_block_end(struct trailer_block *trailer_block); +int blank_line_before_trailer_block(struct trailer_block *trailer_block); + +void trailer_block_release(struct trailer_block *trailer_block); + +void trailer_config_init(void); +void free_trailers(struct list_head *trailers); +void new_trailers_clear(struct list_head *trailers); + +void format_trailers(struct list_head *head, + const struct process_trailer_options *opts, + struct strbuf *out); /* - * Format the trailers from the commit msg "msg" into the strbuf "out". - * Note two caveats about "opts": - * - * - this is primarily a helper for pretty.c, and not - * all of the flags are supported. - * - * - this differs from process_trailers slightly in that we always format - * only the trailer block itself, even if the "only_trailers" option is not - * set. + * Convenience function to format the trailers from the commit msg "msg" into + * the strbuf "out". Reuses format_trailers internally. */ -void format_trailers_from_commit(struct strbuf *out, const char *msg, - const struct process_trailer_options *opts); +void format_trailers_from_commit(const char *msg, + const struct process_trailer_options *opts, + struct strbuf *out); /* * An interface for iterating over the trailers found in a particular commit @@ -118,9 +113,22 @@ struct trailer_iterator { struct strbuf key; struct strbuf val; + /* + * Raw line (e.g., "foo: bar baz") before being parsed as a trailer + * key/val pair. This field can contain non-trailer lines because it's + * valid for a trailer block to contain such lines (i.e., we only + * require 25% of the lines in a trailer block to be trailer lines). + */ + struct strbuf raw; + + /* + * 1 if the raw line was parsed as a separate key/val pair. + */ + int is_trailer; + /* private */ struct { - struct trailer_info info; + struct trailer_block *trailer_block; size_t cur; } internal; }; diff --git a/transport-helper.c b/transport-helper.c index e34a8f47cf..6764463873 100644 --- a/transport-helper.c +++ b/transport-helper.c @@ -626,7 +626,8 @@ static int process_connect_service(struct transport *transport, ret = run_connect(transport, &cmdbuf); } else if (data->stateless_connect && (get_protocol_version_config() == protocol_v2) && - !strcmp("git-upload-pack", name)) { + (!strcmp("git-upload-pack", name) || + !strcmp("git-upload-archive", name))) { strbuf_addf(&cmdbuf, "stateless-connect %s\n", name); ret = run_connect(transport, &cmdbuf); if (ret) @@ -643,6 +644,7 @@ static int process_connect(struct transport *transport, struct helper_data *data = transport->data; const char *name; const char *exec; + int ret; name = for_push ? "git-receive-pack" : "git-upload-pack"; if (for_push) @@ -650,7 +652,10 @@ static int process_connect(struct transport *transport, else exec = data->transport_options.uploadpack; - return process_connect_service(transport, name, exec); + ret = process_connect_service(transport, name, exec); + if (ret) + do_take_over(transport); + return ret; } static int connect_helper(struct transport *transport, const char *name, @@ -660,14 +665,14 @@ static int connect_helper(struct transport *transport, const char *name, /* Get_helper so connect is inited. */ get_helper(transport); - if (!data->connect) - die(_("operation not supported by protocol")); if (!process_connect_service(transport, name, exec)) die(_("can't connect to subservice %s"), name); fd[0] = data->helper->out; fd[1] = data->helper->in; + + do_take_over(transport); return 0; } @@ -682,10 +687,8 @@ static int fetch_refs(struct transport *transport, get_helper(transport); - if (process_connect(transport, 0)) { - do_take_over(transport); + if (process_connect(transport, 0)) return transport->vtable->fetch_refs(transport, nr_heads, to_fetch); - } /* * If we reach here, then the server, the client, and/or the transport @@ -1142,10 +1145,8 @@ static int push_refs(struct transport *transport, { struct helper_data *data = transport->data; - if (process_connect(transport, 1)) { - do_take_over(transport); + if (process_connect(transport, 1)) return transport->vtable->push_refs(transport, remote_refs, flags); - } if (!remote_refs) { fprintf(stderr, @@ -1186,11 +1187,9 @@ static struct ref *get_refs_list(struct transport *transport, int for_push, { get_helper(transport); - if (process_connect(transport, for_push)) { - do_take_over(transport); + if (process_connect(transport, for_push)) return transport->vtable->get_refs_list(transport, for_push, transport_options); - } return get_refs_list_using_list(transport, for_push); } @@ -1274,10 +1273,8 @@ static int get_bundle_uri(struct transport *transport) { get_helper(transport); - if (process_connect(transport, 0)) { - do_take_over(transport); + if (process_connect(transport, 0)) return transport->vtable->get_bundle_uri(transport); - } return -1; } diff --git a/transport.c b/transport.c index bd7899e9bf..df518ead70 100644 --- a/transport.c +++ b/transport.c @@ -1467,6 +1467,7 @@ int transport_push(struct repository *r, if (porcelain && !push_ret) puts("Done"); else if (!quiet && !ret && !transport_refs_pushed(remote_refs)) + /* stable plumbing output; do not modify or localize */ fprintf(stderr, "Everything up-to-date\n"); done: diff --git a/tree-walk.c b/tree-walk.c index b517792ba2..dc1870b6ad 100644 --- a/tree-walk.c +++ b/tree-walk.c @@ -11,35 +11,19 @@ #include "json-writer.h" #include "environment.h" -static const char *get_mode(const char *str, unsigned int *modep) -{ - unsigned char c; - unsigned int mode = 0; - - if (*str == ' ') - return NULL; - - while ((c = *str++) != ' ') { - if (c < '0' || c > '7') - return NULL; - mode = (mode << 3) + (c - '0'); - } - *modep = mode; - return str; -} - static int decode_tree_entry(struct tree_desc *desc, const char *buf, unsigned long size, struct strbuf *err) { const char *path; - unsigned int mode, len; - const unsigned hashsz = the_hash_algo->rawsz; + unsigned int len; + uint16_t mode; + const unsigned hashsz = desc->algo->rawsz; if (size < hashsz + 3 || buf[size - (hashsz + 1)]) { strbuf_addstr(err, _("too-short tree object")); return -1; } - path = get_mode(buf, &mode); + path = parse_mode(buf, &mode); if (!path) { strbuf_addstr(err, _("malformed mode in tree entry")); return -1; @@ -54,15 +38,19 @@ static int decode_tree_entry(struct tree_desc *desc, const char *buf, unsigned l desc->entry.path = path; desc->entry.mode = (desc->flags & TREE_DESC_RAW_MODES) ? mode : canon_mode(mode); desc->entry.pathlen = len - 1; - oidread(&desc->entry.oid, (const unsigned char *)path + len); + oidread_algop(&desc->entry.oid, (const unsigned char *)path + len, + desc->algo); return 0; } -static int init_tree_desc_internal(struct tree_desc *desc, const void *buffer, - unsigned long size, struct strbuf *err, +static int init_tree_desc_internal(struct tree_desc *desc, + const struct object_id *oid, + const void *buffer, unsigned long size, + struct strbuf *err, enum tree_desc_flags flags) { + desc->algo = (oid && oid->algo) ? &hash_algos[oid->algo] : the_hash_algo; desc->buffer = buffer; desc->size = size; desc->flags = flags; @@ -71,19 +59,21 @@ static int init_tree_desc_internal(struct tree_desc *desc, const void *buffer, return 0; } -void init_tree_desc(struct tree_desc *desc, const void *buffer, unsigned long size) +void init_tree_desc(struct tree_desc *desc, const struct object_id *tree_oid, + const void *buffer, unsigned long size) { struct strbuf err = STRBUF_INIT; - if (init_tree_desc_internal(desc, buffer, size, &err, 0)) + if (init_tree_desc_internal(desc, tree_oid, buffer, size, &err, 0)) die("%s", err.buf); strbuf_release(&err); } -int init_tree_desc_gently(struct tree_desc *desc, const void *buffer, unsigned long size, +int init_tree_desc_gently(struct tree_desc *desc, const struct object_id *oid, + const void *buffer, unsigned long size, enum tree_desc_flags flags) { struct strbuf err = STRBUF_INIT; - int result = init_tree_desc_internal(desc, buffer, size, &err, flags); + int result = init_tree_desc_internal(desc, oid, buffer, size, &err, flags); if (result) error("%s", err.buf); strbuf_release(&err); @@ -102,7 +92,7 @@ void *fill_tree_descriptor(struct repository *r, if (!buf) die("unable to read tree %s", oid_to_hex(oid)); } - init_tree_desc(desc, buf, size); + init_tree_desc(desc, oid, buf, size); return buf; } @@ -119,7 +109,7 @@ static void entry_extract(struct tree_desc *t, struct name_entry *a) static int update_tree_entry_internal(struct tree_desc *desc, struct strbuf *err) { const void *buf = desc->buffer; - const unsigned char *end = (const unsigned char *)desc->entry.path + desc->entry.pathlen + 1 + the_hash_algo->rawsz; + const unsigned char *end = (const unsigned char *)desc->entry.path + desc->entry.pathlen + 1 + desc->algo->rawsz; unsigned long size = desc->size; unsigned long len = end - (const unsigned char *)buf; @@ -633,7 +623,7 @@ int get_tree_entry(struct repository *r, retval = -1; } else { struct tree_desc t; - init_tree_desc(&t, tree, size); + init_tree_desc(&t, tree_oid, tree, size); retval = find_tree_entry(r, &t, name, oid, mode); } free(tree); @@ -676,7 +666,7 @@ enum get_oid_result get_tree_entry_follow_symlinks(struct repository *r, struct tree_desc t; int follows_remaining = GET_TREE_ENTRY_FOLLOW_SYMLINKS_MAX_LINKS; - init_tree_desc(&t, NULL, 0UL); + init_tree_desc(&t, NULL, NULL, 0UL); strbuf_addstr(&namebuf, name); oidcpy(¤t_tree_oid, tree_oid); @@ -712,7 +702,7 @@ enum get_oid_result get_tree_entry_follow_symlinks(struct repository *r, goto done; /* descend */ - init_tree_desc(&t, tree, size); + init_tree_desc(&t, ¤t_tree_oid, tree, size); } /* Handle symlinks to e.g. a//b by removing leading slashes */ @@ -746,7 +736,7 @@ enum get_oid_result get_tree_entry_follow_symlinks(struct repository *r, free(parent->tree); parents_nr--; parent = &parents[parents_nr - 1]; - init_tree_desc(&t, parent->tree, parent->size); + init_tree_desc(&t, &parent->oid, parent->tree, parent->size); strbuf_remove(&namebuf, 0, remainder ? 3 : 2); continue; } @@ -826,7 +816,7 @@ enum get_oid_result get_tree_entry_follow_symlinks(struct repository *r, contents_start = contents; parent = &parents[parents_nr - 1]; - init_tree_desc(&t, parent->tree, parent->size); + init_tree_desc(&t, &parent->oid, parent->tree, parent->size); strbuf_splice(&namebuf, 0, len, contents_start, link_len); if (remainder) diff --git a/tree-walk.h b/tree-walk.h index a6bfa3da3a..0b1067fbc5 100644 --- a/tree-walk.h +++ b/tree-walk.h @@ -24,6 +24,7 @@ struct name_entry { * A semi-opaque data structure used to maintain the current state of the walk. */ struct tree_desc { + const struct git_hash_algo *algo; /* * pointer into the memory representation of the tree. It always * points at the current entry being visited. @@ -83,9 +84,11 @@ int update_tree_entry_gently(struct tree_desc *); * size parameters are assumed to be the same as the buffer and size * members of `struct tree`. */ -void init_tree_desc(struct tree_desc *desc, const void *buf, unsigned long size); +void init_tree_desc(struct tree_desc *desc, const struct object_id *tree_oid, + const void *buf, unsigned long size); -int init_tree_desc_gently(struct tree_desc *desc, const void *buf, unsigned long size, +int init_tree_desc_gently(struct tree_desc *desc, const struct object_id *oid, + const void *buf, unsigned long size, enum tree_desc_flags flags); /* @@ -29,7 +29,7 @@ int read_tree_at(struct repository *r, if (parse_tree(tree)) return -1; - init_tree_desc(&desc, tree->buffer, tree->size); + init_tree_desc(&desc, &tree->object.oid, tree->buffer, tree->size); while (tree_entry(&desc, &entry)) { if (retval != all_entries_interesting) { @@ -45,7 +45,7 @@ static int process_tree(struct walker *walker, struct tree *tree) if (parse_tree(tree)) return -1; - init_tree_desc(&desc, tree->buffer, tree->size); + init_tree_desc(&desc, &tree->object.oid, tree->buffer, tree->size); while (tree_entry(&desc, &entry)) { struct object *obj = NULL; diff --git a/worktree.c b/worktree.c index 1399d452ac..b02a05a74a 100644 --- a/worktree.c +++ b/worktree.c @@ -12,18 +12,23 @@ #include "wt-status.h" #include "config.h" +void free_worktree(struct worktree *worktree) +{ + if (!worktree) + return; + free(worktree->path); + free(worktree->id); + free(worktree->head_ref); + free(worktree->lock_reason); + free(worktree->prune_reason); + free(worktree); +} + void free_worktrees(struct worktree **worktrees) { int i = 0; - - for (i = 0; worktrees[i]; i++) { - free(worktrees[i]->path); - free(worktrees[i]->id); - free(worktrees[i]->head_ref); - free(worktrees[i]->lock_reason); - free(worktrees[i]->prune_reason); - free(worktrees[i]); - } + for (i = 0; worktrees[i]; i++) + free_worktree(worktrees[i]); free (worktrees); } @@ -51,7 +56,7 @@ static void add_head_info(struct worktree *wt) /** * get the main worktree */ -static struct worktree *get_main_worktree(void) +static struct worktree *get_main_worktree(int skip_reading_head) { struct worktree *worktree = NULL; struct strbuf worktree_path = STRBUF_INIT; @@ -70,11 +75,13 @@ static struct worktree *get_main_worktree(void) */ worktree->is_bare = (is_bare_repository_cfg == 1) || is_bare_repository(); - add_head_info(worktree); + if (!skip_reading_head) + add_head_info(worktree); return worktree; } -static struct worktree *get_linked_worktree(const char *id) +struct worktree *get_linked_worktree(const char *id, + int skip_reading_head) { struct worktree *worktree = NULL; struct strbuf path = STRBUF_INIT; @@ -93,7 +100,8 @@ static struct worktree *get_linked_worktree(const char *id) CALLOC_ARRAY(worktree, 1); worktree->path = strbuf_detach(&worktree_path, NULL); worktree->id = xstrdup(id); - add_head_info(worktree); + if (!skip_reading_head) + add_head_info(worktree); done: strbuf_release(&path); @@ -118,7 +126,14 @@ static void mark_current_worktree(struct worktree **worktrees) free(git_dir); } -struct worktree **get_worktrees(void) +/* + * NEEDSWORK: This function exists so that we can look up metadata of a + * worktree without trying to access any of its internals like the refdb. It + * would be preferable to instead have a corruption-tolerant function for + * retrieving worktree metadata that could be used when the worktree is known + * to not be in a healthy state, e.g. when creating or repairing it. + */ +static struct worktree **get_worktrees_internal(int skip_reading_head) { struct worktree **list = NULL; struct strbuf path = STRBUF_INIT; @@ -128,7 +143,7 @@ struct worktree **get_worktrees(void) ALLOC_ARRAY(list, alloc); - list[counter++] = get_main_worktree(); + list[counter++] = get_main_worktree(skip_reading_head); strbuf_addf(&path, "%s/worktrees", get_git_common_dir()); dir = opendir(path.buf); @@ -137,7 +152,7 @@ struct worktree **get_worktrees(void) while ((d = readdir_skip_dot_and_dotdot(dir)) != NULL) { struct worktree *linked = NULL; - if ((linked = get_linked_worktree(d->d_name))) { + if ((linked = get_linked_worktree(d->d_name, skip_reading_head))) { ALLOC_GROW(list, counter + 1, alloc); list[counter++] = linked; } @@ -151,6 +166,11 @@ struct worktree **get_worktrees(void) return list; } +struct worktree **get_worktrees(void) +{ + return get_worktrees_internal(0); +} + const char *get_worktree_git_dir(const struct worktree *wt) { if (!wt) @@ -591,7 +611,7 @@ static void repair_noop(int iserr UNUSED, void repair_worktrees(worktree_repair_fn fn, void *cb_data) { - struct worktree **worktrees = get_worktrees(); + struct worktree **worktrees = get_worktrees_internal(1); struct worktree **wt = worktrees + 1; /* +1 skips main worktree */ if (!fn) diff --git a/worktree.h b/worktree.h index ce45b66de9..f14784a2ff 100644 --- a/worktree.h +++ b/worktree.h @@ -58,6 +58,13 @@ struct worktree *find_worktree(struct worktree **list, const char *arg); /* + * Look up the worktree corresponding to `id`, or NULL of no such worktree + * exists. + */ +struct worktree *get_linked_worktree(const char *id, + int skip_reading_head); + +/* * Return the worktree corresponding to `path`, or NULL if no such worktree * exists. */ @@ -135,6 +142,11 @@ void repair_worktrees(worktree_repair_fn, void *cb_data); void repair_worktree_at_path(const char *, worktree_repair_fn, void *cb_data); /* + * Free up the memory for a worktree. + */ +void free_worktree(struct worktree *); + +/* * Free up the memory for worktree(s) */ void free_worktrees(struct worktree **); diff --git a/write-or-die.c b/write-or-die.c index 42a2dc73cd..3942152865 100644 --- a/write-or-die.c +++ b/write-or-die.c @@ -19,20 +19,17 @@ void maybe_flush_or_die(FILE *f, const char *desc) { static int skip_stdout_flush = -1; - struct stat st; - char *cp; if (f == stdout) { if (skip_stdout_flush < 0) { - /* NEEDSWORK: make this a normal Boolean */ - cp = getenv("GIT_FLUSH"); - if (cp) - skip_stdout_flush = (atoi(cp) == 0); - else if ((fstat(fileno(stdout), &st) == 0) && - S_ISREG(st.st_mode)) - skip_stdout_flush = 1; - else - skip_stdout_flush = 0; + skip_stdout_flush = git_env_bool("GIT_FLUSH", -1); + if (skip_stdout_flush < 0) { + struct stat st; + if (fstat(fileno(stdout), &st)) + skip_stdout_flush = 0; + else + skip_stdout_flush = S_ISREG(st.st_mode); + } } if (skip_stdout_flush && !ferror(f)) return; |