diff options
| author | Junio C Hamano <gitster@pobox.com> | 2026-01-13 06:13:12 -0800 |
|---|---|---|
| committer | Junio C Hamano <gitster@pobox.com> | 2026-01-13 06:13:12 -0800 |
| commit | 9e4a786c3db10205fbc8c6a0aa1f14c4ca325760 (patch) | |
| tree | 8977722181bf9b8eb485d94ff7257a8e03f866fa | |
| parent | f0ef5b6d9bcc258e4cbef93839d1b7465d5212b9 (diff) | |
| parent | d205234cb05a5e330c0f7f5b3ea764533a74d69e (diff) | |
| download | git-9e4a786c3db10205fbc8c6a0aa1f14c4ca325760.tar.xz | |
Merge branch 'ps/history' into pw/replay-drop-empty
* ps/history: (186 commits)
builtin/history: implement "reword" subcommand
builtin: add new "history" command
wt-status: provide function to expose status for trees
replay: support updating detached HEAD
replay: support empty commit ranges
replay: small set of cleanups
builtin/replay: move core logic into "libgit.a"
builtin/replay: extract core logic to replay revisions
The 15th batch
t3650: add more regression tests for failure conditions
replay: die if we cannot parse object
replay: improve code comment and die message
replay: die descriptively when invalid commit-ish is given
replay: find *onto only after testing for ref name
replay: remove dead code and rearrange
The 14th batch
The 13th batch
config: use git_parse_int() in git_config_get_expiry_in_days()
receive-pack: convert receive hooks to hook API
receive-pack: convert update hooks to new API
...
187 files changed, 5794 insertions, 2586 deletions
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 27ebf2c8cc..f2e93f5461 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -443,7 +443,7 @@ jobs: - run: ci/install-dependencies.sh - run: useradd builder --create-home - run: chown -R builder . - - run: sudo --preserve-env --set-home --user=builder ci/run-build-and-tests.sh + - run: chmod a+w $GITHUB_ENV && sudo --preserve-env --set-home --user=builder ci/run-build-and-tests.sh - name: print test failures if: failure() && env.FAILED_TEST_ARTIFACTS != '' run: sudo --preserve-env --set-home --user=builder ci/print-test-failures.sh diff --git a/.gitignore b/.gitignore index 78a45cb5be..24635cf2d6 100644 --- a/.gitignore +++ b/.gitignore @@ -79,6 +79,7 @@ /git-grep /git-hash-object /git-help +/git-history /git-hook /git-http-backend /git-http-fetch diff --git a/Documentation/Makefile b/Documentation/Makefile index 47208269a2..2699f0b24a 100644 --- a/Documentation/Makefile +++ b/Documentation/Makefile @@ -143,6 +143,7 @@ TECH_DOCS += technical/shallow TECH_DOCS += technical/sparse-checkout TECH_DOCS += technical/sparse-index TECH_DOCS += technical/trivial-merge +TECH_DOCS += technical/unambiguous-types TECH_DOCS += technical/unit-tests SP_ARTICLES += $(TECH_DOCS) SP_ARTICLES += technical/api-index diff --git a/Documentation/RelNotes/2.53.0.adoc b/Documentation/RelNotes/2.53.0.adoc index c4dfeb1c23..9e8384a4c1 100644 --- a/Documentation/RelNotes/2.53.0.adoc +++ b/Documentation/RelNotes/2.53.0.adoc @@ -20,6 +20,20 @@ UI, Workflows & Features * Add a new manual that describes the data model. + * "git fast-import" learns "--strip-if-invalid" option to drop + invalid cryptographic signature from objects. + + * The use of "revision" (a connected set of commits) has been + clarified in the "git replay" documentation. + + * A help message from "git branch" now mentions "git help" instead of + "man" when suggesting to read some documentation. + + * "git repo struct" learned to take "-z" as a synonym to "--format=nul". + + * More object database related information are shown in "git repo + structure" output. + Performance, Internal Implementation, Development Support etc. -------------------------------------------------------------- @@ -38,6 +52,46 @@ Performance, Internal Implementation, Development Support etc. * A part of code paths that deals with loose objects has been cleaned up. + * "make strip" has been taught to strip "scalar" as well as "git". + + * Dockerised jobs at the GitHub Actions CI have been taught to show + more details of failed tests. + + * Code refactoring around object database sources. + + * Halve the memory consumed by artificial filepairs created during + "git diff --find-copioes-harder", also making the operation run + faster. + + * The "git_istream" abstraction has been revamped to make it easier + to interface with pluggable object database design. + + * Rewrite the only use of "mktemp()" that is subject to TOCTOU race + and Stop using the insecure "mktemp()" function. + (merge 10bba537c4 rs/ban-mktemp later to maint). + + * In-code comment update to clarify that single-letter options are + outside of the scope of command line completion script. + (merge dc8a00fafe jc/completion-no-single-letter-options later to maint). + + * MEMZERO_ARRAY() helper is introduced to avoid clearing only the + first N bytes of an N-element array whose elements are larger than + a byte. + + * "git diff-files -R --find-copies-harder" has been taught to use + the potential copy sources from the index correctly. + + * Require C99 style flexible array member support from all platforms. + + * The code path that enumerates promisor objects have been optimized + to skip pointlessly parsing blob objects. + + * Prepare test suite for Git for Windows that supports symbolic + links. + + * Use hook API to replace ad-hoc invocation of hook scripts with the + run_command() API. + Fixes since v2.52 ----------------- @@ -105,8 +159,96 @@ Fixes since v2.52 * Various issues detected by Asan have been corrected. (merge a031b6181a jk/asan-bonanza later to maint). + * "git config get --path" segfaulted on an ":(optional)path" that + does not exist, which has been corrected. + (merge 0bd16856ff jc/optional-path later to maint). + + * The "--committer-date-is-author-date" option of "git am/rebase" is + a misguided one. The documentation is updated to discourage its + use. + (merge fbf3d0669f kh/doc-committer-date-is-author-date later to maint). + + * The option help text given by "git config unset -h" described + the "--all" option to "replace", not "unset", multiple variables, + which has been corrected. + (merge 18bf67b753 rs/config-unset-opthelp-fix later to maint). + + * The error message given by "git config set", when the variable + being updated has more than one values defined, used old style "git + config" syntax with an incorrect option in its hint, both of which + have been corrected. + (merge df963f0df4 rs/config-set-multi-error-message-fix later to maint). + + * "git replay" forgot to omit the "gpgsig-sha256" extended header + from the resulting commit the same way it omits "gpgsig", which has + been corrected. + (merge 9f3a115087 pw/replay-exclude-gpgsig-fix later to maint). + + * A few tests have been updated to work under the shell compatible + mode of zsh. + (merge a92f243a94 bc/zsh-testsuite later to maint). + + * The way patience diff finds LCS has been optimized. + (merge c7e3b8085b yc/xdiff-patience-optim later to maint). + + * Recent optimization to "last-modified" command introduced use of + uninitialized block of memory, which has been corrected. + (merge fe4e60759b tc/last-modified-active-paths-optimization later to maint). + + * "git last-modified" used to mishandle "--" to mark the beginning of + pathspec, which has been corrected. + (merge 05491b90ce js/last-modified-with-sparse-checkouts later to maint). + + * Emulation code clean-up. + (merge 42aa7603aa gf/win32-pthread-cond-init later to maint). + + * "git submodule add" to add a submodule under <name> segfaulted, + when a submodule.<name>.something is already in .gitmodules file + without defining where its submodule.<name>.path is, which has been + corrected. + (merge dd8e8c786e jc/submodule-add later to maint). + + * "git fetch" that involves fetching tags, when a tag being fetched + needs to overwrite existing one, failed to fetch other tags, which + has been corrected. + (merge b7b17ec8a6 kn/fix-fetch-backfill-tag-with-batched-ref-updates later to maint). + + * Document "rev-list --filter-provided-objects" better. + (merge 6d8dc99478 jt/doc-rev-list-filter-provided-objects later to maint). + + * Even when there is no changes in the packfile and no need to + recompute bitmaps, "git repack" recomputed and updated the MIDX + file, which has been corrected. + (merge 6ce9d558ce ps/repack-avoid-noop-midx-rewrite later to maint). + + * Update HTTP tests to adjust for changes in curl 8.18.0 + (merge 17f4b01da7 jk/test-curl-updates later to maint). + + * Workaround the "iconv" shipped as part of macOS, which is broken + handling stateful ISO/IEC 2022 encoded strings. + (merge cee341e9dd rs/macos-iconv-workaround later to maint). + * Other code cleanup, docfix, build fix, etc. (merge 46207a54cc qj/doc-http-bad-want-response later to maint). (merge df90eccd93 kh/doc-commit-extra-references later to maint). (merge f18aa68861 rs/xmkstemp-simplify later to maint). (merge fddba8f737 ja/doc-synopsis-style later to maint). + (merge 22ce0cb639 en/xdiff-cleanup-2 later to maint). + (merge 8ef7355a8f je/doc-pull later to maint). + (merge 48176f953f jc/capability-leak later to maint). + (merge 8cbbdc92f7 kh/doc-pre-commit-fix later to maint). + (merge d4bc39a4d9 mh/doc-config-gui-gcwarning later to maint). + (merge 41d425008a kh/doc-send-email-paragraph-fix later to maint). + (merge d4b732899e jc/macports-darwinports later to maint). + (merge bab391761d kj/pull-options-decl-cleanup later to maint). + (merge 007b8994d4 rs/t4014-git-version-string-fix later to maint). + (merge 4ce170c522 ds/doc-scalar-config later to maint). + (merge a0c813951a jc/doc-commit-signoff-config later to maint). + (merge 8ee262985a ja/doc-misc-fixes later to maint). + (merge 1722c2244b mh/doc-core-attributesfile later to maint). + (merge c469ca26c5 dk/ci-rust-fix later to maint). + (merge 12f0be0857 gf/clear-path-cache-cleanup later to maint). + (merge 949df6ed6b js/test-func-comment-fix later to maint). + (merge 93f894c001 bc/checkout-error-message-fix later to maint). + (merge abf05d856f rs/show-branch-prio-queue later to maint). + (merge 06188ea5f3 rs/parse-config-expiry-simplify later to maint). diff --git a/Documentation/config/core.adoc b/Documentation/config/core.adoc index 01202da7cd..9bc9de29d9 100644 --- a/Documentation/config/core.adoc +++ b/Documentation/config/core.adoc @@ -492,10 +492,9 @@ core.askPass:: command-line argument and write the password on its STDOUT. core.attributesFile:: - In addition to `.gitattributes` (per-directory) and - `.git/info/attributes`, Git looks into this file for attributes - (see linkgit:gitattributes[5]). Path expansions are made the same - way as for `core.excludesFile`. Its default value is + Specifies the pathname to the file that contains attributes (see + linkgit:gitattributes[5]), in addition to `.gitattributes` (per-directory) + and `.git/info/attributes`. Its default value is `$XDG_CONFIG_HOME/git/attributes`. If `$XDG_CONFIG_HOME` is either not set or empty, `$HOME/.config/git/attributes` is used instead. diff --git a/Documentation/config/gui.adoc b/Documentation/config/gui.adoc index 171be774d2..1565c0af19 100644 --- a/Documentation/config/gui.adoc +++ b/Documentation/config/gui.adoc @@ -55,3 +55,8 @@ gui.blamehistoryctx:: linkgit:gitk[1] for the selected commit, when the `Show History Context` menu item is invoked from 'git gui blame'. If this variable is set to zero, the whole history is shown. + +gui.GCWarning:: + Determines whether linkgit:git-gui[1] should prompt for garbage + collection when git detects a large number of loose objects in + the repository. The default value is "true". diff --git a/Documentation/git-am.adoc b/Documentation/git-am.adoc index b23b4fba20..0c94776e29 100644 --- a/Documentation/git-am.adoc +++ b/Documentation/git-am.adoc @@ -162,6 +162,13 @@ Valid <action> for the `--whitespace` option are: commit creation as the committer date. This allows the user to lie about the committer date by using the same value as the author date. ++ +WARNING: The history walking machinery assumes that commits have +non-decreasing commit timestamps. You should consider if you really need +to use this option. Then you should only use this option to override the +committer date when applying commits on top of a base which commit is +older (in terms of the commit date) than the oldest patch you are +applying. --ignore-date:: By default the command records the date from the e-mail diff --git a/Documentation/git-checkout.adoc b/Documentation/git-checkout.adoc index 6f281b298e..43ccf47cf6 100644 --- a/Documentation/git-checkout.adoc +++ b/Documentation/git-checkout.adoc @@ -509,7 +509,7 @@ ARGUMENT DISAMBIGUATION ----------------------- When you run `git checkout <something>`, Git tries to guess whether -`<something>` is intended to be a branch, a commit, or a set of file(s), +_<something>_ is intended to be a branch, a commit, or a set of file(s), and then either switches to that branch or commit, or restores the specified files. diff --git a/Documentation/git-fast-import.adoc b/Documentation/git-fast-import.adoc index b74179a6c8..479c4081da 100644 --- a/Documentation/git-fast-import.adoc +++ b/Documentation/git-fast-import.adoc @@ -66,15 +66,26 @@ fast-import stream! This option is enabled automatically for remote-helpers that use the `import` capability, as they are already trusted to run their own code. ---signed-tags=(verbatim|warn-verbatim|warn-strip|strip|abort):: - Specify how to handle signed tags. Behaves in the same way - as the same option in linkgit:git-fast-export[1], except that - default is 'verbatim' (instead of 'abort'). +`--signed-tags=(verbatim|warn-verbatim|warn-strip|strip|abort)`:: + Specify how to handle signed tags. Behaves in the same way as + the `--signed-commits=<mode>` below, except that the + `strip-if-invalid` mode is not yet supported. Like for signed + commits, the default mode is `verbatim`. ---signed-commits=(verbatim|warn-verbatim|warn-strip|strip|abort):: - Specify how to handle signed commits. Behaves in the same way - as the same option in linkgit:git-fast-export[1], except that - default is 'verbatim' (instead of 'abort'). +`--signed-commits=<mode>`:: + Specify how to handle signed commits. The following <mode>s + are supported: ++ +* `verbatim`, which is the default, will silently import commit + signatures. +* `warn-verbatim` will import them, but will display a warning. +* `abort` will make this program die when encountering a signed + commit. +* `strip` will silently make the commits unsigned. +* `warn-strip` will make them unsigned, but will display a warning. +* `strip-if-invalid` will check signatures and, if they are invalid, + will strip them and display a warning. The validation is performed + in the same way as linkgit:git-verify-commit[1] does it. Options for Frontends ~~~~~~~~~~~~~~~~~~~~~ diff --git a/Documentation/git-history.adoc b/Documentation/git-history.adoc new file mode 100644 index 0000000000..154e262b76 --- /dev/null +++ b/Documentation/git-history.adoc @@ -0,0 +1,73 @@ +git-history(1) +============== + +NAME +---- +git-history - EXPERIMENTAL: Rewrite history + +SYNOPSIS +-------- +[synopsis] +git history reword <commit> [--ref-action=(branches|head|print)] + +DESCRIPTION +----------- + +Rewrite history by rearranging or modifying specific commits in the +history. + +THIS COMMAND IS EXPERIMENTAL. THE BEHAVIOR MAY CHANGE. + +This command is related to linkgit:git-rebase[1] in that both commands can be +used to rewrite history. There are a couple of major differences though: + +* linkgit:git-history[1] can work in a bare repository as it does not need to + touch either the index or the worktree. +* linkgit:git-history[1] does not execute any linkgit:githooks[5] at the + current point in time. This may change in the future. +* linkgit:git-history[1] by default updates all branches that are descendants + of the original commit to point to the rewritten commit. + +Overall, linkgit:git-history[1] aims to provide a more opinionated way to modify +your commit history that is simpler to use compared to linkgit:git-rebase[1] in +general. + +Use linkgit:git-rebase[1] if you want to reapply a range of commits onto a +different base, or interactive rebases if you want to edit a range of commits +at once. + +LIMITATIONS +----------- + +This command does not (yet) work with histories that contain merges. You +should use linkgit:git-rebase[1] with the `--rebase-merges` flag instead. + +Furthermore, the command does not support operations that can result in merge +conflicts. This limitation is by design as history rewrites are not intended to +be stateful operations. The limitation can be lifted once (if) Git learns about +first-class conflicts. + +COMMANDS +-------- + +The following commands are available to rewrite history in different ways: + +`reword <commit>`:: + Rewrite the commit message of the specified commit. All the other + details of this commit remain unchanged. This command will spawn an + editor with the current message of that commit. + +OPTIONS +------- + +`--ref-action=(branches|head|print)`:: + Control which references will be updated by the command, if any. With + `branches`, all local branches that point to commits which are + descendants of the original commit will be rewritten. With `head`, only + the current `HEAD` reference will be rewritten. With `print`, all + updates as they would be performed with `branches` are printed in a + format that can be consumed by linkgit:git-update-ref[1]. + +GIT +--- +Part of the linkgit:git[1] suite diff --git a/Documentation/git-pull.adoc b/Documentation/git-pull.adoc index 248f6c3f39..88f4fd3926 100644 --- a/Documentation/git-pull.adoc +++ b/Documentation/git-pull.adoc @@ -37,8 +37,8 @@ You can also set the configuration options `pull.rebase`, `pull.squash`, or `pull.ff` with your preferred behaviour. If there's a merge conflict during the merge or rebase that you don't -want to handle, you can safely abort it with `git merge --abort` or `git ---rebase abort`. +want to handle, you can safely abort it with `git merge --abort` or +`git rebase --abort`. OPTIONS ------- diff --git a/Documentation/git-rebase.adoc b/Documentation/git-rebase.adoc index 005caf6164..e177808004 100644 --- a/Documentation/git-rebase.adoc +++ b/Documentation/git-rebase.adoc @@ -87,7 +87,7 @@ of the to-be-rebased branch. However, `ORIG_HEAD` is not guaranteed to still point to that commit at the end of the rebase if other commands that change `ORIG_HEAD` (like `git reset`) are used during the rebase. The previous branch tip, however, is accessible using the reflog of the current branch (i.e. `@{1}`, -see linkgit:gitrevisions[7]. +see linkgit:gitrevisions[7]). TRANSPLANTING A TOPIC BRANCH WITH --ONTO ---------------------------------------- @@ -474,6 +474,13 @@ See also INCOMPATIBLE OPTIONS below. Instead of using the current time as the committer date, use the author date of the commit being rebased as the committer date. This option implies `--force-rebase`. ++ +WARNING: The history walking machinery assumes that commits have +non-decreasing commit timestamps. You should consider if you really need +to use this option. Then you should only use this option to override the +committer date when rebasing commits on top of a base which commit is +older (in terms of the commit date) than the oldest commit you are +applying (in terms of the author date). --ignore-date:: --reset-author-date:: diff --git a/Documentation/git-repack.adoc b/Documentation/git-repack.adoc index d12c4985f6..673ce91083 100644 --- a/Documentation/git-repack.adoc +++ b/Documentation/git-repack.adoc @@ -77,14 +77,14 @@ to the new separate pack will be written. Only useful with `--cruft -d`. --max-cruft-size=<n>:: - Overrides `--max-pack-size` for cruft packs. Inherits the value of + Override `--max-pack-size` for cruft packs. Inherits the value of `--max-pack-size` (if any) by default. See the documentation for `--max-pack-size` for more details. --combine-cruft-below-size=<n>:: When generating cruft packs without pruning, only repack - existing cruft packs whose size is strictly less than `<n>`, - where `<n>` represents a number of bytes, which can optionally + existing cruft packs whose size is strictly less than `<n>` + bytes, which can optionally be suffixed with "k", "m", or "g". Cruft packs whose size is greater than or equal to `<n>` are left as-is and not repacked. Useful when you want to avoid repacking large cruft pack(s) in diff --git a/Documentation/git-replay.adoc b/Documentation/git-replay.adoc index dcb26e8a8e..4c61f3aa1f 100644 --- a/Documentation/git-replay.adoc +++ b/Documentation/git-replay.adoc @@ -9,17 +9,17 @@ git-replay - EXPERIMENTAL: Replay commits on a new base, works with bare repos t SYNOPSIS -------- [verse] -(EXPERIMENTAL!) 'git replay' ([--contained] --onto <newbase> | --advance <branch>) [--ref-action[=<mode>]] <revision-range>... +(EXPERIMENTAL!) 'git replay' ([--contained] --onto <newbase> | --advance <branch>) [--ref-action[=<mode>]] <revision-range> DESCRIPTION ----------- -Takes ranges of commits and replays them onto a new location. Leaves +Takes a range of commits and replays them onto a new location. Leaves the working tree and the index untouched. By default, updates the relevant references using an atomic transaction (all refs update or none). Use `--ref-action=print` to avoid automatic ref updates and instead get update commands that can be piped to `git update-ref --stdin` -(see the OUTPUT section below). +(see the <<output,OUTPUT>> section below). THIS COMMAND IS EXPERIMENTAL. THE BEHAVIOR MAY CHANGE. @@ -42,6 +42,10 @@ The history is replayed on top of the <branch> and <branch> is updated to point at the tip of the resulting history. This is different from `--onto`, which uses the target only as a starting point without updating it. +--contained:: + Update all branches that point at commits in + <revision-range>. Requires `--onto`. + --ref-action[=<mode>]:: Control how references are updated. The mode can be: + @@ -55,14 +59,14 @@ which uses the target only as a starting point without updating it. The default mode can be configured via the `replay.refAction` configuration variable. <revision-range>:: - Range of commits to replay. More than one <revision-range> can - be passed, but in `--advance <branch>` mode, they should have - a single tip, so that it's clear where <branch> should point - to. See "Specifying Ranges" in linkgit:git-rev-parse[1] and the - "Commit Limiting" options below. + Range of commits to replay; see "Specifying Ranges" in + linkgit:git-rev-parse[1]. In `--advance <branch>` mode, the + range should have a single tip, so that it's clear to which tip the + advanced <branch> should point. include::rev-list-options.adoc[] +[[output]] OUTPUT ------ @@ -81,6 +85,10 @@ the shape of the history being replayed. When using `--advance`, the number of refs updated is always one, but for `--onto`, it can be one or more (rebasing multiple branches simultaneously is supported). +There is no stderr output on conflicts; see the <<exit-status,EXIT +STATUS>> section below. + +[[exit-status]] EXIT STATUS ----------- diff --git a/Documentation/git-repo.adoc b/Documentation/git-repo.adoc index 70f0a6d2e4..7d70270dfa 100644 --- a/Documentation/git-repo.adoc +++ b/Documentation/git-repo.adoc @@ -8,8 +8,8 @@ git-repo - Retrieve information about the repository SYNOPSIS -------- [synopsis] -git repo info [--format=(keyvalue|nul)] [-z] [--all | <key>...] -git repo structure [--format=(table|keyvalue|nul)] +git repo info [--format=(keyvalue|nul) | -z] [--all | <key>...] +git repo structure [--format=(table|keyvalue|nul) | -z] DESCRIPTION ----------- @@ -19,7 +19,7 @@ THIS COMMAND IS EXPERIMENTAL. THE BEHAVIOR MAY CHANGE. COMMANDS -------- -`info [--format=(keyvalue|nul)] [-z] [--all | <key>...]`:: +`info [--format=(keyvalue|nul) | -z] [--all | <key>...]`:: Retrieve metadata-related information about the current repository. Only the requested data will be returned based on their keys (see "INFO KEYS" section below). @@ -44,13 +44,14 @@ supported: + `-z` is an alias for `--format=nul`. -`structure [--format=(table|keyvalue|nul)]`:: +`structure [--format=(table|keyvalue|nul) | -z]`:: Retrieve statistics about the current repository structure. The following kinds of information are reported: + * Reference counts categorized by type * Reachable object counts categorized by type - +* Total inflated size of reachable objects by type +* Total disk size of reachable objects by type + The output format can be chosen through the flag `--format`. Three formats are supported: @@ -72,6 +73,8 @@ supported: the delimiter between the key and value instead of '='. Unlike the `keyvalue` format, values containing "unusual" characters are never quoted. ++ +`-z` is an alias for `--format=nul`. INFO KEYS --------- diff --git a/Documentation/git-send-email.adoc b/Documentation/git-send-email.adoc index 263b977353..ebe8853e9f 100644 --- a/Documentation/git-send-email.adoc +++ b/Documentation/git-send-email.adoc @@ -208,7 +208,7 @@ Sending for your own case. Default is the value of `sendemail.smtpEncryption`. --smtp-domain=<FQDN>:: - Specifies the Fully Qualified Domain Name (FQDN) used in the + Specify the Fully Qualified Domain Name (FQDN) used in the HELO/EHLO command to the SMTP server. Some servers require the FQDN to match your IP address. If not set, `git send-email` attempts to determine your FQDN automatically. Default is the value of @@ -245,7 +245,7 @@ a password is obtained using linkgit:git-credential[1]. Disable SMTP authentication. Short hand for `--smtp-auth=none`. --smtp-server=<host>:: - If set, specifies the outgoing SMTP server to use (e.g. + Specify the outgoing SMTP server to use (e.g. `smtp.example.com` or a raw IP address). If unspecified, and if `--sendmail-cmd` is also unspecified, the default is to search for `sendmail` in `/usr/sbin`, `/usr/lib` and `$PATH` if such a @@ -258,7 +258,7 @@ command names. For those use cases, consider using `--sendmail-cmd` instead. --smtp-server-port=<port>:: - Specifies a port different from the default port (SMTP + Specify a port different from the default port (SMTP servers typically listen to smtp port 25, but may also listen to submission port 587, or the common SSL smtp port 465); symbolic port names (e.g. `submission` instead of 587) @@ -266,7 +266,7 @@ instead. `sendemail.smtpServerPort` configuration variable. --smtp-server-option=<option>:: - If set, specifies the outgoing SMTP server option to use. + Specify the outgoing SMTP server option to use. Default value can be specified by the `sendemail.smtpServerOption` configuration option. + @@ -277,7 +277,7 @@ must be used for each option. --smtp-ssl:: Legacy alias for `--smtp-encryption ssl`. ---smtp-ssl-cert-path:: +--smtp-ssl-cert-path <path>:: Path to a store of trusted CA certificates for SMTP SSL/TLS certificate validation (either a directory that has been processed by `c_rehash`, or a single file containing one or more PEM format @@ -321,7 +321,6 @@ for instructions. If disabled with `--no-use-imap-only`, the emails will be sent like usual. Disabled by default, but the `sendemail.useImapOnly` configuration variable can be used to enable it. - + This feature requires setting up `git imap-send`. See linkgit:git-imap-send[1] for instructions. @@ -347,11 +346,11 @@ Automating --no-to:: --no-cc:: --no-bcc:: - Clears any list of `To:`, `Cc:`, `Bcc:` addresses previously + Clear any list of `To:`, `Cc:`, `Bcc:` addresses previously set via config. --no-identity:: - Clears the previously read value of `sendemail.identity` set + Clear the previously read value of `sendemail.identity` set via config, if any. --to-cmd=<command>:: @@ -510,12 +509,12 @@ have been specified, in which case default to `compose`. Currently, validation means the following: + -- - * Invoke the sendemail-validate hook if present (see linkgit:githooks[5]). - * Warn of patches that contain lines longer than - 998 characters unless a suitable transfer encoding - (`auto`, `base64`, or `quoted-printable`) is used; - this is due to SMTP limits as described by - https://www.ietf.org/rfc/rfc5322.txt. +* Invoke the sendemail-validate hook if present (see linkgit:githooks[5]). +* Warn of patches that contain lines longer than + 998 characters unless a suitable transfer encoding + (`auto`, `base64`, or `quoted-printable`) is used; + this is due to SMTP limits as described by + https://www.ietf.org/rfc/rfc5322.txt. -- + Default is the value of `sendemail.validate`; if this is not set, diff --git a/Documentation/git-worktree.adoc b/Documentation/git-worktree.adoc index f272f79783..d74ad7b0e9 100644 --- a/Documentation/git-worktree.adoc +++ b/Documentation/git-worktree.adoc @@ -104,7 +104,7 @@ associated with a new unborn branch named _<branch>_ (after passed to the command. In the event the repository has a remote and `--guess-remote` is used, but no remote or local branches exist, then the command fails with a warning reminding the user to fetch from their remote -first (or override by using `-f/--force`). +first (or override by using `-f`/`--force`). `list`:: diff --git a/Documentation/gitdatamodel.adoc b/Documentation/gitdatamodel.adoc index 3614f5960e..dcfdff0346 100644 --- a/Documentation/gitdatamodel.adoc +++ b/Documentation/gitdatamodel.adoc @@ -235,8 +235,6 @@ there will no longer be a branch that points at the old commit. The old commit is recorded in the current branch's <<reflogs,reflog>>, so it is still "reachable", but when the reflog entry expires it may become unreachable and get deleted. - -the old commit will usually not be reachable, so it may be deleted eventually. Reachable objects will never be deleted. [[index]] diff --git a/Documentation/gitfaq.adoc b/Documentation/gitfaq.adoc index f2917d142c..8d3647d359 100644 --- a/Documentation/gitfaq.adoc +++ b/Documentation/gitfaq.adoc @@ -83,6 +83,25 @@ Windows would be the configuration `"C:\Program Files\Vim\gvim.exe" --nofork`, which quotes the filename with spaces and specifies the `--nofork` option to avoid backgrounding the process. +[[sign-off]] +Why not have `commit.signoff` and other configuration variables?:: + Git intentionally does not (and will not) provide a + configuration variable, such as `commit.signoff`, to + automatically add `--signoff` by default. The reason is to + protect the legal and intentional significance of a sign-off. + If there were more automated and widely publicized ways for + sign-offs to be appended, it would become easier for someone + to argue later that a "Signed-off-by" trailer was just added + out of habit or by automation, without the committer's full + awareness or intent to certify their agreement with the + Developer Certificate of Origin (DCO) or a similar statement. + This could undermine the sign-off’s credibility in legal or + contractual situations. ++ +There exists `format.signoff`, but that is a historical mistake, and +it is not an excuse to add more mistakes of the same kind on top. + + Credentials ----------- diff --git a/Documentation/githooks.adoc b/Documentation/githooks.adoc index 0397dec64d..056553788d 100644 --- a/Documentation/githooks.adoc +++ b/Documentation/githooks.adoc @@ -103,17 +103,14 @@ invoked before obtaining the proposed commit log message and making a commit. Exiting with a non-zero status from this script causes the `git commit` command to abort before creating a commit. -The default 'pre-commit' hook, when enabled, catches introduction -of lines with trailing whitespaces and aborts the commit when -such a line is found. - All the `git commit` hooks are invoked with the environment variable `GIT_EDITOR=:` if the command will not bring up an editor to modify the commit message. -The default 'pre-commit' hook, when enabled--and with the -`hooks.allownonascii` config option unset or set to false--prevents -the use of non-ASCII filenames. +The default 'pre-commit' hook, when enabled, prevents the introduction +of non-ASCII filenames and lines with trailing whitespace. The non-ASCII +check can be turned off by setting the `hooks.allownonascii` config +option to `true`. pre-merge-commit ~~~~~~~~~~~~~~~~ diff --git a/Documentation/meson.build b/Documentation/meson.build index f02dbc20cb..fd2e8cc02d 100644 --- a/Documentation/meson.build +++ b/Documentation/meson.build @@ -64,6 +64,7 @@ manpages = { 'git-gui.adoc' : 1, 'git-hash-object.adoc' : 1, 'git-help.adoc' : 1, + 'git-history.adoc' : 1, 'git-hook.adoc' : 1, 'git-http-backend.adoc' : 1, 'git-http-fetch.adoc' : 1, diff --git a/Documentation/rev-list-options.adoc b/Documentation/rev-list-options.adoc index d9665d82c8..453ec59057 100644 --- a/Documentation/rev-list-options.adoc +++ b/Documentation/rev-list-options.adoc @@ -983,7 +983,9 @@ to name units in KiB, MiB, or GiB. For example, `blob:limit=1k` is the same as 'blob:limit=1024'. + The form `--filter=object:type=(tag|commit|tree|blob)` omits all objects -which are not of the requested type. +which are not of the requested type. Note that explicitly provided objects +ignore filters and are always printed unless `--filter-provided-objects` is +also specified. + The form `--filter=sparse:oid=<blob-ish>` uses a sparse-checkout specification contained in the blob (or blob-expression) _<blob-ish>_ diff --git a/Documentation/scalar.adoc b/Documentation/scalar.adoc index f81b2832f8..5252fb134a 100644 --- a/Documentation/scalar.adoc +++ b/Documentation/scalar.adoc @@ -197,6 +197,170 @@ delete <enlistment>:: This subcommand lets you delete an existing Scalar enlistment from your local file system, unregistering the repository. +RECOMMENDED CONFIG VALUES +------------------------- + +As part of both `scalar clone` and `scalar register`, certain Git config +values are set to optimize for large repositories or cross-platform support. +These options are updated in new Git versions according to the best known +advice for large repositories, and users can get the latest recommendations +by running `scalar reconfigure [--all]`. + +This section lists justifications for the config values that are set in the +latest version. + +am.keepCR=true:: + This setting is important for cross-platform development across Windows + and non-Windows platforms and keeping carriage return (`\r`) characters + in certain workflows. + +commitGraph.changedPaths=true:: + This setting helps the background maintenance steps that compute the + serialized commit-graph to also store changed-path Bloom filters. This + accelerates file history commands and allows users to automatically + benefit without running a foreground command. + +commitGraph.generationVersion=1:: + While the preferred version is 2 for performance reasons, existing users + that had version 1 by default will need special care in upgrading to + version 2. This is likely to change in the future as the upgrade story + solidifies. + +core.autoCRLF=false:: + This removes the transformation of worktree files to add CRLF line + endings when only LF line endings exist. This is removed for performance + reasons. Repositories that use tools that care about CRLF line endings + should commit the necessary files with those line endings instead. + +core.logAllRefUpdates=true:: + This enables the reflog on all branches. While this is a performance + cost for large repositories, it is frequently an important data source + for users to get out of bad situations or to seek support from experts. + +core.safeCRLF=false:: + Similar to `core.autoCRLF=false`, this disables checks around whether + the CRLF conversion is reversible. This is a performance improvement, + but can be dangerous if `core.autoCRLF` is reenabled by the user. + +credential.https://dev.azure.com.useHttpPath=true:: + This setting enables the `credential.useHttpPath` feature only for web + URLs for Azure DevOps. This is important for users interacting with that + service using multiple organizations and thus multiple credential + tokens. + +feature.experimental=false:: + This disables the "experimental" optimizations grouped under this + feature config. The expectation is that all valuable optimizations are + also set explicitly by Scalar config, and any differences are + intentional. Notable differences include several bitmap-related config + options which are disabled for client-focused Scalar repos. + +feature.manyFiles=false:: + This disables the "many files" optimizations grouped under this feature + config. The expectation is that all valuable optimizations are also set + explicitly by Scalar config, and any differences are intentional. + +fetch.showForcedUpdates=false:: + This disables the check at the end of `git fetch` that notifies the user + if the ref update was a forced update (one where the previous position + is not reachable from the latest position). This check can be very + expensive in large repositories, so is disabled and replaced with an + advice message. Set `advice.fetchShowForcedUpdates=false` to disable + this advice message. + +fetch.unpackLimit=1:: + This setting prevents Git from unpacking packfiles into loose objects + as they are downloaded from the server. The default limit of 100 was + intended as a way to prevent performance issues from too many packfiles, + but Scalar uses background maintenance to group packfiles and cover them + with a multi-pack-index, removing this issue. + +fetch.writeCommitGraph=false:: + This config setting was created to help users automatically update their + commit-graph files as they perform fetches. However, this takes time + from foreground fetches and pulls and Scalar uses background maintenance + for this function instead. + +gc.auto=0:: + This disables automatic garbage collection, since Scalar uses background + maintenance to keep the repository data in good shape. + +gui.GCWarning=false:: + Since Scalar disables garbage collection by setting `gc.auto=0`, the + `git-gui` tool may start to warn about this setting. Disable this + warning as Scalar's background maintenance configuration makes the + warning irrelevant. + +index.skipHash=true:: + Disable computing the hash of the index contents as it is being written. + This assists with performance, especially for large index files. + +index.threads=true:: + This tells Git to automatically detect how many threads it should use + when reading the index due to the default value of `core.preloadIndex`, + which enables parallel index reads. This explicit setting also enables + `index.recordOffsetTable=true` to speed up parallel index reads. + +index.version=4:: + This index version adds compression to the path names, reducing the size + of the index in a significant way for large repos. This is an important + performance boost. + +log.excludeDecoration=refs/prefetch/*:: + Since Scalar enables background maintenance with the `incremental` + strategy, this setting avoids polluting `git log` output with refs + stored by the background prefetch operations. + +merge.renames=true:: + When computing merges in large repos, it is particularly important to + detect renames to maximize the potential for a result that will validate + correctly. Users performing merges locally are more likely to be doing + so because a server-side merge (via pull request or similar) resulted in + conflicts. While this is the default setting, it is set specifically to + override a potential change to `diff.renames` which a user may set for + performance reasons. + +merge.stat=false:: + This disables a diff output after computing a merge. This improves + performance of `git merge` for large repos while reducing noisy output. + +pack.useBitmaps=false:: + This disables the use of `.bitmap` files attached to packfiles. Bitmap + files are optimized for server-side use, not client-side use. Scalar + disables this to avoid some performance issues that can occur if a user + accidentally creates `.bitmap` files. + +pack.usePathWalk=true:: + This enables the `--path-walk` option to `git pack-objects` by default. + This can accelerate the computation and compression of packfiles created + by `git push` and other repack operations. + +receive.autoGC=false:: + Similar to `gc.auto`, this setting is disabled in preference of + background maintenance. + +status.aheadBehind=false:: + This disables the ahead/behind calculation that would normally happen + during a `git status` command. This information is frequently ignored by + users but can be expensive to calculate in large repos that receive + thousands of commits per day. The calculation is replaced with an advice + message that can be disabled by disabling the `advice.statusAheadBehind` + config. + +The following settings are different based on which platform is in use: + +core.untrackedCache=(true|false):: + The untracked cache feature is important for performance benefits on + large repositories, but has demonstrated some bugs on Windows + filesystems. Thus, this is set for other platforms but disabled on + Windows. + +http.sslBackend=schannel:: + On Windows, the `openssl` backend has some issues with certain types of + remote providers and certificate types. Override the default setting to + avoid these common problems. + + SEE ALSO -------- linkgit:git-clone[1], linkgit:git-maintenance[1]. diff --git a/Documentation/signoff-option.adoc b/Documentation/signoff-option.adoc index cddfb225d1..6fc2769257 100644 --- a/Documentation/signoff-option.adoc +++ b/Documentation/signoff-option.adoc @@ -16,3 +16,7 @@ endif::git-commit[] + The `--no-signoff` option can be used to countermand an earlier `--signoff` option on the command line. ++ +Git does not (and will not) have a configuration variable to enable +the `--signoff` command line option by default; see the +`commit.signoff` entry in linkgit:gitfaq[7] for more details. diff --git a/Documentation/technical/meson.build b/Documentation/technical/meson.build index faff3964a9..ec07088c57 100644 --- a/Documentation/technical/meson.build +++ b/Documentation/technical/meson.build @@ -32,6 +32,7 @@ articles = [ 'sparse-checkout.adoc', 'sparse-index.adoc', 'trivial-merge.adoc', + 'unambiguous-types.adoc', 'unit-tests.adoc', ] diff --git a/Documentation/technical/unambiguous-types.adoc b/Documentation/technical/unambiguous-types.adoc new file mode 100644 index 0000000000..9a4990847c --- /dev/null +++ b/Documentation/technical/unambiguous-types.adoc @@ -0,0 +1,224 @@ += Unambiguous types + +Most of these mappings are obvious, but there are some nuances and gotchas with +Rust FFI (Foreign Function Interface). + +This document defines clear, one-to-one mappings between primitive types in C, +Rust (and possible other languages in the future). Its purpose is to eliminate +ambiguity in type widths, signedness, and binary representation across +platforms and languages. + +For Git, the only header required to use these unambiguous types in C is +`git-compat-util.h`. + +== Boolean types +[cols="1,1", options="header"] +|=== +| C Type | Rust Type +| bool^1^ | bool +|=== + +== Integer types + +In C, `<stdint.h>` (or an equivalent) must be included. + +[cols="1,1", options="header"] +|=== +| C Type | Rust Type +| uint8_t | u8 +| uint16_t | u16 +| uint32_t | u32 +| uint64_t | u64 + +| int8_t | i8 +| int16_t | i16 +| int32_t | i32 +| int64_t | i64 +|=== + +== Floating-point types + +Rust requires IEEE-754 semantics. +In C, that is typically true, but not guaranteed by the standard. + +[cols="1,1", options="header"] +|=== +| C Type | Rust Type +| float^2^ | f32 +| double^2^ | f64 +|=== + +== Size types + +These types represent pointer-sized integers and are typically defined in +`<stddef.h>` or an equivalent header. + +Size types should be used any time pointer arithmetic is performed e.g. +indexing an array, describing the number of elements in memory, etc... + +[cols="1,1", options="header"] +|=== +| C Type | Rust Type +| size_t^3^ | usize +| ptrdiff_t^3^ | isize +|=== + +== Character types + +This is where C and Rust don't have a clean one-to-one mapping. + +A C `char` and a Rust `u8` share the same bit width, so any C struct containing +a `char` will have the same size as the corresponding Rust struct using `u8`. +In that sense, such structs are safe to pass over the FFI boundary, because +their fields will be laid out identically. However, beyond bit width, C `char` +has additional semantics and platform-dependent behavior that can cause +problems, as discussed below. + +The C language leaves the signedness of `char` implementation defined. Because +our developer build enables -Wsign-compare, comparison of a value of `char` +type with either signed or unsigned integers may trigger warnings from the +compiler. + +Note: Rust's `char` type is an unsigned 32-bit integer that is used to describe +Unicode code points. + +=== Notes +^1^ This is only true if stdbool.h (or equivalent) is used. + +^2^ C does not enforce IEEE-754 compatibility, but Rust expects it. If the +platform/arch for C does not follow IEEE-754 then this equivalence does not +hold. Also, it's assumed that `float` is 32 bits and `double` is 64, but +there may be a strange platform/arch where even this isn't true. + +^3^ C also defines uintptr_t, ssize_t and intptr_t, but these types are +discouraged for FFI purposes. For functions like `read()` and `write()` ssize_t +should be cast to a different, and unambiguous, type before being passed over +the FFI boundary. + + +== Problems with std::ffi::c_* types in Rust +TL;DR: In practice, Rust's `c_*` types aren't guaranteed to match C types for +all possible C compilers, platforms, or architectures, because Rust only +ensures correctness of C types on officially supported targets. These +definitions have changed over time to match more targets which means that the +c_* definitions will differ based on which Rust version Git chooses to use. + +Current list of safe, Rust side, FFI types in Git: + + +* `c_void` +* `CStr` +* `CString` + +Even then, they should be used sparingly, and only where the semantics match +exactly. + +The std::os::raw::c_* directly inherits the problems of core::ffi, which +changes over time and seems to make a best guess at the correct definition for +a given platform/target. This probably isn't a problem for all other platforms +that Rust supports currently, but can anyone say that Rust got it right for all +C compilers of all platforms/targets? + +To give an example: c_long is defined in +footnote:[https://doc.rust-lang.org/1.63.0/src/core/ffi/mod.rs.html#175-189[c_long in 1.63.0]] +footnote:[https://doc.rust-lang.org/1.89.0/src/core/ffi/primitives.rs.html#135-151[c_long in 1.89.0]] + +=== Rust version 1.63.0 + +``` +mod c_long_definition { + cfg_if! { + if #[cfg(all(target_pointer_width = "64", not(windows)))] { + pub type c_long = i64; + pub type NonZero_c_long = crate::num::NonZeroI64; + pub type c_ulong = u64; + pub type NonZero_c_ulong = crate::num::NonZeroU64; + } else { + // The minimal size of `long` in the C standard is 32 bits + pub type c_long = i32; + pub type NonZero_c_long = crate::num::NonZeroI32; + pub type c_ulong = u32; + pub type NonZero_c_ulong = crate::num::NonZeroU32; + } + } +} +``` + +=== Rust version 1.89.0 + +``` +mod c_long_definition { + crate::cfg_select! { + any( + all(target_pointer_width = "64", not(windows)), + // wasm32 Linux ABI uses 64-bit long + all(target_arch = "wasm32", target_os = "linux") + ) => { + pub(super) type c_long = i64; + pub(super) type c_ulong = u64; + } + _ => { + // The minimal size of `long` in the C standard is 32 bits + pub(super) type c_long = i32; + pub(super) type c_ulong = u32; + } + } +} +``` + +Even for the cases where C types are correctly mapped to Rust types via +std::ffi::c_* there are still problems. Let's take c_char for example. On some +platforms it's u8 on others it's i8. + +=== Subtraction underflow in debug mode + +The following code will panic in debug on platforms that define c_char as u8, +but won't if it's an i8. + +``` +let mut x: std::ffi::c_char = 0; +x -= 1; +``` + +=== Inconsistent shift behavior + +`x` will be 0xC0 for platforms that use i8, but will be 0x40 where it's u8. + +``` +let mut x: std::ffi::c_char = 0x80; +x >>= 1; +``` + +=== Equality fails to compile on some platforms + +The following will not compile on platforms that define c_char as i8, but will +if it's u8. You can cast x e.g. `assert_eq!(x as u8, b'a');`, but then you get +a warning on platforms that use u8 and a clean compilation where i8 is used. + +``` +let mut x: std::ffi::c_char = 0x61; +assert_eq!(x, b'a'); +``` + +== Enum types +Rust enum types should not be used as FFI types. Rust enum types are more like +C union types than C enum's. For something like: + +``` +#[repr(C, u8)] +enum Fruit { + Apple, + Banana, + Cherry, +} +``` + +It's easy enough to make sure the Rust enum matches what C would expect, but a +more complex type like. + +``` +enum HashResult { + SHA1([u8; 20]), + SHA256([u8; 32]), +} +``` + +The Rust compiler has to add a discriminant to the enum to distinguish between +the variants. The width, location, and values for that discriminant is up to +the Rust compiler and is not ABI stable. @@ -95,11 +95,21 @@ include shared.mak # and LDFLAGS appropriately. # # Define NO_DARWIN_PORTS if you are building on Darwin/Mac OS X, -# have DarwinPorts installed in /opt/local, but don't want GIT to +# have DarwinPorts (which is an old name for MacPorts) installed +# in /opt/local, but don't want GIT to # link against any libraries installed there. If defined you may # specify your own (or DarwinPort's) include directories and # library directories by defining CFLAGS and LDFLAGS appropriately. # +# Define NO_HOMEBREW if you don't want to use gettext, libiconv and +# msgfmt installed by Homebrew. +# +# Define HOMEBREW_PREFIX if you have Homebrew installed in a non-default +# location on macOS or on Linux and want to use it. +# +# Define USE_HOMEBREW_LIBICONV to link against libiconv installed by +# Homebrew, if present. +# # Define NO_APPLE_COMMON_CRYPTO if you are building on Darwin/Mac OS X # and do not want to use Apple's CommonCrypto library. This allows you # to provide your own OpenSSL library, for example from MacPorts. @@ -981,7 +991,7 @@ SANITIZE_LEAK = SANITIZE_ADDRESS = # For the 'coccicheck' target -SPATCH_INCLUDE_FLAGS = --all-includes +SPATCH_INCLUDE_FLAGS = --all-includes $(addprefix -I ,compat ewah refs sha256 trace2 win32 xdiff) SPATCH_FLAGS = SPATCH_TEST_FLAGS = @@ -1201,6 +1211,7 @@ LIB_OBJS += object-file.o LIB_OBJS += object-name.o LIB_OBJS += object.o LIB_OBJS += odb.o +LIB_OBJS += odb/streaming.o LIB_OBJS += oid-array.o LIB_OBJS += oidmap.o LIB_OBJS += oidset.o @@ -1274,6 +1285,7 @@ LIB_OBJS += repack-geometry.o LIB_OBJS += repack-midx.o LIB_OBJS += repack-promisor.o LIB_OBJS += replace-object.o +LIB_OBJS += replay.o LIB_OBJS += repo-settings.o LIB_OBJS += repository.o LIB_OBJS += rerere.o @@ -1294,7 +1306,6 @@ LIB_OBJS += split-index.o LIB_OBJS += stable-qsort.o LIB_OBJS += statinfo.o LIB_OBJS += strbuf.o -LIB_OBJS += streaming.o LIB_OBJS += string-list.o LIB_OBJS += strmap.o LIB_OBJS += strvec.o @@ -1407,6 +1418,7 @@ BUILTIN_OBJS += builtin/get-tar-commit-id.o BUILTIN_OBJS += builtin/grep.o BUILTIN_OBJS += builtin/hash-object.o BUILTIN_OBJS += builtin/help.o +BUILTIN_OBJS += builtin/history.o BUILTIN_OBJS += builtin/hook.o BUILTIN_OBJS += builtin/index-pack.o BUILTIN_OBJS += builtin/init-db.o @@ -1692,6 +1704,23 @@ ifeq ($(uname_S),Darwin) PTHREAD_LIBS = endif +ifndef NO_HOMEBREW +ifdef HOMEBREW_PREFIX +ifeq ($(shell test -d $(HOMEBREW_PREFIX)/opt/gettext && echo y),y) + BASIC_CFLAGS += -I$(HOMEBREW_PREFIX)/opt/gettext/include + BASIC_LDFLAGS += -L$(HOMEBREW_PREFIX)/opt/gettext/lib +endif +ifeq ($(shell test -x $(HOMEBREW_PREFIX)/opt/gettext/msgfmt && echo y),y) + MSGFMT = $(HOMEBREW_PREFIX)/opt/gettext/msgfmt +endif +ifdef USE_HOMEBREW_LIBICONV +ifeq ($(shell test -d $(HOMEBREW_PREFIX)/opt/libiconv && echo y),y) + ICONVDIR ?= $(HOMEBREW_PREFIX)/opt/libiconv +endif +endif +endif +endif + ifdef NO_LIBGEN_H COMPAT_CFLAGS += -DNO_LIBGEN_H COMPAT_OBJS += compat/basename.o @@ -1919,7 +1948,6 @@ ifdef NO_SETENV endif ifdef NO_MKDTEMP COMPAT_CFLAGS += -DNO_MKDTEMP - COMPAT_OBJS += compat/mkdtemp.o endif ifdef MKDIR_WO_TRAILING_SLASH COMPAT_CFLAGS += -DMKDIR_WO_TRAILING_SLASH @@ -2567,7 +2595,7 @@ please_set_SHELL_PATH_to_a_more_modern_shell: shell_compatibility_test: please_set_SHELL_PATH_to_a_more_modern_shell -strip: $(PROGRAMS) git$X +strip: $(PROGRAMS) git$X scalar$X $(STRIP) $(STRIP_OPTS) $^ ### Target-specific flags and dependencies @@ -3523,7 +3551,7 @@ else COCCICHECK_PATCH_MUST_BE_EMPTY_FILES = $(COCCICHECK_PATCHES_INTREE) endif coccicheck: $(COCCICHECK_PATCH_MUST_BE_EMPTY_FILES) - ! grep -q ^ $(COCCICHECK_PATCH_MUST_BE_EMPTY_FILES) /dev/null + ! grep ^ $(COCCICHECK_PATCH_MUST_BE_EMPTY_FILES) /dev/null # See contrib/coccinelle/README coccicheck-pending: coccicheck-test @@ -3818,7 +3818,7 @@ static int check_preimage(struct apply_state *state, if (*ce && !(*ce)->ce_mode) BUG("ce_mode == 0 for path '%s'", old_name); - if (trust_executable_bit) + if (trust_executable_bit || !S_ISREG(st->st_mode)) st_mode = ce_mode_from_stat(*ce, st->st_mode); else if (*ce) st_mode = (*ce)->ce_mode; diff --git a/archive-tar.c b/archive-tar.c index 73b63ddc41..0fc70d13a8 100644 --- a/archive-tar.c +++ b/archive-tar.c @@ -12,8 +12,8 @@ #include "tar.h" #include "archive.h" #include "odb.h" +#include "odb/streaming.h" #include "strbuf.h" -#include "streaming.h" #include "run-command.h" #include "write-or-die.h" @@ -129,22 +129,20 @@ static void write_trailer(void) */ static int stream_blocked(struct repository *r, const struct object_id *oid) { - struct git_istream *st; - enum object_type type; - unsigned long sz; + struct odb_read_stream *st; char buf[BLOCKSIZE]; ssize_t readlen; - st = open_istream(r, oid, &type, &sz, NULL); + st = odb_read_stream_open(r->objects, oid, NULL); if (!st) return error(_("cannot stream blob %s"), oid_to_hex(oid)); for (;;) { - readlen = read_istream(st, buf, sizeof(buf)); + readlen = odb_read_stream_read(st, buf, sizeof(buf)); if (readlen <= 0) break; do_write_blocked(buf, readlen); } - close_istream(st); + odb_read_stream_close(st); if (!readlen) finish_record(); return readlen; diff --git a/archive-zip.c b/archive-zip.c index bea5bdd43d..97ea8d60d6 100644 --- a/archive-zip.c +++ b/archive-zip.c @@ -10,9 +10,9 @@ #include "gettext.h" #include "git-zlib.h" #include "hex.h" -#include "streaming.h" #include "utf8.h" #include "odb.h" +#include "odb/streaming.h" #include "strbuf.h" #include "userdiff.h" #include "write-or-die.h" @@ -309,7 +309,7 @@ static int write_zip_entry(struct archiver_args *args, enum zip_method method; unsigned char *out; void *deflated = NULL; - struct git_istream *stream = NULL; + struct odb_read_stream *stream = NULL; unsigned long flags = 0; int is_binary = -1; const char *path_without_prefix = path + args->baselen; @@ -347,12 +347,11 @@ static int write_zip_entry(struct archiver_args *args, method = ZIP_METHOD_DEFLATE; if (!buffer) { - enum object_type type; - stream = open_istream(args->repo, oid, &type, &size, - NULL); + stream = odb_read_stream_open(args->repo->objects, oid, NULL); if (!stream) return error(_("cannot stream blob %s"), oid_to_hex(oid)); + size = stream->size; flags |= ZIP_STREAM; out = NULL; } else { @@ -429,7 +428,7 @@ static int write_zip_entry(struct archiver_args *args, ssize_t readlen; for (;;) { - readlen = read_istream(stream, buf, sizeof(buf)); + readlen = odb_read_stream_read(stream, buf, sizeof(buf)); if (readlen <= 0) break; crc = crc32(crc, buf, readlen); @@ -439,7 +438,7 @@ static int write_zip_entry(struct archiver_args *args, buf, readlen); write_or_die(1, buf, readlen); } - close_istream(stream); + odb_read_stream_close(stream); if (readlen) return readlen; @@ -462,7 +461,7 @@ static int write_zip_entry(struct archiver_args *args, zstream.avail_out = sizeof(compressed); for (;;) { - readlen = read_istream(stream, buf, sizeof(buf)); + readlen = odb_read_stream_read(stream, buf, sizeof(buf)); if (readlen <= 0) break; crc = crc32(crc, buf, readlen); @@ -486,7 +485,7 @@ static int write_zip_entry(struct archiver_args *args, } } - close_istream(stream); + odb_read_stream_close(stream); if (readlen) return readlen; @@ -41,4 +41,7 @@ #undef asctime_r #define asctime_r(t, buf) BANNED(asctime_r) +#undef mktemp +#define mktemp(x) BANNED(mktemp) + #endif /* BANNED_H */ @@ -375,7 +375,7 @@ int validate_branchname(const char *name, struct strbuf *ref) if (check_branch_ref(ref, name)) { int code = die_message(_("'%s' is not a valid branch name"), name); advise_if_enabled(ADVICE_REF_SYNTAX, - _("See `man git check-ref-format`")); + _("See 'git help check-ref-format'")); exit(code); } @@ -172,6 +172,7 @@ int cmd_get_tar_commit_id(int argc, const char **argv, const char *prefix, struc int cmd_grep(int argc, const char **argv, const char *prefix, struct repository *repo); int cmd_hash_object(int argc, const char **argv, const char *prefix, struct repository *repo); int cmd_help(int argc, const char **argv, const char *prefix, struct repository *repo); +int cmd_history(int argc, const char **argv, const char *prefix, struct repository *repo); int cmd_hook(int argc, const char **argv, const char *prefix, struct repository *repo); int cmd_index_pack(int argc, const char **argv, const char *prefix, struct repository *repo); int cmd_init_db(int argc, const char **argv, const char *prefix, struct repository *repo); diff --git a/builtin/blame.c b/builtin/blame.c index 27b513d27f..6044973462 100644 --- a/builtin/blame.c +++ b/builtin/blame.c @@ -739,7 +739,8 @@ static int git_blame_config(const char *var, const char *value, ret = git_config_pathname(&str, var, value); if (ret) return ret; - string_list_insert(&ignore_revs_file_list, str); + if (str) + string_list_insert(&ignore_revs_file_list, str); free(str); return 0; } diff --git a/builtin/branch.c b/builtin/branch.c index 9fcf04bebb..c577b5d20f 100644 --- a/builtin/branch.c +++ b/builtin/branch.c @@ -591,7 +591,7 @@ static void copy_or_rename_branch(const char *oldname, const char *newname, int else { int code = die_message(_("invalid branch name: '%s'"), oldname); advise_if_enabled(ADVICE_REF_SYNTAX, - _("See `man git check-ref-format`")); + _("See 'git help check-ref-format'")); exit(code); } } diff --git a/builtin/cat-file.c b/builtin/cat-file.c index 983ecec837..505ddaa12f 100644 --- a/builtin/cat-file.c +++ b/builtin/cat-file.c @@ -18,13 +18,13 @@ #include "list-objects-filter-options.h" #include "parse-options.h" #include "userdiff.h" -#include "streaming.h" #include "oid-array.h" #include "packfile.h" #include "pack-bitmap.h" #include "object-file.h" #include "object-name.h" #include "odb.h" +#include "odb/streaming.h" #include "replace-object.h" #include "promisor-remote.h" #include "mailmap.h" @@ -95,7 +95,7 @@ static int filter_object(const char *path, unsigned mode, static int stream_blob(const struct object_id *oid) { - if (stream_blob_to_fd(1, oid, NULL, 0)) + if (odb_stream_blob_to_fd(the_repository->objects, 1, oid, NULL, 0)) die("unable to stream %s to stdout", oid_to_hex(oid)); return 0; } diff --git a/builtin/checkout.c b/builtin/checkout.c index 66b69df6e6..261699e2f5 100644 --- a/builtin/checkout.c +++ b/builtin/checkout.c @@ -1899,7 +1899,7 @@ static int checkout_main(int argc, const char **argv, const char *prefix, struct object_id rev; if (repo_get_oid_mb(the_repository, opts->from_treeish, &rev)) - die(_("could not resolve %s"), opts->from_treeish); + die(_("could not resolve '%s'"), opts->from_treeish); setup_new_branch_info_and_source_tree(&new_branch_info, opts, &rev, diff --git a/builtin/clone.c b/builtin/clone.c index c990f398ef..b19b302b06 100644 --- a/builtin/clone.c +++ b/builtin/clone.c @@ -1617,7 +1617,7 @@ int cmd_clone(int argc, transport_disconnect(transport); if (option_dissociate) { - close_object_store(the_repository->objects); + odb_close(the_repository->objects); dissociate_from_references(); } diff --git a/builtin/config.c b/builtin/config.c index 75852bd79d..288ebdfdaa 100644 --- a/builtin/config.c +++ b/builtin/config.c @@ -261,6 +261,12 @@ struct strbuf_list { int alloc; }; +/* + * Format the configuration key-value pair (`key_`, `value_`) and + * append it into strbuf `buf`. Returns a negative value on failure, + * 0 on success, 1 on a missing optional value (i.e., telling the + * caller to pretend that <key_,value_> did not exist). + */ static int format_config(const struct config_display_options *opts, struct strbuf *buf, const char *key_, const char *value_, const struct key_value_info *kvi) @@ -299,7 +305,10 @@ static int format_config(const struct config_display_options *opts, char *v; if (git_config_pathname(&v, key_, value_) < 0) return -1; - strbuf_addstr(buf, v); + if (v) + strbuf_addstr(buf, v); + else + return 1; /* :(optional)no-such-file */ free((char *)v); } else if (opts->type == TYPE_EXPIRY_DATE) { timestamp_t t; @@ -344,6 +353,7 @@ static int collect_config(const char *key_, const char *value_, struct collect_config_data *data = cb; struct strbuf_list *values = data->values; const struct key_value_info *kvi = ctx->kvi; + int status; if (!(data->get_value_flags & GET_VALUE_KEY_REGEXP) && strcmp(key_, data->key)) @@ -361,8 +371,15 @@ static int collect_config(const char *key_, const char *value_, ALLOC_GROW(values->items, values->nr + 1, values->alloc); strbuf_init(&values->items[values->nr], 0); - return format_config(data->display_opts, &values->items[values->nr++], - key_, value_, kvi); + status = format_config(data->display_opts, &values->items[values->nr++], + key_, value_, kvi); + if (status < 0) + return status; + if (status) { + strbuf_release(&values->items[--values->nr]); + status = 0; + } + return status; } static int get_value(const struct config_location_options *opts, @@ -438,15 +455,23 @@ static int get_value(const struct config_location_options *opts, if (!values.nr && display_opts->default_value) { struct key_value_info kvi = KVI_INIT; struct strbuf *item; + int status; kvi_from_param(&kvi); ALLOC_GROW(values.items, values.nr + 1, values.alloc); item = &values.items[values.nr++]; strbuf_init(item, 0); - if (format_config(display_opts, item, key_, - display_opts->default_value, &kvi) < 0) + + status = format_config(display_opts, item, key_, + display_opts->default_value, &kvi); + if (status < 0) die(_("failed to format default config value: %s"), display_opts->default_value); + if (status) { + /* default was a missing optional value */ + values.nr--; + strbuf_release(item); + } } ret = !values.nr; @@ -714,11 +739,13 @@ static int get_urlmatch(const struct config_location_options *opts, for_each_string_list_item(item, &values) { struct urlmatch_current_candidate_value *matched = item->util; struct strbuf buf = STRBUF_INIT; + int status; - format_config(&display_opts, &buf, item->string, - matched->value_is_null ? NULL : matched->value.buf, - &matched->kvi); - fwrite(buf.buf, 1, buf.len, stdout); + status = format_config(&display_opts, &buf, item->string, + matched->value_is_null ? NULL : matched->value.buf, + &matched->kvi); + if (!status) + fwrite(buf.buf, 1, buf.len, stdout); strbuf_release(&buf); strbuf_release(&matched->value); @@ -985,7 +1012,7 @@ static int cmd_config_set(int argc, const char **argv, const char *prefix, argv[0], comment, value); if (ret == CONFIG_NOTHING_SET) error(_("cannot overwrite multiple values with a single value\n" - " Use a regexp, --add or --replace-all to change %s."), argv[0]); + " Use --value=<pattern>, --append or --all to change %s."), argv[0]); } location_options_release(&location_opts); @@ -1003,8 +1030,8 @@ static int cmd_config_unset(int argc, const char **argv, const char *prefix, struct option opts[] = { CONFIG_LOCATION_OPTIONS(location_opts), OPT_GROUP(N_("Filter")), - OPT_BIT(0, "all", &flags, N_("replace multi-valued config option with new value"), CONFIG_FLAGS_MULTI_REPLACE), - OPT_STRING(0, "value", &value_pattern, N_("pattern"), N_("show config with values matching the pattern")), + OPT_BIT(0, "all", &flags, N_("unset all multi-valued config options"), CONFIG_FLAGS_MULTI_REPLACE), + OPT_STRING(0, "value", &value_pattern, N_("pattern"), N_("unset multi-valued config options with matching values")), OPT_BIT(0, "fixed-value", &flags, N_("use string equality when comparing values to value pattern"), CONFIG_FLAGS_FIXED_VALUE), OPT_END(), }; diff --git a/builtin/fast-export.c b/builtin/fast-export.c index 0421360ab7..b90da5e616 100644 --- a/builtin/fast-export.c +++ b/builtin/fast-export.c @@ -797,10 +797,7 @@ static void handle_commit(struct commit *commit, struct rev_info *rev, (int)(committer_end - committer), committer); if (signatures.nr) { switch (signed_commit_mode) { - case SIGN_ABORT: - die(_("encountered signed commit %s; use " - "--signed-commits=<mode> to handle it"), - oid_to_hex(&commit->object.oid)); + /* Exporting modes */ case SIGN_WARN_VERBATIM: warning(_("exporting %"PRIuMAX" signature(s) for commit %s"), (uintmax_t)signatures.nr, oid_to_hex(&commit->object.oid)); @@ -811,12 +808,25 @@ static void handle_commit(struct commit *commit, struct rev_info *rev, print_signature(item->string, item->util); } break; + + /* Stripping modes */ case SIGN_WARN_STRIP: warning(_("stripping signature(s) from commit %s"), oid_to_hex(&commit->object.oid)); /* fallthru */ case SIGN_STRIP: break; + + /* Aborting modes */ + case SIGN_ABORT: + die(_("encountered signed commit %s; use " + "--signed-commits=<mode> to handle it"), + oid_to_hex(&commit->object.oid)); + case SIGN_STRIP_IF_INVALID: + die(_("'strip-if-invalid' is not a valid mode for " + "git fast-export with --signed-commits=<mode>")); + default: + BUG("invalid signed_commit_mode value %d", signed_commit_mode); } string_list_clear(&signatures, 0); } @@ -935,16 +945,15 @@ static void handle_tag(const char *name, struct tag *tag) size_t sig_offset = parse_signed_buffer(message, message_size); if (sig_offset < message_size) switch (signed_tag_mode) { - case SIGN_ABORT: - die(_("encountered signed tag %s; use " - "--signed-tags=<mode> to handle it"), - oid_to_hex(&tag->object.oid)); + /* Exporting modes */ case SIGN_WARN_VERBATIM: warning(_("exporting signed tag %s"), oid_to_hex(&tag->object.oid)); /* fallthru */ case SIGN_VERBATIM: break; + + /* Stripping modes */ case SIGN_WARN_STRIP: warning(_("stripping signature from tag %s"), oid_to_hex(&tag->object.oid)); @@ -952,6 +961,17 @@ static void handle_tag(const char *name, struct tag *tag) case SIGN_STRIP: message_size = sig_offset; break; + + /* Aborting modes */ + case SIGN_ABORT: + die(_("encountered signed tag %s; use " + "--signed-tags=<mode> to handle it"), + oid_to_hex(&tag->object.oid)); + case SIGN_STRIP_IF_INVALID: + die(_("'strip-if-invalid' is not a valid mode for " + "git fast-export with --signed-tags=<mode>")); + default: + BUG("invalid signed_commit_mode value %d", signed_commit_mode); } } diff --git a/builtin/fast-import.c b/builtin/fast-import.c index 4cd0b079b6..7849005ccb 100644 --- a/builtin/fast-import.c +++ b/builtin/fast-import.c @@ -2772,7 +2772,7 @@ static void add_gpgsig_to_commit(struct strbuf *commit_data, { struct string_list siglines = STRING_LIST_INIT_NODUP; - if (!sig->hash_algo) + if (!sig || !sig->hash_algo) return; strbuf_addstr(commit_data, header); @@ -2815,6 +2815,57 @@ static void import_one_signature(struct signature_data *sig_sha1, die(_("parse_one_signature() returned unknown hash algo")); } +static void finalize_commit_buffer(struct strbuf *new_data, + struct signature_data *sig_sha1, + struct signature_data *sig_sha256, + struct strbuf *msg) +{ + add_gpgsig_to_commit(new_data, "gpgsig ", sig_sha1); + add_gpgsig_to_commit(new_data, "gpgsig-sha256 ", sig_sha256); + + strbuf_addch(new_data, '\n'); + strbuf_addbuf(new_data, msg); +} + +static void handle_strip_if_invalid(struct strbuf *new_data, + struct signature_data *sig_sha1, + struct signature_data *sig_sha256, + struct strbuf *msg) +{ + struct strbuf tmp_buf = STRBUF_INIT; + struct signature_check signature_check = { 0 }; + int ret; + + /* Check signature in a temporary commit buffer */ + strbuf_addbuf(&tmp_buf, new_data); + finalize_commit_buffer(&tmp_buf, sig_sha1, sig_sha256, msg); + ret = verify_commit_buffer(tmp_buf.buf, tmp_buf.len, &signature_check); + + if (ret) { + const char *signer = signature_check.signer ? + signature_check.signer : _("unknown"); + const char *subject; + int subject_len = find_commit_subject(msg->buf, &subject); + + if (subject_len > 100) + warning(_("stripping invalid signature for commit '%.100s...'\n" + " allegedly by %s"), subject, signer); + else if (subject_len > 0) + warning(_("stripping invalid signature for commit '%.*s'\n" + " allegedly by %s"), subject_len, subject, signer); + else + warning(_("stripping invalid signature for commit\n" + " allegedly by %s"), signer); + + finalize_commit_buffer(new_data, NULL, NULL, msg); + } else { + strbuf_swap(new_data, &tmp_buf); + } + + signature_check_clear(&signature_check); + strbuf_release(&tmp_buf); +} + static void parse_new_commit(const char *arg) { static struct strbuf msg = STRBUF_INIT; @@ -2866,6 +2917,7 @@ static void parse_new_commit(const char *arg) warning(_("importing a commit signature verbatim")); /* fallthru */ case SIGN_VERBATIM: + case SIGN_STRIP_IF_INVALID: import_one_signature(&sig_sha1, &sig_sha256, v); break; @@ -2950,11 +3002,12 @@ static void parse_new_commit(const char *arg) "encoding %s\n", encoding); - add_gpgsig_to_commit(&new_data, "gpgsig ", &sig_sha1); - add_gpgsig_to_commit(&new_data, "gpgsig-sha256 ", &sig_sha256); + if (signed_commit_mode == SIGN_STRIP_IF_INVALID && + (sig_sha1.hash_algo || sig_sha256.hash_algo)) + handle_strip_if_invalid(&new_data, &sig_sha1, &sig_sha256, &msg); + else + finalize_commit_buffer(&new_data, &sig_sha1, &sig_sha256, &msg); - strbuf_addch(&new_data, '\n'); - strbuf_addbuf(&new_data, &msg); free(author); free(committer); free(encoding); @@ -2975,9 +3028,6 @@ static void handle_tag_signature(struct strbuf *msg, const char *name) switch (signed_tag_mode) { /* First, modes that don't change anything */ - case SIGN_ABORT: - die(_("encountered signed tag; use " - "--signed-tags=<mode> to handle it")); case SIGN_WARN_VERBATIM: warning(_("importing a tag signature verbatim for tag '%s'"), name); /* fallthru */ @@ -2994,7 +3044,13 @@ static void handle_tag_signature(struct strbuf *msg, const char *name) strbuf_setlen(msg, sig_offset); break; - /* Third, BUG */ + /* Third, aborting modes */ + case SIGN_ABORT: + die(_("encountered signed tag; use " + "--signed-tags=<mode> to handle it")); + case SIGN_STRIP_IF_INVALID: + die(_("'strip-if-invalid' is not a valid mode for " + "git fast-import with --signed-tags=<mode>")); default: BUG("invalid signed_tag_mode value %d from tag '%s'", signed_tag_mode, name); diff --git a/builtin/fetch.c b/builtin/fetch.c index d1c475a22c..288d3772ea 100644 --- a/builtin/fetch.c +++ b/builtin/fetch.c @@ -1681,6 +1681,36 @@ static void ref_transaction_rejection_handler(const char *refname, *data->retcode = 1; } +/* + * Commit the reference transaction. If it isn't an atomic transaction, handle + * rejected updates as part of using batched updates. + */ +static int commit_ref_transaction(struct ref_transaction **transaction, + bool is_atomic, const char *remote_name, + struct strbuf *err) +{ + int retcode = ref_transaction_commit(*transaction, err); + if (retcode) + goto out; + + if (!is_atomic) { + struct ref_rejection_data data = { + .conflict_msg_shown = 0, + .remote_name = remote_name, + .retcode = &retcode, + }; + + ref_transaction_for_each_rejected_update(*transaction, + ref_transaction_rejection_handler, + &data); + } + +out: + ref_transaction_free(*transaction); + *transaction = NULL; + return retcode; +} + static int do_fetch(struct transport *transport, struct refspec *rs, const struct fetch_config *config) @@ -1853,33 +1883,14 @@ static int do_fetch(struct transport *transport, if (retcode) goto cleanup; - retcode = ref_transaction_commit(transaction, &err); - if (retcode) { - /* - * Explicitly handle transaction cleanup to avoid - * aborting an already closed transaction. - */ - ref_transaction_free(transaction); - transaction = NULL; + retcode = commit_ref_transaction(&transaction, atomic_fetch, + transport->remote->name, &err); + /* + * With '--atomic', bail out if the transaction fails. Without '--atomic', + * continue to fetch head and perform other post-fetch operations. + */ + if (retcode && atomic_fetch) goto cleanup; - } - - if (!atomic_fetch) { - struct ref_rejection_data data = { - .retcode = &retcode, - .conflict_msg_shown = 0, - .remote_name = transport->remote->name, - }; - - ref_transaction_for_each_rejected_update(transaction, - ref_transaction_rejection_handler, - &data); - if (retcode) { - ref_transaction_free(transaction); - transaction = NULL; - goto cleanup; - } - } commit_fetch_head(&fetch_head); @@ -1945,6 +1956,14 @@ static int do_fetch(struct transport *transport, } cleanup: + /* + * When using batched updates, we want to commit the non-rejected + * updates and also handle the rejections. + */ + if (retcode && !atomic_fetch && transaction) + commit_ref_transaction(&transaction, false, + transport->remote->name, &err); + if (retcode) { if (err.len) { error("%s", err.buf); diff --git a/builtin/fsck.c b/builtin/fsck.c index c489582faa..4979bc795e 100644 --- a/builtin/fsck.c +++ b/builtin/fsck.c @@ -13,11 +13,11 @@ #include "fsck.h" #include "parse-options.h" #include "progress.h" -#include "streaming.h" #include "packfile.h" #include "object-file.h" #include "object-name.h" #include "odb.h" +#include "odb/streaming.h" #include "path.h" #include "read-cache-ll.h" #include "replace-object.h" @@ -340,7 +340,8 @@ static void check_unreachable_object(struct object *obj) } f = xfopen(filename, "w"); if (obj->type == OBJ_BLOB) { - if (stream_blob_to_fd(fileno(f), &obj->oid, NULL, 1)) + if (odb_stream_blob_to_fd(the_repository->objects, fileno(f), + &obj->oid, NULL, 1)) die_errno(_("could not write '%s'"), filename); } else fprintf(f, "%s\n", describe_object(&obj->oid)); diff --git a/builtin/gc.c b/builtin/gc.c index e9a76243aa..92c6e7b954 100644 --- a/builtin/gc.c +++ b/builtin/gc.c @@ -1063,7 +1063,7 @@ int cmd_gc(int argc, report_garbage = report_pack_garbage; odb_reprepare(the_repository->objects); if (pack_garbage.nr > 0) { - close_object_store(the_repository->objects); + odb_close(the_repository->objects); clean_pack_garbage(); } diff --git a/builtin/history.c b/builtin/history.c new file mode 100644 index 0000000000..8dcb9a6046 --- /dev/null +++ b/builtin/history.c @@ -0,0 +1,427 @@ +#define USE_THE_REPOSITORY_VARIABLE + +#include "builtin.h" +#include "commit.h" +#include "commit-reach.h" +#include "config.h" +#include "editor.h" +#include "environment.h" +#include "gettext.h" +#include "hex.h" +#include "parse-options.h" +#include "refs.h" +#include "replay.h" +#include "revision.h" +#include "sequencer.h" +#include "strvec.h" +#include "tree.h" +#include "wt-status.h" + +#define GIT_HISTORY_REWORD_USAGE \ + N_("git history reword <commit> [--ref-action=(branches|head|print)]") + +static void change_data_free(void *util, const char *str UNUSED) +{ + struct wt_status_change_data *d = util; + free(d->rename_source); + free(d); +} + +static int fill_commit_message(struct repository *repo, + const struct object_id *old_tree, + const struct object_id *new_tree, + const char *default_message, + const char *action, + struct strbuf *out) +{ + const char *path = git_path_commit_editmsg(); + const char *hint = + _("Please enter the commit message for the %s changes." + " Lines starting\nwith '%s' will be ignored, and an" + " empty message aborts the commit.\n"); + struct wt_status s; + + strbuf_addstr(out, default_message); + strbuf_addch(out, '\n'); + strbuf_commented_addf(out, comment_line_str, hint, action, comment_line_str); + write_file_buf(path, out->buf, out->len); + + wt_status_prepare(repo, &s); + FREE_AND_NULL(s.branch); + s.ahead_behind_flags = AHEAD_BEHIND_QUICK; + s.commit_template = 1; + s.colopts = 0; + s.display_comment_prefix = 1; + s.hints = 0; + s.use_color = 0; + s.whence = FROM_COMMIT; + s.committable = 1; + + s.fp = fopen(git_path_commit_editmsg(), "a"); + if (!s.fp) + return error_errno(_("could not open '%s'"), git_path_commit_editmsg()); + + wt_status_collect_changes_trees(&s, old_tree, new_tree); + wt_status_print(&s); + wt_status_collect_free_buffers(&s); + string_list_clear_func(&s.change, change_data_free); + + strbuf_reset(out); + if (launch_editor(path, out, NULL)) { + fprintf(stderr, _("Aborting commit as launching the editor failed.\n")); + return -1; + } + strbuf_stripspace(out, comment_line_str); + + cleanup_message(out, COMMIT_MSG_CLEANUP_ALL, 0); + + if (!out->len) { + fprintf(stderr, _("Aborting commit due to empty commit message.\n")); + return -1; + } + + return 0; +} + +static int commit_tree_with_edited_message(struct repository *repo, + const char *action, + struct commit *original, + struct commit **out) +{ + const char *exclude_gpgsig[] = { + /* We reencode the message, so the encoding needs to be stripped. */ + "encoding", + /* We need to strip signatures as those will become invalid. */ + "gpgsig", + "gpgsig-sha256", + NULL, + }; + const char *original_message, *original_body, *ptr; + struct commit_extra_header *original_extra_headers = NULL; + struct strbuf commit_message = STRBUF_INIT; + struct object_id rewritten_commit_oid; + struct object_id original_tree_oid; + struct object_id parent_tree_oid; + char *original_author = NULL; + struct commit *parent; + size_t len; + int ret; + + original_tree_oid = repo_get_commit_tree(repo, original)->object.oid; + + parent = original->parents ? original->parents->item : NULL; + if (parent) { + if (repo_parse_commit(repo, parent)) { + ret = error(_("unable to parse parent commit %s"), + oid_to_hex(&parent->object.oid)); + goto out; + } + + parent_tree_oid = repo_get_commit_tree(repo, parent)->object.oid; + } else { + oidcpy(&parent_tree_oid, repo->hash_algo->empty_tree); + } + + /* We retain authorship of the original commit. */ + original_message = repo_logmsg_reencode(repo, original, NULL, NULL); + ptr = find_commit_header(original_message, "author", &len); + if (ptr) + original_author = xmemdupz(ptr, len); + find_commit_subject(original_message, &original_body); + + ret = fill_commit_message(repo, &parent_tree_oid, &original_tree_oid, + original_body, action, &commit_message); + if (ret < 0) + goto out; + + original_extra_headers = read_commit_extra_headers(original, exclude_gpgsig); + + ret = commit_tree_extended(commit_message.buf, commit_message.len, &original_tree_oid, + original->parents, &rewritten_commit_oid, original_author, + NULL, NULL, original_extra_headers); + if (ret < 0) + goto out; + + *out = lookup_commit_or_die(&rewritten_commit_oid, "rewritten commit"); + +out: + free_commit_extra_headers(original_extra_headers); + strbuf_release(&commit_message); + free(original_author); + return ret; +} + +enum ref_action { + REF_ACTION_DEFAULT, + REF_ACTION_BRANCHES, + REF_ACTION_HEAD, + REF_ACTION_PRINT, +}; + +static int parse_ref_action(const struct option *opt, const char *value, int unset) +{ + enum ref_action *action = opt->value; + + BUG_ON_OPT_NEG_NOARG(unset, value); + if (!strcmp(value, "branches")) { + *action = REF_ACTION_BRANCHES; + } else if (!strcmp(value, "head")) { + *action = REF_ACTION_HEAD; + } else if (!strcmp(value, "print")) { + *action = REF_ACTION_PRINT; + } else { + return error(_("%s expects one of 'branches', 'head' or 'print'"), + opt->long_name); + } + + return 0; +} + +static int handle_reference_updates(enum ref_action action, + struct repository *repo, + struct commit *original, + struct commit *rewritten, + const char *reflog_msg) +{ + const struct name_decoration *decoration; + struct replay_revisions_options opts = { 0 }; + struct replay_result result = { 0 }; + struct ref_transaction *transaction = NULL; + struct strvec args = STRVEC_INIT; + struct strbuf err = STRBUF_INIT; + struct commit *head = NULL; + struct rev_info revs; + char hex[GIT_MAX_HEXSZ + 1]; + bool detached_head; + int head_flags = 0; + int ret; + + refs_read_ref_full(get_main_ref_store(repo), "HEAD", + RESOLVE_REF_NO_RECURSE, NULL, &head_flags); + detached_head = !(head_flags & REF_ISSYMREF); + + repo_init_revisions(repo, &revs, NULL); + strvec_push(&args, "ignored"); + strvec_push(&args, "--reverse"); + strvec_push(&args, "--topo-order"); + strvec_push(&args, "--full-history"); + + /* We only want to see commits that are descendants of the old commit. */ + strvec_pushf(&args, "--ancestry-path=%s", + oid_to_hex(&original->object.oid)); + + /* + * Ancestry path may also show ancestors of the old commit, but we + * don't want to see those, either. + */ + strvec_pushf(&args, "^%s", oid_to_hex(&original->object.oid)); + + /* + * When we're asked to update HEAD we need to verify that the commit + * that we want to rewrite is actually an ancestor of it and, if so, + * update it. Otherwise we'll update (or print) all descendant + * branches. + */ + if (action == REF_ACTION_HEAD) { + struct commit_list *from_list = NULL; + + head = lookup_commit_reference_by_name("HEAD"); + if (!head) { + ret = error(_("cannot look up HEAD")); + goto out; + } + + commit_list_insert(original, &from_list); + ret = repo_is_descendant_of(repo, head, from_list); + free_commit_list(from_list); + + if (ret < 0) { + ret = error(_("cannot determine descendance")); + goto out; + } else if (!ret) { + ret = error(_("rewritten commit must be an ancestor " + "of HEAD when using --ref-action=head")); + goto out; + } + + strvec_push(&args, "HEAD"); + } else { + strvec_push(&args, "--branches"); + strvec_push(&args, "HEAD"); + } + + setup_revisions_from_strvec(&args, &revs, NULL); + if (args.nr != 1) + BUG("revisions were set up with invalid argument"); + + opts.onto = oid_to_hex_r(hex, &rewritten->object.oid); + + ret = replay_revisions(&revs, &opts, &result); + if (ret) + goto out; + + switch (action) { + case REF_ACTION_BRANCHES: + case REF_ACTION_HEAD: + transaction = ref_store_transaction_begin(get_main_ref_store(repo), 0, &err); + if (!transaction) { + ret = error(_("failed to begin ref transaction: %s"), err.buf); + goto out; + } + + for (size_t i = 0; i < result.updates_nr; i++) { + ret = ref_transaction_update(transaction, + result.updates[i].refname, + &result.updates[i].new_oid, + &result.updates[i].old_oid, + NULL, NULL, 0, reflog_msg, &err); + if (ret) { + ret = error(_("failed to update ref '%s': %s"), + result.updates[i].refname, err.buf); + goto out; + } + } + + /* + * `replay_revisions()` only updates references that are + * ancestors of `rewritten`, so we need to manually + * handle updating references that point to `original`. + */ + for (decoration = get_name_decoration(&original->object); + decoration; + decoration = decoration->next) + { + if (decoration->type != DECORATION_REF_LOCAL && + decoration->type != DECORATION_REF_HEAD) + continue; + + if (action == REF_ACTION_HEAD && + decoration->type != DECORATION_REF_HEAD) + continue; + + /* + * We only need to update HEAD separately in case it's + * detached. If it's not we'd already update the branch + * it is pointing to. + */ + if (action == REF_ACTION_BRANCHES && + decoration->type == DECORATION_REF_HEAD && + !detached_head) + continue; + + ret = ref_transaction_update(transaction, + decoration->name, + &rewritten->object.oid, + &original->object.oid, + NULL, NULL, 0, reflog_msg, &err); + if (ret) { + ret = error(_("failed to update ref '%s': %s"), + decoration->name, err.buf); + goto out; + } + } + + if (ref_transaction_commit(transaction, &err)) { + ret = error(_("failed to commit ref transaction: %s"), err.buf); + goto out; + } + + break; + case REF_ACTION_PRINT: + for (size_t i = 0; i < result.updates_nr; i++) + printf("update %s %s %s\n", + result.updates[i].refname, + oid_to_hex(&result.updates[i].new_oid), + oid_to_hex(&result.updates[i].old_oid)); + break; + default: + BUG("unsupported ref action %d", action); + } + + ret = 0; + +out: + ref_transaction_free(transaction); + replay_result_release(&result); + release_revisions(&revs); + strbuf_release(&err); + strvec_clear(&args); + return ret; +} + +static int cmd_history_reword(int argc, + const char **argv, + const char *prefix, + struct repository *repo) +{ + const char * const usage[] = { + GIT_HISTORY_REWORD_USAGE, + NULL, + }; + enum ref_action action = REF_ACTION_DEFAULT; + struct option options[] = { + OPT_CALLBACK_F(0, "ref-action", &action, N_("<action>"), + N_("control ref update behavior (branches|head|print)"), + PARSE_OPT_NONEG, parse_ref_action), + OPT_END(), + }; + struct strbuf reflog_msg = STRBUF_INIT; + struct commit *original, *rewritten; + int ret; + + argc = parse_options(argc, argv, prefix, options, usage, 0); + if (argc != 1) { + ret = error(_("command expects a single revision")); + goto out; + } + repo_config(repo, git_default_config, NULL); + + if (action == REF_ACTION_DEFAULT) + action = REF_ACTION_BRANCHES; + + original = lookup_commit_reference_by_name(argv[0]); + if (!original) { + ret = error(_("commit cannot be found: %s"), argv[0]); + goto out; + } + + ret = commit_tree_with_edited_message(repo, "reworded", original, &rewritten); + if (ret < 0) { + ret = error(_("failed writing reworded commit")); + goto out; + } + + strbuf_addf(&reflog_msg, "reword: updating %s", argv[0]); + + ret = handle_reference_updates(action, repo, original, rewritten, + reflog_msg.buf); + if (ret < 0) { + ret = error(_("failed replaying descendants")); + goto out; + } + + ret = 0; + +out: + strbuf_release(&reflog_msg); + return ret; +} + +int cmd_history(int argc, + const char **argv, + const char *prefix, + struct repository *repo) +{ + const char * const usage[] = { + GIT_HISTORY_REWORD_USAGE, + NULL, + }; + parse_opt_subcommand_fn *fn = NULL; + struct option options[] = { + OPT_SUBCOMMAND("reword", &fn, cmd_history_reword), + OPT_END(), + }; + + argc = parse_options(argc, argv, prefix, options, usage, 0); + return fn(argc, argv, prefix, repo); +} diff --git a/builtin/hook.c b/builtin/hook.c index 7afec380d2..73e7b8c2e8 100644 --- a/builtin/hook.c +++ b/builtin/hook.c @@ -44,6 +44,12 @@ static int run(int argc, const char **argv, const char *prefix, goto usage; /* + * All current "hook run" use-cases require ungrouped child output. + * If this changes, a hook run argument can be added to toggle it. + */ + opt.ungroup = 1; + + /* * Having a -- for "run" when providing <hook-args> is * mandatory. */ diff --git a/builtin/index-pack.c b/builtin/index-pack.c index 2b78ba7fe4..a7e901e49c 100644 --- a/builtin/index-pack.c +++ b/builtin/index-pack.c @@ -16,12 +16,12 @@ #include "progress.h" #include "fsck.h" #include "strbuf.h" -#include "streaming.h" #include "thread-utils.h" #include "packfile.h" #include "pack-revindex.h" #include "object-file.h" #include "odb.h" +#include "odb/streaming.h" #include "oid-array.h" #include "oidset.h" #include "path.h" @@ -762,7 +762,7 @@ static void find_ref_delta_children(const struct object_id *oid, struct compare_data { struct object_entry *entry; - struct git_istream *st; + struct odb_read_stream *st; unsigned char *buf; unsigned long buf_size; }; @@ -779,7 +779,7 @@ static int compare_objects(const unsigned char *buf, unsigned long size, } while (size) { - ssize_t len = read_istream(data->st, data->buf, size); + ssize_t len = odb_read_stream_read(data->st, data->buf, size); if (len == 0) die(_("SHA1 COLLISION FOUND WITH %s !"), oid_to_hex(&data->entry->idx.oid)); @@ -798,8 +798,6 @@ static int compare_objects(const unsigned char *buf, unsigned long size, static int check_collison(struct object_entry *entry) { struct compare_data data; - enum object_type type; - unsigned long size; if (entry->size <= repo_settings_get_big_file_threshold(the_repository) || entry->type != OBJ_BLOB) @@ -807,15 +805,14 @@ static int check_collison(struct object_entry *entry) memset(&data, 0, sizeof(data)); data.entry = entry; - data.st = open_istream(the_repository, &entry->idx.oid, &type, &size, - NULL); + data.st = odb_read_stream_open(the_repository->objects, &entry->idx.oid, NULL); if (!data.st) return -1; - if (size != entry->size || type != entry->type) + if (data.st->size != entry->size || data.st->type != entry->type) die(_("SHA1 COLLISION FOUND WITH %s !"), oid_to_hex(&entry->idx.oid)); unpack_data(entry, compare_objects, &data); - close_istream(data.st); + odb_read_stream_close(data.st); free(data.buf); return 0; } @@ -1640,7 +1637,7 @@ static void final(const char *final_pack_name, const char *curr_pack_name, rename_tmp_packfile(&final_index_name, curr_index_name, &index_name, hash, "idx", 1); - if (do_fsck_object) + if (do_fsck_object && startup_info->have_repository) packfile_store_load_pack(the_repository->objects->packfiles, final_index_name, 0); @@ -2110,8 +2107,23 @@ int cmd_index_pack(int argc, else close(input_fd); - if (do_fsck_object && fsck_finish(&fsck_options)) - die(_("fsck error in pack objects")); + if (do_fsck_object) { + /* + * We cannot perform queued consistency checks when running + * outside of a repository because those require us to read + * from the object database, which is uninitialized. + * + * TODO: we may eventually set up an in-memory object database, + * which would allow us to perform these queued checks. + */ + if (!startup_info->have_repository && + fsck_has_queued_checks(&fsck_options)) + die(_("cannot perform queued object checks outside " + "of a repository")); + + if (fsck_finish(&fsck_options)) + die(_("fsck error in pack objects")); + } free(opts.anomaly); free(objects); diff --git a/builtin/last-modified.c b/builtin/last-modified.c index b0ecbdc540..c80f0535f6 100644 --- a/builtin/last-modified.c +++ b/builtin/last-modified.c @@ -327,7 +327,7 @@ static void process_parent(struct last_modified *lm, if (!(parent->object.flags & PARENT1)) active_paths_free(lm, parent); - memset(lm->scratch->words, 0x0, lm->scratch->word_alloc); + MEMZERO_ARRAY(lm->scratch->words, lm->scratch->word_alloc); diff_queue_clear(&diff_queued_diff); } @@ -525,7 +525,8 @@ int cmd_last_modified(int argc, const char **argv, const char *prefix, argc = parse_options(argc, argv, prefix, last_modified_options, last_modified_usage, - PARSE_OPT_KEEP_ARGV0 | PARSE_OPT_KEEP_UNKNOWN_OPT); + PARSE_OPT_KEEP_ARGV0 | PARSE_OPT_KEEP_UNKNOWN_OPT | + PARSE_OPT_KEEP_DASHDASH); repo_config(repo, git_default_config, NULL); diff --git a/builtin/log.c b/builtin/log.c index c8319b8af3..d4cf9c59c8 100644 --- a/builtin/log.c +++ b/builtin/log.c @@ -16,6 +16,7 @@ #include "refs.h" #include "object-name.h" #include "odb.h" +#include "odb/streaming.h" #include "pager.h" #include "color.h" #include "commit.h" @@ -35,7 +36,6 @@ #include "parse-options.h" #include "line-log.h" #include "branch.h" -#include "streaming.h" #include "version.h" #include "mailmap.h" #include "progress.h" @@ -584,7 +584,7 @@ static int show_blob_object(const struct object_id *oid, struct rev_info *rev, c fflush(rev->diffopt.file); if (!rev->diffopt.flags.textconv_set_via_cmdline || !rev->diffopt.flags.allow_textconv) - return stream_blob_to_fd(1, oid, NULL, 0); + return odb_stream_blob_to_fd(the_repository->objects, 1, oid, NULL, 0); if (get_oid_with_context(the_repository, obj_name, GET_OID_RECORD_PATH, @@ -594,7 +594,7 @@ static int show_blob_object(const struct object_id *oid, struct rev_info *rev, c !textconv_object(the_repository, obj_context.path, obj_context.mode, &oidc, 1, &buf, &size)) { object_context_release(&obj_context); - return stream_blob_to_fd(1, oid, NULL, 0); + return odb_stream_blob_to_fd(the_repository->objects, 1, oid, NULL, 0); } if (!buf) diff --git a/builtin/pack-objects.c b/builtin/pack-objects.c index 7937106ec5..1ce8d6ee21 100644 --- a/builtin/pack-objects.c +++ b/builtin/pack-objects.c @@ -22,7 +22,6 @@ #include "pack-objects.h" #include "progress.h" #include "refs.h" -#include "streaming.h" #include "thread-utils.h" #include "pack-bitmap.h" #include "delta-islands.h" @@ -33,6 +32,7 @@ #include "packfile.h" #include "object-file.h" #include "odb.h" +#include "odb/streaming.h" #include "replace-object.h" #include "dir.h" #include "midx.h" @@ -404,7 +404,7 @@ static unsigned long do_compress(void **pptr, unsigned long size) return stream.total_out; } -static unsigned long write_large_blob_data(struct git_istream *st, struct hashfile *f, +static unsigned long write_large_blob_data(struct odb_read_stream *st, struct hashfile *f, const struct object_id *oid) { git_zstream stream; @@ -417,7 +417,7 @@ static unsigned long write_large_blob_data(struct git_istream *st, struct hashfi for (;;) { ssize_t readlen; int zret = Z_OK; - readlen = read_istream(st, ibuf, sizeof(ibuf)); + readlen = odb_read_stream_read(st, ibuf, sizeof(ibuf)); if (readlen == -1) die(_("unable to read %s"), oid_to_hex(oid)); @@ -513,17 +513,19 @@ static unsigned long write_no_reuse_object(struct hashfile *f, struct object_ent unsigned hdrlen; enum object_type type; void *buf; - struct git_istream *st = NULL; + struct odb_read_stream *st = NULL; const unsigned hashsz = the_hash_algo->rawsz; if (!usable_delta) { if (oe_type(entry) == OBJ_BLOB && oe_size_greater_than(&to_pack, entry, repo_settings_get_big_file_threshold(the_repository)) && - (st = open_istream(the_repository, &entry->idx.oid, &type, - &size, NULL)) != NULL) + (st = odb_read_stream_open(the_repository->objects, &entry->idx.oid, + NULL)) != NULL) { buf = NULL; - else { + type = st->type; + size = st->size; + } else { buf = odb_read_object(the_repository->objects, &entry->idx.oid, &type, &size); @@ -577,7 +579,7 @@ static unsigned long write_no_reuse_object(struct hashfile *f, struct object_ent dheader[--pos] = 128 | (--ofs & 127); if (limit && hdrlen + sizeof(dheader) - pos + datalen + hashsz >= limit) { if (st) - close_istream(st); + odb_read_stream_close(st); free(buf); return 0; } @@ -591,7 +593,7 @@ static unsigned long write_no_reuse_object(struct hashfile *f, struct object_ent */ if (limit && hdrlen + hashsz + datalen + hashsz >= limit) { if (st) - close_istream(st); + odb_read_stream_close(st); free(buf); return 0; } @@ -601,7 +603,7 @@ static unsigned long write_no_reuse_object(struct hashfile *f, struct object_ent } else { if (limit && hdrlen + datalen + hashsz >= limit) { if (st) - close_istream(st); + odb_read_stream_close(st); free(buf); return 0; } @@ -609,7 +611,7 @@ static unsigned long write_no_reuse_object(struct hashfile *f, struct object_ent } if (st) { datalen = write_large_blob_data(st, f, &entry->idx.oid); - close_istream(st); + odb_read_stream_close(st); } else { hashwrite(f, buf, datalen); free(buf); diff --git a/builtin/pull.c b/builtin/pull.c index 5ebd529620..3ff748e0b3 100644 --- a/builtin/pull.c +++ b/builtin/pull.c @@ -119,148 +119,6 @@ static int opt_show_forced_updates = -1; static const char *set_upstream; static struct strvec opt_fetch = STRVEC_INIT; -static struct option pull_options[] = { - /* Shared options */ - OPT__VERBOSITY(&opt_verbosity), - OPT_PASSTHRU(0, "progress", &opt_progress, NULL, - N_("force progress reporting"), - PARSE_OPT_NOARG), - OPT_CALLBACK_F(0, "recurse-submodules", - &recurse_submodules_cli, N_("on-demand"), - N_("control for recursive fetching of submodules"), - PARSE_OPT_OPTARG, option_fetch_parse_recurse_submodules), - - /* Options passed to git-merge or git-rebase */ - OPT_GROUP(N_("Options related to merging")), - OPT_CALLBACK_F('r', "rebase", &opt_rebase, - "(false|true|merges|interactive)", - N_("incorporate changes by rebasing rather than merging"), - PARSE_OPT_OPTARG, parse_opt_rebase), - OPT_PASSTHRU('n', NULL, &opt_diffstat, NULL, - N_("do not show a diffstat at the end of the merge"), - PARSE_OPT_NOARG | PARSE_OPT_NONEG), - OPT_PASSTHRU(0, "stat", &opt_diffstat, NULL, - N_("show a diffstat at the end of the merge"), - PARSE_OPT_NOARG), - OPT_PASSTHRU(0, "summary", &opt_diffstat, NULL, - N_("(synonym to --stat)"), - PARSE_OPT_NOARG | PARSE_OPT_HIDDEN), - OPT_PASSTHRU(0, "compact-summary", &opt_diffstat, NULL, - N_("show a compact-summary at the end of the merge"), - PARSE_OPT_NOARG), - OPT_PASSTHRU(0, "log", &opt_log, N_("n"), - N_("add (at most <n>) entries from shortlog to merge commit message"), - PARSE_OPT_OPTARG), - OPT_PASSTHRU(0, "signoff", &opt_signoff, NULL, - N_("add a Signed-off-by trailer"), - PARSE_OPT_OPTARG), - OPT_PASSTHRU(0, "squash", &opt_squash, NULL, - N_("create a single commit instead of doing a merge"), - PARSE_OPT_NOARG), - OPT_PASSTHRU(0, "commit", &opt_commit, NULL, - N_("perform a commit if the merge succeeds (default)"), - PARSE_OPT_NOARG), - OPT_PASSTHRU(0, "edit", &opt_edit, NULL, - N_("edit message before committing"), - PARSE_OPT_NOARG), - OPT_CLEANUP(&cleanup_arg), - OPT_PASSTHRU(0, "ff", &opt_ff, NULL, - N_("allow fast-forward"), - PARSE_OPT_NOARG), - OPT_PASSTHRU(0, "ff-only", &opt_ff, NULL, - N_("abort if fast-forward is not possible"), - PARSE_OPT_NOARG | PARSE_OPT_NONEG), - OPT_PASSTHRU(0, "verify", &opt_verify, NULL, - N_("control use of pre-merge-commit and commit-msg hooks"), - PARSE_OPT_NOARG), - OPT_PASSTHRU(0, "verify-signatures", &opt_verify_signatures, NULL, - N_("verify that the named commit has a valid GPG signature"), - PARSE_OPT_NOARG), - OPT_BOOL(0, "autostash", &opt_autostash, - N_("automatically stash/stash pop before and after")), - OPT_PASSTHRU_ARGV('s', "strategy", &opt_strategies, N_("strategy"), - N_("merge strategy to use"), - 0), - OPT_PASSTHRU_ARGV('X', "strategy-option", &opt_strategy_opts, - N_("option=value"), - N_("option for selected merge strategy"), - 0), - OPT_PASSTHRU('S', "gpg-sign", &opt_gpg_sign, N_("key-id"), - N_("GPG sign commit"), - PARSE_OPT_OPTARG), - OPT_SET_INT(0, "allow-unrelated-histories", - &opt_allow_unrelated_histories, - N_("allow merging unrelated histories"), 1), - - /* Options passed to git-fetch */ - OPT_GROUP(N_("Options related to fetching")), - OPT_PASSTHRU(0, "all", &opt_all, NULL, - N_("fetch from all remotes"), - PARSE_OPT_NOARG), - OPT_PASSTHRU('a', "append", &opt_append, NULL, - N_("append to .git/FETCH_HEAD instead of overwriting"), - PARSE_OPT_NOARG), - OPT_PASSTHRU(0, "upload-pack", &opt_upload_pack, N_("path"), - N_("path to upload pack on remote end"), - 0), - OPT__FORCE(&opt_force, N_("force overwrite of local branch"), 0), - OPT_PASSTHRU('t', "tags", &opt_tags, NULL, - N_("fetch all tags and associated objects"), - PARSE_OPT_NOARG), - OPT_PASSTHRU('p', "prune", &opt_prune, NULL, - N_("prune remote-tracking branches no longer on remote"), - PARSE_OPT_NOARG), - OPT_PASSTHRU('j', "jobs", &max_children, N_("n"), - N_("number of submodules pulled in parallel"), - PARSE_OPT_OPTARG), - OPT_BOOL(0, "dry-run", &opt_dry_run, - N_("dry run")), - OPT_PASSTHRU('k', "keep", &opt_keep, NULL, - N_("keep downloaded pack"), - PARSE_OPT_NOARG), - OPT_PASSTHRU(0, "depth", &opt_depth, N_("depth"), - N_("deepen history of shallow clone"), - 0), - OPT_PASSTHRU_ARGV(0, "shallow-since", &opt_fetch, N_("time"), - N_("deepen history of shallow repository based on time"), - 0), - OPT_PASSTHRU_ARGV(0, "shallow-exclude", &opt_fetch, N_("ref"), - N_("deepen history of shallow clone, excluding ref"), - 0), - OPT_PASSTHRU_ARGV(0, "deepen", &opt_fetch, N_("n"), - N_("deepen history of shallow clone"), - 0), - OPT_PASSTHRU(0, "unshallow", &opt_unshallow, NULL, - N_("convert to a complete repository"), - PARSE_OPT_NONEG | PARSE_OPT_NOARG), - OPT_PASSTHRU(0, "update-shallow", &opt_update_shallow, NULL, - N_("accept refs that update .git/shallow"), - PARSE_OPT_NOARG), - OPT_PASSTHRU(0, "refmap", &opt_refmap, N_("refmap"), - N_("specify fetch refmap"), - PARSE_OPT_NONEG), - OPT_PASSTHRU_ARGV('o', "server-option", &opt_fetch, - N_("server-specific"), - N_("option to transmit"), - 0), - OPT_PASSTHRU('4', "ipv4", &opt_ipv4, NULL, - N_("use IPv4 addresses only"), - PARSE_OPT_NOARG), - OPT_PASSTHRU('6', "ipv6", &opt_ipv6, NULL, - N_("use IPv6 addresses only"), - PARSE_OPT_NOARG), - OPT_PASSTHRU_ARGV(0, "negotiation-tip", &opt_fetch, N_("revision"), - N_("report that we have only objects reachable from this object"), - 0), - OPT_BOOL(0, "show-forced-updates", &opt_show_forced_updates, - N_("check for forced-updates on all updated branches")), - OPT_PASSTHRU(0, "set-upstream", &set_upstream, NULL, - N_("set upstream for git pull/fetch"), - PARSE_OPT_NOARG), - - OPT_END() -}; - /** * Pushes "-q" or "-v" switches into arr to match the opt_verbosity level. */ @@ -1008,6 +866,147 @@ int cmd_pull(int argc, int can_ff; int divergent; int ret; + static struct option pull_options[] = { + /* Shared options */ + OPT__VERBOSITY(&opt_verbosity), + OPT_PASSTHRU(0, "progress", &opt_progress, NULL, + N_("force progress reporting"), + PARSE_OPT_NOARG), + OPT_CALLBACK_F(0, "recurse-submodules", + &recurse_submodules_cli, N_("on-demand"), + N_("control for recursive fetching of submodules"), + PARSE_OPT_OPTARG, option_fetch_parse_recurse_submodules), + + /* Options passed to git-merge or git-rebase */ + OPT_GROUP(N_("Options related to merging")), + OPT_CALLBACK_F('r', "rebase", &opt_rebase, + "(false|true|merges|interactive)", + N_("incorporate changes by rebasing rather than merging"), + PARSE_OPT_OPTARG, parse_opt_rebase), + OPT_PASSTHRU('n', NULL, &opt_diffstat, NULL, + N_("do not show a diffstat at the end of the merge"), + PARSE_OPT_NOARG | PARSE_OPT_NONEG), + OPT_PASSTHRU(0, "stat", &opt_diffstat, NULL, + N_("show a diffstat at the end of the merge"), + PARSE_OPT_NOARG), + OPT_PASSTHRU(0, "summary", &opt_diffstat, NULL, + N_("(synonym to --stat)"), + PARSE_OPT_NOARG | PARSE_OPT_HIDDEN), + OPT_PASSTHRU(0, "compact-summary", &opt_diffstat, NULL, + N_("show a compact-summary at the end of the merge"), + PARSE_OPT_NOARG), + OPT_PASSTHRU(0, "log", &opt_log, N_("n"), + N_("add (at most <n>) entries from shortlog to merge commit message"), + PARSE_OPT_OPTARG), + OPT_PASSTHRU(0, "signoff", &opt_signoff, NULL, + N_("add a Signed-off-by trailer"), + PARSE_OPT_OPTARG), + OPT_PASSTHRU(0, "squash", &opt_squash, NULL, + N_("create a single commit instead of doing a merge"), + PARSE_OPT_NOARG), + OPT_PASSTHRU(0, "commit", &opt_commit, NULL, + N_("perform a commit if the merge succeeds (default)"), + PARSE_OPT_NOARG), + OPT_PASSTHRU(0, "edit", &opt_edit, NULL, + N_("edit message before committing"), + PARSE_OPT_NOARG), + OPT_CLEANUP(&cleanup_arg), + OPT_PASSTHRU(0, "ff", &opt_ff, NULL, + N_("allow fast-forward"), + PARSE_OPT_NOARG), + OPT_PASSTHRU(0, "ff-only", &opt_ff, NULL, + N_("abort if fast-forward is not possible"), + PARSE_OPT_NOARG | PARSE_OPT_NONEG), + OPT_PASSTHRU(0, "verify", &opt_verify, NULL, + N_("control use of pre-merge-commit and commit-msg hooks"), + PARSE_OPT_NOARG), + OPT_PASSTHRU(0, "verify-signatures", &opt_verify_signatures, NULL, + N_("verify that the named commit has a valid GPG signature"), + PARSE_OPT_NOARG), + OPT_BOOL(0, "autostash", &opt_autostash, + N_("automatically stash/stash pop before and after")), + OPT_PASSTHRU_ARGV('s', "strategy", &opt_strategies, N_("strategy"), + N_("merge strategy to use"), + 0), + OPT_PASSTHRU_ARGV('X', "strategy-option", &opt_strategy_opts, + N_("option=value"), + N_("option for selected merge strategy"), + 0), + OPT_PASSTHRU('S', "gpg-sign", &opt_gpg_sign, N_("key-id"), + N_("GPG sign commit"), + PARSE_OPT_OPTARG), + OPT_SET_INT(0, "allow-unrelated-histories", + &opt_allow_unrelated_histories, + N_("allow merging unrelated histories"), 1), + + /* Options passed to git-fetch */ + OPT_GROUP(N_("Options related to fetching")), + OPT_PASSTHRU(0, "all", &opt_all, NULL, + N_("fetch from all remotes"), + PARSE_OPT_NOARG), + OPT_PASSTHRU('a', "append", &opt_append, NULL, + N_("append to .git/FETCH_HEAD instead of overwriting"), + PARSE_OPT_NOARG), + OPT_PASSTHRU(0, "upload-pack", &opt_upload_pack, N_("path"), + N_("path to upload pack on remote end"), + 0), + OPT__FORCE(&opt_force, N_("force overwrite of local branch"), 0), + OPT_PASSTHRU('t', "tags", &opt_tags, NULL, + N_("fetch all tags and associated objects"), + PARSE_OPT_NOARG), + OPT_PASSTHRU('p', "prune", &opt_prune, NULL, + N_("prune remote-tracking branches no longer on remote"), + PARSE_OPT_NOARG), + OPT_PASSTHRU('j', "jobs", &max_children, N_("n"), + N_("number of submodules pulled in parallel"), + PARSE_OPT_OPTARG), + OPT_BOOL(0, "dry-run", &opt_dry_run, + N_("dry run")), + OPT_PASSTHRU('k', "keep", &opt_keep, NULL, + N_("keep downloaded pack"), + PARSE_OPT_NOARG), + OPT_PASSTHRU(0, "depth", &opt_depth, N_("depth"), + N_("deepen history of shallow clone"), + 0), + OPT_PASSTHRU_ARGV(0, "shallow-since", &opt_fetch, N_("time"), + N_("deepen history of shallow repository based on time"), + 0), + OPT_PASSTHRU_ARGV(0, "shallow-exclude", &opt_fetch, N_("ref"), + N_("deepen history of shallow clone, excluding ref"), + 0), + OPT_PASSTHRU_ARGV(0, "deepen", &opt_fetch, N_("n"), + N_("deepen history of shallow clone"), + 0), + OPT_PASSTHRU(0, "unshallow", &opt_unshallow, NULL, + N_("convert to a complete repository"), + PARSE_OPT_NONEG | PARSE_OPT_NOARG), + OPT_PASSTHRU(0, "update-shallow", &opt_update_shallow, NULL, + N_("accept refs that update .git/shallow"), + PARSE_OPT_NOARG), + OPT_PASSTHRU(0, "refmap", &opt_refmap, N_("refmap"), + N_("specify fetch refmap"), + PARSE_OPT_NONEG), + OPT_PASSTHRU_ARGV('o', "server-option", &opt_fetch, + N_("server-specific"), + N_("option to transmit"), + 0), + OPT_PASSTHRU('4', "ipv4", &opt_ipv4, NULL, + N_("use IPv4 addresses only"), + PARSE_OPT_NOARG), + OPT_PASSTHRU('6', "ipv6", &opt_ipv6, NULL, + N_("use IPv6 addresses only"), + PARSE_OPT_NOARG), + OPT_PASSTHRU_ARGV(0, "negotiation-tip", &opt_fetch, N_("revision"), + N_("report that we have only objects reachable from this object"), + 0), + OPT_BOOL(0, "show-forced-updates", &opt_show_forced_updates, + N_("check for forced-updates on all updated branches")), + OPT_PASSTHRU(0, "set-upstream", &set_upstream, NULL, + N_("set upstream for git pull/fetch"), + PARSE_OPT_NOARG), + + OPT_END() + }; if (!getenv("GIT_REFLOG_ACTION")) set_reflog_message(argc, argv); diff --git a/builtin/receive-pack.c b/builtin/receive-pack.c index e8ee0e7321..ef1f77be8c 100644 --- a/builtin/receive-pack.c +++ b/builtin/receive-pack.c @@ -34,7 +34,6 @@ #include "object-file.h" #include "object-name.h" #include "odb.h" -#include "path.h" #include "protocol.h" #include "commit-reach.h" #include "server-info.h" @@ -42,6 +41,7 @@ #include "trace2.h" #include "worktree.h" #include "shallow.h" +#include "setup.h" #include "parse-options.h" static const char * const receive_pack_usage[] = { @@ -177,8 +177,9 @@ static int receive_pack_config(const char *var, const char *value, if (git_config_pathname(&path, var, value)) return -1; - strbuf_addf(&fsck_msg_types, "%cskiplist=%s", - fsck_msg_types.len ? ',' : '=', path); + if (path) + strbuf_addf(&fsck_msg_types, "%cskiplist=%s", + fsck_msg_types.len ? ',' : '=', path); free(path); return 0; } @@ -748,7 +749,7 @@ static int check_cert_push_options(const struct string_list *push_options) return retval; } -static void prepare_push_cert_sha1(struct child_process *proc) +static void prepare_push_cert_sha1(struct run_hooks_opt *opt) { static int already_done; @@ -774,23 +775,23 @@ static void prepare_push_cert_sha1(struct child_process *proc) nonce_status = check_nonce(sigcheck.payload); } if (!is_null_oid(&push_cert_oid)) { - strvec_pushf(&proc->env, "GIT_PUSH_CERT=%s", + strvec_pushf(&opt->env, "GIT_PUSH_CERT=%s", oid_to_hex(&push_cert_oid)); - strvec_pushf(&proc->env, "GIT_PUSH_CERT_SIGNER=%s", + strvec_pushf(&opt->env, "GIT_PUSH_CERT_SIGNER=%s", sigcheck.signer ? sigcheck.signer : ""); - strvec_pushf(&proc->env, "GIT_PUSH_CERT_KEY=%s", + strvec_pushf(&opt->env, "GIT_PUSH_CERT_KEY=%s", sigcheck.key ? sigcheck.key : ""); - strvec_pushf(&proc->env, "GIT_PUSH_CERT_STATUS=%c", + strvec_pushf(&opt->env, "GIT_PUSH_CERT_STATUS=%c", sigcheck.result); if (push_cert_nonce) { - strvec_pushf(&proc->env, + strvec_pushf(&opt->env, "GIT_PUSH_CERT_NONCE=%s", push_cert_nonce); - strvec_pushf(&proc->env, + strvec_pushf(&opt->env, "GIT_PUSH_CERT_NONCE_STATUS=%s", nonce_status); if (nonce_status == NONCE_SLOP) - strvec_pushf(&proc->env, + strvec_pushf(&opt->env, "GIT_PUSH_CERT_NONCE_SLOP=%ld", nonce_stamp_slop); } @@ -802,119 +803,74 @@ struct receive_hook_feed_state { struct ref_push_report *report; int skip_broken; struct strbuf buf; - const struct string_list *push_options; }; -typedef int (*feed_fn)(void *, const char **, size_t *); -static int run_and_feed_hook(const char *hook_name, feed_fn feed, - struct receive_hook_feed_state *feed_state) +static int feed_receive_hook_cb(int hook_stdin_fd, void *pp_cb UNUSED, void *pp_task_cb) { - struct child_process proc = CHILD_PROCESS_INIT; - struct async muxer; - int code; - const char *hook_path = find_hook(the_repository, hook_name); + struct receive_hook_feed_state *state = pp_task_cb; + struct command *cmd = state->cmd; + unsigned int lines_batch_size = 500; - if (!hook_path) - return 0; + strbuf_reset(&state->buf); - strvec_push(&proc.args, hook_path); - proc.in = -1; - proc.stdout_to_stderr = 1; - proc.trace2_hook_name = hook_name; + /* batch lines to avoid going through run-command's poll loop for each line */ + for (unsigned int i = 0; i < lines_batch_size; i++) { + while (cmd && + state->skip_broken && (cmd->error_string || cmd->did_not_exist)) + cmd = cmd->next; - if (feed_state->push_options) { - size_t i; - for (i = 0; i < feed_state->push_options->nr; i++) - strvec_pushf(&proc.env, - "GIT_PUSH_OPTION_%"PRIuMAX"=%s", - (uintmax_t)i, - feed_state->push_options->items[i].string); - strvec_pushf(&proc.env, "GIT_PUSH_OPTION_COUNT=%"PRIuMAX"", - (uintmax_t)feed_state->push_options->nr); - } else - strvec_pushf(&proc.env, "GIT_PUSH_OPTION_COUNT"); + if (!cmd) + break; /* no more commands left */ - if (tmp_objdir) - strvec_pushv(&proc.env, tmp_objdir_env(tmp_objdir)); + if (!state->report) + state->report = cmd->report; - if (use_sideband) { - memset(&muxer, 0, sizeof(muxer)); - muxer.proc = copy_to_sideband; - muxer.in = -1; - code = start_async(&muxer); - if (code) - return code; - proc.err = muxer.in; - } + if (state->report) { + struct object_id *old_oid; + struct object_id *new_oid; + const char *ref_name; - prepare_push_cert_sha1(&proc); + old_oid = state->report->old_oid ? state->report->old_oid : &cmd->old_oid; + new_oid = state->report->new_oid ? state->report->new_oid : &cmd->new_oid; + ref_name = state->report->ref_name ? state->report->ref_name : cmd->ref_name; - code = start_command(&proc); - if (code) { - if (use_sideband) - finish_async(&muxer); - return code; + strbuf_addf(&state->buf, "%s %s %s\n", + oid_to_hex(old_oid), oid_to_hex(new_oid), + ref_name); + + state->report = state->report->next; + if (!state->report) + cmd = cmd->next; + } else { + strbuf_addf(&state->buf, "%s %s %s\n", + oid_to_hex(&cmd->old_oid), oid_to_hex(&cmd->new_oid), + cmd->ref_name); + cmd = cmd->next; + } } - sigchain_push(SIGPIPE, SIG_IGN); + state->cmd = cmd; - while (1) { - const char *buf; - size_t n; - if (feed(feed_state, &buf, &n)) - break; - if (write_in_full(proc.in, buf, n) < 0) - break; + if (state->buf.len > 0) { + int ret = write_in_full(hook_stdin_fd, state->buf.buf, state->buf.len); + if (ret < 0) { + if (errno == EPIPE) + return 1; /* child closed pipe */ + return ret; + } } - close(proc.in); - if (use_sideband) - finish_async(&muxer); - sigchain_pop(SIGPIPE); - - return finish_command(&proc); + return state->cmd ? 0 : 1; /* 0 = more to come, 1 = EOF */ } -static int feed_receive_hook(void *state_, const char **bufp, size_t *sizep) +static void hook_output_to_sideband(struct strbuf *output, void *cb_data UNUSED) { - struct receive_hook_feed_state *state = state_; - struct command *cmd = state->cmd; - - while (cmd && - state->skip_broken && (cmd->error_string || cmd->did_not_exist)) - cmd = cmd->next; - if (!cmd) - return -1; /* EOF */ - if (!bufp) - return 0; /* OK, can feed something. */ - strbuf_reset(&state->buf); - if (!state->report) - state->report = cmd->report; - if (state->report) { - struct object_id *old_oid; - struct object_id *new_oid; - const char *ref_name; + if (!output) + BUG("output must be non-NULL"); - old_oid = state->report->old_oid ? state->report->old_oid : &cmd->old_oid; - new_oid = state->report->new_oid ? state->report->new_oid : &cmd->new_oid; - ref_name = state->report->ref_name ? state->report->ref_name : cmd->ref_name; - strbuf_addf(&state->buf, "%s %s %s\n", - oid_to_hex(old_oid), oid_to_hex(new_oid), - ref_name); - state->report = state->report->next; - if (!state->report) - state->cmd = cmd->next; - } else { - strbuf_addf(&state->buf, "%s %s %s\n", - oid_to_hex(&cmd->old_oid), oid_to_hex(&cmd->new_oid), - cmd->ref_name); - state->cmd = cmd->next; - } - if (bufp) { - *bufp = state->buf.buf; - *sizep = state->buf.len; - } - return 0; + /* buffer might be empty for keepalives */ + if (output->len) + send_sideband(1, 2, output->buf, output->len, use_sideband); } static int run_receive_hook(struct command *commands, @@ -922,47 +878,65 @@ static int run_receive_hook(struct command *commands, int skip_broken, const struct string_list *push_options) { - struct receive_hook_feed_state state; - int status; + struct run_hooks_opt opt = RUN_HOOKS_OPT_INIT; + struct command *iter = commands; + struct receive_hook_feed_state feed_state; + int ret; - strbuf_init(&state.buf, 0); - state.cmd = commands; - state.skip_broken = skip_broken; - state.report = NULL; - if (feed_receive_hook(&state, NULL, NULL)) + /* if there are no valid commands, don't invoke the hook at all. */ + while (iter && skip_broken && (iter->error_string || iter->did_not_exist)) + iter = iter->next; + if (!iter) return 0; - state.cmd = commands; - state.push_options = push_options; - status = run_and_feed_hook(hook_name, feed_receive_hook, &state); - strbuf_release(&state.buf); - return status; + + if (push_options) { + for (int i = 0; i < push_options->nr; i++) + strvec_pushf(&opt.env, "GIT_PUSH_OPTION_%d=%s", i, + push_options->items[i].string); + strvec_pushf(&opt.env, "GIT_PUSH_OPTION_COUNT=%"PRIuMAX"", + (uintmax_t)push_options->nr); + } else { + strvec_push(&opt.env, "GIT_PUSH_OPTION_COUNT"); + } + + if (tmp_objdir) + strvec_pushv(&opt.env, tmp_objdir_env(tmp_objdir)); + + prepare_push_cert_sha1(&opt); + + /* set up sideband printer */ + if (use_sideband) + opt.consume_output = hook_output_to_sideband; + + /* set up stdin callback */ + feed_state.cmd = commands; + feed_state.skip_broken = skip_broken; + feed_state.report = NULL; + strbuf_init(&feed_state.buf, 0); + opt.feed_pipe_cb_data = &feed_state; + opt.feed_pipe = feed_receive_hook_cb; + + ret = run_hooks_opt(the_repository, hook_name, &opt); + + strbuf_release(&feed_state.buf); + + return ret; } static int run_update_hook(struct command *cmd) { - struct child_process proc = CHILD_PROCESS_INIT; - int code; - const char *hook_path = find_hook(the_repository, "update"); - - if (!hook_path) - return 0; - - strvec_push(&proc.args, hook_path); - strvec_push(&proc.args, cmd->ref_name); - strvec_push(&proc.args, oid_to_hex(&cmd->old_oid)); - strvec_push(&proc.args, oid_to_hex(&cmd->new_oid)); + struct run_hooks_opt opt = RUN_HOOKS_OPT_INIT; - proc.no_stdin = 1; - proc.stdout_to_stderr = 1; - proc.err = use_sideband ? -1 : 0; - proc.trace2_hook_name = "update"; + strvec_pushl(&opt.args, + cmd->ref_name, + oid_to_hex(&cmd->old_oid), + oid_to_hex(&cmd->new_oid), + NULL); - code = start_command(&proc); - if (code) - return code; if (use_sideband) - copy_to_sideband(proc.err, -1, NULL); - return finish_command(&proc); + opt.consume_output = hook_output_to_sideband; + + return run_hooks_opt(the_repository, "update", &opt); } static struct command *find_command_by_refname(struct command *list, @@ -1639,33 +1613,20 @@ out: static void run_update_post_hook(struct command *commands) { struct command *cmd; - struct child_process proc = CHILD_PROCESS_INIT; - const char *hook; - - hook = find_hook(the_repository, "post-update"); - if (!hook) - return; + struct run_hooks_opt opt = RUN_HOOKS_OPT_INIT; for (cmd = commands; cmd; cmd = cmd->next) { if (cmd->error_string || cmd->did_not_exist) continue; - if (!proc.args.nr) - strvec_push(&proc.args, hook); - strvec_push(&proc.args, cmd->ref_name); + strvec_push(&opt.args, cmd->ref_name); } - if (!proc.args.nr) + if (!opt.args.nr) return; - proc.no_stdin = 1; - proc.stdout_to_stderr = 1; - proc.err = use_sideband ? -1 : 0; - proc.trace2_hook_name = "post-update"; + if (use_sideband) + opt.consume_output = hook_output_to_sideband; - if (!start_command(&proc)) { - if (use_sideband) - copy_to_sideband(proc.err, -1, NULL); - finish_command(&proc); - } + run_hooks_opt(the_repository, "post-update", &opt); } static void check_aliased_update_internal(struct command *cmd, diff --git a/builtin/repack.c b/builtin/repack.c index cfdb4c0920..d9012141f6 100644 --- a/builtin/repack.c +++ b/builtin/repack.c @@ -488,7 +488,7 @@ int cmd_repack(int argc, string_list_sort(&names); - close_object_store(repo->objects); + odb_close(repo->objects); /* * Ok we have prepared all new packfiles. diff --git a/builtin/replay.c b/builtin/replay.c index 6606a2c94b..2cdde830a8 100644 --- a/builtin/replay.c +++ b/builtin/replay.c @@ -2,294 +2,22 @@ * "git replay" builtin command */ -#define USE_THE_REPOSITORY_VARIABLE -#define DISABLE_SIGN_COMPARE_WARNINGS - #include "git-compat-util.h" #include "builtin.h" #include "config.h" -#include "environment.h" #include "hex.h" -#include "lockfile.h" -#include "merge-ort.h" #include "object-name.h" #include "parse-options.h" #include "refs.h" +#include "replay.h" #include "revision.h" -#include "strmap.h" -#include <oidset.h> -#include <tree.h> enum ref_action_mode { REF_ACTION_UPDATE, REF_ACTION_PRINT, }; -static const char *short_commit_name(struct repository *repo, - struct commit *commit) -{ - return repo_find_unique_abbrev(repo, &commit->object.oid, - DEFAULT_ABBREV); -} - -static struct commit *peel_committish(struct repository *repo, const char *name) -{ - struct object *obj; - struct object_id oid; - - if (repo_get_oid(repo, name, &oid)) - return NULL; - obj = parse_object(repo, &oid); - return (struct commit *)repo_peel_to_type(repo, name, 0, obj, - OBJ_COMMIT); -} - -static char *get_author(const char *message) -{ - size_t len; - const char *a; - - a = find_commit_header(message, "author", &len); - if (a) - return xmemdupz(a, len); - - return NULL; -} - -static struct commit *create_commit(struct repository *repo, - struct tree *tree, - struct commit *based_on, - struct commit *parent) -{ - struct object_id ret; - struct object *obj = NULL; - struct commit_list *parents = NULL; - char *author; - char *sign_commit = NULL; /* FIXME: cli users might want to sign again */ - struct commit_extra_header *extra = NULL; - struct strbuf msg = STRBUF_INIT; - const char *out_enc = get_commit_output_encoding(); - const char *message = repo_logmsg_reencode(repo, based_on, - NULL, out_enc); - const char *orig_message = NULL; - const char *exclude_gpgsig[] = { "gpgsig", NULL }; - - commit_list_insert(parent, &parents); - extra = read_commit_extra_headers(based_on, exclude_gpgsig); - find_commit_subject(message, &orig_message); - strbuf_addstr(&msg, orig_message); - author = get_author(message); - reset_ident_date(); - if (commit_tree_extended(msg.buf, msg.len, &tree->object.oid, parents, - &ret, author, NULL, sign_commit, extra)) { - error(_("failed to write commit object")); - goto out; - } - - obj = parse_object(repo, &ret); - -out: - repo_unuse_commit_buffer(the_repository, based_on, message); - free_commit_extra_headers(extra); - free_commit_list(parents); - strbuf_release(&msg); - free(author); - return (struct commit *)obj; -} - -struct ref_info { - struct commit *onto; - struct strset positive_refs; - struct strset negative_refs; - int positive_refexprs; - int negative_refexprs; -}; - -static void get_ref_information(struct repository *repo, - struct rev_cmdline_info *cmd_info, - struct ref_info *ref_info) -{ - int i; - - ref_info->onto = NULL; - strset_init(&ref_info->positive_refs); - strset_init(&ref_info->negative_refs); - ref_info->positive_refexprs = 0; - ref_info->negative_refexprs = 0; - - /* - * When the user specifies e.g. - * git replay origin/main..mybranch - * git replay ^origin/next mybranch1 mybranch2 - * we want to be able to determine where to replay the commits. In - * these examples, the branches are probably based on an old version - * of either origin/main or origin/next, so we want to replay on the - * newest version of that branch. In contrast we would want to error - * out if they ran - * git replay ^origin/master ^origin/next mybranch - * git replay mybranch~2..mybranch - * the first of those because there's no unique base to choose, and - * the second because they'd likely just be replaying commits on top - * of the same commit and not making any difference. - */ - for (i = 0; i < cmd_info->nr; i++) { - struct rev_cmdline_entry *e = cmd_info->rev + i; - struct object_id oid; - const char *refexpr = e->name; - char *fullname = NULL; - int can_uniquely_dwim = 1; - - if (*refexpr == '^') - refexpr++; - if (repo_dwim_ref(repo, refexpr, strlen(refexpr), &oid, &fullname, 0) != 1) - can_uniquely_dwim = 0; - - if (e->flags & BOTTOM) { - if (can_uniquely_dwim) - strset_add(&ref_info->negative_refs, fullname); - if (!ref_info->negative_refexprs) - ref_info->onto = lookup_commit_reference_gently(repo, - &e->item->oid, 1); - ref_info->negative_refexprs++; - } else { - if (can_uniquely_dwim) - strset_add(&ref_info->positive_refs, fullname); - ref_info->positive_refexprs++; - } - - free(fullname); - } -} - -static void determine_replay_mode(struct repository *repo, - struct rev_cmdline_info *cmd_info, - const char *onto_name, - char **advance_name, - struct commit **onto, - struct strset **update_refs) -{ - struct ref_info rinfo; - - get_ref_information(repo, cmd_info, &rinfo); - if (!rinfo.positive_refexprs) - die(_("need some commits to replay")); - - die_for_incompatible_opt2(!!onto_name, "--onto", - !!*advance_name, "--advance"); - if (onto_name) { - *onto = peel_committish(repo, onto_name); - if (rinfo.positive_refexprs < - strset_get_size(&rinfo.positive_refs)) - die(_("all positive revisions given must be references")); - } else if (*advance_name) { - struct object_id oid; - char *fullname = NULL; - - *onto = peel_committish(repo, *advance_name); - if (repo_dwim_ref(repo, *advance_name, strlen(*advance_name), - &oid, &fullname, 0) == 1) { - free(*advance_name); - *advance_name = fullname; - } else { - die(_("argument to --advance must be a reference")); - } - if (rinfo.positive_refexprs > 1) - die(_("cannot advance target with multiple sources because ordering would be ill-defined")); - } else { - int positive_refs_complete = ( - rinfo.positive_refexprs == - strset_get_size(&rinfo.positive_refs)); - int negative_refs_complete = ( - rinfo.negative_refexprs == - strset_get_size(&rinfo.negative_refs)); - /* - * We need either positive_refs_complete or - * negative_refs_complete, but not both. - */ - if (rinfo.negative_refexprs > 0 && - positive_refs_complete == negative_refs_complete) - die(_("cannot implicitly determine whether this is an --advance or --onto operation")); - if (negative_refs_complete) { - struct hashmap_iter iter; - struct strmap_entry *entry; - const char *last_key = NULL; - - if (rinfo.negative_refexprs == 0) - die(_("all positive revisions given must be references")); - else if (rinfo.negative_refexprs > 1) - die(_("cannot implicitly determine whether this is an --advance or --onto operation")); - else if (rinfo.positive_refexprs > 1) - die(_("cannot advance target with multiple source branches because ordering would be ill-defined")); - - /* Only one entry, but we have to loop to get it */ - strset_for_each_entry(&rinfo.negative_refs, - &iter, entry) { - last_key = entry->key; - } - - free(*advance_name); - *advance_name = xstrdup_or_null(last_key); - } else { /* positive_refs_complete */ - if (rinfo.negative_refexprs > 1) - die(_("cannot implicitly determine correct base for --onto")); - if (rinfo.negative_refexprs == 1) - *onto = rinfo.onto; - } - } - if (!*advance_name) { - *update_refs = xcalloc(1, sizeof(**update_refs)); - **update_refs = rinfo.positive_refs; - memset(&rinfo.positive_refs, 0, sizeof(**update_refs)); - } - strset_clear(&rinfo.negative_refs); - strset_clear(&rinfo.positive_refs); -} - -static struct commit *mapped_commit(kh_oid_map_t *replayed_commits, - struct commit *commit, - struct commit *fallback) -{ - khint_t pos = kh_get_oid_map(replayed_commits, commit->object.oid); - if (pos == kh_end(replayed_commits)) - return fallback; - return kh_value(replayed_commits, pos); -} - -static struct commit *pick_regular_commit(struct repository *repo, - struct commit *pickme, - kh_oid_map_t *replayed_commits, - struct commit *onto, - struct merge_options *merge_opt, - struct merge_result *result) -{ - struct commit *base, *replayed_base; - struct tree *pickme_tree, *base_tree; - - base = pickme->parents->item; - replayed_base = mapped_commit(replayed_commits, base, onto); - - result->tree = repo_get_commit_tree(repo, replayed_base); - pickme_tree = repo_get_commit_tree(repo, pickme); - base_tree = repo_get_commit_tree(repo, base); - - merge_opt->branch1 = short_commit_name(repo, replayed_base); - merge_opt->branch2 = short_commit_name(repo, pickme); - merge_opt->ancestor = xstrfmt("parent of %s", merge_opt->branch2); - - merge_incore_nonrecursive(merge_opt, - base_tree, - result->tree, - pickme_tree, - result); - - free((char*)merge_opt->ancestor); - merge_opt->ancestor = NULL; - if (!result->clean) - return NULL; - return create_commit(repo, result->tree, pickme, replayed_base); -} - static enum ref_action_mode parse_ref_action_mode(const char *ref_action, const char *source) { if (!ref_action || !strcmp(ref_action, "update")) @@ -343,21 +71,11 @@ int cmd_replay(int argc, const char *prefix, struct repository *repo) { - const char *advance_name_opt = NULL; - char *advance_name = NULL; - struct commit *onto = NULL; - const char *onto_name = NULL; - int contained = 0; + struct replay_revisions_options opts = { 0 }; + struct replay_result result = { 0 }; const char *ref_action = NULL; enum ref_action_mode ref_mode; - struct rev_info revs; - struct commit *last_commit = NULL; - struct commit *commit; - struct merge_options merge_opt; - struct merge_result result; - struct strset *update_refs = NULL; - kh_oid_map_t *replayed_commits; struct ref_transaction *transaction = NULL; struct strbuf transaction_err = STRBUF_INIT; struct strbuf reflog_msg = STRBUF_INIT; @@ -366,18 +84,18 @@ int cmd_replay(int argc, const char *const replay_usage[] = { N_("(EXPERIMENTAL!) git replay " "([--contained] --onto <newbase> | --advance <branch>) " - "[--ref-action[=<mode>]] <revision-range>..."), + "[--ref-action[=<mode>]] <revision-range>"), NULL }; struct option replay_options[] = { - OPT_STRING(0, "advance", &advance_name_opt, + OPT_STRING(0, "advance", &opts.advance, N_("branch"), N_("make replay advance given branch")), - OPT_STRING(0, "onto", &onto_name, + OPT_STRING(0, "onto", &opts.onto, N_("revision"), N_("replay onto given commit")), - OPT_BOOL(0, "contained", &contained, - N_("advance all branches contained in revision-range")), + OPT_BOOL(0, "contained", &opts.contained, + N_("update all branches that point at commits in <revision-range>")), OPT_STRING(0, "ref-action", &ref_action, N_("mode"), N_("control ref update behavior (update|print)")), @@ -387,19 +105,19 @@ int cmd_replay(int argc, argc = parse_options(argc, argv, prefix, replay_options, replay_usage, PARSE_OPT_KEEP_ARGV0 | PARSE_OPT_KEEP_UNKNOWN_OPT); - if (!onto_name && !advance_name_opt) { + if (!opts.onto && !opts.advance) { error(_("option --onto or --advance is mandatory")); usage_with_options(replay_usage, replay_options); } - die_for_incompatible_opt2(!!advance_name_opt, "--advance", - contained, "--contained"); + die_for_incompatible_opt2(!!opts.advance, "--advance", + opts.contained, "--contained"); + die_for_incompatible_opt2(!!opts.advance, "--advance", + !!opts.onto, "--onto"); /* Parse ref action mode from command line or config */ ref_mode = get_ref_action_mode(repo, ref_action); - advance_name = xstrdup_or_null(advance_name_opt); - repo_init_revisions(repo, &revs, prefix); /* @@ -451,15 +169,19 @@ int cmd_replay(int argc, revs.simplify_history = 0; } - determine_replay_mode(repo, &revs.cmdline, onto_name, &advance_name, - &onto, &update_refs); + ret = replay_revisions(&revs, &opts, &result); + if (ret) + goto cleanup; /* Build reflog message */ - if (advance_name_opt) - strbuf_addf(&reflog_msg, "replay --advance %s", advance_name_opt); - else - strbuf_addf(&reflog_msg, "replay --onto %s", - oid_to_hex(&onto->object.oid)); + if (opts.advance) { + strbuf_addf(&reflog_msg, "replay --advance %s", opts.advance); + } else { + struct object_id oid; + if (repo_get_oid_committish(repo, opts.onto, &oid)) + BUG("--onto commit should have been resolved beforehand already"); + strbuf_addf(&reflog_msg, "replay --onto %s", oid_to_hex(&oid)); + } /* Initialize ref transaction if using update mode */ if (ref_mode == REF_ACTION_UPDATE) { @@ -472,81 +194,19 @@ int cmd_replay(int argc, } } - if (!onto) /* FIXME: Should handle replaying down to root commit */ - die("Replaying down to root commit is not supported yet!"); - - if (prepare_revision_walk(&revs) < 0) { - ret = error(_("error preparing revisions")); - goto cleanup; - } - - init_basic_merge_options(&merge_opt, repo); - memset(&result, 0, sizeof(result)); - merge_opt.show_rename_progress = 0; - last_commit = onto; - replayed_commits = kh_init_oid_map(); - while ((commit = get_revision(&revs))) { - const struct name_decoration *decoration; - khint_t pos; - int hr; - - if (!commit->parents) - die(_("replaying down to root commit is not supported yet!")); - if (commit->parents->next) - die(_("replaying merge commits is not supported yet!")); - - last_commit = pick_regular_commit(repo, commit, replayed_commits, - onto, &merge_opt, &result); - if (!last_commit) - break; - - /* Record commit -> last_commit mapping */ - pos = kh_put_oid_map(replayed_commits, commit->object.oid, &hr); - if (hr == 0) - BUG("Duplicate rewritten commit: %s\n", - oid_to_hex(&commit->object.oid)); - kh_value(replayed_commits, pos) = last_commit; - - /* Update any necessary branches */ - if (advance_name) - continue; - decoration = get_name_decoration(&commit->object); - if (!decoration) - continue; - while (decoration) { - if (decoration->type == DECORATION_REF_LOCAL && - (contained || strset_contains(update_refs, - decoration->name))) { - if (handle_ref_update(ref_mode, transaction, - decoration->name, - &last_commit->object.oid, - &commit->object.oid, - reflog_msg.buf, - &transaction_err) < 0) { - ret = error(_("failed to update ref '%s': %s"), - decoration->name, transaction_err.buf); - goto cleanup; - } - } - decoration = decoration->next; - } - } - - /* In --advance mode, advance the target ref */ - if (result.clean == 1 && advance_name) { - if (handle_ref_update(ref_mode, transaction, advance_name, - &last_commit->object.oid, - &onto->object.oid, - reflog_msg.buf, - &transaction_err) < 0) { + for (size_t i = 0; i < result.updates_nr; i++) { + ret = handle_ref_update(ref_mode, transaction, result.updates[i].refname, + &result.updates[i].new_oid, &result.updates[i].old_oid, + reflog_msg.buf, &transaction_err); + if (ret) { ret = error(_("failed to update ref '%s': %s"), - advance_name, transaction_err.buf); + result.updates[i].refname, transaction_err.buf); goto cleanup; } } /* Commit the ref transaction if we have one */ - if (transaction && result.clean == 1) { + if (transaction) { if (ref_transaction_commit(transaction, &transaction_err)) { ret = error(_("failed to commit ref transaction: %s"), transaction_err.buf); @@ -554,24 +214,18 @@ int cmd_replay(int argc, } } - merge_finalize(&merge_opt, &result); - kh_destroy_oid_map(replayed_commits); - if (update_refs) { - strset_clear(update_refs); - free(update_refs); - } - ret = result.clean; + ret = 0; cleanup: if (transaction) ref_transaction_free(transaction); + replay_result_release(&result); strbuf_release(&transaction_err); strbuf_release(&reflog_msg); release_revisions(&revs); - free(advance_name); /* Return */ if (ret < 0) exit(128); - return ret ? 0 : 1; + return ret; } diff --git a/builtin/repo.c b/builtin/repo.c index 2a653bd3ea..0ea045abc1 100644 --- a/builtin/repo.c +++ b/builtin/repo.c @@ -2,6 +2,8 @@ #include "builtin.h" #include "environment.h" +#include "hex.h" +#include "odb.h" #include "parse-options.h" #include "path-walk.h" #include "progress.h" @@ -15,8 +17,8 @@ #include "utf8.h" static const char *const repo_usage[] = { - "git repo info [--format=(keyvalue|nul)] [-z] [--all | <key>...]", - "git repo structure [--format=(table|keyvalue|nul)]", + "git repo info [--format=(keyvalue|nul) | -z] [--all | <key>...]", + "git repo structure [--format=(table|keyvalue|nul) | -z]", NULL }; @@ -202,13 +204,19 @@ struct ref_stats { size_t others; }; -struct object_stats { +struct object_values { size_t tags; size_t commits; size_t trees; size_t blobs; }; +struct object_stats { + struct object_values type_counts; + struct object_values inflated_sizes; + struct object_values disk_sizes; +}; + struct repo_structure { struct ref_stats refs; struct object_stats objects; @@ -219,6 +227,7 @@ struct stats_table { int name_col_width; int value_col_width; + int unit_col_width; }; /* @@ -226,6 +235,7 @@ struct stats_table { */ struct stats_table_entry { char *value; + const char *unit; }; static void stats_table_vaddf(struct stats_table *table, @@ -246,11 +256,18 @@ static void stats_table_vaddf(struct stats_table *table, if (name_width > table->name_col_width) table->name_col_width = name_width; - if (entry) { + if (!entry) + return; + if (entry->value) { int value_width = utf8_strwidth(entry->value); if (value_width > table->value_col_width) table->value_col_width = value_width; } + if (entry->unit) { + int unit_width = utf8_strwidth(entry->unit); + if (unit_width > table->unit_col_width) + table->unit_col_width = unit_width; + } } static void stats_table_addf(struct stats_table *table, const char *format, ...) @@ -269,7 +286,21 @@ static void stats_table_count_addf(struct stats_table *table, size_t value, va_list ap; CALLOC_ARRAY(entry, 1); - entry->value = xstrfmt("%" PRIuMAX, (uintmax_t)value); + humanise_count(value, &entry->value, &entry->unit); + + va_start(ap, format); + stats_table_vaddf(table, entry, format, ap); + va_end(ap); +} + +static void stats_table_size_addf(struct stats_table *table, size_t value, + const char *format, ...) +{ + struct stats_table_entry *entry; + va_list ap; + + CALLOC_ARRAY(entry, 1); + humanise_bytes(value, &entry->value, &entry->unit, HUMANISE_COMPACT); va_start(ap, format); stats_table_vaddf(table, entry, format, ap); @@ -281,9 +312,9 @@ static inline size_t get_total_reference_count(struct ref_stats *stats) return stats->branches + stats->remotes + stats->tags + stats->others; } -static inline size_t get_total_object_count(struct object_stats *stats) +static inline size_t get_total_object_values(struct object_values *values) { - return stats->tags + stats->commits + stats->trees + stats->blobs; + return values->tags + values->commits + values->trees + values->blobs; } static void stats_table_setup_structure(struct stats_table *table, @@ -291,7 +322,9 @@ static void stats_table_setup_structure(struct stats_table *table, { struct object_stats *objects = &stats->objects; struct ref_stats *refs = &stats->refs; - size_t object_total; + size_t inflated_object_total; + size_t object_count_total; + size_t disk_object_total; size_t ref_total; ref_total = get_total_reference_count(refs); @@ -302,34 +335,66 @@ static void stats_table_setup_structure(struct stats_table *table, stats_table_count_addf(table, refs->remotes, " * %s", _("Remotes")); stats_table_count_addf(table, refs->others, " * %s", _("Others")); - object_total = get_total_object_count(objects); + object_count_total = get_total_object_values(&objects->type_counts); stats_table_addf(table, ""); stats_table_addf(table, "* %s", _("Reachable objects")); - stats_table_count_addf(table, object_total, " * %s", _("Count")); - stats_table_count_addf(table, objects->commits, " * %s", _("Commits")); - stats_table_count_addf(table, objects->trees, " * %s", _("Trees")); - stats_table_count_addf(table, objects->blobs, " * %s", _("Blobs")); - stats_table_count_addf(table, objects->tags, " * %s", _("Tags")); + stats_table_count_addf(table, object_count_total, " * %s", _("Count")); + stats_table_count_addf(table, objects->type_counts.commits, + " * %s", _("Commits")); + stats_table_count_addf(table, objects->type_counts.trees, + " * %s", _("Trees")); + stats_table_count_addf(table, objects->type_counts.blobs, + " * %s", _("Blobs")); + stats_table_count_addf(table, objects->type_counts.tags, + " * %s", _("Tags")); + + inflated_object_total = get_total_object_values(&objects->inflated_sizes); + stats_table_size_addf(table, inflated_object_total, + " * %s", _("Inflated size")); + stats_table_size_addf(table, objects->inflated_sizes.commits, + " * %s", _("Commits")); + stats_table_size_addf(table, objects->inflated_sizes.trees, + " * %s", _("Trees")); + stats_table_size_addf(table, objects->inflated_sizes.blobs, + " * %s", _("Blobs")); + stats_table_size_addf(table, objects->inflated_sizes.tags, + " * %s", _("Tags")); + + disk_object_total = get_total_object_values(&objects->disk_sizes); + stats_table_size_addf(table, disk_object_total, + " * %s", _("Disk size")); + stats_table_size_addf(table, objects->disk_sizes.commits, + " * %s", _("Commits")); + stats_table_size_addf(table, objects->disk_sizes.trees, + " * %s", _("Trees")); + stats_table_size_addf(table, objects->disk_sizes.blobs, + " * %s", _("Blobs")); + stats_table_size_addf(table, objects->disk_sizes.tags, + " * %s", _("Tags")); } static void stats_table_print_structure(const struct stats_table *table) { const char *name_col_title = _("Repository structure"); const char *value_col_title = _("Value"); - int name_col_width = utf8_strwidth(name_col_title); - int value_col_width = utf8_strwidth(value_col_title); + int title_name_width = utf8_strwidth(name_col_title); + int title_value_width = utf8_strwidth(value_col_title); + int name_col_width = table->name_col_width; + int value_col_width = table->value_col_width; + int unit_col_width = table->unit_col_width; struct string_list_item *item; struct strbuf buf = STRBUF_INIT; - if (table->name_col_width > name_col_width) - name_col_width = table->name_col_width; - if (table->value_col_width > value_col_width) - value_col_width = table->value_col_width; + if (title_name_width > name_col_width) + name_col_width = title_name_width; + if (title_value_width > value_col_width + unit_col_width + 1) + value_col_width = title_value_width - unit_col_width; strbuf_addstr(&buf, "| "); strbuf_utf8_align(&buf, ALIGN_LEFT, name_col_width, name_col_title); strbuf_addstr(&buf, " | "); - strbuf_utf8_align(&buf, ALIGN_LEFT, value_col_width, value_col_title); + strbuf_utf8_align(&buf, ALIGN_LEFT, + value_col_width + unit_col_width + 1, value_col_title); strbuf_addstr(&buf, " |"); printf("%s\n", buf.buf); @@ -337,17 +402,20 @@ static void stats_table_print_structure(const struct stats_table *table) for (int i = 0; i < name_col_width; i++) putchar('-'); printf(" | "); - for (int i = 0; i < value_col_width; i++) + for (int i = 0; i < value_col_width + unit_col_width + 1; i++) putchar('-'); printf(" |\n"); for_each_string_list_item(item, &table->rows) { struct stats_table_entry *entry = item->util; const char *value = ""; + const char *unit = ""; if (entry) { struct stats_table_entry *entry = item->util; value = entry->value; + if (entry->unit) + unit = entry->unit; } strbuf_reset(&buf); @@ -355,6 +423,8 @@ static void stats_table_print_structure(const struct stats_table *table) strbuf_utf8_align(&buf, ALIGN_LEFT, name_col_width, item->string); strbuf_addstr(&buf, " | "); strbuf_utf8_align(&buf, ALIGN_RIGHT, value_col_width, value); + strbuf_addch(&buf, ' '); + strbuf_utf8_align(&buf, ALIGN_LEFT, unit_col_width, unit); strbuf_addstr(&buf, " |"); printf("%s\n", buf.buf); } @@ -389,13 +459,31 @@ static void structure_keyvalue_print(struct repo_structure *stats, (uintmax_t)stats->refs.others, value_delim); printf("objects.commits.count%c%" PRIuMAX "%c", key_delim, - (uintmax_t)stats->objects.commits, value_delim); + (uintmax_t)stats->objects.type_counts.commits, value_delim); printf("objects.trees.count%c%" PRIuMAX "%c", key_delim, - (uintmax_t)stats->objects.trees, value_delim); + (uintmax_t)stats->objects.type_counts.trees, value_delim); printf("objects.blobs.count%c%" PRIuMAX "%c", key_delim, - (uintmax_t)stats->objects.blobs, value_delim); + (uintmax_t)stats->objects.type_counts.blobs, value_delim); printf("objects.tags.count%c%" PRIuMAX "%c", key_delim, - (uintmax_t)stats->objects.tags, value_delim); + (uintmax_t)stats->objects.type_counts.tags, value_delim); + + printf("objects.commits.inflated_size%c%" PRIuMAX "%c", key_delim, + (uintmax_t)stats->objects.inflated_sizes.commits, value_delim); + printf("objects.trees.inflated_size%c%" PRIuMAX "%c", key_delim, + (uintmax_t)stats->objects.inflated_sizes.trees, value_delim); + printf("objects.blobs.inflated_size%c%" PRIuMAX "%c", key_delim, + (uintmax_t)stats->objects.inflated_sizes.blobs, value_delim); + printf("objects.tags.inflated_size%c%" PRIuMAX "%c", key_delim, + (uintmax_t)stats->objects.inflated_sizes.tags, value_delim); + + printf("objects.commits.disk_size%c%" PRIuMAX "%c", key_delim, + (uintmax_t)stats->objects.disk_sizes.commits, value_delim); + printf("objects.trees.disk_size%c%" PRIuMAX "%c", key_delim, + (uintmax_t)stats->objects.disk_sizes.trees, value_delim); + printf("objects.blobs.disk_size%c%" PRIuMAX "%c", key_delim, + (uintmax_t)stats->objects.disk_sizes.blobs, value_delim); + printf("objects.tags.disk_size%c%" PRIuMAX "%c", key_delim, + (uintmax_t)stats->objects.disk_sizes.tags, value_delim); fflush(stdout); } @@ -460,6 +548,7 @@ static void structure_count_references(struct ref_stats *stats, } struct count_objects_data { + struct object_database *odb; struct object_stats *stats; struct progress *progress; }; @@ -469,26 +558,53 @@ static int count_objects(const char *path UNUSED, struct oid_array *oids, { struct count_objects_data *data = cb_data; struct object_stats *stats = data->stats; + size_t inflated_total = 0; + size_t disk_total = 0; size_t object_count; + for (size_t i = 0; i < oids->nr; i++) { + struct object_info oi = OBJECT_INFO_INIT; + unsigned long inflated; + off_t disk; + + oi.sizep = &inflated; + oi.disk_sizep = &disk; + + if (odb_read_object_info_extended(data->odb, &oids->oid[i], &oi, + OBJECT_INFO_SKIP_FETCH_OBJECT | + OBJECT_INFO_QUICK) < 0) + continue; + + inflated_total += inflated; + disk_total += disk; + } + switch (type) { case OBJ_TAG: - stats->tags += oids->nr; + stats->type_counts.tags += oids->nr; + stats->inflated_sizes.tags += inflated_total; + stats->disk_sizes.tags += disk_total; break; case OBJ_COMMIT: - stats->commits += oids->nr; + stats->type_counts.commits += oids->nr; + stats->inflated_sizes.commits += inflated_total; + stats->disk_sizes.commits += disk_total; break; case OBJ_TREE: - stats->trees += oids->nr; + stats->type_counts.trees += oids->nr; + stats->inflated_sizes.trees += inflated_total; + stats->disk_sizes.trees += disk_total; break; case OBJ_BLOB: - stats->blobs += oids->nr; + stats->type_counts.blobs += oids->nr; + stats->inflated_sizes.blobs += inflated_total; + stats->disk_sizes.blobs += disk_total; break; default: BUG("invalid object type"); } - object_count = get_total_object_count(stats); + object_count = get_total_object_values(&stats->type_counts); display_progress(data->progress, object_count); return 0; @@ -500,6 +616,7 @@ static void structure_count_objects(struct object_stats *stats, { struct path_walk_info info = PATH_WALK_INFO_INIT; struct count_objects_data data = { + .odb = repo->objects, .stats = stats, }; @@ -529,6 +646,10 @@ static int cmd_repo_structure(int argc, const char **argv, const char *prefix, OPT_CALLBACK_F(0, "format", &format, N_("format"), N_("output format"), PARSE_OPT_NONEG, parse_format_cb), + OPT_CALLBACK_F('z', NULL, &format, NULL, + N_("synonym for --format=nul"), + PARSE_OPT_NONEG | PARSE_OPT_NOARG, + parse_format_cb), OPT_BOOL(0, "progress", &show_progress, N_("show progress")), OPT_END() }; diff --git a/builtin/show-branch.c b/builtin/show-branch.c index 10475a6b5e..f3ebc1d4ea 100644 --- a/builtin/show-branch.c +++ b/builtin/show-branch.c @@ -18,6 +18,7 @@ #include "commit-slab.h" #include "date.h" #include "wildmatch.h" +#include "prio-queue.h" static const char*const show_branch_usage[] = { N_("git show-branch [-a | --all] [-r | --remotes] [--topo-order | --date-order]\n" @@ -59,11 +60,10 @@ static const char *get_color_reset_code(void) return ""; } -static struct commit *interesting(struct commit_list *list) +static struct commit *interesting(struct prio_queue *queue) { - while (list) { - struct commit *commit = list->item; - list = list->next; + for (size_t i = 0; i < queue->nr; i++) { + struct commit *commit = queue->array[i].data; if (commit->object.flags & UNINTERESTING) continue; return commit; @@ -222,17 +222,18 @@ static int mark_seen(struct commit *commit, struct commit_list **seen_p) return 0; } -static void join_revs(struct commit_list **list_p, +static void join_revs(struct prio_queue *queue, struct commit_list **seen_p, int num_rev, int extra) { int all_mask = ((1u << (REV_SHIFT + num_rev)) - 1); int all_revs = all_mask & ~((1u << REV_SHIFT) - 1); - while (*list_p) { + while (queue->nr) { struct commit_list *parents; - int still_interesting = !!interesting(*list_p); - struct commit *commit = pop_commit(list_p); + int still_interesting = !!interesting(queue); + struct commit *commit = prio_queue_peek(queue); + bool get_pending = true; int flags = commit->object.flags & all_mask; if (!still_interesting && extra <= 0) @@ -253,8 +254,14 @@ static void join_revs(struct commit_list **list_p, if (mark_seen(p, seen_p) && !still_interesting) extra--; p->object.flags |= flags; - commit_list_insert_by_date(p, list_p); + if (get_pending) + prio_queue_replace(queue, p); + else + prio_queue_put(queue, p); + get_pending = false; } + if (get_pending) + prio_queue_get(queue); } /* @@ -639,7 +646,8 @@ int cmd_show_branch(int ac, { struct commit *rev[MAX_REVS], *commit; char *reflog_msg[MAX_REVS] = {0}; - struct commit_list *list = NULL, *seen = NULL; + struct commit_list *seen = NULL; + struct prio_queue queue = { compare_commits_by_commit_date }; unsigned int rev_mask[MAX_REVS]; int num_rev, i, extra = 0; int all_heads = 0, all_remotes = 0; @@ -883,14 +891,14 @@ int cmd_show_branch(int ac, */ commit->object.flags |= flag; if (commit->object.flags == flag) - commit_list_insert_by_date(commit, &list); + prio_queue_put(&queue, commit); rev[num_rev] = commit; } for (i = 0; i < num_rev; i++) rev_mask[i] = rev[i]->object.flags; if (0 <= extra) - join_revs(&list, &seen, num_rev, extra); + join_revs(&queue, &seen, num_rev, extra); commit_list_sort_by_date(&seen); @@ -1001,7 +1009,7 @@ out: for (size_t i = 0; i < ARRAY_SIZE(reflog_msg); i++) free(reflog_msg[i]); free_commit_list(seen); - free_commit_list(list); + clear_prio_queue(&queue); free(args_copy); free(head); return ret; diff --git a/builtin/submodule--helper.c b/builtin/submodule--helper.c index 35f6cf735e..d537ab087a 100644 --- a/builtin/submodule--helper.c +++ b/builtin/submodule--helper.c @@ -1903,6 +1903,13 @@ static int determine_submodule_update_strategy(struct repository *r, const char *val; int ret; + /* + * NEEDSWORK: audit and ensure that update_submodule() has right + * to assume that submodule_from_path() above will always succeed. + */ + if (!sub) + BUG("update_submodule assumes a submodule exists at path (%s)", + path); key = xstrfmt("submodule.%s.update", sub->name); if (update) { @@ -3527,14 +3534,15 @@ static int module_add(int argc, const char **argv, const char *prefix, } } - if(!add_data.sm_name) + if (!add_data.sm_name) add_data.sm_name = add_data.sm_path; existing = submodule_from_name(the_repository, null_oid(the_hash_algo), add_data.sm_name); - if (existing && strcmp(existing->path, add_data.sm_path)) { + if (existing && existing->path && + strcmp(existing->path, add_data.sm_path)) { if (!force) { die(_("submodule name '%s' already used for path '%s'"), add_data.sm_name, existing->path); diff --git a/builtin/upload-archive.c b/builtin/upload-archive.c index 97d7c9522f..25312bb2a5 100644 --- a/builtin/upload-archive.c +++ b/builtin/upload-archive.c @@ -4,8 +4,8 @@ #define USE_THE_REPOSITORY_VARIABLE #include "builtin.h" #include "archive.h" -#include "path.h" #include "pkt-line.h" +#include "setup.h" #include "sideband.h" #include "run-command.h" #include "strvec.h" diff --git a/builtin/upload-pack.c b/builtin/upload-pack.c index c2bbc035ab..30498fafea 100644 --- a/builtin/upload-pack.c +++ b/builtin/upload-pack.c @@ -5,11 +5,11 @@ #include "gettext.h" #include "pkt-line.h" #include "parse-options.h" -#include "path.h" #include "protocol.h" #include "replace-object.h" #include "upload-pack.h" #include "serve.h" +#include "setup.h" #include "commit.h" #include "environment.h" diff --git a/chdir-notify.c b/chdir-notify.c index 0d7bc04607..f8bfe3cbef 100644 --- a/chdir-notify.c +++ b/chdir-notify.c @@ -25,6 +25,24 @@ void chdir_notify_register(const char *name, list_add_tail(&e->list, &chdir_notify_entries); } +void chdir_notify_unregister(const char *name, chdir_notify_callback cb, + void *data) +{ + struct list_head *pos, *p; + + list_for_each_safe(pos, p, &chdir_notify_entries) { + struct chdir_notify_entry *e = + list_entry(pos, struct chdir_notify_entry, list); + + if (e->cb != cb || e->data != data || !e->name != !name || + (e->name && strcmp(e->name, name))) + continue; + + list_del(pos); + free(e); + } +} + static void reparent_cb(const char *name, const char *old_cwd, const char *new_cwd, diff --git a/chdir-notify.h b/chdir-notify.h index 366e4c1ee9..81eb69d846 100644 --- a/chdir-notify.h +++ b/chdir-notify.h @@ -41,6 +41,8 @@ typedef void (*chdir_notify_callback)(const char *name, const char *new_cwd, void *data); void chdir_notify_register(const char *name, chdir_notify_callback cb, void *data); +void chdir_notify_unregister(const char *name, chdir_notify_callback cb, + void *data); void chdir_notify_reparent(const char *name, char **path); /* diff --git a/command-list.txt b/command-list.txt index accd3d0c4b..f9005cf459 100644 --- a/command-list.txt +++ b/command-list.txt @@ -115,6 +115,7 @@ git-grep mainporcelain info git-gui mainporcelain git-hash-object plumbingmanipulators git-help ancillaryinterrogators complete +git-history mainporcelain history git-hook purehelpers git-http-backend synchingrepositories git-http-fetch synchelpers @@ -1315,7 +1315,8 @@ free_return: free(buf); } -int check_commit_signature(const struct commit *commit, struct signature_check *sigc) +int verify_commit_buffer(const char *buffer, size_t size, + struct signature_check *sigc) { struct strbuf payload = STRBUF_INIT; struct strbuf signature = STRBUF_INIT; @@ -1323,7 +1324,8 @@ int check_commit_signature(const struct commit *commit, struct signature_check * sigc->result = 'N'; - if (parse_signed_commit(commit, &payload, &signature, the_hash_algo) <= 0) + if (parse_buffer_signed_by_header(buffer, size, &payload, + &signature, the_hash_algo) <= 0) goto out; sigc->payload_type = SIGNATURE_PAYLOAD_COMMIT; @@ -1337,6 +1339,17 @@ int check_commit_signature(const struct commit *commit, struct signature_check * return ret; } +int check_commit_signature(const struct commit *commit, struct signature_check *sigc) +{ + unsigned long size; + const char *buffer = repo_get_commit_buffer(the_repository, commit, &size); + int ret = verify_commit_buffer(buffer, size, sigc); + + repo_unuse_commit_buffer(the_repository, commit, buffer); + + return ret; +} + void verify_merge_signature(struct commit *commit, int verbosity, int check_trust) { @@ -1965,6 +1978,9 @@ int run_commit_hook(int editor_is_used, const char *index_file, strvec_push(&opt.args, arg); va_end(args); + /* All commit hook use-cases require ungrouping child output. */ + opt.ungroup = 1; + opt.invoked_hook = invoked_hook; return run_hooks_opt(the_repository, name, &opt); } @@ -333,6 +333,13 @@ int remove_signature(struct strbuf *buf); */ int check_commit_signature(const struct commit *commit, struct signature_check *sigc); +/* + * Same as check_commit_signature() but accepts a commit buffer and + * its size, instead of a `struct commit *`. + */ +int verify_commit_buffer(const char *buffer, size_t size, + struct signature_check *sigc); + /* record author-date for each commit object */ struct author_date_slab; void record_author_date(struct author_date_slab *author_date, diff --git a/compat/mingw-posix.h b/compat/mingw-posix.h index 631a208684..0939feff27 100644 --- a/compat/mingw-posix.h +++ b/compat/mingw-posix.h @@ -241,9 +241,6 @@ int mingw_chdir(const char *dirname); int mingw_chmod(const char *filename, int mode); #define chmod mingw_chmod -char *mingw_mktemp(char *template); -#define mktemp mingw_mktemp - char *mingw_getcwd(char *pointer, int len); #define getcwd mingw_getcwd diff --git a/compat/mingw.c b/compat/mingw.c index 90ba5cea9d..f09b49ff21 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -629,6 +629,7 @@ int mingw_open (const char *filename, int oflags, ...) int fd, create = (oflags & (O_CREAT | O_EXCL)) == (O_CREAT | O_EXCL); wchar_t wfilename[MAX_PATH]; open_fn_t open_fn; + WIN32_FILE_ATTRIBUTE_DATA fdata; DECLARE_PROC_ADDR(ntdll.dll, NTSTATUS, NTAPI, RtlGetLastNtStatus, void); @@ -653,6 +654,19 @@ int mingw_open (const char *filename, int oflags, ...) else if (xutftowcs_path(wfilename, filename) < 0) return -1; + /* + * When `symlink` exists and is a symbolic link pointing to a + * non-existing file, `_wopen(symlink, O_CREAT | O_EXCL)` would + * create that file. Not what we want: Linux would say `EEXIST` + * in that instance, which is therefore what Git expects. + */ + if (create && + GetFileAttributesExW(wfilename, GetFileExInfoStandard, &fdata) && + (fdata.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT)) { + errno = EEXIST; + return -1; + } + fd = open_fn(wfilename, oflags, mode); /* @@ -1164,18 +1178,6 @@ unsigned int sleep (unsigned int seconds) return 0; } -char *mingw_mktemp(char *template) -{ - wchar_t wtemplate[MAX_PATH]; - if (xutftowcs_path(wtemplate, template) < 0) - return NULL; - if (!_wmktemp(wtemplate)) - return NULL; - if (xwcstoutf(template, wtemplate, strlen(template) + 1) < 0) - return NULL; - return template; -} - int mkstemp(char *template) { return git_mkstemp_mode(template, 0600); diff --git a/compat/mkdtemp.c b/compat/mkdtemp.c deleted file mode 100644 index 1136119592..0000000000 --- a/compat/mkdtemp.c +++ /dev/null @@ -1,8 +0,0 @@ -#include "../git-compat-util.h" - -char *gitmkdtemp(char *template) -{ - if (!*mktemp(template) || mkdir(template, 0700)) - return NULL; - return template; -} diff --git a/compat/posix.h b/compat/posix.h index 067a00f33b..245386fa4a 100644 --- a/compat/posix.h +++ b/compat/posix.h @@ -329,8 +329,7 @@ int gitsetenv(const char *, const char *, int); #endif #ifdef NO_MKDTEMP -#define mkdtemp gitmkdtemp -char *gitmkdtemp(char *); +#define mkdtemp git_mkdtemp #endif #ifdef NO_UNSETENV diff --git a/compat/simple-ipc/ipc-win32.c b/compat/simple-ipc/ipc-win32.c index a8fc812adf..4a3e7df9c7 100644 --- a/compat/simple-ipc/ipc-win32.c +++ b/compat/simple-ipc/ipc-win32.c @@ -686,7 +686,7 @@ static LPSECURITY_ATTRIBUTES get_sa(struct my_sa_data *d) goto fail; } - memset(ea, 0, NR_EA * sizeof(EXPLICIT_ACCESS)); + MEMZERO_ARRAY(ea, NR_EA); ea[0].grfAccessPermissions = GENERIC_READ | GENERIC_WRITE; ea[0].grfAccessMode = SET_ACCESS; diff --git a/compat/win32/pthread.h b/compat/win32/pthread.h index 859e1d9021..ccacc5a53b 100644 --- a/compat/win32/pthread.h +++ b/compat/win32/pthread.h @@ -34,7 +34,7 @@ typedef int pthread_mutexattr_t; #define pthread_cond_t CONDITION_VARIABLE -#define pthread_cond_init(a,b) InitializeConditionVariable((a)) +#define pthread_cond_init(a,b) return_0((InitializeConditionVariable((a)), 0)) #define pthread_cond_destroy(a) do {} while (0) #define pthread_cond_signal WakeConditionVariable #define pthread_cond_broadcast WakeAllConditionVariable @@ -1291,6 +1291,7 @@ int git_config_pathname(char **dest, const char *var, const char *value) if (is_optional && is_missing_file(path)) { free(path); + *dest = NULL; return 0; } @@ -1953,7 +1954,7 @@ int git_configset_get_maybe_bool(struct config_set *set, const char *key, int *d return 1; } -int git_configset_get_pathname(struct config_set *set, const char *key, char **dest) +static int git_configset_get_pathname(struct config_set *set, const char *key, char **dest) { const char *value; if (!git_configset_get_value(set, key, &value, NULL)) @@ -2434,14 +2435,14 @@ int repo_config_get_expiry_in_days(struct repository *r, const char *key, timestamp_t *expiry, timestamp_t now) { const char *expiry_string; - intmax_t days; + int days; timestamp_t when; if (repo_config_get_string_tmp(r, key, &expiry_string)) return 1; /* no such thing */ - if (git_parse_signed(expiry_string, &days, maximum_signed_value_of_type(int))) { - const int scale = 86400; + if (git_parse_int(expiry_string, &days)) { + const intmax_t scale = 86400; *expiry = now - days * scale; return 0; } @@ -564,7 +564,6 @@ int git_configset_get_ulong(struct config_set *cs, const char *key, unsigned lon int git_configset_get_bool(struct config_set *cs, const char *key, int *dest); int git_configset_get_bool_or_int(struct config_set *cs, const char *key, int *is_bool, int *dest); int git_configset_get_maybe_bool(struct config_set *cs, const char *key, int *dest); -int git_configset_get_pathname(struct config_set *cs, const char *key, char **dest); /** * Run only the discover part of the repo_config_get_*() functions diff --git a/config.mak.uname b/config.mak.uname index 1691c6ae6e..38b35af366 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -124,6 +124,7 @@ ifeq ($(uname_S),Darwin) # - MacOS 10.0.* and MacOS 10.1.0 = Darwin 1.* # - MacOS 10.x.* = Darwin (x+4).* for (1 <= x) # i.e. "begins with [15678] and a dot" means "10.4.* or older". + DARWIN_MAJOR_VERSION = $(shell expr "$(uname_R)" : '\([0-9]*\)\.') ifeq ($(shell expr "$(uname_R)" : '[15678]\.'),2) OLD_ICONV = UnfortunatelyYes NO_APPLE_COMMON_CRYPTO = YesPlease @@ -149,28 +150,13 @@ ifeq ($(uname_S),Darwin) CSPRNG_METHOD = arc4random USE_ENHANCED_BASIC_REGULAR_EXPRESSIONS = YesPlease - # Workaround for `gettext` being keg-only and not even being linked via - # `brew link --force gettext`, should be obsolete as of - # https://github.com/Homebrew/homebrew-core/pull/53489 - ifeq ($(shell test -d /usr/local/opt/gettext/ && echo y),y) - BASIC_CFLAGS += -I/usr/local/include -I/usr/local/opt/gettext/include - BASIC_LDFLAGS += -L/usr/local/lib -L/usr/local/opt/gettext/lib - ifeq ($(shell test -x /usr/local/opt/gettext/bin/msgfmt && echo y),y) - MSGFMT = /usr/local/opt/gettext/bin/msgfmt - endif - # On newer ARM-based machines the default installation path has changed to - # /opt/homebrew. Include it in our search paths so that the user does not - # have to configure this manually. - # - # Note that we do not employ the same workaround as above where we manually - # add gettext. The issue was fixed more than three years ago by now, and at - # that point there haven't been any ARM-based Macs yet. - else ifeq ($(shell test -d /opt/homebrew/ && echo y),y) - BASIC_CFLAGS += -I/opt/homebrew/include - BASIC_LDFLAGS += -L/opt/homebrew/lib - ifeq ($(shell test -x /opt/homebrew/bin/msgfmt && echo y),y) - MSGFMT = /opt/homebrew/bin/msgfmt - endif + ifeq ($(uname_M),arm64) + HOMEBREW_PREFIX = /opt/homebrew + else + HOMEBREW_PREFIX = /usr/local + endif + ifeq ($(shell test "$(DARWIN_MAJOR_VERSION)" -ge 24 && echo 1),1) + USE_HOMEBREW_LIBICONV = UnfortunatelyYes endif # The builtin FSMonitor on MacOS builds upon Simple-IPC. Both require @@ -240,6 +240,8 @@ static void process_capabilities(struct packet_reader *reader, size_t *linelen) size_t nul_location = strlen(line); if (nul_location == *linelen) return; + + free(server_capabilities_v1); server_capabilities_v1 = xstrdup(line + nul_location + 1); *linelen = nul_location; diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt index 479163ab5c..28877feb9d 100644 --- a/contrib/buildsystems/CMakeLists.txt +++ b/contrib/buildsystems/CMakeLists.txt @@ -411,10 +411,6 @@ if(NOT HAVE_SETENV) list(APPEND compat_SOURCES compat/setenv.c) endif() -if(NOT HAVE_MKDTEMP) - list(APPEND compat_SOURCES compat/mkdtemp.c) -endif() - if(NOT HAVE_PREAD) list(APPEND compat_SOURCES compat/pread.c) endif() diff --git a/contrib/coccinelle/array.cocci b/contrib/coccinelle/array.cocci index 27a3b479c9..d306f6a21e 100644 --- a/contrib/coccinelle/array.cocci +++ b/contrib/coccinelle/array.cocci @@ -101,3 +101,23 @@ expression dst, src, n; -ALLOC_ARRAY(dst, n); -COPY_ARRAY(dst, src, n); +DUP_ARRAY(dst, src, n); + +@@ +type T; +T *ptr; +expression n; +@@ +- memset(ptr, \( 0x0 \| 0 \), n * \( sizeof(T) +- \| sizeof(*ptr) +- \) ) ++ MEMZERO_ARRAY(ptr, n) + +@@ +type T; +T[] ptr; +expression n; +@@ +- memset(ptr, \( 0x0 \| 0 \), n * \( sizeof(T) +- \| sizeof(*ptr) +- \) ) ++ MEMZERO_ARRAY(ptr, n) diff --git a/contrib/coccinelle/meson.build b/contrib/coccinelle/meson.build index dc3f73c2e7..ae7f5b5460 100644 --- a/contrib/coccinelle/meson.build +++ b/contrib/coccinelle/meson.build @@ -50,6 +50,11 @@ foreach header : headers_to_check coccinelle_headers += meson.project_source_root() / header endforeach +coccinelle_includes = [] +foreach path : ['compat', 'ewah', 'refs', 'sha256', 'trace2', 'win32', 'xdiff'] + coccinelle_includes += ['-I', meson.project_source_root() / path] +endforeach + patches = [ ] foreach source : coccinelle_sources patches += custom_target( @@ -58,6 +63,7 @@ foreach source : coccinelle_sources '--all-includes', '--sp-file', concatenated_rules, '--patch', meson.project_source_root(), + coccinelle_includes, '@INPUT@', ], input: meson.project_source_root() / source, diff --git a/contrib/completion/git-completion.bash b/contrib/completion/git-completion.bash index 73abea31b4..538dff1ee5 100644 --- a/contrib/completion/git-completion.bash +++ b/contrib/completion/git-completion.bash @@ -13,7 +13,8 @@ # *) git email aliases for git-send-email # *) tree paths within 'ref:path/to/file' expressions # *) file paths within current working directory and index -# *) common --long-options +# *) common --long-options but not single-letter options +# *) arguments to long and single-letter options # # To use these routines: # diff --git a/diff-delta.c b/diff-delta.c index 71d37368d6..43c339f010 100644 --- a/diff-delta.c +++ b/diff-delta.c @@ -171,7 +171,7 @@ struct delta_index * create_delta_index(const void *buf, unsigned long bufsize) mem = hash + hsize; entry = mem; - memset(hash, 0, hsize * sizeof(*hash)); + MEMZERO_ARRAY(hash, hsize); /* allocate an array to count hash entries */ hash_count = calloc(hsize, sizeof(*hash_count)); diff --git a/diff-lib.c b/diff-lib.c index b8f8f3bc31..5307390ff3 100644 --- a/diff-lib.c +++ b/diff-lib.c @@ -226,8 +226,12 @@ void run_diff_files(struct rev_info *revs, unsigned int option) continue; } - if (ce_uptodate(ce) || ce_skip_worktree(ce)) + if (ce_uptodate(ce) || ce_skip_worktree(ce)) { + if (revs->diffopt.flags.find_copies_harder) + diff_same(&revs->diffopt, ce->ce_mode, + &ce->oid, ce->name); continue; + } /* * When CE_VALID is set (via "update-index --assume-unchanged" @@ -272,8 +276,10 @@ void run_diff_files(struct rev_info *revs, unsigned int option) if (!changed && !dirty_submodule) { ce_mark_uptodate(ce); mark_fsmonitor_valid(istate, ce); - if (!revs->diffopt.flags.find_copies_harder) - continue; + if (revs->diffopt.flags.find_copies_harder) + diff_same(&revs->diffopt, newmode, + &ce->oid, ce->name); + continue; } oldmode = ce->ce_mode; old_oid = &ce->oid; @@ -418,13 +424,12 @@ static int show_modified(struct rev_info *revs, } oldmode = old_entry->ce_mode; - if (mode == oldmode && oideq(oid, &old_entry->oid) && !dirty_submodule && - !revs->diffopt.flags.find_copies_harder) - return 0; - - diff_change(&revs->diffopt, oldmode, mode, - &old_entry->oid, oid, 1, !is_null_oid(oid), - old_entry->name, 0, dirty_submodule); + if (mode != oldmode || !oideq(oid, &old_entry->oid) || dirty_submodule) + diff_change(&revs->diffopt, oldmode, mode, + &old_entry->oid, oid, 1, !is_null_oid(oid), + old_entry->name, 0, dirty_submodule); + else if (revs->diffopt.flags.find_copies_harder) + diff_same(&revs->diffopt, mode, oid, old_entry->name); return 0; } @@ -7399,6 +7399,26 @@ void diff_change(struct diff_options *options, concatpath, old_dirty_submodule, new_dirty_submodule); } +void diff_same(struct diff_options *options, + unsigned mode, + const struct object_id *oid, + const char *concatpath) +{ + struct diff_filespec *one; + + if (S_ISGITLINK(mode) && is_submodule_ignored(concatpath, options)) + return; + + if (options->prefix && + strncmp(concatpath, options->prefix, options->prefix_length)) + return; + + one = alloc_filespec(concatpath); + fill_filespec(one, oid, 1, mode); + one->count++; + diff_queue(&diff_queued_diff, one, one); +} + struct diff_filepair *diff_unmerge(struct diff_options *options, const char *path) { struct diff_filepair *pair; @@ -572,6 +572,11 @@ void diff_change(struct diff_options *, const char *fullpath, unsigned dirty_submodule1, unsigned dirty_submodule2); +void diff_same(struct diff_options *, + unsigned mode, + const struct object_id *oid, + const char *fullpath); + struct diff_filepair *diff_unmerge(struct diff_options *, const char *path); void compute_diffstat(struct diff_options *options, struct diffstat_t *diffstat, diff --git a/diffcore-delta.c b/diffcore-delta.c index ba6cbee76b..2de9e9ccff 100644 --- a/diffcore-delta.c +++ b/diffcore-delta.c @@ -56,7 +56,7 @@ static struct spanhash_top *spanhash_rehash(struct spanhash_top *orig) st_mult(sizeof(struct spanhash), sz))); new_spanhash->alloc_log2 = orig->alloc_log2 + 1; new_spanhash->free = INITIAL_FREE(new_spanhash->alloc_log2); - memset(new_spanhash->data, 0, sizeof(struct spanhash) * sz); + MEMZERO_ARRAY(new_spanhash->data, sz); for (i = 0; i < osz; i++) { struct spanhash *o = &(orig->data[i]); int bucket; @@ -135,7 +135,7 @@ static struct spanhash_top *hash_chars(struct repository *r, st_mult(sizeof(struct spanhash), (size_t)1 << i))); hash->alloc_log2 = i; hash->free = INITIAL_FREE(i); - memset(hash->data, 0, sizeof(struct spanhash) * ((size_t)1 << i)); + MEMZERO_ARRAY(hash->data, ((size_t)1 << i)); n = 0; accum1 = accum2 = 0; @@ -2,13 +2,13 @@ #include "git-compat-util.h" #include "odb.h" +#include "odb/streaming.h" #include "dir.h" #include "environment.h" #include "gettext.h" #include "hex.h" #include "name-hash.h" #include "sparse-index.h" -#include "streaming.h" #include "submodule.h" #include "symlinks.h" #include "progress.h" @@ -139,7 +139,7 @@ static int streaming_write_entry(const struct cache_entry *ce, char *path, if (fd < 0) return -1; - result |= stream_blob_to_fd(fd, &ce->oid, filter, 1); + result |= odb_stream_blob_to_fd(the_repository->objects, fd, &ce->oid, filter, 1); *fstat_done = fstat_checkout_output(fd, state, statbuf); result |= close(fd); diff --git a/ewah/bitmap.c b/ewah/bitmap.c index 55928dada8..bf878bf876 100644 --- a/ewah/bitmap.c +++ b/ewah/bitmap.c @@ -46,8 +46,7 @@ static void bitmap_grow(struct bitmap *self, size_t word_alloc) { size_t old_size = self->word_alloc; ALLOC_GROW(self->words, word_alloc, self->word_alloc); - memset(self->words + old_size, 0x0, - (self->word_alloc - old_size) * sizeof(eword_t)); + MEMZERO_ARRAY(self->words + old_size, (self->word_alloc - old_size)); } void bitmap_set(struct bitmap *self, size_t pos) @@ -192,8 +191,8 @@ void bitmap_or_ewah(struct bitmap *self, struct ewah_bitmap *other) if (self->word_alloc < other_final) { self->word_alloc = other_final; REALLOC_ARRAY(self->words, self->word_alloc); - memset(self->words + original_size, 0x0, - (self->word_alloc - original_size) * sizeof(eword_t)); + MEMZERO_ARRAY(self->words + original_size, + (self->word_alloc - original_size)); } ewah_iterator_init(&it, other); diff --git a/fetch-pack.c b/fetch-pack.c index 78c45d4a15..40316c9a34 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -1865,8 +1865,9 @@ int fetch_pack_fsck_config(const char *var, const char *value, if (git_config_pathname(&path, var, value)) return -1; - strbuf_addf(msg_types, "%cskiplist=%s", - msg_types->len ? ',' : '=', path); + if (path) + strbuf_addf(msg_types, "%cskiplist=%s", + msg_types->len ? ',' : '=', path); free(path); return 0; } @@ -1379,6 +1379,12 @@ int fsck_finish(struct fsck_options *options) return ret; } +bool fsck_has_queued_checks(struct fsck_options *options) +{ + return !oidset_equal(&options->gitmodules_found, &options->gitmodules_done) || + !oidset_equal(&options->gitattributes_found, &options->gitattributes_done); +} + void fsck_options_clear(struct fsck_options *options) { free(options->msg_type); @@ -1398,14 +1404,16 @@ int git_fsck_config(const char *var, const char *value, if (strcmp(var, "fsck.skiplist") == 0) { char *path; - struct strbuf sb = STRBUF_INIT; if (git_config_pathname(&path, var, value)) return -1; - strbuf_addf(&sb, "skiplist=%s", path); - free(path); - fsck_set_msg_types(options, sb.buf); - strbuf_release(&sb); + if (path) { + struct strbuf sb = STRBUF_INIT; + strbuf_addf(&sb, "skiplist=%s", path); + free(path); + fsck_set_msg_types(options, sb.buf); + strbuf_release(&sb); + } return 0; } @@ -249,6 +249,13 @@ int fsck_tag_standalone(const struct object_id *oid, const char *buffer, int fsck_finish(struct fsck_options *options); /* + * Check whether there are any checks that have been queued up and that still + * need to be run. Returns `false` iff `fsck_finish()` wouldn't perform any + * actions, `true` otherwise. + */ +bool fsck_has_queued_checks(struct fsck_options *options); + +/* * Clear the fsck_options struct, freeing any allocated memory. */ void fsck_options_clear(struct fsck_options *options); diff --git a/git-compat-util.h b/git-compat-util.h index 398e0fac4f..b0673d1a45 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -38,37 +38,8 @@ struct strbuf; DISABLE_WARNING(-Wsign-compare) #endif -#ifndef FLEX_ARRAY -/* - * See if our compiler is known to support flexible array members. - */ - -/* - * Check vendor specific quirks first, before checking the - * __STDC_VERSION__, as vendor compilers can lie and we need to be - * able to work them around. Note that by not defining FLEX_ARRAY - * here, we can fall back to use the "safer but a bit wasteful" one - * later. - */ -#if defined(__SUNPRO_C) && (__SUNPRO_C <= 0x580) -#elif defined(__GNUC__) -# if (__GNUC__ >= 3) -# define FLEX_ARRAY /* empty */ -# else -# define FLEX_ARRAY 0 /* older GNU extension */ -# endif -#elif defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) -# define FLEX_ARRAY /* empty */ -#endif - -/* - * Otherwise, default to safer but a bit wasteful traditional style - */ -#ifndef FLEX_ARRAY -# define FLEX_ARRAY 1 -#endif -#endif - +#undef FLEX_ARRAY +#define FLEX_ARRAY /* empty - weather balloon to require C99 FAM */ /* * BUILD_ASSERT_OR_ZERO - assert a build-time dependency, as an expression. @@ -726,6 +697,7 @@ static inline uint64_t u64_add(uint64_t a, uint64_t b) #define ALLOC_ARRAY(x, alloc) (x) = xmalloc(st_mult(sizeof(*(x)), (alloc))) #define CALLOC_ARRAY(x, alloc) (x) = xcalloc((alloc), sizeof(*(x))) #define REALLOC_ARRAY(x, alloc) (x) = xrealloc((x), st_mult(sizeof(*(x)), (alloc))) +#define MEMZERO_ARRAY(x, alloc) memset((x), 0x0, st_mult(sizeof(*(x)), (alloc))) #define COPY_ARRAY(dst, src, n) copy_array((dst), (src), (n), sizeof(*(dst)) + \ BARF_UNLESS_COPYABLE((dst), (src))) @@ -586,6 +586,7 @@ static struct cmd_struct commands[] = { { "grep", cmd_grep, RUN_SETUP_GENTLY }, { "hash-object", cmd_hash_object }, { "help", cmd_help }, + { "history", cmd_history, RUN_SETUP }, { "hook", cmd_hook, RUN_SETUP }, { "index-pack", cmd_index_pack, RUN_SETUP_GENTLY | NO_PARSEOPT }, { "init", cmd_init_db }, diff --git a/gpg-interface.c b/gpg-interface.c index f680ed38c0..47222bf31b 100644 --- a/gpg-interface.c +++ b/gpg-interface.c @@ -794,8 +794,16 @@ static int git_gpg_config(const char *var, const char *value, fmtname = "ssh"; if (fmtname) { + char *program; + int status; + fmt = get_format_by_name(fmtname); - return git_config_pathname((char **) &fmt->program, var, value); + status = git_config_pathname(&program, var, value); + if (status) + return status; + if (program) + fmt->program = program; + return status; } return 0; @@ -1146,6 +1154,8 @@ int parse_sign_mode(const char *arg, enum sign_mode *mode) *mode = SIGN_WARN_STRIP; else if (!strcmp(arg, "strip")) *mode = SIGN_STRIP; + else if (!strcmp(arg, "strip-if-invalid")) + *mode = SIGN_STRIP_IF_INVALID; else return -1; return 0; diff --git a/gpg-interface.h b/gpg-interface.h index ead1ed6967..789d1ffac4 100644 --- a/gpg-interface.h +++ b/gpg-interface.h @@ -111,6 +111,7 @@ enum sign_mode { SIGN_VERBATIM, SIGN_WARN_STRIP, SIGN_STRIP, + SIGN_STRIP_IF_INVALID, }; /* @@ -194,7 +194,7 @@ void hashmap_partial_clear_(struct hashmap *map, ssize_t entry_offset) return; if (entry_offset >= 0) /* called by hashmap_clear_entries */ free_individual_entries(map, entry_offset); - memset(map->table, 0, map->tablesize * sizeof(struct hashmap_entry *)); + MEMZERO_ARRAY(map->table, map->tablesize); map->shrink_at = 0; map->private_size = 0; } @@ -55,7 +55,7 @@ int hook_exists(struct repository *r, const char *name) static int pick_next_hook(struct child_process *cp, struct strbuf *out UNUSED, void *pp_cb, - void **pp_task_cb UNUSED) + void **pp_task_cb) { struct hook_cb_data *hook_cb = pp_cb; const char *hook_path = hook_cb->hook_path; @@ -65,11 +65,22 @@ static int pick_next_hook(struct child_process *cp, cp->no_stdin = 1; strvec_pushv(&cp->env, hook_cb->options->env.v); + + if (hook_cb->options->path_to_stdin && hook_cb->options->feed_pipe) + BUG("options path_to_stdin and feed_pipe are mutually exclusive"); + /* reopen the file for stdin; run_command closes it. */ if (hook_cb->options->path_to_stdin) { cp->no_stdin = 0; cp->in = xopen(hook_cb->options->path_to_stdin, O_RDONLY); } + + if (hook_cb->options->feed_pipe) { + cp->no_stdin = 0; + /* start_command() will allocate a pipe / stdin fd for us */ + cp->in = -1; + } + cp->stdout_to_stderr = 1; cp->trace2_hook_name = hook_cb->hook_name; cp->dir = hook_cb->options->dir; @@ -78,6 +89,12 @@ static int pick_next_hook(struct child_process *cp, strvec_pushv(&cp->args, hook_cb->options->args.v); /* + * Provide per-hook internal state via task_cb for easy access, so + * hook callbacks don't have to go through hook_cb->options. + */ + *pp_task_cb = hook_cb->options->feed_pipe_cb_data; + + /* * This pick_next_hook() will be called again, we're only * running one hook, so indicate that no more work will be * done. @@ -136,10 +153,12 @@ int run_hooks_opt(struct repository *r, const char *hook_name, .tr2_label = hook_name, .processes = 1, - .ungroup = 1, + .ungroup = options->ungroup, .get_next_task = pick_next_hook, .start_failure = notify_start_failure, + .feed_pipe = options->feed_pipe, + .consume_output = options->consume_output, .task_finished = notify_hook_finished, .data = &cb_data, @@ -148,6 +167,9 @@ int run_hooks_opt(struct repository *r, const char *hook_name, if (!options) BUG("a struct run_hooks_opt must be provided to run_hooks"); + if (options->path_to_stdin && options->feed_pipe) + BUG("options path_to_stdin and feed_pipe are mutually exclusive"); + if (options->invoked_hook) *options->invoked_hook = 0; @@ -177,6 +199,9 @@ int run_hooks(struct repository *r, const char *hook_name) { struct run_hooks_opt opt = RUN_HOOKS_OPT_INIT; + /* All use-cases of this API require ungrouping. */ + opt.ungroup = 1; + return run_hooks_opt(r, hook_name, &opt); } @@ -1,6 +1,7 @@ #ifndef HOOK_H #define HOOK_H #include "strvec.h" +#include "run-command.h" struct repository; @@ -34,9 +35,59 @@ struct run_hooks_opt int *invoked_hook; /** + * Allow hooks to set run_processes_parallel() 'ungroup' behavior. + */ + unsigned int ungroup:1; + + /** * Path to file which should be piped to stdin for each hook. */ const char *path_to_stdin; + + /** + * Callback used to incrementally feed a child hook stdin pipe. + * + * Useful especially if a hook consumes large quantities of data + * (e.g. a list of all refs in a client push), so feeding it via + * in-memory strings or slurping to/from files is inefficient. + * While the callback allows piecemeal writing, it can also be + * used for smaller inputs, where it gets called only once. + * + * Add hook callback initalization context to `feed_pipe_ctx`. + * Add hook callback internal state to `feed_pipe_cb_data`. + * + */ + feed_pipe_fn feed_pipe; + + /** + * Opaque data pointer used to pass context to `feed_pipe_fn`. + * + * It can be accessed via the second callback arg 'pp_cb': + * ((struct hook_cb_data *) pp_cb)->hook_cb->options->feed_pipe_ctx; + * + * The caller is responsible for managing the memory for this data. + * Only useful when using `run_hooks_opt.feed_pipe`, otherwise ignore it. + */ + void *feed_pipe_ctx; + + /** + * Opaque data pointer used to keep internal state across callback calls. + * + * It can be accessed directly via the third callback arg 'pp_task_cb': + * struct ... *state = pp_task_cb; + * + * The caller is responsible for managing the memory for this data. + * Only useful when using `run_hooks_opt.feed_pipe`, otherwise ignore it. + */ + void *feed_pipe_cb_data; + + /* + * Populate this to capture output and prevent it from being printed to + * stderr. This will be passed directly through to + * run_command:run_parallel_processes(). See t/helper/test-run-command.c + * for an example. + */ + consume_output_fn consume_output; }; #define RUN_HOOKS_OPT_INIT { \ diff --git a/http-backend.c b/http-backend.c index 273ed7266f..24f0dc119a 100644 --- a/http-backend.c +++ b/http-backend.c @@ -16,6 +16,7 @@ #include "run-command.h" #include "string-list.h" #include "url.h" +#include "setup.h" #include "strvec.h" #include "packfile.h" #include "odb.h" diff --git a/http-push.c b/http-push.c index d86ce77119..60a9b75620 100644 --- a/http-push.c +++ b/http-push.c @@ -1725,6 +1725,7 @@ int cmd_main(int argc, const char **argv) int i; int new_refs; struct ref *ref, *local_refs = NULL; + const char *gitdir; CALLOC_ARRAY(repo, 1); @@ -1787,7 +1788,7 @@ int cmd_main(int argc, const char **argv) if (delete_branch && rs.nr != 1) die("You must specify only one branch name when deleting a remote branch"); - setup_git_directory(); + gitdir = setup_git_directory(); memset(remote_dir_exists, -1, 256); @@ -1941,7 +1942,7 @@ int cmd_main(int argc, const char **argv) if (!push_all && !is_null_oid(&ref->old_oid)) strvec_pushf(&commit_argv, "^%s", oid_to_hex(&ref->old_oid)); - repo_init_revisions(the_repository, &revs, setup_git_directory()); + repo_init_revisions(the_repository, &revs, gitdir); setup_revisions_from_strvec(&commit_argv, &revs, NULL); revs.edge_hint = 0; /* just in case */ diff --git a/linear-assignment.c b/linear-assignment.c index 5416cbcf40..97b4f75058 100644 --- a/linear-assignment.c +++ b/linear-assignment.c @@ -20,8 +20,8 @@ void compute_assignment(int column_count, int row_count, int *cost, int i, j, phase; if (column_count < 2) { - memset(column2row, 0, sizeof(int) * column_count); - memset(row2column, 0, sizeof(int) * row_count); + MEMZERO_ARRAY(column2row, column_count); + MEMZERO_ARRAY(row2column, row_count); return; } diff --git a/meson.build b/meson.build index f1b3615659..3a1d12caa4 100644 --- a/meson.build +++ b/meson.build @@ -397,6 +397,7 @@ libgit_sources = [ 'object-name.c', 'object.c', 'odb.c', + 'odb/streaming.c', 'oid-array.c', 'oidmap.c', 'oidset.c', @@ -470,6 +471,7 @@ libgit_sources = [ 'repack-midx.c', 'repack-promisor.c', 'replace-object.c', + 'replay.c', 'repo-settings.c', 'repository.c', 'rerere.c', @@ -490,7 +492,6 @@ libgit_sources = [ 'stable-qsort.c', 'statinfo.c', 'strbuf.c', - 'streaming.c', 'string-list.c', 'strmap.c', 'strvec.c', @@ -609,6 +610,7 @@ builtin_sources = [ 'builtin/grep.c', 'builtin/hash-object.c', 'builtin/help.c', + 'builtin/history.c', 'builtin/hook.c', 'builtin/index-pack.c', 'builtin/init-db.c', @@ -1064,7 +1066,7 @@ if iconv.found() } ''' - if compiler.run(iconv_omits_bom_source, + if meson.can_run_host_binaries() and compiler.run(iconv_omits_bom_source, dependencies: iconv, name: 'iconv omits BOM', ).returncode() != 0 @@ -1401,7 +1403,7 @@ checkfuncs = { 'strlcpy' : ['strlcpy.c'], 'strtoull' : [], 'setenv' : ['setenv.c'], - 'mkdtemp' : ['mkdtemp.c'], + 'mkdtemp' : [], 'initgroups' : [], 'strtoumax' : ['strtoumax.c', 'strtoimax.c'], 'pread' : ['pread.c'], @@ -1492,7 +1494,7 @@ if not has_bsd_sysctl endif endif -if not meson.is_cross_build() and compiler.run(''' +if meson.can_run_host_binaries() and compiler.run(''' #include <stdio.h> int main(int argc, const char **argv) diff --git a/midx-write.c b/midx-write.c index 23e61cb000..ce459b02c3 100644 --- a/midx-write.c +++ b/midx-write.c @@ -1014,6 +1014,65 @@ static void clear_midx_files(struct odb_source *source, strbuf_release(&buf); } +static bool midx_needs_update(struct multi_pack_index *midx, struct write_midx_context *ctx) +{ + struct strset packs = STRSET_INIT; + struct strbuf buf = STRBUF_INIT; + bool needed = true; + + /* + * Ignore incremental updates for now. The assumption is that any + * incremental update would be either empty (in which case we will bail + * out later) or it would actually cover at least one new pack. + */ + if (ctx->incremental) + goto out; + + /* + * Otherwise, we need to verify that the packs covered by the existing + * MIDX match the packs that we already have. The logic to do so is way + * more complicated than it has any right to be. This is because: + * + * - We cannot assume any ordering. + * + * - The MIDX packs may not be loaded at all, and loading them would + * be wasteful. So we need to use the pack names tracked by the + * MIDX itself. + * + * - The MIDX pack names are tracking the ".idx" files, whereas the + * packs themselves are tracking the ".pack" files. So we need to + * strip suffixes. + */ + if (ctx->nr != midx->num_packs + midx->num_packs_in_base) + goto out; + + for (uint32_t i = 0; i < ctx->nr; i++) { + strbuf_reset(&buf); + strbuf_addstr(&buf, pack_basename(ctx->info[i].p)); + strbuf_strip_suffix(&buf, ".pack"); + + if (!strset_add(&packs, buf.buf)) + BUG("same pack added twice?"); + } + + for (uint32_t i = 0; i < ctx->nr; i++) { + strbuf_reset(&buf); + strbuf_addstr(&buf, midx->pack_names[i]); + strbuf_strip_suffix(&buf, ".idx"); + + if (!strset_contains(&packs, buf.buf)) + goto out; + strset_remove(&packs, buf.buf); + } + + needed = false; + +out: + strbuf_release(&buf); + strset_clear(&packs); + return needed; +} + static int write_midx_internal(struct odb_source *source, struct string_list *packs_to_include, struct string_list *packs_to_drop, @@ -1031,6 +1090,7 @@ static int write_midx_internal(struct odb_source *source, struct write_midx_context ctx = { .preferred_pack_idx = NO_PREFERRED_PACK, }; + struct multi_pack_index *midx_to_free = NULL; int bitmapped_packs_concat_len = 0; int pack_name_concat_len = 0; int dropped_packs = 0; @@ -1111,27 +1171,39 @@ static int write_midx_internal(struct odb_source *source, for_each_file_in_pack_dir(source->path, add_pack_to_midx, &ctx); stop_progress(&ctx.progress); - if ((ctx.m && ctx.nr == ctx.m->num_packs + ctx.m->num_packs_in_base) && - !ctx.incremental && - !(packs_to_include || packs_to_drop)) { - struct bitmap_index *bitmap_git; - int bitmap_exists; - int want_bitmap = flags & MIDX_WRITE_BITMAP; + if (!packs_to_drop) { + /* + * If there is no MIDX then either it doesn't exist, or we're + * doing a geometric repack. Try to load it from the source to + * tell these two cases apart. + */ + struct multi_pack_index *midx = ctx.m; + if (!midx) + midx = midx_to_free = load_multi_pack_index(ctx.source); + + if (midx && !midx_needs_update(midx, &ctx)) { + struct bitmap_index *bitmap_git; + int bitmap_exists; + int want_bitmap = flags & MIDX_WRITE_BITMAP; - bitmap_git = prepare_midx_bitmap_git(ctx.m); - bitmap_exists = bitmap_git && bitmap_is_midx(bitmap_git); - free_bitmap_index(bitmap_git); + bitmap_git = prepare_midx_bitmap_git(midx); + bitmap_exists = bitmap_git && bitmap_is_midx(bitmap_git); + free_bitmap_index(bitmap_git); - if (bitmap_exists || !want_bitmap) { - /* - * The correct MIDX already exists, and so does a - * corresponding bitmap (or one wasn't requested). - */ - if (!want_bitmap) - clear_midx_files_ext(source, "bitmap", NULL); - result = 0; - goto cleanup; + if (bitmap_exists || !want_bitmap) { + /* + * The correct MIDX already exists, and so does a + * corresponding bitmap (or one wasn't requested). + */ + if (!want_bitmap) + clear_midx_files_ext(source, "bitmap", NULL); + result = 0; + goto cleanup; + } } + + close_midx(midx_to_free); + midx_to_free = NULL; } if (ctx.incremental && !ctx.nr) { @@ -1458,7 +1530,7 @@ static int write_midx_internal(struct odb_source *source, } if (ctx.m || ctx.base_midx) - close_object_store(ctx.repo->objects); + odb_close(ctx.repo->objects); if (commit_lock_file(&lk) < 0) die_errno(_("could not write multi-pack-index")); @@ -1487,6 +1559,7 @@ cleanup: free(keep_hashes); } strbuf_release(&midx_name); + close_midx(midx_to_free); trace2_region_leave("midx", "write_midx_internal", r); @@ -686,7 +686,7 @@ int midx_preferred_pack(struct multi_pack_index *m, uint32_t *pack_int_id) { if (m->preferred_pack_idx == -1) { uint32_t midx_pos; - if (load_midx_revindex(m) < 0) { + if (load_midx_revindex(m)) { m->preferred_pack_idx = -2; return -1; } diff --git a/object-file.c b/object-file.c index 84c9249dab..6280e42f34 100644 --- a/object-file.c +++ b/object-file.c @@ -20,13 +20,13 @@ #include "object-file-convert.h" #include "object-file.h" #include "odb.h" +#include "odb/streaming.h" #include "oidtree.h" #include "pack.h" #include "packfile.h" #include "path.h" #include "read-cache-ll.h" #include "setup.h" -#include "streaming.h" #include "tempfile.h" #include "tmp-objdir.h" @@ -132,29 +132,27 @@ int check_object_signature(struct repository *r, const struct object_id *oid, int stream_object_signature(struct repository *r, const struct object_id *oid) { struct object_id real_oid; - unsigned long size; - enum object_type obj_type; - struct git_istream *st; + struct odb_read_stream *st; struct git_hash_ctx c; char hdr[MAX_HEADER_LEN]; int hdrlen; - st = open_istream(r, oid, &obj_type, &size, NULL); + st = odb_read_stream_open(r->objects, oid, NULL); if (!st) return -1; /* Generate the header */ - hdrlen = format_object_header(hdr, sizeof(hdr), obj_type, size); + hdrlen = format_object_header(hdr, sizeof(hdr), st->type, st->size); /* Sha1.. */ r->hash_algo->init_fn(&c); git_hash_update(&c, hdr, hdrlen); for (;;) { char buf[1024 * 16]; - ssize_t readlen = read_istream(st, buf, sizeof(buf)); + ssize_t readlen = odb_read_stream_read(st, buf, sizeof(buf)); if (readlen < 0) { - close_istream(st); + odb_read_stream_close(st); return -1; } if (!readlen) @@ -162,7 +160,7 @@ int stream_object_signature(struct repository *r, const struct object_id *oid) git_hash_update(&c, buf, readlen); } git_hash_final_oid(&real_oid, &c); - close_istream(st); + odb_read_stream_close(st); return !oideq(oid, &real_oid) ? -1 : 0; } @@ -234,9 +232,9 @@ static void *map_fd(int fd, const char *path, unsigned long *size) return map; } -void *odb_source_loose_map_object(struct odb_source *source, - const struct object_id *oid, - unsigned long *size) +static void *odb_source_loose_map_object(struct odb_source *source, + const struct object_id *oid, + unsigned long *size) { const char *p; int fd = open_loose_object(source->loose, oid, &p); @@ -246,11 +244,29 @@ void *odb_source_loose_map_object(struct odb_source *source, return map_fd(fd, p, size); } -enum unpack_loose_header_result unpack_loose_header(git_zstream *stream, - unsigned char *map, - unsigned long mapsize, - void *buffer, - unsigned long bufsiz) +enum unpack_loose_header_result { + ULHR_OK, + ULHR_BAD, + ULHR_TOO_LONG, +}; + +/** + * unpack_loose_header() initializes the data stream needed to unpack + * a loose object header. + * + * Returns: + * + * - ULHR_OK on success + * - ULHR_BAD on error + * - ULHR_TOO_LONG if the header was too long + * + * It will only parse up to MAX_HEADER_LEN bytes. + */ +static enum unpack_loose_header_result unpack_loose_header(git_zstream *stream, + unsigned char *map, + unsigned long mapsize, + void *buffer, + unsigned long bufsiz) { int status; @@ -329,11 +345,18 @@ static void *unpack_loose_rest(git_zstream *stream, } /* + * parse_loose_header() parses the starting "<type> <len>\0" of an + * object. If it doesn't follow that format -1 is returned. To check + * the validity of the <type> populate the "typep" in the "struct + * object_info". It will be OBJ_BAD if the object type is unknown. The + * parsed <len> can be retrieved via "oi->sizep", and from there + * passed to unpack_loose_rest(). + * * We used to just use "sscanf()", but that's actually way * too permissive for what we want to check. So do an anal * object header parse by hand. */ -int parse_loose_header(const char *hdr, struct object_info *oi) +static int parse_loose_header(const char *hdr, struct object_info *oi) { const char *type_buf = hdr; size_t size; @@ -403,7 +426,7 @@ int odb_source_loose_read_object_info(struct odb_source *source, unsigned long size_scratch; enum object_type type_scratch; - if (oi->delta_base_oid) + if (oi && oi->delta_base_oid) oidclr(oi->delta_base_oid, source->odb->repo->hash_algo); /* @@ -414,13 +437,13 @@ int odb_source_loose_read_object_info(struct odb_source *source, * return value implicitly indicates whether the * object even exists. */ - if (!oi->typep && !oi->sizep && !oi->contentp) { + if (!oi || (!oi->typep && !oi->sizep && !oi->contentp)) { struct stat st; - if (!oi->disk_sizep && (flags & OBJECT_INFO_QUICK)) + if ((!oi || !oi->disk_sizep) && (flags & OBJECT_INFO_QUICK)) return quick_has_loose(source->loose, oid) ? 0 : -1; if (stat_loose_object(source->loose, oid, &st, &path) < 0) return -1; - if (oi->disk_sizep) + if (oi && oi->disk_sizep) *oi->disk_sizep = st.st_size; return 0; } @@ -1980,3 +2003,127 @@ void odb_source_loose_free(struct odb_source_loose *loose) loose_object_map_clear(&loose->map); free(loose); } + +struct odb_loose_read_stream { + struct odb_read_stream base; + git_zstream z; + enum { + ODB_LOOSE_READ_STREAM_INUSE, + ODB_LOOSE_READ_STREAM_DONE, + ODB_LOOSE_READ_STREAM_ERROR, + } z_state; + void *mapped; + unsigned long mapsize; + char hdr[32]; + int hdr_avail; + int hdr_used; +}; + +static ssize_t read_istream_loose(struct odb_read_stream *_st, char *buf, size_t sz) +{ + struct odb_loose_read_stream *st = (struct odb_loose_read_stream *)_st; + size_t total_read = 0; + + switch (st->z_state) { + case ODB_LOOSE_READ_STREAM_DONE: + return 0; + case ODB_LOOSE_READ_STREAM_ERROR: + return -1; + default: + break; + } + + if (st->hdr_used < st->hdr_avail) { + size_t to_copy = st->hdr_avail - st->hdr_used; + if (sz < to_copy) + to_copy = sz; + memcpy(buf, st->hdr + st->hdr_used, to_copy); + st->hdr_used += to_copy; + total_read += to_copy; + } + + while (total_read < sz) { + int status; + + st->z.next_out = (unsigned char *)buf + total_read; + st->z.avail_out = sz - total_read; + status = git_inflate(&st->z, Z_FINISH); + + total_read = st->z.next_out - (unsigned char *)buf; + + if (status == Z_STREAM_END) { + git_inflate_end(&st->z); + st->z_state = ODB_LOOSE_READ_STREAM_DONE; + break; + } + if (status != Z_OK && (status != Z_BUF_ERROR || total_read < sz)) { + git_inflate_end(&st->z); + st->z_state = ODB_LOOSE_READ_STREAM_ERROR; + return -1; + } + } + return total_read; +} + +static int close_istream_loose(struct odb_read_stream *_st) +{ + struct odb_loose_read_stream *st = (struct odb_loose_read_stream *)_st; + if (st->z_state == ODB_LOOSE_READ_STREAM_INUSE) + git_inflate_end(&st->z); + munmap(st->mapped, st->mapsize); + return 0; +} + +int odb_source_loose_read_object_stream(struct odb_read_stream **out, + struct odb_source *source, + const struct object_id *oid) +{ + struct object_info oi = OBJECT_INFO_INIT; + struct odb_loose_read_stream *st; + unsigned long mapsize; + void *mapped; + + mapped = odb_source_loose_map_object(source, oid, &mapsize); + if (!mapped) + return -1; + + /* + * Note: we must allocate this structure early even though we may still + * fail. This is because we need to initialize the zlib stream, and it + * is not possible to copy the stream around after the fact because it + * has self-referencing pointers. + */ + CALLOC_ARRAY(st, 1); + + switch (unpack_loose_header(&st->z, mapped, mapsize, st->hdr, + sizeof(st->hdr))) { + case ULHR_OK: + break; + case ULHR_BAD: + case ULHR_TOO_LONG: + goto error; + } + + oi.sizep = &st->base.size; + oi.typep = &st->base.type; + + if (parse_loose_header(st->hdr, &oi) < 0 || st->base.type < 0) + goto error; + + st->mapped = mapped; + st->mapsize = mapsize; + st->hdr_used = strlen(st->hdr) + 1; + st->hdr_avail = st->z.total_out; + st->z_state = ODB_LOOSE_READ_STREAM_INUSE; + st->base.close = close_istream_loose; + st->base.read = read_istream_loose; + + *out = &st->base; + + return 0; +error: + git_inflate_end(&st->z); + munmap(st->mapped, st->mapsize); + free(st); + return -1; +} diff --git a/object-file.h b/object-file.h index eeffa67bbd..1229d5f675 100644 --- a/object-file.h +++ b/object-file.h @@ -16,6 +16,8 @@ enum { int index_fd(struct index_state *istate, struct object_id *oid, int fd, struct stat *st, enum object_type type, const char *path, unsigned flags); int index_path(struct index_state *istate, struct object_id *oid, const char *path, struct stat *st, unsigned flags); +struct object_info; +struct odb_read_stream; struct odb_source; struct odb_source_loose { @@ -47,9 +49,9 @@ int odb_source_loose_read_object_info(struct odb_source *source, const struct object_id *oid, struct object_info *oi, int flags); -void *odb_source_loose_map_object(struct odb_source *source, - const struct object_id *oid, - unsigned long *size); +int odb_source_loose_read_object_stream(struct odb_read_stream **out, + struct odb_source *source, + const struct object_id *oid); /* * Return true iff an object database source has a loose object @@ -143,40 +145,6 @@ int for_each_loose_object(struct object_database *odb, int format_object_header(char *str, size_t size, enum object_type type, size_t objsize); -/** - * unpack_loose_header() initializes the data stream needed to unpack - * a loose object header. - * - * Returns: - * - * - ULHR_OK on success - * - ULHR_BAD on error - * - ULHR_TOO_LONG if the header was too long - * - * It will only parse up to MAX_HEADER_LEN bytes. - */ -enum unpack_loose_header_result { - ULHR_OK, - ULHR_BAD, - ULHR_TOO_LONG, -}; -enum unpack_loose_header_result unpack_loose_header(git_zstream *stream, - unsigned char *map, - unsigned long mapsize, - void *buffer, - unsigned long bufsiz); - -/** - * parse_loose_header() parses the starting "<type> <len>\0" of an - * object. If it doesn't follow that format -1 is returned. To check - * the validity of the <type> populate the "typep" in the "struct - * object_info". It will be OBJ_BAD if the object type is unknown. The - * parsed <len> can be retrieved via "oi->sizep", and from there - * passed to unpack_loose_rest(). - */ -struct object_info; -int parse_loose_header(const char *hdr, struct object_info *oi); - int force_object_loose(struct odb_source *source, const struct object_id *oid, time_t mtime); @@ -328,7 +328,7 @@ struct object *parse_object_with_flags(struct repository *r, return &commit->object; } - if ((!obj || obj->type == OBJ_BLOB) && + if ((!obj || obj->type == OBJ_NONE || obj->type == OBJ_BLOB) && odb_read_object_info(r->objects, oid, NULL) == OBJ_BLOB) { if (!skip_hash && stream_object_signature(r, repl) < 0) { error(_("hash mismatch %s"), oid_to_hex(oid)); @@ -344,7 +344,7 @@ struct object *parse_object_with_flags(struct repository *r, * have the on-disk object with the correct type. */ if (skip_hash && discard_tree && - (!obj || obj->type == OBJ_TREE) && + (!obj || obj->type == OBJ_NONE || obj->type == OBJ_TREE) && odb_read_object_info(r->objects, oid, NULL) == OBJ_TREE) { return &lookup_tree(r, oid)->object; } @@ -1,5 +1,6 @@ #include "git-compat-util.h" #include "abspath.h" +#include "chdir-notify.h" #include "commit-graph.h" #include "config.h" #include "dir.h" @@ -9,6 +10,7 @@ #include "khash.h" #include "lockfile.h" #include "loose.h" +#include "midx.h" #include "object-file-convert.h" #include "object-file.h" #include "odb.h" @@ -22,6 +24,7 @@ #include "strbuf.h" #include "strvec.h" #include "submodule.h" +#include "tmp-objdir.h" #include "trace2.h" #include "write-or-die.h" @@ -86,17 +89,20 @@ int odb_mkstemp(struct object_database *odb, /* * Return non-zero iff the path is usable as an alternate object database. */ -static int alt_odb_usable(struct object_database *o, const char *path, - const char *normalized_objdir) +static bool odb_is_source_usable(struct object_database *o, const char *path) { int r; + struct strbuf normalized_objdir = STRBUF_INIT; + bool usable = false; + + strbuf_realpath(&normalized_objdir, o->sources->path, 1); /* Detect cases where alternate disappeared */ if (!is_directory(path)) { error(_("object directory %s does not exist; " "check .git/objects/info/alternates"), path); - return 0; + goto out; } /* @@ -113,37 +119,108 @@ static int alt_odb_usable(struct object_database *o, const char *path, kh_value(o->source_by_path, p) = o->sources; } - if (fspatheq(path, normalized_objdir)) - return 0; + if (fspatheq(path, normalized_objdir.buf)) + goto out; if (kh_get_odb_path_map(o->source_by_path, path) < kh_end(o->source_by_path)) - return 0; + goto out; + + usable = true; - return 1; +out: + strbuf_release(&normalized_objdir); + return usable; } -/* - * Prepare alternate object database registry. - * - * The variable alt_odb_list points at the list of struct - * odb_source. The elements on this list come from - * non-empty elements from colon separated ALTERNATE_DB_ENVIRONMENT - * environment variable, and $GIT_OBJECT_DIRECTORY/info/alternates, - * whose contents is similar to that environment variable but can be - * LF separated. Its base points at a statically allocated buffer that - * contains "/the/directory/corresponding/to/.git/objects/...", while - * its name points just after the slash at the end of ".git/objects/" - * in the example above, and has enough space to hold all hex characters - * of the object ID, an extra slash for the first level indirection, and - * the terminating NUL. - */ -static void read_info_alternates(struct object_database *odb, - const char *relative_base, - int depth); +static void parse_alternates(const char *string, + int sep, + const char *relative_base, + struct strvec *out) +{ + struct strbuf pathbuf = STRBUF_INIT; + struct strbuf buf = STRBUF_INIT; + + if (!string || !*string) + return; + + while (*string) { + const char *end; -struct odb_source *odb_source_new(struct object_database *odb, - const char *path, - bool local) + strbuf_reset(&buf); + strbuf_reset(&pathbuf); + + if (*string == '#') { + /* comment; consume up to next separator */ + end = strchrnul(string, sep); + } else if (*string == '"' && !unquote_c_style(&buf, string, &end)) { + /* + * quoted path; unquote_c_style has copied the + * data for us and set "end". Broken quoting (e.g., + * an entry that doesn't end with a quote) falls + * back to the unquoted case below. + */ + } else { + /* normal, unquoted path */ + end = strchrnul(string, sep); + strbuf_add(&buf, string, end - string); + } + + if (*end) + end++; + string = end; + + if (!buf.len) + continue; + + if (!is_absolute_path(buf.buf) && relative_base) { + strbuf_realpath(&pathbuf, relative_base, 1); + strbuf_addch(&pathbuf, '/'); + } + strbuf_addbuf(&pathbuf, &buf); + + strbuf_reset(&buf); + if (!strbuf_realpath(&buf, pathbuf.buf, 0)) { + error(_("unable to normalize alternate object path: %s"), + pathbuf.buf); + continue; + } + + /* + * The trailing slash after the directory name is given by + * this function at the end. Remove duplicates. + */ + while (buf.len && buf.buf[buf.len - 1] == '/') + strbuf_setlen(&buf, buf.len - 1); + + strvec_push(out, buf.buf); + } + + strbuf_release(&pathbuf); + strbuf_release(&buf); +} + +static void odb_source_read_alternates(struct odb_source *source, + struct strvec *out) +{ + struct strbuf buf = STRBUF_INIT; + char *path; + + path = xstrfmt("%s/info/alternates", source->path); + if (strbuf_read_file(&buf, path, 1024) < 0) { + warn_on_fopen_errors(path); + free(path); + return; + } + parse_alternates(buf.buf, '\n', source->path, out); + + strbuf_release(&buf); + free(path); +} + + +static struct odb_source *odb_source_new(struct object_database *odb, + const char *path, + bool local) { struct odb_source *source; @@ -156,44 +233,19 @@ struct odb_source *odb_source_new(struct object_database *odb, return source; } -static struct odb_source *link_alt_odb_entry(struct object_database *odb, - const char *dir, - const char *relative_base, - int depth) +static struct odb_source *odb_add_alternate_recursively(struct object_database *odb, + const char *source, + int depth) { struct odb_source *alternate = NULL; - struct strbuf pathbuf = STRBUF_INIT; - struct strbuf tmp = STRBUF_INIT; + struct strvec sources = STRVEC_INIT; khiter_t pos; int ret; - if (!is_absolute_path(dir) && relative_base) { - strbuf_realpath(&pathbuf, relative_base, 1); - strbuf_addch(&pathbuf, '/'); - } - strbuf_addstr(&pathbuf, dir); - - if (!strbuf_realpath(&tmp, pathbuf.buf, 0)) { - error(_("unable to normalize alternate object path: %s"), - pathbuf.buf); - goto error; - } - strbuf_swap(&pathbuf, &tmp); - - /* - * The trailing slash after the directory name is given by - * this function at the end. Remove duplicates. - */ - while (pathbuf.len && pathbuf.buf[pathbuf.len - 1] == '/') - strbuf_setlen(&pathbuf, pathbuf.len - 1); - - strbuf_reset(&tmp); - strbuf_realpath(&tmp, odb->sources->path, 1); - - if (!alt_odb_usable(odb, pathbuf.buf, tmp.buf)) + if (!odb_is_source_usable(odb, source)) goto error; - alternate = odb_source_new(odb, pathbuf.buf, false); + alternate = odb_source_new(odb, source, false); /* add the alternate entry */ *odb->sources_tail = alternate; @@ -205,104 +257,42 @@ static struct odb_source *link_alt_odb_entry(struct object_database *odb, kh_value(odb->source_by_path, pos) = alternate; /* recursively add alternates */ - read_info_alternates(odb, alternate->path, depth + 1); - - error: - strbuf_release(&tmp); - strbuf_release(&pathbuf); - return alternate; -} - -static const char *parse_alt_odb_entry(const char *string, - int sep, - struct strbuf *out) -{ - const char *end; - - strbuf_reset(out); - - if (*string == '#') { - /* comment; consume up to next separator */ - end = strchrnul(string, sep); - } else if (*string == '"' && !unquote_c_style(out, string, &end)) { - /* - * quoted path; unquote_c_style has copied the - * data for us and set "end". Broken quoting (e.g., - * an entry that doesn't end with a quote) falls - * back to the unquoted case below. - */ - } else { - /* normal, unquoted path */ - end = strchrnul(string, sep); - strbuf_add(out, string, end - string); - } - - if (*end) - end++; - return end; -} - -static void link_alt_odb_entries(struct object_database *odb, const char *alt, - int sep, const char *relative_base, int depth) -{ - struct strbuf dir = STRBUF_INIT; - - if (!alt || !*alt) - return; - - if (depth > 5) { + odb_source_read_alternates(alternate, &sources); + if (sources.nr && depth + 1 > 5) { error(_("%s: ignoring alternate object stores, nesting too deep"), - relative_base); - return; - } - - while (*alt) { - alt = parse_alt_odb_entry(alt, sep, &dir); - if (!dir.len) - continue; - link_alt_odb_entry(odb, dir.buf, relative_base, depth); - } - strbuf_release(&dir); -} - -static void read_info_alternates(struct object_database *odb, - const char *relative_base, - int depth) -{ - char *path; - struct strbuf buf = STRBUF_INIT; - - path = xstrfmt("%s/info/alternates", relative_base); - if (strbuf_read_file(&buf, path, 1024) < 0) { - warn_on_fopen_errors(path); - free(path); - return; + source); + } else { + for (size_t i = 0; i < sources.nr; i++) + odb_add_alternate_recursively(odb, sources.v[i], depth + 1); } - link_alt_odb_entries(odb, buf.buf, '\n', relative_base, depth); - strbuf_release(&buf); - free(path); + error: + strvec_clear(&sources); + return alternate; } -void odb_add_to_alternates_file(struct object_database *odb, - const char *dir) +static int odb_source_write_alternate(struct odb_source *source, + const char *alternate) { struct lock_file lock = LOCK_INIT; - char *alts = repo_git_path(odb->repo, "objects/info/alternates"); + char *path = xstrfmt("%s/%s", source->path, "info/alternates"); FILE *in, *out; int found = 0; + int ret; - hold_lock_file_for_update(&lock, alts, LOCK_DIE_ON_ERROR); + hold_lock_file_for_update(&lock, path, LOCK_DIE_ON_ERROR); out = fdopen_lock_file(&lock, "w"); - if (!out) - die_errno(_("unable to fdopen alternates lockfile")); + if (!out) { + ret = error_errno(_("unable to fdopen alternates lockfile")); + goto out; + } - in = fopen(alts, "r"); + in = fopen(path, "r"); if (in) { struct strbuf line = STRBUF_INIT; while (strbuf_getline(&line, in) != EOF) { - if (!strcmp(dir, line.buf)) { + if (!strcmp(alternate, line.buf)) { found = 1; break; } @@ -311,20 +301,36 @@ void odb_add_to_alternates_file(struct object_database *odb, strbuf_release(&line); fclose(in); + } else if (errno != ENOENT) { + ret = error_errno(_("unable to read alternates file")); + goto out; } - else if (errno != ENOENT) - die_errno(_("unable to read alternates file")); if (found) { rollback_lock_file(&lock); } else { - fprintf_or_die(out, "%s\n", dir); - if (commit_lock_file(&lock)) - die_errno(_("unable to move new alternates file into place")); - if (odb->loaded_alternates) - link_alt_odb_entries(odb, dir, '\n', NULL, 0); + fprintf_or_die(out, "%s\n", alternate); + if (commit_lock_file(&lock)) { + ret = error_errno(_("unable to move new alternates file into place")); + goto out; + } } - free(alts); + + ret = 0; + +out: + free(path); + return ret; +} + +void odb_add_to_alternates_file(struct object_database *odb, + const char *dir) +{ + int ret = odb_source_write_alternate(odb->sources, dir); + if (ret < 0) + die(NULL); + if (odb->loaded_alternates) + odb_add_alternate_recursively(odb, dir, 0); } struct odb_source *odb_add_to_alternates_memory(struct object_database *odb, @@ -335,7 +341,7 @@ struct odb_source *odb_add_to_alternates_memory(struct object_database *odb, * overwritten when they are. */ odb_prepare_alternates(odb); - return link_alt_odb_entry(odb, dir, NULL, 0); + return odb_add_alternate_recursively(odb, dir, 0); } struct odb_source *odb_set_temporary_primary_source(struct object_database *odb, @@ -359,7 +365,7 @@ struct odb_source *odb_set_temporary_primary_source(struct object_database *odb, * Disable ref updates while a temporary odb is active, since * the objects in the database may roll back. */ - source->disable_ref_updates = 1; + odb->repo->disable_ref_updates = true; source->will_destroy = will_destroy; source->next = odb->sources; odb->sources = source; @@ -386,6 +392,7 @@ void odb_restore_primary_source(struct object_database *odb, if (cur_source->next != restore_source) BUG("we expect the old primary object store to be the first alternate"); + odb->repo->disable_ref_updates = false; odb->sources = restore_source; odb_source_free(cur_source); } @@ -605,13 +612,19 @@ int odb_for_each_alternate(struct object_database *odb, void odb_prepare_alternates(struct object_database *odb) { + struct strvec sources = STRVEC_INIT; + if (odb->loaded_alternates) return; - link_alt_odb_entries(odb, odb->alternate_db, PATH_SEP, NULL, 0); + parse_alternates(odb->alternate_db, PATH_SEP, NULL, &sources); + odb_source_read_alternates(odb->sources, &sources); + for (size_t i = 0; i < sources.nr; i++) + odb_add_alternate_recursively(odb, sources.v[i], 0); - read_info_alternates(odb, odb->sources->path, 0); odb->loaded_alternates = 1; + + strvec_clear(&sources); } int odb_has_alternates(struct object_database *odb) @@ -664,36 +677,31 @@ static int do_oid_object_info_extended(struct object_database *odb, const struct object_id *oid, struct object_info *oi, unsigned flags) { - static struct object_info blank_oi = OBJECT_INFO_INIT; const struct cached_object *co; - struct pack_entry e; - int rtype; const struct object_id *real = oid; int already_retried = 0; - if (flags & OBJECT_INFO_LOOKUP_REPLACE) real = lookup_replace_object(odb->repo, oid); if (is_null_oid(real)) return -1; - if (!oi) - oi = &blank_oi; - co = find_cached_object(odb, real); if (co) { - if (oi->typep) - *(oi->typep) = co->type; - if (oi->sizep) - *(oi->sizep) = co->size; - if (oi->disk_sizep) - *(oi->disk_sizep) = 0; - if (oi->delta_base_oid) - oidclr(oi->delta_base_oid, odb->repo->hash_algo); - if (oi->contentp) - *oi->contentp = xmemdupz(co->buf, co->size); - oi->whence = OI_CACHED; + if (oi) { + if (oi->typep) + *(oi->typep) = co->type; + if (oi->sizep) + *(oi->sizep) = co->size; + if (oi->disk_sizep) + *(oi->disk_sizep) = 0; + if (oi->delta_base_oid) + oidclr(oi->delta_base_oid, odb->repo->hash_algo); + if (oi->contentp) + *oi->contentp = xmemdupz(co->buf, co->size); + oi->whence = OI_CACHED; + } return 0; } @@ -702,8 +710,8 @@ static int do_oid_object_info_extended(struct object_database *odb, while (1) { struct odb_source *source; - if (find_pack_entry(odb->repo, real, &e)) - break; + if (!packfile_store_read_object_info(odb->packfiles, real, oi, flags)) + return 0; /* Most likely it's a loose object. */ for (source = odb->sources; source; source = source->next) @@ -713,8 +721,8 @@ static int do_oid_object_info_extended(struct object_database *odb, /* Not a loose object; someone else may have just packed it. */ if (!(flags & OBJECT_INFO_QUICK)) { odb_reprepare(odb->repo->objects); - if (find_pack_entry(odb->repo, real, &e)) - break; + if (!packfile_store_read_object_info(odb->packfiles, real, oi, flags)) + return 0; } /* @@ -747,25 +755,6 @@ static int do_oid_object_info_extended(struct object_database *odb, } return -1; } - - if (oi == &blank_oi) - /* - * We know that the caller doesn't actually need the - * information below, so return early. - */ - return 0; - rtype = packed_object_info(odb->repo, e.p, e.offset, oi); - if (rtype < 0) { - mark_bad_packed_object(e.p, real); - return do_oid_object_info_extended(odb, real, oi, 0); - } else if (oi->whence == OI_PACKED) { - oi->u.packed.offset = e.offset; - oi->u.packed.pack = e.p; - oi->u.packed.is_delta = (rtype == OBJ_REF_DELTA || - rtype == OBJ_OFS_DELTA); - } - - return 0; } static int oid_object_info_convert(struct repository *r, @@ -1032,18 +1021,79 @@ int odb_write_object_stream(struct object_database *odb, return odb_source_loose_write_stream(odb->sources, stream, len, oid); } -struct object_database *odb_new(struct repository *repo) +static void odb_update_commondir(const char *name UNUSED, + const char *old_cwd, + const char *new_cwd, + void *cb_data) +{ + struct object_database *odb = cb_data; + struct tmp_objdir *tmp_objdir; + struct odb_source *source; + + tmp_objdir = tmp_objdir_unapply_primary_odb(); + + /* + * In theory, we only have to do this for the primary object source, as + * alternates' paths are always resolved to an absolute path. + */ + for (source = odb->sources; source; source = source->next) { + char *path; + + if (is_absolute_path(source->path)) + continue; + + path = reparent_relative_path(old_cwd, new_cwd, + source->path); + + free(source->path); + source->path = path; + } + + if (tmp_objdir) + tmp_objdir_reapply_primary_odb(tmp_objdir, old_cwd, new_cwd); +} + +struct object_database *odb_new(struct repository *repo, + const char *primary_source, + const char *secondary_sources) { struct object_database *o = xmalloc(sizeof(*o)); + char *to_free = NULL; memset(o, 0, sizeof(*o)); o->repo = repo; o->packfiles = packfile_store_new(o); pthread_mutex_init(&o->replace_mutex, NULL); string_list_init_dup(&o->submodule_source_paths); + + if (!primary_source) + primary_source = to_free = xstrfmt("%s/objects", repo->commondir); + o->sources = odb_source_new(o, primary_source, true); + o->sources_tail = &o->sources->next; + o->alternate_db = xstrdup_or_null(secondary_sources); + + free(to_free); + + chdir_notify_register(NULL, odb_update_commondir, o); + return o; } +void odb_close(struct object_database *o) +{ + struct odb_source *source; + + packfile_store_close(o->packfiles); + + for (source = o->sources; source; source = source->next) { + if (source->midx) + close_midx(source->midx); + source->midx = NULL; + } + + close_commit_graph(o); +} + static void odb_free_sources(struct object_database *o) { while (o->sources) { @@ -1057,30 +1107,29 @@ static void odb_free_sources(struct object_database *o) o->source_by_path = NULL; } -void odb_clear(struct object_database *o) +void odb_free(struct object_database *o) { - FREE_AND_NULL(o->alternate_db); + if (!o) + return; + + free(o->alternate_db); oidmap_clear(&o->replace_map, 1); pthread_mutex_destroy(&o->replace_mutex); - free_commit_graph(o->commit_graph); - o->commit_graph = NULL; - o->commit_graph_attempted = 0; - odb_free_sources(o); - o->sources_tail = NULL; - o->loaded_alternates = 0; for (size_t i = 0; i < o->cached_object_nr; i++) free((char *) o->cached_objects[i].value.buf); - FREE_AND_NULL(o->cached_objects); + free(o->cached_objects); - close_object_store(o); + odb_close(o); packfile_store_free(o->packfiles); - o->packfiles = NULL; - string_list_clear(&o->submodule_source_paths, 0); + + chdir_notify_unregister(NULL, odb_update_commondir, o); + + free(o); } void odb_reprepare(struct object_database *o) @@ -67,13 +67,6 @@ struct odb_source { bool local; /* - * This is a temporary object store created by the tmp_objdir - * facility. Disable ref updates since the objects in the store - * might be discarded on rollback. - */ - int disable_ref_updates; - - /* * This object store is ephemeral, so there is no need to fsync. */ int will_destroy; @@ -85,10 +78,6 @@ struct odb_source { char *path; }; -struct odb_source *odb_source_new(struct object_database *odb, - const char *path, - bool local); - struct packed_git; struct packfile_store; struct cached_object_entry; @@ -166,8 +155,30 @@ struct object_database { struct string_list submodule_source_paths; }; -struct object_database *odb_new(struct repository *repo); -void odb_clear(struct object_database *o); +/* + * Create a new object database for the given repository. + * + * If the primary source parameter is set it will override the usual primary + * object directory derived from the repository's common directory. The + * alternate sources are expected to be a PATH_SEP-separated list of secondary + * sources. Note that these alternate sources will be added in addition to, not + * instead of, the alternates identified by the primary source. + * + * Returns the newly created object database. + */ +struct object_database *odb_new(struct repository *repo, + const char *primary_source, + const char *alternate_sources); + +/* Free the object database and release all resources. */ +void odb_free(struct object_database *o); + +/* + * Close the object database and all of its sources so that any held resources + * will be released. The database can still be used after closing it, in which + * case these resources may be reallocated. + */ +void odb_close(struct object_database *o); /* * Clear caches, reload alternates and then reload object sources so that new diff --git a/odb/streaming.c b/odb/streaming.c new file mode 100644 index 0000000000..745cd486fb --- /dev/null +++ b/odb/streaming.c @@ -0,0 +1,293 @@ +/* + * Copyright (c) 2011, Google Inc. + */ + +#include "git-compat-util.h" +#include "convert.h" +#include "environment.h" +#include "repository.h" +#include "object-file.h" +#include "odb.h" +#include "odb/streaming.h" +#include "replace-object.h" +#include "packfile.h" + +#define FILTER_BUFFER (1024*16) + +/***************************************************************** + * + * Filtered stream + * + *****************************************************************/ + +struct odb_filtered_read_stream { + struct odb_read_stream base; + struct odb_read_stream *upstream; + struct stream_filter *filter; + char ibuf[FILTER_BUFFER]; + char obuf[FILTER_BUFFER]; + int i_end, i_ptr; + int o_end, o_ptr; + int input_finished; +}; + +static int close_istream_filtered(struct odb_read_stream *_fs) +{ + struct odb_filtered_read_stream *fs = (struct odb_filtered_read_stream *)_fs; + free_stream_filter(fs->filter); + return odb_read_stream_close(fs->upstream); +} + +static ssize_t read_istream_filtered(struct odb_read_stream *_fs, char *buf, + size_t sz) +{ + struct odb_filtered_read_stream *fs = (struct odb_filtered_read_stream *)_fs; + size_t filled = 0; + + while (sz) { + /* do we already have filtered output? */ + if (fs->o_ptr < fs->o_end) { + size_t to_move = fs->o_end - fs->o_ptr; + if (sz < to_move) + to_move = sz; + memcpy(buf + filled, fs->obuf + fs->o_ptr, to_move); + fs->o_ptr += to_move; + sz -= to_move; + filled += to_move; + continue; + } + fs->o_end = fs->o_ptr = 0; + + /* do we have anything to feed the filter with? */ + if (fs->i_ptr < fs->i_end) { + size_t to_feed = fs->i_end - fs->i_ptr; + size_t to_receive = FILTER_BUFFER; + if (stream_filter(fs->filter, + fs->ibuf + fs->i_ptr, &to_feed, + fs->obuf, &to_receive)) + return -1; + fs->i_ptr = fs->i_end - to_feed; + fs->o_end = FILTER_BUFFER - to_receive; + continue; + } + + /* tell the filter to drain upon no more input */ + if (fs->input_finished) { + size_t to_receive = FILTER_BUFFER; + if (stream_filter(fs->filter, + NULL, NULL, + fs->obuf, &to_receive)) + return -1; + fs->o_end = FILTER_BUFFER - to_receive; + if (!fs->o_end) + break; + continue; + } + fs->i_end = fs->i_ptr = 0; + + /* refill the input from the upstream */ + if (!fs->input_finished) { + fs->i_end = odb_read_stream_read(fs->upstream, fs->ibuf, FILTER_BUFFER); + if (fs->i_end < 0) + return -1; + if (fs->i_end) + continue; + } + fs->input_finished = 1; + } + return filled; +} + +static struct odb_read_stream *attach_stream_filter(struct odb_read_stream *st, + struct stream_filter *filter) +{ + struct odb_filtered_read_stream *fs; + + CALLOC_ARRAY(fs, 1); + fs->base.close = close_istream_filtered; + fs->base.read = read_istream_filtered; + fs->upstream = st; + fs->filter = filter; + fs->base.size = -1; /* unknown */ + fs->base.type = st->type; + + return &fs->base; +} + +/***************************************************************** + * + * In-core stream + * + *****************************************************************/ + +struct odb_incore_read_stream { + struct odb_read_stream base; + char *buf; /* from odb_read_object_info_extended() */ + unsigned long read_ptr; +}; + +static int close_istream_incore(struct odb_read_stream *_st) +{ + struct odb_incore_read_stream *st = (struct odb_incore_read_stream *)_st; + free(st->buf); + return 0; +} + +static ssize_t read_istream_incore(struct odb_read_stream *_st, char *buf, size_t sz) +{ + struct odb_incore_read_stream *st = (struct odb_incore_read_stream *)_st; + size_t read_size = sz; + size_t remainder = st->base.size - st->read_ptr; + + if (remainder <= read_size) + read_size = remainder; + if (read_size) { + memcpy(buf, st->buf + st->read_ptr, read_size); + st->read_ptr += read_size; + } + return read_size; +} + +static int open_istream_incore(struct odb_read_stream **out, + struct object_database *odb, + const struct object_id *oid) +{ + struct object_info oi = OBJECT_INFO_INIT; + struct odb_incore_read_stream stream = { + .base.close = close_istream_incore, + .base.read = read_istream_incore, + }; + struct odb_incore_read_stream *st; + int ret; + + oi.typep = &stream.base.type; + oi.sizep = &stream.base.size; + oi.contentp = (void **)&stream.buf; + ret = odb_read_object_info_extended(odb, oid, &oi, + OBJECT_INFO_DIE_IF_CORRUPT); + if (ret) + return ret; + + CALLOC_ARRAY(st, 1); + *st = stream; + *out = &st->base; + + return 0; +} + +/***************************************************************************** + * static helpers variables and functions for users of streaming interface + *****************************************************************************/ + +static int istream_source(struct odb_read_stream **out, + struct object_database *odb, + const struct object_id *oid) +{ + struct odb_source *source; + + if (!packfile_store_read_object_stream(out, odb->packfiles, oid)) + return 0; + + odb_prepare_alternates(odb); + for (source = odb->sources; source; source = source->next) + if (!odb_source_loose_read_object_stream(out, source, oid)) + return 0; + + return open_istream_incore(out, odb, oid); +} + +/**************************************************************** + * Users of streaming interface + ****************************************************************/ + +int odb_read_stream_close(struct odb_read_stream *st) +{ + int r = st->close(st); + free(st); + return r; +} + +ssize_t odb_read_stream_read(struct odb_read_stream *st, void *buf, size_t sz) +{ + return st->read(st, buf, sz); +} + +struct odb_read_stream *odb_read_stream_open(struct object_database *odb, + const struct object_id *oid, + struct stream_filter *filter) +{ + struct odb_read_stream *st; + const struct object_id *real = lookup_replace_object(odb->repo, oid); + int ret = istream_source(&st, odb, real); + + if (ret) + return NULL; + + if (filter) { + /* Add "&& !is_null_stream_filter(filter)" for performance */ + struct odb_read_stream *nst = attach_stream_filter(st, filter); + if (!nst) { + odb_read_stream_close(st); + return NULL; + } + st = nst; + } + + return st; +} + +int odb_stream_blob_to_fd(struct object_database *odb, + int fd, + const struct object_id *oid, + struct stream_filter *filter, + int can_seek) +{ + struct odb_read_stream *st; + ssize_t kept = 0; + int result = -1; + + st = odb_read_stream_open(odb, oid, filter); + if (!st) { + if (filter) + free_stream_filter(filter); + return result; + } + if (st->type != OBJ_BLOB) + goto close_and_exit; + for (;;) { + char buf[1024 * 16]; + ssize_t wrote, holeto; + ssize_t readlen = odb_read_stream_read(st, buf, sizeof(buf)); + + if (readlen < 0) + goto close_and_exit; + if (!readlen) + break; + if (can_seek && sizeof(buf) == readlen) { + for (holeto = 0; holeto < readlen; holeto++) + if (buf[holeto]) + break; + if (readlen == holeto) { + kept += holeto; + continue; + } + } + + if (kept && lseek(fd, kept, SEEK_CUR) == (off_t) -1) + goto close_and_exit; + else + kept = 0; + wrote = write_in_full(fd, buf, readlen); + + if (wrote < 0) + goto close_and_exit; + } + if (kept && (lseek(fd, kept - 1, SEEK_CUR) == (off_t) -1 || + xwrite(fd, "", 1) != 1)) + goto close_and_exit; + result = 0; + + close_and_exit: + odb_read_stream_close(st); + return result; +} diff --git a/odb/streaming.h b/odb/streaming.h new file mode 100644 index 0000000000..c7861f7e13 --- /dev/null +++ b/odb/streaming.h @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2011, Google Inc. + */ +#ifndef STREAMING_H +#define STREAMING_H 1 + +#include "object.h" + +struct object_database; +struct odb_read_stream; +struct stream_filter; + +typedef int (*odb_read_stream_close_fn)(struct odb_read_stream *); +typedef ssize_t (*odb_read_stream_read_fn)(struct odb_read_stream *, char *, size_t); + +/* + * A stream that can be used to read an object from the object database without + * loading all of it into memory. + */ +struct odb_read_stream { + odb_read_stream_close_fn close; + odb_read_stream_read_fn read; + enum object_type type; + unsigned long size; /* inflated size of full object */ +}; + +/* + * Create a new object stream for the given object database. An optional filter + * can be used to transform the object's content. + * + * Returns the stream on success, a `NULL` pointer otherwise. + */ +struct odb_read_stream *odb_read_stream_open(struct object_database *odb, + const struct object_id *oid, + struct stream_filter *filter); + +/* + * Close the given read stream and release all resources associated with it. + * Returns 0 on success, a negative error code otherwise. + */ +int odb_read_stream_close(struct odb_read_stream *stream); + +/* + * Read data from the stream into the buffer. Returns 0 on EOF and the number + * of bytes read on success. Returns a negative error code in case reading from + * the stream fails. + */ +ssize_t odb_read_stream_read(struct odb_read_stream *stream, void *buf, size_t len); + +/* + * Look up the object by its ID and write the full contents to the file + * descriptor. The object must be a blob, or the function will fail. When + * provided, the filter is used to transform the blob contents. + * + * `can_seek` should be set to 1 in case the given file descriptor can be + * seek(3p)'d on. This is used to support files with holes in case a + * significant portion of the blob contains NUL bytes. + * + * Returns a negative error code on failure, 0 on success. + */ +int odb_stream_blob_to_fd(struct object_database *odb, + int fd, + const struct object_id *oid, + struct stream_filter *filter, + int can_seek); + +#endif /* STREAMING_H */ @@ -16,6 +16,22 @@ int oidset_contains(const struct oidset *set, const struct object_id *oid) return pos != kh_end(&set->set); } +bool oidset_equal(const struct oidset *a, const struct oidset *b) +{ + struct oidset_iter iter; + struct object_id *a_oid; + + if (oidset_size(a) != oidset_size(b)) + return false; + + oidset_iter_init(a, &iter); + while ((a_oid = oidset_iter_next(&iter))) + if (!oidset_contains(b, a_oid)) + return false; + + return true; +} + int oidset_insert(struct oidset *set, const struct object_id *oid) { int added; @@ -39,6 +39,11 @@ void oidset_init(struct oidset *set, size_t initial_size); int oidset_contains(const struct oidset *set, const struct object_id *oid); /** + * Returns true iff `a` and `b` contain the exact same OIDs. + */ +bool oidset_equal(const struct oidset *a, const struct oidset *b); + +/** * Insert the oid into the set; a copy is made, so "oid" does not need * to persist after this function is called. * @@ -94,11 +99,11 @@ void oidset_parse_file_carefully(struct oidset *set, const char *path, oidset_parse_tweak_fn fn, void *cbdata); struct oidset_iter { - kh_oid_set_t *set; + const kh_oid_set_t *set; khiter_t iter; }; -static inline void oidset_iter_init(struct oidset *set, +static inline void oidset_iter_init(const struct oidset *set, struct oidset_iter *iter) { iter->set = &set->set; diff --git a/pack-revindex.c b/pack-revindex.c index d0791cc493..8598b941c8 100644 --- a/pack-revindex.c +++ b/pack-revindex.c @@ -75,7 +75,7 @@ static void sort_revindex(struct revindex_entry *entries, unsigned n, off_t max) for (bits = 0; max >> bits; bits += DIGIT_SIZE) { unsigned i; - memset(pos, 0, BUCKETS * sizeof(*pos)); + MEMZERO_ARRAY(pos, BUCKETS); /* * We want pos[i] to store the index of the last element that diff --git a/pack-revindex.h b/pack-revindex.h index 422c2487ae..0042892091 100644 --- a/pack-revindex.h +++ b/pack-revindex.h @@ -72,7 +72,8 @@ int verify_pack_revindex(struct packed_git *p); * multi-pack index by mmap-ing it and assigning pointers in the * multi_pack_index to point at it. * - * A negative number is returned on error. + * A negative number is returned on error. A positive number is returned in + * case the multi-pack-index does not have a reverse index. */ int load_midx_revindex(struct multi_pack_index *m); diff --git a/packfile.c b/packfile.c index 9cc11b6dc5..23a7f8a191 100644 --- a/packfile.c +++ b/packfile.c @@ -20,6 +20,7 @@ #include "tree.h" #include "object-file.h" #include "odb.h" +#include "odb/streaming.h" #include "midx.h" #include "commit-graph.h" #include "pack-revindex.h" @@ -443,21 +444,6 @@ void close_pack(struct packed_git *p) oidset_clear(&p->bad_objects); } -void close_object_store(struct object_database *o) -{ - struct odb_source *source; - - packfile_store_close(o->packfiles); - - for (source = o->sources; source; source = source->next) { - if (source->midx) - close_midx(source->midx); - source->midx = NULL; - } - - close_commit_graph(o); -} - void unlink_pack_path(const char *pack_name, int force_delete) { static const char *exts[] = {".idx", ".pack", ".rev", ".keep", ".bitmap", ".promisor", ".mtimes"}; @@ -900,22 +886,6 @@ struct packed_git *packfile_store_load_pack(struct packfile_store *store, return p; } -int packfile_store_freshen_object(struct packfile_store *store, - const struct object_id *oid) -{ - struct pack_entry e; - if (!find_pack_entry(store->odb->repo, oid, &e)) - return 0; - if (e.p->is_cruft) - return 0; - if (e.p->freshened) - return 1; - if (utime(e.p->pack_name, NULL)) - return 0; - e.p->freshened = 1; - return 1; -} - void (*report_garbage)(unsigned seen_bits, const char *path); static void report_helper(const struct string_list *list, @@ -2120,7 +2090,9 @@ static int fill_pack_entry(const struct object_id *oid, return 1; } -int find_pack_entry(struct repository *r, const struct object_id *oid, struct pack_entry *e) +static int find_pack_entry(struct repository *r, + const struct object_id *oid, + struct pack_entry *e) { struct packfile_list_entry *l; @@ -2145,6 +2117,56 @@ int find_pack_entry(struct repository *r, const struct object_id *oid, struct pa return 0; } +int packfile_store_freshen_object(struct packfile_store *store, + const struct object_id *oid) +{ + struct pack_entry e; + if (!find_pack_entry(store->odb->repo, oid, &e)) + return 0; + if (e.p->is_cruft) + return 0; + if (e.p->freshened) + return 1; + if (utime(e.p->pack_name, NULL)) + return 0; + e.p->freshened = 1; + return 1; +} + +int packfile_store_read_object_info(struct packfile_store *store, + const struct object_id *oid, + struct object_info *oi, + unsigned flags UNUSED) +{ + struct pack_entry e; + int rtype; + + if (!find_pack_entry(store->odb->repo, oid, &e)) + return 1; + + /* + * We know that the caller doesn't actually need the + * information below, so return early. + */ + if (!oi) + return 0; + + rtype = packed_object_info(store->odb->repo, e.p, e.offset, oi); + if (rtype < 0) { + mark_bad_packed_object(e.p, oid); + return -1; + } + + if (oi->whence == OI_PACKED) { + oi->u.packed.offset = e.offset; + oi->u.packed.pack = e.p; + oi->u.packed.is_delta = (rtype == OBJ_REF_DELTA || + rtype == OBJ_OFS_DELTA); + } + + return 0; +} + static void maybe_invalidate_kept_pack_cache(struct repository *r, unsigned flags) { @@ -2310,7 +2332,8 @@ static int add_promisor_object(const struct object_id *oid, we_parsed_object = 0; } else { we_parsed_object = 1; - obj = parse_object(pack->repo, oid); + obj = parse_object_with_flags(pack->repo, oid, + PARSE_OBJECT_SKIP_HASH_CHECK); } if (!obj) @@ -2415,3 +2438,130 @@ void packfile_store_close(struct packfile_store *store) close_pack(e->pack); } } + +struct odb_packed_read_stream { + struct odb_read_stream base; + struct packed_git *pack; + git_zstream z; + enum { + ODB_PACKED_READ_STREAM_UNINITIALIZED, + ODB_PACKED_READ_STREAM_INUSE, + ODB_PACKED_READ_STREAM_DONE, + ODB_PACKED_READ_STREAM_ERROR, + } z_state; + off_t pos; +}; + +static ssize_t read_istream_pack_non_delta(struct odb_read_stream *_st, char *buf, + size_t sz) +{ + struct odb_packed_read_stream *st = (struct odb_packed_read_stream *)_st; + size_t total_read = 0; + + switch (st->z_state) { + case ODB_PACKED_READ_STREAM_UNINITIALIZED: + memset(&st->z, 0, sizeof(st->z)); + git_inflate_init(&st->z); + st->z_state = ODB_PACKED_READ_STREAM_INUSE; + break; + case ODB_PACKED_READ_STREAM_DONE: + return 0; + case ODB_PACKED_READ_STREAM_ERROR: + return -1; + case ODB_PACKED_READ_STREAM_INUSE: + break; + } + + while (total_read < sz) { + int status; + struct pack_window *window = NULL; + unsigned char *mapped; + + mapped = use_pack(st->pack, &window, + st->pos, &st->z.avail_in); + + st->z.next_out = (unsigned char *)buf + total_read; + st->z.avail_out = sz - total_read; + st->z.next_in = mapped; + status = git_inflate(&st->z, Z_FINISH); + + st->pos += st->z.next_in - mapped; + total_read = st->z.next_out - (unsigned char *)buf; + unuse_pack(&window); + + if (status == Z_STREAM_END) { + git_inflate_end(&st->z); + st->z_state = ODB_PACKED_READ_STREAM_DONE; + break; + } + + /* + * Unlike the loose object case, we do not have to worry here + * about running out of input bytes and spinning infinitely. If + * we get Z_BUF_ERROR due to too few input bytes, then we'll + * replenish them in the next use_pack() call when we loop. If + * we truly hit the end of the pack (i.e., because it's corrupt + * or truncated), then use_pack() catches that and will die(). + */ + if (status != Z_OK && status != Z_BUF_ERROR) { + git_inflate_end(&st->z); + st->z_state = ODB_PACKED_READ_STREAM_ERROR; + return -1; + } + } + return total_read; +} + +static int close_istream_pack_non_delta(struct odb_read_stream *_st) +{ + struct odb_packed_read_stream *st = (struct odb_packed_read_stream *)_st; + if (st->z_state == ODB_PACKED_READ_STREAM_INUSE) + git_inflate_end(&st->z); + return 0; +} + +int packfile_store_read_object_stream(struct odb_read_stream **out, + struct packfile_store *store, + const struct object_id *oid) +{ + struct odb_packed_read_stream *stream; + struct pack_window *window = NULL; + struct object_info oi = OBJECT_INFO_INIT; + enum object_type in_pack_type; + unsigned long size; + + oi.sizep = &size; + + if (packfile_store_read_object_info(store, oid, &oi, 0) || + oi.u.packed.is_delta || + repo_settings_get_big_file_threshold(store->odb->repo) >= size) + return -1; + + in_pack_type = unpack_object_header(oi.u.packed.pack, + &window, + &oi.u.packed.offset, + &size); + unuse_pack(&window); + switch (in_pack_type) { + default: + return -1; /* we do not do deltas for now */ + case OBJ_COMMIT: + case OBJ_TREE: + case OBJ_BLOB: + case OBJ_TAG: + break; + } + + CALLOC_ARRAY(stream, 1); + stream->base.close = close_istream_pack_non_delta; + stream->base.read = read_istream_pack_non_delta; + stream->base.type = in_pack_type; + stream->base.size = size; + stream->z_state = ODB_PACKED_READ_STREAM_UNINITIALIZED; + stream->pack = oi.u.packed.pack; + stream->pos = oi.u.packed.offset; + + *out = &stream->base; + + return 0; +} diff --git a/packfile.h b/packfile.h index 0e178fb279..59d162a3f4 100644 --- a/packfile.h +++ b/packfile.h @@ -9,6 +9,7 @@ /* in odb.h */ struct object_info; +struct odb_read_stream; struct packed_git { struct pack_window *windows; @@ -177,6 +178,21 @@ void packfile_store_add_pack(struct packfile_store *store, for (struct packfile_list_entry *e = packfile_store_get_packs(repo->objects->packfiles); \ ((p) = (e ? e->pack : NULL)); e = e->next) +int packfile_store_read_object_stream(struct odb_read_stream **out, + struct packfile_store *store, + const struct object_id *oid); + +/* + * Try to read the object identified by its ID from the object store and + * populate the object info with its data. Returns 1 in case the object was + * not found, 0 if it was and read successfully, and a negative error code in + * case the object was corrupted. + */ +int packfile_store_read_object_info(struct packfile_store *store, + const struct object_id *oid, + struct object_info *oi, + unsigned flags); + /* * Get all packs managed by the given store, including packfiles that are * referenced by multi-pack indices. @@ -299,7 +315,6 @@ struct object_database; unsigned char *use_pack(struct packed_git *, struct pack_window **, off_t, unsigned long *); void close_pack_windows(struct packed_git *); void close_pack(struct packed_git *); -void close_object_store(struct object_database *o); void unuse_pack(struct pack_window **); void clear_delta_base_cache(void); struct packed_git *add_packed_git(struct repository *r, const char *path, @@ -377,7 +392,6 @@ const struct packed_git *has_packed_and_bad(struct repository *, const struct ob * Iff a pack file in the given repository contains the object named by sha1, * return true and store its location to e. */ -int find_pack_entry(struct repository *r, const struct object_id *oid, struct pack_entry *e); int find_kept_pack_entry(struct repository *r, const struct object_id *oid, unsigned flags, struct pack_entry *e); int has_object_pack(struct repository *r, const struct object_id *oid); diff --git a/parallel-checkout.c b/parallel-checkout.c index fba6aa65a6..0bf4bd6d4a 100644 --- a/parallel-checkout.c +++ b/parallel-checkout.c @@ -13,7 +13,7 @@ #include "read-cache-ll.h" #include "run-command.h" #include "sigchain.h" -#include "streaming.h" +#include "odb/streaming.h" #include "symlinks.h" #include "thread-utils.h" #include "trace2.h" @@ -281,7 +281,8 @@ static int write_pc_item_to_fd(struct parallel_checkout_item *pc_item, int fd, filter = get_stream_filter_ca(&pc_item->ca, &pc_item->ce->oid); if (filter) { - if (stream_blob_to_fd(fd, &pc_item->ce->oid, filter, 1)) { + if (odb_stream_blob_to_fd(the_repository->objects, fd, + &pc_item->ce->oid, filter, 1)) { /* On error, reset fd to try writing without streaming */ if (reset_fd(fd, path)) return -1; @@ -738,106 +738,6 @@ return_null: return NULL; } -/* - * First, one directory to try is determined by the following algorithm. - * - * (0) If "strict" is given, the path is used as given and no DWIM is - * done. Otherwise: - * (1) "~/path" to mean path under the running user's home directory; - * (2) "~user/path" to mean path under named user's home directory; - * (3) "relative/path" to mean cwd relative directory; or - * (4) "/absolute/path" to mean absolute directory. - * - * Unless "strict" is given, we check "%s/.git", "%s", "%s.git/.git", "%s.git" - * in this order. We select the first one that is a valid git repository, and - * chdir() to it. If none match, or we fail to chdir, we return NULL. - * - * If all goes well, we return the directory we used to chdir() (but - * before ~user is expanded), avoiding getcwd() resolving symbolic - * links. User relative paths are also returned as they are given, - * except DWIM suffixing. - */ -const char *enter_repo(const char *path, unsigned flags) -{ - static struct strbuf validated_path = STRBUF_INIT; - static struct strbuf used_path = STRBUF_INIT; - - if (!path) - return NULL; - - if (!(flags & ENTER_REPO_STRICT)) { - static const char *suffix[] = { - "/.git", "", ".git/.git", ".git", NULL, - }; - const char *gitfile; - int len = strlen(path); - int i; - while ((1 < len) && (path[len-1] == '/')) - len--; - - /* - * We can handle arbitrary-sized buffers, but this remains as a - * sanity check on untrusted input. - */ - if (PATH_MAX <= len) - return NULL; - - strbuf_reset(&used_path); - strbuf_reset(&validated_path); - strbuf_add(&used_path, path, len); - strbuf_add(&validated_path, path, len); - - if (used_path.buf[0] == '~') { - char *newpath = interpolate_path(used_path.buf, 0); - if (!newpath) - return NULL; - strbuf_attach(&used_path, newpath, strlen(newpath), - strlen(newpath)); - } - for (i = 0; suffix[i]; i++) { - struct stat st; - size_t baselen = used_path.len; - strbuf_addstr(&used_path, suffix[i]); - if (!stat(used_path.buf, &st) && - (S_ISREG(st.st_mode) || - (S_ISDIR(st.st_mode) && is_git_directory(used_path.buf)))) { - strbuf_addstr(&validated_path, suffix[i]); - break; - } - strbuf_setlen(&used_path, baselen); - } - if (!suffix[i]) - return NULL; - gitfile = read_gitfile(used_path.buf); - if (!(flags & ENTER_REPO_ANY_OWNER_OK)) - die_upon_dubious_ownership(gitfile, NULL, used_path.buf); - if (gitfile) { - strbuf_reset(&used_path); - strbuf_addstr(&used_path, gitfile); - } - if (chdir(used_path.buf)) - return NULL; - path = validated_path.buf; - } - else { - const char *gitfile = read_gitfile(path); - if (!(flags & ENTER_REPO_ANY_OWNER_OK)) - die_upon_dubious_ownership(gitfile, NULL, path); - if (gitfile) - path = gitfile; - if (chdir(path)) - return NULL; - } - - if (is_git_directory(".")) { - set_git_dir(".", 0); - check_repository_format(NULL); - return path; - } - - return NULL; -} - int calc_shared_perm(struct repository *repo, int mode) { @@ -146,21 +146,6 @@ int adjust_shared_perm(struct repository *repo, const char *path); char *interpolate_path(const char *path, int real_home); -/* The bits are as follows: - * - * - ENTER_REPO_STRICT: callers that require exact paths (as opposed - * to allowing known suffixes like ".git", ".git/.git" to be - * omitted) can set this bit. - * - * - ENTER_REPO_ANY_OWNER_OK: callers that are willing to run without - * ownership check can set this bit. - */ -enum { - ENTER_REPO_STRICT = (1<<0), - ENTER_REPO_ANY_OWNER_OK = (1<<1), -}; - -const char *enter_repo(const char *path, unsigned flags); const char *remove_leading_path(const char *in, const char *prefix); const char *relative_path(const char *in, const char *prefix, struct strbuf *sb); int normalize_path_copy_len(char *dst, const char *src, int *prefix_len); @@ -2422,68 +2422,72 @@ static int ref_update_reject_duplicates(struct string_list *refnames, return 0; } -static int run_transaction_hook(struct ref_transaction *transaction, - const char *state) +struct transaction_feed_cb_data { + size_t index; + struct strbuf buf; +}; + +static int transaction_hook_feed_stdin(int hook_stdin_fd, void *pp_cb, void *pp_task_cb) { - struct child_process proc = CHILD_PROCESS_INIT; - struct strbuf buf = STRBUF_INIT; - const char *hook; - int ret = 0; + struct hook_cb_data *hook_cb = pp_cb; + struct ref_transaction *transaction = hook_cb->options->feed_pipe_ctx; + struct transaction_feed_cb_data *feed_cb_data = pp_task_cb; + struct strbuf *buf = &feed_cb_data->buf; + struct ref_update *update; + size_t i = feed_cb_data->index++; + int ret; - hook = find_hook(transaction->ref_store->repo, "reference-transaction"); - if (!hook) - return ret; + if (i >= transaction->nr) + return 1; /* No more refs to process */ - strvec_pushl(&proc.args, hook, state, NULL); - proc.in = -1; - proc.stdout_to_stderr = 1; - proc.trace2_hook_name = "reference-transaction"; + update = transaction->updates[i]; - ret = start_command(&proc); - if (ret) - return ret; + if (update->flags & REF_LOG_ONLY) + return 0; - sigchain_push(SIGPIPE, SIG_IGN); + strbuf_reset(buf); - for (size_t i = 0; i < transaction->nr; i++) { - struct ref_update *update = transaction->updates[i]; + if (!(update->flags & REF_HAVE_OLD)) + strbuf_addf(buf, "%s ", oid_to_hex(null_oid(the_hash_algo))); + else if (update->old_target) + strbuf_addf(buf, "ref:%s ", update->old_target); + else + strbuf_addf(buf, "%s ", oid_to_hex(&update->old_oid)); - if (update->flags & REF_LOG_ONLY) - continue; + if (!(update->flags & REF_HAVE_NEW)) + strbuf_addf(buf, "%s ", oid_to_hex(null_oid(the_hash_algo))); + else if (update->new_target) + strbuf_addf(buf, "ref:%s ", update->new_target); + else + strbuf_addf(buf, "%s ", oid_to_hex(&update->new_oid)); - strbuf_reset(&buf); + strbuf_addf(buf, "%s\n", update->refname); - if (!(update->flags & REF_HAVE_OLD)) - strbuf_addf(&buf, "%s ", oid_to_hex(null_oid(the_hash_algo))); - else if (update->old_target) - strbuf_addf(&buf, "ref:%s ", update->old_target); - else - strbuf_addf(&buf, "%s ", oid_to_hex(&update->old_oid)); + ret = write_in_full(hook_stdin_fd, buf->buf, buf->len); + if (ret < 0 && errno != EPIPE) + return ret; - if (!(update->flags & REF_HAVE_NEW)) - strbuf_addf(&buf, "%s ", oid_to_hex(null_oid(the_hash_algo))); - else if (update->new_target) - strbuf_addf(&buf, "ref:%s ", update->new_target); - else - strbuf_addf(&buf, "%s ", oid_to_hex(&update->new_oid)); + return 0; /* no more input to feed */ +} + +static int run_transaction_hook(struct ref_transaction *transaction, + const char *state) +{ + struct run_hooks_opt opt = RUN_HOOKS_OPT_INIT; + struct transaction_feed_cb_data feed_ctx = { 0 }; + int ret = 0; - strbuf_addf(&buf, "%s\n", update->refname); + strvec_push(&opt.args, state); - if (write_in_full(proc.in, buf.buf, buf.len) < 0) { - if (errno != EPIPE) { - /* Don't leak errno outside this API */ - errno = 0; - ret = -1; - } - break; - } - } + opt.feed_pipe = transaction_hook_feed_stdin; + opt.feed_pipe_ctx = transaction; + opt.feed_pipe_cb_data = &feed_ctx; - close(proc.in); - sigchain_pop(SIGPIPE); - strbuf_release(&buf); + strbuf_init(&feed_ctx.buf, 0); + + ret = run_hooks_opt(transaction->ref_store->repo, "reference-transaction", &opt); - ret |= finish_command(&proc); + strbuf_release(&feed_ctx.buf); return ret; } @@ -2508,7 +2512,7 @@ int ref_transaction_prepare(struct ref_transaction *transaction, break; } - if (refs->repo->objects->sources->disable_ref_updates) { + if (refs->repo->disable_ref_updates) { strbuf_addstr(err, _("ref updates forbidden inside quarantine environment")); return -1; diff --git a/refs/debug.c b/refs/debug.c index 3e31228c9a..639db0f26e 100644 --- a/refs/debug.c +++ b/refs/debug.c @@ -139,7 +139,7 @@ static int debug_optimize_required(struct ref_store *ref_store, struct debug_ref_store *drefs = (struct debug_ref_store *)ref_store; int res = drefs->refs->be->optimize_required(drefs->refs, opts, required); trace_printf_key(&trace_refs, "optimize_required: %s, res: %d\n", - required ? "yes" : "no", res); + *required ? "yes" : "no", res); return res; } diff --git a/replay.c b/replay.c new file mode 100644 index 0000000000..94fb76384b --- /dev/null +++ b/replay.c @@ -0,0 +1,371 @@ +#define USE_THE_REPOSITORY_VARIABLE + +#include "git-compat-util.h" +#include "environment.h" +#include "hex.h" +#include "merge-ort.h" +#include "object-name.h" +#include "refs.h" +#include "replay.h" +#include "revision.h" +#include "strmap.h" +#include "tree.h" + +static const char *short_commit_name(struct repository *repo, + struct commit *commit) +{ + return repo_find_unique_abbrev(repo, &commit->object.oid, + DEFAULT_ABBREV); +} + +static struct commit *peel_committish(struct repository *repo, + const char *name, + const char *mode) +{ + struct object *obj; + struct object_id oid; + + if (repo_get_oid(repo, name, &oid)) + die(_("'%s' is not a valid commit-ish for %s"), name, mode); + obj = parse_object_or_die(repo, &oid, name); + return (struct commit *)repo_peel_to_type(repo, name, 0, obj, + OBJ_COMMIT); +} + +static char *get_author(const char *message) +{ + size_t len; + const char *a; + + a = find_commit_header(message, "author", &len); + if (a) + return xmemdupz(a, len); + + return NULL; +} + +static struct commit *create_commit(struct repository *repo, + struct tree *tree, + struct commit *based_on, + struct commit *parent) +{ + struct object_id ret; + struct object *obj = NULL; + struct commit_list *parents = NULL; + char *author; + char *sign_commit = NULL; /* FIXME: cli users might want to sign again */ + struct commit_extra_header *extra = NULL; + struct strbuf msg = STRBUF_INIT; + const char *out_enc = get_commit_output_encoding(); + const char *message = repo_logmsg_reencode(repo, based_on, + NULL, out_enc); + const char *orig_message = NULL; + const char *exclude_gpgsig[] = { "gpgsig", "gpgsig-sha256", NULL }; + + commit_list_insert(parent, &parents); + extra = read_commit_extra_headers(based_on, exclude_gpgsig); + find_commit_subject(message, &orig_message); + strbuf_addstr(&msg, orig_message); + author = get_author(message); + reset_ident_date(); + if (commit_tree_extended(msg.buf, msg.len, &tree->object.oid, parents, + &ret, author, NULL, sign_commit, extra)) { + error(_("failed to write commit object")); + goto out; + } + + obj = parse_object(repo, &ret); + +out: + repo_unuse_commit_buffer(repo, based_on, message); + free_commit_extra_headers(extra); + free_commit_list(parents); + strbuf_release(&msg); + free(author); + return (struct commit *)obj; +} + +struct ref_info { + struct commit *onto; + struct strset positive_refs; + struct strset negative_refs; + size_t positive_refexprs; + size_t negative_refexprs; +}; + +static void get_ref_information(struct repository *repo, + struct rev_cmdline_info *cmd_info, + struct ref_info *ref_info) +{ + ref_info->onto = NULL; + strset_init(&ref_info->positive_refs); + strset_init(&ref_info->negative_refs); + ref_info->positive_refexprs = 0; + ref_info->negative_refexprs = 0; + + /* + * When the user specifies e.g. + * git replay origin/main..mybranch + * git replay ^origin/next mybranch1 mybranch2 + * we want to be able to determine where to replay the commits. In + * these examples, the branches are probably based on an old version + * of either origin/main or origin/next, so we want to replay on the + * newest version of that branch. In contrast we would want to error + * out if they ran + * git replay ^origin/master ^origin/next mybranch + * git replay mybranch~2..mybranch + * the first of those because there's no unique base to choose, and + * the second because they'd likely just be replaying commits on top + * of the same commit and not making any difference. + */ + for (size_t i = 0; i < cmd_info->nr; i++) { + struct rev_cmdline_entry *e = cmd_info->rev + i; + struct object_id oid; + const char *refexpr = e->name; + char *fullname = NULL; + int can_uniquely_dwim = 1; + + if (*refexpr == '^') + refexpr++; + if (repo_dwim_ref(repo, refexpr, strlen(refexpr), &oid, &fullname, 0) != 1) + can_uniquely_dwim = 0; + + if (e->flags & BOTTOM) { + if (can_uniquely_dwim) + strset_add(&ref_info->negative_refs, fullname); + if (!ref_info->negative_refexprs) + ref_info->onto = lookup_commit_reference_gently(repo, + &e->item->oid, 1); + ref_info->negative_refexprs++; + } else { + if (can_uniquely_dwim) + strset_add(&ref_info->positive_refs, fullname); + ref_info->positive_refexprs++; + } + + free(fullname); + } +} + +static void set_up_replay_mode(struct repository *repo, + struct rev_cmdline_info *cmd_info, + const char *onto_name, + bool *detached_head, + char **advance_name, + struct commit **onto, + struct strset **update_refs) +{ + struct ref_info rinfo; + int head_flags = 0; + + refs_read_ref_full(get_main_ref_store(repo), "HEAD", + RESOLVE_REF_NO_RECURSE, NULL, &head_flags); + *detached_head = !(head_flags & REF_ISSYMREF); + + get_ref_information(repo, cmd_info, &rinfo); + if (!rinfo.positive_refexprs) + die(_("need some commits to replay")); + + if (!onto_name == !*advance_name) + BUG("one and only one of onto_name and *advance_name must be given"); + + if (onto_name) { + *onto = peel_committish(repo, onto_name, "--onto"); + if (rinfo.positive_refexprs < + strset_get_size(&rinfo.positive_refs)) + die(_("all positive revisions given must be references")); + *update_refs = xcalloc(1, sizeof(**update_refs)); + **update_refs = rinfo.positive_refs; + memset(&rinfo.positive_refs, 0, sizeof(**update_refs)); + } else { + struct object_id oid; + char *fullname = NULL; + + if (!*advance_name) + BUG("expected either onto_name or *advance_name in this function"); + + if (repo_dwim_ref(repo, *advance_name, strlen(*advance_name), + &oid, &fullname, 0) == 1) { + free(*advance_name); + *advance_name = fullname; + } else { + die(_("argument to --advance must be a reference")); + } + *onto = peel_committish(repo, *advance_name, "--advance"); + if (rinfo.positive_refexprs > 1) + die(_("cannot advance target with multiple sources because ordering would be ill-defined")); + } + strset_clear(&rinfo.negative_refs); + strset_clear(&rinfo.positive_refs); +} + +static struct commit *mapped_commit(kh_oid_map_t *replayed_commits, + struct commit *commit, + struct commit *fallback) +{ + khint_t pos = kh_get_oid_map(replayed_commits, commit->object.oid); + if (pos == kh_end(replayed_commits)) + return fallback; + return kh_value(replayed_commits, pos); +} + +static struct commit *pick_regular_commit(struct repository *repo, + struct commit *pickme, + kh_oid_map_t *replayed_commits, + struct commit *onto, + struct merge_options *merge_opt, + struct merge_result *result) +{ + struct commit *base, *replayed_base; + struct tree *pickme_tree, *base_tree; + + base = pickme->parents->item; + replayed_base = mapped_commit(replayed_commits, base, onto); + + result->tree = repo_get_commit_tree(repo, replayed_base); + pickme_tree = repo_get_commit_tree(repo, pickme); + base_tree = repo_get_commit_tree(repo, base); + + merge_opt->branch1 = short_commit_name(repo, replayed_base); + merge_opt->branch2 = short_commit_name(repo, pickme); + merge_opt->ancestor = xstrfmt("parent of %s", merge_opt->branch2); + + merge_incore_nonrecursive(merge_opt, + base_tree, + result->tree, + pickme_tree, + result); + + free((char*)merge_opt->ancestor); + merge_opt->ancestor = NULL; + if (!result->clean) + return NULL; + return create_commit(repo, result->tree, pickme, replayed_base); +} + +void replay_result_release(struct replay_result *result) +{ + for (size_t i = 0; i < result->updates_nr; i++) + free(result->updates[i].refname); + free(result->updates); +} + +static void replay_result_queue_update(struct replay_result *result, + const char *refname, + const struct object_id *old_oid, + const struct object_id *new_oid) +{ + ALLOC_GROW(result->updates, result->updates_nr + 1, result->updates_alloc); + result->updates[result->updates_nr].refname = xstrdup(refname); + result->updates[result->updates_nr].old_oid = *old_oid; + result->updates[result->updates_nr].new_oid = *new_oid; + result->updates_nr++; +} + +int replay_revisions(struct rev_info *revs, + struct replay_revisions_options *opts, + struct replay_result *out) +{ + kh_oid_map_t *replayed_commits = NULL; + struct strset *update_refs = NULL; + struct commit *last_commit = NULL; + struct commit *commit; + struct commit *onto = NULL; + struct merge_options merge_opt; + struct merge_result result = { + .clean = 1, + }; + bool detached_head; + char *advance; + int ret; + + advance = xstrdup_or_null(opts->advance); + set_up_replay_mode(revs->repo, &revs->cmdline, opts->onto, + &detached_head, &advance, &onto, &update_refs); + + /* FIXME: Should allow replaying commits with the first as a root commit */ + + if (prepare_revision_walk(revs) < 0) { + ret = error(_("error preparing revisions")); + goto out; + } + + init_basic_merge_options(&merge_opt, revs->repo); + merge_opt.show_rename_progress = 0; + last_commit = onto; + replayed_commits = kh_init_oid_map(); + while ((commit = get_revision(revs))) { + const struct name_decoration *decoration; + khint_t pos; + int hr; + + if (!commit->parents) + die(_("replaying down from root commit is not supported yet!")); + if (commit->parents->next) + die(_("replaying merge commits is not supported yet!")); + + last_commit = pick_regular_commit(revs->repo, commit, replayed_commits, + onto, &merge_opt, &result); + if (!last_commit) + break; + + /* Record commit -> last_commit mapping */ + pos = kh_put_oid_map(replayed_commits, commit->object.oid, &hr); + if (hr == 0) + BUG("Duplicate rewritten commit: %s\n", + oid_to_hex(&commit->object.oid)); + kh_value(replayed_commits, pos) = last_commit; + + /* Update any necessary branches */ + if (advance) + continue; + + for (decoration = get_name_decoration(&commit->object); + decoration; + decoration = decoration->next) + { + if (decoration->type != DECORATION_REF_LOCAL && + decoration->type != DECORATION_REF_HEAD) + continue; + + /* + * We only need to update HEAD separately in case it's + * detached. If it's not we'd already update the branch + * it is pointing to. + */ + if (decoration->type == DECORATION_REF_HEAD && !detached_head) + continue; + + if (!opts->contained && + !strset_contains(update_refs, decoration->name)) + continue; + + replay_result_queue_update(out, decoration->name, + &commit->object.oid, + &last_commit->object.oid); + } + } + + if (!result.clean) { + ret = 1; + goto out; + } + + /* In --advance mode, advance the target ref */ + if (advance) + replay_result_queue_update(out, advance, + &onto->object.oid, + &last_commit->object.oid); + + ret = 0; + +out: + if (update_refs) { + strset_clear(update_refs); + free(update_refs); + } + kh_destroy_oid_map(replayed_commits); + merge_finalize(&merge_opt, &result); + free(advance); + return ret; +} diff --git a/replay.h b/replay.h new file mode 100644 index 0000000000..d8407dc7f7 --- /dev/null +++ b/replay.h @@ -0,0 +1,61 @@ +#ifndef REPLAY_H +#define REPLAY_H + +#include "hash.h" + +struct repository; +struct rev_info; + +/* + * A set of options that can be passed to `replay_revisions()`. + */ +struct replay_revisions_options { + /* + * Starting point at which to create the new commits; must be a branch + * name. The branch will be updated to point to the rewritten commits. + * This option is mutually exclusive with `onto`. + */ + const char *advance; + + /* + * Starting point at which to create the new commits; must be a + * committish. References pointing at decendants of `onto` will be + * updated to point to the new commits. + */ + const char *onto; + + /* + * Update branches that point at commits in the given revision range. + * Requires `onto` to be set. + */ + int contained; +}; + +/* This struct is used as an out-parameter by `replay_revisions()`. */ +struct replay_result { + /* + * The set of reference updates that are caused by replaying the + * commits. + */ + struct replay_ref_update { + char *refname; + struct object_id old_oid; + struct object_id new_oid; + } *updates; + size_t updates_nr, updates_alloc; +}; + +void replay_result_release(struct replay_result *result); + +/* + * Replay a set of commits onto a new location. Leaves both the working tree, + * index and references untouched. Reference updates caused by the replay will + * be recorded in the `updates` out pointer. + * + * Returns 0 on success, 1 on conflict and a negative error code otherwise. + */ +int replay_revisions(struct rev_info *revs, + struct replay_revisions_options *opts, + struct replay_result *out); + +#endif diff --git a/repository.c b/repository.c index 6aaa7ba008..c7e75215ac 100644 --- a/repository.c +++ b/repository.c @@ -52,7 +52,6 @@ static void set_default_hash_algo(struct repository *repo) void initialize_repository(struct repository *repo) { - repo->objects = odb_new(repo); repo->remote_state = remote_state_new(); repo->parsed_objects = parsed_object_pool_new(repo); ALLOC_ARRAY(repo->index, 1); @@ -160,29 +159,19 @@ void repo_set_gitdir(struct repository *repo, * until after xstrdup(root). Then we can free it. */ char *old_gitdir = repo->gitdir; - char *objects_path = NULL; repo->gitdir = xstrdup(gitfile ? gitfile : root); free(old_gitdir); repo_set_commondir(repo, o->commondir); - expand_base_dir(&objects_path, o->object_dir, - repo->commondir, "objects"); - if (!repo->objects->sources) { - repo->objects->sources = odb_source_new(repo->objects, - objects_path, true); - repo->objects->sources_tail = &repo->objects->sources->next; - free(objects_path); - } else { - free(repo->objects->sources->path); - repo->objects->sources->path = objects_path; - } + if (!repo->objects) + repo->objects = odb_new(repo, o->object_dir, o->alternate_db); + else if (!o->skip_initializing_odb) + BUG("cannot reinitialize an already-initialized object directory"); - repo->objects->sources->disable_ref_updates = o->disable_ref_updates; + repo->disable_ref_updates = o->disable_ref_updates; - free(repo->objects->alternate_db); - repo->objects->alternate_db = xstrdup_or_null(o->alternate_db); expand_base_dir(&repo->graft_file, o->graft_file, repo->commondir, "info/grafts"); expand_base_dir(&repo->index_file, o->index_file, @@ -361,7 +350,6 @@ out: static void repo_clear_path_cache(struct repo_path_cache *cache) { FREE_AND_NULL(cache->squash_msg); - FREE_AND_NULL(cache->squash_msg); FREE_AND_NULL(cache->merge_msg); FREE_AND_NULL(cache->merge_rr); FREE_AND_NULL(cache->merge_mode); @@ -382,8 +370,8 @@ void repo_clear(struct repository *repo) FREE_AND_NULL(repo->worktree); FREE_AND_NULL(repo->submodule_prefix); - odb_clear(repo->objects); - FREE_AND_NULL(repo->objects); + odb_free(repo->objects); + repo->objects = NULL; parsed_object_pool_clear(repo->parsed_objects); FREE_AND_NULL(repo->parsed_objects); diff --git a/repository.h b/repository.h index 5808a5d610..6063c4b846 100644 --- a/repository.h +++ b/repository.h @@ -72,6 +72,13 @@ struct repository { struct ref_store *refs_private; /* + * Disable ref updates. This is especially used in contexts where + * transactions may still be rolled back so that we don't start to + * reference objects that may vanish. + */ + bool disable_ref_updates; + + /* * A strmap of ref_stores, stored by submodule name, accessible via * `repo_get_submodule_ref_store()`. */ @@ -187,7 +194,8 @@ struct set_gitdir_args { const char *graft_file; const char *index_file; const char *alternate_db; - int disable_ref_updates; + bool disable_ref_updates; + bool skip_initializing_odb; }; void repo_set_gitdir(struct repository *repo, const char *root, diff --git a/run-command.c b/run-command.c index ed9575bd6a..2d3c2ac55c 100644 --- a/run-command.c +++ b/run-command.c @@ -743,7 +743,7 @@ fail_pipe: fflush(NULL); if (cmd->close_object_store) - close_object_store(the_repository->objects); + odb_close(the_repository->objects); #ifndef GIT_WINDOWS_NATIVE { @@ -1478,15 +1478,32 @@ enum child_state { GIT_CP_WAIT_CLEANUP, }; +struct parallel_child { + enum child_state state; + struct child_process process; + struct strbuf err; + void *data; +}; + +static int child_is_working(const struct parallel_child *pp_child) +{ + return pp_child->state == GIT_CP_WORKING; +} + +static int child_is_ready_for_cleanup(const struct parallel_child *pp_child) +{ + return child_is_working(pp_child) && !pp_child->process.in; +} + +static int child_is_receiving_input(const struct parallel_child *pp_child) +{ + return child_is_working(pp_child) && pp_child->process.in > 0; +} + struct parallel_processes { size_t nr_processes; - struct { - enum child_state state; - struct child_process process; - struct strbuf err; - void *data; - } *children; + struct parallel_child *children; /* * The struct pollfd is logically part of *children, * but the system call expects it as its own array. @@ -1509,7 +1526,7 @@ static void kill_children(const struct parallel_processes *pp, int signo) { for (size_t i = 0; i < opts->processes; i++) - if (pp->children[i].state == GIT_CP_WORKING) + if (child_is_working(&pp->children[i])) kill(pp->children[i].process.pid, signo); } @@ -1578,7 +1595,10 @@ static void pp_cleanup(struct parallel_processes *pp, * When get_next_task added messages to the buffer in its last * iteration, the buffered output is non empty. */ - strbuf_write(&pp->buffered_output, stderr); + if (opts->consume_output) + opts->consume_output(&pp->buffered_output, opts->data); + else + strbuf_write(&pp->buffered_output, stderr); strbuf_release(&pp->buffered_output); sigchain_pop_common(); @@ -1652,6 +1672,44 @@ static int pp_start_one(struct parallel_processes *pp, return 0; } +static void pp_buffer_stdin(struct parallel_processes *pp, + const struct run_process_parallel_opts *opts) +{ + /* Buffer stdin for each pipe. */ + for (size_t i = 0; i < opts->processes; i++) { + struct child_process *proc = &pp->children[i].process; + int ret; + + if (!child_is_receiving_input(&pp->children[i])) + continue; + + /* + * child input is provided via path_to_stdin when the feed_pipe cb is + * missing, so we just signal an EOF. + */ + if (!opts->feed_pipe) { + close(proc->in); + proc->in = 0; + continue; + } + + /** + * Feed the pipe: + * ret < 0 means error + * ret == 0 means there is more data to be fed + * ret > 0 means feeding finished + */ + ret = opts->feed_pipe(proc->in, opts->data, pp->children[i].data); + if (ret < 0) + die_errno("feed_pipe"); + + if (ret) { + close(proc->in); + proc->in = 0; + } + } +} + static void pp_buffer_stderr(struct parallel_processes *pp, const struct run_process_parallel_opts *opts, int output_timeout) @@ -1665,7 +1723,7 @@ static void pp_buffer_stderr(struct parallel_processes *pp, /* Buffer output from all pipes. */ for (size_t i = 0; i < opts->processes; i++) { - if (pp->children[i].state == GIT_CP_WORKING && + if (child_is_working(&pp->children[i]) && pp->pfd[i].revents & (POLLIN | POLLHUP)) { int n = strbuf_read_once(&pp->children[i].err, pp->children[i].process.err, 0); @@ -1679,13 +1737,17 @@ static void pp_buffer_stderr(struct parallel_processes *pp, } } -static void pp_output(const struct parallel_processes *pp) +static void pp_output(const struct parallel_processes *pp, + const struct run_process_parallel_opts *opts) { size_t i = pp->output_owner; - if (pp->children[i].state == GIT_CP_WORKING && + if (child_is_working(&pp->children[i]) && pp->children[i].err.len) { - strbuf_write(&pp->children[i].err, stderr); + if (opts->consume_output) + opts->consume_output(&pp->children[i].err, opts->data); + else + strbuf_write(&pp->children[i].err, stderr); strbuf_reset(&pp->children[i].err); } } @@ -1722,6 +1784,7 @@ static int pp_collect_finished(struct parallel_processes *pp, pp->children[i].state = GIT_CP_FREE; if (pp->pfd) pp->pfd[i].fd = -1; + pp->children[i].process.in = 0; child_process_init(&pp->children[i].process); if (opts->ungroup) { @@ -1732,11 +1795,15 @@ static int pp_collect_finished(struct parallel_processes *pp, } else { const size_t n = opts->processes; - strbuf_write(&pp->children[i].err, stderr); + /* Output errors, then all other finished child processes */ + if (opts->consume_output) { + opts->consume_output(&pp->children[i].err, opts->data); + opts->consume_output(&pp->buffered_output, opts->data); + } else { + strbuf_write(&pp->children[i].err, stderr); + strbuf_write(&pp->buffered_output, stderr); + } strbuf_reset(&pp->children[i].err); - - /* Output all other finished child processes */ - strbuf_write(&pp->buffered_output, stderr); strbuf_reset(&pp->buffered_output); /* @@ -1748,7 +1815,7 @@ static int pp_collect_finished(struct parallel_processes *pp, * running process time. */ for (i = 0; i < n; i++) - if (pp->children[(pp->output_owner + i) % n].state == GIT_CP_WORKING) + if (child_is_working(&pp->children[(pp->output_owner + i) % n])) break; pp->output_owner = (pp->output_owner + i) % n; } @@ -1756,6 +1823,27 @@ static int pp_collect_finished(struct parallel_processes *pp, return result; } +static void pp_handle_child_IO(struct parallel_processes *pp, + const struct run_process_parallel_opts *opts, + int output_timeout) +{ + /* + * First push input, if any (it might no-op), to child tasks to avoid them blocking + * after input. This also prevents deadlocks when ungrouping below, if a child blocks + * while the parent also waits for them to finish. + */ + pp_buffer_stdin(pp, opts); + + if (opts->ungroup) { + for (size_t i = 0; i < opts->processes; i++) + if (child_is_ready_for_cleanup(&pp->children[i])) + pp->children[i].state = GIT_CP_WAIT_CLEANUP; + } else { + pp_buffer_stderr(pp, opts, output_timeout); + pp_output(pp, opts); + } +} + void run_processes_parallel(const struct run_process_parallel_opts *opts) { int i, code; @@ -1775,6 +1863,16 @@ void run_processes_parallel(const struct run_process_parallel_opts *opts) "max:%"PRIuMAX, (uintmax_t)opts->processes); + if (opts->ungroup && opts->consume_output) + BUG("ungroup and reading output are mutualy exclusive"); + + /* + * Child tasks might receive input via stdin, terminating early (or not), so + * ignore the default SIGPIPE which gets handled by each feed_pipe_fn which + * actually writes the data to children stdin fds. + */ + sigchain_push(SIGPIPE, SIG_IGN); + pp_init(&pp, opts, &pp_sig); while (1) { for (i = 0; @@ -1792,13 +1890,7 @@ void run_processes_parallel(const struct run_process_parallel_opts *opts) } if (!pp.nr_processes) break; - if (opts->ungroup) { - for (size_t i = 0; i < opts->processes; i++) - pp.children[i].state = GIT_CP_WAIT_CLEANUP; - } else { - pp_buffer_stderr(&pp, opts, output_timeout); - pp_output(&pp); - } + pp_handle_child_IO(&pp, opts, output_timeout); code = pp_collect_finished(&pp, opts); if (code) { pp.shutdown = 1; @@ -1809,6 +1901,8 @@ void run_processes_parallel(const struct run_process_parallel_opts *opts) pp_cleanup(&pp, opts); + sigchain_pop(SIGPIPE); + if (do_trace2) trace2_region_leave(tr2_category, tr2_label, NULL); } diff --git a/run-command.h b/run-command.h index 0df25e445f..7093252863 100644 --- a/run-command.h +++ b/run-command.h @@ -421,6 +421,32 @@ typedef int (*start_failure_fn)(struct strbuf *out, void *pp_task_cb); /** + * This callback is repeatedly called on every child process who requests + * start_command() to create a pipe by setting child_process.in < 0. + * + * pp_cb is the callback cookie as passed into run_processes_parallel, and + * pp_task_cb is the callback cookie as passed into get_next_task_fn. + * + * Returns < 0 for error + * Returns == 0 when there is more data to be fed (will be called again) + * Returns > 0 when finished (child closed fd or no more data to be fed) + */ +typedef int (*feed_pipe_fn)(int child_in, + void *pp_cb, + void *pp_task_cb); + +/** + * If this callback is provided, output is collated into a new pipe instead + * of the process stderr. Then `consume_output_fn` will be called repeatedly + * with output contained in the `output` arg. It will also be called with an + * empty `output` to allow for keepalives or similar operations if necessary. + * + * pp_cb is the callback cookie as passed into run_processes_parallel. + * No task cookie is provided because the callback receives collated output. + */ +typedef void (*consume_output_fn)(struct strbuf *output, void *pp_cb); + +/** * This callback is called on every child process that finished processing. * * See run_processes_parallel() below for a discussion of the "struct @@ -473,6 +499,18 @@ struct run_process_parallel_opts */ start_failure_fn start_failure; + /* + * feed_pipe: see feed_pipe_fn() above. This can be NULL to omit any + * special handling. + */ + feed_pipe_fn feed_pipe; + + /* + * consume_output: see consume_output_fn() above. This can be NULL + * to omit any special handling. + */ + consume_output_fn consume_output; + /** * task_finished: See task_finished_fn() above. This can be * NULL to omit any special handling. @@ -19,6 +19,7 @@ #include "help.h" #include "setup.h" #include "trace2.h" +#include "path.h" static void setup_enlistment_directory(int argc, const char **argv, const char * const *usagestr, @@ -95,7 +96,17 @@ struct scalar_config { int overwrite_on_reconfigure; }; -static int set_scalar_config(const struct scalar_config *config, int reconfigure) +static int set_scalar_config(const char *key, const char *value) +{ + char *file = repo_git_path(the_repository, "config"); + int res = repo_config_set_multivar_in_file_gently(the_repository, file, + key, value, NULL, + " # set by scalar", 0); + free(file); + return res; +} + +static int set_config_if_missing(const struct scalar_config *config, int reconfigure) { char *value = NULL; int res; @@ -103,7 +114,7 @@ static int set_scalar_config(const struct scalar_config *config, int reconfigure if ((reconfigure && config->overwrite_on_reconfigure) || repo_config_get_string(the_repository, config->key, &value)) { trace2_data_string("scalar", the_repository, config->key, "created"); - res = repo_config_set_gently(the_repository, config->key, config->value); + res = set_scalar_config(config->key, config->value); } else { trace2_data_string("scalar", the_repository, config->key, "exists"); res = 0; @@ -121,14 +132,38 @@ static int have_fsmonitor_support(void) static int set_recommended_config(int reconfigure) { + /* + * Be sure to update Documentation/scalar.adoc if you add, update, + * or remove any of these recommended settings. + */ struct scalar_config config[] = { - /* Required */ - { "am.keepCR", "true", 1 }, - { "core.FSCache", "true", 1 }, - { "core.multiPackIndex", "true", 1 }, - { "core.preloadIndex", "true", 1 }, + { "am.keepCR", "true" }, + { "commitGraph.changedPaths", "true" }, + { "commitGraph.generationVersion", "1" }, + { "core.autoCRLF", "false" }, + { "core.logAllRefUpdates", "true" }, + { "core.safeCRLF", "false" }, + { "credential.https://dev.azure.com.useHttpPath", "true" }, + { "feature.experimental", "false" }, + { "feature.manyFiles", "false" }, + { "fetch.showForcedUpdates", "false" }, + { "fetch.unpackLimit", "1" }, + { "fetch.writeCommitGraph", "false" }, + { "gc.auto", "0" }, + { "gui.GCWarning", "false" }, + { "index.skipHash", "true", 1 /* Fix previous setting. */ }, + { "index.threads", "true"}, + { "index.version", "4" }, + { "merge.renames", "true" }, + { "merge.stat", "false" }, + { "pack.useBitmaps", "false" }, + { "pack.usePathWalk", "true" }, + { "receive.autoGC", "false" }, + { "status.aheadBehind", "false" }, + + /* platform-specific */ #ifndef WIN32 - { "core.untrackedCache", "true", 1 }, + { "core.untrackedCache", "true" }, #else /* * Unfortunately, Scalar's Functional Tests demonstrated @@ -142,50 +177,25 @@ static int set_recommended_config(int reconfigure) * Therefore, with a sad heart, we disable this very useful * feature on Windows. */ - { "core.untrackedCache", "false", 1 }, -#endif - { "core.logAllRefUpdates", "true", 1 }, - { "credential.https://dev.azure.com.useHttpPath", "true", 1 }, - { "credential.validate", "false", 1 }, /* GCM4W-only */ - { "gc.auto", "0", 1 }, - { "gui.GCWarning", "false", 1 }, - { "index.skipHash", "false", 1 }, - { "index.threads", "true", 1 }, - { "index.version", "4", 1 }, - { "merge.stat", "false", 1 }, - { "merge.renames", "true", 1 }, - { "pack.useBitmaps", "false", 1 }, - { "pack.useSparse", "true", 1 }, - { "receive.autoGC", "false", 1 }, - { "feature.manyFiles", "false", 1 }, - { "feature.experimental", "false", 1 }, - { "fetch.unpackLimit", "1", 1 }, - { "fetch.writeCommitGraph", "false", 1 }, -#ifdef WIN32 - { "http.sslBackend", "schannel", 1 }, + { "core.untrackedCache", "false" }, + + /* Other Windows-specific required settings: */ + { "http.sslBackend", "schannel" }, #endif - /* Optional */ - { "status.aheadBehind", "false" }, - { "commitGraph.changedPaths", "true" }, - { "commitGraph.generationVersion", "1" }, - { "core.autoCRLF", "false" }, - { "core.safeCRLF", "false" }, - { "fetch.showForcedUpdates", "false" }, - { "pack.usePathWalk", "true" }, { NULL, NULL }, }; int i; char *value; for (i = 0; config[i].key; i++) { - if (set_scalar_config(config + i, reconfigure)) + if (set_config_if_missing(config + i, reconfigure)) return error(_("could not configure %s=%s"), config[i].key, config[i].value); } if (have_fsmonitor_support()) { struct scalar_config fsmonitor = { "core.fsmonitor", "true" }; - if (set_scalar_config(&fsmonitor, reconfigure)) + if (set_config_if_missing(&fsmonitor, reconfigure)) return error(_("could not configure %s=%s"), fsmonitor.key, fsmonitor.value); } @@ -197,9 +207,8 @@ static int set_recommended_config(int reconfigure) if (repo_config_get_string(the_repository, "log.excludeDecoration", &value)) { trace2_data_string("scalar", the_repository, "log.excludeDecoration", "created"); - if (repo_config_set_multivar_gently(the_repository, "log.excludeDecoration", - "refs/prefetch/*", - CONFIG_REGEX_NONE, 0)) + if (set_scalar_config("log.excludeDecoration", + "refs/prefetch/*")) return error(_("could not configure " "log.excludeDecoration")); } else { @@ -931,7 +940,7 @@ static int cmd_delete(int argc, const char **argv) if (dir_inside_of(cwd, enlistment.buf) >= 0) res = error(_("refusing to delete current working directory")); else { - close_object_store(the_repository->objects); + odb_close(the_repository->objects); res = delete_enlistment(&enlistment); } strbuf_release(&enlistment); diff --git a/sequencer.c b/sequencer.c index 5476d39ba9..71ed31c774 100644 --- a/sequencer.c +++ b/sequencer.c @@ -1292,32 +1292,40 @@ int update_head_with_reflog(const struct commit *old_head, return ret; } +static int pipe_from_strbuf(int hook_stdin_fd, void *pp_cb, void *pp_task_cb UNUSED) +{ + struct hook_cb_data *hook_cb = pp_cb; + struct strbuf *to_pipe = hook_cb->options->feed_pipe_ctx; + int ret; + + if (!to_pipe) + BUG("pipe_from_strbuf called without feed_pipe_ctx"); + + ret = write_in_full(hook_stdin_fd, to_pipe->buf, to_pipe->len); + if (ret < 0 && errno != EPIPE) + return ret; + + return 1; /* done writing */ +} + static int run_rewrite_hook(const struct object_id *oldoid, const struct object_id *newoid) { - struct child_process proc = CHILD_PROCESS_INIT; + struct run_hooks_opt opt = RUN_HOOKS_OPT_INIT; int code; struct strbuf sb = STRBUF_INIT; - const char *hook_path = find_hook(the_repository, "post-rewrite"); - if (!hook_path) - return 0; + strbuf_addf(&sb, "%s %s\n", oid_to_hex(oldoid), oid_to_hex(newoid)); - strvec_pushl(&proc.args, hook_path, "amend", NULL); - proc.in = -1; - proc.stdout_to_stderr = 1; - proc.trace2_hook_name = "post-rewrite"; + opt.feed_pipe_ctx = &sb; + opt.feed_pipe = pipe_from_strbuf; + + strvec_push(&opt.args, "amend"); + + code = run_hooks_opt(the_repository, "post-rewrite", &opt); - code = start_command(&proc); - if (code) - return code; - strbuf_addf(&sb, "%s %s\n", oid_to_hex(oldoid), oid_to_hex(newoid)); - sigchain_push(SIGPIPE, SIG_IGN); - write_in_full(proc.in, sb.buf, sb.len); - close(proc.in); strbuf_release(&sb); - sigchain_pop(SIGPIPE); - return finish_command(&proc); + return code; } void commit_post_rewrite(struct repository *r, @@ -22,7 +22,6 @@ #include "chdir-notify.h" #include "path.h" #include "quote.h" -#include "tmp-objdir.h" #include "trace.h" #include "trace2.h" #include "worktree.h" @@ -1002,6 +1001,83 @@ cleanup_return: return error_code ? NULL : path; } +static void setup_git_env_internal(const char *git_dir, + bool skip_initializing_odb) +{ + char *git_replace_ref_base; + const char *shallow_file; + const char *replace_ref_base; + struct set_gitdir_args args = { NULL }; + struct strvec to_free = STRVEC_INIT; + + args.commondir = getenv_safe(&to_free, GIT_COMMON_DIR_ENVIRONMENT); + args.object_dir = getenv_safe(&to_free, DB_ENVIRONMENT); + args.graft_file = getenv_safe(&to_free, GRAFT_ENVIRONMENT); + args.index_file = getenv_safe(&to_free, INDEX_ENVIRONMENT); + args.alternate_db = getenv_safe(&to_free, ALTERNATE_DB_ENVIRONMENT); + if (getenv(GIT_QUARANTINE_ENVIRONMENT)) + args.disable_ref_updates = true; + args.skip_initializing_odb = skip_initializing_odb; + + repo_set_gitdir(the_repository, git_dir, &args); + strvec_clear(&to_free); + + if (getenv(NO_REPLACE_OBJECTS_ENVIRONMENT)) + disable_replace_refs(); + replace_ref_base = getenv(GIT_REPLACE_REF_BASE_ENVIRONMENT); + git_replace_ref_base = xstrdup(replace_ref_base ? replace_ref_base + : "refs/replace/"); + update_ref_namespace(NAMESPACE_REPLACE, git_replace_ref_base); + + shallow_file = getenv(GIT_SHALLOW_FILE_ENVIRONMENT); + if (shallow_file) + set_alternate_shallow_file(the_repository, shallow_file, 0); + + if (git_env_bool(NO_LAZY_FETCH_ENVIRONMENT, 0)) + fetch_if_missing = 0; +} + +void setup_git_env(const char *git_dir) +{ + setup_git_env_internal(git_dir, false); +} + +static void set_git_dir_1(const char *path, bool skip_initializing_odb) +{ + xsetenv(GIT_DIR_ENVIRONMENT, path, 1); + setup_git_env_internal(path, skip_initializing_odb); +} + +static void update_relative_gitdir(const char *name UNUSED, + const char *old_cwd, + const char *new_cwd, + void *data UNUSED) +{ + char *path = reparent_relative_path(old_cwd, new_cwd, + repo_get_git_dir(the_repository)); + trace_printf_key(&trace_setup_key, + "setup: move $GIT_DIR to '%s'", + path); + set_git_dir_1(path, true); + free(path); +} + +static void set_git_dir(const char *path, int make_realpath) +{ + struct strbuf realpath = STRBUF_INIT; + + if (make_realpath) { + strbuf_realpath(&realpath, path, 1); + path = realpath.buf; + } + + set_git_dir_1(path, false); + if (!is_absolute_path(path)) + chdir_notify_register(NULL, update_relative_gitdir, NULL); + + strbuf_release(&realpath); +} + static const char *setup_explicit_git_dir(const char *gitdirenv, struct strbuf *cwd, struct repository_format *repo_fmt, @@ -1248,7 +1324,7 @@ static int safe_directory_cb(const char *key, const char *value, } else { char *allowed = NULL; - if (!git_config_pathname(&allowed, key, value)) { + if (!git_config_pathname(&allowed, key, value) && allowed) { char *normalized = NULL; /* @@ -1628,79 +1704,85 @@ enum discovery_result discover_git_directory_reason(struct strbuf *commondir, return result; } -void setup_git_env(const char *git_dir) +const char *enter_repo(const char *path, unsigned flags) { - char *git_replace_ref_base; - const char *shallow_file; - const char *replace_ref_base; - struct set_gitdir_args args = { NULL }; - struct strvec to_free = STRVEC_INIT; - - args.commondir = getenv_safe(&to_free, GIT_COMMON_DIR_ENVIRONMENT); - args.object_dir = getenv_safe(&to_free, DB_ENVIRONMENT); - args.graft_file = getenv_safe(&to_free, GRAFT_ENVIRONMENT); - args.index_file = getenv_safe(&to_free, INDEX_ENVIRONMENT); - args.alternate_db = getenv_safe(&to_free, ALTERNATE_DB_ENVIRONMENT); - if (getenv(GIT_QUARANTINE_ENVIRONMENT)) { - args.disable_ref_updates = 1; - } - - repo_set_gitdir(the_repository, git_dir, &args); - strvec_clear(&to_free); - - if (getenv(NO_REPLACE_OBJECTS_ENVIRONMENT)) - disable_replace_refs(); - replace_ref_base = getenv(GIT_REPLACE_REF_BASE_ENVIRONMENT); - git_replace_ref_base = xstrdup(replace_ref_base ? replace_ref_base - : "refs/replace/"); - update_ref_namespace(NAMESPACE_REPLACE, git_replace_ref_base); + static struct strbuf validated_path = STRBUF_INIT; + static struct strbuf used_path = STRBUF_INIT; - shallow_file = getenv(GIT_SHALLOW_FILE_ENVIRONMENT); - if (shallow_file) - set_alternate_shallow_file(the_repository, shallow_file, 0); - - if (git_env_bool(NO_LAZY_FETCH_ENVIRONMENT, 0)) - fetch_if_missing = 0; -} - -static void set_git_dir_1(const char *path) -{ - xsetenv(GIT_DIR_ENVIRONMENT, path, 1); - setup_git_env(path); -} + if (!path) + return NULL; -static void update_relative_gitdir(const char *name UNUSED, - const char *old_cwd, - const char *new_cwd, - void *data UNUSED) -{ - char *path = reparent_relative_path(old_cwd, new_cwd, - repo_get_git_dir(the_repository)); - struct tmp_objdir *tmp_objdir = tmp_objdir_unapply_primary_odb(); + if (!(flags & ENTER_REPO_STRICT)) { + static const char *suffix[] = { + "/.git", "", ".git/.git", ".git", NULL, + }; + const char *gitfile; + int len = strlen(path); + int i; + while ((1 < len) && (path[len-1] == '/')) + len--; - trace_printf_key(&trace_setup_key, - "setup: move $GIT_DIR to '%s'", - path); - set_git_dir_1(path); - if (tmp_objdir) - tmp_objdir_reapply_primary_odb(tmp_objdir, old_cwd, new_cwd); - free(path); -} + /* + * We can handle arbitrary-sized buffers, but this remains as a + * sanity check on untrusted input. + */ + if (PATH_MAX <= len) + return NULL; -void set_git_dir(const char *path, int make_realpath) -{ - struct strbuf realpath = STRBUF_INIT; + strbuf_reset(&used_path); + strbuf_reset(&validated_path); + strbuf_add(&used_path, path, len); + strbuf_add(&validated_path, path, len); - if (make_realpath) { - strbuf_realpath(&realpath, path, 1); - path = realpath.buf; + if (used_path.buf[0] == '~') { + char *newpath = interpolate_path(used_path.buf, 0); + if (!newpath) + return NULL; + strbuf_attach(&used_path, newpath, strlen(newpath), + strlen(newpath)); + } + for (i = 0; suffix[i]; i++) { + struct stat st; + size_t baselen = used_path.len; + strbuf_addstr(&used_path, suffix[i]); + if (!stat(used_path.buf, &st) && + (S_ISREG(st.st_mode) || + (S_ISDIR(st.st_mode) && is_git_directory(used_path.buf)))) { + strbuf_addstr(&validated_path, suffix[i]); + break; + } + strbuf_setlen(&used_path, baselen); + } + if (!suffix[i]) + return NULL; + gitfile = read_gitfile(used_path.buf); + if (!(flags & ENTER_REPO_ANY_OWNER_OK)) + die_upon_dubious_ownership(gitfile, NULL, used_path.buf); + if (gitfile) { + strbuf_reset(&used_path); + strbuf_addstr(&used_path, gitfile); + } + if (chdir(used_path.buf)) + return NULL; + path = validated_path.buf; + } + else { + const char *gitfile = read_gitfile(path); + if (!(flags & ENTER_REPO_ANY_OWNER_OK)) + die_upon_dubious_ownership(gitfile, NULL, path); + if (gitfile) + path = gitfile; + if (chdir(path)) + return NULL; } - set_git_dir_1(path); - if (!is_absolute_path(path)) - chdir_notify_register(NULL, update_relative_gitdir, NULL); + if (is_git_directory(".")) { + set_git_dir(".", 0); + check_repository_format(NULL); + return path; + } - strbuf_release(&realpath); + return NULL; } static int git_work_tree_initialized; @@ -94,9 +94,46 @@ static inline int discover_git_directory(struct strbuf *commondir, return 0; } -void set_git_dir(const char *path, int make_realpath); void set_git_work_tree(const char *tree); +/* Flags that can be passed to `enter_repo()`. */ +enum { + /* + * Callers that require exact paths (as opposed to allowing known + * suffixes like ".git", ".git/.git" to be omitted) can set this bit. + */ + ENTER_REPO_STRICT = (1<<0), + + /* + * Callers that are willing to run without ownership check can set this + * bit. + */ + ENTER_REPO_ANY_OWNER_OK = (1<<1), +}; + +/* + * Discover and enter a repository. + * + * First, one directory to try is determined by the following algorithm. + * + * (0) If "strict" is given, the path is used as given and no DWIM is + * done. Otherwise: + * (1) "~/path" to mean path under the running user's home directory; + * (2) "~user/path" to mean path under named user's home directory; + * (3) "relative/path" to mean cwd relative directory; or + * (4) "/absolute/path" to mean absolute directory. + * + * Unless "strict" is given, we check "%s/.git", "%s", "%s.git/.git", "%s.git" + * in this order. We select the first one that is a valid git repository, and + * chdir() to it. If none match, or we fail to chdir, we return NULL. + * + * If all goes well, we return the directory we used to chdir() (but + * before ~user is expanded), avoiding getcwd() resolving symbolic + * links. User relative paths are also returned as they are given, + * except DWIM suffixing. + */ +const char *enter_repo(const char *path, unsigned flags); + const char *setup_git_directory_gently(int *); const char *setup_git_directory(void); char *prefix_path(const char *prefix, int len, const char *path); @@ -709,7 +709,7 @@ void assign_shallow_commits_to_refs(struct shallow_info *info, if (used) { int bitmap_size = DIV_ROUND_UP(pi.nr_bits, 32) * sizeof(uint32_t); - memset(used, 0, sizeof(*used) * info->shallow->nr); + MEMZERO_ARRAY(used, info->shallow->nr); for (i = 0; i < nr_shallow; i++) { const struct commit *c = lookup_commit(the_repository, &oid[shallow[i]]); @@ -774,7 +774,7 @@ static void post_assign_shallow(struct shallow_info *info, trace_printf_key(&trace_shallow, "shallow: post_assign_shallow\n"); if (ref_status) - memset(ref_status, 0, sizeof(*ref_status) * info->ref->nr); + MEMZERO_ARRAY(ref_status, info->ref->nr); /* Remove unreachable shallow commits from "theirs" */ for (i = dst = 0; i < info->nr_theirs; i++) { diff --git a/src/cargo-meson.sh b/src/cargo-meson.sh index 3998db0435..38728a3711 100755 --- a/src/cargo-meson.sh +++ b/src/cargo-meson.sh @@ -26,7 +26,7 @@ then exit $RET fi -case "$(cargo -vV | sed -s 's/^host: \(.*\)$/\1/')" in +case "$(cargo -vV | sed -n 's/^host: \(.*\)$/\1/p')" in *-windows-*) LIBNAME=gitcore.lib;; *) @@ -836,47 +836,83 @@ void strbuf_addstr_urlencode(struct strbuf *sb, const char *s, strbuf_add_urlencode(sb, s, strlen(s), allow_unencoded_fn); } -static void strbuf_humanise(struct strbuf *buf, off_t bytes, - int humanise_rate) +void humanise_count(size_t count, char **value, const char **unit) { + if (count >= 1000000000) { + size_t x = count + 5000000; /* for rounding */ + *value = xstrfmt(_("%u.%2.2u"), (unsigned)(x / 1000000000), + (unsigned)(x % 1000000000 / 10000000)); + /* TRANSLATORS: SI decimal prefix symbol for 10^9 */ + *unit = _("G"); + } else if (count >= 1000000) { + size_t x = count + 5000; /* for rounding */ + *value = xstrfmt(_("%u.%2.2u"), (unsigned)(x / 1000000), + (unsigned)(x % 1000000 / 10000)); + /* TRANSLATORS: SI decimal prefix symbol for 10^6 */ + *unit = _("M"); + } else if (count >= 1000) { + size_t x = count + 5; /* for rounding */ + *value = xstrfmt(_("%u.%2.2u"), (unsigned)(x / 1000), + (unsigned)(x % 1000 / 10)); + /* TRANSLATORS: SI decimal prefix symbol for 10^3 */ + *unit = _("k"); + } else { + *value = xstrfmt("%u", (unsigned)count); + *unit = NULL; + } +} + +void humanise_bytes(off_t bytes, char **value, const char **unit, + unsigned flags) +{ + int humanise_rate = flags & HUMANISE_RATE; + if (bytes > 1 << 30) { - strbuf_addf(buf, - humanise_rate == 0 ? - /* TRANSLATORS: IEC 80000-13:2008 gibibyte */ - _("%u.%2.2u GiB") : - /* TRANSLATORS: IEC 80000-13:2008 gibibyte/second */ - _("%u.%2.2u GiB/s"), - (unsigned)(bytes >> 30), - (unsigned)(bytes & ((1 << 30) - 1)) / 10737419); + *value = xstrfmt(_("%u.%2.2u"), (unsigned)(bytes >> 30), + (unsigned)(bytes & ((1 << 30) - 1)) / 10737419); + /* TRANSLATORS: IEC 80000-13:2008 gibibyte/second and gibibyte */ + *unit = humanise_rate ? _("GiB/s") : _("GiB"); } else if (bytes > 1 << 20) { - unsigned x = bytes + 5243; /* for rounding */ - strbuf_addf(buf, - humanise_rate == 0 ? - /* TRANSLATORS: IEC 80000-13:2008 mebibyte */ - _("%u.%2.2u MiB") : - /* TRANSLATORS: IEC 80000-13:2008 mebibyte/second */ - _("%u.%2.2u MiB/s"), - x >> 20, ((x & ((1 << 20) - 1)) * 100) >> 20); + unsigned x = bytes + 5243; /* for rounding */ + *value = xstrfmt(_("%u.%2.2u"), x >> 20, + ((x & ((1 << 20) - 1)) * 100) >> 20); + /* TRANSLATORS: IEC 80000-13:2008 mebibyte/second and mebibyte */ + *unit = humanise_rate ? _("MiB/s") : _("MiB"); } else if (bytes > 1 << 10) { - unsigned x = bytes + 5; /* for rounding */ - strbuf_addf(buf, - humanise_rate == 0 ? - /* TRANSLATORS: IEC 80000-13:2008 kibibyte */ - _("%u.%2.2u KiB") : - /* TRANSLATORS: IEC 80000-13:2008 kibibyte/second */ - _("%u.%2.2u KiB/s"), - x >> 10, ((x & ((1 << 10) - 1)) * 100) >> 10); + unsigned x = bytes + 5; /* for rounding */ + *value = xstrfmt(_("%u.%2.2u"), x >> 10, + ((x & ((1 << 10) - 1)) * 100) >> 10); + /* TRANSLATORS: IEC 80000-13:2008 kibibyte/second and kibibyte */ + *unit = humanise_rate ? _("KiB/s") : _("KiB"); } else { - strbuf_addf(buf, - humanise_rate == 0 ? - /* TRANSLATORS: IEC 80000-13:2008 byte */ - Q_("%u byte", "%u bytes", bytes) : + *value = xstrfmt("%u", (unsigned)bytes); + if (flags & HUMANISE_COMPACT) + /* TRANSLATORS: IEC 80000-13:2008 byte/second and byte */ + *unit = humanise_rate ? _("B/s") : _("B"); + else + *unit = humanise_rate ? /* TRANSLATORS: IEC 80000-13:2008 byte/second */ - Q_("%u byte/s", "%u bytes/s", bytes), - (unsigned)bytes); + Q_("byte/s", "bytes/s", bytes) : + /* TRANSLATORS: IEC 80000-13:2008 byte */ + Q_("byte", "bytes", bytes); } } +static void strbuf_humanise(struct strbuf *buf, off_t bytes, unsigned flags) +{ + char *value; + const char *unit; + + humanise_bytes(bytes, &value, &unit, flags); + + /* + * TRANSLATORS: The first argument is the number string. The second + * argument is the unit string (i.e. "12.34 MiB/s"). + */ + strbuf_addf(buf, _("%s %s"), value, unit); + free(value); +} + void strbuf_humanise_bytes(struct strbuf *buf, off_t bytes) { strbuf_humanise(buf, bytes, 0); @@ -884,7 +920,7 @@ void strbuf_humanise_bytes(struct strbuf *buf, off_t bytes) void strbuf_humanise_rate(struct strbuf *buf, off_t bytes) { - strbuf_humanise(buf, bytes, 1); + strbuf_humanise(buf, bytes, HUMANISE_RATE); } int printf_ln(const char *fmt, ...) @@ -367,6 +367,31 @@ void strbuf_addbuf_percentquote(struct strbuf *dst, const struct strbuf *src); */ void strbuf_add_percentencode(struct strbuf *dst, const char *src, int flags); +enum humanise_flags { + /* + * Use rate based units for humanised values. + */ + HUMANISE_RATE = (1 << 0), + /* + * Use compact "B" unit symbol instead of "byte/bytes" for humanised + * values. + */ + HUMANISE_COMPACT = (1 << 1), +}; + +/** + * Converts the given byte size into a downscaled human-readable value and + * corresponding unit as two separate strings. + */ +void humanise_bytes(off_t bytes, char **value, const char **unit, + unsigned flags); + +/** + * Converts the given count into a downscaled human-readable value and + * corresponding unit as two separate strings. + */ +void humanise_count(size_t count, char **value, const char **unit); + /** * Append the given byte size as a human-readable string (i.e. 12.23 KiB, * 3.50 MiB). diff --git a/streaming.c b/streaming.c deleted file mode 100644 index 00ad649ae3..0000000000 --- a/streaming.c +++ /dev/null @@ -1,561 +0,0 @@ -/* - * Copyright (c) 2011, Google Inc. - */ - -#define USE_THE_REPOSITORY_VARIABLE - -#include "git-compat-util.h" -#include "convert.h" -#include "environment.h" -#include "streaming.h" -#include "repository.h" -#include "object-file.h" -#include "odb.h" -#include "replace-object.h" -#include "packfile.h" - -typedef int (*open_istream_fn)(struct git_istream *, - struct repository *, - const struct object_id *, - enum object_type *); -typedef int (*close_istream_fn)(struct git_istream *); -typedef ssize_t (*read_istream_fn)(struct git_istream *, char *, size_t); - -#define FILTER_BUFFER (1024*16) - -struct filtered_istream { - struct git_istream *upstream; - struct stream_filter *filter; - char ibuf[FILTER_BUFFER]; - char obuf[FILTER_BUFFER]; - int i_end, i_ptr; - int o_end, o_ptr; - int input_finished; -}; - -struct git_istream { - open_istream_fn open; - close_istream_fn close; - read_istream_fn read; - - unsigned long size; /* inflated size of full object */ - git_zstream z; - enum { z_unused, z_used, z_done, z_error } z_state; - - union { - struct { - char *buf; /* from odb_read_object_info_extended() */ - unsigned long read_ptr; - } incore; - - struct { - void *mapped; - unsigned long mapsize; - char hdr[32]; - int hdr_avail; - int hdr_used; - } loose; - - struct { - struct packed_git *pack; - off_t pos; - } in_pack; - - struct filtered_istream filtered; - } u; -}; - -/***************************************************************** - * - * Common helpers - * - *****************************************************************/ - -static void close_deflated_stream(struct git_istream *st) -{ - if (st->z_state == z_used) - git_inflate_end(&st->z); -} - - -/***************************************************************** - * - * Filtered stream - * - *****************************************************************/ - -static int close_istream_filtered(struct git_istream *st) -{ - free_stream_filter(st->u.filtered.filter); - return close_istream(st->u.filtered.upstream); -} - -static ssize_t read_istream_filtered(struct git_istream *st, char *buf, - size_t sz) -{ - struct filtered_istream *fs = &(st->u.filtered); - size_t filled = 0; - - while (sz) { - /* do we already have filtered output? */ - if (fs->o_ptr < fs->o_end) { - size_t to_move = fs->o_end - fs->o_ptr; - if (sz < to_move) - to_move = sz; - memcpy(buf + filled, fs->obuf + fs->o_ptr, to_move); - fs->o_ptr += to_move; - sz -= to_move; - filled += to_move; - continue; - } - fs->o_end = fs->o_ptr = 0; - - /* do we have anything to feed the filter with? */ - if (fs->i_ptr < fs->i_end) { - size_t to_feed = fs->i_end - fs->i_ptr; - size_t to_receive = FILTER_BUFFER; - if (stream_filter(fs->filter, - fs->ibuf + fs->i_ptr, &to_feed, - fs->obuf, &to_receive)) - return -1; - fs->i_ptr = fs->i_end - to_feed; - fs->o_end = FILTER_BUFFER - to_receive; - continue; - } - - /* tell the filter to drain upon no more input */ - if (fs->input_finished) { - size_t to_receive = FILTER_BUFFER; - if (stream_filter(fs->filter, - NULL, NULL, - fs->obuf, &to_receive)) - return -1; - fs->o_end = FILTER_BUFFER - to_receive; - if (!fs->o_end) - break; - continue; - } - fs->i_end = fs->i_ptr = 0; - - /* refill the input from the upstream */ - if (!fs->input_finished) { - fs->i_end = read_istream(fs->upstream, fs->ibuf, FILTER_BUFFER); - if (fs->i_end < 0) - return -1; - if (fs->i_end) - continue; - } - fs->input_finished = 1; - } - return filled; -} - -static struct git_istream *attach_stream_filter(struct git_istream *st, - struct stream_filter *filter) -{ - struct git_istream *ifs = xmalloc(sizeof(*ifs)); - struct filtered_istream *fs = &(ifs->u.filtered); - - ifs->close = close_istream_filtered; - ifs->read = read_istream_filtered; - fs->upstream = st; - fs->filter = filter; - fs->i_end = fs->i_ptr = 0; - fs->o_end = fs->o_ptr = 0; - fs->input_finished = 0; - ifs->size = -1; /* unknown */ - return ifs; -} - -/***************************************************************** - * - * Loose object stream - * - *****************************************************************/ - -static ssize_t read_istream_loose(struct git_istream *st, char *buf, size_t sz) -{ - size_t total_read = 0; - - switch (st->z_state) { - case z_done: - return 0; - case z_error: - return -1; - default: - break; - } - - if (st->u.loose.hdr_used < st->u.loose.hdr_avail) { - size_t to_copy = st->u.loose.hdr_avail - st->u.loose.hdr_used; - if (sz < to_copy) - to_copy = sz; - memcpy(buf, st->u.loose.hdr + st->u.loose.hdr_used, to_copy); - st->u.loose.hdr_used += to_copy; - total_read += to_copy; - } - - while (total_read < sz) { - int status; - - st->z.next_out = (unsigned char *)buf + total_read; - st->z.avail_out = sz - total_read; - status = git_inflate(&st->z, Z_FINISH); - - total_read = st->z.next_out - (unsigned char *)buf; - - if (status == Z_STREAM_END) { - git_inflate_end(&st->z); - st->z_state = z_done; - break; - } - if (status != Z_OK && (status != Z_BUF_ERROR || total_read < sz)) { - git_inflate_end(&st->z); - st->z_state = z_error; - return -1; - } - } - return total_read; -} - -static int close_istream_loose(struct git_istream *st) -{ - close_deflated_stream(st); - munmap(st->u.loose.mapped, st->u.loose.mapsize); - return 0; -} - -static int open_istream_loose(struct git_istream *st, struct repository *r, - const struct object_id *oid, - enum object_type *type) -{ - struct object_info oi = OBJECT_INFO_INIT; - struct odb_source *source; - - oi.sizep = &st->size; - oi.typep = type; - - odb_prepare_alternates(r->objects); - for (source = r->objects->sources; source; source = source->next) { - st->u.loose.mapped = odb_source_loose_map_object(source, oid, - &st->u.loose.mapsize); - if (st->u.loose.mapped) - break; - } - if (!st->u.loose.mapped) - return -1; - - switch (unpack_loose_header(&st->z, st->u.loose.mapped, - st->u.loose.mapsize, st->u.loose.hdr, - sizeof(st->u.loose.hdr))) { - case ULHR_OK: - break; - case ULHR_BAD: - case ULHR_TOO_LONG: - goto error; - } - if (parse_loose_header(st->u.loose.hdr, &oi) < 0 || *type < 0) - goto error; - - st->u.loose.hdr_used = strlen(st->u.loose.hdr) + 1; - st->u.loose.hdr_avail = st->z.total_out; - st->z_state = z_used; - st->close = close_istream_loose; - st->read = read_istream_loose; - - return 0; -error: - git_inflate_end(&st->z); - munmap(st->u.loose.mapped, st->u.loose.mapsize); - return -1; -} - - -/***************************************************************** - * - * Non-delta packed object stream - * - *****************************************************************/ - -static ssize_t read_istream_pack_non_delta(struct git_istream *st, char *buf, - size_t sz) -{ - size_t total_read = 0; - - switch (st->z_state) { - case z_unused: - memset(&st->z, 0, sizeof(st->z)); - git_inflate_init(&st->z); - st->z_state = z_used; - break; - case z_done: - return 0; - case z_error: - return -1; - case z_used: - break; - } - - while (total_read < sz) { - int status; - struct pack_window *window = NULL; - unsigned char *mapped; - - mapped = use_pack(st->u.in_pack.pack, &window, - st->u.in_pack.pos, &st->z.avail_in); - - st->z.next_out = (unsigned char *)buf + total_read; - st->z.avail_out = sz - total_read; - st->z.next_in = mapped; - status = git_inflate(&st->z, Z_FINISH); - - st->u.in_pack.pos += st->z.next_in - mapped; - total_read = st->z.next_out - (unsigned char *)buf; - unuse_pack(&window); - - if (status == Z_STREAM_END) { - git_inflate_end(&st->z); - st->z_state = z_done; - break; - } - - /* - * Unlike the loose object case, we do not have to worry here - * about running out of input bytes and spinning infinitely. If - * we get Z_BUF_ERROR due to too few input bytes, then we'll - * replenish them in the next use_pack() call when we loop. If - * we truly hit the end of the pack (i.e., because it's corrupt - * or truncated), then use_pack() catches that and will die(). - */ - if (status != Z_OK && status != Z_BUF_ERROR) { - git_inflate_end(&st->z); - st->z_state = z_error; - return -1; - } - } - return total_read; -} - -static int close_istream_pack_non_delta(struct git_istream *st) -{ - close_deflated_stream(st); - return 0; -} - -static int open_istream_pack_non_delta(struct git_istream *st, - struct repository *r UNUSED, - const struct object_id *oid UNUSED, - enum object_type *type UNUSED) -{ - struct pack_window *window; - enum object_type in_pack_type; - - window = NULL; - - in_pack_type = unpack_object_header(st->u.in_pack.pack, - &window, - &st->u.in_pack.pos, - &st->size); - unuse_pack(&window); - switch (in_pack_type) { - default: - return -1; /* we do not do deltas for now */ - case OBJ_COMMIT: - case OBJ_TREE: - case OBJ_BLOB: - case OBJ_TAG: - break; - } - st->z_state = z_unused; - st->close = close_istream_pack_non_delta; - st->read = read_istream_pack_non_delta; - - return 0; -} - - -/***************************************************************** - * - * In-core stream - * - *****************************************************************/ - -static int close_istream_incore(struct git_istream *st) -{ - free(st->u.incore.buf); - return 0; -} - -static ssize_t read_istream_incore(struct git_istream *st, char *buf, size_t sz) -{ - size_t read_size = sz; - size_t remainder = st->size - st->u.incore.read_ptr; - - if (remainder <= read_size) - read_size = remainder; - if (read_size) { - memcpy(buf, st->u.incore.buf + st->u.incore.read_ptr, read_size); - st->u.incore.read_ptr += read_size; - } - return read_size; -} - -static int open_istream_incore(struct git_istream *st, struct repository *r, - const struct object_id *oid, enum object_type *type) -{ - struct object_info oi = OBJECT_INFO_INIT; - - st->u.incore.read_ptr = 0; - st->close = close_istream_incore; - st->read = read_istream_incore; - - oi.typep = type; - oi.sizep = &st->size; - oi.contentp = (void **)&st->u.incore.buf; - return odb_read_object_info_extended(r->objects, oid, &oi, - OBJECT_INFO_DIE_IF_CORRUPT); -} - -/***************************************************************************** - * static helpers variables and functions for users of streaming interface - *****************************************************************************/ - -static int istream_source(struct git_istream *st, - struct repository *r, - const struct object_id *oid, - enum object_type *type) -{ - unsigned long size; - int status; - struct object_info oi = OBJECT_INFO_INIT; - - oi.typep = type; - oi.sizep = &size; - status = odb_read_object_info_extended(r->objects, oid, &oi, 0); - if (status < 0) - return status; - - switch (oi.whence) { - case OI_LOOSE: - st->open = open_istream_loose; - return 0; - case OI_PACKED: - if (!oi.u.packed.is_delta && - repo_settings_get_big_file_threshold(the_repository) < size) { - st->u.in_pack.pack = oi.u.packed.pack; - st->u.in_pack.pos = oi.u.packed.offset; - st->open = open_istream_pack_non_delta; - return 0; - } - /* fallthru */ - default: - st->open = open_istream_incore; - return 0; - } -} - -/**************************************************************** - * Users of streaming interface - ****************************************************************/ - -int close_istream(struct git_istream *st) -{ - int r = st->close(st); - free(st); - return r; -} - -ssize_t read_istream(struct git_istream *st, void *buf, size_t sz) -{ - return st->read(st, buf, sz); -} - -struct git_istream *open_istream(struct repository *r, - const struct object_id *oid, - enum object_type *type, - unsigned long *size, - struct stream_filter *filter) -{ - struct git_istream *st = xmalloc(sizeof(*st)); - const struct object_id *real = lookup_replace_object(r, oid); - int ret = istream_source(st, r, real, type); - - if (ret) { - free(st); - return NULL; - } - - if (st->open(st, r, real, type)) { - if (open_istream_incore(st, r, real, type)) { - free(st); - return NULL; - } - } - if (filter) { - /* Add "&& !is_null_stream_filter(filter)" for performance */ - struct git_istream *nst = attach_stream_filter(st, filter); - if (!nst) { - close_istream(st); - return NULL; - } - st = nst; - } - - *size = st->size; - return st; -} - -int stream_blob_to_fd(int fd, const struct object_id *oid, struct stream_filter *filter, - int can_seek) -{ - struct git_istream *st; - enum object_type type; - unsigned long sz; - ssize_t kept = 0; - int result = -1; - - st = open_istream(the_repository, oid, &type, &sz, filter); - if (!st) { - if (filter) - free_stream_filter(filter); - return result; - } - if (type != OBJ_BLOB) - goto close_and_exit; - for (;;) { - char buf[1024 * 16]; - ssize_t wrote, holeto; - ssize_t readlen = read_istream(st, buf, sizeof(buf)); - - if (readlen < 0) - goto close_and_exit; - if (!readlen) - break; - if (can_seek && sizeof(buf) == readlen) { - for (holeto = 0; holeto < readlen; holeto++) - if (buf[holeto]) - break; - if (readlen == holeto) { - kept += holeto; - continue; - } - } - - if (kept && lseek(fd, kept, SEEK_CUR) == (off_t) -1) - goto close_and_exit; - else - kept = 0; - wrote = write_in_full(fd, buf, readlen); - - if (wrote < 0) - goto close_and_exit; - } - if (kept && (lseek(fd, kept - 1, SEEK_CUR) == (off_t) -1 || - xwrite(fd, "", 1) != 1)) - goto close_and_exit; - result = 0; - - close_and_exit: - close_istream(st); - return result; -} diff --git a/streaming.h b/streaming.h deleted file mode 100644 index bd27f59e57..0000000000 --- a/streaming.h +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright (c) 2011, Google Inc. - */ -#ifndef STREAMING_H -#define STREAMING_H 1 - -#include "object.h" - -/* opaque */ -struct git_istream; -struct stream_filter; - -struct git_istream *open_istream(struct repository *, const struct object_id *, - enum object_type *, unsigned long *, - struct stream_filter *); -int close_istream(struct git_istream *); -ssize_t read_istream(struct git_istream *, void *, size_t); - -int stream_blob_to_fd(int fd, const struct object_id *, struct stream_filter *, int can_seek); - -#endif /* STREAMING_H */ diff --git a/subprojects/.gitignore b/subprojects/.gitignore index 63ea916ef5..2bb68c8794 100644 --- a/subprojects/.gitignore +++ b/subprojects/.gitignore @@ -1 +1,2 @@ /*/ +.wraplock diff --git a/t/helper/test-repository.c b/t/helper/test-repository.c index 63c37de33d..9ba94cdffa 100644 --- a/t/helper/test-repository.c +++ b/t/helper/test-repository.c @@ -17,10 +17,6 @@ static void test_parse_commit_in_graph(const char *gitdir, const char *worktree, struct commit *c; struct commit_list *parent; - setup_git_env(gitdir); - - repo_clear(the_repository); - if (repo_init(&r, gitdir, worktree)) die("Couldn't init repo"); @@ -47,10 +43,6 @@ static void test_get_commit_tree_in_graph(const char *gitdir, struct commit *c; struct tree *tree; - setup_git_env(gitdir); - - repo_clear(the_repository); - if (repo_init(&r, gitdir, worktree)) die("Couldn't init repo"); @@ -75,24 +67,20 @@ static void test_get_commit_tree_in_graph(const char *gitdir, int cmd__repository(int argc, const char **argv) { - int nongit_ok = 0; - - setup_git_directory_gently(&nongit_ok); - if (argc < 2) die("must have at least 2 arguments"); if (!strcmp(argv[1], "parse_commit_in_graph")) { struct object_id oid; if (argc < 5) die("not enough arguments"); - if (parse_oid_hex(argv[4], &oid, &argv[4])) + if (parse_oid_hex_any(argv[4], &oid, &argv[4]) == GIT_HASH_UNKNOWN) die("cannot parse oid '%s'", argv[4]); test_parse_commit_in_graph(argv[2], argv[3], &oid); } else if (!strcmp(argv[1], "get_commit_tree_in_graph")) { struct object_id oid; if (argc < 5) die("not enough arguments"); - if (parse_oid_hex(argv[4], &oid, &argv[4])) + if (parse_oid_hex_any(argv[4], &oid, &argv[4]) == GIT_HASH_UNKNOWN) die("cannot parse oid '%s'", argv[4]); test_get_commit_tree_in_graph(argv[2], argv[3], &oid); } else { diff --git a/t/helper/test-run-command.c b/t/helper/test-run-command.c index 3719f23cc2..49eace8dce 100644 --- a/t/helper/test-run-command.c +++ b/t/helper/test-run-command.c @@ -23,19 +23,26 @@ static int number_callbacks; static int parallel_next(struct child_process *cp, struct strbuf *err, void *cb, - void **task_cb UNUSED) + void **task_cb) { struct child_process *d = cb; if (number_callbacks >= 4) return 0; strvec_pushv(&cp->args, d->args.v); + cp->in = d->in; + cp->no_stdin = d->no_stdin; if (err) strbuf_addstr(err, "preloaded output of a child\n"); else fprintf(stderr, "preloaded output of a child\n"); number_callbacks++; + + /* test_stdin callback will use this to count remaining lines */ + *task_cb = xmalloc(sizeof(int)); + *(int*)(*task_cb) = 2; + return 1; } @@ -51,18 +58,61 @@ static int no_job(struct child_process *cp UNUSED, return 0; } +static void test_divert_output(struct strbuf *output, void *cb UNUSED) +{ + FILE *output_file; + + output_file = fopen("./output_file", "a"); + + strbuf_write(output, output_file); + fclose(output_file); +} + static int task_finished(int result UNUSED, struct strbuf *err, void *pp_cb UNUSED, - void *pp_task_cb UNUSED) + void *pp_task_cb) { if (err) strbuf_addstr(err, "asking for a quick stop\n"); else fprintf(stderr, "asking for a quick stop\n"); + + FREE_AND_NULL(pp_task_cb); + return 1; } +static int task_finished_quiet(int result UNUSED, + struct strbuf *err UNUSED, + void *pp_cb UNUSED, + void *pp_task_cb) +{ + FREE_AND_NULL(pp_task_cb); + return 0; +} + +static int test_stdin_pipe_feed(int hook_stdin_fd, void *cb UNUSED, void *task_cb) +{ + int *lines_remaining = task_cb; + + if (*lines_remaining) { + struct strbuf buf = STRBUF_INIT; + strbuf_addf(&buf, "sample stdin %d\n", --(*lines_remaining)); + if (write_in_full(hook_stdin_fd, buf.buf, buf.len) < 0) { + if (errno == EPIPE) { + /* child closed stdin, nothing more to do */ + strbuf_release(&buf); + return 1; + } + die_errno("write"); + } + strbuf_release(&buf); + } + + return !(*lines_remaining); +} + struct testsuite { struct string_list tests, failed; int next; @@ -157,6 +207,8 @@ static int testsuite(int argc, const char **argv) struct run_process_parallel_opts opts = { .get_next_task = next_test, .start_failure = test_failed, + .feed_pipe = test_stdin_pipe_feed, + .consume_output = test_divert_output, .task_finished = test_finished, .data = &suite, }; @@ -460,12 +512,23 @@ int cmd__run_command(int argc, const char **argv) if (!strcmp(argv[1], "run-command-parallel")) { opts.get_next_task = parallel_next; + opts.task_finished = task_finished_quiet; } else if (!strcmp(argv[1], "run-command-abort")) { opts.get_next_task = parallel_next; opts.task_finished = task_finished; } else if (!strcmp(argv[1], "run-command-no-jobs")) { opts.get_next_task = no_job; opts.task_finished = task_finished; + } else if (!strcmp(argv[1], "run-command-stdin")) { + proc.in = -1; + proc.no_stdin = 0; + opts.get_next_task = parallel_next; + opts.task_finished = task_finished_quiet; + opts.feed_pipe = test_stdin_pipe_feed; + } else if (!strcmp(argv[1], "run-command-divert-output")) { + opts.get_next_task = parallel_next; + opts.consume_output = test_divert_output; + opts.task_finished = task_finished_quiet; } else { ret = 1; fprintf(stderr, "check usage\n"); diff --git a/t/helper/test-simple-ipc.c b/t/helper/test-simple-ipc.c index 03cc5eea2c..442ad6b16f 100644 --- a/t/helper/test-simple-ipc.c +++ b/t/helper/test-simple-ipc.c @@ -603,7 +603,12 @@ int cmd__simple_ipc(int argc, const char **argv) OPT_INTEGER(0, "bytecount", &cl_args.bytecount, N_("number of bytes")), OPT_INTEGER(0, "batchsize", &cl_args.batchsize, N_("number of requests per thread")), - OPT_STRING(0, "byte", &bytevalue, N_("byte"), N_("ballast character")), + /* + * The "byte" string here is not marked for translation and + * instead relies on translation in strbuf.c:humanise_bytes() to + * avoid conflict with the plural form. + */ + OPT_STRING(0, "byte", &bytevalue, "byte", N_("ballast character")), OPT_STRING(0, "token", &cl_args.token, N_("token"), N_("command token to send to the server")), OPT_END() diff --git a/t/meson.build b/t/meson.build index 7c994d4643..c9f92450dc 100644 --- a/t/meson.build +++ b/t/meson.build @@ -185,6 +185,7 @@ integration_tests = [ 't1308-config-set.sh', 't1309-early-config.sh', 't1310-config-default.sh', + 't1311-config-optional.sh', 't1350-config-hooks-path.sh', 't1400-update-ref.sh', 't1401-symbolic-ref.sh', @@ -386,6 +387,8 @@ integration_tests = [ 't3436-rebase-more-options.sh', 't3437-rebase-fixup-options.sh', 't3438-rebase-broken-files.sh', + 't3450-history.sh', + 't3451-history-reword.sh', 't3500-cherry.sh', 't3501-revert-cherry-pick.sh', 't3502-cherry-pick-merge.sh', @@ -689,6 +692,7 @@ integration_tests = [ 't5562-http-backend-content-length.sh', 't5563-simple-http-auth.sh', 't5564-http-proxy.sh', + 't5565-push-multiple.sh', 't5570-git-daemon.sh', 't5571-pre-push-hook.sh', 't5572-pull-submodule.sh', diff --git a/t/perf/p6010-merge-base.sh b/t/perf/p6010-merge-base.sh index 54f52fa23e..08212dd037 100755 --- a/t/perf/p6010-merge-base.sh +++ b/t/perf/p6010-merge-base.sh @@ -83,9 +83,9 @@ build_history2 () { test_expect_success 'setup' ' max_level=15 && build_history $max_level | git fast-import --export-marks=marks && - git tag one && + git branch one && build_history2 $max_level | git fast-import --import-marks=marks --force && - git tag two && + git branch two && git gc && git log --format=%H --no-merges >expect ' @@ -98,4 +98,8 @@ test_expect_success 'verify result' ' test_cmp expect actual ' +test_perf 'git show-branch' ' + git show-branch one two +' + test_done diff --git a/t/t0001-init.sh b/t/t0001-init.sh index 618da080dc..e4d32bb4d2 100755 --- a/t/t0001-init.sh +++ b/t/t0001-init.sh @@ -425,7 +425,11 @@ test_expect_success SYMLINKS 're-init to move gitdir symlink' ' git init --separate-git-dir ../realgitdir ) && echo "gitdir: $(pwd)/realgitdir" >expected && - test_cmp expected newdir/.git && + case "$GIT_TEST_CMP" in + # `git diff --no-index` does not resolve symlinks + *--no-index*) cmp expected newdir/.git;; + *) test_cmp expected newdir/.git;; + esac && test_cmp expected newdir/here && test_path_is_dir realgitdir/refs ' diff --git a/t/t0061-run-command.sh b/t/t0061-run-command.sh index 76d4936a87..74529e219e 100755 --- a/t/t0061-run-command.sh +++ b/t/t0061-run-command.sh @@ -164,6 +164,44 @@ test_expect_success 'run_command runs ungrouped in parallel with more tasks than test_line_count = 4 err ' +test_expect_success 'run_command can divert output' ' + test_when_finished rm output_file && + test-tool run-command run-command-divert-output 3 sh -c "printf \"%s\n%s\n\" Hello World" 2>actual && + test_must_be_empty actual && + test_cmp expect output_file +' + +test_expect_success 'run_command listens to stdin' ' + cat >expect <<-\EOF && + preloaded output of a child + listening for stdin: + sample stdin 1 + sample stdin 0 + preloaded output of a child + listening for stdin: + sample stdin 1 + sample stdin 0 + preloaded output of a child + listening for stdin: + sample stdin 1 + sample stdin 0 + preloaded output of a child + listening for stdin: + sample stdin 1 + sample stdin 0 + EOF + + write_script stdin-script <<-\EOF && + echo "listening for stdin:" + while read line + do + echo "$line" + done + EOF + test-tool run-command run-command-stdin 2 ./stdin-script 2>actual && + test_cmp expect actual +' + cat >expect <<-EOF preloaded output of a child asking for a quick stop diff --git a/t/t0301-credential-cache.sh b/t/t0301-credential-cache.sh index dc30289f75..6f7cfd9e33 100755 --- a/t/t0301-credential-cache.sh +++ b/t/t0301-credential-cache.sh @@ -123,7 +123,8 @@ test_expect_success SYMLINKS 'use user socket if user directory is a symlink to rmdir \"\$HOME/dir/\" && rm \"\$HOME/.git-credential-cache\" " && - mkdir -p -m 700 "$HOME/dir/" && + mkdir -p "$HOME/dir/" && + chmod 700 "$HOME/dir/" && ln -s "$HOME/dir" "$HOME/.git-credential-cache" && check approve cache <<-\EOF && protocol=https diff --git a/t/t0600-reffiles-backend.sh b/t/t0600-reffiles-backend.sh index b11126ed47..74bfa2e9ba 100755 --- a/t/t0600-reffiles-backend.sh +++ b/t/t0600-reffiles-backend.sh @@ -467,7 +467,7 @@ test_expect_success POSIXPERM 'git reflog expire honors core.sharedRepository' ' esac ' -test_expect_success SYMLINKS 'symref transaction supports symlinks' ' +test_expect_success SYMLINKS,!MINGW 'symref transaction supports symlinks' ' test_when_finished "git symbolic-ref -d TEST_SYMREF_HEAD" && git update-ref refs/heads/new @ && test_config core.prefersymlinkrefs true && diff --git a/t/t0614-reftable-fsck.sh b/t/t0614-reftable-fsck.sh index 85cc47d67e..677eb9143c 100755 --- a/t/t0614-reftable-fsck.sh +++ b/t/t0614-reftable-fsck.sh @@ -20,7 +20,7 @@ test_expect_success "no errors reported on a well formed repository" ' done && # The repository should end up with multiple tables. - test_line_count ">" 1 .git/reftable/tables.list && + test_line_count -gt 1 .git/reftable/tables.list && git refs verify 2>err && test_must_be_empty err diff --git a/t/t1006-cat-file.sh b/t/t1006-cat-file.sh index 1f61b666a7..0eee3bb878 100755 --- a/t/t1006-cat-file.sh +++ b/t/t1006-cat-file.sh @@ -1048,18 +1048,28 @@ test_expect_success 'git cat-file --batch-check --follow-symlinks works for out- echo .. >>expect && echo HEAD:dir/subdir/out-of-repo-link-dir | git cat-file --batch-check --follow-symlinks >actual && test_cmp expect actual && - echo symlink 3 >expect && - echo ../ >>expect && + if test_have_prereq MINGW,SYMLINKS + then + test_write_lines "symlink 2" .. + else + test_write_lines "symlink 3" ../ + fi >expect && echo HEAD:dir/subdir/out-of-repo-link-dir-trailing | git cat-file --batch-check --follow-symlinks >actual && test_cmp expect actual ' test_expect_success 'git cat-file --batch-check --follow-symlinks works for symlinks with internal ..' ' - echo HEAD: | git cat-file --batch-check >expect && - echo HEAD:up-down | git cat-file --batch-check --follow-symlinks >actual && - test_cmp expect actual && - echo HEAD:up-down-trailing | git cat-file --batch-check --follow-symlinks >actual && - test_cmp expect actual && + if test_have_prereq !MINGW + then + # The `up-down` and `up-down-trailing` symlinks are normalized + # in MSYS in `winsymlinks` mode and are therefore in a + # different shape than Git expects them. + echo HEAD: | git cat-file --batch-check >expect && + echo HEAD:up-down | git cat-file --batch-check --follow-symlinks >actual && + test_cmp expect actual && + echo HEAD:up-down-trailing | git cat-file --batch-check --follow-symlinks >actual && + test_cmp expect actual + fi && echo HEAD:up-down-file | git cat-file --batch-check --follow-symlinks >actual && test_cmp found actual && echo symlink 7 >expect && diff --git a/t/t1305-config-include.sh b/t/t1305-config-include.sh index 8ff2b0c232..6e51f892f3 100755 --- a/t/t1305-config-include.sh +++ b/t/t1305-config-include.sh @@ -286,7 +286,7 @@ test_expect_success SYMLINKS 'conditional include, relative path with symlinks' ) ' -test_expect_success SYMLINKS 'conditional include, gitdir matching symlink' ' +test_expect_success SYMLINKS,!MINGW 'conditional include, gitdir matching symlink' ' ln -s foo bar && ( cd bar && @@ -298,7 +298,7 @@ test_expect_success SYMLINKS 'conditional include, gitdir matching symlink' ' ) ' -test_expect_success SYMLINKS 'conditional include, gitdir matching symlink, icase' ' +test_expect_success SYMLINKS,!MINGW 'conditional include, gitdir matching symlink, icase' ' ( cd bar && echo "[includeIf \"gitdir/i:BAR/\"]path=bar8" >>.git/config && diff --git a/t/t1311-config-optional.sh b/t/t1311-config-optional.sh new file mode 100755 index 0000000000..fbbacfc67b --- /dev/null +++ b/t/t1311-config-optional.sh @@ -0,0 +1,38 @@ +#!/bin/sh +# +# Copyright (c) 2025 Google LLC +# + +test_description=':(optional) paths' + +. ./test-lib.sh + +test_expect_success 'var=:(optional)path-exists' ' + test_config a.path ":(optional)path-exists" && + >path-exists && + echo path-exists >expect && + + git config get --path a.path >actual && + test_cmp expect actual +' + +test_expect_success 'missing optional value is ignored' ' + test_config a.path ":(optional)no-such-path" && + # Using --show-scope ensures we skip writing not only the value + # but also any meta-information about the ignored key. + test_must_fail git config get --show-scope --path a.path >actual && + test_line_count = 0 actual +' + +test_expect_success 'missing optional value is ignored in multi-value config' ' + test_when_finished "git config unset --all a.path" && + git config set --append a.path ":(optional)path-exists" && + git config set --append a.path ":(optional)no-such-path" && + >path-exists && + echo path-exists >expect && + + git config --get --path a.path >actual && + test_cmp expect actual +' + +test_done diff --git a/t/t1901-repo-structure.sh b/t/t1901-repo-structure.sh index 36a71a144e..17ff164b05 100755 --- a/t/t1901-repo-structure.sh +++ b/t/t1901-repo-structure.sh @@ -4,27 +4,54 @@ test_description='test git repo structure' . ./test-lib.sh +object_type_disk_usage() { + disk_usage_opt="--disk-usage" + + if test "$2" = "true" + then + disk_usage_opt="--disk-usage=human" + fi + + if test "$1" = "all" + then + git rev-list --all --objects $disk_usage_opt + else + git rev-list --all --objects $disk_usage_opt \ + --filter=object:type=$1 --filter-provided-objects + fi +} + test_expect_success 'empty repository' ' test_when_finished "rm -rf repo" && git init repo && ( cd repo && cat >expect <<-\EOF && - | Repository structure | Value | - | -------------------- | ----- | - | * References | | - | * Count | 0 | - | * Branches | 0 | - | * Tags | 0 | - | * Remotes | 0 | - | * Others | 0 | - | | | - | * Reachable objects | | - | * Count | 0 | - | * Commits | 0 | - | * Trees | 0 | - | * Blobs | 0 | - | * Tags | 0 | + | Repository structure | Value | + | -------------------- | ------ | + | * References | | + | * Count | 0 | + | * Branches | 0 | + | * Tags | 0 | + | * Remotes | 0 | + | * Others | 0 | + | | | + | * Reachable objects | | + | * Count | 0 | + | * Commits | 0 | + | * Trees | 0 | + | * Blobs | 0 | + | * Tags | 0 | + | * Inflated size | 0 B | + | * Commits | 0 B | + | * Trees | 0 B | + | * Blobs | 0 B | + | * Tags | 0 B | + | * Disk size | 0 B | + | * Commits | 0 B | + | * Trees | 0 B | + | * Blobs | 0 B | + | * Tags | 0 B | EOF git repo structure >out 2>err && @@ -34,12 +61,12 @@ test_expect_success 'empty repository' ' ) ' -test_expect_success 'repository with references and objects' ' +test_expect_success SHA1 'repository with references and objects' ' test_when_finished "rm -rf repo" && git init repo && ( cd repo && - test_commit_bulk 42 && + test_commit_bulk 1005 && git tag -a foo -m bar && oid="$(git rev-parse HEAD)" && @@ -48,22 +75,35 @@ test_expect_success 'repository with references and objects' ' # Also creates a commit, tree, and blob. git notes add -m foo && - cat >expect <<-\EOF && - | Repository structure | Value | - | -------------------- | ----- | - | * References | | - | * Count | 4 | - | * Branches | 1 | - | * Tags | 1 | - | * Remotes | 1 | - | * Others | 1 | - | | | - | * Reachable objects | | - | * Count | 130 | - | * Commits | 43 | - | * Trees | 43 | - | * Blobs | 43 | - | * Tags | 1 | + # The tags disk size is handled specially due to the + # git-rev-list(1) --disk-usage=human option printing the full + # "byte/bytes" unit string instead of just "B". + cat >expect <<-EOF && + | Repository structure | Value | + | -------------------- | ---------- | + | * References | | + | * Count | 4 | + | * Branches | 1 | + | * Tags | 1 | + | * Remotes | 1 | + | * Others | 1 | + | | | + | * Reachable objects | | + | * Count | 3.02 k | + | * Commits | 1.01 k | + | * Trees | 1.01 k | + | * Blobs | 1.01 k | + | * Tags | 1 | + | * Inflated size | 16.03 MiB | + | * Commits | 217.92 KiB | + | * Trees | 15.81 MiB | + | * Blobs | 11.68 KiB | + | * Tags | 132 B | + | * Disk size | $(object_type_disk_usage all true) | + | * Commits | $(object_type_disk_usage commit true) | + | * Trees | $(object_type_disk_usage tree true) | + | * Blobs | $(object_type_disk_usage blob true) | + | * Tags | $(object_type_disk_usage tag) B | EOF git repo structure >out 2>err && @@ -73,7 +113,7 @@ test_expect_success 'repository with references and objects' ' ) ' -test_expect_success 'keyvalue and nul format' ' +test_expect_success SHA1 'keyvalue and nul format' ' test_when_finished "rm -rf repo" && git init repo && ( @@ -81,7 +121,7 @@ test_expect_success 'keyvalue and nul format' ' test_commit_bulk 42 && git tag -a foo -m bar && - cat >expect <<-\EOF && + cat >expect <<-EOF && references.branches.count=1 references.tags.count=1 references.remotes.count=0 @@ -90,6 +130,14 @@ test_expect_success 'keyvalue and nul format' ' objects.trees.count=42 objects.blobs.count=42 objects.tags.count=1 + objects.commits.inflated_size=9225 + objects.trees.inflated_size=28554 + objects.blobs.inflated_size=453 + objects.tags.inflated_size=132 + objects.commits.disk_size=$(object_type_disk_usage commit) + objects.trees.disk_size=$(object_type_disk_usage tree) + objects.blobs.disk_size=$(object_type_disk_usage blob) + objects.tags.disk_size=$(object_type_disk_usage tag) EOF git repo structure --format=keyvalue >out 2>err && @@ -102,6 +150,13 @@ test_expect_success 'keyvalue and nul format' ' git repo structure --format=nul >out 2>err && test_cmp expect_nul out && + test_line_count = 0 err && + + # "-z", as a synonym to "--format=nul", participates in the + # usual "last one wins" rule. + git repo structure --format=table -z >out 2>err && + + test_cmp expect_nul out && test_line_count = 0 err ) ' diff --git a/t/t3200-branch.sh b/t/t3200-branch.sh index f3e720dc10..c58e505c43 100755 --- a/t/t3200-branch.sh +++ b/t/t3200-branch.sh @@ -1707,9 +1707,9 @@ test_expect_success '--track overrides branch.autoSetupMerge' ' ' test_expect_success 'errors if given a bad branch name' ' - cat <<-\EOF >expect && - fatal: '\''foo..bar'\'' is not a valid branch name - hint: See `man git check-ref-format` + cat <<-EOF >expect && + fatal: ${SQ}foo..bar${SQ} is not a valid branch name + hint: See ${SQ}git help check-ref-format${SQ} hint: Disable this message with "git config set advice.refSyntax false" EOF test_must_fail git branch foo..bar >actual 2>&1 && diff --git a/t/t3450-history.sh b/t/t3450-history.sh new file mode 100755 index 0000000000..f513463b92 --- /dev/null +++ b/t/t3450-history.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +test_description='tests for git-history command' + +. ./test-lib.sh + +test_expect_success 'does nothing without any arguments' ' + test_must_fail git history 2>err && + test_grep "need a subcommand" err +' + +test_expect_success 'raises an error with unknown argument' ' + test_must_fail git history garbage 2>err && + test_grep "unknown subcommand: .garbage." err +' + +test_done diff --git a/t/t3451-history-reword.sh b/t/t3451-history-reword.sh new file mode 100755 index 0000000000..3594421b68 --- /dev/null +++ b/t/t3451-history-reword.sh @@ -0,0 +1,391 @@ +#!/bin/sh + +test_description='tests for git-history reword subcommand' + +. ./test-lib.sh +. "$TEST_DIRECTORY/lib-log-graph.sh" + +reword_with_message () { + cat >message && + write_script fake-editor.sh <<-\EOF && + cp message "$1" + EOF + test_set_editor "$(pwd)"/fake-editor.sh && + git history reword "$@" && + rm fake-editor.sh message +} + +expect_graph () { + cat >expect && + lib_test_cmp_graph --graph --format=%s "$@" +} + +expect_log () { + git log --format="%s" "$@" >actual && + cat >expect && + test_cmp expect actual +} + +test_expect_success 'can reword tip of a branch' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + test_commit first && + test_commit second && + test_commit third && + + git symbolic-ref HEAD >expect && + reword_with_message HEAD <<-EOF && + third reworded + EOF + git symbolic-ref HEAD >actual && + test_cmp expect actual && + + expect_log <<-\EOF && + third reworded + second + first + EOF + + git reflog >reflog && + test_grep "reword: updating HEAD" reflog + ) +' + +test_expect_success 'can reword commit in the middle' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + test_commit first && + test_commit second && + test_commit third && + + git symbolic-ref HEAD >expect && + reword_with_message HEAD~ <<-EOF && + second reworded + EOF + git symbolic-ref HEAD >actual && + test_cmp expect actual && + + expect_log <<-\EOF + third + second reworded + first + EOF + ) +' + +test_expect_success 'can reword commit in the middle even on detached head' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + test_commit first && + test_commit second && + test_commit third_on_main && + git checkout --detach HEAD^ && + test_commit third_on_head && + + reword_with_message HEAD~ <<-EOF && + second reworded + EOF + + expect_graph HEAD --branches <<-\EOF + * third_on_head + | * third_on_main + |/ + * second reworded + * first + EOF + ) +' + +test_expect_success 'can reword the detached head' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + test_commit first && + test_commit second && + git checkout --detach HEAD && + test_commit third && + + reword_with_message HEAD <<-EOF && + third reworded + EOF + + expect_log <<-\EOF + third reworded + second + first + EOF + ) +' + +test_expect_success 'can reword root commit' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + test_commit first && + test_commit second && + test_commit third && + reword_with_message HEAD~2 <<-EOF && + first reworded + EOF + + expect_log <<-\EOF + third + second + first reworded + EOF + ) +' + +test_expect_success 'can reword in a bare repo' ' + test_when_finished "rm -rf repo repo.git" && + git init repo && + test_commit -C repo first && + git clone --bare repo repo.git && + ( + cd repo.git && + reword_with_message HEAD <<-EOF && + reworded + EOF + + expect_log <<-\EOF + reworded + EOF + ) +' + +test_expect_success 'can reword a commit on a different branch' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + test_commit base && + git branch theirs && + test_commit ours && + git switch theirs && + test_commit theirs && + + git rev-parse ours >ours-before && + reword_with_message theirs <<-EOF && + Reworded theirs + EOF + git rev-parse ours >ours-after && + test_cmp ours-before ours-after && + + expect_graph --branches <<-\EOF + * Reworded theirs + | * ours + |/ + * base + EOF + ) +' + +test_expect_success 'can reword a merge commit' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + test_commit base && + git branch branch && + test_commit ours && + git switch branch && + test_commit theirs && + git switch - && + git merge theirs && + + # It is not possible to replay merge commits embedded in the + # history (yet). + test_must_fail git history reword HEAD~ 2>err && + test_grep "replaying merge commits is not supported yet" err && + + # But it is possible to reword a merge commit directly. + reword_with_message HEAD <<-EOF && + Reworded merge commit + EOF + expect_graph <<-\EOF + * Reworded merge commit + |\ + | * theirs + * | ours + |/ + * base + EOF + ) +' + +test_expect_success '--ref-action=print prints ref updates without modifying repo' ' + test_when_finished "rm -rf repo" && + git init repo --initial-branch=main && + ( + cd repo && + test_commit base && + git branch branch && + test_commit ours && + git switch branch && + test_commit theirs && + + git refs list >refs-expect && + reword_with_message --ref-action=print base >updates <<-\EOF && + reworded commit + EOF + git refs list >refs-actual && + test_cmp refs-expect refs-actual && + + test_grep "update refs/heads/branch" updates && + test_grep "update refs/heads/main" updates && + git update-ref --stdin <updates && + expect_log --branches <<-\EOF + theirs + ours + reworded commit + EOF + ) +' + +test_expect_success '--ref-action=head updates only HEAD' ' + test_when_finished "rm -rf repo" && + git init repo --initial-branch=main && + ( + cd repo && + test_commit base && + git branch branch && + test_commit theirs && + git switch branch && + test_commit ours && + + # When told to update HEAD, only, the command will refuse to + # rewrite commits that are not an ancestor of HEAD. + test_must_fail git history reword --ref-action=head theirs 2>err && + test_grep "rewritten commit must be an ancestor of HEAD" err && + + reword_with_message --ref-action=head base >updates <<-\EOF && + reworded base + EOF + expect_log HEAD <<-\EOF && + ours + reworded base + EOF + expect_log main <<-\EOF + theirs + base + EOF + ) +' + +test_expect_success 'editor shows proper status' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + test_commit first && + + write_script fake-editor.sh <<-\EOF && + cp "$1" . && + printf "\namend a comment\n" >>"$1" + EOF + test_set_editor "$(pwd)"/fake-editor.sh && + git history reword HEAD && + + cat >expect <<-EOF && + first + + # Please enter the commit message for the reworded changes. Lines starting + # with ${SQ}#${SQ} will be ignored, and an empty message aborts the commit. + # Changes to be committed: + # new file: first.t + # + EOF + test_cmp expect COMMIT_EDITMSG && + + test_commit_message HEAD <<-\EOF + first + + amend a comment + EOF + ) +' + +# For now, git-history(1) does not yet execute any hooks. This is subject to +# change in the future, and if it does this test here is expected to start +# failing. In other words, this test is not an endorsement of the current +# status quo. +test_expect_success 'hooks are not executed for rewritten commits' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + test_commit first && + test_commit second && + test_commit third && + + ORIG_PATH="$(pwd)" && + export ORIG_PATH && + for hook in prepare-commit-msg pre-commit post-commit post-rewrite commit-msg + do + write_script .git/hooks/$hook <<-\EOF || exit 1 + touch "$ORIG_PATH/hooks.log + EOF + done && + + reword_with_message HEAD~ <<-EOF && + second reworded + EOF + + cat >expect <<-EOF && + third + second reworded + first + EOF + git log --format=%s >actual && + test_cmp expect actual && + + test_path_is_missing hooks.log + ) +' + +test_expect_success 'aborts with empty commit message' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + test_commit first && + + ! reword_with_message HEAD 2>err </dev/null && + test_grep "Aborting commit due to empty commit message." err + ) +' + +test_expect_success 'retains changes in the worktree and index' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + touch a b && + git add . && + git commit -m "initial commit" && + echo foo >a && + echo bar >b && + git add b && + reword_with_message HEAD <<-EOF && + message + EOF + cat >expect <<-\EOF && + M a + M b + ?? actual + ?? expect + EOF + git status --porcelain >actual && + test_cmp expect actual + ) +' + +test_done diff --git a/t/t3650-replay-basics.sh b/t/t3650-replay-basics.sh index cf3aacf355..c862aa39f3 100755 --- a/t/t3650-replay-basics.sh +++ b/t/t3650-replay-basics.sh @@ -43,6 +43,13 @@ test_expect_success 'setup' ' test_commit L && test_commit M && + git switch --detach topic4 && + test_commit N && + test_commit O && + git switch -c topic-with-merge topic4 && + test_merge P O --no-ff && + git switch main && + git switch -c conflict B && test_commit C.conflict C.t conflict ' @@ -51,6 +58,53 @@ test_expect_success 'setup bare' ' git clone --bare . bare ' +test_expect_success 'argument to --advance must be a reference' ' + echo "fatal: argument to --advance must be a reference" >expect && + oid=$(git rev-parse main) && + test_must_fail git replay --advance=$oid topic1..topic2 2>actual && + test_cmp expect actual +' + +test_expect_success '--onto with invalid commit-ish' ' + printf "fatal: ${SQ}refs/not-valid${SQ} is not " >expect && + printf "a valid commit-ish for --onto\n" >>expect && + test_must_fail git replay --onto=refs/not-valid topic1..topic2 2>actual && + test_cmp expect actual +' + +test_expect_success 'option --onto or --advance is mandatory' ' + echo "error: option --onto or --advance is mandatory" >expect && + test_might_fail git replay -h >>expect && + test_must_fail git replay topic1..topic2 2>actual && + test_cmp expect actual +' + +test_expect_success 'no base or negative ref gives no-replaying down to root error' ' + echo "fatal: replaying down from root commit is not supported yet!" >expect && + test_must_fail git replay --onto=topic1 topic2 2>actual && + test_cmp expect actual +' + +test_expect_success 'options --advance and --contained cannot be used together' ' + printf "fatal: options ${SQ}--advance${SQ} " >expect && + printf "and ${SQ}--contained${SQ} cannot be used together\n" >>expect && + test_must_fail git replay --advance=main --contained \ + topic1..topic2 2>actual && + test_cmp expect actual +' + +test_expect_success 'cannot advance target ... ordering would be ill-defined' ' + echo "fatal: cannot advance target with multiple sources because ordering would be ill-defined" >expect && + test_must_fail git replay --advance=main main topic1 topic2 2>actual && + test_cmp expect actual +' + +test_expect_success 'replaying merge commits is not supported yet' ' + echo "fatal: replaying merge commits is not supported yet!" >expect && + test_must_fail git replay --advance=main main..topic-with-merge 2>actual && + test_cmp expect actual +' + test_expect_success 'using replay to rebase two branches, one on top of other' ' git replay --ref-action=print --onto main topic1..topic2 >result && @@ -195,6 +249,15 @@ test_expect_success 'using replay on bare repo to rebase multiple divergent bran done ' +test_expect_success 'using replay to update detached HEAD' ' + current_head=$(git branch --show-current) && + test_when_finished git switch "$current_head" && + git switch --detach && + test_commit something && + git replay --ref-action=print --onto HEAD~2 --ref-action=print HEAD~..HEAD >updates && + test_grep "update HEAD " updates +' + test_expect_success 'merge.directoryRenames=false' ' # create a test case that stress-tests the rename caching git switch -c rename-onto && diff --git a/t/t4007-rename-3.sh b/t/t4007-rename-3.sh index 3fc81bcd76..1012a370dd 100755 --- a/t/t4007-rename-3.sh +++ b/t/t4007-rename-3.sh @@ -67,7 +67,28 @@ test_expect_success 'copy, limited to a subtree' ' ' test_expect_success 'tweak work tree' ' - rm -f path0/COPYING && + rm -f path0/COPYING +' + +cat >expected <<EOF +:100644 100644 $blob $blob C100 path1/COPYING path0/COPYING +EOF + +# The cache has path0/COPYING and path1/COPYING, the working tree only +# path1/COPYING. This is a deletion -- we don't treat deduplication +# specially. In reverse it should be detected as a copy, though. +test_expect_success 'copy detection, files to index' ' + git diff-files -C --find-copies-harder -R >current && + compare_diff_raw current expected +' + +test_expect_success 'copy detection, files to preloaded index' ' + GIT_TEST_PRELOAD_INDEX=1 \ + git diff-files -C --find-copies-harder -R >current && + compare_diff_raw current expected +' + +test_expect_success 'tweak index' ' git update-index --remove path0/COPYING ' # In the tree, there is only path0/COPYING. In the cache, path0 does diff --git a/t/t4014-format-patch.sh b/t/t4014-format-patch.sh index 2782b1fc18..21d6d0cd9e 100755 --- a/t/t4014-format-patch.sh +++ b/t/t4014-format-patch.sh @@ -980,7 +980,7 @@ test_expect_success 'format-patch --ignore-if-in-upstream HEAD' ' test_expect_success 'get git version' ' git_version=$(git --version) && - git_version=${git_version##* } + git_version=${git_version#git version } ' signature() { diff --git a/t/t5302-pack-index.sh b/t/t5302-pack-index.sh index 413c99274c..9697448cb2 100755 --- a/t/t5302-pack-index.sh +++ b/t/t5302-pack-index.sh @@ -293,4 +293,20 @@ test_expect_success 'too-large packs report the breach' ' grep "maximum allowed size (20 bytes)" err ' +# git-index-pack(1) uses the default hash algorithm outside of the repository, +# and it has no way to tell it otherwise. So we can only run this test with the +# default hash algorithm, as it would otherwise fail to parse the tree. +test_expect_success DEFAULT_HASH_ALGORITHM 'index-pack --fsck-objects outside of a repo' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + printf "100644 blob $(test_oid 001)\t.gitattributes\n" >tree && + git mktree --missing <tree >tree-oid && + git pack-objects <tree-oid pack && + test_must_fail nongit git index-pack --fsck-objects "$(pwd)"/pack-*.pack 2>err && + test_grep "cannot perform queued object checks outside of a repository" err + ) +' + test_done diff --git a/t/t5319-multi-pack-index.sh b/t/t5319-multi-pack-index.sh index 93f319a4b2..794f8b5ab4 100755 --- a/t/t5319-multi-pack-index.sh +++ b/t/t5319-multi-pack-index.sh @@ -350,9 +350,73 @@ test_expect_success 'preferred pack from existing MIDX without bitmaps' ' # the new MIDX git multi-pack-index write --preferred-pack=pack-$pack.pack ) +' + +test_expect_success 'preferred pack cannot be determined without bitmap' ' + test_when_finished "rm -fr preferred-can-be-queried" && + git init preferred-can-be-queried && + ( + cd preferred-can-be-queried && + test_commit initial && + git repack -Adl --write-midx --no-write-bitmap-index && + test_must_fail test-tool read-midx --preferred-pack .git/objects 2>err && + test_grep "could not determine MIDX preferred pack" err && + git repack -Adl --write-midx --write-bitmap-index && + test-tool read-midx --preferred-pack .git/objects + ) +' + +test_midx_is_retained () { + test-tool chmtime =0 .git/objects/pack/multi-pack-index && + ls -l .git/objects/pack/multi-pack-index >expect && + git multi-pack-index write "$@" && + ls -l .git/objects/pack/multi-pack-index >actual && + test_cmp expect actual +} + +test_midx_is_rewritten () { + test-tool chmtime =0 .git/objects/pack/multi-pack-index && + ls -l .git/objects/pack/multi-pack-index >expect && + git multi-pack-index write "$@" && + ls -l .git/objects/pack/multi-pack-index >actual && + ! test_cmp expect actual +} + +test_expect_success 'up-to-date multi-pack-index is retained' ' + test_when_finished "rm -fr midx-up-to-date" && + git init midx-up-to-date && + ( + cd midx-up-to-date && + + # Write the initial pack that contains the most objects. + test_commit first && + test_commit second && + git repack -Ad --write-midx && + test_midx_is_retained && + + # Writing a new bitmap index should cause us to regenerate the MIDX. + test_midx_is_rewritten --bitmap && + test_midx_is_retained --bitmap && + # Ensure that writing a new packfile causes us to rewrite the index. + test_commit incremental && + git repack -d && + test_midx_is_rewritten && + test_midx_is_retained && + + for pack in .git/objects/pack/*.idx + do + basename "$pack" || exit 1 + done >stdin && + test_line_count = 2 stdin && + test_midx_is_retained --stdin-packs <stdin && + head -n1 stdin >stdin.trimmed && + test_midx_is_rewritten --stdin-packs <stdin.trimmed + ) ' +test_done + test_expect_success 'verify multi-pack-index success' ' git multi-pack-index verify --object-dir=$objdir ' diff --git a/t/t5510-fetch.sh b/t/t5510-fetch.sh index b7059cccaa..ce1c23684e 100755 --- a/t/t5510-fetch.sh +++ b/t/t5510-fetch.sh @@ -1552,6 +1552,7 @@ test_expect_success CASE_INSENSITIVE_FS,REFFILES 'D/F conflict on case insensiti ' test_expect_success REFFILES 'D/F conflict on case sensitive filesystem with lock' ' + test_when_finished rm -rf base repo && ( git init --ref-format=reftable base && cd base && @@ -1577,6 +1578,155 @@ test_expect_success REFFILES 'D/F conflict on case sensitive filesystem with loc ) ' +test_expect_success 'fetch --tags fetches existing tags' ' + test_when_finished rm -rf base repo && + + git init base && + git -C base commit --allow-empty -m "empty-commit" && + + git clone --bare base repo && + + git -C base tag tag-1 && + git -C repo for-each-ref >out && + test_grep ! "tag-1" out && + git -C repo fetch --tags && + git -C repo for-each-ref >out && + test_grep "tag-1" out +' + +test_expect_success 'fetch --tags fetches non-conflicting tags' ' + test_when_finished rm -rf base repo && + + git init base && + git -C base commit --allow-empty -m "empty-commit" && + git -C base tag tag-1 && + + git clone --bare base repo && + + git -C base tag tag-2 && + git -C repo for-each-ref >out && + test_grep ! "tag-2" out && + + git -C base commit --allow-empty -m "second empty-commit" && + git -C base tag -f tag-1 && + + test_must_fail git -C repo fetch --tags 2>out && + test_grep "tag-1 (would clobber existing tag)" out && + git -C repo for-each-ref >out && + test_grep "tag-2" out +' + +test_expect_success "backfill tags when providing a refspec" ' + test_when_finished rm -rf source target && + + git init source && + git -C source commit --allow-empty --message common && + git clone file://"$(pwd)"/source target && + ( + cd source && + test_commit history && + test_commit fetch-me + ) && + + # The "history" tag is backfilled even though we requested + # to only fetch HEAD + git -C target fetch origin HEAD:branch && + git -C target tag -l >actual && + cat >expect <<-\EOF && + fetch-me + history + EOF + test_cmp expect actual +' + +test_expect_success REFFILES "FETCH_HEAD is updated even if ref updates fail" ' + test_when_finished rm -rf base repo && + + git init base && + ( + cd base && + test_commit "updated" && + + git update-ref refs/heads/foo @ && + git update-ref refs/heads/branch @ + ) && + + git init --bare repo && + ( + cd repo && + rm -f FETCH_HEAD && + git remote add origin ../base && + >refs/heads/foo.lock && + test_must_fail git fetch -f origin "refs/heads/*:refs/heads/*" 2>err && + test_grep "error: fetching ref refs/heads/foo failed: reference already exists" err && + test_grep "branch ${SQ}branch${SQ} of ../base" FETCH_HEAD && + test_grep "branch ${SQ}foo${SQ} of ../base" FETCH_HEAD + ) +' + +test_expect_success "upstream tracking info is added with --set-upstream" ' + test_when_finished rm -rf base repo && + + git init --initial-branch=main base && + test_commit -C base "updated" && + + git init --bare --initial-branch=main repo && + ( + cd repo && + git remote add origin ../base && + git fetch origin --set-upstream main && + git config get branch.main.remote >actual && + echo "origin" >expect && + test_cmp expect actual + ) +' + +test_expect_success REFFILES "upstream tracking info is added even with conflicts" ' + test_when_finished rm -rf base repo && + + git init --initial-branch=main base && + test_commit -C base "updated" && + + git init --bare --initial-branch=main repo && + ( + cd repo && + git remote add origin ../base && + test_must_fail git config get branch.main.remote && + + mkdir -p refs/remotes/origin && + >refs/remotes/origin/main.lock && + test_must_fail git fetch origin --set-upstream main && + git config get branch.main.remote >actual && + echo "origin" >expect && + test_cmp expect actual + ) +' + +test_expect_success REFFILES "HEAD is updated even with conflicts" ' + test_when_finished rm -rf base repo && + + git init base && + ( + cd base && + test_commit "updated" && + + git update-ref refs/heads/foo @ && + git update-ref refs/heads/branch @ + ) && + + git init --bare repo && + ( + cd repo && + git remote add origin ../base && + + test_path_is_missing refs/remotes/origin/HEAD && + mkdir -p refs/remotes/origin && + >refs/remotes/origin/branch.lock && + test_must_fail git fetch origin && + test -f refs/remotes/origin/HEAD + ) +' + . "$TEST_DIRECTORY"/lib-httpd.sh start_httpd diff --git a/t/t5551-http-fetch-smart.sh b/t/t5551-http-fetch-smart.sh index b0d4ea7801..73cf531580 100755 --- a/t/t5551-http-fetch-smart.sh +++ b/t/t5551-http-fetch-smart.sh @@ -333,12 +333,12 @@ test_expect_success 'dumb clone via http-backend respects namespace' ' test_expect_success 'cookies stored in http.cookiefile when http.savecookies set' ' cat >cookies.txt <<-\EOF && - 127.0.0.1 FALSE /smart_cookies/ FALSE 0 othername othervalue + 127.0.0.1 FALSE /smart_cookies FALSE 0 othername othervalue EOF sort >expect_cookies.txt <<-\EOF && - 127.0.0.1 FALSE /smart_cookies/ FALSE 0 othername othervalue - 127.0.0.1 FALSE /smart_cookies/repo.git/ FALSE 0 name value - 127.0.0.1 FALSE /smart_cookies/repo.git/info/ FALSE 0 name value + 127.0.0.1 FALSE /smart_cookies FALSE 0 othername othervalue + 127.0.0.1 FALSE /smart_cookies/repo.git FALSE 0 name value + 127.0.0.1 FALSE /smart_cookies/repo.git/info FALSE 0 name value EOF git config http.cookiefile cookies.txt && git config http.savecookies true && @@ -351,8 +351,11 @@ test_expect_success 'cookies stored in http.cookiefile when http.savecookies set tag -m "foo" cookie-tag && git fetch $HTTPD_URL/smart_cookies/repo.git cookie-tag && - grep "^[^#]" cookies.txt | sort >cookies_stripped.txt && - test_cmp expect_cookies.txt cookies_stripped.txt + # Strip trailing slashes from cookie paths to handle output from both + # old curl ("/smart_cookies/") and new ("/smart_cookies"). + HT=" " && + grep "^[^#]" cookies.txt | sed "s,/$HT,$HT," | sort >cookies_clean.txt && + test_cmp expect_cookies.txt cookies_clean.txt ' test_expect_success 'transfer.hiderefs works over smart-http' ' diff --git a/t/t5563-simple-http-auth.sh b/t/t5563-simple-http-auth.sh index 317f33af5a..c1febbae9d 100755 --- a/t/t5563-simple-http-auth.sh +++ b/t/t5563-simple-http-auth.sh @@ -469,7 +469,7 @@ test_expect_success 'access using basic auth with wwwauth header empty continuat EOF ' -test_expect_success 'access using basic auth with wwwauth header mixed line-endings' ' +test_expect_success 'access using basic auth with wwwauth header mixed continuations' ' test_when_finished "per_test_cleanup" && set_credential_reply get <<-EOF && @@ -490,7 +490,7 @@ test_expect_success 'access using basic auth with wwwauth header mixed line-endi printf "id=default response=WWW-Authenticate: FooBar param1=\"value1\"\r\n" >>"$CHALLENGE" && printf "id=default response= \r\n" >>"$CHALLENGE" && printf "id=default response=\tparam2=\"value2\"\r\n" >>"$CHALLENGE" && - printf "id=default response=WWW-Authenticate: Basic realm=\"example.com\"" >>"$CHALLENGE" && + printf "id=default response=WWW-Authenticate: Basic realm=\"example.com\"\r\n" >>"$CHALLENGE" && test_config_global credential.helper test-helper && git ls-remote "$HTTPD_URL/custom_auth/repo.git" && diff --git a/t/t5564-http-proxy.sh b/t/t5564-http-proxy.sh index c3903faf2d..3bcbdef409 100755 --- a/t/t5564-http-proxy.sh +++ b/t/t5564-http-proxy.sh @@ -40,10 +40,10 @@ test_expect_success 'clone can prompt for proxy password' ' start_socks() { mkfifo socks_output && - { + ( "$PERL_PATH" "$TEST_DIRECTORY/socks4-proxy.pl" "$1" >socks_output & echo $! > "$TRASH_DIRECTORY/socks.pid" - } && + ) && read line <socks_output && test "$line" = ready } diff --git a/t/t5565-push-multiple.sh b/t/t5565-push-multiple.sh new file mode 100755 index 0000000000..7e93668566 --- /dev/null +++ b/t/t5565-push-multiple.sh @@ -0,0 +1,39 @@ +#!/bin/sh + +test_description='push to group' + +. ./test-lib.sh + +test_expect_success setup ' + for i in 1 2 3 + do + git init dest-$i && + git -C dest-$i symbolic-ref HEAD refs/heads/not-a-branch || + return 1 + done && + test_tick && + git commit --allow-empty -m "initial" && + git config set --append remote.them.pushurl "file://$(pwd)/dest-1" && + git config set --append remote.them.pushurl "file://$(pwd)/dest-2" && + git config set --append remote.them.pushurl "file://$(pwd)/dest-3" && + git config set --append remote.them.push "+refs/heads/*:refs/heads/*" +' + +test_expect_success 'push to group' ' + git push them && + j= && + for i in 1 2 3 + do + git -C dest-$i for-each-ref >actual-$i && + if test -n "$j" + then + test_cmp actual-$j actual-$i + else + cat actual-$i + fi && + j=$i || + return 1 + done +' + +test_done diff --git a/t/t6423-merge-rename-directories.sh b/t/t6423-merge-rename-directories.sh index 533ac85dc8..53535a8ebf 100755 --- a/t/t6423-merge-rename-directories.sh +++ b/t/t6423-merge-rename-directories.sh @@ -5158,13 +5158,18 @@ test_setup_12m () { git switch B && git rm dir/subdir/file && mkdir dir && - ln -s /dev/null dir/subdir && + if test_have_prereq MINGW + then + cmd //c 'mklink dir\subdir NUL' + else + ln -s /dev/null dir/subdir + fi && git add . && git commit -m "B" ) } -test_expect_success '12m: Change parent of renamed-dir to symlink on other side' ' +test_expect_success SYMLINKS '12m: Change parent of renamed-dir to symlink on other side' ' test_setup_12m && ( cd 12m && diff --git a/t/t7400-submodule-basic.sh b/t/t7400-submodule-basic.sh index e6b551daad..65fcfae93a 100755 --- a/t/t7400-submodule-basic.sh +++ b/t/t7400-submodule-basic.sh @@ -48,6 +48,25 @@ test_expect_success 'submodule deinit works on empty repository' ' git submodule deinit --all ' +test_expect_success 'submodule add with incomplete .gitmodules' ' + test_when_finished "rm -f expect actual" && + test_when_finished "git config remove-section submodule.one" && + test_when_finished "git rm -f one .gitmodules" && + git init one && + git -C one commit --allow-empty -m one-initial && + git config -f .gitmodules submodule.one.ignore all && + + git submodule add ./one && + + for var in ignore path url + do + git config -f .gitmodules --get "submodule.one.$var" || + return 1 + done >actual && + test_write_lines all one ./one >expect && + test_cmp expect actual +' + test_expect_success 'setup - initial commit' ' >t && git add t && diff --git a/t/t7703-repack-geometric.sh b/t/t7703-repack-geometric.sh index 9fc1626fbf..98806cdb6f 100755 --- a/t/t7703-repack-geometric.sh +++ b/t/t7703-repack-geometric.sh @@ -287,6 +287,41 @@ test_expect_success '--geometric with pack.packSizeLimit' ' ) ' +test_expect_success '--geometric --write-midx retains up-to-date MIDX without bitmap index' ' + test_when_finished "rm -fr repo" && + git init repo && + ( + cd repo && + test_commit initial && + + test_path_is_missing .git/objects/pack/multi-pack-index && + git repack --geometric=2 --write-midx --no-write-bitmap-index && + test_path_is_file .git/objects/pack/multi-pack-index && + test-tool chmtime =0 .git/objects/pack/multi-pack-index && + + ls -l .git/objects/pack/ >expect && + git repack --geometric=2 --write-midx --no-write-bitmap-index && + ls -l .git/objects/pack/ >actual && + test_cmp expect actual + ) +' + +test_expect_success '--geometric --write-midx retains up-to-date MIDX with bitmap index' ' + test_when_finished "rm -fr repo" && + git init repo && + test_commit -C repo initial && + + test_path_is_missing repo/.git/objects/pack/multi-pack-index && + git -C repo repack --geometric=2 --write-midx --write-bitmap-index && + test_path_is_file repo/.git/objects/pack/multi-pack-index && + test-tool chmtime =0 repo/.git/objects/pack/multi-pack-index && + + ls -l repo/.git/objects/pack/ >expect && + git -C repo repack --geometric=2 --write-midx --write-bitmap-index && + ls -l repo/.git/objects/pack/ >actual && + test_cmp expect actual +' + test_expect_success '--geometric --write-midx with packfiles in main and alternate ODB' ' test_when_finished "rm -fr shared member" && diff --git a/t/t7800-difftool.sh b/t/t7800-difftool.sh index 9b74db5563..bf0f67378d 100755 --- a/t/t7800-difftool.sh +++ b/t/t7800-difftool.sh @@ -752,11 +752,11 @@ test_expect_success SYMLINKS 'difftool --dir-diff handles modified symlinks' ' c EOF git difftool --symlinks --dir-diff --extcmd ls >output && - grep -v ^/ output >actual && + grep -v ":\$" output >actual && test_cmp expect actual && git difftool --no-symlinks --dir-diff --extcmd ls >output && - grep -v ^/ output >actual && + grep -v ":\$" output >actual && test_cmp expect actual && # The left side contains symlink "c" that points to "b" @@ -786,11 +786,11 @@ test_expect_success SYMLINKS 'difftool --dir-diff handles modified symlinks' ' EOF git difftool --symlinks --dir-diff --extcmd ls >output && - grep -v ^/ output >actual && + grep -v ":\$" output >actual && test_cmp expect actual && git difftool --no-symlinks --dir-diff --extcmd ls >output && - grep -v ^/ output >actual && + grep -v ":\$" output >actual && test_cmp expect actual ' diff --git a/t/t8020-last-modified.sh b/t/t8020-last-modified.sh index a4c1114ee2..50f4312f71 100755 --- a/t/t8020-last-modified.sh +++ b/t/t8020-last-modified.sh @@ -78,6 +78,14 @@ test_expect_success 'last-modified subdir' ' EOF ' +test_expect_success 'last-modified in sparse checkout' ' + test_when_finished "git sparse-checkout disable" && + git sparse-checkout set b && + check_last_modified -- a <<-\EOF + 3 a + EOF +' + test_expect_success 'last-modified subdir recursive' ' check_last_modified -r a <<-\EOF 3 a/b/file diff --git a/t/t9210-scalar.sh b/t/t9210-scalar.sh index bd6f0c40d2..009437a5f3 100755 --- a/t/t9210-scalar.sh +++ b/t/t9210-scalar.sh @@ -202,14 +202,17 @@ test_expect_success 'scalar clone --no-... opts' ' test_expect_success 'scalar reconfigure' ' git init one/src && scalar register one && - git -C one/src config core.preloadIndex false && + git -C one/src config unset gui.gcwarning && scalar reconfigure one && - test true = "$(git -C one/src config core.preloadIndex)" && - git -C one/src config core.preloadIndex false && + test false = "$(git -C one/src config gui.gcwarning)" && + git -C one/src config unset gui.gcwarning && rm one/src/cron.txt && GIT_TRACE2_EVENT="$(pwd)/reconfigure" scalar reconfigure -a && test_path_is_file one/src/cron.txt && - test true = "$(git -C one/src config core.preloadIndex)" && + test false = "$(git -C one/src config gui.gcwarning)" && + test_grep "GCWarning = false # set by scalar" one/src/.git/config && + test_grep "excludeDecoration = refs/prefetch/\* # set by scalar" one/src/.git/config && + test_subcommand git maintenance start <reconfigure && test_subcommand ! git maintenance unregister --force <reconfigure && @@ -231,25 +234,29 @@ test_expect_success 'scalar reconfigure --all with includeIf.onbranch' ' git init $num/src && scalar register $num/src && git -C $num/src config includeif."onbranch:foo".path something && - git -C $num/src config core.preloadIndex false || return 1 + git -C $num/src config unset gui.gcwarning || return 1 done && scalar reconfigure --all && for num in $repos do - test true = "$(git -C $num/src config core.preloadIndex)" || return 1 + test false = "$(git -C $num/src config gui.gcwarning)" || return 1 done ' test_expect_success 'scalar reconfigure --all with detached HEADs' ' + # This test demonstrates an issue with index.skipHash=true and + # this test variable for the split index. Disable the test variable. + sane_unset GIT_TEST_SPLIT_INDEX && + repos="two three four" && for num in $repos do rm -rf $num/src && git init $num/src && scalar register $num/src && - git -C $num/src config core.preloadIndex false && + git -C $num/src config unset gui.gcwarning && test_commit -C $num/src initial && git -C $num/src switch --detach HEAD || return 1 done && @@ -258,7 +265,7 @@ test_expect_success 'scalar reconfigure --all with detached HEADs' ' for num in $repos do - test true = "$(git -C $num/src config core.preloadIndex)" || return 1 + test false = "$(git -C $num/src config gui.gcwarning)" || return 1 done ' @@ -290,7 +297,7 @@ test_expect_success 'scalar supports -c/-C' ' git init sub && scalar -C sub -c status.aheadBehind=bogus register && test -z "$(git -C sub config --local status.aheadBehind)" && - test true = "$(git -C sub config core.preloadIndex)" + test false = "$(git -C sub config gui.gcwarning)" ' test_expect_success '`scalar [...] <dir>` errors out when dir is missing' ' diff --git a/t/t9305-fast-import-signatures.sh b/t/t9305-fast-import-signatures.sh index c2b4271658..022dae02e4 100755 --- a/t/t9305-fast-import-signatures.sh +++ b/t/t9305-fast-import-signatures.sh @@ -79,7 +79,7 @@ test_expect_success GPG 'setup a commit with dual OpenPGP signatures on its SHA- echo B >explicit-sha256/B && git -C explicit-sha256 add B && test_tick && - git -C explicit-sha256 commit -S -m "signed" B && + git -C explicit-sha256 commit -S -m "signed commit" B && SHA256_B=$(git -C explicit-sha256 rev-parse dual-signed) && # Create the corresponding SHA-1 commit @@ -103,4 +103,71 @@ test_expect_success GPG 'strip both OpenPGP signatures with --signed-commits=war test_line_count = 2 out ' +test_expect_success GPG 'import commit with no signature with --signed-commits=strip-if-invalid' ' + git fast-export main >output && + git -C new fast-import --quiet --signed-commits=strip-if-invalid <output >log 2>&1 && + test_must_be_empty log +' + +test_expect_success GPG 'keep valid OpenPGP signature with --signed-commits=strip-if-invalid' ' + rm -rf new && + git init new && + + git fast-export --signed-commits=verbatim openpgp-signing >output && + git -C new fast-import --quiet --signed-commits=strip-if-invalid <output >log 2>&1 && + IMPORTED=$(git -C new rev-parse --verify refs/heads/openpgp-signing) && + test $OPENPGP_SIGNING = $IMPORTED && + git -C new cat-file commit "$IMPORTED" >actual && + test_grep -E "^gpgsig(-sha256)? " actual && + test_must_be_empty log +' + +test_expect_success GPG 'strip signature invalidated by message change with --signed-commits=strip-if-invalid' ' + rm -rf new && + git init new && + + git fast-export --signed-commits=verbatim openpgp-signing >output && + + # Change the commit message, which invalidates the signature. + # The commit message length should not change though, otherwise the + # corresponding `data <length>` command would have to be changed too. + sed "s/OpenPGP signed commit/OpenPGP forged commit/" output >modified && + + git -C new fast-import --quiet --signed-commits=strip-if-invalid <modified >log 2>&1 && + + IMPORTED=$(git -C new rev-parse --verify refs/heads/openpgp-signing) && + test $OPENPGP_SIGNING != $IMPORTED && + git -C new cat-file commit "$IMPORTED" >actual && + test_grep ! -E "^gpgsig" actual && + test_grep "stripping invalid signature" log +' + +test_expect_success GPGSM 'keep valid X.509 signature with --signed-commits=strip-if-invalid' ' + rm -rf new && + git init new && + + git fast-export --signed-commits=verbatim x509-signing >output && + git -C new fast-import --quiet --signed-commits=strip-if-invalid <output >log 2>&1 && + IMPORTED=$(git -C new rev-parse --verify refs/heads/x509-signing) && + test $X509_SIGNING = $IMPORTED && + git -C new cat-file commit "$IMPORTED" >actual && + test_grep -E "^gpgsig(-sha256)? " actual && + test_must_be_empty log +' + +test_expect_success GPGSSH 'keep valid SSH signature with --signed-commits=strip-if-invalid' ' + rm -rf new && + git init new && + + test_config -C new gpg.ssh.allowedSignersFile "${GPGSSH_ALLOWED_SIGNERS}" && + + git fast-export --signed-commits=verbatim ssh-signing >output && + git -C new fast-import --quiet --signed-commits=strip-if-invalid <output >log 2>&1 && + IMPORTED=$(git -C new rev-parse --verify refs/heads/ssh-signing) && + test $SSH_SIGNING = $IMPORTED && + git -C new cat-file commit "$IMPORTED" >actual && + test_grep -E "^gpgsig(-sha256)? " actual && + test_must_be_empty log +' + test_done diff --git a/t/t9700/test.pl b/t/t9700/test.pl index 58a9b328d5..570b0c5680 100755 --- a/t/t9700/test.pl +++ b/t/t9700/test.pl @@ -117,7 +117,12 @@ close TEMPFILE; unlink $tmpfile; # paths -is($r->repo_path, $abs_repo_dir . "/.git", "repo_path"); +my $abs_git_dir = $abs_repo_dir . "/.git"; +if ($^O eq 'msys' or $^O eq 'cygwin') { + $abs_git_dir = `cygpath -am "$abs_repo_dir/.git"`; + $abs_git_dir =~ s/\r?\n?$//; +} +is($r->repo_path, $abs_git_dir, "repo_path"); is($r->wc_path, $abs_repo_dir . "/", "wc_path"); is($r->wc_subdir, "", "wc_subdir initial"); $r->wc_chdir("directory1"); @@ -127,7 +132,7 @@ is($r->config("test.string"), "value", "config after wc_chdir"); # Object generation in sub directory chdir("directory2"); my $r2 = Git->repository(); -is($r2->repo_path, $abs_repo_dir . "/.git", "repo_path (2)"); +is($r2->repo_path, $abs_git_dir, "repo_path (2)"); is($r2->wc_path, $abs_repo_dir . "/", "wc_path (2)"); is($r2->wc_subdir, "directory2/", "wc_subdir initial (2)"); diff --git a/t/test-lib-functions.sh b/t/test-lib-functions.sh index 52d7759bf5..14e238d24d 100644 --- a/t/test-lib-functions.sh +++ b/t/test-lib-functions.sh @@ -1724,7 +1724,7 @@ test_detect_hash () { esac } -# Detect the hash algorithm in use. +# Detect the ref format in use. test_detect_ref_format () { echo "${GIT_TEST_DEFAULT_REF_FORMAT:-files}" } diff --git a/transport.c b/transport.c index c7f06a7382..6d0f02be5d 100644 --- a/transport.c +++ b/transport.c @@ -1316,65 +1316,66 @@ static void die_with_unpushed_submodules(struct string_list *needs_pushing) die(_("Aborting.")); } -static int run_pre_push_hook(struct transport *transport, - struct ref *remote_refs) -{ - int ret = 0, x; - struct ref *r; - struct child_process proc = CHILD_PROCESS_INIT; +struct feed_pre_push_hook_data { struct strbuf buf; - const char *hook_path = find_hook(the_repository, "pre-push"); + const struct ref *refs; +}; - if (!hook_path) - return 0; +static int pre_push_hook_feed_stdin(int hook_stdin_fd, void *pp_cb UNUSED, void *pp_task_cb) +{ + struct feed_pre_push_hook_data *data = pp_task_cb; + const struct ref *r = data->refs; + int ret = 0; - strvec_push(&proc.args, hook_path); - strvec_push(&proc.args, transport->remote->name); - strvec_push(&proc.args, transport->url); + if (!r) + return 1; /* no more refs */ - proc.in = -1; - proc.trace2_hook_name = "pre-push"; + data->refs = r->next; - if (start_command(&proc)) { - finish_command(&proc); - return -1; + switch (r->status) { + case REF_STATUS_REJECT_NONFASTFORWARD: + case REF_STATUS_REJECT_REMOTE_UPDATED: + case REF_STATUS_REJECT_STALE: + case REF_STATUS_UPTODATE: + return 0; /* skip refs which won't be pushed */ + default: + break; } - sigchain_push(SIGPIPE, SIG_IGN); + if (!r->peer_ref) + return 0; - strbuf_init(&buf, 256); + strbuf_reset(&data->buf); + strbuf_addf(&data->buf, "%s %s %s %s\n", + r->peer_ref->name, oid_to_hex(&r->new_oid), + r->name, oid_to_hex(&r->old_oid)); - for (r = remote_refs; r; r = r->next) { - if (!r->peer_ref) continue; - if (r->status == REF_STATUS_REJECT_NONFASTFORWARD) continue; - if (r->status == REF_STATUS_REJECT_STALE) continue; - if (r->status == REF_STATUS_REJECT_REMOTE_UPDATED) continue; - if (r->status == REF_STATUS_UPTODATE) continue; + ret = write_in_full(hook_stdin_fd, data->buf.buf, data->buf.len); + if (ret < 0 && errno != EPIPE) + return ret; /* We do not mind if a hook does not read all refs. */ - strbuf_reset(&buf); - strbuf_addf( &buf, "%s %s %s %s\n", - r->peer_ref->name, oid_to_hex(&r->new_oid), - r->name, oid_to_hex(&r->old_oid)); + return 0; +} - if (write_in_full(proc.in, buf.buf, buf.len) < 0) { - /* We do not mind if a hook does not read all refs. */ - if (errno != EPIPE) - ret = -1; - break; - } - } +static int run_pre_push_hook(struct transport *transport, + struct ref *remote_refs) +{ + struct run_hooks_opt opt = RUN_HOOKS_OPT_INIT; + struct feed_pre_push_hook_data data; + int ret = 0; + + strvec_push(&opt.args, transport->remote->name); + strvec_push(&opt.args, transport->url); - strbuf_release(&buf); + strbuf_init(&data.buf, 0); + data.refs = remote_refs; - x = close(proc.in); - if (!ret) - ret = x; + opt.feed_pipe = pre_push_hook_feed_stdin; + opt.feed_pipe_cb_data = &data; - sigchain_pop(SIGPIPE); + ret = run_hooks_opt(the_repository, "pre-push", &opt); - x = finish_command(&proc); - if (!ret) - ret = x; + strbuf_release(&data.buf); return ret; } @@ -429,7 +429,11 @@ int xmkstemp(char *filename_template) #undef TMP_MAX #define TMP_MAX 16384 -int git_mkstemps_mode(char *pattern, int suffix_len, int mode) +/* + * Returns -1 on error, 0 if it created a directory, or an open file + * descriptor to the created regular file. + */ +static int git_mkdstemps_mode(char *pattern, int suffix_len, int mode, bool dir) { static const char letters[] = "abcdefghijklmnopqrstuvwxyz" @@ -471,7 +475,10 @@ int git_mkstemps_mode(char *pattern, int suffix_len, int mode) v /= num_letters; } - fd = open(pattern, O_CREAT | O_EXCL | O_RDWR, mode); + if (dir) + fd = mkdir(pattern, mode); + else + fd = open(pattern, O_CREAT | O_EXCL | O_RDWR, mode); if (fd >= 0) return fd; /* @@ -486,6 +493,16 @@ int git_mkstemps_mode(char *pattern, int suffix_len, int mode) return -1; } +char *git_mkdtemp(char *pattern) +{ + return git_mkdstemps_mode(pattern, 0, 0700, true) ? NULL : pattern; +} + +int git_mkstemps_mode(char *pattern, int suffix_len, int mode) +{ + return git_mkdstemps_mode(pattern, suffix_len, mode, false); +} + int git_mkstemp_mode(char *pattern, int mode) { /* mkstemp is just mkstemps with no suffix */ @@ -37,6 +37,8 @@ int xsnprintf(char *dst, size_t max, const char *fmt, ...); int xgethostname(char *buf, size_t len); +char *git_mkdtemp(char *pattern); + /* set default permissions by passing mode arguments to open(2) */ int git_mkstemps_mode(char *pattern, int suffix_len, int mode); int git_mkstemp_mode(char *pattern, int mode); diff --git a/wt-status.c b/wt-status.c index e12adb26b9..95942399f8 100644 --- a/wt-status.c +++ b/wt-status.c @@ -612,6 +612,30 @@ static void wt_status_collect_updated_cb(struct diff_queue_struct *q, } } +void wt_status_collect_changes_trees(struct wt_status *s, + const struct object_id *old_treeish, + const struct object_id *new_treeish) +{ + struct diff_options opts = { 0 }; + + repo_diff_setup(s->repo, &opts); + opts.output_format = DIFF_FORMAT_CALLBACK; + opts.format_callback = wt_status_collect_updated_cb; + opts.format_callback_data = s; + opts.detect_rename = s->detect_rename >= 0 ? s->detect_rename : opts.detect_rename; + opts.rename_limit = s->rename_limit >= 0 ? s->rename_limit : opts.rename_limit; + opts.rename_score = s->rename_score >= 0 ? s->rename_score : opts.rename_score; + opts.flags.recursive = 1; + diff_setup_done(&opts); + + diff_tree_oid(old_treeish, new_treeish, "", &opts); + diffcore_std(&opts); + diff_flush(&opts); + wt_status_get_state(s->repo, &s->state, 0); + + diff_free(&opts); +} + static void wt_status_collect_changes_worktree(struct wt_status *s) { struct rev_info rev; diff --git a/wt-status.h b/wt-status.h index e40a27214a..e9fe32e98c 100644 --- a/wt-status.h +++ b/wt-status.h @@ -153,6 +153,15 @@ void wt_status_add_cut_line(struct wt_status *s); void wt_status_prepare(struct repository *r, struct wt_status *s); void wt_status_print(struct wt_status *s); void wt_status_collect(struct wt_status *s); + +/* + * Collect all changes between the two trees. Changes will be displayed as if + * they were staged into the index. + */ +void wt_status_collect_changes_trees(struct wt_status *s, + const struct object_id *old_treeish, + const struct object_id *new_treeish); + /* * Frees the buffers allocated by wt_status_collect. */ diff --git a/xdiff-interface.c b/xdiff-interface.c index 4971f722b3..1a35556380 100644 --- a/xdiff-interface.c +++ b/xdiff-interface.c @@ -300,7 +300,7 @@ void xdiff_clear_find_func(xdemitconf_t *xecfg) unsigned long xdiff_hash_string(const char *s, size_t len, long flags) { - return xdl_hash_record(&s, s + len, flags); + return xdl_hash_record((uint8_t const**)&s, (uint8_t const*)s + len, flags); } int xdiff_compare_lines(const char *l1, long s1, diff --git a/xdiff/xdiffi.c b/xdiff/xdiffi.c index 6f3998ee54..4376f943db 100644 --- a/xdiff/xdiffi.c +++ b/xdiff/xdiffi.c @@ -22,9 +22,9 @@ #include "xinclude.h" -static unsigned long get_hash(xdfile_t *xdf, long index) +static size_t get_hash(xdfile_t *xdf, long index) { - return xdf->recs[xdf->rindex[index]].ha; + return xdf->recs[xdf->reference_index[index]].minimal_perfect_hash; } #define XDL_MAX_COST_MIN 256 @@ -278,10 +278,10 @@ int xdl_recs_cmp(xdfile_t *xdf1, long off1, long lim1, */ if (off1 == lim1) { for (; off2 < lim2; off2++) - xdf2->changed[xdf2->rindex[off2]] = true; + xdf2->changed[xdf2->reference_index[off2]] = true; } else if (off2 == lim2) { for (; off1 < lim1; off1++) - xdf1->changed[xdf1->rindex[off1]] = true; + xdf1->changed[xdf1->reference_index[off1]] = true; } else { xdpsplit_t spl; spl.i1 = spl.i2 = 0; @@ -385,7 +385,7 @@ static xdchange_t *xdl_add_change(xdchange_t *xscr, long i1, long i2, long chg1, static int recs_match(xrecord_t *rec1, xrecord_t *rec2) { - return (rec1->ha == rec2->ha); + return rec1->minimal_perfect_hash == rec2->minimal_perfect_hash; } /* @@ -403,11 +403,10 @@ static int recs_match(xrecord_t *rec1, xrecord_t *rec2) */ static int get_indent(xrecord_t *rec) { - long i; int ret = 0; - for (i = 0; i < rec->size; i++) { - char c = rec->ptr[i]; + for (size_t i = 0; i < rec->size; i++) { + char c = (char) rec->ptr[i]; if (!XDL_ISSPACE(c)) return ret; @@ -484,7 +483,7 @@ static void measure_split(const xdfile_t *xdf, long split, { long i; - if (split >= xdf->nrec) { + if (split >= (long)xdf->nrec) { m->end_of_file = 1; m->indent = -1; } else { @@ -507,7 +506,7 @@ static void measure_split(const xdfile_t *xdf, long split, m->post_blank = 0; m->post_indent = -1; - for (i = split + 1; i < xdf->nrec; i++) { + for (i = split + 1; i < (long)xdf->nrec; i++) { m->post_indent = get_indent(&xdf->recs[i]); if (m->post_indent != -1) break; @@ -718,7 +717,7 @@ static void group_init(xdfile_t *xdf, struct xdlgroup *g) */ static inline int group_next(xdfile_t *xdf, struct xdlgroup *g) { - if (g->end == xdf->nrec) + if (g->end == (long)xdf->nrec) return -1; g->start = g->end + 1; @@ -751,7 +750,7 @@ static inline int group_previous(xdfile_t *xdf, struct xdlgroup *g) */ static int group_slide_down(xdfile_t *xdf, struct xdlgroup *g) { - if (g->end < xdf->nrec && + if (g->end < (long)xdf->nrec && recs_match(&xdf->recs[g->start], &xdf->recs[g->end])) { xdf->changed[g->start++] = false; xdf->changed[g->end++] = true; @@ -993,11 +992,11 @@ static void xdl_mark_ignorable_lines(xdchange_t *xscr, xdfenv_t *xe, long flags) rec = &xe->xdf1.recs[xch->i1]; for (i = 0; i < xch->chg1 && ignore; i++) - ignore = xdl_blankline(rec[i].ptr, rec[i].size, flags); + ignore = xdl_blankline((const char *)rec[i].ptr, (long)rec[i].size, flags); rec = &xe->xdf2.recs[xch->i2]; for (i = 0; i < xch->chg2 && ignore; i++) - ignore = xdl_blankline(rec[i].ptr, rec[i].size, flags); + ignore = xdl_blankline((const char *)rec[i].ptr, (long)rec[i].size, flags); xch->ignore = ignore; } @@ -1008,7 +1007,7 @@ static int record_matches_regex(xrecord_t *rec, xpparam_t const *xpp) { size_t i; for (i = 0; i < xpp->ignore_regex_nr; i++) - if (!regexec_buf(xpp->ignore_regex[i], rec->ptr, rec->size, 1, + if (!regexec_buf(xpp->ignore_regex[i], (const char *)rec->ptr, rec->size, 1, ®match, 0)) return 1; diff --git a/xdiff/xemit.c b/xdiff/xemit.c index b2f1f30cd3..04f7e9193b 100644 --- a/xdiff/xemit.c +++ b/xdiff/xemit.c @@ -27,7 +27,7 @@ static int xdl_emit_record(xdfile_t *xdf, long ri, char const *pre, xdemitcb_t * { xrecord_t *rec = &xdf->recs[ri]; - if (xdl_emit_diffrec(rec->ptr, rec->size, pre, strlen(pre), ecb) < 0) + if (xdl_emit_diffrec((char const *)rec->ptr, (long)rec->size, pre, strlen(pre), ecb) < 0) return -1; return 0; @@ -113,8 +113,8 @@ static long match_func_rec(xdfile_t *xdf, xdemitconf_t const *xecfg, long ri, xrecord_t *rec = &xdf->recs[ri]; if (!xecfg->find_func) - return def_ff(rec->ptr, rec->size, buf, sz); - return xecfg->find_func(rec->ptr, rec->size, buf, sz, xecfg->find_func_priv); + return def_ff((const char *)rec->ptr, (long)rec->size, buf, sz); + return xecfg->find_func((const char *)rec->ptr, (long)rec->size, buf, sz, xecfg->find_func_priv); } static int is_func_rec(xdfile_t *xdf, xdemitconf_t const *xecfg, long ri) @@ -137,7 +137,7 @@ static long get_func_line(xdfenv_t *xe, xdemitconf_t const *xecfg, buf = func_line ? func_line->buf : dummy; size = func_line ? sizeof(func_line->buf) : sizeof(dummy); - for (l = start; l != limit && 0 <= l && l < xe->xdf1.nrec; l += step) { + for (l = start; l != limit && 0 <= l && l < (long)xe->xdf1.nrec; l += step) { long len = match_func_rec(&xe->xdf1, xecfg, l, buf, size); if (len >= 0) { if (func_line) @@ -151,7 +151,7 @@ static long get_func_line(xdfenv_t *xe, xdemitconf_t const *xecfg, static int is_empty_rec(xdfile_t *xdf, long ri) { xrecord_t *rec = &xdf->recs[ri]; - long i = 0; + size_t i = 0; for (; i < rec->size && XDL_ISSPACE(rec->ptr[i]); i++); @@ -179,14 +179,14 @@ pre_context_calculation: long fs1, i1 = xch->i1; /* Appended chunk? */ - if (i1 >= xe->xdf1.nrec) { + if (i1 >= (long)xe->xdf1.nrec) { long i2 = xch->i2; /* * We don't need additional context if * a whole function was added. */ - while (i2 < xe->xdf2.nrec) { + while (i2 < (long)xe->xdf2.nrec) { if (is_func_rec(&xe->xdf2, xecfg, i2)) goto post_context_calculation; i2++; @@ -196,7 +196,7 @@ pre_context_calculation: * Otherwise get more context from the * pre-image. */ - i1 = xe->xdf1.nrec - 1; + i1 = (long)xe->xdf1.nrec - 1; } fs1 = get_func_line(xe, xecfg, NULL, i1, -1); @@ -228,8 +228,8 @@ pre_context_calculation: post_context_calculation: lctx = xecfg->ctxlen; - lctx = XDL_MIN(lctx, xe->xdf1.nrec - (xche->i1 + xche->chg1)); - lctx = XDL_MIN(lctx, xe->xdf2.nrec - (xche->i2 + xche->chg2)); + lctx = XDL_MIN(lctx, (long)xe->xdf1.nrec - (xche->i1 + xche->chg1)); + lctx = XDL_MIN(lctx, (long)xe->xdf2.nrec - (xche->i2 + xche->chg2)); e1 = xche->i1 + xche->chg1 + lctx; e2 = xche->i2 + xche->chg2 + lctx; @@ -237,13 +237,13 @@ pre_context_calculation: if (xecfg->flags & XDL_EMIT_FUNCCONTEXT) { long fe1 = get_func_line(xe, xecfg, NULL, xche->i1 + xche->chg1, - xe->xdf1.nrec); + (long)xe->xdf1.nrec); while (fe1 > 0 && is_empty_rec(&xe->xdf1, fe1 - 1)) fe1--; if (fe1 < 0) - fe1 = xe->xdf1.nrec; + fe1 = (long)xe->xdf1.nrec; if (fe1 > e1) { - e2 = XDL_MIN(e2 + (fe1 - e1), xe->xdf2.nrec); + e2 = XDL_MIN(e2 + (fe1 - e1), (long)xe->xdf2.nrec); e1 = fe1; } @@ -254,7 +254,7 @@ pre_context_calculation: */ if (xche->next) { long l = XDL_MIN(xche->next->i1, - xe->xdf1.nrec - 1); + (long)xe->xdf1.nrec - 1); if (l - xecfg->ctxlen <= e1 || get_func_line(xe, xecfg, NULL, l, e1) < 0) { xche = xche->next; diff --git a/xdiff/xhistogram.c b/xdiff/xhistogram.c index 6dc450b1fe..5ae1282c27 100644 --- a/xdiff/xhistogram.c +++ b/xdiff/xhistogram.c @@ -90,7 +90,7 @@ struct region { static int cmp_recs(xrecord_t *r1, xrecord_t *r2) { - return r1->ha == r2->ha; + return r1->minimal_perfect_hash == r2->minimal_perfect_hash; } @@ -98,7 +98,7 @@ static int cmp_recs(xrecord_t *r1, xrecord_t *r2) (cmp_recs(REC(i->env, s1, l1), REC(i->env, s2, l2))) #define TABLE_HASH(index, side, line) \ - XDL_HASHLONG((REC(index->env, side, line))->ha, index->table_bits) + XDL_HASHLONG((REC(index->env, side, line))->minimal_perfect_hash, index->table_bits) static int scanA(struct histindex *index, int line1, int count1) { diff --git a/xdiff/xmerge.c b/xdiff/xmerge.c index fd600cbb5d..29dad98c49 100644 --- a/xdiff/xmerge.c +++ b/xdiff/xmerge.c @@ -101,8 +101,8 @@ static int xdl_merge_cmp_lines(xdfenv_t *xe1, int i1, xdfenv_t *xe2, int i2, xrecord_t *rec2 = xe2->xdf2.recs + i2; for (i = 0; i < line_count; i++) { - int result = xdl_recmatch(rec1[i].ptr, rec1[i].size, - rec2[i].ptr, rec2[i].size, flags); + int result = xdl_recmatch((const char *)rec1[i].ptr, (long)rec1[i].size, + (const char *)rec2[i].ptr, (long)rec2[i].size, flags); if (!result) return -1; } @@ -119,11 +119,11 @@ static int xdl_recs_copy_0(int use_orig, xdfenv_t *xe, int i, int count, int nee if (count < 1) return 0; - for (i = 0; i < count; size += recs[i++].size) + for (i = 0; i < count; size += (int)recs[i++].size) if (dest) memcpy(dest + size, recs[i].ptr, recs[i].size); if (add_nl) { - i = recs[count - 1].size; + i = (int)recs[count - 1].size; if (i == 0 || recs[count - 1].ptr[i - 1] != '\n') { if (needs_cr) { if (dest) @@ -156,9 +156,9 @@ static int xdl_orig_copy(xdfenv_t *xe, int i, int count, int needs_cr, int add_n */ static int is_eol_crlf(xdfile_t *file, int i) { - long size; + size_t size; - if (i < file->nrec - 1) + if (i < (long)file->nrec - 1) /* All lines before the last *must* end in LF */ return (size = file->recs[i].size) > 1 && file->recs[i].ptr[size - 2] == '\r'; @@ -317,15 +317,15 @@ static int xdl_fill_merge_buffer(xdfenv_t *xe1, const char *name1, continue; i = m->i1 + m->chg1; } - size += xdl_recs_copy(xe1, i, xe1->xdf2.nrec - i, 0, 0, + size += xdl_recs_copy(xe1, i, (int)xe1->xdf2.nrec - i, 0, 0, dest ? dest + size : NULL); return size; } static int recmatch(xrecord_t *rec1, xrecord_t *rec2, unsigned long flags) { - return xdl_recmatch(rec1->ptr, rec1->size, - rec2->ptr, rec2->size, flags); + return xdl_recmatch((const char *)rec1->ptr, (long)rec1->size, + (const char *)rec2->ptr, (long)rec2->size, flags); } /* @@ -382,10 +382,10 @@ static int xdl_refine_conflicts(xdfenv_t *xe1, xdfenv_t *xe2, xdmerge_t *m, * we have a very simple mmfile structure. */ t1.ptr = (char *)xe1->xdf2.recs[m->i1].ptr; - t1.size = xe1->xdf2.recs[m->i1 + m->chg1 - 1].ptr + t1.size = (char *)xe1->xdf2.recs[m->i1 + m->chg1 - 1].ptr + xe1->xdf2.recs[m->i1 + m->chg1 - 1].size - t1.ptr; t2.ptr = (char *)xe2->xdf2.recs[m->i2].ptr; - t2.size = xe2->xdf2.recs[m->i2 + m->chg2 - 1].ptr + t2.size = (char *)xe2->xdf2.recs[m->i2 + m->chg2 - 1].ptr + xe2->xdf2.recs[m->i2 + m->chg2 - 1].size - t2.ptr; if (xdl_do_diff(&t1, &t2, xpp, &xe) < 0) return -1; @@ -440,8 +440,8 @@ static int line_contains_alnum(const char *ptr, long size) static int lines_contain_alnum(xdfenv_t *xe, int i, int chg) { for (; chg; chg--, i++) - if (line_contains_alnum(xe->xdf2.recs[i].ptr, - xe->xdf2.recs[i].size)) + if (line_contains_alnum((const char *)xe->xdf2.recs[i].ptr, + (long)xe->xdf2.recs[i].size)) return 1; return 0; } @@ -622,7 +622,7 @@ static int xdl_do_merge(xdfenv_t *xe1, xdchange_t *xscr1, changes = c; i0 = xscr1->i1; i1 = xscr1->i2; - i2 = xscr1->i1 + xe2->xdf2.nrec - xe2->xdf1.nrec; + i2 = xscr1->i1 + (long)xe2->xdf2.nrec - (long)xe2->xdf1.nrec; chg0 = xscr1->chg1; chg1 = xscr1->chg2; chg2 = xscr1->chg1; @@ -637,7 +637,7 @@ static int xdl_do_merge(xdfenv_t *xe1, xdchange_t *xscr1, if (!changes) changes = c; i0 = xscr2->i1; - i1 = xscr2->i1 + xe1->xdf2.nrec - xe1->xdf1.nrec; + i1 = xscr2->i1 + (long)xe1->xdf2.nrec - (long)xe1->xdf1.nrec; i2 = xscr2->i2; chg0 = xscr2->chg1; chg1 = xscr2->chg1; diff --git a/xdiff/xpatience.c b/xdiff/xpatience.c index 669b653580..9580d18032 100644 --- a/xdiff/xpatience.c +++ b/xdiff/xpatience.c @@ -48,7 +48,7 @@ struct hashmap { int nr, alloc; struct entry { - unsigned long hash; + size_t minimal_perfect_hash; /* * 0 = unused entry, 1 = first line, 2 = second, etc. * line2 is NON_UNIQUE if the line is not unique @@ -101,10 +101,10 @@ static void insert_record(xpparam_t const *xpp, int line, struct hashmap *map, * So we multiply ha by 2 in the hope that the hashing was * "unique enough". */ - int index = (int)((record->ha << 1) % map->alloc); + int index = (int)((record->minimal_perfect_hash << 1) % map->alloc); while (map->entries[index].line1) { - if (map->entries[index].hash != record->ha) { + if (map->entries[index].minimal_perfect_hash != record->minimal_perfect_hash) { if (++index >= map->alloc) index = 0; continue; @@ -120,8 +120,8 @@ static void insert_record(xpparam_t const *xpp, int line, struct hashmap *map, if (pass == 2) return; map->entries[index].line1 = line; - map->entries[index].hash = record->ha; - map->entries[index].anchor = is_anchor(xpp, map->env->xdf1.recs[line - 1].ptr); + map->entries[index].minimal_perfect_hash = record->minimal_perfect_hash; + map->entries[index].anchor = is_anchor(xpp, (const char *)map->env->xdf1.recs[line - 1].ptr); if (!map->first) map->first = map->entries + index; if (map->last) { @@ -211,7 +211,10 @@ static int find_longest_common_sequence(struct hashmap *map, struct entry **res) for (entry = map->first; entry; entry = entry->next) { if (!entry->line2 || entry->line2 == NON_UNIQUE) continue; - i = binary_search(sequence, longest, entry); + if (longest == 0 || entry->line2 > sequence[longest - 1]->line2) + i = longest - 1; + else + i = binary_search(sequence, longest, entry); entry->previous = i < 0 ? NULL : sequence[i]; ++i; if (i <= anchor_i) @@ -248,7 +251,7 @@ static int match(struct hashmap *map, int line1, int line2) { xrecord_t *record1 = &map->env->xdf1.recs[line1 - 1]; xrecord_t *record2 = &map->env->xdf2.recs[line2 - 1]; - return record1->ha == record2->ha; + return record1->minimal_perfect_hash == record2->minimal_perfect_hash; } static int patience_diff(xpparam_t const *xpp, xdfenv_t *env, @@ -370,5 +373,5 @@ static int patience_diff(xpparam_t const *xpp, xdfenv_t *env, int xdl_do_patience_diff(xpparam_t const *xpp, xdfenv_t *env) { - return patience_diff(xpp, env, 1, env->xdf1.nrec, 1, env->xdf2.nrec); + return patience_diff(xpp, env, 1, (int)env->xdf1.nrec, 1, (int)env->xdf2.nrec); } diff --git a/xdiff/xprepare.c b/xdiff/xprepare.c index 192334f1b7..34c82e4f8e 100644 --- a/xdiff/xprepare.c +++ b/xdiff/xprepare.c @@ -93,14 +93,14 @@ static void xdl_free_classifier(xdlclassifier_t *cf) { static int xdl_classify_record(unsigned int pass, xdlclassifier_t *cf, xrecord_t *rec) { - long hi; + size_t hi; xdlclass_t *rcrec; - hi = (long) XDL_HASHLONG(rec->ha, cf->hbits); + hi = XDL_HASHLONG(rec->line_hash, cf->hbits); for (rcrec = cf->rchash[hi]; rcrec; rcrec = rcrec->next) - if (rcrec->rec.ha == rec->ha && - xdl_recmatch(rcrec->rec.ptr, rcrec->rec.size, - rec->ptr, rec->size, cf->flags)) + if (rcrec->rec.line_hash == rec->line_hash && + xdl_recmatch((const char *)rcrec->rec.ptr, (long)rcrec->rec.size, + (const char *)rec->ptr, (long)rec->size, cf->flags)) break; if (!rcrec) { @@ -120,7 +120,7 @@ static int xdl_classify_record(unsigned int pass, xdlclassifier_t *cf, xrecord_t (pass == 1) ? rcrec->len1++ : rcrec->len2++; - rec->ha = (unsigned long) rcrec->idx; + rec->minimal_perfect_hash = (size_t)rcrec->idx; return 0; } @@ -128,7 +128,7 @@ static int xdl_classify_record(unsigned int pass, xdlclassifier_t *cf, xrecord_t static void xdl_free_ctx(xdfile_t *xdf) { - xdl_free(xdf->rindex); + xdl_free(xdf->reference_index); xdl_free(xdf->changed - 1); xdl_free(xdf->recs); } @@ -137,11 +137,11 @@ static void xdl_free_ctx(xdfile_t *xdf) static int xdl_prepare_ctx(unsigned int pass, mmfile_t *mf, long narec, xpparam_t const *xpp, xdlclassifier_t *cf, xdfile_t *xdf) { long bsize; - unsigned long hav; - char const *blk, *cur, *top, *prev; + uint64_t hav; + uint8_t const *blk, *cur, *top, *prev; xrecord_t *crec; - xdf->rindex = NULL; + xdf->reference_index = NULL; xdf->changed = NULL; xdf->recs = NULL; @@ -153,12 +153,12 @@ static int xdl_prepare_ctx(unsigned int pass, mmfile_t *mf, long narec, xpparam_ for (top = blk + bsize; cur < top; ) { prev = cur; hav = xdl_hash_record(&cur, top, xpp->flags); - if (XDL_ALLOC_GROW(xdf->recs, xdf->nrec + 1, narec)) + if (XDL_ALLOC_GROW(xdf->recs, (long)xdf->nrec + 1, narec)) goto abort; crec = &xdf->recs[xdf->nrec++]; crec->ptr = prev; - crec->size = (long) (cur - prev); - crec->ha = hav; + crec->size = cur - prev; + crec->line_hash = hav; if (xdl_classify_record(pass, cf, crec) < 0) goto abort; } @@ -169,7 +169,7 @@ static int xdl_prepare_ctx(unsigned int pass, mmfile_t *mf, long narec, xpparam_ if ((XDF_DIFF_ALG(xpp->flags) != XDF_PATIENCE_DIFF) && (XDF_DIFF_ALG(xpp->flags) != XDF_HISTOGRAM_DIFF)) { - if (!XDL_ALLOC_ARRAY(xdf->rindex, xdf->nrec + 1)) + if (!XDL_ALLOC_ARRAY(xdf->reference_index, xdf->nrec + 1)) goto abort; } @@ -264,7 +264,7 @@ static bool xdl_clean_mmatch(uint8_t const *action, long i, long s, long e) { * might be potentially discarded if they appear in a run of discardable. */ static int xdl_cleanup_records(xdlclassifier_t *cf, xdfile_t *xdf1, xdfile_t *xdf2) { - long i, nm, nreff, mlim; + long i, nm, mlim; xrecord_t *recs; xdlclass_t *rcrec; uint8_t *action1 = NULL, *action2 = NULL; @@ -287,18 +287,18 @@ static int xdl_cleanup_records(xdlclassifier_t *cf, xdfile_t *xdf1, xdfile_t *xd /* * Initialize temporary arrays with DISCARD, KEEP, or INVESTIGATE. */ - if ((mlim = xdl_bogosqrt(xdf1->nrec)) > XDL_MAX_EQLIMIT) + if ((mlim = xdl_bogosqrt((long)xdf1->nrec)) > XDL_MAX_EQLIMIT) mlim = XDL_MAX_EQLIMIT; for (i = xdf1->dstart, recs = &xdf1->recs[xdf1->dstart]; i <= xdf1->dend; i++, recs++) { - rcrec = cf->rcrecs[recs->ha]; + rcrec = cf->rcrecs[recs->minimal_perfect_hash]; nm = rcrec ? rcrec->len2 : 0; action1[i] = (nm == 0) ? DISCARD: (nm >= mlim && !need_min) ? INVESTIGATE: KEEP; } - if ((mlim = xdl_bogosqrt(xdf2->nrec)) > XDL_MAX_EQLIMIT) + if ((mlim = xdl_bogosqrt((long)xdf2->nrec)) > XDL_MAX_EQLIMIT) mlim = XDL_MAX_EQLIMIT; for (i = xdf2->dstart, recs = &xdf2->recs[xdf2->dstart]; i <= xdf2->dend; i++, recs++) { - rcrec = cf->rcrecs[recs->ha]; + rcrec = cf->rcrecs[recs->minimal_perfect_hash]; nm = rcrec ? rcrec->len1 : 0; action2[i] = (nm == 0) ? DISCARD: (nm >= mlim && !need_min) ? INVESTIGATE: KEEP; } @@ -307,29 +307,29 @@ static int xdl_cleanup_records(xdlclassifier_t *cf, xdfile_t *xdf1, xdfile_t *xd * Use temporary arrays to decide if changed[i] should remain * false, or become true. */ - for (nreff = 0, i = xdf1->dstart, recs = &xdf1->recs[xdf1->dstart]; + xdf1->nreff = 0; + for (i = xdf1->dstart, recs = &xdf1->recs[xdf1->dstart]; i <= xdf1->dend; i++, recs++) { if (action1[i] == KEEP || (action1[i] == INVESTIGATE && !xdl_clean_mmatch(action1, i, xdf1->dstart, xdf1->dend))) { - xdf1->rindex[nreff++] = i; + xdf1->reference_index[xdf1->nreff++] = i; /* changed[i] remains false, i.e. keep */ } else xdf1->changed[i] = true; /* i.e. discard */ } - xdf1->nreff = nreff; - for (nreff = 0, i = xdf2->dstart, recs = &xdf2->recs[xdf2->dstart]; + xdf2->nreff = 0; + for (i = xdf2->dstart, recs = &xdf2->recs[xdf2->dstart]; i <= xdf2->dend; i++, recs++) { if (action2[i] == KEEP || (action2[i] == INVESTIGATE && !xdl_clean_mmatch(action2, i, xdf2->dstart, xdf2->dend))) { - xdf2->rindex[nreff++] = i; + xdf2->reference_index[xdf2->nreff++] = i; /* changed[i] remains false, i.e. keep */ } else xdf2->changed[i] = true; /* i.e. discard */ } - xdf2->nreff = nreff; cleanup: xdl_free(action1); @@ -348,9 +348,9 @@ static int xdl_trim_ends(xdfile_t *xdf1, xdfile_t *xdf2) { recs1 = xdf1->recs; recs2 = xdf2->recs; - for (i = 0, lim = XDL_MIN(xdf1->nrec, xdf2->nrec); i < lim; + for (i = 0, lim = (long)XDL_MIN(xdf1->nrec, xdf2->nrec); i < lim; i++, recs1++, recs2++) - if (recs1->ha != recs2->ha) + if (recs1->minimal_perfect_hash != recs2->minimal_perfect_hash) break; xdf1->dstart = xdf2->dstart = i; @@ -358,11 +358,11 @@ static int xdl_trim_ends(xdfile_t *xdf1, xdfile_t *xdf2) { recs1 = xdf1->recs + xdf1->nrec - 1; recs2 = xdf2->recs + xdf2->nrec - 1; for (lim -= i, i = 0; i < lim; i++, recs1--, recs2--) - if (recs1->ha != recs2->ha) + if (recs1->minimal_perfect_hash != recs2->minimal_perfect_hash) break; - xdf1->dend = xdf1->nrec - i - 1; - xdf2->dend = xdf2->nrec - i - 1; + xdf1->dend = (long)xdf1->nrec - i - 1; + xdf2->dend = (long)xdf2->nrec - i - 1; return 0; } diff --git a/xdiff/xtypes.h b/xdiff/xtypes.h index f145abba3e..979586f20a 100644 --- a/xdiff/xtypes.h +++ b/xdiff/xtypes.h @@ -39,18 +39,19 @@ typedef struct s_chastore { } chastore_t; typedef struct s_xrecord { - char const *ptr; - long size; - unsigned long ha; + uint8_t const *ptr; + size_t size; + uint64_t line_hash; + size_t minimal_perfect_hash; } xrecord_t; typedef struct s_xdfile { xrecord_t *recs; - long nrec; - long dstart, dend; + size_t nrec; + ptrdiff_t dstart, dend; bool *changed; - long *rindex; - long nreff; + size_t *reference_index; + size_t nreff; } xdfile_t; typedef struct s_xdfenv { diff --git a/xdiff/xutils.c b/xdiff/xutils.c index 447e66c719..77ee1ad9c8 100644 --- a/xdiff/xutils.c +++ b/xdiff/xutils.c @@ -249,11 +249,11 @@ int xdl_recmatch(const char *l1, long s1, const char *l2, long s2, long flags) return 1; } -unsigned long xdl_hash_record_with_whitespace(char const **data, - char const *top, long flags) { - unsigned long ha = 5381; - char const *ptr = *data; - int cr_at_eol_only = (flags & XDF_WHITESPACE_FLAGS) == XDF_IGNORE_CR_AT_EOL; +uint64_t xdl_hash_record_with_whitespace(uint8_t const **data, + uint8_t const *top, uint64_t flags) { + uint64_t ha = 5381; + uint8_t const *ptr = *data; + bool cr_at_eol_only = (flags & XDF_WHITESPACE_FLAGS) == XDF_IGNORE_CR_AT_EOL; for (; ptr < top && *ptr != '\n'; ptr++) { if (cr_at_eol_only) { @@ -263,8 +263,8 @@ unsigned long xdl_hash_record_with_whitespace(char const **data, continue; } else if (XDL_ISSPACE(*ptr)) { - const char *ptr2 = ptr; - int at_eol; + const uint8_t *ptr2 = ptr; + bool at_eol; while (ptr + 1 < top && XDL_ISSPACE(ptr[1]) && ptr[1] != '\n') ptr++; @@ -274,20 +274,20 @@ unsigned long xdl_hash_record_with_whitespace(char const **data, else if (flags & XDF_IGNORE_WHITESPACE_CHANGE && !at_eol) { ha += (ha << 5); - ha ^= (unsigned long) ' '; + ha ^= (uint64_t) ' '; } else if (flags & XDF_IGNORE_WHITESPACE_AT_EOL && !at_eol) { while (ptr2 != ptr + 1) { ha += (ha << 5); - ha ^= (unsigned long) *ptr2; + ha ^= (uint64_t) *ptr2; ptr2++; } } continue; } ha += (ha << 5); - ha ^= (unsigned long) *ptr; + ha ^= (uint64_t) *ptr; } *data = ptr < top ? ptr + 1: ptr; @@ -304,9 +304,9 @@ unsigned long xdl_hash_record_with_whitespace(char const **data, #define REASSOC_FENCE(x, y) #endif -unsigned long xdl_hash_record_verbatim(char const **data, char const *top) { - unsigned long ha = 5381, c0, c1; - char const *ptr = *data; +uint64_t xdl_hash_record_verbatim(uint8_t const **data, uint8_t const *top) { + uint64_t ha = 5381, c0, c1; + uint8_t const *ptr = *data; #if 0 /* * The baseline form of the optimized loop below. This is the djb2 @@ -314,7 +314,7 @@ unsigned long xdl_hash_record_verbatim(char const **data, char const *top) { */ for (; ptr < top && *ptr != '\n'; ptr++) { ha += (ha << 5); - ha += (unsigned long) *ptr; + ha += (uint64_t) *ptr; } *data = ptr < top ? ptr + 1: ptr; #else @@ -465,10 +465,10 @@ int xdl_fall_back_diff(xdfenv_t *diff_env, xpparam_t const *xpp, xdfenv_t env; subfile1.ptr = (char *)diff_env->xdf1.recs[line1 - 1].ptr; - subfile1.size = diff_env->xdf1.recs[line1 + count1 - 2].ptr + + subfile1.size = (char *)diff_env->xdf1.recs[line1 + count1 - 2].ptr + diff_env->xdf1.recs[line1 + count1 - 2].size - subfile1.ptr; subfile2.ptr = (char *)diff_env->xdf2.recs[line2 - 1].ptr; - subfile2.size = diff_env->xdf2.recs[line2 + count2 - 2].ptr + + subfile2.size = (char *)diff_env->xdf2.recs[line2 + count2 - 2].ptr + diff_env->xdf2.recs[line2 + count2 - 2].size - subfile2.ptr; if (xdl_do_diff(&subfile1, &subfile2, xpp, &env) < 0) return -1; diff --git a/xdiff/xutils.h b/xdiff/xutils.h index 13f6831047..615b4a9d35 100644 --- a/xdiff/xutils.h +++ b/xdiff/xutils.h @@ -34,9 +34,9 @@ void *xdl_cha_alloc(chastore_t *cha); long xdl_guess_lines(mmfile_t *mf, long sample); int xdl_blankline(const char *line, long size, long flags); int xdl_recmatch(const char *l1, long s1, const char *l2, long s2, long flags); -unsigned long xdl_hash_record_verbatim(char const **data, char const *top); -unsigned long xdl_hash_record_with_whitespace(char const **data, char const *top, long flags); -static inline unsigned long xdl_hash_record(char const **data, char const *top, long flags) +uint64_t xdl_hash_record_verbatim(uint8_t const **data, uint8_t const *top); +uint64_t xdl_hash_record_with_whitespace(uint8_t const **data, uint8_t const *top, uint64_t flags); +static inline uint64_t xdl_hash_record(uint8_t const **data, uint8_t const *top, uint64_t flags) { if (flags & XDF_WHITESPACE_FLAGS) return xdl_hash_record_with_whitespace(data, top, flags); |