From 66833f0e70c473ca6c4e6a79d34e879d8b40ba9d Mon Sep 17 00:00:00 2001 From: Ævar Arnfjörð Bjarmason Date: Thu, 9 Sep 2021 19:24:37 -0400 Subject: pack-write: refactor renaming in finish_tmp_packfile() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refactor the renaming in finish_tmp_packfile() into a helper function. The callers are now expected to pass a "name_buffer" ending in "pack-OID." instead of the previous "pack-", we then append "pack", "idx" or "rev" to it. By doing the strbuf_setlen() in rename_tmp_packfile() we reuse the buffer and avoid the repeated allocations we'd get if that function had its own temporary "struct strbuf". This approach of reusing the buffer does make the last user in pack-object.c's write_pack_file() slightly awkward, since we needlessly do a strbuf_setlen() before calling strbuf_release() for consistency. In subsequent changes we'll move that bitmap writing code around, so let's not skip the strbuf_setlen() now. The previous strbuf_reset() idiom originated with 5889271114a (finish_tmp_packfile():use strbuf for pathname construction, 2014-03-03), which in turn was a minimal adjustment of pre-strbuf code added in 0e990530ae (finish_tmp_packfile(): a helper function, 2011-10-28). Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Taylor Blau Signed-off-by: Junio C Hamano --- builtin/pack-objects.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'builtin/pack-objects.c') diff --git a/builtin/pack-objects.c b/builtin/pack-objects.c index de00adbb9e..d42de63fda 100644 --- a/builtin/pack-objects.c +++ b/builtin/pack-objects.c @@ -1237,7 +1237,8 @@ static void write_pack_file(void) warning_errno(_("failed utime() on %s"), pack_tmp_name); } - strbuf_addf(&tmpname, "%s-", base_name); + strbuf_addf(&tmpname, "%s-%s.", base_name, + hash_to_hex(hash)); if (write_bitmap_index) { bitmap_writer_set_checksum(hash); @@ -1250,8 +1251,9 @@ static void write_pack_file(void) &pack_idx_opts, hash); if (write_bitmap_index) { - strbuf_addf(&tmpname, "%s.bitmap", hash_to_hex(hash)); + size_t tmpname_len = tmpname.len; + strbuf_addstr(&tmpname, "bitmap"); stop_progress(&progress_state); bitmap_writer_show_progress(progress); @@ -1260,6 +1262,7 @@ static void write_pack_file(void) bitmap_writer_finish(written_list, nr_written, tmpname.buf, write_bitmap_options); write_bitmap_index = 0; + strbuf_setlen(&tmpname, tmpname_len); } strbuf_release(&tmpname); -- cgit v1.3 From 2ec02dd5a8261bc837b961ef36788081ded5c2bc Mon Sep 17 00:00:00 2001 From: Ævar Arnfjörð Bjarmason Date: Thu, 9 Sep 2021 19:24:56 -0400 Subject: pack-write: split up finish_tmp_packfile() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Split up the finish_tmp_packfile() function and use the split-up version in pack-objects.c in preparation for moving the step of renaming the *.idx file later as part of a function change. Since the only other caller of finish_tmp_packfile() was in bulk-checkin.c, and it won't be needing a change to its *.idx renaming, provide a thin wrapper for the old function as a static function in that file. If other callers end up needing the simpler version it could be moved back to "pack-write.c" and "pack.h". Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Taylor Blau Signed-off-by: Junio C Hamano --- builtin/pack-objects.c | 7 +++++-- bulk-checkin.c | 16 ++++++++++++++++ pack-write.c | 22 +++++++++++++--------- pack.h | 7 +++++-- 4 files changed, 39 insertions(+), 13 deletions(-) (limited to 'builtin/pack-objects.c') diff --git a/builtin/pack-objects.c b/builtin/pack-objects.c index d42de63fda..0a46584922 100644 --- a/builtin/pack-objects.c +++ b/builtin/pack-objects.c @@ -1217,6 +1217,7 @@ static void write_pack_file(void) if (!pack_to_stdout) { struct stat st; struct strbuf tmpname = STRBUF_INIT; + char *idx_tmp_name = NULL; /* * Packs are runtime accessed in their mtime @@ -1246,9 +1247,10 @@ static void write_pack_file(void) &to_pack, written_list, nr_written); } - finish_tmp_packfile(&tmpname, pack_tmp_name, + stage_tmp_packfiles(&tmpname, pack_tmp_name, written_list, nr_written, - &pack_idx_opts, hash); + &pack_idx_opts, hash, &idx_tmp_name); + rename_tmp_packfile_idx(&tmpname, &idx_tmp_name); if (write_bitmap_index) { size_t tmpname_len = tmpname.len; @@ -1265,6 +1267,7 @@ static void write_pack_file(void) strbuf_setlen(&tmpname, tmpname_len); } + free(idx_tmp_name); strbuf_release(&tmpname); free(pack_tmp_name); puts(hash_to_hex(hash)); diff --git a/bulk-checkin.c b/bulk-checkin.c index c19d471f0b..8785b2ac80 100644 --- a/bulk-checkin.c +++ b/bulk-checkin.c @@ -23,6 +23,22 @@ static struct bulk_checkin_state { uint32_t nr_written; } state; +static void finish_tmp_packfile(struct strbuf *basename, + const char *pack_tmp_name, + struct pack_idx_entry **written_list, + uint32_t nr_written, + struct pack_idx_option *pack_idx_opts, + unsigned char hash[]) +{ + char *idx_tmp_name = NULL; + + stage_tmp_packfiles(basename, pack_tmp_name, written_list, nr_written, + pack_idx_opts, hash, &idx_tmp_name); + rename_tmp_packfile_idx(basename, &idx_tmp_name); + + free(idx_tmp_name); +} + static void finish_bulk_checkin(struct bulk_checkin_state *state) { unsigned char hash[GIT_MAX_RAWSZ]; diff --git a/pack-write.c b/pack-write.c index 51157916f5..32ebf0cdf7 100644 --- a/pack-write.c +++ b/pack-write.c @@ -474,21 +474,28 @@ static void rename_tmp_packfile(struct strbuf *name_prefix, const char *source, strbuf_setlen(name_prefix, name_prefix_len); } -void finish_tmp_packfile(struct strbuf *name_buffer, +void rename_tmp_packfile_idx(struct strbuf *name_buffer, + char **idx_tmp_name) +{ + rename_tmp_packfile(name_buffer, *idx_tmp_name, "idx"); +} + +void stage_tmp_packfiles(struct strbuf *name_buffer, const char *pack_tmp_name, struct pack_idx_entry **written_list, uint32_t nr_written, struct pack_idx_option *pack_idx_opts, - unsigned char hash[]) + unsigned char hash[], + char **idx_tmp_name) { - const char *idx_tmp_name, *rev_tmp_name = NULL; + const char *rev_tmp_name = NULL; if (adjust_shared_perm(pack_tmp_name)) die_errno("unable to make temporary pack file readable"); - idx_tmp_name = write_idx_file(NULL, written_list, nr_written, - pack_idx_opts, hash); - if (adjust_shared_perm(idx_tmp_name)) + *idx_tmp_name = (char *)write_idx_file(NULL, written_list, nr_written, + pack_idx_opts, hash); + if (adjust_shared_perm(*idx_tmp_name)) die_errno("unable to make temporary index file readable"); rev_tmp_name = write_rev_file(NULL, written_list, nr_written, hash, @@ -497,9 +504,6 @@ void finish_tmp_packfile(struct strbuf *name_buffer, rename_tmp_packfile(name_buffer, pack_tmp_name, "pack"); if (rev_tmp_name) rename_tmp_packfile(name_buffer, rev_tmp_name, "rev"); - rename_tmp_packfile(name_buffer, idx_tmp_name, "idx"); - - free((void *)idx_tmp_name); } void write_promisor_file(const char *promisor_name, struct ref **sought, int nr_sought) diff --git a/pack.h b/pack.h index 1c17254c0a..b22bfc4a18 100644 --- a/pack.h +++ b/pack.h @@ -110,11 +110,14 @@ int encode_in_pack_object_header(unsigned char *hdr, int hdr_len, int read_pack_header(int fd, struct pack_header *); struct hashfile *create_tmp_packfile(char **pack_tmp_name); -void finish_tmp_packfile(struct strbuf *name_buffer, +void stage_tmp_packfiles(struct strbuf *name_buffer, const char *pack_tmp_name, struct pack_idx_entry **written_list, uint32_t nr_written, struct pack_idx_option *pack_idx_opts, - unsigned char sha1[]); + unsigned char hash[], + char **idx_tmp_name); +void rename_tmp_packfile_idx(struct strbuf *basename, + char **idx_tmp_name); #endif -- cgit v1.3 From 4bc1fd6e3941be74027594efad3d2358a93702df Mon Sep 17 00:00:00 2001 From: Ævar Arnfjörð Bjarmason Date: Thu, 9 Sep 2021 19:25:00 -0400 Subject: pack-objects: rename .idx files into place after .bitmap files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In preceding commits the race of renaming .idx files in place before .rev files and other auxiliary files was fixed in pack-write.c's finish_tmp_packfile(), builtin/repack.c's "struct exts", and builtin/index-pack.c's final(). As noted in the change to pack-write.c we left in place the issue of writing *.bitmap files after the *.idx, let's fix that issue. See 7cc8f971085 (pack-objects: implement bitmap writing, 2013-12-21) for commentary at the time when *.bitmap was implemented about how those files are written out, nothing in that commit contradicts what's being done here. Note that this commit and preceding ones only close any race condition with *.idx files being written before their auxiliary files if we're optimistic about our lack of fsync()-ing in this are not tripping us over. See the thread at [1] for a rabbit hole of various discussions about filesystem races in the face of doing and not doing fsync() (and if doing fsync(), not doing it properly). We may want to fsync the containing directory once after renaming the *.idx file into place, but that is outside the scope of this series. 1. https://lore.kernel.org/git/8735qgkvv1.fsf@evledraar.gmail.com/ Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Taylor Blau Signed-off-by: Junio C Hamano --- builtin/pack-objects.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'builtin/pack-objects.c') diff --git a/builtin/pack-objects.c b/builtin/pack-objects.c index 0a46584922..7a3632828c 100644 --- a/builtin/pack-objects.c +++ b/builtin/pack-objects.c @@ -1250,7 +1250,6 @@ static void write_pack_file(void) stage_tmp_packfiles(&tmpname, pack_tmp_name, written_list, nr_written, &pack_idx_opts, hash, &idx_tmp_name); - rename_tmp_packfile_idx(&tmpname, &idx_tmp_name); if (write_bitmap_index) { size_t tmpname_len = tmpname.len; @@ -1267,6 +1266,8 @@ static void write_pack_file(void) strbuf_setlen(&tmpname, tmpname_len); } + rename_tmp_packfile_idx(&tmpname, &idx_tmp_name); + free(idx_tmp_name); strbuf_release(&tmpname); free(pack_tmp_name); -- cgit v1.3