From 0f4d6cada83dc6bd6b6b24dc0d2b3e6460c645cb Mon Sep 17 00:00:00 2001 From: "brian m. carlson" Date: Tue, 19 Feb 2019 00:04:54 +0000 Subject: pack-bitmap: make bitmap header handling hash agnostic Increase the checksum field in struct bitmap_disk_header to be GIT_MAX_RAWSZ bytes in length and ensure that we hash the proper number of bytes out when computing the bitmap checksum. Signed-off-by: brian m. carlson Signed-off-by: Junio C Hamano --- pack-bitmap.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'pack-bitmap.c') diff --git a/pack-bitmap.c b/pack-bitmap.c index 4695aaf6b4..b53f37243c 100644 --- a/pack-bitmap.c +++ b/pack-bitmap.c @@ -163,7 +163,7 @@ static int load_bitmap_header(struct bitmap_index *index) } index->entry_count = ntohl(header->entry_count); - index->map_pos += sizeof(*header); + index->map_pos += sizeof(*header) - GIT_MAX_RAWSZ + the_hash_algo->rawsz; return 0; } -- cgit v1.3 From 53636539d37007099f14249a5dae4c05dc72ace4 Mon Sep 17 00:00:00 2001 From: "brian m. carlson" Date: Tue, 19 Feb 2019 00:04:55 +0000 Subject: pack-bitmap: convert struct stored_bitmap to object_id Convert struct stored_bitmap to use struct object_id. Signed-off-by: brian m. carlson Signed-off-by: Junio C Hamano --- pack-bitmap.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'pack-bitmap.c') diff --git a/pack-bitmap.c b/pack-bitmap.c index b53f37243c..c760913cea 100644 --- a/pack-bitmap.c +++ b/pack-bitmap.c @@ -18,7 +18,7 @@ * commit. */ struct stored_bitmap { - unsigned char sha1[20]; + struct object_id oid; struct ewah_bitmap *root; struct stored_bitmap *xor; int flags; @@ -181,9 +181,9 @@ static struct stored_bitmap *store_bitmap(struct bitmap_index *index, stored->root = root; stored->xor = xor_with; stored->flags = flags; - hashcpy(stored->sha1, sha1); + oidread(&stored->oid, sha1); - hash_pos = kh_put_sha1(index->bitmaps, stored->sha1, &ret); + hash_pos = kh_put_sha1(index->bitmaps, stored->oid.hash, &ret); /* a 0 return code means the insertion succeeded with no changes, * because the SHA1 already existed on the map. this is bad, there @@ -1080,7 +1080,7 @@ int rebuild_existing_bitmaps(struct bitmap_index *bitmap_git, lookup_stored_bitmap(stored), rebuild)) { hash_pos = kh_put_sha1(reused_bitmaps, - stored->sha1, + stored->oid.hash, &hash_ret); kh_value(reused_bitmaps, hash_pos) = bitmap_to_ewah(rebuild); -- cgit v1.3 From 0dd4924891664e9e9970e427e84f902491fcd3d3 Mon Sep 17 00:00:00 2001 From: "brian m. carlson" Date: Tue, 19 Feb 2019 00:04:56 +0000 Subject: pack-bitmap: replace sha1_to_hex Replace the uses of sha1_to_hex in the pack bitmap code with hash_to_hex to allow the use of SHA-256 as well. Rename a few variables since they are no longer limited to SHA-1. Signed-off-by: brian m. carlson Signed-off-by: Junio C Hamano --- pack-bitmap-write.c | 6 +++--- pack-bitmap.c | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) (limited to 'pack-bitmap.c') diff --git a/pack-bitmap-write.c b/pack-bitmap-write.c index c82fb01fd7..802ed62677 100644 --- a/pack-bitmap-write.c +++ b/pack-bitmap-write.c @@ -142,13 +142,13 @@ static inline void reset_all_seen(void) seen_objects_nr = 0; } -static uint32_t find_object_pos(const unsigned char *sha1) +static uint32_t find_object_pos(const unsigned char *hash) { - struct object_entry *entry = packlist_find(writer.to_pack, sha1, NULL); + struct object_entry *entry = packlist_find(writer.to_pack, hash, NULL); if (!entry) { die("Failed to write bitmap index. Packfile doesn't have full closure " - "(object %s is missing)", sha1_to_hex(sha1)); + "(object %s is missing)", hash_to_hex(hash)); } return oe_in_pack_pos(writer.to_pack, entry); diff --git a/pack-bitmap.c b/pack-bitmap.c index c760913cea..6d6fa68563 100644 --- a/pack-bitmap.c +++ b/pack-bitmap.c @@ -169,7 +169,7 @@ static int load_bitmap_header(struct bitmap_index *index) static struct stored_bitmap *store_bitmap(struct bitmap_index *index, struct ewah_bitmap *root, - const unsigned char *sha1, + const unsigned char *hash, struct stored_bitmap *xor_with, int flags) { @@ -181,7 +181,7 @@ static struct stored_bitmap *store_bitmap(struct bitmap_index *index, stored->root = root; stored->xor = xor_with; stored->flags = flags; - oidread(&stored->oid, sha1); + oidread(&stored->oid, hash); hash_pos = kh_put_sha1(index->bitmaps, stored->oid.hash, &ret); @@ -189,7 +189,7 @@ static struct stored_bitmap *store_bitmap(struct bitmap_index *index, * because the SHA1 already existed on the map. this is bad, there * shouldn't be duplicated commits in the index */ if (ret == 0) { - error("Duplicate entry in bitmap index: %s", sha1_to_hex(sha1)); + error("Duplicate entry in bitmap index: %s", hash_to_hex(hash)); return NULL; } @@ -805,7 +805,7 @@ int reuse_partial_packfile_from_bitmap(struct bitmap_index *bitmap_git, fprintf(stderr, "Failed to reuse at %d (%016llx)\n", reuse_objects, result->words[i]); - fprintf(stderr, " %s\n", sha1_to_hex(sha1)); + fprintf(stderr, " %s\n", hash_to_hex(sha1)); } #endif -- cgit v1.3 From 9941e920e0b5a79c5b7859cb59b9ab866a77b25f Mon Sep 17 00:00:00 2001 From: "brian m. carlson" Date: Tue, 19 Feb 2019 00:04:57 +0000 Subject: pack-bitmap: switch hard-coded constants to the_hash_algo Switch two hard-coded uses of 20 to references to the_hash_algo. Signed-off-by: brian m. carlson Signed-off-by: Junio C Hamano --- pack-bitmap.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'pack-bitmap.c') diff --git a/pack-bitmap.c b/pack-bitmap.c index 6d6fa68563..603492c237 100644 --- a/pack-bitmap.c +++ b/pack-bitmap.c @@ -138,7 +138,7 @@ static int load_bitmap_header(struct bitmap_index *index) { struct bitmap_disk_header *header = (void *)index->map; - if (index->map_size < sizeof(*header) + 20) + if (index->map_size < sizeof(*header) + the_hash_algo->rawsz) return error("Corrupted bitmap index (missing header data)"); if (memcmp(header->magic, BITMAP_IDX_SIGNATURE, sizeof(BITMAP_IDX_SIGNATURE)) != 0) @@ -157,7 +157,7 @@ static int load_bitmap_header(struct bitmap_index *index) "(Git requires BITMAP_OPT_FULL_DAG)"); if (flags & BITMAP_OPT_HASH_CACHE) { - unsigned char *end = index->map + index->map_size - 20; + unsigned char *end = index->map + index->map_size - the_hash_algo->rawsz; index->hashes = ((uint32_t *)end) - index->pack->num_objects; } } -- cgit v1.3 From 3c7714485dc8adc810b6c52058992cfc767dfcb5 Mon Sep 17 00:00:00 2001 From: "brian m. carlson" Date: Tue, 19 Feb 2019 00:04:58 +0000 Subject: pack-bitmap: switch hash tables to use struct object_id Instead of storing unsigned char pointers in the hash tables, switch to storing instances of struct object_id. Update several internal functions and one external function to take pointers to struct object_id. Signed-off-by: brian m. carlson Signed-off-by: Junio C Hamano --- builtin/pack-objects.c | 6 +++--- pack-bitmap.c | 58 +++++++++++++++++++++++++------------------------- pack-bitmap.h | 2 +- 3 files changed, 33 insertions(+), 33 deletions(-) (limited to 'pack-bitmap.c') diff --git a/builtin/pack-objects.c b/builtin/pack-objects.c index a154fc29f6..2d9a3bdc9d 100644 --- a/builtin/pack-objects.c +++ b/builtin/pack-objects.c @@ -1487,6 +1487,7 @@ static int can_reuse_delta(const unsigned char *base_sha1, struct object_entry **base_out) { struct object_entry *base; + struct object_id base_oid; if (!base_sha1) return 0; @@ -1508,10 +1509,9 @@ static int can_reuse_delta(const unsigned char *base_sha1, * even if it was buried too deep in history to make it into the * packing list. */ - if (thin && bitmap_has_sha1_in_uninteresting(bitmap_git, base_sha1)) { + oidread(&base_oid, base_sha1); + if (thin && bitmap_has_oid_in_uninteresting(bitmap_git, &base_oid)) { if (use_delta_islands) { - struct object_id base_oid; - hashcpy(base_oid.hash, base_sha1); if (!in_same_island(&delta->idx.oid, &base_oid)) return 0; } diff --git a/pack-bitmap.c b/pack-bitmap.c index 603492c237..70d51f4f50 100644 --- a/pack-bitmap.c +++ b/pack-bitmap.c @@ -60,8 +60,8 @@ struct bitmap_index { struct ewah_bitmap *blobs; struct ewah_bitmap *tags; - /* Map from SHA1 -> `stored_bitmap` for all the bitmapped commits */ - khash_sha1 *bitmaps; + /* Map from object ID -> `stored_bitmap` for all the bitmapped commits */ + kh_oid_map_t *bitmaps; /* Number of bitmapped commits */ uint32_t entry_count; @@ -80,7 +80,7 @@ struct bitmap_index { struct object **objects; uint32_t *hashes; uint32_t count, alloc; - khash_sha1_pos *positions; + kh_oid_pos_t *positions; } ext_index; /* Bitmap result of the last performed walk */ @@ -183,7 +183,7 @@ static struct stored_bitmap *store_bitmap(struct bitmap_index *index, stored->flags = flags; oidread(&stored->oid, hash); - hash_pos = kh_put_sha1(index->bitmaps, stored->oid.hash, &ret); + hash_pos = kh_put_oid_map(index->bitmaps, stored->oid, &ret); /* a 0 return code means the insertion succeeded with no changes, * because the SHA1 already existed on the map. this is bad, there @@ -306,8 +306,8 @@ static int load_pack_bitmap(struct bitmap_index *bitmap_git) { assert(bitmap_git->map); - bitmap_git->bitmaps = kh_init_sha1(); - bitmap_git->ext_index.positions = kh_init_sha1_pos(); + bitmap_git->bitmaps = kh_init_oid_map(); + bitmap_git->ext_index.positions = kh_init_oid_pos(); load_pack_revindex(bitmap_git->pack); if (!(bitmap_git->commits = read_bitmap_1(bitmap_git)) || @@ -362,10 +362,10 @@ struct include_data { }; static inline int bitmap_position_extended(struct bitmap_index *bitmap_git, - const unsigned char *sha1) + const struct object_id *oid) { - khash_sha1_pos *positions = bitmap_git->ext_index.positions; - khiter_t pos = kh_get_sha1_pos(positions, sha1); + khash_oid_pos *positions = bitmap_git->ext_index.positions; + khiter_t pos = kh_get_oid_pos(positions, *oid); if (pos < kh_end(positions)) { int bitmap_pos = kh_value(positions, pos); @@ -376,9 +376,9 @@ static inline int bitmap_position_extended(struct bitmap_index *bitmap_git, } static inline int bitmap_position_packfile(struct bitmap_index *bitmap_git, - const unsigned char *sha1) + const struct object_id *oid) { - off_t offset = find_pack_entry_one(sha1, bitmap_git->pack); + off_t offset = find_pack_entry_one(oid->hash, bitmap_git->pack); if (!offset) return -1; @@ -386,10 +386,10 @@ static inline int bitmap_position_packfile(struct bitmap_index *bitmap_git, } static int bitmap_position(struct bitmap_index *bitmap_git, - const unsigned char *sha1) + const struct object_id *oid) { - int pos = bitmap_position_packfile(bitmap_git, sha1); - return (pos >= 0) ? pos : bitmap_position_extended(bitmap_git, sha1); + int pos = bitmap_position_packfile(bitmap_git, oid); + return (pos >= 0) ? pos : bitmap_position_extended(bitmap_git, oid); } static int ext_index_add_object(struct bitmap_index *bitmap_git, @@ -401,7 +401,7 @@ static int ext_index_add_object(struct bitmap_index *bitmap_git, int hash_ret; int bitmap_pos; - hash_pos = kh_put_sha1_pos(eindex->positions, object->oid.hash, &hash_ret); + hash_pos = kh_put_oid_pos(eindex->positions, object->oid, &hash_ret); if (hash_ret > 0) { if (eindex->count >= eindex->alloc) { eindex->alloc = (eindex->alloc + 16) * 3 / 2; @@ -431,7 +431,7 @@ static void show_object(struct object *object, const char *name, void *data_) struct bitmap_show_data *data = data_; int bitmap_pos; - bitmap_pos = bitmap_position(data->bitmap_git, object->oid.hash); + bitmap_pos = bitmap_position(data->bitmap_git, &object->oid); if (bitmap_pos < 0) bitmap_pos = ext_index_add_object(data->bitmap_git, object, @@ -446,7 +446,7 @@ static void show_commit(struct commit *commit, void *data) static int add_to_include_set(struct bitmap_index *bitmap_git, struct include_data *data, - const unsigned char *sha1, + const struct object_id *oid, int bitmap_pos) { khiter_t hash_pos; @@ -457,7 +457,7 @@ static int add_to_include_set(struct bitmap_index *bitmap_git, if (bitmap_get(data->base, bitmap_pos)) return 0; - hash_pos = kh_get_sha1(bitmap_git->bitmaps, sha1); + hash_pos = kh_get_oid_map(bitmap_git->bitmaps, *oid); if (hash_pos < kh_end(bitmap_git->bitmaps)) { struct stored_bitmap *st = kh_value(bitmap_git->bitmaps, hash_pos); bitmap_or_ewah(data->base, lookup_stored_bitmap(st)); @@ -473,13 +473,13 @@ static int should_include(struct commit *commit, void *_data) struct include_data *data = _data; int bitmap_pos; - bitmap_pos = bitmap_position(data->bitmap_git, commit->object.oid.hash); + bitmap_pos = bitmap_position(data->bitmap_git, &commit->object.oid); if (bitmap_pos < 0) bitmap_pos = ext_index_add_object(data->bitmap_git, (struct object *)commit, NULL); - if (!add_to_include_set(data->bitmap_git, data, commit->object.oid.hash, + if (!add_to_include_set(data->bitmap_git, data, &commit->object.oid, bitmap_pos)) { struct commit_list *parent = commit->parents; @@ -517,7 +517,7 @@ static struct bitmap *find_objects(struct bitmap_index *bitmap_git, roots = roots->next; if (object->type == OBJ_COMMIT) { - khiter_t pos = kh_get_sha1(bitmap_git->bitmaps, object->oid.hash); + khiter_t pos = kh_get_oid_map(bitmap_git->bitmaps, object->oid); if (pos < kh_end(bitmap_git->bitmaps)) { struct stored_bitmap *st = kh_value(bitmap_git->bitmaps, pos); @@ -559,7 +559,7 @@ static struct bitmap *find_objects(struct bitmap_index *bitmap_git, int pos; roots = roots->next; - pos = bitmap_position(bitmap_git, object->oid.hash); + pos = bitmap_position(bitmap_git, &object->oid); if (pos < 0 || base == NULL || !bitmap_get(base, pos)) { object->flags &= ~UNINTERESTING; @@ -925,7 +925,7 @@ static void test_show_object(struct object *object, const char *name, struct bitmap_test_data *tdata = data; int bitmap_pos; - bitmap_pos = bitmap_position(tdata->bitmap_git, object->oid.hash); + bitmap_pos = bitmap_position(tdata->bitmap_git, &object->oid); if (bitmap_pos < 0) die("Object not in bitmap: %s\n", oid_to_hex(&object->oid)); @@ -939,7 +939,7 @@ static void test_show_commit(struct commit *commit, void *data) int bitmap_pos; bitmap_pos = bitmap_position(tdata->bitmap_git, - commit->object.oid.hash); + &commit->object.oid); if (bitmap_pos < 0) die("Object not in bitmap: %s\n", oid_to_hex(&commit->object.oid)); @@ -966,7 +966,7 @@ void test_bitmap_walk(struct rev_info *revs) bitmap_git->version, bitmap_git->entry_count); root = revs->pending.objects[0].item; - pos = kh_get_sha1(bitmap_git->bitmaps, root->oid.hash); + pos = kh_get_oid_map(bitmap_git->bitmaps, root->oid); if (pos < kh_end(bitmap_git->bitmaps)) { struct stored_bitmap *st = kh_value(bitmap_git->bitmaps, pos); @@ -1108,7 +1108,7 @@ void free_bitmap_index(struct bitmap_index *b) ewah_pool_free(b->trees); ewah_pool_free(b->blobs); ewah_pool_free(b->tags); - kh_destroy_sha1(b->bitmaps); + kh_destroy_oid_map(b->bitmaps); free(b->ext_index.objects); free(b->ext_index.hashes); bitmap_free(b->result); @@ -1116,8 +1116,8 @@ void free_bitmap_index(struct bitmap_index *b) free(b); } -int bitmap_has_sha1_in_uninteresting(struct bitmap_index *bitmap_git, - const unsigned char *sha1) +int bitmap_has_oid_in_uninteresting(struct bitmap_index *bitmap_git, + const struct object_id *oid) { int pos; @@ -1126,7 +1126,7 @@ int bitmap_has_sha1_in_uninteresting(struct bitmap_index *bitmap_git, if (!bitmap_git->haves) return 0; /* walk had no "haves" */ - pos = bitmap_position_packfile(bitmap_git, sha1); + pos = bitmap_position_packfile(bitmap_git, oid); if (pos < 0) return 0; diff --git a/pack-bitmap.h b/pack-bitmap.h index 344ba23af9..ee9792264c 100644 --- a/pack-bitmap.h +++ b/pack-bitmap.h @@ -59,7 +59,7 @@ void free_bitmap_index(struct bitmap_index *); * queried to see if a particular object was reachable from any of the * objects flagged as UNINTERESTING. */ -int bitmap_has_sha1_in_uninteresting(struct bitmap_index *, const unsigned char *sha1); +int bitmap_has_oid_in_uninteresting(struct bitmap_index *, const struct object_id *oid); void bitmap_writer_show_progress(int show); void bitmap_writer_set_checksum(unsigned char *sha1); -- cgit v1.3