From 4d703a1a9016cd0a08994ddf7fc2f4739f223112 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Sat, 23 Dec 2006 02:33:25 -0500 Subject: Replace unpack_entry_gently with unpack_entry. The unpack_entry_gently function currently has only two callers: the delta base resolution in sha1_file.c and the main loop of pack-check.c. Both of these must change to using unpack_entry directly when we implement sliding window mmap logic, so I'm doing it earlier to help break down the change set. This may cause a slight performance decrease for delta base resolution as well as for pack-check.c's verify_packfile(), as the pack use counter will be incremented and decremented for every object that is unpacked. Signed-off-by: Shawn O. Pearce Signed-off-by: Junio C Hamano --- cache.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cache.h') diff --git a/cache.h b/cache.h index 29dd290c92..e0a26e9d04 100644 --- a/cache.h +++ b/cache.h @@ -394,7 +394,7 @@ extern struct packed_git *add_packed_git(char *, int, int); extern int num_packed_objects(const struct packed_git *p); extern int nth_packed_object_sha1(const struct packed_git *, int, unsigned char*); extern unsigned long find_pack_entry_one(const unsigned char *, struct packed_git *); -extern void *unpack_entry_gently(struct packed_git *, unsigned long, char *, unsigned long *); +extern void *unpack_entry(struct packed_git *, unsigned long, char *, unsigned long *); extern unsigned long unpack_object_header_gently(const unsigned char *buf, unsigned long len, enum object_type *type, unsigned long *sizep); extern void packed_object_info_detail(struct packed_git *, unsigned long, char *, unsigned long *, unsigned long *, unsigned int *, unsigned char *); -- cgit v1.3 From 77ccc5bbd1bd403abd5f552be7210073bea856a6 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Sat, 23 Dec 2006 02:33:35 -0500 Subject: Introduce new config option for mmap limit. Rather than hardcoding the maximum number of bytes which can be mmapped from pack files we should make this value configurable, allowing the end user to increase or decrease this limit on a per-repository basis depending on the size of the repository and the capabilities of their operating system. In general users should not need to manually tune such a low-level setting within the core code, but being able to artifically limit the number of bytes which we can mmap at once from pack files will make it easier to craft test cases for the new mmap sliding window implementation. Signed-off-by: Shawn O. Pearce Signed-off-by: Junio C Hamano --- Documentation/config.txt | 9 +++++++++ cache.h | 1 + config.c | 5 +++++ environment.c | 1 + sha1_file.c | 3 +-- 5 files changed, 17 insertions(+), 2 deletions(-) (limited to 'cache.h') diff --git a/Documentation/config.txt b/Documentation/config.txt index 178e0e1e20..28fe6942cf 100644 --- a/Documentation/config.txt +++ b/Documentation/config.txt @@ -118,6 +118,15 @@ core.legacyheaders:: database directly (where the "http://" and "rsync://" protocols count as direct access). +core.packedGitLimit:: + Maximum number of bytes to map simultaneously into memory + from pack files. If Git needs to access more than this many + bytes at once to complete an operation it will unmap existing + regions to reclaim virtual address space within the process. + Default is 256 MiB, which should be reasonable for all + users/operating systems, except on largest Git projects. + You probably do not need to adjust this value. + alias.*:: Command aliases for the gitlink:git[1] command wrapper - e.g. after defining "alias.last = cat-file commit HEAD", the invocation diff --git a/cache.h b/cache.h index e0a26e9d04..816239bea0 100644 --- a/cache.h +++ b/cache.h @@ -196,6 +196,7 @@ extern int warn_ambiguous_refs; extern int shared_repository; extern const char *apply_default_whitespace; extern int zlib_compression_level; +extern size_t packed_git_limit; #define GIT_REPO_VERSION 0 extern int repository_format_version; diff --git a/config.c b/config.c index fcccf7e2a4..0c21286cb2 100644 --- a/config.c +++ b/config.c @@ -298,6 +298,11 @@ int git_default_config(const char *var, const char *value) return 0; } + if (!strcmp(var, "core.packedgitlimit")) { + packed_git_limit = git_config_int(var, value); + return 0; + } + if (!strcmp(var, "user.name")) { strlcpy(git_default_name, value, sizeof(git_default_name)); return 0; diff --git a/environment.c b/environment.c index a1502c4e87..a3ddae68a7 100644 --- a/environment.c +++ b/environment.c @@ -23,6 +23,7 @@ char *git_log_output_encoding; int shared_repository = PERM_UMASK; const char *apply_default_whitespace; int zlib_compression_level = Z_DEFAULT_COMPRESSION; +size_t packed_git_limit = 256 * 1024 * 1024; int pager_in_use; int pager_use_color = 1; diff --git a/sha1_file.c b/sha1_file.c index 4824a5d4d8..4183f595ed 100644 --- a/sha1_file.c +++ b/sha1_file.c @@ -397,7 +397,6 @@ static char *find_sha1_file(const unsigned char *sha1, struct stat *st) return NULL; } -#define PACK_MAX_SZ (1<<26) static int pack_used_ctr; static unsigned long pack_mapped; struct packed_git *packed_git; @@ -490,7 +489,7 @@ int use_packed_git(struct packed_git *p) struct pack_header *hdr; pack_mapped += p->pack_size; - while (PACK_MAX_SZ < pack_mapped && unuse_one_packed_git()) + while (packed_git_limit < pack_mapped && unuse_one_packed_git()) ; /* nothing */ fd = open(p->pack_name, O_RDONLY); if (fd < 0) -- cgit v1.3 From c41ee586dc95b757cdff4deae10a30a691ba758b Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Sat, 23 Dec 2006 02:33:44 -0500 Subject: Refactor packed_git to prepare for sliding mmap windows. The idea behind the sliding mmap window pack reader implementation is to have multiple mmap regions active against the same pack file, thereby allowing the process to mmap in only the active/hot sections of the pack and reduce overall virtual address space usage. To implement this we need to refactor the mmap related data (pack_base, pack_use_cnt) out of struct packed_git and move them into a new struct pack_window. We are refactoring the code to support a single struct pack_window per packfile, thereby emulating the prior behavior of mmap'ing the entire pack file. Signed-off-by: Shawn O. Pearce Signed-off-by: Junio C Hamano --- builtin-pack-objects.c | 4 ++-- cache.h | 13 ++++++++++--- pack-check.c | 6 +++--- sha1_file.c | 51 +++++++++++++++++++++++++------------------------- 4 files changed, 40 insertions(+), 34 deletions(-) (limited to 'cache.h') diff --git a/builtin-pack-objects.c b/builtin-pack-objects.c index 9e15beb3ba..4a00a1206f 100644 --- a/builtin-pack-objects.c +++ b/builtin-pack-objects.c @@ -438,7 +438,7 @@ static unsigned long write_object(struct sha1file *f, } use_packed_git(p); - buf = (char *) p->pack_base + buf = p->windows->base + entry->in_pack_offset + entry->in_pack_header_size; datalen = find_packed_object_size(p, entry->in_pack_offset) @@ -943,7 +943,7 @@ static void check_object(struct object_entry *entry) struct object_entry *base_entry = NULL; use_packed_git(p); - buf = p->pack_base; + buf = p->windows->base; buf += entry->in_pack_offset; /* We want in_pack_type even if we do not reuse delta. diff --git a/cache.h b/cache.h index 816239bea0..ae7bceca50 100644 --- a/cache.h +++ b/cache.h @@ -336,14 +336,21 @@ extern struct alternate_object_database { } *alt_odb_list; extern void prepare_alt_odb(void); +struct pack_window { + struct pack_window *next; + unsigned char *base; + off_t offset; + size_t len; + unsigned int last_used; + unsigned int inuse_cnt; +}; + extern struct packed_git { struct packed_git *next; unsigned long index_size; unsigned long pack_size; + struct pack_window *windows; unsigned int *index_base; - void *pack_base; - unsigned int pack_last_used; - unsigned int pack_use_cnt; int pack_local; unsigned char sha1[20]; /* something like ".git/objects/pack/xxxxx.pack" */ diff --git a/pack-check.c b/pack-check.c index 491bad2ae4..761cc852e9 100644 --- a/pack-check.c +++ b/pack-check.c @@ -13,7 +13,8 @@ static int verify_packfile(struct packed_git *p) int nr_objects, err, i; /* Header consistency check */ - hdr = p->pack_base; + pack_base = p->windows->base; + hdr = (struct pack_header*)pack_base; if (hdr->hdr_signature != htonl(PACK_SIGNATURE)) return error("Packfile %s signature mismatch", p->pack_name); if (!pack_version_ok(hdr->hdr_version)) @@ -26,7 +27,6 @@ static int verify_packfile(struct packed_git *p) num_packed_objects(p)); SHA1_Init(&ctx); - pack_base = p->pack_base; SHA1_Update(&ctx, pack_base, pack_size - 20); SHA1_Final(sha1, &ctx); if (hashcmp(sha1, (unsigned char *)pack_base + pack_size - 20)) @@ -78,7 +78,7 @@ static void show_pack_info(struct packed_git *p) int nr_objects, i; unsigned int chain_histogram[MAX_CHAIN]; - hdr = p->pack_base; + hdr = (struct pack_header*)p->windows->base; nr_objects = ntohl(hdr->hdr_entries); memset(chain_histogram, 0, sizeof(chain_histogram)); diff --git a/sha1_file.c b/sha1_file.c index 4183f595ed..a9f374e60a 100644 --- a/sha1_file.c +++ b/sha1_file.c @@ -455,21 +455,23 @@ static int unuse_one_packed_git(void) struct packed_git *p, *lru = NULL; for (p = packed_git; p; p = p->next) { - if (p->pack_use_cnt || !p->pack_base) + if (!p->windows || p->windows->inuse_cnt) continue; - if (!lru || p->pack_last_used < lru->pack_last_used) + if (!lru || p->windows->last_used < lru->windows->last_used) lru = p; } if (!lru) return 0; - munmap(lru->pack_base, lru->pack_size); - lru->pack_base = NULL; + munmap(lru->windows->base, lru->windows->len); + free(lru->windows); + lru->windows = NULL; return 1; } void unuse_packed_git(struct packed_git *p) { - p->pack_use_cnt--; + if (p->windows) + p->windows->inuse_cnt--; } int use_packed_git(struct packed_git *p) @@ -482,10 +484,10 @@ int use_packed_git(struct packed_git *p) die("packfile %s not a regular file", p->pack_name); p->pack_size = st.st_size; } - if (!p->pack_base) { + if (!p->windows) { int fd; struct stat st; - void *map; + struct pack_window *win; struct pack_header *hdr; pack_mapped += p->pack_size; @@ -500,16 +502,18 @@ int use_packed_git(struct packed_git *p) } if (st.st_size != p->pack_size) die("packfile %s size mismatch.", p->pack_name); - map = mmap(NULL, p->pack_size, PROT_READ, MAP_PRIVATE, fd, 0); + win = xcalloc(1, sizeof(*win)); + win->len = p->pack_size; + win->base = mmap(NULL, p->pack_size, PROT_READ, MAP_PRIVATE, fd, 0); close(fd); - if (map == MAP_FAILED) + if (win->base == MAP_FAILED) die("packfile %s cannot be mapped.", p->pack_name); - p->pack_base = map; + p->windows = win; /* Check if we understand this pack file. If we don't we're * likely too old to handle it. */ - hdr = map; + hdr = (struct pack_header*)win->base; if (hdr->hdr_signature != htonl(PACK_SIGNATURE)) die("packfile %s isn't actually a pack.", p->pack_name); if (!pack_version_ok(hdr->hdr_version)) @@ -522,13 +526,13 @@ int use_packed_git(struct packed_git *p) */ if (hashcmp((unsigned char *)(p->index_base) + p->index_size - 40, - (unsigned char *)p->pack_base + + p->windows->base + p->pack_size - 20)) { die("packfile %s does not match index.", p->pack_name); } } - p->pack_last_used = pack_used_ctr++; - p->pack_use_cnt++; + p->windows->last_used = pack_used_ctr++; + p->windows->inuse_cnt++; return 0; } @@ -558,9 +562,7 @@ struct packed_git *add_packed_git(char *path, int path_len, int local) p->pack_size = st.st_size; p->index_base = idx_map; p->next = NULL; - p->pack_base = NULL; - p->pack_last_used = 0; - p->pack_use_cnt = 0; + p->windows = NULL; p->pack_local = local; if ((path_len > 44) && !get_sha1_hex(path + path_len - 44, sha1)) hashcpy(p->sha1, sha1); @@ -591,9 +593,7 @@ struct packed_git *parse_pack_index_file(const unsigned char *sha1, char *idx_pa p->pack_size = 0; p->index_base = idx_map; p->next = NULL; - p->pack_base = NULL; - p->pack_last_used = 0; - p->pack_use_cnt = 0; + p->windows = NULL; hashcpy(p->sha1, sha1); return p; } @@ -882,7 +882,7 @@ static unsigned long get_delta_base(struct packed_git *p, unsigned long delta_obj_offset, unsigned long *base_obj_offset) { - unsigned char *base_info = (unsigned char *) p->pack_base + offset; + unsigned char *base_info = p->windows->base + offset; unsigned long base_offset; /* there must be at least 20 bytes left regardless of delta type */ @@ -949,7 +949,7 @@ static int packed_delta_info(struct packed_git *p, memset(&stream, 0, sizeof(stream)); - stream.next_in = (unsigned char *) p->pack_base + offset; + stream.next_in = p->windows->base + offset; stream.avail_in = p->pack_size - offset; stream.next_out = delta_head; stream.avail_out = sizeof(delta_head); @@ -984,8 +984,7 @@ static unsigned long unpack_object_header(struct packed_git *p, unsigned long of if (p->pack_size <= offset) die("object offset outside of pack file"); - used = unpack_object_header_gently((unsigned char *)p->pack_base + - offset, + used = unpack_object_header_gently(p->windows->base + offset, p->pack_size - offset, type, sizep); if (!used) die("object offset outside of pack file"); @@ -1031,7 +1030,7 @@ void packed_object_info_detail(struct packed_git *p, if (p->pack_size <= offset + 20) die("pack file %s records an incomplete delta base", p->pack_name); - next_sha1 = (unsigned char *) p->pack_base + offset; + next_sha1 = p->windows->base + offset; if (*delta_chain_length == 0) hashcpy(base_sha1, next_sha1); offset = find_pack_entry_one(next_sha1, p); @@ -1081,7 +1080,7 @@ static void *unpack_compressed_entry(struct packed_git *p, buffer = xmalloc(size + 1); buffer[size] = 0; memset(&stream, 0, sizeof(stream)); - stream.next_in = (unsigned char*)p->pack_base + offset; + stream.next_in = p->windows->base + offset; stream.avail_in = p->pack_size - offset; stream.next_out = buffer; stream.avail_out = size; -- cgit v1.3 From 2dc3a234094afc61c367be3e6018722cd9d92ea9 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Sat, 23 Dec 2006 02:33:47 -0500 Subject: Use off_t for index and pack file lengths. Since the index_size and pack_size members of struct packed_git are the lengths of those corresponding files we should use the off_t size of the operating system to store these file lengths, rather than an unsigned long. This would help in the future should we ever resurrect Junio's 64 bit index implementation. Signed-off-by: Shawn O. Pearce Signed-off-by: Junio C Hamano --- cache.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'cache.h') diff --git a/cache.h b/cache.h index ae7bceca50..cc872440ef 100644 --- a/cache.h +++ b/cache.h @@ -347,10 +347,10 @@ struct pack_window { extern struct packed_git { struct packed_git *next; - unsigned long index_size; - unsigned long pack_size; struct pack_window *windows; unsigned int *index_base; + off_t index_size; + off_t pack_size; int pack_local; unsigned char sha1[20]; /* something like ".git/objects/pack/xxxxx.pack" */ -- cgit v1.3 From 75025ccdb79b5d6b290c630f8777c9f7bb9d257c Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Sat, 23 Dec 2006 02:33:55 -0500 Subject: Create read_or_die utility routine. Like write_or_die read_or_die reads the entire length requested or it kills the current process with a die call. Signed-off-by: Shawn O. Pearce Signed-off-by: Junio C Hamano --- cache.h | 1 + write_or_die.c | 16 ++++++++++++++++ 2 files changed, 17 insertions(+) (limited to 'cache.h') diff --git a/cache.h b/cache.h index cc872440ef..e6b716495d 100644 --- a/cache.h +++ b/cache.h @@ -428,6 +428,7 @@ extern char *git_commit_encoding; extern char *git_log_output_encoding; extern int copy_fd(int ifd, int ofd); +extern void read_or_die(int fd, void *buf, size_t count); extern void write_or_die(int fd, const void *buf, size_t count); extern int write_or_whine(int fd, const void *buf, size_t count, const char *msg); diff --git a/write_or_die.c b/write_or_die.c index bfe4eeb649..8cf6486025 100644 --- a/write_or_die.c +++ b/write_or_die.c @@ -1,5 +1,21 @@ #include "cache.h" +void read_or_die(int fd, void *buf, size_t count) +{ + char *p = buf; + ssize_t loaded; + + while (count > 0) { + loaded = xread(fd, p, count); + if (loaded == 0) + die("unexpected end of file"); + else if (loaded < 0) + die("read error (%s)", strerror(errno)); + count -= loaded; + p += loaded; + } +} + void write_or_die(int fd, const void *buf, size_t count) { const char *p = buf; -- cgit v1.3 From 9bc879c1ced505089e2a1e420d32599bb15b35b5 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Sat, 23 Dec 2006 02:34:01 -0500 Subject: Refactor how we open pack files to prepare for multiple windows. To efficiently support mmaping of multiple regions of the same pack file we want to keep the pack's file descriptor open while we are actively working with that pack. So we are now keeping that file descriptor in packed_git.pack_fd and closing it only after we unmap the last window. This is going to increase the number of file descriptors that are in use at once, however that will be bounded by the total number of pack files present and therefore should not be very high. It is a small tradeoff which we may need to revisit after some testing can be done on various repositories and systems. For code clarity we also want to seperate out the implementation of how we open a pack file from the implementation which locates a suitable window (or makes a new one) from the given pack file. Since this is a rather large delta I'm taking advantage of doing it now, in a fairly isolated change. When we open a pack file we need to examine the header and trailer without having a mmap in place, as we may only need to mmap the middle section of this particular pack. Consequently the verification code has been refactored to make use of the new read_or_die function. Signed-off-by: Shawn O. Pearce Signed-off-by: Junio C Hamano --- cache.h | 1 + sha1_file.c | 86 +++++++++++++++++++++++++++++++++---------------------------- 2 files changed, 47 insertions(+), 40 deletions(-) (limited to 'cache.h') diff --git a/cache.h b/cache.h index e6b716495d..f8a89280fd 100644 --- a/cache.h +++ b/cache.h @@ -351,6 +351,7 @@ extern struct packed_git { unsigned int *index_base; off_t index_size; off_t pack_size; + int pack_fd; int pack_local; unsigned char sha1[20]; /* something like ".git/objects/pack/xxxxx.pack" */ diff --git a/sha1_file.c b/sha1_file.c index a9f374e60a..79d2d9dacd 100644 --- a/sha1_file.c +++ b/sha1_file.c @@ -465,6 +465,8 @@ static int unuse_one_packed_git(void) munmap(lru->windows->base, lru->windows->len); free(lru->windows); lru->windows = NULL; + close(p->pack_fd); + p->pack_fd = -1; return 1; } @@ -474,62 +476,64 @@ void unuse_packed_git(struct packed_git *p) p->windows->inuse_cnt--; } -int use_packed_git(struct packed_git *p) +static void open_packed_git(struct packed_git *p) { + struct stat st; + struct pack_header hdr; + unsigned char sha1[20]; + unsigned char *idx_sha1; + + p->pack_fd = open(p->pack_name, O_RDONLY); + if (p->pack_fd < 0 || fstat(p->pack_fd, &st)) + die("packfile %s cannot be opened", p->pack_name); + + /* If we created the struct before we had the pack we lack size. */ if (!p->pack_size) { - struct stat st; - /* We created the struct before we had the pack */ - stat(p->pack_name, &st); if (!S_ISREG(st.st_mode)) die("packfile %s not a regular file", p->pack_name); p->pack_size = st.st_size; - } + } else if (p->pack_size != st.st_size) + die("packfile %s size changed", p->pack_name); + + /* Verify we recognize this pack file format. */ + read_or_die(p->pack_fd, &hdr, sizeof(hdr)); + if (hdr.hdr_signature != htonl(PACK_SIGNATURE)) + die("file %s is not a GIT packfile", p->pack_name); + if (!pack_version_ok(hdr.hdr_version)) + die("packfile %s is version %u and not supported" + " (try upgrading GIT to a newer version)", + p->pack_name, ntohl(hdr.hdr_version)); + + /* Verify the pack matches its index. */ + if (num_packed_objects(p) != ntohl(hdr.hdr_entries)) + die("packfile %s claims to have %u objects" + " while index size indicates %u objects", + p->pack_name, ntohl(hdr.hdr_entries), + num_packed_objects(p)); + if (lseek(p->pack_fd, p->pack_size - sizeof(sha1), SEEK_SET) == -1) + die("end of packfile %s is unavailable", p->pack_name); + read_or_die(p->pack_fd, sha1, sizeof(sha1)); + idx_sha1 = ((unsigned char *)p->index_base) + p->index_size - 40; + if (hashcmp(sha1, idx_sha1)) + die("packfile %s does not match index", p->pack_name); +} + +int use_packed_git(struct packed_git *p) +{ + if (p->pack_fd == -1) + open_packed_git(p); if (!p->windows) { - int fd; - struct stat st; struct pack_window *win; - struct pack_header *hdr; pack_mapped += p->pack_size; while (packed_git_limit < pack_mapped && unuse_one_packed_git()) ; /* nothing */ - fd = open(p->pack_name, O_RDONLY); - if (fd < 0) - die("packfile %s cannot be opened", p->pack_name); - if (fstat(fd, &st)) { - close(fd); - die("packfile %s cannot be opened", p->pack_name); - } - if (st.st_size != p->pack_size) - die("packfile %s size mismatch.", p->pack_name); win = xcalloc(1, sizeof(*win)); win->len = p->pack_size; - win->base = mmap(NULL, p->pack_size, PROT_READ, MAP_PRIVATE, fd, 0); - close(fd); + win->base = mmap(NULL, p->pack_size, PROT_READ, MAP_PRIVATE, p->pack_fd, 0); if (win->base == MAP_FAILED) die("packfile %s cannot be mapped.", p->pack_name); p->windows = win; - - /* Check if we understand this pack file. If we don't we're - * likely too old to handle it. - */ - hdr = (struct pack_header*)win->base; - if (hdr->hdr_signature != htonl(PACK_SIGNATURE)) - die("packfile %s isn't actually a pack.", p->pack_name); - if (!pack_version_ok(hdr->hdr_version)) - die("packfile %s is version %i and not supported" - " (try upgrading GIT to a newer version)", - p->pack_name, ntohl(hdr->hdr_version)); - - /* Check if the pack file matches with the index file. - * this is cheap. - */ - if (hashcmp((unsigned char *)(p->index_base) + - p->index_size - 40, - p->windows->base + - p->pack_size - 20)) { - die("packfile %s does not match index.", p->pack_name); - } } p->windows->last_used = pack_used_ctr++; p->windows->inuse_cnt++; @@ -563,6 +567,7 @@ struct packed_git *add_packed_git(char *path, int path_len, int local) p->index_base = idx_map; p->next = NULL; p->windows = NULL; + p->pack_fd = -1; p->pack_local = local; if ((path_len > 44) && !get_sha1_hex(path + path_len - 44, sha1)) hashcpy(p->sha1, sha1); @@ -594,6 +599,7 @@ struct packed_git *parse_pack_index_file(const unsigned char *sha1, char *idx_pa p->index_base = idx_map; p->next = NULL; p->windows = NULL; + p->pack_fd = -1; hashcpy(p->sha1, sha1); return p; } -- cgit v1.3 From 03e79c88aa34ce188eb4fb7509e6d127c79c507d Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Sat, 23 Dec 2006 02:34:08 -0500 Subject: Replace use_packed_git with window cursors. Part of the implementation concept of the sliding mmap window for pack access is to permit multiple windows per pack to be mapped independently. Since the inuse_cnt is associated with the mmap and not with the file, this value is in struct pack_window and needs to be incremented/decremented for each pack_window accessed by any code. To faciliate that implementation we need to replace all uses of use_packed_git() and unuse_packed_git() with a different API that follows struct pack_window objects rather than struct packed_git. The way this works is when we need to start accessing a pack for the first time we should setup a new window 'cursor' by declaring a local and setting it to NULL: struct pack_windows *w_curs = NULL; To obtain the memory region which contains a specific section of the pack file we invoke use_pack(), supplying the address of our current window cursor: unsigned int len; unsigned char *addr = use_pack(p, &w_curs, offset, &len); the returned address `addr` will be the first byte at `offset` within the pack file. The optional variable len will also be updated with the number of bytes remaining following the address. Multiple calls to use_pack() with the same window cursor will update the window cursor, moving it from one window to another when necessary. In this way each window cursor variable maintains only one struct pack_window inuse at a time. Finally before exiting the scope which originally declared the window cursor we must invoke unuse_pack() to unuse the current window (which may be different from the one that was first obtained from use_pack): unuse_pack(&w_curs); This implementation is still not complete with regards to multiple windows, as only one window per pack file is supported right now. Signed-off-by: Shawn O. Pearce Signed-off-by: Junio C Hamano --- builtin-pack-objects.c | 16 ++++--- cache.h | 4 +- pack-check.c | 22 +++++----- sha1_file.c | 110 ++++++++++++++++++++++++++++--------------------- 4 files changed, 85 insertions(+), 67 deletions(-) (limited to 'cache.h') diff --git a/builtin-pack-objects.c b/builtin-pack-objects.c index 4a00a1206f..6d7ae7f1ae 100644 --- a/builtin-pack-objects.c +++ b/builtin-pack-objects.c @@ -416,6 +416,7 @@ static unsigned long write_object(struct sha1file *f, } else { struct packed_git *p = entry->in_pack; + struct pack_window *w_curs = NULL; if (entry->delta) { obj_type = (allow_ofs_delta && entry->delta->offset) ? @@ -437,16 +438,14 @@ static unsigned long write_object(struct sha1file *f, hdrlen += 20; } - use_packed_git(p); - buf = p->windows->base - + entry->in_pack_offset - + entry->in_pack_header_size; + buf = use_pack(p, &w_curs, entry->in_pack_offset + + entry->in_pack_header_size, NULL); datalen = find_packed_object_size(p, entry->in_pack_offset) - entry->in_pack_header_size; if (!pack_to_stdout && check_inflate(buf, datalen, entry->size)) die("corrupt delta in pack %s", sha1_to_hex(entry->sha1)); sha1write(f, buf, datalen); - unuse_packed_git(p); + unuse_pack(&w_curs); reused++; } if (entry->delta) @@ -937,14 +936,13 @@ static void check_object(struct object_entry *entry) if (entry->in_pack && !entry->preferred_base) { struct packed_git *p = entry->in_pack; + struct pack_window *w_curs = NULL; unsigned long left = p->pack_size - entry->in_pack_offset; unsigned long size, used; unsigned char *buf; struct object_entry *base_entry = NULL; - use_packed_git(p); - buf = p->windows->base; - buf += entry->in_pack_offset; + buf = use_pack(p, &w_curs, entry->in_pack_offset, NULL); /* We want in_pack_type even if we do not reuse delta. * There is no point not reusing non-delta representations. @@ -990,7 +988,7 @@ static void check_object(struct object_entry *entry) if (base_name) base_entry = locate_object_entry(base_name); } - unuse_packed_git(p); + unuse_pack(&w_curs); entry->in_pack_header_size = used; if (base_entry) { diff --git a/cache.h b/cache.h index f8a89280fd..c927f29d31 100644 --- a/cache.h +++ b/cache.h @@ -397,8 +397,8 @@ extern void install_packed_git(struct packed_git *pack); extern struct packed_git *find_sha1_pack(const unsigned char *sha1, struct packed_git *packs); -extern int use_packed_git(struct packed_git *); -extern void unuse_packed_git(struct packed_git *); +extern unsigned char* use_pack(struct packed_git *, struct pack_window **, unsigned long, unsigned int *); +extern void unuse_pack(struct pack_window **); extern struct packed_git *add_packed_git(char *, int, int); extern int num_packed_objects(const struct packed_git *p); extern int nth_packed_object_sha1(const struct packed_git *, int, unsigned char*); diff --git a/pack-check.c b/pack-check.c index 761cc852e9..972916f406 100644 --- a/pack-check.c +++ b/pack-check.c @@ -1,7 +1,8 @@ #include "cache.h" #include "pack.h" -static int verify_packfile(struct packed_git *p) +static int verify_packfile(struct packed_git *p, + struct pack_window **w_curs) { unsigned long index_size = p->index_size; void *index_base = p->index_base; @@ -13,7 +14,7 @@ static int verify_packfile(struct packed_git *p) int nr_objects, err, i; /* Header consistency check */ - pack_base = p->windows->base; + pack_base = use_pack(p, w_curs, 0, NULL); hdr = (struct pack_header*)pack_base; if (hdr->hdr_signature != htonl(PACK_SIGNATURE)) return error("Packfile %s signature mismatch", p->pack_name); @@ -72,13 +73,14 @@ static int verify_packfile(struct packed_git *p) #define MAX_CHAIN 40 -static void show_pack_info(struct packed_git *p) +static void show_pack_info(struct packed_git *p, + struct pack_window **w_curs) { struct pack_header *hdr; int nr_objects, i; unsigned int chain_histogram[MAX_CHAIN]; - hdr = (struct pack_header*)p->windows->base; + hdr = (struct pack_header*)use_pack(p, w_curs, 0, NULL); nr_objects = ntohl(hdr->hdr_entries); memset(chain_histogram, 0, sizeof(chain_histogram)); @@ -142,18 +144,18 @@ int verify_pack(struct packed_git *p, int verbose) if (!ret) { /* Verify pack file */ - use_packed_git(p); - ret = verify_packfile(p); - unuse_packed_git(p); + struct pack_window *w_curs = NULL; + ret = verify_packfile(p, &w_curs); + unuse_pack(&w_curs); } if (verbose) { if (ret) printf("%s: bad\n", p->pack_name); else { - use_packed_git(p); - show_pack_info(p); - unuse_packed_git(p); + struct pack_window *w_curs = NULL; + show_pack_info(p, &w_curs); + unuse_pack(&w_curs); printf("%s: ok\n", p->pack_name); } } diff --git a/sha1_file.c b/sha1_file.c index 79d2d9dacd..4d80527baf 100644 --- a/sha1_file.c +++ b/sha1_file.c @@ -470,10 +470,13 @@ static int unuse_one_packed_git(void) return 1; } -void unuse_packed_git(struct packed_git *p) +void unuse_pack(struct pack_window **w_cursor) { - if (p->windows) - p->windows->inuse_cnt--; + struct pack_window *w = *w_cursor; + if (w) { + w->inuse_cnt--; + *w_cursor = NULL; + } } static void open_packed_git(struct packed_git *p) @@ -518,13 +521,16 @@ static void open_packed_git(struct packed_git *p) die("packfile %s does not match index", p->pack_name); } -int use_packed_git(struct packed_git *p) +unsigned char* use_pack(struct packed_git *p, + struct pack_window **w_cursor, + unsigned long offset, + unsigned int *left) { + struct pack_window *win = p->windows; + if (p->pack_fd == -1) open_packed_git(p); - if (!p->windows) { - struct pack_window *win; - + if (!win) { pack_mapped += p->pack_size; while (packed_git_limit < pack_mapped && unuse_one_packed_git()) ; /* nothing */ @@ -535,9 +541,14 @@ int use_packed_git(struct packed_git *p) die("packfile %s cannot be mapped.", p->pack_name); p->windows = win; } - p->windows->last_used = pack_used_ctr++; - p->windows->inuse_cnt++; - return 0; + if (win != *w_cursor) { + win->last_used = pack_used_ctr++; + win->inuse_cnt++; + *w_cursor = win; + } + if (left) + *left = win->len - offset; + return win->base + offset; } struct packed_git *add_packed_git(char *path, int path_len, int local) @@ -883,12 +894,13 @@ void * unpack_sha1_file(void *map, unsigned long mapsize, char *type, unsigned l } static unsigned long get_delta_base(struct packed_git *p, + struct pack_window **w_curs, unsigned long offset, enum object_type kind, unsigned long delta_obj_offset, unsigned long *base_obj_offset) { - unsigned char *base_info = p->windows->base + offset; + unsigned char *base_info = use_pack(p, w_curs, offset, NULL); unsigned long base_offset; /* there must be at least 20 bytes left regardless of delta type */ @@ -928,6 +940,7 @@ static int packed_object_info(struct packed_git *p, unsigned long offset, char *type, unsigned long *sizep); static int packed_delta_info(struct packed_git *p, + struct pack_window **w_curs, unsigned long offset, enum object_type kind, unsigned long obj_offset, @@ -936,7 +949,8 @@ static int packed_delta_info(struct packed_git *p, { unsigned long base_offset; - offset = get_delta_base(p, offset, kind, obj_offset, &base_offset); + offset = get_delta_base(p, w_curs, offset, kind, + obj_offset, &base_offset); /* We choose to only get the type of the base object and * ignore potentially corrupt pack file that expects the delta @@ -955,8 +969,7 @@ static int packed_delta_info(struct packed_git *p, memset(&stream, 0, sizeof(stream)); - stream.next_in = p->windows->base + offset; - stream.avail_in = p->pack_size - offset; + stream.next_in = use_pack(p, w_curs, offset, &stream.avail_in); stream.next_out = delta_head; stream.avail_out = sizeof(delta_head); @@ -982,16 +995,18 @@ static int packed_delta_info(struct packed_git *p, return 0; } -static unsigned long unpack_object_header(struct packed_git *p, unsigned long offset, - enum object_type *type, unsigned long *sizep) +static unsigned long unpack_object_header(struct packed_git *p, + struct pack_window **w_curs, + unsigned long offset, + enum object_type *type, + unsigned long *sizep) { + unsigned char *base; + unsigned int left; unsigned long used; - if (p->pack_size <= offset) - die("object offset outside of pack file"); - - used = unpack_object_header_gently(p->windows->base + offset, - p->pack_size - offset, type, sizep); + base = use_pack(p, w_curs, offset, &left); + used = unpack_object_header_gently(base, left, type, sizep); if (!used) die("object offset outside of pack file"); @@ -1006,13 +1021,14 @@ void packed_object_info_detail(struct packed_git *p, unsigned int *delta_chain_length, unsigned char *base_sha1) { + struct pack_window *w_curs = NULL; unsigned long obj_offset, val; unsigned char *next_sha1; enum object_type kind; *delta_chain_length = 0; obj_offset = offset; - offset = unpack_object_header(p, offset, &kind, size); + offset = unpack_object_header(p, &w_curs, offset, &kind, size); for (;;) { switch (kind) { @@ -1025,25 +1041,24 @@ void packed_object_info_detail(struct packed_git *p, case OBJ_TAG: strcpy(type, type_names[kind]); *store_size = 0; /* notyet */ + unuse_pack(&w_curs); return; case OBJ_OFS_DELTA: - get_delta_base(p, offset, kind, obj_offset, &offset); + get_delta_base(p, &w_curs, offset, kind, + obj_offset, &offset); if (*delta_chain_length == 0) { /* TODO: find base_sha1 as pointed by offset */ } break; case OBJ_REF_DELTA: - if (p->pack_size <= offset + 20) - die("pack file %s records an incomplete delta base", - p->pack_name); - next_sha1 = p->windows->base + offset; + next_sha1 = use_pack(p, &w_curs, offset, NULL); if (*delta_chain_length == 0) hashcpy(base_sha1, next_sha1); offset = find_pack_entry_one(next_sha1, p); break; } obj_offset = offset; - offset = unpack_object_header(p, offset, &kind, &val); + offset = unpack_object_header(p, &w_curs, offset, &kind, &val); (*delta_chain_length)++; } } @@ -1051,20 +1066,26 @@ void packed_object_info_detail(struct packed_git *p, static int packed_object_info(struct packed_git *p, unsigned long offset, char *type, unsigned long *sizep) { + struct pack_window *w_curs = NULL; unsigned long size, obj_offset = offset; enum object_type kind; + int r; - offset = unpack_object_header(p, offset, &kind, &size); + offset = unpack_object_header(p, &w_curs, offset, &kind, &size); switch (kind) { case OBJ_OFS_DELTA: case OBJ_REF_DELTA: - return packed_delta_info(p, offset, kind, obj_offset, type, sizep); + r = packed_delta_info(p, &w_curs, offset, kind, + obj_offset, type, sizep); + unuse_pack(&w_curs); + return r; case OBJ_COMMIT: case OBJ_TREE: case OBJ_BLOB: case OBJ_TAG: strcpy(type, type_names[kind]); + unuse_pack(&w_curs); break; default: die("pack %s contains unknown object type %d", @@ -1076,6 +1097,7 @@ static int packed_object_info(struct packed_git *p, unsigned long offset, } static void *unpack_compressed_entry(struct packed_git *p, + struct pack_window **w_curs, unsigned long offset, unsigned long size) { @@ -1086,8 +1108,7 @@ static void *unpack_compressed_entry(struct packed_git *p, buffer = xmalloc(size + 1); buffer[size] = 0; memset(&stream, 0, sizeof(stream)); - stream.next_in = p->windows->base + offset; - stream.avail_in = p->pack_size - offset; + stream.next_in = use_pack(p, w_curs, offset, &stream.avail_in); stream.next_out = buffer; stream.avail_out = size; @@ -1103,6 +1124,7 @@ static void *unpack_compressed_entry(struct packed_git *p, } static void *unpack_delta_entry(struct packed_git *p, + struct pack_window **w_curs, unsigned long offset, unsigned long delta_size, enum object_type kind, @@ -1113,13 +1135,14 @@ static void *unpack_delta_entry(struct packed_git *p, void *delta_data, *result, *base; unsigned long result_size, base_size, base_offset; - offset = get_delta_base(p, offset, kind, obj_offset, &base_offset); + offset = get_delta_base(p, w_curs, offset, kind, + obj_offset, &base_offset); base = unpack_entry(p, base_offset, type, &base_size); if (!base) die("failed to read delta base object at %lu from %s", base_offset, p->pack_name); - delta_data = unpack_compressed_entry(p, offset, delta_size); + delta_data = unpack_compressed_entry(p, w_curs, offset, delta_size); result = patch_delta(base, base_size, delta_data, delta_size, &result_size); @@ -1134,17 +1157,17 @@ static void *unpack_delta_entry(struct packed_git *p, void *unpack_entry(struct packed_git *p, unsigned long offset, char *type, unsigned long *sizep) { + struct pack_window *w_curs = NULL; unsigned long size, obj_offset = offset; enum object_type kind; void *retval; - if (use_packed_git(p)) - die("cannot map packed file"); - offset = unpack_object_header(p, offset, &kind, &size); + offset = unpack_object_header(p, &w_curs, offset, &kind, &size); switch (kind) { case OBJ_OFS_DELTA: case OBJ_REF_DELTA: - retval = unpack_delta_entry(p, offset, size, kind, obj_offset, type, sizep); + retval = unpack_delta_entry(p, &w_curs, offset, size, + kind, obj_offset, type, sizep); break; case OBJ_COMMIT: case OBJ_TREE: @@ -1152,12 +1175,12 @@ void *unpack_entry(struct packed_git *p, unsigned long offset, case OBJ_TAG: strcpy(type, type_names[kind]); *sizep = size; - retval = unpack_compressed_entry(p, offset, size); + retval = unpack_compressed_entry(p, &w_curs, offset, size); break; default: die("unknown object type %i in %s", kind, p->pack_name); } - unuse_packed_git(p); + unuse_pack(&w_curs); return retval; } @@ -1284,7 +1307,6 @@ static int sha1_loose_object_info(const unsigned char *sha1, char *type, unsigne int sha1_object_info(const unsigned char *sha1, char *type, unsigned long *sizep) { - int status; struct pack_entry e; if (!find_pack_entry(sha1, &e, NULL)) { @@ -1292,11 +1314,7 @@ int sha1_object_info(const unsigned char *sha1, char *type, unsigned long *sizep if (!find_pack_entry(sha1, &e, NULL)) return sha1_loose_object_info(sha1, type, sizep); } - if (use_packed_git(e.p)) - die("cannot map packed file"); - status = packed_object_info(e.p, e.offset, type, sizep); - unuse_packed_git(e.p); - return status; + return packed_object_info(e.p, e.offset, type, sizep); } static void *read_packed_sha1(const unsigned char *sha1, char *type, unsigned long *size) -- cgit v1.3 From 60bb8b1453e2f93d13e7bb44e8e46c085d2dd752 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Sat, 23 Dec 2006 02:34:28 -0500 Subject: Fully activate the sliding window pack access. This finally turns on the sliding window behavior for packfile data access by mapping limited size windows and chaining them under the packed_git->windows list. We consider a given byte offset to be within the window only if there would be at least 20 bytes (one hash worth of data) accessible after the requested offset. This range selection relates to the contract that use_pack() makes with its callers, allowing them to access one hash or one object header without needing to call use_pack() for every byte of data obtained. In the worst case scenario we will map the same page of data twice into memory: once at the end of one window and once again at the start of the next window. This duplicate page mapping will happen only when an object header or a delta base reference is spanned over the end of a window and is always limited to just one page of duplication, as no sane operating system will ever have a page size smaller than a hash. I am assuming that the possible wasted page of virtual address space is going to perform faster than the alternatives, which would be to copy the object header or ref delta into a temporary buffer prior to parsing, or to check the window range on every byte during header parsing. We may decide to revisit this decision in the future since this is just a gut instinct decision and has not actually been proven out by experimental testing. Signed-off-by: Shawn O. Pearce Signed-off-by: Junio C Hamano --- Documentation/config.txt | 11 ++++++++ cache.h | 1 + config.c | 10 ++++++++ environment.c | 1 + sha1_file.c | 66 ++++++++++++++++++++++++++++++++++++++---------- 5 files changed, 76 insertions(+), 13 deletions(-) (limited to 'cache.h') diff --git a/Documentation/config.txt b/Documentation/config.txt index 28fe6942cf..d71653dc65 100644 --- a/Documentation/config.txt +++ b/Documentation/config.txt @@ -118,6 +118,17 @@ core.legacyheaders:: database directly (where the "http://" and "rsync://" protocols count as direct access). +core.packedGitWindowSize:: + Number of bytes of a pack file to map into memory in a + single mapping operation. Larger window sizes may allow + your system to process a smaller number of large pack files + more quickly. Smaller window sizes will negatively affect + performance due to increased calls to the opreating system's + memory manager, but may improve performance when accessing + a large number of large pack files. Default is 32 MiB, + which should be reasonable for all users/operating systems. + You probably do not need to adjust this value. + core.packedGitLimit:: Maximum number of bytes to map simultaneously into memory from pack files. If Git needs to access more than this many diff --git a/cache.h b/cache.h index c927f29d31..c7d7e4dcc0 100644 --- a/cache.h +++ b/cache.h @@ -196,6 +196,7 @@ extern int warn_ambiguous_refs; extern int shared_repository; extern const char *apply_default_whitespace; extern int zlib_compression_level; +extern size_t packed_git_window_size; extern size_t packed_git_limit; #define GIT_REPO_VERSION 0 diff --git a/config.c b/config.c index 0c21286cb2..21f166e608 100644 --- a/config.c +++ b/config.c @@ -298,6 +298,16 @@ int git_default_config(const char *var, const char *value) return 0; } + if (!strcmp(var, "core.packedgitwindowsize")) { + int pgsz = getpagesize(); + packed_git_window_size = git_config_int(var, value); + packed_git_window_size /= pgsz; + if (!packed_git_window_size) + packed_git_window_size = 1; + packed_git_window_size *= pgsz; + return 0; + } + if (!strcmp(var, "core.packedgitlimit")) { packed_git_limit = git_config_int(var, value); return 0; diff --git a/environment.c b/environment.c index a3ddae68a7..e559fd69c7 100644 --- a/environment.c +++ b/environment.c @@ -23,6 +23,7 @@ char *git_log_output_encoding; int shared_repository = PERM_UMASK; const char *apply_default_whitespace; int zlib_compression_level = Z_DEFAULT_COMPRESSION; +size_t packed_git_window_size = 32 * 1024 * 1024; size_t packed_git_limit = 256 * 1024 * 1024; int pager_in_use; int pager_use_color = 1; diff --git a/sha1_file.c b/sha1_file.c index 8e14a5a882..548535c844 100644 --- a/sha1_file.c +++ b/sha1_file.c @@ -397,8 +397,9 @@ static char *find_sha1_file(const unsigned char *sha1, struct stat *st) return NULL; } -static int pack_used_ctr; -static unsigned long pack_mapped; +static unsigned int pack_used_ctr; +static size_t pack_mapped; +static size_t page_size; struct packed_git *packed_git; static int check_packed_git_idx(const char *path, unsigned long *idx_size_, @@ -536,31 +537,70 @@ static void open_packed_git(struct packed_git *p) die("packfile %s does not match index", p->pack_name); } +static int in_window(struct pack_window *win, unsigned long offset) +{ + /* We must promise at least 20 bytes (one hash) after the + * offset is available from this window, otherwise the offset + * is not actually in this window and a different window (which + * has that one hash excess) must be used. This is to support + * the object header and delta base parsing routines below. + */ + off_t win_off = win->offset; + return win_off <= offset + && (offset + 20) <= (win_off + win->len); +} + unsigned char* use_pack(struct packed_git *p, struct pack_window **w_cursor, unsigned long offset, unsigned int *left) { - struct pack_window *win = p->windows; + struct pack_window *win = *w_cursor; if (p->pack_fd == -1) open_packed_git(p); - if (!win) { - pack_mapped += p->pack_size; - while (packed_git_limit < pack_mapped && unuse_one_window()) - ; /* nothing */ - win = xcalloc(1, sizeof(*win)); - win->len = p->pack_size; - win->base = mmap(NULL, p->pack_size, PROT_READ, MAP_PRIVATE, p->pack_fd, 0); - if (win->base == MAP_FAILED) - die("packfile %s cannot be mapped.", p->pack_name); - p->windows = win; + + /* Since packfiles end in a hash of their content and its + * pointless to ask for an offset into the middle of that + * hash, and the in_window function above wouldn't match + * don't allow an offset too close to the end of the file. + */ + if (offset > (p->pack_size - 20)) + die("offset beyond end of packfile (truncated pack?)"); + + if (!win || !in_window(win, offset)) { + if (win) + win->inuse_cnt--; + for (win = p->windows; win; win = win->next) { + if (in_window(win, offset)) + break; + } + if (!win) { + if (!page_size) + page_size = getpagesize(); + win = xcalloc(1, sizeof(*win)); + win->offset = (offset / page_size) * page_size; + win->len = p->pack_size - win->offset; + if (win->len > packed_git_window_size) + win->len = packed_git_window_size; + pack_mapped += win->len; + while (packed_git_limit < pack_mapped && unuse_one_window()) + ; /* nothing */ + win->base = mmap(NULL, win->len, + PROT_READ, MAP_PRIVATE, + p->pack_fd, win->offset); + if (win->base == MAP_FAILED) + die("packfile %s cannot be mapped.", p->pack_name); + win->next = p->windows; + p->windows = win; + } } if (win != *w_cursor) { win->last_used = pack_used_ctr++; win->inuse_cnt++; *w_cursor = win; } + offset -= win->offset; if (left) *left = win->len - offset; return win->base + offset; -- cgit v1.3 From a53128b60162d7a84adca4206540df5b8e3d9dc8 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Sat, 23 Dec 2006 02:34:47 -0500 Subject: Create pack_report() as a debugging aid. Much like the alloc_report() function can be useful to report on object allocation statistics while debugging the new pack_report() function can be useful to report on the behavior of the mmap window code used for packfile access. Signed-off-by: Shawn O. Pearce Signed-off-by: Junio C Hamano --- cache.h | 1 + sha1_file.c | 31 +++++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+) (limited to 'cache.h') diff --git a/cache.h b/cache.h index c7d7e4dcc0..a5fc23235e 100644 --- a/cache.h +++ b/cache.h @@ -398,6 +398,7 @@ extern void install_packed_git(struct packed_git *pack); extern struct packed_git *find_sha1_pack(const unsigned char *sha1, struct packed_git *packs); +extern void pack_report(); extern unsigned char* use_pack(struct packed_git *, struct pack_window **, unsigned long, unsigned int *); extern void unuse_pack(struct pack_window **); extern struct packed_git *add_packed_git(char *, int, int); diff --git a/sha1_file.c b/sha1_file.c index 01a2f8779e..8de8ce0a72 100644 --- a/sha1_file.c +++ b/sha1_file.c @@ -398,10 +398,34 @@ static char *find_sha1_file(const unsigned char *sha1, struct stat *st) } static unsigned int pack_used_ctr; +static unsigned int pack_mmap_calls; +static unsigned int peak_pack_open_windows; +static unsigned int pack_open_windows; +static size_t peak_pack_mapped; static size_t pack_mapped; static size_t page_size; struct packed_git *packed_git; +void pack_report() +{ + fprintf(stderr, + "pack_report: getpagesize() = %10lu\n" + "pack_report: core.packedGitWindowSize = %10lu\n" + "pack_report: core.packedGitLimit = %10lu\n", + page_size, + packed_git_window_size, + packed_git_limit); + fprintf(stderr, + "pack_report: pack_used_ctr = %10u\n" + "pack_report: pack_mmap_calls = %10u\n" + "pack_report: pack_open_windows = %10u / %10u\n" + "pack_report: pack_mapped = %10lu / %10lu\n", + pack_used_ctr, + pack_mmap_calls, + pack_open_windows, peak_pack_open_windows, + pack_mapped, peak_pack_mapped); +} + static int check_packed_git_idx(const char *path, unsigned long *idx_size_, void **idx_map_) { @@ -492,6 +516,7 @@ static int unuse_one_window(struct packed_git *current) } } free(lru_w); + pack_open_windows--; return 1; } return 0; @@ -605,6 +630,12 @@ unsigned char* use_pack(struct packed_git *p, die("packfile %s cannot be mapped: %s", p->pack_name, strerror(errno)); + pack_mmap_calls++; + pack_open_windows++; + if (pack_mapped > peak_pack_mapped) + peak_pack_mapped = pack_mapped; + if (pack_open_windows > peak_pack_open_windows) + peak_pack_open_windows = pack_open_windows; win->next = p->windows; p->windows = win; } -- cgit v1.3