From e42797f5b6d5ed3b8c894d89493e285c40d58dc8 Mon Sep 17 00:00:00 2001 From: Nicolas Pitre Date: Mon, 23 Oct 2006 14:50:18 -0400 Subject: enable index-pack streaming capability A new flag, --stdin, allows for a pack to be received over a stream. When this flag is provided, the pack content is written to either the named pack file or directly to the object repository under the same name as produced by git-repack. The pack index is written as well with the corresponding base name, unless the index name is overriden with -o. With this patch, git-index-pack could be used instead of git-unpack-objects when fetching remote objects but only with non "thin" packs for now. Signed-off-by: Nicolas Pitre Signed-off-by: Junio C Hamano --- Documentation/git-index-pack.txt | 8 ++- index-pack.c | 133 ++++++++++++++++++++++++++++++--------- 2 files changed, 111 insertions(+), 30 deletions(-) diff --git a/Documentation/git-index-pack.txt b/Documentation/git-index-pack.txt index 71ce557276..db7af5859e 100644 --- a/Documentation/git-index-pack.txt +++ b/Documentation/git-index-pack.txt @@ -8,7 +8,7 @@ git-index-pack - Build pack index file for an existing packed archive SYNOPSIS -------- -'git-index-pack' [-o ] +'git-index-pack' [-o ] { | --stdin [] } DESCRIPTION @@ -29,6 +29,12 @@ OPTIONS fails if the name of packed archive does not end with .pack). +--stdin:: + When this flag is provided, the pack is read from stdin + instead and a copy is then written to . If + is not specified, the pack is written to + objects/pack/ directory of the current git repository with + a default name determined from the pack content. Author ------ diff --git a/index-pack.c b/index-pack.c index e33f60524f..cecdd2680c 100644 --- a/index-pack.c +++ b/index-pack.c @@ -8,7 +8,7 @@ #include "tree.h" static const char index_pack_usage[] = -"git-index-pack [-o index-file] pack-file"; +"git-index-pack [-o ] { | --stdin [] }"; struct object_entry { @@ -37,17 +37,18 @@ struct delta_entry union delta_base base; }; -static const char *pack_name; static struct object_entry *objects; static struct delta_entry *deltas; static int nr_objects; static int nr_deltas; +static int from_stdin; + /* We always read in 4kB chunks. */ static unsigned char input_buffer[4096]; static unsigned long input_offset, input_len, consumed_bytes; static SHA_CTX input_ctx; -static int input_fd; +static int input_fd, output_fd, mmap_fd; /* * Make sure at least "min" bytes are available in the buffer, and @@ -60,6 +61,8 @@ static void * fill(int min) if (min > sizeof(input_buffer)) die("cannot fill %d bytes", min); if (input_offset) { + if (output_fd >= 0) + write_or_die(output_fd, input_buffer, input_offset); SHA1_Update(&input_ctx, input_buffer, input_offset); memcpy(input_buffer, input_buffer + input_offset, input_len); input_offset = 0; @@ -86,13 +89,31 @@ static void use(int bytes) consumed_bytes += bytes; } -static void open_pack_file(void) +static const char * open_pack_file(const char *pack_name) { - input_fd = open(pack_name, O_RDONLY); - if (input_fd < 0) - die("cannot open packfile '%s': %s", pack_name, - strerror(errno)); + if (from_stdin) { + input_fd = 0; + if (!pack_name) { + static char tmpfile[PATH_MAX]; + snprintf(tmpfile, sizeof(tmpfile), + "%s/pack_XXXXXX", get_object_directory()); + output_fd = mkstemp(tmpfile); + pack_name = xstrdup(tmpfile); + } else + output_fd = open(pack_name, O_CREAT|O_EXCL|O_RDWR, 0600); + if (output_fd < 0) + die("unable to create %s: %s\n", pack_name, strerror(errno)); + mmap_fd = output_fd; + } else { + input_fd = open(pack_name, O_RDONLY); + if (input_fd < 0) + die("cannot open packfile '%s': %s", + pack_name, strerror(errno)); + output_fd = -1; + mmap_fd = input_fd; + } SHA1_Init(&input_ctx); + return pack_name; } static void parse_pack_header(void) @@ -101,10 +122,9 @@ static void parse_pack_header(void) /* Header consistency check */ if (hdr->hdr_signature != htonl(PACK_SIGNATURE)) - die("packfile '%s' signature mismatch", pack_name); + die("pack signature mismatch"); if (!pack_version_ok(hdr->hdr_version)) - die("packfile '%s' version %d unsupported", - pack_name, ntohl(hdr->hdr_version)); + die("pack version %d unsupported", ntohl(hdr->hdr_version)); nr_objects = ntohl(hdr->hdr_entries); use(sizeof(struct pack_header)); @@ -122,8 +142,7 @@ static void bad_object(unsigned long offset, const char *format, ...) va_start(params, format); vsnprintf(buf, sizeof(buf), format, params); va_end(params); - die("packfile '%s': bad object at offset %lu: %s", - pack_name, offset, buf); + die("pack has bad object at offset %lu: %s", offset, buf); } static void *unpack_entry_data(unsigned long offset, unsigned long size) @@ -222,9 +241,9 @@ static void * get_data_from_pack(struct object_entry *obj) int st; map = mmap(NULL, len + pg_offset, PROT_READ, MAP_PRIVATE, - input_fd, from - pg_offset); + mmap_fd, from - pg_offset); if (map == MAP_FAILED) - die("cannot mmap packfile '%s': %s", pack_name, strerror(errno)); + die("cannot mmap pack file: %s", strerror(errno)); data = xmalloc(obj->size); memset(&stream, 0, sizeof(stream)); stream.next_out = data; @@ -382,14 +401,16 @@ static void parse_pack_objects(unsigned char *sha1) SHA1_Update(&input_ctx, input_buffer, input_offset); SHA1_Final(sha1, &input_ctx); if (hashcmp(fill(20), sha1)) - die("packfile '%s' SHA1 mismatch", pack_name); + die("pack is corrupted (SHA1 mismatch)"); use(20); + if (output_fd >= 0) + write_or_die(output_fd, input_buffer, input_offset); /* If input_fd is a file, we should have reached its end now. */ if (fstat(input_fd, &st)) - die("cannot fstat packfile '%s': %s", pack_name, strerror(errno)); + die("cannot fstat packfile: %s", strerror(errno)); if (S_ISREG(st.st_mode) && st.st_size != consumed_bytes) - die("packfile '%s' has junk at the end", pack_name); + die("pack has junk at the end"); /* Sort deltas by base SHA1/offset for fast searching */ qsort(deltas, nr_deltas, sizeof(struct delta_entry), @@ -435,7 +456,7 @@ static void parse_pack_objects(unsigned char *sha1) for (i = 0; i < nr_deltas; i++) { if (deltas[i].obj->real_type == OBJ_REF_DELTA || deltas[i].obj->real_type == OBJ_OFS_DELTA) - die("packfile '%s' has unresolved deltas", pack_name); + die("pack has unresolved deltas"); } } @@ -450,12 +471,12 @@ static int sha1_compare(const void *_a, const void *_b) * On entry *sha1 contains the pack content SHA1 hash, on exit it is * the SHA1 hash of sorted object names. */ -static void write_index_file(const char *index_name, unsigned char *sha1) +static const char * write_index_file(const char *index_name, unsigned char *sha1) { struct sha1file *f; struct object_entry **sorted_by_sha, **list, **last; unsigned int array[256]; - int i; + int i, fd; SHA_CTX ctx; if (nr_objects) { @@ -472,8 +493,19 @@ static void write_index_file(const char *index_name, unsigned char *sha1) else sorted_by_sha = list = last = NULL; - unlink(index_name); - f = sha1create("%s", index_name); + if (!index_name) { + static char tmpfile[PATH_MAX]; + snprintf(tmpfile, sizeof(tmpfile), + "%s/index_XXXXXX", get_object_directory()); + fd = mkstemp(tmpfile); + index_name = xstrdup(tmpfile); + } else { + unlink(index_name); + fd = open(index_name, O_CREAT|O_EXCL|O_WRONLY, 0600); + } + if (fd < 0) + die("unable to create %s: %s", index_name, strerror(errno)); + f = sha1fd(fd, index_name); /* * Write the first-level table (the list is sorted, @@ -513,12 +545,52 @@ static void write_index_file(const char *index_name, unsigned char *sha1) sha1close(f, NULL, 1); free(sorted_by_sha); SHA1_Final(sha1, &ctx); + return index_name; +} + +static void final(const char *final_pack_name, const char *curr_pack_name, + const char *final_index_name, const char *curr_index_name, + unsigned char *sha1) +{ + char name[PATH_MAX]; + int err; + + if (!from_stdin) { + close(input_fd); + } else { + err = close(output_fd); + if (err) + die("error while closing pack file: %s", strerror(errno)); + chmod(curr_pack_name, 0444); + } + + if (final_pack_name != curr_pack_name) { + if (!final_pack_name) { + snprintf(name, sizeof(name), "%s/pack/pack-%s.pack", + get_object_directory(), sha1_to_hex(sha1)); + final_pack_name = name; + } + if (move_temp_to_file(curr_pack_name, final_pack_name)) + die("cannot store pack file"); + } + + chmod(curr_index_name, 0444); + if (final_index_name != curr_index_name) { + if (!final_index_name) { + snprintf(name, sizeof(name), "%s/pack/pack-%s.idx", + get_object_directory(), sha1_to_hex(sha1)); + final_index_name = name; + } + if (move_temp_to_file(curr_index_name, final_index_name)) + die("cannot store index file"); + } } int main(int argc, char **argv) { int i; - char *index_name = NULL; + const char *curr_pack, *pack_name = NULL; + const char *curr_index, *index_name = NULL; char *index_name_buf = NULL; unsigned char sha1[20]; @@ -526,7 +598,9 @@ int main(int argc, char **argv) const char *arg = argv[i]; if (*arg == '-') { - if (!strcmp(arg, "-o")) { + if (!strcmp(arg, "--stdin")) { + from_stdin = 1; + } else if (!strcmp(arg, "-o")) { if (index_name || (i+1) >= argc) usage(index_pack_usage); index_name = argv[++i]; @@ -540,9 +614,9 @@ int main(int argc, char **argv) pack_name = arg; } - if (!pack_name) + if (!pack_name && !from_stdin) usage(index_pack_usage); - if (!index_name) { + if (!index_name && pack_name) { int len = strlen(pack_name); if (!has_extension(pack_name, ".pack")) die("packfile name '%s' does not end with '.pack'", @@ -553,13 +627,14 @@ int main(int argc, char **argv) index_name = index_name_buf; } - open_pack_file(); + curr_pack = open_pack_file(pack_name); parse_pack_header(); objects = xcalloc(nr_objects + 1, sizeof(struct object_entry)); deltas = xcalloc(nr_objects, sizeof(struct delta_entry)); parse_pack_objects(sha1); free(deltas); - write_index_file(index_name, sha1); + curr_index = write_index_file(index_name, sha1); + final(pack_name, curr_pack, index_name, curr_index, sha1); free(objects); free(index_name_buf); -- cgit v1.3 From 636171cb80255682bdfc9bf5a98c9e66d4c0444a Mon Sep 17 00:00:00 2001 From: Nicolas Pitre Date: Wed, 25 Oct 2006 23:28:17 -0400 Subject: make index-pack able to complete thin packs. A new flag, --fix-thin, instructs git-index-pack to append any missing objects to a thin pack to make it self contained and indexable. Of course objects missing from the pack must be present elsewhere in the local repository. Signed-off-by: Nicolas Pitre Signed-off-by: Junio C Hamano --- Documentation/git-index-pack.txt | 14 ++- index-pack.c | 254 ++++++++++++++++++++++++++++++++------- 2 files changed, 224 insertions(+), 44 deletions(-) diff --git a/Documentation/git-index-pack.txt b/Documentation/git-index-pack.txt index db7af5859e..c58287d682 100644 --- a/Documentation/git-index-pack.txt +++ b/Documentation/git-index-pack.txt @@ -8,7 +8,8 @@ git-index-pack - Build pack index file for an existing packed archive SYNOPSIS -------- -'git-index-pack' [-o ] { | --stdin [] } +'git-index-pack' [-o ] +'git-index-pack' --stdin [--fix-thin] [-o ] [] DESCRIPTION @@ -36,6 +37,17 @@ OPTIONS objects/pack/ directory of the current git repository with a default name determined from the pack content. +--fix-thin:: + It is possible for gitlink:git-pack-objects[1] to build + "thin" pack, which records objects in deltified form based on + objects not included in the pack to reduce network traffic. + Those objects are expected to be present on the receiving end + and they must be included in the pack for that pack to be self + contained and indexable. Without this option any attempt to + index a thin pack will fail. This option only makes sense in + conjonction with --stdin. + + Author ------ Written by Sergey Vlasov diff --git a/index-pack.c b/index-pack.c index cecdd2680c..9086bbf154 100644 --- a/index-pack.c +++ b/index-pack.c @@ -8,7 +8,7 @@ #include "tree.h" static const char index_pack_usage[] = -"git-index-pack [-o ] { | --stdin [] }"; +"git-index-pack [-o ] { | --stdin [--fix-thin] [] }"; struct object_entry { @@ -33,14 +33,15 @@ union delta_base { struct delta_entry { - struct object_entry *obj; union delta_base base; + int obj_no; }; static struct object_entry *objects; static struct delta_entry *deltas; static int nr_objects; static int nr_deltas; +static int nr_resolved_deltas; static int from_stdin; @@ -50,6 +51,18 @@ static unsigned long input_offset, input_len, consumed_bytes; static SHA_CTX input_ctx; static int input_fd, output_fd, mmap_fd; +/* Discard current buffer used content. */ +static void flush() +{ + if (input_offset) { + if (output_fd >= 0) + write_or_die(output_fd, input_buffer, input_offset); + SHA1_Update(&input_ctx, input_buffer, input_offset); + memcpy(input_buffer, input_buffer + input_offset, input_len); + input_offset = 0; + } +} + /* * Make sure at least "min" bytes are available in the buffer, and * return the pointer to the buffer. @@ -60,13 +73,7 @@ static void * fill(int min) return input_buffer + input_offset; if (min > sizeof(input_buffer)) die("cannot fill %d bytes", min); - if (input_offset) { - if (output_fd >= 0) - write_or_die(output_fd, input_buffer, input_offset); - SHA1_Update(&input_ctx, input_buffer, input_offset); - memcpy(input_buffer, input_buffer + input_offset, input_len); - input_offset = 0; - } + flush(); do { int ret = xread(input_fd, input_buffer + input_len, sizeof(input_buffer) - input_len); @@ -323,10 +330,9 @@ static void sha1_object(const void *data, unsigned long size, SHA1_Final(sha1, &ctx); } -static void resolve_delta(struct delta_entry *delta, void *base_data, +static void resolve_delta(struct object_entry *delta_obj, void *base_data, unsigned long base_size, enum object_type type) { - struct object_entry *obj = delta->obj; void *delta_data; unsigned long delta_size; void *result; @@ -334,29 +340,34 @@ static void resolve_delta(struct delta_entry *delta, void *base_data, union delta_base delta_base; int j, first, last; - obj->real_type = type; - delta_data = get_data_from_pack(obj); - delta_size = obj->size; + delta_obj->real_type = type; + delta_data = get_data_from_pack(delta_obj); + delta_size = delta_obj->size; result = patch_delta(base_data, base_size, delta_data, delta_size, &result_size); free(delta_data); if (!result) - bad_object(obj->offset, "failed to apply delta"); - sha1_object(result, result_size, type, obj->sha1); + bad_object(delta_obj->offset, "failed to apply delta"); + sha1_object(result, result_size, type, delta_obj->sha1); + nr_resolved_deltas++; - hashcpy(delta_base.sha1, obj->sha1); + hashcpy(delta_base.sha1, delta_obj->sha1); if (!find_delta_childs(&delta_base, &first, &last)) { - for (j = first; j <= last; j++) - if (deltas[j].obj->type == OBJ_REF_DELTA) - resolve_delta(&deltas[j], result, result_size, type); + for (j = first; j <= last; j++) { + struct object_entry *child = objects + deltas[j].obj_no; + if (child->real_type == OBJ_REF_DELTA) + resolve_delta(child, result, result_size, type); + } } memset(&delta_base, 0, sizeof(delta_base)); - delta_base.offset = obj->offset; + delta_base.offset = delta_obj->offset; if (!find_delta_childs(&delta_base, &first, &last)) { - for (j = first; j <= last; j++) - if (deltas[j].obj->type == OBJ_OFS_DELTA) - resolve_delta(&deltas[j], result, result_size, type); + for (j = first; j <= last; j++) { + struct object_entry *child = objects + deltas[j].obj_no; + if (child->real_type == OBJ_OFS_DELTA) + resolve_delta(child, result, result_size, type); + } } free(result); @@ -389,7 +400,7 @@ static void parse_pack_objects(unsigned char *sha1) obj->real_type = obj->type; if (obj->type == OBJ_REF_DELTA || obj->type == OBJ_OFS_DELTA) { nr_deltas++; - delta->obj = obj; + delta->obj_no = i; delta++; } else sha1_object(data, obj->size, obj->type, obj->sha1); @@ -398,18 +409,15 @@ static void parse_pack_objects(unsigned char *sha1) objects[i].offset = consumed_bytes; /* Check pack integrity */ - SHA1_Update(&input_ctx, input_buffer, input_offset); + flush(); SHA1_Final(sha1, &input_ctx); if (hashcmp(fill(20), sha1)) die("pack is corrupted (SHA1 mismatch)"); - use(20); - if (output_fd >= 0) - write_or_die(output_fd, input_buffer, input_offset); /* If input_fd is a file, we should have reached its end now. */ if (fstat(input_fd, &st)) die("cannot fstat packfile: %s", strerror(errno)); - if (S_ISREG(st.st_mode) && st.st_size != consumed_bytes) + if (S_ISREG(st.st_mode) && st.st_size != consumed_bytes + 20) die("pack has junk at the end"); /* Sort deltas by base SHA1/offset for fast searching */ @@ -440,24 +448,161 @@ static void parse_pack_objects(unsigned char *sha1) continue; data = get_data_from_pack(obj); if (ref) - for (j = ref_first; j <= ref_last; j++) - if (deltas[j].obj->type == OBJ_REF_DELTA) - resolve_delta(&deltas[j], data, + for (j = ref_first; j <= ref_last; j++) { + struct object_entry *child = objects + deltas[j].obj_no; + if (child->real_type == OBJ_REF_DELTA) + resolve_delta(child, data, obj->size, obj->type); + } if (ofs) - for (j = ofs_first; j <= ofs_last; j++) - if (deltas[j].obj->type == OBJ_OFS_DELTA) - resolve_delta(&deltas[j], data, + for (j = ofs_first; j <= ofs_last; j++) { + struct object_entry *child = objects + deltas[j].obj_no; + if (child->real_type == OBJ_OFS_DELTA) + resolve_delta(child, data, obj->size, obj->type); + } free(data); } +} + +static int write_compressed(int fd, void *in, unsigned int size) +{ + z_stream stream; + unsigned long maxsize; + void *out; + + memset(&stream, 0, sizeof(stream)); + deflateInit(&stream, zlib_compression_level); + maxsize = deflateBound(&stream, size); + out = xmalloc(maxsize); + + /* Compress it */ + stream.next_in = in; + stream.avail_in = size; + stream.next_out = out; + stream.avail_out = maxsize; + while (deflate(&stream, Z_FINISH) == Z_OK); + deflateEnd(&stream); + + size = stream.total_out; + write_or_die(fd, out, size); + free(out); + return size; +} + +static void append_obj_to_pack(void *buf, + unsigned long size, enum object_type type) +{ + struct object_entry *obj = &objects[nr_objects++]; + unsigned char header[10]; + unsigned long s = size; + int n = 0; + unsigned char c = (type << 4) | (s & 15); + s >>= 4; + while (s) { + header[n++] = c | 0x80; + c = s & 0x7f; + s >>= 7; + } + header[n++] = c; + write_or_die(output_fd, header, n); + obj[1].offset = obj[0].offset + n; + obj[1].offset += write_compressed(output_fd, buf, size); + sha1_object(buf, size, type, obj->sha1); +} + +static int delta_pos_compare(const void *_a, const void *_b) +{ + struct delta_entry *a = *(struct delta_entry **)_a; + struct delta_entry *b = *(struct delta_entry **)_b; + return a->obj_no - b->obj_no; +} - /* Check for unresolved deltas */ +static void fix_unresolved_deltas(int nr_unresolved) +{ + struct delta_entry **sorted_by_pos; + int i, n = 0; + + /* + * Since many unresolved deltas may well be themselves base objects + * for more unresolved deltas, we really want to include the + * smallest number of base objects that would cover as much delta + * as possible by picking the + * trunc deltas first, allowing for other deltas to resolve without + * additional base objects. Since most base objects are to be found + * before deltas depending on them, a good heuristic is to start + * resolving deltas in the same order as their position in the pack. + */ + sorted_by_pos = xmalloc(nr_unresolved * sizeof(*sorted_by_pos)); for (i = 0; i < nr_deltas; i++) { - if (deltas[i].obj->real_type == OBJ_REF_DELTA || - deltas[i].obj->real_type == OBJ_OFS_DELTA) - die("pack has unresolved deltas"); + if (objects[deltas[i].obj_no].real_type != OBJ_REF_DELTA) + continue; + sorted_by_pos[n++] = &deltas[i]; } + qsort(sorted_by_pos, n, sizeof(*sorted_by_pos), delta_pos_compare); + + for (i = 0; i < n; i++) { + struct delta_entry *d = sorted_by_pos[i]; + void *data; + unsigned long size; + char type[10]; + enum object_type obj_type; + int j, first, last; + + if (objects[d->obj_no].real_type != OBJ_REF_DELTA) + continue; + data = read_sha1_file(d->base.sha1, type, &size); + if (!data) + continue; + if (!strcmp(type, blob_type)) obj_type = OBJ_BLOB; + else if (!strcmp(type, tree_type)) obj_type = OBJ_TREE; + else if (!strcmp(type, commit_type)) obj_type = OBJ_COMMIT; + else if (!strcmp(type, tag_type)) obj_type = OBJ_TAG; + else die("base object %s is of type '%s'", + sha1_to_hex(d->base.sha1), type); + + find_delta_childs(&d->base, &first, &last); + for (j = first; j <= last; j++) { + struct object_entry *child = objects + deltas[j].obj_no; + if (child->real_type == OBJ_REF_DELTA) + resolve_delta(child, data, size, obj_type); + } + + append_obj_to_pack(data, size, obj_type); + free(data); + } + free(sorted_by_pos); +} + +static void readjust_pack_header_and_sha1(unsigned char *sha1) +{ + struct pack_header hdr; + SHA_CTX ctx; + int size; + + /* Rewrite pack header with updated object number */ + if (lseek(output_fd, 0, SEEK_SET) != 0) + die("cannot seek back: %s", strerror(errno)); + if (xread(output_fd, &hdr, sizeof(hdr)) != sizeof(hdr)) + die("cannot read pack header back: %s", strerror(errno)); + hdr.hdr_entries = htonl(nr_objects); + if (lseek(output_fd, 0, SEEK_SET) != 0) + die("cannot seek back: %s", strerror(errno)); + write_or_die(output_fd, &hdr, sizeof(hdr)); + if (lseek(output_fd, 0, SEEK_SET) != 0) + die("cannot seek back: %s", strerror(errno)); + + /* Recompute and store the new pack's SHA1 */ + SHA1_Init(&ctx); + do { + unsigned char *buf[4096]; + size = xread(output_fd, buf, sizeof(buf)); + if (size < 0) + die("cannot read pack data back: %s", strerror(errno)); + SHA1_Update(&ctx, buf, size); + } while (size > 0); + SHA1_Final(sha1, &ctx); + write_or_die(output_fd, sha1, 20); } static int sha1_compare(const void *_a, const void *_b) @@ -588,7 +733,7 @@ static void final(const char *final_pack_name, const char *curr_pack_name, int main(int argc, char **argv) { - int i; + int i, fix_thin_pack = 0; const char *curr_pack, *pack_name = NULL; const char *curr_index, *index_name = NULL; char *index_name_buf = NULL; @@ -600,6 +745,8 @@ int main(int argc, char **argv) if (*arg == '-') { if (!strcmp(arg, "--stdin")) { from_stdin = 1; + } else if (!strcmp(arg, "--fix-thin")) { + fix_thin_pack = 1; } else if (!strcmp(arg, "-o")) { if (index_name || (i+1) >= argc) usage(index_pack_usage); @@ -616,6 +763,8 @@ int main(int argc, char **argv) if (!pack_name && !from_stdin) usage(index_pack_usage); + if (fix_thin_pack && !from_stdin) + die("--fix-thin cannot be used without --stdin"); if (!index_name && pack_name) { int len = strlen(pack_name); if (!has_extension(pack_name, ".pack")) @@ -629,9 +778,28 @@ int main(int argc, char **argv) curr_pack = open_pack_file(pack_name); parse_pack_header(); - objects = xcalloc(nr_objects + 1, sizeof(struct object_entry)); - deltas = xcalloc(nr_objects, sizeof(struct delta_entry)); + objects = xmalloc((nr_objects + 1) * sizeof(struct object_entry)); + deltas = xmalloc(nr_objects * sizeof(struct delta_entry)); parse_pack_objects(sha1); + if (nr_deltas != nr_resolved_deltas) { + if (fix_thin_pack) { + int nr_unresolved = nr_deltas - nr_resolved_deltas; + if (nr_unresolved <= 0) + die("confusion beyond insanity"); + objects = xrealloc(objects, + (nr_objects + nr_unresolved + 1) + * sizeof(*objects)); + fix_unresolved_deltas(nr_unresolved); + readjust_pack_header_and_sha1(sha1); + } + if (nr_deltas != nr_resolved_deltas) + die("pack has %d unresolved deltas", + nr_deltas - nr_resolved_deltas); + } else { + /* Flush remaining pack final 20-byte SHA1. */ + use(20); + flush(); + } free(deltas); curr_index = write_index_file(index_name, sha1); final(pack_name, curr_pack, index_name, curr_index, sha1); -- cgit v1.3 From 3c9af366469524920864bbc1af4dcfd72029c314 Mon Sep 17 00:00:00 2001 From: Nicolas Pitre Date: Wed, 25 Oct 2006 23:32:59 -0400 Subject: add progress status to index-pack This is more interesting to look at when performing a big fetch. Signed-off-by: Nicolas Pitre Signed-off-by: Junio C Hamano --- Documentation/git-index-pack.txt | 7 ++-- index-pack.c | 74 +++++++++++++++++++++++++++++++++++++--- 2 files changed, 75 insertions(+), 6 deletions(-) diff --git a/Documentation/git-index-pack.txt b/Documentation/git-index-pack.txt index c58287d682..9fa4847d56 100644 --- a/Documentation/git-index-pack.txt +++ b/Documentation/git-index-pack.txt @@ -8,8 +8,8 @@ git-index-pack - Build pack index file for an existing packed archive SYNOPSIS -------- -'git-index-pack' [-o ] -'git-index-pack' --stdin [--fix-thin] [-o ] [] +'git-index-pack' [-v] [-o ] +'git-index-pack' --stdin [--fix-thin] [-v] [-o ] [] DESCRIPTION @@ -22,6 +22,9 @@ objects/pack/ directory of a git repository. OPTIONS ------- +-v:: + Be verbose about what is going on, including progress status. + -o :: Write the generated pack index into the specified file. Without this option the name of pack index diff --git a/index-pack.c b/index-pack.c index 9086bbf154..2046b37e4b 100644 --- a/index-pack.c +++ b/index-pack.c @@ -6,9 +6,11 @@ #include "commit.h" #include "tag.h" #include "tree.h" +#include +#include static const char index_pack_usage[] = -"git-index-pack [-o ] { | --stdin [--fix-thin] [] }"; +"git-index-pack [-v] [-o ] { | --stdin [--fix-thin] [] }"; struct object_entry { @@ -44,6 +46,42 @@ static int nr_deltas; static int nr_resolved_deltas; static int from_stdin; +static int verbose; + +static volatile sig_atomic_t progress_update; + +static void progress_interval(int signum) +{ + progress_update = 1; +} + +static void setup_progress_signal(void) +{ + struct sigaction sa; + struct itimerval v; + + memset(&sa, 0, sizeof(sa)); + sa.sa_handler = progress_interval; + sigemptyset(&sa.sa_mask); + sa.sa_flags = SA_RESTART; + sigaction(SIGALRM, &sa, NULL); + + v.it_interval.tv_sec = 1; + v.it_interval.tv_usec = 0; + v.it_value = v.it_interval; + setitimer(ITIMER_REAL, &v, NULL); + +} + +static unsigned display_progress(unsigned n, unsigned total, unsigned last_pc) +{ + unsigned percent = n * 100 / total; + if (percent != last_pc || progress_update) { + fprintf(stderr, "%4u%% (%u/%u) done\r", percent, n, total); + progress_update = 0; + } + return percent; +} /* We always read in 4kB chunks. */ static unsigned char input_buffer[4096]; @@ -135,7 +173,6 @@ static void parse_pack_header(void) nr_objects = ntohl(hdr->hdr_entries); use(sizeof(struct pack_header)); - /*fprintf(stderr, "Indexing %d objects\n", nr_objects);*/ } static void bad_object(unsigned long offset, const char *format, @@ -383,7 +420,7 @@ static int compare_delta_entry(const void *a, const void *b) /* Parse all objects and return the pack content SHA1 hash */ static void parse_pack_objects(unsigned char *sha1) { - int i; + int i, percent = -1; struct delta_entry *delta = deltas; void *data; struct stat st; @@ -394,6 +431,8 @@ static void parse_pack_objects(unsigned char *sha1) * - calculate SHA1 of all non-delta objects; * - remember base SHA1 for all deltas. */ + if (verbose) + fprintf(stderr, "Indexing %d objects.\n", nr_objects); for (i = 0; i < nr_objects; i++) { struct object_entry *obj = &objects[i]; data = unpack_raw_entry(obj, &delta->base); @@ -405,8 +444,12 @@ static void parse_pack_objects(unsigned char *sha1) } else sha1_object(data, obj->size, obj->type, obj->sha1); free(data); + if (verbose) + percent = display_progress(i+1, nr_objects, percent); } objects[i].offset = consumed_bytes; + if (verbose) + fputc('\n', stderr); /* Check pack integrity */ flush(); @@ -420,6 +463,9 @@ static void parse_pack_objects(unsigned char *sha1) if (S_ISREG(st.st_mode) && st.st_size != consumed_bytes + 20) die("pack has junk at the end"); + if (!nr_deltas) + return; + /* Sort deltas by base SHA1/offset for fast searching */ qsort(deltas, nr_deltas, sizeof(struct delta_entry), compare_delta_entry); @@ -432,6 +478,8 @@ static void parse_pack_objects(unsigned char *sha1) * recursively checking if the resulting object is used as a base * for some more deltas. */ + if (verbose) + fprintf(stderr, "Resolving %d deltas.\n", nr_deltas); for (i = 0; i < nr_objects; i++) { struct object_entry *obj = &objects[i]; union delta_base base; @@ -462,7 +510,12 @@ static void parse_pack_objects(unsigned char *sha1) obj->size, obj->type); } free(data); + if (verbose) + percent = display_progress(nr_resolved_deltas, + nr_deltas, percent); } + if (verbose && nr_resolved_deltas == nr_deltas) + fputc('\n', stderr); } static int write_compressed(int fd, void *in, unsigned int size) @@ -521,7 +574,7 @@ static int delta_pos_compare(const void *_a, const void *_b) static void fix_unresolved_deltas(int nr_unresolved) { struct delta_entry **sorted_by_pos; - int i, n = 0; + int i, n = 0, percent = -1; /* * Since many unresolved deltas may well be themselves base objects @@ -570,8 +623,13 @@ static void fix_unresolved_deltas(int nr_unresolved) append_obj_to_pack(data, size, obj_type); free(data); + if (verbose) + percent = display_progress(nr_resolved_deltas, + nr_deltas, percent); } free(sorted_by_pos); + if (verbose) + fputc('\n', stderr); } static void readjust_pack_header_and_sha1(unsigned char *sha1) @@ -747,6 +805,8 @@ int main(int argc, char **argv) from_stdin = 1; } else if (!strcmp(arg, "--fix-thin")) { fix_thin_pack = 1; + } else if (!strcmp(arg, "-v")) { + verbose = 1; } else if (!strcmp(arg, "-o")) { if (index_name || (i+1) >= argc) usage(index_pack_usage); @@ -780,16 +840,22 @@ int main(int argc, char **argv) parse_pack_header(); objects = xmalloc((nr_objects + 1) * sizeof(struct object_entry)); deltas = xmalloc(nr_objects * sizeof(struct delta_entry)); + if (verbose) + setup_progress_signal(); parse_pack_objects(sha1); if (nr_deltas != nr_resolved_deltas) { if (fix_thin_pack) { int nr_unresolved = nr_deltas - nr_resolved_deltas; + int nr_objects_initial = nr_objects; if (nr_unresolved <= 0) die("confusion beyond insanity"); objects = xrealloc(objects, (nr_objects + nr_unresolved + 1) * sizeof(*objects)); fix_unresolved_deltas(nr_unresolved); + if (verbose) + fprintf(stderr, "%d objects were added to complete this thin pack.\n", + nr_objects - nr_objects_initial); readjust_pack_header_and_sha1(sha1); } if (nr_deltas != nr_resolved_deltas) -- cgit v1.3 From 9bee24785133ba3c2361b17f8c20019ab57b6f72 Mon Sep 17 00:00:00 2001 From: Nicolas Pitre Date: Wed, 25 Oct 2006 23:31:53 -0400 Subject: mimic unpack-objects when --stdin is used with index-pack It appears that git-unpack-objects writes the last part of the input buffer to stdout after the pack has been parsed. This looks a bit suspicious since the last fill() might have filled the buffer up to the 4096 byte limit and more data might still be pending on stdin, but since this is about being a drop-in replacement for unpack-objects let's simply duplicate the same behavior for now. [jc: with fix-up appeared in Nico's sleep] Signed-off-by: Nicolas Pitre Signed-off-by: Junio C Hamano --- index-pack.c | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/index-pack.c b/index-pack.c index 2046b37e4b..5c747a62ba 100644 --- a/index-pack.c +++ b/index-pack.c @@ -456,11 +456,12 @@ static void parse_pack_objects(unsigned char *sha1) SHA1_Final(sha1, &input_ctx); if (hashcmp(fill(20), sha1)) die("pack is corrupted (SHA1 mismatch)"); + use(20); /* If input_fd is a file, we should have reached its end now. */ if (fstat(input_fd, &st)) die("cannot fstat packfile: %s", strerror(errno)); - if (S_ISREG(st.st_mode) && st.st_size != consumed_bytes + 20) + if (S_ISREG(st.st_mode) && st.st_size != consumed_bytes) die("pack has junk at the end"); if (!nr_deltas) @@ -765,6 +766,18 @@ static void final(const char *final_pack_name, const char *curr_pack_name, if (err) die("error while closing pack file: %s", strerror(errno)); chmod(curr_pack_name, 0444); + + /* + * Let's just mimic git-unpack-objects here and write + * the last part of the buffer to stdout. + */ + while (input_len) { + err = xwrite(1, input_buffer + input_offset, input_len); + if (err <= 0) + break; + input_len -= err; + input_offset += err; + } } if (final_pack_name != curr_pack_name) { @@ -863,7 +876,6 @@ int main(int argc, char **argv) nr_deltas - nr_resolved_deltas); } else { /* Flush remaining pack final 20-byte SHA1. */ - use(20); flush(); } free(deltas); @@ -872,7 +884,8 @@ int main(int argc, char **argv) free(objects); free(index_name_buf); - printf("%s\n", sha1_to_hex(sha1)); + if (!from_stdin) + printf("%s\n", sha1_to_hex(sha1)); return 0; } -- cgit v1.3 From d9c20ba13dfca737373ba466d2a718cafdc17f92 Mon Sep 17 00:00:00 2001 From: Nicolas Pitre Date: Fri, 27 Oct 2006 15:42:17 -0400 Subject: enhance clone and fetch -k experience Now that index-pack can be streamed with a pack, it is probably a good idea to use it directly instead of creating a temporary file and running index-pack afterwards. This way index-pack can abort early whenever a corruption is encountered even if the pack has not been fully downloaded, it can display a progress percentage as it knows how much to expects, and it is a bit faster since the pack indexing is partially done as data is received. Using fetch -k doesn't need to disable thin pack generation on the remote end either. Signed-off-by: Nicolas Pitre Signed-off-by: Junio C Hamano --- fetch-clone.c | 219 ++++------------------------------------------------------ fetch-pack.c | 2 - 2 files changed, 15 insertions(+), 206 deletions(-) diff --git a/fetch-clone.c b/fetch-clone.c index 76b99afcdb..96cdab43c9 100644 --- a/fetch-clone.c +++ b/fetch-clone.c @@ -3,97 +3,6 @@ #include "pkt-line.h" #include "sideband.h" #include -#include - -static int finish_pack(const char *pack_tmp_name, const char *me) -{ - int pipe_fd[2]; - pid_t pid; - char idx[PATH_MAX]; - char final[PATH_MAX]; - char hash[41]; - unsigned char sha1[20]; - char *cp; - int err = 0; - - if (pipe(pipe_fd) < 0) - die("%s: unable to set up pipe", me); - - strcpy(idx, pack_tmp_name); /* ".git/objects/pack-XXXXXX" */ - cp = strrchr(idx, '/'); - memcpy(cp, "/pidx", 5); - - pid = fork(); - if (pid < 0) - die("%s: unable to fork off git-index-pack", me); - if (!pid) { - close(0); - dup2(pipe_fd[1], 1); - close(pipe_fd[0]); - close(pipe_fd[1]); - execl_git_cmd("index-pack", "-o", idx, pack_tmp_name, NULL); - error("cannot exec git-index-pack <%s> <%s>", - idx, pack_tmp_name); - exit(1); - } - close(pipe_fd[1]); - if (read(pipe_fd[0], hash, 40) != 40) { - error("%s: unable to read from git-index-pack", me); - err = 1; - } - close(pipe_fd[0]); - - for (;;) { - int status, code; - - if (waitpid(pid, &status, 0) < 0) { - if (errno == EINTR) - continue; - error("waitpid failed (%s)", strerror(errno)); - goto error_die; - } - if (WIFSIGNALED(status)) { - int sig = WTERMSIG(status); - error("git-index-pack died of signal %d", sig); - goto error_die; - } - if (!WIFEXITED(status)) { - error("git-index-pack died of unnatural causes %d", - status); - goto error_die; - } - code = WEXITSTATUS(status); - if (code) { - error("git-index-pack died with error code %d", code); - goto error_die; - } - if (err) - goto error_die; - break; - } - hash[40] = 0; - if (get_sha1_hex(hash, sha1)) { - error("git-index-pack reported nonsense '%s'", hash); - goto error_die; - } - /* Now we have pack in pack_tmp_name[], and - * idx in idx[]; rename them to their final names. - */ - snprintf(final, sizeof(final), - "%s/pack/pack-%s.pack", get_object_directory(), hash); - move_temp_to_file(pack_tmp_name, final); - chmod(final, 0444); - snprintf(final, sizeof(final), - "%s/pack/pack-%s.idx", get_object_directory(), hash); - move_temp_to_file(idx, final); - chmod(final, 0444); - return 0; - - error_die: - unlink(idx); - unlink(pack_tmp_name); - exit(1); -} static pid_t setup_sideband(int sideband, const char *me, int fd[2], int xd[2]) { @@ -128,7 +37,7 @@ static pid_t setup_sideband(int sideband, const char *me, int fd[2], int xd[2]) return side_pid; } -int receive_unpack_pack(int xd[2], const char *me, int quiet, int sideband) +static int get_pack(int xd[2], const char *me, int sideband, const char **argv) { int status; pid_t pid, side_pid; @@ -142,135 +51,37 @@ int receive_unpack_pack(int xd[2], const char *me, int quiet, int sideband) dup2(fd[0], 0); close(fd[0]); close(fd[1]); - execl_git_cmd("unpack-objects", quiet ? "-q" : NULL, NULL); - die("git-unpack-objects exec failed"); + execv_git_cmd(argv); + die("%s exec failed", argv[0]); } close(fd[0]); close(fd[1]); while (waitpid(pid, &status, 0) < 0) { if (errno != EINTR) - die("waiting for git-unpack-objects: %s", - strerror(errno)); + die("waiting for %s: %s", argv[0], strerror(errno)); } if (WIFEXITED(status)) { int code = WEXITSTATUS(status); if (code) - die("git-unpack-objects died with error code %d", - code); + die("%s died with error code %d", argv[0], code); return 0; } if (WIFSIGNALED(status)) { int sig = WTERMSIG(status); - die("git-unpack-objects died of signal %d", sig); + die("%s died of signal %d", argv[0], sig); } - die("git-unpack-objects died of unnatural causes %d", status); + die("%s died of unnatural causes %d", argv[0], status); } -/* - * We average out the download speed over this many "events", where - * an event is a minimum of about half a second. That way, we get - * a reasonably stable number. - */ -#define NR_AVERAGE (4) - -/* - * A "binary msec" is a power-of-two-msec, aka 1/1024th of a second. - * Keeping the time in that format means that "bytes / msecs" means - * the same as kB/s (modulo rounding). - * - * 1000512 is a magic number (usecs in a second, rounded up by half - * of 1024, to make "rounding" come out right ;) - */ -#define usec_to_binarymsec(x) ((int)(x) / (1000512 >> 10)) +int receive_unpack_pack(int xd[2], const char *me, int quiet, int sideband) +{ + const char *argv[3] = { "unpack-objects", quiet ? "-q" : NULL, NULL }; + return get_pack(xd, me, sideband, argv); +} int receive_keep_pack(int xd[2], const char *me, int quiet, int sideband) { - char tmpfile[PATH_MAX]; - int ofd, ifd, fd[2]; - unsigned long total; - static struct timeval prev_tv; - struct average { - unsigned long bytes; - unsigned long time; - } download[NR_AVERAGE] = { {0, 0}, }; - unsigned long avg_bytes, avg_time; - int idx = 0; - - setup_sideband(sideband, me, fd, xd); - - ifd = fd[0]; - snprintf(tmpfile, sizeof(tmpfile), - "%s/pack/tmp-XXXXXX", get_object_directory()); - ofd = mkstemp(tmpfile); - if (ofd < 0) - return error("unable to create temporary file %s", tmpfile); - - gettimeofday(&prev_tv, NULL); - total = 0; - avg_bytes = 0; - avg_time = 0; - while (1) { - char buf[8192]; - ssize_t sz, wsz, pos; - sz = read(ifd, buf, sizeof(buf)); - if (sz == 0) - break; - if (sz < 0) { - if (errno != EINTR && errno != EAGAIN) { - error("error reading pack (%s)", strerror(errno)); - close(ofd); - unlink(tmpfile); - return -1; - } - sz = 0; - } - pos = 0; - while (pos < sz) { - wsz = write(ofd, buf + pos, sz - pos); - if (wsz < 0) { - error("error writing pack (%s)", - strerror(errno)); - close(ofd); - unlink(tmpfile); - return -1; - } - pos += wsz; - } - total += sz; - if (!quiet) { - static unsigned long last; - struct timeval tv; - unsigned long diff = total - last; - /* not really "msecs", but a power-of-two millisec (1/1024th of a sec) */ - unsigned long msecs; - - gettimeofday(&tv, NULL); - msecs = tv.tv_sec - prev_tv.tv_sec; - msecs <<= 10; - msecs += usec_to_binarymsec(tv.tv_usec - prev_tv.tv_usec); - - if (msecs > 500) { - prev_tv = tv; - last = total; - - /* Update averages ..*/ - avg_bytes += diff; - avg_time += msecs; - avg_bytes -= download[idx].bytes; - avg_time -= download[idx].time; - download[idx].bytes = diff; - download[idx].time = msecs; - idx++; - if (idx >= NR_AVERAGE) - idx = 0; - - fprintf(stderr, "%4lu.%03luMB (%lu kB/s) \r", - total >> 20, - 1000*((total >> 10) & 1023)>>10, - avg_bytes / avg_time ); - } - } - } - close(ofd); - return finish_pack(tmpfile, me); + const char *argv[5] = { "index-pack", "--stdin", "--fix-thin", + quiet ? NULL : "-v", NULL }; + return get_pack(xd, me, sideband, argv); } diff --git a/fetch-pack.c b/fetch-pack.c index 474d54520e..8720ed42e9 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -518,8 +518,6 @@ int main(int argc, char **argv) } if (!dest) usage(fetch_pack_usage); - if (keep_pack) - use_thin_pack = 0; pid = git_connect(fd, dest, exec); if (pid < 0) return 1; -- cgit v1.3 From b89c4e93cc1939493c2bb9b6c3f60eabaf653eff Mon Sep 17 00:00:00 2001 From: Nicolas Pitre Date: Fri, 27 Oct 2006 16:14:23 -0400 Subject: index-pack: minor fixes to comment and function name Use proper english. Be more exact in one comment. [jc: I threw in a bit of style clean-up as well] Signed-off-by: Nicolas Pitre Signed-off-by: Junio C Hamano --- index-pack.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/index-pack.c b/index-pack.c index 5c747a62ba..866a054056 100644 --- a/index-pack.c +++ b/index-pack.c @@ -105,7 +105,7 @@ static void flush() * Make sure at least "min" bytes are available in the buffer, and * return the pointer to the buffer. */ -static void * fill(int min) +static void *fill(int min) { if (min <= input_len) return input_buffer + input_offset; @@ -134,7 +134,7 @@ static void use(int bytes) consumed_bytes += bytes; } -static const char * open_pack_file(const char *pack_name) +static const char *open_pack_file(const char *pack_name) { if (from_stdin) { input_fd = 0; @@ -275,7 +275,7 @@ static void *unpack_raw_entry(struct object_entry *obj, union delta_base *delta_ return unpack_entry_data(obj->offset, obj->size); } -static void * get_data_from_pack(struct object_entry *obj) +static void *get_data_from_pack(struct object_entry *obj) { unsigned long from = obj[0].offset + obj[0].hdr_size; unsigned long len = obj[1].offset - from; @@ -324,8 +324,8 @@ static int find_delta(const union delta_base *base) return -first-1; } -static int find_delta_childs(const union delta_base *base, - int *first_index, int *last_index) +static int find_delta_children(const union delta_base *base, + int *first_index, int *last_index) { int first = find_delta(base); int last = first; @@ -389,7 +389,7 @@ static void resolve_delta(struct object_entry *delta_obj, void *base_data, nr_resolved_deltas++; hashcpy(delta_base.sha1, delta_obj->sha1); - if (!find_delta_childs(&delta_base, &first, &last)) { + if (!find_delta_children(&delta_base, &first, &last)) { for (j = first; j <= last; j++) { struct object_entry *child = objects + deltas[j].obj_no; if (child->real_type == OBJ_REF_DELTA) @@ -399,7 +399,7 @@ static void resolve_delta(struct object_entry *delta_obj, void *base_data, memset(&delta_base, 0, sizeof(delta_base)); delta_base.offset = delta_obj->offset; - if (!find_delta_childs(&delta_base, &first, &last)) { + if (!find_delta_children(&delta_base, &first, &last)) { for (j = first; j <= last; j++) { struct object_entry *child = objects + deltas[j].obj_no; if (child->real_type == OBJ_OFS_DELTA) @@ -429,7 +429,7 @@ static void parse_pack_objects(unsigned char *sha1) * First pass: * - find locations of all objects; * - calculate SHA1 of all non-delta objects; - * - remember base SHA1 for all deltas. + * - remember base (SHA1 or offset) for all deltas. */ if (verbose) fprintf(stderr, "Indexing %d objects.\n", nr_objects); @@ -489,10 +489,10 @@ static void parse_pack_objects(unsigned char *sha1) if (obj->type == OBJ_REF_DELTA || obj->type == OBJ_OFS_DELTA) continue; hashcpy(base.sha1, obj->sha1); - ref = !find_delta_childs(&base, &ref_first, &ref_last); + ref = !find_delta_children(&base, &ref_first, &ref_last); memset(&base, 0, sizeof(base)); base.offset = obj->offset; - ofs = !find_delta_childs(&base, &ofs_first, &ofs_last); + ofs = !find_delta_children(&base, &ofs_first, &ofs_last); if (!ref && !ofs) continue; data = get_data_from_pack(obj); @@ -615,7 +615,7 @@ static void fix_unresolved_deltas(int nr_unresolved) else die("base object %s is of type '%s'", sha1_to_hex(d->base.sha1), type); - find_delta_childs(&d->base, &first, &last); + find_delta_children(&d->base, &first, &last); for (j = first; j <= last; j++) { struct object_entry *child = objects + deltas[j].obj_no; if (child->real_type == OBJ_REF_DELTA) @@ -675,7 +675,7 @@ static int sha1_compare(const void *_a, const void *_b) * On entry *sha1 contains the pack content SHA1 hash, on exit it is * the SHA1 hash of sorted object names. */ -static const char * write_index_file(const char *index_name, unsigned char *sha1) +static const char *write_index_file(const char *index_name, unsigned char *sha1) { struct sha1file *f; struct object_entry **sorted_by_sha, **list, **last; -- cgit v1.3 From c7740a943ec896247ebc5514b6be02710caf3c53 Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Sun, 29 Oct 2006 00:47:56 -0700 Subject: send-pack --keep: do not explode into loose objects on the receiving end. This adds "keep-pack" extension to send-pack vs receive pack protocol, and makes the receiver invoke "index-pack --stdin --fix-thin". With this, you can ask send-pack not to explode the result into loose objects on the receiving end. I've patched has_sha1_file() to re-check for added packs just like is done in read_sha1_file() for now, but I think the static "re-prepare" interface for packs was a mistake. Creation of a new pack inside a process that needs to read objects in them back ought to be a rare event, so we are better off making the callers (such as receive-pack that calls "index-pack --stdin --fix-thin") explicitly call re-prepare. That way we do not have to penalize ordinary users of read_sha1_file() and has_sha1_file(). We would need to fix this someday. Signed-off-by: Junio C Hamano --- receive-pack.c | 19 ++++++++++++++++--- send-pack.c | 23 ++++++++++++++++++----- sha1_file.c | 7 +++++-- t/t5400-send-pack.sh | 9 +++++++++ 4 files changed, 48 insertions(+), 10 deletions(-) diff --git a/receive-pack.c b/receive-pack.c index ea2dbd4e33..ef50226f4d 100644 --- a/receive-pack.c +++ b/receive-pack.c @@ -8,10 +8,14 @@ static const char receive_pack_usage[] = "git-receive-pack "; static const char *unpacker[] = { "unpack-objects", NULL }; +static const char *keep_packer[] = { + "index-pack", "--stdin", "--fix-thin", NULL +}; static int report_status; +static int keep_pack; -static char capabilities[] = "report-status"; +static char capabilities[] = "report-status keep-pack"; static int capabilities_sent; static int show_ref(const char *path, const unsigned char *sha1) @@ -261,6 +265,8 @@ static void read_head_info(void) if (reflen + 82 < len) { if (strstr(refname + reflen + 1, "report-status")) report_status = 1; + if (strstr(refname + reflen + 1, "keep-pack")) + keep_pack = 1; } cmd = xmalloc(sizeof(struct command) + len - 80); hashcpy(cmd->old_sha1, old_sha1); @@ -275,7 +281,14 @@ static void read_head_info(void) static const char *unpack(int *error_code) { - int code = run_command_v_opt(1, unpacker, RUN_GIT_CMD); + int code; + + if (keep_pack) + code = run_command_v_opt(ARRAY_SIZE(keep_packer) - 1, + keep_packer, RUN_GIT_CMD); + else + code = run_command_v_opt(ARRAY_SIZE(unpacker) - 1, + unpacker, RUN_GIT_CMD); *error_code = 0; switch (code) { @@ -335,7 +348,7 @@ int main(int argc, char **argv) if (!dir) usage(receive_pack_usage); - if(!enter_repo(dir, 0)) + if (!enter_repo(dir, 0)) die("'%s': unable to chdir or not a git archive", dir); write_head_info(); diff --git a/send-pack.c b/send-pack.c index 5bb123a376..54d218c03b 100644 --- a/send-pack.c +++ b/send-pack.c @@ -6,13 +6,14 @@ #include "exec_cmd.h" static const char send_pack_usage[] = -"git-send-pack [--all] [--exec=git-receive-pack] [...]\n" +"git-send-pack [--all] [--keep] [--exec=git-receive-pack] [...]\n" " --all and explicit specification are mutually exclusive."; static const char *exec = "git-receive-pack"; static int verbose; static int send_all; static int force_update; static int use_thin_pack; +static int keep_pack; static int is_zero_sha1(const unsigned char *sha1) { @@ -270,6 +271,7 @@ static int send_pack(int in, int out, int nr_refspec, char **refspec) int new_refs; int ret = 0; int ask_for_status_report = 0; + int ask_to_keep_pack = 0; int expect_status_report = 0; /* No funny business with the matcher */ @@ -279,6 +281,8 @@ static int send_pack(int in, int out, int nr_refspec, char **refspec) /* Does the other end support the reporting? */ if (server_supports("report-status")) ask_for_status_report = 1; + if (server_supports("keep-pack") && keep_pack) + ask_to_keep_pack = 1; /* match them up */ if (!remote_tail) @@ -355,12 +359,17 @@ static int send_pack(int in, int out, int nr_refspec, char **refspec) strcpy(old_hex, sha1_to_hex(ref->old_sha1)); new_hex = sha1_to_hex(ref->new_sha1); - if (ask_for_status_report) { - packet_write(out, "%s %s %s%c%s", + if (ask_for_status_report || ask_to_keep_pack) { + packet_write(out, "%s %s %s%c%s%s", old_hex, new_hex, ref->name, 0, - "report-status"); + ask_for_status_report + ? " report-status" : "", + ask_to_keep_pack + ? " keep-pack" : ""); + if (ask_for_status_report) + expect_status_report = 1; ask_for_status_report = 0; - expect_status_report = 1; + ask_to_keep_pack = 0; } else packet_write(out, "%s %s %s", @@ -419,6 +428,10 @@ int main(int argc, char **argv) verbose = 1; continue; } + if (!strcmp(arg, "--keep")) { + keep_pack = 1; + continue; + } if (!strcmp(arg, "--thin")) { use_thin_pack = 1; continue; diff --git a/sha1_file.c b/sha1_file.c index e89d24c015..278ba2f4cd 100644 --- a/sha1_file.c +++ b/sha1_file.c @@ -1292,7 +1292,7 @@ static void *read_packed_sha1(const unsigned char *sha1, char *type, unsigned lo return unpack_entry(&e, type, size); } -void * read_sha1_file(const unsigned char *sha1, char *type, unsigned long *size) +void *read_sha1_file(const unsigned char *sha1, char *type, unsigned long *size) { unsigned long mapsize; void *map, *buf; @@ -1757,7 +1757,10 @@ int has_sha1_file(const unsigned char *sha1) if (find_pack_entry(sha1, &e, NULL)) return 1; - return find_sha1_file(sha1, &st) ? 1 : 0; + if (find_sha1_file(sha1, &st)) + return 1; + reprepare_packed_git(); + return find_pack_entry(sha1, &e, NULL); } /* diff --git a/t/t5400-send-pack.sh b/t/t5400-send-pack.sh index 8afb899717..d831f8dfe3 100755 --- a/t/t5400-send-pack.sh +++ b/t/t5400-send-pack.sh @@ -78,4 +78,13 @@ test_expect_success \ ! diff -u .git/refs/heads/master victim/.git/refs/heads/master ' +test_expect_success 'push with --keep' ' + t=`cd victim && git-rev-parse --verify refs/heads/master` && + git-update-ref refs/heads/master $t && + : > foo && + git add foo && + git commit -m "one more" && + git-send-pack --keep ./victim/.git/ master +' + test_done -- cgit v1.3 From d4ff6d92c3d7020233a9b4990a3545ee54a5c033 Mon Sep 17 00:00:00 2001 From: Shawn Pearce Date: Sun, 29 Oct 2006 04:37:11 -0500 Subject: Allow short pack names to git-pack-objects --unpacked=. This allows us to pass just the file name of a pack rather than the complete path when we want pack-objects to consider its contents as though they were loose objects. This can be helpful if $GIT_OBJECT_DIRECTORY contains shell metacharacters which make it cumbersome to pass complete paths safely in a shell script. Signed-off-by: Shawn O. Pearce Signed-off-by: Junio C Hamano --- sha1_file.c | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/sha1_file.c b/sha1_file.c index e89d24c015..5e6c8b8bbf 100644 --- a/sha1_file.c +++ b/sha1_file.c @@ -1203,6 +1203,24 @@ unsigned long find_pack_entry_one(const unsigned char *sha1, return 0; } +static int matches_pack_name(struct packed_git *p, const char *ig) +{ + const char *last_c, *c; + + if (!strcmp(p->pack_name, ig)) + return 0; + + for (c = p->pack_name, last_c = c; *c;) + if (*c == '/') + last_c = ++c; + else + ++c; + if (!strcmp(last_c, ig)) + return 0; + + return 1; +} + static int find_pack_entry(const unsigned char *sha1, struct pack_entry *e, const char **ignore_packed) { struct packed_git *p; @@ -1214,7 +1232,7 @@ static int find_pack_entry(const unsigned char *sha1, struct pack_entry *e, cons if (ignore_packed) { const char **ig; for (ig = ignore_packed; *ig; ig++) - if (!strcmp(p->pack_name, *ig)) + if (!matches_pack_name(p, *ig)) break; if (*ig) continue; -- cgit v1.3 From ce8590748b918687abc4c7cd2d432dd23f07ae40 Mon Sep 17 00:00:00 2001 From: Shawn Pearce Date: Sun, 29 Oct 2006 04:37:54 -0500 Subject: Only repack active packs by skipping over kept packs. During `git repack -a -d` only repack objects which are loose or which reside in an active (a non-kept) pack. This allows the user to keep large packs as-is without continuous repacking and can be very helpful on large repositories. It should also help us resolve a race condition between `git repack -a -d` and the new pack store functionality in `git-receive-pack`. Kept packs are those which have a corresponding .keep file in $GIT_OBJECT_DIRECTORY/pack. That is pack-X.pack will be kept (not repacked and not deleted) if pack-X.keep exists in the same directory when `git repack -a -d` starts. Currently this feature is not documented and there is no user interface to keep an existing pack. Signed-off-by: Shawn O. Pearce Signed-off-by: Junio C Hamano --- git-repack.sh | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/git-repack.sh b/git-repack.sh index 17e24526c2..f150a558ca 100755 --- a/git-repack.sh +++ b/git-repack.sh @@ -45,11 +45,19 @@ case ",$all_into_one," in args='--unpacked --incremental' ;; ,t,) - args= - - # Redundancy check in all-into-one case is trivial. - existing=`test -d "$PACKDIR" && cd "$PACKDIR" && \ - find . -type f \( -name '*.pack' -o -name '*.idx' \) -print` + if [ -d "$PACKDIR" ]; then + for e in `cd "$PACKDIR" && find . -type f -name '*.pack' \ + | sed -e 's/^\.\///' -e 's/\.pack$//'` + do + if [ -e "$PACKDIR/$e.keep" ]; then + : keep + else + args="$args --unpacked=$e.pack" + existing="$existing $e" + fi + done + fi + [ -z "$args" ] && args='--unpacked --incremental' ;; esac @@ -86,17 +94,16 @@ fi if test "$remove_redundant" = t then - # We know $existing are all redundant only when - # all-into-one is used. - if test "$all_into_one" != '' && test "$existing" != '' + # We know $existing are all redundant. + if [ -n "$existing" ] then sync ( cd "$PACKDIR" && for e in $existing do case "$e" in - ./pack-$name.pack | ./pack-$name.idx) ;; - *) rm -f $e ;; + pack-$name) ;; + *) rm -f "$e.pack" "$e.idx" "$e.keep" ;; esac done ) -- cgit v1.3 From b8077709243138c3d8cc1c096c06a95b250a9001 Mon Sep 17 00:00:00 2001 From: Shawn Pearce Date: Sun, 29 Oct 2006 04:41:59 -0500 Subject: Teach git-index-pack how to keep a pack file. To prevent a race condition between `index-pack --stdin` and `repack -a -d` where the repack deletes the newly created pack file before any refs are updated to reference objects contained within it we mark the pack file as one that should be kept. This removes it from the list of packs that `repack -a -d` will consider for removal. Callers such as `receive-pack` which want to invoke `index-pack` should use this new --keep option to prevent the newly created pack and index file pair from being deleted before they have finished any related ref updates. Only after all ref updates have been finished should the associated .keep file be removed. Signed-off-by: Shawn O. Pearce Signed-off-by: Junio C Hamano --- Documentation/git-index-pack.txt | 24 +++++++++++++++++++--- index-pack.c | 43 +++++++++++++++++++++++++++++++++++++--- 2 files changed, 61 insertions(+), 6 deletions(-) diff --git a/Documentation/git-index-pack.txt b/Documentation/git-index-pack.txt index 9fa4847d56..1235416e09 100644 --- a/Documentation/git-index-pack.txt +++ b/Documentation/git-index-pack.txt @@ -9,7 +9,7 @@ git-index-pack - Build pack index file for an existing packed archive SYNOPSIS -------- 'git-index-pack' [-v] [-o ] -'git-index-pack' --stdin [--fix-thin] [-v] [-o ] [] +'git-index-pack' --stdin [--fix-thin] [--keep] [-v] [-o ] [] DESCRIPTION @@ -38,7 +38,10 @@ OPTIONS instead and a copy is then written to . If is not specified, the pack is written to objects/pack/ directory of the current git repository with - a default name determined from the pack content. + a default name determined from the pack content. If + is not specified consider using --keep to + prevent a race condition between this process and + gitlink::git-repack[1] . --fix-thin:: It is possible for gitlink:git-pack-objects[1] to build @@ -48,7 +51,22 @@ OPTIONS and they must be included in the pack for that pack to be self contained and indexable. Without this option any attempt to index a thin pack will fail. This option only makes sense in - conjonction with --stdin. + conjunction with --stdin. + +--keep:: + Before moving the index into its final destination + create an empty .keep file for the associated pack file. + This option is usually necessary with --stdin to prevent a + simultaneous gitlink:git-repack[1] process from deleting + the newly constructed pack and index before refs can be + updated to use objects contained in the pack. + +--keep='why':: + Like --keep create a .keep file before moving the index into + its final destination, but rather than creating an empty file + place 'why' followed by an LF into the .keep file. The 'why' + message can later be searched for within all .keep files to + locate any which have outlived their usefulness. Author diff --git a/index-pack.c b/index-pack.c index 866a054056..b37dd78729 100644 --- a/index-pack.c +++ b/index-pack.c @@ -10,7 +10,7 @@ #include static const char index_pack_usage[] = -"git-index-pack [-v] [-o ] { | --stdin [--fix-thin] [] }"; +"git-index-pack [-v] [-o ] [{ ---keep | --keep= }] { | --stdin [--fix-thin] [] }"; struct object_entry { @@ -754,6 +754,7 @@ static const char *write_index_file(const char *index_name, unsigned char *sha1) static void final(const char *final_pack_name, const char *curr_pack_name, const char *final_index_name, const char *curr_index_name, + const char *keep_name, const char *keep_msg, unsigned char *sha1) { char name[PATH_MAX]; @@ -780,6 +781,23 @@ static void final(const char *final_pack_name, const char *curr_pack_name, } } + if (keep_msg) { + int keep_fd, keep_msg_len = strlen(keep_msg); + if (!keep_name) { + snprintf(name, sizeof(name), "%s/pack/pack-%s.keep", + get_object_directory(), sha1_to_hex(sha1)); + keep_name = name; + } + keep_fd = open(keep_name, O_RDWR | O_CREAT, 0600); + if (keep_fd < 0) + die("cannot write keep file"); + if (keep_msg_len > 0) { + write_or_die(keep_fd, keep_msg, keep_msg_len); + write_or_die(keep_fd, "\n", 1); + } + close(keep_fd); + } + if (final_pack_name != curr_pack_name) { if (!final_pack_name) { snprintf(name, sizeof(name), "%s/pack/pack-%s.pack", @@ -807,7 +825,8 @@ int main(int argc, char **argv) int i, fix_thin_pack = 0; const char *curr_pack, *pack_name = NULL; const char *curr_index, *index_name = NULL; - char *index_name_buf = NULL; + const char *keep_name = NULL, *keep_msg = NULL; + char *index_name_buf = NULL, *keep_name_buf = NULL; unsigned char sha1[20]; for (i = 1; i < argc; i++) { @@ -818,6 +837,10 @@ int main(int argc, char **argv) from_stdin = 1; } else if (!strcmp(arg, "--fix-thin")) { fix_thin_pack = 1; + } else if (!strcmp(arg, "--keep")) { + keep_msg = ""; + } else if (!strncmp(arg, "--keep=", 7)) { + keep_msg = arg + 7; } else if (!strcmp(arg, "-v")) { verbose = 1; } else if (!strcmp(arg, "-o")) { @@ -848,6 +871,16 @@ int main(int argc, char **argv) strcpy(index_name_buf + len - 5, ".idx"); index_name = index_name_buf; } + if (keep_msg && !keep_name && pack_name) { + int len = strlen(pack_name); + if (!has_extension(pack_name, ".pack")) + die("packfile name '%s' does not end with '.pack'", + pack_name); + keep_name_buf = xmalloc(len); + memcpy(keep_name_buf, pack_name, len - 5); + strcpy(keep_name_buf + len - 5, ".keep"); + keep_name = keep_name_buf; + } curr_pack = open_pack_file(pack_name); parse_pack_header(); @@ -880,9 +913,13 @@ int main(int argc, char **argv) } free(deltas); curr_index = write_index_file(index_name, sha1); - final(pack_name, curr_pack, index_name, curr_index, sha1); + final(pack_name, curr_pack, + index_name, curr_index, + keep_name, keep_msg, + sha1); free(objects); free(index_name_buf); + free(keep_name_buf); if (!from_stdin) printf("%s\n", sha1_to_hex(sha1)); -- cgit v1.3 From 5ca807842fcb709757147df0044e8b4b2946ea22 Mon Sep 17 00:00:00 2001 From: Nicolas Pitre Date: Sun, 29 Oct 2006 21:33:22 -0500 Subject: missing small substitution Signed-off-by: Nicolas Pitre Signed-off-by: Junio C Hamano --- fetch-clone.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fetch-clone.c b/fetch-clone.c index 96cdab43c9..f629d8dacc 100644 --- a/fetch-clone.c +++ b/fetch-clone.c @@ -46,7 +46,7 @@ static int get_pack(int xd[2], const char *me, int sideband, const char **argv) side_pid = setup_sideband(sideband, me, fd, xd); pid = fork(); if (pid < 0) - die("%s: unable to fork off git-unpack-objects", me); + die("%s: unable to fork off %s", me, argv[0]); if (!pid) { dup2(fd[0], 0); close(fd[0]); -- cgit v1.3 From 4508dde16951df00529207f179fee405e8646a64 Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Mon, 30 Oct 2006 16:02:07 -0800 Subject: Revert "send-pack --keep: do not explode into loose objects on the receiving end." This reverts commit c7740a943ec896247ebc5514b6be02710caf3c53. There should be a way to make this controllable from the receiver end. --- receive-pack.c | 19 +++---------------- send-pack.c | 23 +++++------------------ sha1_file.c | 7 ++----- t/t5400-send-pack.sh | 9 --------- 4 files changed, 10 insertions(+), 48 deletions(-) diff --git a/receive-pack.c b/receive-pack.c index ef50226f4d..ea2dbd4e33 100644 --- a/receive-pack.c +++ b/receive-pack.c @@ -8,14 +8,10 @@ static const char receive_pack_usage[] = "git-receive-pack "; static const char *unpacker[] = { "unpack-objects", NULL }; -static const char *keep_packer[] = { - "index-pack", "--stdin", "--fix-thin", NULL -}; static int report_status; -static int keep_pack; -static char capabilities[] = "report-status keep-pack"; +static char capabilities[] = "report-status"; static int capabilities_sent; static int show_ref(const char *path, const unsigned char *sha1) @@ -265,8 +261,6 @@ static void read_head_info(void) if (reflen + 82 < len) { if (strstr(refname + reflen + 1, "report-status")) report_status = 1; - if (strstr(refname + reflen + 1, "keep-pack")) - keep_pack = 1; } cmd = xmalloc(sizeof(struct command) + len - 80); hashcpy(cmd->old_sha1, old_sha1); @@ -281,14 +275,7 @@ static void read_head_info(void) static const char *unpack(int *error_code) { - int code; - - if (keep_pack) - code = run_command_v_opt(ARRAY_SIZE(keep_packer) - 1, - keep_packer, RUN_GIT_CMD); - else - code = run_command_v_opt(ARRAY_SIZE(unpacker) - 1, - unpacker, RUN_GIT_CMD); + int code = run_command_v_opt(1, unpacker, RUN_GIT_CMD); *error_code = 0; switch (code) { @@ -348,7 +335,7 @@ int main(int argc, char **argv) if (!dir) usage(receive_pack_usage); - if (!enter_repo(dir, 0)) + if(!enter_repo(dir, 0)) die("'%s': unable to chdir or not a git archive", dir); write_head_info(); diff --git a/send-pack.c b/send-pack.c index 54d218c03b..5bb123a376 100644 --- a/send-pack.c +++ b/send-pack.c @@ -6,14 +6,13 @@ #include "exec_cmd.h" static const char send_pack_usage[] = -"git-send-pack [--all] [--keep] [--exec=git-receive-pack] [...]\n" +"git-send-pack [--all] [--exec=git-receive-pack] [...]\n" " --all and explicit specification are mutually exclusive."; static const char *exec = "git-receive-pack"; static int verbose; static int send_all; static int force_update; static int use_thin_pack; -static int keep_pack; static int is_zero_sha1(const unsigned char *sha1) { @@ -271,7 +270,6 @@ static int send_pack(int in, int out, int nr_refspec, char **refspec) int new_refs; int ret = 0; int ask_for_status_report = 0; - int ask_to_keep_pack = 0; int expect_status_report = 0; /* No funny business with the matcher */ @@ -281,8 +279,6 @@ static int send_pack(int in, int out, int nr_refspec, char **refspec) /* Does the other end support the reporting? */ if (server_supports("report-status")) ask_for_status_report = 1; - if (server_supports("keep-pack") && keep_pack) - ask_to_keep_pack = 1; /* match them up */ if (!remote_tail) @@ -359,17 +355,12 @@ static int send_pack(int in, int out, int nr_refspec, char **refspec) strcpy(old_hex, sha1_to_hex(ref->old_sha1)); new_hex = sha1_to_hex(ref->new_sha1); - if (ask_for_status_report || ask_to_keep_pack) { - packet_write(out, "%s %s %s%c%s%s", + if (ask_for_status_report) { + packet_write(out, "%s %s %s%c%s", old_hex, new_hex, ref->name, 0, - ask_for_status_report - ? " report-status" : "", - ask_to_keep_pack - ? " keep-pack" : ""); - if (ask_for_status_report) - expect_status_report = 1; + "report-status"); ask_for_status_report = 0; - ask_to_keep_pack = 0; + expect_status_report = 1; } else packet_write(out, "%s %s %s", @@ -428,10 +419,6 @@ int main(int argc, char **argv) verbose = 1; continue; } - if (!strcmp(arg, "--keep")) { - keep_pack = 1; - continue; - } if (!strcmp(arg, "--thin")) { use_thin_pack = 1; continue; diff --git a/sha1_file.c b/sha1_file.c index 278ba2f4cd..e89d24c015 100644 --- a/sha1_file.c +++ b/sha1_file.c @@ -1292,7 +1292,7 @@ static void *read_packed_sha1(const unsigned char *sha1, char *type, unsigned lo return unpack_entry(&e, type, size); } -void *read_sha1_file(const unsigned char *sha1, char *type, unsigned long *size) +void * read_sha1_file(const unsigned char *sha1, char *type, unsigned long *size) { unsigned long mapsize; void *map, *buf; @@ -1757,10 +1757,7 @@ int has_sha1_file(const unsigned char *sha1) if (find_pack_entry(sha1, &e, NULL)) return 1; - if (find_sha1_file(sha1, &st)) - return 1; - reprepare_packed_git(); - return find_pack_entry(sha1, &e, NULL); + return find_sha1_file(sha1, &st) ? 1 : 0; } /* diff --git a/t/t5400-send-pack.sh b/t/t5400-send-pack.sh index d831f8dfe3..8afb899717 100755 --- a/t/t5400-send-pack.sh +++ b/t/t5400-send-pack.sh @@ -78,13 +78,4 @@ test_expect_success \ ! diff -u .git/refs/heads/master victim/.git/refs/heads/master ' -test_expect_success 'push with --keep' ' - t=`cd victim && git-rev-parse --verify refs/heads/master` && - git-update-ref refs/heads/master $t && - : > foo && - git add foo && - git commit -m "one more" && - git-send-pack --keep ./victim/.git/ master -' - test_done -- cgit v1.3 From 861ed12106d7f8e65b90c8a5ed4a026cd71a044d Mon Sep 17 00:00:00 2001 From: Shawn Pearce Date: Mon, 30 Oct 2006 17:34:50 -0500 Subject: Remove unused variable in receive-pack. We aren't using this return code variable for anything so lets just get rid of it to keep this section of code clean. Signed-off-by: Shawn O. Pearce Signed-off-by: Junio C Hamano --- receive-pack.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/receive-pack.c b/receive-pack.c index ea2dbd4e33..675b1c18a3 100644 --- a/receive-pack.c +++ b/receive-pack.c @@ -273,11 +273,10 @@ static void read_head_info(void) } } -static const char *unpack(int *error_code) +static const char *unpack(void) { int code = run_command_v_opt(1, unpacker, RUN_GIT_CMD); - *error_code = 0; switch (code) { case 0: return NULL; @@ -294,7 +293,6 @@ static const char *unpack(int *error_code) case -ERR_RUN_COMMAND_WAITPID_NOEXIT: return "unpacker died strangely"; default: - *error_code = -code; return "unpacker exited with error code"; } } @@ -345,8 +343,7 @@ int main(int argc, char **argv) read_head_info(); if (commands) { - int code; - const char *unpack_status = unpack(&code); + const char *unpack_status = unpack(); if (!unpack_status) execute_commands(); if (report_status) -- cgit v1.3 From bed006fbddf919eed81cf62954e0332a395bf035 Mon Sep 17 00:00:00 2001 From: Nicolas Pitre Date: Wed, 1 Nov 2006 17:06:20 -0500 Subject: Allow pack header preprocessing before unpack-objects/index-pack. Some applications which invoke unpack-objects or index-pack --stdin may want to examine the pack header to determine the number of objects contained in the pack and use that value to determine which executable to invoke to handle the rest of the pack stream. However if the caller consumes the pack header from the input stream then its no longer available for unpack-objects or index-pack --stdin, both of which need the version and object count to process the stream. This change introduces --pack_header=ver,cnt as a command line option that the caller can supply to indicate it has already consumed the pack header and what version and object count were found in that header. As this option is only meant for low level applications such as receive-pack we are not documenting it at this time. Signed-off-by: Nicolas Pitre Signed-off-by: Junio C Hamano --- builtin-unpack-objects.c | 15 +++++++++++++++ index-pack.c | 13 +++++++++++++ 2 files changed, 28 insertions(+) diff --git a/builtin-unpack-objects.c b/builtin-unpack-objects.c index 74a90c1129..e6d7574844 100644 --- a/builtin-unpack-objects.c +++ b/builtin-unpack-objects.c @@ -371,6 +371,21 @@ int cmd_unpack_objects(int argc, const char **argv, const char *prefix) recover = 1; continue; } + if (!strncmp(arg, "--pack_header=", 14)) { + struct pack_header *hdr; + char *c; + + hdr = (struct pack_header *)buffer; + hdr->hdr_signature = htonl(PACK_SIGNATURE); + hdr->hdr_version = htonl(strtoul(arg + 14, &c, 10)); + if (*c != ',') + die("bad %s", arg); + hdr->hdr_entries = htonl(strtoul(c + 1, &c, 10)); + if (*c) + die("bad %s", arg); + len = sizeof(*hdr); + continue; + } usage(unpack_usage); } diff --git a/index-pack.c b/index-pack.c index b37dd78729..a3b55f9b0f 100644 --- a/index-pack.c +++ b/index-pack.c @@ -841,6 +841,19 @@ int main(int argc, char **argv) keep_msg = ""; } else if (!strncmp(arg, "--keep=", 7)) { keep_msg = arg + 7; + } else if (!strncmp(arg, "--pack_header=", 14)) { + struct pack_header *hdr; + char *c; + + hdr = (struct pack_header *)input_buffer; + hdr->hdr_signature = htonl(PACK_SIGNATURE); + hdr->hdr_version = htonl(strtoul(arg + 14, &c, 10)); + if (*c != ',') + die("bad %s", arg); + hdr->hdr_entries = htonl(strtoul(c + 1, &c, 10)); + if (*c) + die("bad %s", arg); + input_len = sizeof(*hdr); } else if (!strcmp(arg, "-v")) { verbose = 1; } else if (!strcmp(arg, "-o")) { -- cgit v1.3 From fc04c412d8d2412e97bb2a664a1746e333dfd9ae Mon Sep 17 00:00:00 2001 From: Shawn Pearce Date: Wed, 1 Nov 2006 17:06:21 -0500 Subject: Teach receive-pack how to keep pack files based on object count. Since keeping a pushed pack or exploding it into loose objects should be a local repository decision this teaches receive-pack to decide if it should call unpack-objects or index-pack --stdin --fix-thin based on the setting of receive.unpackLimit and the number of objects contained in the received pack. If the number of objects (hdr_entries) in the received pack is below the value of receive.unpackLimit (which is 5000 by default) then we unpack-objects as we have in the past. If the hdr_entries >= receive.unpackLimit then we call index-pack and ask it to include our pid and hostname in the .keep file to make it easier to identify why a given pack has been kept in the repository. Currently this leaves every received pack as a kept pack. We really don't want that as received packs will tend to be small. Instead we want to delete the .keep file automatically after all refs have been updated. That is being left as room for future improvement. Signed-off-by: Shawn O. Pearce Signed-off-by: Junio C Hamano --- Documentation/config.txt | 11 +++++- cache.h | 1 + receive-pack.c | 98 ++++++++++++++++++++++++++++++++++++++---------- sha1_file.c | 2 +- 4 files changed, 91 insertions(+), 21 deletions(-) diff --git a/Documentation/config.txt b/Documentation/config.txt index d9e73da2a7..9d3c71c3b8 100644 --- a/Documentation/config.txt +++ b/Documentation/config.txt @@ -301,7 +301,16 @@ imap:: The configuration variables in the 'imap' section are described in gitlink:git-imap-send[1]. -receive.denyNonFastforwads:: +receive.unpackLimit:: + If the number of objects received in a push is below this + limit then the objects will be unpacked into loose object + files. However if the number of received objects equals or + exceeds this limit then the received pack will be stored as + a pack, after adding any missing delta bases. Storing the + pack from a push can make the push operation complete faster, + especially on slow filesystems. + +receive.denyNonFastForwards:: If set to true, git-receive-pack will deny a ref update which is not a fast forward. Use this to prevent such an update via a push, even if that push is forced. This configuration variable is diff --git a/cache.h b/cache.h index e997a85005..6cb7e1d8c0 100644 --- a/cache.h +++ b/cache.h @@ -376,6 +376,7 @@ extern struct packed_git *parse_pack_index_file(const unsigned char *sha1, char *idx_path); extern void prepare_packed_git(void); +extern void reprepare_packed_git(void); extern void install_packed_git(struct packed_git *pack); extern struct packed_git *find_sha1_pack(const unsigned char *sha1, diff --git a/receive-pack.c b/receive-pack.c index ed08660da4..b8d12700d8 100644 --- a/receive-pack.c +++ b/receive-pack.c @@ -1,4 +1,5 @@ #include "cache.h" +#include "pack.h" #include "refs.h" #include "pkt-line.h" #include "run-command.h" @@ -7,9 +8,8 @@ static const char receive_pack_usage[] = "git-receive-pack "; -static const char *unpacker[] = { "unpack-objects", NULL }; - static int deny_non_fast_forwards = 0; +static int unpack_limit = 5000; static int report_status; static char capabilities[] = "report-status"; @@ -25,6 +25,12 @@ static int receive_pack_config(const char *var, const char *value) return 0; } + if (strcmp(var, "receive.unpacklimit") == 0) + { + unpack_limit = git_config_int(var, value); + return 0; + } + return 0; } @@ -227,27 +233,81 @@ static void read_head_info(void) } } +static const char *parse_pack_header(struct pack_header *hdr) +{ + char *c = (char*)hdr; + ssize_t remaining = sizeof(struct pack_header); + do { + ssize_t r = xread(0, c, remaining); + if (r <= 0) + return "eof before pack header was fully read"; + remaining -= r; + c += r; + } while (remaining > 0); + if (hdr->hdr_signature != htonl(PACK_SIGNATURE)) + return "protocol error (pack signature mismatch detected)"; + if (!pack_version_ok(hdr->hdr_version)) + return "protocol error (pack version unsupported)"; + return NULL; +} + static const char *unpack(void) { - int code = run_command_v_opt(1, unpacker, RUN_GIT_CMD); + struct pack_header hdr; + const char *hdr_err; + char hdr_arg[38]; + int code; + + hdr_err = parse_pack_header(&hdr); + if (hdr_err) + return hdr_err; + snprintf(hdr_arg, sizeof(hdr_arg), "--pack_header=%u,%u", + ntohl(hdr.hdr_version), ntohl(hdr.hdr_entries)); + + if (ntohl(hdr.hdr_entries) < unpack_limit) { + const char *unpacker[3]; + unpacker[0] = "unpack-objects"; + unpacker[1] = hdr_arg; + unpacker[2] = NULL; + code = run_command_v_opt(1, unpacker, RUN_GIT_CMD); + } else { + const char *keeper[6]; + char my_host[255], keep_arg[128 + 255]; + + if (gethostname(my_host, sizeof(my_host))) + strcpy(my_host, "localhost"); + snprintf(keep_arg, sizeof(keep_arg), + "--keep=receive-pack %i on %s", + getpid(), my_host); + + keeper[0] = "index-pack"; + keeper[1] = "--stdin"; + keeper[2] = "--fix-thin"; + keeper[3] = hdr_arg; + keeper[4] = keep_arg; + keeper[5] = NULL; + code = run_command_v_opt(1, keeper, RUN_GIT_CMD); + if (!code) + reprepare_packed_git(); + } switch (code) { - case 0: - return NULL; - case -ERR_RUN_COMMAND_FORK: - return "unpack fork failed"; - case -ERR_RUN_COMMAND_EXEC: - return "unpack execute failed"; - case -ERR_RUN_COMMAND_WAITPID: - return "waitpid failed"; - case -ERR_RUN_COMMAND_WAITPID_WRONG_PID: - return "waitpid is confused"; - case -ERR_RUN_COMMAND_WAITPID_SIGNAL: - return "unpacker died of signal"; - case -ERR_RUN_COMMAND_WAITPID_NOEXIT: - return "unpacker died strangely"; - default: - return "unpacker exited with error code"; + case 0: + return NULL; + case -ERR_RUN_COMMAND_FORK: + return "unpack fork failed"; + case -ERR_RUN_COMMAND_EXEC: + return "unpack execute failed"; + case -ERR_RUN_COMMAND_WAITPID: + return "waitpid failed"; + case -ERR_RUN_COMMAND_WAITPID_WRONG_PID: + return "waitpid is confused"; + case -ERR_RUN_COMMAND_WAITPID_SIGNAL: + return "unpacker died of signal"; + case -ERR_RUN_COMMAND_WAITPID_NOEXIT: + return "unpacker died strangely"; + default: + return "unpacker exited with error code"; } } diff --git a/sha1_file.c b/sha1_file.c index 2aa944abd0..6ea59b5588 100644 --- a/sha1_file.c +++ b/sha1_file.c @@ -663,7 +663,7 @@ void prepare_packed_git(void) prepare_packed_git_run_once = 1; } -static void reprepare_packed_git(void) +void reprepare_packed_git(void) { prepare_packed_git_run_once = 0; prepare_packed_git(); -- cgit v1.3 From 920ccbfc3bedd384f3073c342c64fbd028a4ee3a Mon Sep 17 00:00:00 2001 From: Nicolas Pitre Date: Wed, 1 Nov 2006 17:06:22 -0500 Subject: git-fetch can use both --thin and --keep with fetch-pack now Signed-off-by: Nicolas Pitre Signed-off-by: Junio C Hamano --- git-fetch.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/git-fetch.sh b/git-fetch.sh index 539dff6ee0..2b5538f176 100755 --- a/git-fetch.sh +++ b/git-fetch.sh @@ -20,7 +20,7 @@ verbose= update_head_ok= exec= upload_pack= -keep=--thin +keep= while case "$#" in 0) break ;; esac do case "$1" in @@ -370,7 +370,7 @@ fetch_main () { ( : subshell because we muck with IFS IFS=" $LF" ( - git-fetch-pack $exec $keep "$remote" $rref || echo failed "$remote" + git-fetch-pack --thin $exec $keep "$remote" $rref || echo failed "$remote" ) | while read sha1 remote_name do -- cgit v1.3 From da093d37506f034bfb14e6de26cce9c14b1a1216 Mon Sep 17 00:00:00 2001 From: Nicolas Pitre Date: Wed, 1 Nov 2006 17:06:23 -0500 Subject: improve fetch-pack's handling of kept packs Since functions in fetch-clone.c were only used from fetch-pack.c, its content has been merged with fetch-pack.c. This allows for better coupling of features with much simpler implementations. One new thing is that the (abscence of) --thin also enforce it on index-pack now, such that index-pack will abort if a thin pack was _not_ asked for. The -k or --keep, when provided twice, now causes the fetched pack to be left as a kept pack just like receive-pack currently does. Eventually this will be used to close a race against concurrent repacking. Signed-off-by: Nicolas Pitre Signed-off-by: Junio C Hamano --- Documentation/git-fetch-pack.txt | 3 +- Makefile | 2 +- cache.h | 4 -- fetch-clone.c | 87 ------------------------------- fetch-pack.c | 110 ++++++++++++++++++++++++++++++++++++--- 5 files changed, 106 insertions(+), 100 deletions(-) delete mode 100644 fetch-clone.c diff --git a/Documentation/git-fetch-pack.txt b/Documentation/git-fetch-pack.txt index bff9aa6939..3e6cd880b0 100644 --- a/Documentation/git-fetch-pack.txt +++ b/Documentation/git-fetch-pack.txt @@ -32,7 +32,8 @@ OPTIONS -k:: Do not invoke 'git-unpack-objects' on received data, but create a single packfile out of it instead, and store it - in the object database. + in the object database. If provided twice then the pack is + locked against repacking. --exec=:: Use this to specify the path to 'git-upload-pack' on the diff --git a/Makefile b/Makefile index 1cc9f586d6..9bf50bcf24 100644 --- a/Makefile +++ b/Makefile @@ -260,7 +260,7 @@ LIB_OBJS = \ quote.o read-cache.o refs.o run-command.o dir.o object-refs.o \ server-info.o setup.o sha1_file.o sha1_name.o strbuf.o \ tag.o tree.o usage.o config.o environment.o ctype.o copy.o \ - fetch-clone.o revision.o pager.o tree-walk.o xdiff-interface.o \ + revision.o pager.o tree-walk.o xdiff-interface.o \ write_or_die.o trace.o list-objects.o grep.o \ alloc.o merge-file.o path-list.o help.o unpack-trees.o $(DIFF_OBJS) \ color.o wt-status.o archive-zip.o archive-tar.o diff --git a/cache.h b/cache.h index 6cb7e1d8c0..f2ec5c8c14 100644 --- a/cache.h +++ b/cache.h @@ -416,10 +416,6 @@ extern int copy_fd(int ifd, int ofd); extern void write_or_die(int fd, const void *buf, size_t count); extern int write_or_whine(int fd, const void *buf, size_t count, const char *msg); -/* Finish off pack transfer receiving end */ -extern int receive_unpack_pack(int fd[2], const char *me, int quiet, int); -extern int receive_keep_pack(int fd[2], const char *me, int quiet, int); - /* pager.c */ extern void setup_pager(void); extern int pager_in_use; diff --git a/fetch-clone.c b/fetch-clone.c deleted file mode 100644 index f629d8dacc..0000000000 --- a/fetch-clone.c +++ /dev/null @@ -1,87 +0,0 @@ -#include "cache.h" -#include "exec_cmd.h" -#include "pkt-line.h" -#include "sideband.h" -#include - -static pid_t setup_sideband(int sideband, const char *me, int fd[2], int xd[2]) -{ - pid_t side_pid; - - if (!sideband) { - fd[0] = xd[0]; - fd[1] = xd[1]; - return 0; - } - /* xd[] is talking with upload-pack; subprocess reads from - * xd[0], spits out band#2 to stderr, and feeds us band#1 - * through our fd[0]. - */ - if (pipe(fd) < 0) - die("%s: unable to set up pipe", me); - side_pid = fork(); - if (side_pid < 0) - die("%s: unable to fork off sideband demultiplexer", me); - if (!side_pid) { - /* subprocess */ - close(fd[0]); - if (xd[0] != xd[1]) - close(xd[1]); - if (recv_sideband(me, xd[0], fd[1], 2)) - exit(1); - exit(0); - } - close(xd[0]); - close(fd[1]); - fd[1] = xd[1]; - return side_pid; -} - -static int get_pack(int xd[2], const char *me, int sideband, const char **argv) -{ - int status; - pid_t pid, side_pid; - int fd[2]; - - side_pid = setup_sideband(sideband, me, fd, xd); - pid = fork(); - if (pid < 0) - die("%s: unable to fork off %s", me, argv[0]); - if (!pid) { - dup2(fd[0], 0); - close(fd[0]); - close(fd[1]); - execv_git_cmd(argv); - die("%s exec failed", argv[0]); - } - close(fd[0]); - close(fd[1]); - while (waitpid(pid, &status, 0) < 0) { - if (errno != EINTR) - die("waiting for %s: %s", argv[0], strerror(errno)); - } - if (WIFEXITED(status)) { - int code = WEXITSTATUS(status); - if (code) - die("%s died with error code %d", argv[0], code); - return 0; - } - if (WIFSIGNALED(status)) { - int sig = WTERMSIG(status); - die("%s died of signal %d", argv[0], sig); - } - die("%s died of unnatural causes %d", argv[0], status); -} - -int receive_unpack_pack(int xd[2], const char *me, int quiet, int sideband) -{ - const char *argv[3] = { "unpack-objects", quiet ? "-q" : NULL, NULL }; - return get_pack(xd, me, sideband, argv); -} - -int receive_keep_pack(int xd[2], const char *me, int quiet, int sideband) -{ - const char *argv[5] = { "index-pack", "--stdin", "--fix-thin", - quiet ? NULL : "-v", NULL }; - return get_pack(xd, me, sideband, argv); -} diff --git a/fetch-pack.c b/fetch-pack.c index 36ea092d1b..0a169dce85 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -3,6 +3,9 @@ #include "pkt-line.h" #include "commit.h" #include "tag.h" +#include "exec_cmd.h" +#include "sideband.h" +#include static int keep_pack; static int quiet; @@ -416,6 +419,103 @@ static int everything_local(struct ref **refs, int nr_match, char **match) return retval; } +static pid_t setup_sideband(int fd[2], int xd[2]) +{ + pid_t side_pid; + + if (!use_sideband) { + fd[0] = xd[0]; + fd[1] = xd[1]; + return 0; + } + /* xd[] is talking with upload-pack; subprocess reads from + * xd[0], spits out band#2 to stderr, and feeds us band#1 + * through our fd[0]. + */ + if (pipe(fd) < 0) + die("fetch-pack: unable to set up pipe"); + side_pid = fork(); + if (side_pid < 0) + die("fetch-pack: unable to fork off sideband demultiplexer"); + if (!side_pid) { + /* subprocess */ + close(fd[0]); + if (xd[0] != xd[1]) + close(xd[1]); + if (recv_sideband("fetch-pack", xd[0], fd[1], 2)) + exit(1); + exit(0); + } + close(xd[0]); + close(fd[1]); + fd[1] = xd[1]; + return side_pid; +} + +static int get_pack(int xd[2], const char **argv) +{ + int status; + pid_t pid, side_pid; + int fd[2]; + + side_pid = setup_sideband(fd, xd); + pid = fork(); + if (pid < 0) + die("fetch-pack: unable to fork off %s", argv[0]); + if (!pid) { + dup2(fd[0], 0); + close(fd[0]); + close(fd[1]); + execv_git_cmd(argv); + die("%s exec failed", argv[0]); + } + close(fd[0]); + close(fd[1]); + while (waitpid(pid, &status, 0) < 0) { + if (errno != EINTR) + die("waiting for %s: %s", argv[0], strerror(errno)); + } + if (WIFEXITED(status)) { + int code = WEXITSTATUS(status); + if (code) + die("%s died with error code %d", argv[0], code); + return 0; + } + if (WIFSIGNALED(status)) { + int sig = WTERMSIG(status); + die("%s died of signal %d", argv[0], sig); + } + die("%s died of unnatural causes %d", argv[0], status); +} + +static int explode_rx_pack(int xd[2]) +{ + const char *argv[3] = { "unpack-objects", quiet ? "-q" : NULL, NULL }; + return get_pack(xd, argv); +} + +static int keep_rx_pack(int xd[2]) +{ + const char *argv[6]; + char keep_arg[256]; + int n = 0; + + argv[n++] = "index-pack"; + argv[n++] = "--stdin"; + if (!quiet) + argv[n++] = "-v"; + if (use_thin_pack) + argv[n++] = "--fix-thin"; + if (keep_pack > 1) { + int s = sprintf(keep_arg, "--keep=fetch-pack %i on ", getpid()); + if (gethostname(keep_arg + s, sizeof(keep_arg) - s)) + strcpy(keep_arg + s, "localhost"); + argv[n++] = keep_arg; + } + argv[n] = NULL; + return get_pack(xd, argv); +} + static int fetch_pack(int fd[2], int nr_match, char **match) { struct ref *ref; @@ -447,17 +547,13 @@ static int fetch_pack(int fd[2], int nr_match, char **match) goto all_done; } if (find_common(fd, sha1, ref) < 0) - if (!keep_pack) + if (keep_pack != 1) /* When cloning, it is not unusual to have * no common commit. */ fprintf(stderr, "warning: no common commits\n"); - if (keep_pack) - status = receive_keep_pack(fd, "git-fetch-pack", quiet, use_sideband); - else - status = receive_unpack_pack(fd, "git-fetch-pack", quiet, use_sideband); - + status = (keep_pack) ? keep_rx_pack(fd) : explode_rx_pack(fd); if (status) die("git-fetch-pack: fetch failed."); @@ -494,7 +590,7 @@ int main(int argc, char **argv) continue; } if (!strcmp("--keep", arg) || !strcmp("-k", arg)) { - keep_pack = 1; + keep_pack++; continue; } if (!strcmp("--thin", arg)) { -- cgit v1.3 From 9ca4a201eaf0c58dbc7184cb2d5ab01c48cb7447 Mon Sep 17 00:00:00 2001 From: Nicolas Pitre Date: Wed, 1 Nov 2006 17:06:24 -0500 Subject: have index-pack create .keep file more carefully If by chance we receive a pack which content (list of objects) matches another pack that we already have, and if that pack is marked with a .keep file, then we should not overwrite it. Signed-off-by: Nicolas Pitre Signed-off-by: Junio C Hamano --- index-pack.c | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/index-pack.c b/index-pack.c index a3b55f9b0f..8d64a883af 100644 --- a/index-pack.c +++ b/index-pack.c @@ -788,14 +788,17 @@ static void final(const char *final_pack_name, const char *curr_pack_name, get_object_directory(), sha1_to_hex(sha1)); keep_name = name; } - keep_fd = open(keep_name, O_RDWR | O_CREAT, 0600); - if (keep_fd < 0) - die("cannot write keep file"); - if (keep_msg_len > 0) { - write_or_die(keep_fd, keep_msg, keep_msg_len); - write_or_die(keep_fd, "\n", 1); + keep_fd = open(keep_name, O_RDWR|O_CREAT|O_EXCL, 0600); + if (keep_fd < 0) { + if (errno != EEXIST) + die("cannot write keep file"); + } else { + if (keep_msg_len > 0) { + write_or_die(keep_fd, keep_msg, keep_msg_len); + write_or_die(keep_fd, "\n", 1); + } + close(keep_fd); } - close(keep_fd); } if (final_pack_name != curr_pack_name) { -- cgit v1.3 From 576162a45f35e157427300066b0ff566ff698a0f Mon Sep 17 00:00:00 2001 From: Nicolas Pitre Date: Wed, 1 Nov 2006 17:06:25 -0500 Subject: remove .keep pack lock files when done with refs update This makes both git-fetch and git-push (fetch-pack and receive-pack) safe against a possible race with aparallel git-repack -a -d that could prune the new pack while it is not yet referenced, and remove the .keep file after refs have been updated. Signed-off-by: Nicolas Pitre Signed-off-by: Junio C Hamano --- Documentation/git-index-pack.txt | 11 +++++ git-fetch.sh | 10 ++++- index-pack.c | 38 +++++++++------- receive-pack.c | 96 ++++++++++++++++++++++++++++++---------- 4 files changed, 116 insertions(+), 39 deletions(-) diff --git a/Documentation/git-index-pack.txt b/Documentation/git-index-pack.txt index 1235416e09..2229ee86b7 100644 --- a/Documentation/git-index-pack.txt +++ b/Documentation/git-index-pack.txt @@ -69,6 +69,17 @@ OPTIONS locate any which have outlived their usefulness. +Note +---- + +Once the index has been created, the list of object names is sorted +and the SHA1 hash of that list is printed to stdout. If --stdin was +also used then this is prefixed by either "pack\t", or "keep\t" if a +new .keep file was successfully created. This is useful to remove a +.keep file used as a lock to prevent the race with gitlink:git-repack[1] +mentioned above. + + Author ------ Written by Sergey Vlasov diff --git a/git-fetch.sh b/git-fetch.sh index 2b5538f176..7442dd2ca5 100755 --- a/git-fetch.sh +++ b/git-fetch.sh @@ -51,7 +51,7 @@ do verbose=Yes ;; -k|--k|--ke|--kee|--keep) - keep=--keep + keep='-k -k' ;; --reflog-action=*) rloga=`expr "z$1" : 'z-[^=]*=\(.*\)'` @@ -368,6 +368,7 @@ fetch_main () { ;; # we are already done. *) ( : subshell because we muck with IFS + pack_lockfile= IFS=" $LF" ( git-fetch-pack --thin $exec $keep "$remote" $rref || echo failed "$remote" @@ -378,6 +379,12 @@ fetch_main () { failed) echo >&2 "Fetch failure: $remote" exit 1 ;; + # special line coming from index-pack with the pack name + pack) + continue ;; + keep) + pack_lockfile="$GIT_OBJECT_DIRECTORY/pack/pack-$remote_name.keep" + continue ;; esac found= single_force= @@ -408,6 +415,7 @@ fetch_main () { append_fetch_head "$sha1" "$remote" \ "$remote_name" "$remote_nick" "$local_name" "$not_for_merge" done + if [ "$pack_lockfile" ]; then rm -f "$pack_lockfile"; fi ) || exit ;; esac diff --git a/index-pack.c b/index-pack.c index 8d64a883af..042aea8842 100644 --- a/index-pack.c +++ b/index-pack.c @@ -757,6 +757,7 @@ static void final(const char *final_pack_name, const char *curr_pack_name, const char *keep_name, const char *keep_msg, unsigned char *sha1) { + char *report = "pack"; char name[PATH_MAX]; int err; @@ -767,18 +768,6 @@ static void final(const char *final_pack_name, const char *curr_pack_name, if (err) die("error while closing pack file: %s", strerror(errno)); chmod(curr_pack_name, 0444); - - /* - * Let's just mimic git-unpack-objects here and write - * the last part of the buffer to stdout. - */ - while (input_len) { - err = xwrite(1, input_buffer + input_offset, input_len); - if (err <= 0) - break; - input_len -= err; - input_offset += err; - } } if (keep_msg) { @@ -798,6 +787,7 @@ static void final(const char *final_pack_name, const char *curr_pack_name, write_or_die(keep_fd, "\n", 1); } close(keep_fd); + report = "keep"; } } @@ -821,6 +811,27 @@ static void final(const char *final_pack_name, const char *curr_pack_name, if (move_temp_to_file(curr_index_name, final_index_name)) die("cannot store index file"); } + + if (!from_stdin) { + printf("%s\n", sha1_to_hex(sha1)); + } else { + char buf[48]; + int len = snprintf(buf, sizeof(buf), "%s\t%s\n", + report, sha1_to_hex(sha1)); + xwrite(1, buf, len); + + /* + * Let's just mimic git-unpack-objects here and write + * the last part of the input buffer to stdout. + */ + while (input_len) { + err = xwrite(1, input_buffer + input_offset, input_len); + if (err <= 0) + break; + input_len -= err; + input_offset += err; + } + } } int main(int argc, char **argv) @@ -937,8 +948,5 @@ int main(int argc, char **argv) free(index_name_buf); free(keep_name_buf); - if (!from_stdin) - printf("%s\n", sha1_to_hex(sha1)); - return 0; } diff --git a/receive-pack.c b/receive-pack.c index b8d12700d8..d56898c9b2 100644 --- a/receive-pack.c +++ b/receive-pack.c @@ -3,8 +3,10 @@ #include "refs.h" #include "pkt-line.h" #include "run-command.h" +#include "exec_cmd.h" #include "commit.h" #include "object.h" +#include static const char receive_pack_usage[] = "git-receive-pack "; @@ -251,12 +253,13 @@ static const char *parse_pack_header(struct pack_header *hdr) return NULL; } +static const char *pack_lockfile; + static const char *unpack(void) { struct pack_header hdr; const char *hdr_err; char hdr_arg[38]; - int code; hdr_err = parse_pack_header(&hdr); if (hdr_err) @@ -265,33 +268,13 @@ static const char *unpack(void) ntohl(hdr.hdr_version), ntohl(hdr.hdr_entries)); if (ntohl(hdr.hdr_entries) < unpack_limit) { + int code; const char *unpacker[3]; unpacker[0] = "unpack-objects"; unpacker[1] = hdr_arg; unpacker[2] = NULL; code = run_command_v_opt(1, unpacker, RUN_GIT_CMD); - } else { - const char *keeper[6]; - char my_host[255], keep_arg[128 + 255]; - - if (gethostname(my_host, sizeof(my_host))) - strcpy(my_host, "localhost"); - snprintf(keep_arg, sizeof(keep_arg), - "--keep=receive-pack %i on %s", - getpid(), my_host); - - keeper[0] = "index-pack"; - keeper[1] = "--stdin"; - keeper[2] = "--fix-thin"; - keeper[3] = hdr_arg; - keeper[4] = keep_arg; - keeper[5] = NULL; - code = run_command_v_opt(1, keeper, RUN_GIT_CMD); - if (!code) - reprepare_packed_git(); - } - - switch (code) { + switch (code) { case 0: return NULL; case -ERR_RUN_COMMAND_FORK: @@ -308,6 +291,71 @@ static const char *unpack(void) return "unpacker died strangely"; default: return "unpacker exited with error code"; + } + } else { + const char *keeper[6]; + int fd[2], s, len, status; + pid_t pid; + char keep_arg[256]; + char packname[46]; + + s = sprintf(keep_arg, "--keep=receive-pack %i on ", getpid()); + if (gethostname(keep_arg + s, sizeof(keep_arg) - s)) + strcpy(keep_arg + s, "localhost"); + + keeper[0] = "index-pack"; + keeper[1] = "--stdin"; + keeper[2] = "--fix-thin"; + keeper[3] = hdr_arg; + keeper[4] = keep_arg; + keeper[5] = NULL; + + if (pipe(fd) < 0) + return "index-pack pipe failed"; + pid = fork(); + if (pid < 0) + return "index-pack fork failed"; + if (!pid) { + dup2(fd[1], 1); + close(fd[1]); + close(fd[0]); + execv_git_cmd(keeper); + die("execv of index-pack failed"); + } + close(fd[1]); + + /* + * The first thing we expects from index-pack's output + * is "pack\t%40s\n" or "keep\t%40s\n" (46 bytes) where + * %40s is the newly created pack SHA1 name. In the "keep" + * case, we need it to remove the corresponding .keep file + * later on. If we don't get that then tough luck with it. + */ + for (len = 0; + len < 46 && (s = xread(fd[0], packname+len, 46-len)) > 0; + len += s); + close(fd[0]); + if (len == 46 && packname[45] == '\n' && + memcmp(packname, "keep\t", 5) == 0) { + char path[PATH_MAX]; + packname[45] = 0; + snprintf(path, sizeof(path), "%s/pack/pack-%s.keep", + get_object_directory(), packname + 5); + pack_lockfile = xstrdup(path); + } + + /* Then wrap our index-pack process. */ + while (waitpid(pid, &status, 0) < 0) + if (errno != EINTR) + return "waitpid failed"; + if (WIFEXITED(status)) { + int code = WEXITSTATUS(status); + if (code) + return "index-pack exited with error code"; + reprepare_packed_git(); + return NULL; + } + return "index-pack abnormal exit"; } } @@ -363,6 +411,8 @@ int main(int argc, char **argv) const char *unpack_status = unpack(); if (!unpack_status) execute_commands(); + if (pack_lockfile) + unlink(pack_lockfile); if (report_status) report(unpack_status); } -- cgit v1.3