From 84c8d8aec535e1d92c5acc36e84ac8e00f6e0e7a Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Wed, 12 Oct 2005 16:54:19 -0700 Subject: Fix packname hash generation. This changes the generation of hash packfiles have in their names, from "hash of object names as fed to us" to "hash of object names in the resulting pack, in the order they appear in the index file". The new "git-index-pack" command is taught to output the computed hash value to its standard output. With this, we can store downloaded pack in a temporary file without knowing its final name, run git-index-pack to generate idx for it while finding out its final name, and then rename the pack and idx to their final names. Signed-off-by: Junio C Hamano --- pack-objects.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) (limited to 'pack-objects.c') diff --git a/pack-objects.c b/pack-objects.c index 3d622787cc..ef55cab5f3 100644 --- a/pack-objects.c +++ b/pack-objects.c @@ -393,6 +393,7 @@ int main(int argc, char **argv) SHA_CTX ctx; char line[PATH_MAX + 20]; int window = 10, depth = 10, pack_to_stdout = 0; + struct object_entry **list; int i; for (i = 1; i < argc; i++) { @@ -435,7 +436,6 @@ int main(int argc, char **argv) if (pack_to_stdout != !base_name) usage(pack_usage); - SHA1_Init(&ctx); while (fgets(line, sizeof(line), stdin) != NULL) { unsigned int hash; char *p; @@ -451,10 +451,8 @@ int main(int argc, char **argv) continue; hash = hash * 11 + c; } - if (add_object_entry(sha1, hash)) - SHA1_Update(&ctx, sha1, 20); + add_object_entry(sha1, hash); } - SHA1_Final(object_list_sha1, &ctx); if (non_empty && !nr_objects) return 0; get_object_details(); @@ -462,6 +460,14 @@ int main(int argc, char **argv) fprintf(stderr, "Packing %d objects\n", nr_objects); sorted_by_sha = create_sorted_list(sha1_sort); + SHA1_Init(&ctx); + list = sorted_by_sha; + for (i = 0; i < nr_objects; i++) { + struct object_entry *entry = *list++; + SHA1_Update(&ctx, entry->sha1, 20); + } + SHA1_Final(object_list_sha1, &ctx); + sorted_by_type = create_sorted_list(type_size_sort); if (window && depth) find_deltas(sorted_by_type, window+1, depth); -- cgit v1.3 From 64560374cc6c3ebe595540f3425b2b38036cf253 Mon Sep 17 00:00:00 2001 From: Linus Torvalds Date: Thu, 13 Oct 2005 15:38:28 -0700 Subject: Add support for "local" packing This adds the "--local" flag to git-pack-objects, which acts like "--incremental", except that instead of ignoring all packed objects, it only ignores objects that are packed and in an alternate object tree. As a result, it effectively only does a local re-pack: any remote-packed objects will stay in the alternate object directories. Signed-off-by: Linus Torvalds Signed-off-by: Junio C Hamano --- pack-objects.c | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) (limited to 'pack-objects.c') diff --git a/pack-objects.c b/pack-objects.c index ef55cab5f3..8a1ee746e0 100644 --- a/pack-objects.c +++ b/pack-objects.c @@ -5,7 +5,7 @@ #include "pack.h" #include "csum-file.h" -static const char pack_usage[] = "git-pack-objects [--incremental] [--window=N] [--depth=N] {--stdout | base-name} < object-list"; +static const char pack_usage[] = "git-pack-objects [--local] [--incremental] [--window=N] [--depth=N] {--stdout | base-name} < object-list"; struct object_entry { unsigned char sha1[20]; @@ -20,6 +20,7 @@ struct object_entry { static unsigned char object_list_sha1[20]; static int non_empty = 0; +static int local = 0; static int incremental = 0; static struct object_entry **sorted_by_sha, **sorted_by_type; static struct object_entry *objects = NULL; @@ -195,8 +196,20 @@ static int add_object_entry(unsigned char *sha1, unsigned int hash) unsigned int idx = nr_objects; struct object_entry *entry; - if (incremental && has_sha1_pack(sha1)) - return 0; + if (incremental || local) { + struct packed_git *p; + + for (p = packed_git; p; p = p->next) { + struct pack_entry e; + + if (find_pack_entry_one(sha1, &e, p)) { + if (incremental) + return 0; + if (local && !p->pack_local) + return 0; + } + } + } if (idx >= nr_alloc) { unsigned int needed = (idx + 1024) * 3 / 2; @@ -404,6 +417,10 @@ int main(int argc, char **argv) non_empty = 1; continue; } + if (!strcmp("--local", arg)) { + local = 1; + continue; + } if (!strcmp("--incremental", arg)) { incremental = 1; continue; @@ -436,6 +453,7 @@ int main(int argc, char **argv) if (pack_to_stdout != !base_name) usage(pack_usage); + prepare_packed_git(); while (fgets(line, sizeof(line), stdin) != NULL) { unsigned int hash; char *p; -- cgit v1.3 From 4546738b58a0134eef154231b07d60fc174d56e3 Mon Sep 17 00:00:00 2001 From: Linus Torvalds Date: Thu, 13 Oct 2005 11:03:18 -0700 Subject: Unlocalized isspace and friends Do our own ctype.h, just to get the sane semantics: we want locale-independence, _and_ we want the right signed behaviour. Plus we only use a very small subset of ctype.h anyway (isspace, isalpha, isdigit and isalnum). Signed-off-by: Junio C Hamano --- Makefile | 3 ++- apply.c | 1 - cache.h | 26 ++++++++++++++++++++++++++ commit-tree.c | 1 - commit.c | 1 - config.c | 1 - convert-objects.c | 1 - ctype.c | 23 +++++++++++++++++++++++ date.c | 1 - diff-tree.c | 1 - ident.c | 1 - mailsplit.c | 1 - pack-objects.c | 1 - patch-id.c | 1 - refs.c | 1 - update-ref.c | 1 - 16 files changed, 51 insertions(+), 14 deletions(-) create mode 100644 ctype.c (limited to 'pack-objects.c') diff --git a/Makefile b/Makefile index e2e87f6beb..9fe65ba2be 100644 --- a/Makefile +++ b/Makefile @@ -159,7 +159,8 @@ LIB_OBJS = \ object.o pack-check.o patch-delta.o path.o pkt-line.o \ quote.o read-cache.o refs.o run-command.o \ server-info.o setup.o sha1_file.o sha1_name.o strbuf.o \ - tag.o tree.o usage.o config.o environment.o $(DIFF_OBJS) + tag.o tree.o usage.o config.o environment.o ctype.o \ + $(DIFF_OBJS) LIBS = $(LIB_FILE) LIBS += -lz diff --git a/apply.c b/apply.c index 155fbe84da..f4d00f2835 100644 --- a/apply.c +++ b/apply.c @@ -6,7 +6,6 @@ * This applies patches on top of some (arbitrary) version of the SCM. * */ -#include #include #include "cache.h" diff --git a/cache.h b/cache.h index 328658235b..f1d15ab3c9 100644 --- a/cache.h +++ b/cache.h @@ -387,4 +387,30 @@ extern int git_config_bool(const char *, const char *); extern char git_default_email[MAX_GITNAME]; extern char git_default_name[MAX_GITNAME]; +/* Sane ctype - no locale, and works with signed chars */ +#undef isspace +#undef isdigit +#undef isalpha +#undef isalnum +#undef tolower +#undef toupper +extern unsigned char sane_ctype[256]; +#define GIT_SPACE 0x01 +#define GIT_DIGIT 0x02 +#define GIT_ALPHA 0x04 +#define sane_istest(x,mask) ((sane_ctype[(unsigned char)(x)] & (mask)) != 0) +#define isspace(x) sane_istest(x,GIT_SPACE) +#define isdigit(x) sane_istest(x,GIT_DIGIT) +#define isalpha(x) sane_istest(x,GIT_ALPHA) +#define isalnum(x) sane_istest(x,GIT_ALPHA | GIT_DIGIT) +#define tolower(x) sane_case((unsigned char)(x), 0x20) +#define toupper(x) sane_case((unsigned char)(x), 0) + +static inline int sane_case(int x, int high) +{ + if (sane_istest(x, GIT_ALPHA)) + x = (x & ~0x20) | high; + return x; +} + #endif /* CACHE_H */ diff --git a/commit-tree.c b/commit-tree.c index 030fb704e5..ea0fdd44e2 100644 --- a/commit-tree.c +++ b/commit-tree.c @@ -7,7 +7,6 @@ #include #include -#include #define BLOCKING (1ul << 14) diff --git a/commit.c b/commit.c index f735f981bb..8f403180e5 100644 --- a/commit.c +++ b/commit.c @@ -1,4 +1,3 @@ -#include #include "tag.h" #include "commit.h" #include "cache.h" diff --git a/config.c b/config.c index 9b7c6f2942..519fecfee4 100644 --- a/config.c +++ b/config.c @@ -1,4 +1,3 @@ -#include #include "cache.h" diff --git a/convert-objects.c b/convert-objects.c index 9ad0c77678..a892013f0f 100644 --- a/convert-objects.c +++ b/convert-objects.c @@ -1,6 +1,5 @@ #define _XOPEN_SOURCE /* glibc2 needs this */ #include -#include #include "cache.h" struct entry { diff --git a/ctype.c b/ctype.c new file mode 100644 index 0000000000..56bdffa636 --- /dev/null +++ b/ctype.c @@ -0,0 +1,23 @@ +/* + * Sane locale-independent, ASCII ctype. + * + * No surprises, and works with signed and unsigned chars. + */ +#include "cache.h" + +#define SS GIT_SPACE +#define AA GIT_ALPHA +#define DD GIT_DIGIT + +unsigned char sane_ctype[256] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, SS, SS, 0, 0, SS, 0, 0, /* 0-15 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 16-15 */ + SS, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 32-15 */ + DD, DD, DD, DD, DD, DD, DD, DD, DD, DD, 0, 0, 0, 0, 0, 0, /* 48-15 */ + 0, AA, AA, AA, AA, AA, AA, AA, AA, AA, AA, AA, AA, AA, AA, AA, /* 64-15 */ + AA, AA, AA, AA, AA, AA, AA, AA, AA, AA, AA, 0, 0, 0, 0, 0, /* 80-15 */ + 0, AA, AA, AA, AA, AA, AA, AA, AA, AA, AA, AA, AA, AA, AA, AA, /* 96-15 */ + AA, AA, AA, AA, AA, AA, AA, AA, AA, AA, AA, 0, 0, 0, 0, 0, /* 112-15 */ + /* Nothing in the 128.. range */ +}; + diff --git a/date.c b/date.c index b21cadc4d6..63f5a09197 100644 --- a/date.c +++ b/date.c @@ -4,7 +4,6 @@ * Copyright (C) Linus Torvalds, 2005 */ -#include #include #include "cache.h" diff --git a/diff-tree.c b/diff-tree.c index 2203fa56d0..851722037d 100644 --- a/diff-tree.c +++ b/diff-tree.c @@ -1,4 +1,3 @@ -#include #include "cache.h" #include "diff.h" #include "commit.h" diff --git a/ident.c b/ident.c index 7a9f5672eb..1bfbc6ff35 100644 --- a/ident.c +++ b/ident.c @@ -9,7 +9,6 @@ #include #include -#include static char git_default_date[50]; diff --git a/mailsplit.c b/mailsplit.c index 0f8100dcca..189f4ed724 100644 --- a/mailsplit.c +++ b/mailsplit.c @@ -11,7 +11,6 @@ #include #include #include -#include #include #include "cache.h" diff --git a/pack-objects.c b/pack-objects.c index 8a1ee746e0..b3e6152033 100644 --- a/pack-objects.c +++ b/pack-objects.c @@ -1,4 +1,3 @@ -#include #include "cache.h" #include "object.h" #include "delta.h" diff --git a/patch-id.c b/patch-id.c index 960e7cedf9..edbc4aa3e8 100644 --- a/patch-id.c +++ b/patch-id.c @@ -1,4 +1,3 @@ -#include #include "cache.h" static void flush_current_id(int patchlen, unsigned char *id, SHA_CTX *c) diff --git a/refs.c b/refs.c index 5a8cbd4ef3..42240d2769 100644 --- a/refs.c +++ b/refs.c @@ -2,7 +2,6 @@ #include "cache.h" #include -#include /* We allow "recursive" symbolic refs. Only within reason, though */ #define MAXDEPTH 5 diff --git a/update-ref.c b/update-ref.c index 4a1704c1a5..65dc3d6385 100644 --- a/update-ref.c +++ b/update-ref.c @@ -1,6 +1,5 @@ #include "cache.h" #include "refs.h" -#include static const char git_update_ref_usage[] = "git-update-ref []"; -- cgit v1.3 From f3123c4ab3d3698262e59561ac084de45b10365a Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Sat, 22 Oct 2005 01:28:13 -0700 Subject: pack-objects: Allow use of pre-generated pack. git-pack-objects can reuse pack files stored in $GIT_DIR/pack-cache directory, when a necessary pack is found. This is hopefully useful when upload-pack (called from git-daemon) is expected to receive requests for the same set of objects many times (e.g full cloning request of any project, or updates from the set of heads previous day to the latest for a slow moving project). Currently git-pack-objects does *not* keep pack files it creates for reusing. It might be useful to add --update-cache option to it, which would allow it store pack files it created in the pack-cache directory, and prune rarely used ones from it. Signed-off-by: Junio C Hamano --- Makefile | 2 +- cache.h | 1 + copy.c | 37 ++++++++++++++++++++++++++ pack-objects.c | 84 ++++++++++++++++++++++++++++++++++++++++++++++++++-------- 4 files changed, 112 insertions(+), 12 deletions(-) create mode 100644 copy.c (limited to 'pack-objects.c') diff --git a/Makefile b/Makefile index 5b0306d391..701067d435 100644 --- a/Makefile +++ b/Makefile @@ -159,7 +159,7 @@ LIB_OBJS = \ object.o pack-check.o patch-delta.o path.o pkt-line.o \ quote.o read-cache.o refs.o run-command.o \ server-info.o setup.o sha1_file.o sha1_name.o strbuf.o \ - tag.o tree.o usage.o config.o environment.o ctype.o \ + tag.o tree.o usage.o config.o environment.o ctype.o copy.o \ $(DIFF_OBJS) LIBS = $(LIB_FILE) diff --git a/cache.h b/cache.h index d776016822..2e36cc5b8b 100644 --- a/cache.h +++ b/cache.h @@ -413,4 +413,5 @@ static inline int sane_case(int x, int high) return x; } +extern int copy_fd(int ifd, int ofd); #endif /* CACHE_H */ diff --git a/copy.c b/copy.c new file mode 100644 index 0000000000..20092757d3 --- /dev/null +++ b/copy.c @@ -0,0 +1,37 @@ +#include "cache.h" + +int copy_fd(int ifd, int ofd) +{ + while (1) { + int len; + char buffer[8192]; + char *buf = buffer; + len = read(ifd, buffer, sizeof(buffer)); + if (!len) + break; + if (len < 0) { + if (errno == EAGAIN) + continue; + return error("copy-fd: read returned %s", + strerror(errno)); + } + while (1) { + int written = write(ofd, buf, len); + if (written > 0) { + buf += written; + len -= written; + if (!len) + break; + } + if (!written) + return error("copy-fd: write returned 0"); + if (errno == EAGAIN || errno == EINTR) + continue; + return error("copy-fd: write returned %s", + strerror(errno)); + } + } + close(ifd); + return 0; +} + diff --git a/pack-objects.c b/pack-objects.c index b3e6152033..4e941e7392 100644 --- a/pack-objects.c +++ b/pack-objects.c @@ -400,6 +400,71 @@ static void find_deltas(struct object_entry **list, int window, int depth) free(array); } +static void prepare_pack(int window, int depth) +{ + get_object_details(); + + fprintf(stderr, "Packing %d objects\n", nr_objects); + + sorted_by_type = create_sorted_list(type_size_sort); + if (window && depth) + find_deltas(sorted_by_type, window+1, depth); + write_pack_file(); +} + +static int reuse_cached_pack(unsigned char *sha1, int pack_to_stdout) +{ + static const char cache[] = "pack-cache/pack-%s.%s"; + char *cached_pack, *cached_idx; + int ifd, ofd, ifd_ix = -1; + + cached_pack = git_path(cache, sha1_to_hex(sha1), "pack"); + ifd = open(cached_pack, O_RDONLY); + if (ifd < 0) + return 0; + + if (!pack_to_stdout) { + cached_idx = git_path(cache, sha1_to_hex(sha1), "idx"); + ifd_ix = open(cached_idx, O_RDONLY); + if (ifd_ix < 0) { + close(ifd); + return 0; + } + } + + fprintf(stderr, "Reusing %d objects pack %s\n", nr_objects, + sha1_to_hex(sha1)); + + if (pack_to_stdout) { + if (copy_fd(ifd, 1)) + exit(1); + close(ifd); + } + else { + char name[PATH_MAX]; + snprintf(name, sizeof(name), + "%s-%s.%s", base_name, sha1_to_hex(sha1), "pack"); + ofd = open(name, O_CREAT | O_EXCL | O_WRONLY, 0666); + if (ofd < 0) + die("unable to open %s (%s)", name, strerror(errno)); + if (copy_fd(ifd, ofd)) + exit(1); + close(ifd); + + snprintf(name, sizeof(name), + "%s-%s.%s", base_name, sha1_to_hex(sha1), "idx"); + ofd = open(name, O_CREAT | O_EXCL | O_WRONLY, 0666); + if (ofd < 0) + die("unable to open %s (%s)", name, strerror(errno)); + if (copy_fd(ifd_ix, ofd)) + exit(1); + close(ifd_ix); + puts(sha1_to_hex(sha1)); + } + + return 1; +} + int main(int argc, char **argv) { SHA_CTX ctx; @@ -472,9 +537,6 @@ int main(int argc, char **argv) } if (non_empty && !nr_objects) return 0; - get_object_details(); - - fprintf(stderr, "Packing %d objects\n", nr_objects); sorted_by_sha = create_sorted_list(sha1_sort); SHA1_Init(&ctx); @@ -485,14 +547,14 @@ int main(int argc, char **argv) } SHA1_Final(object_list_sha1, &ctx); - sorted_by_type = create_sorted_list(type_size_sort); - if (window && depth) - find_deltas(sorted_by_type, window+1, depth); - - write_pack_file(); - if (!pack_to_stdout) { - write_index_file(); - puts(sha1_to_hex(object_list_sha1)); + if (reuse_cached_pack(object_list_sha1, pack_to_stdout)) + ; + else { + prepare_pack(window, depth); + if (!pack_to_stdout) { + write_index_file(); + puts(sha1_to_hex(object_list_sha1)); + } } return 0; } -- cgit v1.3