From a04f6531099cabbbb94e3d7be0b3db0b718860ee Mon Sep 17 00:00:00 2001 From: Jeff King Date: Thu, 13 Aug 2020 10:57:19 -0400 Subject: Makefile: drop builtins from MSVC pdb list Over the years some more programs have become builtins, but nobody updated this MSVC-specific section of the file (which specifically says that it should not include builtins). Let's bring it up to date. Signed-off-by: Jeff King Signed-off-by: Junio C Hamano --- Makefile | 2 -- 1 file changed, 2 deletions(-) diff --git a/Makefile b/Makefile index 65f8cfb236..271b96348e 100644 --- a/Makefile +++ b/Makefile @@ -2901,7 +2901,6 @@ ifdef MSVC # because it is just a copy/hardlink of git.exe, rather than a unique binary. $(INSTALL) git.pdb '$(DESTDIR_SQ)$(bindir_SQ)' $(INSTALL) git-shell.pdb '$(DESTDIR_SQ)$(bindir_SQ)' - $(INSTALL) git-upload-pack.pdb '$(DESTDIR_SQ)$(bindir_SQ)' $(INSTALL) git-credential-store.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' $(INSTALL) git-daemon.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' $(INSTALL) git-fast-import.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' @@ -2912,7 +2911,6 @@ ifdef MSVC $(INSTALL) git-remote-http.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' $(INSTALL) git-remote-testsvn.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' $(INSTALL) git-sh-i18n--envsubst.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' - $(INSTALL) git-show-index.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' ifndef DEBUG $(INSTALL) $(vcpkg_rel_bin)/*.dll '$(DESTDIR_SQ)$(bindir_SQ)' $(INSTALL) $(vcpkg_rel_bin)/*.pdb '$(DESTDIR_SQ)$(bindir_SQ)' -- cgit v1.3 From b5dd96b70aca364f163f0b3743418779cfe062e6 Mon Sep 17 00:00:00 2001 From: Jeff King Date: Thu, 13 Aug 2020 10:58:55 -0400 Subject: make credential helpers builtins There's no real reason for credential helpers to be separate binaries. I did them this way originally under the notion that helper don't _need_ to be part of Git, and so can be built totally separately (and indeed, the ones in contrib/credential are). But the ones in our main Makefile build on libgit.a, and the resulting binaries are reasonably large. We can slim down our total disk footprint by just making them builtins. This reduces the size of: make strip install from 29MB to 24MB on my Debian system. Note that credential-cache can't operate without support for Unix sockets. Currently we just don't build it at all when NO_UNIX_SOCKETS is set. We could continue that with conditionals in the Makefile and our list of builtins. But instead, let's build a dummy implementation that dies with an informative message. That has two advantages: - it's simpler, because the conditional bits are all kept inside the credential-cache source - a user who is expecting it to exist will be told _why_ they can't use it, rather than getting the "credential-cache is not a git command" error which makes it look like the Git install is broken. Note that our dummy implementation does still respond to "-h" in order to appease t0012 (and this may be a little friendlier for users, as well). Signed-off-by: Jeff King Signed-off-by: Junio C Hamano --- Makefile | 8 +- builtin.h | 3 + builtin/credential-cache--daemon.c | 318 ++++++++++++++++++++++++++++++++++++ builtin/credential-cache.c | 157 ++++++++++++++++++ builtin/credential-store.c | 195 ++++++++++++++++++++++ contrib/buildsystems/CMakeLists.txt | 20 +-- credential-cache--daemon.c | 297 --------------------------------- credential-cache.c | 136 --------------- credential-store.c | 195 ---------------------- git.c | 3 + 10 files changed, 680 insertions(+), 652 deletions(-) create mode 100644 builtin/credential-cache--daemon.c create mode 100644 builtin/credential-cache.c create mode 100644 builtin/credential-store.c delete mode 100644 credential-cache--daemon.c delete mode 100644 credential-cache.c delete mode 100644 credential-store.c diff --git a/Makefile b/Makefile index 271b96348e..5b43c0fafb 100644 --- a/Makefile +++ b/Makefile @@ -672,7 +672,6 @@ EXTRA_PROGRAMS = PROGRAMS += $(EXTRA_PROGRAMS) PROGRAM_OBJS += bugreport.o -PROGRAM_OBJS += credential-store.o PROGRAM_OBJS += daemon.o PROGRAM_OBJS += fast-import.o PROGRAM_OBJS += http-backend.o @@ -1052,6 +1051,9 @@ BUILTIN_OBJS += builtin/checkout-index.o BUILTIN_OBJS += builtin/checkout.o BUILTIN_OBJS += builtin/clean.o BUILTIN_OBJS += builtin/clone.o +BUILTIN_OBJS += builtin/credential-cache.o +BUILTIN_OBJS += builtin/credential-cache--daemon.o +BUILTIN_OBJS += builtin/credential-store.o BUILTIN_OBJS += builtin/column.o BUILTIN_OBJS += builtin/commit-graph.o BUILTIN_OBJS += builtin/commit-tree.o @@ -1634,11 +1636,8 @@ ifdef NO_INET_PTON endif ifdef NO_UNIX_SOCKETS BASIC_CFLAGS += -DNO_UNIX_SOCKETS - EXCLUDED_PROGRAMS += git-credential-cache git-credential-cache--daemon else LIB_OBJS += unix-socket.o - PROGRAM_OBJS += credential-cache.o - PROGRAM_OBJS += credential-cache--daemon.o endif ifdef NO_ICONV @@ -2901,7 +2900,6 @@ ifdef MSVC # because it is just a copy/hardlink of git.exe, rather than a unique binary. $(INSTALL) git.pdb '$(DESTDIR_SQ)$(bindir_SQ)' $(INSTALL) git-shell.pdb '$(DESTDIR_SQ)$(bindir_SQ)' - $(INSTALL) git-credential-store.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' $(INSTALL) git-daemon.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' $(INSTALL) git-fast-import.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' $(INSTALL) git-http-backend.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' diff --git a/builtin.h b/builtin.h index a5ae15bfe5..4a0aed5448 100644 --- a/builtin.h +++ b/builtin.h @@ -138,6 +138,9 @@ int cmd_commit_tree(int argc, const char **argv, const char *prefix); int cmd_config(int argc, const char **argv, const char *prefix); int cmd_count_objects(int argc, const char **argv, const char *prefix); int cmd_credential(int argc, const char **argv, const char *prefix); +int cmd_credential_cache(int argc, const char **argv, const char *prefix); +int cmd_credential_cache_daemon(int argc, const char **argv, const char *prefix); +int cmd_credential_store(int argc, const char **argv, const char *prefix); int cmd_describe(int argc, const char **argv, const char *prefix); int cmd_diff_files(int argc, const char **argv, const char *prefix); int cmd_diff_index(int argc, const char **argv, const char *prefix); diff --git a/builtin/credential-cache--daemon.c b/builtin/credential-cache--daemon.c new file mode 100644 index 0000000000..c61f123a3b --- /dev/null +++ b/builtin/credential-cache--daemon.c @@ -0,0 +1,318 @@ +#include "builtin.h" +#include "parse-options.h" + +#ifndef NO_UNIX_SOCKETS + +#include "config.h" +#include "tempfile.h" +#include "credential.h" +#include "unix-socket.h" + +struct credential_cache_entry { + struct credential item; + timestamp_t expiration; +}; +static struct credential_cache_entry *entries; +static int entries_nr; +static int entries_alloc; + +static void cache_credential(struct credential *c, int timeout) +{ + struct credential_cache_entry *e; + + ALLOC_GROW(entries, entries_nr + 1, entries_alloc); + e = &entries[entries_nr++]; + + /* take ownership of pointers */ + memcpy(&e->item, c, sizeof(*c)); + memset(c, 0, sizeof(*c)); + e->expiration = time(NULL) + timeout; +} + +static struct credential_cache_entry *lookup_credential(const struct credential *c) +{ + int i; + for (i = 0; i < entries_nr; i++) { + struct credential *e = &entries[i].item; + if (credential_match(c, e)) + return &entries[i]; + } + return NULL; +} + +static void remove_credential(const struct credential *c) +{ + struct credential_cache_entry *e; + + e = lookup_credential(c); + if (e) + e->expiration = 0; +} + +static timestamp_t check_expirations(void) +{ + static timestamp_t wait_for_entry_until; + int i = 0; + timestamp_t now = time(NULL); + timestamp_t next = TIME_MAX; + + /* + * Initially give the client 30 seconds to actually contact us + * and store a credential before we decide there's no point in + * keeping the daemon around. + */ + if (!wait_for_entry_until) + wait_for_entry_until = now + 30; + + while (i < entries_nr) { + if (entries[i].expiration <= now) { + entries_nr--; + credential_clear(&entries[i].item); + if (i != entries_nr) + memcpy(&entries[i], &entries[entries_nr], sizeof(*entries)); + /* + * Stick around 30 seconds in case a new credential + * shows up (e.g., because we just removed a failed + * one, and we will soon get the correct one). + */ + wait_for_entry_until = now + 30; + } + else { + if (entries[i].expiration < next) + next = entries[i].expiration; + i++; + } + } + + if (!entries_nr) { + if (wait_for_entry_until <= now) + return 0; + next = wait_for_entry_until; + } + + return next - now; +} + +static int read_request(FILE *fh, struct credential *c, + struct strbuf *action, int *timeout) +{ + static struct strbuf item = STRBUF_INIT; + const char *p; + + strbuf_getline_lf(&item, fh); + if (!skip_prefix(item.buf, "action=", &p)) + return error("client sent bogus action line: %s", item.buf); + strbuf_addstr(action, p); + + strbuf_getline_lf(&item, fh); + if (!skip_prefix(item.buf, "timeout=", &p)) + return error("client sent bogus timeout line: %s", item.buf); + *timeout = atoi(p); + + if (credential_read(c, fh) < 0) + return -1; + return 0; +} + +static void serve_one_client(FILE *in, FILE *out) +{ + struct credential c = CREDENTIAL_INIT; + struct strbuf action = STRBUF_INIT; + int timeout = -1; + + if (read_request(in, &c, &action, &timeout) < 0) + /* ignore error */ ; + else if (!strcmp(action.buf, "get")) { + struct credential_cache_entry *e = lookup_credential(&c); + if (e) { + fprintf(out, "username=%s\n", e->item.username); + fprintf(out, "password=%s\n", e->item.password); + } + } + else if (!strcmp(action.buf, "exit")) { + /* + * It's important that we clean up our socket first, and then + * signal the client only once we have finished the cleanup. + * Calling exit() directly does this, because we clean up in + * our atexit() handler, and then signal the client when our + * process actually ends, which closes the socket and gives + * them EOF. + */ + exit(0); + } + else if (!strcmp(action.buf, "erase")) + remove_credential(&c); + else if (!strcmp(action.buf, "store")) { + if (timeout < 0) + warning("cache client didn't specify a timeout"); + else if (!c.username || !c.password) + warning("cache client gave us a partial credential"); + else { + remove_credential(&c); + cache_credential(&c, timeout); + } + } + else + warning("cache client sent unknown action: %s", action.buf); + + credential_clear(&c); + strbuf_release(&action); +} + +static int serve_cache_loop(int fd) +{ + struct pollfd pfd; + timestamp_t wakeup; + + wakeup = check_expirations(); + if (!wakeup) + return 0; + + pfd.fd = fd; + pfd.events = POLLIN; + if (poll(&pfd, 1, 1000 * wakeup) < 0) { + if (errno != EINTR) + die_errno("poll failed"); + return 1; + } + + if (pfd.revents & POLLIN) { + int client, client2; + FILE *in, *out; + + client = accept(fd, NULL, NULL); + if (client < 0) { + warning_errno("accept failed"); + return 1; + } + client2 = dup(client); + if (client2 < 0) { + warning_errno("dup failed"); + close(client); + return 1; + } + + in = xfdopen(client, "r"); + out = xfdopen(client2, "w"); + serve_one_client(in, out); + fclose(in); + fclose(out); + } + return 1; +} + +static void serve_cache(const char *socket_path, int debug) +{ + int fd; + + fd = unix_stream_listen(socket_path); + if (fd < 0) + die_errno("unable to bind to '%s'", socket_path); + + printf("ok\n"); + fclose(stdout); + if (!debug) { + if (!freopen("/dev/null", "w", stderr)) + die_errno("unable to point stderr to /dev/null"); + } + + while (serve_cache_loop(fd)) + ; /* nothing */ + + close(fd); +} + +static const char permissions_advice[] = N_( +"The permissions on your socket directory are too loose; other\n" +"users may be able to read your cached credentials. Consider running:\n" +"\n" +" chmod 0700 %s"); +static void init_socket_directory(const char *path) +{ + struct stat st; + char *path_copy = xstrdup(path); + char *dir = dirname(path_copy); + + if (!stat(dir, &st)) { + if (st.st_mode & 077) + die(_(permissions_advice), dir); + } else { + /* + * We must be sure to create the directory with the correct mode, + * not just chmod it after the fact; otherwise, there is a race + * condition in which somebody can chdir to it, sleep, then try to open + * our protected socket. + */ + if (safe_create_leading_directories_const(dir) < 0) + die_errno("unable to create directories for '%s'", dir); + if (mkdir(dir, 0700) < 0) + die_errno("unable to mkdir '%s'", dir); + } + + if (chdir(dir)) + /* + * We don't actually care what our cwd is; we chdir here just to + * be a friendly daemon and avoid tying up our original cwd. + * If this fails, it's OK to just continue without that benefit. + */ + ; + + free(path_copy); +} + +int cmd_credential_cache_daemon(int argc, const char **argv, const char *prefix) +{ + struct tempfile *socket_file; + const char *socket_path; + int ignore_sighup = 0; + static const char *usage[] = { + "git-credential-cache--daemon [opts] ", + NULL + }; + int debug = 0; + const struct option options[] = { + OPT_BOOL(0, "debug", &debug, + N_("print debugging messages to stderr")), + OPT_END() + }; + + git_config_get_bool("credentialcache.ignoresighup", &ignore_sighup); + + argc = parse_options(argc, argv, prefix, options, usage, 0); + socket_path = argv[0]; + + if (!socket_path) + usage_with_options(usage, options); + + if (!is_absolute_path(socket_path)) + die("socket directory must be an absolute path"); + + init_socket_directory(socket_path); + socket_file = register_tempfile(socket_path); + + if (ignore_sighup) + signal(SIGHUP, SIG_IGN); + + serve_cache(socket_path, debug); + delete_tempfile(&socket_file); + + return 0; +} + +#else + +int cmd_credential_cache_daemon(int argc, const char **argv, const char *prefix) +{ + const char * const usage[] = { + "git credential-cache--daemon [options] ", + "", + "credential-cache--daemon is disabled in this build of Git", + NULL + }; + struct option options[] = { OPT_END() }; + + argc = parse_options(argc, argv, prefix, options, usage, 0); + die(_("credential-cache--daemon unavailable; no unix socket support")); +} + +#endif /* NO_UNIX_SOCKET */ diff --git a/builtin/credential-cache.c b/builtin/credential-cache.c new file mode 100644 index 0000000000..d0fafdeb9e --- /dev/null +++ b/builtin/credential-cache.c @@ -0,0 +1,157 @@ +#include "builtin.h" +#include "parse-options.h" + +#ifndef NO_UNIX_SOCKETS + +#include "credential.h" +#include "string-list.h" +#include "unix-socket.h" +#include "run-command.h" + +#define FLAG_SPAWN 0x1 +#define FLAG_RELAY 0x2 + +static int send_request(const char *socket, const struct strbuf *out) +{ + int got_data = 0; + int fd = unix_stream_connect(socket); + + if (fd < 0) + return -1; + + if (write_in_full(fd, out->buf, out->len) < 0) + die_errno("unable to write to cache daemon"); + shutdown(fd, SHUT_WR); + + while (1) { + char in[1024]; + int r; + + r = read_in_full(fd, in, sizeof(in)); + if (r == 0 || (r < 0 && errno == ECONNRESET)) + break; + if (r < 0) + die_errno("read error from cache daemon"); + write_or_die(1, in, r); + got_data = 1; + } + close(fd); + return got_data; +} + +static void spawn_daemon(const char *socket) +{ + struct child_process daemon = CHILD_PROCESS_INIT; + const char *argv[] = { NULL, NULL, NULL }; + char buf[128]; + int r; + + argv[0] = "git-credential-cache--daemon"; + argv[1] = socket; + daemon.argv = argv; + daemon.no_stdin = 1; + daemon.out = -1; + + if (start_command(&daemon)) + die_errno("unable to start cache daemon"); + r = read_in_full(daemon.out, buf, sizeof(buf)); + if (r < 0) + die_errno("unable to read result code from cache daemon"); + if (r != 3 || memcmp(buf, "ok\n", 3)) + die("cache daemon did not start: %.*s", r, buf); + close(daemon.out); +} + +static void do_cache(const char *socket, const char *action, int timeout, + int flags) +{ + struct strbuf buf = STRBUF_INIT; + + strbuf_addf(&buf, "action=%s\n", action); + strbuf_addf(&buf, "timeout=%d\n", timeout); + if (flags & FLAG_RELAY) { + if (strbuf_read(&buf, 0, 0) < 0) + die_errno("unable to relay credential"); + } + + if (send_request(socket, &buf) < 0) { + if (errno != ENOENT && errno != ECONNREFUSED) + die_errno("unable to connect to cache daemon"); + if (flags & FLAG_SPAWN) { + spawn_daemon(socket); + if (send_request(socket, &buf) < 0) + die_errno("unable to connect to cache daemon"); + } + } + strbuf_release(&buf); +} + +static char *get_socket_path(void) +{ + struct stat sb; + char *old_dir, *socket; + old_dir = expand_user_path("~/.git-credential-cache", 0); + if (old_dir && !stat(old_dir, &sb) && S_ISDIR(sb.st_mode)) + socket = xstrfmt("%s/socket", old_dir); + else + socket = xdg_cache_home("credential/socket"); + free(old_dir); + return socket; +} + +int cmd_credential_cache(int argc, const char **argv, const char *prefix) +{ + char *socket_path = NULL; + int timeout = 900; + const char *op; + const char * const usage[] = { + "git credential-cache [] ", + NULL + }; + struct option options[] = { + OPT_INTEGER(0, "timeout", &timeout, + "number of seconds to cache credentials"), + OPT_STRING(0, "socket", &socket_path, "path", + "path of cache-daemon socket"), + OPT_END() + }; + + argc = parse_options(argc, argv, prefix, options, usage, 0); + if (!argc) + usage_with_options(usage, options); + op = argv[0]; + + if (!socket_path) + socket_path = get_socket_path(); + if (!socket_path) + die("unable to find a suitable socket path; use --socket"); + + if (!strcmp(op, "exit")) + do_cache(socket_path, op, timeout, 0); + else if (!strcmp(op, "get") || !strcmp(op, "erase")) + do_cache(socket_path, op, timeout, FLAG_RELAY); + else if (!strcmp(op, "store")) + do_cache(socket_path, op, timeout, FLAG_RELAY|FLAG_SPAWN); + else + ; /* ignore unknown operation */ + + return 0; +} + +#else + +int cmd_credential_cache(int argc, const char **argv, const char *prefix) +{ + const char * const usage[] = { + "git credential-cache [options] ", + "", + "credential-cache is disabled in this build of Git", + NULL + }; + struct option options[] = { OPT_END() }; + + argc = parse_options(argc, argv, prefix, options, usage, 0); + die(_("credential-cache unavailable; no unix socket support")); +} + +#endif /* NO_UNIX_SOCKETS */ diff --git a/builtin/credential-store.c b/builtin/credential-store.c new file mode 100644 index 0000000000..5331ab151a --- /dev/null +++ b/builtin/credential-store.c @@ -0,0 +1,195 @@ +#include "builtin.h" +#include "lockfile.h" +#include "credential.h" +#include "string-list.h" +#include "parse-options.h" + +static struct lock_file credential_lock; + +static int parse_credential_file(const char *fn, + struct credential *c, + void (*match_cb)(struct credential *), + void (*other_cb)(struct strbuf *)) +{ + FILE *fh; + struct strbuf line = STRBUF_INIT; + struct credential entry = CREDENTIAL_INIT; + int found_credential = 0; + + fh = fopen(fn, "r"); + if (!fh) { + if (errno != ENOENT && errno != EACCES) + die_errno("unable to open %s", fn); + return found_credential; + } + + while (strbuf_getline_lf(&line, fh) != EOF) { + if (!credential_from_url_gently(&entry, line.buf, 1) && + entry.username && entry.password && + credential_match(c, &entry)) { + found_credential = 1; + if (match_cb) { + match_cb(&entry); + break; + } + } + else if (other_cb) + other_cb(&line); + } + + credential_clear(&entry); + strbuf_release(&line); + fclose(fh); + return found_credential; +} + +static void print_entry(struct credential *c) +{ + printf("username=%s\n", c->username); + printf("password=%s\n", c->password); +} + +static void print_line(struct strbuf *buf) +{ + strbuf_addch(buf, '\n'); + write_or_die(get_lock_file_fd(&credential_lock), buf->buf, buf->len); +} + +static void rewrite_credential_file(const char *fn, struct credential *c, + struct strbuf *extra) +{ + if (hold_lock_file_for_update(&credential_lock, fn, 0) < 0) + die_errno("unable to get credential storage lock"); + if (extra) + print_line(extra); + parse_credential_file(fn, c, NULL, print_line); + if (commit_lock_file(&credential_lock) < 0) + die_errno("unable to write credential store"); +} + +static void store_credential_file(const char *fn, struct credential *c) +{ + struct strbuf buf = STRBUF_INIT; + + strbuf_addf(&buf, "%s://", c->protocol); + strbuf_addstr_urlencode(&buf, c->username, is_rfc3986_unreserved); + strbuf_addch(&buf, ':'); + strbuf_addstr_urlencode(&buf, c->password, is_rfc3986_unreserved); + strbuf_addch(&buf, '@'); + if (c->host) + strbuf_addstr_urlencode(&buf, c->host, is_rfc3986_unreserved); + if (c->path) { + strbuf_addch(&buf, '/'); + strbuf_addstr_urlencode(&buf, c->path, + is_rfc3986_reserved_or_unreserved); + } + + rewrite_credential_file(fn, c, &buf); + strbuf_release(&buf); +} + +static void store_credential(const struct string_list *fns, struct credential *c) +{ + struct string_list_item *fn; + + /* + * Sanity check that what we are storing is actually sensible. + * In particular, we can't make a URL without a protocol field. + * Without either a host or pathname (depending on the scheme), + * we have no primary key. And without a username and password, + * we are not actually storing a credential. + */ + if (!c->protocol || !(c->host || c->path) || !c->username || !c->password) + return; + + for_each_string_list_item(fn, fns) + if (!access(fn->string, F_OK)) { + store_credential_file(fn->string, c); + return; + } + /* + * Write credential to the filename specified by fns->items[0], thus + * creating it + */ + if (fns->nr) + store_credential_file(fns->items[0].string, c); +} + +static void remove_credential(const struct string_list *fns, struct credential *c) +{ + struct string_list_item *fn; + + /* + * Sanity check that we actually have something to match + * against. The input we get is a restrictive pattern, + * so technically a blank credential means "erase everything". + * But it is too easy to accidentally send this, since it is equivalent + * to empty input. So explicitly disallow it, and require that the + * pattern have some actual content to match. + */ + if (!c->protocol && !c->host && !c->path && !c->username) + return; + for_each_string_list_item(fn, fns) + if (!access(fn->string, F_OK)) + rewrite_credential_file(fn->string, c, NULL); +} + +static void lookup_credential(const struct string_list *fns, struct credential *c) +{ + struct string_list_item *fn; + + for_each_string_list_item(fn, fns) + if (parse_credential_file(fn->string, c, print_entry, NULL)) + return; /* Found credential */ +} + +int cmd_credential_store(int argc, const char **argv, const char *prefix) +{ + const char * const usage[] = { + "git credential-store [] ", + NULL + }; + const char *op; + struct credential c = CREDENTIAL_INIT; + struct string_list fns = STRING_LIST_INIT_DUP; + char *file = NULL; + struct option options[] = { + OPT_STRING(0, "file", &file, "path", + "fetch and store credentials in "), + OPT_END() + }; + + umask(077); + + argc = parse_options(argc, (const char **)argv, prefix, options, usage, 0); + if (argc != 1) + usage_with_options(usage, options); + op = argv[0]; + + if (file) { + string_list_append(&fns, file); + } else { + if ((file = expand_user_path("~/.git-credentials", 0))) + string_list_append_nodup(&fns, file); + file = xdg_config_home("credentials"); + if (file) + string_list_append_nodup(&fns, file); + } + if (!fns.nr) + die("unable to set up default path; use --file"); + + if (credential_read(&c, stdin) < 0) + die("unable to read credential"); + + if (!strcmp(op, "get")) + lookup_credential(&fns, &c); + else if (!strcmp(op, "erase")) + remove_credential(&fns, &c); + else if (!strcmp(op, "store")) + store_credential(&fns, &c); + else + ; /* Ignore unknown operation. */ + + string_list_clear(&fns, 0); + return 0; +} diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt index 47215df25b..4be61247e5 100644 --- a/contrib/buildsystems/CMakeLists.txt +++ b/contrib/buildsystems/CMakeLists.txt @@ -501,15 +501,9 @@ unset(CMAKE_REQUIRED_INCLUDES) #programs set(PROGRAMS_BUILT - git git-bugreport git-credential-store git-daemon git-fast-import git-http-backend git-sh-i18n--envsubst + git git-bugreport git-daemon git-fast-import git-http-backend git-sh-i18n--envsubst git-shell git-remote-testsvn) -if(NO_UNIX_SOCKETS) - list(APPEND excluded_progs git-credential-cache git-credential-cache--daemon) -else() - list(APPEND PROGRAMS_BUILT git-credential-cache git-credential-cache--daemon) -endif() - if(NOT CURL_FOUND) list(APPEND excluded_progs git-http-fetch git-http-push) add_compile_definitions(NO_CURL) @@ -633,9 +627,6 @@ target_link_libraries(git common-main) add_executable(git-bugreport ${CMAKE_SOURCE_DIR}/bugreport.c) target_link_libraries(git-bugreport common-main) -add_executable(git-credential-store ${CMAKE_SOURCE_DIR}/credential-store.c) -target_link_libraries(git-credential-store common-main) - add_executable(git-daemon ${CMAKE_SOURCE_DIR}/daemon.c) target_link_libraries(git-daemon common-main) @@ -672,15 +663,6 @@ endif() add_executable(git-remote-testsvn ${CMAKE_SOURCE_DIR}/remote-testsvn.c) target_link_libraries(git-remote-testsvn common-main vcs-svn) -if(NOT NO_UNIX_SOCKETS) - add_executable(git-credential-cache ${CMAKE_SOURCE_DIR}/credential-cache.c) - target_link_libraries(git-credential-cache common-main) - - add_executable(git-credential-cache--daemon ${CMAKE_SOURCE_DIR}/credential-cache--daemon.c) - target_link_libraries(git-credential-cache--daemon common-main) -endif() - - set(git_builtin_extra cherry cherry-pick format-patch fsck-objects init merge-subtree restore show diff --git a/credential-cache--daemon.c b/credential-cache--daemon.c deleted file mode 100644 index ec1271f89c..0000000000 --- a/credential-cache--daemon.c +++ /dev/null @@ -1,297 +0,0 @@ -#include "cache.h" -#include "config.h" -#include "tempfile.h" -#include "credential.h" -#include "unix-socket.h" -#include "parse-options.h" - -struct credential_cache_entry { - struct credential item; - timestamp_t expiration; -}; -static struct credential_cache_entry *entries; -static int entries_nr; -static int entries_alloc; - -static void cache_credential(struct credential *c, int timeout) -{ - struct credential_cache_entry *e; - - ALLOC_GROW(entries, entries_nr + 1, entries_alloc); - e = &entries[entries_nr++]; - - /* take ownership of pointers */ - memcpy(&e->item, c, sizeof(*c)); - memset(c, 0, sizeof(*c)); - e->expiration = time(NULL) + timeout; -} - -static struct credential_cache_entry *lookup_credential(const struct credential *c) -{ - int i; - for (i = 0; i < entries_nr; i++) { - struct credential *e = &entries[i].item; - if (credential_match(c, e)) - return &entries[i]; - } - return NULL; -} - -static void remove_credential(const struct credential *c) -{ - struct credential_cache_entry *e; - - e = lookup_credential(c); - if (e) - e->expiration = 0; -} - -static timestamp_t check_expirations(void) -{ - static timestamp_t wait_for_entry_until; - int i = 0; - timestamp_t now = time(NULL); - timestamp_t next = TIME_MAX; - - /* - * Initially give the client 30 seconds to actually contact us - * and store a credential before we decide there's no point in - * keeping the daemon around. - */ - if (!wait_for_entry_until) - wait_for_entry_until = now + 30; - - while (i < entries_nr) { - if (entries[i].expiration <= now) { - entries_nr--; - credential_clear(&entries[i].item); - if (i != entries_nr) - memcpy(&entries[i], &entries[entries_nr], sizeof(*entries)); - /* - * Stick around 30 seconds in case a new credential - * shows up (e.g., because we just removed a failed - * one, and we will soon get the correct one). - */ - wait_for_entry_until = now + 30; - } - else { - if (entries[i].expiration < next) - next = entries[i].expiration; - i++; - } - } - - if (!entries_nr) { - if (wait_for_entry_until <= now) - return 0; - next = wait_for_entry_until; - } - - return next - now; -} - -static int read_request(FILE *fh, struct credential *c, - struct strbuf *action, int *timeout) -{ - static struct strbuf item = STRBUF_INIT; - const char *p; - - strbuf_getline_lf(&item, fh); - if (!skip_prefix(item.buf, "action=", &p)) - return error("client sent bogus action line: %s", item.buf); - strbuf_addstr(action, p); - - strbuf_getline_lf(&item, fh); - if (!skip_prefix(item.buf, "timeout=", &p)) - return error("client sent bogus timeout line: %s", item.buf); - *timeout = atoi(p); - - if (credential_read(c, fh) < 0) - return -1; - return 0; -} - -static void serve_one_client(FILE *in, FILE *out) -{ - struct credential c = CREDENTIAL_INIT; - struct strbuf action = STRBUF_INIT; - int timeout = -1; - - if (read_request(in, &c, &action, &timeout) < 0) - /* ignore error */ ; - else if (!strcmp(action.buf, "get")) { - struct credential_cache_entry *e = lookup_credential(&c); - if (e) { - fprintf(out, "username=%s\n", e->item.username); - fprintf(out, "password=%s\n", e->item.password); - } - } - else if (!strcmp(action.buf, "exit")) { - /* - * It's important that we clean up our socket first, and then - * signal the client only once we have finished the cleanup. - * Calling exit() directly does this, because we clean up in - * our atexit() handler, and then signal the client when our - * process actually ends, which closes the socket and gives - * them EOF. - */ - exit(0); - } - else if (!strcmp(action.buf, "erase")) - remove_credential(&c); - else if (!strcmp(action.buf, "store")) { - if (timeout < 0) - warning("cache client didn't specify a timeout"); - else if (!c.username || !c.password) - warning("cache client gave us a partial credential"); - else { - remove_credential(&c); - cache_credential(&c, timeout); - } - } - else - warning("cache client sent unknown action: %s", action.buf); - - credential_clear(&c); - strbuf_release(&action); -} - -static int serve_cache_loop(int fd) -{ - struct pollfd pfd; - timestamp_t wakeup; - - wakeup = check_expirations(); - if (!wakeup) - return 0; - - pfd.fd = fd; - pfd.events = POLLIN; - if (poll(&pfd, 1, 1000 * wakeup) < 0) { - if (errno != EINTR) - die_errno("poll failed"); - return 1; - } - - if (pfd.revents & POLLIN) { - int client, client2; - FILE *in, *out; - - client = accept(fd, NULL, NULL); - if (client < 0) { - warning_errno("accept failed"); - return 1; - } - client2 = dup(client); - if (client2 < 0) { - warning_errno("dup failed"); - close(client); - return 1; - } - - in = xfdopen(client, "r"); - out = xfdopen(client2, "w"); - serve_one_client(in, out); - fclose(in); - fclose(out); - } - return 1; -} - -static void serve_cache(const char *socket_path, int debug) -{ - int fd; - - fd = unix_stream_listen(socket_path); - if (fd < 0) - die_errno("unable to bind to '%s'", socket_path); - - printf("ok\n"); - fclose(stdout); - if (!debug) { - if (!freopen("/dev/null", "w", stderr)) - die_errno("unable to point stderr to /dev/null"); - } - - while (serve_cache_loop(fd)) - ; /* nothing */ - - close(fd); -} - -static const char permissions_advice[] = N_( -"The permissions on your socket directory are too loose; other\n" -"users may be able to read your cached credentials. Consider running:\n" -"\n" -" chmod 0700 %s"); -static void init_socket_directory(const char *path) -{ - struct stat st; - char *path_copy = xstrdup(path); - char *dir = dirname(path_copy); - - if (!stat(dir, &st)) { - if (st.st_mode & 077) - die(_(permissions_advice), dir); - } else { - /* - * We must be sure to create the directory with the correct mode, - * not just chmod it after the fact; otherwise, there is a race - * condition in which somebody can chdir to it, sleep, then try to open - * our protected socket. - */ - if (safe_create_leading_directories_const(dir) < 0) - die_errno("unable to create directories for '%s'", dir); - if (mkdir(dir, 0700) < 0) - die_errno("unable to mkdir '%s'", dir); - } - - if (chdir(dir)) - /* - * We don't actually care what our cwd is; we chdir here just to - * be a friendly daemon and avoid tying up our original cwd. - * If this fails, it's OK to just continue without that benefit. - */ - ; - - free(path_copy); -} - -int cmd_main(int argc, const char **argv) -{ - struct tempfile *socket_file; - const char *socket_path; - int ignore_sighup = 0; - static const char *usage[] = { - "git-credential-cache--daemon [opts] ", - NULL - }; - int debug = 0; - const struct option options[] = { - OPT_BOOL(0, "debug", &debug, - N_("print debugging messages to stderr")), - OPT_END() - }; - - git_config_get_bool("credentialcache.ignoresighup", &ignore_sighup); - - argc = parse_options(argc, argv, NULL, options, usage, 0); - socket_path = argv[0]; - - if (!socket_path) - usage_with_options(usage, options); - - if (!is_absolute_path(socket_path)) - die("socket directory must be an absolute path"); - - init_socket_directory(socket_path); - socket_file = register_tempfile(socket_path); - - if (ignore_sighup) - signal(SIGHUP, SIG_IGN); - - serve_cache(socket_path, debug); - delete_tempfile(&socket_file); - - return 0; -} diff --git a/credential-cache.c b/credential-cache.c deleted file mode 100644 index 1cccc3a0b9..0000000000 --- a/credential-cache.c +++ /dev/null @@ -1,136 +0,0 @@ -#include "cache.h" -#include "credential.h" -#include "string-list.h" -#include "parse-options.h" -#include "unix-socket.h" -#include "run-command.h" - -#define FLAG_SPAWN 0x1 -#define FLAG_RELAY 0x2 - -static int send_request(const char *socket, const struct strbuf *out) -{ - int got_data = 0; - int fd = unix_stream_connect(socket); - - if (fd < 0) - return -1; - - if (write_in_full(fd, out->buf, out->len) < 0) - die_errno("unable to write to cache daemon"); - shutdown(fd, SHUT_WR); - - while (1) { - char in[1024]; - int r; - - r = read_in_full(fd, in, sizeof(in)); - if (r == 0 || (r < 0 && errno == ECONNRESET)) - break; - if (r < 0) - die_errno("read error from cache daemon"); - write_or_die(1, in, r); - got_data = 1; - } - close(fd); - return got_data; -} - -static void spawn_daemon(const char *socket) -{ - struct child_process daemon = CHILD_PROCESS_INIT; - const char *argv[] = { NULL, NULL, NULL }; - char buf[128]; - int r; - - argv[0] = "git-credential-cache--daemon"; - argv[1] = socket; - daemon.argv = argv; - daemon.no_stdin = 1; - daemon.out = -1; - - if (start_command(&daemon)) - die_errno("unable to start cache daemon"); - r = read_in_full(daemon.out, buf, sizeof(buf)); - if (r < 0) - die_errno("unable to read result code from cache daemon"); - if (r != 3 || memcmp(buf, "ok\n", 3)) - die("cache daemon did not start: %.*s", r, buf); - close(daemon.out); -} - -static void do_cache(const char *socket, const char *action, int timeout, - int flags) -{ - struct strbuf buf = STRBUF_INIT; - - strbuf_addf(&buf, "action=%s\n", action); - strbuf_addf(&buf, "timeout=%d\n", timeout); - if (flags & FLAG_RELAY) { - if (strbuf_read(&buf, 0, 0) < 0) - die_errno("unable to relay credential"); - } - - if (send_request(socket, &buf) < 0) { - if (errno != ENOENT && errno != ECONNREFUSED) - die_errno("unable to connect to cache daemon"); - if (flags & FLAG_SPAWN) { - spawn_daemon(socket); - if (send_request(socket, &buf) < 0) - die_errno("unable to connect to cache daemon"); - } - } - strbuf_release(&buf); -} - -static char *get_socket_path(void) -{ - struct stat sb; - char *old_dir, *socket; - old_dir = expand_user_path("~/.git-credential-cache", 0); - if (old_dir && !stat(old_dir, &sb) && S_ISDIR(sb.st_mode)) - socket = xstrfmt("%s/socket", old_dir); - else - socket = xdg_cache_home("credential/socket"); - free(old_dir); - return socket; -} - -int cmd_main(int argc, const char **argv) -{ - char *socket_path = NULL; - int timeout = 900; - const char *op; - const char * const usage[] = { - "git credential-cache [] ", - NULL - }; - struct option options[] = { - OPT_INTEGER(0, "timeout", &timeout, - "number of seconds to cache credentials"), - OPT_STRING(0, "socket", &socket_path, "path", - "path of cache-daemon socket"), - OPT_END() - }; - - argc = parse_options(argc, argv, NULL, options, usage, 0); - if (!argc) - usage_with_options(usage, options); - op = argv[0]; - - if (!socket_path) - socket_path = get_socket_path(); - if (!socket_path) - die("unable to find a suitable socket path; use --socket"); - - if (!strcmp(op, "exit")) - do_cache(socket_path, op, timeout, 0); - else if (!strcmp(op, "get") || !strcmp(op, "erase")) - do_cache(socket_path, op, timeout, FLAG_RELAY); - else if (!strcmp(op, "store")) - do_cache(socket_path, op, timeout, FLAG_RELAY|FLAG_SPAWN); - else - ; /* ignore unknown operation */ - - return 0; -} diff --git a/credential-store.c b/credential-store.c deleted file mode 100644 index 294e771681..0000000000 --- a/credential-store.c +++ /dev/null @@ -1,195 +0,0 @@ -#include "cache.h" -#include "lockfile.h" -#include "credential.h" -#include "string-list.h" -#include "parse-options.h" - -static struct lock_file credential_lock; - -static int parse_credential_file(const char *fn, - struct credential *c, - void (*match_cb)(struct credential *), - void (*other_cb)(struct strbuf *)) -{ - FILE *fh; - struct strbuf line = STRBUF_INIT; - struct credential entry = CREDENTIAL_INIT; - int found_credential = 0; - - fh = fopen(fn, "r"); - if (!fh) { - if (errno != ENOENT && errno != EACCES) - die_errno("unable to open %s", fn); - return found_credential; - } - - while (strbuf_getline_lf(&line, fh) != EOF) { - if (!credential_from_url_gently(&entry, line.buf, 1) && - entry.username && entry.password && - credential_match(c, &entry)) { - found_credential = 1; - if (match_cb) { - match_cb(&entry); - break; - } - } - else if (other_cb) - other_cb(&line); - } - - credential_clear(&entry); - strbuf_release(&line); - fclose(fh); - return found_credential; -} - -static void print_entry(struct credential *c) -{ - printf("username=%s\n", c->username); - printf("password=%s\n", c->password); -} - -static void print_line(struct strbuf *buf) -{ - strbuf_addch(buf, '\n'); - write_or_die(get_lock_file_fd(&credential_lock), buf->buf, buf->len); -} - -static void rewrite_credential_file(const char *fn, struct credential *c, - struct strbuf *extra) -{ - if (hold_lock_file_for_update(&credential_lock, fn, 0) < 0) - die_errno("unable to get credential storage lock"); - if (extra) - print_line(extra); - parse_credential_file(fn, c, NULL, print_line); - if (commit_lock_file(&credential_lock) < 0) - die_errno("unable to write credential store"); -} - -static void store_credential_file(const char *fn, struct credential *c) -{ - struct strbuf buf = STRBUF_INIT; - - strbuf_addf(&buf, "%s://", c->protocol); - strbuf_addstr_urlencode(&buf, c->username, is_rfc3986_unreserved); - strbuf_addch(&buf, ':'); - strbuf_addstr_urlencode(&buf, c->password, is_rfc3986_unreserved); - strbuf_addch(&buf, '@'); - if (c->host) - strbuf_addstr_urlencode(&buf, c->host, is_rfc3986_unreserved); - if (c->path) { - strbuf_addch(&buf, '/'); - strbuf_addstr_urlencode(&buf, c->path, - is_rfc3986_reserved_or_unreserved); - } - - rewrite_credential_file(fn, c, &buf); - strbuf_release(&buf); -} - -static void store_credential(const struct string_list *fns, struct credential *c) -{ - struct string_list_item *fn; - - /* - * Sanity check that what we are storing is actually sensible. - * In particular, we can't make a URL without a protocol field. - * Without either a host or pathname (depending on the scheme), - * we have no primary key. And without a username and password, - * we are not actually storing a credential. - */ - if (!c->protocol || !(c->host || c->path) || !c->username || !c->password) - return; - - for_each_string_list_item(fn, fns) - if (!access(fn->string, F_OK)) { - store_credential_file(fn->string, c); - return; - } - /* - * Write credential to the filename specified by fns->items[0], thus - * creating it - */ - if (fns->nr) - store_credential_file(fns->items[0].string, c); -} - -static void remove_credential(const struct string_list *fns, struct credential *c) -{ - struct string_list_item *fn; - - /* - * Sanity check that we actually have something to match - * against. The input we get is a restrictive pattern, - * so technically a blank credential means "erase everything". - * But it is too easy to accidentally send this, since it is equivalent - * to empty input. So explicitly disallow it, and require that the - * pattern have some actual content to match. - */ - if (!c->protocol && !c->host && !c->path && !c->username) - return; - for_each_string_list_item(fn, fns) - if (!access(fn->string, F_OK)) - rewrite_credential_file(fn->string, c, NULL); -} - -static void lookup_credential(const struct string_list *fns, struct credential *c) -{ - struct string_list_item *fn; - - for_each_string_list_item(fn, fns) - if (parse_credential_file(fn->string, c, print_entry, NULL)) - return; /* Found credential */ -} - -int cmd_main(int argc, const char **argv) -{ - const char * const usage[] = { - "git credential-store [] ", - NULL - }; - const char *op; - struct credential c = CREDENTIAL_INIT; - struct string_list fns = STRING_LIST_INIT_DUP; - char *file = NULL; - struct option options[] = { - OPT_STRING(0, "file", &file, "path", - "fetch and store credentials in "), - OPT_END() - }; - - umask(077); - - argc = parse_options(argc, (const char **)argv, NULL, options, usage, 0); - if (argc != 1) - usage_with_options(usage, options); - op = argv[0]; - - if (file) { - string_list_append(&fns, file); - } else { - if ((file = expand_user_path("~/.git-credentials", 0))) - string_list_append_nodup(&fns, file); - file = xdg_config_home("credentials"); - if (file) - string_list_append_nodup(&fns, file); - } - if (!fns.nr) - die("unable to set up default path; use --file"); - - if (credential_read(&c, stdin) < 0) - die("unable to read credential"); - - if (!strcmp(op, "get")) - lookup_credential(&fns, &c); - else if (!strcmp(op, "erase")) - remove_credential(&fns, &c); - else if (!strcmp(op, "store")) - store_credential(&fns, &c); - else - ; /* Ignore unknown operation. */ - - string_list_clear(&fns, 0); - return 0; -} diff --git a/git.c b/git.c index 8bd1d7551d..39a160fa52 100644 --- a/git.c +++ b/git.c @@ -499,6 +499,9 @@ static struct cmd_struct commands[] = { { "config", cmd_config, RUN_SETUP_GENTLY | DELAY_PAGER_CONFIG }, { "count-objects", cmd_count_objects, RUN_SETUP }, { "credential", cmd_credential, RUN_SETUP_GENTLY | NO_PARSEOPT }, + { "credential-cache", cmd_credential_cache }, + { "credential-cache--daemon", cmd_credential_cache_daemon }, + { "credential-store", cmd_credential_store }, { "describe", cmd_describe, RUN_SETUP }, { "diff", cmd_diff, NO_PARSEOPT }, { "diff-files", cmd_diff_files, RUN_SETUP | NEED_WORK_TREE | NO_PARSEOPT }, -- cgit v1.3 From d7a5649c82dfa83ba8d0253e4140f242dca859a7 Mon Sep 17 00:00:00 2001 From: Jeff King Date: Thu, 13 Aug 2020 10:59:36 -0400 Subject: make git-bugreport a builtin There's no reason that bugreport has to be a separate binary. And since it links against libgit.a, it has a rather large disk footprint. Let's make it a builtin, which reduces the size of a stripped installation from 24MB to 22MB. This also simplifies our Makefile a bit. And we can take advantage of builtin niceties like RUN_SETUP_GENTLY. Signed-off-by: Jeff King Signed-off-by: Junio C Hamano --- Makefile | 6 +- bugreport.c | 200 ------------------------------------ builtin.h | 1 + builtin/bugreport.c | 196 +++++++++++++++++++++++++++++++++++ contrib/buildsystems/CMakeLists.txt | 5 +- git.c | 1 + 6 files changed, 200 insertions(+), 209 deletions(-) delete mode 100644 bugreport.c create mode 100644 builtin/bugreport.c diff --git a/Makefile b/Makefile index 5b43c0fafb..acaff6968b 100644 --- a/Makefile +++ b/Makefile @@ -671,7 +671,6 @@ EXTRA_PROGRAMS = # ... and all the rest that could be moved out of bindir to gitexecdir PROGRAMS += $(EXTRA_PROGRAMS) -PROGRAM_OBJS += bugreport.o PROGRAM_OBJS += daemon.o PROGRAM_OBJS += fast-import.o PROGRAM_OBJS += http-backend.o @@ -1041,6 +1040,7 @@ BUILTIN_OBJS += builtin/archive.o BUILTIN_OBJS += builtin/bisect--helper.o BUILTIN_OBJS += builtin/blame.o BUILTIN_OBJS += builtin/branch.o +BUILTIN_OBJS += builtin/bugreport.o BUILTIN_OBJS += builtin/bundle.o BUILTIN_OBJS += builtin/cat-file.o BUILTIN_OBJS += builtin/check-attr.o @@ -2458,10 +2458,6 @@ endif git-%$X: %.o GIT-LDFLAGS $(GITLIBS) $(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) $(LIBS) -git-bugreport$X: bugreport.o GIT-LDFLAGS $(GITLIBS) - $(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) \ - $(LIBS) - git-imap-send$X: imap-send.o $(IMAP_SEND_BUILDDEPS) GIT-LDFLAGS $(GITLIBS) $(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) \ $(IMAP_SEND_LDFLAGS) $(LIBS) diff --git a/bugreport.c b/bugreport.c deleted file mode 100644 index 09579e268d..0000000000 --- a/bugreport.c +++ /dev/null @@ -1,200 +0,0 @@ -#include "cache.h" -#include "parse-options.h" -#include "strbuf.h" -#include "help.h" -#include "compat/compiler.h" -#include "run-command.h" - - -static void get_system_info(struct strbuf *sys_info) -{ - struct utsname uname_info; - char *shell = NULL; - - /* get git version from native cmd */ - strbuf_addstr(sys_info, _("git version:\n")); - get_version_info(sys_info, 1); - - /* system call for other version info */ - strbuf_addstr(sys_info, "uname: "); - if (uname(&uname_info)) - strbuf_addf(sys_info, _("uname() failed with error '%s' (%d)\n"), - strerror(errno), - errno); - else - strbuf_addf(sys_info, "%s %s %s %s\n", - uname_info.sysname, - uname_info.release, - uname_info.version, - uname_info.machine); - - strbuf_addstr(sys_info, _("compiler info: ")); - get_compiler_info(sys_info); - - strbuf_addstr(sys_info, _("libc info: ")); - get_libc_info(sys_info); - - shell = getenv("SHELL"); - strbuf_addf(sys_info, "$SHELL (typically, interactive shell): %s\n", - shell ? shell : ""); -} - -static void get_populated_hooks(struct strbuf *hook_info, int nongit) -{ - /* - * NEEDSWORK: Doesn't look like there is a list of all possible hooks; - * so below is a transcription of `git help hooks`. Later, this should - * be replaced with some programmatically generated list (generated from - * doc or else taken from some library which tells us about all the - * hooks) - */ - static const char *hook[] = { - "applypatch-msg", - "pre-applypatch", - "post-applypatch", - "pre-commit", - "pre-merge-commit", - "prepare-commit-msg", - "commit-msg", - "post-commit", - "pre-rebase", - "post-checkout", - "post-merge", - "pre-push", - "pre-receive", - "update", - "post-receive", - "post-update", - "push-to-checkout", - "pre-auto-gc", - "post-rewrite", - "sendemail-validate", - "fsmonitor-watchman", - "p4-pre-submit", - "post-index-change", - }; - int i; - - if (nongit) { - strbuf_addstr(hook_info, - _("not run from a git repository - no hooks to show\n")); - return; - } - - for (i = 0; i < ARRAY_SIZE(hook); i++) - if (find_hook(hook[i])) - strbuf_addf(hook_info, "%s\n", hook[i]); -} - -static const char * const bugreport_usage[] = { - N_("git bugreport [-o|--output-directory ] [-s|--suffix ]"), - NULL -}; - -static int get_bug_template(struct strbuf *template) -{ - const char template_text[] = N_( -"Thank you for filling out a Git bug report!\n" -"Please answer the following questions to help us understand your issue.\n" -"\n" -"What did you do before the bug happened? (Steps to reproduce your issue)\n" -"\n" -"What did you expect to happen? (Expected behavior)\n" -"\n" -"What happened instead? (Actual behavior)\n" -"\n" -"What's different between what you expected and what actually happened?\n" -"\n" -"Anything else you want to add:\n" -"\n" -"Please review the rest of the bug report below.\n" -"You can delete any lines you don't wish to share.\n"); - - strbuf_addstr(template, _(template_text)); - return 0; -} - -static void get_header(struct strbuf *buf, const char *title) -{ - strbuf_addf(buf, "\n\n[%s]\n", title); -} - -int cmd_main(int argc, const char **argv) -{ - struct strbuf buffer = STRBUF_INIT; - struct strbuf report_path = STRBUF_INIT; - int report = -1; - time_t now = time(NULL); - char *option_output = NULL; - char *option_suffix = "%Y-%m-%d-%H%M"; - int nongit_ok = 0; - const char *prefix = NULL; - const char *user_relative_path = NULL; - - const struct option bugreport_options[] = { - OPT_STRING('o', "output-directory", &option_output, N_("path"), - N_("specify a destination for the bugreport file")), - OPT_STRING('s', "suffix", &option_suffix, N_("format"), - N_("specify a strftime format suffix for the filename")), - OPT_END() - }; - - prefix = setup_git_directory_gently(&nongit_ok); - - argc = parse_options(argc, argv, prefix, bugreport_options, - bugreport_usage, 0); - - /* Prepare the path to put the result */ - strbuf_addstr(&report_path, - prefix_filename(prefix, - option_output ? option_output : "")); - strbuf_complete(&report_path, '/'); - - strbuf_addstr(&report_path, "git-bugreport-"); - strbuf_addftime(&report_path, option_suffix, localtime(&now), 0, 0); - strbuf_addstr(&report_path, ".txt"); - - switch (safe_create_leading_directories(report_path.buf)) { - case SCLD_OK: - case SCLD_EXISTS: - break; - default: - die(_("could not create leading directories for '%s'"), - report_path.buf); - } - - /* Prepare the report contents */ - get_bug_template(&buffer); - - get_header(&buffer, _("System Info")); - get_system_info(&buffer); - - get_header(&buffer, _("Enabled Hooks")); - get_populated_hooks(&buffer, nongit_ok); - - /* fopen doesn't offer us an O_EXCL alternative, except with glibc. */ - report = open(report_path.buf, O_CREAT | O_EXCL | O_WRONLY, 0666); - - if (report < 0) { - UNLEAK(report_path); - die(_("couldn't create a new file at '%s'"), report_path.buf); - } - - if (write_in_full(report, buffer.buf, buffer.len) < 0) - die_errno(_("unable to write to %s"), report_path.buf); - - close(report); - - /* - * We want to print the path relative to the user, but we still need the - * path relative to us to give to the editor. - */ - if (!(prefix && skip_prefix(report_path.buf, prefix, &user_relative_path))) - user_relative_path = report_path.buf; - fprintf(stderr, _("Created new report at '%s'.\n"), - user_relative_path); - - UNLEAK(buffer); - UNLEAK(report_path); - return !!launch_editor(report_path.buf, NULL, NULL); -} diff --git a/builtin.h b/builtin.h index 4a0aed5448..1e78d6c142 100644 --- a/builtin.h +++ b/builtin.h @@ -119,6 +119,7 @@ int cmd_archive(int argc, const char **argv, const char *prefix); int cmd_bisect__helper(int argc, const char **argv, const char *prefix); int cmd_blame(int argc, const char **argv, const char *prefix); int cmd_branch(int argc, const char **argv, const char *prefix); +int cmd_bugreport(int argc, const char **argv, const char *prefix); int cmd_bundle(int argc, const char **argv, const char *prefix); int cmd_cat_file(int argc, const char **argv, const char *prefix); int cmd_checkout(int argc, const char **argv, const char *prefix); diff --git a/builtin/bugreport.c b/builtin/bugreport.c new file mode 100644 index 0000000000..9c920cc065 --- /dev/null +++ b/builtin/bugreport.c @@ -0,0 +1,196 @@ +#include "builtin.h" +#include "parse-options.h" +#include "strbuf.h" +#include "help.h" +#include "compat/compiler.h" +#include "run-command.h" + + +static void get_system_info(struct strbuf *sys_info) +{ + struct utsname uname_info; + char *shell = NULL; + + /* get git version from native cmd */ + strbuf_addstr(sys_info, _("git version:\n")); + get_version_info(sys_info, 1); + + /* system call for other version info */ + strbuf_addstr(sys_info, "uname: "); + if (uname(&uname_info)) + strbuf_addf(sys_info, _("uname() failed with error '%s' (%d)\n"), + strerror(errno), + errno); + else + strbuf_addf(sys_info, "%s %s %s %s\n", + uname_info.sysname, + uname_info.release, + uname_info.version, + uname_info.machine); + + strbuf_addstr(sys_info, _("compiler info: ")); + get_compiler_info(sys_info); + + strbuf_addstr(sys_info, _("libc info: ")); + get_libc_info(sys_info); + + shell = getenv("SHELL"); + strbuf_addf(sys_info, "$SHELL (typically, interactive shell): %s\n", + shell ? shell : ""); +} + +static void get_populated_hooks(struct strbuf *hook_info, int nongit) +{ + /* + * NEEDSWORK: Doesn't look like there is a list of all possible hooks; + * so below is a transcription of `git help hooks`. Later, this should + * be replaced with some programmatically generated list (generated from + * doc or else taken from some library which tells us about all the + * hooks) + */ + static const char *hook[] = { + "applypatch-msg", + "pre-applypatch", + "post-applypatch", + "pre-commit", + "pre-merge-commit", + "prepare-commit-msg", + "commit-msg", + "post-commit", + "pre-rebase", + "post-checkout", + "post-merge", + "pre-push", + "pre-receive", + "update", + "post-receive", + "post-update", + "push-to-checkout", + "pre-auto-gc", + "post-rewrite", + "sendemail-validate", + "fsmonitor-watchman", + "p4-pre-submit", + "post-index-change", + }; + int i; + + if (nongit) { + strbuf_addstr(hook_info, + _("not run from a git repository - no hooks to show\n")); + return; + } + + for (i = 0; i < ARRAY_SIZE(hook); i++) + if (find_hook(hook[i])) + strbuf_addf(hook_info, "%s\n", hook[i]); +} + +static const char * const bugreport_usage[] = { + N_("git bugreport [-o|--output-directory ] [-s|--suffix ]"), + NULL +}; + +static int get_bug_template(struct strbuf *template) +{ + const char template_text[] = N_( +"Thank you for filling out a Git bug report!\n" +"Please answer the following questions to help us understand your issue.\n" +"\n" +"What did you do before the bug happened? (Steps to reproduce your issue)\n" +"\n" +"What did you expect to happen? (Expected behavior)\n" +"\n" +"What happened instead? (Actual behavior)\n" +"\n" +"What's different between what you expected and what actually happened?\n" +"\n" +"Anything else you want to add:\n" +"\n" +"Please review the rest of the bug report below.\n" +"You can delete any lines you don't wish to share.\n"); + + strbuf_addstr(template, _(template_text)); + return 0; +} + +static void get_header(struct strbuf *buf, const char *title) +{ + strbuf_addf(buf, "\n\n[%s]\n", title); +} + +int cmd_bugreport(int argc, const char **argv, const char *prefix) +{ + struct strbuf buffer = STRBUF_INIT; + struct strbuf report_path = STRBUF_INIT; + int report = -1; + time_t now = time(NULL); + char *option_output = NULL; + char *option_suffix = "%Y-%m-%d-%H%M"; + const char *user_relative_path = NULL; + + const struct option bugreport_options[] = { + OPT_STRING('o', "output-directory", &option_output, N_("path"), + N_("specify a destination for the bugreport file")), + OPT_STRING('s', "suffix", &option_suffix, N_("format"), + N_("specify a strftime format suffix for the filename")), + OPT_END() + }; + + argc = parse_options(argc, argv, prefix, bugreport_options, + bugreport_usage, 0); + + /* Prepare the path to put the result */ + strbuf_addstr(&report_path, + prefix_filename(prefix, + option_output ? option_output : "")); + strbuf_complete(&report_path, '/'); + + strbuf_addstr(&report_path, "git-bugreport-"); + strbuf_addftime(&report_path, option_suffix, localtime(&now), 0, 0); + strbuf_addstr(&report_path, ".txt"); + + switch (safe_create_leading_directories(report_path.buf)) { + case SCLD_OK: + case SCLD_EXISTS: + break; + default: + die(_("could not create leading directories for '%s'"), + report_path.buf); + } + + /* Prepare the report contents */ + get_bug_template(&buffer); + + get_header(&buffer, _("System Info")); + get_system_info(&buffer); + + get_header(&buffer, _("Enabled Hooks")); + get_populated_hooks(&buffer, !startup_info->have_repository); + + /* fopen doesn't offer us an O_EXCL alternative, except with glibc. */ + report = open(report_path.buf, O_CREAT | O_EXCL | O_WRONLY, 0666); + + if (report < 0) { + UNLEAK(report_path); + die(_("couldn't create a new file at '%s'"), report_path.buf); + } + + if (write_in_full(report, buffer.buf, buffer.len) < 0) + die_errno(_("unable to write to %s"), report_path.buf); + + close(report); + + /* + * We want to print the path relative to the user, but we still need the + * path relative to us to give to the editor. + */ + if (!(prefix && skip_prefix(report_path.buf, prefix, &user_relative_path))) + user_relative_path = report_path.buf; + fprintf(stderr, _("Created new report at '%s'.\n"), + user_relative_path); + + UNLEAK(buffer); + UNLEAK(report_path); + return !!launch_editor(report_path.buf, NULL, NULL); +} diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt index 4be61247e5..3e211606fd 100644 --- a/contrib/buildsystems/CMakeLists.txt +++ b/contrib/buildsystems/CMakeLists.txt @@ -501,7 +501,7 @@ unset(CMAKE_REQUIRED_INCLUDES) #programs set(PROGRAMS_BUILT - git git-bugreport git-daemon git-fast-import git-http-backend git-sh-i18n--envsubst + git git-daemon git-fast-import git-http-backend git-sh-i18n--envsubst git-shell git-remote-testsvn) if(NOT CURL_FOUND) @@ -624,9 +624,6 @@ list(TRANSFORM git_SOURCES PREPEND "${CMAKE_SOURCE_DIR}/") add_executable(git ${CMAKE_SOURCE_DIR}/git.c ${git_SOURCES}) target_link_libraries(git common-main) -add_executable(git-bugreport ${CMAKE_SOURCE_DIR}/bugreport.c) -target_link_libraries(git-bugreport common-main) - add_executable(git-daemon ${CMAKE_SOURCE_DIR}/daemon.c) target_link_libraries(git-daemon common-main) diff --git a/git.c b/git.c index 39a160fa52..bf790e7f4f 100644 --- a/git.c +++ b/git.c @@ -479,6 +479,7 @@ static struct cmd_struct commands[] = { { "bisect--helper", cmd_bisect__helper, RUN_SETUP }, { "blame", cmd_blame, RUN_SETUP }, { "branch", cmd_branch, RUN_SETUP | DELAY_PAGER_CONFIG }, + { "bugreport", cmd_bugreport, RUN_SETUP_GENTLY }, { "bundle", cmd_bundle, RUN_SETUP_GENTLY | NO_PARSEOPT }, { "cat-file", cmd_cat_file, RUN_SETUP }, { "check-attr", cmd_check_attr, RUN_SETUP }, -- cgit v1.3 From a006f875e2689cb7df543d5950beadb0416d305b Mon Sep 17 00:00:00 2001 From: Jeff King Date: Thu, 13 Aug 2020 10:59:45 -0400 Subject: make git-fast-import a builtin There's no reason that git-fast-import benefits from being a separate binary. And as it links against libgit.a, it has a non-trivial disk footprint. Let's make it a builtin, which reduces the size of a stripped installation from 22MB to 21MB. Signed-off-by: Jeff King Signed-off-by: Junio C Hamano --- Makefile | 3 +- builtin.h | 1 + builtin/fast-import.c | 3648 ++++++++++++++++++++++++++++++++++ contrib/buildsystems/CMakeLists.txt | 5 +- fast-import.c | 3649 ----------------------------------- git.c | 1 + 6 files changed, 3652 insertions(+), 3655 deletions(-) create mode 100644 builtin/fast-import.c delete mode 100644 fast-import.c diff --git a/Makefile b/Makefile index acaff6968b..2b821fc762 100644 --- a/Makefile +++ b/Makefile @@ -672,7 +672,6 @@ EXTRA_PROGRAMS = PROGRAMS += $(EXTRA_PROGRAMS) PROGRAM_OBJS += daemon.o -PROGRAM_OBJS += fast-import.o PROGRAM_OBJS += http-backend.o PROGRAM_OBJS += imap-send.o PROGRAM_OBJS += remote-testsvn.o @@ -1069,6 +1068,7 @@ BUILTIN_OBJS += builtin/diff.o BUILTIN_OBJS += builtin/difftool.o BUILTIN_OBJS += builtin/env--helper.o BUILTIN_OBJS += builtin/fast-export.o +BUILTIN_OBJS += builtin/fast-import.o BUILTIN_OBJS += builtin/fetch-pack.o BUILTIN_OBJS += builtin/fetch.o BUILTIN_OBJS += builtin/fmt-merge-msg.o @@ -2897,7 +2897,6 @@ ifdef MSVC $(INSTALL) git.pdb '$(DESTDIR_SQ)$(bindir_SQ)' $(INSTALL) git-shell.pdb '$(DESTDIR_SQ)$(bindir_SQ)' $(INSTALL) git-daemon.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' - $(INSTALL) git-fast-import.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' $(INSTALL) git-http-backend.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' $(INSTALL) git-http-fetch.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' $(INSTALL) git-http-push.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' diff --git a/builtin.h b/builtin.h index 1e78d6c142..ba954e180c 100644 --- a/builtin.h +++ b/builtin.h @@ -150,6 +150,7 @@ int cmd_diff_tree(int argc, const char **argv, const char *prefix); int cmd_difftool(int argc, const char **argv, const char *prefix); int cmd_env__helper(int argc, const char **argv, const char *prefix); int cmd_fast_export(int argc, const char **argv, const char *prefix); +int cmd_fast_import(int argc, const char **argv, const char *prefix); int cmd_fetch(int argc, const char **argv, const char *prefix); int cmd_fetch_pack(int argc, const char **argv, const char *prefix); int cmd_fmt_merge_msg(int argc, const char **argv, const char *prefix); diff --git a/builtin/fast-import.c b/builtin/fast-import.c new file mode 100644 index 0000000000..8cc5d09c64 --- /dev/null +++ b/builtin/fast-import.c @@ -0,0 +1,3648 @@ +#include "builtin.h" +#include "cache.h" +#include "repository.h" +#include "config.h" +#include "lockfile.h" +#include "object.h" +#include "blob.h" +#include "tree.h" +#include "commit.h" +#include "delta.h" +#include "pack.h" +#include "refs.h" +#include "csum-file.h" +#include "quote.h" +#include "dir.h" +#include "run-command.h" +#include "packfile.h" +#include "object-store.h" +#include "mem-pool.h" +#include "commit-reach.h" +#include "khash.h" + +#define PACK_ID_BITS 16 +#define MAX_PACK_ID ((1<rawsz * 3) + +struct object_entry { + struct pack_idx_entry idx; + struct hashmap_entry ent; + uint32_t type : TYPE_BITS, + pack_id : PACK_ID_BITS, + depth : DEPTH_BITS; +}; + +static int object_entry_hashcmp(const void *map_data, + const struct hashmap_entry *eptr, + const struct hashmap_entry *entry_or_key, + const void *keydata) +{ + const struct object_id *oid = keydata; + const struct object_entry *e1, *e2; + + e1 = container_of(eptr, const struct object_entry, ent); + if (oid) + return oidcmp(&e1->idx.oid, oid); + + e2 = container_of(entry_or_key, const struct object_entry, ent); + return oidcmp(&e1->idx.oid, &e2->idx.oid); +} + +struct object_entry_pool { + struct object_entry_pool *next_pool; + struct object_entry *next_free; + struct object_entry *end; + struct object_entry entries[FLEX_ARRAY]; /* more */ +}; + +struct mark_set { + union { + struct object_id *oids[1024]; + struct object_entry *marked[1024]; + struct mark_set *sets[1024]; + } data; + unsigned int shift; +}; + +struct last_object { + struct strbuf data; + off_t offset; + unsigned int depth; + unsigned no_swap : 1; +}; + +struct atom_str { + struct atom_str *next_atom; + unsigned short str_len; + char str_dat[FLEX_ARRAY]; /* more */ +}; + +struct tree_content; +struct tree_entry { + struct tree_content *tree; + struct atom_str *name; + struct tree_entry_ms { + uint16_t mode; + struct object_id oid; + } versions[2]; +}; + +struct tree_content { + unsigned int entry_capacity; /* must match avail_tree_content */ + unsigned int entry_count; + unsigned int delta_depth; + struct tree_entry *entries[FLEX_ARRAY]; /* more */ +}; + +struct avail_tree_content { + unsigned int entry_capacity; /* must match tree_content */ + struct avail_tree_content *next_avail; +}; + +struct branch { + struct branch *table_next_branch; + struct branch *active_next_branch; + const char *name; + struct tree_entry branch_tree; + uintmax_t last_commit; + uintmax_t num_notes; + unsigned active : 1; + unsigned delete : 1; + unsigned pack_id : PACK_ID_BITS; + struct object_id oid; +}; + +struct tag { + struct tag *next_tag; + const char *name; + unsigned int pack_id; + struct object_id oid; +}; + +struct hash_list { + struct hash_list *next; + struct object_id oid; +}; + +typedef enum { + WHENSPEC_RAW = 1, + WHENSPEC_RAW_PERMISSIVE, + WHENSPEC_RFC2822, + WHENSPEC_NOW +} whenspec_type; + +struct recent_command { + struct recent_command *prev; + struct recent_command *next; + char *buf; +}; + +typedef void (*mark_set_inserter_t)(struct mark_set *s, struct object_id *oid, uintmax_t mark); +typedef void (*each_mark_fn_t)(uintmax_t mark, void *obj, void *cbp); + +/* Configured limits on output */ +static unsigned long max_depth = 50; +static off_t max_packsize; +static int unpack_limit = 100; +static int force_update; + +/* Stats and misc. counters */ +static uintmax_t alloc_count; +static uintmax_t marks_set_count; +static uintmax_t object_count_by_type[1 << TYPE_BITS]; +static uintmax_t duplicate_count_by_type[1 << TYPE_BITS]; +static uintmax_t delta_count_by_type[1 << TYPE_BITS]; +static uintmax_t delta_count_attempts_by_type[1 << TYPE_BITS]; +static unsigned long object_count; +static unsigned long branch_count; +static unsigned long branch_load_count; +static int failure; +static FILE *pack_edges; +static unsigned int show_stats = 1; +static int global_argc; +static const char **global_argv; + +/* Memory pools */ +static struct mem_pool fi_mem_pool = {NULL, 2*1024*1024 - + sizeof(struct mp_block), 0 }; + +/* Atom management */ +static unsigned int atom_table_sz = 4451; +static unsigned int atom_cnt; +static struct atom_str **atom_table; + +/* The .pack file being generated */ +static struct pack_idx_option pack_idx_opts; +static unsigned int pack_id; +static struct hashfile *pack_file; +static struct packed_git *pack_data; +static struct packed_git **all_packs; +static off_t pack_size; + +/* Table of objects we've written. */ +static unsigned int object_entry_alloc = 5000; +static struct object_entry_pool *blocks; +static struct hashmap object_table; +static struct mark_set *marks; +static const char *export_marks_file; +static const char *import_marks_file; +static int import_marks_file_from_stream; +static int import_marks_file_ignore_missing; +static int import_marks_file_done; +static int relative_marks_paths; + +/* Our last blob */ +static struct last_object last_blob = { STRBUF_INIT, 0, 0, 0 }; + +/* Tree management */ +static unsigned int tree_entry_alloc = 1000; +static void *avail_tree_entry; +static unsigned int avail_tree_table_sz = 100; +static struct avail_tree_content **avail_tree_table; +static size_t tree_entry_allocd; +static struct strbuf old_tree = STRBUF_INIT; +static struct strbuf new_tree = STRBUF_INIT; + +/* Branch data */ +static unsigned long max_active_branches = 5; +static unsigned long cur_active_branches; +static unsigned long branch_table_sz = 1039; +static struct branch **branch_table; +static struct branch *active_branches; + +/* Tag data */ +static struct tag *first_tag; +static struct tag *last_tag; + +/* Input stream parsing */ +static whenspec_type whenspec = WHENSPEC_RAW; +static struct strbuf command_buf = STRBUF_INIT; +static int unread_command_buf; +static struct recent_command cmd_hist = {&cmd_hist, &cmd_hist, NULL}; +static struct recent_command *cmd_tail = &cmd_hist; +static struct recent_command *rc_free; +static unsigned int cmd_save = 100; +static uintmax_t next_mark; +static struct strbuf new_data = STRBUF_INIT; +static int seen_data_command; +static int require_explicit_termination; +static int allow_unsafe_features; + +/* Signal handling */ +static volatile sig_atomic_t checkpoint_requested; + +/* Submodule marks */ +static struct string_list sub_marks_from = STRING_LIST_INIT_DUP; +static struct string_list sub_marks_to = STRING_LIST_INIT_DUP; +static kh_oid_map_t *sub_oid_map; + +/* Where to write output of cat-blob commands */ +static int cat_blob_fd = STDOUT_FILENO; + +static void parse_argv(void); +static void parse_get_mark(const char *p); +static void parse_cat_blob(const char *p); +static void parse_ls(const char *p, struct branch *b); + +static void for_each_mark(struct mark_set *m, uintmax_t base, each_mark_fn_t callback, void *p) +{ + uintmax_t k; + if (m->shift) { + for (k = 0; k < 1024; k++) { + if (m->data.sets[k]) + for_each_mark(m->data.sets[k], base + (k << m->shift), callback, p); + } + } else { + for (k = 0; k < 1024; k++) { + if (m->data.marked[k]) + callback(base + k, m->data.marked[k], p); + } + } +} + +static void dump_marks_fn(uintmax_t mark, void *object, void *cbp) { + struct object_entry *e = object; + FILE *f = cbp; + + fprintf(f, ":%" PRIuMAX " %s\n", mark, oid_to_hex(&e->idx.oid)); +} + +static void write_branch_report(FILE *rpt, struct branch *b) +{ + fprintf(rpt, "%s:\n", b->name); + + fprintf(rpt, " status :"); + if (b->active) + fputs(" active", rpt); + if (b->branch_tree.tree) + fputs(" loaded", rpt); + if (is_null_oid(&b->branch_tree.versions[1].oid)) + fputs(" dirty", rpt); + fputc('\n', rpt); + + fprintf(rpt, " tip commit : %s\n", oid_to_hex(&b->oid)); + fprintf(rpt, " old tree : %s\n", + oid_to_hex(&b->branch_tree.versions[0].oid)); + fprintf(rpt, " cur tree : %s\n", + oid_to_hex(&b->branch_tree.versions[1].oid)); + fprintf(rpt, " commit clock: %" PRIuMAX "\n", b->last_commit); + + fputs(" last pack : ", rpt); + if (b->pack_id < MAX_PACK_ID) + fprintf(rpt, "%u", b->pack_id); + fputc('\n', rpt); + + fputc('\n', rpt); +} + +static void write_crash_report(const char *err) +{ + char *loc = git_pathdup("fast_import_crash_%"PRIuMAX, (uintmax_t) getpid()); + FILE *rpt = fopen(loc, "w"); + struct branch *b; + unsigned long lu; + struct recent_command *rc; + + if (!rpt) { + error_errno("can't write crash report %s", loc); + free(loc); + return; + } + + fprintf(stderr, "fast-import: dumping crash report to %s\n", loc); + + fprintf(rpt, "fast-import crash report:\n"); + fprintf(rpt, " fast-import process: %"PRIuMAX"\n", (uintmax_t) getpid()); + fprintf(rpt, " parent process : %"PRIuMAX"\n", (uintmax_t) getppid()); + fprintf(rpt, " at %s\n", show_date(time(NULL), 0, DATE_MODE(ISO8601))); + fputc('\n', rpt); + + fputs("fatal: ", rpt); + fputs(err, rpt); + fputc('\n', rpt); + + fputc('\n', rpt); + fputs("Most Recent Commands Before Crash\n", rpt); + fputs("---------------------------------\n", rpt); + for (rc = cmd_hist.next; rc != &cmd_hist; rc = rc->next) { + if (rc->next == &cmd_hist) + fputs("* ", rpt); + else + fputs(" ", rpt); + fputs(rc->buf, rpt); + fputc('\n', rpt); + } + + fputc('\n', rpt); + fputs("Active Branch LRU\n", rpt); + fputs("-----------------\n", rpt); + fprintf(rpt, " active_branches = %lu cur, %lu max\n", + cur_active_branches, + max_active_branches); + fputc('\n', rpt); + fputs(" pos clock name\n", rpt); + fputs(" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n", rpt); + for (b = active_branches, lu = 0; b; b = b->active_next_branch) + fprintf(rpt, " %2lu) %6" PRIuMAX" %s\n", + ++lu, b->last_commit, b->name); + + fputc('\n', rpt); + fputs("Inactive Branches\n", rpt); + fputs("-----------------\n", rpt); + for (lu = 0; lu < branch_table_sz; lu++) { + for (b = branch_table[lu]; b; b = b->table_next_branch) + write_branch_report(rpt, b); + } + + if (first_tag) { + struct tag *tg; + fputc('\n', rpt); + fputs("Annotated Tags\n", rpt); + fputs("--------------\n", rpt); + for (tg = first_tag; tg; tg = tg->next_tag) { + fputs(oid_to_hex(&tg->oid), rpt); + fputc(' ', rpt); + fputs(tg->name, rpt); + fputc('\n', rpt); + } + } + + fputc('\n', rpt); + fputs("Marks\n", rpt); + fputs("-----\n", rpt); + if (export_marks_file) + fprintf(rpt, " exported to %s\n", export_marks_file); + else + for_each_mark(marks, 0, dump_marks_fn, rpt); + + fputc('\n', rpt); + fputs("-------------------\n", rpt); + fputs("END OF CRASH REPORT\n", rpt); + fclose(rpt); + free(loc); +} + +static void end_packfile(void); +static void unkeep_all_packs(void); +static void dump_marks(void); + +static NORETURN void die_nicely(const char *err, va_list params) +{ + static int zombie; + char message[2 * PATH_MAX]; + + vsnprintf(message, sizeof(message), err, params); + fputs("fatal: ", stderr); + fputs(message, stderr); + fputc('\n', stderr); + + if (!zombie) { + zombie = 1; + write_crash_report(message); + end_packfile(); + unkeep_all_packs(); + dump_marks(); + } + exit(128); +} + +#ifndef SIGUSR1 /* Windows, for example */ + +static void set_checkpoint_signal(void) +{ +} + +#else + +static void checkpoint_signal(int signo) +{ + checkpoint_requested = 1; +} + +static void set_checkpoint_signal(void) +{ + struct sigaction sa; + + memset(&sa, 0, sizeof(sa)); + sa.sa_handler = checkpoint_signal; + sigemptyset(&sa.sa_mask); + sa.sa_flags = SA_RESTART; + sigaction(SIGUSR1, &sa, NULL); +} + +#endif + +static void alloc_objects(unsigned int cnt) +{ + struct object_entry_pool *b; + + b = xmalloc(sizeof(struct object_entry_pool) + + cnt * sizeof(struct object_entry)); + b->next_pool = blocks; + b->next_free = b->entries; + b->end = b->entries + cnt; + blocks = b; + alloc_count += cnt; +} + +static struct object_entry *new_object(struct object_id *oid) +{ + struct object_entry *e; + + if (blocks->next_free == blocks->end) + alloc_objects(object_entry_alloc); + + e = blocks->next_free++; + oidcpy(&e->idx.oid, oid); + return e; +} + +static struct object_entry *find_object(struct object_id *oid) +{ + return hashmap_get_entry_from_hash(&object_table, oidhash(oid), oid, + struct object_entry, ent); +} + +static struct object_entry *insert_object(struct object_id *oid) +{ + struct object_entry *e; + unsigned int hash = oidhash(oid); + + e = hashmap_get_entry_from_hash(&object_table, hash, oid, + struct object_entry, ent); + if (!e) { + e = new_object(oid); + e->idx.offset = 0; + hashmap_entry_init(&e->ent, hash); + hashmap_add(&object_table, &e->ent); + } + + return e; +} + +static void invalidate_pack_id(unsigned int id) +{ + unsigned long lu; + struct tag *t; + struct hashmap_iter iter; + struct object_entry *e; + + hashmap_for_each_entry(&object_table, &iter, e, ent) { + if (e->pack_id == id) + e->pack_id = MAX_PACK_ID; + } + + for (lu = 0; lu < branch_table_sz; lu++) { + struct branch *b; + + for (b = branch_table[lu]; b; b = b->table_next_branch) + if (b->pack_id == id) + b->pack_id = MAX_PACK_ID; + } + + for (t = first_tag; t; t = t->next_tag) + if (t->pack_id == id) + t->pack_id = MAX_PACK_ID; +} + +static unsigned int hc_str(const char *s, size_t len) +{ + unsigned int r = 0; + while (len-- > 0) + r = r * 31 + *s++; + return r; +} + +static char *pool_strdup(const char *s) +{ + size_t len = strlen(s) + 1; + char *r = mem_pool_alloc(&fi_mem_pool, len); + memcpy(r, s, len); + return r; +} + +static void insert_mark(struct mark_set *s, uintmax_t idnum, struct object_entry *oe) +{ + while ((idnum >> s->shift) >= 1024) { + s = mem_pool_calloc(&fi_mem_pool, 1, sizeof(struct mark_set)); + s->shift = marks->shift + 10; + s->data.sets[0] = marks; + marks = s; + } + while (s->shift) { + uintmax_t i = idnum >> s->shift; + idnum -= i << s->shift; + if (!s->data.sets[i]) { + s->data.sets[i] = mem_pool_calloc(&fi_mem_pool, 1, sizeof(struct mark_set)); + s->data.sets[i]->shift = s->shift - 10; + } + s = s->data.sets[i]; + } + if (!s->data.marked[idnum]) + marks_set_count++; + s->data.marked[idnum] = oe; +} + +static void *find_mark(struct mark_set *s, uintmax_t idnum) +{ + uintmax_t orig_idnum = idnum; + struct object_entry *oe = NULL; + if ((idnum >> s->shift) < 1024) { + while (s && s->shift) { + uintmax_t i = idnum >> s->shift; + idnum -= i << s->shift; + s = s->data.sets[i]; + } + if (s) + oe = s->data.marked[idnum]; + } + if (!oe) + die("mark :%" PRIuMAX " not declared", orig_idnum); + return oe; +} + +static struct atom_str *to_atom(const char *s, unsigned short len) +{ + unsigned int hc = hc_str(s, len) % atom_table_sz; + struct atom_str *c; + + for (c = atom_table[hc]; c; c = c->next_atom) + if (c->str_len == len && !strncmp(s, c->str_dat, len)) + return c; + + c = mem_pool_alloc(&fi_mem_pool, sizeof(struct atom_str) + len + 1); + c->str_len = len; + memcpy(c->str_dat, s, len); + c->str_dat[len] = 0; + c->next_atom = atom_table[hc]; + atom_table[hc] = c; + atom_cnt++; + return c; +} + +static struct branch *lookup_branch(const char *name) +{ + unsigned int hc = hc_str(name, strlen(name)) % branch_table_sz; + struct branch *b; + + for (b = branch_table[hc]; b; b = b->table_next_branch) + if (!strcmp(name, b->name)) + return b; + return NULL; +} + +static struct branch *new_branch(const char *name) +{ + unsigned int hc = hc_str(name, strlen(name)) % branch_table_sz; + struct branch *b = lookup_branch(name); + + if (b) + die("Invalid attempt to create duplicate branch: %s", name); + if (check_refname_format(name, REFNAME_ALLOW_ONELEVEL)) + die("Branch name doesn't conform to GIT standards: %s", name); + + b = mem_pool_calloc(&fi_mem_pool, 1, sizeof(struct branch)); + b->name = pool_strdup(name); + b->table_next_branch = branch_table[hc]; + b->branch_tree.versions[0].mode = S_IFDIR; + b->branch_tree.versions[1].mode = S_IFDIR; + b->num_notes = 0; + b->active = 0; + b->pack_id = MAX_PACK_ID; + branch_table[hc] = b; + branch_count++; + return b; +} + +static unsigned int hc_entries(unsigned int cnt) +{ + cnt = cnt & 7 ? (cnt / 8) + 1 : cnt / 8; + return cnt < avail_tree_table_sz ? cnt : avail_tree_table_sz - 1; +} + +static struct tree_content *new_tree_content(unsigned int cnt) +{ + struct avail_tree_content *f, *l = NULL; + struct tree_content *t; + unsigned int hc = hc_entries(cnt); + + for (f = avail_tree_table[hc]; f; l = f, f = f->next_avail) + if (f->entry_capacity >= cnt) + break; + + if (f) { + if (l) + l->next_avail = f->next_avail; + else + avail_tree_table[hc] = f->next_avail; + } else { + cnt = cnt & 7 ? ((cnt / 8) + 1) * 8 : cnt; + f = mem_pool_alloc(&fi_mem_pool, sizeof(*t) + sizeof(t->entries[0]) * cnt); + f->entry_capacity = cnt; + } + + t = (struct tree_content*)f; + t->entry_count = 0; + t->delta_depth = 0; + return t; +} + +static void release_tree_entry(struct tree_entry *e); +static void release_tree_content(struct tree_content *t) +{ + struct avail_tree_content *f = (struct avail_tree_content*)t; + unsigned int hc = hc_entries(f->entry_capacity); + f->next_avail = avail_tree_table[hc]; + avail_tree_table[hc] = f; +} + +static void release_tree_content_recursive(struct tree_content *t) +{ + unsigned int i; + for (i = 0; i < t->entry_count; i++) + release_tree_entry(t->entries[i]); + release_tree_content(t); +} + +static struct tree_content *grow_tree_content( + struct tree_content *t, + int amt) +{ + struct tree_content *r = new_tree_content(t->entry_count + amt); + r->entry_count = t->entry_count; + r->delta_depth = t->delta_depth; + COPY_ARRAY(r->entries, t->entries, t->entry_count); + release_tree_content(t); + return r; +} + +static struct tree_entry *new_tree_entry(void) +{ + struct tree_entry *e; + + if (!avail_tree_entry) { + unsigned int n = tree_entry_alloc; + tree_entry_allocd += n * sizeof(struct tree_entry); + ALLOC_ARRAY(e, n); + avail_tree_entry = e; + while (n-- > 1) { + *((void**)e) = e + 1; + e++; + } + *((void**)e) = NULL; + } + + e = avail_tree_entry; + avail_tree_entry = *((void**)e); + return e; +} + +static void release_tree_entry(struct tree_entry *e) +{ + if (e->tree) + release_tree_content_recursive(e->tree); + *((void**)e) = avail_tree_entry; + avail_tree_entry = e; +} + +static struct tree_content *dup_tree_content(struct tree_content *s) +{ + struct tree_content *d; + struct tree_entry *a, *b; + unsigned int i; + + if (!s) + return NULL; + d = new_tree_content(s->entry_count); + for (i = 0; i < s->entry_count; i++) { + a = s->entries[i]; + b = new_tree_entry(); + memcpy(b, a, sizeof(*a)); + if (a->tree && is_null_oid(&b->versions[1].oid)) + b->tree = dup_tree_content(a->tree); + else + b->tree = NULL; + d->entries[i] = b; + } + d->entry_count = s->entry_count; + d->delta_depth = s->delta_depth; + + return d; +} + +static void start_packfile(void) +{ + struct strbuf tmp_file = STRBUF_INIT; + struct packed_git *p; + struct pack_header hdr; + int pack_fd; + + pack_fd = odb_mkstemp(&tmp_file, "pack/tmp_pack_XXXXXX"); + FLEX_ALLOC_STR(p, pack_name, tmp_file.buf); + strbuf_release(&tmp_file); + + p->pack_fd = pack_fd; + p->do_not_close = 1; + pack_file = hashfd(pack_fd, p->pack_name); + + hdr.hdr_signature = htonl(PACK_SIGNATURE); + hdr.hdr_version = htonl(2); + hdr.hdr_entries = 0; + hashwrite(pack_file, &hdr, sizeof(hdr)); + + pack_data = p; + pack_size = sizeof(hdr); + object_count = 0; + + REALLOC_ARRAY(all_packs, pack_id + 1); + all_packs[pack_id] = p; +} + +static const char *create_index(void) +{ + const char *tmpfile; + struct pack_idx_entry **idx, **c, **last; + struct object_entry *e; + struct object_entry_pool *o; + + /* Build the table of object IDs. */ + ALLOC_ARRAY(idx, object_count); + c = idx; + for (o = blocks; o; o = o->next_pool) + for (e = o->next_free; e-- != o->entries;) + if (pack_id == e->pack_id) + *c++ = &e->idx; + last = idx + object_count; + if (c != last) + die("internal consistency error creating the index"); + + tmpfile = write_idx_file(NULL, idx, object_count, &pack_idx_opts, + pack_data->hash); + free(idx); + return tmpfile; +} + +static char *keep_pack(const char *curr_index_name) +{ + static const char *keep_msg = "fast-import"; + struct strbuf name = STRBUF_INIT; + int keep_fd; + + odb_pack_name(&name, pack_data->hash, "keep"); + keep_fd = odb_pack_keep(name.buf); + if (keep_fd < 0) + die_errno("cannot create keep file"); + write_or_die(keep_fd, keep_msg, strlen(keep_msg)); + if (close(keep_fd)) + die_errno("failed to write keep file"); + + odb_pack_name(&name, pack_data->hash, "pack"); + if (finalize_object_file(pack_data->pack_name, name.buf)) + die("cannot store pack file"); + + odb_pack_name(&name, pack_data->hash, "idx"); + if (finalize_object_file(curr_index_name, name.buf)) + die("cannot store index file"); + free((void *)curr_index_name); + return strbuf_detach(&name, NULL); +} + +static void unkeep_all_packs(void) +{ + struct strbuf name = STRBUF_INIT; + int k; + + for (k = 0; k < pack_id; k++) { + struct packed_git *p = all_packs[k]; + odb_pack_name(&name, p->hash, "keep"); + unlink_or_warn(name.buf); + } + strbuf_release(&name); +} + +static int loosen_small_pack(const struct packed_git *p) +{ + struct child_process unpack = CHILD_PROCESS_INIT; + + if (lseek(p->pack_fd, 0, SEEK_SET) < 0) + die_errno("Failed seeking to start of '%s'", p->pack_name); + + unpack.in = p->pack_fd; + unpack.git_cmd = 1; + unpack.stdout_to_stderr = 1; + strvec_push(&unpack.args, "unpack-objects"); + if (!show_stats) + strvec_push(&unpack.args, "-q"); + + return run_command(&unpack); +} + +static void end_packfile(void) +{ + static int running; + + if (running || !pack_data) + return; + + running = 1; + clear_delta_base_cache(); + if (object_count) { + struct packed_git *new_p; + struct object_id cur_pack_oid; + char *idx_name; + int i; + struct branch *b; + struct tag *t; + + close_pack_windows(pack_data); + finalize_hashfile(pack_file, cur_pack_oid.hash, 0); + fixup_pack_header_footer(pack_data->pack_fd, pack_data->hash, + pack_data->pack_name, object_count, + cur_pack_oid.hash, pack_size); + + if (object_count <= unpack_limit) { + if (!loosen_small_pack(pack_data)) { + invalidate_pack_id(pack_id); + goto discard_pack; + } + } + + close(pack_data->pack_fd); + idx_name = keep_pack(create_index()); + + /* Register the packfile with core git's machinery. */ + new_p = add_packed_git(idx_name, strlen(idx_name), 1); + if (!new_p) + die("core git rejected index %s", idx_name); + all_packs[pack_id] = new_p; + install_packed_git(the_repository, new_p); + free(idx_name); + + /* Print the boundary */ + if (pack_edges) { + fprintf(pack_edges, "%s:", new_p->pack_name); + for (i = 0; i < branch_table_sz; i++) { + for (b = branch_table[i]; b; b = b->table_next_branch) { + if (b->pack_id == pack_id) + fprintf(pack_edges, " %s", + oid_to_hex(&b->oid)); + } + } + for (t = first_tag; t; t = t->next_tag) { + if (t->pack_id == pack_id) + fprintf(pack_edges, " %s", + oid_to_hex(&t->oid)); + } + fputc('\n', pack_edges); + fflush(pack_edges); + } + + pack_id++; + } + else { +discard_pack: + close(pack_data->pack_fd); + unlink_or_warn(pack_data->pack_name); + } + FREE_AND_NULL(pack_data); + running = 0; + + /* We can't carry a delta across packfiles. */ + strbuf_release(&last_blob.data); + last_blob.offset = 0; + last_blob.depth = 0; +} + +static void cycle_packfile(void) +{ + end_packfile(); + start_packfile(); +} + +static int store_object( + enum object_type type, + struct strbuf *dat, + struct last_object *last, + struct object_id *oidout, + uintmax_t mark) +{ + void *out, *delta; + struct object_entry *e; + unsigned char hdr[96]; + struct object_id oid; + unsigned long hdrlen, deltalen; + git_hash_ctx c; + git_zstream s; + + hdrlen = xsnprintf((char *)hdr, sizeof(hdr), "%s %lu", + type_name(type), (unsigned long)dat->len) + 1; + the_hash_algo->init_fn(&c); + the_hash_algo->update_fn(&c, hdr, hdrlen); + the_hash_algo->update_fn(&c, dat->buf, dat->len); + the_hash_algo->final_fn(oid.hash, &c); + if (oidout) + oidcpy(oidout, &oid); + + e = insert_object(&oid); + if (mark) + insert_mark(marks, mark, e); + if (e->idx.offset) { + duplicate_count_by_type[type]++; + return 1; + } else if (find_sha1_pack(oid.hash, + get_all_packs(the_repository))) { + e->type = type; + e->pack_id = MAX_PACK_ID; + e->idx.offset = 1; /* just not zero! */ + duplicate_count_by_type[type]++; + return 1; + } + + if (last && last->data.len && last->data.buf && last->depth < max_depth + && dat->len > the_hash_algo->rawsz) { + + delta_count_attempts_by_type[type]++; + delta = diff_delta(last->data.buf, last->data.len, + dat->buf, dat->len, + &deltalen, dat->len - the_hash_algo->rawsz); + } else + delta = NULL; + + git_deflate_init(&s, pack_compression_level); + if (delta) { + s.next_in = delta; + s.avail_in = deltalen; + } else { + s.next_in = (void *)dat->buf; + s.avail_in = dat->len; + } + s.avail_out = git_deflate_bound(&s, s.avail_in); + s.next_out = out = xmalloc(s.avail_out); + while (git_deflate(&s, Z_FINISH) == Z_OK) + ; /* nothing */ + git_deflate_end(&s); + + /* Determine if we should auto-checkpoint. */ + if ((max_packsize + && (pack_size + PACK_SIZE_THRESHOLD + s.total_out) > max_packsize) + || (pack_size + PACK_SIZE_THRESHOLD + s.total_out) < pack_size) { + + /* This new object needs to *not* have the current pack_id. */ + e->pack_id = pack_id + 1; + cycle_packfile(); + + /* We cannot carry a delta into the new pack. */ + if (delta) { + FREE_AND_NULL(delta); + + git_deflate_init(&s, pack_compression_level); + s.next_in = (void *)dat->buf; + s.avail_in = dat->len; + s.avail_out = git_deflate_bound(&s, s.avail_in); + s.next_out = out = xrealloc(out, s.avail_out); + while (git_deflate(&s, Z_FINISH) == Z_OK) + ; /* nothing */ + git_deflate_end(&s); + } + } + + e->type = type; + e->pack_id = pack_id; + e->idx.offset = pack_size; + object_count++; + object_count_by_type[type]++; + + crc32_begin(pack_file); + + if (delta) { + off_t ofs = e->idx.offset - last->offset; + unsigned pos = sizeof(hdr) - 1; + + delta_count_by_type[type]++; + e->depth = last->depth + 1; + + hdrlen = encode_in_pack_object_header(hdr, sizeof(hdr), + OBJ_OFS_DELTA, deltalen); + hashwrite(pack_file, hdr, hdrlen); + pack_size += hdrlen; + + hdr[pos] = ofs & 127; + while (ofs >>= 7) + hdr[--pos] = 128 | (--ofs & 127); + hashwrite(pack_file, hdr + pos, sizeof(hdr) - pos); + pack_size += sizeof(hdr) - pos; + } else { + e->depth = 0; + hdrlen = encode_in_pack_object_header(hdr, sizeof(hdr), + type, dat->len); + hashwrite(pack_file, hdr, hdrlen); + pack_size += hdrlen; + } + + hashwrite(pack_file, out, s.total_out); + pack_size += s.total_out; + + e->idx.crc32 = crc32_end(pack_file); + + free(out); + free(delta); + if (last) { + if (last->no_swap) { + last->data = *dat; + } else { + strbuf_swap(&last->data, dat); + } + last->offset = e->idx.offset; + last->depth = e->depth; + } + return 0; +} + +static void truncate_pack(struct hashfile_checkpoint *checkpoint) +{ + if (hashfile_truncate(pack_file, checkpoint)) + die_errno("cannot truncate pack to skip duplicate"); + pack_size = checkpoint->offset; +} + +static void stream_blob(uintmax_t len, struct object_id *oidout, uintmax_t mark) +{ + size_t in_sz = 64 * 1024, out_sz = 64 * 1024; + unsigned char *in_buf = xmalloc(in_sz); + unsigned char *out_buf = xmalloc(out_sz); + struct object_entry *e; + struct object_id oid; + unsigned long hdrlen; + off_t offset; + git_hash_ctx c; + git_zstream s; + struct hashfile_checkpoint checkpoint; + int status = Z_OK; + + /* Determine if we should auto-checkpoint. */ + if ((max_packsize + && (pack_size + PACK_SIZE_THRESHOLD + len) > max_packsize) + || (pack_size + PACK_SIZE_THRESHOLD + len) < pack_size) + cycle_packfile(); + + hashfile_checkpoint(pack_file, &checkpoint); + offset = checkpoint.offset; + + hdrlen = xsnprintf((char *)out_buf, out_sz, "blob %" PRIuMAX, len) + 1; + + the_hash_algo->init_fn(&c); + the_hash_algo->update_fn(&c, out_buf, hdrlen); + + crc32_begin(pack_file); + + git_deflate_init(&s, pack_compression_level); + + hdrlen = encode_in_pack_object_header(out_buf, out_sz, OBJ_BLOB, len); + + s.next_out = out_buf + hdrlen; + s.avail_out = out_sz - hdrlen; + + while (status != Z_STREAM_END) { + if (0 < len && !s.avail_in) { + size_t cnt = in_sz < len ? in_sz : (size_t)len; + size_t n = fread(in_buf, 1, cnt, stdin); + if (!n && feof(stdin)) + die("EOF in data (%" PRIuMAX " bytes remaining)", len); + + the_hash_algo->update_fn(&c, in_buf, n); + s.next_in = in_buf; + s.avail_in = n; + len -= n; + } + + status = git_deflate(&s, len ? 0 : Z_FINISH); + + if (!s.avail_out || status == Z_STREAM_END) { + size_t n = s.next_out - out_buf; + hashwrite(pack_file, out_buf, n); + pack_size += n; + s.next_out = out_buf; + s.avail_out = out_sz; + } + + switch (status) { + case Z_OK: + case Z_BUF_ERROR: + case Z_STREAM_END: + continue; + default: + die("unexpected deflate failure: %d", status); + } + } + git_deflate_end(&s); + the_hash_algo->final_fn(oid.hash, &c); + + if (oidout) + oidcpy(oidout, &oid); + + e = insert_object(&oid); + + if (mark) + insert_mark(marks, mark, e); + + if (e->idx.offset) { + duplicate_count_by_type[OBJ_BLOB]++; + truncate_pack(&checkpoint); + + } else if (find_sha1_pack(oid.hash, + get_all_packs(the_repository))) { + e->type = OBJ_BLOB; + e->pack_id = MAX_PACK_ID; + e->idx.offset = 1; /* just not zero! */ + duplicate_count_by_type[OBJ_BLOB]++; + truncate_pack(&checkpoint); + + } else { + e->depth = 0; + e->type = OBJ_BLOB; + e->pack_id = pack_id; + e->idx.offset = offset; + e->idx.crc32 = crc32_end(pack_file); + object_count++; + object_count_by_type[OBJ_BLOB]++; + } + + free(in_buf); + free(out_buf); +} + +/* All calls must be guarded by find_object() or find_mark() to + * ensure the 'struct object_entry' passed was written by this + * process instance. We unpack the entry by the offset, avoiding + * the need for the corresponding .idx file. This unpacking rule + * works because we only use OBJ_REF_DELTA within the packfiles + * created by fast-import. + * + * oe must not be NULL. Such an oe usually comes from giving + * an unknown SHA-1 to find_object() or an undefined mark to + * find_mark(). Callers must test for this condition and use + * the standard read_sha1_file() when it happens. + * + * oe->pack_id must not be MAX_PACK_ID. Such an oe is usually from + * find_mark(), where the mark was reloaded from an existing marks + * file and is referencing an object that this fast-import process + * instance did not write out to a packfile. Callers must test for + * this condition and use read_sha1_file() instead. + */ +static void *gfi_unpack_entry( + struct object_entry *oe, + unsigned long *sizep) +{ + enum object_type type; + struct packed_git *p = all_packs[oe->pack_id]; + if (p == pack_data && p->pack_size < (pack_size + the_hash_algo->rawsz)) { + /* The object is stored in the packfile we are writing to + * and we have modified it since the last time we scanned + * back to read a previously written object. If an old + * window covered [p->pack_size, p->pack_size + rawsz) its + * data is stale and is not valid. Closing all windows + * and updating the packfile length ensures we can read + * the newly written data. + */ + close_pack_windows(p); + hashflush(pack_file); + + /* We have to offer rawsz bytes additional on the end of + * the packfile as the core unpacker code assumes the + * footer is present at the file end and must promise + * at least rawsz bytes within any window it maps. But + * we don't actually create the footer here. + */ + p->pack_size = pack_size + the_hash_algo->rawsz; + } + return unpack_entry(the_repository, p, oe->idx.offset, &type, sizep); +} + +static const char *get_mode(const char *str, uint16_t *modep) +{ + unsigned char c; + uint16_t mode = 0; + + while ((c = *str++) != ' ') { + if (c < '0' || c > '7') + return NULL; + mode = (mode << 3) + (c - '0'); + } + *modep = mode; + return str; +} + +static void load_tree(struct tree_entry *root) +{ + struct object_id *oid = &root->versions[1].oid; + struct object_entry *myoe; + struct tree_content *t; + unsigned long size; + char *buf; + const char *c; + + root->tree = t = new_tree_content(8); + if (is_null_oid(oid)) + return; + + myoe = find_object(oid); + if (myoe && myoe->pack_id != MAX_PACK_ID) { + if (myoe->type != OBJ_TREE) + die("Not a tree: %s", oid_to_hex(oid)); + t->delta_depth = myoe->depth; + buf = gfi_unpack_entry(myoe, &size); + if (!buf) + die("Can't load tree %s", oid_to_hex(oid)); + } else { + enum object_type type; + buf = read_object_file(oid, &type, &size); + if (!buf || type != OBJ_TREE) + die("Can't load tree %s", oid_to_hex(oid)); + } + + c = buf; + while (c != (buf + size)) { + struct tree_entry *e = new_tree_entry(); + + if (t->entry_count == t->entry_capacity) + root->tree = t = grow_tree_content(t, t->entry_count); + t->entries[t->entry_count++] = e; + + e->tree = NULL; + c = get_mode(c, &e->versions[1].mode); + if (!c) + die("Corrupt mode in %s", oid_to_hex(oid)); + e->versions[0].mode = e->versions[1].mode; + e->name = to_atom(c, strlen(c)); + c += e->name->str_len + 1; + hashcpy(e->versions[0].oid.hash, (unsigned char *)c); + hashcpy(e->versions[1].oid.hash, (unsigned char *)c); + c += the_hash_algo->rawsz; + } + free(buf); +} + +static int tecmp0 (const void *_a, const void *_b) +{ + struct tree_entry *a = *((struct tree_entry**)_a); + struct tree_entry *b = *((struct tree_entry**)_b); + return base_name_compare( + a->name->str_dat, a->name->str_len, a->versions[0].mode, + b->name->str_dat, b->name->str_len, b->versions[0].mode); +} + +static int tecmp1 (const void *_a, const void *_b) +{ + struct tree_entry *a = *((struct tree_entry**)_a); + struct tree_entry *b = *((struct tree_entry**)_b); + return base_name_compare( + a->name->str_dat, a->name->str_len, a->versions[1].mode, + b->name->str_dat, b->name->str_len, b->versions[1].mode); +} + +static void mktree(struct tree_content *t, int v, struct strbuf *b) +{ + size_t maxlen = 0; + unsigned int i; + + if (!v) + QSORT(t->entries, t->entry_count, tecmp0); + else + QSORT(t->entries, t->entry_count, tecmp1); + + for (i = 0; i < t->entry_count; i++) { + if (t->entries[i]->versions[v].mode) + maxlen += t->entries[i]->name->str_len + 34; + } + + strbuf_reset(b); + strbuf_grow(b, maxlen); + for (i = 0; i < t->entry_count; i++) { + struct tree_entry *e = t->entries[i]; + if (!e->versions[v].mode) + continue; + strbuf_addf(b, "%o %s%c", + (unsigned int)(e->versions[v].mode & ~NO_DELTA), + e->name->str_dat, '\0'); + strbuf_add(b, e->versions[v].oid.hash, the_hash_algo->rawsz); + } +} + +static void store_tree(struct tree_entry *root) +{ + struct tree_content *t; + unsigned int i, j, del; + struct last_object lo = { STRBUF_INIT, 0, 0, /* no_swap */ 1 }; + struct object_entry *le = NULL; + + if (!is_null_oid(&root->versions[1].oid)) + return; + + if (!root->tree) + load_tree(root); + t = root->tree; + + for (i = 0; i < t->entry_count; i++) { + if (t->entries[i]->tree) + store_tree(t->entries[i]); + } + + if (!(root->versions[0].mode & NO_DELTA)) + le = find_object(&root->versions[0].oid); + if (S_ISDIR(root->versions[0].mode) && le && le->pack_id == pack_id) { + mktree(t, 0, &old_tree); + lo.data = old_tree; + lo.offset = le->idx.offset; + lo.depth = t->delta_depth; + } + + mktree(t, 1, &new_tree); + store_object(OBJ_TREE, &new_tree, &lo, &root->versions[1].oid, 0); + + t->delta_depth = lo.depth; + for (i = 0, j = 0, del = 0; i < t->entry_count; i++) { + struct tree_entry *e = t->entries[i]; + if (e->versions[1].mode) { + e->versions[0].mode = e->versions[1].mode; + oidcpy(&e->versions[0].oid, &e->versions[1].oid); + t->entries[j++] = e; + } else { + release_tree_entry(e); + del++; + } + } + t->entry_count -= del; +} + +static void tree_content_replace( + struct tree_entry *root, + const struct object_id *oid, + const uint16_t mode, + struct tree_content *newtree) +{ + if (!S_ISDIR(mode)) + die("Root cannot be a non-directory"); + oidclr(&root->versions[0].oid); + oidcpy(&root->versions[1].oid, oid); + if (root->tree) + release_tree_content_recursive(root->tree); + root->tree = newtree; +} + +static int tree_content_set( + struct tree_entry *root, + const char *p, + const struct object_id *oid, + const uint16_t mode, + struct tree_content *subtree) +{ + struct tree_content *t; + const char *slash1; + unsigned int i, n; + struct tree_entry *e; + + slash1 = strchrnul(p, '/'); + n = slash1 - p; + if (!n) + die("Empty path component found in input"); + if (!*slash1 && !S_ISDIR(mode) && subtree) + die("Non-directories cannot have subtrees"); + + if (!root->tree) + load_tree(root); + t = root->tree; + for (i = 0; i < t->entry_count; i++) { + e = t->entries[i]; + if (e->name->str_len == n && !fspathncmp(p, e->name->str_dat, n)) { + if (!*slash1) { + if (!S_ISDIR(mode) + && e->versions[1].mode == mode + && oideq(&e->versions[1].oid, oid)) + return 0; + e->versions[1].mode = mode; + oidcpy(&e->versions[1].oid, oid); + if (e->tree) + release_tree_content_recursive(e->tree); + e->tree = subtree; + + /* + * We need to leave e->versions[0].sha1 alone + * to avoid modifying the preimage tree used + * when writing out the parent directory. + * But after replacing the subdir with a + * completely different one, it's not a good + * delta base any more, and besides, we've + * thrown away the tree entries needed to + * make a delta against it. + * + * So let's just explicitly disable deltas + * for the subtree. + */ + if (S_ISDIR(e->versions[0].mode)) + e->versions[0].mode |= NO_DELTA; + + oidclr(&root->versions[1].oid); + return 1; + } + if (!S_ISDIR(e->versions[1].mode)) { + e->tree = new_tree_content(8); + e->versions[1].mode = S_IFDIR; + } + if (!e->tree) + load_tree(e); + if (tree_content_set(e, slash1 + 1, oid, mode, subtree)) { + oidclr(&root->versions[1].oid); + return 1; + } + return 0; + } + } + + if (t->entry_count == t->entry_capacity) + root->tree = t = grow_tree_content(t, t->entry_count); + e = new_tree_entry(); + e->name = to_atom(p, n); + e->versions[0].mode = 0; + oidclr(&e->versions[0].oid); + t->entries[t->entry_count++] = e; + if (*slash1) { + e->tree = new_tree_content(8); + e->versions[1].mode = S_IFDIR; + tree_content_set(e, slash1 + 1, oid, mode, subtree); + } else { + e->tree = subtree; + e->versions[1].mode = mode; + oidcpy(&e->versions[1].oid, oid); + } + oidclr(&root->versions[1].oid); + return 1; +} + +static int tree_content_remove( + struct tree_entry *root, + const char *p, + struct tree_entry *backup_leaf, + int allow_root) +{ + struct tree_content *t; + const char *slash1; + unsigned int i, n; + struct tree_entry *e; + + slash1 = strchrnul(p, '/'); + n = slash1 - p; + + if (!root->tree) + load_tree(root); + + if (!*p && allow_root) { + e = root; + goto del_entry; + } + + t = root->tree; + for (i = 0; i < t->entry_count; i++) { + e = t->entries[i]; + if (e->name->str_len == n && !fspathncmp(p, e->name->str_dat, n)) { + if (*slash1 && !S_ISDIR(e->versions[1].mode)) + /* + * If p names a file in some subdirectory, and a + * file or symlink matching the name of the + * parent directory of p exists, then p cannot + * exist and need not be deleted. + */ + return 1; + if (!*slash1 || !S_ISDIR(e->versions[1].mode)) + goto del_entry; + if (!e->tree) + load_tree(e); + if (tree_content_remove(e, slash1 + 1, backup_leaf, 0)) { + for (n = 0; n < e->tree->entry_count; n++) { + if (e->tree->entries[n]->versions[1].mode) { + oidclr(&root->versions[1].oid); + return 1; + } + } + backup_leaf = NULL; + goto del_entry; + } + return 0; + } + } + return 0; + +del_entry: + if (backup_leaf) + memcpy(backup_leaf, e, sizeof(*backup_leaf)); + else if (e->tree) + release_tree_content_recursive(e->tree); + e->tree = NULL; + e->versions[1].mode = 0; + oidclr(&e->versions[1].oid); + oidclr(&root->versions[1].oid); + return 1; +} + +static int tree_content_get( + struct tree_entry *root, + const char *p, + struct tree_entry *leaf, + int allow_root) +{ + struct tree_content *t; + const char *slash1; + unsigned int i, n; + struct tree_entry *e; + + slash1 = strchrnul(p, '/'); + n = slash1 - p; + if (!n && !allow_root) + die("Empty path component found in input"); + + if (!root->tree) + load_tree(root); + + if (!n) { + e = root; + goto found_entry; + } + + t = root->tree; + for (i = 0; i < t->entry_count; i++) { + e = t->entries[i]; + if (e->name->str_len == n && !fspathncmp(p, e->name->str_dat, n)) { + if (!*slash1) + goto found_entry; + if (!S_ISDIR(e->versions[1].mode)) + return 0; + if (!e->tree) + load_tree(e); + return tree_content_get(e, slash1 + 1, leaf, 0); + } + } + return 0; + +found_entry: + memcpy(leaf, e, sizeof(*leaf)); + if (e->tree && is_null_oid(&e->versions[1].oid)) + leaf->tree = dup_tree_content(e->tree); + else + leaf->tree = NULL; + return 1; +} + +static int update_branch(struct branch *b) +{ + static const char *msg = "fast-import"; + struct ref_transaction *transaction; + struct object_id old_oid; + struct strbuf err = STRBUF_INIT; + + if (is_null_oid(&b->oid)) { + if (b->delete) + delete_ref(NULL, b->name, NULL, 0); + return 0; + } + if (read_ref(b->name, &old_oid)) + oidclr(&old_oid); + if (!force_update && !is_null_oid(&old_oid)) { + struct commit *old_cmit, *new_cmit; + + old_cmit = lookup_commit_reference_gently(the_repository, + &old_oid, 0); + new_cmit = lookup_commit_reference_gently(the_repository, + &b->oid, 0); + if (!old_cmit || !new_cmit) + return error("Branch %s is missing commits.", b->name); + + if (!in_merge_bases(old_cmit, new_cmit)) { + warning("Not updating %s" + " (new tip %s does not contain %s)", + b->name, oid_to_hex(&b->oid), + oid_to_hex(&old_oid)); + return -1; + } + } + transaction = ref_transaction_begin(&err); + if (!transaction || + ref_transaction_update(transaction, b->name, &b->oid, &old_oid, + 0, msg, &err) || + ref_transaction_commit(transaction, &err)) { + ref_transaction_free(transaction); + error("%s", err.buf); + strbuf_release(&err); + return -1; + } + ref_transaction_free(transaction); + strbuf_release(&err); + return 0; +} + +static void dump_branches(void) +{ + unsigned int i; + struct branch *b; + + for (i = 0; i < branch_table_sz; i++) { + for (b = branch_table[i]; b; b = b->table_next_branch) + failure |= update_branch(b); + } +} + +static void dump_tags(void) +{ + static const char *msg = "fast-import"; + struct tag *t; + struct strbuf ref_name = STRBUF_INIT; + struct strbuf err = STRBUF_INIT; + struct ref_transaction *transaction; + + transaction = ref_transaction_begin(&err); + if (!transaction) { + failure |= error("%s", err.buf); + goto cleanup; + } + for (t = first_tag; t; t = t->next_tag) { + strbuf_reset(&ref_name); + strbuf_addf(&ref_name, "refs/tags/%s", t->name); + + if (ref_transaction_update(transaction, ref_name.buf, + &t->oid, NULL, 0, msg, &err)) { + failure |= error("%s", err.buf); + goto cleanup; + } + } + if (ref_transaction_commit(transaction, &err)) + failure |= error("%s", err.buf); + + cleanup: + ref_transaction_free(transaction); + strbuf_release(&ref_name); + strbuf_release(&err); +} + +static void dump_marks(void) +{ + struct lock_file mark_lock = LOCK_INIT; + FILE *f; + + if (!export_marks_file || (import_marks_file && !import_marks_file_done)) + return; + + if (safe_create_leading_directories_const(export_marks_file)) { + failure |= error_errno("unable to create leading directories of %s", + export_marks_file); + return; + } + + if (hold_lock_file_for_update(&mark_lock, export_marks_file, 0) < 0) { + failure |= error_errno("Unable to write marks file %s", + export_marks_file); + return; + } + + f = fdopen_lock_file(&mark_lock, "w"); + if (!f) { + int saved_errno = errno; + rollback_lock_file(&mark_lock); + failure |= error("Unable to write marks file %s: %s", + export_marks_file, strerror(saved_errno)); + return; + } + + for_each_mark(marks, 0, dump_marks_fn, f); + if (commit_lock_file(&mark_lock)) { + failure |= error_errno("Unable to write file %s", + export_marks_file); + return; + } +} + +static void insert_object_entry(struct mark_set *s, struct object_id *oid, uintmax_t mark) +{ + struct object_entry *e; + e = find_object(oid); + if (!e) { + enum object_type type = oid_object_info(the_repository, + oid, NULL); + if (type < 0) + die("object not found: %s", oid_to_hex(oid)); + e = insert_object(oid); + e->type = type; + e->pack_id = MAX_PACK_ID; + e->idx.offset = 1; /* just not zero! */ + } + insert_mark(s, mark, e); +} + +static void insert_oid_entry(struct mark_set *s, struct object_id *oid, uintmax_t mark) +{ + insert_mark(s, mark, xmemdupz(oid, sizeof(*oid))); +} + +static void read_mark_file(struct mark_set *s, FILE *f, mark_set_inserter_t inserter) +{ + char line[512]; + while (fgets(line, sizeof(line), f)) { + uintmax_t mark; + char *end; + struct object_id oid; + + /* Ensure SHA-1 objects are padded with zeros. */ + memset(oid.hash, 0, sizeof(oid.hash)); + + end = strchr(line, '\n'); + if (line[0] != ':' || !end) + die("corrupt mark line: %s", line); + *end = 0; + mark = strtoumax(line + 1, &end, 10); + if (!mark || end == line + 1 + || *end != ' ' + || get_oid_hex_any(end + 1, &oid) == GIT_HASH_UNKNOWN) + die("corrupt mark line: %s", line); + inserter(s, &oid, mark); + } +} + +static void read_marks(void) +{ + FILE *f = fopen(import_marks_file, "r"); + if (f) + ; + else if (import_marks_file_ignore_missing && errno == ENOENT) + goto done; /* Marks file does not exist */ + else + die_errno("cannot read '%s'", import_marks_file); + read_mark_file(marks, f, insert_object_entry); + fclose(f); +done: + import_marks_file_done = 1; +} + + +static int read_next_command(void) +{ + static int stdin_eof = 0; + + if (stdin_eof) { + unread_command_buf = 0; + return EOF; + } + + for (;;) { + if (unread_command_buf) { + unread_command_buf = 0; + } else { + struct recent_command *rc; + + stdin_eof = strbuf_getline_lf(&command_buf, stdin); + if (stdin_eof) + return EOF; + + if (!seen_data_command + && !starts_with(command_buf.buf, "feature ") + && !starts_with(command_buf.buf, "option ")) { + parse_argv(); + } + + rc = rc_free; + if (rc) + rc_free = rc->next; + else { + rc = cmd_hist.next; + cmd_hist.next = rc->next; + cmd_hist.next->prev = &cmd_hist; + free(rc->buf); + } + + rc->buf = xstrdup(command_buf.buf); + rc->prev = cmd_tail; + rc->next = cmd_hist.prev; + rc->prev->next = rc; + cmd_tail = rc; + } + if (command_buf.buf[0] == '#') + continue; + return 0; + } +} + +static void skip_optional_lf(void) +{ + int term_char = fgetc(stdin); + if (term_char != '\n' && term_char != EOF) + ungetc(term_char, stdin); +} + +static void parse_mark(void) +{ + const char *v; + if (skip_prefix(command_buf.buf, "mark :", &v)) { + next_mark = strtoumax(v, NULL, 10); + read_next_command(); + } + else + next_mark = 0; +} + +static void parse_original_identifier(void) +{ + const char *v; + if (skip_prefix(command_buf.buf, "original-oid ", &v)) + read_next_command(); +} + +static int parse_data(struct strbuf *sb, uintmax_t limit, uintmax_t *len_res) +{ + const char *data; + strbuf_reset(sb); + + if (!skip_prefix(command_buf.buf, "data ", &data)) + die("Expected 'data n' command, found: %s", command_buf.buf); + + if (skip_prefix(data, "<<", &data)) { + char *term = xstrdup(data); + size_t term_len = command_buf.len - (data - command_buf.buf); + + for (;;) { + if (strbuf_getline_lf(&command_buf, stdin) == EOF) + die("EOF in data (terminator '%s' not found)", term); + if (term_len == command_buf.len + && !strcmp(term, command_buf.buf)) + break; + strbuf_addbuf(sb, &command_buf); + strbuf_addch(sb, '\n'); + } + free(term); + } + else { + uintmax_t len = strtoumax(data, NULL, 10); + size_t n = 0, length = (size_t)len; + + if (limit && limit < len) { + *len_res = len; + return 0; + } + if (length < len) + die("data is too large to use in this context"); + + while (n < length) { + size_t s = strbuf_fread(sb, length - n, stdin); + if (!s && feof(stdin)) + die("EOF in data (%lu bytes remaining)", + (unsigned long)(length - n)); + n += s; + } + } + + skip_optional_lf(); + return 1; +} + +static int validate_raw_date(const char *src, struct strbuf *result, int strict) +{ + const char *orig_src = src; + char *endp; + unsigned long num; + + errno = 0; + + num = strtoul(src, &endp, 10); + /* + * NEEDSWORK: perhaps check for reasonable values? For example, we + * could error on values representing times more than a + * day in the future. + */ + if (errno || endp == src || *endp != ' ') + return -1; + + src = endp + 1; + if (*src != '-' && *src != '+') + return -1; + + num = strtoul(src + 1, &endp, 10); + /* + * NEEDSWORK: check for brokenness other than num > 1400, such as + * (num % 100) >= 60, or ((num % 100) % 15) != 0 ? + */ + if (errno || endp == src + 1 || *endp || /* did not parse */ + (strict && (1400 < num)) /* parsed a broken timezone */ + ) + return -1; + + strbuf_addstr(result, orig_src); + return 0; +} + +static char *parse_ident(const char *buf) +{ + const char *ltgt; + size_t name_len; + struct strbuf ident = STRBUF_INIT; + + /* ensure there is a space delimiter even if there is no name */ + if (*buf == '<') + --buf; + + ltgt = buf + strcspn(buf, "<>"); + if (*ltgt != '<') + die("Missing < in ident string: %s", buf); + if (ltgt != buf && ltgt[-1] != ' ') + die("Missing space before < in ident string: %s", buf); + ltgt = ltgt + 1 + strcspn(ltgt + 1, "<>"); + if (*ltgt != '>') + die("Missing > in ident string: %s", buf); + ltgt++; + if (*ltgt != ' ') + die("Missing space after > in ident string: %s", buf); + ltgt++; + name_len = ltgt - buf; + strbuf_add(&ident, buf, name_len); + + switch (whenspec) { + case WHENSPEC_RAW: + if (validate_raw_date(ltgt, &ident, 1) < 0) + die("Invalid raw date \"%s\" in ident: %s", ltgt, buf); + break; + case WHENSPEC_RAW_PERMISSIVE: + if (validate_raw_date(ltgt, &ident, 0) < 0) + die("Invalid raw date \"%s\" in ident: %s", ltgt, buf); + break; + case WHENSPEC_RFC2822: + if (parse_date(ltgt, &ident) < 0) + die("Invalid rfc2822 date \"%s\" in ident: %s", ltgt, buf); + break; + case WHENSPEC_NOW: + if (strcmp("now", ltgt)) + die("Date in ident must be 'now': %s", buf); + datestamp(&ident); + break; + } + + return strbuf_detach(&ident, NULL); +} + +static void parse_and_store_blob( + struct last_object *last, + struct object_id *oidout, + uintmax_t mark) +{ + static struct strbuf buf = STRBUF_INIT; + uintmax_t len; + + if (parse_data(&buf, big_file_threshold, &len)) + store_object(OBJ_BLOB, &buf, last, oidout, mark); + else { + if (last) { + strbuf_release(&last->data); + last->offset = 0; + last->depth = 0; + } + stream_blob(len, oidout, mark); + skip_optional_lf(); + } +} + +static void parse_new_blob(void) +{ + read_next_command(); + parse_mark(); + parse_original_identifier(); + parse_and_store_blob(&last_blob, NULL, next_mark); +} + +static void unload_one_branch(void) +{ + while (cur_active_branches + && cur_active_branches >= max_active_branches) { + uintmax_t min_commit = ULONG_MAX; + struct branch *e, *l = NULL, *p = NULL; + + for (e = active_branches; e; e = e->active_next_branch) { + if (e->last_commit < min_commit) { + p = l; + min_commit = e->last_commit; + } + l = e; + } + + if (p) { + e = p->active_next_branch; + p->active_next_branch = e->active_next_branch; + } else { + e = active_branches; + active_branches = e->active_next_branch; + } + e->active = 0; + e->active_next_branch = NULL; + if (e->branch_tree.tree) { + release_tree_content_recursive(e->branch_tree.tree); + e->branch_tree.tree = NULL; + } + cur_active_branches--; + } +} + +static void load_branch(struct branch *b) +{ + load_tree(&b->branch_tree); + if (!b->active) { + b->active = 1; + b->active_next_branch = active_branches; + active_branches = b; + cur_active_branches++; + branch_load_count++; + } +} + +static unsigned char convert_num_notes_to_fanout(uintmax_t num_notes) +{ + unsigned char fanout = 0; + while ((num_notes >>= 8)) + fanout++; + return fanout; +} + +static void construct_path_with_fanout(const char *hex_sha1, + unsigned char fanout, char *path) +{ + unsigned int i = 0, j = 0; + if (fanout >= the_hash_algo->rawsz) + die("Too large fanout (%u)", fanout); + while (fanout) { + path[i++] = hex_sha1[j++]; + path[i++] = hex_sha1[j++]; + path[i++] = '/'; + fanout--; + } + memcpy(path + i, hex_sha1 + j, the_hash_algo->hexsz - j); + path[i + the_hash_algo->hexsz - j] = '\0'; +} + +static uintmax_t do_change_note_fanout( + struct tree_entry *orig_root, struct tree_entry *root, + char *hex_oid, unsigned int hex_oid_len, + char *fullpath, unsigned int fullpath_len, + unsigned char fanout) +{ + struct tree_content *t; + struct tree_entry *e, leaf; + unsigned int i, tmp_hex_oid_len, tmp_fullpath_len; + uintmax_t num_notes = 0; + struct object_id oid; + /* hex oid + '/' between each pair of hex digits + NUL */ + char realpath[GIT_MAX_HEXSZ + ((GIT_MAX_HEXSZ / 2) - 1) + 1]; + const unsigned hexsz = the_hash_algo->hexsz; + + if (!root->tree) + load_tree(root); + t = root->tree; + + for (i = 0; t && i < t->entry_count; i++) { + e = t->entries[i]; + tmp_hex_oid_len = hex_oid_len + e->name->str_len; + tmp_fullpath_len = fullpath_len; + + /* + * We're interested in EITHER existing note entries (entries + * with exactly 40 hex chars in path, not including directory + * separators), OR directory entries that may contain note + * entries (with < 40 hex chars in path). + * Also, each path component in a note entry must be a multiple + * of 2 chars. + */ + if (!e->versions[1].mode || + tmp_hex_oid_len > hexsz || + e->name->str_len % 2) + continue; + + /* This _may_ be a note entry, or a subdir containing notes */ + memcpy(hex_oid + hex_oid_len, e->name->str_dat, + e->name->str_len); + if (tmp_fullpath_len) + fullpath[tmp_fullpath_len++] = '/'; + memcpy(fullpath + tmp_fullpath_len, e->name->str_dat, + e->name->str_len); + tmp_fullpath_len += e->name->str_len; + fullpath[tmp_fullpath_len] = '\0'; + + if (tmp_hex_oid_len == hexsz && !get_oid_hex(hex_oid, &oid)) { + /* This is a note entry */ + if (fanout == 0xff) { + /* Counting mode, no rename */ + num_notes++; + continue; + } + construct_path_with_fanout(hex_oid, fanout, realpath); + if (!strcmp(fullpath, realpath)) { + /* Note entry is in correct location */ + num_notes++; + continue; + } + + /* Rename fullpath to realpath */ + if (!tree_content_remove(orig_root, fullpath, &leaf, 0)) + die("Failed to remove path %s", fullpath); + tree_content_set(orig_root, realpath, + &leaf.versions[1].oid, + leaf.versions[1].mode, + leaf.tree); + } else if (S_ISDIR(e->versions[1].mode)) { + /* This is a subdir that may contain note entries */ + num_notes += do_change_note_fanout(orig_root, e, + hex_oid, tmp_hex_oid_len, + fullpath, tmp_fullpath_len, fanout); + } + + /* The above may have reallocated the current tree_content */ + t = root->tree; + } + return num_notes; +} + +static uintmax_t change_note_fanout(struct tree_entry *root, + unsigned char fanout) +{ + /* + * The size of path is due to one slash between every two hex digits, + * plus the terminating NUL. Note that there is no slash at the end, so + * the number of slashes is one less than half the number of hex + * characters. + */ + char hex_oid[GIT_MAX_HEXSZ], path[GIT_MAX_HEXSZ + (GIT_MAX_HEXSZ / 2) - 1 + 1]; + return do_change_note_fanout(root, root, hex_oid, 0, path, 0, fanout); +} + +static int parse_mapped_oid_hex(const char *hex, struct object_id *oid, const char **end) +{ + int algo; + khiter_t it; + + /* Make SHA-1 object IDs have all-zero padding. */ + memset(oid->hash, 0, sizeof(oid->hash)); + + algo = parse_oid_hex_any(hex, oid, end); + if (algo == GIT_HASH_UNKNOWN) + return -1; + + it = kh_get_oid_map(sub_oid_map, *oid); + /* No such object? */ + if (it == kh_end(sub_oid_map)) { + /* If we're using the same algorithm, pass it through. */ + if (hash_algos[algo].format_id == the_hash_algo->format_id) + return 0; + return -1; + } + oidcpy(oid, kh_value(sub_oid_map, it)); + return 0; +} + +/* + * Given a pointer into a string, parse a mark reference: + * + * idnum ::= ':' bigint; + * + * Return the first character after the value in *endptr. + * + * Complain if the following character is not what is expected, + * either a space or end of the string. + */ +static uintmax_t parse_mark_ref(const char *p, char **endptr) +{ + uintmax_t mark; + + assert(*p == ':'); + p++; + mark = strtoumax(p, endptr, 10); + if (*endptr == p) + die("No value after ':' in mark: %s", command_buf.buf); + return mark; +} + +/* + * Parse the mark reference, and complain if this is not the end of + * the string. + */ +static uintmax_t parse_mark_ref_eol(const char *p) +{ + char *end; + uintmax_t mark; + + mark = parse_mark_ref(p, &end); + if (*end != '\0') + die("Garbage after mark: %s", command_buf.buf); + return mark; +} + +/* + * Parse the mark reference, demanding a trailing space. Return a + * pointer to the space. + */ +static uintmax_t parse_mark_ref_space(const char **p) +{ + uintmax_t mark; + char *end; + + mark = parse_mark_ref(*p, &end); + if (*end++ != ' ') + die("Missing space after mark: %s", command_buf.buf); + *p = end; + return mark; +} + +static void file_change_m(const char *p, struct branch *b) +{ + static struct strbuf uq = STRBUF_INIT; + const char *endp; + struct object_entry *oe; + struct object_id oid; + uint16_t mode, inline_data = 0; + + p = get_mode(p, &mode); + if (!p) + die("Corrupt mode: %s", command_buf.buf); + switch (mode) { + case 0644: + case 0755: + mode |= S_IFREG; + case S_IFREG | 0644: + case S_IFREG | 0755: + case S_IFLNK: + case S_IFDIR: + case S_IFGITLINK: + /* ok */ + break; + default: + die("Corrupt mode: %s", command_buf.buf); + } + + if (*p == ':') { + oe = find_mark(marks, parse_mark_ref_space(&p)); + oidcpy(&oid, &oe->idx.oid); + } else if (skip_prefix(p, "inline ", &p)) { + inline_data = 1; + oe = NULL; /* not used with inline_data, but makes gcc happy */ + } else { + if (parse_mapped_oid_hex(p, &oid, &p)) + die("Invalid dataref: %s", command_buf.buf); + oe = find_object(&oid); + if (*p++ != ' ') + die("Missing space after SHA1: %s", command_buf.buf); + } + + strbuf_reset(&uq); + if (!unquote_c_style(&uq, p, &endp)) { + if (*endp) + die("Garbage after path in: %s", command_buf.buf); + p = uq.buf; + } + + /* Git does not track empty, non-toplevel directories. */ + if (S_ISDIR(mode) && is_empty_tree_oid(&oid) && *p) { + tree_content_remove(&b->branch_tree, p, NULL, 0); + return; + } + + if (S_ISGITLINK(mode)) { + if (inline_data) + die("Git links cannot be specified 'inline': %s", + command_buf.buf); + else if (oe) { + if (oe->type != OBJ_COMMIT) + die("Not a commit (actually a %s): %s", + type_name(oe->type), command_buf.buf); + } + /* + * Accept the sha1 without checking; it expected to be in + * another repository. + */ + } else if (inline_data) { + if (S_ISDIR(mode)) + die("Directories cannot be specified 'inline': %s", + command_buf.buf); + if (p != uq.buf) { + strbuf_addstr(&uq, p); + p = uq.buf; + } + while (read_next_command() != EOF) { + const char *v; + if (skip_prefix(command_buf.buf, "cat-blob ", &v)) + parse_cat_blob(v); + else { + parse_and_store_blob(&last_blob, &oid, 0); + break; + } + } + } else { + enum object_type expected = S_ISDIR(mode) ? + OBJ_TREE: OBJ_BLOB; + enum object_type type = oe ? oe->type : + oid_object_info(the_repository, &oid, + NULL); + if (type < 0) + die("%s not found: %s", + S_ISDIR(mode) ? "Tree" : "Blob", + command_buf.buf); + if (type != expected) + die("Not a %s (actually a %s): %s", + type_name(expected), type_name(type), + command_buf.buf); + } + + if (!*p) { + tree_content_replace(&b->branch_tree, &oid, mode, NULL); + return; + } + tree_content_set(&b->branch_tree, p, &oid, mode, NULL); +} + +static void file_change_d(const char *p, struct branch *b) +{ + static struct strbuf uq = STRBUF_INIT; + const char *endp; + + strbuf_reset(&uq); + if (!unquote_c_style(&uq, p, &endp)) { + if (*endp) + die("Garbage after path in: %s", command_buf.buf); + p = uq.buf; + } + tree_content_remove(&b->branch_tree, p, NULL, 1); +} + +static void file_change_cr(const char *s, struct branch *b, int rename) +{ + const char *d; + static struct strbuf s_uq = STRBUF_INIT; + static struct strbuf d_uq = STRBUF_INIT; + const char *endp; + struct tree_entry leaf; + + strbuf_reset(&s_uq); + if (!unquote_c_style(&s_uq, s, &endp)) { + if (*endp != ' ') + die("Missing space after source: %s", command_buf.buf); + } else { + endp = strchr(s, ' '); + if (!endp) + die("Missing space after source: %s", command_buf.buf); + strbuf_add(&s_uq, s, endp - s); + } + s = s_uq.buf; + + endp++; + if (!*endp) + die("Missing dest: %s", command_buf.buf); + + d = endp; + strbuf_reset(&d_uq); + if (!unquote_c_style(&d_uq, d, &endp)) { + if (*endp) + die("Garbage after dest in: %s", command_buf.buf); + d = d_uq.buf; + } + + memset(&leaf, 0, sizeof(leaf)); + if (rename) + tree_content_remove(&b->branch_tree, s, &leaf, 1); + else + tree_content_get(&b->branch_tree, s, &leaf, 1); + if (!leaf.versions[1].mode) + die("Path %s not in branch", s); + if (!*d) { /* C "path/to/subdir" "" */ + tree_content_replace(&b->branch_tree, + &leaf.versions[1].oid, + leaf.versions[1].mode, + leaf.tree); + return; + } + tree_content_set(&b->branch_tree, d, + &leaf.versions[1].oid, + leaf.versions[1].mode, + leaf.tree); +} + +static void note_change_n(const char *p, struct branch *b, unsigned char *old_fanout) +{ + static struct strbuf uq = STRBUF_INIT; + struct object_entry *oe; + struct branch *s; + struct object_id oid, commit_oid; + char path[GIT_MAX_RAWSZ * 3]; + uint16_t inline_data = 0; + unsigned char new_fanout; + + /* + * When loading a branch, we don't traverse its tree to count the real + * number of notes (too expensive to do this for all non-note refs). + * This means that recently loaded notes refs might incorrectly have + * b->num_notes == 0, and consequently, old_fanout might be wrong. + * + * Fix this by traversing the tree and counting the number of notes + * when b->num_notes == 0. If the notes tree is truly empty, the + * calculation should not take long. + */ + if (b->num_notes == 0 && *old_fanout == 0) { + /* Invoke change_note_fanout() in "counting mode". */ + b->num_notes = change_note_fanout(&b->branch_tree, 0xff); + *old_fanout = convert_num_notes_to_fanout(b->num_notes); + } + + /* Now parse the notemodify command. */ + /* or 'inline' */ + if (*p == ':') { + oe = find_mark(marks, parse_mark_ref_space(&p)); + oidcpy(&oid, &oe->idx.oid); + } else if (skip_prefix(p, "inline ", &p)) { + inline_data = 1; + oe = NULL; /* not used with inline_data, but makes gcc happy */ + } else { + if (parse_mapped_oid_hex(p, &oid, &p)) + die("Invalid dataref: %s", command_buf.buf); + oe = find_object(&oid); + if (*p++ != ' ') + die("Missing space after SHA1: %s", command_buf.buf); + } + + /* */ + s = lookup_branch(p); + if (s) { + if (is_null_oid(&s->oid)) + die("Can't add a note on empty branch."); + oidcpy(&commit_oid, &s->oid); + } else if (*p == ':') { + uintmax_t commit_mark = parse_mark_ref_eol(p); + struct object_entry *commit_oe = find_mark(marks, commit_mark); + if (commit_oe->type != OBJ_COMMIT) + die("Mark :%" PRIuMAX " not a commit", commit_mark); + oidcpy(&commit_oid, &commit_oe->idx.oid); + } else if (!get_oid(p, &commit_oid)) { + unsigned long size; + char *buf = read_object_with_reference(the_repository, + &commit_oid, + commit_type, &size, + &commit_oid); + if (!buf || size < the_hash_algo->hexsz + 6) + die("Not a valid commit: %s", p); + free(buf); + } else + die("Invalid ref name or SHA1 expression: %s", p); + + if (inline_data) { + if (p != uq.buf) { + strbuf_addstr(&uq, p); + p = uq.buf; + } + read_next_command(); + parse_and_store_blob(&last_blob, &oid, 0); + } else if (oe) { + if (oe->type != OBJ_BLOB) + die("Not a blob (actually a %s): %s", + type_name(oe->type), command_buf.buf); + } else if (!is_null_oid(&oid)) { + enum object_type type = oid_object_info(the_repository, &oid, + NULL); + if (type < 0) + die("Blob not found: %s", command_buf.buf); + if (type != OBJ_BLOB) + die("Not a blob (actually a %s): %s", + type_name(type), command_buf.buf); + } + + construct_path_with_fanout(oid_to_hex(&commit_oid), *old_fanout, path); + if (tree_content_remove(&b->branch_tree, path, NULL, 0)) + b->num_notes--; + + if (is_null_oid(&oid)) + return; /* nothing to insert */ + + b->num_notes++; + new_fanout = convert_num_notes_to_fanout(b->num_notes); + construct_path_with_fanout(oid_to_hex(&commit_oid), new_fanout, path); + tree_content_set(&b->branch_tree, path, &oid, S_IFREG | 0644, NULL); +} + +static void file_change_deleteall(struct branch *b) +{ + release_tree_content_recursive(b->branch_tree.tree); + oidclr(&b->branch_tree.versions[0].oid); + oidclr(&b->branch_tree.versions[1].oid); + load_tree(&b->branch_tree); + b->num_notes = 0; +} + +static void parse_from_commit(struct branch *b, char *buf, unsigned long size) +{ + if (!buf || size < the_hash_algo->hexsz + 6) + die("Not a valid commit: %s", oid_to_hex(&b->oid)); + if (memcmp("tree ", buf, 5) + || get_oid_hex(buf + 5, &b->branch_tree.versions[1].oid)) + die("The commit %s is corrupt", oid_to_hex(&b->oid)); + oidcpy(&b->branch_tree.versions[0].oid, + &b->branch_tree.versions[1].oid); +} + +static void parse_from_existing(struct branch *b) +{ + if (is_null_oid(&b->oid)) { + oidclr(&b->branch_tree.versions[0].oid); + oidclr(&b->branch_tree.versions[1].oid); + } else { + unsigned long size; + char *buf; + + buf = read_object_with_reference(the_repository, + &b->oid, commit_type, &size, + &b->oid); + parse_from_commit(b, buf, size); + free(buf); + } +} + +static int parse_objectish(struct branch *b, const char *objectish) +{ + struct branch *s; + struct object_id oid; + + oidcpy(&oid, &b->branch_tree.versions[1].oid); + + s = lookup_branch(objectish); + if (b == s) + die("Can't create a branch from itself: %s", b->name); + else if (s) { + struct object_id *t = &s->branch_tree.versions[1].oid; + oidcpy(&b->oid, &s->oid); + oidcpy(&b->branch_tree.versions[0].oid, t); + oidcpy(&b->branch_tree.versions[1].oid, t); + } else if (*objectish == ':') { + uintmax_t idnum = parse_mark_ref_eol(objectish); + struct object_entry *oe = find_mark(marks, idnum); + if (oe->type != OBJ_COMMIT) + die("Mark :%" PRIuMAX " not a commit", idnum); + if (!oideq(&b->oid, &oe->idx.oid)) { + oidcpy(&b->oid, &oe->idx.oid); + if (oe->pack_id != MAX_PACK_ID) { + unsigned long size; + char *buf = gfi_unpack_entry(oe, &size); + parse_from_commit(b, buf, size); + free(buf); + } else + parse_from_existing(b); + } + } else if (!get_oid(objectish, &b->oid)) { + parse_from_existing(b); + if (is_null_oid(&b->oid)) + b->delete = 1; + } + else + die("Invalid ref name or SHA1 expression: %s", objectish); + + if (b->branch_tree.tree && !oideq(&oid, &b->branch_tree.versions[1].oid)) { + release_tree_content_recursive(b->branch_tree.tree); + b->branch_tree.tree = NULL; + } + + read_next_command(); + return 1; +} + +static int parse_from(struct branch *b) +{ + const char *from; + + if (!skip_prefix(command_buf.buf, "from ", &from)) + return 0; + + return parse_objectish(b, from); +} + +static int parse_objectish_with_prefix(struct branch *b, const char *prefix) +{ + const char *base; + + if (!skip_prefix(command_buf.buf, prefix, &base)) + return 0; + + return parse_objectish(b, base); +} + +static struct hash_list *parse_merge(unsigned int *count) +{ + struct hash_list *list = NULL, **tail = &list, *n; + const char *from; + struct branch *s; + + *count = 0; + while (skip_prefix(command_buf.buf, "merge ", &from)) { + n = xmalloc(sizeof(*n)); + s = lookup_branch(from); + if (s) + oidcpy(&n->oid, &s->oid); + else if (*from == ':') { + uintmax_t idnum = parse_mark_ref_eol(from); + struct object_entry *oe = find_mark(marks, idnum); + if (oe->type != OBJ_COMMIT) + die("Mark :%" PRIuMAX " not a commit", idnum); + oidcpy(&n->oid, &oe->idx.oid); + } else if (!get_oid(from, &n->oid)) { + unsigned long size; + char *buf = read_object_with_reference(the_repository, + &n->oid, + commit_type, + &size, &n->oid); + if (!buf || size < the_hash_algo->hexsz + 6) + die("Not a valid commit: %s", from); + free(buf); + } else + die("Invalid ref name or SHA1 expression: %s", from); + + n->next = NULL; + *tail = n; + tail = &n->next; + + (*count)++; + read_next_command(); + } + return list; +} + +static void parse_new_commit(const char *arg) +{ + static struct strbuf msg = STRBUF_INIT; + struct branch *b; + char *author = NULL; + char *committer = NULL; + char *encoding = NULL; + struct hash_list *merge_list = NULL; + unsigned int merge_count; + unsigned char prev_fanout, new_fanout; + const char *v; + + b = lookup_branch(arg); + if (!b) + b = new_branch(arg); + + read_next_command(); + parse_mark(); + parse_original_identifier(); + if (skip_prefix(command_buf.buf, "author ", &v)) { + author = parse_ident(v); + read_next_command(); + } + if (skip_prefix(command_buf.buf, "committer ", &v)) { + committer = parse_ident(v); + read_next_command(); + } + if (!committer) + die("Expected committer but didn't get one"); + if (skip_prefix(command_buf.buf, "encoding ", &v)) { + encoding = xstrdup(v); + read_next_command(); + } + parse_data(&msg, 0, NULL); + read_next_command(); + parse_from(b); + merge_list = parse_merge(&merge_count); + + /* ensure the branch is active/loaded */ + if (!b->branch_tree.tree || !max_active_branches) { + unload_one_branch(); + load_branch(b); + } + + prev_fanout = convert_num_notes_to_fanout(b->num_notes); + + /* file_change* */ + while (command_buf.len > 0) { + if (skip_prefix(command_buf.buf, "M ", &v)) + file_change_m(v, b); + else if (skip_prefix(command_buf.buf, "D ", &v)) + file_change_d(v, b); + else if (skip_prefix(command_buf.buf, "R ", &v)) + file_change_cr(v, b, 1); + else if (skip_prefix(command_buf.buf, "C ", &v)) + file_change_cr(v, b, 0); + else if (skip_prefix(command_buf.buf, "N ", &v)) + note_change_n(v, b, &prev_fanout); + else if (!strcmp("deleteall", command_buf.buf)) + file_change_deleteall(b); + else if (skip_prefix(command_buf.buf, "ls ", &v)) + parse_ls(v, b); + else if (skip_prefix(command_buf.buf, "cat-blob ", &v)) + parse_cat_blob(v); + else { + unread_command_buf = 1; + break; + } + if (read_next_command() == EOF) + break; + } + + new_fanout = convert_num_notes_to_fanout(b->num_notes); + if (new_fanout != prev_fanout) + b->num_notes = change_note_fanout(&b->branch_tree, new_fanout); + + /* build the tree and the commit */ + store_tree(&b->branch_tree); + oidcpy(&b->branch_tree.versions[0].oid, + &b->branch_tree.versions[1].oid); + + strbuf_reset(&new_data); + strbuf_addf(&new_data, "tree %s\n", + oid_to_hex(&b->branch_tree.versions[1].oid)); + if (!is_null_oid(&b->oid)) + strbuf_addf(&new_data, "parent %s\n", + oid_to_hex(&b->oid)); + while (merge_list) { + struct hash_list *next = merge_list->next; + strbuf_addf(&new_data, "parent %s\n", + oid_to_hex(&merge_list->oid)); + free(merge_list); + merge_list = next; + } + strbuf_addf(&new_data, + "author %s\n" + "committer %s\n", + author ? author : committer, committer); + if (encoding) + strbuf_addf(&new_data, + "encoding %s\n", + encoding); + strbuf_addch(&new_data, '\n'); + strbuf_addbuf(&new_data, &msg); + free(author); + free(committer); + free(encoding); + + if (!store_object(OBJ_COMMIT, &new_data, NULL, &b->oid, next_mark)) + b->pack_id = pack_id; + b->last_commit = object_count_by_type[OBJ_COMMIT]; +} + +static void parse_new_tag(const char *arg) +{ + static struct strbuf msg = STRBUF_INIT; + const char *from; + char *tagger; + struct branch *s; + struct tag *t; + uintmax_t from_mark = 0; + struct object_id oid; + enum object_type type; + const char *v; + + t = mem_pool_alloc(&fi_mem_pool, sizeof(struct tag)); + memset(t, 0, sizeof(struct tag)); + t->name = pool_strdup(arg); + if (last_tag) + last_tag->next_tag = t; + else + first_tag = t; + last_tag = t; + read_next_command(); + parse_mark(); + + /* from ... */ + if (!skip_prefix(command_buf.buf, "from ", &from)) + die("Expected from command, got %s", command_buf.buf); + s = lookup_branch(from); + if (s) { + if (is_null_oid(&s->oid)) + die("Can't tag an empty branch."); + oidcpy(&oid, &s->oid); + type = OBJ_COMMIT; + } else if (*from == ':') { + struct object_entry *oe; + from_mark = parse_mark_ref_eol(from); + oe = find_mark(marks, from_mark); + type = oe->type; + oidcpy(&oid, &oe->idx.oid); + } else if (!get_oid(from, &oid)) { + struct object_entry *oe = find_object(&oid); + if (!oe) { + type = oid_object_info(the_repository, &oid, NULL); + if (type < 0) + die("Not a valid object: %s", from); + } else + type = oe->type; + } else + die("Invalid ref name or SHA1 expression: %s", from); + read_next_command(); + + /* original-oid ... */ + parse_original_identifier(); + + /* tagger ... */ + if (skip_prefix(command_buf.buf, "tagger ", &v)) { + tagger = parse_ident(v); + read_next_command(); + } else + tagger = NULL; + + /* tag payload/message */ + parse_data(&msg, 0, NULL); + + /* build the tag object */ + strbuf_reset(&new_data); + + strbuf_addf(&new_data, + "object %s\n" + "type %s\n" + "tag %s\n", + oid_to_hex(&oid), type_name(type), t->name); + if (tagger) + strbuf_addf(&new_data, + "tagger %s\n", tagger); + strbuf_addch(&new_data, '\n'); + strbuf_addbuf(&new_data, &msg); + free(tagger); + + if (store_object(OBJ_TAG, &new_data, NULL, &t->oid, next_mark)) + t->pack_id = MAX_PACK_ID; + else + t->pack_id = pack_id; +} + +static void parse_reset_branch(const char *arg) +{ + struct branch *b; + const char *tag_name; + + b = lookup_branch(arg); + if (b) { + oidclr(&b->oid); + oidclr(&b->branch_tree.versions[0].oid); + oidclr(&b->branch_tree.versions[1].oid); + if (b->branch_tree.tree) { + release_tree_content_recursive(b->branch_tree.tree); + b->branch_tree.tree = NULL; + } + } + else + b = new_branch(arg); + read_next_command(); + parse_from(b); + if (b->delete && skip_prefix(b->name, "refs/tags/", &tag_name)) { + /* + * Elsewhere, we call dump_branches() before dump_tags(), + * and dump_branches() will handle ref deletions first, so + * in order to make sure the deletion actually takes effect, + * we need to remove the tag from our list of tags to update. + * + * NEEDSWORK: replace list of tags with hashmap for faster + * deletion? + */ + struct tag *t, *prev = NULL; + for (t = first_tag; t; t = t->next_tag) { + if (!strcmp(t->name, tag_name)) + break; + prev = t; + } + if (t) { + if (prev) + prev->next_tag = t->next_tag; + else + first_tag = t->next_tag; + if (!t->next_tag) + last_tag = prev; + /* There is no mem_pool_free(t) function to call. */ + } + } + if (command_buf.len > 0) + unread_command_buf = 1; +} + +static void cat_blob_write(const char *buf, unsigned long size) +{ + if (write_in_full(cat_blob_fd, buf, size) < 0) + die_errno("Write to frontend failed"); +} + +static void cat_blob(struct object_entry *oe, struct object_id *oid) +{ + struct strbuf line = STRBUF_INIT; + unsigned long size; + enum object_type type = 0; + char *buf; + + if (!oe || oe->pack_id == MAX_PACK_ID) { + buf = read_object_file(oid, &type, &size); + } else { + type = oe->type; + buf = gfi_unpack_entry(oe, &size); + } + + /* + * Output based on batch_one_object() from cat-file.c. + */ + if (type <= 0) { + strbuf_reset(&line); + strbuf_addf(&line, "%s missing\n", oid_to_hex(oid)); + cat_blob_write(line.buf, line.len); + strbuf_release(&line); + free(buf); + return; + } + if (!buf) + die("Can't read object %s", oid_to_hex(oid)); + if (type != OBJ_BLOB) + die("Object %s is a %s but a blob was expected.", + oid_to_hex(oid), type_name(type)); + strbuf_reset(&line); + strbuf_addf(&line, "%s %s %"PRIuMAX"\n", oid_to_hex(oid), + type_name(type), (uintmax_t)size); + cat_blob_write(line.buf, line.len); + strbuf_release(&line); + cat_blob_write(buf, size); + cat_blob_write("\n", 1); + if (oe && oe->pack_id == pack_id) { + last_blob.offset = oe->idx.offset; + strbuf_attach(&last_blob.data, buf, size, size); + last_blob.depth = oe->depth; + } else + free(buf); +} + +static void parse_get_mark(const char *p) +{ + struct object_entry *oe; + char output[GIT_MAX_HEXSZ + 2]; + + /* get-mark SP LF */ + if (*p != ':') + die("Not a mark: %s", p); + + oe = find_mark(marks, parse_mark_ref_eol(p)); + if (!oe) + die("Unknown mark: %s", command_buf.buf); + + xsnprintf(output, sizeof(output), "%s\n", oid_to_hex(&oe->idx.oid)); + cat_blob_write(output, the_hash_algo->hexsz + 1); +} + +static void parse_cat_blob(const char *p) +{ + struct object_entry *oe; + struct object_id oid; + + /* cat-blob SP LF */ + if (*p == ':') { + oe = find_mark(marks, parse_mark_ref_eol(p)); + if (!oe) + die("Unknown mark: %s", command_buf.buf); + oidcpy(&oid, &oe->idx.oid); + } else { + if (parse_mapped_oid_hex(p, &oid, &p)) + die("Invalid dataref: %s", command_buf.buf); + if (*p) + die("Garbage after SHA1: %s", command_buf.buf); + oe = find_object(&oid); + } + + cat_blob(oe, &oid); +} + +static struct object_entry *dereference(struct object_entry *oe, + struct object_id *oid) +{ + unsigned long size; + char *buf = NULL; + const unsigned hexsz = the_hash_algo->hexsz; + + if (!oe) { + enum object_type type = oid_object_info(the_repository, oid, + NULL); + if (type < 0) + die("object not found: %s", oid_to_hex(oid)); + /* cache it! */ + oe = insert_object(oid); + oe->type = type; + oe->pack_id = MAX_PACK_ID; + oe->idx.offset = 1; + } + switch (oe->type) { + case OBJ_TREE: /* easy case. */ + return oe; + case OBJ_COMMIT: + case OBJ_TAG: + break; + default: + die("Not a tree-ish: %s", command_buf.buf); + } + + if (oe->pack_id != MAX_PACK_ID) { /* in a pack being written */ + buf = gfi_unpack_entry(oe, &size); + } else { + enum object_type unused; + buf = read_object_file(oid, &unused, &size); + } + if (!buf) + die("Can't load object %s", oid_to_hex(oid)); + + /* Peel one layer. */ + switch (oe->type) { + case OBJ_TAG: + if (size < hexsz + strlen("object ") || + get_oid_hex(buf + strlen("object "), oid)) + die("Invalid SHA1 in tag: %s", command_buf.buf); + break; + case OBJ_COMMIT: + if (size < hexsz + strlen("tree ") || + get_oid_hex(buf + strlen("tree "), oid)) + die("Invalid SHA1 in commit: %s", command_buf.buf); + } + + free(buf); + return find_object(oid); +} + +static void insert_mapped_mark(uintmax_t mark, void *object, void *cbp) +{ + struct object_id *fromoid = object; + struct object_id *tooid = find_mark(cbp, mark); + int ret; + khiter_t it; + + it = kh_put_oid_map(sub_oid_map, *fromoid, &ret); + /* We've already seen this object. */ + if (ret == 0) + return; + kh_value(sub_oid_map, it) = tooid; +} + +static void build_mark_map_one(struct mark_set *from, struct mark_set *to) +{ + for_each_mark(from, 0, insert_mapped_mark, to); +} + +static void build_mark_map(struct string_list *from, struct string_list *to) +{ + struct string_list_item *fromp, *top; + + sub_oid_map = kh_init_oid_map(); + + for_each_string_list_item(fromp, from) { + top = string_list_lookup(to, fromp->string); + if (!fromp->util) { + die(_("Missing from marks for submodule '%s'"), fromp->string); + } else if (!top || !top->util) { + die(_("Missing to marks for submodule '%s'"), fromp->string); + } + build_mark_map_one(fromp->util, top->util); + } +} + +static struct object_entry *parse_treeish_dataref(const char **p) +{ + struct object_id oid; + struct object_entry *e; + + if (**p == ':') { /* */ + e = find_mark(marks, parse_mark_ref_space(p)); + if (!e) + die("Unknown mark: %s", command_buf.buf); + oidcpy(&oid, &e->idx.oid); + } else { /* */ + if (parse_mapped_oid_hex(*p, &oid, p)) + die("Invalid dataref: %s", command_buf.buf); + e = find_object(&oid); + if (*(*p)++ != ' ') + die("Missing space after tree-ish: %s", command_buf.buf); + } + + while (!e || e->type != OBJ_TREE) + e = dereference(e, &oid); + return e; +} + +static void print_ls(int mode, const unsigned char *hash, const char *path) +{ + static struct strbuf line = STRBUF_INIT; + + /* See show_tree(). */ + const char *type = + S_ISGITLINK(mode) ? commit_type : + S_ISDIR(mode) ? tree_type : + blob_type; + + if (!mode) { + /* missing SP path LF */ + strbuf_reset(&line); + strbuf_addstr(&line, "missing "); + quote_c_style(path, &line, NULL, 0); + strbuf_addch(&line, '\n'); + } else { + /* mode SP type SP object_name TAB path LF */ + strbuf_reset(&line); + strbuf_addf(&line, "%06o %s %s\t", + mode & ~NO_DELTA, type, hash_to_hex(hash)); + quote_c_style(path, &line, NULL, 0); + strbuf_addch(&line, '\n'); + } + cat_blob_write(line.buf, line.len); +} + +static void parse_ls(const char *p, struct branch *b) +{ + struct tree_entry *root = NULL; + struct tree_entry leaf = {NULL}; + + /* ls SP ( SP)? */ + if (*p == '"') { + if (!b) + die("Not in a commit: %s", command_buf.buf); + root = &b->branch_tree; + } else { + struct object_entry *e = parse_treeish_dataref(&p); + root = new_tree_entry(); + oidcpy(&root->versions[1].oid, &e->idx.oid); + if (!is_null_oid(&root->versions[1].oid)) + root->versions[1].mode = S_IFDIR; + load_tree(root); + } + if (*p == '"') { + static struct strbuf uq = STRBUF_INIT; + const char *endp; + strbuf_reset(&uq); + if (unquote_c_style(&uq, p, &endp)) + die("Invalid path: %s", command_buf.buf); + if (*endp) + die("Garbage after path in: %s", command_buf.buf); + p = uq.buf; + } + tree_content_get(root, p, &leaf, 1); + /* + * A directory in preparation would have a sha1 of zero + * until it is saved. Save, for simplicity. + */ + if (S_ISDIR(leaf.versions[1].mode)) + store_tree(&leaf); + + print_ls(leaf.versions[1].mode, leaf.versions[1].oid.hash, p); + if (leaf.tree) + release_tree_content_recursive(leaf.tree); + if (!b || root != &b->branch_tree) + release_tree_entry(root); +} + +static void checkpoint(void) +{ + checkpoint_requested = 0; + if (object_count) { + cycle_packfile(); + } + dump_branches(); + dump_tags(); + dump_marks(); +} + +static void parse_checkpoint(void) +{ + checkpoint_requested = 1; + skip_optional_lf(); +} + +static void parse_progress(void) +{ + fwrite(command_buf.buf, 1, command_buf.len, stdout); + fputc('\n', stdout); + fflush(stdout); + skip_optional_lf(); +} + +static void parse_alias(void) +{ + struct object_entry *e; + struct branch b; + + skip_optional_lf(); + read_next_command(); + + /* mark ... */ + parse_mark(); + if (!next_mark) + die(_("Expected 'mark' command, got %s"), command_buf.buf); + + /* to ... */ + memset(&b, 0, sizeof(b)); + if (!parse_objectish_with_prefix(&b, "to ")) + die(_("Expected 'to' command, got %s"), command_buf.buf); + e = find_object(&b.oid); + assert(e); + insert_mark(marks, next_mark, e); +} + +static char* make_fast_import_path(const char *path) +{ + if (!relative_marks_paths || is_absolute_path(path)) + return xstrdup(path); + return git_pathdup("info/fast-import/%s", path); +} + +static void option_import_marks(const char *marks, + int from_stream, int ignore_missing) +{ + if (import_marks_file) { + if (from_stream) + die("Only one import-marks command allowed per stream"); + + /* read previous mark file */ + if(!import_marks_file_from_stream) + read_marks(); + } + + import_marks_file = make_fast_import_path(marks); + import_marks_file_from_stream = from_stream; + import_marks_file_ignore_missing = ignore_missing; +} + +static void option_date_format(const char *fmt) +{ + if (!strcmp(fmt, "raw")) + whenspec = WHENSPEC_RAW; + else if (!strcmp(fmt, "raw-permissive")) + whenspec = WHENSPEC_RAW_PERMISSIVE; + else if (!strcmp(fmt, "rfc2822")) + whenspec = WHENSPEC_RFC2822; + else if (!strcmp(fmt, "now")) + whenspec = WHENSPEC_NOW; + else + die("unknown --date-format argument %s", fmt); +} + +static unsigned long ulong_arg(const char *option, const char *arg) +{ + char *endptr; + unsigned long rv = strtoul(arg, &endptr, 0); + if (strchr(arg, '-') || endptr == arg || *endptr) + die("%s: argument must be a non-negative integer", option); + return rv; +} + +static void option_depth(const char *depth) +{ + max_depth = ulong_arg("--depth", depth); + if (max_depth > MAX_DEPTH) + die("--depth cannot exceed %u", MAX_DEPTH); +} + +static void option_active_branches(const char *branches) +{ + max_active_branches = ulong_arg("--active-branches", branches); +} + +static void option_export_marks(const char *marks) +{ + export_marks_file = make_fast_import_path(marks); +} + +static void option_cat_blob_fd(const char *fd) +{ + unsigned long n = ulong_arg("--cat-blob-fd", fd); + if (n > (unsigned long) INT_MAX) + die("--cat-blob-fd cannot exceed %d", INT_MAX); + cat_blob_fd = (int) n; +} + +static void option_export_pack_edges(const char *edges) +{ + if (pack_edges) + fclose(pack_edges); + pack_edges = xfopen(edges, "a"); +} + +static void option_rewrite_submodules(const char *arg, struct string_list *list) +{ + struct mark_set *ms; + FILE *fp; + char *s = xstrdup(arg); + char *f = strchr(s, ':'); + if (!f) + die(_("Expected format name:filename for submodule rewrite option")); + *f = '\0'; + f++; + ms = xcalloc(1, sizeof(*ms)); + string_list_insert(list, s)->util = ms; + + fp = fopen(f, "r"); + if (!fp) + die_errno("cannot read '%s'", f); + read_mark_file(ms, fp, insert_oid_entry); + fclose(fp); +} + +static int parse_one_option(const char *option) +{ + if (skip_prefix(option, "max-pack-size=", &option)) { + unsigned long v; + if (!git_parse_ulong(option, &v)) + return 0; + if (v < 8192) { + warning("max-pack-size is now in bytes, assuming --max-pack-size=%lum", v); + v *= 1024 * 1024; + } else if (v < 1024 * 1024) { + warning("minimum max-pack-size is 1 MiB"); + v = 1024 * 1024; + } + max_packsize = v; + } else if (skip_prefix(option, "big-file-threshold=", &option)) { + unsigned long v; + if (!git_parse_ulong(option, &v)) + return 0; + big_file_threshold = v; + } else if (skip_prefix(option, "depth=", &option)) { + option_depth(option); + } else if (skip_prefix(option, "active-branches=", &option)) { + option_active_branches(option); + } else if (skip_prefix(option, "export-pack-edges=", &option)) { + option_export_pack_edges(option); + } else if (!strcmp(option, "quiet")) { + show_stats = 0; + } else if (!strcmp(option, "stats")) { + show_stats = 1; + } else if (!strcmp(option, "allow-unsafe-features")) { + ; /* already handled during early option parsing */ + } else { + return 0; + } + + return 1; +} + +static void check_unsafe_feature(const char *feature, int from_stream) +{ + if (from_stream && !allow_unsafe_features) + die(_("feature '%s' forbidden in input without --allow-unsafe-features"), + feature); +} + +static int parse_one_feature(const char *feature, int from_stream) +{ + const char *arg; + + if (skip_prefix(feature, "date-format=", &arg)) { + option_date_format(arg); + } else if (skip_prefix(feature, "import-marks=", &arg)) { + check_unsafe_feature("import-marks", from_stream); + option_import_marks(arg, from_stream, 0); + } else if (skip_prefix(feature, "import-marks-if-exists=", &arg)) { + check_unsafe_feature("import-marks-if-exists", from_stream); + option_import_marks(arg, from_stream, 1); + } else if (skip_prefix(feature, "export-marks=", &arg)) { + check_unsafe_feature(feature, from_stream); + option_export_marks(arg); + } else if (!strcmp(feature, "alias")) { + ; /* Don't die - this feature is supported */ + } else if (skip_prefix(feature, "rewrite-submodules-to=", &arg)) { + option_rewrite_submodules(arg, &sub_marks_to); + } else if (skip_prefix(feature, "rewrite-submodules-from=", &arg)) { + option_rewrite_submodules(arg, &sub_marks_from); + } else if (skip_prefix(feature, "rewrite-submodules-from=", &arg)) { + } else if (!strcmp(feature, "get-mark")) { + ; /* Don't die - this feature is supported */ + } else if (!strcmp(feature, "cat-blob")) { + ; /* Don't die - this feature is supported */ + } else if (!strcmp(feature, "relative-marks")) { + relative_marks_paths = 1; + } else if (!strcmp(feature, "no-relative-marks")) { + relative_marks_paths = 0; + } else if (!strcmp(feature, "done")) { + require_explicit_termination = 1; + } else if (!strcmp(feature, "force")) { + force_update = 1; + } else if (!strcmp(feature, "notes") || !strcmp(feature, "ls")) { + ; /* do nothing; we have the feature */ + } else { + return 0; + } + + return 1; +} + +static void parse_feature(const char *feature) +{ + if (seen_data_command) + die("Got feature command '%s' after data command", feature); + + if (parse_one_feature(feature, 1)) + return; + + die("This version of fast-import does not support feature %s.", feature); +} + +static void parse_option(const char *option) +{ + if (seen_data_command) + die("Got option command '%s' after data command", option); + + if (parse_one_option(option)) + return; + + die("This version of fast-import does not support option: %s", option); +} + +static void git_pack_config(void) +{ + int indexversion_value; + int limit; + unsigned long packsizelimit_value; + + if (!git_config_get_ulong("pack.depth", &max_depth)) { + if (max_depth > MAX_DEPTH) + max_depth = MAX_DEPTH; + } + if (!git_config_get_int("pack.indexversion", &indexversion_value)) { + pack_idx_opts.version = indexversion_value; + if (pack_idx_opts.version > 2) + git_die_config("pack.indexversion", + "bad pack.indexversion=%"PRIu32, pack_idx_opts.version); + } + if (!git_config_get_ulong("pack.packsizelimit", &packsizelimit_value)) + max_packsize = packsizelimit_value; + + if (!git_config_get_int("fastimport.unpacklimit", &limit)) + unpack_limit = limit; + else if (!git_config_get_int("transfer.unpacklimit", &limit)) + unpack_limit = limit; + + git_config(git_default_config, NULL); +} + +static const char fast_import_usage[] = +"git fast-import [--date-format=] [--max-pack-size=] [--big-file-threshold=] [--depth=] [--active-branches=] [--export-marks=]"; + +static void parse_argv(void) +{ + unsigned int i; + + for (i = 1; i < global_argc; i++) { + const char *a = global_argv[i]; + + if (*a != '-' || !strcmp(a, "--")) + break; + + if (!skip_prefix(a, "--", &a)) + die("unknown option %s", a); + + if (parse_one_option(a)) + continue; + + if (parse_one_feature(a, 0)) + continue; + + if (skip_prefix(a, "cat-blob-fd=", &a)) { + option_cat_blob_fd(a); + continue; + } + + die("unknown option --%s", a); + } + if (i != global_argc) + usage(fast_import_usage); + + seen_data_command = 1; + if (import_marks_file) + read_marks(); + build_mark_map(&sub_marks_from, &sub_marks_to); +} + +int cmd_fast_import(int argc, const char **argv, const char *prefix) +{ + unsigned int i; + + if (argc == 2 && !strcmp(argv[1], "-h")) + usage(fast_import_usage); + + reset_pack_idx_option(&pack_idx_opts); + git_pack_config(); + + alloc_objects(object_entry_alloc); + strbuf_init(&command_buf, 0); + atom_table = xcalloc(atom_table_sz, sizeof(struct atom_str*)); + branch_table = xcalloc(branch_table_sz, sizeof(struct branch*)); + avail_tree_table = xcalloc(avail_tree_table_sz, sizeof(struct avail_tree_content*)); + marks = mem_pool_calloc(&fi_mem_pool, 1, sizeof(struct mark_set)); + + hashmap_init(&object_table, object_entry_hashcmp, NULL, 0); + + /* + * We don't parse most options until after we've seen the set of + * "feature" lines at the start of the stream (which allows the command + * line to override stream data). But we must do an early parse of any + * command-line options that impact how we interpret the feature lines. + */ + for (i = 1; i < argc; i++) { + const char *arg = argv[i]; + if (*arg != '-' || !strcmp(arg, "--")) + break; + if (!strcmp(arg, "--allow-unsafe-features")) + allow_unsafe_features = 1; + } + + global_argc = argc; + global_argv = argv; + + rc_free = mem_pool_alloc(&fi_mem_pool, cmd_save * sizeof(*rc_free)); + for (i = 0; i < (cmd_save - 1); i++) + rc_free[i].next = &rc_free[i + 1]; + rc_free[cmd_save - 1].next = NULL; + + start_packfile(); + set_die_routine(die_nicely); + set_checkpoint_signal(); + while (read_next_command() != EOF) { + const char *v; + if (!strcmp("blob", command_buf.buf)) + parse_new_blob(); + else if (skip_prefix(command_buf.buf, "commit ", &v)) + parse_new_commit(v); + else if (skip_prefix(command_buf.buf, "tag ", &v)) + parse_new_tag(v); + else if (skip_prefix(command_buf.buf, "reset ", &v)) + parse_reset_branch(v); + else if (skip_prefix(command_buf.buf, "ls ", &v)) + parse_ls(v, NULL); + else if (skip_prefix(command_buf.buf, "cat-blob ", &v)) + parse_cat_blob(v); + else if (skip_prefix(command_buf.buf, "get-mark ", &v)) + parse_get_mark(v); + else if (!strcmp("checkpoint", command_buf.buf)) + parse_checkpoint(); + else if (!strcmp("done", command_buf.buf)) + break; + else if (!strcmp("alias", command_buf.buf)) + parse_alias(); + else if (starts_with(command_buf.buf, "progress ")) + parse_progress(); + else if (skip_prefix(command_buf.buf, "feature ", &v)) + parse_feature(v); + else if (skip_prefix(command_buf.buf, "option git ", &v)) + parse_option(v); + else if (starts_with(command_buf.buf, "option ")) + /* ignore non-git options*/; + else + die("Unsupported command: %s", command_buf.buf); + + if (checkpoint_requested) + checkpoint(); + } + + /* argv hasn't been parsed yet, do so */ + if (!seen_data_command) + parse_argv(); + + if (require_explicit_termination && feof(stdin)) + die("stream ends early"); + + end_packfile(); + + dump_branches(); + dump_tags(); + unkeep_all_packs(); + dump_marks(); + + if (pack_edges) + fclose(pack_edges); + + if (show_stats) { + uintmax_t total_count = 0, duplicate_count = 0; + for (i = 0; i < ARRAY_SIZE(object_count_by_type); i++) + total_count += object_count_by_type[i]; + for (i = 0; i < ARRAY_SIZE(duplicate_count_by_type); i++) + duplicate_count += duplicate_count_by_type[i]; + + fprintf(stderr, "%s statistics:\n", argv[0]); + fprintf(stderr, "---------------------------------------------------------------------\n"); + fprintf(stderr, "Alloc'd objects: %10" PRIuMAX "\n", alloc_count); + fprintf(stderr, "Total objects: %10" PRIuMAX " (%10" PRIuMAX " duplicates )\n", total_count, duplicate_count); + fprintf(stderr, " blobs : %10" PRIuMAX " (%10" PRIuMAX " duplicates %10" PRIuMAX " deltas of %10" PRIuMAX" attempts)\n", object_count_by_type[OBJ_BLOB], duplicate_count_by_type[OBJ_BLOB], delta_count_by_type[OBJ_BLOB], delta_count_attempts_by_type[OBJ_BLOB]); + fprintf(stderr, " trees : %10" PRIuMAX " (%10" PRIuMAX " duplicates %10" PRIuMAX " deltas of %10" PRIuMAX" attempts)\n", object_count_by_type[OBJ_TREE], duplicate_count_by_type[OBJ_TREE], delta_count_by_type[OBJ_TREE], delta_count_attempts_by_type[OBJ_TREE]); + fprintf(stderr, " commits: %10" PRIuMAX " (%10" PRIuMAX " duplicates %10" PRIuMAX " deltas of %10" PRIuMAX" attempts)\n", object_count_by_type[OBJ_COMMIT], duplicate_count_by_type[OBJ_COMMIT], delta_count_by_type[OBJ_COMMIT], delta_count_attempts_by_type[OBJ_COMMIT]); + fprintf(stderr, " tags : %10" PRIuMAX " (%10" PRIuMAX " duplicates %10" PRIuMAX " deltas of %10" PRIuMAX" attempts)\n", object_count_by_type[OBJ_TAG], duplicate_count_by_type[OBJ_TAG], delta_count_by_type[OBJ_TAG], delta_count_attempts_by_type[OBJ_TAG]); + fprintf(stderr, "Total branches: %10lu (%10lu loads )\n", branch_count, branch_load_count); + fprintf(stderr, " marks: %10" PRIuMAX " (%10" PRIuMAX " unique )\n", (((uintmax_t)1) << marks->shift) * 1024, marks_set_count); + fprintf(stderr, " atoms: %10u\n", atom_cnt); + fprintf(stderr, "Memory total: %10" PRIuMAX " KiB\n", (tree_entry_allocd + fi_mem_pool.pool_alloc + alloc_count*sizeof(struct object_entry))/1024); + fprintf(stderr, " pools: %10lu KiB\n", (unsigned long)((tree_entry_allocd + fi_mem_pool.pool_alloc) /1024)); + fprintf(stderr, " objects: %10" PRIuMAX " KiB\n", (alloc_count*sizeof(struct object_entry))/1024); + fprintf(stderr, "---------------------------------------------------------------------\n"); + pack_report(); + fprintf(stderr, "---------------------------------------------------------------------\n"); + fprintf(stderr, "\n"); + } + + return failure ? 1 : 0; +} diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt index 3e211606fd..4a6f135b16 100644 --- a/contrib/buildsystems/CMakeLists.txt +++ b/contrib/buildsystems/CMakeLists.txt @@ -501,7 +501,7 @@ unset(CMAKE_REQUIRED_INCLUDES) #programs set(PROGRAMS_BUILT - git git-daemon git-fast-import git-http-backend git-sh-i18n--envsubst + git git-daemon git-http-backend git-sh-i18n--envsubst git-shell git-remote-testsvn) if(NOT CURL_FOUND) @@ -627,9 +627,6 @@ target_link_libraries(git common-main) add_executable(git-daemon ${CMAKE_SOURCE_DIR}/daemon.c) target_link_libraries(git-daemon common-main) -add_executable(git-fast-import ${CMAKE_SOURCE_DIR}/fast-import.c) -target_link_libraries(git-fast-import common-main) - add_executable(git-http-backend ${CMAKE_SOURCE_DIR}/http-backend.c) target_link_libraries(git-http-backend common-main) diff --git a/fast-import.c b/fast-import.c deleted file mode 100644 index ce47794db6..0000000000 --- a/fast-import.c +++ /dev/null @@ -1,3649 +0,0 @@ -#include "builtin.h" -#include "cache.h" -#include "repository.h" -#include "config.h" -#include "lockfile.h" -#include "object.h" -#include "blob.h" -#include "tree.h" -#include "commit.h" -#include "delta.h" -#include "pack.h" -#include "refs.h" -#include "csum-file.h" -#include "quote.h" -#include "dir.h" -#include "run-command.h" -#include "packfile.h" -#include "object-store.h" -#include "mem-pool.h" -#include "commit-reach.h" -#include "khash.h" - -#define PACK_ID_BITS 16 -#define MAX_PACK_ID ((1<rawsz * 3) - -struct object_entry { - struct pack_idx_entry idx; - struct hashmap_entry ent; - uint32_t type : TYPE_BITS, - pack_id : PACK_ID_BITS, - depth : DEPTH_BITS; -}; - -static int object_entry_hashcmp(const void *map_data, - const struct hashmap_entry *eptr, - const struct hashmap_entry *entry_or_key, - const void *keydata) -{ - const struct object_id *oid = keydata; - const struct object_entry *e1, *e2; - - e1 = container_of(eptr, const struct object_entry, ent); - if (oid) - return oidcmp(&e1->idx.oid, oid); - - e2 = container_of(entry_or_key, const struct object_entry, ent); - return oidcmp(&e1->idx.oid, &e2->idx.oid); -} - -struct object_entry_pool { - struct object_entry_pool *next_pool; - struct object_entry *next_free; - struct object_entry *end; - struct object_entry entries[FLEX_ARRAY]; /* more */ -}; - -struct mark_set { - union { - struct object_id *oids[1024]; - struct object_entry *marked[1024]; - struct mark_set *sets[1024]; - } data; - unsigned int shift; -}; - -struct last_object { - struct strbuf data; - off_t offset; - unsigned int depth; - unsigned no_swap : 1; -}; - -struct atom_str { - struct atom_str *next_atom; - unsigned short str_len; - char str_dat[FLEX_ARRAY]; /* more */ -}; - -struct tree_content; -struct tree_entry { - struct tree_content *tree; - struct atom_str *name; - struct tree_entry_ms { - uint16_t mode; - struct object_id oid; - } versions[2]; -}; - -struct tree_content { - unsigned int entry_capacity; /* must match avail_tree_content */ - unsigned int entry_count; - unsigned int delta_depth; - struct tree_entry *entries[FLEX_ARRAY]; /* more */ -}; - -struct avail_tree_content { - unsigned int entry_capacity; /* must match tree_content */ - struct avail_tree_content *next_avail; -}; - -struct branch { - struct branch *table_next_branch; - struct branch *active_next_branch; - const char *name; - struct tree_entry branch_tree; - uintmax_t last_commit; - uintmax_t num_notes; - unsigned active : 1; - unsigned delete : 1; - unsigned pack_id : PACK_ID_BITS; - struct object_id oid; -}; - -struct tag { - struct tag *next_tag; - const char *name; - unsigned int pack_id; - struct object_id oid; -}; - -struct hash_list { - struct hash_list *next; - struct object_id oid; -}; - -typedef enum { - WHENSPEC_RAW = 1, - WHENSPEC_RAW_PERMISSIVE, - WHENSPEC_RFC2822, - WHENSPEC_NOW -} whenspec_type; - -struct recent_command { - struct recent_command *prev; - struct recent_command *next; - char *buf; -}; - -typedef void (*mark_set_inserter_t)(struct mark_set *s, struct object_id *oid, uintmax_t mark); -typedef void (*each_mark_fn_t)(uintmax_t mark, void *obj, void *cbp); - -/* Configured limits on output */ -static unsigned long max_depth = 50; -static off_t max_packsize; -static int unpack_limit = 100; -static int force_update; - -/* Stats and misc. counters */ -static uintmax_t alloc_count; -static uintmax_t marks_set_count; -static uintmax_t object_count_by_type[1 << TYPE_BITS]; -static uintmax_t duplicate_count_by_type[1 << TYPE_BITS]; -static uintmax_t delta_count_by_type[1 << TYPE_BITS]; -static uintmax_t delta_count_attempts_by_type[1 << TYPE_BITS]; -static unsigned long object_count; -static unsigned long branch_count; -static unsigned long branch_load_count; -static int failure; -static FILE *pack_edges; -static unsigned int show_stats = 1; -static int global_argc; -static const char **global_argv; - -/* Memory pools */ -static struct mem_pool fi_mem_pool = {NULL, 2*1024*1024 - - sizeof(struct mp_block), 0 }; - -/* Atom management */ -static unsigned int atom_table_sz = 4451; -static unsigned int atom_cnt; -static struct atom_str **atom_table; - -/* The .pack file being generated */ -static struct pack_idx_option pack_idx_opts; -static unsigned int pack_id; -static struct hashfile *pack_file; -static struct packed_git *pack_data; -static struct packed_git **all_packs; -static off_t pack_size; - -/* Table of objects we've written. */ -static unsigned int object_entry_alloc = 5000; -static struct object_entry_pool *blocks; -static struct hashmap object_table; -static struct mark_set *marks; -static const char *export_marks_file; -static const char *import_marks_file; -static int import_marks_file_from_stream; -static int import_marks_file_ignore_missing; -static int import_marks_file_done; -static int relative_marks_paths; - -/* Our last blob */ -static struct last_object last_blob = { STRBUF_INIT, 0, 0, 0 }; - -/* Tree management */ -static unsigned int tree_entry_alloc = 1000; -static void *avail_tree_entry; -static unsigned int avail_tree_table_sz = 100; -static struct avail_tree_content **avail_tree_table; -static size_t tree_entry_allocd; -static struct strbuf old_tree = STRBUF_INIT; -static struct strbuf new_tree = STRBUF_INIT; - -/* Branch data */ -static unsigned long max_active_branches = 5; -static unsigned long cur_active_branches; -static unsigned long branch_table_sz = 1039; -static struct branch **branch_table; -static struct branch *active_branches; - -/* Tag data */ -static struct tag *first_tag; -static struct tag *last_tag; - -/* Input stream parsing */ -static whenspec_type whenspec = WHENSPEC_RAW; -static struct strbuf command_buf = STRBUF_INIT; -static int unread_command_buf; -static struct recent_command cmd_hist = {&cmd_hist, &cmd_hist, NULL}; -static struct recent_command *cmd_tail = &cmd_hist; -static struct recent_command *rc_free; -static unsigned int cmd_save = 100; -static uintmax_t next_mark; -static struct strbuf new_data = STRBUF_INIT; -static int seen_data_command; -static int require_explicit_termination; -static int allow_unsafe_features; - -/* Signal handling */ -static volatile sig_atomic_t checkpoint_requested; - -/* Submodule marks */ -static struct string_list sub_marks_from = STRING_LIST_INIT_DUP; -static struct string_list sub_marks_to = STRING_LIST_INIT_DUP; -static kh_oid_map_t *sub_oid_map; - -/* Where to write output of cat-blob commands */ -static int cat_blob_fd = STDOUT_FILENO; - -static void parse_argv(void); -static void parse_get_mark(const char *p); -static void parse_cat_blob(const char *p); -static void parse_ls(const char *p, struct branch *b); - -static void for_each_mark(struct mark_set *m, uintmax_t base, each_mark_fn_t callback, void *p) -{ - uintmax_t k; - if (m->shift) { - for (k = 0; k < 1024; k++) { - if (m->data.sets[k]) - for_each_mark(m->data.sets[k], base + (k << m->shift), callback, p); - } - } else { - for (k = 0; k < 1024; k++) { - if (m->data.marked[k]) - callback(base + k, m->data.marked[k], p); - } - } -} - -static void dump_marks_fn(uintmax_t mark, void *object, void *cbp) { - struct object_entry *e = object; - FILE *f = cbp; - - fprintf(f, ":%" PRIuMAX " %s\n", mark, oid_to_hex(&e->idx.oid)); -} - -static void write_branch_report(FILE *rpt, struct branch *b) -{ - fprintf(rpt, "%s:\n", b->name); - - fprintf(rpt, " status :"); - if (b->active) - fputs(" active", rpt); - if (b->branch_tree.tree) - fputs(" loaded", rpt); - if (is_null_oid(&b->branch_tree.versions[1].oid)) - fputs(" dirty", rpt); - fputc('\n', rpt); - - fprintf(rpt, " tip commit : %s\n", oid_to_hex(&b->oid)); - fprintf(rpt, " old tree : %s\n", - oid_to_hex(&b->branch_tree.versions[0].oid)); - fprintf(rpt, " cur tree : %s\n", - oid_to_hex(&b->branch_tree.versions[1].oid)); - fprintf(rpt, " commit clock: %" PRIuMAX "\n", b->last_commit); - - fputs(" last pack : ", rpt); - if (b->pack_id < MAX_PACK_ID) - fprintf(rpt, "%u", b->pack_id); - fputc('\n', rpt); - - fputc('\n', rpt); -} - -static void write_crash_report(const char *err) -{ - char *loc = git_pathdup("fast_import_crash_%"PRIuMAX, (uintmax_t) getpid()); - FILE *rpt = fopen(loc, "w"); - struct branch *b; - unsigned long lu; - struct recent_command *rc; - - if (!rpt) { - error_errno("can't write crash report %s", loc); - free(loc); - return; - } - - fprintf(stderr, "fast-import: dumping crash report to %s\n", loc); - - fprintf(rpt, "fast-import crash report:\n"); - fprintf(rpt, " fast-import process: %"PRIuMAX"\n", (uintmax_t) getpid()); - fprintf(rpt, " parent process : %"PRIuMAX"\n", (uintmax_t) getppid()); - fprintf(rpt, " at %s\n", show_date(time(NULL), 0, DATE_MODE(ISO8601))); - fputc('\n', rpt); - - fputs("fatal: ", rpt); - fputs(err, rpt); - fputc('\n', rpt); - - fputc('\n', rpt); - fputs("Most Recent Commands Before Crash\n", rpt); - fputs("---------------------------------\n", rpt); - for (rc = cmd_hist.next; rc != &cmd_hist; rc = rc->next) { - if (rc->next == &cmd_hist) - fputs("* ", rpt); - else - fputs(" ", rpt); - fputs(rc->buf, rpt); - fputc('\n', rpt); - } - - fputc('\n', rpt); - fputs("Active Branch LRU\n", rpt); - fputs("-----------------\n", rpt); - fprintf(rpt, " active_branches = %lu cur, %lu max\n", - cur_active_branches, - max_active_branches); - fputc('\n', rpt); - fputs(" pos clock name\n", rpt); - fputs(" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n", rpt); - for (b = active_branches, lu = 0; b; b = b->active_next_branch) - fprintf(rpt, " %2lu) %6" PRIuMAX" %s\n", - ++lu, b->last_commit, b->name); - - fputc('\n', rpt); - fputs("Inactive Branches\n", rpt); - fputs("-----------------\n", rpt); - for (lu = 0; lu < branch_table_sz; lu++) { - for (b = branch_table[lu]; b; b = b->table_next_branch) - write_branch_report(rpt, b); - } - - if (first_tag) { - struct tag *tg; - fputc('\n', rpt); - fputs("Annotated Tags\n", rpt); - fputs("--------------\n", rpt); - for (tg = first_tag; tg; tg = tg->next_tag) { - fputs(oid_to_hex(&tg->oid), rpt); - fputc(' ', rpt); - fputs(tg->name, rpt); - fputc('\n', rpt); - } - } - - fputc('\n', rpt); - fputs("Marks\n", rpt); - fputs("-----\n", rpt); - if (export_marks_file) - fprintf(rpt, " exported to %s\n", export_marks_file); - else - for_each_mark(marks, 0, dump_marks_fn, rpt); - - fputc('\n', rpt); - fputs("-------------------\n", rpt); - fputs("END OF CRASH REPORT\n", rpt); - fclose(rpt); - free(loc); -} - -static void end_packfile(void); -static void unkeep_all_packs(void); -static void dump_marks(void); - -static NORETURN void die_nicely(const char *err, va_list params) -{ - static int zombie; - char message[2 * PATH_MAX]; - - vsnprintf(message, sizeof(message), err, params); - fputs("fatal: ", stderr); - fputs(message, stderr); - fputc('\n', stderr); - - if (!zombie) { - zombie = 1; - write_crash_report(message); - end_packfile(); - unkeep_all_packs(); - dump_marks(); - } - exit(128); -} - -#ifndef SIGUSR1 /* Windows, for example */ - -static void set_checkpoint_signal(void) -{ -} - -#else - -static void checkpoint_signal(int signo) -{ - checkpoint_requested = 1; -} - -static void set_checkpoint_signal(void) -{ - struct sigaction sa; - - memset(&sa, 0, sizeof(sa)); - sa.sa_handler = checkpoint_signal; - sigemptyset(&sa.sa_mask); - sa.sa_flags = SA_RESTART; - sigaction(SIGUSR1, &sa, NULL); -} - -#endif - -static void alloc_objects(unsigned int cnt) -{ - struct object_entry_pool *b; - - b = xmalloc(sizeof(struct object_entry_pool) - + cnt * sizeof(struct object_entry)); - b->next_pool = blocks; - b->next_free = b->entries; - b->end = b->entries + cnt; - blocks = b; - alloc_count += cnt; -} - -static struct object_entry *new_object(struct object_id *oid) -{ - struct object_entry *e; - - if (blocks->next_free == blocks->end) - alloc_objects(object_entry_alloc); - - e = blocks->next_free++; - oidcpy(&e->idx.oid, oid); - return e; -} - -static struct object_entry *find_object(struct object_id *oid) -{ - return hashmap_get_entry_from_hash(&object_table, oidhash(oid), oid, - struct object_entry, ent); -} - -static struct object_entry *insert_object(struct object_id *oid) -{ - struct object_entry *e; - unsigned int hash = oidhash(oid); - - e = hashmap_get_entry_from_hash(&object_table, hash, oid, - struct object_entry, ent); - if (!e) { - e = new_object(oid); - e->idx.offset = 0; - hashmap_entry_init(&e->ent, hash); - hashmap_add(&object_table, &e->ent); - } - - return e; -} - -static void invalidate_pack_id(unsigned int id) -{ - unsigned long lu; - struct tag *t; - struct hashmap_iter iter; - struct object_entry *e; - - hashmap_for_each_entry(&object_table, &iter, e, ent) { - if (e->pack_id == id) - e->pack_id = MAX_PACK_ID; - } - - for (lu = 0; lu < branch_table_sz; lu++) { - struct branch *b; - - for (b = branch_table[lu]; b; b = b->table_next_branch) - if (b->pack_id == id) - b->pack_id = MAX_PACK_ID; - } - - for (t = first_tag; t; t = t->next_tag) - if (t->pack_id == id) - t->pack_id = MAX_PACK_ID; -} - -static unsigned int hc_str(const char *s, size_t len) -{ - unsigned int r = 0; - while (len-- > 0) - r = r * 31 + *s++; - return r; -} - -static char *pool_strdup(const char *s) -{ - size_t len = strlen(s) + 1; - char *r = mem_pool_alloc(&fi_mem_pool, len); - memcpy(r, s, len); - return r; -} - -static void insert_mark(struct mark_set *s, uintmax_t idnum, struct object_entry *oe) -{ - while ((idnum >> s->shift) >= 1024) { - s = mem_pool_calloc(&fi_mem_pool, 1, sizeof(struct mark_set)); - s->shift = marks->shift + 10; - s->data.sets[0] = marks; - marks = s; - } - while (s->shift) { - uintmax_t i = idnum >> s->shift; - idnum -= i << s->shift; - if (!s->data.sets[i]) { - s->data.sets[i] = mem_pool_calloc(&fi_mem_pool, 1, sizeof(struct mark_set)); - s->data.sets[i]->shift = s->shift - 10; - } - s = s->data.sets[i]; - } - if (!s->data.marked[idnum]) - marks_set_count++; - s->data.marked[idnum] = oe; -} - -static void *find_mark(struct mark_set *s, uintmax_t idnum) -{ - uintmax_t orig_idnum = idnum; - struct object_entry *oe = NULL; - if ((idnum >> s->shift) < 1024) { - while (s && s->shift) { - uintmax_t i = idnum >> s->shift; - idnum -= i << s->shift; - s = s->data.sets[i]; - } - if (s) - oe = s->data.marked[idnum]; - } - if (!oe) - die("mark :%" PRIuMAX " not declared", orig_idnum); - return oe; -} - -static struct atom_str *to_atom(const char *s, unsigned short len) -{ - unsigned int hc = hc_str(s, len) % atom_table_sz; - struct atom_str *c; - - for (c = atom_table[hc]; c; c = c->next_atom) - if (c->str_len == len && !strncmp(s, c->str_dat, len)) - return c; - - c = mem_pool_alloc(&fi_mem_pool, sizeof(struct atom_str) + len + 1); - c->str_len = len; - memcpy(c->str_dat, s, len); - c->str_dat[len] = 0; - c->next_atom = atom_table[hc]; - atom_table[hc] = c; - atom_cnt++; - return c; -} - -static struct branch *lookup_branch(const char *name) -{ - unsigned int hc = hc_str(name, strlen(name)) % branch_table_sz; - struct branch *b; - - for (b = branch_table[hc]; b; b = b->table_next_branch) - if (!strcmp(name, b->name)) - return b; - return NULL; -} - -static struct branch *new_branch(const char *name) -{ - unsigned int hc = hc_str(name, strlen(name)) % branch_table_sz; - struct branch *b = lookup_branch(name); - - if (b) - die("Invalid attempt to create duplicate branch: %s", name); - if (check_refname_format(name, REFNAME_ALLOW_ONELEVEL)) - die("Branch name doesn't conform to GIT standards: %s", name); - - b = mem_pool_calloc(&fi_mem_pool, 1, sizeof(struct branch)); - b->name = pool_strdup(name); - b->table_next_branch = branch_table[hc]; - b->branch_tree.versions[0].mode = S_IFDIR; - b->branch_tree.versions[1].mode = S_IFDIR; - b->num_notes = 0; - b->active = 0; - b->pack_id = MAX_PACK_ID; - branch_table[hc] = b; - branch_count++; - return b; -} - -static unsigned int hc_entries(unsigned int cnt) -{ - cnt = cnt & 7 ? (cnt / 8) + 1 : cnt / 8; - return cnt < avail_tree_table_sz ? cnt : avail_tree_table_sz - 1; -} - -static struct tree_content *new_tree_content(unsigned int cnt) -{ - struct avail_tree_content *f, *l = NULL; - struct tree_content *t; - unsigned int hc = hc_entries(cnt); - - for (f = avail_tree_table[hc]; f; l = f, f = f->next_avail) - if (f->entry_capacity >= cnt) - break; - - if (f) { - if (l) - l->next_avail = f->next_avail; - else - avail_tree_table[hc] = f->next_avail; - } else { - cnt = cnt & 7 ? ((cnt / 8) + 1) * 8 : cnt; - f = mem_pool_alloc(&fi_mem_pool, sizeof(*t) + sizeof(t->entries[0]) * cnt); - f->entry_capacity = cnt; - } - - t = (struct tree_content*)f; - t->entry_count = 0; - t->delta_depth = 0; - return t; -} - -static void release_tree_entry(struct tree_entry *e); -static void release_tree_content(struct tree_content *t) -{ - struct avail_tree_content *f = (struct avail_tree_content*)t; - unsigned int hc = hc_entries(f->entry_capacity); - f->next_avail = avail_tree_table[hc]; - avail_tree_table[hc] = f; -} - -static void release_tree_content_recursive(struct tree_content *t) -{ - unsigned int i; - for (i = 0; i < t->entry_count; i++) - release_tree_entry(t->entries[i]); - release_tree_content(t); -} - -static struct tree_content *grow_tree_content( - struct tree_content *t, - int amt) -{ - struct tree_content *r = new_tree_content(t->entry_count + amt); - r->entry_count = t->entry_count; - r->delta_depth = t->delta_depth; - COPY_ARRAY(r->entries, t->entries, t->entry_count); - release_tree_content(t); - return r; -} - -static struct tree_entry *new_tree_entry(void) -{ - struct tree_entry *e; - - if (!avail_tree_entry) { - unsigned int n = tree_entry_alloc; - tree_entry_allocd += n * sizeof(struct tree_entry); - ALLOC_ARRAY(e, n); - avail_tree_entry = e; - while (n-- > 1) { - *((void**)e) = e + 1; - e++; - } - *((void**)e) = NULL; - } - - e = avail_tree_entry; - avail_tree_entry = *((void**)e); - return e; -} - -static void release_tree_entry(struct tree_entry *e) -{ - if (e->tree) - release_tree_content_recursive(e->tree); - *((void**)e) = avail_tree_entry; - avail_tree_entry = e; -} - -static struct tree_content *dup_tree_content(struct tree_content *s) -{ - struct tree_content *d; - struct tree_entry *a, *b; - unsigned int i; - - if (!s) - return NULL; - d = new_tree_content(s->entry_count); - for (i = 0; i < s->entry_count; i++) { - a = s->entries[i]; - b = new_tree_entry(); - memcpy(b, a, sizeof(*a)); - if (a->tree && is_null_oid(&b->versions[1].oid)) - b->tree = dup_tree_content(a->tree); - else - b->tree = NULL; - d->entries[i] = b; - } - d->entry_count = s->entry_count; - d->delta_depth = s->delta_depth; - - return d; -} - -static void start_packfile(void) -{ - struct strbuf tmp_file = STRBUF_INIT; - struct packed_git *p; - struct pack_header hdr; - int pack_fd; - - pack_fd = odb_mkstemp(&tmp_file, "pack/tmp_pack_XXXXXX"); - FLEX_ALLOC_STR(p, pack_name, tmp_file.buf); - strbuf_release(&tmp_file); - - p->pack_fd = pack_fd; - p->do_not_close = 1; - pack_file = hashfd(pack_fd, p->pack_name); - - hdr.hdr_signature = htonl(PACK_SIGNATURE); - hdr.hdr_version = htonl(2); - hdr.hdr_entries = 0; - hashwrite(pack_file, &hdr, sizeof(hdr)); - - pack_data = p; - pack_size = sizeof(hdr); - object_count = 0; - - REALLOC_ARRAY(all_packs, pack_id + 1); - all_packs[pack_id] = p; -} - -static const char *create_index(void) -{ - const char *tmpfile; - struct pack_idx_entry **idx, **c, **last; - struct object_entry *e; - struct object_entry_pool *o; - - /* Build the table of object IDs. */ - ALLOC_ARRAY(idx, object_count); - c = idx; - for (o = blocks; o; o = o->next_pool) - for (e = o->next_free; e-- != o->entries;) - if (pack_id == e->pack_id) - *c++ = &e->idx; - last = idx + object_count; - if (c != last) - die("internal consistency error creating the index"); - - tmpfile = write_idx_file(NULL, idx, object_count, &pack_idx_opts, - pack_data->hash); - free(idx); - return tmpfile; -} - -static char *keep_pack(const char *curr_index_name) -{ - static const char *keep_msg = "fast-import"; - struct strbuf name = STRBUF_INIT; - int keep_fd; - - odb_pack_name(&name, pack_data->hash, "keep"); - keep_fd = odb_pack_keep(name.buf); - if (keep_fd < 0) - die_errno("cannot create keep file"); - write_or_die(keep_fd, keep_msg, strlen(keep_msg)); - if (close(keep_fd)) - die_errno("failed to write keep file"); - - odb_pack_name(&name, pack_data->hash, "pack"); - if (finalize_object_file(pack_data->pack_name, name.buf)) - die("cannot store pack file"); - - odb_pack_name(&name, pack_data->hash, "idx"); - if (finalize_object_file(curr_index_name, name.buf)) - die("cannot store index file"); - free((void *)curr_index_name); - return strbuf_detach(&name, NULL); -} - -static void unkeep_all_packs(void) -{ - struct strbuf name = STRBUF_INIT; - int k; - - for (k = 0; k < pack_id; k++) { - struct packed_git *p = all_packs[k]; - odb_pack_name(&name, p->hash, "keep"); - unlink_or_warn(name.buf); - } - strbuf_release(&name); -} - -static int loosen_small_pack(const struct packed_git *p) -{ - struct child_process unpack = CHILD_PROCESS_INIT; - - if (lseek(p->pack_fd, 0, SEEK_SET) < 0) - die_errno("Failed seeking to start of '%s'", p->pack_name); - - unpack.in = p->pack_fd; - unpack.git_cmd = 1; - unpack.stdout_to_stderr = 1; - strvec_push(&unpack.args, "unpack-objects"); - if (!show_stats) - strvec_push(&unpack.args, "-q"); - - return run_command(&unpack); -} - -static void end_packfile(void) -{ - static int running; - - if (running || !pack_data) - return; - - running = 1; - clear_delta_base_cache(); - if (object_count) { - struct packed_git *new_p; - struct object_id cur_pack_oid; - char *idx_name; - int i; - struct branch *b; - struct tag *t; - - close_pack_windows(pack_data); - finalize_hashfile(pack_file, cur_pack_oid.hash, 0); - fixup_pack_header_footer(pack_data->pack_fd, pack_data->hash, - pack_data->pack_name, object_count, - cur_pack_oid.hash, pack_size); - - if (object_count <= unpack_limit) { - if (!loosen_small_pack(pack_data)) { - invalidate_pack_id(pack_id); - goto discard_pack; - } - } - - close(pack_data->pack_fd); - idx_name = keep_pack(create_index()); - - /* Register the packfile with core git's machinery. */ - new_p = add_packed_git(idx_name, strlen(idx_name), 1); - if (!new_p) - die("core git rejected index %s", idx_name); - all_packs[pack_id] = new_p; - install_packed_git(the_repository, new_p); - free(idx_name); - - /* Print the boundary */ - if (pack_edges) { - fprintf(pack_edges, "%s:", new_p->pack_name); - for (i = 0; i < branch_table_sz; i++) { - for (b = branch_table[i]; b; b = b->table_next_branch) { - if (b->pack_id == pack_id) - fprintf(pack_edges, " %s", - oid_to_hex(&b->oid)); - } - } - for (t = first_tag; t; t = t->next_tag) { - if (t->pack_id == pack_id) - fprintf(pack_edges, " %s", - oid_to_hex(&t->oid)); - } - fputc('\n', pack_edges); - fflush(pack_edges); - } - - pack_id++; - } - else { -discard_pack: - close(pack_data->pack_fd); - unlink_or_warn(pack_data->pack_name); - } - FREE_AND_NULL(pack_data); - running = 0; - - /* We can't carry a delta across packfiles. */ - strbuf_release(&last_blob.data); - last_blob.offset = 0; - last_blob.depth = 0; -} - -static void cycle_packfile(void) -{ - end_packfile(); - start_packfile(); -} - -static int store_object( - enum object_type type, - struct strbuf *dat, - struct last_object *last, - struct object_id *oidout, - uintmax_t mark) -{ - void *out, *delta; - struct object_entry *e; - unsigned char hdr[96]; - struct object_id oid; - unsigned long hdrlen, deltalen; - git_hash_ctx c; - git_zstream s; - - hdrlen = xsnprintf((char *)hdr, sizeof(hdr), "%s %lu", - type_name(type), (unsigned long)dat->len) + 1; - the_hash_algo->init_fn(&c); - the_hash_algo->update_fn(&c, hdr, hdrlen); - the_hash_algo->update_fn(&c, dat->buf, dat->len); - the_hash_algo->final_fn(oid.hash, &c); - if (oidout) - oidcpy(oidout, &oid); - - e = insert_object(&oid); - if (mark) - insert_mark(marks, mark, e); - if (e->idx.offset) { - duplicate_count_by_type[type]++; - return 1; - } else if (find_sha1_pack(oid.hash, - get_all_packs(the_repository))) { - e->type = type; - e->pack_id = MAX_PACK_ID; - e->idx.offset = 1; /* just not zero! */ - duplicate_count_by_type[type]++; - return 1; - } - - if (last && last->data.len && last->data.buf && last->depth < max_depth - && dat->len > the_hash_algo->rawsz) { - - delta_count_attempts_by_type[type]++; - delta = diff_delta(last->data.buf, last->data.len, - dat->buf, dat->len, - &deltalen, dat->len - the_hash_algo->rawsz); - } else - delta = NULL; - - git_deflate_init(&s, pack_compression_level); - if (delta) { - s.next_in = delta; - s.avail_in = deltalen; - } else { - s.next_in = (void *)dat->buf; - s.avail_in = dat->len; - } - s.avail_out = git_deflate_bound(&s, s.avail_in); - s.next_out = out = xmalloc(s.avail_out); - while (git_deflate(&s, Z_FINISH) == Z_OK) - ; /* nothing */ - git_deflate_end(&s); - - /* Determine if we should auto-checkpoint. */ - if ((max_packsize - && (pack_size + PACK_SIZE_THRESHOLD + s.total_out) > max_packsize) - || (pack_size + PACK_SIZE_THRESHOLD + s.total_out) < pack_size) { - - /* This new object needs to *not* have the current pack_id. */ - e->pack_id = pack_id + 1; - cycle_packfile(); - - /* We cannot carry a delta into the new pack. */ - if (delta) { - FREE_AND_NULL(delta); - - git_deflate_init(&s, pack_compression_level); - s.next_in = (void *)dat->buf; - s.avail_in = dat->len; - s.avail_out = git_deflate_bound(&s, s.avail_in); - s.next_out = out = xrealloc(out, s.avail_out); - while (git_deflate(&s, Z_FINISH) == Z_OK) - ; /* nothing */ - git_deflate_end(&s); - } - } - - e->type = type; - e->pack_id = pack_id; - e->idx.offset = pack_size; - object_count++; - object_count_by_type[type]++; - - crc32_begin(pack_file); - - if (delta) { - off_t ofs = e->idx.offset - last->offset; - unsigned pos = sizeof(hdr) - 1; - - delta_count_by_type[type]++; - e->depth = last->depth + 1; - - hdrlen = encode_in_pack_object_header(hdr, sizeof(hdr), - OBJ_OFS_DELTA, deltalen); - hashwrite(pack_file, hdr, hdrlen); - pack_size += hdrlen; - - hdr[pos] = ofs & 127; - while (ofs >>= 7) - hdr[--pos] = 128 | (--ofs & 127); - hashwrite(pack_file, hdr + pos, sizeof(hdr) - pos); - pack_size += sizeof(hdr) - pos; - } else { - e->depth = 0; - hdrlen = encode_in_pack_object_header(hdr, sizeof(hdr), - type, dat->len); - hashwrite(pack_file, hdr, hdrlen); - pack_size += hdrlen; - } - - hashwrite(pack_file, out, s.total_out); - pack_size += s.total_out; - - e->idx.crc32 = crc32_end(pack_file); - - free(out); - free(delta); - if (last) { - if (last->no_swap) { - last->data = *dat; - } else { - strbuf_swap(&last->data, dat); - } - last->offset = e->idx.offset; - last->depth = e->depth; - } - return 0; -} - -static void truncate_pack(struct hashfile_checkpoint *checkpoint) -{ - if (hashfile_truncate(pack_file, checkpoint)) - die_errno("cannot truncate pack to skip duplicate"); - pack_size = checkpoint->offset; -} - -static void stream_blob(uintmax_t len, struct object_id *oidout, uintmax_t mark) -{ - size_t in_sz = 64 * 1024, out_sz = 64 * 1024; - unsigned char *in_buf = xmalloc(in_sz); - unsigned char *out_buf = xmalloc(out_sz); - struct object_entry *e; - struct object_id oid; - unsigned long hdrlen; - off_t offset; - git_hash_ctx c; - git_zstream s; - struct hashfile_checkpoint checkpoint; - int status = Z_OK; - - /* Determine if we should auto-checkpoint. */ - if ((max_packsize - && (pack_size + PACK_SIZE_THRESHOLD + len) > max_packsize) - || (pack_size + PACK_SIZE_THRESHOLD + len) < pack_size) - cycle_packfile(); - - hashfile_checkpoint(pack_file, &checkpoint); - offset = checkpoint.offset; - - hdrlen = xsnprintf((char *)out_buf, out_sz, "blob %" PRIuMAX, len) + 1; - - the_hash_algo->init_fn(&c); - the_hash_algo->update_fn(&c, out_buf, hdrlen); - - crc32_begin(pack_file); - - git_deflate_init(&s, pack_compression_level); - - hdrlen = encode_in_pack_object_header(out_buf, out_sz, OBJ_BLOB, len); - - s.next_out = out_buf + hdrlen; - s.avail_out = out_sz - hdrlen; - - while (status != Z_STREAM_END) { - if (0 < len && !s.avail_in) { - size_t cnt = in_sz < len ? in_sz : (size_t)len; - size_t n = fread(in_buf, 1, cnt, stdin); - if (!n && feof(stdin)) - die("EOF in data (%" PRIuMAX " bytes remaining)", len); - - the_hash_algo->update_fn(&c, in_buf, n); - s.next_in = in_buf; - s.avail_in = n; - len -= n; - } - - status = git_deflate(&s, len ? 0 : Z_FINISH); - - if (!s.avail_out || status == Z_STREAM_END) { - size_t n = s.next_out - out_buf; - hashwrite(pack_file, out_buf, n); - pack_size += n; - s.next_out = out_buf; - s.avail_out = out_sz; - } - - switch (status) { - case Z_OK: - case Z_BUF_ERROR: - case Z_STREAM_END: - continue; - default: - die("unexpected deflate failure: %d", status); - } - } - git_deflate_end(&s); - the_hash_algo->final_fn(oid.hash, &c); - - if (oidout) - oidcpy(oidout, &oid); - - e = insert_object(&oid); - - if (mark) - insert_mark(marks, mark, e); - - if (e->idx.offset) { - duplicate_count_by_type[OBJ_BLOB]++; - truncate_pack(&checkpoint); - - } else if (find_sha1_pack(oid.hash, - get_all_packs(the_repository))) { - e->type = OBJ_BLOB; - e->pack_id = MAX_PACK_ID; - e->idx.offset = 1; /* just not zero! */ - duplicate_count_by_type[OBJ_BLOB]++; - truncate_pack(&checkpoint); - - } else { - e->depth = 0; - e->type = OBJ_BLOB; - e->pack_id = pack_id; - e->idx.offset = offset; - e->idx.crc32 = crc32_end(pack_file); - object_count++; - object_count_by_type[OBJ_BLOB]++; - } - - free(in_buf); - free(out_buf); -} - -/* All calls must be guarded by find_object() or find_mark() to - * ensure the 'struct object_entry' passed was written by this - * process instance. We unpack the entry by the offset, avoiding - * the need for the corresponding .idx file. This unpacking rule - * works because we only use OBJ_REF_DELTA within the packfiles - * created by fast-import. - * - * oe must not be NULL. Such an oe usually comes from giving - * an unknown SHA-1 to find_object() or an undefined mark to - * find_mark(). Callers must test for this condition and use - * the standard read_sha1_file() when it happens. - * - * oe->pack_id must not be MAX_PACK_ID. Such an oe is usually from - * find_mark(), where the mark was reloaded from an existing marks - * file and is referencing an object that this fast-import process - * instance did not write out to a packfile. Callers must test for - * this condition and use read_sha1_file() instead. - */ -static void *gfi_unpack_entry( - struct object_entry *oe, - unsigned long *sizep) -{ - enum object_type type; - struct packed_git *p = all_packs[oe->pack_id]; - if (p == pack_data && p->pack_size < (pack_size + the_hash_algo->rawsz)) { - /* The object is stored in the packfile we are writing to - * and we have modified it since the last time we scanned - * back to read a previously written object. If an old - * window covered [p->pack_size, p->pack_size + rawsz) its - * data is stale and is not valid. Closing all windows - * and updating the packfile length ensures we can read - * the newly written data. - */ - close_pack_windows(p); - hashflush(pack_file); - - /* We have to offer rawsz bytes additional on the end of - * the packfile as the core unpacker code assumes the - * footer is present at the file end and must promise - * at least rawsz bytes within any window it maps. But - * we don't actually create the footer here. - */ - p->pack_size = pack_size + the_hash_algo->rawsz; - } - return unpack_entry(the_repository, p, oe->idx.offset, &type, sizep); -} - -static const char *get_mode(const char *str, uint16_t *modep) -{ - unsigned char c; - uint16_t mode = 0; - - while ((c = *str++) != ' ') { - if (c < '0' || c > '7') - return NULL; - mode = (mode << 3) + (c - '0'); - } - *modep = mode; - return str; -} - -static void load_tree(struct tree_entry *root) -{ - struct object_id *oid = &root->versions[1].oid; - struct object_entry *myoe; - struct tree_content *t; - unsigned long size; - char *buf; - const char *c; - - root->tree = t = new_tree_content(8); - if (is_null_oid(oid)) - return; - - myoe = find_object(oid); - if (myoe && myoe->pack_id != MAX_PACK_ID) { - if (myoe->type != OBJ_TREE) - die("Not a tree: %s", oid_to_hex(oid)); - t->delta_depth = myoe->depth; - buf = gfi_unpack_entry(myoe, &size); - if (!buf) - die("Can't load tree %s", oid_to_hex(oid)); - } else { - enum object_type type; - buf = read_object_file(oid, &type, &size); - if (!buf || type != OBJ_TREE) - die("Can't load tree %s", oid_to_hex(oid)); - } - - c = buf; - while (c != (buf + size)) { - struct tree_entry *e = new_tree_entry(); - - if (t->entry_count == t->entry_capacity) - root->tree = t = grow_tree_content(t, t->entry_count); - t->entries[t->entry_count++] = e; - - e->tree = NULL; - c = get_mode(c, &e->versions[1].mode); - if (!c) - die("Corrupt mode in %s", oid_to_hex(oid)); - e->versions[0].mode = e->versions[1].mode; - e->name = to_atom(c, strlen(c)); - c += e->name->str_len + 1; - hashcpy(e->versions[0].oid.hash, (unsigned char *)c); - hashcpy(e->versions[1].oid.hash, (unsigned char *)c); - c += the_hash_algo->rawsz; - } - free(buf); -} - -static int tecmp0 (const void *_a, const void *_b) -{ - struct tree_entry *a = *((struct tree_entry**)_a); - struct tree_entry *b = *((struct tree_entry**)_b); - return base_name_compare( - a->name->str_dat, a->name->str_len, a->versions[0].mode, - b->name->str_dat, b->name->str_len, b->versions[0].mode); -} - -static int tecmp1 (const void *_a, const void *_b) -{ - struct tree_entry *a = *((struct tree_entry**)_a); - struct tree_entry *b = *((struct tree_entry**)_b); - return base_name_compare( - a->name->str_dat, a->name->str_len, a->versions[1].mode, - b->name->str_dat, b->name->str_len, b->versions[1].mode); -} - -static void mktree(struct tree_content *t, int v, struct strbuf *b) -{ - size_t maxlen = 0; - unsigned int i; - - if (!v) - QSORT(t->entries, t->entry_count, tecmp0); - else - QSORT(t->entries, t->entry_count, tecmp1); - - for (i = 0; i < t->entry_count; i++) { - if (t->entries[i]->versions[v].mode) - maxlen += t->entries[i]->name->str_len + 34; - } - - strbuf_reset(b); - strbuf_grow(b, maxlen); - for (i = 0; i < t->entry_count; i++) { - struct tree_entry *e = t->entries[i]; - if (!e->versions[v].mode) - continue; - strbuf_addf(b, "%o %s%c", - (unsigned int)(e->versions[v].mode & ~NO_DELTA), - e->name->str_dat, '\0'); - strbuf_add(b, e->versions[v].oid.hash, the_hash_algo->rawsz); - } -} - -static void store_tree(struct tree_entry *root) -{ - struct tree_content *t; - unsigned int i, j, del; - struct last_object lo = { STRBUF_INIT, 0, 0, /* no_swap */ 1 }; - struct object_entry *le = NULL; - - if (!is_null_oid(&root->versions[1].oid)) - return; - - if (!root->tree) - load_tree(root); - t = root->tree; - - for (i = 0; i < t->entry_count; i++) { - if (t->entries[i]->tree) - store_tree(t->entries[i]); - } - - if (!(root->versions[0].mode & NO_DELTA)) - le = find_object(&root->versions[0].oid); - if (S_ISDIR(root->versions[0].mode) && le && le->pack_id == pack_id) { - mktree(t, 0, &old_tree); - lo.data = old_tree; - lo.offset = le->idx.offset; - lo.depth = t->delta_depth; - } - - mktree(t, 1, &new_tree); - store_object(OBJ_TREE, &new_tree, &lo, &root->versions[1].oid, 0); - - t->delta_depth = lo.depth; - for (i = 0, j = 0, del = 0; i < t->entry_count; i++) { - struct tree_entry *e = t->entries[i]; - if (e->versions[1].mode) { - e->versions[0].mode = e->versions[1].mode; - oidcpy(&e->versions[0].oid, &e->versions[1].oid); - t->entries[j++] = e; - } else { - release_tree_entry(e); - del++; - } - } - t->entry_count -= del; -} - -static void tree_content_replace( - struct tree_entry *root, - const struct object_id *oid, - const uint16_t mode, - struct tree_content *newtree) -{ - if (!S_ISDIR(mode)) - die("Root cannot be a non-directory"); - oidclr(&root->versions[0].oid); - oidcpy(&root->versions[1].oid, oid); - if (root->tree) - release_tree_content_recursive(root->tree); - root->tree = newtree; -} - -static int tree_content_set( - struct tree_entry *root, - const char *p, - const struct object_id *oid, - const uint16_t mode, - struct tree_content *subtree) -{ - struct tree_content *t; - const char *slash1; - unsigned int i, n; - struct tree_entry *e; - - slash1 = strchrnul(p, '/'); - n = slash1 - p; - if (!n) - die("Empty path component found in input"); - if (!*slash1 && !S_ISDIR(mode) && subtree) - die("Non-directories cannot have subtrees"); - - if (!root->tree) - load_tree(root); - t = root->tree; - for (i = 0; i < t->entry_count; i++) { - e = t->entries[i]; - if (e->name->str_len == n && !fspathncmp(p, e->name->str_dat, n)) { - if (!*slash1) { - if (!S_ISDIR(mode) - && e->versions[1].mode == mode - && oideq(&e->versions[1].oid, oid)) - return 0; - e->versions[1].mode = mode; - oidcpy(&e->versions[1].oid, oid); - if (e->tree) - release_tree_content_recursive(e->tree); - e->tree = subtree; - - /* - * We need to leave e->versions[0].sha1 alone - * to avoid modifying the preimage tree used - * when writing out the parent directory. - * But after replacing the subdir with a - * completely different one, it's not a good - * delta base any more, and besides, we've - * thrown away the tree entries needed to - * make a delta against it. - * - * So let's just explicitly disable deltas - * for the subtree. - */ - if (S_ISDIR(e->versions[0].mode)) - e->versions[0].mode |= NO_DELTA; - - oidclr(&root->versions[1].oid); - return 1; - } - if (!S_ISDIR(e->versions[1].mode)) { - e->tree = new_tree_content(8); - e->versions[1].mode = S_IFDIR; - } - if (!e->tree) - load_tree(e); - if (tree_content_set(e, slash1 + 1, oid, mode, subtree)) { - oidclr(&root->versions[1].oid); - return 1; - } - return 0; - } - } - - if (t->entry_count == t->entry_capacity) - root->tree = t = grow_tree_content(t, t->entry_count); - e = new_tree_entry(); - e->name = to_atom(p, n); - e->versions[0].mode = 0; - oidclr(&e->versions[0].oid); - t->entries[t->entry_count++] = e; - if (*slash1) { - e->tree = new_tree_content(8); - e->versions[1].mode = S_IFDIR; - tree_content_set(e, slash1 + 1, oid, mode, subtree); - } else { - e->tree = subtree; - e->versions[1].mode = mode; - oidcpy(&e->versions[1].oid, oid); - } - oidclr(&root->versions[1].oid); - return 1; -} - -static int tree_content_remove( - struct tree_entry *root, - const char *p, - struct tree_entry *backup_leaf, - int allow_root) -{ - struct tree_content *t; - const char *slash1; - unsigned int i, n; - struct tree_entry *e; - - slash1 = strchrnul(p, '/'); - n = slash1 - p; - - if (!root->tree) - load_tree(root); - - if (!*p && allow_root) { - e = root; - goto del_entry; - } - - t = root->tree; - for (i = 0; i < t->entry_count; i++) { - e = t->entries[i]; - if (e->name->str_len == n && !fspathncmp(p, e->name->str_dat, n)) { - if (*slash1 && !S_ISDIR(e->versions[1].mode)) - /* - * If p names a file in some subdirectory, and a - * file or symlink matching the name of the - * parent directory of p exists, then p cannot - * exist and need not be deleted. - */ - return 1; - if (!*slash1 || !S_ISDIR(e->versions[1].mode)) - goto del_entry; - if (!e->tree) - load_tree(e); - if (tree_content_remove(e, slash1 + 1, backup_leaf, 0)) { - for (n = 0; n < e->tree->entry_count; n++) { - if (e->tree->entries[n]->versions[1].mode) { - oidclr(&root->versions[1].oid); - return 1; - } - } - backup_leaf = NULL; - goto del_entry; - } - return 0; - } - } - return 0; - -del_entry: - if (backup_leaf) - memcpy(backup_leaf, e, sizeof(*backup_leaf)); - else if (e->tree) - release_tree_content_recursive(e->tree); - e->tree = NULL; - e->versions[1].mode = 0; - oidclr(&e->versions[1].oid); - oidclr(&root->versions[1].oid); - return 1; -} - -static int tree_content_get( - struct tree_entry *root, - const char *p, - struct tree_entry *leaf, - int allow_root) -{ - struct tree_content *t; - const char *slash1; - unsigned int i, n; - struct tree_entry *e; - - slash1 = strchrnul(p, '/'); - n = slash1 - p; - if (!n && !allow_root) - die("Empty path component found in input"); - - if (!root->tree) - load_tree(root); - - if (!n) { - e = root; - goto found_entry; - } - - t = root->tree; - for (i = 0; i < t->entry_count; i++) { - e = t->entries[i]; - if (e->name->str_len == n && !fspathncmp(p, e->name->str_dat, n)) { - if (!*slash1) - goto found_entry; - if (!S_ISDIR(e->versions[1].mode)) - return 0; - if (!e->tree) - load_tree(e); - return tree_content_get(e, slash1 + 1, leaf, 0); - } - } - return 0; - -found_entry: - memcpy(leaf, e, sizeof(*leaf)); - if (e->tree && is_null_oid(&e->versions[1].oid)) - leaf->tree = dup_tree_content(e->tree); - else - leaf->tree = NULL; - return 1; -} - -static int update_branch(struct branch *b) -{ - static const char *msg = "fast-import"; - struct ref_transaction *transaction; - struct object_id old_oid; - struct strbuf err = STRBUF_INIT; - - if (is_null_oid(&b->oid)) { - if (b->delete) - delete_ref(NULL, b->name, NULL, 0); - return 0; - } - if (read_ref(b->name, &old_oid)) - oidclr(&old_oid); - if (!force_update && !is_null_oid(&old_oid)) { - struct commit *old_cmit, *new_cmit; - - old_cmit = lookup_commit_reference_gently(the_repository, - &old_oid, 0); - new_cmit = lookup_commit_reference_gently(the_repository, - &b->oid, 0); - if (!old_cmit || !new_cmit) - return error("Branch %s is missing commits.", b->name); - - if (!in_merge_bases(old_cmit, new_cmit)) { - warning("Not updating %s" - " (new tip %s does not contain %s)", - b->name, oid_to_hex(&b->oid), - oid_to_hex(&old_oid)); - return -1; - } - } - transaction = ref_transaction_begin(&err); - if (!transaction || - ref_transaction_update(transaction, b->name, &b->oid, &old_oid, - 0, msg, &err) || - ref_transaction_commit(transaction, &err)) { - ref_transaction_free(transaction); - error("%s", err.buf); - strbuf_release(&err); - return -1; - } - ref_transaction_free(transaction); - strbuf_release(&err); - return 0; -} - -static void dump_branches(void) -{ - unsigned int i; - struct branch *b; - - for (i = 0; i < branch_table_sz; i++) { - for (b = branch_table[i]; b; b = b->table_next_branch) - failure |= update_branch(b); - } -} - -static void dump_tags(void) -{ - static const char *msg = "fast-import"; - struct tag *t; - struct strbuf ref_name = STRBUF_INIT; - struct strbuf err = STRBUF_INIT; - struct ref_transaction *transaction; - - transaction = ref_transaction_begin(&err); - if (!transaction) { - failure |= error("%s", err.buf); - goto cleanup; - } - for (t = first_tag; t; t = t->next_tag) { - strbuf_reset(&ref_name); - strbuf_addf(&ref_name, "refs/tags/%s", t->name); - - if (ref_transaction_update(transaction, ref_name.buf, - &t->oid, NULL, 0, msg, &err)) { - failure |= error("%s", err.buf); - goto cleanup; - } - } - if (ref_transaction_commit(transaction, &err)) - failure |= error("%s", err.buf); - - cleanup: - ref_transaction_free(transaction); - strbuf_release(&ref_name); - strbuf_release(&err); -} - -static void dump_marks(void) -{ - struct lock_file mark_lock = LOCK_INIT; - FILE *f; - - if (!export_marks_file || (import_marks_file && !import_marks_file_done)) - return; - - if (safe_create_leading_directories_const(export_marks_file)) { - failure |= error_errno("unable to create leading directories of %s", - export_marks_file); - return; - } - - if (hold_lock_file_for_update(&mark_lock, export_marks_file, 0) < 0) { - failure |= error_errno("Unable to write marks file %s", - export_marks_file); - return; - } - - f = fdopen_lock_file(&mark_lock, "w"); - if (!f) { - int saved_errno = errno; - rollback_lock_file(&mark_lock); - failure |= error("Unable to write marks file %s: %s", - export_marks_file, strerror(saved_errno)); - return; - } - - for_each_mark(marks, 0, dump_marks_fn, f); - if (commit_lock_file(&mark_lock)) { - failure |= error_errno("Unable to write file %s", - export_marks_file); - return; - } -} - -static void insert_object_entry(struct mark_set *s, struct object_id *oid, uintmax_t mark) -{ - struct object_entry *e; - e = find_object(oid); - if (!e) { - enum object_type type = oid_object_info(the_repository, - oid, NULL); - if (type < 0) - die("object not found: %s", oid_to_hex(oid)); - e = insert_object(oid); - e->type = type; - e->pack_id = MAX_PACK_ID; - e->idx.offset = 1; /* just not zero! */ - } - insert_mark(s, mark, e); -} - -static void insert_oid_entry(struct mark_set *s, struct object_id *oid, uintmax_t mark) -{ - insert_mark(s, mark, xmemdupz(oid, sizeof(*oid))); -} - -static void read_mark_file(struct mark_set *s, FILE *f, mark_set_inserter_t inserter) -{ - char line[512]; - while (fgets(line, sizeof(line), f)) { - uintmax_t mark; - char *end; - struct object_id oid; - - /* Ensure SHA-1 objects are padded with zeros. */ - memset(oid.hash, 0, sizeof(oid.hash)); - - end = strchr(line, '\n'); - if (line[0] != ':' || !end) - die("corrupt mark line: %s", line); - *end = 0; - mark = strtoumax(line + 1, &end, 10); - if (!mark || end == line + 1 - || *end != ' ' - || get_oid_hex_any(end + 1, &oid) == GIT_HASH_UNKNOWN) - die("corrupt mark line: %s", line); - inserter(s, &oid, mark); - } -} - -static void read_marks(void) -{ - FILE *f = fopen(import_marks_file, "r"); - if (f) - ; - else if (import_marks_file_ignore_missing && errno == ENOENT) - goto done; /* Marks file does not exist */ - else - die_errno("cannot read '%s'", import_marks_file); - read_mark_file(marks, f, insert_object_entry); - fclose(f); -done: - import_marks_file_done = 1; -} - - -static int read_next_command(void) -{ - static int stdin_eof = 0; - - if (stdin_eof) { - unread_command_buf = 0; - return EOF; - } - - for (;;) { - if (unread_command_buf) { - unread_command_buf = 0; - } else { - struct recent_command *rc; - - stdin_eof = strbuf_getline_lf(&command_buf, stdin); - if (stdin_eof) - return EOF; - - if (!seen_data_command - && !starts_with(command_buf.buf, "feature ") - && !starts_with(command_buf.buf, "option ")) { - parse_argv(); - } - - rc = rc_free; - if (rc) - rc_free = rc->next; - else { - rc = cmd_hist.next; - cmd_hist.next = rc->next; - cmd_hist.next->prev = &cmd_hist; - free(rc->buf); - } - - rc->buf = xstrdup(command_buf.buf); - rc->prev = cmd_tail; - rc->next = cmd_hist.prev; - rc->prev->next = rc; - cmd_tail = rc; - } - if (command_buf.buf[0] == '#') - continue; - return 0; - } -} - -static void skip_optional_lf(void) -{ - int term_char = fgetc(stdin); - if (term_char != '\n' && term_char != EOF) - ungetc(term_char, stdin); -} - -static void parse_mark(void) -{ - const char *v; - if (skip_prefix(command_buf.buf, "mark :", &v)) { - next_mark = strtoumax(v, NULL, 10); - read_next_command(); - } - else - next_mark = 0; -} - -static void parse_original_identifier(void) -{ - const char *v; - if (skip_prefix(command_buf.buf, "original-oid ", &v)) - read_next_command(); -} - -static int parse_data(struct strbuf *sb, uintmax_t limit, uintmax_t *len_res) -{ - const char *data; - strbuf_reset(sb); - - if (!skip_prefix(command_buf.buf, "data ", &data)) - die("Expected 'data n' command, found: %s", command_buf.buf); - - if (skip_prefix(data, "<<", &data)) { - char *term = xstrdup(data); - size_t term_len = command_buf.len - (data - command_buf.buf); - - for (;;) { - if (strbuf_getline_lf(&command_buf, stdin) == EOF) - die("EOF in data (terminator '%s' not found)", term); - if (term_len == command_buf.len - && !strcmp(term, command_buf.buf)) - break; - strbuf_addbuf(sb, &command_buf); - strbuf_addch(sb, '\n'); - } - free(term); - } - else { - uintmax_t len = strtoumax(data, NULL, 10); - size_t n = 0, length = (size_t)len; - - if (limit && limit < len) { - *len_res = len; - return 0; - } - if (length < len) - die("data is too large to use in this context"); - - while (n < length) { - size_t s = strbuf_fread(sb, length - n, stdin); - if (!s && feof(stdin)) - die("EOF in data (%lu bytes remaining)", - (unsigned long)(length - n)); - n += s; - } - } - - skip_optional_lf(); - return 1; -} - -static int validate_raw_date(const char *src, struct strbuf *result, int strict) -{ - const char *orig_src = src; - char *endp; - unsigned long num; - - errno = 0; - - num = strtoul(src, &endp, 10); - /* - * NEEDSWORK: perhaps check for reasonable values? For example, we - * could error on values representing times more than a - * day in the future. - */ - if (errno || endp == src || *endp != ' ') - return -1; - - src = endp + 1; - if (*src != '-' && *src != '+') - return -1; - - num = strtoul(src + 1, &endp, 10); - /* - * NEEDSWORK: check for brokenness other than num > 1400, such as - * (num % 100) >= 60, or ((num % 100) % 15) != 0 ? - */ - if (errno || endp == src + 1 || *endp || /* did not parse */ - (strict && (1400 < num)) /* parsed a broken timezone */ - ) - return -1; - - strbuf_addstr(result, orig_src); - return 0; -} - -static char *parse_ident(const char *buf) -{ - const char *ltgt; - size_t name_len; - struct strbuf ident = STRBUF_INIT; - - /* ensure there is a space delimiter even if there is no name */ - if (*buf == '<') - --buf; - - ltgt = buf + strcspn(buf, "<>"); - if (*ltgt != '<') - die("Missing < in ident string: %s", buf); - if (ltgt != buf && ltgt[-1] != ' ') - die("Missing space before < in ident string: %s", buf); - ltgt = ltgt + 1 + strcspn(ltgt + 1, "<>"); - if (*ltgt != '>') - die("Missing > in ident string: %s", buf); - ltgt++; - if (*ltgt != ' ') - die("Missing space after > in ident string: %s", buf); - ltgt++; - name_len = ltgt - buf; - strbuf_add(&ident, buf, name_len); - - switch (whenspec) { - case WHENSPEC_RAW: - if (validate_raw_date(ltgt, &ident, 1) < 0) - die("Invalid raw date \"%s\" in ident: %s", ltgt, buf); - break; - case WHENSPEC_RAW_PERMISSIVE: - if (validate_raw_date(ltgt, &ident, 0) < 0) - die("Invalid raw date \"%s\" in ident: %s", ltgt, buf); - break; - case WHENSPEC_RFC2822: - if (parse_date(ltgt, &ident) < 0) - die("Invalid rfc2822 date \"%s\" in ident: %s", ltgt, buf); - break; - case WHENSPEC_NOW: - if (strcmp("now", ltgt)) - die("Date in ident must be 'now': %s", buf); - datestamp(&ident); - break; - } - - return strbuf_detach(&ident, NULL); -} - -static void parse_and_store_blob( - struct last_object *last, - struct object_id *oidout, - uintmax_t mark) -{ - static struct strbuf buf = STRBUF_INIT; - uintmax_t len; - - if (parse_data(&buf, big_file_threshold, &len)) - store_object(OBJ_BLOB, &buf, last, oidout, mark); - else { - if (last) { - strbuf_release(&last->data); - last->offset = 0; - last->depth = 0; - } - stream_blob(len, oidout, mark); - skip_optional_lf(); - } -} - -static void parse_new_blob(void) -{ - read_next_command(); - parse_mark(); - parse_original_identifier(); - parse_and_store_blob(&last_blob, NULL, next_mark); -} - -static void unload_one_branch(void) -{ - while (cur_active_branches - && cur_active_branches >= max_active_branches) { - uintmax_t min_commit = ULONG_MAX; - struct branch *e, *l = NULL, *p = NULL; - - for (e = active_branches; e; e = e->active_next_branch) { - if (e->last_commit < min_commit) { - p = l; - min_commit = e->last_commit; - } - l = e; - } - - if (p) { - e = p->active_next_branch; - p->active_next_branch = e->active_next_branch; - } else { - e = active_branches; - active_branches = e->active_next_branch; - } - e->active = 0; - e->active_next_branch = NULL; - if (e->branch_tree.tree) { - release_tree_content_recursive(e->branch_tree.tree); - e->branch_tree.tree = NULL; - } - cur_active_branches--; - } -} - -static void load_branch(struct branch *b) -{ - load_tree(&b->branch_tree); - if (!b->active) { - b->active = 1; - b->active_next_branch = active_branches; - active_branches = b; - cur_active_branches++; - branch_load_count++; - } -} - -static unsigned char convert_num_notes_to_fanout(uintmax_t num_notes) -{ - unsigned char fanout = 0; - while ((num_notes >>= 8)) - fanout++; - return fanout; -} - -static void construct_path_with_fanout(const char *hex_sha1, - unsigned char fanout, char *path) -{ - unsigned int i = 0, j = 0; - if (fanout >= the_hash_algo->rawsz) - die("Too large fanout (%u)", fanout); - while (fanout) { - path[i++] = hex_sha1[j++]; - path[i++] = hex_sha1[j++]; - path[i++] = '/'; - fanout--; - } - memcpy(path + i, hex_sha1 + j, the_hash_algo->hexsz - j); - path[i + the_hash_algo->hexsz - j] = '\0'; -} - -static uintmax_t do_change_note_fanout( - struct tree_entry *orig_root, struct tree_entry *root, - char *hex_oid, unsigned int hex_oid_len, - char *fullpath, unsigned int fullpath_len, - unsigned char fanout) -{ - struct tree_content *t; - struct tree_entry *e, leaf; - unsigned int i, tmp_hex_oid_len, tmp_fullpath_len; - uintmax_t num_notes = 0; - struct object_id oid; - /* hex oid + '/' between each pair of hex digits + NUL */ - char realpath[GIT_MAX_HEXSZ + ((GIT_MAX_HEXSZ / 2) - 1) + 1]; - const unsigned hexsz = the_hash_algo->hexsz; - - if (!root->tree) - load_tree(root); - t = root->tree; - - for (i = 0; t && i < t->entry_count; i++) { - e = t->entries[i]; - tmp_hex_oid_len = hex_oid_len + e->name->str_len; - tmp_fullpath_len = fullpath_len; - - /* - * We're interested in EITHER existing note entries (entries - * with exactly 40 hex chars in path, not including directory - * separators), OR directory entries that may contain note - * entries (with < 40 hex chars in path). - * Also, each path component in a note entry must be a multiple - * of 2 chars. - */ - if (!e->versions[1].mode || - tmp_hex_oid_len > hexsz || - e->name->str_len % 2) - continue; - - /* This _may_ be a note entry, or a subdir containing notes */ - memcpy(hex_oid + hex_oid_len, e->name->str_dat, - e->name->str_len); - if (tmp_fullpath_len) - fullpath[tmp_fullpath_len++] = '/'; - memcpy(fullpath + tmp_fullpath_len, e->name->str_dat, - e->name->str_len); - tmp_fullpath_len += e->name->str_len; - fullpath[tmp_fullpath_len] = '\0'; - - if (tmp_hex_oid_len == hexsz && !get_oid_hex(hex_oid, &oid)) { - /* This is a note entry */ - if (fanout == 0xff) { - /* Counting mode, no rename */ - num_notes++; - continue; - } - construct_path_with_fanout(hex_oid, fanout, realpath); - if (!strcmp(fullpath, realpath)) { - /* Note entry is in correct location */ - num_notes++; - continue; - } - - /* Rename fullpath to realpath */ - if (!tree_content_remove(orig_root, fullpath, &leaf, 0)) - die("Failed to remove path %s", fullpath); - tree_content_set(orig_root, realpath, - &leaf.versions[1].oid, - leaf.versions[1].mode, - leaf.tree); - } else if (S_ISDIR(e->versions[1].mode)) { - /* This is a subdir that may contain note entries */ - num_notes += do_change_note_fanout(orig_root, e, - hex_oid, tmp_hex_oid_len, - fullpath, tmp_fullpath_len, fanout); - } - - /* The above may have reallocated the current tree_content */ - t = root->tree; - } - return num_notes; -} - -static uintmax_t change_note_fanout(struct tree_entry *root, - unsigned char fanout) -{ - /* - * The size of path is due to one slash between every two hex digits, - * plus the terminating NUL. Note that there is no slash at the end, so - * the number of slashes is one less than half the number of hex - * characters. - */ - char hex_oid[GIT_MAX_HEXSZ], path[GIT_MAX_HEXSZ + (GIT_MAX_HEXSZ / 2) - 1 + 1]; - return do_change_note_fanout(root, root, hex_oid, 0, path, 0, fanout); -} - -static int parse_mapped_oid_hex(const char *hex, struct object_id *oid, const char **end) -{ - int algo; - khiter_t it; - - /* Make SHA-1 object IDs have all-zero padding. */ - memset(oid->hash, 0, sizeof(oid->hash)); - - algo = parse_oid_hex_any(hex, oid, end); - if (algo == GIT_HASH_UNKNOWN) - return -1; - - it = kh_get_oid_map(sub_oid_map, *oid); - /* No such object? */ - if (it == kh_end(sub_oid_map)) { - /* If we're using the same algorithm, pass it through. */ - if (hash_algos[algo].format_id == the_hash_algo->format_id) - return 0; - return -1; - } - oidcpy(oid, kh_value(sub_oid_map, it)); - return 0; -} - -/* - * Given a pointer into a string, parse a mark reference: - * - * idnum ::= ':' bigint; - * - * Return the first character after the value in *endptr. - * - * Complain if the following character is not what is expected, - * either a space or end of the string. - */ -static uintmax_t parse_mark_ref(const char *p, char **endptr) -{ - uintmax_t mark; - - assert(*p == ':'); - p++; - mark = strtoumax(p, endptr, 10); - if (*endptr == p) - die("No value after ':' in mark: %s", command_buf.buf); - return mark; -} - -/* - * Parse the mark reference, and complain if this is not the end of - * the string. - */ -static uintmax_t parse_mark_ref_eol(const char *p) -{ - char *end; - uintmax_t mark; - - mark = parse_mark_ref(p, &end); - if (*end != '\0') - die("Garbage after mark: %s", command_buf.buf); - return mark; -} - -/* - * Parse the mark reference, demanding a trailing space. Return a - * pointer to the space. - */ -static uintmax_t parse_mark_ref_space(const char **p) -{ - uintmax_t mark; - char *end; - - mark = parse_mark_ref(*p, &end); - if (*end++ != ' ') - die("Missing space after mark: %s", command_buf.buf); - *p = end; - return mark; -} - -static void file_change_m(const char *p, struct branch *b) -{ - static struct strbuf uq = STRBUF_INIT; - const char *endp; - struct object_entry *oe; - struct object_id oid; - uint16_t mode, inline_data = 0; - - p = get_mode(p, &mode); - if (!p) - die("Corrupt mode: %s", command_buf.buf); - switch (mode) { - case 0644: - case 0755: - mode |= S_IFREG; - case S_IFREG | 0644: - case S_IFREG | 0755: - case S_IFLNK: - case S_IFDIR: - case S_IFGITLINK: - /* ok */ - break; - default: - die("Corrupt mode: %s", command_buf.buf); - } - - if (*p == ':') { - oe = find_mark(marks, parse_mark_ref_space(&p)); - oidcpy(&oid, &oe->idx.oid); - } else if (skip_prefix(p, "inline ", &p)) { - inline_data = 1; - oe = NULL; /* not used with inline_data, but makes gcc happy */ - } else { - if (parse_mapped_oid_hex(p, &oid, &p)) - die("Invalid dataref: %s", command_buf.buf); - oe = find_object(&oid); - if (*p++ != ' ') - die("Missing space after SHA1: %s", command_buf.buf); - } - - strbuf_reset(&uq); - if (!unquote_c_style(&uq, p, &endp)) { - if (*endp) - die("Garbage after path in: %s", command_buf.buf); - p = uq.buf; - } - - /* Git does not track empty, non-toplevel directories. */ - if (S_ISDIR(mode) && is_empty_tree_oid(&oid) && *p) { - tree_content_remove(&b->branch_tree, p, NULL, 0); - return; - } - - if (S_ISGITLINK(mode)) { - if (inline_data) - die("Git links cannot be specified 'inline': %s", - command_buf.buf); - else if (oe) { - if (oe->type != OBJ_COMMIT) - die("Not a commit (actually a %s): %s", - type_name(oe->type), command_buf.buf); - } - /* - * Accept the sha1 without checking; it expected to be in - * another repository. - */ - } else if (inline_data) { - if (S_ISDIR(mode)) - die("Directories cannot be specified 'inline': %s", - command_buf.buf); - if (p != uq.buf) { - strbuf_addstr(&uq, p); - p = uq.buf; - } - while (read_next_command() != EOF) { - const char *v; - if (skip_prefix(command_buf.buf, "cat-blob ", &v)) - parse_cat_blob(v); - else { - parse_and_store_blob(&last_blob, &oid, 0); - break; - } - } - } else { - enum object_type expected = S_ISDIR(mode) ? - OBJ_TREE: OBJ_BLOB; - enum object_type type = oe ? oe->type : - oid_object_info(the_repository, &oid, - NULL); - if (type < 0) - die("%s not found: %s", - S_ISDIR(mode) ? "Tree" : "Blob", - command_buf.buf); - if (type != expected) - die("Not a %s (actually a %s): %s", - type_name(expected), type_name(type), - command_buf.buf); - } - - if (!*p) { - tree_content_replace(&b->branch_tree, &oid, mode, NULL); - return; - } - tree_content_set(&b->branch_tree, p, &oid, mode, NULL); -} - -static void file_change_d(const char *p, struct branch *b) -{ - static struct strbuf uq = STRBUF_INIT; - const char *endp; - - strbuf_reset(&uq); - if (!unquote_c_style(&uq, p, &endp)) { - if (*endp) - die("Garbage after path in: %s", command_buf.buf); - p = uq.buf; - } - tree_content_remove(&b->branch_tree, p, NULL, 1); -} - -static void file_change_cr(const char *s, struct branch *b, int rename) -{ - const char *d; - static struct strbuf s_uq = STRBUF_INIT; - static struct strbuf d_uq = STRBUF_INIT; - const char *endp; - struct tree_entry leaf; - - strbuf_reset(&s_uq); - if (!unquote_c_style(&s_uq, s, &endp)) { - if (*endp != ' ') - die("Missing space after source: %s", command_buf.buf); - } else { - endp = strchr(s, ' '); - if (!endp) - die("Missing space after source: %s", command_buf.buf); - strbuf_add(&s_uq, s, endp - s); - } - s = s_uq.buf; - - endp++; - if (!*endp) - die("Missing dest: %s", command_buf.buf); - - d = endp; - strbuf_reset(&d_uq); - if (!unquote_c_style(&d_uq, d, &endp)) { - if (*endp) - die("Garbage after dest in: %s", command_buf.buf); - d = d_uq.buf; - } - - memset(&leaf, 0, sizeof(leaf)); - if (rename) - tree_content_remove(&b->branch_tree, s, &leaf, 1); - else - tree_content_get(&b->branch_tree, s, &leaf, 1); - if (!leaf.versions[1].mode) - die("Path %s not in branch", s); - if (!*d) { /* C "path/to/subdir" "" */ - tree_content_replace(&b->branch_tree, - &leaf.versions[1].oid, - leaf.versions[1].mode, - leaf.tree); - return; - } - tree_content_set(&b->branch_tree, d, - &leaf.versions[1].oid, - leaf.versions[1].mode, - leaf.tree); -} - -static void note_change_n(const char *p, struct branch *b, unsigned char *old_fanout) -{ - static struct strbuf uq = STRBUF_INIT; - struct object_entry *oe; - struct branch *s; - struct object_id oid, commit_oid; - char path[GIT_MAX_RAWSZ * 3]; - uint16_t inline_data = 0; - unsigned char new_fanout; - - /* - * When loading a branch, we don't traverse its tree to count the real - * number of notes (too expensive to do this for all non-note refs). - * This means that recently loaded notes refs might incorrectly have - * b->num_notes == 0, and consequently, old_fanout might be wrong. - * - * Fix this by traversing the tree and counting the number of notes - * when b->num_notes == 0. If the notes tree is truly empty, the - * calculation should not take long. - */ - if (b->num_notes == 0 && *old_fanout == 0) { - /* Invoke change_note_fanout() in "counting mode". */ - b->num_notes = change_note_fanout(&b->branch_tree, 0xff); - *old_fanout = convert_num_notes_to_fanout(b->num_notes); - } - - /* Now parse the notemodify command. */ - /* or 'inline' */ - if (*p == ':') { - oe = find_mark(marks, parse_mark_ref_space(&p)); - oidcpy(&oid, &oe->idx.oid); - } else if (skip_prefix(p, "inline ", &p)) { - inline_data = 1; - oe = NULL; /* not used with inline_data, but makes gcc happy */ - } else { - if (parse_mapped_oid_hex(p, &oid, &p)) - die("Invalid dataref: %s", command_buf.buf); - oe = find_object(&oid); - if (*p++ != ' ') - die("Missing space after SHA1: %s", command_buf.buf); - } - - /* */ - s = lookup_branch(p); - if (s) { - if (is_null_oid(&s->oid)) - die("Can't add a note on empty branch."); - oidcpy(&commit_oid, &s->oid); - } else if (*p == ':') { - uintmax_t commit_mark = parse_mark_ref_eol(p); - struct object_entry *commit_oe = find_mark(marks, commit_mark); - if (commit_oe->type != OBJ_COMMIT) - die("Mark :%" PRIuMAX " not a commit", commit_mark); - oidcpy(&commit_oid, &commit_oe->idx.oid); - } else if (!get_oid(p, &commit_oid)) { - unsigned long size; - char *buf = read_object_with_reference(the_repository, - &commit_oid, - commit_type, &size, - &commit_oid); - if (!buf || size < the_hash_algo->hexsz + 6) - die("Not a valid commit: %s", p); - free(buf); - } else - die("Invalid ref name or SHA1 expression: %s", p); - - if (inline_data) { - if (p != uq.buf) { - strbuf_addstr(&uq, p); - p = uq.buf; - } - read_next_command(); - parse_and_store_blob(&last_blob, &oid, 0); - } else if (oe) { - if (oe->type != OBJ_BLOB) - die("Not a blob (actually a %s): %s", - type_name(oe->type), command_buf.buf); - } else if (!is_null_oid(&oid)) { - enum object_type type = oid_object_info(the_repository, &oid, - NULL); - if (type < 0) - die("Blob not found: %s", command_buf.buf); - if (type != OBJ_BLOB) - die("Not a blob (actually a %s): %s", - type_name(type), command_buf.buf); - } - - construct_path_with_fanout(oid_to_hex(&commit_oid), *old_fanout, path); - if (tree_content_remove(&b->branch_tree, path, NULL, 0)) - b->num_notes--; - - if (is_null_oid(&oid)) - return; /* nothing to insert */ - - b->num_notes++; - new_fanout = convert_num_notes_to_fanout(b->num_notes); - construct_path_with_fanout(oid_to_hex(&commit_oid), new_fanout, path); - tree_content_set(&b->branch_tree, path, &oid, S_IFREG | 0644, NULL); -} - -static void file_change_deleteall(struct branch *b) -{ - release_tree_content_recursive(b->branch_tree.tree); - oidclr(&b->branch_tree.versions[0].oid); - oidclr(&b->branch_tree.versions[1].oid); - load_tree(&b->branch_tree); - b->num_notes = 0; -} - -static void parse_from_commit(struct branch *b, char *buf, unsigned long size) -{ - if (!buf || size < the_hash_algo->hexsz + 6) - die("Not a valid commit: %s", oid_to_hex(&b->oid)); - if (memcmp("tree ", buf, 5) - || get_oid_hex(buf + 5, &b->branch_tree.versions[1].oid)) - die("The commit %s is corrupt", oid_to_hex(&b->oid)); - oidcpy(&b->branch_tree.versions[0].oid, - &b->branch_tree.versions[1].oid); -} - -static void parse_from_existing(struct branch *b) -{ - if (is_null_oid(&b->oid)) { - oidclr(&b->branch_tree.versions[0].oid); - oidclr(&b->branch_tree.versions[1].oid); - } else { - unsigned long size; - char *buf; - - buf = read_object_with_reference(the_repository, - &b->oid, commit_type, &size, - &b->oid); - parse_from_commit(b, buf, size); - free(buf); - } -} - -static int parse_objectish(struct branch *b, const char *objectish) -{ - struct branch *s; - struct object_id oid; - - oidcpy(&oid, &b->branch_tree.versions[1].oid); - - s = lookup_branch(objectish); - if (b == s) - die("Can't create a branch from itself: %s", b->name); - else if (s) { - struct object_id *t = &s->branch_tree.versions[1].oid; - oidcpy(&b->oid, &s->oid); - oidcpy(&b->branch_tree.versions[0].oid, t); - oidcpy(&b->branch_tree.versions[1].oid, t); - } else if (*objectish == ':') { - uintmax_t idnum = parse_mark_ref_eol(objectish); - struct object_entry *oe = find_mark(marks, idnum); - if (oe->type != OBJ_COMMIT) - die("Mark :%" PRIuMAX " not a commit", idnum); - if (!oideq(&b->oid, &oe->idx.oid)) { - oidcpy(&b->oid, &oe->idx.oid); - if (oe->pack_id != MAX_PACK_ID) { - unsigned long size; - char *buf = gfi_unpack_entry(oe, &size); - parse_from_commit(b, buf, size); - free(buf); - } else - parse_from_existing(b); - } - } else if (!get_oid(objectish, &b->oid)) { - parse_from_existing(b); - if (is_null_oid(&b->oid)) - b->delete = 1; - } - else - die("Invalid ref name or SHA1 expression: %s", objectish); - - if (b->branch_tree.tree && !oideq(&oid, &b->branch_tree.versions[1].oid)) { - release_tree_content_recursive(b->branch_tree.tree); - b->branch_tree.tree = NULL; - } - - read_next_command(); - return 1; -} - -static int parse_from(struct branch *b) -{ - const char *from; - - if (!skip_prefix(command_buf.buf, "from ", &from)) - return 0; - - return parse_objectish(b, from); -} - -static int parse_objectish_with_prefix(struct branch *b, const char *prefix) -{ - const char *base; - - if (!skip_prefix(command_buf.buf, prefix, &base)) - return 0; - - return parse_objectish(b, base); -} - -static struct hash_list *parse_merge(unsigned int *count) -{ - struct hash_list *list = NULL, **tail = &list, *n; - const char *from; - struct branch *s; - - *count = 0; - while (skip_prefix(command_buf.buf, "merge ", &from)) { - n = xmalloc(sizeof(*n)); - s = lookup_branch(from); - if (s) - oidcpy(&n->oid, &s->oid); - else if (*from == ':') { - uintmax_t idnum = parse_mark_ref_eol(from); - struct object_entry *oe = find_mark(marks, idnum); - if (oe->type != OBJ_COMMIT) - die("Mark :%" PRIuMAX " not a commit", idnum); - oidcpy(&n->oid, &oe->idx.oid); - } else if (!get_oid(from, &n->oid)) { - unsigned long size; - char *buf = read_object_with_reference(the_repository, - &n->oid, - commit_type, - &size, &n->oid); - if (!buf || size < the_hash_algo->hexsz + 6) - die("Not a valid commit: %s", from); - free(buf); - } else - die("Invalid ref name or SHA1 expression: %s", from); - - n->next = NULL; - *tail = n; - tail = &n->next; - - (*count)++; - read_next_command(); - } - return list; -} - -static void parse_new_commit(const char *arg) -{ - static struct strbuf msg = STRBUF_INIT; - struct branch *b; - char *author = NULL; - char *committer = NULL; - char *encoding = NULL; - struct hash_list *merge_list = NULL; - unsigned int merge_count; - unsigned char prev_fanout, new_fanout; - const char *v; - - b = lookup_branch(arg); - if (!b) - b = new_branch(arg); - - read_next_command(); - parse_mark(); - parse_original_identifier(); - if (skip_prefix(command_buf.buf, "author ", &v)) { - author = parse_ident(v); - read_next_command(); - } - if (skip_prefix(command_buf.buf, "committer ", &v)) { - committer = parse_ident(v); - read_next_command(); - } - if (!committer) - die("Expected committer but didn't get one"); - if (skip_prefix(command_buf.buf, "encoding ", &v)) { - encoding = xstrdup(v); - read_next_command(); - } - parse_data(&msg, 0, NULL); - read_next_command(); - parse_from(b); - merge_list = parse_merge(&merge_count); - - /* ensure the branch is active/loaded */ - if (!b->branch_tree.tree || !max_active_branches) { - unload_one_branch(); - load_branch(b); - } - - prev_fanout = convert_num_notes_to_fanout(b->num_notes); - - /* file_change* */ - while (command_buf.len > 0) { - if (skip_prefix(command_buf.buf, "M ", &v)) - file_change_m(v, b); - else if (skip_prefix(command_buf.buf, "D ", &v)) - file_change_d(v, b); - else if (skip_prefix(command_buf.buf, "R ", &v)) - file_change_cr(v, b, 1); - else if (skip_prefix(command_buf.buf, "C ", &v)) - file_change_cr(v, b, 0); - else if (skip_prefix(command_buf.buf, "N ", &v)) - note_change_n(v, b, &prev_fanout); - else if (!strcmp("deleteall", command_buf.buf)) - file_change_deleteall(b); - else if (skip_prefix(command_buf.buf, "ls ", &v)) - parse_ls(v, b); - else if (skip_prefix(command_buf.buf, "cat-blob ", &v)) - parse_cat_blob(v); - else { - unread_command_buf = 1; - break; - } - if (read_next_command() == EOF) - break; - } - - new_fanout = convert_num_notes_to_fanout(b->num_notes); - if (new_fanout != prev_fanout) - b->num_notes = change_note_fanout(&b->branch_tree, new_fanout); - - /* build the tree and the commit */ - store_tree(&b->branch_tree); - oidcpy(&b->branch_tree.versions[0].oid, - &b->branch_tree.versions[1].oid); - - strbuf_reset(&new_data); - strbuf_addf(&new_data, "tree %s\n", - oid_to_hex(&b->branch_tree.versions[1].oid)); - if (!is_null_oid(&b->oid)) - strbuf_addf(&new_data, "parent %s\n", - oid_to_hex(&b->oid)); - while (merge_list) { - struct hash_list *next = merge_list->next; - strbuf_addf(&new_data, "parent %s\n", - oid_to_hex(&merge_list->oid)); - free(merge_list); - merge_list = next; - } - strbuf_addf(&new_data, - "author %s\n" - "committer %s\n", - author ? author : committer, committer); - if (encoding) - strbuf_addf(&new_data, - "encoding %s\n", - encoding); - strbuf_addch(&new_data, '\n'); - strbuf_addbuf(&new_data, &msg); - free(author); - free(committer); - free(encoding); - - if (!store_object(OBJ_COMMIT, &new_data, NULL, &b->oid, next_mark)) - b->pack_id = pack_id; - b->last_commit = object_count_by_type[OBJ_COMMIT]; -} - -static void parse_new_tag(const char *arg) -{ - static struct strbuf msg = STRBUF_INIT; - const char *from; - char *tagger; - struct branch *s; - struct tag *t; - uintmax_t from_mark = 0; - struct object_id oid; - enum object_type type; - const char *v; - - t = mem_pool_alloc(&fi_mem_pool, sizeof(struct tag)); - memset(t, 0, sizeof(struct tag)); - t->name = pool_strdup(arg); - if (last_tag) - last_tag->next_tag = t; - else - first_tag = t; - last_tag = t; - read_next_command(); - parse_mark(); - - /* from ... */ - if (!skip_prefix(command_buf.buf, "from ", &from)) - die("Expected from command, got %s", command_buf.buf); - s = lookup_branch(from); - if (s) { - if (is_null_oid(&s->oid)) - die("Can't tag an empty branch."); - oidcpy(&oid, &s->oid); - type = OBJ_COMMIT; - } else if (*from == ':') { - struct object_entry *oe; - from_mark = parse_mark_ref_eol(from); - oe = find_mark(marks, from_mark); - type = oe->type; - oidcpy(&oid, &oe->idx.oid); - } else if (!get_oid(from, &oid)) { - struct object_entry *oe = find_object(&oid); - if (!oe) { - type = oid_object_info(the_repository, &oid, NULL); - if (type < 0) - die("Not a valid object: %s", from); - } else - type = oe->type; - } else - die("Invalid ref name or SHA1 expression: %s", from); - read_next_command(); - - /* original-oid ... */ - parse_original_identifier(); - - /* tagger ... */ - if (skip_prefix(command_buf.buf, "tagger ", &v)) { - tagger = parse_ident(v); - read_next_command(); - } else - tagger = NULL; - - /* tag payload/message */ - parse_data(&msg, 0, NULL); - - /* build the tag object */ - strbuf_reset(&new_data); - - strbuf_addf(&new_data, - "object %s\n" - "type %s\n" - "tag %s\n", - oid_to_hex(&oid), type_name(type), t->name); - if (tagger) - strbuf_addf(&new_data, - "tagger %s\n", tagger); - strbuf_addch(&new_data, '\n'); - strbuf_addbuf(&new_data, &msg); - free(tagger); - - if (store_object(OBJ_TAG, &new_data, NULL, &t->oid, next_mark)) - t->pack_id = MAX_PACK_ID; - else - t->pack_id = pack_id; -} - -static void parse_reset_branch(const char *arg) -{ - struct branch *b; - const char *tag_name; - - b = lookup_branch(arg); - if (b) { - oidclr(&b->oid); - oidclr(&b->branch_tree.versions[0].oid); - oidclr(&b->branch_tree.versions[1].oid); - if (b->branch_tree.tree) { - release_tree_content_recursive(b->branch_tree.tree); - b->branch_tree.tree = NULL; - } - } - else - b = new_branch(arg); - read_next_command(); - parse_from(b); - if (b->delete && skip_prefix(b->name, "refs/tags/", &tag_name)) { - /* - * Elsewhere, we call dump_branches() before dump_tags(), - * and dump_branches() will handle ref deletions first, so - * in order to make sure the deletion actually takes effect, - * we need to remove the tag from our list of tags to update. - * - * NEEDSWORK: replace list of tags with hashmap for faster - * deletion? - */ - struct tag *t, *prev = NULL; - for (t = first_tag; t; t = t->next_tag) { - if (!strcmp(t->name, tag_name)) - break; - prev = t; - } - if (t) { - if (prev) - prev->next_tag = t->next_tag; - else - first_tag = t->next_tag; - if (!t->next_tag) - last_tag = prev; - /* There is no mem_pool_free(t) function to call. */ - } - } - if (command_buf.len > 0) - unread_command_buf = 1; -} - -static void cat_blob_write(const char *buf, unsigned long size) -{ - if (write_in_full(cat_blob_fd, buf, size) < 0) - die_errno("Write to frontend failed"); -} - -static void cat_blob(struct object_entry *oe, struct object_id *oid) -{ - struct strbuf line = STRBUF_INIT; - unsigned long size; - enum object_type type = 0; - char *buf; - - if (!oe || oe->pack_id == MAX_PACK_ID) { - buf = read_object_file(oid, &type, &size); - } else { - type = oe->type; - buf = gfi_unpack_entry(oe, &size); - } - - /* - * Output based on batch_one_object() from cat-file.c. - */ - if (type <= 0) { - strbuf_reset(&line); - strbuf_addf(&line, "%s missing\n", oid_to_hex(oid)); - cat_blob_write(line.buf, line.len); - strbuf_release(&line); - free(buf); - return; - } - if (!buf) - die("Can't read object %s", oid_to_hex(oid)); - if (type != OBJ_BLOB) - die("Object %s is a %s but a blob was expected.", - oid_to_hex(oid), type_name(type)); - strbuf_reset(&line); - strbuf_addf(&line, "%s %s %"PRIuMAX"\n", oid_to_hex(oid), - type_name(type), (uintmax_t)size); - cat_blob_write(line.buf, line.len); - strbuf_release(&line); - cat_blob_write(buf, size); - cat_blob_write("\n", 1); - if (oe && oe->pack_id == pack_id) { - last_blob.offset = oe->idx.offset; - strbuf_attach(&last_blob.data, buf, size, size); - last_blob.depth = oe->depth; - } else - free(buf); -} - -static void parse_get_mark(const char *p) -{ - struct object_entry *oe; - char output[GIT_MAX_HEXSZ + 2]; - - /* get-mark SP LF */ - if (*p != ':') - die("Not a mark: %s", p); - - oe = find_mark(marks, parse_mark_ref_eol(p)); - if (!oe) - die("Unknown mark: %s", command_buf.buf); - - xsnprintf(output, sizeof(output), "%s\n", oid_to_hex(&oe->idx.oid)); - cat_blob_write(output, the_hash_algo->hexsz + 1); -} - -static void parse_cat_blob(const char *p) -{ - struct object_entry *oe; - struct object_id oid; - - /* cat-blob SP LF */ - if (*p == ':') { - oe = find_mark(marks, parse_mark_ref_eol(p)); - if (!oe) - die("Unknown mark: %s", command_buf.buf); - oidcpy(&oid, &oe->idx.oid); - } else { - if (parse_mapped_oid_hex(p, &oid, &p)) - die("Invalid dataref: %s", command_buf.buf); - if (*p) - die("Garbage after SHA1: %s", command_buf.buf); - oe = find_object(&oid); - } - - cat_blob(oe, &oid); -} - -static struct object_entry *dereference(struct object_entry *oe, - struct object_id *oid) -{ - unsigned long size; - char *buf = NULL; - const unsigned hexsz = the_hash_algo->hexsz; - - if (!oe) { - enum object_type type = oid_object_info(the_repository, oid, - NULL); - if (type < 0) - die("object not found: %s", oid_to_hex(oid)); - /* cache it! */ - oe = insert_object(oid); - oe->type = type; - oe->pack_id = MAX_PACK_ID; - oe->idx.offset = 1; - } - switch (oe->type) { - case OBJ_TREE: /* easy case. */ - return oe; - case OBJ_COMMIT: - case OBJ_TAG: - break; - default: - die("Not a tree-ish: %s", command_buf.buf); - } - - if (oe->pack_id != MAX_PACK_ID) { /* in a pack being written */ - buf = gfi_unpack_entry(oe, &size); - } else { - enum object_type unused; - buf = read_object_file(oid, &unused, &size); - } - if (!buf) - die("Can't load object %s", oid_to_hex(oid)); - - /* Peel one layer. */ - switch (oe->type) { - case OBJ_TAG: - if (size < hexsz + strlen("object ") || - get_oid_hex(buf + strlen("object "), oid)) - die("Invalid SHA1 in tag: %s", command_buf.buf); - break; - case OBJ_COMMIT: - if (size < hexsz + strlen("tree ") || - get_oid_hex(buf + strlen("tree "), oid)) - die("Invalid SHA1 in commit: %s", command_buf.buf); - } - - free(buf); - return find_object(oid); -} - -static void insert_mapped_mark(uintmax_t mark, void *object, void *cbp) -{ - struct object_id *fromoid = object; - struct object_id *tooid = find_mark(cbp, mark); - int ret; - khiter_t it; - - it = kh_put_oid_map(sub_oid_map, *fromoid, &ret); - /* We've already seen this object. */ - if (ret == 0) - return; - kh_value(sub_oid_map, it) = tooid; -} - -static void build_mark_map_one(struct mark_set *from, struct mark_set *to) -{ - for_each_mark(from, 0, insert_mapped_mark, to); -} - -static void build_mark_map(struct string_list *from, struct string_list *to) -{ - struct string_list_item *fromp, *top; - - sub_oid_map = kh_init_oid_map(); - - for_each_string_list_item(fromp, from) { - top = string_list_lookup(to, fromp->string); - if (!fromp->util) { - die(_("Missing from marks for submodule '%s'"), fromp->string); - } else if (!top || !top->util) { - die(_("Missing to marks for submodule '%s'"), fromp->string); - } - build_mark_map_one(fromp->util, top->util); - } -} - -static struct object_entry *parse_treeish_dataref(const char **p) -{ - struct object_id oid; - struct object_entry *e; - - if (**p == ':') { /* */ - e = find_mark(marks, parse_mark_ref_space(p)); - if (!e) - die("Unknown mark: %s", command_buf.buf); - oidcpy(&oid, &e->idx.oid); - } else { /* */ - if (parse_mapped_oid_hex(*p, &oid, p)) - die("Invalid dataref: %s", command_buf.buf); - e = find_object(&oid); - if (*(*p)++ != ' ') - die("Missing space after tree-ish: %s", command_buf.buf); - } - - while (!e || e->type != OBJ_TREE) - e = dereference(e, &oid); - return e; -} - -static void print_ls(int mode, const unsigned char *hash, const char *path) -{ - static struct strbuf line = STRBUF_INIT; - - /* See show_tree(). */ - const char *type = - S_ISGITLINK(mode) ? commit_type : - S_ISDIR(mode) ? tree_type : - blob_type; - - if (!mode) { - /* missing SP path LF */ - strbuf_reset(&line); - strbuf_addstr(&line, "missing "); - quote_c_style(path, &line, NULL, 0); - strbuf_addch(&line, '\n'); - } else { - /* mode SP type SP object_name TAB path LF */ - strbuf_reset(&line); - strbuf_addf(&line, "%06o %s %s\t", - mode & ~NO_DELTA, type, hash_to_hex(hash)); - quote_c_style(path, &line, NULL, 0); - strbuf_addch(&line, '\n'); - } - cat_blob_write(line.buf, line.len); -} - -static void parse_ls(const char *p, struct branch *b) -{ - struct tree_entry *root = NULL; - struct tree_entry leaf = {NULL}; - - /* ls SP ( SP)? */ - if (*p == '"') { - if (!b) - die("Not in a commit: %s", command_buf.buf); - root = &b->branch_tree; - } else { - struct object_entry *e = parse_treeish_dataref(&p); - root = new_tree_entry(); - oidcpy(&root->versions[1].oid, &e->idx.oid); - if (!is_null_oid(&root->versions[1].oid)) - root->versions[1].mode = S_IFDIR; - load_tree(root); - } - if (*p == '"') { - static struct strbuf uq = STRBUF_INIT; - const char *endp; - strbuf_reset(&uq); - if (unquote_c_style(&uq, p, &endp)) - die("Invalid path: %s", command_buf.buf); - if (*endp) - die("Garbage after path in: %s", command_buf.buf); - p = uq.buf; - } - tree_content_get(root, p, &leaf, 1); - /* - * A directory in preparation would have a sha1 of zero - * until it is saved. Save, for simplicity. - */ - if (S_ISDIR(leaf.versions[1].mode)) - store_tree(&leaf); - - print_ls(leaf.versions[1].mode, leaf.versions[1].oid.hash, p); - if (leaf.tree) - release_tree_content_recursive(leaf.tree); - if (!b || root != &b->branch_tree) - release_tree_entry(root); -} - -static void checkpoint(void) -{ - checkpoint_requested = 0; - if (object_count) { - cycle_packfile(); - } - dump_branches(); - dump_tags(); - dump_marks(); -} - -static void parse_checkpoint(void) -{ - checkpoint_requested = 1; - skip_optional_lf(); -} - -static void parse_progress(void) -{ - fwrite(command_buf.buf, 1, command_buf.len, stdout); - fputc('\n', stdout); - fflush(stdout); - skip_optional_lf(); -} - -static void parse_alias(void) -{ - struct object_entry *e; - struct branch b; - - skip_optional_lf(); - read_next_command(); - - /* mark ... */ - parse_mark(); - if (!next_mark) - die(_("Expected 'mark' command, got %s"), command_buf.buf); - - /* to ... */ - memset(&b, 0, sizeof(b)); - if (!parse_objectish_with_prefix(&b, "to ")) - die(_("Expected 'to' command, got %s"), command_buf.buf); - e = find_object(&b.oid); - assert(e); - insert_mark(marks, next_mark, e); -} - -static char* make_fast_import_path(const char *path) -{ - if (!relative_marks_paths || is_absolute_path(path)) - return xstrdup(path); - return git_pathdup("info/fast-import/%s", path); -} - -static void option_import_marks(const char *marks, - int from_stream, int ignore_missing) -{ - if (import_marks_file) { - if (from_stream) - die("Only one import-marks command allowed per stream"); - - /* read previous mark file */ - if(!import_marks_file_from_stream) - read_marks(); - } - - import_marks_file = make_fast_import_path(marks); - import_marks_file_from_stream = from_stream; - import_marks_file_ignore_missing = ignore_missing; -} - -static void option_date_format(const char *fmt) -{ - if (!strcmp(fmt, "raw")) - whenspec = WHENSPEC_RAW; - else if (!strcmp(fmt, "raw-permissive")) - whenspec = WHENSPEC_RAW_PERMISSIVE; - else if (!strcmp(fmt, "rfc2822")) - whenspec = WHENSPEC_RFC2822; - else if (!strcmp(fmt, "now")) - whenspec = WHENSPEC_NOW; - else - die("unknown --date-format argument %s", fmt); -} - -static unsigned long ulong_arg(const char *option, const char *arg) -{ - char *endptr; - unsigned long rv = strtoul(arg, &endptr, 0); - if (strchr(arg, '-') || endptr == arg || *endptr) - die("%s: argument must be a non-negative integer", option); - return rv; -} - -static void option_depth(const char *depth) -{ - max_depth = ulong_arg("--depth", depth); - if (max_depth > MAX_DEPTH) - die("--depth cannot exceed %u", MAX_DEPTH); -} - -static void option_active_branches(const char *branches) -{ - max_active_branches = ulong_arg("--active-branches", branches); -} - -static void option_export_marks(const char *marks) -{ - export_marks_file = make_fast_import_path(marks); -} - -static void option_cat_blob_fd(const char *fd) -{ - unsigned long n = ulong_arg("--cat-blob-fd", fd); - if (n > (unsigned long) INT_MAX) - die("--cat-blob-fd cannot exceed %d", INT_MAX); - cat_blob_fd = (int) n; -} - -static void option_export_pack_edges(const char *edges) -{ - if (pack_edges) - fclose(pack_edges); - pack_edges = xfopen(edges, "a"); -} - -static void option_rewrite_submodules(const char *arg, struct string_list *list) -{ - struct mark_set *ms; - FILE *fp; - char *s = xstrdup(arg); - char *f = strchr(s, ':'); - if (!f) - die(_("Expected format name:filename for submodule rewrite option")); - *f = '\0'; - f++; - ms = xcalloc(1, sizeof(*ms)); - string_list_insert(list, s)->util = ms; - - fp = fopen(f, "r"); - if (!fp) - die_errno("cannot read '%s'", f); - read_mark_file(ms, fp, insert_oid_entry); - fclose(fp); -} - -static int parse_one_option(const char *option) -{ - if (skip_prefix(option, "max-pack-size=", &option)) { - unsigned long v; - if (!git_parse_ulong(option, &v)) - return 0; - if (v < 8192) { - warning("max-pack-size is now in bytes, assuming --max-pack-size=%lum", v); - v *= 1024 * 1024; - } else if (v < 1024 * 1024) { - warning("minimum max-pack-size is 1 MiB"); - v = 1024 * 1024; - } - max_packsize = v; - } else if (skip_prefix(option, "big-file-threshold=", &option)) { - unsigned long v; - if (!git_parse_ulong(option, &v)) - return 0; - big_file_threshold = v; - } else if (skip_prefix(option, "depth=", &option)) { - option_depth(option); - } else if (skip_prefix(option, "active-branches=", &option)) { - option_active_branches(option); - } else if (skip_prefix(option, "export-pack-edges=", &option)) { - option_export_pack_edges(option); - } else if (!strcmp(option, "quiet")) { - show_stats = 0; - } else if (!strcmp(option, "stats")) { - show_stats = 1; - } else if (!strcmp(option, "allow-unsafe-features")) { - ; /* already handled during early option parsing */ - } else { - return 0; - } - - return 1; -} - -static void check_unsafe_feature(const char *feature, int from_stream) -{ - if (from_stream && !allow_unsafe_features) - die(_("feature '%s' forbidden in input without --allow-unsafe-features"), - feature); -} - -static int parse_one_feature(const char *feature, int from_stream) -{ - const char *arg; - - if (skip_prefix(feature, "date-format=", &arg)) { - option_date_format(arg); - } else if (skip_prefix(feature, "import-marks=", &arg)) { - check_unsafe_feature("import-marks", from_stream); - option_import_marks(arg, from_stream, 0); - } else if (skip_prefix(feature, "import-marks-if-exists=", &arg)) { - check_unsafe_feature("import-marks-if-exists", from_stream); - option_import_marks(arg, from_stream, 1); - } else if (skip_prefix(feature, "export-marks=", &arg)) { - check_unsafe_feature(feature, from_stream); - option_export_marks(arg); - } else if (!strcmp(feature, "alias")) { - ; /* Don't die - this feature is supported */ - } else if (skip_prefix(feature, "rewrite-submodules-to=", &arg)) { - option_rewrite_submodules(arg, &sub_marks_to); - } else if (skip_prefix(feature, "rewrite-submodules-from=", &arg)) { - option_rewrite_submodules(arg, &sub_marks_from); - } else if (skip_prefix(feature, "rewrite-submodules-from=", &arg)) { - } else if (!strcmp(feature, "get-mark")) { - ; /* Don't die - this feature is supported */ - } else if (!strcmp(feature, "cat-blob")) { - ; /* Don't die - this feature is supported */ - } else if (!strcmp(feature, "relative-marks")) { - relative_marks_paths = 1; - } else if (!strcmp(feature, "no-relative-marks")) { - relative_marks_paths = 0; - } else if (!strcmp(feature, "done")) { - require_explicit_termination = 1; - } else if (!strcmp(feature, "force")) { - force_update = 1; - } else if (!strcmp(feature, "notes") || !strcmp(feature, "ls")) { - ; /* do nothing; we have the feature */ - } else { - return 0; - } - - return 1; -} - -static void parse_feature(const char *feature) -{ - if (seen_data_command) - die("Got feature command '%s' after data command", feature); - - if (parse_one_feature(feature, 1)) - return; - - die("This version of fast-import does not support feature %s.", feature); -} - -static void parse_option(const char *option) -{ - if (seen_data_command) - die("Got option command '%s' after data command", option); - - if (parse_one_option(option)) - return; - - die("This version of fast-import does not support option: %s", option); -} - -static void git_pack_config(void) -{ - int indexversion_value; - int limit; - unsigned long packsizelimit_value; - - if (!git_config_get_ulong("pack.depth", &max_depth)) { - if (max_depth > MAX_DEPTH) - max_depth = MAX_DEPTH; - } - if (!git_config_get_int("pack.indexversion", &indexversion_value)) { - pack_idx_opts.version = indexversion_value; - if (pack_idx_opts.version > 2) - git_die_config("pack.indexversion", - "bad pack.indexversion=%"PRIu32, pack_idx_opts.version); - } - if (!git_config_get_ulong("pack.packsizelimit", &packsizelimit_value)) - max_packsize = packsizelimit_value; - - if (!git_config_get_int("fastimport.unpacklimit", &limit)) - unpack_limit = limit; - else if (!git_config_get_int("transfer.unpacklimit", &limit)) - unpack_limit = limit; - - git_config(git_default_config, NULL); -} - -static const char fast_import_usage[] = -"git fast-import [--date-format=] [--max-pack-size=] [--big-file-threshold=] [--depth=] [--active-branches=] [--export-marks=]"; - -static void parse_argv(void) -{ - unsigned int i; - - for (i = 1; i < global_argc; i++) { - const char *a = global_argv[i]; - - if (*a != '-' || !strcmp(a, "--")) - break; - - if (!skip_prefix(a, "--", &a)) - die("unknown option %s", a); - - if (parse_one_option(a)) - continue; - - if (parse_one_feature(a, 0)) - continue; - - if (skip_prefix(a, "cat-blob-fd=", &a)) { - option_cat_blob_fd(a); - continue; - } - - die("unknown option --%s", a); - } - if (i != global_argc) - usage(fast_import_usage); - - seen_data_command = 1; - if (import_marks_file) - read_marks(); - build_mark_map(&sub_marks_from, &sub_marks_to); -} - -int cmd_main(int argc, const char **argv) -{ - unsigned int i; - - if (argc == 2 && !strcmp(argv[1], "-h")) - usage(fast_import_usage); - - setup_git_directory(); - reset_pack_idx_option(&pack_idx_opts); - git_pack_config(); - - alloc_objects(object_entry_alloc); - strbuf_init(&command_buf, 0); - atom_table = xcalloc(atom_table_sz, sizeof(struct atom_str*)); - branch_table = xcalloc(branch_table_sz, sizeof(struct branch*)); - avail_tree_table = xcalloc(avail_tree_table_sz, sizeof(struct avail_tree_content*)); - marks = mem_pool_calloc(&fi_mem_pool, 1, sizeof(struct mark_set)); - - hashmap_init(&object_table, object_entry_hashcmp, NULL, 0); - - /* - * We don't parse most options until after we've seen the set of - * "feature" lines at the start of the stream (which allows the command - * line to override stream data). But we must do an early parse of any - * command-line options that impact how we interpret the feature lines. - */ - for (i = 1; i < argc; i++) { - const char *arg = argv[i]; - if (*arg != '-' || !strcmp(arg, "--")) - break; - if (!strcmp(arg, "--allow-unsafe-features")) - allow_unsafe_features = 1; - } - - global_argc = argc; - global_argv = argv; - - rc_free = mem_pool_alloc(&fi_mem_pool, cmd_save * sizeof(*rc_free)); - for (i = 0; i < (cmd_save - 1); i++) - rc_free[i].next = &rc_free[i + 1]; - rc_free[cmd_save - 1].next = NULL; - - start_packfile(); - set_die_routine(die_nicely); - set_checkpoint_signal(); - while (read_next_command() != EOF) { - const char *v; - if (!strcmp("blob", command_buf.buf)) - parse_new_blob(); - else if (skip_prefix(command_buf.buf, "commit ", &v)) - parse_new_commit(v); - else if (skip_prefix(command_buf.buf, "tag ", &v)) - parse_new_tag(v); - else if (skip_prefix(command_buf.buf, "reset ", &v)) - parse_reset_branch(v); - else if (skip_prefix(command_buf.buf, "ls ", &v)) - parse_ls(v, NULL); - else if (skip_prefix(command_buf.buf, "cat-blob ", &v)) - parse_cat_blob(v); - else if (skip_prefix(command_buf.buf, "get-mark ", &v)) - parse_get_mark(v); - else if (!strcmp("checkpoint", command_buf.buf)) - parse_checkpoint(); - else if (!strcmp("done", command_buf.buf)) - break; - else if (!strcmp("alias", command_buf.buf)) - parse_alias(); - else if (starts_with(command_buf.buf, "progress ")) - parse_progress(); - else if (skip_prefix(command_buf.buf, "feature ", &v)) - parse_feature(v); - else if (skip_prefix(command_buf.buf, "option git ", &v)) - parse_option(v); - else if (starts_with(command_buf.buf, "option ")) - /* ignore non-git options*/; - else - die("Unsupported command: %s", command_buf.buf); - - if (checkpoint_requested) - checkpoint(); - } - - /* argv hasn't been parsed yet, do so */ - if (!seen_data_command) - parse_argv(); - - if (require_explicit_termination && feof(stdin)) - die("stream ends early"); - - end_packfile(); - - dump_branches(); - dump_tags(); - unkeep_all_packs(); - dump_marks(); - - if (pack_edges) - fclose(pack_edges); - - if (show_stats) { - uintmax_t total_count = 0, duplicate_count = 0; - for (i = 0; i < ARRAY_SIZE(object_count_by_type); i++) - total_count += object_count_by_type[i]; - for (i = 0; i < ARRAY_SIZE(duplicate_count_by_type); i++) - duplicate_count += duplicate_count_by_type[i]; - - fprintf(stderr, "%s statistics:\n", argv[0]); - fprintf(stderr, "---------------------------------------------------------------------\n"); - fprintf(stderr, "Alloc'd objects: %10" PRIuMAX "\n", alloc_count); - fprintf(stderr, "Total objects: %10" PRIuMAX " (%10" PRIuMAX " duplicates )\n", total_count, duplicate_count); - fprintf(stderr, " blobs : %10" PRIuMAX " (%10" PRIuMAX " duplicates %10" PRIuMAX " deltas of %10" PRIuMAX" attempts)\n", object_count_by_type[OBJ_BLOB], duplicate_count_by_type[OBJ_BLOB], delta_count_by_type[OBJ_BLOB], delta_count_attempts_by_type[OBJ_BLOB]); - fprintf(stderr, " trees : %10" PRIuMAX " (%10" PRIuMAX " duplicates %10" PRIuMAX " deltas of %10" PRIuMAX" attempts)\n", object_count_by_type[OBJ_TREE], duplicate_count_by_type[OBJ_TREE], delta_count_by_type[OBJ_TREE], delta_count_attempts_by_type[OBJ_TREE]); - fprintf(stderr, " commits: %10" PRIuMAX " (%10" PRIuMAX " duplicates %10" PRIuMAX " deltas of %10" PRIuMAX" attempts)\n", object_count_by_type[OBJ_COMMIT], duplicate_count_by_type[OBJ_COMMIT], delta_count_by_type[OBJ_COMMIT], delta_count_attempts_by_type[OBJ_COMMIT]); - fprintf(stderr, " tags : %10" PRIuMAX " (%10" PRIuMAX " duplicates %10" PRIuMAX " deltas of %10" PRIuMAX" attempts)\n", object_count_by_type[OBJ_TAG], duplicate_count_by_type[OBJ_TAG], delta_count_by_type[OBJ_TAG], delta_count_attempts_by_type[OBJ_TAG]); - fprintf(stderr, "Total branches: %10lu (%10lu loads )\n", branch_count, branch_load_count); - fprintf(stderr, " marks: %10" PRIuMAX " (%10" PRIuMAX " unique )\n", (((uintmax_t)1) << marks->shift) * 1024, marks_set_count); - fprintf(stderr, " atoms: %10u\n", atom_cnt); - fprintf(stderr, "Memory total: %10" PRIuMAX " KiB\n", (tree_entry_allocd + fi_mem_pool.pool_alloc + alloc_count*sizeof(struct object_entry))/1024); - fprintf(stderr, " pools: %10lu KiB\n", (unsigned long)((tree_entry_allocd + fi_mem_pool.pool_alloc) /1024)); - fprintf(stderr, " objects: %10" PRIuMAX " KiB\n", (alloc_count*sizeof(struct object_entry))/1024); - fprintf(stderr, "---------------------------------------------------------------------\n"); - pack_report(); - fprintf(stderr, "---------------------------------------------------------------------\n"); - fprintf(stderr, "\n"); - } - - return failure ? 1 : 0; -} diff --git a/git.c b/git.c index bf790e7f4f..01c456edce 100644 --- a/git.c +++ b/git.c @@ -511,6 +511,7 @@ static struct cmd_struct commands[] = { { "difftool", cmd_difftool, RUN_SETUP_GENTLY }, { "env--helper", cmd_env__helper }, { "fast-export", cmd_fast_export, RUN_SETUP }, + { "fast-import", cmd_fast_import, RUN_SETUP | NO_PARSEOPT }, { "fetch", cmd_fetch, RUN_SETUP }, { "fetch-pack", cmd_fetch_pack, RUN_SETUP | NO_PARSEOPT }, { "fmt-merge-msg", cmd_fmt_merge_msg, RUN_SETUP }, -- cgit v1.3 From fc47391e2478b26d33cae033f13405b6ee6af97b Mon Sep 17 00:00:00 2001 From: Jeff King Date: Thu, 13 Aug 2020 11:00:17 -0400 Subject: drop vcs-svn experiment The code in vcs-svn was started in 2010 as an attempt to build a remote-helper for interacting with svn repositories (as opposed to git-svn). However, we never got as far as shipping a mature remote helper, and the last substantive commit was e99d012a6bc in 2012. We do have a git-remote-testsvn, and it is even installed as part of "make install". But given the name, it seems unlikely to be used by anybody (you'd have to explicitly "git clone testsvn::$url", and there have been zero mentions of that on the mailing list since 2013, and even that includes the phrase "you might need to hack a bit to get it working properly"[1]). We also ship contrib/svn-fe, which builds on the vcs-svn work. However, it does not seem to build out of the box for me, as the link step misses some required libraries for using libgit.a. Curiously, the original build breakage bisects for me to eff80a9fd9 (Allow custom "comment char", 2013-01-16), which seems unrelated. There was an attempt to fix it in da011cb0e7 (contrib/svn-fe: fix Makefile, 2014-08-28), but on my system that only switches the error message. So it seems like the result is not really usable by anybody in practice. It would be wonderful if somebody wanted to pick up the topic again, and potentially it's worth carrying around for that reason. But the flip side is that people doing tree-wide operations have to deal with this code. And you can see the list with (replace "HEAD" with this commit as appropriate): { echo "--" git diff-tree --diff-filter=D -r --name-only HEAD^ HEAD } | git log --no-merges --oneline e99d012a6bc.. --stdin which shows 58 times somebody had to deal with the code, generally due to a compile or test failure, or a tree-wide style fix or API change. Let's drop it and let anybody who wants to pick it up do so by resurrecting it from the git history. As a bonus, this also reduces the size of a stripped installation of Git from 21MB to 19MB. [1] https://lore.kernel.org/git/CALkWK0mPHzKfzFKKpZkfAus3YVC9NFYDbFnt+5JQYVKipk3bQQ@mail.gmail.com/ Signed-off-by: Jeff King Signed-off-by: Junio C Hamano --- .gitignore | 1 - Makefile | 26 +- contrib/buildsystems/CMakeLists.txt | 26 +- contrib/svn-fe/.gitignore | 4 - contrib/svn-fe/Makefile | 105 ---- contrib/svn-fe/svn-fe.c | 18 - contrib/svn-fe/svn-fe.txt | 71 --- contrib/svn-fe/svnrdump_sim.py | 68 --- remote-testsvn.c | 341 ----------- t/helper/.gitignore | 2 - t/helper/test-line-buffer.c | 81 --- t/helper/test-svn-fe.c | 52 -- t/t0081-line-buffer.sh | 90 --- t/t9010-svn-fe.sh | 1105 ----------------------------------- t/t9011-svn-da.sh | 248 -------- t/t9020-remote-svn.sh | 95 --- t/test-lib-functions.sh | 2 +- vcs-svn/LICENSE | 32 - vcs-svn/fast_export.c | 365 ------------ vcs-svn/line_buffer.c | 126 ---- vcs-svn/line_buffer.txt | 77 --- vcs-svn/sliding_window.c | 79 --- vcs-svn/svndiff.c | 309 ---------- vcs-svn/svndump.c | 540 ----------------- 24 files changed, 7 insertions(+), 3856 deletions(-) delete mode 100644 contrib/svn-fe/.gitignore delete mode 100644 contrib/svn-fe/Makefile delete mode 100644 contrib/svn-fe/svn-fe.c delete mode 100644 contrib/svn-fe/svn-fe.txt delete mode 100755 contrib/svn-fe/svnrdump_sim.py delete mode 100644 remote-testsvn.c delete mode 100644 t/helper/test-line-buffer.c delete mode 100644 t/helper/test-svn-fe.c delete mode 100755 t/t0081-line-buffer.sh delete mode 100755 t/t9010-svn-fe.sh delete mode 100755 t/t9011-svn-da.sh delete mode 100755 t/t9020-remote-svn.sh delete mode 100644 vcs-svn/LICENSE delete mode 100644 vcs-svn/fast_export.c delete mode 100644 vcs-svn/line_buffer.c delete mode 100644 vcs-svn/line_buffer.txt delete mode 100644 vcs-svn/sliding_window.c delete mode 100644 vcs-svn/svndiff.c delete mode 100644 vcs-svn/svndump.c diff --git a/.gitignore b/.gitignore index ee509a2ad2..9673e792db 100644 --- a/.gitignore +++ b/.gitignore @@ -134,7 +134,6 @@ /git-remote-fd /git-remote-ext /git-remote-testpy -/git-remote-testsvn /git-repack /git-replace /git-request-pull diff --git a/Makefile b/Makefile index 2b821fc762..314179d918 100644 --- a/Makefile +++ b/Makefile @@ -569,7 +569,6 @@ BUILT_INS = COMPAT_CFLAGS = COMPAT_OBJS = XDIFF_OBJS = -VCSSVN_OBJS = GENERATED_H = EXTRA_CPPFLAGS = FUZZ_OBJS = @@ -674,7 +673,6 @@ PROGRAMS += $(EXTRA_PROGRAMS) PROGRAM_OBJS += daemon.o PROGRAM_OBJS += http-backend.o PROGRAM_OBJS += imap-send.o -PROGRAM_OBJS += remote-testsvn.o PROGRAM_OBJS += sh-i18n--envsubst.o PROGRAM_OBJS += shell.o @@ -746,8 +744,6 @@ TEST_BUILTINS_OBJS += test-xml-encode.o # Do not add more tests here unless they have extra dependencies. Add # them in TEST_BUILTINS_OBJS above. TEST_PROGRAMS_NEED_X += test-fake-ssh -TEST_PROGRAMS_NEED_X += test-line-buffer -TEST_PROGRAMS_NEED_X += test-svn-fe TEST_PROGRAMS_NEED_X += test-tool TEST_PROGRAMS = $(patsubst %,t/helper/%$X,$(TEST_PROGRAMS_NEED_X)) @@ -803,7 +799,6 @@ TEST_SHELL_PATH = $(SHELL_PATH) LIB_FILE = libgit.a XDIFF_LIB = xdiff/lib.a -VCSSVN_LIB = vcs-svn/lib.a GENERATED_H += config-list.h GENERATED_H += command-list.h @@ -2345,16 +2340,9 @@ XDIFF_OBJS += xdiff/xpatience.o XDIFF_OBJS += xdiff/xprepare.o XDIFF_OBJS += xdiff/xutils.o -VCSSVN_OBJS += vcs-svn/fast_export.o -VCSSVN_OBJS += vcs-svn/line_buffer.o -VCSSVN_OBJS += vcs-svn/sliding_window.o -VCSSVN_OBJS += vcs-svn/svndiff.o -VCSSVN_OBJS += vcs-svn/svndump.o - TEST_OBJS := $(patsubst %$X,%.o,$(TEST_PROGRAMS)) $(patsubst %,t/helper/%,$(TEST_BUILTINS_OBJS)) OBJECTS := $(LIB_OBJS) $(BUILTIN_OBJS) $(PROGRAM_OBJS) $(TEST_OBJS) \ $(XDIFF_OBJS) \ - $(VCSSVN_OBJS) \ $(FUZZ_OBJS) \ common-main.o \ git.o @@ -2469,10 +2457,6 @@ git-http-push$X: http.o http-push.o GIT-LDFLAGS $(GITLIBS) $(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) \ $(CURL_LIBCURL) $(EXPAT_LIBEXPAT) $(LIBS) -git-remote-testsvn$X: remote-testsvn.o GIT-LDFLAGS $(GITLIBS) $(VCSSVN_LIB) - $(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) $(LIBS) \ - $(VCSSVN_LIB) - $(REMOTE_CURL_ALIASES): $(REMOTE_CURL_PRIMARY) $(QUIET_LNCP)$(RM) $@ && \ ln $< $@ 2>/dev/null || \ @@ -2489,9 +2473,6 @@ $(LIB_FILE): $(LIB_OBJS) $(XDIFF_LIB): $(XDIFF_OBJS) $(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^ -$(VCSSVN_LIB): $(VCSSVN_OBJS) - $(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^ - export DEFAULT_EDITOR DEFAULT_PAGER Documentation/GIT-EXCLUDED-PROGRAMS: FORCE @@ -2766,10 +2747,6 @@ perf: all .PHONY: test perf -t/helper/test-line-buffer$X: $(VCSSVN_LIB) - -t/helper/test-svn-fe$X: $(VCSSVN_LIB) - .PRECIOUS: $(TEST_OBJS) t/helper/test-tool$X: $(patsubst %,t/helper/%,$(TEST_BUILTINS_OBJS)) @@ -2902,7 +2879,6 @@ ifdef MSVC $(INSTALL) git-http-push.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' $(INSTALL) git-imap-send.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' $(INSTALL) git-remote-http.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' - $(INSTALL) git-remote-testsvn.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' $(INSTALL) git-sh-i18n--envsubst.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' ifndef DEBUG $(INSTALL) $(vcpkg_rel_bin)/*.dll '$(DESTDIR_SQ)$(bindir_SQ)' @@ -3103,7 +3079,7 @@ cocciclean: clean: profile-clean coverage-clean cocciclean $(RM) *.res $(RM) $(OBJECTS) - $(RM) $(LIB_FILE) $(XDIFF_LIB) $(VCSSVN_LIB) + $(RM) $(LIB_FILE) $(XDIFF_LIB) $(RM) $(ALL_PROGRAMS) $(SCRIPT_LIB) $(BUILT_INS) git$X $(RM) $(TEST_PROGRAMS) $(RM) $(FUZZ_PROGRAMS) diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt index 4a6f135b16..5007f173f1 100644 --- a/contrib/buildsystems/CMakeLists.txt +++ b/contrib/buildsystems/CMakeLists.txt @@ -502,7 +502,7 @@ unset(CMAKE_REQUIRED_INCLUDES) #programs set(PROGRAMS_BUILT git git-daemon git-http-backend git-sh-i18n--envsubst - git-shell git-remote-testsvn) + git-shell) if(NOT CURL_FOUND) list(APPEND excluded_progs git-http-fetch git-http-push) @@ -568,12 +568,6 @@ parse_makefile_for_sources(libxdiff_SOURCES "XDIFF_OBJS") list(TRANSFORM libxdiff_SOURCES PREPEND "${CMAKE_SOURCE_DIR}/") add_library(xdiff STATIC ${libxdiff_SOURCES}) -#libvcs-svn -parse_makefile_for_sources(libvcs-svn_SOURCES "VCSSVN_OBJS") - -list(TRANSFORM libvcs-svn_SOURCES PREPEND "${CMAKE_SOURCE_DIR}/") -add_library(vcs-svn STATIC ${libvcs-svn_SOURCES}) - if(WIN32) if(NOT MSVC)#use windres when compiling with gcc and clang add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/git.res @@ -654,9 +648,6 @@ if(CURL_FOUND) endif() endif() -add_executable(git-remote-testsvn ${CMAKE_SOURCE_DIR}/remote-testsvn.c) -target_link_libraries(git-remote-testsvn common-main vcs-svn) - set(git_builtin_extra cherry cherry-pick format-patch fsck-objects init merge-subtree restore show @@ -832,12 +823,6 @@ if(BUILD_TESTING) add_executable(test-fake-ssh ${CMAKE_SOURCE_DIR}/t/helper/test-fake-ssh.c) target_link_libraries(test-fake-ssh common-main) -add_executable(test-line-buffer ${CMAKE_SOURCE_DIR}/t/helper/test-line-buffer.c) -target_link_libraries(test-line-buffer common-main vcs-svn) - -add_executable(test-svn-fe ${CMAKE_SOURCE_DIR}/t/helper/test-svn-fe.c) -target_link_libraries(test-svn-fe common-main vcs-svn) - #test-tool parse_makefile_for_sources(test-tool_SOURCES "TEST_BUILTINS_OBJS") @@ -845,13 +830,13 @@ list(TRANSFORM test-tool_SOURCES PREPEND "${CMAKE_SOURCE_DIR}/t/helper/") add_executable(test-tool ${CMAKE_SOURCE_DIR}/t/helper/test-tool.c ${test-tool_SOURCES}) target_link_libraries(test-tool common-main) -set_target_properties(test-fake-ssh test-line-buffer test-svn-fe test-tool +set_target_properties(test-fake-ssh test-tool PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/t/helper) if(MSVC) - set_target_properties(test-fake-ssh test-line-buffer test-svn-fe test-tool + set_target_properties(test-fake-ssh test-tool PROPERTIES RUNTIME_OUTPUT_DIRECTORY_DEBUG ${CMAKE_BINARY_DIR}/t/helper) - set_target_properties(test-fake-ssh test-line-buffer test-svn-fe test-tool + set_target_properties(test-fake-ssh test-tool PROPERTIES RUNTIME_OUTPUT_DIRECTORY_RELEASE ${CMAKE_BINARY_DIR}/t/helper) endif() @@ -860,7 +845,7 @@ set(wrapper_scripts git git-upload-pack git-receive-pack git-upload-archive git-shell git-remote-ext) set(wrapper_test_scripts - test-fake-ssh test-line-buffer test-svn-fe test-tool) + test-fake-ssh test-tool) foreach(script ${wrapper_scripts}) @@ -961,7 +946,6 @@ if(NOT ${CMAKE_BINARY_DIR}/CMakeCache.txt STREQUAL ${CACHE_PATH}) file(COPY ${CMAKE_SOURCE_DIR}/mergetools/tkdiff DESTINATION ${CMAKE_BINARY_DIR}/mergetools/) file(COPY ${CMAKE_SOURCE_DIR}/contrib/completion/git-prompt.sh DESTINATION ${CMAKE_BINARY_DIR}/contrib/completion/) file(COPY ${CMAKE_SOURCE_DIR}/contrib/completion/git-completion.bash DESTINATION ${CMAKE_BINARY_DIR}/contrib/completion/) - file(COPY ${CMAKE_SOURCE_DIR}/contrib/svn-fe/svnrdump_sim.py DESTINATION ${CMAKE_BINARY_DIR}/contrib/svn-fe/) endif() file(GLOB test_scipts "${CMAKE_SOURCE_DIR}/t/t[0-9]*.sh") diff --git a/contrib/svn-fe/.gitignore b/contrib/svn-fe/.gitignore deleted file mode 100644 index 02a7791585..0000000000 --- a/contrib/svn-fe/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -/*.xml -/*.1 -/*.html -/svn-fe diff --git a/contrib/svn-fe/Makefile b/contrib/svn-fe/Makefile deleted file mode 100644 index e8651aaf4b..0000000000 --- a/contrib/svn-fe/Makefile +++ /dev/null @@ -1,105 +0,0 @@ -all:: svn-fe$X - -CC = cc -RM = rm -f -MV = mv - -CFLAGS = -g -O2 -Wall -LDFLAGS = -EXTLIBS = -lz - -include ../../config.mak.uname --include ../../config.mak.autogen --include ../../config.mak - -ifeq ($(uname_S),Darwin) - ifndef NO_FINK - ifeq ($(shell test -d /sw/lib && echo y),y) - CFLAGS += -I/sw/include - LDFLAGS += -L/sw/lib - endif - endif - ifndef NO_DARWIN_PORTS - ifeq ($(shell test -d /opt/local/lib && echo y),y) - CFLAGS += -I/opt/local/include - LDFLAGS += -L/opt/local/lib - endif - endif -endif - -ifndef NO_OPENSSL - EXTLIBS += -lssl - ifdef NEEDS_CRYPTO_WITH_SSL - EXTLIBS += -lcrypto - endif -endif - -ifndef NO_PTHREADS - CFLAGS += $(PTHREADS_CFLAGS) - EXTLIBS += $(PTHREAD_LIBS) -endif - -ifdef HAVE_CLOCK_GETTIME - CFLAGS += -DHAVE_CLOCK_GETTIME - EXTLIBS += -lrt -endif - -ifdef NEEDS_LIBICONV - EXTLIBS += -liconv -endif - -GIT_LIB = ../../libgit.a -VCSSVN_LIB = ../../vcs-svn/lib.a -XDIFF_LIB = ../../xdiff/lib.a - -LIBS = $(VCSSVN_LIB) $(GIT_LIB) $(XDIFF_LIB) - -QUIET_SUBDIR0 = +$(MAKE) -C # space to separate -C and subdir -QUIET_SUBDIR1 = - -ifneq ($(findstring $(MAKEFLAGS),w),w) -PRINT_DIR = --no-print-directory -else # "make -w" -NO_SUBDIR = : -endif - -ifneq ($(findstring $(MAKEFLAGS),s),s) -ifndef V - QUIET_CC = @echo ' ' CC $@; - QUIET_LINK = @echo ' ' LINK $@; - QUIET_SUBDIR0 = +@subdir= - QUIET_SUBDIR1 = ;$(NO_SUBDIR) echo ' ' SUBDIR $$subdir; \ - $(MAKE) $(PRINT_DIR) -C $$subdir -endif -endif - -svn-fe$X: svn-fe.o $(VCSSVN_LIB) $(XDIFF_LIB) $(GIT_LIB) - $(QUIET_LINK)$(CC) $(CFLAGS) $(LDFLAGS) $(EXTLIBS) -o $@ svn-fe.o $(LIBS) - -svn-fe.o: svn-fe.c ../../vcs-svn/svndump.h - $(QUIET_CC)$(CC) $(CFLAGS) -I../../vcs-svn -o $*.o -c $< - -svn-fe.html: svn-fe.txt - $(QUIET_SUBDIR0)../../Documentation $(QUIET_SUBDIR1) \ - MAN_TXT=../contrib/svn-fe/svn-fe.txt \ - ../contrib/svn-fe/$@ - -svn-fe.1: svn-fe.txt - $(QUIET_SUBDIR0)../../Documentation $(QUIET_SUBDIR1) \ - MAN_TXT=../contrib/svn-fe/svn-fe.txt \ - ../contrib/svn-fe/$@ - $(MV) ../../Documentation/svn-fe.1 . - -../../vcs-svn/lib.a: FORCE - $(QUIET_SUBDIR0)../.. $(QUIET_SUBDIR1) vcs-svn/lib.a - -../../xdiff/lib.a: FORCE - $(QUIET_SUBDIR0)../.. $(QUIET_SUBDIR1) xdiff/lib.a - -../../libgit.a: FORCE - $(QUIET_SUBDIR0)../.. $(QUIET_SUBDIR1) libgit.a - -clean: - $(RM) svn-fe$X svn-fe.o svn-fe.html svn-fe.xml svn-fe.1 - -.PHONY: all clean FORCE diff --git a/contrib/svn-fe/svn-fe.c b/contrib/svn-fe/svn-fe.c deleted file mode 100644 index f363505abb..0000000000 --- a/contrib/svn-fe/svn-fe.c +++ /dev/null @@ -1,18 +0,0 @@ -/* - * This file is in the public domain. - * You may freely use, modify, distribute, and relicense it. - */ - -#include -#include "svndump.h" - -int main(int argc, char **argv) -{ - if (svndump_init(NULL)) - return 1; - svndump_read((argc > 1) ? argv[1] : NULL, "refs/heads/master", - "refs/notes/svn/revs"); - svndump_deinit(); - svndump_reset(); - return 0; -} diff --git a/contrib/svn-fe/svn-fe.txt b/contrib/svn-fe/svn-fe.txt deleted file mode 100644 index 19333fc8df..0000000000 --- a/contrib/svn-fe/svn-fe.txt +++ /dev/null @@ -1,71 +0,0 @@ -svn-fe(1) -========= - -NAME ----- -svn-fe - convert an SVN "dumpfile" to a fast-import stream - -SYNOPSIS --------- -[verse] -mkfifo backchannel && -svnadmin dump --deltas REPO | - svn-fe [url] 3backchannel - -DESCRIPTION ------------ - -Converts a Subversion dumpfile into input suitable for -git-fast-import(1) and similar importers. REPO is a path to a -Subversion repository mirrored on the local disk. Remote Subversion -repositories can be mirrored on local disk using the `svnsync` -command. - -Note: this tool is very young. The details of its commandline -interface may change in backward incompatible ways. - -INPUT FORMAT ------------- -Subversion's repository dump format is documented in full in -`notes/dump-load-format.txt` from the Subversion source tree. -Files in this format can be generated using the 'svnadmin dump' or -'svk admin dump' command. - -OUTPUT FORMAT -------------- -The fast-import format is documented by the git-fast-import(1) -manual page. - -NOTES ------ -Subversion dumps do not record a separate author and committer for -each revision, nor do they record a separate display name and email -address for each author. Like git-svn(1), 'svn-fe' will use the name - ---------- -user ---------- - -as committer, where 'user' is the value of the `svn:author` property -and 'UUID' the repository's identifier. - -To support incremental imports, 'svn-fe' puts a `git-svn-id` line at -the end of each commit log message if passed a URL on the command -line. This line has the form `git-svn-id: URL@REVNO UUID`. - -The resulting repository will generally require further processing -to put each project in its own repository and to separate the history -of each branch. The 'git filter-repo --subdirectory-filter' command -may be useful for this purpose. - -BUGS ----- -Empty directories and unknown properties are silently discarded. - -The exit status does not reflect whether an error was detected. - -SEE ALSO --------- -git-svn(1), svn2git(1), svk(1), git-filter-repo(1), git-fast-import(1), -https://svn.apache.org/repos/asf/subversion/trunk/notes/dump-load-format.txt diff --git a/contrib/svn-fe/svnrdump_sim.py b/contrib/svn-fe/svnrdump_sim.py deleted file mode 100755 index 8a3cee6175..0000000000 --- a/contrib/svn-fe/svnrdump_sim.py +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env python -""" -Simulates svnrdump by replaying an existing dump from a file, taking care -of the specified revision range. -To simulate incremental imports the environment variable SVNRMAX can be set -to the highest revision that should be available. -""" -import sys -import os - -if sys.hexversion < 0x02040000: - # The limiter is the ValueError() calls. This may be too conservative - sys.stderr.write("svnrdump-sim.py: requires Python 2.4 or later.\n") - sys.exit(1) - - -def getrevlimit(): - var = 'SVNRMAX' - if var in os.environ: - return os.environ[var] - return None - - -def writedump(url, lower, upper): - if url.startswith('sim://'): - filename = url[6:] - if filename[-1] == '/': - filename = filename[:-1] # remove terminating slash - else: - raise ValueError('sim:// url required') - f = open(filename, 'r') - state = 'header' - wroterev = False - while(True): - l = f.readline() - if l == '': - break - if state == 'header' and l.startswith('Revision-number: '): - state = 'prefix' - if state == 'prefix' and l == 'Revision-number: %s\n' % lower: - state = 'selection' - if not upper == 'HEAD' and state == 'selection' and \ - l == 'Revision-number: %s\n' % upper: - break - - if state == 'header' or state == 'selection': - if state == 'selection': - wroterev = True - sys.stdout.write(l) - return wroterev - -if __name__ == "__main__": - if not (len(sys.argv) in (3, 4, 5)): - print("usage: %s dump URL -rLOWER:UPPER") - sys.exit(1) - if not sys.argv[1] == 'dump': - raise NotImplementedError('only "dump" is supported.') - url = sys.argv[2] - r = ('0', 'HEAD') - if len(sys.argv) == 4 and sys.argv[3][0:2] == '-r': - r = sys.argv[3][2:].lstrip().split(':') - if not getrevlimit() is None: - r[1] = getrevlimit() - if writedump(url, r[0], r[1]): - ret = 0 - else: - ret = 1 - sys.exit(ret) diff --git a/remote-testsvn.c b/remote-testsvn.c deleted file mode 100644 index 636b2b62a6..0000000000 --- a/remote-testsvn.c +++ /dev/null @@ -1,341 +0,0 @@ -#include "cache.h" -#include "refs.h" -#include "remote.h" -#include "object-store.h" -#include "strbuf.h" -#include "url.h" -#include "exec-cmd.h" -#include "run-command.h" -#include "vcs-svn/svndump.h" -#include "notes.h" -#include "strvec.h" - -static const char *url; -static int dump_from_file; -static const char *private_ref; -static char *remote_ref; -static const char *marksfilename, *notes_ref; -struct rev_note { unsigned int rev_nr; }; - -static int cmd_capabilities(const char *line); -static int cmd_import(const char *line); -static int cmd_list(const char *line); - -typedef int (*input_command_handler)(const char *); -struct input_command_entry { - const char *name; - input_command_handler fn; - unsigned char batchable; /* whether the command starts or is part of a batch */ -}; - -static const struct input_command_entry input_command_list[] = { - { "capabilities", cmd_capabilities, 0 }, - { "import", cmd_import, 1 }, - { "list", cmd_list, 0 }, - { NULL, NULL } -}; - -static int cmd_capabilities(const char *line) -{ - printf("import\n"); - printf("bidi-import\n"); - printf("refspec %s:%s\n\n", remote_ref, private_ref); - fflush(stdout); - return 0; -} - -static void terminate_batch(void) -{ - /* terminate a current batch's fast-import stream */ - printf("done\n"); - fflush(stdout); -} - -/* NOTE: 'ref' refers to a git reference, while 'rev' refers to a svn revision. */ -static char *read_ref_note(const struct object_id *oid) -{ - const struct object_id *note_oid; - char *msg = NULL; - unsigned long msglen; - enum object_type type; - - init_notes(NULL, notes_ref, NULL, 0); - if (!(note_oid = get_note(NULL, oid))) - return NULL; /* note tree not found */ - if (!(msg = read_object_file(note_oid, &type, &msglen))) - error("Empty notes tree. %s", notes_ref); - else if (!msglen || type != OBJ_BLOB) { - error("Note contains unusable content. " - "Is something else using this notes tree? %s", notes_ref); - FREE_AND_NULL(msg); - } - free_notes(NULL); - return msg; -} - -static int parse_rev_note(const char *msg, struct rev_note *res) -{ - const char *key, *value, *end; - size_t len; - - while (*msg) { - end = strchrnul(msg, '\n'); - len = end - msg; - - key = "Revision-number: "; - if (starts_with(msg, key)) { - long i; - char *end; - value = msg + strlen(key); - i = strtol(value, &end, 0); - if (end == value || i < 0 || i > UINT32_MAX) - return -1; - res->rev_nr = i; - return 0; - } - msg += len + 1; - } - /* didn't find it */ - return -1; -} - -static int note2mark_cb(const struct object_id *object_oid, - const struct object_id *note_oid, char *note_path, - void *cb_data) -{ - FILE *file = (FILE *)cb_data; - char *msg; - unsigned long msglen; - enum object_type type; - struct rev_note note; - - if (!(msg = read_object_file(note_oid, &type, &msglen)) || - !msglen || type != OBJ_BLOB) { - free(msg); - return 1; - } - if (parse_rev_note(msg, ¬e)) - return 2; - if (fprintf(file, ":%d %s\n", note.rev_nr, oid_to_hex(object_oid)) < 1) - return 3; - return 0; -} - -static void regenerate_marks(void) -{ - int ret; - FILE *marksfile = xfopen(marksfilename, "w+"); - - ret = for_each_note(NULL, 0, note2mark_cb, marksfile); - if (ret) - die("Regeneration of marks failed, returned %d.", ret); - fclose(marksfile); -} - -static void check_or_regenerate_marks(int latestrev) -{ - FILE *marksfile; - struct strbuf sb = STRBUF_INIT; - struct strbuf line = STRBUF_INIT; - int found = 0; - - if (latestrev < 1) - return; - - init_notes(NULL, notes_ref, NULL, 0); - marksfile = fopen(marksfilename, "r"); - if (!marksfile) { - regenerate_marks(); - marksfile = xfopen(marksfilename, "r"); - fclose(marksfile); - } else { - strbuf_addf(&sb, ":%d ", latestrev); - while (strbuf_getline_lf(&line, marksfile) != EOF) { - if (starts_with(line.buf, sb.buf)) { - found++; - break; - } - } - fclose(marksfile); - if (!found) - regenerate_marks(); - } - free_notes(NULL); - strbuf_release(&sb); - strbuf_release(&line); -} - -static int cmd_import(const char *line) -{ - int code; - int dumpin_fd; - char *note_msg; - struct object_id head_oid; - unsigned int startrev; - struct child_process svndump_proc = CHILD_PROCESS_INIT; - const char *command = "svnrdump"; - - if (read_ref(private_ref, &head_oid)) - startrev = 0; - else { - note_msg = read_ref_note(&head_oid); - if(note_msg == NULL) { - warning("No note found for %s.", private_ref); - startrev = 0; - } else { - struct rev_note note = { 0 }; - if (parse_rev_note(note_msg, ¬e)) - die("Revision number couldn't be parsed from note."); - startrev = note.rev_nr + 1; - free(note_msg); - } - } - check_or_regenerate_marks(startrev - 1); - - if (dump_from_file) { - dumpin_fd = open(url, O_RDONLY); - if(dumpin_fd < 0) - die_errno("Couldn't open svn dump file %s.", url); - } else { - svndump_proc.out = -1; - strvec_push(&svndump_proc.args, command); - strvec_push(&svndump_proc.args, "dump"); - strvec_push(&svndump_proc.args, url); - strvec_pushf(&svndump_proc.args, "-r%u:HEAD", startrev); - - code = start_command(&svndump_proc); - if (code) - die("Unable to start %s, code %d", command, code); - dumpin_fd = svndump_proc.out; - } - /* setup marks file import/export */ - printf("feature import-marks-if-exists=%s\n" - "feature export-marks=%s\n", marksfilename, marksfilename); - - svndump_init_fd(dumpin_fd, STDIN_FILENO); - svndump_read(url, private_ref, notes_ref); - svndump_deinit(); - svndump_reset(); - - close(dumpin_fd); - if (!dump_from_file) { - code = finish_command(&svndump_proc); - if (code) - warning("%s, returned %d", command, code); - } - - return 0; -} - -static int cmd_list(const char *line) -{ - printf("? %s\n\n", remote_ref); - fflush(stdout); - return 0; -} - -static int do_command(struct strbuf *line) -{ - const struct input_command_entry *p = input_command_list; - static struct string_list batchlines = STRING_LIST_INIT_DUP; - static const struct input_command_entry *batch_cmd; - /* - * commands can be grouped together in a batch. - * Batches are ended by \n. If no batch is active the program ends. - * During a batch all lines are buffered and passed to the handler function - * when the batch is terminated. - */ - if (line->len == 0) { - if (batch_cmd) { - struct string_list_item *item; - for_each_string_list_item(item, &batchlines) - batch_cmd->fn(item->string); - terminate_batch(); - batch_cmd = NULL; - string_list_clear(&batchlines, 0); - return 0; /* end of the batch, continue reading other commands. */ - } - return 1; /* end of command stream, quit */ - } - if (batch_cmd) { - if (!starts_with(batch_cmd->name, line->buf)) - die("Active %s batch interrupted by %s", batch_cmd->name, line->buf); - /* buffer batch lines */ - string_list_append(&batchlines, line->buf); - return 0; - } - - for (p = input_command_list; p->name; p++) { - if (starts_with(line->buf, p->name) && (strlen(p->name) == line->len || - line->buf[strlen(p->name)] == ' ')) { - if (p->batchable) { - batch_cmd = p; - string_list_append(&batchlines, line->buf); - return 0; - } - return p->fn(line->buf); - } - } - die("Unknown command '%s'\n", line->buf); - return 0; -} - -int cmd_main(int argc, const char **argv) -{ - struct strbuf buf = STRBUF_INIT, url_sb = STRBUF_INIT, - private_ref_sb = STRBUF_INIT, marksfilename_sb = STRBUF_INIT, - notes_ref_sb = STRBUF_INIT; - static struct remote *remote; - const char *url_in, *remote_ref_short; - - setup_git_directory(); - if (argc < 2 || argc > 3) { - usage("git-remote-svn []"); - return 1; - } - - remote_ref_short = git_default_branch_name(); - remote_ref = xstrfmt("refs/heads/%s", remote_ref_short); - - remote = remote_get(argv[1]); - url_in = (argc == 3) ? argv[2] : remote->url[0]; - - if (starts_with(url_in, "file://")) { - dump_from_file = 1; - url = url_decode(url_in + sizeof("file://")-1); - } else { - dump_from_file = 0; - end_url_with_slash(&url_sb, url_in); - url = url_sb.buf; - } - - strbuf_addf(&private_ref_sb, "refs/svn/%s/%s", - remote->name, remote_ref_short); - private_ref = private_ref_sb.buf; - - strbuf_addf(¬es_ref_sb, "refs/notes/%s/revs", remote->name); - notes_ref = notes_ref_sb.buf; - - strbuf_addf(&marksfilename_sb, "%s/info/fast-import/remote-svn/%s.marks", - get_git_dir(), remote->name); - marksfilename = marksfilename_sb.buf; - - while (1) { - if (strbuf_getline_lf(&buf, stdin) == EOF) { - if (ferror(stdin)) - die("Error reading command stream"); - else - die("Unexpected end of command stream"); - } - if (do_command(&buf)) - break; - strbuf_reset(&buf); - } - - strbuf_release(&buf); - strbuf_release(&url_sb); - strbuf_release(&private_ref_sb); - strbuf_release(¬es_ref_sb); - strbuf_release(&marksfilename_sb); - return 0; -} diff --git a/t/helper/.gitignore b/t/helper/.gitignore index 48c7bb0bbb..8c2ddcce95 100644 --- a/t/helper/.gitignore +++ b/t/helper/.gitignore @@ -1,4 +1,2 @@ /test-tool /test-fake-ssh -/test-line-buffer -/test-svn-fe diff --git a/t/helper/test-line-buffer.c b/t/helper/test-line-buffer.c deleted file mode 100644 index 078dd7e29d..0000000000 --- a/t/helper/test-line-buffer.c +++ /dev/null @@ -1,81 +0,0 @@ -/* - * test-line-buffer.c: code to exercise the svn importer's input helper - */ - -#include "git-compat-util.h" -#include "strbuf.h" -#include "vcs-svn/line_buffer.h" - -static uint32_t strtouint32(const char *s) -{ - char *end; - uintmax_t n = strtoumax(s, &end, 10); - if (*s == '\0' || *end != '\0') - die("invalid count: %s", s); - return (uint32_t) n; -} - -static void handle_command(const char *command, const char *arg, struct line_buffer *buf) -{ - if (starts_with(command, "binary ")) { - struct strbuf sb = STRBUF_INIT; - strbuf_addch(&sb, '>'); - buffer_read_binary(buf, &sb, strtouint32(arg)); - fwrite(sb.buf, 1, sb.len, stdout); - strbuf_release(&sb); - } else if (starts_with(command, "copy ")) { - buffer_copy_bytes(buf, strtouint32(arg)); - } else if (starts_with(command, "skip ")) { - buffer_skip_bytes(buf, strtouint32(arg)); - } else { - die("unrecognized command: %s", command); - } -} - -static void handle_line(const char *line, struct line_buffer *stdin_buf) -{ - const char *arg = strchr(line, ' '); - if (!arg) - die("no argument in line: %s", line); - handle_command(line, arg + 1, stdin_buf); -} - -int cmd_main(int argc, const char **argv) -{ - struct line_buffer stdin_buf = LINE_BUFFER_INIT; - struct line_buffer file_buf = LINE_BUFFER_INIT; - struct line_buffer *input = &stdin_buf; - const char *filename; - char *s; - - if (argc == 1) - filename = NULL; - else if (argc == 2) - filename = argv[1]; - else - usage("test-line-buffer [file | &fd] < script"); - - if (buffer_init(&stdin_buf, NULL)) - die_errno("open error"); - if (filename) { - if (*filename == '&') { - if (buffer_fdinit(&file_buf, strtouint32(filename + 1))) - die_errno("error opening fd %s", filename + 1); - } else { - if (buffer_init(&file_buf, filename)) - die_errno("error opening %s", filename); - } - input = &file_buf; - } - - while ((s = buffer_read_line(&stdin_buf))) - handle_line(s, input); - - if (filename && buffer_deinit(&file_buf)) - die("error reading from %s", filename); - if (buffer_deinit(&stdin_buf)) - die("input error"); - if (ferror(stdout)) - die("output error"); - return 0; -} diff --git a/t/helper/test-svn-fe.c b/t/helper/test-svn-fe.c deleted file mode 100644 index 7667c0803f..0000000000 --- a/t/helper/test-svn-fe.c +++ /dev/null @@ -1,52 +0,0 @@ -/* - * test-svn-fe: Code to exercise the svn import lib - */ - -#include "git-compat-util.h" -#include "vcs-svn/svndump.h" -#include "vcs-svn/svndiff.h" -#include "vcs-svn/sliding_window.h" -#include "vcs-svn/line_buffer.h" - -static const char test_svnfe_usage[] = - "test-svn-fe ( | [-d] )"; - -static int apply_delta(int argc, const char **argv) -{ - struct line_buffer preimage = LINE_BUFFER_INIT; - struct line_buffer delta = LINE_BUFFER_INIT; - struct sliding_view preimage_view = SLIDING_VIEW_INIT(&preimage, -1); - - if (argc != 5) - usage(test_svnfe_usage); - - if (buffer_init(&preimage, argv[2])) - die_errno("cannot open preimage"); - if (buffer_init(&delta, argv[3])) - die_errno("cannot open delta"); - if (svndiff0_apply(&delta, (off_t) strtoumax(argv[4], NULL, 0), - &preimage_view, stdout)) - return 1; - if (buffer_deinit(&preimage)) - die_errno("cannot close preimage"); - if (buffer_deinit(&delta)) - die_errno("cannot close delta"); - strbuf_release(&preimage_view.buf); - return 0; -} - -int cmd_main(int argc, const char **argv) -{ - if (argc == 2) { - if (svndump_init(argv[1])) - return 1; - svndump_read(NULL, "refs/heads/master", "refs/notes/svn/revs"); - svndump_deinit(); - svndump_reset(); - return 0; - } - - if (argc >= 2 && !strcmp(argv[1], "-d")) - return apply_delta(argc, argv); - usage(test_svnfe_usage); -} diff --git a/t/t0081-line-buffer.sh b/t/t0081-line-buffer.sh deleted file mode 100755 index ce92e6acad..0000000000 --- a/t/t0081-line-buffer.sh +++ /dev/null @@ -1,90 +0,0 @@ -#!/bin/sh - -test_description="Test the svn importer's input handling routines. - -These tests provide some simple checks that the line_buffer API -behaves as advertised. - -While at it, check that input of newlines and null bytes are handled -correctly. -" -. ./test-lib.sh - -test_expect_success 'hello world' ' - echo ">HELLO" >expect && - test-line-buffer <<-\EOF >actual && - binary 6 - HELLO - EOF - test_cmp expect actual -' - -test_expect_success '0-length read, send along greeting' ' - echo ">HELLO" >expect && - test-line-buffer <<-\EOF >actual && - binary 0 - copy 6 - HELLO - EOF - test_cmp expect actual -' - -test_expect_success !MINGW 'read from file descriptor' ' - rm -f input && - echo hello >expect && - echo hello >input && - echo copy 6 | - test-line-buffer "&4" 4actual && - test_cmp expect actual -' - -test_expect_success 'skip, copy null byte' ' - echo Q | q_to_nul >expect && - q_to_nul <<-\EOF | test-line-buffer >actual && - skip 2 - Q - copy 2 - Q - EOF - test_cmp expect actual -' - -test_expect_success 'read null byte' ' - echo ">QhelloQ" | q_to_nul >expect && - q_to_nul <<-\EOF | test-line-buffer >actual && - binary 8 - QhelloQ - EOF - test_cmp expect actual -' - -test_expect_success 'long reads are truncated' ' - echo ">foo" >expect && - test-line-buffer <<-\EOF >actual && - binary 5 - foo - EOF - test_cmp expect actual -' - -test_expect_success 'long copies are truncated' ' - printf "%s\n" ">" foo >expect && - test-line-buffer <<-\EOF >actual && - binary 1 - - copy 5 - foo - EOF - test_cmp expect actual -' - -test_expect_success 'long binary reads are truncated' ' - echo ">foo" >expect && - test-line-buffer <<-\EOF >actual && - binary 5 - foo - EOF - test_cmp expect actual -' - -test_done diff --git a/t/t9010-svn-fe.sh b/t/t9010-svn-fe.sh deleted file mode 100755 index 83f8f5cacb..0000000000 --- a/t/t9010-svn-fe.sh +++ /dev/null @@ -1,1105 +0,0 @@ -#!/bin/sh - -test_description='check svn dumpfile importer' - -. ./test-lib.sh - -if test_have_prereq !PIPE -then - skip_all="svn dumpfile importer testing requires the PIPE prerequisite" - test_done -fi - -reinit_git () { - rm -fr .git && - rm -f stream backflow && - git init && - mkfifo stream backflow -} - -try_dump () { - input=$1 && - maybe_fail_svnfe=${2:+test_$2} && - maybe_fail_fi=${3:+test_$3} && - - { - $maybe_fail_svnfe test-svn-fe "$input" >stream 3backflow && - wait $! -} - -properties () { - while test "$#" -ne 0 - do - property="$1" && - value="$2" && - printf "%s\n" "K ${#property}" && - printf "%s\n" "$property" && - printf "%s\n" "V ${#value}" && - printf "%s\n" "$value" && - shift 2 || - return 1 - done -} - -text_no_props () { - text="$1 -" && - printf "%s\n" "Prop-content-length: 10" && - printf "%s\n" "Text-content-length: ${#text}" && - printf "%s\n" "Content-length: $((${#text} + 10))" && - printf "%s\n" "" "PROPS-END" && - printf "%s\n" "$text" -} - -test_expect_success 'empty dump' ' - reinit_git && - echo "SVN-fs-dump-format-version: 2" >input && - try_dump input -' - -test_expect_success 'v4 dumps not supported' ' - reinit_git && - echo "SVN-fs-dump-format-version: 4" >v4.dump && - try_dump v4.dump must_fail -' - -test_expect_failure 'empty revision' ' - reinit_git && - printf "rev : %s\n" "" "" >expect && - cat >emptyrev.dump <<-\EOF && - SVN-fs-dump-format-version: 3 - - Revision-number: 1 - Prop-content-length: 0 - Content-length: 0 - - Revision-number: 2 - Prop-content-length: 0 - Content-length: 0 - - EOF - try_dump emptyrev.dump && - git log -p --format="rev <%an, %ae>: %s" HEAD >actual && - test_cmp expect actual -' - -test_expect_success 'empty properties' ' - reinit_git && - printf "rev : %s\n" "" "" >expect && - cat >emptyprop.dump <<-\EOF && - SVN-fs-dump-format-version: 3 - - Revision-number: 1 - Prop-content-length: 10 - Content-length: 10 - - PROPS-END - - Revision-number: 2 - Prop-content-length: 10 - Content-length: 10 - - PROPS-END - EOF - try_dump emptyprop.dump && - git log -p --format="rev <%an, %ae>: %s" HEAD >actual && - test_cmp expect actual -' - -test_expect_success 'author name and commit message' ' - reinit_git && - echo "" >expect.author && - cat >message <<-\EOF && - A concise summary of the change - - A detailed description of the change, why it is needed, what - was broken and why applying this is the best course of action. - - * file.c - Details pertaining to an individual file. - EOF - { - properties \ - svn:author author@example.com \ - svn:log "$(cat message)" && - echo PROPS-END - } >props && - { - echo "SVN-fs-dump-format-version: 3" && - echo && - echo "Revision-number: 1" && - echo Prop-content-length: $(wc -c log.dump && - try_dump log.dump && - git log -p --format="%B" HEAD >actual.log && - git log --format="<%an, %ae>" >actual.author && - test_cmp message actual.log && - test_cmp expect.author actual.author -' - -test_expect_success 'unsupported properties are ignored' ' - reinit_git && - echo author >expect && - cat >extraprop.dump <<-\EOF && - SVN-fs-dump-format-version: 3 - - Revision-number: 1 - Prop-content-length: 56 - Content-length: 56 - - K 8 - nonsense - V 1 - y - K 10 - svn:author - V 6 - author - PROPS-END - EOF - try_dump extraprop.dump && - git log -p --format=%an HEAD >actual && - test_cmp expect actual -' - -test_expect_failure 'timestamp and empty file' ' - echo author@example.com >expect.author && - echo 1999-01-01 >expect.date && - echo file >expect.files && - reinit_git && - { - properties \ - svn:author author@example.com \ - svn:date "1999-01-01T00:01:002.000000Z" \ - svn:log "add empty file" && - echo PROPS-END - } >props && - { - cat <<-EOF && - SVN-fs-dump-format-version: 3 - - Revision-number: 1 - EOF - echo Prop-content-length: $(wc -c emptyfile.dump && - try_dump emptyfile.dump && - git log --format=%an HEAD >actual.author && - git log --date=short --format=%ad HEAD >actual.date && - git ls-tree -r --name-only HEAD >actual.files && - test_cmp expect.author actual.author && - test_cmp expect.date actual.date && - test_cmp expect.files actual.files && - git checkout HEAD empty-file && - test_must_be_empty file -' - -test_expect_success 'directory with files' ' - reinit_git && - printf "%s\n" directory/file1 directory/file2 >expect.files && - echo hi >hi && - echo hello >hello && - { - properties \ - svn:author author@example.com \ - svn:date "1999-02-01T00:01:002.000000Z" \ - svn:log "add directory with some files in it" && - echo PROPS-END - } >props && - { - cat <<-EOF && - SVN-fs-dump-format-version: 3 - - Revision-number: 1 - EOF - echo Prop-content-length: $(wc -c directory.dump && - try_dump directory.dump && - - git ls-tree -r --name-only HEAD >actual.files && - git checkout HEAD directory && - test_cmp expect.files actual.files && - test_cmp hello directory/file1 && - test_cmp hi directory/file2 -' - -test_expect_success 'branch name with backslash' ' - reinit_git && - sort <<-\EOF >expect.branch-files && - trunk/file1 - trunk/file2 - "branches/UpdateFOPto094\\/file1" - "branches/UpdateFOPto094\\/file2" - EOF - - echo hi >hi && - echo hello >hello && - { - properties \ - svn:author author@example.com \ - svn:date "1999-02-02T00:01:02.000000Z" \ - svn:log "add directory with some files in it" && - echo PROPS-END - } >props.setup && - { - properties \ - svn:author brancher@example.com \ - svn:date "2007-12-06T21:38:34.000000Z" \ - svn:log "Updating fop to .94 and adjust fo-stylesheets" && - echo PROPS-END - } >props.branch && - { - cat <<-EOF && - SVN-fs-dump-format-version: 3 - - Revision-number: 1 - EOF - echo Prop-content-length: $(wc -c branch.dump && - try_dump branch.dump && - - git ls-tree -r --name-only HEAD | - sort >actual.branch-files && - test_cmp expect.branch-files actual.branch-files -' - -test_expect_success 'node without action' ' - reinit_git && - cat >inaction.dump <<-\EOF && - SVN-fs-dump-format-version: 3 - - Revision-number: 1 - Prop-content-length: 10 - Content-length: 10 - - PROPS-END - - Node-path: directory - Node-kind: dir - Prop-content-length: 10 - Content-length: 10 - - PROPS-END - EOF - try_dump inaction.dump must_fail -' - -test_expect_success 'action: add node without text' ' - reinit_git && - cat >textless.dump <<-\EOF && - SVN-fs-dump-format-version: 3 - - Revision-number: 1 - Prop-content-length: 10 - Content-length: 10 - - PROPS-END - - Node-path: textless - Node-kind: file - Node-action: add - Prop-content-length: 10 - Content-length: 10 - - PROPS-END - EOF - try_dump textless.dump must_fail -' - -test_expect_failure 'change file mode but keep old content' ' - reinit_git && - cat >expect <<-\EOF && - OBJID - :120000 100644 OBJID OBJID T greeting - OBJID - :100644 120000 OBJID OBJID T greeting - OBJID - :000000 100644 OBJID OBJID A greeting - EOF - echo "link hello" >expect.blob && - echo hello >hello && - cat >filemode.dump <<-\EOF && - SVN-fs-dump-format-version: 3 - - Revision-number: 1 - Prop-content-length: 10 - Content-length: 10 - - PROPS-END - - Node-path: greeting - Node-kind: file - Node-action: add - Prop-content-length: 10 - Text-content-length: 11 - Content-length: 21 - - PROPS-END - link hello - - Revision-number: 2 - Prop-content-length: 10 - Content-length: 10 - - PROPS-END - - Node-path: greeting - Node-kind: file - Node-action: change - Prop-content-length: 33 - Content-length: 33 - - K 11 - svn:special - V 1 - * - PROPS-END - - Revision-number: 3 - Prop-content-length: 10 - Content-length: 10 - - PROPS-END - - Node-path: greeting - Node-kind: file - Node-action: change - Prop-content-length: 10 - Content-length: 10 - - PROPS-END - EOF - try_dump filemode.dump && - { - git rev-list HEAD | - git diff-tree --root --stdin | - sed "s/$OID_REGEX/OBJID/g" - } >actual && - git show HEAD:greeting >actual.blob && - git show HEAD^:greeting >actual.target && - test_cmp expect actual && - test_cmp expect.blob actual.blob && - test_cmp hello actual.target -' - -test_expect_success 'NUL in property value' ' - reinit_git && - echo "commit message" >expect.message && - { - properties \ - unimportant "something with a NUL (Q)" \ - svn:log "commit message" && - echo PROPS-END - } | - q_to_nul >props && - { - cat <<-\EOF && - SVN-fs-dump-format-version: 3 - - Revision-number: 1 - EOF - echo Prop-content-length: $(wc -c nulprop.dump && - try_dump nulprop.dump && - git diff-tree --always -s --format=%s HEAD >actual.message && - test_cmp expect.message actual.message -' - -test_expect_success 'NUL in log message, file content, and property name' ' - # Caveat: svnadmin 1.6.16 (r1073529) truncates at \0 in the - # svn:specialQnotreally example. - reinit_git && - cat >expect <<-\EOF && - OBJID - :100644 100644 OBJID OBJID M greeting - OBJID - :000000 100644 OBJID OBJID A greeting - EOF - printf "\n%s\n" "something with an ASCII NUL (Q)" >expect.message && - printf "%s\n" "helQo" >expect.hello1 && - printf "%s\n" "link hello" >expect.hello2 && - { - properties svn:log "something with an ASCII NUL (Q)" && - echo PROPS-END - } | - q_to_nul >props && - { - q_to_nul <<-\EOF && - SVN-fs-dump-format-version: 3 - - Revision-number: 1 - Prop-content-length: 10 - Content-length: 10 - - PROPS-END - - Node-path: greeting - Node-kind: file - Node-action: add - Prop-content-length: 10 - Text-content-length: 6 - Content-length: 16 - - PROPS-END - helQo - - Revision-number: 2 - EOF - echo Prop-content-length: $(wc -c 8bitclean.dump && - try_dump 8bitclean.dump && - { - git rev-list HEAD | - git diff-tree --root --stdin | - sed "s/$OID_REGEX/OBJID/g" - } >actual && - { - git cat-file commit HEAD | nul_to_q && - echo - } | - sed -ne "/^\$/,\$ p" >actual.message && - git cat-file blob HEAD^:greeting | nul_to_q >actual.hello1 && - git cat-file blob HEAD:greeting | nul_to_q >actual.hello2 && - test_cmp expect actual && - test_cmp expect.message actual.message && - test_cmp expect.hello1 actual.hello1 && - test_cmp expect.hello2 actual.hello2 -' - -test_expect_success 'change file mode and reiterate content' ' - reinit_git && - cat >expect <<-\EOF && - OBJID - :120000 100644 OBJID OBJID T greeting - OBJID - :100644 120000 OBJID OBJID T greeting - OBJID - :000000 100644 OBJID OBJID A greeting - EOF - echo "link hello" >expect.blob && - echo hello >hello && - cat >filemode2.dump <<-\EOF && - SVN-fs-dump-format-version: 3 - - Revision-number: 1 - Prop-content-length: 10 - Content-length: 10 - - PROPS-END - - Node-path: greeting - Node-kind: file - Node-action: add - Prop-content-length: 10 - Text-content-length: 11 - Content-length: 21 - - PROPS-END - link hello - - Revision-number: 2 - Prop-content-length: 10 - Content-length: 10 - - PROPS-END - - Node-path: greeting - Node-kind: file - Node-action: change - Prop-content-length: 33 - Text-content-length: 11 - Content-length: 44 - - K 11 - svn:special - V 1 - * - PROPS-END - link hello - - Revision-number: 3 - Prop-content-length: 10 - Content-length: 10 - - PROPS-END - - Node-path: greeting - Node-kind: file - Node-action: change - Prop-content-length: 10 - Text-content-length: 11 - Content-length: 21 - - PROPS-END - link hello - EOF - try_dump filemode2.dump && - { - git rev-list HEAD | - git diff-tree --root --stdin | - sed "s/$OID_REGEX/OBJID/g" - } >actual && - git show HEAD:greeting >actual.blob && - git show HEAD^:greeting >actual.target && - test_cmp expect actual && - test_cmp expect.blob actual.blob && - test_cmp hello actual.target -' - -test_expect_success 'deltas supported' ' - reinit_git && - { - # (old) h + (inline) ello + (old) \n - printf "SVNQ%b%b%s" "Q\003\006\005\004" "\001Q\0204\001\002" "ello" | - q_to_nul - } >delta && - { - properties \ - svn:author author@example.com \ - svn:date "1999-01-05T00:01:002.000000Z" \ - svn:log "add greeting" && - echo PROPS-END - } >props && - { - properties \ - svn:author author@example.com \ - svn:date "1999-01-06T00:01:002.000000Z" \ - svn:log "change it" && - echo PROPS-END - } >props2 && - { - echo SVN-fs-dump-format-version: 3 && - echo && - echo Revision-number: 1 && - echo Prop-content-length: $(wc -c delta.dump && - try_dump delta.dump -' - -test_expect_success 'property deltas supported' ' - reinit_git && - cat >expect <<-\EOF && - OBJID - :100755 100644 OBJID OBJID M script.sh - EOF - { - properties \ - svn:author author@example.com \ - svn:date "1999-03-06T00:01:002.000000Z" \ - svn:log "make an executable, or chmod -x it" && - echo PROPS-END - } >revprops && - { - echo SVN-fs-dump-format-version: 3 && - echo && - echo Revision-number: 1 && - echo Prop-content-length: $(wc -c propdelta.dump && - try_dump propdelta.dump && - { - git rev-list HEAD | - git diff-tree --stdin | - sed "s/$OID_REGEX/OBJID/g" - } >actual && - test_cmp expect actual -' - -test_expect_success 'properties on /' ' - reinit_git && - cat <<-\EOF >expect && - OBJID - OBJID - :000000 100644 OBJID OBJID A greeting - EOF - sed -e "s/X$//" <<-\EOF >changeroot.dump && - SVN-fs-dump-format-version: 3 - - Revision-number: 1 - Prop-content-length: 10 - Content-length: 10 - - PROPS-END - - Node-path: greeting - Node-kind: file - Node-action: add - Text-content-length: 0 - Prop-content-length: 10 - Content-length: 10 - - PROPS-END - - Revision-number: 2 - Prop-content-length: 10 - Content-length: 10 - - PROPS-END - - Node-path: X - Node-kind: dir - Node-action: change - Prop-delta: true - Prop-content-length: 43 - Content-length: 43 - - K 10 - svn:ignore - V 11 - build-area - - PROPS-END - EOF - try_dump changeroot.dump && - { - git rev-list HEAD | - git diff-tree --root --always --stdin | - sed "s/$OID_REGEX/OBJID/g" - } >actual && - test_cmp expect actual -' - -test_expect_success 'deltas for typechange' ' - reinit_git && - cat >expect <<-\EOF && - OBJID - :120000 100644 OBJID OBJID T test-file - OBJID - :100755 120000 OBJID OBJID T test-file - OBJID - :000000 100755 OBJID OBJID A test-file - EOF - cat >deleteprop.dump <<-\EOF && - SVN-fs-dump-format-version: 3 - - Revision-number: 1 - Prop-content-length: 10 - Content-length: 10 - - PROPS-END - - Node-path: test-file - Node-kind: file - Node-action: add - Prop-delta: true - Prop-content-length: 35 - Text-content-length: 17 - Content-length: 52 - - K 14 - svn:executable - V 0 - - PROPS-END - link testing 123 - - Revision-number: 2 - Prop-content-length: 10 - Content-length: 10 - - PROPS-END - - Node-path: test-file - Node-kind: file - Node-action: change - Prop-delta: true - Prop-content-length: 53 - Text-content-length: 17 - Content-length: 70 - - K 11 - svn:special - V 1 - * - D 14 - svn:executable - PROPS-END - link testing 231 - - Revision-number: 3 - Prop-content-length: 10 - Content-length: 10 - - PROPS-END - - Node-path: test-file - Node-kind: file - Node-action: change - Prop-delta: true - Prop-content-length: 27 - Text-content-length: 17 - Content-length: 44 - - D 11 - svn:special - PROPS-END - link testing 321 - EOF - try_dump deleteprop.dump && - { - git rev-list HEAD | - git diff-tree --root --stdin | - sed "s/$OID_REGEX/OBJID/g" - } >actual && - test_cmp expect actual -' - -test_expect_success 'deltas need not consume the whole preimage' ' - reinit_git && - cat >expect <<-\EOF && - OBJID - :120000 100644 OBJID OBJID T postimage - OBJID - :100644 120000 OBJID OBJID T postimage - OBJID - :000000 100644 OBJID OBJID A postimage - EOF - echo "first preimage" >expect.1 && - printf target >expect.2 && - printf lnk >expect.3 && - { - printf "SVNQ%b%b%b" "QQ\017\001\017" "\0217" "first preimage\n" | - q_to_nul - } >delta.1 && - { - properties svn:special "*" && - echo PROPS-END - } >symlink.props && - { - printf "SVNQ%b%b%b" "Q\002\013\004\012" "\0201\001\001\0211" "lnk target" | - q_to_nul - } >delta.2 && - { - printf "SVNQ%b%b" "Q\004\003\004Q" "\001Q\002\002" | - q_to_nul - } >delta.3 && - { - cat <<-\EOF && - SVN-fs-dump-format-version: 3 - - Revision-number: 1 - Prop-content-length: 10 - Content-length: 10 - - PROPS-END - - Node-path: postimage - Node-kind: file - Node-action: add - Text-delta: true - Prop-content-length: 10 - EOF - echo Text-content-length: $(wc -c deltapartial.dump && - try_dump deltapartial.dump && - { - git rev-list HEAD | - git diff-tree --root --stdin | - sed "s/$OID_REGEX/OBJID/g" - } >actual && - test_cmp expect actual && - git show HEAD:postimage >actual.3 && - git show HEAD^:postimage >actual.2 && - git show HEAD^^:postimage >actual.1 && - test_cmp expect.1 actual.1 && - test_cmp expect.2 actual.2 && - test_cmp expect.3 actual.3 -' - -test_expect_success 'no hang for delta trying to read past end of preimage' ' - reinit_git && - { - # COPY 1 - printf "SVNQ%b%b" "Q\001\001\002Q" "\001Q" | - q_to_nul - } >greedy.delta && - { - cat <<-\EOF && - SVN-fs-dump-format-version: 3 - - Revision-number: 1 - Prop-content-length: 10 - Content-length: 10 - - PROPS-END - - Node-path: bootstrap - Node-kind: file - Node-action: add - Text-delta: true - Prop-content-length: 10 - EOF - echo Text-content-length: $(wc -c greedydelta.dump && - try_dump greedydelta.dump must_fail might_fail -' - -test_expect_success 'set up svn repo' ' - svnconf=$PWD/svnconf && - mkdir -p "$svnconf" && - - if - svnadmin -h >/dev/null 2>&1 && - svnadmin create simple-svn && - svnadmin load simple-svn <"$TEST_DIRECTORY/t9135/svn.dump" && - svn export --config-dir "$svnconf" "file://$PWD/simple-svn" simple-svnco - then - test_set_prereq SVNREPO - fi -' - -test_expect_success SVNREPO 't9135/svn.dump' ' - mkdir -p simple-git && - ( - cd simple-git && - reinit_git && - try_dump "$TEST_DIRECTORY/t9135/svn.dump" - ) && - ( - cd simple-svnco && - git init && - git add . && - git fetch ../simple-git master && - git diff --exit-code FETCH_HEAD - ) -' - -test_done diff --git a/t/t9011-svn-da.sh b/t/t9011-svn-da.sh deleted file mode 100755 index ab1ef28fd9..0000000000 --- a/t/t9011-svn-da.sh +++ /dev/null @@ -1,248 +0,0 @@ -#!/bin/sh - -test_description='test parsing of svndiff0 files - -Using the "test-svn-fe -d" helper, check that svn-fe correctly -interprets deltas using various facilities (some from the spec, -some only learned from practice). -' -. ./test-lib.sh - ->empty -printf foo >preimage - -test_expect_success 'reject empty delta' ' - test_must_fail test-svn-fe -d preimage empty 0 -' - -test_expect_success 'delta can empty file' ' - printf "SVNQ" | q_to_nul >clear.delta && - test-svn-fe -d preimage clear.delta 4 >actual && - test_must_be_empty actual -' - -test_expect_success 'reject svndiff2' ' - printf "SVN\002" >bad.filetype && - test_must_fail test-svn-fe -d preimage bad.filetype 4 -' - -test_expect_success 'one-window empty delta' ' - printf "SVNQ%s" "QQQQQ" | q_to_nul >clear.onewindow && - test-svn-fe -d preimage clear.onewindow 9 >actual && - test_must_be_empty actual -' - -test_expect_success 'reject incomplete window header' ' - printf "SVNQ%s" "QQQQQ" | q_to_nul >clear.onewindow && - printf "SVNQ%s" "QQ" | q_to_nul >clear.partialwindow && - test_must_fail test-svn-fe -d preimage clear.onewindow 6 && - test_must_fail test-svn-fe -d preimage clear.partialwindow 6 -' - -test_expect_success 'reject declared delta longer than actual delta' ' - printf "SVNQ%s" "QQQQQ" | q_to_nul >clear.onewindow && - printf "SVNQ%s" "QQ" | q_to_nul >clear.partialwindow && - test_must_fail test-svn-fe -d preimage clear.onewindow 14 && - test_must_fail test-svn-fe -d preimage clear.partialwindow 9 -' - -test_expect_success 'two-window empty delta' ' - printf "SVNQ%s%s" "QQQQQ" "QQQQQ" | q_to_nul >clear.twowindow && - test-svn-fe -d preimage clear.twowindow 14 >actual && - test_must_fail test-svn-fe -d preimage clear.twowindow 13 && - test_must_be_empty actual -' - -test_expect_success 'noisy zeroes' ' - printf "SVNQ%s" \ - "RRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRQQQQQ" | - tr R "\200" | - q_to_nul >clear.noisy && - len=$(wc -c clear.badmagic && - test_must_fail test-svn-fe -d preimage clear.badmagic 5 -' - -test_expect_success 'reject truncated integer' ' - printf "SVNQ%s%s" "QQQQQ" "QQQQRRQ" | - tr R "\200" | - q_to_nul >clear.fullint && - printf "SVNQ%s%s" "QQQQQ" "QQQQRR" | - tr RT "\201" | - q_to_nul >clear.partialint && - test_must_fail test-svn-fe -d preimage clear.fullint 15 && - test-svn-fe -d preimage clear.fullint 16 && - test_must_fail test-svn-fe -d preimage clear.partialint 15 -' - -test_expect_success 'nonempty (but unused) preimage view' ' - printf "SVNQ%b" "Q\003QQQ" | q_to_nul >clear.readpreimage && - test-svn-fe -d preimage clear.readpreimage 9 >actual && - test_must_be_empty actual -' - -test_expect_success 'preimage view: right endpoint cannot backtrack' ' - printf "SVNQ%b%b" "Q\003QQQ" "Q\002QQQ" | - q_to_nul >clear.backtrack && - test_must_fail test-svn-fe -d preimage clear.backtrack 14 -' - -test_expect_success 'preimage view: left endpoint can advance' ' - printf "SVNQ%b%b" "Q\003QQQ" "\001\002QQQ" | - q_to_nul >clear.preshrink && - printf "SVNQ%b%b" "Q\003QQQ" "\001\001QQQ" | - q_to_nul >clear.shrinkbacktrack && - test-svn-fe -d preimage clear.preshrink 14 >actual && - test_must_fail test-svn-fe -d preimage clear.shrinkbacktrack 14 && - test_must_be_empty actual -' - -test_expect_success 'preimage view: offsets compared by value' ' - printf "SVNQ%b%b" "\001\001QQQ" "\0200Q\003QQQ" | - q_to_nul >clear.noisybacktrack && - printf "SVNQ%b%b" "\001\001QQQ" "\0200\001\002QQQ" | - q_to_nul >clear.noisyadvance && - test_must_fail test-svn-fe -d preimage clear.noisybacktrack 15 && - test-svn-fe -d preimage clear.noisyadvance 15 && - test_must_be_empty actual -' - -test_expect_success 'preimage view: reject truncated preimage' ' - printf "SVNQ%b" "\010QQQQ" | q_to_nul >clear.lateemptyread && - printf "SVNQ%b" "\010\001QQQ" | q_to_nul >clear.latenonemptyread && - printf "SVNQ%b" "\001\010QQQ" | q_to_nul >clear.longread && - test_must_fail test-svn-fe -d preimage clear.lateemptyread 9 && - test_must_fail test-svn-fe -d preimage clear.latenonemptyread 9 && - test_must_fail test-svn-fe -d preimage clear.longread 9 -' - -test_expect_success 'forbid unconsumed inline data' ' - printf "SVNQ%b%s%b%s" "QQQQ\003" "bar" "QQQQ\001" "x" | - q_to_nul >inline.clear && - test_must_fail test-svn-fe -d preimage inline.clear 18 >actual -' - -test_expect_success 'reject truncated inline data' ' - printf "SVNQ%b%s" "QQQQ\003" "b" | q_to_nul >inline.trunc && - test_must_fail test-svn-fe -d preimage inline.trunc 10 -' - -test_expect_success 'reject truncated inline data (after instruction section)' ' - printf "SVNQ%b%b%s" "QQ\001\001\003" "\0201" "b" | q_to_nul >insn.trunc && - test_must_fail test-svn-fe -d preimage insn.trunc 11 -' - -test_expect_success 'copyfrom_data' ' - echo hi >expect && - printf "SVNQ%b%b%b" "QQ\003\001\003" "\0203" "hi\n" | q_to_nul >copydat && - test-svn-fe -d preimage copydat 13 >actual && - test_cmp expect actual -' - -test_expect_success 'multiple copyfrom_data' ' - echo hi >expect && - printf "SVNQ%b%b%b%b%b" "QQ\003\002\003" "\0201\0202" "hi\n" \ - "QQQ\002Q" "\0200Q" | q_to_nul >copy.multi && - len=$(wc -c actual && - test_cmp expect actual -' - -test_expect_success 'incomplete multiple insn' ' - printf "SVNQ%b%b%b" "QQ\003\002\003" "\0203\0200" "hi\n" | - q_to_nul >copy.partial && - len=$(wc -c copy.incomplete && - len=$(wc -c expect && - printf "SVNQ%b%b%s" "QQ\006\004\003" "\0203\0100\003Q" "foo" | - q_to_nul >copytarget.repeat && - len=$(wc -c actual && - test_cmp expect actual -' - -test_expect_success 'copyfrom target out of order' ' - printf foooof >expect && - printf "SVNQ%b%b%s" \ - "QQ\006\007\003" "\0203\0101\002\0101\001\0101Q" "foo" | - q_to_nul >copytarget.reverse && - len=$(wc -c actual && - test_cmp expect actual -' - -test_expect_success 'catch copyfrom future' ' - printf "SVNQ%b%b%s" "QQ\004\004\003" "\0202\0101\002\0201" "XYZ" | - q_to_nul >copytarget.infuture && - len=$(wc -c expect && - printf "SVNQ%b%b%s" "QQ\014\004\003" "\0202\0111Q\0201" "XYZ" | - q_to_nul >copytarget.sustain && - len=$(wc -c actual && - test_cmp expect actual -' - -test_expect_success 'catch copy that overflows' ' - printf "SVNQ%b%b%s" "QQ\003\003\001" "\0201\0177Q" X | - q_to_nul >copytarget.overflow && - len=$(wc -c expect && - printf "SVNQ%b%b" "Q\003\003\002Q" "\003Q" | q_to_nul >copysource.all && - test-svn-fe -d preimage copysource.all 11 >actual && - test_cmp expect actual -' - -test_expect_success 'copy backwards' ' - printf oof >expect && - printf "SVNQ%b%b" "Q\003\003\006Q" "\001\002\001\001\001Q" | - q_to_nul >copysource.rev && - test-svn-fe -d preimage copysource.rev 15 >actual && - test_cmp expect actual -' - -test_expect_success 'offsets are relative to window' ' - printf fo >expect && - printf "SVNQ%b%b%b%b" "Q\003\001\002Q" "\001Q" \ - "\002\001\001\002Q" "\001Q" | - q_to_nul >copysource.two && - test-svn-fe -d preimage copysource.two 18 >actual && - test_cmp expect actual -' - -test_expect_success 'example from notes/svndiff' ' - printf aaaaccccdddddddd >expect && - printf aaaabbbbcccc >source && - printf "SVNQ%b%b%s" "Q\014\020\007\001" \ - "\004Q\004\010\0201\0107\010" d | - q_to_nul >delta.example && - len=$(wc -c actual && - test_cmp expect actual -' - -test_done diff --git a/t/t9020-remote-svn.sh b/t/t9020-remote-svn.sh deleted file mode 100755 index 754c4a3284..0000000000 --- a/t/t9020-remote-svn.sh +++ /dev/null @@ -1,95 +0,0 @@ -#!/bin/sh - -test_description='tests remote-svn' - -. ./test-lib.sh - -MARKSPATH=.git/info/fast-import/remote-svn - -if ! test_have_prereq PYTHON -then - skip_all='skipping remote-svn tests, python not available' - test_done -fi - -# Override svnrdump with our simulator -PATH="$HOME:$PATH" -export PATH PYTHON_PATH GIT_BUILD_DIR - -write_script "$HOME/svnrdump" <<\EOF -exec "$PYTHON_PATH" "$GIT_BUILD_DIR/contrib/svn-fe/svnrdump_sim.py" "$@" -EOF - -init_git () { - rm -fr .git && - git init && - #git remote add svnsim testsvn::sim:///$TEST_DIRECTORY/t9020/example.svnrdump - # let's reuse an existing dump file!? - git remote add svnsim "testsvn::sim://$TEST_DIRECTORY/t9154/svn.dump" - git remote add svnfile "testsvn::file://$TEST_DIRECTORY/t9154/svn.dump" -} - -if test -e "$GIT_BUILD_DIR/git-remote-testsvn" -then - test_set_prereq REMOTE_SVN -fi - -test_debug ' - git --version - type git - type svnrdump -' - -test_expect_success REMOTE_SVN 'simple fetch' ' - init_git && - git fetch svnsim && - test_cmp .git/refs/svn/svnsim/master .git/refs/remotes/svnsim/master && - cp .git/refs/remotes/svnsim/master master.good -' - -test_debug ' - git show-ref -s refs/svn/svnsim/master - git show-ref -s refs/remotes/svnsim/master -' - -test_expect_success REMOTE_SVN 'repeated fetch, nothing shall change' ' - git fetch svnsim && - test_cmp master.good .git/refs/remotes/svnsim/master -' - -test_expect_success REMOTE_SVN 'fetch from a file:// url gives the same result' ' - git fetch svnfile -' - -test_expect_failure REMOTE_SVN 'the sha1 differ because the git-svn-id line in the commit msg contains the url' ' - test_cmp .git/refs/remotes/svnfile/master .git/refs/remotes/svnsim/master -' - -test_expect_success REMOTE_SVN 'mark-file regeneration' ' - # filter out any other marks, that can not be regenerated. Only up to 3 digit revisions are allowed here - grep ":[0-9]\{1,3\} " $MARKSPATH/svnsim.marks > $MARKSPATH/svnsim.marks.old && - rm $MARKSPATH/svnsim.marks && - git fetch svnsim && - test_cmp $MARKSPATH/svnsim.marks.old $MARKSPATH/svnsim.marks -' - -test_expect_success REMOTE_SVN 'incremental imports must lead to the same head' ' - SVNRMAX=3 && - export SVNRMAX && - init_git && - git fetch svnsim && - test_cmp .git/refs/svn/svnsim/master .git/refs/remotes/svnsim/master && - unset SVNRMAX && - git fetch svnsim && - test_cmp master.good .git/refs/remotes/svnsim/master -' - -test_expect_success REMOTE_SVN 'respects configured default initial branch' ' - git -c init.defaultBranch=trunk remote add -f trunk \ - "testsvn::file://$TEST_DIRECTORY/t9154/svn.dump" && - git rev-parse --verify refs/remotes/trunk/trunk -' - -test_debug 'git branch -a' - -test_done diff --git a/t/test-lib-functions.sh b/t/test-lib-functions.sh index 596c549cdd..6d9ea1bb67 100644 --- a/t/test-lib-functions.sh +++ b/t/test-lib-functions.sh @@ -820,7 +820,7 @@ test_must_fail_acceptable () { fi case "$1" in - git|__git*|test-tool|test-svn-fe|test_terminal) + git|__git*|test-tool|test_terminal) return 0 ;; *) diff --git a/vcs-svn/LICENSE b/vcs-svn/LICENSE deleted file mode 100644 index eb91858b82..0000000000 --- a/vcs-svn/LICENSE +++ /dev/null @@ -1,32 +0,0 @@ -Copyright (C) 2010 David Barr . -All rights reserved. - -Copyright (C) 2010 Jonathan Nieder . - -Copyright (C) 2005 Stefan Hegny, hydrografix Consulting GmbH, -Frankfurt/Main, Germany -and others, see http://svn2cc.sarovar.org - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions -are met: -1. Redistributions of source code must retain the above copyright - notice(s), this list of conditions and the following disclaimer - unmodified other than the allowable addition of one or more - copyright notices. -2. Redistributions in binary form must reproduce the above copyright - notice(s), this list of conditions and the following disclaimer in - the documentation and/or other materials provided with the - distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY -EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR -BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE -OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, -EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vcs-svn/fast_export.c b/vcs-svn/fast_export.c deleted file mode 100644 index b5b8913cb0..0000000000 --- a/vcs-svn/fast_export.c +++ /dev/null @@ -1,365 +0,0 @@ -/* - * Licensed under a two-clause BSD-style license. - * See LICENSE for details. - */ - -#include "cache.h" -#include "quote.h" -#include "fast_export.h" -#include "strbuf.h" -#include "svndiff.h" -#include "sliding_window.h" -#include "line_buffer.h" - -#define MAX_GITSVN_LINE_LEN 4096 - -static uint32_t first_commit_done; -static struct line_buffer postimage = LINE_BUFFER_INIT; -static struct line_buffer report_buffer = LINE_BUFFER_INIT; - -/* NEEDSWORK: move to fast_export_init() */ -static int init_postimage(void) -{ - static int postimage_initialized; - if (postimage_initialized) - return 0; - postimage_initialized = 1; - return buffer_tmpfile_init(&postimage); -} - -void fast_export_init(int fd) -{ - first_commit_done = 0; - if (buffer_fdinit(&report_buffer, fd)) - die_errno("cannot read from file descriptor %d", fd); -} - -void fast_export_deinit(void) -{ - if (buffer_deinit(&report_buffer)) - die_errno("error closing fast-import feedback stream"); -} - -void fast_export_delete(const char *path) -{ - putchar('D'); - putchar(' '); - quote_c_style(path, NULL, stdout, 0); - putchar('\n'); -} - -static void fast_export_truncate(const char *path, uint32_t mode) -{ - fast_export_modify(path, mode, "inline"); - printf("data 0\n\n"); -} - -void fast_export_modify(const char *path, uint32_t mode, const char *dataref) -{ - /* Mode must be 100644, 100755, 120000, or 160000. */ - if (!dataref) { - fast_export_truncate(path, mode); - return; - } - printf("M %06"PRIo32" %s ", mode, dataref); - quote_c_style(path, NULL, stdout, 0); - putchar('\n'); -} - -void fast_export_begin_note(uint32_t revision, const char *author, - const char *log, timestamp_t timestamp, const char *note_ref) -{ - static int firstnote = 1; - size_t loglen = strlen(log); - printf("commit %s\n", note_ref); - printf("committer %s <%s@%s> %"PRItime" +0000\n", author, author, "local", timestamp); - printf("data %"PRIuMAX"\n", (uintmax_t)loglen); - fwrite(log, loglen, 1, stdout); - if (firstnote) { - if (revision > 1) - printf("from %s^0", note_ref); - firstnote = 0; - } - fputc('\n', stdout); -} - -void fast_export_note(const char *committish, const char *dataref) -{ - printf("N %s %s\n", dataref, committish); -} - -static char gitsvnline[MAX_GITSVN_LINE_LEN]; -void fast_export_begin_commit(uint32_t revision, const char *author, - const struct strbuf *log, - const char *uuid, const char *url, - timestamp_t timestamp, const char *local_ref) -{ - static const struct strbuf empty = STRBUF_INIT; - if (!log) - log = ∅ - if (*uuid && *url) { - snprintf(gitsvnline, MAX_GITSVN_LINE_LEN, - "\n\ngit-svn-id: %s@%"PRIu32" %s\n", - url, revision, uuid); - } else { - *gitsvnline = '\0'; - } - printf("commit %s\n", local_ref); - printf("mark :%"PRIu32"\n", revision); - printf("committer %s <%s@%s> %"PRItime" +0000\n", - *author ? author : "nobody", - *author ? author : "nobody", - *uuid ? uuid : "local", timestamp); - printf("data %"PRIuMAX"\n", - (uintmax_t) (log->len + strlen(gitsvnline))); - fwrite(log->buf, log->len, 1, stdout); - printf("%s\n", gitsvnline); - if (!first_commit_done) { - if (revision > 1) - printf("from :%"PRIu32"\n", revision - 1); - first_commit_done = 1; - } -} - -void fast_export_end_commit(uint32_t revision) -{ - printf("progress Imported commit %"PRIu32".\n\n", revision); -} - -static void ls_from_rev(uint32_t rev, const char *path) -{ - /* ls :5 path/to/old/file */ - printf("ls :%"PRIu32" ", rev); - quote_c_style(path, NULL, stdout, 0); - putchar('\n'); - fflush(stdout); -} - -static void ls_from_active_commit(const char *path) -{ - /* ls "path/to/file" */ - printf("ls \""); - quote_c_style(path, NULL, stdout, 1); - printf("\"\n"); - fflush(stdout); -} - -static const char *get_response_line(void) -{ - const char *line = buffer_read_line(&report_buffer); - if (line) - return line; - if (buffer_ferror(&report_buffer)) - die_errno("error reading from fast-import"); - die("unexpected end of fast-import feedback"); -} - -static void die_short_read(struct line_buffer *input) -{ - if (buffer_ferror(input)) - die_errno("error reading dump file"); - die("invalid dump: unexpected end of file"); -} - -static int parse_cat_response_line(const char *header, off_t *len) -{ - uintmax_t n; - const char *type; - const char *end; - - if (ends_with(header, " missing")) - return error("cat-blob reports missing blob: %s", header); - type = strstr(header, " blob "); - if (!type) - return error("cat-blob header has wrong object type: %s", header); - n = strtoumax(type + strlen(" blob "), (char **) &end, 10); - if (end == type + strlen(" blob ")) - return error("cat-blob header does not contain length: %s", header); - if (memchr(type + strlen(" blob "), '-', end - type - strlen(" blob "))) - return error("cat-blob header contains negative length: %s", header); - if (n == UINTMAX_MAX || n > maximum_signed_value_of_type(off_t)) - return error("blob too large for current definition of off_t"); - *len = n; - if (*end) - return error("cat-blob header contains garbage after length: %s", header); - return 0; -} - -static void check_preimage_overflow(off_t a, off_t b) -{ - if (signed_add_overflows(a, b)) - die("blob too large for current definition of off_t"); -} - -static long apply_delta(off_t len, struct line_buffer *input, - const char *old_data, uint32_t old_mode) -{ - long ret; - struct sliding_view preimage = SLIDING_VIEW_INIT(&report_buffer, 0); - FILE *out; - - if (init_postimage() || !(out = buffer_tmpfile_rewind(&postimage))) - die("cannot open temporary file for blob retrieval"); - if (old_data) { - const char *response; - printf("cat-blob %s\n", old_data); - fflush(stdout); - response = get_response_line(); - if (parse_cat_response_line(response, &preimage.max_off)) - die("invalid cat-blob response: %s", response); - check_preimage_overflow(preimage.max_off, 1); - } - if (old_mode == S_IFLNK) { - strbuf_addstr(&preimage.buf, "link "); - check_preimage_overflow(preimage.max_off, strlen("link ")); - preimage.max_off += strlen("link "); - check_preimage_overflow(preimage.max_off, 1); - } - if (svndiff0_apply(input, len, &preimage, out)) - die("cannot apply delta"); - if (old_data) { - /* Read the remainder of preimage and trailing newline. */ - assert(!signed_add_overflows(preimage.max_off, 1)); - preimage.max_off++; /* room for newline */ - if (move_window(&preimage, preimage.max_off - 1, 1)) - die("cannot seek to end of input"); - if (preimage.buf.buf[0] != '\n') - die("missing newline after cat-blob response"); - } - ret = buffer_tmpfile_prepare_to_read(&postimage); - if (ret < 0) - die("cannot read temporary file for blob retrieval"); - strbuf_release(&preimage.buf); - return ret; -} - -void fast_export_buf_to_data(const struct strbuf *data) -{ - printf("data %"PRIuMAX"\n", (uintmax_t)data->len); - fwrite(data->buf, data->len, 1, stdout); - fputc('\n', stdout); -} - -void fast_export_data(uint32_t mode, off_t len, struct line_buffer *input) -{ - assert(len >= 0); - if (mode == S_IFLNK) { - /* svn symlink blobs start with "link " */ - if (len < 5) - die("invalid dump: symlink too short for \"link\" prefix"); - len -= 5; - if (buffer_skip_bytes(input, 5) != 5) - die_short_read(input); - } - printf("data %"PRIuMAX"\n", (uintmax_t) len); - if (buffer_copy_bytes(input, len) != len) - die_short_read(input); - fputc('\n', stdout); -} - -static int parse_ls_response(const char *response, uint32_t *mode, - struct strbuf *dataref) -{ - const char *tab; - const char *response_end; - - assert(response); - response_end = response + strlen(response); - - if (*response == 'm') { /* Missing. */ - errno = ENOENT; - return -1; - } - - /* Mode. */ - if (response_end - response < (signed) strlen("100644") || - response[strlen("100644")] != ' ') - die("invalid ls response: missing mode: %s", response); - *mode = 0; - for (; *response != ' '; response++) { - char ch = *response; - if (ch < '0' || ch > '7') - die("invalid ls response: mode is not octal: %s", response); - *mode *= 8; - *mode += ch - '0'; - } - - /* ' blob ' or ' tree ' */ - if (response_end - response < (signed) strlen(" blob ") || - (response[1] != 'b' && response[1] != 't')) - die("unexpected ls response: not a tree or blob: %s", response); - response += strlen(" blob "); - - /* Dataref. */ - tab = memchr(response, '\t', response_end - response); - if (!tab) - die("invalid ls response: missing tab: %s", response); - strbuf_add(dataref, response, tab - response); - return 0; -} - -int fast_export_ls_rev(uint32_t rev, const char *path, - uint32_t *mode, struct strbuf *dataref) -{ - ls_from_rev(rev, path); - return parse_ls_response(get_response_line(), mode, dataref); -} - -int fast_export_ls(const char *path, uint32_t *mode, struct strbuf *dataref) -{ - ls_from_active_commit(path); - return parse_ls_response(get_response_line(), mode, dataref); -} - -const char *fast_export_read_path(const char *path, uint32_t *mode_out) -{ - int err; - static struct strbuf buf = STRBUF_INIT; - - strbuf_reset(&buf); - err = fast_export_ls(path, mode_out, &buf); - if (err) { - if (errno != ENOENT) - BUG("unexpected fast_export_ls error: %s", - strerror(errno)); - /* Treat missing paths as directories. */ - *mode_out = S_IFDIR; - return NULL; - } - return buf.buf; -} - -void fast_export_copy(uint32_t revision, const char *src, const char *dst) -{ - int err; - uint32_t mode; - static struct strbuf data = STRBUF_INIT; - - strbuf_reset(&data); - err = fast_export_ls_rev(revision, src, &mode, &data); - if (err) { - if (errno != ENOENT) - BUG("unexpected fast_export_ls_rev error: %s", - strerror(errno)); - fast_export_delete(dst); - return; - } - fast_export_modify(dst, mode, data.buf); -} - -void fast_export_blob_delta(uint32_t mode, - uint32_t old_mode, const char *old_data, - off_t len, struct line_buffer *input) -{ - long postimage_len; - - assert(len >= 0); - postimage_len = apply_delta(len, input, old_data, old_mode); - if (mode == S_IFLNK) { - buffer_skip_bytes(&postimage, strlen("link ")); - postimage_len -= strlen("link "); - } - printf("data %ld\n", postimage_len); - buffer_copy_bytes(&postimage, postimage_len); - fputc('\n', stdout); -} diff --git a/vcs-svn/line_buffer.c b/vcs-svn/line_buffer.c deleted file mode 100644 index e416caf8a4..0000000000 --- a/vcs-svn/line_buffer.c +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Licensed under a two-clause BSD-style license. - * See LICENSE for details. - */ - -#include "git-compat-util.h" -#include "line_buffer.h" -#include "strbuf.h" - -#define COPY_BUFFER_LEN 4096 - -int buffer_init(struct line_buffer *buf, const char *filename) -{ - buf->infile = filename ? fopen(filename, "r") : stdin; - if (!buf->infile) - return -1; - return 0; -} - -int buffer_fdinit(struct line_buffer *buf, int fd) -{ - buf->infile = fdopen(fd, "r"); - if (!buf->infile) - return -1; - return 0; -} - -int buffer_tmpfile_init(struct line_buffer *buf) -{ - buf->infile = tmpfile(); - if (!buf->infile) - return -1; - return 0; -} - -int buffer_deinit(struct line_buffer *buf) -{ - int err; - if (buf->infile == stdin) - return ferror(buf->infile); - err = ferror(buf->infile); - err |= fclose(buf->infile); - return err; -} - -FILE *buffer_tmpfile_rewind(struct line_buffer *buf) -{ - rewind(buf->infile); - return buf->infile; -} - -long buffer_tmpfile_prepare_to_read(struct line_buffer *buf) -{ - long pos = ftell(buf->infile); - if (pos < 0) - return error_errno("ftell error"); - if (fseek(buf->infile, 0, SEEK_SET)) - return error_errno("seek error"); - return pos; -} - -int buffer_ferror(struct line_buffer *buf) -{ - return ferror(buf->infile); -} - -int buffer_read_char(struct line_buffer *buf) -{ - return fgetc(buf->infile); -} - -/* Read a line without trailing newline. */ -char *buffer_read_line(struct line_buffer *buf) -{ - char *end; - if (!fgets(buf->line_buffer, sizeof(buf->line_buffer), buf->infile)) - /* Error or data exhausted. */ - return NULL; - end = buf->line_buffer + strlen(buf->line_buffer); - if (end[-1] == '\n') - end[-1] = '\0'; - else if (feof(buf->infile)) - ; /* No newline at end of file. That's fine. */ - else - /* - * Line was too long. - * There is probably a saner way to deal with this, - * but for now let's return an error. - */ - return NULL; - return buf->line_buffer; -} - -size_t buffer_read_binary(struct line_buffer *buf, - struct strbuf *sb, size_t size) -{ - return strbuf_fread(sb, size, buf->infile); -} - -off_t buffer_copy_bytes(struct line_buffer *buf, off_t nbytes) -{ - char byte_buffer[COPY_BUFFER_LEN]; - off_t done = 0; - while (done < nbytes && !feof(buf->infile) && !ferror(buf->infile)) { - off_t len = nbytes - done; - size_t in = len < COPY_BUFFER_LEN ? len : COPY_BUFFER_LEN; - in = fread(byte_buffer, 1, in, buf->infile); - done += in; - fwrite(byte_buffer, 1, in, stdout); - if (ferror(stdout)) - return done + buffer_skip_bytes(buf, nbytes - done); - } - return done; -} - -off_t buffer_skip_bytes(struct line_buffer *buf, off_t nbytes) -{ - char byte_buffer[COPY_BUFFER_LEN]; - off_t done = 0; - while (done < nbytes && !feof(buf->infile) && !ferror(buf->infile)) { - off_t len = nbytes - done; - size_t in = len < COPY_BUFFER_LEN ? len : COPY_BUFFER_LEN; - done += fread(byte_buffer, 1, in, buf->infile); - } - return done; -} diff --git a/vcs-svn/line_buffer.txt b/vcs-svn/line_buffer.txt deleted file mode 100644 index 8e139eb22d..0000000000 --- a/vcs-svn/line_buffer.txt +++ /dev/null @@ -1,77 +0,0 @@ -line_buffer API -=============== - -The line_buffer library provides a convenient interface for -mostly-line-oriented input. - -Each line is not permitted to exceed 10000 bytes. The provided -functions are not thread-safe or async-signal-safe, and like -`fgets()`, they generally do not function correctly if interrupted -by a signal without SA_RESTART set. - -Calling sequence ----------------- - -The calling program: - - - initializes a `struct line_buffer` to LINE_BUFFER_INIT - - specifies a file to read with `buffer_init` - - processes input with `buffer_read_line`, `buffer_skip_bytes`, - and `buffer_copy_bytes` - - closes the file with `buffer_deinit`, perhaps to start over and - read another file. - -When finished, the caller can use `buffer_reset` to deallocate -resources. - -Using temporary files ---------------------- - -Temporary files provide a place to store data that should not outlive -the calling program. A program - - - initializes a `struct line_buffer` to LINE_BUFFER_INIT - - requests a temporary file with `buffer_tmpfile_init` - - acquires an output handle by calling `buffer_tmpfile_rewind` - - uses standard I/O functions like `fprintf` and `fwrite` to fill - the temporary file - - declares writing is over with `buffer_tmpfile_prepare_to_read` - - can re-read what was written with `buffer_read_line`, - `buffer_copy_bytes`, and so on - - can reuse the temporary file by calling `buffer_tmpfile_rewind` - again - - removes the temporary file with `buffer_deinit`, perhaps to - reuse the line_buffer for some other file. - -When finished, the calling program can use `buffer_reset` to deallocate -resources. - -Functions ---------- - -`buffer_init`, `buffer_fdinit`:: - Open the named file or file descriptor for input. - buffer_init(buf, NULL) prepares to read from stdin. - On failure, returns -1 (with errno indicating the nature - of the failure). - -`buffer_deinit`:: - Stop reading from the current file (closing it unless - it was stdin). Returns nonzero if `fclose` fails or - the error indicator was set. - -`buffer_read_line`:: - Read a line and strip off the trailing newline. - On failure or end of file, returns NULL. - -`buffer_copy_bytes`:: - Read `len` bytes of input and dump them to the standard output - stream. Returns early for error or end of file. - -`buffer_skip_bytes`:: - Discards `len` bytes from the input stream (stopping early - if necessary because of an error or eof). Return value is - the number of bytes successfully read. - -`buffer_reset`:: - Deallocates non-static buffers. diff --git a/vcs-svn/sliding_window.c b/vcs-svn/sliding_window.c deleted file mode 100644 index 06d273c9e8..0000000000 --- a/vcs-svn/sliding_window.c +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Licensed under a two-clause BSD-style license. - * See LICENSE for details. - */ - -#include "git-compat-util.h" -#include "sliding_window.h" -#include "line_buffer.h" -#include "strbuf.h" - -static int input_error(struct line_buffer *file) -{ - if (!buffer_ferror(file)) - return error("delta preimage ends early"); - return error_errno("cannot read delta preimage"); -} - -static int skip_or_whine(struct line_buffer *file, off_t gap) -{ - if (buffer_skip_bytes(file, gap) != gap) - return input_error(file); - return 0; -} - -static int read_to_fill_or_whine(struct line_buffer *file, - struct strbuf *buf, size_t width) -{ - buffer_read_binary(file, buf, width - buf->len); - if (buf->len != width) - return input_error(file); - return 0; -} - -static int check_offset_overflow(off_t offset, uintmax_t len) -{ - if (len > maximum_signed_value_of_type(off_t)) - return error("unrepresentable length in delta: " - "%"PRIuMAX" > OFF_MAX", len); - if (signed_add_overflows(offset, (off_t) len)) - return error("unrepresentable offset in delta: " - "%"PRIuMAX" + %"PRIuMAX" > OFF_MAX", - (uintmax_t) offset, len); - return 0; -} - -int move_window(struct sliding_view *view, off_t off, size_t width) -{ - off_t file_offset; - assert(view); - assert(view->width <= view->buf.len); - assert(!check_offset_overflow(view->off, view->buf.len)); - - if (check_offset_overflow(off, width)) - return -1; - if (off < view->off || off + width < view->off + view->width) - return error("invalid delta: window slides left"); - if (view->max_off >= 0 && view->max_off < off + (off_t) width) - return error("delta preimage ends early"); - - file_offset = view->off + view->buf.len; - if (off < file_offset) { - /* Move the overlapping region into place. */ - strbuf_remove(&view->buf, 0, off - view->off); - } else { - /* Seek ahead to skip the gap. */ - if (skip_or_whine(view->file, off - file_offset)) - return -1; - strbuf_setlen(&view->buf, 0); - } - - if (view->buf.len > width) - ; /* Already read. */ - else if (read_to_fill_or_whine(view->file, &view->buf, width)) - return -1; - - view->off = off; - view->width = width; - return 0; -} diff --git a/vcs-svn/svndiff.c b/vcs-svn/svndiff.c deleted file mode 100644 index 75c753162a..0000000000 --- a/vcs-svn/svndiff.c +++ /dev/null @@ -1,309 +0,0 @@ -/* - * Licensed under a two-clause BSD-style license. - * See LICENSE for details. - */ - -#include "git-compat-util.h" -#include "sliding_window.h" -#include "line_buffer.h" -#include "svndiff.h" - -/* - * svndiff0 applier - * - * See http://svn.apache.org/repos/asf/subversion/trunk/notes/svndiff. - * - * svndiff0 ::= 'SVN\0' window* - * window ::= int int int int int instructions inline_data; - * instructions ::= instruction*; - * instruction ::= view_selector int int - * | copyfrom_data int - * | packed_view_selector int - * | packed_copyfrom_data - * ; - * view_selector ::= copyfrom_source - * | copyfrom_target - * ; - * copyfrom_source ::= # binary 00 000000; - * copyfrom_target ::= # binary 01 000000; - * copyfrom_data ::= # binary 10 000000; - * packed_view_selector ::= # view_selector OR-ed with 6 bit value; - * packed_copyfrom_data ::= # copyfrom_data OR-ed with 6 bit value; - * int ::= highdigit* lowdigit; - * highdigit ::= # binary 1000 0000 OR-ed with 7 bit value; - * lowdigit ::= # 7 bit value; - */ - -#define INSN_MASK 0xc0 -#define INSN_COPYFROM_SOURCE 0x00 -#define INSN_COPYFROM_TARGET 0x40 -#define INSN_COPYFROM_DATA 0x80 -#define OPERAND_MASK 0x3f - -#define VLI_CONTINUE 0x80 -#define VLI_DIGIT_MASK 0x7f -#define VLI_BITS_PER_DIGIT 7 - -struct window { - struct sliding_view *in; - struct strbuf out; - struct strbuf instructions; - struct strbuf data; -}; - -#define WINDOW_INIT(w) { (w), STRBUF_INIT, STRBUF_INIT, STRBUF_INIT } - -static void window_release(struct window *ctx) -{ - strbuf_release(&ctx->out); - strbuf_release(&ctx->instructions); - strbuf_release(&ctx->data); -} - -static int write_strbuf(struct strbuf *sb, FILE *out) -{ - if (fwrite(sb->buf, 1, sb->len, out) == sb->len) /* Success. */ - return 0; - return error_errno("cannot write delta postimage"); -} - -static int error_short_read(struct line_buffer *input) -{ - if (buffer_ferror(input)) - return error_errno("error reading delta"); - return error("invalid delta: unexpected end of file"); -} - -static int read_chunk(struct line_buffer *delta, off_t *delta_len, - struct strbuf *buf, size_t len) -{ - assert(*delta_len >= 0); - strbuf_reset(buf); - if (len > (uintmax_t) *delta_len || - buffer_read_binary(delta, buf, len) != len) - return error_short_read(delta); - *delta_len -= buf->len; - return 0; -} - -static int read_magic(struct line_buffer *in, off_t *len) -{ - static const char magic[] = {'S', 'V', 'N', '\0'}; - struct strbuf sb = STRBUF_INIT; - - if (read_chunk(in, len, &sb, sizeof(magic))) { - strbuf_release(&sb); - return -1; - } - if (memcmp(sb.buf, magic, sizeof(magic))) { - strbuf_release(&sb); - return error("invalid delta: unrecognized file type"); - } - strbuf_release(&sb); - return 0; -} - -static int read_int(struct line_buffer *in, uintmax_t *result, off_t *len) -{ - uintmax_t rv = 0; - off_t sz; - for (sz = *len; sz; sz--) { - const int ch = buffer_read_char(in); - if (ch == EOF) - break; - - rv <<= VLI_BITS_PER_DIGIT; - rv += (ch & VLI_DIGIT_MASK); - if (ch & VLI_CONTINUE) - continue; - - *result = rv; - *len = sz - 1; - return 0; - } - return error_short_read(in); -} - -static int parse_int(const char **buf, size_t *result, const char *end) -{ - size_t rv = 0; - const char *pos; - for (pos = *buf; pos != end; pos++) { - unsigned char ch = *pos; - - rv <<= VLI_BITS_PER_DIGIT; - rv += (ch & VLI_DIGIT_MASK); - if (ch & VLI_CONTINUE) - continue; - - *result = rv; - *buf = pos + 1; - return 0; - } - return error("invalid delta: unexpected end of instructions section"); -} - -static int read_offset(struct line_buffer *in, off_t *result, off_t *len) -{ - uintmax_t val; - if (read_int(in, &val, len)) - return -1; - if (val > maximum_signed_value_of_type(off_t)) - return error("unrepresentable offset in delta: %"PRIuMAX"", val); - *result = val; - return 0; -} - -static int read_length(struct line_buffer *in, size_t *result, off_t *len) -{ - uintmax_t val; - if (read_int(in, &val, len)) - return -1; - if (val > SIZE_MAX) - return error("unrepresentable length in delta: %"PRIuMAX"", val); - *result = val; - return 0; -} - -static int copyfrom_source(struct window *ctx, const char **instructions, - size_t nbytes, const char *insns_end) -{ - size_t offset; - if (parse_int(instructions, &offset, insns_end)) - return -1; - if (unsigned_add_overflows(offset, nbytes) || - offset + nbytes > ctx->in->width) - return error("invalid delta: copies source data outside view"); - strbuf_add(&ctx->out, ctx->in->buf.buf + offset, nbytes); - return 0; -} - -static int copyfrom_target(struct window *ctx, const char **instructions, - size_t nbytes, const char *instructions_end) -{ - size_t offset; - if (parse_int(instructions, &offset, instructions_end)) - return -1; - if (offset >= ctx->out.len) - return error("invalid delta: copies from the future"); - for (; nbytes > 0; nbytes--) - strbuf_addch(&ctx->out, ctx->out.buf[offset++]); - return 0; -} - -static int copyfrom_data(struct window *ctx, size_t *data_pos, size_t nbytes) -{ - const size_t pos = *data_pos; - if (unsigned_add_overflows(pos, nbytes) || - pos + nbytes > ctx->data.len) - return error("invalid delta: copies unavailable inline data"); - strbuf_add(&ctx->out, ctx->data.buf + pos, nbytes); - *data_pos += nbytes; - return 0; -} - -static int parse_first_operand(const char **buf, size_t *out, const char *end) -{ - size_t result = (unsigned char) *(*buf)++ & OPERAND_MASK; - if (result) { /* immediate operand */ - *out = result; - return 0; - } - return parse_int(buf, out, end); -} - -static int execute_one_instruction(struct window *ctx, - const char **instructions, size_t *data_pos) -{ - unsigned int instruction; - const char *insns_end = ctx->instructions.buf + ctx->instructions.len; - size_t nbytes; - assert(ctx); - assert(instructions && *instructions); - assert(data_pos); - - instruction = (unsigned char) **instructions; - if (parse_first_operand(instructions, &nbytes, insns_end)) - return -1; - switch (instruction & INSN_MASK) { - case INSN_COPYFROM_SOURCE: - return copyfrom_source(ctx, instructions, nbytes, insns_end); - case INSN_COPYFROM_TARGET: - return copyfrom_target(ctx, instructions, nbytes, insns_end); - case INSN_COPYFROM_DATA: - return copyfrom_data(ctx, data_pos, nbytes); - default: - return error("invalid delta: unrecognized instruction"); - } -} - -static int apply_window_in_core(struct window *ctx) -{ - const char *instructions; - size_t data_pos = 0; - - /* - * Fill ctx->out.buf using data from the source, target, - * and inline data views. - */ - for (instructions = ctx->instructions.buf; - instructions != ctx->instructions.buf + ctx->instructions.len; - ) - if (execute_one_instruction(ctx, &instructions, &data_pos)) - return -1; - if (data_pos != ctx->data.len) - return error("invalid delta: does not copy all inline data"); - return 0; -} - -static int apply_one_window(struct line_buffer *delta, off_t *delta_len, - struct sliding_view *preimage, FILE *out) -{ - int rv = -1; - struct window ctx = WINDOW_INIT(preimage); - size_t out_len; - size_t instructions_len; - size_t data_len; - assert(delta_len); - - /* "source view" offset and length already handled; */ - if (read_length(delta, &out_len, delta_len) || - read_length(delta, &instructions_len, delta_len) || - read_length(delta, &data_len, delta_len) || - read_chunk(delta, delta_len, &ctx.instructions, instructions_len) || - read_chunk(delta, delta_len, &ctx.data, data_len)) - goto error_out; - strbuf_grow(&ctx.out, out_len); - if (apply_window_in_core(&ctx)) - goto error_out; - if (ctx.out.len != out_len) { - rv = error("invalid delta: incorrect postimage length"); - goto error_out; - } - if (write_strbuf(&ctx.out, out)) - goto error_out; - rv = 0; -error_out: - window_release(&ctx); - return rv; -} - -int svndiff0_apply(struct line_buffer *delta, off_t delta_len, - struct sliding_view *preimage, FILE *postimage) -{ - assert(delta && preimage && postimage && delta_len >= 0); - - if (read_magic(delta, &delta_len)) - return -1; - while (delta_len) { /* For each window: */ - off_t pre_off = -1; - size_t pre_len; - - if (read_offset(delta, &pre_off, &delta_len) || - read_length(delta, &pre_len, &delta_len) || - move_window(preimage, pre_off, pre_len) || - apply_one_window(delta, &delta_len, preimage, postimage)) - return -1; - } - return 0; -} diff --git a/vcs-svn/svndump.c b/vcs-svn/svndump.c deleted file mode 100644 index 08d136b8cc..0000000000 --- a/vcs-svn/svndump.c +++ /dev/null @@ -1,540 +0,0 @@ -/* - * Parse and rearrange a svnadmin dump. - * Create the dump with: - * svnadmin dump --incremental -r: >outfile - * - * Licensed under a two-clause BSD-style license. - * See LICENSE for details. - */ - -#include "cache.h" -#include "fast_export.h" -#include "line_buffer.h" -#include "strbuf.h" -#include "svndump.h" - -/* - * Compare start of string to literal of equal length; - * must be guarded by length test. - */ -#define constcmp(s, ref) memcmp(s, ref, sizeof(ref) - 1) - -#define REPORT_FILENO 3 - -#define NODEACT_REPLACE 4 -#define NODEACT_DELETE 3 -#define NODEACT_ADD 2 -#define NODEACT_CHANGE 1 -#define NODEACT_UNKNOWN 0 - -/* States: */ -#define DUMP_CTX 0 /* dump metadata */ -#define REV_CTX 1 /* revision metadata */ -#define NODE_CTX 2 /* node metadata */ -#define INTERNODE_CTX 3 /* between nodes */ - -#define DATE_RFC2822_LEN 31 - -static struct line_buffer input = LINE_BUFFER_INIT; - -static struct { - uint32_t action, srcRev, type; - off_t prop_length, text_length; - struct strbuf src, dst; - uint32_t text_delta, prop_delta; -} node_ctx; - -static struct { - uint32_t revision; - timestamp_t timestamp; - struct strbuf log, author, note; -} rev_ctx; - -static struct { - uint32_t version; - struct strbuf uuid, url; -} dump_ctx; - -static void reset_node_ctx(char *fname) -{ - node_ctx.type = 0; - node_ctx.action = NODEACT_UNKNOWN; - node_ctx.prop_length = -1; - node_ctx.text_length = -1; - strbuf_reset(&node_ctx.src); - node_ctx.srcRev = 0; - strbuf_reset(&node_ctx.dst); - if (fname) - strbuf_addstr(&node_ctx.dst, fname); - node_ctx.text_delta = 0; - node_ctx.prop_delta = 0; -} - -static void reset_rev_ctx(uint32_t revision) -{ - rev_ctx.revision = revision; - rev_ctx.timestamp = 0; - strbuf_reset(&rev_ctx.log); - strbuf_reset(&rev_ctx.author); - strbuf_reset(&rev_ctx.note); -} - -static void reset_dump_ctx(const char *url) -{ - strbuf_reset(&dump_ctx.url); - if (url) - strbuf_addstr(&dump_ctx.url, url); - dump_ctx.version = 1; - strbuf_reset(&dump_ctx.uuid); -} - -static void handle_property(const struct strbuf *key_buf, - struct strbuf *val, - uint32_t *type_set) -{ - const char *key = key_buf->buf; - size_t keylen = key_buf->len; - - switch (keylen + 1) { - case sizeof("svn:log"): - if (constcmp(key, "svn:log")) - break; - if (!val) - die("invalid dump: unsets svn:log"); - strbuf_swap(&rev_ctx.log, val); - break; - case sizeof("svn:author"): - if (constcmp(key, "svn:author")) - break; - if (!val) - strbuf_reset(&rev_ctx.author); - else - strbuf_swap(&rev_ctx.author, val); - break; - case sizeof("svn:date"): - if (constcmp(key, "svn:date")) - break; - if (!val) - die("invalid dump: unsets svn:date"); - if (parse_date_basic(val->buf, &rev_ctx.timestamp, NULL)) - warning("invalid timestamp: %s", val->buf); - break; - case sizeof("svn:executable"): - case sizeof("svn:special"): - if (keylen == strlen("svn:executable") && - constcmp(key, "svn:executable")) - break; - if (keylen == strlen("svn:special") && - constcmp(key, "svn:special")) - break; - if (*type_set) { - if (!val) - return; - die("invalid dump: sets type twice"); - } - if (!val) { - node_ctx.type = S_IFREG | 0644; - return; - } - *type_set = 1; - node_ctx.type = keylen == strlen("svn:executable") ? - (S_IFREG | 0755) : - S_IFLNK; - } -} - -static void die_short_read(void) -{ - if (buffer_ferror(&input)) - die_errno("error reading dump file"); - die("invalid dump: unexpected end of file"); -} - -static void read_props(void) -{ - static struct strbuf key = STRBUF_INIT; - static struct strbuf val = STRBUF_INIT; - const char *t; - /* - * NEEDSWORK: to support simple mode changes like - * K 11 - * svn:special - * V 1 - * * - * D 14 - * svn:executable - * we keep track of whether a mode has been set and reset to - * plain file only if not. We should be keeping track of the - * symlink and executable bits separately instead. - */ - uint32_t type_set = 0; - while ((t = buffer_read_line(&input)) && strcmp(t, "PROPS-END")) { - uint32_t len; - const char type = t[0]; - int ch; - - if (!type || t[1] != ' ') - die("invalid property line: %s", t); - len = atoi(&t[2]); - strbuf_reset(&val); - buffer_read_binary(&input, &val, len); - if (val.len < len) - die_short_read(); - - /* Discard trailing newline. */ - ch = buffer_read_char(&input); - if (ch == EOF) - die_short_read(); - if (ch != '\n') - die("invalid dump: expected newline after %s", val.buf); - - switch (type) { - case 'K': - strbuf_swap(&key, &val); - continue; - case 'D': - handle_property(&val, NULL, &type_set); - continue; - case 'V': - handle_property(&key, &val, &type_set); - strbuf_reset(&key); - continue; - default: - die("invalid property line: %s", t); - } - } -} - -static void handle_node(void) -{ - const uint32_t type = node_ctx.type; - const int have_props = node_ctx.prop_length != -1; - const int have_text = node_ctx.text_length != -1; - /* - * Old text for this node: - * NULL - directory or bug - * empty_blob - empty - * "" - data retrievable from fast-import - */ - static const char *const empty_blob = "::empty::"; - const char *old_data = NULL; - uint32_t old_mode = S_IFREG | 0644; - - if (node_ctx.action == NODEACT_DELETE) { - if (have_text || have_props || node_ctx.srcRev) - die("invalid dump: deletion node has " - "copyfrom info, text, or properties"); - fast_export_delete(node_ctx.dst.buf); - return; - } - if (node_ctx.action == NODEACT_REPLACE) { - fast_export_delete(node_ctx.dst.buf); - node_ctx.action = NODEACT_ADD; - } - if (node_ctx.srcRev) { - fast_export_copy(node_ctx.srcRev, node_ctx.src.buf, node_ctx.dst.buf); - if (node_ctx.action == NODEACT_ADD) - node_ctx.action = NODEACT_CHANGE; - } - if (have_text && type == S_IFDIR) - die("invalid dump: directories cannot have text attached"); - - /* - * Find old content (old_data) and decide on the new mode. - */ - if (node_ctx.action == NODEACT_CHANGE && !*node_ctx.dst.buf) { - if (type != S_IFDIR) - die("invalid dump: root of tree is not a regular file"); - old_data = NULL; - } else if (node_ctx.action == NODEACT_CHANGE) { - uint32_t mode; - old_data = fast_export_read_path(node_ctx.dst.buf, &mode); - if (mode == S_IFDIR && type != S_IFDIR) - die("invalid dump: cannot modify a directory into a file"); - if (mode != S_IFDIR && type == S_IFDIR) - die("invalid dump: cannot modify a file into a directory"); - node_ctx.type = mode; - old_mode = mode; - } else if (node_ctx.action == NODEACT_ADD) { - if (type == S_IFDIR) - old_data = NULL; - else if (have_text) - old_data = empty_blob; - else - die("invalid dump: adds node without text"); - } else { - die("invalid dump: Node-path block lacks Node-action"); - } - - /* - * Adjust mode to reflect properties. - */ - if (have_props) { - if (!node_ctx.prop_delta) - node_ctx.type = type; - if (node_ctx.prop_length) - read_props(); - } - - /* - * Save the result. - */ - if (type == S_IFDIR) /* directories are not tracked. */ - return; - assert(old_data); - if (old_data == empty_blob) - /* For the fast_export_* functions, NULL means empty. */ - old_data = NULL; - if (!have_text) { - fast_export_modify(node_ctx.dst.buf, node_ctx.type, old_data); - return; - } - if (!node_ctx.text_delta) { - fast_export_modify(node_ctx.dst.buf, node_ctx.type, "inline"); - fast_export_data(node_ctx.type, node_ctx.text_length, &input); - return; - } - fast_export_modify(node_ctx.dst.buf, node_ctx.type, "inline"); - fast_export_blob_delta(node_ctx.type, old_mode, old_data, - node_ctx.text_length, &input); -} - -static void begin_revision(const char *remote_ref) -{ - if (!rev_ctx.revision) /* revision 0 gets no git commit. */ - return; - fast_export_begin_commit(rev_ctx.revision, rev_ctx.author.buf, - &rev_ctx.log, dump_ctx.uuid.buf, dump_ctx.url.buf, - rev_ctx.timestamp, remote_ref); -} - -static void end_revision(const char *note_ref) -{ - struct strbuf mark = STRBUF_INIT; - if (rev_ctx.revision) { - fast_export_end_commit(rev_ctx.revision); - fast_export_begin_note(rev_ctx.revision, "remote-svn", - "Note created by remote-svn.", rev_ctx.timestamp, note_ref); - strbuf_addf(&mark, ":%"PRIu32, rev_ctx.revision); - fast_export_note(mark.buf, "inline"); - fast_export_buf_to_data(&rev_ctx.note); - strbuf_release(&mark); - } -} - -void svndump_read(const char *url, const char *local_ref, const char *notes_ref) -{ - char *val; - char *t; - uint32_t active_ctx = DUMP_CTX; - uint32_t len; - - reset_dump_ctx(url); - while ((t = buffer_read_line(&input))) { - val = strchr(t, ':'); - if (!val) - continue; - val++; - if (*val != ' ') - continue; - val++; - - /* strlen(key) + 1 */ - switch (val - t - 1) { - case sizeof("SVN-fs-dump-format-version"): - if (constcmp(t, "SVN-fs-dump-format-version")) - continue; - dump_ctx.version = atoi(val); - if (dump_ctx.version > 3) - die("expected svn dump format version <= 3, found %"PRIu32, - dump_ctx.version); - break; - case sizeof("UUID"): - if (constcmp(t, "UUID")) - continue; - strbuf_reset(&dump_ctx.uuid); - strbuf_addstr(&dump_ctx.uuid, val); - break; - case sizeof("Revision-number"): - if (constcmp(t, "Revision-number")) - continue; - if (active_ctx == NODE_CTX) - handle_node(); - if (active_ctx == REV_CTX) - begin_revision(local_ref); - if (active_ctx != DUMP_CTX) - end_revision(notes_ref); - active_ctx = REV_CTX; - reset_rev_ctx(atoi(val)); - strbuf_addf(&rev_ctx.note, "%s\n", t); - break; - case sizeof("Node-path"): - if (constcmp(t, "Node-")) - continue; - if (!constcmp(t + strlen("Node-"), "path")) { - if (active_ctx == NODE_CTX) - handle_node(); - if (active_ctx == REV_CTX) - begin_revision(local_ref); - active_ctx = NODE_CTX; - reset_node_ctx(val); - strbuf_addf(&rev_ctx.note, "%s\n", t); - break; - } - if (constcmp(t + strlen("Node-"), "kind")) - continue; - strbuf_addf(&rev_ctx.note, "%s\n", t); - if (!strcmp(val, "dir")) - node_ctx.type = S_IFDIR; - else if (!strcmp(val, "file")) - node_ctx.type = S_IFREG | 0644; - else - fprintf(stderr, "Unknown node-kind: %s\n", val); - break; - case sizeof("Node-action"): - if (constcmp(t, "Node-action")) - continue; - strbuf_addf(&rev_ctx.note, "%s\n", t); - if (!strcmp(val, "delete")) { - node_ctx.action = NODEACT_DELETE; - } else if (!strcmp(val, "add")) { - node_ctx.action = NODEACT_ADD; - } else if (!strcmp(val, "change")) { - node_ctx.action = NODEACT_CHANGE; - } else if (!strcmp(val, "replace")) { - node_ctx.action = NODEACT_REPLACE; - } else { - fprintf(stderr, "Unknown node-action: %s\n", val); - node_ctx.action = NODEACT_UNKNOWN; - } - break; - case sizeof("Node-copyfrom-path"): - if (constcmp(t, "Node-copyfrom-path")) - continue; - strbuf_reset(&node_ctx.src); - strbuf_addstr(&node_ctx.src, val); - strbuf_addf(&rev_ctx.note, "%s\n", t); - break; - case sizeof("Node-copyfrom-rev"): - if (constcmp(t, "Node-copyfrom-rev")) - continue; - node_ctx.srcRev = atoi(val); - strbuf_addf(&rev_ctx.note, "%s\n", t); - break; - case sizeof("Text-content-length"): - if (constcmp(t, "Text") && constcmp(t, "Prop")) - continue; - if (constcmp(t + 4, "-content-length")) - continue; - { - char *end; - uintmax_t len; - - len = strtoumax(val, &end, 10); - if (!isdigit(*val) || *end) - die("invalid dump: non-numeric length %s", val); - if (len > maximum_signed_value_of_type(off_t)) - die("unrepresentable length in dump: %s", val); - - if (*t == 'T') - node_ctx.text_length = (off_t) len; - else - node_ctx.prop_length = (off_t) len; - break; - } - case sizeof("Text-delta"): - if (!constcmp(t, "Text-delta")) { - node_ctx.text_delta = !strcmp(val, "true"); - break; - } - if (constcmp(t, "Prop-delta")) - continue; - node_ctx.prop_delta = !strcmp(val, "true"); - break; - case sizeof("Content-length"): - if (constcmp(t, "Content-length")) - continue; - len = atoi(val); - t = buffer_read_line(&input); - if (!t) - die_short_read(); - if (*t) - die("invalid dump: expected blank line after content length header"); - if (active_ctx == REV_CTX) { - read_props(); - } else if (active_ctx == NODE_CTX) { - handle_node(); - active_ctx = INTERNODE_CTX; - } else { - fprintf(stderr, "Unexpected content length header: %"PRIu32"\n", len); - if (buffer_skip_bytes(&input, len) != len) - die_short_read(); - } - } - } - if (buffer_ferror(&input)) - die_short_read(); - if (active_ctx == NODE_CTX) - handle_node(); - if (active_ctx == REV_CTX) - begin_revision(local_ref); - if (active_ctx != DUMP_CTX) - end_revision(notes_ref); -} - -static void init(int report_fd) -{ - fast_export_init(report_fd); - strbuf_init(&dump_ctx.uuid, 4096); - strbuf_init(&dump_ctx.url, 4096); - strbuf_init(&rev_ctx.log, 4096); - strbuf_init(&rev_ctx.author, 4096); - strbuf_init(&rev_ctx.note, 4096); - strbuf_init(&node_ctx.src, 4096); - strbuf_init(&node_ctx.dst, 4096); - reset_dump_ctx(NULL); - reset_rev_ctx(0); - reset_node_ctx(NULL); - return; -} - -int svndump_init(const char *filename) -{ - if (buffer_init(&input, filename)) - return error_errno("cannot open %s", filename ? filename : "NULL"); - init(REPORT_FILENO); - return 0; -} - -int svndump_init_fd(int in_fd, int back_fd) -{ - if(buffer_fdinit(&input, xdup(in_fd))) - return error_errno("cannot open fd %d", in_fd); - init(xdup(back_fd)); - return 0; -} - -void svndump_deinit(void) -{ - fast_export_deinit(); - reset_dump_ctx(NULL); - reset_rev_ctx(0); - reset_node_ctx(NULL); - strbuf_release(&rev_ctx.log); - strbuf_release(&rev_ctx.author); - strbuf_release(&rev_ctx.note); - strbuf_release(&node_ctx.src); - strbuf_release(&node_ctx.dst); - if (buffer_deinit(&input)) - fprintf(stderr, "Input error\n"); - if (ferror(stdout)) - fprintf(stderr, "Output error\n"); -} - -void svndump_reset(void) -{ - strbuf_release(&dump_ctx.uuid); - strbuf_release(&dump_ctx.url); - strbuf_release(&rev_ctx.log); - strbuf_release(&rev_ctx.author); -} -- cgit v1.3