From 1d389ab65dc6867d3059d60bb2b3951bc01a185e Mon Sep 17 00:00:00 2001 From: Nick Hengeveld Date: Mon, 10 Oct 2005 23:22:01 -0700 Subject: Add support for parallel HTTP transfers Add support for parallel HTTP transfers. Prefetch populates a queue of objects to transfer and starts feeding requests to an active request queue for processing; fetch_object keeps the active queue moving while the specified object is being transferred. The size of the active queue can be restricted using -r and defaults to 5 concurrent transfers. Requests for objects that are not prefetched are also processed via the active queue. Signed-off-by: Nick Hengeveld Signed-off-by: Junio C Hamano --- http-fetch.c | 810 +++++++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 587 insertions(+), 223 deletions(-) (limited to 'http-fetch.c') diff --git a/http-fetch.c b/http-fetch.c index 71a8c60b56..0b01877b5c 100644 --- a/http-fetch.c +++ b/http-fetch.c @@ -6,6 +6,8 @@ #include #include +#define DEFAULT_MAX_REQUESTS 5 + #if LIBCURL_VERSION_NUM < 0x070704 #define curl_global_cleanup() do { /* nothing */ } while(0) #endif @@ -16,13 +18,16 @@ #define PREV_BUF_SIZE 4096 #define RANGE_HEADER_SIZE 30 -static CURL *curl; +static int max_requests = DEFAULT_MAX_REQUESTS; +static int active_requests = 0; +static int data_received; + +static CURLM *curlm; +static CURL *curl_default; static struct curl_slist *no_pragma_header; static struct curl_slist *no_range_header; static char curl_errorstr[CURL_ERROR_SIZE]; -static char *initial_base; - struct alt_base { char *base; @@ -33,11 +38,46 @@ struct alt_base static struct alt_base *alt = NULL; -static SHA_CTX c; -static z_stream stream; +enum transfer_state { + WAITING, + ABORTED, + ACTIVE, + COMPLETE, +}; -static int local; -static int zret; +struct transfer_request +{ + unsigned char sha1[20]; + struct alt_base *repo; + char *url; + char filename[PATH_MAX]; + char tmpfile[PATH_MAX]; + int local; + enum transfer_state state; + CURLcode curl_result; + char errorstr[CURL_ERROR_SIZE]; + long http_code; + unsigned char real_sha1[20]; + SHA_CTX c; + z_stream stream; + int zret; + int rename; + struct active_request_slot *slot; + struct transfer_request *next; +}; + +struct active_request_slot +{ + CURL *curl; + FILE *local; + int in_use; + int done; + CURLcode curl_result; + struct active_request_slot *next; +}; + +static struct transfer_request *request_queue_head = NULL; +static struct active_request_slot *active_queue_head = NULL; static int curl_ssl_verify; static char *ssl_cert; @@ -60,6 +100,7 @@ static size_t fwrite_buffer(void *ptr, size_t eltsize, size_t nmemb, size = buffer->size - buffer->posn; memcpy(buffer->buffer + buffer->posn, ptr, size); buffer->posn += size; + data_received++; return size; } @@ -69,28 +110,28 @@ static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb, unsigned char expn[4096]; size_t size = eltsize * nmemb; int posn = 0; + struct transfer_request *request = (struct transfer_request *)data; do { - ssize_t retval = write(local, ptr + posn, size - posn); + ssize_t retval = write(request->local, + ptr + posn, size - posn); if (retval < 0) return posn; posn += retval; } while (posn < size); - stream.avail_in = size; - stream.next_in = ptr; + request->stream.avail_in = size; + request->stream.next_in = ptr; do { - stream.next_out = expn; - stream.avail_out = sizeof(expn); - zret = inflate(&stream, Z_SYNC_FLUSH); - SHA1_Update(&c, expn, sizeof(expn) - stream.avail_out); - } while (stream.avail_in && zret == Z_OK); + request->stream.next_out = expn; + request->stream.avail_out = sizeof(expn); + request->zret = inflate(&request->stream, Z_SYNC_FLUSH); + SHA1_Update(&request->c, expn, + sizeof(expn) - request->stream.avail_out); + } while (request->stream.avail_in && request->zret == Z_OK); + data_received++; return size; } -void prefetch(unsigned char *sha1) -{ -} - int relink_or_rename(char *old, char *new) { int ret; @@ -110,10 +151,369 @@ int relink_or_rename(char *old, char *new) { return 0; } +void process_curl_messages(); +void process_request_queue(); + +struct active_request_slot *get_active_slot() +{ + struct active_request_slot *slot = active_queue_head; + struct active_request_slot *newslot; + int num_transfers; + + /* Wait for a slot to open up if the queue is full */ + while (active_requests >= max_requests) { + curl_multi_perform(curlm, &num_transfers); + if (num_transfers < active_requests) { + process_curl_messages(); + } + } + + while (slot != NULL && slot->in_use) { + slot = slot->next; + } + if (slot == NULL) { + newslot = xmalloc(sizeof(*newslot)); + newslot->curl = curl_easy_duphandle(curl_default); + newslot->in_use = 0; + newslot->next = NULL; + + slot = active_queue_head; + if (slot == NULL) { + active_queue_head = newslot; + } else { + while (slot->next != NULL) { + slot = slot->next; + } + slot->next = newslot; + } + slot = newslot; + } + + active_requests++; + slot->in_use = 1; + slot->done = 0; + slot->local = NULL; + curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header); + curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_range_header); + curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, curl_errorstr); + + return slot; +} + +int start_active_slot(struct active_request_slot *slot) +{ + CURLMcode curlm_result = curl_multi_add_handle(curlm, slot->curl); + + if (curlm_result != CURLM_OK && + curlm_result != CURLM_CALL_MULTI_PERFORM) { + active_requests--; + slot->in_use = 0; + return 0; + } + + return 1; +} + +void run_active_slot(struct active_request_slot *slot) +{ + int num_transfers; + long last_pos = 0; + long current_pos; + fd_set readfds; + fd_set writefds; + fd_set excfds; + int max_fd; + struct timeval select_timeout; + CURLMcode curlm_result; + + while (!slot->done) { + data_received = 0; + do { + curlm_result = curl_multi_perform(curlm, + &num_transfers); + } while (curlm_result == CURLM_CALL_MULTI_PERFORM); + if (num_transfers < active_requests) { + process_curl_messages(); + process_request_queue(); + } + + if (!data_received && slot->local != NULL) { + current_pos = ftell(slot->local); + if (current_pos > last_pos) + data_received++; + last_pos = current_pos; + } + + if (!slot->done && !data_received) { + max_fd = 0; + FD_ZERO(&readfds); + FD_ZERO(&writefds); + FD_ZERO(&excfds); + select_timeout.tv_sec = 0; + select_timeout.tv_usec = 50000; + select(max_fd, &readfds, &writefds, + &excfds, &select_timeout); + } + } +} + +void start_request(struct transfer_request *request) +{ + char *hex = sha1_to_hex(request->sha1); + char prevfile[PATH_MAX]; + char *url; + char *posn; + int prevlocal; + unsigned char prev_buf[PREV_BUF_SIZE]; + ssize_t prev_read = 0; + long prev_posn = 0; + char range[RANGE_HEADER_SIZE]; + struct curl_slist *range_header = NULL; + struct active_request_slot *slot; + + snprintf(prevfile, sizeof(prevfile), "%s.prev", request->filename); + unlink(prevfile); + rename(request->tmpfile, prevfile); + unlink(request->tmpfile); + + request->local = open(request->tmpfile, + O_WRONLY | O_CREAT | O_EXCL, 0666); + if (request->local < 0) { + request->state = ABORTED; + error("Couldn't create temporary file %s for %s: %s\n", + request->tmpfile, request->filename, strerror(errno)); + return; + } + + memset(&request->stream, 0, sizeof(request->stream)); + + inflateInit(&request->stream); + + SHA1_Init(&request->c); + + url = xmalloc(strlen(request->repo->base) + 50); + request->url = xmalloc(strlen(request->repo->base) + 50); + strcpy(url, request->repo->base); + posn = url + strlen(request->repo->base); + strcpy(posn, "objects/"); + posn += 8; + memcpy(posn, hex, 2); + posn += 2; + *(posn++) = '/'; + strcpy(posn, hex + 2); + strcpy(request->url, url); + + /* If a previous temp file is present, process what was already + fetched. */ + prevlocal = open(prevfile, O_RDONLY); + if (prevlocal != -1) { + do { + prev_read = read(prevlocal, prev_buf, PREV_BUF_SIZE); + if (prev_read>0) { + if (fwrite_sha1_file(prev_buf, + 1, + prev_read, + request) == prev_read) { + prev_posn += prev_read; + } else { + prev_read = -1; + } + } + } while (prev_read > 0); + close(prevlocal); + } + unlink(prevfile); + + /* Reset inflate/SHA1 if there was an error reading the previous temp + file; also rewind to the beginning of the local file. */ + if (prev_read == -1) { + memset(&request->stream, 0, sizeof(request->stream)); + inflateInit(&request->stream); + SHA1_Init(&request->c); + if (prev_posn>0) { + prev_posn = 0; + lseek(request->local, SEEK_SET, 0); + ftruncate(request->local, 0); + } + } + + slot = get_active_slot(); + curl_easy_setopt(slot->curl, CURLOPT_FILE, request); + curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_sha1_file); + curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, request->errorstr); + curl_easy_setopt(slot->curl, CURLOPT_URL, url); + + /* If we have successfully processed data from a previous fetch + attempt, only fetch the data we don't already have. */ + if (prev_posn>0) { + if (get_verbosely) + fprintf(stderr, + "Resuming fetch of object %s at byte %ld\n", + hex, prev_posn); + sprintf(range, "Range: bytes=%ld-", prev_posn); + range_header = curl_slist_append(range_header, range); + curl_easy_setopt(slot->curl, + CURLOPT_HTTPHEADER, range_header); + } + + /* Try to add to multi handle, abort the request on error */ + if (!start_active_slot(slot)) { + request->state = ABORTED; + close(request->local); + free(request->url); + return; + } + + request->slot = slot; + request->state = ACTIVE; +} + +void finish_request(struct transfer_request *request) +{ + fchmod(request->local, 0444); + close(request->local); + + if (request->http_code == 416) { + fprintf(stderr, "Warning: requested range invalid; we may already have all the data.\n"); + } else if (request->curl_result != CURLE_OK) { + return; + } + + inflateEnd(&request->stream); + SHA1_Final(request->real_sha1, &request->c); + if (request->zret != Z_STREAM_END) { + unlink(request->tmpfile); + return; + } + if (memcmp(request->sha1, request->real_sha1, 20)) { + unlink(request->tmpfile); + return; + } + request->rename = + relink_or_rename(request->tmpfile, request->filename); + + if (request->rename == 0) + pull_say("got %s\n", sha1_to_hex(request->sha1)); +} + +void release_request(struct transfer_request *request) +{ + struct transfer_request *entry = request_queue_head; + + if (request == request_queue_head) { + request_queue_head = request->next; + } else { + while (entry->next != NULL && entry->next != request) + entry = entry->next; + if (entry->next == request) + entry->next = entry->next->next; + } + + free(request->url); + free(request); +} + +void process_curl_messages() +{ + int num_messages; + struct active_request_slot *slot; + struct transfer_request *request = NULL; + CURLMsg *curl_message = curl_multi_info_read(curlm, &num_messages); + + while (curl_message != NULL) { + if (curl_message->msg == CURLMSG_DONE) { + slot = active_queue_head; + while (slot != NULL && + slot->curl != curl_message->easy_handle) + slot = slot->next; + if (slot != NULL) { + curl_multi_remove_handle(curlm, slot->curl); + active_requests--; + slot->done = 1; + slot->in_use = 0; + slot->curl_result = curl_message->data.result; + request = request_queue_head; + while (request != NULL && + request->slot != slot) + request = request->next; + } else { + fprintf(stderr, "Received DONE message for unknown request!\n"); + } + if (request != NULL) { + request->curl_result = + curl_message->data.result; + curl_easy_getinfo(slot->curl, + CURLINFO_HTTP_CODE, + &request->http_code); + request->slot = NULL; + + /* Use alternates if necessary */ + if (request->http_code == 404 && + request->repo->next != NULL) { + request->repo = request->repo->next; + start_request(request); + } else { + finish_request(request); + request->state = COMPLETE; + } + } + } else { + fprintf(stderr, "Unknown CURL message received: %d\n", + (int)curl_message->msg); + } + curl_message = curl_multi_info_read(curlm, &num_messages); + } +} + +void process_request_queue() +{ + struct transfer_request *request = request_queue_head; + int num_transfers; + + while (active_requests < max_requests && request != NULL) { + if (request->state == WAITING) { + start_request(request); + curl_multi_perform(curlm, &num_transfers); + } + request = request->next; + } +} + +void prefetch(unsigned char *sha1) +{ + struct transfer_request *newreq; + struct transfer_request *tail; + char *filename = sha1_file_name(sha1); + + newreq = xmalloc(sizeof(*newreq)); + memcpy(newreq->sha1, sha1, 20); + newreq->repo = alt; + newreq->url = NULL; + newreq->local = -1; + newreq->state = WAITING; + snprintf(newreq->filename, sizeof(newreq->filename), "%s", filename); + snprintf(newreq->tmpfile, sizeof(newreq->tmpfile), + "%s.temp", filename); + newreq->next = NULL; + + if (request_queue_head == NULL) { + request_queue_head = newreq; + } else { + tail = request_queue_head; + while (tail->next != NULL) { + tail = tail->next; + } + tail->next = newreq; + } + process_request_queue(); + process_curl_messages(); +} + static int got_alternates = 0; static int fetch_index(struct alt_base *repo, unsigned char *sha1) { + char *hex = sha1_to_hex(sha1); char *filename; char *url; char tmpfile[PATH_MAX]; @@ -121,20 +521,18 @@ static int fetch_index(struct alt_base *repo, unsigned char *sha1) long prev_posn = 0; char range[RANGE_HEADER_SIZE]; struct curl_slist *range_header = NULL; - CURLcode curl_result; FILE *indexfile; + struct active_request_slot *slot; if (has_pack_index(sha1)) return 0; if (get_verbosely) - fprintf(stderr, "Getting index for pack %s\n", - sha1_to_hex(sha1)); + fprintf(stderr, "Getting index for pack %s\n", hex); url = xmalloc(strlen(repo->base) + 64); - sprintf(url, "%s/objects/pack/pack-%s.idx", - repo->base, sha1_to_hex(sha1)); + sprintf(url, "%s/objects/pack/pack-%s.idx", repo->base, hex); filename = sha1_pack_index_name(sha1); snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename); @@ -143,12 +541,12 @@ static int fetch_index(struct alt_base *repo, unsigned char *sha1) return error("Unable to open local file %s for pack index", filename); - curl_easy_setopt(curl, CURLOPT_FILE, indexfile); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fwrite); - curl_easy_setopt(curl, CURLOPT_URL, url); - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, no_pragma_header); - curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, curl_errorstr); - + slot = get_active_slot(); + curl_easy_setopt(slot->curl, CURLOPT_FILE, indexfile); + curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite); + curl_easy_setopt(slot->curl, CURLOPT_URL, url); + slot->local = indexfile; + /* If there is data present from a previous transfer attempt, resume where it left off */ prev_posn = ftell(indexfile); @@ -156,20 +554,21 @@ static int fetch_index(struct alt_base *repo, unsigned char *sha1) if (get_verbosely) fprintf(stderr, "Resuming fetch of index for pack %s at byte %ld\n", - sha1_to_hex(sha1), prev_posn); + hex, prev_posn); sprintf(range, "Range: bytes=%ld-", prev_posn); range_header = curl_slist_append(range_header, range); - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, range_header); + curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header); } - /* Clear out the Range: header after performing the request, so - other curl requests don't inherit inappropriate header data */ - curl_result = curl_easy_perform(curl); - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, no_range_header); - if (curl_result != 0) { - fclose(indexfile); - return error("Unable to get pack index %s\n%s", url, - curl_errorstr); + if (start_active_slot(slot)) { + run_active_slot(slot); + if (slot->curl_result != CURLE_OK) { + fclose(indexfile); + return error("Unable to get pack index %s\n%s", url, + curl_errorstr); + } + } else { + return error("Unable to start request"); } fclose(indexfile); @@ -205,6 +604,9 @@ static int fetch_alternates(char *base) char *data; int i = 0; int http_specific = 1; + struct alt_base *tail = alt; + + struct active_request_slot *slot; if (got_alternates) return 0; data = xmalloc(4096); @@ -218,22 +620,31 @@ static int fetch_alternates(char *base) url = xmalloc(strlen(base) + 31); sprintf(url, "%s/objects/info/http-alternates", base); - curl_easy_setopt(curl, CURLOPT_FILE, &buffer); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fwrite_buffer); - curl_easy_setopt(curl, CURLOPT_URL, url); - - if (curl_easy_perform(curl) || !buffer.posn) { - http_specific = 0; - - sprintf(url, "%s/objects/info/alternates", base); - - curl_easy_setopt(curl, CURLOPT_FILE, &buffer); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fwrite_buffer); - curl_easy_setopt(curl, CURLOPT_URL, url); - - if (curl_easy_perform(curl)) { - return 0; + slot = get_active_slot(); + curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer); + curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer); + curl_easy_setopt(slot->curl, CURLOPT_URL, url); + if (start_active_slot(slot)) { + run_active_slot(slot); + if (slot->curl_result != CURLE_OK || !buffer.posn) { + http_specific = 0; + + sprintf(url, "%s/objects/info/alternates", base); + + slot = get_active_slot(); + curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer); + curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, + fwrite_buffer); + curl_easy_setopt(slot->curl, CURLOPT_URL, url); + if (start_active_slot(slot)) { + run_active_slot(slot); + if (slot->curl_result != CURLE_OK) { + return 0; + } + } } + } else { + return 0; } data[buffer.posn] = '\0'; @@ -283,11 +694,13 @@ static int fetch_alternates(char *base) fprintf(stderr, "Also look at %s\n", target); newalt = xmalloc(sizeof(*newalt)); - newalt->next = alt; + newalt->next = NULL; newalt->base = target; newalt->got_indices = 0; newalt->packs = NULL; - alt = newalt; + while (tail->next != NULL) + tail = tail->next; + tail->next = newalt; ret++; } } @@ -306,6 +719,8 @@ static int fetch_indices(struct alt_base *repo) char *data; int i = 0; + struct active_request_slot *slot; + if (repo->got_indices) return 0; @@ -320,14 +735,18 @@ static int fetch_indices(struct alt_base *repo) url = xmalloc(strlen(repo->base) + 21); sprintf(url, "%s/objects/info/packs", repo->base); - curl_easy_setopt(curl, CURLOPT_FILE, &buffer); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fwrite_buffer); - curl_easy_setopt(curl, CURLOPT_URL, url); - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, NULL); - curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, curl_errorstr); - - if (curl_easy_perform(curl)) - return error("%s", curl_errorstr); + slot = get_active_slot(); + curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer); + curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer); + curl_easy_setopt(slot->curl, CURLOPT_URL, url); + curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL); + if (start_active_slot(slot)) { + run_active_slot(slot); + if (slot->curl_result != CURLE_OK) + return error("%s", curl_errorstr); + } else { + return error("Unable to start request"); + } while (i < buffer.posn) { switch (data[i]) { @@ -364,7 +783,8 @@ static int fetch_pack(struct alt_base *repo, unsigned char *sha1) long prev_posn = 0; char range[RANGE_HEADER_SIZE]; struct curl_slist *range_header = NULL; - CURLcode curl_result; + + struct active_request_slot *slot; if (fetch_indices(repo)) return -1; @@ -390,11 +810,11 @@ static int fetch_pack(struct alt_base *repo, unsigned char *sha1) return error("Unable to open local file %s for pack", filename); - curl_easy_setopt(curl, CURLOPT_FILE, packfile); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fwrite); - curl_easy_setopt(curl, CURLOPT_URL, url); - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, no_pragma_header); - curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, curl_errorstr); + slot = get_active_slot(); + curl_easy_setopt(slot->curl, CURLOPT_FILE, packfile); + curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite); + curl_easy_setopt(slot->curl, CURLOPT_URL, url); + slot->local = packfile; /* If there is data present from a previous transfer attempt, resume where it left off */ @@ -406,17 +826,18 @@ static int fetch_pack(struct alt_base *repo, unsigned char *sha1) sha1_to_hex(target->sha1), prev_posn); sprintf(range, "Range: bytes=%ld-", prev_posn); range_header = curl_slist_append(range_header, range); - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, range_header); + curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header); } - /* Clear out the Range: header after performing the request, so - other curl requests don't inherit inappropriate header data */ - curl_result = curl_easy_perform(curl); - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, no_range_header); - if (curl_result != 0) { - fclose(packfile); - return error("Unable to get pack file %s\n%s", url, - curl_errorstr); + if (start_active_slot(slot)) { + run_active_slot(slot); + if (slot->curl_result != CURLE_OK) { + fclose(packfile); + return error("Unable to get pack file %s\n%s", url, + curl_errorstr); + } + } else { + return error("Unable to start request"); } fclose(packfile); @@ -441,155 +862,73 @@ static int fetch_pack(struct alt_base *repo, unsigned char *sha1) static int fetch_object(struct alt_base *repo, unsigned char *sha1) { char *hex = sha1_to_hex(sha1); - char *filename = sha1_file_name(sha1); - unsigned char real_sha1[20]; - char tmpfile[PATH_MAX]; - char prevfile[PATH_MAX]; int ret; - char *url; - char *posn; - int prevlocal; - unsigned char prev_buf[PREV_BUF_SIZE]; - ssize_t prev_read = 0; - long prev_posn = 0; - char range[RANGE_HEADER_SIZE]; - struct curl_slist *range_header = NULL; - CURLcode curl_result; - - snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename); - snprintf(prevfile, sizeof(prevfile), "%s.prev", filename); - - if (unlink(prevfile) && (errno != ENOENT)) - return error("Failed to unlink %s (%s)", - prevfile, strerror(errno)); - if (rename(tmpfile, prevfile) && (errno != ENOENT)) - return error("Failed to rename %s to %s (%s)", - tmpfile, prevfile, strerror(errno)); - - local = open(tmpfile, O_WRONLY | O_CREAT | O_EXCL, 0666); - - /* Note: if another instance starts now, it will turn our new - tmpfile into its prevfile. */ - - if (local < 0) - return error("Couldn't create temporary file %s for %s: %s\n", - tmpfile, filename, strerror(errno)); - - memset(&stream, 0, sizeof(stream)); - - inflateInit(&stream); - - SHA1_Init(&c); - - curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1); - curl_easy_setopt(curl, CURLOPT_FILE, NULL); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fwrite_sha1_file); - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, no_pragma_header); - curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, curl_errorstr); - - url = xmalloc(strlen(repo->base) + 50); - strcpy(url, repo->base); - posn = url + strlen(repo->base); - strcpy(posn, "objects/"); - posn += 8; - memcpy(posn, hex, 2); - posn += 2; - *(posn++) = '/'; - strcpy(posn, hex + 2); + struct transfer_request *request = request_queue_head; + int num_transfers; + + while (request != NULL && memcmp(request->sha1, sha1, 20)) + request = request->next; + if (request == NULL) + return error("Couldn't find request for %s in the queue", hex); + + while (request->state == WAITING) { + curl_multi_perform(curlm, &num_transfers); + if (num_transfers < active_requests) { + process_curl_messages(); + process_request_queue(); + } + } - curl_easy_setopt(curl, CURLOPT_URL, url); + if (request->state == ACTIVE) + run_active_slot(request->slot); - /* If a previous temp file is present, process what was already - fetched. */ - prevlocal = open(prevfile, O_RDONLY); - if (prevlocal != -1) { - do { - prev_read = read(prevlocal, prev_buf, PREV_BUF_SIZE); - if (prev_read>0) { - if (fwrite_sha1_file(prev_buf, - 1, - prev_read, - NULL) == prev_read) { - prev_posn += prev_read; - } else { - prev_read = -1; - } - } - } while (prev_read > 0); - close(prevlocal); + if (request->state == ABORTED) { + release_request(request); + return error("Request for %s aborted", hex); } - unlink(prevfile); - /* Reset inflate/SHA1 if there was an error reading the previous temp - file; also rewind to the beginning of the local file. */ - if (prev_read == -1) { - memset(&stream, 0, sizeof(stream)); - inflateInit(&stream); - SHA1_Init(&c); - if (prev_posn>0) { - prev_posn = 0; - lseek(local, SEEK_SET, 0); - ftruncate(local, 0); - } + if (request->curl_result != CURLE_OK && request->http_code != 416) { + ret = error("%s", request->errorstr); + release_request(request); + return ret; } - /* If we have successfully processed data from a previous fetch - attempt, only fetch the data we don't already have. */ - if (prev_posn>0) { - if (get_verbosely) - fprintf(stderr, - "Resuming fetch of object %s at byte %ld\n", - hex, prev_posn); - sprintf(range, "Range: bytes=%ld-", prev_posn); - range_header = curl_slist_append(range_header, range); - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, range_header); + if (request->zret != Z_STREAM_END) { + ret = error("File %s (%s) corrupt\n", hex, request->url); + release_request(request); + return ret; } - /* Clear out the Range: header after performing the request, so - other curl requests don't inherit inappropriate header data */ - curl_result = curl_easy_perform(curl); - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, no_range_header); - if (curl_result != 0) { - return error("%s", curl_errorstr); + if (memcmp(request->sha1, request->real_sha1, 20)) { + release_request(request); + return error("File %s has bad hash\n", hex); } - fchmod(local, 0444); - close(local); - inflateEnd(&stream); - SHA1_Final(real_sha1, &c); - if (zret != Z_STREAM_END) { - unlink(tmpfile); - return error("File %s (%s) corrupt\n", hex, url); - } - if (memcmp(sha1, real_sha1, 20)) { - unlink(tmpfile); - return error("File %s has bad hash\n", hex); + if (request->rename < 0) { + ret = error("unable to write sha1 filename %s: %s", + request->filename, + strerror(request->rename)); + release_request(request); + return ret; } - ret = relink_or_rename(tmpfile, filename); - if (ret) - return error("unable to write sha1 filename %s: %s", - filename, strerror(ret)); - pull_say("got %s\n", hex); + release_request(request); return 0; } int fetch(unsigned char *sha1) { struct alt_base *altbase = alt; + + if (!fetch_object(altbase, sha1)) + return 0; while (altbase) { - if (!fetch_object(altbase, sha1)) - return 0; if (!fetch_pack(altbase, sha1)) return 0; - if (fetch_alternates(altbase->base) > 0) { - altbase = alt; - continue; - } altbase = altbase->next; } return error("Unable to find %s under %s\n", sha1_to_hex(sha1), - initial_base); + alt->base); } int fetch_ref(char *ref, unsigned char *sha1) @@ -597,17 +936,13 @@ int fetch_ref(char *ref, unsigned char *sha1) char *url, *posn; char hex[42]; struct buffer buffer; - char *base = initial_base; + char *base = alt->base; + struct active_request_slot *slot; buffer.size = 41; buffer.posn = 0; buffer.buffer = hex; hex[41] = '\0'; - curl_easy_setopt(curl, CURLOPT_FILE, &buffer); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fwrite_buffer); - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, NULL); - curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, curl_errorstr); - url = xmalloc(strlen(base) + 6 + strlen(ref)); strcpy(url, base); posn = url + strlen(base); @@ -615,11 +950,19 @@ int fetch_ref(char *ref, unsigned char *sha1) posn += 5; strcpy(posn, ref); - curl_easy_setopt(curl, CURLOPT_URL, url); - - if (curl_easy_perform(curl)) - return error("Couldn't get %s for %s\n%s", - url, ref, curl_errorstr); + slot = get_active_slot(); + curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer); + curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer); + curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL); + curl_easy_setopt(slot->curl, CURLOPT_URL, url); + if (start_active_slot(slot)) { + run_active_slot(slot); + if (slot->curl_result != CURLE_OK) + return error("Couldn't get %s for %s\n%s", + url, ref, curl_errorstr); + } else { + return error("Unable to start request"); + } hex[40] = '\0'; get_sha1_hex(hex, sha1); @@ -631,6 +974,7 @@ int main(int argc, char **argv) char *commit_id; char *url; int arg = 1; + struct active_request_slot *slot; while (arg < argc && argv[arg][0] == '-') { if (argv[arg][1] == 't') { @@ -648,6 +992,11 @@ int main(int argc, char **argv) arg++; } else if (!strcmp(argv[arg], "--recover")) { get_recover = 1; + } else if (argv[arg][1] == 'r') { + max_requests = atoi(argv[arg + 1]); + if (max_requests < 1) + max_requests = DEFAULT_MAX_REQUESTS; + arg++; } arg++; } @@ -660,44 +1009,59 @@ int main(int argc, char **argv) curl_global_init(CURL_GLOBAL_ALL); - curl = curl_easy_init(); + curlm = curl_multi_init(); + if (curlm == NULL) { + fprintf(stderr, "Error creating curl multi handle.\n"); + return 1; + } no_pragma_header = curl_slist_append(no_pragma_header, "Pragma:"); no_range_header = curl_slist_append(no_range_header, "Range:"); + curl_default = curl_easy_init(); + curl_ssl_verify = getenv("GIT_SSL_NO_VERIFY") ? 0 : 1; - curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, curl_ssl_verify); + curl_easy_setopt(curl_default, CURLOPT_SSL_VERIFYPEER, curl_ssl_verify); #if LIBCURL_VERSION_NUM >= 0x070907 - curl_easy_setopt(curl, CURLOPT_NETRC, CURL_NETRC_OPTIONAL); + curl_easy_setopt(curl_default, CURLOPT_NETRC, CURL_NETRC_OPTIONAL); #endif if ((ssl_cert = getenv("GIT_SSL_CERT")) != NULL) { - curl_easy_setopt(curl, CURLOPT_SSLCERT, ssl_cert); + curl_easy_setopt(curl_default, CURLOPT_SSLCERT, ssl_cert); } #if LIBCURL_VERSION_NUM >= 0x070902 if ((ssl_key = getenv("GIT_SSL_KEY")) != NULL) { - curl_easy_setopt(curl, CURLOPT_SSLKEY, ssl_key); + curl_easy_setopt(curl_default, CURLOPT_SSLKEY, ssl_key); } #endif #if LIBCURL_VERSION_NUM >= 0x070908 if ((ssl_capath = getenv("GIT_SSL_CAPATH")) != NULL) { - curl_easy_setopt(curl, CURLOPT_CAPATH, ssl_capath); + curl_easy_setopt(curl_default, CURLOPT_CAPATH, ssl_capath); } #endif if ((ssl_cainfo = getenv("GIT_SSL_CAINFO")) != NULL) { - curl_easy_setopt(curl, CURLOPT_CAINFO, ssl_cainfo); + curl_easy_setopt(curl_default, CURLOPT_CAINFO, ssl_cainfo); } + curl_easy_setopt(curl_default, CURLOPT_FAILONERROR, 1); alt = xmalloc(sizeof(*alt)); alt->base = url; alt->got_indices = 0; alt->packs = NULL; alt->next = NULL; - initial_base = url; + fetch_alternates(alt->base); if (pull(commit_id)) return 1; curl_slist_free_all(no_pragma_header); + curl_slist_free_all(no_range_header); + curl_easy_cleanup(curl_default); + slot = active_queue_head; + while (slot != NULL) { + curl_easy_cleanup(slot->curl); + slot = slot->next; + } + curl_multi_cleanup(curlm); curl_global_cleanup(); return 0; } -- cgit v1.3-5-g9baa From a7a8d3786e6b60a472d0f88f778cacaead122a6c Mon Sep 17 00:00:00 2001 From: Nick Hengeveld Date: Mon, 10 Oct 2005 23:22:01 -0700 Subject: Only compile parallel HTTP support with CURL >= 7.9.8 Only compile parallel HTTP support with CURL >= 7.9.8 Signed-off-by: Nick Hengeveld Signed-off-by: Junio C Hamano --- http-fetch.c | 62 +++++++++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 57 insertions(+), 5 deletions(-) (limited to 'http-fetch.c') diff --git a/http-fetch.c b/http-fetch.c index 0b01877b5c..b034a4508a 100644 --- a/http-fetch.c +++ b/http-fetch.c @@ -6,7 +6,10 @@ #include #include +#if LIBCURL_VERSION_NUM >= 0x070908 +#define USE_CURL_MULTI #define DEFAULT_MAX_REQUESTS 5 +#endif #if LIBCURL_VERSION_NUM < 0x070704 #define curl_global_cleanup() do { /* nothing */ } while(0) @@ -18,11 +21,13 @@ #define PREV_BUF_SIZE 4096 #define RANGE_HEADER_SIZE 30 -static int max_requests = DEFAULT_MAX_REQUESTS; static int active_requests = 0; static int data_received; +#ifdef USE_CURL_MULTI +static int max_requests = DEFAULT_MAX_REQUESTS; static CURLM *curlm; +#endif static CURL *curl_default; static struct curl_slist *no_pragma_header; static struct curl_slist *no_range_header; @@ -151,13 +156,17 @@ int relink_or_rename(char *old, char *new) { return 0; } +#ifdef USE_CURL_MULTI void process_curl_messages(); void process_request_queue(); +#endif struct active_request_slot *get_active_slot() { struct active_request_slot *slot = active_queue_head; struct active_request_slot *newslot; + +#ifdef USE_CURL_MULTI int num_transfers; /* Wait for a slot to open up if the queue is full */ @@ -167,6 +176,7 @@ struct active_request_slot *get_active_slot() process_curl_messages(); } } +#endif while (slot != NULL && slot->in_use) { slot = slot->next; @@ -202,6 +212,7 @@ struct active_request_slot *get_active_slot() int start_active_slot(struct active_request_slot *slot) { +#ifdef USE_CURL_MULTI CURLMcode curlm_result = curl_multi_add_handle(curlm, slot->curl); if (curlm_result != CURLM_OK && @@ -210,12 +221,13 @@ int start_active_slot(struct active_request_slot *slot) slot->in_use = 0; return 0; } - +#endif return 1; } void run_active_slot(struct active_request_slot *slot) { +#ifdef USE_CURL_MULTI int num_transfers; long last_pos = 0; long current_pos; @@ -255,6 +267,10 @@ void run_active_slot(struct active_request_slot *slot) &excfds, &select_timeout); } } +#else + slot->curl_result = curl_easy_perform(slot->curl); + active_requests--; +#endif } void start_request(struct transfer_request *request) @@ -356,7 +372,7 @@ void start_request(struct transfer_request *request) CURLOPT_HTTPHEADER, range_header); } - /* Try to add to multi handle, abort the request on error */ + /* Try to get the request started, abort the request on error */ if (!start_active_slot(slot)) { request->state = ABORTED; close(request->local); @@ -413,6 +429,7 @@ void release_request(struct transfer_request *request) free(request); } +#ifdef USE_CURL_MULTI void process_curl_messages() { int num_messages; @@ -478,6 +495,7 @@ void process_request_queue() request = request->next; } } +#endif void prefetch(unsigned char *sha1) { @@ -505,8 +523,10 @@ void prefetch(unsigned char *sha1) } tail->next = newreq; } +#ifdef USE_CURL_MULTI process_request_queue(); process_curl_messages(); +#endif } static int got_alternates = 0; @@ -864,13 +884,14 @@ static int fetch_object(struct alt_base *repo, unsigned char *sha1) char *hex = sha1_to_hex(sha1); int ret; struct transfer_request *request = request_queue_head; - int num_transfers; while (request != NULL && memcmp(request->sha1, sha1, 20)) request = request->next; if (request == NULL) return error("Couldn't find request for %s in the queue", hex); +#ifdef USE_CURL_MULTI + int num_transfers; while (request->state == WAITING) { curl_multi_perform(curlm, &num_transfers); if (num_transfers < active_requests) { @@ -878,9 +899,30 @@ static int fetch_object(struct alt_base *repo, unsigned char *sha1) process_request_queue(); } } +#else + start_request(request); +#endif - if (request->state == ACTIVE) + while (request->state == ACTIVE) { run_active_slot(request->slot); +#ifndef USE_CURL_MULTI + request->curl_result = request->slot->curl_result; + curl_easy_getinfo(request->slot->curl, + CURLINFO_HTTP_CODE, + &request->http_code); + request->slot = NULL; + + /* Use alternates if necessary */ + if (request->http_code == 404 && + request->repo->next != NULL) { + request->repo = request->repo->next; + start_request(request); + } else { + finish_request(request); + request->state = COMPLETE; + } +#endif + } if (request->state == ABORTED) { release_request(request); @@ -992,16 +1034,22 @@ int main(int argc, char **argv) arg++; } else if (!strcmp(argv[arg], "--recover")) { get_recover = 1; +#ifdef USE_CURL_MULTI } else if (argv[arg][1] == 'r') { max_requests = atoi(argv[arg + 1]); if (max_requests < 1) max_requests = DEFAULT_MAX_REQUESTS; arg++; +#endif } arg++; } if (argc < arg + 2) { +#ifdef USE_CURL_MULTI + usage("git-http-fetch [-c] [-t] [-a] [-d] [-v] [-r concurrent-request-limit] [--recover] [-w ref] commit-id url"); +#else usage("git-http-fetch [-c] [-t] [-a] [-d] [-v] [--recover] [-w ref] commit-id url"); +#endif return 1; } commit_id = argv[arg]; @@ -1009,11 +1057,13 @@ int main(int argc, char **argv) curl_global_init(CURL_GLOBAL_ALL); +#ifdef USE_CURL_MULTI curlm = curl_multi_init(); if (curlm == NULL) { fprintf(stderr, "Error creating curl multi handle.\n"); return 1; } +#endif no_pragma_header = curl_slist_append(no_pragma_header, "Pragma:"); no_range_header = curl_slist_append(no_range_header, "Range:"); @@ -1061,7 +1111,9 @@ int main(int argc, char **argv) curl_easy_cleanup(slot->curl); slot = slot->next; } +#ifdef USE_CURL_MULTI curl_multi_cleanup(curlm); +#endif curl_global_cleanup(); return 0; } -- cgit v1.3-5-g9baa From 380792390e05e744f9d7eefbc35d1db80e44e27a Mon Sep 17 00:00:00 2001 From: Nick Hengeveld Date: Mon, 10 Oct 2005 23:22:01 -0700 Subject: Set the parallel HTTP request limit via an environment variable Use an environment variable rather than a command-line argument to set the parallel HTTP request limit. This allows the setting to work whether git-http-fetch is run directly or via git-fetch. Signed-off-by: Nick Hengeveld Signed-off-by: Junio C Hamano --- http-fetch.c | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) (limited to 'http-fetch.c') diff --git a/http-fetch.c b/http-fetch.c index b034a4508a..dd9ea4ca17 100644 --- a/http-fetch.c +++ b/http-fetch.c @@ -1034,22 +1034,11 @@ int main(int argc, char **argv) arg++; } else if (!strcmp(argv[arg], "--recover")) { get_recover = 1; -#ifdef USE_CURL_MULTI - } else if (argv[arg][1] == 'r') { - max_requests = atoi(argv[arg + 1]); - if (max_requests < 1) - max_requests = DEFAULT_MAX_REQUESTS; - arg++; -#endif } arg++; } if (argc < arg + 2) { -#ifdef USE_CURL_MULTI - usage("git-http-fetch [-c] [-t] [-a] [-d] [-v] [-r concurrent-request-limit] [--recover] [-w ref] commit-id url"); -#else usage("git-http-fetch [-c] [-t] [-a] [-d] [-v] [--recover] [-w ref] commit-id url"); -#endif return 1; } commit_id = argv[arg]; @@ -1058,6 +1047,12 @@ int main(int argc, char **argv) curl_global_init(CURL_GLOBAL_ALL); #ifdef USE_CURL_MULTI + char *http_max_requests = getenv("GIT_HTTP_MAX_REQUESTS"); + if (http_max_requests != NULL) + max_requests = atoi(http_max_requests); + if (max_requests < 1) + max_requests = DEFAULT_MAX_REQUESTS; + curlm = curl_multi_init(); if (curlm == NULL) { fprintf(stderr, "Error creating curl multi handle.\n"); -- cgit v1.3-5-g9baa From 11f0dafe2be419240c0006c3e9112cbad3568baf Mon Sep 17 00:00:00 2001 From: Nick Hengeveld Date: Mon, 10 Oct 2005 23:22:01 -0700 Subject: [PATCH] Don't fetch objects that exist in the local repository Be sure not to fetch objects that already exist in the local repository. The main process loop no longer performs this check, http-fetch now checks prior to starting a new request queue entry and when fetch_object() is called, and local-fetch now checks when fetch_object() is called. As discussed in this thread: http://marc.theaimsgroup.com/?t=112854890500001 Signed-off-by: Nick Hengeveld --- fetch.c | 2 +- http-fetch.c | 10 +++++++++- local-fetch.c | 5 ++++- 3 files changed, 14 insertions(+), 3 deletions(-) (limited to 'http-fetch.c') diff --git a/fetch.c b/fetch.c index 3e073d3584..73bde07aea 100644 --- a/fetch.c +++ b/fetch.c @@ -165,7 +165,7 @@ static int loop(void) * the queue because we needed to fetch it first. */ if (! (obj->flags & TO_SCAN)) { - if (!has_sha1_file(obj->sha1) && fetch(obj->sha1)) { + if (fetch(obj->sha1)) { report_missing(obj->type ? obj->type : "object", obj->sha1); diff --git a/http-fetch.c b/http-fetch.c index dd9ea4ca17..d1e4593ba7 100644 --- a/http-fetch.c +++ b/http-fetch.c @@ -489,7 +489,10 @@ void process_request_queue() while (active_requests < max_requests && request != NULL) { if (request->state == WAITING) { - start_request(request); + if (has_sha1_file(request->sha1)) + release_request(request); + else + start_request(request); curl_multi_perform(curlm, &num_transfers); } request = request->next; @@ -890,6 +893,11 @@ static int fetch_object(struct alt_base *repo, unsigned char *sha1) if (request == NULL) return error("Couldn't find request for %s in the queue", hex); + if (has_sha1_file(request->sha1)) { + release_request(request); + return 0; + } + #ifdef USE_CURL_MULTI int num_transfers; while (request->state == WAITING) { diff --git a/local-fetch.c b/local-fetch.c index a57386ca6a..87a93de02f 100644 --- a/local-fetch.c +++ b/local-fetch.c @@ -166,7 +166,10 @@ static int fetch_file(const unsigned char *sha1) int fetch(unsigned char *sha1) { - return fetch_file(sha1) && fetch_pack(sha1); + if (has_sha1_file(sha1)) + return 0; + else + return fetch_file(sha1) && fetch_pack(sha1); } int fetch_ref(char *ref, unsigned char *sha1) -- cgit v1.3-5-g9baa From 031260064840e6e8c6bc36d7856a091ba80330c2 Mon Sep 17 00:00:00 2001 From: Nick Hengeveld Date: Mon, 10 Oct 2005 23:22:01 -0700 Subject: Restore functionality to allow proxies to cache objects The parallel request changes didn't properly implement the previous patch to allow caching of retrieved objects by proxy servers. Restore the previous functionality such that by default requests include the "Pragma: no-cache" header, and this header is removed on requests for pack indexes, packs, and objects. Signed-off-by: Nick Hengeveld Signed-off-by: Junio C Hamano --- http-fetch.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'http-fetch.c') diff --git a/http-fetch.c b/http-fetch.c index d1e4593ba7..5d0e3e3923 100644 --- a/http-fetch.c +++ b/http-fetch.c @@ -29,6 +29,7 @@ static int max_requests = DEFAULT_MAX_REQUESTS; static CURLM *curlm; #endif static CURL *curl_default; +static struct curl_slist *pragma_header; static struct curl_slist *no_pragma_header; static struct curl_slist *no_range_header; static char curl_errorstr[CURL_ERROR_SIZE]; @@ -203,7 +204,7 @@ struct active_request_slot *get_active_slot() slot->in_use = 1; slot->done = 0; slot->local = NULL; - curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header); + curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, pragma_header); curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_range_header); curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, curl_errorstr); @@ -358,6 +359,7 @@ void start_request(struct transfer_request *request) curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_sha1_file); curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, request->errorstr); curl_easy_setopt(slot->curl, CURLOPT_URL, url); + curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header); /* If we have successfully processed data from a previous fetch attempt, only fetch the data we don't already have. */ @@ -568,6 +570,7 @@ static int fetch_index(struct alt_base *repo, unsigned char *sha1) curl_easy_setopt(slot->curl, CURLOPT_FILE, indexfile); curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite); curl_easy_setopt(slot->curl, CURLOPT_URL, url); + curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header); slot->local = indexfile; /* If there is data present from a previous transfer attempt, @@ -837,6 +840,7 @@ static int fetch_pack(struct alt_base *repo, unsigned char *sha1) curl_easy_setopt(slot->curl, CURLOPT_FILE, packfile); curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite); curl_easy_setopt(slot->curl, CURLOPT_URL, url); + curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header); slot->local = packfile; /* If there is data present from a previous transfer attempt, @@ -1067,6 +1071,7 @@ int main(int argc, char **argv) return 1; } #endif + pragma_header = curl_slist_append(pragma_header, "Pragma: no-cache"); no_pragma_header = curl_slist_append(no_pragma_header, "Pragma:"); no_range_header = curl_slist_append(no_range_header, "Range:"); @@ -1106,6 +1111,7 @@ int main(int argc, char **argv) if (pull(commit_id)) return 1; + curl_slist_free_all(pragma_header); curl_slist_free_all(no_pragma_header); curl_slist_free_all(no_range_header); curl_easy_cleanup(curl_default); -- cgit v1.3-5-g9baa From b721e01f6efd84b04c6946553cd566f2707c964d Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Mon, 10 Oct 2005 23:22:01 -0700 Subject: Use the same move_temp_to_file in git-http-fetch. The http commit walker cannot use the same temporary file creation code because it needs to use predictable temporary filename for partial fetch continuation purposes, but the code to move the temporary file to the final location should be usable from the ordinary object creation codepath. Export move_temp_to_file from sha1_file.c and use it, while losing the custom relink_or_rename function from http-fetch.c. Also the temporary object file creation part needs to make sure the leading path exists, in preparation of the really lazy fan-out directory creation. Signed-off-by: Junio C Hamano --- cache.h | 1 + http-fetch.c | 48 ++++++++++++++++++------------------------------ sha1_file.c | 2 +- 3 files changed, 20 insertions(+), 31 deletions(-) (limited to 'http-fetch.c') diff --git a/cache.h b/cache.h index 0571282e8c..64cbcac5b8 100644 --- a/cache.h +++ b/cache.h @@ -223,6 +223,7 @@ extern int read_tree(void *buffer, unsigned long size, int stage, const char **p extern int write_sha1_from_fd(const unsigned char *sha1, int fd, char *buffer, size_t bufsize, size_t *bufposn); extern int write_sha1_to_fd(int fd, const unsigned char *sha1); +extern int move_temp_to_file(const char *tmpfile, char *filename); extern int has_sha1_pack(const unsigned char *sha1); extern int has_sha1_file(const unsigned char *sha1); diff --git a/http-fetch.c b/http-fetch.c index 5d0e3e3923..5821c9e5e9 100644 --- a/http-fetch.c +++ b/http-fetch.c @@ -138,25 +138,6 @@ static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb, return size; } -int relink_or_rename(char *old, char *new) { - int ret; - - ret = link(old, new); - if (ret < 0) { - /* Same Coda hack as in write_sha1_file(sha1_file.c) */ - ret = errno; - if (ret == EXDEV && !rename(old, new)) - return 0; - } - unlink(old); - if (ret) { - if (ret != EEXIST) - return ret; - } - - return 0; -} - #ifdef USE_CURL_MULTI void process_curl_messages(); void process_request_queue(); @@ -295,6 +276,20 @@ void start_request(struct transfer_request *request) request->local = open(request->tmpfile, O_WRONLY | O_CREAT | O_EXCL, 0666); + /* This could have failed due to the "lazy directory creation"; + * try to mkdir the last path component. + */ + if (request->local < 0 && errno == ENOENT) { + char *dir = strrchr(request->tmpfile, '/'); + if (dir) { + *dir = 0; + mkdir(request->tmpfile, 0777); + *dir = '/'; + } + request->local = open(request->tmpfile, + O_WRONLY | O_CREAT | O_EXCL, 0666); + } + if (request->local < 0) { request->state = ABORTED; error("Couldn't create temporary file %s for %s: %s\n", @@ -408,7 +403,7 @@ void finish_request(struct transfer_request *request) return; } request->rename = - relink_or_rename(request->tmpfile, request->filename); + move_temp_to_file(request->tmpfile, request->filename); if (request->rename == 0) pull_say("got %s\n", sha1_to_hex(request->sha1)); @@ -542,7 +537,6 @@ static int fetch_index(struct alt_base *repo, unsigned char *sha1) char *filename; char *url; char tmpfile[PATH_MAX]; - int ret; long prev_posn = 0; char range[RANGE_HEADER_SIZE]; struct curl_slist *range_header = NULL; @@ -599,12 +593,7 @@ static int fetch_index(struct alt_base *repo, unsigned char *sha1) fclose(indexfile); - ret = relink_or_rename(tmpfile, filename); - if (ret) - return error("unable to write index filename %s: %s", - filename, strerror(ret)); - - return 0; + return move_temp_to_file(tmpfile, filename); } static int setup_index(struct alt_base *repo, unsigned char *sha1) @@ -869,10 +858,9 @@ static int fetch_pack(struct alt_base *repo, unsigned char *sha1) fclose(packfile); - ret = relink_or_rename(tmpfile, filename); + ret = move_temp_to_file(tmpfile, filename); if (ret) - return error("unable to write pack filename %s: %s", - filename, strerror(ret)); + return ret; lst = &repo->packs; while (*lst != target) diff --git a/sha1_file.c b/sha1_file.c index baaa4c00da..6e3ea232ee 100644 --- a/sha1_file.c +++ b/sha1_file.c @@ -1287,7 +1287,7 @@ static int link_temp_to_file(const char *tmpfile, char *filename) /* * Move the just written object into its final resting place */ -static int move_temp_to_file(const char *tmpfile, char *filename) +int move_temp_to_file(const char *tmpfile, char *filename) { int ret = link_temp_to_file(tmpfile, filename); if (ret) { -- cgit v1.3-5-g9baa From dc1b5ea8781644203213e83b74cf3b432d88d067 Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Mon, 10 Oct 2005 23:22:02 -0700 Subject: Remove unused 'got_alternates' variable. The function fetch_alternates() is called only once from the main now. Signed-off-by: Junio C Hamano --- http-fetch.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) (limited to 'http-fetch.c') diff --git a/http-fetch.c b/http-fetch.c index 5821c9e5e9..e537591edb 100644 --- a/http-fetch.c +++ b/http-fetch.c @@ -529,8 +529,6 @@ void prefetch(unsigned char *sha1) #endif } -static int got_alternates = 0; - static int fetch_index(struct alt_base *repo, unsigned char *sha1) { char *hex = sha1_to_hex(sha1); @@ -622,8 +620,7 @@ static int fetch_alternates(char *base) struct alt_base *tail = alt; struct active_request_slot *slot; - if (got_alternates) - return 0; + data = xmalloc(4096); buffer.size = 4095; buffer.posn = 0; @@ -721,7 +718,6 @@ static int fetch_alternates(char *base) } i = posn + 1; } - got_alternates = 1; return ret; } -- cgit v1.3-5-g9baa From 94fa447ace56bc6514bdd66cafe8f56f0e3ce936 Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Mon, 10 Oct 2005 23:22:02 -0700 Subject: Quote reference names while fetching with curl. curl_escape ought to do this, but we should not let it quote slashes (nobody said refs/tags cannot have subdirectories), so we roll our own safer version. With this, the last part of git-clone from Martin's moodle repository that used to fail now works, which reads: $ git-http-fetch -v -a -w 'tags/MOODLE_15_MERGED **INVALID**' \ 'tags/MOODLE_15_MERGED **INVALID**' \ http://locke.catalyst.net.nz/git/moodle.git/ Signed-off-by: Junio C Hamano --- http-fetch.c | 55 +++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 47 insertions(+), 8 deletions(-) (limited to 'http-fetch.c') diff --git a/http-fetch.c b/http-fetch.c index e537591edb..0aba891754 100644 --- a/http-fetch.c +++ b/http-fetch.c @@ -969,9 +969,54 @@ int fetch(unsigned char *sha1) alt->base); } +static inline int needs_quote(int ch) +{ + switch (ch) { + case '/': case '-': case '.': + case 'A'...'Z': case 'a'...'z': case '0'...'9': + return 0; + default: + return 1; + } +} + +static inline int hex(int v) +{ + if (v < 10) return '0' + v; + else return 'A' + v - 10; +} + +static char *quote_ref_url(const char *base, const char *ref) +{ + const char *cp; + char *dp, *qref; + int len, baselen, ch; + + baselen = strlen(base); + len = baselen + 6; /* "refs/" + NUL */ + for (cp = ref; (ch = *cp) != 0; cp++, len++) + if (needs_quote(ch)) + len += 2; /* extra two hex plus replacement % */ + qref = xmalloc(len); + memcpy(qref, base, baselen); + memcpy(qref + baselen, "refs/", 5); + for (cp = ref, dp = qref + baselen + 5; (ch = *cp) != 0; cp++) { + if (needs_quote(ch)) { + *dp++ = '%'; + *dp++ = hex((ch >> 4) & 0xF); + *dp++ = hex(ch & 0xF); + } + else + *dp++ = ch; + } + *dp = 0; + + return qref; +} + int fetch_ref(char *ref, unsigned char *sha1) { - char *url, *posn; + char *url; char hex[42]; struct buffer buffer; char *base = alt->base; @@ -981,13 +1026,7 @@ int fetch_ref(char *ref, unsigned char *sha1) buffer.buffer = hex; hex[41] = '\0'; - url = xmalloc(strlen(base) + 6 + strlen(ref)); - strcpy(url, base); - posn = url + strlen(base); - strcpy(posn, "refs/"); - posn += 5; - strcpy(posn, ref); - + url = quote_ref_url(base, ref); slot = get_active_slot(); curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer); curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer); -- cgit v1.3-5-g9baa From bc8f26527064e8bd9123c293741a0a3762b8a64e Mon Sep 17 00:00:00 2001 From: Sergey Vlasov Date: Thu, 13 Oct 2005 10:49:53 -0700 Subject: git-http-fetch: Remove size limit for objects/info/{packs,alternates} git-http-fetch received objects/info/packs into a fixed-size buffer and started to fail when this file became larger than the buffer. Change it to grow the buffer dynamically, and do the same thing for objects/info/alternates. Also add missing free() calls for these buffers. Signed-off-by: Sergey Vlasov Signed-off-by: Junio C Hamano --- http-fetch.c | 43 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 36 insertions(+), 7 deletions(-) (limited to 'http-fetch.c') diff --git a/http-fetch.c b/http-fetch.c index 0aba891754..c6daf6a01f 100644 --- a/http-fetch.c +++ b/http-fetch.c @@ -110,6 +110,22 @@ static size_t fwrite_buffer(void *ptr, size_t eltsize, size_t nmemb, return size; } +static size_t fwrite_buffer_dynamic(const void *ptr, size_t eltsize, + size_t nmemb, struct buffer *buffer) +{ + size_t size = eltsize * nmemb; + if (size > buffer->size - buffer->posn) { + buffer->size = buffer->size * 3 / 2; + if (buffer->size < buffer->posn + size) + buffer->size = buffer->posn + size; + buffer->buffer = xrealloc(buffer->buffer, buffer->size); + } + memcpy(buffer->buffer + buffer->posn, ptr, size); + buffer->posn += size; + data_received++; + return size; +} + static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb, void *data) { @@ -618,11 +634,12 @@ static int fetch_alternates(char *base) int i = 0; int http_specific = 1; struct alt_base *tail = alt; + static const char null_byte = '\0'; struct active_request_slot *slot; data = xmalloc(4096); - buffer.size = 4095; + buffer.size = 4096; buffer.posn = 0; buffer.buffer = data; @@ -634,7 +651,8 @@ static int fetch_alternates(char *base) slot = get_active_slot(); curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer); - curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer); + curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, + fwrite_buffer_dynamic); curl_easy_setopt(slot->curl, CURLOPT_URL, url); if (start_active_slot(slot)) { run_active_slot(slot); @@ -646,20 +664,24 @@ static int fetch_alternates(char *base) slot = get_active_slot(); curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer); curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, - fwrite_buffer); + fwrite_buffer_dynamic); curl_easy_setopt(slot->curl, CURLOPT_URL, url); if (start_active_slot(slot)) { run_active_slot(slot); if (slot->curl_result != CURLE_OK) { + free(buffer.buffer); return 0; } } } } else { + free(buffer.buffer); return 0; } - data[buffer.posn] = '\0'; + fwrite_buffer_dynamic(&null_byte, 1, 1, &buffer); + buffer.posn--; + data = buffer.buffer; while (i < buffer.posn) { int posn = i; @@ -718,7 +740,8 @@ static int fetch_alternates(char *base) } i = posn + 1; } - + + free(buffer.buffer); return ret; } @@ -748,17 +771,22 @@ static int fetch_indices(struct alt_base *repo) slot = get_active_slot(); curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer); - curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer); + curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, + fwrite_buffer_dynamic); curl_easy_setopt(slot->curl, CURLOPT_URL, url); curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL); if (start_active_slot(slot)) { run_active_slot(slot); - if (slot->curl_result != CURLE_OK) + if (slot->curl_result != CURLE_OK) { + free(buffer.buffer); return error("%s", curl_errorstr); + } } else { + free(buffer.buffer); return error("Unable to start request"); } + data = buffer.buffer; while (i < buffer.posn) { switch (data[i]) { case 'P': @@ -778,6 +806,7 @@ static int fetch_indices(struct alt_base *repo) i++; } + free(buffer.buffer); repo->got_indices = 1; return 0; } -- cgit v1.3-5-g9baa From d402d5566fdf226697a386dfb9858e5d954e9b91 Mon Sep 17 00:00:00 2001 From: Nick Hengeveld Date: Fri, 14 Oct 2005 17:16:01 -0700 Subject: Use config file settings for http Use "http." config file settings if they exist. Environment variables still work, and they will override config file settings. Signed-off-by: Nick Hengeveld Signed-off-by: Junio C Hamano --- http-fetch.c | 109 +++++++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 88 insertions(+), 21 deletions(-) (limited to 'http-fetch.c') diff --git a/http-fetch.c b/http-fetch.c index c6daf6a01f..784aedfc70 100644 --- a/http-fetch.c +++ b/http-fetch.c @@ -25,7 +25,7 @@ static int active_requests = 0; static int data_received; #ifdef USE_CURL_MULTI -static int max_requests = DEFAULT_MAX_REQUESTS; +static int max_requests = -1; static CURLM *curlm; #endif static CURL *curl_default; @@ -85,11 +85,11 @@ struct active_request_slot static struct transfer_request *request_queue_head = NULL; static struct active_request_slot *active_queue_head = NULL; -static int curl_ssl_verify; -static char *ssl_cert; -static char *ssl_key; -static char *ssl_capath; -static char *ssl_cainfo; +static int curl_ssl_verify = -1; +static char *ssl_cert = NULL; +static char *ssl_key = NULL; +static char *ssl_capath = NULL; +static char *ssl_cainfo = NULL; struct buffer { @@ -98,6 +98,60 @@ struct buffer void *buffer; }; +static int http_options(const char *var, const char *value) +{ + if (!strcmp("http.sslverify", var)) { + if (curl_ssl_verify == -1) { + curl_ssl_verify = git_config_bool(var, value); + } + return 0; + } + + if (!strcmp("http.sslcert", var)) { + if (ssl_cert == NULL) { + ssl_cert = xmalloc(strlen(value)+1); + strcpy(ssl_cert, value); + } + return 0; + } +#if LIBCURL_VERSION_NUM >= 0x070902 + if (!strcmp("http.sslkey", var)) { + if (ssl_key == NULL) { + ssl_key = xmalloc(strlen(value)+1); + strcpy(ssl_key, value); + } + return 0; + } +#endif +#if LIBCURL_VERSION_NUM >= 0x070908 + if (!strcmp("http.sslcapath", var)) { + if (ssl_capath == NULL) { + ssl_capath = xmalloc(strlen(value)+1); + strcpy(ssl_capath, value); + } + return 0; + } +#endif + if (!strcmp("http.sslcainfo", var)) { + if (ssl_cainfo == NULL) { + ssl_cainfo = xmalloc(strlen(value)+1); + strcpy(ssl_cainfo, value); + } + return 0; + } + +#ifdef USE_CURL_MULTI + if (!strcmp("http.maxrequests", var)) { + if (max_requests == -1) + max_requests = git_config_int(var, value); + return 0; + } +#endif + + /* Fall back on the default ones */ + return git_default_config(var, value); +} + static size_t fwrite_buffer(void *ptr, size_t eltsize, size_t nmemb, struct buffer *buffer) { @@ -1114,8 +1168,6 @@ int main(int argc, char **argv) char *http_max_requests = getenv("GIT_HTTP_MAX_REQUESTS"); if (http_max_requests != NULL) max_requests = atoi(http_max_requests); - if (max_requests < 1) - max_requests = DEFAULT_MAX_REQUESTS; curlm = curl_multi_init(); if (curlm == NULL) { @@ -1123,34 +1175,49 @@ int main(int argc, char **argv) return 1; } #endif + + if (getenv("GIT_SSL_NO_VERIFY")) + curl_ssl_verify = 0; + + ssl_cert = getenv("GIT_SSL_CERT"); +#if LIBCURL_VERSION_NUM >= 0x070902 + ssl_key = getenv("GIT_SSL_KEY"); +#endif +#if LIBCURL_VERSION_NUM >= 0x070908 + ssl_capath = getenv("GIT_SSL_CAPATH"); +#endif + ssl_cainfo = getenv("GIT_SSL_CAINFO"); + + git_config(http_options); + + if (curl_ssl_verify == -1) + curl_ssl_verify = 1; + +#ifdef USE_CURL_MULTI + if (max_requests < 1) + max_requests = DEFAULT_MAX_REQUESTS; +#endif + pragma_header = curl_slist_append(pragma_header, "Pragma: no-cache"); no_pragma_header = curl_slist_append(no_pragma_header, "Pragma:"); no_range_header = curl_slist_append(no_range_header, "Range:"); curl_default = curl_easy_init(); - curl_ssl_verify = getenv("GIT_SSL_NO_VERIFY") ? 0 : 1; curl_easy_setopt(curl_default, CURLOPT_SSL_VERIFYPEER, curl_ssl_verify); #if LIBCURL_VERSION_NUM >= 0x070907 curl_easy_setopt(curl_default, CURLOPT_NETRC, CURL_NETRC_OPTIONAL); #endif - if ((ssl_cert = getenv("GIT_SSL_CERT")) != NULL) { + if (ssl_cert != NULL) curl_easy_setopt(curl_default, CURLOPT_SSLCERT, ssl_cert); - } -#if LIBCURL_VERSION_NUM >= 0x070902 - if ((ssl_key = getenv("GIT_SSL_KEY")) != NULL) { + if (ssl_key != NULL) curl_easy_setopt(curl_default, CURLOPT_SSLKEY, ssl_key); - } -#endif -#if LIBCURL_VERSION_NUM >= 0x070908 - if ((ssl_capath = getenv("GIT_SSL_CAPATH")) != NULL) { + if (ssl_capath != NULL) curl_easy_setopt(curl_default, CURLOPT_CAPATH, ssl_capath); - } -#endif - if ((ssl_cainfo = getenv("GIT_SSL_CAINFO")) != NULL) { + if (ssl_cainfo != NULL) curl_easy_setopt(curl_default, CURLOPT_CAINFO, ssl_cainfo); - } + curl_easy_setopt(curl_default, CURLOPT_FAILONERROR, 1); alt = xmalloc(sizeof(*alt)); -- cgit v1.3-5-g9baa From 7baa3e8694d823ce9731172c700bf0d780ae283e Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sat, 15 Oct 2005 11:10:46 -0700 Subject: Some curl versions lack curl_easy_duphandle() Hi, On Fri, 14 Oct 2005, Junio C Hamano wrote: > Johannes Schindelin writes: > > > This patch looks bigger than it really is: The code to get the > > default handle was refactored into a function, and is called > > instead of curl_easy_duphandle() if that does not exist. > > I'd like to take Nick's config file patch first, which > unfortunately interferes with your patch. I'd hate to ask you > this, but could you rebase it on top of Nick's patch, [...] No need to hate it. Here comes the rebased patch, and this time, I actually tested it a bit. Signed-off-by: Junio C Hamano --- http-fetch.c | 60 ++++++++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 44 insertions(+), 16 deletions(-) (limited to 'http-fetch.c') diff --git a/http-fetch.c b/http-fetch.c index 784aedfc70..40bd0b430b 100644 --- a/http-fetch.c +++ b/http-fetch.c @@ -18,6 +18,10 @@ #define curl_global_init(a) do { /* nothing */ } while(0) #endif +#if LIBCURL_VERSION_NUM < 0x070c04 +#define NO_CURL_EASY_DUPHANDLE +#endif + #define PREV_BUF_SIZE 4096 #define RANGE_HEADER_SIZE 30 @@ -28,7 +32,9 @@ static int data_received; static int max_requests = -1; static CURLM *curlm; #endif +#ifndef NO_CURL_EASY_DUPHANDLE static CURL *curl_default; +#endif static struct curl_slist *pragma_header; static struct curl_slist *no_pragma_header; static struct curl_slist *no_range_header; @@ -87,8 +93,12 @@ static struct active_request_slot *active_queue_head = NULL; static int curl_ssl_verify = -1; static char *ssl_cert = NULL; +#if LIBCURL_VERSION_NUM >= 0x070902 static char *ssl_key = NULL; +#endif +#if LIBCURL_VERSION_NUM >= 0x070908 static char *ssl_capath = NULL; +#endif static char *ssl_cainfo = NULL; struct buffer @@ -213,6 +223,32 @@ void process_curl_messages(); void process_request_queue(); #endif +static CURL* get_curl_handle() +{ + CURL* result = curl_easy_init(); + + curl_easy_setopt(result, CURLOPT_SSL_VERIFYPEER, curl_ssl_verify); +#if LIBCURL_VERSION_NUM >= 0x070907 + curl_easy_setopt(result, CURLOPT_NETRC, CURL_NETRC_OPTIONAL); +#endif + + if (ssl_cert != NULL) + curl_easy_setopt(result, CURLOPT_SSLCERT, ssl_cert); +#if LIBCURL_VERSION_NUM >= 0x070902 + if (ssl_key != NULL) + curl_easy_setopt(result, CURLOPT_SSLKEY, ssl_key); +#endif +#if LIBCURL_VERSION_NUM >= 0x070908 + if (ssl_capath != NULL) + curl_easy_setopt(result, CURLOPT_CAPATH, ssl_capath); +#endif + if (ssl_cainfo != NULL) + curl_easy_setopt(result, CURLOPT_CAINFO, ssl_cainfo); + curl_easy_setopt(result, CURLOPT_FAILONERROR, 1); + + return result; +} + struct active_request_slot *get_active_slot() { struct active_request_slot *slot = active_queue_head; @@ -235,7 +271,11 @@ struct active_request_slot *get_active_slot() } if (slot == NULL) { newslot = xmalloc(sizeof(*newslot)); +#ifdef NO_CURL_EASY_DUPHANDLE + newslot->curl = get_curl_handle(); +#else newslot->curl = curl_easy_duphandle(curl_default); +#endif newslot->in_use = 0; newslot->next = NULL; @@ -1202,24 +1242,10 @@ int main(int argc, char **argv) no_pragma_header = curl_slist_append(no_pragma_header, "Pragma:"); no_range_header = curl_slist_append(no_range_header, "Range:"); - curl_default = curl_easy_init(); - - curl_easy_setopt(curl_default, CURLOPT_SSL_VERIFYPEER, curl_ssl_verify); -#if LIBCURL_VERSION_NUM >= 0x070907 - curl_easy_setopt(curl_default, CURLOPT_NETRC, CURL_NETRC_OPTIONAL); +#ifndef NO_CURL_EASY_DUPHANDLE + curl_default = get_curl_handle(); #endif - if (ssl_cert != NULL) - curl_easy_setopt(curl_default, CURLOPT_SSLCERT, ssl_cert); - if (ssl_key != NULL) - curl_easy_setopt(curl_default, CURLOPT_SSLKEY, ssl_key); - if (ssl_capath != NULL) - curl_easy_setopt(curl_default, CURLOPT_CAPATH, ssl_capath); - if (ssl_cainfo != NULL) - curl_easy_setopt(curl_default, CURLOPT_CAINFO, ssl_cainfo); - - curl_easy_setopt(curl_default, CURLOPT_FAILONERROR, 1); - alt = xmalloc(sizeof(*alt)); alt->base = url; alt->got_indices = 0; @@ -1233,7 +1259,9 @@ int main(int argc, char **argv) curl_slist_free_all(pragma_header); curl_slist_free_all(no_pragma_header); curl_slist_free_all(no_range_header); +#ifndef NO_CURL_EASY_DUPHANDLE curl_easy_cleanup(curl_default); +#endif slot = active_queue_head; while (slot != NULL) { curl_easy_cleanup(slot->curl); -- cgit v1.3-5-g9baa From f80376c5976143a1788ecc4f7ca86ac0ec8b9874 Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Sat, 15 Oct 2005 11:13:55 -0700 Subject: Show curl error a bit better. Signed-off-by: Junio C Hamano --- http-fetch.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'http-fetch.c') diff --git a/http-fetch.c b/http-fetch.c index 40bd0b430b..da4d2f2c69 100644 --- a/http-fetch.c +++ b/http-fetch.c @@ -1049,7 +1049,9 @@ static int fetch_object(struct alt_base *repo, unsigned char *sha1) } if (request->curl_result != CURLE_OK && request->http_code != 416) { - ret = error("%s", request->errorstr); + ret = error("%s (curl_result = %d, http_code = %ld, sha1 = %s)", + request->errorstr, request->curl_result, + request->http_code, hex); release_request(request); return ret; } -- cgit v1.3-5-g9baa From f5dce80611ed865d2d5d1e76c2c3ae64bf99fe99 Mon Sep 17 00:00:00 2001 From: Peter Hagervall Date: Sun, 16 Oct 2005 00:01:08 -0700 Subject: Sparse fixes for http-fetch This patch cleans out all sparse warnings from http-fetch.c I'm a bit uncomfortable with adding extra #ifdefs to avoid either 'mixing declaration with code' or 'unused variable' warnings, but I figured that since those functions are already littered with #ifdefs I might just get away with it. Comments? [jc: I adjusted Peter's patch to address uncomfortableness issues.] Signed-off-by: Peter Hagervall Signed-off-by: Junio C Hamano --- http-fetch.c | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) (limited to 'http-fetch.c') diff --git a/http-fetch.c b/http-fetch.c index da4d2f2c69..efa6e82329 100644 --- a/http-fetch.c +++ b/http-fetch.c @@ -219,11 +219,11 @@ static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb, } #ifdef USE_CURL_MULTI -void process_curl_messages(); -void process_request_queue(); +static void process_curl_messages(void); +static void process_request_queue(void); #endif -static CURL* get_curl_handle() +static CURL* get_curl_handle(void) { CURL* result = curl_easy_init(); @@ -249,7 +249,7 @@ static CURL* get_curl_handle() return result; } -struct active_request_slot *get_active_slot() +static struct active_request_slot *get_active_slot(void) { struct active_request_slot *slot = active_queue_head; struct active_request_slot *newslot; @@ -302,7 +302,7 @@ struct active_request_slot *get_active_slot() return slot; } -int start_active_slot(struct active_request_slot *slot) +static int start_active_slot(struct active_request_slot *slot) { #ifdef USE_CURL_MULTI CURLMcode curlm_result = curl_multi_add_handle(curlm, slot->curl); @@ -317,7 +317,7 @@ int start_active_slot(struct active_request_slot *slot) return 1; } -void run_active_slot(struct active_request_slot *slot) +static void run_active_slot(struct active_request_slot *slot) { #ifdef USE_CURL_MULTI int num_transfers; @@ -365,7 +365,7 @@ void run_active_slot(struct active_request_slot *slot) #endif } -void start_request(struct transfer_request *request) +static void start_request(struct transfer_request *request) { char *hex = sha1_to_hex(request->sha1); char prevfile[PATH_MAX]; @@ -491,7 +491,7 @@ void start_request(struct transfer_request *request) request->state = ACTIVE; } -void finish_request(struct transfer_request *request) +static void finish_request(struct transfer_request *request) { fchmod(request->local, 0444); close(request->local); @@ -519,7 +519,7 @@ void finish_request(struct transfer_request *request) pull_say("got %s\n", sha1_to_hex(request->sha1)); } -void release_request(struct transfer_request *request) +static void release_request(struct transfer_request *request) { struct transfer_request *entry = request_queue_head; @@ -537,7 +537,7 @@ void release_request(struct transfer_request *request) } #ifdef USE_CURL_MULTI -void process_curl_messages() +void process_curl_messages(void) { int num_messages; struct active_request_slot *slot; @@ -589,7 +589,7 @@ void process_curl_messages() } } -void process_request_queue() +void process_request_queue(void) { struct transfer_request *request = request_queue_head; int num_transfers; @@ -1010,8 +1010,8 @@ static int fetch_object(struct alt_base *repo, unsigned char *sha1) } #ifdef USE_CURL_MULTI - int num_transfers; while (request->state == WAITING) { + int num_transfers; curl_multi_perform(curlm, &num_transfers); if (num_transfers < active_requests) { process_curl_messages(); @@ -1207,9 +1207,11 @@ int main(int argc, char **argv) curl_global_init(CURL_GLOBAL_ALL); #ifdef USE_CURL_MULTI - char *http_max_requests = getenv("GIT_HTTP_MAX_REQUESTS"); - if (http_max_requests != NULL) - max_requests = atoi(http_max_requests); + { + char *http_max_requests = getenv("GIT_HTTP_MAX_REQUESTS"); + if (http_max_requests != NULL) + max_requests = atoi(http_max_requests); + } curlm = curl_multi_init(); if (curlm == NULL) { -- cgit v1.3-5-g9baa From e0004e286c50364f134c951754c34e815489ead0 Mon Sep 17 00:00:00 2001 From: Nick Hengeveld Date: Wed, 19 Oct 2005 14:27:01 -0700 Subject: Support for HTTP transfer timeouts based on transfer speed Add configuration settings to abort HTTP requests if the transfer rate drops below a threshold for a specified length of time. Environment variables override config file settings. Signed-off-by: Nick Hengeveld Signed-off-by: Junio C Hamano --- http-fetch.c | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) (limited to 'http-fetch.c') diff --git a/http-fetch.c b/http-fetch.c index efa6e82329..a7dc2cc3bd 100644 --- a/http-fetch.c +++ b/http-fetch.c @@ -100,6 +100,8 @@ static char *ssl_key = NULL; static char *ssl_capath = NULL; #endif static char *ssl_cainfo = NULL; +static long curl_low_speed_limit = -1; +static long curl_low_speed_time = -1; struct buffer { @@ -158,6 +160,17 @@ static int http_options(const char *var, const char *value) } #endif + if (!strcmp("http.lowspeedlimit", var)) { + if (curl_low_speed_limit == -1) + curl_low_speed_limit = (long)git_config_int(var, value); + return 0; + } + if (!strcmp("http.lowspeedtime", var)) { + if (curl_low_speed_time == -1) + curl_low_speed_time = (long)git_config_int(var, value); + return 0; + } + /* Fall back on the default ones */ return git_default_config(var, value); } @@ -246,6 +259,13 @@ static CURL* get_curl_handle(void) curl_easy_setopt(result, CURLOPT_CAINFO, ssl_cainfo); curl_easy_setopt(result, CURLOPT_FAILONERROR, 1); + if (curl_low_speed_limit > 0 && curl_low_speed_time > 0) { + curl_easy_setopt(result, CURLOPT_LOW_SPEED_LIMIT, + curl_low_speed_limit); + curl_easy_setopt(result, CURLOPT_LOW_SPEED_TIME, + curl_low_speed_time); + } + return result; } @@ -1177,6 +1197,8 @@ int main(int argc, char **argv) char *url; int arg = 1; struct active_request_slot *slot; + char *low_speed_limit; + char *low_speed_time; while (arg < argc && argv[arg][0] == '-') { if (argv[arg][1] == 't') { @@ -1232,6 +1254,13 @@ int main(int argc, char **argv) #endif ssl_cainfo = getenv("GIT_SSL_CAINFO"); + low_speed_limit = getenv("GIT_HTTP_LOW_SPEED_LIMIT"); + if (low_speed_limit != NULL) + curl_low_speed_limit = strtol(low_speed_limit, NULL, 10); + low_speed_time = getenv("GIT_HTTP_LOW_SPEED_TIME"); + if (low_speed_time != NULL) + curl_low_speed_time = strtol(low_speed_time, NULL, 10); + git_config(http_options); if (curl_ssl_verify == -1) -- cgit v1.3-5-g9baa From f1a906a387a2155069e02b09c33dabd6d058ede5 Mon Sep 17 00:00:00 2001 From: Nick Hengeveld Date: Fri, 21 Oct 2005 12:06:10 -0700 Subject: [PATCH 1/3] Clean up CURL handles in unused request slots Clean up CURL handles in unused request slots Signed-off-by: Nick Hengeveld Signed-off-by: Junio C Hamano --- http-fetch.c | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) (limited to 'http-fetch.c') diff --git a/http-fetch.c b/http-fetch.c index a7dc2cc3bd..d26fae8472 100644 --- a/http-fetch.c +++ b/http-fetch.c @@ -291,11 +291,7 @@ static struct active_request_slot *get_active_slot(void) } if (slot == NULL) { newslot = xmalloc(sizeof(*newslot)); -#ifdef NO_CURL_EASY_DUPHANDLE - newslot->curl = get_curl_handle(); -#else - newslot->curl = curl_easy_duphandle(curl_default); -#endif + newslot->curl = NULL; newslot->in_use = 0; newslot->next = NULL; @@ -311,6 +307,14 @@ static struct active_request_slot *get_active_slot(void) slot = newslot; } + if (slot->curl == NULL) { +#ifdef NO_CURL_EASY_DUPHANDLE + slot->curl = get_curl_handle(); +#else + slot->curl = curl_easy_duphandle(curl_default); +#endif + } + active_requests++; slot->in_use = 1; slot->done = 0; @@ -612,6 +616,7 @@ void process_curl_messages(void) void process_request_queue(void) { struct transfer_request *request = request_queue_head; + struct active_request_slot *slot = active_queue_head; int num_transfers; while (active_requests < max_requests && request != NULL) { @@ -624,6 +629,14 @@ void process_request_queue(void) } request = request->next; } + + while (slot != NULL) { + if (!slot->in_use && slot->curl != NULL) { + curl_easy_cleanup(slot->curl); + slot->curl = NULL; + } + slot = slot->next; + } } #endif @@ -1297,7 +1310,8 @@ int main(int argc, char **argv) #endif slot = active_queue_head; while (slot != NULL) { - curl_easy_cleanup(slot->curl); + if (slot->curl != NULL) + curl_easy_cleanup(slot->curl); slot = slot->next; } #ifdef USE_CURL_MULTI -- cgit v1.3-5-g9baa From f7eb290fa0620cebc033e80216c74ab9d1573a6c Mon Sep 17 00:00:00 2001 From: Nick Hengeveld Date: Fri, 21 Oct 2005 12:06:20 -0700 Subject: [PATCH 2/3] Switched back to loading alternates as needed Switched back to loading alternates as needed Signed-off-by: Nick Hengeveld Signed-off-by: Junio C Hamano --- http-fetch.c | 45 +++++++++++++++++++++++++++++---------------- 1 file changed, 29 insertions(+), 16 deletions(-) (limited to 'http-fetch.c') diff --git a/http-fetch.c b/http-fetch.c index d26fae8472..ed1053ade3 100644 --- a/http-fetch.c +++ b/http-fetch.c @@ -25,6 +25,7 @@ #define PREV_BUF_SIZE 4096 #define RANGE_HEADER_SIZE 30 +static int got_alternates = 0; static int active_requests = 0; static int data_received; @@ -85,6 +86,7 @@ struct active_request_slot int in_use; int done; CURLcode curl_result; + long http_code; struct active_request_slot *next; }; @@ -235,6 +237,7 @@ static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb, static void process_curl_messages(void); static void process_request_queue(void); #endif +static int fetch_alternates(char *base); static CURL* get_curl_handle(void) { @@ -580,6 +583,9 @@ void process_curl_messages(void) slot->done = 1; slot->in_use = 0; slot->curl_result = curl_message->data.result; + curl_easy_getinfo(slot->curl, + CURLINFO_HTTP_CODE, + &slot->http_code); request = request_queue_head; while (request != NULL && request->slot != slot) @@ -590,19 +596,20 @@ void process_curl_messages(void) if (request != NULL) { request->curl_result = curl_message->data.result; - curl_easy_getinfo(slot->curl, - CURLINFO_HTTP_CODE, - &request->http_code); + request->http_code = slot->http_code; request->slot = NULL; + request->state = COMPLETE; /* Use alternates if necessary */ - if (request->http_code == 404 && - request->repo->next != NULL) { - request->repo = request->repo->next; - start_request(request); + if (request->http_code == 404) { + fetch_alternates(alt->base); + if (request->repo->next != NULL) { + request->repo = + request->repo->next; + start_request(request); + } } else { finish_request(request); - request->state = COMPLETE; } } } else { @@ -765,6 +772,9 @@ static int fetch_alternates(char *base) struct active_request_slot *slot; + if (got_alternates) + return 0; + data = xmalloc(4096); buffer.size = 4096; buffer.posn = 0; @@ -797,6 +807,8 @@ static int fetch_alternates(char *base) run_active_slot(slot); if (slot->curl_result != CURLE_OK) { free(buffer.buffer); + if (slot->http_code == 404) + got_alternates = 1; return 0; } } @@ -868,6 +880,7 @@ static int fetch_alternates(char *base) i = posn + 1; } + got_alternates = 1; free(buffer.buffer); return ret; } @@ -1059,16 +1072,16 @@ static int fetch_object(struct alt_base *repo, unsigned char *sha1) run_active_slot(request->slot); #ifndef USE_CURL_MULTI request->curl_result = request->slot->curl_result; - curl_easy_getinfo(request->slot->curl, - CURLINFO_HTTP_CODE, - &request->http_code); + request->http_code = request->slot->http_code; request->slot = NULL; /* Use alternates if necessary */ - if (request->http_code == 404 && - request->repo->next != NULL) { - request->repo = request->repo->next; - start_request(request); + if (request->http_code == 404) { + fetch_alternates(alt->base); + if (request->repo->next != NULL) { + request->repo = request->repo->next; + start_request(request); + } } else { finish_request(request); request->state = COMPLETE; @@ -1121,6 +1134,7 @@ int fetch(unsigned char *sha1) while (altbase) { if (!fetch_pack(altbase, sha1)) return 0; + fetch_alternates(alt->base); altbase = altbase->next; } return error("Unable to find %s under %s\n", sha1_to_hex(sha1), @@ -1297,7 +1311,6 @@ int main(int argc, char **argv) alt->got_indices = 0; alt->packs = NULL; alt->next = NULL; - fetch_alternates(alt->base); if (pull(commit_id)) return 1; -- cgit v1.3-5-g9baa From 7b9ae53ea3680f5b0d17a2dd40b39a17f1fcc5cf Mon Sep 17 00:00:00 2001 From: Nick Hengeveld Date: Fri, 21 Oct 2005 12:06:27 -0700 Subject: [PATCH 3/3] Allow running requests to finish after a pull error Allow running requests to finish after a pull error Signed-off-by: Nick Hengeveld Signed-off-by: Junio C Hamano --- http-fetch.c | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) (limited to 'http-fetch.c') diff --git a/http-fetch.c b/http-fetch.c index ed1053ade3..1ee1df20d6 100644 --- a/http-fetch.c +++ b/http-fetch.c @@ -1226,6 +1226,8 @@ int main(int argc, char **argv) struct active_request_slot *slot; char *low_speed_limit; char *low_speed_time; + char *wait_url; + int rc = 0; while (arg < argc && argv[arg][0] == '-') { if (argv[arg][1] == 't') { @@ -1313,7 +1315,7 @@ int main(int argc, char **argv) alt->next = NULL; if (pull(commit_id)) - return 1; + rc = 1; curl_slist_free_all(pragma_header); curl_slist_free_all(no_pragma_header); @@ -1323,6 +1325,15 @@ int main(int argc, char **argv) #endif slot = active_queue_head; while (slot != NULL) { + if (slot->in_use) { + if (get_verbosely) { + curl_easy_getinfo(slot->curl, + CURLINFO_EFFECTIVE_URL, + &wait_url); + fprintf(stderr, "Waiting for %s\n", wait_url); + } + run_active_slot(slot); + } if (slot->curl != NULL) curl_easy_cleanup(slot->curl); slot = slot->next; @@ -1331,5 +1342,5 @@ int main(int argc, char **argv) curl_multi_cleanup(curlm); #endif curl_global_cleanup(); - return 0; + return rc; } -- cgit v1.3-5-g9baa From e2029eb963bab6efeff48a7e1ded93842a257717 Mon Sep 17 00:00:00 2001 From: Petr Baudis Date: Fri, 21 Oct 2005 18:18:46 +0200 Subject: Silence confusing and false-positive curl error message git-http-fetch spits out curl 404 error message when unable to fetch an object, but that's confusing since no error really happened and the object is usually found in a pack it tries right after that. And if the object still cannot be retrieved, it will say another error message anyway. OTOH other HTTP errors (403 etc) are likely fatal and the user should be still informed about them. Signed-off-by: Petr Baudis Signed-off-by: Junio C Hamano --- http-fetch.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'http-fetch.c') diff --git a/http-fetch.c b/http-fetch.c index 1ee1df20d6..a1b03cd9c8 100644 --- a/http-fetch.c +++ b/http-fetch.c @@ -1095,9 +1095,12 @@ static int fetch_object(struct alt_base *repo, unsigned char *sha1) } if (request->curl_result != CURLE_OK && request->http_code != 416) { - ret = error("%s (curl_result = %d, http_code = %ld, sha1 = %s)", - request->errorstr, request->curl_result, - request->http_code, hex); + if (request->http_code == 404) + ret = -1; /* Be silent, it is probably in a pack. */ + else + ret = error("%s (curl_result = %d, http_code = %ld, sha1 = %s)", + request->errorstr, request->curl_result, + request->http_code, hex); release_request(request); return ret; } -- cgit v1.3-5-g9baa