From 6dcc05ffc3ead0745d19decd0e8ecd65edc9d414 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Mon, 7 Apr 2025 15:16:13 +0200 Subject: reftable: fix formatting of the license header The license headers used across the reftable library doesn't follow our typical coding style for multi-line comments. Fix it. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- reftable/stack.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'reftable/stack.c') diff --git a/reftable/stack.c b/reftable/stack.c index 6dac015b47..bc3cfa9117 100644 --- a/reftable/stack.c +++ b/reftable/stack.c @@ -1,10 +1,10 @@ /* -Copyright 2020 Google LLC - -Use of this source code is governed by a BSD-style -license that can be found in the LICENSE file or at -https://developers.google.com/open-source/licenses/bsd -*/ + * Copyright 2020 Google LLC + * + * Use of this source code is governed by a BSD-style + * license that can be found in the LICENSE file or at + * https://developers.google.com/open-source/licenses/bsd + */ #include "stack.h" -- cgit v1.3 From b648bd654947db06e5549d724f46d3660ac11e19 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Mon, 7 Apr 2025 15:16:14 +0200 Subject: reftable/reader: rename data structure to "table" The `struct reftable_reader` subsystem encapsulates a table that has been read from the disk. As such, the current name of that structure is somewhat hard to understand as it only talks about the fact that we read something from disk, without really giving an indicator _what_ that is. Furthermore, this naming schema doesn't really fit well into how the other structures are named: `reftable_merged_table`, `reftable_stack`, `reftable_block` and `reftable_record` are all named after what they encapsulate. Rename the subsystem to `reftable_table`, which directly gives a hint that the data structure is about handling the individual tables part of the stack. While this change results in a lot of churn, it prepares for us exposing the APIs to third-party callers now that the reftable library is a standalone library that can be linked against by other projects. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- Makefile | 4 +- meson.build | 2 +- reftable/iter.c | 10 +- reftable/iter.h | 4 +- reftable/merged.c | 26 +- reftable/merged.h | 4 +- reftable/reader.c | 879 ------------------------------------ reftable/reader.h | 67 --- reftable/reftable-merged.h | 6 +- reftable/reftable-reader.h | 72 --- reftable/reftable-table.h | 72 +++ reftable/stack.c | 168 +++---- reftable/stack.h | 4 +- reftable/table.c | 879 ++++++++++++++++++++++++++++++++++++ reftable/table.h | 67 +++ t/helper/test-reftable.c | 14 +- t/meson.build | 2 +- t/unit-tests/t-reftable-merged.c | 74 +-- t/unit-tests/t-reftable-reader.c | 96 ---- t/unit-tests/t-reftable-readwrite.c | 96 ++-- t/unit-tests/t-reftable-stack.c | 66 +-- t/unit-tests/t-reftable-table.c | 96 ++++ 22 files changed, 1354 insertions(+), 1354 deletions(-) delete mode 100644 reftable/reader.c delete mode 100644 reftable/reader.h delete mode 100644 reftable/reftable-reader.h create mode 100644 reftable/reftable-table.h create mode 100644 reftable/table.c create mode 100644 reftable/table.h delete mode 100644 t/unit-tests/t-reftable-reader.c create mode 100644 t/unit-tests/t-reftable-table.c (limited to 'reftable/stack.c') diff --git a/Makefile b/Makefile index 22d632da4e..58bd9dc645 100644 --- a/Makefile +++ b/Makefile @@ -1374,10 +1374,10 @@ UNIT_TEST_PROGRAMS += t-reftable-basics UNIT_TEST_PROGRAMS += t-reftable-block UNIT_TEST_PROGRAMS += t-reftable-merged UNIT_TEST_PROGRAMS += t-reftable-pq -UNIT_TEST_PROGRAMS += t-reftable-reader UNIT_TEST_PROGRAMS += t-reftable-readwrite UNIT_TEST_PROGRAMS += t-reftable-record UNIT_TEST_PROGRAMS += t-reftable-stack +UNIT_TEST_PROGRAMS += t-reftable-table UNIT_TEST_PROGRAMS += t-trailer UNIT_TEST_PROGRAMS += t-urlmatch-normalization UNIT_TEST_PROGS = $(patsubst %,$(UNIT_TEST_BIN)/%$X,$(UNIT_TEST_PROGRAMS)) @@ -2729,10 +2729,10 @@ REFTABLE_OBJS += reftable/blocksource.o REFTABLE_OBJS += reftable/iter.o REFTABLE_OBJS += reftable/merged.o REFTABLE_OBJS += reftable/pq.o -REFTABLE_OBJS += reftable/reader.o REFTABLE_OBJS += reftable/record.o REFTABLE_OBJS += reftable/stack.o REFTABLE_OBJS += reftable/system.o +REFTABLE_OBJS += reftable/table.o REFTABLE_OBJS += reftable/tree.o REFTABLE_OBJS += reftable/writer.o diff --git a/meson.build b/meson.build index efe2871c9d..61986cc48d 100644 --- a/meson.build +++ b/meson.build @@ -410,10 +410,10 @@ libgit_sources = [ 'reftable/iter.c', 'reftable/merged.c', 'reftable/pq.c', - 'reftable/reader.c', 'reftable/record.c', 'reftable/stack.c', 'reftable/system.c', + 'reftable/table.c', 'reftable/tree.c', 'reftable/writer.c', 'remote.c', diff --git a/reftable/iter.c b/reftable/iter.c index 44a155e756..7376f263c9 100644 --- a/reftable/iter.c +++ b/reftable/iter.c @@ -12,8 +12,8 @@ #include "block.h" #include "constants.h" -#include "reader.h" #include "reftable-error.h" +#include "table.h" int iterator_seek(struct reftable_iterator *it, struct reftable_record *want) { @@ -130,8 +130,8 @@ static int indexed_table_ref_iter_next_block(struct indexed_table_ref_iter *it) reftable_block_done(&it->block_reader.block); off = it->offsets[it->offset_idx++]; - err = reader_init_block_reader(it->r, &it->block_reader, off, - BLOCK_TYPE_REF); + err = table_init_block_reader(it->table, &it->block_reader, off, + BLOCK_TYPE_REF); if (err < 0) { return err; } @@ -181,7 +181,7 @@ static int indexed_table_ref_iter_next(void *p, struct reftable_record *rec) } int indexed_table_ref_iter_new(struct indexed_table_ref_iter **dest, - struct reftable_reader *r, uint8_t *oid, + struct reftable_table *t, uint8_t *oid, int oid_len, uint64_t *offsets, int offset_len) { struct indexed_table_ref_iter empty = INDEXED_TABLE_REF_ITER_INIT; @@ -195,7 +195,7 @@ int indexed_table_ref_iter_new(struct indexed_table_ref_iter **dest, } *itr = empty; - itr->r = r; + itr->table = t; err = reftable_buf_add(&itr->oid, oid, oid_len); if (err < 0) diff --git a/reftable/iter.h b/reftable/iter.h index bd217405f5..add5c9472f 100644 --- a/reftable/iter.h +++ b/reftable/iter.h @@ -59,7 +59,7 @@ void iterator_from_filtering_ref_iterator(struct reftable_iterator *, * but using the object index. */ struct indexed_table_ref_iter { - struct reftable_reader *r; + struct reftable_table *table; struct reftable_buf oid; /* mutable */ @@ -83,7 +83,7 @@ void iterator_from_indexed_table_ref_iter(struct reftable_iterator *it, /* Takes ownership of `offsets` */ int indexed_table_ref_iter_new(struct indexed_table_ref_iter **dest, - struct reftable_reader *r, uint8_t *oid, + struct reftable_table *t, uint8_t *oid, int oid_len, uint64_t *offsets, int offset_len); #endif diff --git a/reftable/merged.c b/reftable/merged.c index 1829a08175..d5b974d660 100644 --- a/reftable/merged.c +++ b/reftable/merged.c @@ -11,11 +11,11 @@ #include "constants.h" #include "iter.h" #include "pq.h" -#include "reader.h" #include "record.h" #include "reftable-merged.h" #include "reftable-error.h" #include "system.h" +#include "table.h" struct merged_subiter { struct reftable_iterator iter; @@ -192,7 +192,7 @@ static void iterator_from_merged_iter(struct reftable_iterator *it, } int reftable_merged_table_new(struct reftable_merged_table **dest, - struct reftable_reader **readers, size_t n, + struct reftable_table **tables, size_t n, enum reftable_hash hash_id) { struct reftable_merged_table *m = NULL; @@ -200,10 +200,10 @@ int reftable_merged_table_new(struct reftable_merged_table **dest, uint64_t first_min = 0; for (size_t i = 0; i < n; i++) { - uint64_t min = reftable_reader_min_update_index(readers[i]); - uint64_t max = reftable_reader_max_update_index(readers[i]); + uint64_t min = reftable_table_min_update_index(tables[i]); + uint64_t max = reftable_table_max_update_index(tables[i]); - if (reftable_reader_hash_id(readers[i]) != hash_id) { + if (reftable_table_hash_id(tables[i]) != hash_id) { return REFTABLE_FORMAT_ERROR; } if (i == 0 || min < first_min) { @@ -218,8 +218,8 @@ int reftable_merged_table_new(struct reftable_merged_table **dest, if (!m) return REFTABLE_OUT_OF_MEMORY_ERROR; - m->readers = readers; - m->readers_len = n; + m->tables = tables; + m->tables_len = n; m->min = first_min; m->max = last_max; m->hash_id = hash_id; @@ -254,20 +254,20 @@ int merged_table_init_iter(struct reftable_merged_table *mt, struct merged_iter *mi = NULL; int ret; - if (mt->readers_len) { - REFTABLE_CALLOC_ARRAY(subiters, mt->readers_len); + if (mt->tables_len) { + REFTABLE_CALLOC_ARRAY(subiters, mt->tables_len); if (!subiters) { ret = REFTABLE_OUT_OF_MEMORY_ERROR; goto out; } } - for (size_t i = 0; i < mt->readers_len; i++) { + for (size_t i = 0; i < mt->tables_len; i++) { ret = reftable_record_init(&subiters[i].rec, typ); if (ret < 0) goto out; - ret = reader_init_iter(mt->readers[i], &subiters[i].iter, typ); + ret = table_init_iter(mt->tables[i], &subiters[i].iter, typ); if (ret < 0) goto out; } @@ -280,14 +280,14 @@ int merged_table_init_iter(struct reftable_merged_table *mt, mi->advance_index = -1; mi->suppress_deletions = mt->suppress_deletions; mi->subiters = subiters; - mi->subiters_len = mt->readers_len; + mi->subiters_len = mt->tables_len; iterator_from_merged_iter(it, mi); ret = 0; out: if (ret < 0) { - for (size_t i = 0; subiters && i < mt->readers_len; i++) { + for (size_t i = 0; subiters && i < mt->tables_len; i++) { reftable_iterator_destroy(&subiters[i].iter); reftable_record_release(&subiters[i].rec); } diff --git a/reftable/merged.h b/reftable/merged.h index ae92409d34..4317e5f5f6 100644 --- a/reftable/merged.h +++ b/reftable/merged.h @@ -13,8 +13,8 @@ #include "reftable-basics.h" struct reftable_merged_table { - struct reftable_reader **readers; - size_t readers_len; + struct reftable_table **tables; + size_t tables_len; enum reftable_hash hash_id; /* If unset, produce deletions. This is useful for compaction. For the diff --git a/reftable/reader.c b/reftable/reader.c deleted file mode 100644 index 155a26c30f..0000000000 --- a/reftable/reader.c +++ /dev/null @@ -1,879 +0,0 @@ -/* - * Copyright 2020 Google LLC - * - * Use of this source code is governed by a BSD-style - * license that can be found in the LICENSE file or at - * https://developers.google.com/open-source/licenses/bsd - */ - -#include "reader.h" - -#include "system.h" -#include "block.h" -#include "constants.h" -#include "iter.h" -#include "record.h" -#include "reftable-error.h" - -uint64_t block_source_size(struct reftable_block_source *source) -{ - return source->ops->size(source->arg); -} - -ssize_t block_source_read_block(struct reftable_block_source *source, - struct reftable_block *dest, uint64_t off, - uint32_t size) -{ - ssize_t result = source->ops->read_block(source->arg, dest, off, size); - dest->source = *source; - return result; -} - -void block_source_close(struct reftable_block_source *source) -{ - if (!source->ops) { - return; - } - - source->ops->close(source->arg); - source->ops = NULL; -} - -static struct reftable_reader_offsets * -reader_offsets_for(struct reftable_reader *r, uint8_t typ) -{ - switch (typ) { - case BLOCK_TYPE_REF: - return &r->ref_offsets; - case BLOCK_TYPE_LOG: - return &r->log_offsets; - case BLOCK_TYPE_OBJ: - return &r->obj_offsets; - } - abort(); -} - -static int reader_get_block(struct reftable_reader *r, - struct reftable_block *dest, uint64_t off, - uint32_t sz) -{ - ssize_t bytes_read; - if (off >= r->size) - return 0; - if (off + sz > r->size) - sz = r->size - off; - - bytes_read = block_source_read_block(&r->source, dest, off, sz); - if (bytes_read < 0) - return (int)bytes_read; - - return 0; -} - -enum reftable_hash reftable_reader_hash_id(struct reftable_reader *r) -{ - return r->hash_id; -} - -const char *reader_name(struct reftable_reader *r) -{ - return r->name; -} - -static int parse_footer(struct reftable_reader *r, uint8_t *footer, - uint8_t *header) -{ - uint8_t *f = footer; - uint8_t first_block_typ; - int err = 0; - uint32_t computed_crc; - uint32_t file_crc; - - if (memcmp(f, "REFT", 4)) { - err = REFTABLE_FORMAT_ERROR; - goto done; - } - f += 4; - - if (memcmp(footer, header, header_size(r->version))) { - err = REFTABLE_FORMAT_ERROR; - goto done; - } - - f++; - r->block_size = reftable_get_be24(f); - - f += 3; - r->min_update_index = reftable_get_be64(f); - f += 8; - r->max_update_index = reftable_get_be64(f); - f += 8; - - if (r->version == 1) { - r->hash_id = REFTABLE_HASH_SHA1; - } else { - switch (reftable_get_be32(f)) { - case REFTABLE_FORMAT_ID_SHA1: - r->hash_id = REFTABLE_HASH_SHA1; - break; - case REFTABLE_FORMAT_ID_SHA256: - r->hash_id = REFTABLE_HASH_SHA256; - break; - default: - err = REFTABLE_FORMAT_ERROR; - goto done; - } - - f += 4; - } - - r->ref_offsets.index_offset = reftable_get_be64(f); - f += 8; - - r->obj_offsets.offset = reftable_get_be64(f); - f += 8; - - r->object_id_len = r->obj_offsets.offset & ((1 << 5) - 1); - r->obj_offsets.offset >>= 5; - - r->obj_offsets.index_offset = reftable_get_be64(f); - f += 8; - r->log_offsets.offset = reftable_get_be64(f); - f += 8; - r->log_offsets.index_offset = reftable_get_be64(f); - f += 8; - - computed_crc = crc32(0, footer, f - footer); - file_crc = reftable_get_be32(f); - f += 4; - if (computed_crc != file_crc) { - err = REFTABLE_FORMAT_ERROR; - goto done; - } - - first_block_typ = header[header_size(r->version)]; - r->ref_offsets.is_present = (first_block_typ == BLOCK_TYPE_REF); - r->ref_offsets.offset = 0; - r->log_offsets.is_present = (first_block_typ == BLOCK_TYPE_LOG || - r->log_offsets.offset > 0); - r->obj_offsets.is_present = r->obj_offsets.offset > 0; - if (r->obj_offsets.is_present && !r->object_id_len) { - err = REFTABLE_FORMAT_ERROR; - goto done; - } - - err = 0; -done: - return err; -} - -struct table_iter { - struct reftable_reader *r; - uint8_t typ; - uint64_t block_off; - struct block_reader br; - struct block_iter bi; - int is_finished; -}; - -static int table_iter_init(struct table_iter *ti, struct reftable_reader *r) -{ - struct block_iter bi = BLOCK_ITER_INIT; - memset(ti, 0, sizeof(*ti)); - reftable_reader_incref(r); - ti->r = r; - ti->bi = bi; - return 0; -} - -static int table_iter_next_in_block(struct table_iter *ti, - struct reftable_record *rec) -{ - int res = block_iter_next(&ti->bi, rec); - if (res == 0 && reftable_record_type(rec) == BLOCK_TYPE_REF) { - rec->u.ref.update_index += ti->r->min_update_index; - } - - return res; -} - -static void table_iter_block_done(struct table_iter *ti) -{ - block_reader_release(&ti->br); - block_iter_reset(&ti->bi); -} - -static int32_t extract_block_size(uint8_t *data, uint8_t *typ, uint64_t off, - int version) -{ - int32_t result = 0; - - if (off == 0) { - data += header_size(version); - } - - *typ = data[0]; - if (reftable_is_block_type(*typ)) { - result = reftable_get_be24(data + 1); - } - return result; -} - -int reader_init_block_reader(struct reftable_reader *r, struct block_reader *br, - uint64_t next_off, uint8_t want_typ) -{ - int32_t guess_block_size = r->block_size ? r->block_size : - DEFAULT_BLOCK_SIZE; - struct reftable_block block = { NULL }; - uint8_t block_typ = 0; - int err = 0; - uint32_t header_off = next_off ? 0 : header_size(r->version); - int32_t block_size = 0; - - if (next_off >= r->size) - return 1; - - err = reader_get_block(r, &block, next_off, guess_block_size); - if (err < 0) - goto done; - - block_size = extract_block_size(block.data, &block_typ, next_off, - r->version); - if (block_size < 0) { - err = block_size; - goto done; - } - if (want_typ != BLOCK_TYPE_ANY && block_typ != want_typ) { - err = 1; - goto done; - } - - if (block_size > guess_block_size) { - reftable_block_done(&block); - err = reader_get_block(r, &block, next_off, block_size); - if (err < 0) { - goto done; - } - } - - err = block_reader_init(br, &block, header_off, r->block_size, - hash_size(r->hash_id)); -done: - reftable_block_done(&block); - - return err; -} - -static void table_iter_close(struct table_iter *ti) -{ - table_iter_block_done(ti); - block_iter_close(&ti->bi); - reftable_reader_decref(ti->r); -} - -static int table_iter_next_block(struct table_iter *ti) -{ - uint64_t next_block_off = ti->block_off + ti->br.full_block_size; - int err; - - err = reader_init_block_reader(ti->r, &ti->br, next_block_off, ti->typ); - if (err > 0) - ti->is_finished = 1; - if (err) - return err; - - ti->block_off = next_block_off; - ti->is_finished = 0; - block_iter_seek_start(&ti->bi, &ti->br); - - return 0; -} - -static int table_iter_next(struct table_iter *ti, struct reftable_record *rec) -{ - if (reftable_record_type(rec) != ti->typ) - return REFTABLE_API_ERROR; - - while (1) { - int err; - - if (ti->is_finished) - return 1; - - /* - * Check whether the current block still has more records. If - * so, return it. If the iterator returns positive then the - * current block has been exhausted. - */ - err = table_iter_next_in_block(ti, rec); - if (err <= 0) - return err; - - /* - * Otherwise, we need to continue to the next block in the - * table and retry. If there are no more blocks then the - * iterator is drained. - */ - err = table_iter_next_block(ti); - if (err) { - ti->is_finished = 1; - return err; - } - } -} - -static int table_iter_seek_to(struct table_iter *ti, uint64_t off, uint8_t typ) -{ - int err; - - err = reader_init_block_reader(ti->r, &ti->br, off, typ); - if (err != 0) - return err; - - ti->typ = block_reader_type(&ti->br); - ti->block_off = off; - block_iter_seek_start(&ti->bi, &ti->br); - ti->is_finished = 0; - return 0; -} - -static int table_iter_seek_start(struct table_iter *ti, uint8_t typ, int index) -{ - struct reftable_reader_offsets *offs = reader_offsets_for(ti->r, typ); - uint64_t off = offs->offset; - if (index) { - off = offs->index_offset; - if (off == 0) { - return 1; - } - typ = BLOCK_TYPE_INDEX; - } - - return table_iter_seek_to(ti, off, typ); -} - -static int table_iter_seek_linear(struct table_iter *ti, - struct reftable_record *want) -{ - struct reftable_buf want_key = REFTABLE_BUF_INIT; - struct reftable_buf got_key = REFTABLE_BUF_INIT; - struct reftable_record rec; - int err; - - err = reftable_record_init(&rec, reftable_record_type(want)); - if (err < 0) - goto done; - - err = reftable_record_key(want, &want_key); - if (err < 0) - goto done; - - /* - * First we need to locate the block that must contain our record. To - * do so we scan through blocks linearly until we find the first block - * whose first key is bigger than our wanted key. Once we have found - * that block we know that the key must be contained in the preceding - * block. - * - * This algorithm is somewhat unfortunate because it means that we - * always have to seek one block too far and then back up. But as we - * can only decode the _first_ key of a block but not its _last_ key we - * have no other way to do this. - */ - while (1) { - struct table_iter next = *ti; - - /* - * We must be careful to not modify underlying data of `ti` - * because we may find that `next` does not contain our desired - * block, but that `ti` does. In that case, we would discard - * `next` and continue with `ti`. - * - * This also means that we cannot reuse allocated memory for - * `next` here. While it would be great if we could, it should - * in practice not be too bad given that we should only ever - * end up doing linear seeks with at most three blocks. As soon - * as we have more than three blocks we would have an index, so - * we would not do a linear search there anymore. - */ - memset(&next.br.block, 0, sizeof(next.br.block)); - next.br.zstream = NULL; - next.br.uncompressed_data = NULL; - next.br.uncompressed_cap = 0; - - err = table_iter_next_block(&next); - if (err < 0) - goto done; - if (err > 0) - break; - - err = block_reader_first_key(&next.br, &got_key); - if (err < 0) - goto done; - - if (reftable_buf_cmp(&got_key, &want_key) > 0) { - table_iter_block_done(&next); - break; - } - - table_iter_block_done(ti); - *ti = next; - } - - /* - * We have located the block that must contain our record, so we seek - * the wanted key inside of it. If the block does not contain our key - * we know that the corresponding record does not exist. - */ - err = block_iter_seek_key(&ti->bi, &ti->br, &want_key); - if (err < 0) - goto done; - err = 0; - -done: - reftable_record_release(&rec); - reftable_buf_release(&want_key); - reftable_buf_release(&got_key); - return err; -} - -static int table_iter_seek_indexed(struct table_iter *ti, - struct reftable_record *rec) -{ - struct reftable_record want_index = { - .type = BLOCK_TYPE_INDEX, .u.idx = { .last_key = REFTABLE_BUF_INIT } - }; - struct reftable_record index_result = { - .type = BLOCK_TYPE_INDEX, - .u.idx = { .last_key = REFTABLE_BUF_INIT }, - }; - int err; - - err = reftable_record_key(rec, &want_index.u.idx.last_key); - if (err < 0) - goto done; - - /* - * The index may consist of multiple levels, where each level may have - * multiple index blocks. We start by doing a linear search in the - * highest layer that identifies the relevant index block as well as - * the record inside that block that corresponds to our wanted key. - */ - err = table_iter_seek_linear(ti, &want_index); - if (err < 0) - goto done; - - /* - * Traverse down the levels until we find a non-index entry. - */ - while (1) { - /* - * In case we seek a record that does not exist the index iter - * will tell us that the iterator is over. This works because - * the last index entry of the current level will contain the - * last key it knows about. So in case our seeked key is larger - * than the last indexed key we know that it won't exist. - * - * There is one subtlety in the layout of the index section - * that makes this work as expected: the highest-level index is - * at end of the section and will point backwards and thus we - * start reading from the end of the index section, not the - * beginning. - * - * If that wasn't the case and the order was reversed then the - * linear seek would seek into the lower levels and traverse - * all levels of the index only to find out that the key does - * not exist. - */ - err = table_iter_next(ti, &index_result); - if (err != 0) - goto done; - - err = table_iter_seek_to(ti, index_result.u.idx.offset, 0); - if (err != 0) - goto done; - - err = block_iter_seek_key(&ti->bi, &ti->br, &want_index.u.idx.last_key); - if (err < 0) - goto done; - - if (ti->typ == reftable_record_type(rec)) { - err = 0; - break; - } - - if (ti->typ != BLOCK_TYPE_INDEX) { - err = REFTABLE_FORMAT_ERROR; - goto done; - } - } - -done: - reftable_record_release(&want_index); - reftable_record_release(&index_result); - return err; -} - -static int table_iter_seek(struct table_iter *ti, - struct reftable_record *want) -{ - uint8_t typ = reftable_record_type(want); - struct reftable_reader_offsets *offs = reader_offsets_for(ti->r, typ); - int err; - - err = table_iter_seek_start(ti, reftable_record_type(want), - !!offs->index_offset); - if (err < 0) - goto out; - - if (offs->index_offset) - err = table_iter_seek_indexed(ti, want); - else - err = table_iter_seek_linear(ti, want); - if (err) - goto out; - -out: - return err; -} - -static int table_iter_seek_void(void *ti, struct reftable_record *want) -{ - return table_iter_seek(ti, want); -} - -static int table_iter_next_void(void *ti, struct reftable_record *rec) -{ - return table_iter_next(ti, rec); -} - -static void table_iter_close_void(void *ti) -{ - table_iter_close(ti); -} - -static struct reftable_iterator_vtable table_iter_vtable = { - .seek = &table_iter_seek_void, - .next = &table_iter_next_void, - .close = &table_iter_close_void, -}; - -static void iterator_from_table_iter(struct reftable_iterator *it, - struct table_iter *ti) -{ - assert(!it->ops); - it->iter_arg = ti; - it->ops = &table_iter_vtable; -} - -int reader_init_iter(struct reftable_reader *r, - struct reftable_iterator *it, - uint8_t typ) -{ - struct reftable_reader_offsets *offs = reader_offsets_for(r, typ); - - if (offs->is_present) { - struct table_iter *ti; - REFTABLE_ALLOC_ARRAY(ti, 1); - if (!ti) - return REFTABLE_OUT_OF_MEMORY_ERROR; - - table_iter_init(ti, r); - iterator_from_table_iter(it, ti); - } else { - iterator_set_empty(it); - } - - return 0; -} - -int reftable_reader_init_ref_iterator(struct reftable_reader *r, - struct reftable_iterator *it) -{ - return reader_init_iter(r, it, BLOCK_TYPE_REF); -} - -int reftable_reader_init_log_iterator(struct reftable_reader *r, - struct reftable_iterator *it) -{ - return reader_init_iter(r, it, BLOCK_TYPE_LOG); -} - -int reftable_reader_new(struct reftable_reader **out, - struct reftable_block_source *source, char const *name) -{ - struct reftable_block footer = { 0 }; - struct reftable_block header = { 0 }; - struct reftable_reader *r; - uint64_t file_size = block_source_size(source); - uint32_t read_size; - ssize_t bytes_read; - int err; - - REFTABLE_CALLOC_ARRAY(r, 1); - if (!r) { - err = REFTABLE_OUT_OF_MEMORY_ERROR; - goto done; - } - - /* - * We need one extra byte to read the type of first block. We also - * pretend to always be reading v2 of the format because it is larger. - */ - read_size = header_size(2) + 1; - if (read_size > file_size) { - err = REFTABLE_FORMAT_ERROR; - goto done; - } - - bytes_read = block_source_read_block(source, &header, 0, read_size); - if (bytes_read < 0 || (size_t)bytes_read != read_size) { - err = REFTABLE_IO_ERROR; - goto done; - } - - if (memcmp(header.data, "REFT", 4)) { - err = REFTABLE_FORMAT_ERROR; - goto done; - } - r->version = header.data[4]; - if (r->version != 1 && r->version != 2) { - err = REFTABLE_FORMAT_ERROR; - goto done; - } - - r->size = file_size - footer_size(r->version); - r->source = *source; - r->name = reftable_strdup(name); - if (!r->name) { - err = REFTABLE_OUT_OF_MEMORY_ERROR; - goto done; - } - r->hash_id = 0; - r->refcount = 1; - - bytes_read = block_source_read_block(source, &footer, r->size, - footer_size(r->version)); - if (bytes_read < 0 || (size_t)bytes_read != footer_size(r->version)) { - err = REFTABLE_IO_ERROR; - goto done; - } - - err = parse_footer(r, footer.data, header.data); - if (err) - goto done; - - *out = r; - -done: - reftable_block_done(&footer); - reftable_block_done(&header); - if (err) { - if (r) - reftable_free(r->name); - reftable_free(r); - block_source_close(source); - } - return err; -} - -void reftable_reader_incref(struct reftable_reader *r) -{ - r->refcount++; -} - -void reftable_reader_decref(struct reftable_reader *r) -{ - if (!r) - return; - if (--r->refcount) - return; - block_source_close(&r->source); - REFTABLE_FREE_AND_NULL(r->name); - reftable_free(r); -} - -static int reftable_reader_refs_for_indexed(struct reftable_reader *r, - struct reftable_iterator *it, - uint8_t *oid) -{ - struct reftable_record want = { - .type = BLOCK_TYPE_OBJ, - .u.obj = { - .hash_prefix = oid, - .hash_prefix_len = r->object_id_len, - }, - }; - struct reftable_iterator oit = { NULL }; - struct reftable_record got = { - .type = BLOCK_TYPE_OBJ, - .u.obj = { 0 }, - }; - int err = 0; - struct indexed_table_ref_iter *itr = NULL; - - /* Look through the reverse index. */ - err = reader_init_iter(r, &oit, BLOCK_TYPE_OBJ); - if (err < 0) - goto done; - - err = iterator_seek(&oit, &want); - if (err != 0) - goto done; - - /* read out the reftable_obj_record */ - err = iterator_next(&oit, &got); - if (err < 0) - goto done; - - if (err > 0 || memcmp(want.u.obj.hash_prefix, got.u.obj.hash_prefix, - r->object_id_len)) { - /* didn't find it; return empty iterator */ - iterator_set_empty(it); - err = 0; - goto done; - } - - err = indexed_table_ref_iter_new(&itr, r, oid, hash_size(r->hash_id), - got.u.obj.offsets, - got.u.obj.offset_len); - if (err < 0) - goto done; - got.u.obj.offsets = NULL; - iterator_from_indexed_table_ref_iter(it, itr); - -done: - reftable_iterator_destroy(&oit); - reftable_record_release(&got); - return err; -} - -static int reftable_reader_refs_for_unindexed(struct reftable_reader *r, - struct reftable_iterator *it, - uint8_t *oid) -{ - struct table_iter *ti; - struct filtering_ref_iterator *filter = NULL; - struct filtering_ref_iterator empty = FILTERING_REF_ITERATOR_INIT; - uint32_t oid_len = hash_size(r->hash_id); - int err; - - REFTABLE_ALLOC_ARRAY(ti, 1); - if (!ti) { - err = REFTABLE_OUT_OF_MEMORY_ERROR; - goto out; - } - - table_iter_init(ti, r); - err = table_iter_seek_start(ti, BLOCK_TYPE_REF, 0); - if (err < 0) - goto out; - - filter = reftable_malloc(sizeof(*filter)); - if (!filter) { - err = REFTABLE_OUT_OF_MEMORY_ERROR; - goto out; - } - *filter = empty; - - err = reftable_buf_add(&filter->oid, oid, oid_len); - if (err < 0) - goto out; - - iterator_from_table_iter(&filter->it, ti); - - iterator_from_filtering_ref_iterator(it, filter); - - err = 0; - -out: - if (err < 0) { - if (ti) - table_iter_close(ti); - reftable_free(ti); - } - return err; -} - -int reftable_reader_refs_for(struct reftable_reader *r, - struct reftable_iterator *it, uint8_t *oid) -{ - if (r->obj_offsets.is_present) - return reftable_reader_refs_for_indexed(r, it, oid); - return reftable_reader_refs_for_unindexed(r, it, oid); -} - -uint64_t reftable_reader_max_update_index(struct reftable_reader *r) -{ - return r->max_update_index; -} - -uint64_t reftable_reader_min_update_index(struct reftable_reader *r) -{ - return r->min_update_index; -} - -int reftable_reader_print_blocks(const char *tablename) -{ - struct { - const char *name; - int type; - } sections[] = { - { - .name = "ref", - .type = BLOCK_TYPE_REF, - }, - { - .name = "obj", - .type = BLOCK_TYPE_OBJ, - }, - { - .name = "log", - .type = BLOCK_TYPE_LOG, - }, - }; - struct reftable_block_source src = { 0 }; - struct reftable_reader *r = NULL; - struct table_iter ti = { 0 }; - size_t i; - int err; - - err = reftable_block_source_from_file(&src, tablename); - if (err < 0) - goto done; - - err = reftable_reader_new(&r, &src, tablename); - if (err < 0) - goto done; - - table_iter_init(&ti, r); - - printf("header:\n"); - printf(" block_size: %d\n", r->block_size); - - for (i = 0; i < sizeof(sections) / sizeof(*sections); i++) { - err = table_iter_seek_start(&ti, sections[i].type, 0); - if (err < 0) - goto done; - if (err > 0) - continue; - - printf("%s:\n", sections[i].name); - - while (1) { - printf(" - length: %u\n", ti.br.block_len); - printf(" restarts: %u\n", ti.br.restart_count); - - err = table_iter_next_block(&ti); - if (err < 0) - goto done; - if (err > 0) - break; - } - } - -done: - reftable_reader_decref(r); - table_iter_close(&ti); - return err; -} diff --git a/reftable/reader.h b/reftable/reader.h deleted file mode 100644 index c9dccbd6c9..0000000000 --- a/reftable/reader.h +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright 2020 Google LLC - * - * Use of this source code is governed by a BSD-style - * license that can be found in the LICENSE file or at - * https://developers.google.com/open-source/licenses/bsd - */ - -#ifndef READER_H -#define READER_H - -#include "block.h" -#include "record.h" -#include "reftable-iterator.h" -#include "reftable-reader.h" - -uint64_t block_source_size(struct reftable_block_source *source); - -ssize_t block_source_read_block(struct reftable_block_source *source, - struct reftable_block *dest, uint64_t off, - uint32_t size); -void block_source_close(struct reftable_block_source *source); - -/* metadata for a block type */ -struct reftable_reader_offsets { - int is_present; - uint64_t offset; - uint64_t index_offset; -}; - -/* The state for reading a reftable file. */ -struct reftable_reader { - /* for convenience, associate a name with the instance. */ - char *name; - struct reftable_block_source source; - - /* Size of the file, excluding the footer. */ - uint64_t size; - - /* The hash function used for ref records. */ - enum reftable_hash hash_id; - - uint32_t block_size; - uint64_t min_update_index; - uint64_t max_update_index; - /* Length of the OID keys in the 'o' section */ - int object_id_len; - int version; - - struct reftable_reader_offsets ref_offsets; - struct reftable_reader_offsets obj_offsets; - struct reftable_reader_offsets log_offsets; - - uint64_t refcount; -}; - -const char *reader_name(struct reftable_reader *r); - -int reader_init_iter(struct reftable_reader *r, - struct reftable_iterator *it, - uint8_t typ); - -/* initialize a block reader to read from `r` */ -int reader_init_block_reader(struct reftable_reader *r, struct block_reader *br, - uint64_t next_off, uint8_t want_typ); - -#endif diff --git a/reftable/reftable-merged.h b/reftable/reftable-merged.h index 445756e475..e5af846b32 100644 --- a/reftable/reftable-merged.h +++ b/reftable/reftable-merged.h @@ -26,14 +26,14 @@ /* A merged table is implements seeking/iterating over a stack of tables. */ struct reftable_merged_table; -struct reftable_reader; +struct reftable_table; /* - * reftable_merged_table_new creates a new merged table. The readers must be + * reftable_merged_table_new creates a new merged table. The tables must be * kept alive as long as the merged table is still in use. */ int reftable_merged_table_new(struct reftable_merged_table **dest, - struct reftable_reader **readers, size_t n, + struct reftable_table **tables, size_t n, enum reftable_hash hash_id); /* Initialize a merged table iterator for reading refs. */ diff --git a/reftable/reftable-reader.h b/reftable/reftable-reader.h deleted file mode 100644 index 9b6a8ac071..0000000000 --- a/reftable/reftable-reader.h +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright 2020 Google LLC - * - * Use of this source code is governed by a BSD-style - * license that can be found in the LICENSE file or at - * https://developers.google.com/open-source/licenses/bsd - */ - -#ifndef REFTABLE_READER_H -#define REFTABLE_READER_H - -#include "reftable-iterator.h" -#include "reftable-blocksource.h" - -/* - * Reading single tables - * - * The follow routines are for reading single files. For an - * application-level interface, skip ahead to struct - * reftable_merged_table and struct reftable_stack. - */ - -/* The reader struct is a handle to an open reftable file. */ -struct reftable_reader; - -/* reftable_reader_new opens a reftable for reading. If successful, - * returns 0 code and sets pp. The name is used for creating a - * stack. Typically, it is the basename of the file. The block source - * `src` is owned by the reader, and is closed on calling - * reftable_reader_destroy(). On error, the block source `src` is - * closed as well. - */ -int reftable_reader_new(struct reftable_reader **pp, - struct reftable_block_source *src, const char *name); - -/* - * Manage the reference count of the reftable reader. A newly initialized - * reader starts with a refcount of 1 and will be deleted once the refcount has - * reached 0. - * - * This is required because readers may have longer lifetimes than the stack - * they belong to. The stack may for example be reloaded while the old tables - * are still being accessed by an iterator. - */ -void reftable_reader_incref(struct reftable_reader *reader); -void reftable_reader_decref(struct reftable_reader *reader); - -/* Initialize a reftable iterator for reading refs. */ -int reftable_reader_init_ref_iterator(struct reftable_reader *r, - struct reftable_iterator *it); - -/* Initialize a reftable iterator for reading logs. */ -int reftable_reader_init_log_iterator(struct reftable_reader *r, - struct reftable_iterator *it); - -/* returns the hash ID used in this table. */ -enum reftable_hash reftable_reader_hash_id(struct reftable_reader *r); - -/* return an iterator for the refs pointing to `oid`. */ -int reftable_reader_refs_for(struct reftable_reader *r, - struct reftable_iterator *it, uint8_t *oid); - -/* return the max_update_index for a table */ -uint64_t reftable_reader_max_update_index(struct reftable_reader *r); - -/* return the min_update_index for a table */ -uint64_t reftable_reader_min_update_index(struct reftable_reader *r); - -/* print blocks onto stdout for debugging. */ -int reftable_reader_print_blocks(const char *tablename); - -#endif diff --git a/reftable/reftable-table.h b/reftable/reftable-table.h new file mode 100644 index 0000000000..9437902672 --- /dev/null +++ b/reftable/reftable-table.h @@ -0,0 +1,72 @@ +/* + * Copyright 2020 Google LLC + * + * Use of this source code is governed by a BSD-style + * license that can be found in the LICENSE file or at + * https://developers.google.com/open-source/licenses/bsd + */ + +#ifndef REFTABLE_TABLE_H +#define REFTABLE_TABLE_H + +#include "reftable-iterator.h" +#include "reftable-blocksource.h" + +/* + * Reading single tables + * + * The follow routines are for reading single files. For an + * application-level interface, skip ahead to struct + * reftable_merged_table and struct reftable_stack. + */ + +/* The table struct is a handle to an open reftable file. */ +struct reftable_table; + +/* reftable_table_new opens a reftable for reading. If successful, + * returns 0 code and sets pp. The name is used for creating a + * stack. Typically, it is the basename of the file. The block source + * `src` is owned by the table, and is closed on calling + * reftable_table_destroy(). On error, the block source `src` is + * closed as well. + */ +int reftable_table_new(struct reftable_table **out, + struct reftable_block_source *src, const char *name); + +/* + * Manage the reference count of the reftable table. A newly initialized + * table starts with a refcount of 1 and will be deleted once the refcount has + * reached 0. + * + * This is required because tables may have longer lifetimes than the stack + * they belong to. The stack may for example be reloaded while the old tables + * are still being accessed by an iterator. + */ +void reftable_table_incref(struct reftable_table *table); +void reftable_table_decref(struct reftable_table *table); + +/* Initialize a reftable iterator for reading refs. */ +int reftable_table_init_ref_iterator(struct reftable_table *t, + struct reftable_iterator *it); + +/* Initialize a reftable iterator for reading logs. */ +int reftable_table_init_log_iterator(struct reftable_table *t, + struct reftable_iterator *it); + +/* returns the hash ID used in this table. */ +enum reftable_hash reftable_table_hash_id(struct reftable_table *t); + +/* return an iterator for the refs pointing to `oid`. */ +int reftable_table_refs_for(struct reftable_table *t, + struct reftable_iterator *it, uint8_t *oid); + +/* return the max_update_index for a table */ +uint64_t reftable_table_max_update_index(struct reftable_table *t); + +/* return the min_update_index for a table */ +uint64_t reftable_table_min_update_index(struct reftable_table *t); + +/* print blocks onto stdout for debugging. */ +int reftable_table_print_blocks(const char *tablename); + +#endif diff --git a/reftable/stack.c b/reftable/stack.c index bc3cfa9117..cc48e725b1 100644 --- a/reftable/stack.c +++ b/reftable/stack.c @@ -11,10 +11,10 @@ #include "system.h" #include "constants.h" #include "merged.h" -#include "reader.h" #include "reftable-error.h" #include "reftable-record.h" #include "reftable-merged.h" +#include "table.h" #include "writer.h" static int stack_try_add(struct reftable_stack *st, @@ -248,11 +248,11 @@ void reftable_stack_destroy(struct reftable_stack *st) REFTABLE_FREE_AND_NULL(names); } - if (st->readers) { + if (st->tables) { struct reftable_buf filename = REFTABLE_BUF_INIT; - for (size_t i = 0; i < st->readers_len; i++) { - const char *name = reader_name(st->readers[i]); + for (size_t i = 0; i < st->tables_len; i++) { + const char *name = reftable_table_name(st->tables[i]); int try_unlinking = 1; reftable_buf_reset(&filename); @@ -260,7 +260,7 @@ void reftable_stack_destroy(struct reftable_stack *st) if (stack_filename(&filename, st, name) < 0) try_unlinking = 0; } - reftable_reader_decref(st->readers[i]); + reftable_table_decref(st->tables[i]); if (try_unlinking && filename.len) { /* On Windows, can only unlink after closing. */ @@ -269,8 +269,8 @@ void reftable_stack_destroy(struct reftable_stack *st) } reftable_buf_release(&filename); - st->readers_len = 0; - REFTABLE_FREE_AND_NULL(st->readers); + st->tables_len = 0; + REFTABLE_FREE_AND_NULL(st->tables); } if (st->list_fd >= 0) { @@ -284,14 +284,14 @@ void reftable_stack_destroy(struct reftable_stack *st) free_names(names); } -static struct reftable_reader **stack_copy_readers(struct reftable_stack *st, - size_t cur_len) +static struct reftable_table **stack_copy_tables(struct reftable_stack *st, + size_t cur_len) { - struct reftable_reader **cur = reftable_calloc(cur_len, sizeof(*cur)); + struct reftable_table **cur = reftable_calloc(cur_len, sizeof(*cur)); if (!cur) return NULL; for (size_t i = 0; i < cur_len; i++) - cur[i] = st->readers[i]; + cur[i] = st->tables[i]; return cur; } @@ -299,19 +299,19 @@ static int reftable_stack_reload_once(struct reftable_stack *st, const char **names, int reuse_open) { - size_t cur_len = !st->merged ? 0 : st->merged->readers_len; - struct reftable_reader **cur = NULL; - struct reftable_reader **reused = NULL; - struct reftable_reader **new_readers = NULL; + size_t cur_len = !st->merged ? 0 : st->merged->tables_len; + struct reftable_table **cur = NULL; + struct reftable_table **reused = NULL; + struct reftable_table **new_tables = NULL; size_t reused_len = 0, reused_alloc = 0, names_len; - size_t new_readers_len = 0; + size_t new_tables_len = 0; struct reftable_merged_table *new_merged = NULL; struct reftable_buf table_path = REFTABLE_BUF_INIT; int err = 0; size_t i; if (cur_len) { - cur = stack_copy_readers(st, cur_len); + cur = stack_copy_tables(st, cur_len); if (!cur) { err = REFTABLE_OUT_OF_MEMORY_ERROR; goto done; @@ -321,28 +321,28 @@ static int reftable_stack_reload_once(struct reftable_stack *st, names_len = names_length(names); if (names_len) { - new_readers = reftable_calloc(names_len, sizeof(*new_readers)); - if (!new_readers) { + new_tables = reftable_calloc(names_len, sizeof(*new_tables)); + if (!new_tables) { err = REFTABLE_OUT_OF_MEMORY_ERROR; goto done; } } while (*names) { - struct reftable_reader *rd = NULL; + struct reftable_table *table = NULL; const char *name = *names++; /* this is linear; we assume compaction keeps the number of tables under control so this is not quadratic. */ for (i = 0; reuse_open && i < cur_len; i++) { if (cur[i] && 0 == strcmp(cur[i]->name, name)) { - rd = cur[i]; + table = cur[i]; cur[i] = NULL; /* * When reloading the stack fails, we end up - * releasing all new readers. This also - * includes the reused readers, even though + * releasing all new tables. This also + * includes the reused tables, even though * they are still in used by the old stack. We * thus need to keep them alive here, which we * do by bumping their refcount. @@ -354,13 +354,13 @@ static int reftable_stack_reload_once(struct reftable_stack *st, err = REFTABLE_OUT_OF_MEMORY_ERROR; goto done; } - reused[reused_len++] = rd; - reftable_reader_incref(rd); + reused[reused_len++] = table; + reftable_table_incref(table); break; } } - if (!rd) { + if (!table) { struct reftable_block_source src = { NULL }; err = stack_filename(&table_path, st, name); @@ -372,36 +372,36 @@ static int reftable_stack_reload_once(struct reftable_stack *st, if (err < 0) goto done; - err = reftable_reader_new(&rd, &src, name); + err = reftable_table_new(&table, &src, name); if (err < 0) goto done; } - new_readers[new_readers_len] = rd; - new_readers_len++; + new_tables[new_tables_len] = table; + new_tables_len++; } /* success! */ - err = reftable_merged_table_new(&new_merged, new_readers, - new_readers_len, st->opts.hash_id); + err = reftable_merged_table_new(&new_merged, new_tables, + new_tables_len, st->opts.hash_id); if (err < 0) goto done; /* - * Close the old, non-reused readers and proactively try to unlink + * Close the old, non-reused tables and proactively try to unlink * them. This is done for systems like Windows, where the underlying - * file of such an open reader wouldn't have been possible to be + * file of such an open table wouldn't have been possible to be * unlinked by the compacting process. */ for (i = 0; i < cur_len; i++) { if (cur[i]) { - const char *name = reader_name(cur[i]); + const char *name = reftable_table_name(cur[i]); err = stack_filename(&table_path, st, name); if (err < 0) goto done; - reftable_reader_decref(cur[i]); + reftable_table_decref(cur[i]); unlink(table_path.buf); } } @@ -412,25 +412,25 @@ static int reftable_stack_reload_once(struct reftable_stack *st, new_merged->suppress_deletions = 1; st->merged = new_merged; - if (st->readers) - reftable_free(st->readers); - st->readers = new_readers; - st->readers_len = new_readers_len; - new_readers = NULL; - new_readers_len = 0; + if (st->tables) + reftable_free(st->tables); + st->tables = new_tables; + st->tables_len = new_tables_len; + new_tables = NULL; + new_tables_len = 0; /* - * Decrement the refcount of reused readers again. This only needs to + * Decrement the refcount of reused tables again. This only needs to * happen on the successful case, because on the unsuccessful one we - * decrement their refcount via `new_readers`. + * decrement their refcount via `new_tables`. */ for (i = 0; i < reused_len; i++) - reftable_reader_decref(reused[i]); + reftable_table_decref(reused[i]); done: - for (i = 0; i < new_readers_len; i++) - reftable_reader_decref(new_readers[i]); - reftable_free(new_readers); + for (i = 0; i < new_tables_len; i++) + reftable_table_decref(new_tables[i]); + reftable_free(new_tables); reftable_free(reused); reftable_free(cur); reftable_buf_release(&table_path); @@ -615,10 +615,10 @@ static int stack_uptodate(struct reftable_stack *st) /* * It's fine for "tables.list" to not exist. In that * case, we have to refresh when the loaded stack has - * any readers. + * any tables. */ if (errno == ENOENT) - return !!st->readers_len; + return !!st->tables_len; return REFTABLE_IO_ERROR; } @@ -637,19 +637,19 @@ static int stack_uptodate(struct reftable_stack *st) if (err < 0) return err; - for (size_t i = 0; i < st->readers_len; i++) { + for (size_t i = 0; i < st->tables_len; i++) { if (!names[i]) { err = 1; goto done; } - if (strcmp(st->readers[i]->name, names[i])) { + if (strcmp(st->tables[i]->name, names[i])) { err = 1; goto done; } } - if (names[st->merged->readers_len]) { + if (names[st->merged->tables_len]) { err = 1; goto done; } @@ -792,8 +792,8 @@ int reftable_addition_commit(struct reftable_addition *add) if (add->new_tables_len == 0) goto done; - for (i = 0; i < add->stack->merged->readers_len; i++) { - if ((err = reftable_buf_addstr(&table_list, add->stack->readers[i]->name)) < 0 || + for (i = 0; i < add->stack->merged->tables_len; i++) { + if ((err = reftable_buf_addstr(&table_list, add->stack->tables[i]->name)) < 0 || (err = reftable_buf_addstr(&table_list, "\n")) < 0) goto done; } @@ -1000,9 +1000,9 @@ done: uint64_t reftable_stack_next_update_index(struct reftable_stack *st) { - int sz = st->merged->readers_len; + int sz = st->merged->tables_len; if (sz > 0) - return reftable_reader_max_update_index(st->readers[sz - 1]) + + return reftable_table_max_update_index(st->tables[sz - 1]) + 1; return 1; } @@ -1021,8 +1021,8 @@ static int stack_compact_locked(struct reftable_stack *st, struct reftable_tmpfile tab_file = REFTABLE_TMPFILE_INIT; int err = 0; - err = format_name(&next_name, reftable_reader_min_update_index(st->readers[first]), - reftable_reader_max_update_index(st->readers[last])); + err = format_name(&next_name, reftable_table_min_update_index(st->tables[first]), + reftable_table_max_update_index(st->tables[last])); if (err < 0) goto done; @@ -1087,13 +1087,13 @@ static int stack_write_compact(struct reftable_stack *st, int err = 0; for (size_t i = first; i <= last; i++) - st->stats.bytes += st->readers[i]->size; - err = reftable_writer_set_limits(wr, st->readers[first]->min_update_index, - st->readers[last]->max_update_index); + st->stats.bytes += st->tables[i]->size; + err = reftable_writer_set_limits(wr, st->tables[first]->min_update_index, + st->tables[last]->max_update_index); if (err < 0) goto done; - err = reftable_merged_table_new(&mt, st->readers + first, subtabs_len, + err = reftable_merged_table_new(&mt, st->tables + first, subtabs_len, st->opts.hash_id); if (err < 0) goto done; @@ -1250,7 +1250,7 @@ static int stack_compact_range(struct reftable_stack *st, table_locks[i] = REFTABLE_FLOCK_INIT; for (i = last + 1; i > first; i--) { - err = stack_filename(&table_name, st, reader_name(st->readers[i - 1])); + err = stack_filename(&table_name, st, reftable_table_name(st->tables[i - 1])); if (err < 0) goto done; @@ -1376,7 +1376,7 @@ static int stack_compact_range(struct reftable_stack *st, * compacted in the updated "tables.list" file. */ for (size_t i = 0; names[i]; i++) { - if (strcmp(names[i], st->readers[first]->name)) + if (strcmp(names[i], st->tables[first]->name)) continue; /* @@ -1386,8 +1386,8 @@ static int stack_compact_range(struct reftable_stack *st, * have compacted them. */ for (size_t j = 1; j < last - first + 1; j++) { - const char *old = first + j < st->merged->readers_len ? - st->readers[first + j]->name : NULL; + const char *old = first + j < st->merged->tables_len ? + st->tables[first + j]->name : NULL; const char *new = names[i + j]; /* @@ -1427,16 +1427,16 @@ static int stack_compact_range(struct reftable_stack *st, * `fd_read_lines()` uses a `NULL` sentinel to indicate that * the array is at its end. As we use `free_names()` to free * the array, we need to include this sentinel value here and - * thus have to allocate `readers_len + 1` many entries. + * thus have to allocate `tables_len + 1` many entries. */ - REFTABLE_CALLOC_ARRAY(names, st->merged->readers_len + 1); + REFTABLE_CALLOC_ARRAY(names, st->merged->tables_len + 1); if (!names) { err = REFTABLE_OUT_OF_MEMORY_ERROR; goto done; } - for (size_t i = 0; i < st->merged->readers_len; i++) { - names[i] = reftable_strdup(st->readers[i]->name); + for (size_t i = 0; i < st->merged->tables_len; i++) { + names[i] = reftable_strdup(st->tables[i]->name); if (!names[i]) { err = REFTABLE_OUT_OF_MEMORY_ERROR; goto done; @@ -1451,8 +1451,8 @@ static int stack_compact_range(struct reftable_stack *st, * it into place now. */ if (!is_empty_table) { - err = format_name(&new_table_name, st->readers[first]->min_update_index, - st->readers[last]->max_update_index); + err = format_name(&new_table_name, st->tables[first]->min_update_index, + st->tables[last]->max_update_index); if (err < 0) goto done; @@ -1559,7 +1559,7 @@ done: int reftable_stack_compact_all(struct reftable_stack *st, struct reftable_log_expiry_config *config) { - size_t last = st->merged->readers_len ? st->merged->readers_len - 1 : 0; + size_t last = st->merged->tables_len ? st->merged->tables_len - 1 : 0; return stack_compact_range(st, 0, last, config, 0); } @@ -1650,12 +1650,12 @@ static uint64_t *stack_table_sizes_for_compaction(struct reftable_stack *st) int overhead = header_size(version) - 1; uint64_t *sizes; - REFTABLE_CALLOC_ARRAY(sizes, st->merged->readers_len); + REFTABLE_CALLOC_ARRAY(sizes, st->merged->tables_len); if (!sizes) return NULL; - for (size_t i = 0; i < st->merged->readers_len; i++) - sizes[i] = st->readers[i]->size - overhead; + for (size_t i = 0; i < st->merged->tables_len; i++) + sizes[i] = st->tables[i]->size - overhead; return sizes; } @@ -1665,14 +1665,14 @@ int reftable_stack_auto_compact(struct reftable_stack *st) struct segment seg; uint64_t *sizes; - if (st->merged->readers_len < 2) + if (st->merged->tables_len < 2) return 0; sizes = stack_table_sizes_for_compaction(st); if (!sizes) return REFTABLE_OUT_OF_MEMORY_ERROR; - seg = suggest_compaction_segment(sizes, st->merged->readers_len, + seg = suggest_compaction_segment(sizes, st->merged->tables_len, st->opts.auto_compaction_factor); reftable_free(sizes); @@ -1763,7 +1763,7 @@ static void remove_maybe_stale_table(struct reftable_stack *st, uint64_t max, int err = 0; uint64_t update_idx = 0; struct reftable_block_source src = { NULL }; - struct reftable_reader *rd = NULL; + struct reftable_table *table = NULL; struct reftable_buf table_path = REFTABLE_BUF_INIT; err = stack_filename(&table_path, st, name); @@ -1774,12 +1774,12 @@ static void remove_maybe_stale_table(struct reftable_stack *st, uint64_t max, if (err < 0) goto done; - err = reftable_reader_new(&rd, &src, name); + err = reftable_table_new(&table, &src, name); if (err < 0) goto done; - update_idx = reftable_reader_max_update_index(rd); - reftable_reader_decref(rd); + update_idx = reftable_table_max_update_index(table); + reftable_table_decref(table); if (update_idx <= max) { unlink(table_path.buf); @@ -1803,8 +1803,8 @@ static int reftable_stack_clean_locked(struct reftable_stack *st) if (!is_table_name(d->d_name)) continue; - for (size_t i = 0; !found && i < st->readers_len; i++) - found = !strcmp(reader_name(st->readers[i]), d->d_name); + for (size_t i = 0; !found && i < st->tables_len; i++) + found = !strcmp(reftable_table_name(st->tables[i]), d->d_name); if (found) continue; diff --git a/reftable/stack.h b/reftable/stack.h index d91f5ce607..bc28f2998a 100644 --- a/reftable/stack.h +++ b/reftable/stack.h @@ -22,8 +22,8 @@ struct reftable_stack { struct reftable_write_options opts; - struct reftable_reader **readers; - size_t readers_len; + struct reftable_table **tables; + size_t tables_len; struct reftable_merged_table *merged; struct reftable_compaction_stats stats; }; diff --git a/reftable/table.c b/reftable/table.c new file mode 100644 index 0000000000..440fb559ad --- /dev/null +++ b/reftable/table.c @@ -0,0 +1,879 @@ +/* + * Copyright 2020 Google LLC + * + * Use of this source code is governed by a BSD-style + * license that can be found in the LICENSE file or at + * https://developers.google.com/open-source/licenses/bsd + */ + +#include "table.h" + +#include "system.h" +#include "block.h" +#include "constants.h" +#include "iter.h" +#include "record.h" +#include "reftable-error.h" + +uint64_t block_source_size(struct reftable_block_source *source) +{ + return source->ops->size(source->arg); +} + +ssize_t block_source_read_block(struct reftable_block_source *source, + struct reftable_block *dest, uint64_t off, + uint32_t size) +{ + ssize_t result = source->ops->read_block(source->arg, dest, off, size); + dest->source = *source; + return result; +} + +void block_source_close(struct reftable_block_source *source) +{ + if (!source->ops) { + return; + } + + source->ops->close(source->arg); + source->ops = NULL; +} + +static struct reftable_table_offsets * +table_offsets_for(struct reftable_table *t, uint8_t typ) +{ + switch (typ) { + case BLOCK_TYPE_REF: + return &t->ref_offsets; + case BLOCK_TYPE_LOG: + return &t->log_offsets; + case BLOCK_TYPE_OBJ: + return &t->obj_offsets; + } + abort(); +} + +static int table_get_block(struct reftable_table *t, + struct reftable_block *dest, uint64_t off, + uint32_t sz) +{ + ssize_t bytes_read; + if (off >= t->size) + return 0; + if (off + sz > t->size) + sz = t->size - off; + + bytes_read = block_source_read_block(&t->source, dest, off, sz); + if (bytes_read < 0) + return (int)bytes_read; + + return 0; +} + +enum reftable_hash reftable_table_hash_id(struct reftable_table *t) +{ + return t->hash_id; +} + +const char *reftable_table_name(struct reftable_table *t) +{ + return t->name; +} + +static int parse_footer(struct reftable_table *t, uint8_t *footer, + uint8_t *header) +{ + uint8_t *f = footer; + uint8_t first_block_typ; + int err = 0; + uint32_t computed_crc; + uint32_t file_crc; + + if (memcmp(f, "REFT", 4)) { + err = REFTABLE_FORMAT_ERROR; + goto done; + } + f += 4; + + if (memcmp(footer, header, header_size(t->version))) { + err = REFTABLE_FORMAT_ERROR; + goto done; + } + + f++; + t->block_size = reftable_get_be24(f); + + f += 3; + t->min_update_index = reftable_get_be64(f); + f += 8; + t->max_update_index = reftable_get_be64(f); + f += 8; + + if (t->version == 1) { + t->hash_id = REFTABLE_HASH_SHA1; + } else { + switch (reftable_get_be32(f)) { + case REFTABLE_FORMAT_ID_SHA1: + t->hash_id = REFTABLE_HASH_SHA1; + break; + case REFTABLE_FORMAT_ID_SHA256: + t->hash_id = REFTABLE_HASH_SHA256; + break; + default: + err = REFTABLE_FORMAT_ERROR; + goto done; + } + + f += 4; + } + + t->ref_offsets.index_offset = reftable_get_be64(f); + f += 8; + + t->obj_offsets.offset = reftable_get_be64(f); + f += 8; + + t->object_id_len = t->obj_offsets.offset & ((1 << 5) - 1); + t->obj_offsets.offset >>= 5; + + t->obj_offsets.index_offset = reftable_get_be64(f); + f += 8; + t->log_offsets.offset = reftable_get_be64(f); + f += 8; + t->log_offsets.index_offset = reftable_get_be64(f); + f += 8; + + computed_crc = crc32(0, footer, f - footer); + file_crc = reftable_get_be32(f); + f += 4; + if (computed_crc != file_crc) { + err = REFTABLE_FORMAT_ERROR; + goto done; + } + + first_block_typ = header[header_size(t->version)]; + t->ref_offsets.is_present = (first_block_typ == BLOCK_TYPE_REF); + t->ref_offsets.offset = 0; + t->log_offsets.is_present = (first_block_typ == BLOCK_TYPE_LOG || + t->log_offsets.offset > 0); + t->obj_offsets.is_present = t->obj_offsets.offset > 0; + if (t->obj_offsets.is_present && !t->object_id_len) { + err = REFTABLE_FORMAT_ERROR; + goto done; + } + + err = 0; +done: + return err; +} + +struct table_iter { + struct reftable_table *table; + uint8_t typ; + uint64_t block_off; + struct block_reader br; + struct block_iter bi; + int is_finished; +}; + +static int table_iter_init(struct table_iter *ti, struct reftable_table *t) +{ + struct block_iter bi = BLOCK_ITER_INIT; + memset(ti, 0, sizeof(*ti)); + reftable_table_incref(t); + ti->table = t; + ti->bi = bi; + return 0; +} + +static int table_iter_next_in_block(struct table_iter *ti, + struct reftable_record *rec) +{ + int res = block_iter_next(&ti->bi, rec); + if (res == 0 && reftable_record_type(rec) == BLOCK_TYPE_REF) { + rec->u.ref.update_index += ti->table->min_update_index; + } + + return res; +} + +static void table_iter_block_done(struct table_iter *ti) +{ + block_reader_release(&ti->br); + block_iter_reset(&ti->bi); +} + +static int32_t extract_block_size(uint8_t *data, uint8_t *typ, uint64_t off, + int version) +{ + int32_t result = 0; + + if (off == 0) { + data += header_size(version); + } + + *typ = data[0]; + if (reftable_is_block_type(*typ)) { + result = reftable_get_be24(data + 1); + } + return result; +} + +int table_init_block_reader(struct reftable_table *t, struct block_reader *br, + uint64_t next_off, uint8_t want_typ) +{ + int32_t guess_block_size = t->block_size ? t->block_size : + DEFAULT_BLOCK_SIZE; + struct reftable_block block = { NULL }; + uint8_t block_typ = 0; + int err = 0; + uint32_t header_off = next_off ? 0 : header_size(t->version); + int32_t block_size = 0; + + if (next_off >= t->size) + return 1; + + err = table_get_block(t, &block, next_off, guess_block_size); + if (err < 0) + goto done; + + block_size = extract_block_size(block.data, &block_typ, next_off, + t->version); + if (block_size < 0) { + err = block_size; + goto done; + } + if (want_typ != BLOCK_TYPE_ANY && block_typ != want_typ) { + err = 1; + goto done; + } + + if (block_size > guess_block_size) { + reftable_block_done(&block); + err = table_get_block(t, &block, next_off, block_size); + if (err < 0) { + goto done; + } + } + + err = block_reader_init(br, &block, header_off, t->block_size, + hash_size(t->hash_id)); +done: + reftable_block_done(&block); + + return err; +} + +static void table_iter_close(struct table_iter *ti) +{ + table_iter_block_done(ti); + block_iter_close(&ti->bi); + reftable_table_decref(ti->table); +} + +static int table_iter_next_block(struct table_iter *ti) +{ + uint64_t next_block_off = ti->block_off + ti->br.full_block_size; + int err; + + err = table_init_block_reader(ti->table, &ti->br, next_block_off, ti->typ); + if (err > 0) + ti->is_finished = 1; + if (err) + return err; + + ti->block_off = next_block_off; + ti->is_finished = 0; + block_iter_seek_start(&ti->bi, &ti->br); + + return 0; +} + +static int table_iter_next(struct table_iter *ti, struct reftable_record *rec) +{ + if (reftable_record_type(rec) != ti->typ) + return REFTABLE_API_ERROR; + + while (1) { + int err; + + if (ti->is_finished) + return 1; + + /* + * Check whether the current block still has more records. If + * so, return it. If the iterator returns positive then the + * current block has been exhausted. + */ + err = table_iter_next_in_block(ti, rec); + if (err <= 0) + return err; + + /* + * Otherwise, we need to continue to the next block in the + * table and retry. If there are no more blocks then the + * iterator is drained. + */ + err = table_iter_next_block(ti); + if (err) { + ti->is_finished = 1; + return err; + } + } +} + +static int table_iter_seek_to(struct table_iter *ti, uint64_t off, uint8_t typ) +{ + int err; + + err = table_init_block_reader(ti->table, &ti->br, off, typ); + if (err != 0) + return err; + + ti->typ = block_reader_type(&ti->br); + ti->block_off = off; + block_iter_seek_start(&ti->bi, &ti->br); + ti->is_finished = 0; + return 0; +} + +static int table_iter_seek_start(struct table_iter *ti, uint8_t typ, int index) +{ + struct reftable_table_offsets *offs = table_offsets_for(ti->table, typ); + uint64_t off = offs->offset; + if (index) { + off = offs->index_offset; + if (off == 0) { + return 1; + } + typ = BLOCK_TYPE_INDEX; + } + + return table_iter_seek_to(ti, off, typ); +} + +static int table_iter_seek_linear(struct table_iter *ti, + struct reftable_record *want) +{ + struct reftable_buf want_key = REFTABLE_BUF_INIT; + struct reftable_buf got_key = REFTABLE_BUF_INIT; + struct reftable_record rec; + int err; + + err = reftable_record_init(&rec, reftable_record_type(want)); + if (err < 0) + goto done; + + err = reftable_record_key(want, &want_key); + if (err < 0) + goto done; + + /* + * First we need to locate the block that must contain our record. To + * do so we scan through blocks linearly until we find the first block + * whose first key is bigger than our wanted key. Once we have found + * that block we know that the key must be contained in the preceding + * block. + * + * This algorithm is somewhat unfortunate because it means that we + * always have to seek one block too far and then back up. But as we + * can only decode the _first_ key of a block but not its _last_ key we + * have no other way to do this. + */ + while (1) { + struct table_iter next = *ti; + + /* + * We must be careful to not modify underlying data of `ti` + * because we may find that `next` does not contain our desired + * block, but that `ti` does. In that case, we would discard + * `next` and continue with `ti`. + * + * This also means that we cannot reuse allocated memory for + * `next` here. While it would be great if we could, it should + * in practice not be too bad given that we should only ever + * end up doing linear seeks with at most three blocks. As soon + * as we have more than three blocks we would have an index, so + * we would not do a linear search there anymore. + */ + memset(&next.br.block, 0, sizeof(next.br.block)); + next.br.zstream = NULL; + next.br.uncompressed_data = NULL; + next.br.uncompressed_cap = 0; + + err = table_iter_next_block(&next); + if (err < 0) + goto done; + if (err > 0) + break; + + err = block_reader_first_key(&next.br, &got_key); + if (err < 0) + goto done; + + if (reftable_buf_cmp(&got_key, &want_key) > 0) { + table_iter_block_done(&next); + break; + } + + table_iter_block_done(ti); + *ti = next; + } + + /* + * We have located the block that must contain our record, so we seek + * the wanted key inside of it. If the block does not contain our key + * we know that the corresponding record does not exist. + */ + err = block_iter_seek_key(&ti->bi, &ti->br, &want_key); + if (err < 0) + goto done; + err = 0; + +done: + reftable_record_release(&rec); + reftable_buf_release(&want_key); + reftable_buf_release(&got_key); + return err; +} + +static int table_iter_seek_indexed(struct table_iter *ti, + struct reftable_record *rec) +{ + struct reftable_record want_index = { + .type = BLOCK_TYPE_INDEX, .u.idx = { .last_key = REFTABLE_BUF_INIT } + }; + struct reftable_record index_result = { + .type = BLOCK_TYPE_INDEX, + .u.idx = { .last_key = REFTABLE_BUF_INIT }, + }; + int err; + + err = reftable_record_key(rec, &want_index.u.idx.last_key); + if (err < 0) + goto done; + + /* + * The index may consist of multiple levels, where each level may have + * multiple index blocks. We start by doing a linear search in the + * highest layer that identifies the relevant index block as well as + * the record inside that block that corresponds to our wanted key. + */ + err = table_iter_seek_linear(ti, &want_index); + if (err < 0) + goto done; + + /* + * Traverse down the levels until we find a non-index entry. + */ + while (1) { + /* + * In case we seek a record that does not exist the index iter + * will tell us that the iterator is over. This works because + * the last index entry of the current level will contain the + * last key it knows about. So in case our seeked key is larger + * than the last indexed key we know that it won't exist. + * + * There is one subtlety in the layout of the index section + * that makes this work as expected: the highest-level index is + * at end of the section and will point backwards and thus we + * start reading from the end of the index section, not the + * beginning. + * + * If that wasn't the case and the order was reversed then the + * linear seek would seek into the lower levels and traverse + * all levels of the index only to find out that the key does + * not exist. + */ + err = table_iter_next(ti, &index_result); + if (err != 0) + goto done; + + err = table_iter_seek_to(ti, index_result.u.idx.offset, 0); + if (err != 0) + goto done; + + err = block_iter_seek_key(&ti->bi, &ti->br, &want_index.u.idx.last_key); + if (err < 0) + goto done; + + if (ti->typ == reftable_record_type(rec)) { + err = 0; + break; + } + + if (ti->typ != BLOCK_TYPE_INDEX) { + err = REFTABLE_FORMAT_ERROR; + goto done; + } + } + +done: + reftable_record_release(&want_index); + reftable_record_release(&index_result); + return err; +} + +static int table_iter_seek(struct table_iter *ti, + struct reftable_record *want) +{ + uint8_t typ = reftable_record_type(want); + struct reftable_table_offsets *offs = table_offsets_for(ti->table, typ); + int err; + + err = table_iter_seek_start(ti, reftable_record_type(want), + !!offs->index_offset); + if (err < 0) + goto out; + + if (offs->index_offset) + err = table_iter_seek_indexed(ti, want); + else + err = table_iter_seek_linear(ti, want); + if (err) + goto out; + +out: + return err; +} + +static int table_iter_seek_void(void *ti, struct reftable_record *want) +{ + return table_iter_seek(ti, want); +} + +static int table_iter_next_void(void *ti, struct reftable_record *rec) +{ + return table_iter_next(ti, rec); +} + +static void table_iter_close_void(void *ti) +{ + table_iter_close(ti); +} + +static struct reftable_iterator_vtable table_iter_vtable = { + .seek = &table_iter_seek_void, + .next = &table_iter_next_void, + .close = &table_iter_close_void, +}; + +static void iterator_from_table_iter(struct reftable_iterator *it, + struct table_iter *ti) +{ + assert(!it->ops); + it->iter_arg = ti; + it->ops = &table_iter_vtable; +} + +int table_init_iter(struct reftable_table *t, + struct reftable_iterator *it, + uint8_t typ) +{ + struct reftable_table_offsets *offs = table_offsets_for(t, typ); + + if (offs->is_present) { + struct table_iter *ti; + REFTABLE_ALLOC_ARRAY(ti, 1); + if (!ti) + return REFTABLE_OUT_OF_MEMORY_ERROR; + + table_iter_init(ti, t); + iterator_from_table_iter(it, ti); + } else { + iterator_set_empty(it); + } + + return 0; +} + +int reftable_table_init_ref_iterator(struct reftable_table *t, + struct reftable_iterator *it) +{ + return table_init_iter(t, it, BLOCK_TYPE_REF); +} + +int reftable_table_init_log_iterator(struct reftable_table *t, + struct reftable_iterator *it) +{ + return table_init_iter(t, it, BLOCK_TYPE_LOG); +} + +int reftable_table_new(struct reftable_table **out, + struct reftable_block_source *source, char const *name) +{ + struct reftable_block footer = { 0 }; + struct reftable_block header = { 0 }; + struct reftable_table *t; + uint64_t file_size = block_source_size(source); + uint32_t read_size; + ssize_t bytes_read; + int err; + + REFTABLE_CALLOC_ARRAY(t, 1); + if (!t) { + err = REFTABLE_OUT_OF_MEMORY_ERROR; + goto done; + } + + /* + * We need one extra byte to read the type of first block. We also + * pretend to always be reading v2 of the format because it is larger. + */ + read_size = header_size(2) + 1; + if (read_size > file_size) { + err = REFTABLE_FORMAT_ERROR; + goto done; + } + + bytes_read = block_source_read_block(source, &header, 0, read_size); + if (bytes_read < 0 || (size_t)bytes_read != read_size) { + err = REFTABLE_IO_ERROR; + goto done; + } + + if (memcmp(header.data, "REFT", 4)) { + err = REFTABLE_FORMAT_ERROR; + goto done; + } + t->version = header.data[4]; + if (t->version != 1 && t->version != 2) { + err = REFTABLE_FORMAT_ERROR; + goto done; + } + + t->size = file_size - footer_size(t->version); + t->source = *source; + t->name = reftable_strdup(name); + if (!t->name) { + err = REFTABLE_OUT_OF_MEMORY_ERROR; + goto done; + } + t->hash_id = 0; + t->refcount = 1; + + bytes_read = block_source_read_block(source, &footer, t->size, + footer_size(t->version)); + if (bytes_read < 0 || (size_t)bytes_read != footer_size(t->version)) { + err = REFTABLE_IO_ERROR; + goto done; + } + + err = parse_footer(t, footer.data, header.data); + if (err) + goto done; + + *out = t; + +done: + reftable_block_done(&footer); + reftable_block_done(&header); + if (err) { + if (t) + reftable_free(t->name); + reftable_free(t); + block_source_close(source); + } + return err; +} + +void reftable_table_incref(struct reftable_table *t) +{ + t->refcount++; +} + +void reftable_table_decref(struct reftable_table *t) +{ + if (!t) + return; + if (--t->refcount) + return; + block_source_close(&t->source); + REFTABLE_FREE_AND_NULL(t->name); + reftable_free(t); +} + +static int reftable_table_refs_for_indexed(struct reftable_table *t, + struct reftable_iterator *it, + uint8_t *oid) +{ + struct reftable_record want = { + .type = BLOCK_TYPE_OBJ, + .u.obj = { + .hash_prefix = oid, + .hash_prefix_len = t->object_id_len, + }, + }; + struct reftable_iterator oit = { NULL }; + struct reftable_record got = { + .type = BLOCK_TYPE_OBJ, + .u.obj = { 0 }, + }; + int err = 0; + struct indexed_table_ref_iter *itr = NULL; + + /* Look through the reverse index. */ + err = table_init_iter(t, &oit, BLOCK_TYPE_OBJ); + if (err < 0) + goto done; + + err = iterator_seek(&oit, &want); + if (err != 0) + goto done; + + /* read out the reftable_obj_record */ + err = iterator_next(&oit, &got); + if (err < 0) + goto done; + + if (err > 0 || memcmp(want.u.obj.hash_prefix, got.u.obj.hash_prefix, + t->object_id_len)) { + /* didn't find it; return empty iterator */ + iterator_set_empty(it); + err = 0; + goto done; + } + + err = indexed_table_ref_iter_new(&itr, t, oid, hash_size(t->hash_id), + got.u.obj.offsets, + got.u.obj.offset_len); + if (err < 0) + goto done; + got.u.obj.offsets = NULL; + iterator_from_indexed_table_ref_iter(it, itr); + +done: + reftable_iterator_destroy(&oit); + reftable_record_release(&got); + return err; +} + +static int reftable_table_refs_for_unindexed(struct reftable_table *t, + struct reftable_iterator *it, + uint8_t *oid) +{ + struct table_iter *ti; + struct filtering_ref_iterator *filter = NULL; + struct filtering_ref_iterator empty = FILTERING_REF_ITERATOR_INIT; + uint32_t oid_len = hash_size(t->hash_id); + int err; + + REFTABLE_ALLOC_ARRAY(ti, 1); + if (!ti) { + err = REFTABLE_OUT_OF_MEMORY_ERROR; + goto out; + } + + table_iter_init(ti, t); + err = table_iter_seek_start(ti, BLOCK_TYPE_REF, 0); + if (err < 0) + goto out; + + filter = reftable_malloc(sizeof(*filter)); + if (!filter) { + err = REFTABLE_OUT_OF_MEMORY_ERROR; + goto out; + } + *filter = empty; + + err = reftable_buf_add(&filter->oid, oid, oid_len); + if (err < 0) + goto out; + + iterator_from_table_iter(&filter->it, ti); + + iterator_from_filtering_ref_iterator(it, filter); + + err = 0; + +out: + if (err < 0) { + if (ti) + table_iter_close(ti); + reftable_free(ti); + } + return err; +} + +int reftable_table_refs_for(struct reftable_table *t, + struct reftable_iterator *it, uint8_t *oid) +{ + if (t->obj_offsets.is_present) + return reftable_table_refs_for_indexed(t, it, oid); + return reftable_table_refs_for_unindexed(t, it, oid); +} + +uint64_t reftable_table_max_update_index(struct reftable_table *t) +{ + return t->max_update_index; +} + +uint64_t reftable_table_min_update_index(struct reftable_table *t) +{ + return t->min_update_index; +} + +int reftable_table_print_blocks(const char *tablename) +{ + struct { + const char *name; + int type; + } sections[] = { + { + .name = "ref", + .type = BLOCK_TYPE_REF, + }, + { + .name = "obj", + .type = BLOCK_TYPE_OBJ, + }, + { + .name = "log", + .type = BLOCK_TYPE_LOG, + }, + }; + struct reftable_block_source src = { 0 }; + struct reftable_table *table = NULL; + struct table_iter ti = { 0 }; + size_t i; + int err; + + err = reftable_block_source_from_file(&src, tablename); + if (err < 0) + goto done; + + err = reftable_table_new(&table, &src, tablename); + if (err < 0) + goto done; + + table_iter_init(&ti, table); + + printf("header:\n"); + printf(" block_size: %d\n", table->block_size); + + for (i = 0; i < sizeof(sections) / sizeof(*sections); i++) { + err = table_iter_seek_start(&ti, sections[i].type, 0); + if (err < 0) + goto done; + if (err > 0) + continue; + + printf("%s:\n", sections[i].name); + + while (1) { + printf(" - length: %u\n", ti.br.block_len); + printf(" restarts: %u\n", ti.br.restart_count); + + err = table_iter_next_block(&ti); + if (err < 0) + goto done; + if (err > 0) + break; + } + } + +done: + reftable_table_decref(table); + table_iter_close(&ti); + return err; +} diff --git a/reftable/table.h b/reftable/table.h new file mode 100644 index 0000000000..9cd8f80a20 --- /dev/null +++ b/reftable/table.h @@ -0,0 +1,67 @@ +/* + * Copyright 2020 Google LLC + * + * Use of this source code is governed by a BSD-style + * license that can be found in the LICENSE file or at + * https://developers.google.com/open-source/licenses/bsd + */ + +#ifndef TABLE_H +#define TABLE_H + +#include "block.h" +#include "record.h" +#include "reftable-iterator.h" +#include "reftable-table.h" + +uint64_t block_source_size(struct reftable_block_source *source); + +ssize_t block_source_read_block(struct reftable_block_source *source, + struct reftable_block *dest, uint64_t off, + uint32_t size); +void block_source_close(struct reftable_block_source *source); + +/* metadata for a block type */ +struct reftable_table_offsets { + int is_present; + uint64_t offset; + uint64_t index_offset; +}; + +/* The state for reading a reftable file. */ +struct reftable_table { + /* for convenience, associate a name with the instance. */ + char *name; + struct reftable_block_source source; + + /* Size of the file, excluding the footer. */ + uint64_t size; + + /* The hash function used for ref records. */ + enum reftable_hash hash_id; + + uint32_t block_size; + uint64_t min_update_index; + uint64_t max_update_index; + /* Length of the OID keys in the 'o' section */ + int object_id_len; + int version; + + struct reftable_table_offsets ref_offsets; + struct reftable_table_offsets obj_offsets; + struct reftable_table_offsets log_offsets; + + uint64_t refcount; +}; + +const char *reftable_table_name(struct reftable_table *t); + +int table_init_iter(struct reftable_table *t, + struct reftable_iterator *it, + uint8_t typ); + +/* initialize a block reader to read from `t` */ +int table_init_block_reader(struct reftable_table *t, struct block_reader *br, + uint64_t next_off, uint8_t want_typ); + +#endif diff --git a/t/helper/test-reftable.c b/t/helper/test-reftable.c index 3c72ed985b..f8f1956f4f 100644 --- a/t/helper/test-reftable.c +++ b/t/helper/test-reftable.c @@ -4,8 +4,8 @@ #include "reftable/system.h" #include "reftable/reftable-error.h" #include "reftable/reftable-merged.h" -#include "reftable/reftable-reader.h" #include "reftable/reftable-stack.h" +#include "reftable/reftable-table.h" #include "test-tool.h" static void print_help(void) @@ -126,19 +126,19 @@ static int dump_reftable(const char *tablename) { struct reftable_block_source src = { 0 }; struct reftable_merged_table *mt = NULL; - struct reftable_reader *r = NULL; + struct reftable_table *table = NULL; int err; err = reftable_block_source_from_file(&src, tablename); if (err < 0) goto done; - err = reftable_reader_new(&r, &src, tablename); + err = reftable_table_new(&table, &src, tablename); if (err < 0) goto done; - err = reftable_merged_table_new(&mt, &r, 1, - reftable_reader_hash_id(r)); + err = reftable_merged_table_new(&mt, &table, 1, + reftable_table_hash_id(table)); if (err < 0) goto done; @@ -146,7 +146,7 @@ static int dump_reftable(const char *tablename) done: reftable_merged_table_free(mt); - reftable_reader_decref(r); + reftable_table_decref(table); return err; } @@ -184,7 +184,7 @@ int cmd__dump_reftable(int argc, const char **argv) arg = argv[1]; if (opt_dump_blocks) { - err = reftable_reader_print_blocks(arg); + err = reftable_table_print_blocks(arg); } else if (opt_dump_table) { err = dump_reftable(arg); } else if (opt_dump_stack) { diff --git a/t/meson.build b/t/meson.build index a59da26be3..ded928d9e6 100644 --- a/t/meson.build +++ b/t/meson.build @@ -56,10 +56,10 @@ unit_test_programs = [ 'unit-tests/t-reftable-block.c', 'unit-tests/t-reftable-merged.c', 'unit-tests/t-reftable-pq.c', - 'unit-tests/t-reftable-reader.c', 'unit-tests/t-reftable-readwrite.c', 'unit-tests/t-reftable-record.c', 'unit-tests/t-reftable-stack.c', + 'unit-tests/t-reftable-table.c', 'unit-tests/t-trailer.c', 'unit-tests/t-urlmatch-normalization.c', ] diff --git a/t/unit-tests/t-reftable-merged.c b/t/unit-tests/t-reftable-merged.c index 60836f80d6..fed6beb85c 100644 --- a/t/unit-tests/t-reftable-merged.c +++ b/t/unit-tests/t-reftable-merged.c @@ -11,7 +11,7 @@ https://developers.google.com/open-source/licenses/bsd #include "reftable/blocksource.h" #include "reftable/constants.h" #include "reftable/merged.h" -#include "reftable/reader.h" +#include "reftable/table.h" #include "reftable/reftable-error.h" #include "reftable/reftable-merged.h" #include "reftable/reftable-writer.h" @@ -19,7 +19,7 @@ https://developers.google.com/open-source/licenses/bsd static struct reftable_merged_table * merged_table_from_records(struct reftable_ref_record **refs, struct reftable_block_source **source, - struct reftable_reader ***readers, const size_t *sizes, + struct reftable_table ***tables, const size_t *sizes, struct reftable_buf *buf, const size_t n) { struct reftable_merged_table *mt = NULL; @@ -28,8 +28,8 @@ merged_table_from_records(struct reftable_ref_record **refs, }; int err; - REFTABLE_CALLOC_ARRAY(*readers, n); - check(*readers != NULL); + REFTABLE_CALLOC_ARRAY(*tables, n); + check(*tables != NULL); REFTABLE_CALLOC_ARRAY(*source, n); check(*source != NULL); @@ -37,21 +37,21 @@ merged_table_from_records(struct reftable_ref_record **refs, t_reftable_write_to_buf(&buf[i], refs[i], sizes[i], NULL, 0, &opts); block_source_from_buf(&(*source)[i], &buf[i]); - err = reftable_reader_new(&(*readers)[i], &(*source)[i], - "name"); + err = reftable_table_new(&(*tables)[i], &(*source)[i], + "name"); check(!err); } - err = reftable_merged_table_new(&mt, *readers, n, REFTABLE_HASH_SHA1); + err = reftable_merged_table_new(&mt, *tables, n, REFTABLE_HASH_SHA1); check(!err); return mt; } -static void readers_destroy(struct reftable_reader **readers, const size_t n) +static void tables_destroy(struct reftable_table **tables, const size_t n) { for (size_t i = 0; i < n; i++) - reftable_reader_decref(readers[i]); - reftable_free(readers); + reftable_table_decref(tables[i]); + reftable_free(tables); } static void t_merged_single_record(void) @@ -77,9 +77,9 @@ static void t_merged_single_record(void) size_t sizes[] = { ARRAY_SIZE(r1), ARRAY_SIZE(r2), ARRAY_SIZE(r3) }; struct reftable_buf bufs[3] = { REFTABLE_BUF_INIT, REFTABLE_BUF_INIT, REFTABLE_BUF_INIT }; struct reftable_block_source *bs = NULL; - struct reftable_reader **readers = NULL; + struct reftable_table **tables = NULL; struct reftable_merged_table *mt = - merged_table_from_records(refs, &bs, &readers, sizes, bufs, 3); + merged_table_from_records(refs, &bs, &tables, sizes, bufs, 3); struct reftable_ref_record ref = { 0 }; struct reftable_iterator it = { 0 }; int err; @@ -94,7 +94,7 @@ static void t_merged_single_record(void) check(reftable_ref_record_equal(&r2[0], &ref, REFTABLE_HASH_SIZE_SHA1)); reftable_ref_record_release(&ref); reftable_iterator_destroy(&it); - readers_destroy(readers, 3); + tables_destroy(tables, 3); reftable_merged_table_free(mt); for (size_t i = 0; i < ARRAY_SIZE(bufs); i++) reftable_buf_release(&bufs[i]); @@ -154,9 +154,9 @@ static void t_merged_refs(void) size_t sizes[3] = { ARRAY_SIZE(r1), ARRAY_SIZE(r2), ARRAY_SIZE(r3) }; struct reftable_buf bufs[3] = { REFTABLE_BUF_INIT, REFTABLE_BUF_INIT, REFTABLE_BUF_INIT }; struct reftable_block_source *bs = NULL; - struct reftable_reader **readers = NULL; + struct reftable_table **tables = NULL; struct reftable_merged_table *mt = - merged_table_from_records(refs, &bs, &readers, sizes, bufs, 3); + merged_table_from_records(refs, &bs, &tables, sizes, bufs, 3); struct reftable_iterator it = { 0 }; int err; struct reftable_ref_record *out = NULL; @@ -193,7 +193,7 @@ static void t_merged_refs(void) for (i = 0; i < 3; i++) reftable_buf_release(&bufs[i]); - readers_destroy(readers, 3); + tables_destroy(tables, 3); reftable_merged_table_free(mt); reftable_free(bs); } @@ -238,12 +238,12 @@ static void t_merged_seek_multiple_times(void) REFTABLE_BUF_INIT, REFTABLE_BUF_INIT, }; struct reftable_block_source *sources = NULL; - struct reftable_reader **readers = NULL; + struct reftable_table **tables = NULL; struct reftable_ref_record rec = { 0 }; struct reftable_iterator it = { 0 }; struct reftable_merged_table *mt; - mt = merged_table_from_records(refs, &sources, &readers, sizes, bufs, 2); + mt = merged_table_from_records(refs, &sources, &tables, sizes, bufs, 2); merged_table_init_iter(mt, &it, BLOCK_TYPE_REF); for (size_t i = 0; i < 5; i++) { @@ -266,7 +266,7 @@ static void t_merged_seek_multiple_times(void) for (size_t i = 0; i < ARRAY_SIZE(bufs); i++) reftable_buf_release(&bufs[i]); - readers_destroy(readers, ARRAY_SIZE(refs)); + tables_destroy(tables, ARRAY_SIZE(refs)); reftable_ref_record_release(&rec); reftable_iterator_destroy(&it); reftable_merged_table_free(mt); @@ -313,13 +313,13 @@ static void t_merged_seek_multiple_times_without_draining(void) REFTABLE_BUF_INIT, REFTABLE_BUF_INIT, }; struct reftable_block_source *sources = NULL; - struct reftable_reader **readers = NULL; + struct reftable_table **tables = NULL; struct reftable_ref_record rec = { 0 }; struct reftable_iterator it = { 0 }; struct reftable_merged_table *mt; int err; - mt = merged_table_from_records(refs, &sources, &readers, sizes, bufs, 2); + mt = merged_table_from_records(refs, &sources, &tables, sizes, bufs, 2); merged_table_init_iter(mt, &it, BLOCK_TYPE_REF); err = reftable_iterator_seek_ref(&it, "b"); @@ -338,7 +338,7 @@ static void t_merged_seek_multiple_times_without_draining(void) for (size_t i = 0; i < ARRAY_SIZE(bufs); i++) reftable_buf_release(&bufs[i]); - readers_destroy(readers, ARRAY_SIZE(refs)); + tables_destroy(tables, ARRAY_SIZE(refs)); reftable_ref_record_release(&rec); reftable_iterator_destroy(&it); reftable_merged_table_free(mt); @@ -348,7 +348,7 @@ static void t_merged_seek_multiple_times_without_draining(void) static struct reftable_merged_table * merged_table_from_log_records(struct reftable_log_record **logs, struct reftable_block_source **source, - struct reftable_reader ***readers, const size_t *sizes, + struct reftable_table ***tables, const size_t *sizes, struct reftable_buf *buf, const size_t n) { struct reftable_merged_table *mt = NULL; @@ -358,8 +358,8 @@ merged_table_from_log_records(struct reftable_log_record **logs, }; int err; - REFTABLE_CALLOC_ARRAY(*readers, n); - check(*readers != NULL); + REFTABLE_CALLOC_ARRAY(*tables, n); + check(*tables != NULL); REFTABLE_CALLOC_ARRAY(*source, n); check(*source != NULL); @@ -367,12 +367,12 @@ merged_table_from_log_records(struct reftable_log_record **logs, t_reftable_write_to_buf(&buf[i], NULL, 0, logs[i], sizes[i], &opts); block_source_from_buf(&(*source)[i], &buf[i]); - err = reftable_reader_new(&(*readers)[i], &(*source)[i], - "name"); + err = reftable_table_new(&(*tables)[i], &(*source)[i], + "name"); check(!err); } - err = reftable_merged_table_new(&mt, *readers, n, REFTABLE_HASH_SHA1); + err = reftable_merged_table_new(&mt, *tables, n, REFTABLE_HASH_SHA1); check(!err); return mt; } @@ -435,9 +435,9 @@ static void t_merged_logs(void) size_t sizes[3] = { ARRAY_SIZE(r1), ARRAY_SIZE(r2), ARRAY_SIZE(r3) }; struct reftable_buf bufs[3] = { REFTABLE_BUF_INIT, REFTABLE_BUF_INIT, REFTABLE_BUF_INIT }; struct reftable_block_source *bs = NULL; - struct reftable_reader **readers = NULL; + struct reftable_table **tables = NULL; struct reftable_merged_table *mt = merged_table_from_log_records( - logs, &bs, &readers, sizes, bufs, 3); + logs, &bs, &tables, sizes, bufs, 3); struct reftable_iterator it = { 0 }; int err; struct reftable_log_record *out = NULL; @@ -485,7 +485,7 @@ static void t_merged_logs(void) for (i = 0; i < 3; i++) reftable_buf_release(&bufs[i]); - readers_destroy(readers, 3); + tables_destroy(tables, 3); reftable_merged_table_free(mt); reftable_free(bs); } @@ -502,7 +502,7 @@ static void t_default_write_opts(void) int err; struct reftable_block_source source = { 0 }; uint32_t hash_id; - struct reftable_reader *rd = NULL; + struct reftable_table *table = NULL; struct reftable_merged_table *merged = NULL; reftable_writer_set_limits(w, 1, 1); @@ -516,18 +516,18 @@ static void t_default_write_opts(void) block_source_from_buf(&source, &buf); - err = reftable_reader_new(&rd, &source, "filename"); + err = reftable_table_new(&table, &source, "filename"); check(!err); - hash_id = reftable_reader_hash_id(rd); + hash_id = reftable_table_hash_id(table); check_int(hash_id, ==, REFTABLE_HASH_SHA1); - err = reftable_merged_table_new(&merged, &rd, 1, REFTABLE_HASH_SHA256); + err = reftable_merged_table_new(&merged, &table, 1, REFTABLE_HASH_SHA256); check_int(err, ==, REFTABLE_FORMAT_ERROR); - err = reftable_merged_table_new(&merged, &rd, 1, REFTABLE_HASH_SHA1); + err = reftable_merged_table_new(&merged, &table, 1, REFTABLE_HASH_SHA1); check(!err); - reftable_reader_decref(rd); + reftable_table_decref(table); reftable_merged_table_free(merged); reftable_buf_release(&buf); } diff --git a/t/unit-tests/t-reftable-reader.c b/t/unit-tests/t-reftable-reader.c deleted file mode 100644 index 546df6005e..0000000000 --- a/t/unit-tests/t-reftable-reader.c +++ /dev/null @@ -1,96 +0,0 @@ -#include "test-lib.h" -#include "lib-reftable.h" -#include "reftable/blocksource.h" -#include "reftable/reader.h" - -static int t_reader_seek_once(void) -{ - struct reftable_ref_record records[] = { - { - .refname = (char *) "refs/heads/main", - .value_type = REFTABLE_REF_VAL1, - .value.val1 = { 42 }, - }, - }; - struct reftable_block_source source = { 0 }; - struct reftable_ref_record ref = { 0 }; - struct reftable_iterator it = { 0 }; - struct reftable_reader *reader; - struct reftable_buf buf = REFTABLE_BUF_INIT; - int ret; - - t_reftable_write_to_buf(&buf, records, ARRAY_SIZE(records), NULL, 0, NULL); - block_source_from_buf(&source, &buf); - - ret = reftable_reader_new(&reader, &source, "name"); - check(!ret); - - reftable_reader_init_ref_iterator(reader, &it); - ret = reftable_iterator_seek_ref(&it, ""); - check(!ret); - ret = reftable_iterator_next_ref(&it, &ref); - check(!ret); - - ret = reftable_ref_record_equal(&ref, &records[0], REFTABLE_HASH_SIZE_SHA1); - check_int(ret, ==, 1); - - ret = reftable_iterator_next_ref(&it, &ref); - check_int(ret, ==, 1); - - reftable_ref_record_release(&ref); - reftable_iterator_destroy(&it); - reftable_reader_decref(reader); - reftable_buf_release(&buf); - return 0; -} - -static int t_reader_reseek(void) -{ - struct reftable_ref_record records[] = { - { - .refname = (char *) "refs/heads/main", - .value_type = REFTABLE_REF_VAL1, - .value.val1 = { 42 }, - }, - }; - struct reftable_block_source source = { 0 }; - struct reftable_ref_record ref = { 0 }; - struct reftable_iterator it = { 0 }; - struct reftable_reader *reader; - struct reftable_buf buf = REFTABLE_BUF_INIT; - int ret; - - t_reftable_write_to_buf(&buf, records, ARRAY_SIZE(records), NULL, 0, NULL); - block_source_from_buf(&source, &buf); - - ret = reftable_reader_new(&reader, &source, "name"); - check(!ret); - - reftable_reader_init_ref_iterator(reader, &it); - - for (size_t i = 0; i < 5; i++) { - ret = reftable_iterator_seek_ref(&it, ""); - check(!ret); - ret = reftable_iterator_next_ref(&it, &ref); - check(!ret); - - ret = reftable_ref_record_equal(&ref, &records[0], REFTABLE_HASH_SIZE_SHA1); - check_int(ret, ==, 1); - - ret = reftable_iterator_next_ref(&it, &ref); - check_int(ret, ==, 1); - } - - reftable_ref_record_release(&ref); - reftable_iterator_destroy(&it); - reftable_reader_decref(reader); - reftable_buf_release(&buf); - return 0; -} - -int cmd_main(int argc UNUSED, const char *argv[] UNUSED) -{ - TEST(t_reader_seek_once(), "reader can seek once"); - TEST(t_reader_reseek(), "reader can reseek multiple times"); - return test_done(); -} diff --git a/t/unit-tests/t-reftable-readwrite.c b/t/unit-tests/t-reftable-readwrite.c index c9626831da..c4c27242ba 100644 --- a/t/unit-tests/t-reftable-readwrite.c +++ b/t/unit-tests/t-reftable-readwrite.c @@ -12,9 +12,9 @@ https://developers.google.com/open-source/licenses/bsd #include "lib-reftable.h" #include "reftable/basics.h" #include "reftable/blocksource.h" -#include "reftable/reader.h" #include "reftable/reftable-error.h" #include "reftable/reftable-writer.h" +#include "reftable/table.h" #include "strbuf.h" static const int update_index = 5; @@ -204,7 +204,7 @@ static void t_log_write_read(void) struct reftable_ref_record ref = { 0 }; struct reftable_log_record log = { 0 }; struct reftable_iterator it = { 0 }; - struct reftable_reader *reader; + struct reftable_table *table; struct reftable_block_source source = { 0 }; struct reftable_buf buf = REFTABLE_BUF_INIT; struct reftable_writer *w = t_reftable_strbuf_writer(&buf, &opts); @@ -254,10 +254,10 @@ static void t_log_write_read(void) block_source_from_buf(&source, &buf); - err = reftable_reader_new(&reader, &source, "file.log"); + err = reftable_table_new(&table, &source, "file.log"); check(!err); - err = reftable_reader_init_ref_iterator(reader, &it); + err = reftable_table_init_ref_iterator(table, &it); check(!err); err = reftable_iterator_seek_ref(&it, names[N - 1]); @@ -273,7 +273,7 @@ static void t_log_write_read(void) reftable_iterator_destroy(&it); reftable_ref_record_release(&ref); - err = reftable_reader_init_log_iterator(reader, &it); + err = reftable_table_init_log_iterator(table, &it); check(!err); err = reftable_iterator_seek_log(&it, ""); check(!err); @@ -294,7 +294,7 @@ static void t_log_write_read(void) /* cleanup. */ reftable_buf_release(&buf); free_names(names); - reftable_reader_decref(reader); + reftable_table_decref(table); } static void t_log_zlib_corruption(void) @@ -303,7 +303,7 @@ static void t_log_zlib_corruption(void) .block_size = 256, }; struct reftable_iterator it = { 0 }; - struct reftable_reader *reader; + struct reftable_table *table; struct reftable_block_source source = { 0 }; struct reftable_buf buf = REFTABLE_BUF_INIT; struct reftable_writer *w = t_reftable_strbuf_writer(&buf, &opts); @@ -345,10 +345,10 @@ static void t_log_zlib_corruption(void) block_source_from_buf(&source, &buf); - err = reftable_reader_new(&reader, &source, "file.log"); + err = reftable_table_new(&table, &source, "file.log"); check(!err); - err = reftable_reader_init_log_iterator(reader, &it); + err = reftable_table_init_log_iterator(table, &it); check(!err); err = reftable_iterator_seek_log(&it, "refname"); check_int(err, ==, REFTABLE_ZLIB_ERROR); @@ -356,7 +356,7 @@ static void t_log_zlib_corruption(void) reftable_iterator_destroy(&it); /* cleanup. */ - reftable_reader_decref(reader); + reftable_table_decref(table); reftable_buf_release(&buf); } @@ -367,7 +367,7 @@ static void t_table_read_write_sequential(void) int N = 50; struct reftable_iterator it = { 0 }; struct reftable_block_source source = { 0 }; - struct reftable_reader *reader; + struct reftable_table *table; int err = 0; int j = 0; @@ -375,10 +375,10 @@ static void t_table_read_write_sequential(void) block_source_from_buf(&source, &buf); - err = reftable_reader_new(&reader, &source, "file.ref"); + err = reftable_table_new(&table, &source, "file.ref"); check(!err); - err = reftable_reader_init_ref_iterator(reader, &it); + err = reftable_table_init_ref_iterator(table, &it); check(!err); err = reftable_iterator_seek_ref(&it, ""); check(!err); @@ -396,7 +396,7 @@ static void t_table_read_write_sequential(void) check_int(j, ==, N); reftable_iterator_destroy(&it); - reftable_reader_decref(reader); + reftable_table_decref(table); reftable_buf_release(&buf); free_names(names); } @@ -417,7 +417,7 @@ static void t_table_read_api(void) char **names; struct reftable_buf buf = REFTABLE_BUF_INIT; int N = 50; - struct reftable_reader *reader; + struct reftable_table *table; struct reftable_block_source source = { 0 }; int err; struct reftable_log_record log = { 0 }; @@ -427,10 +427,10 @@ static void t_table_read_api(void) block_source_from_buf(&source, &buf); - err = reftable_reader_new(&reader, &source, "file.ref"); + err = reftable_table_new(&table, &source, "file.ref"); check(!err); - err = reftable_reader_init_ref_iterator(reader, &it); + err = reftable_table_init_ref_iterator(table, &it); check(!err); err = reftable_iterator_seek_ref(&it, names[0]); check(!err); @@ -441,7 +441,7 @@ static void t_table_read_api(void) reftable_buf_release(&buf); free_names(names); reftable_iterator_destroy(&it); - reftable_reader_decref(reader); + reftable_table_decref(table); reftable_buf_release(&buf); } @@ -450,7 +450,7 @@ static void t_table_read_write_seek(int index, enum reftable_hash hash_id) char **names; struct reftable_buf buf = REFTABLE_BUF_INIT; int N = 50; - struct reftable_reader *reader; + struct reftable_table *table; struct reftable_block_source source = { 0 }; int err; int i = 0; @@ -463,18 +463,18 @@ static void t_table_read_write_seek(int index, enum reftable_hash hash_id) block_source_from_buf(&source, &buf); - err = reftable_reader_new(&reader, &source, "file.ref"); + err = reftable_table_new(&table, &source, "file.ref"); check(!err); - check_int(hash_id, ==, reftable_reader_hash_id(reader)); + check_int(hash_id, ==, reftable_table_hash_id(table)); if (!index) { - reader->ref_offsets.index_offset = 0; + table->ref_offsets.index_offset = 0; } else { - check_int(reader->ref_offsets.index_offset, >, 0); + check_int(table->ref_offsets.index_offset, >, 0); } for (i = 1; i < N; i++) { - err = reftable_reader_init_ref_iterator(reader, &it); + err = reftable_table_init_ref_iterator(table, &it); check(!err); err = reftable_iterator_seek_ref(&it, names[i]); check(!err); @@ -491,7 +491,7 @@ static void t_table_read_write_seek(int index, enum reftable_hash hash_id) check(!reftable_buf_addstr(&pastLast, names[N - 1])); check(!reftable_buf_addstr(&pastLast, "/")); - err = reftable_reader_init_ref_iterator(reader, &it); + err = reftable_table_init_ref_iterator(table, &it); check(!err); err = reftable_iterator_seek_ref(&it, pastLast.buf); if (err == 0) { @@ -507,7 +507,7 @@ static void t_table_read_write_seek(int index, enum reftable_hash hash_id) reftable_buf_release(&buf); free_names(names); - reftable_reader_decref(reader); + reftable_table_decref(table); } static void t_table_read_write_seek_linear(void) @@ -535,7 +535,7 @@ static void t_table_refs_for(int indexed) .block_size = 256, }; struct reftable_ref_record ref = { 0 }; - struct reftable_reader *reader; + struct reftable_table *table; struct reftable_block_source source = { 0 }; struct reftable_buf buf = REFTABLE_BUF_INIT; struct reftable_writer *w = t_reftable_strbuf_writer(&buf, &opts); @@ -585,18 +585,18 @@ static void t_table_refs_for(int indexed) block_source_from_buf(&source, &buf); - err = reftable_reader_new(&reader, &source, "file.ref"); + err = reftable_table_new(&table, &source, "file.ref"); check(!err); if (!indexed) - reader->obj_offsets.is_present = 0; + table->obj_offsets.is_present = 0; - err = reftable_reader_init_ref_iterator(reader, &it); + err = reftable_table_init_ref_iterator(table, &it); check(!err); err = reftable_iterator_seek_ref(&it, ""); check(!err); reftable_iterator_destroy(&it); - err = reftable_reader_refs_for(reader, &it, want_hash); + err = reftable_table_refs_for(table, &it, want_hash); check(!err); for (j = 0; ; j++) { @@ -613,7 +613,7 @@ static void t_table_refs_for(int indexed) reftable_buf_release(&buf); free_names(want_names); reftable_iterator_destroy(&it); - reftable_reader_decref(reader); + reftable_table_decref(table); } static void t_table_refs_for_no_index(void) @@ -632,7 +632,7 @@ static void t_write_empty_table(void) struct reftable_buf buf = REFTABLE_BUF_INIT; struct reftable_writer *w = t_reftable_strbuf_writer(&buf, &opts); struct reftable_block_source source = { 0 }; - struct reftable_reader *rd = NULL; + struct reftable_table *table = NULL; struct reftable_ref_record rec = { 0 }; struct reftable_iterator it = { 0 }; int err; @@ -647,10 +647,10 @@ static void t_write_empty_table(void) block_source_from_buf(&source, &buf); - err = reftable_reader_new(&rd, &source, "filename"); + err = reftable_table_new(&table, &source, "filename"); check(!err); - err = reftable_reader_init_ref_iterator(rd, &it); + err = reftable_table_init_ref_iterator(table, &it); check(!err); err = reftable_iterator_seek_ref(&it, ""); check(!err); @@ -659,7 +659,7 @@ static void t_write_empty_table(void) check_int(err, >, 0); reftable_iterator_destroy(&it); - reftable_reader_decref(rd); + reftable_table_decref(table); reftable_buf_release(&buf); } @@ -803,7 +803,7 @@ static void t_write_multiple_indices(void) struct reftable_iterator it = { 0 }; const struct reftable_stats *stats; struct reftable_writer *writer; - struct reftable_reader *reader; + struct reftable_table *table; char buf[128]; int err, i; @@ -852,21 +852,21 @@ static void t_write_multiple_indices(void) check_int(stats->log_stats.index_offset, >, 0); block_source_from_buf(&source, &writer_buf); - err = reftable_reader_new(&reader, &source, "filename"); + err = reftable_table_new(&table, &source, "filename"); check(!err); /* * Seeking the log uses the log index now. In case there is any * confusion regarding indices we would notice here. */ - err = reftable_reader_init_log_iterator(reader, &it); + err = reftable_table_init_log_iterator(table, &it); check(!err); err = reftable_iterator_seek_log(&it, ""); check(!err); reftable_iterator_destroy(&it); reftable_writer_free(writer); - reftable_reader_decref(reader); + reftable_table_decref(table); reftable_buf_release(&writer_buf); } @@ -880,7 +880,7 @@ static void t_write_multi_level_index(void) struct reftable_iterator it = { 0 }; const struct reftable_stats *stats; struct reftable_writer *writer; - struct reftable_reader *reader; + struct reftable_table *table; int err; writer = t_reftable_strbuf_writer(&writer_buf, &opts); @@ -909,20 +909,20 @@ static void t_write_multi_level_index(void) check_int(stats->ref_stats.max_index_level, ==, 2); block_source_from_buf(&source, &writer_buf); - err = reftable_reader_new(&reader, &source, "filename"); + err = reftable_table_new(&table, &source, "filename"); check(!err); /* * Seeking the last ref should work as expected. */ - err = reftable_reader_init_ref_iterator(reader, &it); + err = reftable_table_init_ref_iterator(table, &it); check(!err); err = reftable_iterator_seek_ref(&it, "refs/heads/199"); check(!err); reftable_iterator_destroy(&it); reftable_writer_free(writer); - reftable_reader_decref(reader); + reftable_table_decref(table); reftable_buf_release(&writer_buf); reftable_buf_release(&buf); } @@ -931,11 +931,11 @@ static void t_corrupt_table_empty(void) { struct reftable_buf buf = REFTABLE_BUF_INIT; struct reftable_block_source source = { 0 }; - struct reftable_reader *reader; + struct reftable_table *table; int err; block_source_from_buf(&source, &buf); - err = reftable_reader_new(&reader, &source, "file.log"); + err = reftable_table_new(&table, &source, "file.log"); check_int(err, ==, REFTABLE_FORMAT_ERROR); } @@ -944,12 +944,12 @@ static void t_corrupt_table(void) uint8_t zeros[1024] = { 0 }; struct reftable_buf buf = REFTABLE_BUF_INIT; struct reftable_block_source source = { 0 }; - struct reftable_reader *reader; + struct reftable_table *table; int err; check(!reftable_buf_add(&buf, zeros, sizeof(zeros))); block_source_from_buf(&source, &buf); - err = reftable_reader_new(&reader, &source, "file.log"); + err = reftable_table_new(&table, &source, "file.log"); check_int(err, ==, REFTABLE_FORMAT_ERROR); reftable_buf_release(&buf); diff --git a/t/unit-tests/t-reftable-stack.c b/t/unit-tests/t-reftable-stack.c index c3f0059c34..2f49c97519 100644 --- a/t/unit-tests/t-reftable-stack.c +++ b/t/unit-tests/t-reftable-stack.c @@ -12,9 +12,9 @@ https://developers.google.com/open-source/licenses/bsd #include "lib-reftable.h" #include "dir.h" #include "reftable/merged.h" -#include "reftable/reader.h" #include "reftable/reftable-error.h" #include "reftable/stack.h" +#include "reftable/table.h" #include "strbuf.h" #include "tempfile.h" #include @@ -176,7 +176,7 @@ static void t_reftable_stack_add_one(void) err = reftable_stack_read_ref(st, ref.refname, &dest); check(!err); check(reftable_ref_record_equal(&ref, &dest, REFTABLE_HASH_SIZE_SHA1)); - check_int(st->readers_len, >, 0); + check_int(st->tables_len, >, 0); #ifndef GIT_WINDOWS_NATIVE check(!reftable_buf_addstr(&scratch, dir)); @@ -189,7 +189,7 @@ static void t_reftable_stack_add_one(void) check(!reftable_buf_addstr(&scratch, dir)); check(!reftable_buf_addstr(&scratch, "/")); /* do not try at home; not an external API for reftable. */ - check(!reftable_buf_addstr(&scratch, st->readers[0]->name)); + check(!reftable_buf_addstr(&scratch, st->tables[0]->name)); err = stat(scratch.buf, &stat_result); check(!err); check_int((stat_result.st_mode & 0777), ==, opts.default_permissions); @@ -402,9 +402,9 @@ static void t_reftable_stack_transaction_api_performs_auto_compaction(void) * all tables in the stack. */ if (i != n) - check_int(st->merged->readers_len, ==, i + 1); + check_int(st->merged->tables_len, ==, i + 1); else - check_int(st->merged->readers_len, ==, 1); + check_int(st->merged->tables_len, ==, 1); } reftable_stack_destroy(st); @@ -430,7 +430,7 @@ static void t_reftable_stack_auto_compaction_fails_gracefully(void) err = reftable_stack_add(st, write_test_ref, &ref); check(!err); - check_int(st->merged->readers_len, ==, 1); + check_int(st->merged->tables_len, ==, 1); check_int(st->stats.attempts, ==, 0); check_int(st->stats.failures, ==, 0); @@ -441,14 +441,14 @@ static void t_reftable_stack_auto_compaction_fails_gracefully(void) */ check(!reftable_buf_addstr(&table_path, dir)); check(!reftable_buf_addstr(&table_path, "/")); - check(!reftable_buf_addstr(&table_path, st->readers[0]->name)); + check(!reftable_buf_addstr(&table_path, st->tables[0]->name)); check(!reftable_buf_addstr(&table_path, ".lock")); write_file_buf(table_path.buf, "", 0); ref.update_index = 2; err = reftable_stack_add(st, write_test_ref, &ref); check(!err); - check_int(st->merged->readers_len, ==, 2); + check_int(st->merged->tables_len, ==, 2); check_int(st->stats.attempts, ==, 1); check_int(st->stats.failures, ==, 1); @@ -592,7 +592,7 @@ static void t_reftable_stack_add(void) check(!reftable_buf_addstr(&path, dir)); check(!reftable_buf_addstr(&path, "/")); /* do not try at home; not an external API for reftable. */ - check(!reftable_buf_addstr(&path, st->readers[0]->name)); + check(!reftable_buf_addstr(&path, st->tables[0]->name)); err = stat(path.buf, &stat_result); check(!err); check_int((stat_result.st_mode & 0777), ==, opts.default_permissions); @@ -1026,7 +1026,7 @@ static void t_reftable_stack_auto_compaction(void) err = reftable_stack_auto_compact(st); check(!err); - check(i < 2 || st->merged->readers_len < 2 * fastlogN(i, 2)); + check(i < 2 || st->merged->tables_len < 2 * fastlogN(i, 2)); } check_int(reftable_stack_compaction_stats(st)->entries_written, <, @@ -1061,7 +1061,7 @@ static void t_reftable_stack_auto_compaction_factor(void) err = reftable_stack_add(st, &write_test_ref, &ref); check(!err); - check(i < 5 || st->merged->readers_len < 5 * fastlogN(i, 5)); + check(i < 5 || st->merged->tables_len < 5 * fastlogN(i, 5)); } reftable_stack_destroy(st); @@ -1082,7 +1082,7 @@ static void t_reftable_stack_auto_compaction_with_locked_tables(void) check(!err); write_n_ref_tables(st, 5); - check_int(st->merged->readers_len, ==, 5); + check_int(st->merged->tables_len, ==, 5); /* * Given that all tables we have written should be roughly the same @@ -1091,7 +1091,7 @@ static void t_reftable_stack_auto_compaction_with_locked_tables(void) */ check(!reftable_buf_addstr(&buf, dir)); check(!reftable_buf_addstr(&buf, "/")); - check(!reftable_buf_addstr(&buf, st->readers[2]->name)); + check(!reftable_buf_addstr(&buf, st->tables[2]->name)); check(!reftable_buf_addstr(&buf, ".lock")); write_file_buf(buf.buf, "", 0); @@ -1104,7 +1104,7 @@ static void t_reftable_stack_auto_compaction_with_locked_tables(void) err = reftable_stack_auto_compact(st); check(!err); check_int(st->stats.failures, ==, 0); - check_int(st->merged->readers_len, ==, 4); + check_int(st->merged->tables_len, ==, 4); reftable_stack_destroy(st); reftable_buf_release(&buf); @@ -1149,9 +1149,9 @@ static void t_reftable_stack_add_performs_auto_compaction(void) * all tables in the stack. */ if (i != n) - check_int(st->merged->readers_len, ==, i + 1); + check_int(st->merged->tables_len, ==, i + 1); else - check_int(st->merged->readers_len, ==, 1); + check_int(st->merged->tables_len, ==, 1); } reftable_stack_destroy(st); @@ -1172,12 +1172,12 @@ static void t_reftable_stack_compaction_with_locked_tables(void) check(!err); write_n_ref_tables(st, 3); - check_int(st->merged->readers_len, ==, 3); + check_int(st->merged->tables_len, ==, 3); /* Lock one of the tables that we're about to compact. */ check(!reftable_buf_addstr(&buf, dir)); check(!reftable_buf_addstr(&buf, "/")); - check(!reftable_buf_addstr(&buf, st->readers[1]->name)); + check(!reftable_buf_addstr(&buf, st->tables[1]->name)); check(!reftable_buf_addstr(&buf, ".lock")); write_file_buf(buf.buf, "", 0); @@ -1188,7 +1188,7 @@ static void t_reftable_stack_compaction_with_locked_tables(void) err = reftable_stack_compact_all(st, NULL); check_int(err, ==, REFTABLE_LOCK_ERROR); check_int(st->stats.failures, ==, 1); - check_int(st->merged->readers_len, ==, 3); + check_int(st->merged->tables_len, ==, 3); reftable_stack_destroy(st); reftable_buf_release(&buf); @@ -1222,10 +1222,10 @@ static void t_reftable_stack_compaction_concurrent(void) static void unclean_stack_close(struct reftable_stack *st) { /* break abstraction boundary to simulate unclean shutdown. */ - for (size_t i = 0; i < st->readers_len; i++) - reftable_reader_decref(st->readers[i]); - st->readers_len = 0; - REFTABLE_FREE_AND_NULL(st->readers); + for (size_t i = 0; i < st->tables_len; i++) + reftable_table_decref(st->tables[i]); + st->tables_len = 0; + REFTABLE_FREE_AND_NULL(st->tables); } static void t_reftable_stack_compaction_concurrent_clean(void) @@ -1275,7 +1275,7 @@ static void t_reftable_stack_read_across_reload(void) err = reftable_new_stack(&st1, dir, &opts); check(!err); write_n_ref_tables(st1, 2); - check_int(st1->merged->readers_len, ==, 2); + check_int(st1->merged->tables_len, ==, 2); reftable_stack_init_ref_iterator(st1, &it); err = reftable_iterator_seek_ref(&it, ""); check(!err); @@ -1283,10 +1283,10 @@ static void t_reftable_stack_read_across_reload(void) /* Set up a second stack for the same directory and compact it. */ err = reftable_new_stack(&st2, dir, &opts); check(!err); - check_int(st2->merged->readers_len, ==, 2); + check_int(st2->merged->tables_len, ==, 2); err = reftable_stack_compact_all(st2, NULL); check(!err); - check_int(st2->merged->readers_len, ==, 1); + check_int(st2->merged->tables_len, ==, 1); /* * Verify that we can continue to use the old iterator even after we @@ -1294,7 +1294,7 @@ static void t_reftable_stack_read_across_reload(void) */ err = reftable_stack_reload(st1); check(!err); - check_int(st1->merged->readers_len, ==, 1); + check_int(st1->merged->tables_len, ==, 1); err = reftable_iterator_next_ref(&it, &rec); check(!err); check_str(rec.refname, "refs/heads/branch-0000"); @@ -1325,19 +1325,19 @@ static void t_reftable_stack_reload_with_missing_table(void) err = reftable_new_stack(&st, dir, &opts); check(!err); write_n_ref_tables(st, 2); - check_int(st->merged->readers_len, ==, 2); + check_int(st->merged->tables_len, ==, 2); reftable_stack_init_ref_iterator(st, &it); err = reftable_iterator_seek_ref(&it, ""); check(!err); /* * Update the tables.list file with some garbage data, while reusing - * our old readers. This should trigger a partial reload of the stack, - * where we try to reuse our old readers. + * our old tables. This should trigger a partial reload of the stack, + * where we try to reuse our old tables. */ - check(!reftable_buf_addstr(&content, st->readers[0]->name)); + check(!reftable_buf_addstr(&content, st->tables[0]->name)); check(!reftable_buf_addstr(&content, "\n")); - check(!reftable_buf_addstr(&content, st->readers[1]->name)); + check(!reftable_buf_addstr(&content, st->tables[1]->name)); check(!reftable_buf_addstr(&content, "\n")); check(!reftable_buf_addstr(&content, "garbage\n")); check(!reftable_buf_addstr(&table_path, st->list_file)); @@ -1348,7 +1348,7 @@ static void t_reftable_stack_reload_with_missing_table(void) err = reftable_stack_reload(st); check_int(err, ==, -4); - check_int(st->merged->readers_len, ==, 2); + check_int(st->merged->tables_len, ==, 2); /* * Even though the reload has failed, we should be able to continue diff --git a/t/unit-tests/t-reftable-table.c b/t/unit-tests/t-reftable-table.c new file mode 100644 index 0000000000..77c59dbf46 --- /dev/null +++ b/t/unit-tests/t-reftable-table.c @@ -0,0 +1,96 @@ +#include "test-lib.h" +#include "lib-reftable.h" +#include "reftable/blocksource.h" +#include "reftable/table.h" + +static int t_table_seek_once(void) +{ + struct reftable_ref_record records[] = { + { + .refname = (char *) "refs/heads/main", + .value_type = REFTABLE_REF_VAL1, + .value.val1 = { 42 }, + }, + }; + struct reftable_block_source source = { 0 }; + struct reftable_ref_record ref = { 0 }; + struct reftable_iterator it = { 0 }; + struct reftable_table *table; + struct reftable_buf buf = REFTABLE_BUF_INIT; + int ret; + + t_reftable_write_to_buf(&buf, records, ARRAY_SIZE(records), NULL, 0, NULL); + block_source_from_buf(&source, &buf); + + ret = reftable_table_new(&table, &source, "name"); + check(!ret); + + reftable_table_init_ref_iterator(table, &it); + ret = reftable_iterator_seek_ref(&it, ""); + check(!ret); + ret = reftable_iterator_next_ref(&it, &ref); + check(!ret); + + ret = reftable_ref_record_equal(&ref, &records[0], REFTABLE_HASH_SIZE_SHA1); + check_int(ret, ==, 1); + + ret = reftable_iterator_next_ref(&it, &ref); + check_int(ret, ==, 1); + + reftable_ref_record_release(&ref); + reftable_iterator_destroy(&it); + reftable_table_decref(table); + reftable_buf_release(&buf); + return 0; +} + +static int t_table_reseek(void) +{ + struct reftable_ref_record records[] = { + { + .refname = (char *) "refs/heads/main", + .value_type = REFTABLE_REF_VAL1, + .value.val1 = { 42 }, + }, + }; + struct reftable_block_source source = { 0 }; + struct reftable_ref_record ref = { 0 }; + struct reftable_iterator it = { 0 }; + struct reftable_table *table; + struct reftable_buf buf = REFTABLE_BUF_INIT; + int ret; + + t_reftable_write_to_buf(&buf, records, ARRAY_SIZE(records), NULL, 0, NULL); + block_source_from_buf(&source, &buf); + + ret = reftable_table_new(&table, &source, "name"); + check(!ret); + + reftable_table_init_ref_iterator(table, &it); + + for (size_t i = 0; i < 5; i++) { + ret = reftable_iterator_seek_ref(&it, ""); + check(!ret); + ret = reftable_iterator_next_ref(&it, &ref); + check(!ret); + + ret = reftable_ref_record_equal(&ref, &records[0], REFTABLE_HASH_SIZE_SHA1); + check_int(ret, ==, 1); + + ret = reftable_iterator_next_ref(&it, &ref); + check_int(ret, ==, 1); + } + + reftable_ref_record_release(&ref); + reftable_iterator_destroy(&it); + reftable_table_decref(table); + reftable_buf_release(&buf); + return 0; +} + +int cmd_main(int argc UNUSED, const char *argv[] UNUSED) +{ + TEST(t_table_seek_once(), "table can seek once"); + TEST(t_table_reseek(), "table can reseek multiple times"); + return test_done(); +} -- cgit v1.3 From 0f8ee94b636b5ab183c62b8fdd26c1611c2b86f4 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Mon, 7 Apr 2025 15:16:27 +0200 Subject: reftable/constants: make block types part of the public interface Now that reftable blocks can be read individually via the public interface it becomes necessary for callers to be able to distinguish the different types of blocks. Expose the relevant constants. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- reftable/block.c | 4 ++-- reftable/constants.h | 6 +----- reftable/iter.c | 10 +++++----- reftable/merged.c | 4 ++-- reftable/record.c | 40 ++++++++++++++++++++-------------------- reftable/reftable-constants.h | 18 ++++++++++++++++++ reftable/stack.c | 8 ++++---- reftable/table.c | 40 ++++++++++++++++++++-------------------- reftable/writer.c | 22 +++++++++++----------- t/unit-tests/t-reftable-block.c | 22 +++++++++++----------- t/unit-tests/t-reftable-merged.c | 12 ++++++------ t/unit-tests/t-reftable-pq.c | 10 +++++----- t/unit-tests/t-reftable-record.c | 40 ++++++++++++++++++++-------------------- t/unit-tests/t-reftable-table.c | 12 ++++++------ 14 files changed, 131 insertions(+), 117 deletions(-) create mode 100644 reftable/reftable-constants.h (limited to 'reftable/stack.c') diff --git a/reftable/block.c b/reftable/block.c index 08e22170d5..795815b476 100644 --- a/reftable/block.c +++ b/reftable/block.c @@ -160,7 +160,7 @@ int block_writer_finish(struct block_writer *w) * Log records are stored zlib-compressed. Note that the compression * also spans over the restart points we have just written. */ - if (block_writer_type(w) == BLOCK_TYPE_LOG) { + if (block_writer_type(w) == REFTABLE_BLOCK_TYPE_LOG) { int block_header_skip = 4 + w->header_off; uLongf src_len = w->next - block_header_skip, compressed_len; int ret; @@ -254,7 +254,7 @@ int reftable_block_init(struct reftable_block *block, goto done; } - if (block_type == BLOCK_TYPE_LOG) { + if (block_type == REFTABLE_BLOCK_TYPE_LOG) { uint32_t block_header_skip = 4 + header_size; uLong dst_len = block_size - block_header_skip; uLong src_len = block->block_data.len - block_header_skip; diff --git a/reftable/constants.h b/reftable/constants.h index 091728cf03..e3b1aaa516 100644 --- a/reftable/constants.h +++ b/reftable/constants.h @@ -9,11 +9,7 @@ #ifndef CONSTANTS_H #define CONSTANTS_H -#define BLOCK_TYPE_LOG 'g' -#define BLOCK_TYPE_INDEX 'i' -#define BLOCK_TYPE_REF 'r' -#define BLOCK_TYPE_OBJ 'o' -#define BLOCK_TYPE_ANY 0 +#include "reftable-constants.h" #define MAX_RESTARTS ((1 << 16) - 1) #define DEFAULT_BLOCK_SIZE 4096 diff --git a/reftable/iter.c b/reftable/iter.c index a79c90d9ba..2ecc52b336 100644 --- a/reftable/iter.c +++ b/reftable/iter.c @@ -131,7 +131,7 @@ static int indexed_table_ref_iter_next_block(struct indexed_table_ref_iter *it) block_source_release_data(&it->block.block_data); off = it->offsets[it->offset_idx++]; - err = table_init_block(it->table, &it->block, off, BLOCK_TYPE_REF); + err = table_init_block(it->table, &it->block, off, REFTABLE_BLOCK_TYPE_REF); if (err < 0) { return err; } @@ -246,7 +246,7 @@ int reftable_iterator_seek_ref(struct reftable_iterator *it, const char *name) { struct reftable_record want = { - .type = BLOCK_TYPE_REF, + .type = REFTABLE_BLOCK_TYPE_REF, .u.ref = { .refname = (char *)name, }, @@ -258,7 +258,7 @@ int reftable_iterator_next_ref(struct reftable_iterator *it, struct reftable_ref_record *ref) { struct reftable_record rec = { - .type = BLOCK_TYPE_REF, + .type = REFTABLE_BLOCK_TYPE_REF, .u = { .ref = *ref }, @@ -272,7 +272,7 @@ int reftable_iterator_seek_log_at(struct reftable_iterator *it, const char *name, uint64_t update_index) { struct reftable_record want = { - .type = BLOCK_TYPE_LOG, + .type = REFTABLE_BLOCK_TYPE_LOG, .u.log = { .refname = (char *)name, .update_index = update_index, @@ -291,7 +291,7 @@ int reftable_iterator_next_log(struct reftable_iterator *it, struct reftable_log_record *log) { struct reftable_record rec = { - .type = BLOCK_TYPE_LOG, + .type = REFTABLE_BLOCK_TYPE_LOG, .u = { .log = *log, }, diff --git a/reftable/merged.c b/reftable/merged.c index d5b974d660..733de07454 100644 --- a/reftable/merged.c +++ b/reftable/merged.c @@ -301,13 +301,13 @@ out: int reftable_merged_table_init_ref_iterator(struct reftable_merged_table *mt, struct reftable_iterator *it) { - return merged_table_init_iter(mt, it, BLOCK_TYPE_REF); + return merged_table_init_iter(mt, it, REFTABLE_BLOCK_TYPE_REF); } int reftable_merged_table_init_log_iterator(struct reftable_merged_table *mt, struct reftable_iterator *it) { - return merged_table_init_iter(mt, it, BLOCK_TYPE_LOG); + return merged_table_init_iter(mt, it, REFTABLE_BLOCK_TYPE_LOG); } enum reftable_hash reftable_merged_table_hash_id(struct reftable_merged_table *mt) diff --git a/reftable/record.c b/reftable/record.c index 26cd834d40..ed00a72441 100644 --- a/reftable/record.c +++ b/reftable/record.c @@ -69,10 +69,10 @@ int put_var_int(struct string_view *dest, uint64_t value) int reftable_is_block_type(uint8_t typ) { switch (typ) { - case BLOCK_TYPE_REF: - case BLOCK_TYPE_LOG: - case BLOCK_TYPE_OBJ: - case BLOCK_TYPE_INDEX: + case REFTABLE_BLOCK_TYPE_REF: + case REFTABLE_BLOCK_TYPE_LOG: + case REFTABLE_BLOCK_TYPE_OBJ: + case REFTABLE_BLOCK_TYPE_INDEX: return 1; } return 0; @@ -462,7 +462,7 @@ static int reftable_ref_record_cmp_void(const void *_a, const void *_b) static struct reftable_record_vtable reftable_ref_record_vtable = { .key = &reftable_ref_record_key, - .type = BLOCK_TYPE_REF, + .type = REFTABLE_BLOCK_TYPE_REF, .copy_from = &reftable_ref_record_copy_from, .val_type = &reftable_ref_record_val_type, .encode = &reftable_ref_record_encode, @@ -664,7 +664,7 @@ static int reftable_obj_record_cmp_void(const void *_a, const void *_b) static struct reftable_record_vtable reftable_obj_record_vtable = { .key = &reftable_obj_record_key, - .type = BLOCK_TYPE_OBJ, + .type = REFTABLE_BLOCK_TYPE_OBJ, .copy_from = &reftable_obj_record_copy_from, .val_type = &reftable_obj_record_val_type, .encode = &reftable_obj_record_encode, @@ -1035,7 +1035,7 @@ static int reftable_log_record_is_deletion_void(const void *p) static struct reftable_record_vtable reftable_log_record_vtable = { .key = &reftable_log_record_key, - .type = BLOCK_TYPE_LOG, + .type = REFTABLE_BLOCK_TYPE_LOG, .copy_from = &reftable_log_record_copy_from, .val_type = &reftable_log_record_val_type, .encode = &reftable_log_record_encode, @@ -1137,7 +1137,7 @@ static int reftable_index_record_cmp(const void *_a, const void *_b) static struct reftable_record_vtable reftable_index_record_vtable = { .key = &reftable_index_record_key, - .type = BLOCK_TYPE_INDEX, + .type = REFTABLE_BLOCK_TYPE_INDEX, .copy_from = &reftable_index_record_copy_from, .val_type = &reftable_index_record_val_type, .encode = &reftable_index_record_encode, @@ -1280,13 +1280,13 @@ int reftable_log_record_is_deletion(const struct reftable_log_record *log) static void *reftable_record_data(struct reftable_record *rec) { switch (rec->type) { - case BLOCK_TYPE_REF: + case REFTABLE_BLOCK_TYPE_REF: return &rec->u.ref; - case BLOCK_TYPE_LOG: + case REFTABLE_BLOCK_TYPE_LOG: return &rec->u.log; - case BLOCK_TYPE_INDEX: + case REFTABLE_BLOCK_TYPE_INDEX: return &rec->u.idx; - case BLOCK_TYPE_OBJ: + case REFTABLE_BLOCK_TYPE_OBJ: return &rec->u.obj; } abort(); @@ -1296,13 +1296,13 @@ static struct reftable_record_vtable * reftable_record_vtable(struct reftable_record *rec) { switch (rec->type) { - case BLOCK_TYPE_REF: + case REFTABLE_BLOCK_TYPE_REF: return &reftable_ref_record_vtable; - case BLOCK_TYPE_LOG: + case REFTABLE_BLOCK_TYPE_LOG: return &reftable_log_record_vtable; - case BLOCK_TYPE_INDEX: + case REFTABLE_BLOCK_TYPE_INDEX: return &reftable_index_record_vtable; - case BLOCK_TYPE_OBJ: + case REFTABLE_BLOCK_TYPE_OBJ: return &reftable_obj_record_vtable; } abort(); @@ -1314,11 +1314,11 @@ int reftable_record_init(struct reftable_record *rec, uint8_t typ) rec->type = typ; switch (typ) { - case BLOCK_TYPE_REF: - case BLOCK_TYPE_LOG: - case BLOCK_TYPE_OBJ: + case REFTABLE_BLOCK_TYPE_REF: + case REFTABLE_BLOCK_TYPE_LOG: + case REFTABLE_BLOCK_TYPE_OBJ: return 0; - case BLOCK_TYPE_INDEX: + case REFTABLE_BLOCK_TYPE_INDEX: reftable_buf_init(&rec->u.idx.last_key); return 0; default: diff --git a/reftable/reftable-constants.h b/reftable/reftable-constants.h new file mode 100644 index 0000000000..4ae9ba4bac --- /dev/null +++ b/reftable/reftable-constants.h @@ -0,0 +1,18 @@ +/* + * Copyright 2020 Google LLC + * + * Use of this source code is governed by a BSD-style + * license that can be found in the LICENSE file or at + * https://developers.google.com/open-source/licenses/bsd + */ + +#ifndef REFTABLE_CONSTANTS_H +#define REFTABLE_CONSTANTS_H + +#define REFTABLE_BLOCK_TYPE_LOG 'g' +#define REFTABLE_BLOCK_TYPE_INDEX 'i' +#define REFTABLE_BLOCK_TYPE_REF 'r' +#define REFTABLE_BLOCK_TYPE_OBJ 'o' +#define REFTABLE_BLOCK_TYPE_ANY 0 + +#endif /* REFTABLE_CONSTANTS_H */ diff --git a/reftable/stack.c b/reftable/stack.c index cc48e725b1..4caf96aa1d 100644 --- a/reftable/stack.c +++ b/reftable/stack.c @@ -203,14 +203,14 @@ int reftable_stack_init_ref_iterator(struct reftable_stack *st, struct reftable_iterator *it) { return merged_table_init_iter(reftable_stack_merged_table(st), - it, BLOCK_TYPE_REF); + it, REFTABLE_BLOCK_TYPE_REF); } int reftable_stack_init_log_iterator(struct reftable_stack *st, struct reftable_iterator *it) { return merged_table_init_iter(reftable_stack_merged_table(st), - it, BLOCK_TYPE_LOG); + it, REFTABLE_BLOCK_TYPE_LOG); } struct reftable_merged_table * @@ -1098,7 +1098,7 @@ static int stack_write_compact(struct reftable_stack *st, if (err < 0) goto done; - err = merged_table_init_iter(mt, &it, BLOCK_TYPE_REF); + err = merged_table_init_iter(mt, &it, REFTABLE_BLOCK_TYPE_REF); if (err < 0) goto done; @@ -1126,7 +1126,7 @@ static int stack_write_compact(struct reftable_stack *st, } reftable_iterator_destroy(&it); - err = merged_table_init_iter(mt, &it, BLOCK_TYPE_LOG); + err = merged_table_init_iter(mt, &it, REFTABLE_BLOCK_TYPE_LOG); if (err < 0) goto done; diff --git a/reftable/table.c b/reftable/table.c index a2a0c7a1d2..1af3942322 100644 --- a/reftable/table.c +++ b/reftable/table.c @@ -20,11 +20,11 @@ static struct reftable_table_offsets * table_offsets_for(struct reftable_table *t, uint8_t typ) { switch (typ) { - case BLOCK_TYPE_REF: + case REFTABLE_BLOCK_TYPE_REF: return &t->ref_offsets; - case BLOCK_TYPE_LOG: + case REFTABLE_BLOCK_TYPE_LOG: return &t->log_offsets; - case BLOCK_TYPE_OBJ: + case REFTABLE_BLOCK_TYPE_OBJ: return &t->obj_offsets; } abort(); @@ -112,9 +112,9 @@ static int parse_footer(struct reftable_table *t, uint8_t *footer, } first_block_typ = header[header_size(t->version)]; - t->ref_offsets.is_present = (first_block_typ == BLOCK_TYPE_REF); + t->ref_offsets.is_present = (first_block_typ == REFTABLE_BLOCK_TYPE_REF); t->ref_offsets.offset = 0; - t->log_offsets.is_present = (first_block_typ == BLOCK_TYPE_LOG || + t->log_offsets.is_present = (first_block_typ == REFTABLE_BLOCK_TYPE_LOG || t->log_offsets.offset > 0); t->obj_offsets.is_present = t->obj_offsets.offset > 0; if (t->obj_offsets.is_present && !t->object_id_len) { @@ -150,7 +150,7 @@ static int table_iter_next_in_block(struct table_iter *ti, struct reftable_record *rec) { int res = block_iter_next(&ti->bi, rec); - if (res == 0 && reftable_record_type(rec) == BLOCK_TYPE_REF) { + if (res == 0 && reftable_record_type(rec) == REFTABLE_BLOCK_TYPE_REF) { rec->u.ref.update_index += ti->table->min_update_index; } @@ -177,7 +177,7 @@ int table_init_block(struct reftable_table *t, struct reftable_block *block, if (err < 0) goto done; - if (want_typ != BLOCK_TYPE_ANY && block->block_type != want_typ) { + if (want_typ != REFTABLE_BLOCK_TYPE_ANY && block->block_type != want_typ) { err = 1; goto done; } @@ -270,7 +270,7 @@ static int table_iter_seek_start(struct table_iter *ti, uint8_t typ, int index) if (off == 0) { return 1; } - typ = BLOCK_TYPE_INDEX; + typ = REFTABLE_BLOCK_TYPE_INDEX; } return table_iter_seek_to(ti, off, typ); @@ -366,10 +366,10 @@ static int table_iter_seek_indexed(struct table_iter *ti, struct reftable_record *rec) { struct reftable_record want_index = { - .type = BLOCK_TYPE_INDEX, .u.idx = { .last_key = REFTABLE_BUF_INIT } + .type = REFTABLE_BLOCK_TYPE_INDEX, .u.idx = { .last_key = REFTABLE_BUF_INIT } }; struct reftable_record index_result = { - .type = BLOCK_TYPE_INDEX, + .type = REFTABLE_BLOCK_TYPE_INDEX, .u.idx = { .last_key = REFTABLE_BUF_INIT }, }; int err; @@ -429,7 +429,7 @@ static int table_iter_seek_indexed(struct table_iter *ti, break; } - if (ti->typ != BLOCK_TYPE_INDEX) { + if (ti->typ != REFTABLE_BLOCK_TYPE_INDEX) { err = REFTABLE_FORMAT_ERROR; goto done; } @@ -517,13 +517,13 @@ int table_init_iter(struct reftable_table *t, int reftable_table_init_ref_iterator(struct reftable_table *t, struct reftable_iterator *it) { - return table_init_iter(t, it, BLOCK_TYPE_REF); + return table_init_iter(t, it, REFTABLE_BLOCK_TYPE_REF); } int reftable_table_init_log_iterator(struct reftable_table *t, struct reftable_iterator *it) { - return table_init_iter(t, it, BLOCK_TYPE_LOG); + return table_init_iter(t, it, REFTABLE_BLOCK_TYPE_LOG); } int reftable_table_new(struct reftable_table **out, @@ -625,7 +625,7 @@ static int reftable_table_refs_for_indexed(struct reftable_table *t, uint8_t *oid) { struct reftable_record want = { - .type = BLOCK_TYPE_OBJ, + .type = REFTABLE_BLOCK_TYPE_OBJ, .u.obj = { .hash_prefix = oid, .hash_prefix_len = t->object_id_len, @@ -633,14 +633,14 @@ static int reftable_table_refs_for_indexed(struct reftable_table *t, }; struct reftable_iterator oit = { NULL }; struct reftable_record got = { - .type = BLOCK_TYPE_OBJ, + .type = REFTABLE_BLOCK_TYPE_OBJ, .u.obj = { 0 }, }; int err = 0; struct indexed_table_ref_iter *itr = NULL; /* Look through the reverse index. */ - err = table_init_iter(t, &oit, BLOCK_TYPE_OBJ); + err = table_init_iter(t, &oit, REFTABLE_BLOCK_TYPE_OBJ); if (err < 0) goto done; @@ -692,7 +692,7 @@ static int reftable_table_refs_for_unindexed(struct reftable_table *t, } table_iter_init(ti, t); - err = table_iter_seek_start(ti, BLOCK_TYPE_REF, 0); + err = table_iter_seek_start(ti, REFTABLE_BLOCK_TYPE_REF, 0); if (err < 0) goto out; @@ -748,15 +748,15 @@ int reftable_table_print_blocks(const char *tablename) } sections[] = { { .name = "ref", - .type = BLOCK_TYPE_REF, + .type = REFTABLE_BLOCK_TYPE_REF, }, { .name = "obj", - .type = BLOCK_TYPE_OBJ, + .type = REFTABLE_BLOCK_TYPE_OBJ, }, { .name = "log", - .type = BLOCK_TYPE_LOG, + .type = REFTABLE_BLOCK_TYPE_LOG, }, }; struct reftable_block_source src = { 0 }; diff --git a/reftable/writer.c b/reftable/writer.c index f0accfd0c3..0954c29bcf 100644 --- a/reftable/writer.c +++ b/reftable/writer.c @@ -172,7 +172,7 @@ int reftable_writer_new(struct reftable_writer **out, wp->write_arg = writer_arg; wp->opts = opts; wp->flush = flush_func; - writer_reinit_block_writer(wp, BLOCK_TYPE_REF); + writer_reinit_block_writer(wp, REFTABLE_BLOCK_TYPE_REF); *out = wp; @@ -347,7 +347,7 @@ int reftable_writer_add_ref(struct reftable_writer *w, struct reftable_ref_record *ref) { struct reftable_record rec = { - .type = BLOCK_TYPE_REF, + .type = REFTABLE_BLOCK_TYPE_REF, .u = { .ref = *ref }, @@ -411,13 +411,13 @@ static int reftable_writer_add_log_verbatim(struct reftable_writer *w, struct reftable_log_record *log) { struct reftable_record rec = { - .type = BLOCK_TYPE_LOG, + .type = REFTABLE_BLOCK_TYPE_LOG, .u = { .log = *log, }, }; if (w->block_writer && - block_writer_type(w->block_writer) == BLOCK_TYPE_REF) { + block_writer_type(w->block_writer) == REFTABLE_BLOCK_TYPE_REF) { int err = writer_finish_public_section(w); if (err < 0) return err; @@ -537,7 +537,7 @@ static int writer_finish_section(struct reftable_writer *w) max_level++; index_start = w->next; - err = writer_reinit_block_writer(w, BLOCK_TYPE_INDEX); + err = writer_reinit_block_writer(w, REFTABLE_BLOCK_TYPE_INDEX); if (err < 0) return err; @@ -549,7 +549,7 @@ static int writer_finish_section(struct reftable_writer *w) w->index_cap = 0; for (i = 0; i < idx_len; i++) { struct reftable_record rec = { - .type = BLOCK_TYPE_INDEX, + .type = REFTABLE_BLOCK_TYPE_INDEX, .u = { .idx = idx[i], }, @@ -614,7 +614,7 @@ static void write_object_record(void *void_arg, void *key) struct write_record_arg *arg = void_arg; struct obj_index_tree_node *entry = key; struct reftable_record - rec = { .type = BLOCK_TYPE_OBJ, + rec = { .type = REFTABLE_BLOCK_TYPE_OBJ, .u.obj = { .hash_prefix = (uint8_t *)entry->hash.buf, .hash_prefix_len = arg->w->stats.object_id_len, @@ -632,7 +632,7 @@ static void write_object_record(void *void_arg, void *key) if (arg->err < 0) goto done; - arg->err = writer_reinit_block_writer(arg->w, BLOCK_TYPE_OBJ); + arg->err = writer_reinit_block_writer(arg->w, REFTABLE_BLOCK_TYPE_OBJ); if (arg->err < 0) goto done; @@ -670,7 +670,7 @@ static int writer_dump_object_index(struct reftable_writer *w) infix_walk(w->obj_index_tree, &update_common, &common); w->stats.object_id_len = common.max + 1; - err = writer_reinit_block_writer(w, BLOCK_TYPE_OBJ); + err = writer_reinit_block_writer(w, REFTABLE_BLOCK_TYPE_OBJ); if (err < 0) return err; @@ -694,7 +694,7 @@ static int writer_finish_public_section(struct reftable_writer *w) err = writer_finish_section(w); if (err < 0) return err; - if (typ == BLOCK_TYPE_REF && !w->opts.skip_index_objects && + if (typ == REFTABLE_BLOCK_TYPE_REF && !w->opts.skip_index_objects && w->stats.ref_stats.index_blocks > 0) { err = writer_dump_object_index(w); if (err < 0) @@ -799,7 +799,7 @@ static int writer_flush_nonempty_block(struct reftable_writer *w) * By default, all records except for log records are padded to the * block size. */ - if (!w->opts.unpadded && typ != BLOCK_TYPE_LOG) + if (!w->opts.unpadded && typ != REFTABLE_BLOCK_TYPE_LOG) padding = w->opts.block_size - raw_bytes; bstats = writer_reftable_block_stats(w, typ); diff --git a/t/unit-tests/t-reftable-block.c b/t/unit-tests/t-reftable-block.c index 5577a5769e..7dbd93601c 100644 --- a/t/unit-tests/t-reftable-block.c +++ b/t/unit-tests/t-reftable-block.c @@ -24,7 +24,7 @@ static void t_ref_block_read_write(void) .last_key = REFTABLE_BUF_INIT, }; struct reftable_record rec = { - .type = BLOCK_TYPE_REF, + .type = REFTABLE_BLOCK_TYPE_REF, }; size_t i = 0; int ret; @@ -37,7 +37,7 @@ static void t_ref_block_read_write(void) check(block_data.buf != NULL); block_data.len = block_size; - ret = block_writer_init(&bw, BLOCK_TYPE_REF, (uint8_t *) block_data.buf, block_size, + ret = block_writer_init(&bw, REFTABLE_BLOCK_TYPE_REF, (uint8_t *) block_data.buf, block_size, header_off, hash_size(REFTABLE_HASH_SHA1)); check(!ret); @@ -118,7 +118,7 @@ static void t_log_block_read_write(void) .last_key = REFTABLE_BUF_INIT, }; struct reftable_record rec = { - .type = BLOCK_TYPE_LOG, + .type = REFTABLE_BLOCK_TYPE_LOG, }; size_t i = 0; int ret; @@ -131,7 +131,7 @@ static void t_log_block_read_write(void) check(block_data.buf != NULL); block_data.len = block_size; - ret = block_writer_init(&bw, BLOCK_TYPE_LOG, (uint8_t *) block_data.buf, block_size, + ret = block_writer_init(&bw, REFTABLE_BLOCK_TYPE_LOG, (uint8_t *) block_data.buf, block_size, header_off, hash_size(REFTABLE_HASH_SHA1)); check(!ret); @@ -208,7 +208,7 @@ static void t_obj_block_read_write(void) .last_key = REFTABLE_BUF_INIT, }; struct reftable_record rec = { - .type = BLOCK_TYPE_OBJ, + .type = REFTABLE_BLOCK_TYPE_OBJ, }; size_t i = 0; int ret; @@ -221,7 +221,7 @@ static void t_obj_block_read_write(void) check(block_data.buf != NULL); block_data.len = block_size; - ret = block_writer_init(&bw, BLOCK_TYPE_OBJ, (uint8_t *) block_data.buf, block_size, + ret = block_writer_init(&bw, REFTABLE_BLOCK_TYPE_OBJ, (uint8_t *) block_data.buf, block_size, header_off, hash_size(REFTABLE_HASH_SHA1)); check(!ret); @@ -291,7 +291,7 @@ static void t_index_block_read_write(void) .last_key = REFTABLE_BUF_INIT, }; struct reftable_record rec = { - .type = BLOCK_TYPE_INDEX, + .type = REFTABLE_BLOCK_TYPE_INDEX, .u.idx.last_key = REFTABLE_BUF_INIT, }; size_t i = 0; @@ -305,7 +305,7 @@ static void t_index_block_read_write(void) check(block_data.buf != NULL); block_data.len = block_size; - ret = block_writer_init(&bw, BLOCK_TYPE_INDEX, (uint8_t *) block_data.buf, block_size, + ret = block_writer_init(&bw, REFTABLE_BLOCK_TYPE_INDEX, (uint8_t *) block_data.buf, block_size, header_off, hash_size(REFTABLE_HASH_SHA1)); check(!ret); @@ -315,7 +315,7 @@ static void t_index_block_read_write(void) snprintf(buf, sizeof(buf), "branch%02"PRIuMAX, (uintmax_t)i); reftable_buf_init(&recs[i].u.idx.last_key); - recs[i].type = BLOCK_TYPE_INDEX; + recs[i].type = REFTABLE_BLOCK_TYPE_INDEX; check(!reftable_buf_addstr(&recs[i].u.idx.last_key, buf)); recs[i].u.idx.offset = i; @@ -389,13 +389,13 @@ static void t_block_iterator(void) REFTABLE_CALLOC_ARRAY(data.buf, data.len); check(data.buf != NULL); - err = block_writer_init(&writer, BLOCK_TYPE_REF, (uint8_t *) data.buf, data.len, + err = block_writer_init(&writer, REFTABLE_BLOCK_TYPE_REF, (uint8_t *) data.buf, data.len, 0, hash_size(REFTABLE_HASH_SHA1)); check(!err); for (size_t i = 0; i < ARRAY_SIZE(expected_refs); i++) { expected_refs[i] = (struct reftable_record) { - .type = BLOCK_TYPE_REF, + .type = REFTABLE_BLOCK_TYPE_REF, .u.ref = { .value_type = REFTABLE_REF_VAL1, .refname = xstrfmt("refs/heads/branch-%02"PRIuMAX, (uintmax_t)i), diff --git a/t/unit-tests/t-reftable-merged.c b/t/unit-tests/t-reftable-merged.c index fed6beb85c..18c3251a56 100644 --- a/t/unit-tests/t-reftable-merged.c +++ b/t/unit-tests/t-reftable-merged.c @@ -84,7 +84,7 @@ static void t_merged_single_record(void) struct reftable_iterator it = { 0 }; int err; - err = merged_table_init_iter(mt, &it, BLOCK_TYPE_REF); + err = merged_table_init_iter(mt, &it, REFTABLE_BLOCK_TYPE_REF); check(!err); err = reftable_iterator_seek_ref(&it, "a"); check(!err); @@ -164,7 +164,7 @@ static void t_merged_refs(void) size_t cap = 0; size_t i; - err = merged_table_init_iter(mt, &it, BLOCK_TYPE_REF); + err = merged_table_init_iter(mt, &it, REFTABLE_BLOCK_TYPE_REF); check(!err); err = reftable_iterator_seek_ref(&it, "a"); check(!err); @@ -244,7 +244,7 @@ static void t_merged_seek_multiple_times(void) struct reftable_merged_table *mt; mt = merged_table_from_records(refs, &sources, &tables, sizes, bufs, 2); - merged_table_init_iter(mt, &it, BLOCK_TYPE_REF); + merged_table_init_iter(mt, &it, REFTABLE_BLOCK_TYPE_REF); for (size_t i = 0; i < 5; i++) { int err = reftable_iterator_seek_ref(&it, "c"); @@ -320,7 +320,7 @@ static void t_merged_seek_multiple_times_without_draining(void) int err; mt = merged_table_from_records(refs, &sources, &tables, sizes, bufs, 2); - merged_table_init_iter(mt, &it, BLOCK_TYPE_REF); + merged_table_init_iter(mt, &it, REFTABLE_BLOCK_TYPE_REF); err = reftable_iterator_seek_ref(&it, "b"); check(!err); @@ -445,7 +445,7 @@ static void t_merged_logs(void) size_t cap = 0; size_t i; - err = merged_table_init_iter(mt, &it, BLOCK_TYPE_LOG); + err = merged_table_init_iter(mt, &it, REFTABLE_BLOCK_TYPE_LOG); check(!err); err = reftable_iterator_seek_log(&it, "a"); check(!err); @@ -469,7 +469,7 @@ static void t_merged_logs(void) check(reftable_log_record_equal(want[i], &out[i], REFTABLE_HASH_SIZE_SHA1)); - err = merged_table_init_iter(mt, &it, BLOCK_TYPE_LOG); + err = merged_table_init_iter(mt, &it, REFTABLE_BLOCK_TYPE_LOG); check(!err); err = reftable_iterator_seek_log_at(&it, "a", 2); check(!err); diff --git a/t/unit-tests/t-reftable-pq.c b/t/unit-tests/t-reftable-pq.c index c128fe8616..fb5a4eb187 100644 --- a/t/unit-tests/t-reftable-pq.c +++ b/t/unit-tests/t-reftable-pq.c @@ -34,7 +34,7 @@ static void t_pq_record(void) char *last = NULL; for (i = 0; i < N; i++) { - check(!reftable_record_init(&recs[i], BLOCK_TYPE_REF)); + check(!reftable_record_init(&recs[i], REFTABLE_BLOCK_TYPE_REF)); recs[i].u.ref.refname = xstrfmt("%02"PRIuMAX, (uintmax_t)i); } @@ -57,7 +57,7 @@ static void t_pq_record(void) merged_iter_pqueue_check(&pq); check(pq_entry_equal(&top, &e)); - check(reftable_record_type(e.rec) == BLOCK_TYPE_REF); + check(reftable_record_type(e.rec) == REFTABLE_BLOCK_TYPE_REF); if (last) check_int(strcmp(last, e.rec->u.ref.refname), <, 0); last = e.rec->u.ref.refname; @@ -76,7 +76,7 @@ static void t_pq_index(void) size_t N = ARRAY_SIZE(recs), i; for (i = 0; i < N; i++) { - check(!reftable_record_init(&recs[i], BLOCK_TYPE_REF)); + check(!reftable_record_init(&recs[i], REFTABLE_BLOCK_TYPE_REF)); recs[i].u.ref.refname = (char *) "refs/heads/master"; } @@ -100,7 +100,7 @@ static void t_pq_index(void) merged_iter_pqueue_check(&pq); check(pq_entry_equal(&top, &e)); - check(reftable_record_type(e.rec) == BLOCK_TYPE_REF); + check(reftable_record_type(e.rec) == REFTABLE_BLOCK_TYPE_REF); check_int(e.index, ==, i); if (last) check_str(last, e.rec->u.ref.refname); @@ -117,7 +117,7 @@ static void t_merged_iter_pqueue_top(void) size_t N = ARRAY_SIZE(recs), i; for (i = 0; i < N; i++) { - check(!reftable_record_init(&recs[i], BLOCK_TYPE_REF)); + check(!reftable_record_init(&recs[i], REFTABLE_BLOCK_TYPE_REF)); recs[i].u.ref.refname = (char *) "refs/heads/master"; } diff --git a/t/unit-tests/t-reftable-record.c b/t/unit-tests/t-reftable-record.c index 5954966373..553a007664 100644 --- a/t/unit-tests/t-reftable-record.c +++ b/t/unit-tests/t-reftable-record.c @@ -84,17 +84,17 @@ static void t_reftable_ref_record_comparison(void) { struct reftable_record in[3] = { { - .type = BLOCK_TYPE_REF, + .type = REFTABLE_BLOCK_TYPE_REF, .u.ref.refname = (char *) "refs/heads/master", .u.ref.value_type = REFTABLE_REF_VAL1, }, { - .type = BLOCK_TYPE_REF, + .type = REFTABLE_BLOCK_TYPE_REF, .u.ref.refname = (char *) "refs/heads/master", .u.ref.value_type = REFTABLE_REF_DELETION, }, { - .type = BLOCK_TYPE_REF, + .type = REFTABLE_BLOCK_TYPE_REF, .u.ref.refname = (char *) "HEAD", .u.ref.value_type = REFTABLE_REF_SYMREF, .u.ref.value.symref = (char *) "refs/heads/master", @@ -141,10 +141,10 @@ static void t_reftable_ref_record_roundtrip(void) for (int i = REFTABLE_REF_DELETION; i < REFTABLE_NR_REF_VALUETYPES; i++) { struct reftable_record in = { - .type = BLOCK_TYPE_REF, + .type = REFTABLE_BLOCK_TYPE_REF, .u.ref.value_type = i, }; - struct reftable_record out = { .type = BLOCK_TYPE_REF }; + struct reftable_record out = { .type = REFTABLE_BLOCK_TYPE_REF }; struct reftable_buf key = REFTABLE_BUF_INIT; uint8_t buffer[1024] = { 0 }; struct string_view dest = { @@ -198,17 +198,17 @@ static void t_reftable_log_record_comparison(void) { struct reftable_record in[3] = { { - .type = BLOCK_TYPE_LOG, + .type = REFTABLE_BLOCK_TYPE_LOG, .u.log.refname = (char *) "refs/heads/master", .u.log.update_index = 42, }, { - .type = BLOCK_TYPE_LOG, + .type = REFTABLE_BLOCK_TYPE_LOG, .u.log.refname = (char *) "refs/heads/master", .u.log.update_index = 22, }, { - .type = BLOCK_TYPE_LOG, + .type = REFTABLE_BLOCK_TYPE_LOG, .u.log.refname = (char *) "refs/heads/main", .u.log.update_index = 22, }, @@ -297,7 +297,7 @@ static void t_reftable_log_record_roundtrip(void) check(!reftable_log_record_is_deletion(&in[2])); for (size_t i = 0; i < ARRAY_SIZE(in); i++) { - struct reftable_record rec = { .type = BLOCK_TYPE_LOG }; + struct reftable_record rec = { .type = REFTABLE_BLOCK_TYPE_LOG }; struct reftable_buf key = REFTABLE_BUF_INIT; uint8_t buffer[1024] = { 0 }; struct string_view dest = { @@ -306,7 +306,7 @@ static void t_reftable_log_record_roundtrip(void) }; /* populate out, to check for leaks. */ struct reftable_record out = { - .type = BLOCK_TYPE_LOG, + .type = REFTABLE_BLOCK_TYPE_LOG, .u.log = { .refname = xstrdup("old name"), .value_type = REFTABLE_LOG_UPDATE, @@ -384,21 +384,21 @@ static void t_reftable_obj_record_comparison(void) uint64_t offsets[] = { 0, 16, 32, 48, 64, 80, 96, 112}; struct reftable_record in[3] = { { - .type = BLOCK_TYPE_OBJ, + .type = REFTABLE_BLOCK_TYPE_OBJ, .u.obj.hash_prefix = id_bytes, .u.obj.hash_prefix_len = 7, .u.obj.offsets = offsets, .u.obj.offset_len = 8, }, { - .type = BLOCK_TYPE_OBJ, + .type = REFTABLE_BLOCK_TYPE_OBJ, .u.obj.hash_prefix = id_bytes, .u.obj.hash_prefix_len = 7, .u.obj.offsets = offsets, .u.obj.offset_len = 5, }, { - .type = BLOCK_TYPE_OBJ, + .type = REFTABLE_BLOCK_TYPE_OBJ, .u.obj.hash_prefix = id_bytes, .u.obj.hash_prefix_len = 5, }, @@ -450,13 +450,13 @@ static void t_reftable_obj_record_roundtrip(void) .len = sizeof(buffer), }; struct reftable_record in = { - .type = BLOCK_TYPE_OBJ, + .type = REFTABLE_BLOCK_TYPE_OBJ, .u = { .obj = recs[i], }, }; struct reftable_buf key = REFTABLE_BUF_INIT; - struct reftable_record out = { .type = BLOCK_TYPE_OBJ }; + struct reftable_record out = { .type = REFTABLE_BLOCK_TYPE_OBJ }; int n, m; uint8_t extra; @@ -482,17 +482,17 @@ static void t_reftable_index_record_comparison(void) { struct reftable_record in[3] = { { - .type = BLOCK_TYPE_INDEX, + .type = REFTABLE_BLOCK_TYPE_INDEX, .u.idx.offset = 22, .u.idx.last_key = REFTABLE_BUF_INIT, }, { - .type = BLOCK_TYPE_INDEX, + .type = REFTABLE_BLOCK_TYPE_INDEX, .u.idx.offset = 32, .u.idx.last_key = REFTABLE_BUF_INIT, }, { - .type = BLOCK_TYPE_INDEX, + .type = REFTABLE_BLOCK_TYPE_INDEX, .u.idx.offset = 32, .u.idx.last_key = REFTABLE_BUF_INIT, }, @@ -523,7 +523,7 @@ static void t_reftable_index_record_comparison(void) static void t_reftable_index_record_roundtrip(void) { struct reftable_record in = { - .type = BLOCK_TYPE_INDEX, + .type = REFTABLE_BLOCK_TYPE_INDEX, .u.idx = { .offset = 42, .last_key = REFTABLE_BUF_INIT, @@ -537,7 +537,7 @@ static void t_reftable_index_record_roundtrip(void) struct reftable_buf scratch = REFTABLE_BUF_INIT; struct reftable_buf key = REFTABLE_BUF_INIT; struct reftable_record out = { - .type = BLOCK_TYPE_INDEX, + .type = REFTABLE_BLOCK_TYPE_INDEX, .u.idx = { .last_key = REFTABLE_BUF_INIT }, }; int n, m; diff --git a/t/unit-tests/t-reftable-table.c b/t/unit-tests/t-reftable-table.c index ba39cdf9a7..7e1eb533d0 100644 --- a/t/unit-tests/t-reftable-table.c +++ b/t/unit-tests/t-reftable-table.c @@ -106,33 +106,33 @@ static int t_table_block_iterator(void) uint16_t record_count; } expected_blocks[] = { { - .block_type = BLOCK_TYPE_REF, + .block_type = REFTABLE_BLOCK_TYPE_REF, .header_off = 24, .restart_count = 10, .record_count = 158, }, { - .block_type = BLOCK_TYPE_REF, + .block_type = REFTABLE_BLOCK_TYPE_REF, .restart_count = 10, .record_count = 159, }, { - .block_type = BLOCK_TYPE_REF, + .block_type = REFTABLE_BLOCK_TYPE_REF, .restart_count = 10, .record_count = 159, }, { - .block_type = BLOCK_TYPE_REF, + .block_type = REFTABLE_BLOCK_TYPE_REF, .restart_count = 2, .record_count = 24, }, { - .block_type = BLOCK_TYPE_INDEX, + .block_type = REFTABLE_BLOCK_TYPE_INDEX, .restart_count = 1, .record_count = 4, }, { - .block_type = BLOCK_TYPE_OBJ, + .block_type = REFTABLE_BLOCK_TYPE_OBJ, .restart_count = 1, .record_count = 1, }, -- cgit v1.3