averello/git-mirror
1
0
Fork 0
mirror of https://github.com/git/git.git synced 2025-12-12 20:36:24 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'ps/object-source-loose'

Browse Source 
A part of code paths that deals with loose objects has been cleaned
up.

* ps/object-source-loose:
  object-file: refactor writing objects via a stream
  object-file: rename `write_object_file()`
  object-file: refactor freshening of objects
  object-file: rename `has_loose_object()`
  object-file: read objects via the loose object source
  object-file: move loose object map into loose source
  object-file: hide internals when we need to reprepare loose sources
  object-file: move loose object cache into loose source
  object-file: introduce `struct odb_source_loose`
  object-file: move `fetch_if_missing`
  odb: adjust naming to free object sources
  odb: introduce `odb_source_new()`
  odb: fix subtle logic to check whether an alternate is usable
This commit is contained in:
Junio C Hamano
2025-11-24 15:46:41 -08:00
parent 05ce3ab2c6 3e5e360888
commit a545103244
12 changed files with 287 additions and 207 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
builtin/pack-objects.c
View File

@@ -1715,7 +1715,7 @@ static int want_object_in_pack_mtime(const struct object_id *oid,
*/ */
struct odb_source *source = the_repository->objects->sources->next; struct odb_source *source = the_repository->objects->sources->next;
for (; source; source = source->next) for (; source; source = source->next)
if (has_loose_object(source, oid)) if (odb_source_loose_has_object(source, oid))
return 0; return 0;
} }
@@ -3976,7 +3976,7 @@ static void add_cruft_object_entry(const struct object_id *oid, enum object_type
int found = 0; int found = 0;
for (; !found && source; source = source->next) for (; !found && source; source = source->next)
if (has_loose_object(source, oid)) if (odb_source_loose_has_object(source, oid))
found = 1; found = 1;
/* /*

7
builtin/unpack-objects.c
View File

@@ -363,7 +363,7 @@ struct input_zstream_data {
int status; int status;
}; };
static const void *feed_input_zstream(struct input_stream *in_stream, static const void *feed_input_zstream(struct odb_write_stream *in_stream,
unsigned long *readlen) unsigned long *readlen)
{ {
struct input_zstream_data *data = in_stream->data; struct input_zstream_data *data = in_stream->data;
@@ -393,7 +393,7 @@ static void stream_blob(unsigned long size, unsigned nr)
{ {
git_zstream zstream = { 0 }; git_zstream zstream = { 0 };
struct input_zstream_data data = { 0 }; struct input_zstream_data data = { 0 };
struct input_stream in_stream = { struct odb_write_stream in_stream = {
.read = feed_input_zstream, .read = feed_input_zstream,
.data = &data, .data = &data,
}; };
@@ -402,8 +402,7 @@ static void stream_blob(unsigned long size, unsigned nr)
data.zstream = &zstream; data.zstream = &zstream;
git_inflate_init(&zstream); git_inflate_init(&zstream);
if (stream_loose_object(the_repository->objects->sources, if (odb_write_object_stream(the_repository->objects, &in_stream, size, &info->oid))
&in_stream, size, &info->oid))
die(_("failed to write object in stream")); die(_("failed to write object in stream"));
if (data.status != Z_STREAM_END) if (data.status != Z_STREAM_END)

19
loose.c
View File

@@ -1,6 +1,7 @@
#include "git-compat-util.h" #include "git-compat-util.h"
#include "hash.h" #include "hash.h"
#include "path.h" #include "path.h"
#include "object-file.h"
#include "odb.h" #include "odb.h"
#include "hex.h" #include "hex.h"
#include "repository.h" #include "repository.h"
@@ -48,13 +49,13 @@ static int insert_loose_map(struct odb_source *source,
const struct object_id *oid, const struct object_id *oid,
const struct object_id *compat_oid) const struct object_id *compat_oid)
{ {
struct loose_object_map *map = source->loose_map; struct loose_object_map *map = source->loose->map;
int inserted = 0; int inserted = 0;
inserted |= insert_oid_pair(map->to_compat, oid, compat_oid); inserted |= insert_oid_pair(map->to_compat, oid, compat_oid);
inserted |= insert_oid_pair(map->to_storage, compat_oid, oid); inserted |= insert_oid_pair(map->to_storage, compat_oid, oid);
if (inserted) if (inserted)
oidtree_insert(source->loose_objects_cache, compat_oid); oidtree_insert(source->loose->cache, compat_oid);
return inserted; return inserted;
} }
@@ -64,11 +65,11 @@ static int load_one_loose_object_map(struct repository *repo, struct odb_source
struct strbuf buf = STRBUF_INIT, path = STRBUF_INIT; struct strbuf buf = STRBUF_INIT, path = STRBUF_INIT;
FILE *fp; FILE *fp;
if (!source->loose_map) if (!source->loose->map)
loose_object_map_init(&source->loose_map); loose_object_map_init(&source->loose->map);
if (!source->loose_objects_cache) { if (!source->loose->cache) {
ALLOC_ARRAY(source->loose_objects_cache, 1); ALLOC_ARRAY(source->loose->cache, 1);
oidtree_init(source->loose_objects_cache); oidtree_init(source->loose->cache);
} }
insert_loose_map(source, repo->hash_algo->empty_tree, repo->compat_hash_algo->empty_tree); insert_loose_map(source, repo->hash_algo->empty_tree, repo->compat_hash_algo->empty_tree);
@@ -124,7 +125,7 @@ int repo_read_loose_object_map(struct repository *repo)
int repo_write_loose_object_map(struct repository *repo) int repo_write_loose_object_map(struct repository *repo)
{ {
kh_oid_map_t *map = repo->objects->sources->loose_map->to_compat; kh_oid_map_t *map = repo->objects->sources->loose->map->to_compat;
struct lock_file lock; struct lock_file lock;
int fd; int fd;
khiter_t iter; khiter_t iter;
@@ -230,7 +231,7 @@ int repo_loose_object_map_oid(struct repository *repo,
khiter_t pos; khiter_t pos;
for (source = repo->objects->sources; source; source = source->next) { for (source = repo->objects->sources; source; source = source->next) {
struct loose_object_map *loose_map = source->loose_map; struct loose_object_map *loose_map = source->loose->map;
if (!loose_map) if (!loose_map)
continue; continue;
map = (to == repo->compat_hash_algo) ? map = (to == repo->compat_hash_algo) ?

175
object-file.c
View File

@@ -99,8 +99,8 @@ static int check_and_freshen_source(struct odb_source *source,
return check_and_freshen_file(path.buf, freshen); return check_and_freshen_file(path.buf, freshen);
} }
int has_loose_object(struct odb_source *source, int odb_source_loose_has_object(struct odb_source *source,
const struct object_id *oid) const struct object_id *oid)
{ {
return check_and_freshen_source(source, oid, 0); return check_and_freshen_source(source, oid, 0);
} }
@@ -167,25 +167,22 @@ int stream_object_signature(struct repository *r, const struct object_id *oid)
} }
/* /*
* Find "oid" as a loose object in the local repository or in an alternate. * Find "oid" as a loose object in given source.
* Returns 0 on success, negative on failure. * Returns 0 on success, negative on failure.
* *
* The "path" out-parameter will give the path of the object we found (if any). * The "path" out-parameter will give the path of the object we found (if any).
* Note that it may point to static storage and is only valid until another * Note that it may point to static storage and is only valid until another
* call to stat_loose_object(). * call to stat_loose_object().
*/ */
static int stat_loose_object(struct repository *r, const struct object_id *oid, static int stat_loose_object(struct odb_source_loose *loose,
const struct object_id *oid,
struct stat *st, const char **path) struct stat *st, const char **path)
{ {
struct odb_source *source;
static struct strbuf buf = STRBUF_INIT; static struct strbuf buf = STRBUF_INIT;
odb_prepare_alternates(r->objects); *path = odb_loose_path(loose->source, &buf, oid);
for (source = r->objects->sources; source; source = source->next) { if (!lstat(*path, st))
*path = odb_loose_path(source, &buf, oid); return 0;
if (!lstat(*path, st))
return 0;
}
return -1; return -1;
} }
@@ -194,39 +191,24 @@ static int stat_loose_object(struct repository *r, const struct object_id *oid,
* Like stat_loose_object(), but actually open the object and return the * Like stat_loose_object(), but actually open the object and return the
* descriptor. See the caveats on the "path" parameter above. * descriptor. See the caveats on the "path" parameter above.
*/ */
static int open_loose_object(struct repository *r, static int open_loose_object(struct odb_source_loose *loose,
const struct object_id *oid, const char **path) const struct object_id *oid, const char **path)
{ {
int fd;
struct odb_source *source;
int most_interesting_errno = ENOENT;
static struct strbuf buf = STRBUF_INIT; static struct strbuf buf = STRBUF_INIT;
int fd;
odb_prepare_alternates(r->objects); *path = odb_loose_path(loose->source, &buf, oid);
for (source = r->objects->sources; source; source = source->next) { fd = git_open(*path);
*path = odb_loose_path(source, &buf, oid); if (fd >= 0)
fd = git_open(*path); return fd;
if (fd >= 0)
return fd;
if (most_interesting_errno == ENOENT)
most_interesting_errno = errno;
}
errno = most_interesting_errno;
return -1; return -1;
} }
static int quick_has_loose(struct repository *r, static int quick_has_loose(struct odb_source_loose *loose,
const struct object_id *oid) const struct object_id *oid)
{ {
struct odb_source *source; return !!oidtree_contains(odb_source_loose_cache(loose->source, oid), oid);
odb_prepare_alternates(r->objects);
for (source = r->objects->sources; source; source = source->next) {
if (oidtree_contains(odb_loose_cache(source, oid), oid))
return 1;
}
return 0;
} }
/* /*
@@ -252,12 +234,12 @@ static void *map_fd(int fd, const char *path, unsigned long *size)
return map; return map;
} }
void *map_loose_object(struct repository *r, void *odb_source_loose_map_object(struct odb_source *source,
const struct object_id *oid, const struct object_id *oid,
unsigned long *size) unsigned long *size)
{ {
const char *p; const char *p;
int fd = open_loose_object(r, oid, &p); int fd = open_loose_object(source->loose, oid, &p);
if (fd < 0) if (fd < 0)
return NULL; return NULL;
@@ -407,9 +389,9 @@ int parse_loose_header(const char *hdr, struct object_info *oi)
return 0; return 0;
} }
int loose_object_info(struct repository *r, int odb_source_loose_read_object_info(struct odb_source *source,
const struct object_id *oid, const struct object_id *oid,
struct object_info *oi, int flags) struct object_info *oi, int flags)
{ {
int status = 0; int status = 0;
int fd; int fd;
@@ -422,7 +404,7 @@ int loose_object_info(struct repository *r,
enum object_type type_scratch; enum object_type type_scratch;
if (oi->delta_base_oid) if (oi->delta_base_oid)
oidclr(oi->delta_base_oid, r->hash_algo); oidclr(oi->delta_base_oid, source->odb->repo->hash_algo);
/* /*
* If we don't care about type or size, then we don't * If we don't care about type or size, then we don't
@@ -435,15 +417,15 @@ int loose_object_info(struct repository *r,
if (!oi->typep && !oi->sizep && !oi->contentp) { if (!oi->typep && !oi->sizep && !oi->contentp) {
struct stat st; struct stat st;
if (!oi->disk_sizep && (flags & OBJECT_INFO_QUICK)) if (!oi->disk_sizep && (flags & OBJECT_INFO_QUICK))
return quick_has_loose(r, oid) ? 0 : -1; return quick_has_loose(source->loose, oid) ? 0 : -1;
if (stat_loose_object(r, oid, &st, &path) < 0) if (stat_loose_object(source->loose, oid, &st, &path) < 0)
return -1; return -1;
if (oi->disk_sizep) if (oi->disk_sizep)
*oi->disk_sizep = st.st_size; *oi->disk_sizep = st.st_size;
return 0; return 0;
} }
fd = open_loose_object(r, oid, &path); fd = open_loose_object(source->loose, oid, &path);
if (fd < 0) { if (fd < 0) {
if (errno != ENOENT) if (errno != ENOENT)
error_errno(_("unable to open loose object %s"), oid_to_hex(oid)); error_errno(_("unable to open loose object %s"), oid_to_hex(oid));
@@ -986,35 +968,15 @@ static int write_loose_object(struct odb_source *source,
FOF_SKIP_COLLISION_CHECK); FOF_SKIP_COLLISION_CHECK);
} }
static int freshen_loose_object(struct object_database *odb, int odb_source_loose_freshen_object(struct odb_source *source,
const struct object_id *oid) const struct object_id *oid)
{ {
odb_prepare_alternates(odb); return !!check_and_freshen_source(source, oid, 1);
for (struct odb_source *source = odb->sources; source; source = source->next)
if (check_and_freshen_source(source, oid, 1))
return 1;
return 0;
} }
static int freshen_packed_object(struct object_database *odb, int odb_source_loose_write_stream(struct odb_source *source,
const struct object_id *oid) struct odb_write_stream *in_stream, size_t len,
{ struct object_id *oid)
struct pack_entry e;
if (!find_pack_entry(odb->repo, oid, &e))
return 0;
if (e.p->is_cruft)
return 0;
if (e.p->freshened)
return 1;
if (!freshen_file(e.p->pack_name))
return 0;
e.p->freshened = 1;
return 1;
}
int stream_loose_object(struct odb_source *source,
struct input_stream *in_stream, size_t len,
struct object_id *oid)
{ {
const struct git_hash_algo *compat = source->odb->repo->compat_hash_algo; const struct git_hash_algo *compat = source->odb->repo->compat_hash_algo;
struct object_id compat_oid; struct object_id compat_oid;
@@ -1091,12 +1053,10 @@ int stream_loose_object(struct odb_source *source,
die(_("deflateEnd on stream object failed (%d)"), ret); die(_("deflateEnd on stream object failed (%d)"), ret);
close_loose_object(source, fd, tmp_file.buf); close_loose_object(source, fd, tmp_file.buf);
if (freshen_packed_object(source->odb, oid) || if (odb_freshen_object(source->odb, oid)) {
freshen_loose_object(source->odb, oid)) {
unlink_or_warn(tmp_file.buf); unlink_or_warn(tmp_file.buf);
goto cleanup; goto cleanup;
} }
odb_loose_path(source, &filename, oid); odb_loose_path(source, &filename, oid);
/* We finally know the object path, and create the missing dir. */ /* We finally know the object path, and create the missing dir. */
@@ -1124,10 +1084,10 @@ cleanup:
return err; return err;
} }
int write_object_file(struct odb_source *source, int odb_source_loose_write_object(struct odb_source *source,
const void *buf, unsigned long len, const void *buf, unsigned long len,
enum object_type type, struct object_id *oid, enum object_type type, struct object_id *oid,
struct object_id *compat_oid_in, unsigned flags) struct object_id *compat_oid_in, unsigned flags)
{ {
const struct git_hash_algo *algo = source->odb->repo->hash_algo; const struct git_hash_algo *algo = source->odb->repo->hash_algo;
const struct git_hash_algo *compat = source->odb->repo->compat_hash_algo; const struct git_hash_algo *compat = source->odb->repo->compat_hash_algo;
@@ -1155,8 +1115,7 @@ int write_object_file(struct odb_source *source,
* it out into .git/objects/??/?{38} file. * it out into .git/objects/??/?{38} file.
*/ */
write_object_file_prepare(algo, buf, len, type, oid, hdr, &hdrlen); write_object_file_prepare(algo, buf, len, type, oid, hdr, &hdrlen);
if (freshen_packed_object(source->odb, oid) || if (odb_freshen_object(source->odb, oid))
freshen_loose_object(source->odb, oid))
return 0; return 0;
if (write_loose_object(source, oid, hdr, hdrlen, buf, len, 0, flags)) if (write_loose_object(source, oid, hdr, hdrlen, buf, len, 0, flags))
return -1; return -1;
@@ -1179,7 +1138,7 @@ int force_object_loose(struct odb_source *source,
int ret; int ret;
for (struct odb_source *s = source->odb->sources; s; s = s->next) for (struct odb_source *s = source->odb->sources; s; s = s->next)
if (has_loose_object(s, oid)) if (odb_source_loose_has_object(s, oid))
return 0; return 0;
oi.typep = &type; oi.typep = &type;
@@ -1806,44 +1765,49 @@ static int append_loose_object(const struct object_id *oid,
return 0; return 0;
} }
struct oidtree *odb_loose_cache(struct odb_source *source, struct oidtree *odb_source_loose_cache(struct odb_source *source,
const struct object_id *oid) const struct object_id *oid)
{ {
int subdir_nr = oid->hash[0]; int subdir_nr = oid->hash[0];
struct strbuf buf = STRBUF_INIT; struct strbuf buf = STRBUF_INIT;
size_t word_bits = bitsizeof(source->loose_objects_subdir_seen[0]); size_t word_bits = bitsizeof(source->loose->subdir_seen[0]);
size_t word_index = subdir_nr / word_bits; size_t word_index = subdir_nr / word_bits;
size_t mask = (size_t)1u << (subdir_nr % word_bits); size_t mask = (size_t)1u << (subdir_nr % word_bits);
uint32_t *bitmap; uint32_t *bitmap;
if (subdir_nr < 0 || if (subdir_nr < 0 ||
(size_t) subdir_nr >= bitsizeof(source->loose_objects_subdir_seen)) (size_t) subdir_nr >= bitsizeof(source->loose->subdir_seen))
BUG("subdir_nr out of range"); BUG("subdir_nr out of range");
bitmap = &source->loose_objects_subdir_seen[word_index]; bitmap = &source->loose->subdir_seen[word_index];
if (*bitmap & mask) if (*bitmap & mask)
return source->loose_objects_cache; return source->loose->cache;
if (!source->loose_objects_cache) { if (!source->loose->cache) {
ALLOC_ARRAY(source->loose_objects_cache, 1); ALLOC_ARRAY(source->loose->cache, 1);
oidtree_init(source->loose_objects_cache); oidtree_init(source->loose->cache);
} }
strbuf_addstr(&buf, source->path); strbuf_addstr(&buf, source->path);
for_each_file_in_obj_subdir(subdir_nr, &buf, for_each_file_in_obj_subdir(subdir_nr, &buf,
source->odb->repo->hash_algo, source->odb->repo->hash_algo,
append_loose_object, append_loose_object,
NULL, NULL, NULL, NULL,
source->loose_objects_cache); source->loose->cache);
*bitmap |= mask; *bitmap |= mask;
strbuf_release(&buf); strbuf_release(&buf);
return source->loose_objects_cache; return source->loose->cache;
} }
void odb_clear_loose_cache(struct odb_source *source) static void odb_source_loose_clear_cache(struct odb_source_loose *loose)
{ {
oidtree_clear(source->loose_objects_cache); oidtree_clear(loose->cache);
FREE_AND_NULL(source->loose_objects_cache); FREE_AND_NULL(loose->cache);
memset(&source->loose_objects_subdir_seen, 0, memset(&loose->subdir_seen, 0,
sizeof(source->loose_objects_subdir_seen)); sizeof(loose->subdir_seen));
}
void odb_source_loose_reprepare(struct odb_source *source)
{
odb_source_loose_clear_cache(source->loose);
} }
static int check_stream_oid(git_zstream *stream, static int check_stream_oid(git_zstream *stream,
@@ -1999,3 +1963,20 @@ void object_file_transaction_commit(struct odb_transaction *transaction)
transaction->odb->transaction = NULL; transaction->odb->transaction = NULL;
free(transaction); free(transaction);
} }
struct odb_source_loose *odb_source_loose_new(struct odb_source *source)
{
struct odb_source_loose *loose;
CALLOC_ARRAY(loose, 1);
loose->source = source;
return loose;
}
void odb_source_loose_free(struct odb_source_loose *loose)
{
if (!loose)
return;
odb_source_loose_clear_cache(loose);
loose_object_map_clear(&loose->map);
free(loose);
}

98
object-file.h
View File

@@ -7,14 +7,6 @@
struct index_state; struct index_state;
/*
* Set this to 0 to prevent odb_read_object_info_extended() from fetching missing
* blobs. This has a difference only if extensions.partialClone is set.
*
* Its default value is 1.
*/
extern int fetch_if_missing;
enum { enum {
INDEX_WRITE_OBJECT = (1 << 0), INDEX_WRITE_OBJECT = (1 << 0),
INDEX_FORMAT_CHECK = (1 << 1), INDEX_FORMAT_CHECK = (1 << 1),
@@ -26,15 +18,65 @@ int index_path(struct index_state *istate, struct object_id *oid, const char *pa
struct odb_source; struct odb_source;
struct odb_source_loose {
struct odb_source *source;
/*
* Used to store the results of readdir(3) calls when we are OK
* sacrificing accuracy due to races for speed. That includes
* object existence with OBJECT_INFO_QUICK, as well as
* our search for unique abbreviated hashes. Don't use it for tasks
* requiring greater accuracy!
*
* Be sure to call odb_load_loose_cache() before using.
*/
uint32_t subdir_seen[8]; /* 256 bits */
struct oidtree *cache;
/* Map between object IDs for loose objects. */
struct loose_object_map *map;
};
struct odb_source_loose *odb_source_loose_new(struct odb_source *source);
void odb_source_loose_free(struct odb_source_loose *loose);
/* Reprepare the loose source by emptying the loose object cache. */
void odb_source_loose_reprepare(struct odb_source *source);
int odb_source_loose_read_object_info(struct odb_source *source,
const struct object_id *oid,
struct object_info *oi, int flags);
void *odb_source_loose_map_object(struct odb_source *source,
const struct object_id *oid,
unsigned long *size);
/*
* Return true iff an object database source has a loose object
* with the specified name. This function does not respect replace
* references.
*/
int odb_source_loose_has_object(struct odb_source *source,
const struct object_id *oid);
int odb_source_loose_freshen_object(struct odb_source *source,
const struct object_id *oid);
int odb_source_loose_write_object(struct odb_source *source,
const void *buf, unsigned long len,
enum object_type type, struct object_id *oid,
struct object_id *compat_oid_in, unsigned flags);
int odb_source_loose_write_stream(struct odb_source *source,
struct odb_write_stream *stream, size_t len,
struct object_id *oid);
/* /*
* Populate and return the loose object cache array corresponding to the * Populate and return the loose object cache array corresponding to the
* given object ID. * given object ID.
*/ */
struct oidtree *odb_loose_cache(struct odb_source *source, struct oidtree *odb_source_loose_cache(struct odb_source *source,
const struct object_id *oid); const struct object_id *oid);
/* Empty the loose object cache for the specified object directory. */
void odb_clear_loose_cache(struct odb_source *source);
/* /*
* Put in `buf` the name of the file in the local object database that * Put in `buf` the name of the file in the local object database that
@@ -44,17 +86,6 @@ const char *odb_loose_path(struct odb_source *source,
struct strbuf *buf, struct strbuf *buf,
const struct object_id *oid); const struct object_id *oid);
/*
* Return true iff an object database source has a loose object
* with the specified name. This function does not respect replace
* references.
*/
int has_loose_object(struct odb_source *source,
const struct object_id *oid);
void *map_loose_object(struct repository *r, const struct object_id *oid,
unsigned long *size);
/* /*
* Iterate over the files in the loose-object parts of the object * Iterate over the files in the loose-object parts of the object
* directory "path", triggering the following callbacks: * directory "path", triggering the following callbacks:
@@ -146,21 +177,6 @@ enum unpack_loose_header_result unpack_loose_header(git_zstream *stream,
struct object_info; struct object_info;
int parse_loose_header(const char *hdr, struct object_info *oi); int parse_loose_header(const char *hdr, struct object_info *oi);
int write_object_file(struct odb_source *source,
const void *buf, unsigned long len,
enum object_type type, struct object_id *oid,
struct object_id *compat_oid_in, unsigned flags);
struct input_stream {
const void *(*read)(struct input_stream *, unsigned long *len);
void *data;
int is_finished;
};
int stream_loose_object(struct odb_source *source,
struct input_stream *in_stream, size_t len,
struct object_id *oid);
int force_object_loose(struct odb_source *source, int force_object_loose(struct odb_source *source,
const struct object_id *oid, time_t mtime); const struct object_id *oid, time_t mtime);
@@ -182,10 +198,6 @@ int check_object_signature(struct repository *r, const struct object_id *oid,
*/ */
int stream_object_signature(struct repository *r, const struct object_id *oid); int stream_object_signature(struct repository *r, const struct object_id *oid);
int loose_object_info(struct repository *r,
const struct object_id *oid,
struct object_info *oi, int flags);
enum finalize_object_file_flags { enum finalize_object_file_flags {
FOF_SKIP_COLLISION_CHECK = 1, FOF_SKIP_COLLISION_CHECK = 1,
}; };

2
object-name.c
View File

@@ -116,7 +116,7 @@ static void find_short_object_filename(struct disambiguate_state *ds)
struct odb_source *source; struct odb_source *source;
for (source = ds->repo->objects->sources; source && !ds->ambiguous; source = source->next) for (source = ds->repo->objects->sources; source && !ds->ambiguous; source = source->next)
oidtree_each(odb_loose_cache(source, &ds->bin_pfx), oidtree_each(odb_source_loose_cache(source, &ds->bin_pfx),
&ds->bin_pfx, ds->len, match_prefix, ds); &ds->bin_pfx, ds->len, match_prefix, ds);
} }

104
odb.c
View File

@@ -86,17 +86,16 @@ int odb_mkstemp(struct object_database *odb,
/* /*
* Return non-zero iff the path is usable as an alternate object database. * Return non-zero iff the path is usable as an alternate object database.
*/ */
static int alt_odb_usable(struct object_database *o, static int alt_odb_usable(struct object_database *o, const char *path,
struct strbuf *path, const char *normalized_objdir)
const char *normalized_objdir, khiter_t *pos)
{ {
int r; int r;
/* Detect cases where alternate disappeared */ /* Detect cases where alternate disappeared */
if (!is_directory(path->buf)) { if (!is_directory(path)) {
error(_("object directory %s does not exist; " error(_("object directory %s does not exist; "
"check .git/objects/info/alternates"), "check .git/objects/info/alternates"),
path->buf); path);
return 0; return 0;
} }
@@ -113,11 +112,14 @@ static int alt_odb_usable(struct object_database *o,
assert(r == 1); /* never used */ assert(r == 1); /* never used */
kh_value(o->source_by_path, p) = o->sources; kh_value(o->source_by_path, p) = o->sources;
} }
if (fspatheq(path->buf, normalized_objdir))
if (fspatheq(path, normalized_objdir))
return 0; return 0;
*pos = kh_put_odb_path_map(o->source_by_path, path->buf, &r);
/* r: 0 = exists, 1 = never used, 2 = deleted */ if (kh_get_odb_path_map(o->source_by_path, path) < kh_end(o->source_by_path))
return r == 0 ? 0 : 1; return 0;
return 1;
} }
/* /*
@@ -139,6 +141,21 @@ static void read_info_alternates(struct object_database *odb,
const char *relative_base, const char *relative_base,
int depth); int depth);
struct odb_source *odb_source_new(struct object_database *odb,
const char *path,
bool local)
{
struct odb_source *source;
CALLOC_ARRAY(source, 1);
source->odb = odb;
source->local = local;
source->path = xstrdup(path);
source->loose = odb_source_loose_new(source);
return source;
}
static struct odb_source *link_alt_odb_entry(struct object_database *odb, static struct odb_source *link_alt_odb_entry(struct object_database *odb,
const char *dir, const char *dir,
const char *relative_base, const char *relative_base,
@@ -148,6 +165,7 @@ static struct odb_source *link_alt_odb_entry(struct object_database *odb,
struct strbuf pathbuf = STRBUF_INIT; struct strbuf pathbuf = STRBUF_INIT;
struct strbuf tmp = STRBUF_INIT; struct strbuf tmp = STRBUF_INIT;
khiter_t pos; khiter_t pos;
int ret;
if (!is_absolute_path(dir) && relative_base) { if (!is_absolute_path(dir) && relative_base) {
strbuf_realpath(&pathbuf, relative_base, 1); strbuf_realpath(&pathbuf, relative_base, 1);
@@ -172,20 +190,18 @@ static struct odb_source *link_alt_odb_entry(struct object_database *odb,
strbuf_reset(&tmp); strbuf_reset(&tmp);
strbuf_realpath(&tmp, odb->sources->path, 1); strbuf_realpath(&tmp, odb->sources->path, 1);
if (!alt_odb_usable(odb, &pathbuf, tmp.buf, &pos)) if (!alt_odb_usable(odb, pathbuf.buf, tmp.buf))
goto error; goto error;
CALLOC_ARRAY(alternate, 1); alternate = odb_source_new(odb, pathbuf.buf, false);
alternate->odb = odb;
alternate->local = false;
/* pathbuf.buf is already in r->objects->source_by_path */
alternate->path = strbuf_detach(&pathbuf, NULL);
/* add the alternate entry */ /* add the alternate entry */
*odb->sources_tail = alternate; *odb->sources_tail = alternate;
odb->sources_tail = &(alternate->next); odb->sources_tail = &(alternate->next);
alternate->next = NULL;
assert(odb->source_by_path); pos = kh_put_odb_path_map(odb->source_by_path, alternate->path, &ret);
if (!ret)
BUG("source must not yet exist");
kh_value(odb->source_by_path, pos) = alternate; kh_value(odb->source_by_path, pos) = alternate;
/* recursively add alternates */ /* recursively add alternates */
@@ -337,9 +353,7 @@ struct odb_source *odb_set_temporary_primary_source(struct object_database *odb,
* Make a new primary odb and link the old primary ODB in as an * Make a new primary odb and link the old primary ODB in as an
* alternate * alternate
*/ */
source = xcalloc(1, sizeof(*source)); source = odb_source_new(odb, dir, false);
source->odb = odb;
source->path = xstrdup(dir);
/* /*
* Disable ref updates while a temporary odb is active, since * Disable ref updates while a temporary odb is active, since
@@ -352,11 +366,10 @@ struct odb_source *odb_set_temporary_primary_source(struct object_database *odb,
return source->next; return source->next;
} }
static void free_object_directory(struct odb_source *source) static void odb_source_free(struct odb_source *source)
{ {
free(source->path); free(source->path);
odb_clear_loose_cache(source); odb_source_loose_free(source->loose);
loose_object_map_clear(&source->loose_map);
free(source); free(source);
} }
@@ -374,7 +387,7 @@ void odb_restore_primary_source(struct object_database *odb,
BUG("we expect the old primary object store to be the first alternate"); BUG("we expect the old primary object store to be the first alternate");
odb->sources = restore_source; odb->sources = restore_source;
free_object_directory(cur_source); odb_source_free(cur_source);
} }
char *compute_alternate_path(const char *path, struct strbuf *err) char *compute_alternate_path(const char *path, struct strbuf *err)
@@ -684,13 +697,18 @@ static int do_oid_object_info_extended(struct object_database *odb,
return 0; return 0;
} }
odb_prepare_alternates(odb);
while (1) { while (1) {
struct odb_source *source;
if (find_pack_entry(odb->repo, real, &e)) if (find_pack_entry(odb->repo, real, &e))
break; break;
/* Most likely it's a loose object. */ /* Most likely it's a loose object. */
if (!loose_object_info(odb->repo, real, oi, flags)) for (source = odb->sources; source; source = source->next)
return 0; if (!odb_source_loose_read_object_info(source, real, oi, flags))
return 0;
/* Not a loose object; someone else may have just packed it. */ /* Not a loose object; someone else may have just packed it. */
if (!(flags & OBJECT_INFO_QUICK)) { if (!(flags & OBJECT_INFO_QUICK)) {
@@ -969,6 +987,22 @@ int odb_has_object(struct object_database *odb, const struct object_id *oid,
return odb_read_object_info_extended(odb, oid, NULL, object_info_flags) >= 0; return odb_read_object_info_extended(odb, oid, NULL, object_info_flags) >= 0;
} }
int odb_freshen_object(struct object_database *odb,
const struct object_id *oid)
{
struct odb_source *source;
if (packfile_store_freshen_object(odb->packfiles, oid))
return 1;
odb_prepare_alternates(odb);
for (source = odb->sources; source; source = source->next)
if (odb_source_loose_freshen_object(source, oid))
return 1;
return 0;
}
void odb_assert_oid_type(struct object_database *odb, void odb_assert_oid_type(struct object_database *odb,
const struct object_id *oid, enum object_type expect) const struct object_id *oid, enum object_type expect)
{ {
@@ -987,7 +1021,15 @@ int odb_write_object_ext(struct object_database *odb,
struct object_id *compat_oid, struct object_id *compat_oid,
unsigned flags) unsigned flags)
{ {
return write_object_file(odb->sources, buf, len, type, oid, compat_oid, flags); return odb_source_loose_write_object(odb->sources, buf, len, type,
oid, compat_oid, flags);
}
int odb_write_object_stream(struct object_database *odb,
struct odb_write_stream *stream, size_t len,
struct object_id *oid)
{
return odb_source_loose_write_stream(odb->sources, stream, len, oid);
} }
struct object_database *odb_new(struct repository *repo) struct object_database *odb_new(struct repository *repo)
@@ -1002,13 +1044,13 @@ struct object_database *odb_new(struct repository *repo)
return o; return o;
} }
static void free_object_directories(struct object_database *o) static void odb_free_sources(struct object_database *o)
{ {
while (o->sources) { while (o->sources) {
struct odb_source *next; struct odb_source *next;
next = o->sources->next; next = o->sources->next;
free_object_directory(o->sources); odb_source_free(o->sources);
o->sources = next; o->sources = next;
} }
kh_destroy_odb_path_map(o->source_by_path); kh_destroy_odb_path_map(o->source_by_path);
@@ -1026,7 +1068,7 @@ void odb_clear(struct object_database *o)
o->commit_graph = NULL; o->commit_graph = NULL;
o->commit_graph_attempted = 0; o->commit_graph_attempted = 0;
free_object_directories(o); odb_free_sources(o);
o->sources_tail = NULL; o->sources_tail = NULL;
o->loaded_alternates = 0; o->loaded_alternates = 0;
@@ -1057,7 +1099,7 @@ void odb_reprepare(struct object_database *o)
odb_prepare_alternates(o); odb_prepare_alternates(o);
for (source = o->sources; source; source = source->next) for (source = o->sources; source; source = source->next)
odb_clear_loose_cache(source); odb_source_loose_reprepare(source);
o->approximate_object_count_valid = 0; o->approximate_object_count_valid = 0;

41
odb.h
View File

@@ -14,6 +14,14 @@ struct strbuf;
struct repository; struct repository;
struct multi_pack_index; struct multi_pack_index;
/*
* Set this to 0 to prevent odb_read_object_info_extended() from fetching missing
* blobs. This has a difference only if extensions.partialClone is set.
*
* Its default value is 1.
*/
extern int fetch_if_missing;
/* /*
* Compute the exact path an alternate is at and returns it. In case of * Compute the exact path an alternate is at and returns it. In case of
* error NULL is returned and the human readable error is added to `err` * error NULL is returned and the human readable error is added to `err`
@@ -40,20 +48,8 @@ struct odb_source {
/* Object database that owns this object source. */ /* Object database that owns this object source. */
struct object_database *odb; struct object_database *odb;
/* /* Private state for loose objects. */
* Used to store the results of readdir(3) calls when we are OK struct odb_source_loose *loose;
* sacrificing accuracy due to races for speed. That includes
* object existence with OBJECT_INFO_QUICK, as well as
* our search for unique abbreviated hashes. Don't use it for tasks
* requiring greater accuracy!
*
* Be sure to call odb_load_loose_cache() before using.
*/
uint32_t loose_objects_subdir_seen[8]; /* 256 bits */
struct oidtree *loose_objects_cache;
/* Map between object IDs for loose objects. */
struct loose_object_map *loose_map;
/* /*
* private data * private data
@@ -89,6 +85,10 @@ struct odb_source {
char *path; char *path;
}; };
struct odb_source *odb_source_new(struct object_database *odb,
const char *path,
bool local);
struct packed_git; struct packed_git;
struct packfile_store; struct packfile_store;
struct cached_object_entry; struct cached_object_entry;
@@ -396,6 +396,9 @@ int odb_has_object(struct object_database *odb,
const struct object_id *oid, const struct object_id *oid,
unsigned flags); unsigned flags);
int odb_freshen_object(struct object_database *odb,
const struct object_id *oid);
void odb_assert_oid_type(struct object_database *odb, void odb_assert_oid_type(struct object_database *odb,
const struct object_id *oid, enum object_type expect); const struct object_id *oid, enum object_type expect);
@@ -489,4 +492,14 @@ static inline int odb_write_object(struct object_database *odb,
return odb_write_object_ext(odb, buf, len, type, oid, NULL, 0); return odb_write_object_ext(odb, buf, len, type, oid, NULL, 0);
} }
struct odb_write_stream {
const void *(*read)(struct odb_write_stream *, unsigned long *len);
void *data;
int is_finished;
};
int odb_write_object_stream(struct object_database *odb,
struct odb_write_stream *stream, size_t len,
struct object_id *oid);
#endif /* ODB_H */ #endif /* ODB_H */

16
packfile.c
View File

@@ -900,6 +900,22 @@ struct packed_git *packfile_store_load_pack(struct packfile_store *store,
return p; return p;
} }
int packfile_store_freshen_object(struct packfile_store *store,
const struct object_id *oid)
{
struct pack_entry e;
if (!find_pack_entry(store->odb->repo, oid, &e))
return 0;
if (e.p->is_cruft)
return 0;
if (e.p->freshened)
return 1;
if (utime(e.p->pack_name, NULL))
return 0;
e.p->freshened = 1;
return 1;
}
void (*report_garbage)(unsigned seen_bits, const char *path); void (*report_garbage)(unsigned seen_bits, const char *path);
static void report_helper(const struct string_list *list, static void report_helper(const struct string_list *list,

3
packfile.h
View File

@@ -191,6 +191,9 @@ struct packfile_list_entry *packfile_store_get_packs(struct packfile_store *stor
struct packed_git *packfile_store_load_pack(struct packfile_store *store, struct packed_git *packfile_store_load_pack(struct packfile_store *store,
const char *idx_path, int local); const char *idx_path, int local);
int packfile_store_freshen_object(struct packfile_store *store,
const struct object_id *oid);
struct pack_window { struct pack_window {
struct pack_window *next; struct pack_window *next;
unsigned char *base; unsigned char *base;

14
repository.c
View File

@@ -160,20 +160,24 @@ void repo_set_gitdir(struct repository *repo,
* until after xstrdup(root). Then we can free it. * until after xstrdup(root). Then we can free it.
*/ */
char *old_gitdir = repo->gitdir; char *old_gitdir = repo->gitdir;
char *objects_path = NULL;
repo->gitdir = xstrdup(gitfile ? gitfile : root); repo->gitdir = xstrdup(gitfile ? gitfile : root);
free(old_gitdir); free(old_gitdir);
repo_set_commondir(repo, o->commondir); repo_set_commondir(repo, o->commondir);
expand_base_dir(&objects_path, o->object_dir,
repo->commondir, "objects");
if (!repo->objects->sources) { if (!repo->objects->sources) {
CALLOC_ARRAY(repo->objects->sources, 1); repo->objects->sources = odb_source_new(repo->objects,
repo->objects->sources->odb = repo->objects; objects_path, true);
repo->objects->sources->local = true;
repo->objects->sources_tail = &repo->objects->sources->next; repo->objects->sources_tail = &repo->objects->sources->next;
free(objects_path);
} else {
free(repo->objects->sources->path);
repo->objects->sources->path = objects_path;
} }
expand_base_dir(&repo->objects->sources->path, o->object_dir,
repo->commondir, "objects");
repo->objects->sources->disable_ref_updates = o->disable_ref_updates; repo->objects->sources->disable_ref_updates = o->disable_ref_updates;

11
streaming.c
View File

@@ -230,12 +230,21 @@ static int open_istream_loose(struct git_istream *st, struct repository *r,
enum object_type *type) enum object_type *type)
{ {
struct object_info oi = OBJECT_INFO_INIT; struct object_info oi = OBJECT_INFO_INIT;
struct odb_source *source;
oi.sizep = &st->size; oi.sizep = &st->size;
oi.typep = type; oi.typep = type;
st->u.loose.mapped = map_loose_object(r, oid, &st->u.loose.mapsize); odb_prepare_alternates(r->objects);
for (source = r->objects->sources; source; source = source->next) {
st->u.loose.mapped = odb_source_loose_map_object(source, oid,
&st->u.loose.mapsize);
if (st->u.loose.mapped)
break;
}
if (!st->u.loose.mapped) if (!st->u.loose.mapped)
return -1; return -1;
switch (unpack_loose_header(&st->z, st->u.loose.mapped, switch (unpack_loose_header(&st->z, st->u.loose.mapped,
st->u.loose.mapsize, st->u.loose.hdr, st->u.loose.mapsize, st->u.loose.hdr,
sizeof(st->u.loose.hdr))) { sizeof(st->u.loose.hdr))) {