RFC: Reftable backend current version

Documentation/config/extensions.txt

@@ -37,3 +37,12 @@ details.
 +
 For historical reasons, `extensions.worktreeConfig` is respected
 regardless of the `core.repositoryFormatVersion` setting.
++
+extensions.refStorage::
+Specify the ref storage mechanism to use. The acceptable values are `files` and
+`reftable`. If not specified, `files` is assumed. It is an error to specify
+this key unless `core.repositoryFormatVersion` is 1.
++
+Note that this setting should only be set by linkgit:git-init[1] or
+linkgit:git-clone[1]. Trying to change it after initialization will not
+work and will produce hard-to-diagnose issues.

Documentation/technical/repository-version.txt

@@ -100,3 +100,10 @@ If set, by default "git config" reads from both "config" and
 multiple working directory mode, "config" file is shared while
 "config.worktree" is per-working directory (i.e., it's in
 GIT_COMMON_DIR/worktrees/<id>/config.worktree)
+
+==== `refStorage`
+
+Specifies the file format for the ref database. Values are `files`
+(for the traditional packed + loose ref format) and `reftable` for the
+binary reftable format. See https://github.com/google/reftable for
+more information.

Makefile

@@ -1118,6 +1118,7 @@ LIB_OBJS += reflog.o
 LIB_OBJS += refs.o
 LIB_OBJS += refs/debug.o
 LIB_OBJS += refs/files-backend.o
+LIB_OBJS += refs/reftable-backend.o
 LIB_OBJS += refs/iterator.o
 LIB_OBJS += refs/packed-backend.o
 LIB_OBJS += refs/ref-cache.o

abspath.c

@@ -41,8 +41,7 @@ static void get_next_component(struct strbuf *next, struct strbuf *remaining)
 /* Skip sequences of multiple path-separators */
 for (start = remaining->buf; is_dir_sep(*start); start++)
 ; /* nothing */
-/* Find end of the path component */
-for (end = start; *end && !is_dir_sep(*end); end++)
+/* Find end of the path component */	for (end = start; *end && !is_dir_sep(*end); end++)
 ; /* nothing */
 
 strbuf_add(next, start, end - start);

builtin/clone.c

@@ -1101,6 +1101,7 @@ int cmd_clone(int argc, const char **argv, const char *prefix)
 }
 
 init_db(git_dir, real_git_dir, option_template, GIT_HASH_UNKNOWN, NULL,
+default_ref_storage(),
 do_not_override_repo_unix_permissions, INIT_DB_QUIET);
 
 if (real_git_dir) {
@@ -1327,7 +1328,7 @@ int cmd_clone(int argc, const char **argv, const char *prefix)
  * let's set ours to the same thing.
  */
 hash_algo = hash_algo_by_ptr(transport_get_hash_algo(transport));
-initialize_repository_version(hash_algo, 1);
+initialize_repository_version(hash_algo, 1, default_ref_storage());
 repo_set_hash_algo(the_repository, hash_algo);
 
 if (mapped_refs) {

builtin/init-db.c

@@ -71,6 +71,7 @@ static const char *const init_db_usage[] = {
 int cmd_init_db(int argc, const char **argv, const char *prefix)
 {
 const char *git_dir;
+const char *ref_storage_format = default_ref_storage();
 const char *real_git_dir = NULL;
 const char *work_tree;
 const char *template_dir = NULL;
@@ -98,6 +99,7 @@ int cmd_init_db(int argc, const char **argv, const char *prefix)
 OPT_END()
 };
 
+fprintf(stderr, "RSF: %s\n", ref_storage_format);
 argc = parse_options(argc, argv, prefix, init_db_options, init_db_usage, 0);
 
 if (real_git_dir && is_bare_repository_cfg == 1)
@@ -236,5 +238,5 @@ int cmd_init_db(int argc, const char **argv, const char *prefix)
 
 flags |= INIT_DB_EXIST_OK;
 return init_db(git_dir, real_git_dir, template_dir, hash_algo,
- initial_branch, init_shared_repository, flags);
+ initial_branch, ref_storage_format, init_shared_repository, flags);
 }

builtin/stash.c

@@ -242,10 +242,16 @@ static int get_stash_info(struct stash_info *info, int argc, const char **argv)
 static int do_clear_stash(void)
 {
 struct object_id obj;
+int result;
 if (repo_get_oid(the_repository, ref_stash, &obj))
 return 0;
 
-return delete_ref(NULL, ref_stash, &obj, 0);
+result = delete_ref(NULL, ref_stash, &obj, 0);
+
+/* Ignore error; this is necessary for reftable, which keeps reflogs
+ * even when refs are deleted. */
+delete_reflog(ref_stash);
+return result;
 }
 
 static int clear_stash(int argc, const char **argv, const char *prefix)

builtin/worktree.c

@@ -25,6 +25,7 @@
 #include "utf8.h"
 #include "worktree.h"
 #include "quote.h"
+#include "../refs/refs-internal.h"
 
 #define BUILTIN_WORKTREE_ADD_USAGE \
 N_("git worktree add [-f] [--detach] [--checkout] [--lock [--reason <string>]]\n" \
@@ -503,9 +504,29 @@ static int add_worktree(const char *path, const char *refname,
  * worktree.
  */
 strbuf_reset(&sb);
-strbuf_addf(&sb, "%s/HEAD", sb_repo.buf);
-write_file(sb.buf, "%s", oid_to_hex(null_oid()));
-strbuf_reset(&sb);
+if (get_main_ref_store(the_repository)->be == &refs_be_reftable) {
+/* XXX this is cut & paste from reftable_init_db. */
+strbuf_addf(&sb, "%s/HEAD", sb_repo.buf);
+write_file(sb.buf, "%s", "ref: refs/heads/.invalid\n");
+strbuf_reset(&sb);
+
+strbuf_addf(&sb, "%s/refs", sb_repo.buf);
+safe_create_dir(sb.buf, 1);
+strbuf_reset(&sb);
+
+strbuf_addf(&sb, "%s/refs/heads", sb_repo.buf);
+write_file(sb.buf, "this repository uses the reftable format");
+strbuf_reset(&sb);
+
+strbuf_addf(&sb, "%s/reftable", sb_repo.buf);
+safe_create_dir(sb.buf, 1);
+strbuf_reset(&sb);
+} else {
+strbuf_addf(&sb, "%s/HEAD", sb_repo.buf);
+write_file(sb.buf, "%s", oid_to_hex(null_oid()));
+strbuf_reset(&sb);
+}
+
 strbuf_addf(&sb, "%s/commondir", sb_repo.buf);
 write_file(sb.buf, "../..");
 

config.mak.uname

@@ -740,7 +740,7 @@ vcxproj:
 
 # Make .vcxproj files and add them
 perl contrib/buildsystems/generate -g Vcxproj
-git add -f git.sln {*,*/lib,t/helper/*}/*.vcxproj
+git add -f git.sln {*,*/lib,*/libreftable,t/helper/*}/*.vcxproj
 
 # Generate the LinkOrCopyBuiltins.targets and LinkOrCopyRemoteHttp.targets file
 (echo '<Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">' && \

contrib/workdir/git-new-workdir

@@ -79,7 +79,7 @@ trap cleanup $siglist
 # create the links to the original repo. explicitly exclude index, HEAD and
 # logs/HEAD from the list since they are purely related to the current working
 # directory, and should not be shared.
-for x in config refs logs/refs objects info hooks packed-refs remotes rr-cache svn
+for x in config refs logs/refs objects info hooks packed-refs remotes rr-cache svn reftable
 do
 # create a containing directory if needed
 case $x in

environment.h

@@ -215,6 +215,9 @@ int odb_pack_keep(const char *name);
 const char *get_log_output_encoding(void);
 const char *get_commit_output_encoding(void);
 
+const char *default_ref_storage(void); 
+
+
 extern const char *git_commit_encoding;
 extern const char *git_log_output_encoding;
 

refs.c

@@ -30,10 +30,15 @@
 #include "commit.h"
 #include "wildmatch.h"
 
+const char *default_ref_storage(void)
+{
+return git_env_bool("GIT_TEST_REFTABLE", 0) ? "reftable" : "files";
+}
+
 /*
  * List of all available backends
  */
-static struct ref_storage_be *refs_backends = &refs_be_files;
+static struct ref_storage_be *refs_backends = &refs_be_reftable;
 
 static struct ref_storage_be *find_ref_storage_backend(const char *name)
 {
@@ -1785,12 +1790,46 @@ static int refs_read_special_head(struct ref_store *ref_store,
 return result;
 }
 
+static int is_special_ref(const char *refname)
+{
+static const char *pseudo_refs[] = {
+"MERGE_HEAD",
+"FETCH_HEAD",
+"MERGE_AUTOSTASH",
+"CHERRY_PICK_HEAD",
+/* HEAD looks like a pseudoref but must be stored in the ref
+ * backend, as it partakes in transactions
+ */
+};
+int i = 0;
+
+/*
+ * Various tests assume that "ALL_CAPS" is not pseudoref and can contain
+ * symrefs.
+ */
+for (i = 0; i < ARRAY_SIZE(pseudo_refs); i++) {
+if (!strcmp(pseudo_refs[i], refname))
+return 1;
+}
+
+/* rebase-apply/rebase-merge functionality stores lots of info
+ besides a object ID in the file system, and expects that it
+ can clean up by deleting the rebase-{apply,merge}/
+ directory
+*/
+if (starts_with(refname, "rebase-apply/") ||
+ starts_with(refname, "rebase-merge/"))
+return 1;
+
+return 0;
+}
+
 int refs_read_raw_ref(struct ref_store *ref_store, const char *refname,
  struct object_id *oid, struct strbuf *referent,
  unsigned int *type, int *failure_errno)
 {
 assert(failure_errno);
-if (!strcmp(refname, "FETCH_HEAD") || !strcmp(refname, "MERGE_HEAD")) {
+if (is_special_ref(refname)) {
 return refs_read_special_head(ref_store, refname, oid, referent,
  type, failure_errno);
 }
@@ -1993,13 +2032,13 @@ static struct ref_store *lookup_ref_store_map(struct hashmap *map,
  * gitdir.
  */
 static struct ref_store *ref_store_init(struct repository *repo,
-const char *gitdir,
+const char *gitdir, const char *be_name,
 unsigned int flags)
 {
-const char *be_name = "files";
-struct ref_storage_be *be = find_ref_storage_backend(be_name);
+struct ref_storage_be *be;
 struct ref_store *refs;
 
+be = find_ref_storage_backend(be_name);
 if (!be)
 BUG("reference backend %s is unknown", be_name);
 
@@ -2015,7 +2054,11 @@ struct ref_store *get_main_ref_store(struct repository *r)
 if (!r->gitdir)
 BUG("attempting to get main_ref_store outside of repository");
 
-r->refs_private = ref_store_init(r, r->gitdir, REF_STORE_ALL_CAPS);
+r->refs_private = ref_store_init(r, r->gitdir,
+ r->ref_storage_format ?
+ r->ref_storage_format :
+ default_ref_storage(),
+ REF_STORE_ALL_CAPS);
 r->refs_private = maybe_debug_wrap_ref_store(r->gitdir, r->refs_private);
 return r->refs_private;
 }
@@ -2083,7 +2126,7 @@ struct ref_store *get_submodule_ref_store(const char *submodule)
 free(subrepo);
 goto done;
 }
-refs = ref_store_init(subrepo, submodule_sb.buf,
+refs = ref_store_init(subrepo, submodule_sb.buf, default_ref_storage(),
  REF_STORE_READ | REF_STORE_ODB);
 register_ref_store_map(&submodule_ref_stores, "submodule",
  refs, submodule);
@@ -2097,6 +2140,7 @@ struct ref_store *get_submodule_ref_store(const char *submodule)
 
 struct ref_store *get_worktree_ref_store(const struct worktree *wt)
 {
+const char *format = default_ref_storage();
 struct ref_store *refs;
 const char *id;
 
@@ -2111,11 +2155,10 @@ struct ref_store *get_worktree_ref_store(const struct worktree *wt)
 if (wt->id)
 refs = ref_store_init(the_repository,
  git_common_path("worktrees/%s", wt->id),
- REF_STORE_ALL_CAPS);
+ format, REF_STORE_ALL_CAPS);
 else
-refs = ref_store_init(the_repository,
- get_git_common_dir(),
- REF_STORE_ALL_CAPS);
+refs = ref_store_init(the_repository, get_git_common_dir(),
+ format, REF_STORE_ALL_CAPS);
 
 if (refs)
 register_ref_store_map(&worktree_ref_stores, "worktree",

refs.h

@@ -11,6 +11,9 @@ struct string_list;
 struct string_list_item;
 struct worktree;
 
+/* Returns the ref storage backend to use by default. */
+const char *default_ref_storage(void);
+
 /*
  * Resolve a reference, recursively following symbolic refererences.
  *

refs/refs-internal.h

@@ -694,6 +694,7 @@ struct ref_storage_be {
 };
 
 extern struct ref_storage_be refs_be_files;
+extern struct ref_storage_be refs_be_reftable;
 extern struct ref_storage_be refs_be_packed;
 
 /*