Re: [RFC/PATCH v4 09/49] Add initial external odb support

[Ben Peart]

On 6/20/2017 3:54 AM, Christian Couder wrote:

From: Jeff King 

Signed-off-by: Christian Couder 


I'd suggest you make the function names consistent with the capabilities 
flags (ie get, put, have) both here in odb_helper.c/h and in 
external_odb.c/h.



+int odb_helper_has_object(struct odb_helper *o, const unsigned char *sha1);
+int odb_helper_fetch_object(struct odb_helper *o, const unsigned char *sha1,
+   int fd);
+
+#endif /* ODB_HELPER_H */


The following patch fixes a few compiler warnings/errors.


diff --git a/odb-helper.c b/odb-helper.c
index 01cd6a713c..ffbbd2fc87 100644
--- a/odb-helper.c
+++ b/odb-helper.c
@@ -118,7 +118,7 @@ static int check_object_process_error(int err,
  unsigned int capability)
 {
if (!err)
-   return;
+   return 0;

if (!strcmp(status, "error")) {
/* The process signaled a problem with the file. */
@@ -192,7 +192,7 @@ static ssize_t 
read_packetized_plain_object_to_fd(struct odb_helper *o,

git_SHA1_Init();

/* First header.. */
-   hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %lu", typename(type), 
size) + 1;
+   hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %"PRIuMAX, 
typename(type), size) + 1;

stream.next_in = (unsigned char *)hdr;
stream.avail_in = hdrlen;
while (git_deflate(, 0) == Z_OK)
@@ -253,7 +253,7 @@ static ssize_t 
read_packetized_plain_object_to_fd(struct odb_helper *o,

return -1;
}
if (total_got != size) {
-   warning("size mismatch from odb helper '%s' for %s (%lu 
!= %lu)",
+   warning("size mismatch from odb helper '%s' for %s (%lu 
!= %"PRIuMAX")",

o->name, sha1_to_hex(sha1), total_got, size);
return -1;
}
@@ -587,7 +587,6 @@ static int have_object_process(struct odb_helper *o)
struct strbuf status = STRBUF_INIT;
const char *cmd = o->cmd;
uint64_t start;
-   char *line;
int packet_len;
int total_got = 0;

@@ -946,7 +945,7 @@ int odb_helper_for_each_object(struct odb_helper *o,
return 0;
 }

-int odb_helper_write_plain_object(struct odb_helper *o,
+static int odb_helper_write_plain_object(struct odb_helper *o,
  const void *buf, size_t len,
  const char *type, unsigned char *sha1)
 {

[RFC/PATCH v4 09/49] Add initial external odb support

[Christian Couder]

From: Jeff King 

Signed-off-by: Christian Couder 
---
 Makefile|   2 +
 cache.h |   9 ++
 external-odb.c  | 115 +++
 external-odb.h  |   8 ++
 odb-helper.c| 245 
 odb-helper.h|  25 +
 sha1_file.c |  79 +++-
 t/t0400-external-odb.sh |  46 +
 8 files changed, 507 insertions(+), 22 deletions(-)
 create mode 100644 external-odb.c
 create mode 100644 external-odb.h
 create mode 100644 odb-helper.c
 create mode 100644 odb-helper.h
 create mode 100755 t/t0400-external-odb.sh

diff --git a/Makefile b/Makefile
index f484801638..b488874d60 100644
--- a/Makefile
+++ b/Makefile
@@ -776,6 +776,7 @@ LIB_OBJS += ewah/ewah_bitmap.o
 LIB_OBJS += ewah/ewah_io.o
 LIB_OBJS += ewah/ewah_rlw.o
 LIB_OBJS += exec_cmd.o
+LIB_OBJS += external-odb.o
 LIB_OBJS += fetch-pack.o
 LIB_OBJS += fsck.o
 LIB_OBJS += gettext.o
@@ -808,6 +809,7 @@ LIB_OBJS += notes-cache.o
 LIB_OBJS += notes-merge.o
 LIB_OBJS += notes-utils.o
 LIB_OBJS += object.o
+LIB_OBJS += odb-helper.o
 LIB_OBJS += oidset.o
 LIB_OBJS += pack-bitmap.o
 LIB_OBJS += pack-bitmap-write.o
diff --git a/cache.h b/cache.h
index d6ba8a2f11..391a69e9c5 100644
--- a/cache.h
+++ b/cache.h
@@ -954,6 +954,12 @@ const char *git_path_shallow(void);
  */
 extern const char *sha1_file_name(const unsigned char *sha1);
 
+/*
+ * Like sha1_file_name, but return the filename within a specific alternate
+ * object directory. Shares the same static buffer with sha1_file_name.
+ */
+extern const char *sha1_file_name_alt(const char *objdir, const unsigned char 
*sha1);
+
 /*
  * Return the name of the (local) packfile with the specified sha1 in
  * its name.  The return value is a pointer to memory that is
@@ -1265,6 +1271,8 @@ extern int do_check_packed_object_crc;
 
 extern int check_sha1_signature(const unsigned char *sha1, void *buf, unsigned 
long size, const char *type);
 
+extern int create_object_tmpfile(struct strbuf *tmp, const char *filename);
+extern void close_sha1_file(int fd);
 extern int finalize_object_file(const char *tmpfile, const char *filename);
 
 extern int has_sha1_pack(const unsigned char *sha1);
@@ -1600,6 +1608,7 @@ extern void read_info_alternates(const char * 
relative_base, int depth);
 extern char *compute_alternate_path(const char *path, struct strbuf *err);
 typedef int alt_odb_fn(struct alternate_object_database *, void *);
 extern int foreach_alt_odb(alt_odb_fn, void*);
+extern void prepare_external_alt_odb(void);
 
 /*
  * Allocate a "struct alternate_object_database" but do _not_ actually
diff --git a/external-odb.c b/external-odb.c
new file mode 100644
index 00..1ccfa99a01
--- /dev/null
+++ b/external-odb.c
@@ -0,0 +1,115 @@
+#include "cache.h"
+#include "external-odb.h"
+#include "odb-helper.h"
+
+static struct odb_helper *helpers;
+static struct odb_helper **helpers_tail = 
+
+static struct odb_helper *find_or_create_helper(const char *name, int len)
+{
+   struct odb_helper *o;
+
+   for (o = helpers; o; o = o->next)
+   if (!strncmp(o->name, name, len) && !o->name[len])
+   return o;
+
+   o = odb_helper_new(name, len);
+   *helpers_tail = o;
+   helpers_tail = >next;
+
+   return o;
+}
+
+static int external_odb_config(const char *var, const char *value, void *data)
+{
+   struct odb_helper *o;
+   const char *key, *dot;
+
+   if (!skip_prefix(var, "odb.", ))
+   return 0;
+   dot = strrchr(key, '.');
+   if (!dot)
+   return 0;
+
+   o = find_or_create_helper(key, dot - key);
+   key = dot + 1;
+
+   if (!strcmp(key, "command"))
+   return git_config_string(>cmd, var, value);
+
+   return 0;
+}
+
+static void external_odb_init(void)
+{
+   static int initialized;
+
+   if (initialized)
+   return;
+   initialized = 1;
+
+   git_config(external_odb_config, NULL);
+}
+
+const char *external_odb_root(void)
+{
+   static const char *root;
+   if (!root)
+   root = git_pathdup("objects/external");
+   return root;
+}
+
+int external_odb_has_object(const unsigned char *sha1)
+{
+   struct odb_helper *o;
+
+   external_odb_init();
+
+   for (o = helpers; o; o = o->next)
+   if (odb_helper_has_object(o, sha1))
+   return 1;
+   return 0;
+}
+
+int external_odb_fetch_object(const unsigned char *sha1)
+{
+   struct odb_helper *o;
+   const char *path;
+
+   if (!external_odb_has_object(sha1))
+   return -1;
+
+   path = sha1_file_name_alt(external_odb_root(), sha1);
+   safe_create_leading_directories_const(path);
+   prepare_external_alt_odb();
+
+   for (o = helpers; o; o = o->next) {
+   struct strbuf tmpfile = STRBUF_INIT;
+   int ret;
+