Skip to content

Commit 2564d99

Browse files
peffgitster
peff
authored and
gitster
committed
tmp-objdir: introduce API for temporary object directories
Once objects are added to the object database by a process, they cannot easily be deleted, as we don't know what other processes may have started referencing them. We have to clean them up with git-gc, which will apply the usual reachability and grace-period checks. This patch provides an alternative: it helps callers create a temporary directory inside the object directory, and a temporary environment which can be passed to sub-programs to ask them to write there (the original object directory remains accessible as an alternate of the temporary one). See tmp-objdir.h for details on the API. Signed-off-by: Jeff King <[email protected]> Signed-off-by: Junio C Hamano <[email protected]>
1 parent 526f108 commit 2564d99

File tree

3 files changed

+328
-0
lines changed

3 files changed

+328
-0
lines changed

‎Makefile

+1
Original file line numberDiff line numberDiff line change
@@ -831,6 +831,7 @@ LIB_OBJS += submodule-config.o
831831
LIB_OBJS += symlinks.o
832832
LIB_OBJS += tag.o
833833
LIB_OBJS += tempfile.o
834+
LIB_OBJS += tmp-objdir.o
834835
LIB_OBJS += trace.o
835836
LIB_OBJS += trailer.o
836837
LIB_OBJS += transport.o

‎tmp-objdir.c

+273
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,273 @@
1+
#include "cache.h"
2+
#include "tmp-objdir.h"
3+
#include "dir.h"
4+
#include "sigchain.h"
5+
#include "string-list.h"
6+
#include "strbuf.h"
7+
#include "argv-array.h"
8+
9+
struct tmp_objdir {
10+
struct strbuf path;
11+
struct argv_array env;
12+
};
13+
14+
/*
15+
* Allow only one tmp_objdir at a time in a running process, which simplifies
16+
* our signal/atexit cleanup routines. It's doubtful callers will ever need
17+
* more than one, and we can expand later if so. You can have many such
18+
* tmp_objdirs simultaneously in many processes, of course.
19+
*/
20+
static struct tmp_objdir *the_tmp_objdir;
21+
22+
static void tmp_objdir_free(struct tmp_objdir *t)
23+
{
24+
strbuf_release(&t->path);
25+
argv_array_clear(&t->env);
26+
free(t);
27+
}
28+
29+
static int tmp_objdir_destroy_1(struct tmp_objdir *t, int on_signal)
30+
{
31+
int err;
32+
33+
if (!t)
34+
return 0;
35+
36+
if (t == the_tmp_objdir)
37+
the_tmp_objdir = NULL;
38+
39+
/*
40+
* This may use malloc via strbuf_grow(), but we should
41+
* have pre-grown t->path sufficiently so that this
42+
* doesn't happen in practice.
43+
*/
44+
err = remove_dir_recursively(&t->path, 0);
45+
46+
/*
47+
* When we are cleaning up due to a signal, we won't bother
48+
* freeing memory; it may cause a deadlock if the signal
49+
* arrived while libc's allocator lock is held.
50+
*/
51+
if (!on_signal)
52+
tmp_objdir_free(t);
53+
return err;
54+
}
55+
56+
int tmp_objdir_destroy(struct tmp_objdir *t)
57+
{
58+
return tmp_objdir_destroy_1(t, 0);
59+
}
60+
61+
static void remove_tmp_objdir(void)
62+
{
63+
tmp_objdir_destroy(the_tmp_objdir);
64+
}
65+
66+
static void remove_tmp_objdir_on_signal(int signo)
67+
{
68+
tmp_objdir_destroy_1(the_tmp_objdir, 1);
69+
sigchain_pop(signo);
70+
raise(signo);
71+
}
72+
73+
/*
74+
* These env_* functions are for setting up the child environment; the
75+
* "replace" variant overrides the value of any existing variable with that
76+
* "key". The "append" variant puts our new value at the end of a list,
77+
* separated by PATH_SEP (which is what separate values in
78+
* GIT_ALTERNATE_OBJECT_DIRECTORIES).
79+
*/
80+
static void env_append(struct argv_array *env, const char *key, const char *val)
81+
{
82+
const char *old = getenv(key);
83+
84+
if (!old)
85+
argv_array_pushf(env, "%s=%s", key, val);
86+
else
87+
argv_array_pushf(env, "%s=%s%c%s", key, old, PATH_SEP, val);
88+
}
89+
90+
static void env_replace(struct argv_array *env, const char *key, const char *val)
91+
{
92+
argv_array_pushf(env, "%s=%s", key, val);
93+
}
94+
95+
static int setup_tmp_objdir(const char *root)
96+
{
97+
char *path;
98+
int ret = 0;
99+
100+
path = xstrfmt("%s/pack", root);
101+
ret = mkdir(path, 0777);
102+
free(path);
103+
104+
return ret;
105+
}
106+
107+
struct tmp_objdir *tmp_objdir_create(void)
108+
{
109+
static int installed_handlers;
110+
struct tmp_objdir *t;
111+
112+
if (the_tmp_objdir)
113+
die("BUG: only one tmp_objdir can be used at a time");
114+
115+
t = xmalloc(sizeof(*t));
116+
strbuf_init(&t->path, 0);
117+
argv_array_init(&t->env);
118+
119+
strbuf_addf(&t->path, "%s/incoming-XXXXXX", get_object_directory());
120+
121+
/*
122+
* Grow the strbuf beyond any filename we expect to be placed in it.
123+
* If tmp_objdir_destroy() is called by a signal handler, then
124+
* we should be able to use the strbuf to remove files without
125+
* having to call malloc.
126+
*/
127+
strbuf_grow(&t->path, 1024);
128+
129+
if (!mkdtemp(t->path.buf)) {
130+
/* free, not destroy, as we never touched the filesystem */
131+
tmp_objdir_free(t);
132+
return NULL;
133+
}
134+
135+
the_tmp_objdir = t;
136+
if (!installed_handlers) {
137+
atexit(remove_tmp_objdir);
138+
sigchain_push_common(remove_tmp_objdir_on_signal);
139+
installed_handlers++;
140+
}
141+
142+
if (setup_tmp_objdir(t->path.buf)) {
143+
tmp_objdir_destroy(t);
144+
return NULL;
145+
}
146+
147+
env_append(&t->env, ALTERNATE_DB_ENVIRONMENT,
148+
absolute_path(get_object_directory()));
149+
env_replace(&t->env, DB_ENVIRONMENT, absolute_path(t->path.buf));
150+
151+
return t;
152+
}
153+
154+
/*
155+
* Make sure we copy packfiles and their associated metafiles in the correct
156+
* order. All of these ends_with checks are slightly expensive to do in
157+
* the midst of a sorting routine, but in practice it shouldn't matter.
158+
* We will have a relatively small number of packfiles to order, and loose
159+
* objects exit early in the first line.
160+
*/
161+
static int pack_copy_priority(const char *name)
162+
{
163+
if (!starts_with(name, "pack"))
164+
return 0;
165+
if (ends_with(name, ".keep"))
166+
return 1;
167+
if (ends_with(name, ".pack"))
168+
return 2;
169+
if (ends_with(name, ".idx"))
170+
return 3;
171+
return 4;
172+
}
173+
174+
static int pack_copy_cmp(const char *a, const char *b)
175+
{
176+
return pack_copy_priority(a) - pack_copy_priority(b);
177+
}
178+
179+
static int read_dir_paths(struct string_list *out, const char *path)
180+
{
181+
DIR *dh;
182+
struct dirent *de;
183+
184+
dh = opendir(path);
185+
if (!dh)
186+
return -1;
187+
188+
while ((de = readdir(dh)))
189+
if (!is_dot_or_dotdot(de->d_name))
190+
string_list_append(out, de->d_name);
191+
192+
closedir(dh);
193+
return 0;
194+
}
195+
196+
static int migrate_paths(struct strbuf *src, struct strbuf *dst);
197+
198+
static int migrate_one(struct strbuf *src, struct strbuf *dst)
199+
{
200+
struct stat st;
201+
202+
if (stat(src->buf, &st) < 0)
203+
return -1;
204+
if (S_ISDIR(st.st_mode)) {
205+
if (!mkdir(dst->buf, 0777)) {
206+
if (adjust_shared_perm(dst->buf))
207+
return -1;
208+
} else if (errno != EEXIST)
209+
return -1;
210+
return migrate_paths(src, dst);
211+
}
212+
return finalize_object_file(src->buf, dst->buf);
213+
}
214+
215+
static int migrate_paths(struct strbuf *src, struct strbuf *dst)
216+
{
217+
size_t src_len = src->len, dst_len = dst->len;
218+
struct string_list paths = STRING_LIST_INIT_DUP;
219+
int i;
220+
int ret = 0;
221+
222+
if (read_dir_paths(&paths, src->buf) < 0)
223+
return -1;
224+
paths.cmp = pack_copy_cmp;
225+
string_list_sort(&paths);
226+
227+
for (i = 0; i < paths.nr; i++) {
228+
const char *name = paths.items[i].string;
229+
230+
strbuf_addf(src, "/%s", name);
231+
strbuf_addf(dst, "/%s", name);
232+
233+
ret |= migrate_one(src, dst);
234+
235+
strbuf_setlen(src, src_len);
236+
strbuf_setlen(dst, dst_len);
237+
}
238+
239+
string_list_clear(&paths, 0);
240+
return ret;
241+
}
242+
243+
int tmp_objdir_migrate(struct tmp_objdir *t)
244+
{
245+
struct strbuf src = STRBUF_INIT, dst = STRBUF_INIT;
246+
int ret;
247+
248+
if (!t)
249+
return 0;
250+
251+
strbuf_addbuf(&src, &t->path);
252+
strbuf_addstr(&dst, get_object_directory());
253+
254+
ret = migrate_paths(&src, &dst);
255+
256+
strbuf_release(&src);
257+
strbuf_release(&dst);
258+
259+
tmp_objdir_destroy(t);
260+
return ret;
261+
}
262+
263+
const char **tmp_objdir_env(const struct tmp_objdir *t)
264+
{
265+
if (!t)
266+
return NULL;
267+
return t->env.argv;
268+
}
269+
270+
void tmp_objdir_add_as_alternate(const struct tmp_objdir *t)
271+
{
272+
add_to_alternates_memory(t->path.buf);
273+
}

‎tmp-objdir.h

+54
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
#ifndef TMP_OBJDIR_H
2+
#define TMP_OBJDIR_H
3+
4+
/*
5+
* This API allows you to create a temporary object directory, advertise it to
6+
* sub-processes via GIT_OBJECT_DIRECTORY and GIT_ALTERNATE_OBJECT_DIRECTORIES,
7+
* and then either migrate its object into the main object directory, or remove
8+
* it. The library handles unexpected signal/exit death by cleaning up the
9+
* temporary directory.
10+
*
11+
* Example:
12+
*
13+
* struct tmp_objdir *t = tmp_objdir_create();
14+
* if (!run_command_v_opt_cd_env(cmd, 0, NULL, tmp_objdir_env(t)) &&
15+
* !tmp_objdir_migrate(t))
16+
* printf("success!\n");
17+
* else
18+
* die("failed...tmp_objdir will clean up for us");
19+
*
20+
*/
21+
22+
struct tmp_objdir;
23+
24+
/*
25+
* Create a new temporary object directory; returns NULL on failure.
26+
*/
27+
struct tmp_objdir *tmp_objdir_create(void);
28+
29+
/*
30+
* Return a list of environment strings, suitable for use with
31+
* child_process.env, that can be passed to child programs to make use of the
32+
* temporary object directory.
33+
*/
34+
const char **tmp_objdir_env(const struct tmp_objdir *);
35+
36+
/*
37+
* Finalize a temporary object directory by migrating its objects into the main
38+
* object database, removing the temporary directory, and freeing any
39+
* associated resources.
40+
*/
41+
int tmp_objdir_migrate(struct tmp_objdir *);
42+
43+
/*
44+
* Destroy a temporary object directory, discarding any objects it contains.
45+
*/
46+
int tmp_objdir_destroy(struct tmp_objdir *);
47+
48+
/*
49+
* Add the temporary object directory as an alternate object store in the
50+
* current process.
51+
*/
52+
void tmp_objdir_add_as_alternate(const struct tmp_objdir *);
53+
54+
#endif /* TMP_OBJDIR_H */

0 commit comments

Comments
 (0)