enhance clone and fetch -k experience
Now that index-pack can be streamed with a pack, it is probably a good idea to use it directly instead of creating a temporary file and running index-pack afterwards. This way index-pack can abort early whenever a corruption is encountered even if the pack has not been fully downloaded, it can display a progress percentage as it knows how much to expects, and it is a bit faster since the pack indexing is partially done as data is received. Using fetch -k doesn't need to disable thin pack generation on the remote end either. Signed-off-by: Nicolas Pitre <nico@cam.org> Signed-off-by: Junio C Hamano <junkio@cox.net>
Nicolas Pitre committed
Oct 27, 2006 at 15:42 UTC
d9c20ba13dfca737373ba466d2a718cafdc17f92
2 files changed
+15
-206
fetch-clone.c
+15
-204
@@ -3,97 +3,6 @@
3
#include "pkt-line.h"
4
#include "sideband.h"
5
#include <sys/wait.h>
6
-#include <sys/time.h>
7
-
8
-static int finish_pack(const char *pack_tmp_name, const char *me)
9
-{
10
- int pipe_fd[2];
11
- pid_t pid;
12
- char idx[PATH_MAX];
13
- char final[PATH_MAX];
14
- char hash[41];
15
- unsigned char sha1[20];
16
- char *cp;
17
- int err = 0;
18
-
19
- if (pipe(pipe_fd) < 0)
20
- die("%s: unable to set up pipe", me);
21
-
22
- strcpy(idx, pack_tmp_name); /* ".git/objects/pack-XXXXXX" */
23
- cp = strrchr(idx, '/');
24
- memcpy(cp, "/pidx", 5);
25
-
26
- pid = fork();
27
- if (pid < 0)
28
- die("%s: unable to fork off git-index-pack", me);
29
- if (!pid) {
30
- close(0);
31
- dup2(pipe_fd[1], 1);
32
- close(pipe_fd[0]);
33
- close(pipe_fd[1]);
34
- execl_git_cmd("index-pack", "-o", idx, pack_tmp_name, NULL);
35
- error("cannot exec git-index-pack <%s> <%s>",
36
- idx, pack_tmp_name);
37
- exit(1);
38
- }
39
- close(pipe_fd[1]);
40
- if (read(pipe_fd[0], hash, 40) != 40) {
41
- error("%s: unable to read from git-index-pack", me);
42
- err = 1;
43
- }
44
- close(pipe_fd[0]);
45
-
46
- for (;;) {
47
- int status, code;
48
-
49
- if (waitpid(pid, &status, 0) < 0) {
50
- if (errno == EINTR)
51
- continue;
52
- error("waitpid failed (%s)", strerror(errno));
53
- goto error_die;
54
- }
55
- if (WIFSIGNALED(status)) {
56
- int sig = WTERMSIG(status);
57
- error("git-index-pack died of signal %d", sig);
58
- goto error_die;
59
- }
60
- if (!WIFEXITED(status)) {
61
- error("git-index-pack died of unnatural causes %d",
62
- status);
63
- goto error_die;
64
- }
65
- code = WEXITSTATUS(status);
66
- if (code) {
67
- error("git-index-pack died with error code %d", code);
68
- goto error_die;
69
- }
70
- if (err)
71
- goto error_die;
72
- break;
73
- }
74
- hash[40] = 0;
75
- if (get_sha1_hex(hash, sha1)) {
76
- error("git-index-pack reported nonsense '%s'", hash);
77
- goto error_die;
78
- }
79
- /* Now we have pack in pack_tmp_name[], and
80
- * idx in idx[]; rename them to their final names.
81
- */
82
- snprintf(final, sizeof(final),
83
- "%s/pack/pack-%s.pack", get_object_directory(), hash);
84
- move_temp_to_file(pack_tmp_name, final);
85
- chmod(final, 0444);
86
- snprintf(final, sizeof(final),
87
- "%s/pack/pack-%s.idx", get_object_directory(), hash);
88
- move_temp_to_file(idx, final);
89
- chmod(final, 0444);
90
- return 0;
91
-
92
- error_die:
93
- unlink(idx);
94
- unlink(pack_tmp_name);
95
- exit(1);
96
-}
6
7
static pid_t setup_sideband(int sideband, const char *me, int fd[2], int xd[2])
8
{
@@ -128,7 +37,7 @@ static pid_t setup_sideband(int sideband, const char *me, int fd[2], int xd[2])
37
return side_pid;
38
}
39
131
-int receive_unpack_pack(int xd[2], const char *me, int quiet, int sideband)
40
+static int get_pack(int xd[2], const char *me, int sideband, const char **argv)
41
{
42
int status;
43
pid_t pid, side_pid;
@@ -142,135 +51,37 @@ int receive_unpack_pack(int xd[2], const char *me, int quiet, int sideband)
51
dup2(fd[0], 0);
52
close(fd[0]);
53
close(fd[1]);
145
- execl_git_cmd("unpack-objects", quiet ? "-q" : NULL, NULL);
146
- die("git-unpack-objects exec failed");
54
+ execv_git_cmd(argv);
55
+ die("%s exec failed", argv[0]);
56
}
57
close(fd[0]);
58
close(fd[1]);
59
while (waitpid(pid, &status, 0) < 0) {
60
if (errno != EINTR)
152
- die("waiting for git-unpack-objects: %s",
153
- strerror(errno));
61
+ die("waiting for %s: %s", argv[0], strerror(errno));
62
}
63
if (WIFEXITED(status)) {
64
int code = WEXITSTATUS(status);
65
if (code)
158
- die("git-unpack-objects died with error code %d",
159
- code);
66
+ die("%s died with error code %d", argv[0], code);
67
return 0;
68
}
69
if (WIFSIGNALED(status)) {
70
int sig = WTERMSIG(status);
164
- die("git-unpack-objects died of signal %d", sig);
71
+ die("%s died of signal %d", argv[0], sig);
72
}
166
- die("git-unpack-objects died of unnatural causes %d", status);
73
+ die("%s died of unnatural causes %d", argv[0], status);
74
}
75
169
-/*
170
- * We average out the download speed over this many "events", where
171
- * an event is a minimum of about half a second. That way, we get
172
- * a reasonably stable number.
173
- */
174
-#define NR_AVERAGE (4)
175
-
176
-/*
177
- * A "binary msec" is a power-of-two-msec, aka 1/1024th of a second.
178
- * Keeping the time in that format means that "bytes / msecs" means
179
- * the same as kB/s (modulo rounding).
180
- *
181
- * 1000512 is a magic number (usecs in a second, rounded up by half
182
- * of 1024, to make "rounding" come out right ;)
183
- */
184
-#define usec_to_binarymsec(x) ((int)(x) / (1000512 >> 10))
76
+int receive_unpack_pack(int xd[2], const char *me, int quiet, int sideband)
77
+{
78
+ const char *argv[3] = { "unpack-objects", quiet ? "-q" : NULL, NULL };
79
+ return get_pack(xd, me, sideband, argv);
80
+}
81
82
int receive_keep_pack(int xd[2], const char *me, int quiet, int sideband)
83
{
188
- char tmpfile[PATH_MAX];
189
- int ofd, ifd, fd[2];
190
- unsigned long total;
191
- static struct timeval prev_tv;
192
- struct average {
193
- unsigned long bytes;
194
- unsigned long time;
195
- } download[NR_AVERAGE] = { {0, 0}, };
196
- unsigned long avg_bytes, avg_time;
197
- int idx = 0;
198
-
199
- setup_sideband(sideband, me, fd, xd);
200
-
201
- ifd = fd[0];
202
- snprintf(tmpfile, sizeof(tmpfile),
203
- "%s/pack/tmp-XXXXXX", get_object_directory());
204
- ofd = mkstemp(tmpfile);
205
- if (ofd < 0)
206
- return error("unable to create temporary file %s", tmpfile);
207
-
208
- gettimeofday(&prev_tv, NULL);
209
- total = 0;
210
- avg_bytes = 0;
211
- avg_time = 0;
212
- while (1) {
213
- char buf[8192];
214
- ssize_t sz, wsz, pos;
215
- sz = read(ifd, buf, sizeof(buf));
216
- if (sz == 0)
217
- break;
218
- if (sz < 0) {
219
- if (errno != EINTR && errno != EAGAIN) {
220
- error("error reading pack (%s)", strerror(errno));
221
- close(ofd);
222
- unlink(tmpfile);
223
- return -1;
224
- }
225
- sz = 0;
226
- }
227
- pos = 0;
228
- while (pos < sz) {
229
- wsz = write(ofd, buf + pos, sz - pos);
230
- if (wsz < 0) {
231
- error("error writing pack (%s)",
232
- strerror(errno));
233
- close(ofd);
234
- unlink(tmpfile);
235
- return -1;
236
- }
237
- pos += wsz;
238
- }
239
- total += sz;
240
- if (!quiet) {
241
- static unsigned long last;
242
- struct timeval tv;
243
- unsigned long diff = total - last;
244
- /* not really "msecs", but a power-of-two millisec (1/1024th of a sec) */
245
- unsigned long msecs;
246
-
247
- gettimeofday(&tv, NULL);
248
- msecs = tv.tv_sec - prev_tv.tv_sec;
249
- msecs <<= 10;
250
- msecs += usec_to_binarymsec(tv.tv_usec - prev_tv.tv_usec);
251
-
252
- if (msecs > 500) {
253
- prev_tv = tv;
254
- last = total;
255
-
256
- /* Update averages ..*/
257
- avg_bytes += diff;
258
- avg_time += msecs;
259
- avg_bytes -= download[idx].bytes;
260
- avg_time -= download[idx].time;
261
- download[idx].bytes = diff;
262
- download[idx].time = msecs;
263
- idx++;
264
- if (idx >= NR_AVERAGE)
265
- idx = 0;
266
-
267
- fprintf(stderr, "%4lu.%03luMB (%lu kB/s) \r",
268
- total >> 20,
269
- 1000*((total >> 10) & 1023)>>10,
270
- avg_bytes / avg_time );
271
- }
272
- }
273
- }
274
- close(ofd);
275
- return finish_pack(tmpfile, me);
84
+ const char *argv[5] = { "index-pack", "--stdin", "--fix-thin",
85
+ quiet ? NULL : "-v", NULL };
86
+ return get_pack(xd, me, sideband, argv);
87
}
fetch-pack.c
-2
@@ -518,8 +518,6 @@ int main(int argc, char **argv)
518
}
519
if (!dest)
520
usage(fetch_pack_usage);
521
- if (keep_pack)
522
- use_thin_pack = 0;
521
pid = git_connect(fd, dest, exec);
522
if (pid < 0)
523
return 1;