@matheusbsbs / matheus-bagliano / commits / d9c20ba13d
Fork 0

enhance clone and fetch -k experience

Now that index-pack can be streamed with a pack, it is probably a good idea to use it directly instead of creating a temporary file and running index-pack afterwards. This way index-pack can abort early whenever a corruption is encountered even if the pack has not been fully downloaded, it can display a progress percentage as it knows how much to expects, and it is a bit faster since the pack indexing is partially done as data is received. Using fetch -k doesn't need to disable thin pack generation on the remote end either. Signed-off-by: Nicolas Pitre <nico@cam.org> Signed-off-by: Junio C Hamano <junkio@cox.net>

Nicolas Pitre committed Oct 27, 2006 at 15:42 UTC d9c20ba13dfca737373ba466d2a718cafdc17f92
2 files changed +15 -206
fetch-clone.c
+15 -204
@@ -3,97 +3,6 @@
3 #include "pkt-line.h"
4 #include "sideband.h"
5 #include <sys/wait.h>
6 -#include <sys/time.h>
7 -
8 -static int finish_pack(const char *pack_tmp_name, const char *me)
9 -{
10 - int pipe_fd[2];
11 - pid_t pid;
12 - char idx[PATH_MAX];
13 - char final[PATH_MAX];
14 - char hash[41];
15 - unsigned char sha1[20];
16 - char *cp;
17 - int err = 0;
18 -
19 - if (pipe(pipe_fd) < 0)
20 - die("%s: unable to set up pipe", me);
21 -
22 - strcpy(idx, pack_tmp_name); /* ".git/objects/pack-XXXXXX" */
23 - cp = strrchr(idx, '/');
24 - memcpy(cp, "/pidx", 5);
25 -
26 - pid = fork();
27 - if (pid < 0)
28 - die("%s: unable to fork off git-index-pack", me);
29 - if (!pid) {
30 - close(0);
31 - dup2(pipe_fd[1], 1);
32 - close(pipe_fd[0]);
33 - close(pipe_fd[1]);
34 - execl_git_cmd("index-pack", "-o", idx, pack_tmp_name, NULL);
35 - error("cannot exec git-index-pack <%s> <%s>",
36 - idx, pack_tmp_name);
37 - exit(1);
38 - }
39 - close(pipe_fd[1]);
40 - if (read(pipe_fd[0], hash, 40) != 40) {
41 - error("%s: unable to read from git-index-pack", me);
42 - err = 1;
43 - }
44 - close(pipe_fd[0]);
45 -
46 - for (;;) {
47 - int status, code;
48 -
49 - if (waitpid(pid, &status, 0) < 0) {
50 - if (errno == EINTR)
51 - continue;
52 - error("waitpid failed (%s)", strerror(errno));
53 - goto error_die;
54 - }
55 - if (WIFSIGNALED(status)) {
56 - int sig = WTERMSIG(status);
57 - error("git-index-pack died of signal %d", sig);
58 - goto error_die;
59 - }
60 - if (!WIFEXITED(status)) {
61 - error("git-index-pack died of unnatural causes %d",
62 - status);
63 - goto error_die;
64 - }
65 - code = WEXITSTATUS(status);
66 - if (code) {
67 - error("git-index-pack died with error code %d", code);
68 - goto error_die;
69 - }
70 - if (err)
71 - goto error_die;
72 - break;
73 - }
74 - hash[40] = 0;
75 - if (get_sha1_hex(hash, sha1)) {
76 - error("git-index-pack reported nonsense '%s'", hash);
77 - goto error_die;
78 - }
79 - /* Now we have pack in pack_tmp_name[], and
80 - * idx in idx[]; rename them to their final names.
81 - */
82 - snprintf(final, sizeof(final),
83 - "%s/pack/pack-%s.pack", get_object_directory(), hash);
84 - move_temp_to_file(pack_tmp_name, final);
85 - chmod(final, 0444);
86 - snprintf(final, sizeof(final),
87 - "%s/pack/pack-%s.idx", get_object_directory(), hash);
88 - move_temp_to_file(idx, final);
89 - chmod(final, 0444);
90 - return 0;
91 -
92 - error_die:
93 - unlink(idx);
94 - unlink(pack_tmp_name);
95 - exit(1);
96 -}
6
7 static pid_t setup_sideband(int sideband, const char *me, int fd[2], int xd[2])
8 {
@@ -128,7 +37,7 @@ static pid_t setup_sideband(int sideband, const char *me, int fd[2], int xd[2])
37 return side_pid;
38 }
39
131 -int receive_unpack_pack(int xd[2], const char *me, int quiet, int sideband)
40 +static int get_pack(int xd[2], const char *me, int sideband, const char **argv)
41 {
42 int status;
43 pid_t pid, side_pid;
@@ -142,135 +51,37 @@ int receive_unpack_pack(int xd[2], const char *me, int quiet, int sideband)
51 dup2(fd[0], 0);
52 close(fd[0]);
53 close(fd[1]);
145 - execl_git_cmd("unpack-objects", quiet ? "-q" : NULL, NULL);
146 - die("git-unpack-objects exec failed");
54 + execv_git_cmd(argv);
55 + die("%s exec failed", argv[0]);
56 }
57 close(fd[0]);
58 close(fd[1]);
59 while (waitpid(pid, &status, 0) < 0) {
60 if (errno != EINTR)
152 - die("waiting for git-unpack-objects: %s",
153 - strerror(errno));
61 + die("waiting for %s: %s", argv[0], strerror(errno));
62 }
63 if (WIFEXITED(status)) {
64 int code = WEXITSTATUS(status);
65 if (code)
158 - die("git-unpack-objects died with error code %d",
159 - code);
66 + die("%s died with error code %d", argv[0], code);
67 return 0;
68 }
69 if (WIFSIGNALED(status)) {
70 int sig = WTERMSIG(status);
164 - die("git-unpack-objects died of signal %d", sig);
71 + die("%s died of signal %d", argv[0], sig);
72 }
166 - die("git-unpack-objects died of unnatural causes %d", status);
73 + die("%s died of unnatural causes %d", argv[0], status);
74 }
75
169 -/*
170 - * We average out the download speed over this many "events", where
171 - * an event is a minimum of about half a second. That way, we get
172 - * a reasonably stable number.
173 - */
174 -#define NR_AVERAGE (4)
175 -
176 -/*
177 - * A "binary msec" is a power-of-two-msec, aka 1/1024th of a second.
178 - * Keeping the time in that format means that "bytes / msecs" means
179 - * the same as kB/s (modulo rounding).
180 - *
181 - * 1000512 is a magic number (usecs in a second, rounded up by half
182 - * of 1024, to make "rounding" come out right ;)
183 - */
184 -#define usec_to_binarymsec(x) ((int)(x) / (1000512 >> 10))
76 +int receive_unpack_pack(int xd[2], const char *me, int quiet, int sideband)
77 +{
78 + const char *argv[3] = { "unpack-objects", quiet ? "-q" : NULL, NULL };
79 + return get_pack(xd, me, sideband, argv);
80 +}
81
82 int receive_keep_pack(int xd[2], const char *me, int quiet, int sideband)
83 {
188 - char tmpfile[PATH_MAX];
189 - int ofd, ifd, fd[2];
190 - unsigned long total;
191 - static struct timeval prev_tv;
192 - struct average {
193 - unsigned long bytes;
194 - unsigned long time;
195 - } download[NR_AVERAGE] = { {0, 0}, };
196 - unsigned long avg_bytes, avg_time;
197 - int idx = 0;
198 -
199 - setup_sideband(sideband, me, fd, xd);
200 -
201 - ifd = fd[0];
202 - snprintf(tmpfile, sizeof(tmpfile),
203 - "%s/pack/tmp-XXXXXX", get_object_directory());
204 - ofd = mkstemp(tmpfile);
205 - if (ofd < 0)
206 - return error("unable to create temporary file %s", tmpfile);
207 -
208 - gettimeofday(&prev_tv, NULL);
209 - total = 0;
210 - avg_bytes = 0;
211 - avg_time = 0;
212 - while (1) {
213 - char buf[8192];
214 - ssize_t sz, wsz, pos;
215 - sz = read(ifd, buf, sizeof(buf));
216 - if (sz == 0)
217 - break;
218 - if (sz < 0) {
219 - if (errno != EINTR && errno != EAGAIN) {
220 - error("error reading pack (%s)", strerror(errno));
221 - close(ofd);
222 - unlink(tmpfile);
223 - return -1;
224 - }
225 - sz = 0;
226 - }
227 - pos = 0;
228 - while (pos < sz) {
229 - wsz = write(ofd, buf + pos, sz - pos);
230 - if (wsz < 0) {
231 - error("error writing pack (%s)",
232 - strerror(errno));
233 - close(ofd);
234 - unlink(tmpfile);
235 - return -1;
236 - }
237 - pos += wsz;
238 - }
239 - total += sz;
240 - if (!quiet) {
241 - static unsigned long last;
242 - struct timeval tv;
243 - unsigned long diff = total - last;
244 - /* not really "msecs", but a power-of-two millisec (1/1024th of a sec) */
245 - unsigned long msecs;
246 -
247 - gettimeofday(&tv, NULL);
248 - msecs = tv.tv_sec - prev_tv.tv_sec;
249 - msecs <<= 10;
250 - msecs += usec_to_binarymsec(tv.tv_usec - prev_tv.tv_usec);
251 -
252 - if (msecs > 500) {
253 - prev_tv = tv;
254 - last = total;
255 -
256 - /* Update averages ..*/
257 - avg_bytes += diff;
258 - avg_time += msecs;
259 - avg_bytes -= download[idx].bytes;
260 - avg_time -= download[idx].time;
261 - download[idx].bytes = diff;
262 - download[idx].time = msecs;
263 - idx++;
264 - if (idx >= NR_AVERAGE)
265 - idx = 0;
266 -
267 - fprintf(stderr, "%4lu.%03luMB (%lu kB/s) \r",
268 - total >> 20,
269 - 1000*((total >> 10) & 1023)>>10,
270 - avg_bytes / avg_time );
271 - }
272 - }
273 - }
274 - close(ofd);
275 - return finish_pack(tmpfile, me);
84 + const char *argv[5] = { "index-pack", "--stdin", "--fix-thin",
85 + quiet ? NULL : "-v", NULL };
86 + return get_pack(xd, me, sideband, argv);
87 }
fetch-pack.c
-2
@@ -518,8 +518,6 @@ int main(int argc, char **argv)
518 }
519 if (!dest)
520 usage(fetch_pack_usage);
521 - if (keep_pack)
522 - use_thin_pack = 0;
521 pid = git_connect(fd, dest, exec);
522 if (pid < 0)
523 return 1;