@matheusbsbs / matheus-bagliano / commits / 7aeaa2fc0a
Fork 0

remote-helpers: add testgit helper

Currently the remote helper infrastructure is only used by the curl helper, which does not give a good impression of how remote helpers can be used to interact with foreign repositories. Since implementing such a helper is non-trivial it would be good to have at least one easy-to-follow example demonstrating how to implement a helper that interacts with a foreign vcs using fast-import/fast-export. The testgit helper can be used to interact with remote git repositories by prefixing the url with "testgit::". Signed-off-by: Junio C Hamano <gitster@pobox.com>

Sverre Rabbelier committed Mar 29, 2010 at 11:48 UTC 7aeaa2fc0abbf439534769e15b3a59a5814cc3d1
7 files changed +456
.gitignore
+1
@@ -112,6 +112,7 @@
112 /git-remote-https
113 /git-remote-ftp
114 /git-remote-ftps
115 +/git-remote-testgit
116 /git-repack
117 /git-replace
118 /git-repo-config
Makefile
+2
@@ -366,6 +366,8 @@ SCRIPT_PERL += git-relink.perl
366 SCRIPT_PERL += git-send-email.perl
367 SCRIPT_PERL += git-svn.perl
368
369 +SCRIPT_PYTHON += git-remote-testgit.py
370 +
371 SCRIPTS = $(patsubst %.sh,%,$(SCRIPT_SH)) \
372 $(patsubst %.perl,%,$(SCRIPT_PERL)) \
373 $(patsubst %.py,%,$(SCRIPT_PYTHON)) \
git-remote-testgit.py new
+233
@@ -0,0 +1,233 @@
1 +#!/usr/bin/env python
2 +
3 +import hashlib
4 +import sys
5 +
6 +from git_remote_helpers.util import die, debug, warn
7 +from git_remote_helpers.git.repo import GitRepo
8 +from git_remote_helpers.git.exporter import GitExporter
9 +from git_remote_helpers.git.importer import GitImporter
10 +from git_remote_helpers.git.non_local import NonLocalGit
11 +
12 +def get_repo(alias, url):
13 + """Returns a git repository object initialized for usage.
14 + """
15 +
16 + repo = GitRepo(url)
17 + repo.get_revs()
18 + repo.get_head()
19 +
20 + hasher = hashlib.sha1()
21 + hasher.update(repo.path)
22 + repo.hash = hasher.hexdigest()
23 +
24 + repo.get_base_path = lambda base: os.path.join(
25 + base, 'info', 'fast-import', repo.hash)
26 +
27 + prefix = 'refs/testgit/%s/' % alias
28 + debug("prefix: '%s'", prefix)
29 +
30 + repo.gitdir = ""
31 + repo.alias = alias
32 + repo.prefix = prefix
33 +
34 + repo.exporter = GitExporter(repo)
35 + repo.importer = GitImporter(repo)
36 + repo.non_local = NonLocalGit(repo)
37 +
38 + return repo
39 +
40 +
41 +def local_repo(repo, path):
42 + """Returns a git repository object initalized for usage.
43 + """
44 +
45 + local = GitRepo(path)
46 +
47 + local.non_local = None
48 + local.gitdir = repo.gitdir
49 + local.alias = repo.alias
50 + local.prefix = repo.prefix
51 + local.hash = repo.hash
52 + local.get_base_path = repo.get_base_path
53 + local.exporter = GitExporter(local)
54 + local.importer = GitImporter(local)
55 +
56 + return local
57 +
58 +
59 +def do_capabilities(repo, args):
60 + """Prints the supported capabilities.
61 + """
62 +
63 + print "import"
64 + print "export"
65 + print "gitdir"
66 + print "refspec refs/heads/*:%s*" % repo.prefix
67 +
68 + print # end capabilities
69 +
70 +
71 +def do_list(repo, args):
72 + """Lists all known references.
73 +
74 + Bug: This will always set the remote head to master for non-local
75 + repositories, since we have no way of determining what the remote
76 + head is at clone time.
77 + """
78 +
79 + for ref in repo.revs:
80 + debug("? refs/heads/%s", ref)
81 + print "? refs/heads/%s" % ref
82 +
83 + if repo.head:
84 + debug("@refs/heads/%s HEAD" % repo.head)
85 + print "@refs/heads/%s HEAD" % repo.head
86 + else:
87 + debug("@refs/heads/master HEAD")
88 + print "@refs/heads/master HEAD"
89 +
90 + print # end list
91 +
92 +
93 +def update_local_repo(repo):
94 + """Updates (or clones) a local repo.
95 + """
96 +
97 + if repo.local:
98 + return repo
99 +
100 + path = repo.non_local.clone(repo.gitdir)
101 + repo.non_local.update(repo.gitdir)
102 + repo = local_repo(repo, path)
103 + return repo
104 +
105 +
106 +def do_import(repo, args):
107 + """Exports a fast-import stream from testgit for git to import.
108 + """
109 +
110 + if len(args) != 1:
111 + die("Import needs exactly one ref")
112 +
113 + if not repo.gitdir:
114 + die("Need gitdir to import")
115 +
116 + repo = update_local_repo(repo)
117 + repo.exporter.export_repo(repo.gitdir)
118 +
119 +
120 +def do_export(repo, args):
121 + """Imports a fast-import stream from git to testgit.
122 + """
123 +
124 + if not repo.gitdir:
125 + die("Need gitdir to export")
126 +
127 + dirname = repo.get_base_path(repo.gitdir)
128 +
129 + if not os.path.exists(dirname):
130 + os.makedirs(dirname)
131 +
132 + path = os.path.join(dirname, 'testgit.marks')
133 + print path
134 + print path if os.path.exists(path) else ""
135 + sys.stdout.flush()
136 +
137 + update_local_repo(repo)
138 + repo.importer.do_import(repo.gitdir)
139 + repo.non_local.push(repo.gitdir)
140 +
141 +
142 +def do_gitdir(repo, args):
143 + """Stores the location of the gitdir.
144 + """
145 +
146 + if not args:
147 + die("gitdir needs an argument")
148 +
149 + repo.gitdir = ' '.join(args)
150 +
151 +
152 +COMMANDS = {
153 + 'capabilities': do_capabilities,
154 + 'list': do_list,
155 + 'import': do_import,
156 + 'export': do_export,
157 + 'gitdir': do_gitdir,
158 +}
159 +
160 +
161 +def sanitize(value):
162 + """Cleans up the url.
163 + """
164 +
165 + if value.startswith('testgit::'):
166 + value = value[9:]
167 +
168 + return value
169 +
170 +
171 +def read_one_line(repo):
172 + """Reads and processes one command.
173 + """
174 +
175 + line = sys.stdin.readline()
176 +
177 + cmdline = line
178 +
179 + if not cmdline:
180 + warn("Unexpected EOF")
181 + return False
182 +
183 + cmdline = cmdline.strip().split()
184 + if not cmdline:
185 + # Blank line means we're about to quit
186 + return False
187 +
188 + cmd = cmdline.pop(0)
189 + debug("Got command '%s' with args '%s'", cmd, ' '.join(cmdline))
190 +
191 + if cmd not in COMMANDS:
192 + die("Unknown command, %s", cmd)
193 +
194 + func = COMMANDS[cmd]
195 + func(repo, cmdline)
196 + sys.stdout.flush()
197 +
198 + return True
199 +
200 +
201 +def main(args):
202 + """Starts a new remote helper for the specified repository.
203 + """
204 +
205 + if len(args) != 3:
206 + die("Expecting exactly three arguments.")
207 + sys.exit(1)
208 +
209 + if os.getenv("GIT_DEBUG_TESTGIT"):
210 + import git_remote_helpers.util
211 + git_remote_helpers.util.DEBUG = True
212 +
213 + alias = sanitize(args[1])
214 + url = sanitize(args[2])
215 +
216 + if not alias.isalnum():
217 + warn("non-alnum alias '%s'", alias)
218 + alias = "tmp"
219 +
220 + args[1] = alias
221 + args[2] = url
222 +
223 + repo = get_repo(alias, url)
224 +
225 + debug("Got arguments %s", args[1:])
226 +
227 + more = True
228 +
229 + while (more):
230 + more = read_one_line(repo)
231 +
232 +if __name__ == '__main__':
233 + sys.exit(main(sys.argv))
git_remote_helpers/git/exporter.py new
+51
@@ -0,0 +1,51 @@
1 +import os
2 +import subprocess
3 +import sys
4 +
5 +
6 +class GitExporter(object):
7 + """An exporter for testgit repositories.
8 +
9 + The exporter simply delegates to git fast-export.
10 + """
11 +
12 + def __init__(self, repo):
13 + """Creates a new exporter for the specified repo.
14 + """
15 +
16 + self.repo = repo
17 +
18 + def export_repo(self, base):
19 + """Exports a fast-export stream for the given directory.
20 +
21 + Simply delegates to git fast-epxort and pipes it through sed
22 + to make the refs show up under the prefix rather than the
23 + default refs/heads. This is to demonstrate how the export
24 + data can be stored under it's own ref (using the refspec
25 + capability).
26 + """
27 +
28 + dirname = self.repo.get_base_path(base)
29 + path = os.path.abspath(os.path.join(dirname, 'testgit.marks'))
30 +
31 + if not os.path.exists(dirname):
32 + os.makedirs(dirname)
33 +
34 + print "feature relative-marks"
35 + if os.path.exists(os.path.join(dirname, 'git.marks')):
36 + print "feature import-marks=%s/git.marks" % self.repo.hash
37 + print "feature export-marks=%s/git.marks" % self.repo.hash
38 + sys.stdout.flush()
39 +
40 + args = ["git", "--git-dir=" + self.repo.gitpath, "fast-export", "--export-marks=" + path]
41 +
42 + if os.path.exists(path):
43 + args.append("--import-marks=" + path)
44 +
45 + args.append("HEAD")
46 +
47 + p1 = subprocess.Popen(args, stdout=subprocess.PIPE)
48 +
49 + args = ["sed", "s_refs/heads/_" + self.repo.prefix + "_g"]
50 +
51 + subprocess.check_call(args, stdin=p1.stdout)
git_remote_helpers/git/importer.py new
+38
@@ -0,0 +1,38 @@
1 +import os
2 +import subprocess
3 +
4 +
5 +class GitImporter(object):
6 + """An importer for testgit repositories.
7 +
8 + This importer simply delegates to git fast-import.
9 + """
10 +
11 + def __init__(self, repo):
12 + """Creates a new importer for the specified repo.
13 + """
14 +
15 + self.repo = repo
16 +
17 + def do_import(self, base):
18 + """Imports a fast-import stream to the given directory.
19 +
20 + Simply delegates to git fast-import.
21 + """
22 +
23 + dirname = self.repo.get_base_path(base)
24 + if self.repo.local:
25 + gitdir = self.repo.gitpath
26 + else:
27 + gitdir = os.path.abspath(os.path.join(dirname, '.git'))
28 + path = os.path.abspath(os.path.join(dirname, 'git.marks'))
29 +
30 + if not os.path.exists(dirname):
31 + os.makedirs(dirname)
32 +
33 + args = ["git", "--git-dir=" + gitdir, "fast-import", "--quiet", "--export-marks=" + path]
34 +
35 + if os.path.exists(path):
36 + args.append("--import-marks=" + path)
37 +
38 + subprocess.check_call(args)
git_remote_helpers/git/non_local.py new
+61
@@ -0,0 +1,61 @@
1 +import os
2 +import subprocess
3 +
4 +from git_remote_helpers.util import die, warn
5 +
6 +
7 +class NonLocalGit(object):
8 + """Handler to interact with non-local repos.
9 + """
10 +
11 + def __init__(self, repo):
12 + """Creates a new non-local handler for the specified repo.
13 + """
14 +
15 + self.repo = repo
16 +
17 + def clone(self, base):
18 + """Clones the non-local repo to base.
19 +
20 + Does nothing if a clone already exists.
21 + """
22 +
23 + path = os.path.join(self.repo.get_base_path(base), '.git')
24 +
25 + # already cloned
26 + if os.path.exists(path):
27 + return path
28 +
29 + os.makedirs(path)
30 + args = ["git", "clone", "--bare", "--quiet", self.repo.gitpath, path]
31 +
32 + subprocess.check_call(args)
33 +
34 + return path
35 +
36 + def update(self, base):
37 + """Updates checkout of the non-local repo in base.
38 + """
39 +
40 + path = os.path.join(self.repo.get_base_path(base), '.git')
41 +
42 + if not os.path.exists(path):
43 + die("could not find repo at %s", path)
44 +
45 + args = ["git", "--git-dir=" + path, "fetch", "--quiet", self.repo.gitpath]
46 + subprocess.check_call(args)
47 +
48 + args = ["git", "--git-dir=" + path, "update-ref", "refs/heads/master", "FETCH_HEAD"]
49 + subprocess.check_call(args)
50 +
51 + def push(self, base):
52 + """Pushes from the non-local repo to base.
53 + """
54 +
55 + path = os.path.join(self.repo.get_base_path(base), '.git')
56 +
57 + if not os.path.exists(path):
58 + die("could not find repo at %s", path)
59 +
60 + args = ["git", "--git-dir=" + path, "push", "--quiet", self.repo.gitpath]
61 + subprocess.check_call(args)
git_remote_helpers/git/repo.py new
+70
@@ -0,0 +1,70 @@
1 +import os
2 +import subprocess
3 +
4 +def sanitize(rev, sep='\t'):
5 + """Converts a for-each-ref line to a name/value pair.
6 + """
7 +
8 + splitrev = rev.split(sep)
9 + branchval = splitrev[0]
10 + branchname = splitrev[1].strip()
11 + if branchname.startswith("refs/heads/"):
12 + branchname = branchname[11:]
13 +
14 + return branchname, branchval
15 +
16 +def is_remote(url):
17 + """Checks whether the specified value is a remote url.
18 + """
19 +
20 + prefixes = ["http", "file", "git"]
21 +
22 + return any(url.startswith(i) for i in prefixes)
23 +
24 +class GitRepo(object):
25 + """Repo object representing a repo.
26 + """
27 +
28 + def __init__(self, path):
29 + """Initializes a new repo at the given path.
30 + """
31 +
32 + self.path = path
33 + self.head = None
34 + self.revmap = {}
35 + self.local = not is_remote(self.path)
36 +
37 + if(self.path.endswith('.git')):
38 + self.gitpath = self.path
39 + else:
40 + self.gitpath = os.path.join(self.path, '.git')
41 +
42 + if self.local and not os.path.exists(self.gitpath):
43 + os.makedirs(self.gitpath)
44 +
45 + def get_revs(self):
46 + """Fetches all revs from the remote.
47 + """
48 +
49 + args = ["git", "ls-remote", self.gitpath]
50 + path = ".cached_revs"
51 + ofile = open(path, "w")
52 +
53 + subprocess.check_call(args, stdout=ofile)
54 + output = open(path).readlines()
55 + self.revmap = dict(sanitize(i) for i in output)
56 + if "HEAD" in self.revmap:
57 + del self.revmap["HEAD"]
58 + self.revs = self.revmap.keys()
59 + ofile.close()
60 +
61 + def get_head(self):
62 + """Determines the head of a local repo.
63 + """
64 +
65 + if not self.local:
66 + return
67 +
68 + path = os.path.join(self.gitpath, "HEAD")
69 + head = open(path).readline()
70 + self.head, _ = sanitize(head, ' ')