| 1 | # frozen_string_literal: true |
| 2 | |
| 3 | require "open3" |
| 4 | require "fileutils" |
| 5 | require "tmpdir" |
| 6 | require "tempfile" |
| 7 | require "timeout" |
| 8 | require "base64" |
| 9 | |
| 10 | # The git mechanics of an import: mirror-clone a source repo in a sandbox, then |
| 11 | # push every ref (all branches and tags, full history, no rewrites) into an |
| 12 | # existing bare siGit repo. LFS objects are fetched when git-lfs is available, |
| 13 | # and clearly reported as skipped otherwise. |
| 14 | # |
| 15 | # This class is deliberately isolation-focused — every guardrail that keeps a |
| 16 | # hung or malicious remote from wedging a worker lives here: |
| 17 | # - the clone runs in a fresh temp dir that is always removed, |
| 18 | # - every git invocation runs with a hard timeout and is killed (whole process |
| 19 | # group) if it overruns, |
| 20 | # - `GIT_TERMINAL_PROMPT=0` makes auth failures fail fast instead of hanging, |
| 21 | # - the cloned size is capped, so one repo can't fill the disk, |
| 22 | # - credentials are passed via an HTTP header (not the URL, not the stored |
| 23 | # remote config) and scrubbed from any captured output before it is logged. |
| 24 | # |
| 25 | # It does not touch the database or create the destination repo — the caller |
| 26 | # (RepositoryImportJob) owns lifecycle, status, and cleanup. |
| 27 | class RepositoryImportService |
| 28 | class ImportError < StandardError; end |
| 29 | class SizeExceeded < ImportError; end |
| 30 | class CloneTimeout < ImportError; end |
| 31 | |
| 32 | # A single git call may not pin a worker forever. Overridable for very large |
| 33 | # legitimate repos via env. |
| 34 | DEFAULT_TIMEOUT = Integer(ENV.fetch("IMPORT_GIT_TIMEOUT", 1800)) # 30 min |
| 35 | |
| 36 | # Per-repo on-disk cap. GitHub reports size in KB; we also re-check the actual |
| 37 | # clone so URL imports (no reported size) are bounded too. |
| 38 | DEFAULT_MAX_SIZE_KB = Integer(ENV.fetch("IMPORT_MAX_REPO_SIZE_KB", 2_000_000)) # ~2 GB |
| 39 | |
| 40 | Result = Struct.new(:lfs_detected, :lfs_skipped, :size_kb, keyword_init: true) |
| 41 | |
| 42 | # Clones +source_url+ (with optional +credential+ token for private sources) |
| 43 | # and pushes all refs into the existing bare repo at +dest_path+. Yields a |
| 44 | # status symbol (:cloning, :pushing, :fetching_lfs) to +progress+ as it moves |
| 45 | # through the phases. Returns a Result. Raises ImportError (or a subclass) on |
| 46 | # failure; the caller is responsible for cleaning up +dest_path+. |
| 47 | def self.clone_and_push(source_url:, dest_path:, credential: nil, |
| 48 | max_size_kb: DEFAULT_MAX_SIZE_KB, timeout: DEFAULT_TIMEOUT, progress: nil) |
| 49 | new(source_url:, dest_path:, credential:, max_size_kb:, timeout:, progress:).call |
| 50 | end |
| 51 | |
| 52 | def initialize(source_url:, dest_path:, credential:, max_size_kb:, timeout:, progress:) |
| 53 | @source_url = source_url |
| 54 | @dest_path = dest_path |
| 55 | @credential = credential |
| 56 | @max_size_kb = max_size_kb |
| 57 | @timeout = timeout |
| 58 | @progress = progress |
| 59 | end |
| 60 | |
| 61 | def call |
| 62 | Dir.mktmpdir("sigit-import-") do |workdir| |
| 63 | mirror = File.join(workdir, "source.git") |
| 64 | |
| 65 | emit(:cloning) |
| 66 | clone_mirror(mirror) |
| 67 | enforce_size!(mirror) |
| 68 | |
| 69 | emit(:pushing) |
| 70 | push_all_refs(mirror) |
| 71 | |
| 72 | emit(:fetching_lfs) |
| 73 | lfs = handle_lfs(mirror) |
| 74 | |
| 75 | Result.new( |
| 76 | lfs_detected: lfs[:detected], |
| 77 | lfs_skipped: lfs[:skipped], |
| 78 | size_kb: dir_size_kb(@dest_path) |
| 79 | ) |
| 80 | end |
| 81 | end |
| 82 | |
| 83 | private |
| 84 | |
| 85 | def emit(phase) |
| 86 | @progress&.call(phase) |
| 87 | end |
| 88 | |
| 89 | def clone_mirror(dest) |
| 90 | args = [ "git" ] |
| 91 | args += credential_config |
| 92 | args += [ "clone", "--mirror", @source_url, dest ] |
| 93 | |
| 94 | ok, output = run(args, timeout: @timeout) |
| 95 | raise ImportError, "Clone failed: #{first_error_line(output)}" unless ok |
| 96 | end |
| 97 | |
| 98 | # Push every ref verbatim (heads + tags), preserving history exactly. The |
| 99 | # refspec form (rather than `--mirror`) keeps siGit's internal refs, if any, |
| 100 | # from being pruned, and never rewrites commits. |
| 101 | def push_all_refs(mirror) |
| 102 | ok, output = run([ "git", "-C", mirror, "push", @dest_path, "+refs/heads/*:refs/heads/*", "+refs/tags/*:refs/tags/*" ], |
| 103 | timeout: @timeout) |
| 104 | raise ImportError, "Push failed: #{first_error_line(output)}" unless ok |
| 105 | end |
| 106 | |
| 107 | # Best-effort LFS handling. Detection is cheap (does any ref carry a |
| 108 | # `.gitattributes` with `filter=lfs`); fetching requires the git-lfs binary. |
| 109 | # When it's missing we don't fail — the LFS *pointer* files are ordinary git |
| 110 | # blobs and are already imported with the history; only the large binaries are |
| 111 | # skipped, which we report so the UI can say so plainly. |
| 112 | def handle_lfs(mirror) |
| 113 | return { detected: false, skipped: false } unless lfs_used?(mirror) |
| 114 | return { detected: true, skipped: true } unless lfs_available? |
| 115 | |
| 116 | ok, _ = run([ "git", "-C", mirror, "lfs", "fetch", "--all" ], timeout: @timeout) |
| 117 | return { detected: true, skipped: true } unless ok |
| 118 | |
| 119 | # Copy the fetched object store into the destination so siGit holds the |
| 120 | # binaries too. Non-fatal if it doesn't land. |
| 121 | src_lfs = File.join(mirror, "lfs") |
| 122 | if Dir.exist?(src_lfs) |
| 123 | FileUtils.cp_r(src_lfs, File.join(@dest_path, "lfs")) |
| 124 | { detected: true, skipped: false } |
| 125 | else |
| 126 | { detected: true, skipped: true } |
| 127 | end |
| 128 | rescue StandardError => e |
| 129 | Rails.logger.warn("LFS import step failed (continuing): #{e.class}") |
| 130 | { detected: true, skipped: true } |
| 131 | end |
| 132 | |
| 133 | def lfs_used?(mirror) |
| 134 | # Check the tip of every branch for a .gitattributes that enables the lfs |
| 135 | # filter. Cheap and covers the overwhelmingly common case. The blob read |
| 136 | # goes through GitRepositoryService (the single git-read layer) rather than |
| 137 | # shelling out again here. |
| 138 | ok, refs = run([ "git", "-C", mirror, "for-each-ref", "--format=%(refname)", "refs/heads/" ], timeout: 30) |
| 139 | return false unless ok |
| 140 | |
| 141 | refs.each_line.map(&:strip).reject(&:empty?).any? do |ref| |
| 142 | content = GitRepositoryService.file_content(mirror, ref, ".gitattributes") |
| 143 | content&.include?("filter=lfs") |
| 144 | end |
| 145 | end |
| 146 | |
| 147 | def lfs_available? |
| 148 | _o, _e, st = Open3.capture3("git", "lfs", "version") |
| 149 | st.success? |
| 150 | rescue StandardError |
| 151 | false |
| 152 | end |
| 153 | |
| 154 | def enforce_size!(mirror) |
| 155 | size = dir_size_kb(mirror) |
| 156 | return if size.nil? || size <= @max_size_kb |
| 157 | |
| 158 | raise SizeExceeded, |
| 159 | "Repository is #{(size / 1024.0).round} MB, over the #{(@max_size_kb / 1024.0).round} MB import limit." |
| 160 | end |
| 161 | |
| 162 | # ── process execution ──────────────────────────────────────────────────── |
| 163 | |
| 164 | # Runs +cmd+ with a hard timeout. On overrun the whole process group is killed |
| 165 | # so a hung `git` (or a child it spawned) can't linger. Returns |
| 166 | # [success_bool, output_string]; output has any credential scrubbed out. |
| 167 | def run(cmd, timeout:) |
| 168 | out = Tempfile.new("sigit-git-out") |
| 169 | begin |
| 170 | pid = Process.spawn(git_env, *cmd, out: out.path, err: [ :child, :out ], pgroup: true) |
| 171 | begin |
| 172 | Timeout.timeout(timeout) { Process.wait(pid) } |
| 173 | rescue Timeout::Error |
| 174 | kill_group(pid) |
| 175 | raise CloneTimeout, "git operation exceeded #{timeout}s and was terminated." |
| 176 | end |
| 177 | success = $?.success? |
| 178 | [ success, scrub(File.read(out.path)) ] |
| 179 | ensure |
| 180 | out.close! |
| 181 | end |
| 182 | end |
| 183 | |
| 184 | def kill_group(pid) |
| 185 | begin |
| 186 | Process.kill("TERM", -pid) |
| 187 | sleep 2 |
| 188 | Process.kill("KILL", -pid) |
| 189 | rescue Errno::ESRCH, Errno::EPERM |
| 190 | # already gone |
| 191 | end |
| 192 | |
| 193 | begin |
| 194 | Process.wait(pid) |
| 195 | rescue Errno::ECHILD |
| 196 | nil |
| 197 | end |
| 198 | end |
| 199 | |
| 200 | def git_env |
| 201 | { |
| 202 | # Never block on an interactive auth/host prompt — fail fast instead. |
| 203 | "GIT_TERMINAL_PROMPT" => "0", |
| 204 | "GIT_ASKPASS" => "/bin/echo", |
| 205 | "GCM_INTERACTIVE" => "never", |
| 206 | # Don't try to download LFS blobs during the mirror clone; we handle LFS |
| 207 | # explicitly afterwards. |
| 208 | "GIT_LFS_SKIP_SMUDGE" => "1" |
| 209 | } |
| 210 | end |
| 211 | |
| 212 | # Auth for a private source, injected as an HTTP header via `-c` so the token |
| 213 | # never lands in the URL, the stored remote config, or the clone on disk. |
| 214 | def credential_config |
| 215 | return [] if @credential.blank? |
| 216 | |
| 217 | basic = Base64.strict_encode64("x-access-token:#{@credential}") |
| 218 | [ "-c", "http.extraHeader=Authorization: Basic #{basic}" ] |
| 219 | end |
| 220 | |
| 221 | # Remove the credential from any captured text before it can be logged or |
| 222 | # persisted to the import's error_message. |
| 223 | def scrub(text) |
| 224 | return text if @credential.blank? |
| 225 | |
| 226 | text.to_s.gsub(@credential, "***") |
| 227 | end |
| 228 | |
| 229 | def first_error_line(output) |
| 230 | line = output.to_s.each_line.map(&:strip).reject(&:empty?).last |
| 231 | line.presence || "unknown error" |
| 232 | end |
| 233 | |
| 234 | def dir_size_kb(path) |
| 235 | return nil unless Dir.exist?(path) |
| 236 | |
| 237 | out, _e, st = Open3.capture3("du", "-sk", path) |
| 238 | st.success? ? out.split("\t").first.to_i : nil |
| 239 | end |
| 240 | end |