Add one-click "Import from GitHub" (mirror + migrate)

index eb1c6d1..7389b6c 100644 --- a/.env.example +++ b/.env.example @@ -87,6 +87,45 @@ STRIPE_WEBHOOK_SECRET=whsec_your-webhook-signing-secret # SIGITSI_URL=https://sigit.si +# ----------------------------------------------------------------------------- +# Import from GitHub (mirror + migrate) +# A GitHub OAuth App used ONLY for importing repos — separate from the smbCloud +# "Continue with GitHub" sign-in above. Register one at +# https://github.com/settings/developers with the callback: +# <SIGITSI_URL>/import/github/callback +# Without these, import-by-URL (public repos) still works; GitHub repo listing / +# private import does not. +# ----------------------------------------------------------------------------- + +# GITHUB_IMPORT_CLIENT_ID=your-github-oauth-app-client-id +# GITHUB_IMPORT_CLIENT_SECRET=your-github-oauth-app-client-secret + +# Shared secret for the GitHub push webhook (POST /webhooks/github) that triggers +# immediate mirror syncs. Set the same value in the repo/org webhook config. +# GITHUB_WEBHOOK_SECRET=a-long-random-string + +# Import guardrails (all optional; sensible defaults shown). +# IMPORT_MAX_REPO_SIZE_KB=2000000 # per-repo cap (~2 GB) +# IMPORT_MAX_USER_VOLUME_KB=10000000 # per-user in-flight cap (~10 GB) +# IMPORT_MAX_ACTIVE_PER_USER=10 # concurrent imports per user +# IMPORT_GIT_TIMEOUT=1800 # seconds a single clone/push may run +# MIRROR_SYNC_TIMEOUT=900 # seconds a single mirror fetch may run +# MIRROR_SYNC_INTERVAL_SECONDS=3600 # how stale a mirror may get before re-sync + + +# ----------------------------------------------------------------------------- +# Active Record Encryption (required in production) +# Encrypts tokens at rest: the GitHub import OAuth token and any private mirror +# upstream credential. Generate a set with: bin/rails db:encryption:init +# In development/test these are derived from SECRET_KEY_BASE if unset (never used +# for real user data). +# ----------------------------------------------------------------------------- + +# ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY= +# ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY= +# ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT= + + # ----------------------------------------------------------------------------- # Rails # -----------------------------------------------------------------------------

index 6bae39c..4c5fe6c 100644 --- a/app/controllers/git_http_controller.rb +++ b/app/controllers/git_http_controller.rb @@ -24,9 +24,10 @@ class GitHttpController < ActionController::API def info_refs service = params[:service] return head(:forbidden) unless SERVICES.include?(service) + return if reject_mirror_push!(service) return unless authorize!(service) - advertise = git_run([service.delete_prefix("git-"), "--stateless-rpc", "--advertise-refs", @repo.disk_path]) + advertise = git_run([ service.delete_prefix("git-"), "--stateless-rpc", "--advertise-refs", @repo.disk_path ]) return head(:internal_server_error) if advertise.nil? no_cache @@ -42,17 +43,37 @@ class GitHttpController < ActionController::API # POST /:user/:repo.git/git-receive-pack (push) def receive_pack + return if reject_mirror_push!("git-receive-pack") return unless authorize!("git-receive-pack") rpc("receive-pack", "application/x-git-receive-pack-result") end private + # Mirrors are read-only: their upstream is the source of truth, so accepting a + # push would let siGit silently diverge. Reject any receive-pack (push) with a + # message the user actually sees — a git `ERR` pkt-line in the ref + # advertisement, which the client prints as `fatal: remote error: …`. Returns + # true when it handled (rejected) the request. Applies to everyone, including + # the owner, until the mirror is detached. + def reject_mirror_push!(service) + return false unless service == "git-receive-pack" + return false unless @repo&.mirror? + + message = "siGit: #{@repo.full_name} is a read-only mirror of #{@repo.upstream_url}. " \ + "Detach it in the siGit UI to make it writable." + + no_cache + response.content_type = "application/x-git-receive-pack-advertisement" + render body: pkt_line("# service=git-receive-pack\n") + "0000" + pkt_line("ERR #{message}") + true + end + def rpc(service, content_type) input = request.body.read.to_s input = ActiveSupport::Gzip.decompress(input) if gzip_request? - out = git_run([service, "--stateless-rpc", @repo.disk_path], stdin: input) + out = git_run([ service, "--stateless-rpc", @repo.disk_path ], stdin: input) return head(:internal_server_error) if out.nil? no_cache

new file mode 100644 index 0000000..4595aab --- /dev/null +++ b/app/controllers/github_connections_controller.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +# Connects a user's GitHub account for repository imports via our own GitHub +# OAuth app (separate from smbCloud sign-in). The resulting token is stored +# encrypted in GithubConnection and never leaves the server. +class GithubConnectionsController < ApplicationController + before_action :require_sign_in! + before_action :noindex! + + # GET /import/github/connect + # Kicks off the OAuth dance. `visibility=public` narrows the scope to public + # repos only; the default requests `repo` so private repos can be imported. + def connect + unless GithubOauthService.configured? + return redirect_to new_import_path, alert: "GitHub import isn't configured on this server yet." + end + + state = GithubOauthService.generate_state + session[:github_import_state] = state + + include_private = params[:visibility] != "public" + redirect_to GithubOauthService.authorize_url( + redirect_uri: github_import_callback_url, + state: state, + include_private: include_private + ), allow_other_host: true + end + + # GET /import/github/callback + def callback + if params[:error].present? + return redirect_to new_import_path, alert: "GitHub connection was cancelled." + end + + # CSRF: the state we set must round-trip back unchanged. + expected = session.delete(:github_import_state) + if expected.blank? || !ActiveSupport::SecurityUtils.secure_compare(params[:state].to_s, expected) + return redirect_to new_import_path, alert: "GitHub connection failed a security check. Please try again." + end + + token = GithubOauthService.exchange_code( + code: params[:code].to_s, + redirect_uri: github_import_callback_url + ) + + login = GithubApiClient.new(token[:access_token]).login + + connection = current_user.github_connection || current_user.build_github_connection + connection.update!( + access_token: token[:access_token], + scope: token[:scope], + token_type: token[:token_type], + github_login: login, + connected_at: Time.current + ) + + redirect_to new_import_path, notice: "Connected to GitHub#{login ? " as @#{login}" : ''}." + rescue GithubOauthService::ExchangeError, GithubOauthService::ConfigurationError => e + Rails.logger.warn("GitHub import connect failed: #{e.class}") + redirect_to new_import_path, alert: "Couldn't connect to GitHub. Please try again." + end + + # DELETE /import/github/disconnect + def disconnect + current_user.github_connection&.destroy + redirect_to new_import_path, notice: "Disconnected from GitHub." + end +end

new file mode 100644 index 0000000..4d5ee1c --- /dev/null +++ b/app/controllers/github_webhooks_controller.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +# Receives GitHub push webhooks and triggers an immediate mirror sync for any +# mirror whose upstream matches the pushed repo. Optional: mirrors also sync on +# a schedule, so this is just a freshness optimization. +# +# Authenticated by the HMAC signature GitHub sends (X-Hub-Signature-256), keyed +# by GITHUB_WEBHOOK_SECRET. Without a configured secret the endpoint refuses +# requests rather than trusting unsigned input. +class GithubWebhooksController < ActionController::API + # POST /webhooks/github + def create + return head(:service_unavailable) if webhook_secret.blank? + + payload = request.body.read + return head(:unauthorized) unless valid_signature?(payload) + + event = request.headers["X-GitHub-Event"] + # Respond OK to ping so the webhook can be verified in GitHub's UI. + return head(:ok) if event == "ping" + return head(:ok) unless event == "push" + + data = JSON.parse(payload) rescue {} + urls = candidate_urls(data) + return head(:ok) if urls.empty? + + Repository.mirrors.where(upstream_url: urls).find_each do |repo| + MirrorSyncJob.perform_later(repo.id) + end + + head :ok + end + + private + + # The clone URLs GitHub might have stored as our upstream for this repo. + def candidate_urls(data) + repo = data["repository"] || {} + [ repo["clone_url"], repo["git_url"], repo["html_url"], repo["ssh_url"] ].compact.uniq + end + + def valid_signature?(payload) + sig = request.headers["X-Hub-Signature-256"].to_s + return false if sig.blank? + + expected = "sha256=" + OpenSSL::HMAC.hexdigest("sha256", webhook_secret, payload) + ActiveSupport::SecurityUtils.secure_compare(sig, expected) + end + + def webhook_secret + ENV["GITHUB_WEBHOOK_SECRET"].presence + end +end

new file mode 100644 index 0000000..315e7dd --- /dev/null +++ b/app/controllers/imports_controller.rb @@ -0,0 +1,204 @@ +# frozen_string_literal: true + +# The "Import from GitHub" experience: connect GitHub and pick repos, or paste a +# public git URL. Imports run off the request thread in RepositoryImportJob; this +# controller only validates, enforces limits, records RepositoryImport rows, and +# enqueues the work. +class ImportsController < ApplicationController + before_action :require_sign_in! + before_action :noindex! + + # Guardrails against abuse / one user saturating the workers. + MAX_ACTIVE_IMPORTS = Integer(ENV.fetch("IMPORT_MAX_ACTIVE_PER_USER", 10)) + MAX_USER_VOLUME_KB = Integer(ENV.fetch("IMPORT_MAX_USER_VOLUME_KB", 10_000_000)) # ~10 GB in flight + MAX_REPO_SIZE_KB = RepositoryImportService::DEFAULT_MAX_SIZE_KB + + # GET /import + def new + @connection = current_user.github_connection + @recent_imports = current_user.repository_imports.recent.limit(20) + @github_repos = load_github_repos(@connection) if @connection + end + + # GET /import (list-only, e.g. progress polling of all imports) + def index + @imports = current_user.repository_imports.recent.limit(50) + end + + # GET /import/:id — progress for a single import + def show + @import = current_user.repository_imports.find(params[:id]) + rescue ActiveRecord::RecordNotFound + redirect_to new_import_path, alert: "Import not found." + end + + # POST /import + # Either: + # source=github, repos[]=owner/name..., mode=migrate|mirror + # source=url, url=..., name=..., mode=..., visibility=public|private + def create + mode = params[:mode].to_s == "mirror" ? "mirror" : "migrate" + + specs = + if params[:source].to_s == "url" + build_url_spec(mode) + else + build_github_specs(mode) + end + + return if performed? # a builder already redirected with an error + if specs.blank? + return redirect_to new_import_path, alert: "Select at least one repository to import." + end + + enqueue_imports(specs) + end + + # POST /import/:id/retry + def retry + import = current_user.repository_imports.find(params[:id]) + unless import.retryable? + return redirect_to import_path(import), alert: "This import can't be retried." + end + + import.update!(status: "queued", error_message: nil, started_at: nil, finished_at: nil) + RepositoryImportJob.perform_later(import.id) + redirect_to import_path(import), notice: "Retrying import." + rescue ActiveRecord::RecordNotFound + redirect_to new_import_path, alert: "Import not found." + end + + private + + # Builds a single import spec from the by-URL form. Validates the URL (SSRF + # guard) and the target name. Redirects with an error on invalid input. + def build_url_spec(mode) + url = ImportUrlValidator.validate!(params[:url]) + host = URI.parse(url).host + name = params[:name].presence || derive_name_from_url(url) + + [ { source_url: url, source_host: host, mode: mode, + target_name: name, target_private: false, size_kb: nil, + description: nil, default_branch: nil } ] + rescue ImportUrlValidator::InvalidUrl => e + redirect_to new_import_path, alert: e.message + nil + end + + # Builds import specs from selected GitHub repos. Reads fresh metadata from + # GitHub so we capture the real visibility/size/default branch (never trusting + # client-supplied values for security-relevant fields like private). + def build_github_specs(mode) + selected = Array(params[:repos]).reject(&:blank?).uniq + return [] if selected.empty? + + connection = current_user.github_connection + return redirect_to(new_import_path, alert: "Connect GitHub first.") && nil if connection.nil? + + client = connection.api_client + selected.filter_map do |full_name| + meta = client.repository(full_name) + next if meta.nil? + + { + source_url: meta[:clone_url], + source_host: "github.com", + mode: mode, + target_name: meta[:name], + target_private: !!meta[:private], + size_kb: meta[:size_kb], + description: meta[:description], + default_branch: meta[:default_branch] + } + end + rescue GithubApiClient::RateLimited + redirect_to new_import_path, alert: "GitHub rate limit reached. Please try again shortly." + nil + rescue GithubApiClient::Error + redirect_to new_import_path, alert: "Couldn't read repositories from GitHub. Please try again." + nil + end + + # Applies caps, resolves name collisions, creates the RepositoryImport rows, + # and enqueues one job per import. Redirects with a summary. + def enqueue_imports(specs) + active = current_user.repository_imports.active.count + volume = current_user.active_import_size_kb + + created = [] + rejected = [] + + specs.each do |spec| + if spec[:size_kb] && spec[:size_kb] > MAX_REPO_SIZE_KB + rejected << "#{spec[:target_name]} (too large: #{(spec[:size_kb] / 1024.0).round} MB)" + next + end + if active + created.length >= MAX_ACTIVE_IMPORTS + rejected << "#{spec[:target_name]} (import queue full — try again later)" + next + end + if volume + (spec[:size_kb] || 0) > MAX_USER_VOLUME_KB + rejected << "#{spec[:target_name]} (would exceed your in-flight import limit)" + next + end + + import = current_user.repository_imports.create!( + source_url: spec[:source_url], + source_host: spec[:source_host], + mode: spec[:mode], + target_name: unique_repo_name(spec[:target_name]), + target_private: spec[:target_private], + description: spec[:description], + default_branch: spec[:default_branch], + source_size_kb: spec[:size_kb], + status: "queued" + ) + RepositoryImportJob.perform_later(import.id) + created << import + volume += (spec[:size_kb] || 0) + end + + notice = created.any? ? "Queued #{created.length} import#{'s' if created.length != 1}." : nil + alert = rejected.any? ? "Skipped: #{rejected.join('; ')}." : nil + + if created.length == 1 && rejected.empty? + redirect_to import_path(created.first), notice: notice + else + redirect_to new_import_path, notice: notice, alert: alert + end + end + + # Ensures the target name doesn't collide with an existing repo (or another + # queued import) by appending -1, -2, … This makes bulk imports and retries + # idempotent rather than failing on the unique (user, name) index. + def unique_repo_name(base) + base = base.to_s.gsub(/[^a-zA-Z0-9._-]/, "-").gsub(/-{2,}/, "-").gsub(/\A[-.]+|[-.]+\z/, "") + base = "imported-repo" if base.blank? + + taken = current_user.repositories.pluck(:name).map(&:downcase).to_set + taken.merge(current_user.repository_imports.active.pluck(:target_name).map(&:downcase)) + + return base unless taken.include?(base.downcase) + + (1..1000).each do |n| + candidate = "#{base}-#{n}" + return candidate unless taken.include?(candidate.downcase) + end + "#{base}-#{SecureRandom.hex(3)}" + end + + def derive_name_from_url(url) + File.basename(URI.parse(url).path.to_s).sub(/\.git\z/, "").presence || "imported-repo" + end + + # Lists the user's GitHub repos for the picker. Cached briefly so re-rendering + # the page (and Turbo revisits) don't re-hit GitHub. Never blocks the page on a + # rate-limit/outage — returns nil and the view shows a fallback. + def load_github_repos(connection) + Rails.cache.fetch("github_repos/#{connection.id}/#{connection.updated_at.to_i}", expires_in: 5.minutes) do + connection.api_client.list_repositories + end + rescue GithubApiClient::RateLimited, GithubApiClient::Error, GithubApiClient::Unauthorized + nil + end +end

new file mode 100644 index 0000000..4ffdbb7 --- /dev/null +++ b/app/controllers/mirrors_controller.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +# Actions on mirror repositories: detach (convert to a normal writable repo and +# stop syncing) and a manual sync trigger. Owner-only. +class MirrorsController < ApplicationController + before_action :require_sign_in! + before_action :noindex! + before_action :load_repository + before_action :ensure_owner! + + # POST /:username/:repository/mirror/detach + # Flips the mirror into a normal writable repo and stops syncing. Idempotent. + def detach + if @repository.detach_mirror! + redirect_to repository_path(@owner.username, @repository.name), + notice: "Mirror detached. This is now a normal repository you can push to." + else + redirect_to repository_path(@owner.username, @repository.name), + alert: "This repository isn't a mirror." + end + end + + # POST /:username/:repository/mirror/sync + # Enqueue an immediate sync (in addition to the scheduled ones). + def sync + unless @repository.mirror? + return redirect_to repository_path(@owner.username, @repository.name), + alert: "This repository isn't a mirror." + end + + MirrorSyncJob.perform_later(@repository.id) + redirect_to repository_path(@owner.username, @repository.name), + notice: "Sync queued." + end + + private + + def load_repository + @owner = User.find_by!(username: params[:username]) + @repository = @owner.repositories.find_by!(name: params[:repository]) + rescue ActiveRecord::RecordNotFound + render file: Rails.public_path.join("404.html"), status: :not_found, layout: false + end + + def ensure_owner! + unless signed_in? && current_user == @owner + render file: Rails.public_path.join("404.html"), status: :not_found, layout: false + end + end +end

new file mode 100644 index 0000000..363ab53 --- /dev/null +++ b/app/jobs/mirror_sync_job.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +# Syncs one mirror repository from its upstream. Enqueued on a schedule by +# MirrorSyncSchedulerJob and (optionally) by a GitHub push webhook. Marks the +# repo's sync status so the UI can show last-synced time and surface failures. +class MirrorSyncJob < ApplicationJob + queue_as :default + + def perform(repository_id) + repo = Repository.find_by(id: repository_id) + return if repo.nil? || !repo.mirror? + + repo.update!(mirror_status: "syncing") + MirrorSyncService.sync(repo) + repo.update!(mirror_status: "ok", mirror_synced_at: Time.current, mirror_error: nil) + rescue ImportUrlValidator::InvalidUrl => e + # Upstream now resolves somewhere unsafe (e.g. DNS rebinding) — stop, don't + # fetch, and surface it. + repo&.update!(mirror_status: "failed", mirror_error: "Upstream is no longer a valid public URL: #{e.message}") + rescue MirrorSyncService::SyncError => e + repo&.update!(mirror_status: "failed", mirror_error: e.message) + rescue StandardError => e + Rails.logger.error("Mirror sync failed for repo #{repository_id}: #{e.class}: #{e.message}") + repo&.update!(mirror_status: "failed", mirror_error: "Sync failed unexpectedly.") + end +end

new file mode 100644 index 0000000..5864603 --- /dev/null +++ b/app/jobs/mirror_sync_scheduler_job.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +# Periodic sweep that enqueues a MirrorSyncJob for every mirror due for a +# refresh. Runs from Solid Queue's recurring schedule (config/recurring.yml). +# +# Fanning out one job per repo (rather than syncing inline) keeps any single +# slow/hung upstream from blocking the others and lets the queue spread the work +# across workers. "Due" means never-synced or last synced before the interval. +class MirrorSyncSchedulerJob < ApplicationJob + queue_as :default + + # How stale a mirror may get before we re-sync it on the schedule. Webhooks, + # when configured, sync sooner. + SYNC_INTERVAL = ActiveSupport::Duration.build(Integer(ENV.fetch("MIRROR_SYNC_INTERVAL_SECONDS", 3600))) + + # Don't pile up on a mirror that's already syncing. + def perform + cutoff = Time.current - SYNC_INTERVAL + + due = Repository.mirrors + .where.not(mirror_status: "syncing") + .where("mirror_synced_at IS NULL OR mirror_synced_at < ?", cutoff) + + due.find_each do |repo| + MirrorSyncJob.perform_later(repo.id) + end + end +end

new file mode 100644 index 0000000..fab89c1 --- /dev/null +++ b/app/jobs/repository_import_job.rb @@ -0,0 +1,143 @@ +# frozen_string_literal: true + +# Runs a repository import off the request thread: creates the destination repo, +# mirror-clones the source and pushes its history in, records LFS/metadata, and +# either finishes cleanly or rolls back so no half-created repo is left behind. +# +# Idempotency & retries: the job is safe to re-run for a failed import. It first +# clears any partial destination from a previous attempt, so a retry starts from +# a clean slate and can't duplicate or corrupt an existing repo. A name that +# collides with a *different* existing repo fails explicitly rather than +# clobbering it. +class RepositoryImportJob < ApplicationJob + queue_as :default + + # Raised when the target name collides with a different existing repo. Its + # message is user-facing, so it's surfaced verbatim on the import. + class NameCollisionError < StandardError; end + + # Don't auto-retry: an import failure is surfaced to the user as a retryable + # state with a message, and retrying blindly could hammer a bad remote. + def perform(import_id) + import = RepositoryImport.find_by(id: import_id) + return if import.nil? + # Guard against double-delivery: only queued or (re-queued) failed imports + # should run. An already-active/done import is left alone. + return unless %w[queued failed].include?(import.status) + + reset_for_retry(import) + + user = import.user + repo = create_destination!(import) + import.update!(repository: repo) + + result = RepositoryImportService.clone_and_push( + source_url: import.source_url, + dest_path: repo.disk_path, + credential: import_credential(import), + progress: ->(phase) { import.transition_to(phase.to_s) } + ) + + import.transition_to("finalizing") + finalize_repo!(repo, import, result) + + import.update!(lfs_detected: result.lfs_detected, lfs_skipped: result.lfs_skipped, + source_size_kb: result.size_kb || import.source_size_kb) + import.transition_to("done") + rescue StandardError => e + handle_failure(import, e) + end + + private + + # Clear any leftover from a previous failed attempt so a retry is clean. + def reset_for_retry(import) + return if import.repository.nil? + + destroy_repo(import.repository) + import.update!(repository: nil) + end + + # Creates the Repository row and the bare repo on disk. Name collisions with a + # different existing repo are rejected here, explicitly. + def create_destination!(import) + user = import.user + existing = user.repositories.find_by("LOWER(name) = ?", import.target_name.downcase) + if existing + raise NameCollisionError, + "A repository named \"#{import.target_name}\" already exists. Rename it or delete it, then retry." + end + + repo = user.repositories.new( + name: import.target_name, + description: import.description, + is_private: import.target_private, + default_branch: import.default_branch.presence || "main", + kind: "code" + ) + repo.disk_path = GitRepositoryService.repo_path(user.username, repo.name) + repo.save! + + GitRepositoryService.create_bare_repo(user.username, repo.name) + repo + end + + # After the push: point HEAD at the imported default branch (so the README + # renders on the landing page) and, for mirror mode, wire up the read-only + # upstream link and stamp the first successful sync. + def finalize_repo!(repo, import, _result) + head = GitRepositoryService.set_head(repo.disk_path, repo.default_branch) + repo.update!(default_branch: head) if head.present? && head != repo.default_branch + + if import.mirror? + repo.update!( + mirror: true, + upstream_url: import.source_url, + upstream_token: import_credential(import), + mirror_status: "ok", + mirror_synced_at: Time.current + ) + end + end + + # The credential to clone/fetch the source with. GitHub private sources use the + # user's connection token; URL imports are unauthenticated (public only), so a + # private URL without credentials fails fast in the clone. + def import_credential(import) + return nil unless import.source_host == "github.com" + + import.user.github_connection&.access_token + end + + def handle_failure(import, error) + return if import.nil? + + # Leave no half-created repo behind. + if import.repository + destroy_repo(import.repository) + import.update!(repository: nil) + end + + message = + case error + when NameCollisionError, + RepositoryImportService::SizeExceeded, + RepositoryImportService::CloneTimeout, + RepositoryImportService::ImportError + error.message + else + Rails.logger.error("Import ##{import.id} failed: #{error.class}: #{error.message}") + "The import failed unexpectedly. You can retry it." + end + + import.transition_to("failed", error: message) + end + + def destroy_repo(repo) + path = repo.disk_path + repo.destroy + FileUtils.rm_rf(path) if path.present? + rescue StandardError => e + Rails.logger.error("Failed to clean up repo #{repo&.id}: #{e.message}") + end +end

new file mode 100644 index 0000000..ad08d87 --- /dev/null +++ b/app/models/github_connection.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +# A user's connection to their GitHub account for importing repositories. +# +# Distinct from smbCloud "Continue with GitHub" sign-in: that authenticates the +# user to siGit; this holds a GitHub API token (scope `repo` or `public_repo`) +# used to list and clone the user's repos. One connection per user; the token is +# encrypted at rest and never serialized to the client. +class GithubConnection < ApplicationRecord + belongs_to :user + + encrypts :access_token + + validates :access_token, presence: true + validates :user_id, uniqueness: true + + # True when the connection can reach the user's private repositories. + def private_scope? + scope.to_s.split(/[ ,]/).include?("repo") + end + + # A GithubApiClient bound to this connection's token. The token stays server + # side; callers get repo data, never the credential. + def api_client + GithubApiClient.new(access_token) + end +end

index d60dbcd..833ba36 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -6,6 +6,13 @@ class Repository < ApplicationRecord has_many :stargazers, through: :stars, source: :user has_many :pull_requests, dependent: :destroy has_many :issues, dependent: :destroy + has_many :repository_imports, dependent: :nullify + + # Upstream credential for a private mirror, encrypted at rest and never + # returned to the client. Nil for public upstreams and non-mirror repos. + encrypts :upstream_token + + scope :mirrors, -> { where(mirror: true) } scope :models, -> { where(kind: "model") } scope :code, -> { where(kind: "code") } @@ -44,11 +51,47 @@ class Repository < ApplicationRecord end # True if +user+ may push to / open changes against this repo. siGit repos - # have a single owner and no collaborators yet, so write == ownership. + # have a single owner and no collaborators yet, so write == ownership — but a + # mirror is read-only for everyone (including the owner) until it's detached, + # so its upstream stays the single source of truth. def writable_by?(user) + return false if mirror? user.present? && user_id == user.id end + # ── Mirror mode ───────────────────────────────────────────────────────── + + def mirror? + mirror + end + + # Flip a mirror into a normal writable repo and stop syncing. Idempotent: a + # non-mirror repo is left untouched. The upstream URL is kept for reference, + # but the credential is dropped since we no longer fetch with it. + def detach_mirror! + return false unless mirror? + + update!( + mirror: false, + upstream_token: nil, + mirror_status: nil, + mirror_error: nil + ) + true + end + + # A short human status for the mirror badge/health line. + def mirror_state_label + return nil unless mirror? + + case mirror_status + when "syncing" then "Syncing…" + when "failed" then "Sync failed" + else + mirror_synced_at ? "Synced #{mirror_synced_at.strftime('%b %-d, %H:%M UTC')}" : "Awaiting first sync" + end + end + # Read a file's contents at +ref+ (branch, tag, or SHA). Returns the content # String, or nil when the path doesn't exist at that ref. This is the git read # layer the MCP `get_file_contents` tool sits on.

new file mode 100644 index 0000000..e6c83b4 --- /dev/null +++ b/app/models/repository_import.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +# One repository import, from queued through to done or failed. Drives the +# progress UI and makes a failed import retryable. The actual clone/push runs in +# RepositoryImportJob; this record is the state it advances. +class RepositoryImport < ApplicationRecord + belongs_to :user + belongs_to :repository, optional: true + + MODES = %w[migrate mirror].freeze + + # Ordered lifecycle. `cloning`/`pushing`/`fetching_lfs` are the long phases the + # UI narrates; terminal states are `done` and `failed`. + STATUSES = %w[queued cloning pushing fetching_lfs finalizing done failed].freeze + TERMINAL = %w[done failed].freeze + ACTIVE = STATUSES - TERMINAL + + validates :mode, inclusion: { in: MODES } + validates :status, inclusion: { in: STATUSES } + validates :source_url, presence: true + validates :target_name, presence: true + + scope :active, -> { where(status: ACTIVE) } + scope :recent, -> { order(created_at: :desc) } + + def mirror? + mode == "mirror" + end + + def migrate? + mode == "migrate" + end + + def terminal? + TERMINAL.include?(status) + end + + def failed? + status == "failed" + end + + def done? + status == "done" + end + + # A retry is allowed only from a failed state; re-running from any active or + # done state would risk duplicate work against a live import. + def retryable? + failed? + end + + # Human-friendly one-liner for the progress UI. + def status_label + { + "queued" => "Queued", + "cloning" => "Cloning source", + "pushing" => "Pushing to siGit", + "fetching_lfs" => "Fetching LFS objects", + "finalizing" => "Finalizing", + "done" => "Imported", + "failed" => "Failed" + }.fetch(status, status.humanize) + end + + # Advance the state machine, stamping timing on entry/exit. Kept tiny so the + # job body reads as a sequence of `transition_to(...)` calls. + def transition_to(new_status, error: nil) + attrs = { status: new_status } + attrs[:started_at] = Time.current if new_status == "cloning" && started_at.nil? + attrs[:finished_at] = Time.current if TERMINAL.include?(new_status) + attrs[:error_message] = error if error + update!(attrs) + end +end

index 54cb9d2..e8efba9 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -8,6 +8,19 @@ class User < ApplicationRecord has_one :subscription, dependent: :destroy has_many :cloud_usages, dependent: :destroy has_many :cloud_sessions, dependent: :destroy + has_one :github_connection, dependent: :destroy + has_many :repository_imports, dependent: :destroy + + # True when the user has linked their GitHub account for imports. + def github_connected? + github_connection.present? + end + + # Total reported size (KB) of imports currently in flight, used to enforce the + # per-user concurrent import volume cap without counting finished ones. + def active_import_size_kb + repository_imports.active.sum(:source_size_kb) + end # Whether this user may use siGit Code Cloud (the paid cloud tiers). Free / # unsubscribed users run on-device only.

index 536b557..a048c34 100644 --- a/app/services/git_repository_service.rb +++ b/app/services/git_repository_service.rb @@ -31,6 +31,19 @@ class GitRepositoryService end end + # Point HEAD at +branch+ so the landing page and default clone check out the + # right branch. Falls back to the first available branch when +branch+ wasn't + # among the pushed refs (e.g. an imported repo whose default differs). Returns + # the branch HEAD ends up on, or nil if the repo has no branches at all. + def self.set_head(path, branch) + target = branch if branch.present? && branch_exists?(path, branch) + target ||= branches(path).first + return nil if target.nil? + + system("git", "--git-dir", path, "symbolic-ref", "HEAD", "refs/heads/#{target}", exception: false) + target + end + def self.default_branch(path) out, _err, status = Open3.capture3("git", "--git-dir", path, "symbolic-ref", "--short", "HEAD") return "main" unless status.success?

new file mode 100644 index 0000000..2b44bc0 --- /dev/null +++ b/app/services/github_api_client.rb @@ -0,0 +1,126 @@ +# frozen_string_literal: true + +require "net/http" +require "json" + +# Thin GitHub REST client bound to one user's OAuth token, used to list repos +# and read repo metadata for the import flow. Intentionally small — we only need +# a few read endpoints, so this avoids pulling in Octokit. +# +# The token is held only in memory on this instance and sent as a bearer header; +# it is never logged. Rate limits are respected: on a 403/429 with the limit +# exhausted we raise RateLimited carrying the reset time so the caller can back +# off rather than hammer GitHub. +class GithubApiClient + API_ROOT = "https://api.github.com" + + OPEN_TIMEOUT = 5 + READ_TIMEOUT = 15 + + class Error < StandardError; end + class Unauthorized < Error; end + class RateLimited < Error + attr_reader :reset_at + def initialize(message, reset_at:) + super(message) + @reset_at = reset_at + end + end + + def initialize(access_token) + @access_token = access_token + end + + # The authenticated user's login, or nil if the token is bad. Used to label + # the connection and build clone URLs. + def login + me = get("/user") + me["login"] + rescue Unauthorized + nil + end + + # Lists the authenticated user's repositories, newest-pushed first. Paginates + # internally up to +max+ repos (a safety cap so a user with thousands of repos + # can't make us page forever). Returns an array of normalized hashes. + def list_repositories(max: 300, per_page: 50) + repos = [] + page = 1 + loop do + batch = get("/user/repos", affiliation: "owner,collaborator", + sort: "pushed", per_page: per_page, page: page) + break if batch.blank? + + repos.concat(batch.map { |r| normalize_repo(r) }) + break if batch.length < per_page || repos.length >= max + + page += 1 + end + repos.first(max) + end + + # Metadata for a single repo ("owner/name"), normalized. nil if not found. + def repository(full_name) + normalize_repo(get("/repos/#{full_name}")) + rescue Error + nil + end + + private + + def normalize_repo(r) + { + full_name: r["full_name"], + name: r["name"], + description: r["description"], + private: r["private"], + default_branch: r["default_branch"], + clone_url: r["clone_url"], + homepage: r["homepage"], + topics: r["topics"] || [], + size_kb: r["size"], # GitHub reports size in KB + pushed_at: r["pushed_at"], + archived: r["archived"], + fork: r["fork"] + } + end + + def get(path, **query) + uri = URI.parse("#{API_ROOT}#{path}") + uri.query = URI.encode_www_form(query) if query.any? + + req = Net::HTTP::Get.new(uri) + req["Authorization"] = "Bearer #{@access_token}" + req["Accept"] = "application/vnd.github+json" + req["X-GitHub-Api-Version"] = "2022-11-28" + req["User-Agent"] = "sigit-si-importer" + + res = Net::HTTP.start(uri.hostname, uri.port, use_ssl: true, + open_timeout: OPEN_TIMEOUT, read_timeout: READ_TIMEOUT) do |http| + http.request(req) + end + + handle(res) + end + + def handle(res) + case res + when Net::HTTPSuccess + res.body.present? ? JSON.parse(res.body) : nil + when Net::HTTPUnauthorized + raise Unauthorized, "GitHub token is invalid or expired." + when Net::HTTPForbidden, Net::HTTPTooManyRequests + # Distinguish a real rate-limit from a permission error by the remaining + # header, so we only tell the caller to back off when it actually should. + if res["x-ratelimit-remaining"].to_i.zero? && res["x-ratelimit-reset"].present? + reset_at = Time.at(res["x-ratelimit-reset"].to_i) + raise RateLimited.new("GitHub API rate limit reached.", reset_at: reset_at) + end + raise Error, "GitHub denied the request (HTTP #{res.code})." + else + raise Error, "GitHub request failed (HTTP #{res.code})." + end + rescue JSON::ParserError + raise Error, "GitHub returned an unreadable response." + end +end

new file mode 100644 index 0000000..ac7651a --- /dev/null +++ b/app/services/github_oauth_service.rb @@ -0,0 +1,115 @@ +# frozen_string_literal: true + +require "net/http" +require "json" +require "securerandom" + +# The GitHub OAuth app siGit uses to import repositories. +# +# This is a *separate* OAuth app from smbCloud "Continue with GitHub" sign-in. +# Sign-in authenticates the user to siGit (via smbCloud) and gives us no GitHub +# API access. Importing needs a GitHub token that can read the user's repos and +# clone private ones, so we run our own minimal OAuth app: +# +# 1. #authorize_url — send the user to GitHub to grant access. +# 2. GitHub redirects back to our callback with ?code=…&state=… +# 3. #exchange_code — trade the code for an access token, stored encrypted in +# GithubConnection. +# +# Requires: +# GITHUB_IMPORT_CLIENT_ID +# GITHUB_IMPORT_CLIENT_SECRET +# +# The token never reaches the browser; only the server holds it. +class GithubOauthService + class ConfigurationError < StandardError; end + class ExchangeError < StandardError; end + + AUTHORIZE_URL = "https://github.com/login/oauth/authorize" + TOKEN_URL = "https://github.com/login/oauth/access_token" + + # Full read/write to private repos vs. public repos only. We ask for the + # narrowest scope that covers what the user chose to import. + SCOPE_PRIVATE = "repo" + SCOPE_PUBLIC = "public_repo" + + OPEN_TIMEOUT = 5 + READ_TIMEOUT = 15 + + class << self + def configured? + client_id.present? && client_secret.present? + end + + # A CSRF token to round-trip through the `state` param. Stored in the session + # before redirecting and compared on callback. + def generate_state + SecureRandom.urlsafe_base64(24) + end + + # URL to send the user's browser to. +include_private+ picks the scope. + def authorize_url(redirect_uri:, state:, include_private: true) + raise ConfigurationError, "GitHub import OAuth app is not configured." unless configured? + + params = { + client_id: client_id, + redirect_uri: redirect_uri, + scope: include_private ? SCOPE_PRIVATE : SCOPE_PUBLIC, + state: state, + allow_signup: "false" + } + "#{AUTHORIZE_URL}?#{URI.encode_www_form(params)}" + end + + # Exchanges the OAuth +code+ for an access token. Returns a hash: + # { access_token:, scope:, token_type: } + # Raises ExchangeError on any failure (never leaks the secret or the raw + # GitHub error to callers/logs). + def exchange_code(code:, redirect_uri:) + raise ConfigurationError, "GitHub import OAuth app is not configured." unless configured? + + uri = URI.parse(TOKEN_URL) + req = Net::HTTP::Post.new(uri) + req["Accept"] = "application/json" + req.set_form_data( + client_id: client_id, + client_secret: client_secret, + code: code, + redirect_uri: redirect_uri + ) + + res = Net::HTTP.start(uri.hostname, uri.port, use_ssl: true, + open_timeout: OPEN_TIMEOUT, read_timeout: READ_TIMEOUT) do |http| + http.request(req) + end + + unless res.is_a?(Net::HTTPSuccess) + raise ExchangeError, "GitHub rejected the authorization (HTTP #{res.code})." + end + + body = JSON.parse(res.body) + token = body["access_token"] + if token.blank? + # body["error"] is safe to surface (e.g. "bad_verification_code"); it + # contains no secret. + raise ExchangeError, "GitHub did not return an access token (#{body['error'] || 'unknown error'})." + end + + { + access_token: token, + scope: body["scope"].to_s, + token_type: body["token_type"].presence || "bearer" + } + rescue JSON::ParserError + raise ExchangeError, "GitHub returned an unreadable token response." + end + + def client_id + ENV["GITHUB_IMPORT_CLIENT_ID"].presence + end + + def client_secret + ENV["GITHUB_IMPORT_CLIENT_SECRET"].presence + end + end +end

new file mode 100644 index 0000000..32ee375 --- /dev/null +++ b/app/services/import_url_validator.rb @@ -0,0 +1,135 @@ +# frozen_string_literal: true + +require "uri" +require "resolv" +require "ipaddr" + +# Validates a user-supplied git URL before we hand it to `git clone`, closing +# the SSRF hole that "import by URL" would otherwise open: without this, a user +# could point the importer at `http://169.254.169.254/…` (cloud metadata) or an +# internal `http://10.0.0.5/…` service and have the worker fetch it. +# +# Rules: +# - scheme must be http or https (no ssh://, file://, git://, ftp://, …) +# - a host must be present +# - the host must not resolve to a private, loopback, link-local, or otherwise +# non-public address (checked for every A/AAAA record, and for literal IPs) +# - userinfo (user:pass@) is rejected — credentials belong in the token field, +# not smuggled through the URL +# +# Usage: +# ImportUrlValidator.validate!(url) # => normalized URL string, or raises +# ImportUrlValidator.safe?(url) # => true/false +# +# DNS resolution here is advisory (it reduces the attack surface); the clone +# itself still runs sandboxed with a timeout because DNS can rebind between this +# check and the fetch. +class ImportUrlValidator + class InvalidUrl < StandardError; end + + ALLOWED_SCHEMES = %w[http https].freeze + + # CIDR blocks that must never be reachable from an import. Covers loopback, + # RFC1918 private ranges, link-local (incl. 169.254.169.254 metadata), + # carrier-grade NAT, and their IPv6 equivalents (ULA, link-local, mapped v4). + BLOCKED_RANGES = [ + "0.0.0.0/8", # "this" network + "10.0.0.0/8", # private + "100.64.0.0/10", # carrier-grade NAT + "127.0.0.0/8", # loopback + "169.254.0.0/16", # link-local (AWS/GCP metadata lives at 169.254.169.254) + "172.16.0.0/12", # private + "192.0.0.0/24", # IETF protocol assignments + "192.168.0.0/16", # private + "198.18.0.0/15", # benchmarking + "::1/128", # IPv6 loopback + "fc00::/7", # IPv6 unique local + "fe80::/10", # IPv6 link-local + "::ffff:0:0/96" # IPv4-mapped IPv6 (re-checked as v4 below) + ].map { |c| IPAddr.new(c) }.freeze + + class << self + # Returns a normalized URL string when +raw+ is a safe public http(s) git + # URL, otherwise raises InvalidUrl with a user-facing message. + def validate!(raw) + url = raw.to_s.strip + raise InvalidUrl, "Enter a repository URL." if url.empty? + + uri = begin + URI.parse(url) + rescue URI::InvalidURIError + raise InvalidUrl, "That doesn't look like a valid URL." + end + + unless ALLOWED_SCHEMES.include?(uri.scheme) + raise InvalidUrl, "Only http(s) git URLs are supported." + end + + raise InvalidUrl, "The URL is missing a host." if uri.host.blank? + + if uri.userinfo.present? + raise InvalidUrl, "Don't put credentials in the URL; connect the account or use a token instead." + end + + resolve_and_guard!(uri.host) + + url + end + + def safe?(raw) + validate!(raw) + true + rescue InvalidUrl + false + end + + private + + # Rejects the host if it's a blocked literal IP, or if any address it + # resolves to is blocked. A resolution failure is treated as unsafe. + def resolve_and_guard!(host) + addresses = + if literal_ip?(host) + [ host ] + else + resolve(host) + end + + raise InvalidUrl, "Could not resolve that host." if addresses.empty? + + addresses.each do |addr| + ip = IPAddr.new(addr) + # Normalize IPv4-mapped IPv6 (::ffff:10.0.0.1) down to the v4 form so it + # can't slip past the v4 blocklist. + ip = ip.ipv4_mapped? ? ip.native : ip if ip.ipv6? + if blocked?(ip) + raise InvalidUrl, "That host points at a private or internal address, which isn't allowed." + end + end + rescue IPAddr::InvalidAddressError + raise InvalidUrl, "That host resolved to an address we couldn't verify." + end + + def blocked?(ip) + BLOCKED_RANGES.any? { |range| range.include?(ip) } + end + + def literal_ip?(host) + IPAddr.new(host) + true + rescue IPAddr::InvalidAddressError + false + end + + def resolve(host) + Resolv::DNS.open do |dns| + dns.timeouts = 3 + v4 = dns.getresources(host, Resolv::DNS::Resource::IN::A).map { |r| r.address.to_s } + v6 = dns.getresources(host, Resolv::DNS::Resource::IN::AAAA).map { |r| r.address.to_s } + v4 + v6 + end + rescue Resolv::ResolvError, Resolv::ResolvTimeout + [] + end + end +end

new file mode 100644 index 0000000..5b80a2b --- /dev/null +++ b/app/services/mirror_sync_service.rb @@ -0,0 +1,112 @@ +# frozen_string_literal: true + +require "open3" +require "tempfile" +require "timeout" +require "base64" + +# Re-syncs a mirror repository from its upstream: force-fetches all heads and +# tags into the (read-only) bare repo so it tracks the upstream exactly. Because +# mirrors reject local pushes, force-updating refs here can never clobber unique +# local work — the upstream is the single source of truth by design. +# +# Shares the isolation posture of RepositoryImportService: hard timeout with +# process-group kill, no interactive prompts, credentials passed as an HTTP +# header (never the URL or stored config) and scrubbed from captured output. +class MirrorSyncService + class SyncError < StandardError; end + class SyncTimeout < SyncError; end + + DEFAULT_TIMEOUT = Integer(ENV.fetch("MIRROR_SYNC_TIMEOUT", 900)) # 15 min + + def self.sync(repository, timeout: DEFAULT_TIMEOUT) + new(repository, timeout:).call + end + + def initialize(repository, timeout: DEFAULT_TIMEOUT) + @repository = repository + @timeout = timeout + end + + # Fetches upstream into the repo. Raises SyncError on failure. On success the + # caller stamps mirror_synced_at / mirror_status. + def call + raise SyncError, "Not a mirror." unless @repository.mirror? + raise SyncError, "Mirror has no upstream URL." if @repository.upstream_url.blank? + + # Re-validate the upstream every sync: DNS could have been repointed at an + # internal address since the import, so this closes the rebinding window. + ImportUrlValidator.validate!(@repository.upstream_url) + + args = [ "git" ] + args += credential_config + args += [ "-C", @repository.disk_path, "fetch", "--prune", @repository.upstream_url, + "+refs/heads/*:refs/heads/*", "+refs/tags/*:refs/tags/*" ] + + ok, output = run(args, timeout: @timeout) + raise SyncError, "Fetch failed: #{first_error_line(output)}" unless ok + + true + end + + private + + def credential_config + token = @repository.upstream_token + return [] if token.blank? + + basic = Base64.strict_encode64("x-access-token:#{token}") + [ "-c", "http.extraHeader=Authorization: Basic #{basic}" ] + end + + def run(cmd, timeout:) + out = Tempfile.new("sigit-sync-out") + begin + pid = Process.spawn(git_env, *cmd, out: out.path, err: [ :child, :out ], pgroup: true) + begin + Timeout.timeout(timeout) { Process.wait(pid) } + rescue Timeout::Error + kill_group(pid) + raise SyncTimeout, "mirror sync exceeded #{timeout}s and was terminated." + end + [ $?.success?, scrub(File.read(out.path)) ] + ensure + out.close! + end + end + + def kill_group(pid) + begin + Process.kill("TERM", -pid) + sleep 2 + Process.kill("KILL", -pid) + rescue Errno::ESRCH, Errno::EPERM + nil + end + begin + Process.wait(pid) + rescue Errno::ECHILD + nil + end + end + + def git_env + { + "GIT_TERMINAL_PROMPT" => "0", + "GIT_ASKPASS" => "/bin/echo", + "GCM_INTERACTIVE" => "never", + "GIT_LFS_SKIP_SMUDGE" => "1" + } + end + + def scrub(text) + token = @repository.upstream_token + return text if token.blank? + + text.to_s.gsub(token, "***") + end + + def first_error_line(output) + output.to_s.each_line.map(&:strip).reject(&:empty?).last.presence || "unknown error" + end +end

new file mode 100644 index 0000000..ab8419a --- /dev/null +++ b/app/services/repository_import_service.rb @@ -0,0 +1,240 @@ +# frozen_string_literal: true + +require "open3" +require "fileutils" +require "tmpdir" +require "tempfile" +require "timeout" +require "base64" + +# The git mechanics of an import: mirror-clone a source repo in a sandbox, then +# push every ref (all branches and tags, full history, no rewrites) into an +# existing bare siGit repo. LFS objects are fetched when git-lfs is available, +# and clearly reported as skipped otherwise. +# +# This class is deliberately isolation-focused — every guardrail that keeps a +# hung or malicious remote from wedging a worker lives here: +# - the clone runs in a fresh temp dir that is always removed, +# - every git invocation runs with a hard timeout and is killed (whole process +# group) if it overruns, +# - `GIT_TERMINAL_PROMPT=0` makes auth failures fail fast instead of hanging, +# - the cloned size is capped, so one repo can't fill the disk, +# - credentials are passed via an HTTP header (not the URL, not the stored +# remote config) and scrubbed from any captured output before it is logged. +# +# It does not touch the database or create the destination repo — the caller +# (RepositoryImportJob) owns lifecycle, status, and cleanup. +class RepositoryImportService + class ImportError < StandardError; end + class SizeExceeded < ImportError; end + class CloneTimeout < ImportError; end + + # A single git call may not pin a worker forever. Overridable for very large + # legitimate repos via env. + DEFAULT_TIMEOUT = Integer(ENV.fetch("IMPORT_GIT_TIMEOUT", 1800)) # 30 min + + # Per-repo on-disk cap. GitHub reports size in KB; we also re-check the actual + # clone so URL imports (no reported size) are bounded too. + DEFAULT_MAX_SIZE_KB = Integer(ENV.fetch("IMPORT_MAX_REPO_SIZE_KB", 2_000_000)) # ~2 GB + + Result = Struct.new(:lfs_detected, :lfs_skipped, :size_kb, keyword_init: true) + + # Clones +source_url+ (with optional +credential+ token for private sources) + # and pushes all refs into the existing bare repo at +dest_path+. Yields a + # status symbol (:cloning, :pushing, :fetching_lfs) to +progress+ as it moves + # through the phases. Returns a Result. Raises ImportError (or a subclass) on + # failure; the caller is responsible for cleaning up +dest_path+. + def self.clone_and_push(source_url:, dest_path:, credential: nil, + max_size_kb: DEFAULT_MAX_SIZE_KB, timeout: DEFAULT_TIMEOUT, progress: nil) + new(source_url:, dest_path:, credential:, max_size_kb:, timeout:, progress:).call + end + + def initialize(source_url:, dest_path:, credential:, max_size_kb:, timeout:, progress:) + @source_url = source_url + @dest_path = dest_path + @credential = credential + @max_size_kb = max_size_kb + @timeout = timeout + @progress = progress + end + + def call + Dir.mktmpdir("sigit-import-") do |workdir| + mirror = File.join(workdir, "source.git") + + emit(:cloning) + clone_mirror(mirror) + enforce_size!(mirror) + + emit(:pushing) + push_all_refs(mirror) + + emit(:fetching_lfs) + lfs = handle_lfs(mirror) + + Result.new( + lfs_detected: lfs[:detected], + lfs_skipped: lfs[:skipped], + size_kb: dir_size_kb(@dest_path) + ) + end + end + + private + + def emit(phase) + @progress&.call(phase) + end + + def clone_mirror(dest) + args = [ "git" ] + args += credential_config + args += [ "clone", "--mirror", @source_url, dest ] + + ok, output = run(args, timeout: @timeout) + raise ImportError, "Clone failed: #{first_error_line(output)}" unless ok + end + + # Push every ref verbatim (heads + tags), preserving history exactly. The + # refspec form (rather than `--mirror`) keeps siGit's internal refs, if any, + # from being pruned, and never rewrites commits. + def push_all_refs(mirror) + ok, output = run([ "git", "-C", mirror, "push", @dest_path, "+refs/heads/*:refs/heads/*", "+refs/tags/*:refs/tags/*" ], + timeout: @timeout) + raise ImportError, "Push failed: #{first_error_line(output)}" unless ok + end + + # Best-effort LFS handling. Detection is cheap (does any ref carry a + # `.gitattributes` with `filter=lfs`); fetching requires the git-lfs binary. + # When it's missing we don't fail — the LFS *pointer* files are ordinary git + # blobs and are already imported with the history; only the large binaries are + # skipped, which we report so the UI can say so plainly. + def handle_lfs(mirror) + return { detected: false, skipped: false } unless lfs_used?(mirror) + return { detected: true, skipped: true } unless lfs_available? + + ok, _ = run([ "git", "-C", mirror, "lfs", "fetch", "--all" ], timeout: @timeout) + return { detected: true, skipped: true } unless ok + + # Copy the fetched object store into the destination so siGit holds the + # binaries too. Non-fatal if it doesn't land. + src_lfs = File.join(mirror, "lfs") + if Dir.exist?(src_lfs) + FileUtils.cp_r(src_lfs, File.join(@dest_path, "lfs")) + { detected: true, skipped: false } + else + { detected: true, skipped: true } + end + rescue StandardError => e + Rails.logger.warn("LFS import step failed (continuing): #{e.class}") + { detected: true, skipped: true } + end + + def lfs_used?(mirror) + # Check the tip of every branch for a .gitattributes that enables the lfs + # filter. Cheap and covers the overwhelmingly common case. The blob read + # goes through GitRepositoryService (the single git-read layer) rather than + # shelling out again here. + ok, refs = run([ "git", "-C", mirror, "for-each-ref", "--format=%(refname)", "refs/heads/" ], timeout: 30) + return false unless ok + + refs.each_line.map(&:strip).reject(&:empty?).any? do |ref| + content = GitRepositoryService.file_content(mirror, ref, ".gitattributes") + content&.include?("filter=lfs") + end + end + + def lfs_available? + _o, _e, st = Open3.capture3("git", "lfs", "version") + st.success? + rescue StandardError + false + end + + def enforce_size!(mirror) + size = dir_size_kb(mirror) + return if size.nil? || size <= @max_size_kb + + raise SizeExceeded, + "Repository is #{(size / 1024.0).round} MB, over the #{(@max_size_kb / 1024.0).round} MB import limit." + end + + # ── process execution ──────────────────────────────────────────────────── + + # Runs +cmd+ with a hard timeout. On overrun the whole process group is killed + # so a hung `git` (or a child it spawned) can't linger. Returns + # [success_bool, output_string]; output has any credential scrubbed out. + def run(cmd, timeout:) + out = Tempfile.new("sigit-git-out") + begin + pid = Process.spawn(git_env, *cmd, out: out.path, err: [ :child, :out ], pgroup: true) + begin + Timeout.timeout(timeout) { Process.wait(pid) } + rescue Timeout::Error + kill_group(pid) + raise CloneTimeout, "git operation exceeded #{timeout}s and was terminated." + end + success = $?.success? + [ success, scrub(File.read(out.path)) ] + ensure + out.close! + end + end + + def kill_group(pid) + begin + Process.kill("TERM", -pid) + sleep 2 + Process.kill("KILL", -pid) + rescue Errno::ESRCH, Errno::EPERM + # already gone + end + + begin + Process.wait(pid) + rescue Errno::ECHILD + nil + end + end + + def git_env + { + # Never block on an interactive auth/host prompt — fail fast instead. + "GIT_TERMINAL_PROMPT" => "0", + "GIT_ASKPASS" => "/bin/echo", + "GCM_INTERACTIVE" => "never", + # Don't try to download LFS blobs during the mirror clone; we handle LFS + # explicitly afterwards. + "GIT_LFS_SKIP_SMUDGE" => "1" + } + end + + # Auth for a private source, injected as an HTTP header via `-c` so the token + # never lands in the URL, the stored remote config, or the clone on disk. + def credential_config + return [] if @credential.blank? + + basic = Base64.strict_encode64("x-access-token:#{@credential}") + [ "-c", "http.extraHeader=Authorization: Basic #{basic}" ] + end + + # Remove the credential from any captured text before it can be logged or + # persisted to the import's error_message. + def scrub(text) + return text if @credential.blank? + + text.to_s.gsub(@credential, "***") + end + + def first_error_line(output) + line = output.to_s.each_line.map(&:strip).reject(&:empty?).last + line.presence || "unknown error" + end + + def dir_size_kb(path) + return nil unless Dir.exist?(path) + + out, _e, st = Open3.capture3("du", "-sk", path) + st.success? ? out.split("\t").first.to_i : nil + end +end

new file mode 100644 index 0000000..975f15d --- /dev/null +++ b/app/views/imports/_status_badge.html.erb @@ -0,0 +1,18 @@ +<%# + Small status pill for a RepositoryImport. Colors: terminal-done green, + failed red, everything in-flight amber. +%> +<% classes = + if import.done? then "bg-green-900/30 text-green-300 border border-green-800/40" + elsif import.failed? then "bg-red-900/30 text-red-300 border border-red-800/40" + else "bg-amber-900/30 text-amber-300 border border-amber-800/40" + end %> +<span class="inline-flex items-center gap-1.5 text-xs px-2 py-0.5 rounded <%= classes %>"> + <% unless import.terminal? %> + <svg class="w-3 h-3 animate-spin" viewBox="0 0 24 24" fill="none"> + <circle class="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" stroke-width="4"></circle> + <path class="opacity-75" fill="currentColor" d="M4 12a8 8 0 018-8V0C5.4 0 0 5.4 0 12h4z"></path> + </svg> + <% end %> + <%= import.status_label %> +</span>

new file mode 100644 index 0000000..d7386f6 --- /dev/null +++ b/app/views/imports/index.html.erb @@ -0,0 +1,28 @@ +<% content_for :title, "Imports" %> + +<div class="max-w-2xl mx-auto px-4 sm:px-6 py-10"> + <div class="flex items-center justify-between mb-6"> + <h1 class="text-xl font-semibold text-gray-100">Imports</h1> + <%= link_to "Import a repo", new_import_path, class: "btn-primary text-xs py-1.5 px-3" %> + </div> + + <% if @imports.empty? %> + <div class="card px-6 py-12 text-center"> + <p class="text-sm text-gray-400 mb-4">You haven't imported anything yet.</p> + <%= link_to "Import from GitHub", new_import_path, class: "btn-primary" %> + </div> + <% else %> + <div class="card divide-y divide-surface-600"> + <% @imports.each do |import| %> + <%= link_to import_path(import), class: "flex items-center justify-between px-4 py-3 hover:bg-surface-600" do %> + <div class="min-w-0"> + <span class="text-sm text-gray-100"><%= import.target_name %></span> + <span class="text-xs text-gray-500 ml-2"><%= import.mirror? ? "mirror" : "migrate" %></span> + <div class="text-xs text-gray-500"><%= time_ago_in_words(import.created_at) %> ago</div> + </div> + <%= render "imports/status_badge", import: import %> + <% end %> + <% end %> + </div> + <% end %> +</div>

new file mode 100644 index 0000000..f67f2f1 --- /dev/null +++ b/app/views/imports/new.html.erb @@ -0,0 +1,155 @@ +<% content_for :title, "Import from GitHub" %> + +<div class="max-w-3xl mx-auto px-4 sm:px-6 py-10"> + <h1 class="text-xl font-semibold text-gray-100 mb-1">Import a repository</h1> + <p class="text-sm text-gray-400 mb-8"> + Bring a repo to siGit with its full history — every branch and tag, no rewrites. + </p> + + <%# ── MIGRATE vs MIRROR explainer ─────────────────────────────────────── %> + <div class="grid grid-cols-1 sm:grid-cols-2 gap-3 mb-8"> + <div class="card px-4 py-3"> + <div class="text-sm font-medium text-gray-100 mb-0.5">Migrate</div> + <p class="text-xs text-gray-400">Copy the repo once. siGit becomes the source of truth and you can push here.</p> + </div> + <div class="card px-4 py-3"> + <div class="text-sm font-medium text-gray-100 mb-0.5">Mirror <span class="badge-gray ml-1">risk-free</span></div> + <p class="text-xs text-gray-400">Keep GitHub as upstream. siGit auto-syncs and stays read-only until you detach.</p> + </div> + </div> + + <%# ── GitHub section ───────────────────────────────────────────────────── %> + <section class="mb-10"> + <div class="flex items-center justify-between mb-3"> + <h2 class="text-base font-semibold text-gray-100">From GitHub</h2> + <% if @connection %> + <div class="flex items-center gap-3 text-xs text-gray-400"> + <span>Connected<%= " as @#{@connection.github_login}" if @connection.github_login %></span> + <%= button_to "Disconnect", github_import_disconnect_path, method: :delete, + class: "text-red-400 hover:underline", form: { class: "inline" } %> + </div> + <% end %> + </div> + + <% unless GithubOauthService.configured? %> + <div class="card px-4 py-3 text-sm text-amber-300 bg-amber-900/10 border-amber-800/40"> + GitHub import isn't configured on this server. You can still import any public repo by URL below. + </div> + <% end %> + + <% if @connection.nil? %> + <div class="card px-6 py-8 text-center"> + <p class="text-sm text-gray-400 mb-4">Connect GitHub to see and pick your repositories.</p> + <div class="flex flex-col sm:flex-row items-center justify-center gap-3"> + <%= link_to github_import_connect_path, class: "btn-primary" do %> + Connect GitHub (incl. private) + <% end %> + <%= link_to github_import_connect_path(visibility: "public"), class: "btn-ghost" do %> + Public repos only + <% end %> + </div> + <p class="text-xs text-gray-500 mt-3">We request the narrowest scope for what you choose. Your token is stored encrypted and never shown.</p> + </div> + <% elsif @github_repos.nil? %> + <div class="card px-4 py-3 text-sm text-amber-300"> + Couldn't load your GitHub repositories right now (rate limit or a hiccup). Try again shortly. + </div> + <% elsif @github_repos.empty? %> + <div class="card px-4 py-3 text-sm text-gray-400">No repositories found on your GitHub account.</div> + <% else %> + <%= form_with url: imports_path, method: :post, data: { turbo: false }, class: "space-y-4" do %> + <input type="hidden" name="source" value="github"> + + <div class="flex items-center gap-4"> + <span class="form-label mb-0">Mode</span> + <label class="flex items-center gap-1.5 text-sm text-gray-200"> + <input type="radio" name="mode" value="migrate" checked> Migrate + </label> + <label class="flex items-center gap-1.5 text-sm text-gray-200"> + <input type="radio" name="mode" value="mirror"> Mirror + </label> + </div> + + <div class="card divide-y divide-surface-600 max-h-96 overflow-y-auto"> + <% @github_repos.each do |repo| %> + <label class="flex items-center gap-3 px-4 py-2.5 hover:bg-surface-600 cursor-pointer"> + <input type="checkbox" name="repos[]" value="<%= repo[:full_name] %>" class="shrink-0"> + <div class="min-w-0 flex-1"> + <div class="flex items-center gap-2"> + <span class="text-sm font-medium text-gray-100 truncate"><%= repo[:full_name] %></span> + <% if repo[:private] %><span class="badge-gray">Private</span><% end %> + <% if repo[:archived] %><span class="badge-gray">Archived</span><% end %> + </div> + <% if repo[:description].present? %> + <p class="text-xs text-gray-400 truncate"><%= repo[:description] %></p> + <% end %> + </div> + <div class="shrink-0 text-right text-xs text-gray-500"> + <div><%= number_to_human_size((repo[:size_kb] || 0) * 1024) %></div> + <% if repo[:pushed_at].present? %> + <div>pushed <%= time_ago_in_words(Time.parse(repo[:pushed_at])) rescue "—" %> ago</div> + <% end %> + </div> + </label> + <% end %> + </div> + + <div class="flex items-center gap-3"> + <%= submit_tag "Import selected", class: "btn-primary cursor-pointer" %> + <span class="text-xs text-gray-500">Select multiple to queue a bulk import.</span> + </div> + <% end %> + <% end %> + </section> + + <%# ── Import by URL ────────────────────────────────────────────────────── %> + <section class="mb-10"> + <h2 class="text-base font-semibold text-gray-100 mb-3">Or import by URL</h2> + <%= form_with url: imports_path, method: :post, data: { turbo: false }, class: "space-y-4" do %> + <input type="hidden" name="source" value="url"> + <div> + <label class="form-label">Public git URL</label> + <input type="url" name="url" class="form-input" placeholder="https://github.com/owner/repo.git" required> + <p class="text-xs text-gray-500 mt-1">Any public http(s) git URL. Private URLs need the GitHub connection above.</p> + </div> + <div class="flex items-end gap-3"> + <div class="flex-1"> + <label class="form-label">Repository name <span class="text-gray-500">(optional)</span></label> + <input type="text" name="name" class="form-input" placeholder="derived from the URL"> + </div> + <div class="flex items-center gap-3 pb-2.5"> + <label class="flex items-center gap-1.5 text-sm text-gray-200"> + <input type="radio" name="mode" value="migrate" checked> Migrate + </label> + <label class="flex items-center gap-1.5 text-sm text-gray-200"> + <input type="radio" name="mode" value="mirror"> Mirror + </label> + </div> + </div> + <%= submit_tag "Import from URL", class: "btn-secondary cursor-pointer" %> + <% end %> + </section> + + <p class="text-xs text-gray-500 mb-8"> + Issues, pull requests, wikis, and releases are <strong>not imported</strong> in this version — only the + git repository, its history, branches, and tags. + </p> + + <%# ── Recent imports ───────────────────────────────────────────────────── %> + <% if @recent_imports.present? %> + <section> + <h2 class="text-base font-semibold text-gray-100 mb-3">Recent imports</h2> + <div class="card divide-y divide-surface-600"> + <% @recent_imports.each do |import| %> + <%= link_to import_path(import), class: "flex items-center justify-between px-4 py-2.5 hover:bg-surface-600" do %> + <div class="min-w-0"> + <span class="text-sm text-gray-100"><%= import.target_name %></span> + <span class="text-xs text-gray-500 ml-2"><%= import.mirror? ? "mirror" : "migrate" %></span> + </div> + <%= render "imports/status_badge", import: import %> + <% end %> + <% end %> + </div> + </section> + <% end %> +</div>

new file mode 100644 index 0000000..b5fb20a --- /dev/null +++ b/app/views/imports/show.html.erb @@ -0,0 +1,84 @@ +<% content_for :title, "Importing #{@import.target_name}" %> + +<%# Live progress: poll every few seconds until the import reaches a terminal + state. A meta refresh keeps this dependency-free (no JS needed) and stops + once done/failed. %> +<% unless @import.terminal? %> + <% content_for :head do %> + <meta http-equiv="refresh" content="4"> + <% end %> +<% end %> + +<div class="max-w-2xl mx-auto px-4 sm:px-6 py-12"> + <div class="flex items-center justify-between mb-6"> + <div> + <h1 class="text-xl font-semibold text-gray-100"><%= @import.target_name %></h1> + <p class="text-sm text-gray-400"> + <%= @import.mirror? ? "Mirror" : "Migrate" %> · from <%= @import.source_url %> + </p> + </div> + <%= render "imports/status_badge", import: @import %> + </div> + + <%# Ordered phase checklist. %> + <% phases = [ + ["queued", "Queued"], + ["cloning", "Cloning source"], + ["pushing", "Pushing to siGit"], + ["fetching_lfs", "Fetching LFS objects"], + ["finalizing", "Finalizing"], + ["done", "Done"] + ] + order = RepositoryImport::STATUSES + current_idx = order.index(@import.status) || 0 %> + + <div class="card divide-y divide-surface-600"> + <% phases.each do |key, label| + idx = order.index(key) + done = !@import.failed? && idx && idx < current_idx + active = @import.status == key %> + <div class="flex items-center gap-3 px-4 py-3"> + <% if done || (@import.done? && key == "done") %> + <svg class="w-4 h-4 text-green-400" viewBox="0 0 16 16" fill="currentColor"><path d="M13.78 4.22a.75.75 0 0 1 0 1.06l-7.25 7.25a.75.75 0 0 1-1.06 0L1.72 8.84a.75.75 0 1 1 1.06-1.06l3.22 3.22 6.72-6.72a.75.75 0 0 1 1.06 0Z"/></svg> + <% elsif active && !@import.terminal? %> + <svg class="w-4 h-4 text-amber-400 animate-spin" viewBox="0 0 24 24" fill="none"><circle class="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" stroke-width="4"></circle><path class="opacity-75" fill="currentColor" d="M4 12a8 8 0 018-8V0C5.4 0 0 5.4 0 12h4z"></path></svg> + <% else %> + <span class="w-4 h-4 rounded-full border border-surface-500 inline-block"></span> + <% end %> + <span class="text-sm <%= done || active ? 'text-gray-100' : 'text-gray-500' %>"><%= label %></span> + </div> + <% end %> + </div> + + <% if @import.failed? %> + <div class="border border-red-800/40 bg-red-900/20 rounded px-4 py-3 mt-6"> + <p class="text-sm font-medium text-red-300 mb-1">Import failed</p> + <p class="text-sm text-red-400"><%= @import.error_message %></p> + <div class="mt-3"> + <%= button_to "Retry import", retry_import_path(@import), method: :post, class: "btn-secondary cursor-pointer" %> + </div> + </div> + <% end %> + + <% if @import.done? && @import.repository %> + <div class="mt-6 space-y-3"> + <div class="flex items-center gap-3"> + <%= link_to "View repository", repository_path(@import.user.username, @import.repository.name), class: "btn-primary" %> + <% if @import.mirror? %> + <span class="badge-gray">Read-only mirror</span> + <% end %> + </div> + <% if @import.lfs_detected %> + <p class="text-xs <%= @import.lfs_skipped ? 'text-amber-400' : 'text-gray-400' %>"> + <% if @import.lfs_skipped %> + Git LFS was detected but its large objects were <strong>not fetched</strong> (LFS unavailable on this server). + Pointer files and full history were imported. + <% else %> + Git LFS objects were fetched and imported. + <% end %> + </p> + <% end %> + <p class="text-xs text-gray-500">Issues, pull requests, and releases were not imported.</p> + </div> + <% end %> +</div>

index 72fe5d6..37a42a1 100644 --- a/app/views/layouts/application.html.erb +++ b/app/views/layouts/application.html.erb @@ -14,6 +14,7 @@ <%= stylesheet_link_tag "tailwind", "data-turbo-track": "reload" %> <%= javascript_importmap_tags %> <%= csrf_meta_tags %> + <%= yield :head %> </head> <body class="h-full bg-surface-900">

index ead4c12..6fa6793 100644 --- a/app/views/pages/dashboard.html.erb +++ b/app/views/pages/dashboard.html.erb @@ -57,8 +57,15 @@ <path stroke-linecap="round" stroke-linejoin="round" d="M3.75 9.776c.112-.017.227-.026.344-.026h15.812c.117 0 .232.009.344.026m-16.5 0a2.25 2.25 0 0 0-1.883 2.542l.857 6a2.25 2.25 0 0 0 2.227 1.932H19.05a2.25 2.25 0 0 0 2.227-1.932l.857-6a2.25 2.25 0 0 0-1.883-2.542m-16.5 0V6A2.25 2.25 0 0 1 6 3.75h3.879a1.5 1.5 0 0 1 1.06.44l2.122 2.12a1.5 1.5 0 0 0 1.06.44H18A2.25 2.25 0 0 1 20.25 9v.776"/> </svg> - <p class="text-sm text-gray-400 mb-4">No repositories yet.</p> - <%= link_to new_repository_path, class: "btn-primary" do %>Create your first repository<% end %> + <p class="text-sm text-gray-100 font-medium mb-1">Import your first repo from GitHub</p> + <p class="text-sm text-gray-400 mb-5 max-w-sm mx-auto"> + Bring your code over with its full history — or mirror it read-only and keep + GitHub as upstream to try siGit risk-free. + </p> + <div class="flex items-center justify-center gap-3"> + <%= link_to new_import_path, class: "btn-primary" do %>Import from GitHub<% end %> + <%= link_to new_repository_path, class: "btn-ghost" do %>Start from scratch<% end %> + </div> </div> <% end %> </main>

index c7eee46..aa9fa17 100644 --- a/app/views/repositories/show.html.erb +++ b/app/views/repositories/show.html.erb @@ -17,8 +17,33 @@ <% if @repository.is_private %> <span class="badge-gray ml-1">Private</span> <% end %> + <% if @repository.mirror? %> + <span class="badge-gray ml-1">Mirror · read-only</span> + <% end %> </div> + <% if @repository.mirror? %> + <div class="border border-surface-500 bg-surface-700/40 rounded px-4 py-3 mb-4 flex flex-wrap items-center justify-between gap-3"> + <div class="min-w-0"> + <p class="text-sm text-gray-200"> + Mirrored from <span class="text-gray-100 break-all"><%= @repository.upstream_url %></span> + </p> + <p class="text-xs <%= @repository.mirror_status == 'failed' ? 'text-red-400' : 'text-gray-500' %>"> + <%= @repository.mirror_state_label %><%= " — #{@repository.mirror_error}" if @repository.mirror_status == 'failed' && @repository.mirror_error.present? %> + </p> + </div> + <% if signed_in? && current_user == @owner %> + <div class="flex items-center gap-2 shrink-0"> + <%= button_to "Sync now", repository_mirror_sync_path(@owner.username, @repository.name), + method: :post, class: "btn-ghost text-xs py-1 px-2 cursor-pointer" %> + <%= button_to "Detach", repository_mirror_detach_path(@owner.username, @repository.name), + method: :post, class: "btn-secondary text-xs py-1 px-2 cursor-pointer", + form: { data: { turbo_confirm: "Detach this mirror? It becomes a normal writable repo and stops syncing from upstream." } } %> + </div> + <% end %> + </div> + <% end %> + <% if @repository.description.present? %> <p class="text-sm text-gray-400 mb-4"><%= @repository.description %></p> <% end %>

index 675cd1f..865d336 100644 --- a/app/views/shared/_navbar.html.erb +++ b/app/views/shared/_navbar.html.erb @@ -45,6 +45,8 @@ </div> <%= link_to user_profile_path(current_user.username), class: "block px-3 py-1.5 text-sm text-gray-300 hover:bg-surface-600" do %>Your repositories<% end %> + <%= link_to new_import_path, + class: "block px-3 py-1.5 text-sm text-gray-300 hover:bg-surface-600" do %>Import from GitHub<% end %> <%= link_to settings_path, class: "block px-3 py-1.5 text-sm text-gray-300 hover:bg-surface-600" do %>Settings<% end %> <div class="border-t border-surface-600 mt-1">

index cb0241d..9fda9b1 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -50,7 +50,11 @@ Rails.application.configure do # config.cache_store = :mem_cache_store # Replace the default in-process and non-durable queuing backend for Active Job. - # config.active_job.queue_adapter = :resque + # Solid Queue (database-backed) runs the long-running repository imports and + # mirror syncs off the web tier. Recurring tasks are declared in + # config/recurring.yml. + config.active_job.queue_adapter = :solid_queue + config.solid_queue.connects_to = { database: { writing: :queue } } # Ignore bad email addresses and do not raise email delivery errors. # Set this to true and configure the email server for immediate delivery to raise delivery errors.

new file mode 100644 index 0000000..b899354 --- /dev/null +++ b/config/initializers/active_record_encryption.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +# Active Record Encryption keys for data encrypted at rest — currently the +# per-user GitHub OAuth token (GithubConnection#access_token) and the upstream +# mirror credential (Repository#upstream_token). +# +# Keys come from the environment so real secrets never live in the repo: +# +# ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY +# ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY +# ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT +# +# In production these MUST be set (generate with `bin/rails db:encryption:init`); +# we fail loudly at boot if they're missing rather than silently storing tokens +# under a predictable key. In development and test we derive deterministic keys +# from secret_key_base so the app and specs run without extra setup — never used +# for real user tokens. +Rails.application.configure do + enc = config.active_record.encryption + + primary = ENV["ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY"] + deterministic = ENV["ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY"] + salt = ENV["ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT"] + + if primary.blank? || deterministic.blank? || salt.blank? + if Rails.env.production? + raise "Active Record Encryption keys are not configured. Set " \ + "ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY, " \ + "ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY and " \ + "ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT " \ + "(generate with `bin/rails db:encryption:init`)." + end + + base = Rails.application.secret_key_base.to_s + base = "sigitsi-dev-encryption" if base.blank? + derive = ->(label) { Digest::SHA256.hexdigest("#{label}:#{base}")[0, 32] } + primary ||= derive.call("ar-enc-primary") + deterministic ||= derive.call("ar-enc-deterministic") + salt ||= derive.call("ar-enc-salt") + end + + enc.primary_key = primary + enc.deterministic_key = deterministic + enc.key_derivation_salt = salt + # Tokens are opaque and never queried by ciphertext, so non-deterministic + # (randomized IV) encryption is the safer default. + enc.support_unencrypted_data = false +end

new file mode 100644 index 0000000..a6d5110 --- /dev/null +++ b/config/recurring.yml @@ -0,0 +1,15 @@ +# Solid Queue recurring tasks. +# https://github.com/rails/solid_queue#recurring-tasks +# +# The scheduler enqueues a MirrorSyncJob for every mirror repo that is due for a +# refresh (see MirrorSyncSchedulerJob). Webhooks, when configured, sync sooner. + +production: + mirror_sync: + class: MirrorSyncSchedulerJob + schedule: every 15 minutes + +development: + mirror_sync: + class: MirrorSyncSchedulerJob + schedule: every 15 minutes

index 60e4f60..1cf66f1 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -67,6 +67,25 @@ Rails.application.routes.draw do get "/new", to: "repositories#new", as: :new_repository post "/new", to: "repositories#create" + # Import from GitHub (mirror + migrate). Declared before the "/:username" + # matcher so "/import" isn't read as a profile. + # + # The GitHub OAuth connect routes are declared before the resources block so + # "/import/github/*" isn't swallowed by the "/import/:id" show route. This is + # our own GitHub OAuth app (separate from smbCloud sign-in); the token it mints + # can read/clone the user's GitHub repos. + get "/import/github/connect", to: "github_connections#connect", as: :github_import_connect + get "/import/github/callback", to: "github_connections#callback", as: :github_import_callback + delete "/import/github/disconnect", to: "github_connections#disconnect", as: :github_import_disconnect + + resources :imports, path: "import", only: %i[index new create show], + constraints: { id: /\d+/ } do + post :retry, on: :member + end + + # GitHub push webhook → immediate mirror sync (HMAC-authenticated). + post "/webhooks/github", to: "github_webhooks#create" + # Settings get "/settings", to: "users#settings", as: :settings patch "/settings", to: "users#update_settings" @@ -141,6 +160,10 @@ Rails.application.routes.draw do get "/:username/:repository/tree/:branch", to: "repositories#tree", as: :repository_branch post "/:username/:repository/star", to: "stars#create", as: :repository_star, constraints: { repository: /[^\/.][^\/]*/ }, format: false + post "/:username/:repository/mirror/detach", to: "mirrors#detach", as: :repository_mirror_detach, + constraints: { repository: /[^\/.][^\/]*/ }, format: false + post "/:username/:repository/mirror/sync", to: "mirrors#sync", as: :repository_mirror_sync, + constraints: { repository: /[^\/.][^\/]*/ }, format: false # `format: false` keeps dotted names (e.g. "Qwen2.5-3B-Instruct-GGUF") intact # rather than treating the dot as a response-format separator. get "/:username/:repository", to: "repositories#show", as: :repository,

new file mode 100644 index 0000000..2476cd7 --- /dev/null +++ b/db/migrate/20250101000011_create_github_connections.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +# Per-user GitHub OAuth connection used by the "Import from GitHub" flow. +# +# This is deliberately separate from the smbCloud-brokered "Continue with +# GitHub" sign-in: that gives us a smbCloud session, not a GitHub API token. +# Importing needs a real GitHub token (scoped `repo` or `public_repo`) to list +# private repos and clone them, so we run our own minimal OAuth app and store +# the resulting token here — encrypted at rest, one row per user, revocable. +class CreateGithubConnections < ActiveRecord::Migration[8.1] + def change + create_table :github_connections do |t| + t.references :user, null: false, foreign_key: true, index: { unique: true } + t.string :github_login + # Encrypted at rest via ActiveRecord::Encryption (`encrypts :access_token`). + # text, not string: encrypted + base64 ciphertext is longer than the token. + t.text :access_token, null: false + t.string :scope + t.string :token_type, default: "bearer", null: false + t.datetime :connected_at + + t.timestamps + end + end +end

new file mode 100644 index 0000000..85ad7b6 --- /dev/null +++ b/db/migrate/20250101000012_create_repository_imports.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +# Tracks one "import a repo into siGit" operation end to end so the UI can show +# live progress and a failed import can be retried. The heavy lifting (clone, +# push, LFS) runs off the request thread in RepositoryImportJob; this row is the +# durable state machine it drives. +class CreateRepositoryImports < ActiveRecord::Migration[8.1] + def change + create_table :repository_imports do |t| + t.references :user, null: false, foreign_key: true + # Set once the target repo exists on disk; nil while queued/cloning and + # after a cleaned-up failure. Nullified if the repo is later destroyed. + t.references :repository, null: true, foreign_key: { on_delete: :nullify } + + t.string :source_url, null: false # normalized https git URL + t.string :source_host # e.g. "github.com" + t.string :mode, null: false, default: "migrate" # migrate | mirror + t.string :status, null: false, default: "queued" # see RepositoryImport::STATUSES + t.text :error_message + + # Target repo attributes captured up front so a retry is deterministic and + # doesn't depend on re-reading GitHub. + t.string :target_name, null: false + t.boolean :target_private, null: false, default: false + t.string :default_branch + t.text :description + + # Best-effort LFS accounting surfaced in the UI. + t.boolean :lfs_detected, null: false, default: false + t.boolean :lfs_skipped, null: false, default: false + + t.bigint :source_size_kb # reported by GitHub, for size caps + t.datetime :started_at + t.datetime :finished_at + + t.timestamps + end + + add_index :repository_imports, [ :user_id, :status ] + end +end

new file mode 100644 index 0000000..39c1d91 --- /dev/null +++ b/db/migrate/20250101000013_add_mirror_to_repositories.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +# Mirror mode: a repository that keeps an upstream (e.g. GitHub) as the source +# of truth and auto-syncs from it. Mirrors are read-only on siGit (pushes are +# rejected) to avoid divergence, until the owner "detaches" them into a normal +# writable repo. +class AddMirrorToRepositories < ActiveRecord::Migration[8.1] + def change + change_table :repositories, bulk: true do |t| + t.boolean :mirror, null: false, default: false + t.string :upstream_url # the remote we fetch from + # Encrypted upstream credential for private mirrors (ActiveRecord:: + # Encryption). Nil for public upstreams. Never returned to the client. + t.text :upstream_token + t.string :mirror_status # ok | syncing | failed + t.datetime :mirror_synced_at + t.text :mirror_error + end + + add_index :repositories, :mirror + # Due mirrors are polled by the recurring MirrorSyncScheduler; index the + # columns it filters/orders on. + add_index :repositories, [ :mirror, :mirror_synced_at ] + end +end

index 182a008..2bcd350 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.1].define(version: 2025_01_01_000010) do +ActiveRecord::Schema[8.1].define(version: 2025_01_01_000013) do # These are extensions that must be enabled in order to support this database enable_extension "pg_catalog.plpgsql" @@ -56,6 +56,18 @@ ActiveRecord::Schema[8.1].define(version: 2025_01_01_000010) do t.index ["user_id"], name: "index_comments_on_user_id" end + create_table "github_connections", force: :cascade do |t| + t.text "access_token", null: false + t.datetime "connected_at" + t.datetime "created_at", null: false + t.string "github_login" + t.string "scope" + t.string "token_type", default: "bearer", null: false + t.datetime "updated_at", null: false + t.bigint "user_id", null: false + t.index ["user_id"], name: "index_github_connections_on_user_id", unique: true + end + create_table "issues", force: :cascade do |t| t.text "body" t.datetime "closed_at" @@ -96,15 +108,47 @@ ActiveRecord::Schema[8.1].define(version: 2025_01_01_000010) do t.integer "downloads_count", default: 0, null: false t.boolean "is_private", default: false, null: false t.string "kind", default: "code", null: false + t.boolean "mirror", default: false, null: false + t.text "mirror_error" + t.string "mirror_status" + t.datetime "mirror_synced_at" t.string "name", null: false t.integer "stars_count", default: 0, null: false t.datetime "updated_at", null: false + t.text "upstream_token" + t.string "upstream_url" t.bigint "user_id", null: false t.index ["kind"], name: "index_repositories_on_kind" + t.index ["mirror", "mirror_synced_at"], name: "index_repositories_on_mirror_and_mirror_synced_at" + t.index ["mirror"], name: "index_repositories_on_mirror" t.index ["user_id", "name"], name: "index_repositories_on_user_id_and_name", unique: true t.index ["user_id"], name: "index_repositories_on_user_id" end + create_table "repository_imports", force: :cascade do |t| + t.datetime "created_at", null: false + t.string "default_branch" + t.text "description" + t.text "error_message" + t.datetime "finished_at" + t.boolean "lfs_detected", default: false, null: false + t.boolean "lfs_skipped", default: false, null: false + t.string "mode", default: "migrate", null: false + t.bigint "repository_id" + t.string "source_host" + t.bigint "source_size_kb" + t.string "source_url", null: false + t.datetime "started_at" + t.string "status", default: "queued", null: false + t.string "target_name", null: false + t.boolean "target_private", default: false, null: false + t.datetime "updated_at", null: false + t.bigint "user_id", null: false + t.index ["repository_id"], name: "index_repository_imports_on_repository_id" + t.index ["user_id", "status"], name: "index_repository_imports_on_user_id_and_status" + t.index ["user_id"], name: "index_repository_imports_on_user_id" + end + create_table "ssh_keys", force: :cascade do |t| t.datetime "created_at", null: false t.string "fingerprint", null: false @@ -157,11 +201,14 @@ ActiveRecord::Schema[8.1].define(version: 2025_01_01_000010) do add_foreign_key "cloud_sessions", "users" add_foreign_key "cloud_usages", "users" add_foreign_key "comments", "users" + add_foreign_key "github_connections", "users" add_foreign_key "issues", "repositories" add_foreign_key "issues", "users" add_foreign_key "pull_requests", "repositories" add_foreign_key "pull_requests", "users" add_foreign_key "repositories", "users" + add_foreign_key "repository_imports", "repositories", on_delete: :nullify + add_foreign_key "repository_imports", "users" add_foreign_key "ssh_keys", "users" add_foreign_key "stars", "repositories" add_foreign_key "stars", "users"

new file mode 100644 index 0000000..6e0ed35 --- /dev/null +++ b/docs/import-from-github.md @@ -0,0 +1,123 @@ +# Import from GitHub (mirror + migrate) + +One-click import of a Git repository into siGit, in two modes: + +- **Migrate** — copy the repo once; siGit becomes the source of truth and you can + push to it. +- **Mirror** — keep GitHub (or any upstream) as the source of truth; siGit + auto-syncs on a schedule (and on webhook, if configured) and stays **read-only** + until you *detach* it. + +There is also **import by URL** for any public `http(s)` git URL, with no OAuth. + +## How it fits the existing app + +- Repos are still bare repos on disk under `SIGITSI_REPOS_PATH/<user>/<name>.git`, + created via `GitRepositoryService` — the importer reuses that path, it does not + invent new storage. +- The long-running clone/push runs off the request thread in + `RepositoryImportJob` (Solid Queue). Mirror syncs run in `MirrorSyncJob`, + swept by the recurring `MirrorSyncSchedulerJob` (`config/recurring.yml`). +- GitHub API access uses a **dedicated GitHub OAuth app** (env + `GITHUB_IMPORT_CLIENT_ID` / `GITHUB_IMPORT_CLIENT_SECRET`), separate from the + smbCloud "Continue with GitHub" *sign-in* (which yields no GitHub API token). + The token is stored encrypted (`GithubConnection#access_token`, Active Record + Encryption) and never returned to the client. + +## Key files + +| Area | File | +|------|------| +| SSRF URL guard | `app/services/import_url_validator.rb` | +| GitHub OAuth | `app/services/github_oauth_service.rb`, `app/controllers/github_connections_controller.rb` | +| GitHub REST | `app/services/github_api_client.rb` | +| Clone + push | `app/services/repository_import_service.rb` | +| Import orchestration | `app/jobs/repository_import_job.rb`, `app/controllers/imports_controller.rb` | +| Mirror sync | `app/services/mirror_sync_service.rb`, `app/jobs/mirror_sync_job.rb`, `app/jobs/mirror_sync_scheduler_job.rb` | +| Mirror read-only | `app/controllers/git_http_controller.rb` (`reject_mirror_push!`) | +| Detach / manual sync | `app/controllers/mirrors_controller.rb` | +| Webhook | `app/controllers/github_webhooks_controller.rb` | + +## Guardrails + +- **SSRF:** import-by-URL allows only `http(s)`; loopback / RFC1918 / link-local + (incl. `169.254.169.254`) / IPv6 ULA + mapped-v4 are rejected, for literal IPs + and every resolved address. Mirrors re-validate the upstream on every sync + (DNS-rebinding window). +- **Isolation:** clones run in a temp dir that's always removed; every git call + has a hard timeout and its process group is killed on overrun; + `GIT_TERMINAL_PROMPT=0` makes auth failures fail fast. +- **Size/abuse:** per-repo size cap, per-user in-flight volume cap, and a + concurrent-import cap (`IMPORT_MAX_*` env). Oversized repos are skipped with a + message. +- **Secrets:** OAuth + upstream tokens are encrypted at rest, scrubbed from + captured git output, never logged, never sent to the client, dropped on + disconnect/detach. +- **Idempotency:** a failed import leaves no half-created repo (rollback) and is + retryable; retries clear any partial repo; name collisions with a *different* + existing repo fail explicitly; bulk imports auto-suffix colliding names. +- **Rate limits:** `GithubApiClient` raises `RateLimited` with a reset time so + callers back off; the repo list is cached briefly and never blocks the page. + +Not imported in v1 (surfaced in the UI, not silently dropped): issues, pull +requests, wikis, releases, Actions. + +## Configuration + +See `.env.example` (the "Import from GitHub" and "Active Record Encryption" +sections). Minimum for full functionality: + +``` +GITHUB_IMPORT_CLIENT_ID=... +GITHUB_IMPORT_CLIENT_SECRET=... +# production only: +ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY=... +ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY=... +ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT=... # bin/rails db:encryption:init +# optional: GITHUB_WEBHOOK_SECRET, IMPORT_MAX_*, MIRROR_SYNC_* +``` + +Register the GitHub OAuth app with callback `<SIGITSI_URL>/import/github/callback`. + +## Verify locally + +```sh +bin/rails db:prepare +bin/rails tailwindcss:build # so views render in specs + +# Full suite (all green): +bundle exec rspec + +# Just this feature: +bundle exec rspec \ + spec/services/import_url_validator_spec.rb \ + spec/models/github_connection_spec.rb \ + spec/services/repository_import_service_spec.rb \ + spec/jobs/repository_import_job_spec.rb \ + spec/requests/github_connections_spec.rb \ + spec/requests/git_http_mirror_spec.rb +``` + +Manual smoke test (no GitHub app needed — uses import-by-URL against a local +fixture, so nothing hits the network): + +```sh +# 1. Build a tiny bare fixture repo with a branch + tag. +tmp=$(mktemp -d); git init -q -b main "$tmp/w" +git -C "$tmp/w" -c user.email=a@b.c -c user.name=t commit -q --allow-empty -m init +git -C "$tmp/w" tag v1 +git clone -q --bare "$tmp/w" "$tmp/src.git" + +# 2. In `bin/rails console`, queue and run an import synchronously: +# u = User.first +# imp = u.repository_imports.create!(source_url: "<tmp>/src.git", mode: "migrate", +# target_name: "fixture", status: "queued") +# RepositoryImportJob.perform_now(imp.id) +# imp.reload.status # => "done" +# imp.repository.default_branch # => "main"; branches/tags preserved +``` + +For mirror mode, set `mode: "mirror"`; the resulting repo is read-only +(`repo.writable_by?(u) == false`) and a `git push` to it is rejected with a +message. Use the repo page's **Detach** to convert it to a normal writable repo +and stop syncing.

new file mode 100644 index 0000000..a9059c5 --- /dev/null +++ b/spec/jobs/repository_import_job_spec.rb @@ -0,0 +1,147 @@ +# frozen_string_literal: true + +require "rails_helper" +require "tmpdir" +require "fileutils" + +RSpec.describe RepositoryImportJob, type: :job do + let(:user) { User.create!(smbcloud_id: 7, email: "imp@example.com", username: "importer") } + + around do |example| + Dir.mktmpdir("import-job-spec") do |tmp| + @tmp = tmp + @repos = File.join(tmp, "repos") + @source = build_source(File.join(tmp, "src")) + example.run + end + end + + before do + # Point repo storage at the sandbox for both the service and the job. + allow(GitRepositoryService).to receive(:repo_path) do |u, r| + File.join(@repos, u, "#{r}.git") + end + end + + def build_source(dir) + work = File.join(dir, "work") + FileUtils.mkdir_p(work) + system("git", "init", "-q", "-b", "main", work, exception: true) + system("git", "-C", work, "config", "user.email", "t@e.st", exception: true) + system("git", "-C", work, "config", "user.name", "T", exception: true) + File.write(File.join(work, "README.md"), "# Imported\n\nHello.\n") + system("git", "-C", work, "add", ".", exception: true) + system("git", "-C", work, "commit", "-qm", "init", exception: true) + system("git", "-C", work, "tag", "v1", exception: true) + system("git", "-C", work, "checkout", "-q", "-b", "dev", exception: true) + File.write(File.join(work, "d.txt"), "d\n") + system("git", "-C", work, "add", ".", exception: true) + system("git", "-C", work, "commit", "-qm", "dev", exception: true) + system("git", "-C", work, "checkout", "-q", "main", exception: true) + bare = File.join(dir, "source.git") + system("git", "clone", "-q", "--bare", work, bare, exception: true) + bare + end + + def new_import(attrs = {}) + user.repository_imports.create!({ + source_url: @source, source_host: nil, mode: "migrate", + target_name: "imported", target_private: false, default_branch: "main", + status: "queued" + }.merge(attrs)) + end + + it "migrates a repo with all branches, tags, history and a working default branch" do + import = new_import + + described_class.perform_now(import.id) + import.reload + + expect(import.status).to eq("done") + repo = import.repository + expect(repo).to be_present + expect(repo).to be_initialized + + branches = GitRepositoryService.branches(repo.disk_path).sort + expect(branches).to eq(%w[dev main]) + expect(`git --git-dir #{repo.disk_path} tag`.split).to eq(%w[v1]) + expect(GitRepositoryService.default_branch(repo.disk_path)).to eq("main") + # README is reachable on the default branch → landing page will render it. + expect(GitRepositoryService.readme_content(repo.disk_path, "main")).to be_present + end + + it "sets up a read-only mirror in mirror mode" do + import = new_import(mode: "mirror") + + described_class.perform_now(import.id) + import.reload + repo = import.repository + + expect(import.status).to eq("done") + expect(repo.mirror?).to be(true) + expect(repo.upstream_url).to eq(@source) + expect(repo.mirror_status).to eq("ok") + expect(repo.mirror_synced_at).to be_present + expect(repo.writable_by?(user)).to be(false) + end + + it "leaves no half-created repo behind when the import fails, and stays retryable" do + import = new_import + allow(RepositoryImportService).to receive(:clone_and_push) + .and_raise(RepositoryImportService::ImportError, "boom") + + described_class.perform_now(import.id) + import.reload + + expect(import.status).to eq("failed") + expect(import.error_message).to eq("boom") + expect(import.repository).to be_nil + expect(user.repositories.count).to eq(0) + expect(Dir.exist?(File.join(@repos, user.username, "imported.git"))).to be(false) + expect(import.retryable?).to be(true) + end + + it "rejects a name collision with a different existing repo, without touching it" do + existing = user.repositories.create!( + name: "imported", disk_path: GitRepositoryService.repo_path(user.username, "imported"), + default_branch: "main", description: "the original" + ) + import = new_import + + described_class.perform_now(import.id) + import.reload + + expect(import.status).to eq("failed") + expect(import.error_message).to match(/already exists/) + expect(existing.reload.description).to eq("the original") + expect(user.repositories.count).to eq(1) + end + + it "is idempotent against double delivery (already-done import is a no-op)" do + import = new_import + described_class.perform_now(import.id) + described_class.perform_now(import.id) # second delivery + + expect(user.repositories.where(name: "imported").count).to eq(1) + expect(import.reload.status).to eq("done") + end + + it "clears a partial repo from a previous attempt when retrying" do + # Simulate a prior failed attempt that left a repo attached. + partial = user.repositories.create!( + name: "imported", disk_path: GitRepositoryService.repo_path(user.username, "imported"), + default_branch: "main" + ) + FileUtils.mkdir_p(partial.disk_path) + import = new_import(status: "failed", repository: partial) + + described_class.perform_now(import.id) + import.reload + + expect(import.status).to eq("done") + expect(Repository.where(id: partial.id)).to be_empty # old partial gone + expect(import.repository).to be_present + expect(import.repository).to be_initialized # fresh, clean import + expect(user.repositories.where(name: "imported").count).to eq(1) + end +end

new file mode 100644 index 0000000..b4c0bed --- /dev/null +++ b/spec/models/github_connection_spec.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +require "rails_helper" + +RSpec.describe GithubConnection, type: :model do + let(:user) { User.create!(smbcloud_id: 42, email: "gh@example.com", username: "ghuser") } + + it "stores the access token encrypted at rest, not in plaintext" do + conn = user.create_github_connection!(access_token: "gho_secrettoken123", scope: "repo") + + # The model round-trips the real value... + expect(conn.reload.access_token).to eq("gho_secrettoken123") + + # ...but the raw column holds ciphertext, never the plaintext token. + raw = ActiveRecord::Base.connection.select_value( + "SELECT access_token FROM github_connections WHERE id = #{conn.id}" + ) + expect(raw).not_to include("gho_secrettoken123") + expect(raw).to be_present + end + + it "requires an access token" do + conn = user.build_github_connection(access_token: nil) + expect(conn).not_to be_valid + expect(conn.errors[:access_token]).to be_present + end + + it "is unique per user" do + user.create_github_connection!(access_token: "a") + dup = GithubConnection.new(user_id: user.id, access_token: "b") + expect(dup).not_to be_valid + expect(dup.errors[:user_id]).to be_present + end + + it "detects private scope" do + expect(user.build_github_connection(access_token: "a", scope: "repo").private_scope?).to be(true) + expect(user.build_github_connection(access_token: "a", scope: "public_repo").private_scope?).to be(false) + end +end

new file mode 100644 index 0000000..e5f2583 --- /dev/null +++ b/spec/requests/git_http_mirror_spec.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +require "rails_helper" +require "tmpdir" +require "fileutils" + +# Mirror repos are read-only over git: a push (receive-pack) must be rejected +# with a message the user actually sees, while clone/fetch and normal repos are +# unaffected. +RSpec.describe "Git HTTP mirror read-only enforcement", type: :request do + let(:user) { User.create!(smbcloud_id: 55, email: "m@example.com", username: "mirroruser") } + + around do |example| + Dir.mktmpdir("git-http-mirror") do |tmp| + @bare = File.join(tmp, "mirrored.git") + system("git", "init", "--bare", "-q", @bare, exception: true) + example.run + end + end + + let!(:mirror) do + user.repositories.create!( + name: "mirrored", disk_path: @bare, default_branch: "main", + mirror: true, upstream_url: "https://github.com/owner/mirrored.git", mirror_status: "ok" + ) + end + + it "rejects a push to a mirror with a clear git error message" do + get "/mirroruser/mirrored.git/info/refs", params: { service: "git-receive-pack" } + + expect(response).to have_http_status(:ok) + expect(response.body).to include("ERR") + expect(response.body).to include("read-only mirror") + expect(response.body).to include("Detach") + end + + it "also blocks the receive-pack RPC endpoint directly" do + post "/mirroruser/mirrored.git/git-receive-pack" + # The advertisement carries the ERR packet rather than proxying to git. + expect(response.body).to include("read-only mirror") + end + + it "still allows fetch/clone (upload-pack) from a public mirror" do + get "/mirroruser/mirrored.git/info/refs", params: { service: "git-upload-pack" } + expect(response).to have_http_status(:ok) + expect(response.body).not_to include("read-only mirror") + end + + it "does not block pushes on a normal (non-mirror) repo" do + normal_dir = File.join(Dir.tmpdir, "normal-#{SecureRandom.hex(4)}.git") + system("git", "init", "--bare", "-q", normal_dir, exception: true) + user.repositories.create!(name: "normal", disk_path: normal_dir, default_branch: "main") + + get "/mirroruser/normal.git/info/refs", params: { service: "git-receive-pack" } + # No mirror rejection; instead it falls through to auth (401 challenge). + expect(response.body).not_to include("read-only mirror") + expect(response).to have_http_status(:unauthorized) + ensure + FileUtils.rm_rf(normal_dir) if normal_dir + end +end

new file mode 100644 index 0000000..f127614 --- /dev/null +++ b/spec/requests/github_connections_spec.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +require "rails_helper" + +# The GitHub OAuth connect flow for imports: the callback must verify state, +# exchange the code, store the token encrypted, and never leak it to the client. +RSpec.describe "GitHub import connection", type: :request do + let(:user) { User.create!(smbcloud_id: 99, email: "conn@example.com", username: "connuser") } + + before do + # Sign the user in (session-based auth) without going through smbCloud. + allow_any_instance_of(ApplicationController).to receive(:current_user).and_return(user) + allow(GithubOauthService).to receive(:configured?).and_return(true) + allow(GithubOauthService).to receive(:generate_state).and_return("state-xyz") + allow(GithubOauthService).to receive(:authorize_url).and_return("https://github.com/login/oauth/authorize?x=1") + end + + # Establishes the CSRF state in the session, the way the real flow does. + def start_connect + get github_import_connect_path + expect(response).to redirect_to("https://github.com/login/oauth/authorize?x=1") + end + + it "exchanges the code and stores the token encrypted on callback" do + start_connect + allow(GithubOauthService).to receive(:exchange_code) + .and_return(access_token: "gho_thetoken", scope: "repo", token_type: "bearer") + fake_client = instance_double(GithubApiClient, login: "octocat", list_repositories: []) + allow(GithubApiClient).to receive(:new).with("gho_thetoken").and_return(fake_client) + + get github_import_callback_path, params: { code: "abc", state: "state-xyz" } + + expect(response).to redirect_to(new_import_path) + conn = user.reload.github_connection + expect(conn).to be_present + expect(conn.access_token).to eq("gho_thetoken") + expect(conn.github_login).to eq("octocat") + + # Ciphertext at rest, not the plaintext token. + raw = ActiveRecord::Base.connection.select_value( + "SELECT access_token FROM github_connections WHERE id = #{conn.id}" + ) + expect(raw).not_to include("gho_thetoken") + + # The token is never echoed back to the client. + expect(response.body).not_to include("gho_thetoken") + follow_redirect! + expect(response.body).not_to include("gho_thetoken") + end + + it "rejects a callback whose state doesn't match (CSRF)" do + start_connect + expect(GithubOauthService).not_to receive(:exchange_code) + + get github_import_callback_path, params: { code: "abc", state: "wrong-state" } + + expect(response).to redirect_to(new_import_path) + expect(user.reload.github_connection).to be_nil + end + + it "handles a user cancelling the GitHub authorization" do + get github_import_callback_path, params: { error: "access_denied" } + expect(response).to redirect_to(new_import_path) + expect(user.reload.github_connection).to be_nil + end + + it "disconnects and removes the stored token" do + user.create_github_connection!(access_token: "gho_x", scope: "repo") + delete github_import_disconnect_path + expect(user.reload.github_connection).to be_nil + end +end

new file mode 100644 index 0000000..6443b4d --- /dev/null +++ b/spec/services/import_url_validator_spec.rb @@ -0,0 +1,80 @@ +# frozen_string_literal: true + +require "rails_helper" + +# The SSRF guard for import-by-URL. These are the checks that stop a user from +# pointing the importer at cloud metadata or an internal service. +RSpec.describe ImportUrlValidator do + describe ".validate!" do + it "accepts a public https git URL" do + expect(described_class.validate!("https://192.0.2.10/owner/repo.git")) + .to eq("https://192.0.2.10/owner/repo.git") + end + + it "rejects non-http(s) schemes" do + %w[ + ssh://git@github.com/x.git + git://github.com/x.git + file:///etc/passwd + ftp://example.com/x.git + ].each do |url| + expect { described_class.validate!(url) } + .to raise_error(ImportUrlValidator::InvalidUrl), "expected #{url} to be rejected" + end + end + + it "rejects the cloud metadata address" do + expect { described_class.validate!("http://169.254.169.254/latest/meta-data/") } + .to raise_error(ImportUrlValidator::InvalidUrl, /private or internal/) + end + + it "rejects loopback and RFC1918 literal IPs" do + %w[ + http://127.0.0.1/x.git + http://10.0.0.5/x.git + http://192.168.1.1/x.git + http://172.16.9.9/x.git + ].each do |url| + expect { described_class.validate!(url) } + .to raise_error(ImportUrlValidator::InvalidUrl), "expected #{url} to be rejected" + end + end + + it "rejects IPv6 loopback and IPv4-mapped private addresses" do + expect { described_class.validate!("http://[::1]/x.git") } + .to raise_error(ImportUrlValidator::InvalidUrl) + expect { described_class.validate!("http://[::ffff:10.0.0.1]/x.git") } + .to raise_error(ImportUrlValidator::InvalidUrl) + end + + it "rejects credentials embedded in the URL" do + expect { described_class.validate!("https://user:pass@github.com/x.git") } + .to raise_error(ImportUrlValidator::InvalidUrl, /credentials/) + end + + it "rejects a host that resolves to a blocked address" do + allow(described_class).to receive(:resolve).and_return([ "127.0.0.1" ]) + expect { described_class.validate!("https://internal.example.com/x.git") } + .to raise_error(ImportUrlValidator::InvalidUrl, /private or internal/) + end + + it "accepts a host that resolves to a public address" do + allow(described_class).to receive(:resolve).and_return([ "140.82.112.3" ]) + expect(described_class.validate!("https://github.com/rails/rails.git")) + .to eq("https://github.com/rails/rails.git") + end + + it "rejects blank input" do + expect { described_class.validate!("") }.to raise_error(ImportUrlValidator::InvalidUrl) + expect { described_class.validate!(nil) }.to raise_error(ImportUrlValidator::InvalidUrl) + end + end + + describe ".safe?" do + it "is true/false without raising" do + allow(described_class).to receive(:resolve).and_return([ "140.82.112.3" ]) + expect(described_class.safe?("https://github.com/x.git")).to be(true) + expect(described_class.safe?("http://127.0.0.1/x.git")).to be(false) + end + end +end

new file mode 100644 index 0000000..1d5bc24 --- /dev/null +++ b/spec/services/repository_import_service_spec.rb @@ -0,0 +1,95 @@ +# frozen_string_literal: true + +require "rails_helper" +require "tmpdir" +require "fileutils" +require "open3" + +# Exercises the git mechanics against a small local fixture repo (no network): +# a mirror clone + push must carry every branch, every tag, and the full +# history, and the guardrails (size cap, timeout) must fire. +RSpec.describe RepositoryImportService do + # Builds a bare source repo with two branches (main, feature) and a tag (v1), + # returning its path. Yields nothing; caller cleans up the tmpdir. + def build_source(dir) + work = File.join(dir, "work") + FileUtils.mkdir_p(work) + run = ->(*a) { system(*a, exception: true) } + run.call("git", "init", "-q", "-b", "main", work) + run.call("git", "-C", work, "config", "user.email", "t@e.st") + run.call("git", "-C", work, "config", "user.name", "Test") + File.write(File.join(work, "README.md"), "# Fixture\n") + run.call("git", "-C", work, "add", ".") + run.call("git", "-C", work, "commit", "-qm", "initial") + run.call("git", "-C", work, "tag", "v1") + run.call("git", "-C", work, "checkout", "-q", "-b", "feature") + File.write(File.join(work, "f.txt"), "x\n") + run.call("git", "-C", work, "add", ".") + run.call("git", "-C", work, "commit", "-qm", "feature work") + run.call("git", "-C", work, "checkout", "-q", "main") + + bare = File.join(dir, "source.git") + run.call("git", "clone", "-q", "--bare", work, bare) + bare + end + + def empty_bare(dir) + dest = File.join(dir, "dest.git") + system("git", "init", "--bare", "-q", dest, exception: true) + dest + end + + around do |example| + Dir.mktmpdir("import-svc-spec") do |tmp| + @tmp = tmp + example.run + end + end + + it "imports every branch, tag, and the full history, reporting progress" do + source = build_source(@tmp) + dest = empty_bare(@tmp) + phases = [] + + result = described_class.clone_and_push( + source_url: source, dest_path: dest, + progress: ->(p) { phases << p } + ) + + branches = GitRepositoryService.branches(dest).sort + tags = Open3.capture3("git", "--git-dir", dest, "tag").first.split + + expect(branches).to eq(%w[feature main]) + expect(tags).to eq(%w[v1]) + expect(GitRepositoryService.commit_count(dest, "main")).to eq(1) + expect(GitRepositoryService.commit_count(dest, "feature")).to eq(2) + expect(phases).to eq(%i[cloning pushing fetching_lfs]) + expect(result.lfs_detected).to be(false) + expect(result.size_kb).to be_a(Integer) + end + + it "rejects a repo larger than the size cap" do + source = build_source(@tmp) + dest = empty_bare(@tmp) + + expect { + described_class.clone_and_push(source_url: source, dest_path: dest, max_size_kb: 0) + }.to raise_error(RepositoryImportService::SizeExceeded) + end + + it "fails cleanly on an unreachable source instead of hanging" do + dest = empty_bare(@tmp) + expect { + described_class.clone_and_push(source_url: "https://192.0.2.1/nope.git", + dest_path: dest, timeout: 8) + }.to raise_error(RepositoryImportService::ImportError) + end + + it "terminates a git call that overruns its timeout" do + source = build_source(@tmp) + dest = empty_bare(@tmp) + expect { + described_class.clone_and_push(source_url: source, dest_path: dest, timeout: 0.001) + }.to raise_error(RepositoryImportService::CloneTimeout) + end +end