From 3d73f20c1fea304e13693dc6a8e500f8c0f43ad0 Mon Sep 17 00:00:00 2001 From: Azalea Date: Fri, 8 May 2026 15:32:33 +0000 Subject: [PATCH] [+] End-to-end testing --- .gitignore | 1 + Cargo.toml | 4 + README.md | 41 + src/config.rs | 67 ++ src/git.rs | 3 +- src/interactive.rs | 145 ++- src/provider.rs | 16 +- src/sync.rs | 82 +- src/webhook.rs | 34 +- tests/e2e/sequential.rs | 1737 +++++++++++++++++++++++++++++ tests/unit/config.rs | 80 ++ tests/unit/interactive.rs | 36 +- tests/unit/interactive_test_io.rs | 121 ++ tests/unit/sync.rs | 53 + tests/unit/webhook.rs | 47 + 15 files changed, 2405 insertions(+), 62 deletions(-) create mode 100644 tests/e2e/sequential.rs diff --git a/.gitignore b/.gitignore index ea8c4bf..fedaa2b 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ /target +.env diff --git a/Cargo.toml b/Cargo.toml index 773b508..b87aabd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,3 +19,7 @@ tempfile = "3.13" tiny_http = "0.12" toml = "0.8" url = "2.5" + +[[test]] +name = "sequential" +path = "tests/e2e/sequential.rs" diff --git a/README.md b/README.md index a487290..b17c4c8 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,8 @@ Sync only repositories whose names match a regex: refray sync --repo-pattern '^(foo|bar)-' ``` +For persistent per-group filters, set `repo_whitelist` and/or `repo_blacklist` in the config instead. `--repo-pattern` is an extra one-off filter applied on top of the group config. + Retry only repositories that failed during the previous non-dry-run sync: ```sh @@ -148,6 +150,10 @@ If `[webhook].reachability_check_interval_minutes` is configured, `serve` period Each mirror group is treated as a set of equivalent namespaces. Repositories are matched by repository name across all endpoints. +Set `sync_visibility = "all"`, `"private"`, or `"public"` on a mirror group to choose which repository visibility is included in that group. `visibility` still controls the visibility used when `refray` creates a missing repository. + +Set `repo_whitelist = ["..."]` and/or `repo_blacklist = ["..."]` on a mirror group to filter repository names with regular expressions. An empty whitelist includes all repository names, and blacklist matches are excluded after whitelist matches. These name filters are independent from `sync_visibility`; both must match for a repository to be synced. + For every repository name found in any endpoint, `refray` will: 1. Create missing repositories on the other endpoints when `create_missing = true`. @@ -194,6 +200,9 @@ token = { env = "GITEA_TOKEN" } [[mirrors]] name = "personal" +sync_visibility = "all" +repo_whitelist = ["^important-"] +repo_blacklist = ["-archive$"] create_missing = true visibility = "private" allow_force = false @@ -210,6 +219,38 @@ kind = "user" namespace = "azalea" ``` +## Testing + +Run the normal, non-destructive test suite: + +```sh +cargo test +``` + +The sequential live e2e test is ignored by default because it creates and deletes repositories on real provider accounts. Configure four token/username pairs in `.env` or the process environment: + +```sh +GH_USER=... +GH_TOKEN=... +GL_USER=... +GL_TOKEN=... +GT_USER=... +GT_TOKEN=... +FO_USER=... +FO_TOKEN=... +``` + +Optional base URL overrides are `GL_BASE_URL`, `GT_BASE_URL` or `GITEA_BASE_URL`, and `FO_BASE_URL` or `FORGEJO_BASE_URL`. The Gitea and Forgejo defaults are `https://gitea.com` and `https://codeberg.org`. + +Run the destructive e2e test explicitly: + +```sh +REFRAY_E2E_ALLOW_DESTRUCTIVE=1 \ + cargo test --test sequential -- --ignored --test-threads=1 --nocapture +``` + +By default cleanup only deletes repositories named `refray-e2e-*`. To start by deleting every owned repository visible to the configured accounts, set `REFRAY_E2E_CLEAR_ALL_REPOS=DELETE_ALL_OWNED_REPOS`. Provider skips (`REFRAY_E2E_SKIP_GITHUB`, `REFRAY_E2E_SKIP_GITLAB`, `REFRAY_E2E_SKIP_GITEA`, `REFRAY_E2E_SKIP_FORGEJO`) and `REFRAY_E2E_ALLOW_PARTIAL=1` are available for local debugging, but the full support check should run with all four providers. + ## Issues and Pull Requests Issues and pull requests are not mirrored. diff --git a/src/config.rs b/src/config.rs index 17d68df..d3a6f03 100644 --- a/src/config.rs +++ b/src/config.rs @@ -5,6 +5,7 @@ use std::path::{Path, PathBuf}; use anyhow::{Context, Result, anyhow, bail}; use directories::ProjectDirs; +use regex::Regex; use serde::{Deserialize, Serialize}; const APP_NAME: &str = "refray"; @@ -51,6 +52,12 @@ pub enum TokenConfig { pub struct MirrorConfig { pub name: String, pub endpoints: Vec, + #[serde(default)] + pub sync_visibility: SyncVisibility, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub repo_whitelist: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub repo_blacklist: Vec, #[serde(default = "default_true")] pub create_missing: bool, #[serde(default)] @@ -106,6 +113,65 @@ pub enum Visibility { Public, } +#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum SyncVisibility { + #[default] + All, + Private, + Public, +} + +#[derive(Clone, Debug)] +pub struct RepoNameFilter { + whitelist: Vec, + blacklist: Vec, +} + +impl SyncVisibility { + pub fn matches_private(&self, private: bool) -> bool { + match self { + SyncVisibility::All => true, + SyncVisibility::Private => private, + SyncVisibility::Public => !private, + } + } +} + +impl MirrorConfig { + pub fn repo_filter(&self) -> Result { + Ok(RepoNameFilter { + whitelist: compile_repo_patterns(&self.name, "repo_whitelist", &self.repo_whitelist)?, + blacklist: compile_repo_patterns(&self.name, "repo_blacklist", &self.repo_blacklist)?, + }) + } +} + +impl RepoNameFilter { + pub fn matches(&self, repo_name: &str) -> bool { + let whitelisted = self.whitelist.is_empty() + || self + .whitelist + .iter() + .any(|pattern| pattern.is_match(repo_name)); + let blacklisted = self + .blacklist + .iter() + .any(|pattern| pattern.is_match(repo_name)); + whitelisted && !blacklisted + } +} + +fn compile_repo_patterns(mirror: &str, field: &str, patterns: &[String]) -> Result> { + patterns + .iter() + .map(|pattern| { + Regex::new(pattern) + .with_context(|| format!("mirror '{mirror}' has invalid {field} regex '{pattern}'")) + }) + .collect() +} + fn default_true() -> bool { true } @@ -262,6 +328,7 @@ pub fn validate_config(config: &Config) -> Result<()> { bail!("no mirror groups configured"); } for mirror in &config.mirrors { + mirror.repo_filter()?; if mirror.endpoints.len() < 2 { bail!( "mirror '{}' must contain at least two endpoints", diff --git a/src/git.rs b/src/git.rs index d80676d..0983f78 100644 --- a/src/git.rs +++ b/src/git.rs @@ -133,6 +133,7 @@ impl GitMirror { self.run(["fetch", "--prune", &remote.name, &tag_refspec]) } + #[cfg(test)] pub fn cached_remote_refs_match( &self, remote: &RemoteSpec, @@ -589,7 +590,7 @@ impl GitMirror { Ok(rebased.trim().to_string()) } - fn is_ancestor(&self, ancestor: &str, descendant: &str) -> Result { + pub fn is_ancestor(&self, ancestor: &str, descendant: &str) -> Result { let status = self .command() .args(["merge-base", "--is-ancestor", ancestor, descendant]) diff --git a/src/interactive.rs b/src/interactive.rs index 183b8b5..2f35f06 100644 --- a/src/interactive.rs +++ b/src/interactive.rs @@ -9,13 +9,14 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH}; use anyhow::{Context, Result, anyhow}; use console::{Term, style}; use dialoguer::{Confirm, Input, Password, Select, theme::ColorfulTheme}; +use regex::Regex; use reqwest::blocking::Client; use tiny_http::{Request, Response, Server, StatusCode}; use url::Url; use crate::config::{ Config, ConflictResolutionStrategy, EndpointConfig, MirrorConfig, NamespaceKind, ProviderKind, - SiteConfig, TokenConfig, Visibility, WebhookConfig, + SiteConfig, SyncVisibility, TokenConfig, Visibility, WebhookConfig, }; use crate::provider::ProviderClient; use crate::webhook::check_webhook_url_reachable; @@ -37,6 +38,12 @@ struct ParsedProfileUrl { namespace: String, } +#[derive(Clone, Debug, Default)] +struct RepoFilterInput { + whitelist: Vec, + blacklist: Vec, +} + pub fn run_config_wizard(path: &Path) -> Result { let existing_config = path.exists(); let mut config = Config::load_or_default(path)?; @@ -108,10 +115,15 @@ enum WizardAction { fn add_sync_group_styled(config: &mut Config, theme: &ColorfulTheme) -> Result<()> { let endpoints = prompt_sync_group_endpoints_styled(config, theme, &[])?; + let sync_visibility = prompt_sync_visibility_styled(theme, None)?; + let repo_filters = prompt_repo_filters_styled(theme, None)?; let conflict_resolution = prompt_conflict_resolution_styled(theme, None)?; config.upsert_mirror(MirrorConfig { name: next_mirror_name(config), endpoints, + sync_visibility, + repo_whitelist: repo_filters.whitelist, + repo_blacklist: repo_filters.blacklist, create_missing: true, visibility: Visibility::Private, allow_force: false, @@ -435,11 +447,23 @@ fn edit_sync_group_styled(config: &mut Config, theme: &ColorfulTheme) -> Result< style(format!("sync group {}", index + 1)).cyan() ); let existing = config.mirrors[index].endpoints.clone(); + let existing_sync_visibility = config.mirrors[index].sync_visibility.clone(); + let existing_repo_whitelist = config.mirrors[index].repo_whitelist.clone(); + let existing_repo_blacklist = config.mirrors[index].repo_blacklist.clone(); let existing_conflict_resolution = config.mirrors[index].conflict_resolution.clone(); let endpoints = prompt_sync_group_endpoints_styled(config, theme, &existing)?; + let sync_visibility = prompt_sync_visibility_styled(theme, Some(&existing_sync_visibility))?; + let existing_repo_filters = RepoFilterInput { + whitelist: existing_repo_whitelist, + blacklist: existing_repo_blacklist, + }; + let repo_filters = prompt_repo_filters_styled(theme, Some(&existing_repo_filters))?; let conflict_resolution = prompt_conflict_resolution_styled(theme, Some(&existing_conflict_resolution))?; config.mirrors[index].endpoints = endpoints; + config.mirrors[index].sync_visibility = sync_visibility; + config.mirrors[index].repo_whitelist = repo_filters.whitelist; + config.mirrors[index].repo_blacklist = repo_filters.blacklist; config.mirrors[index].conflict_resolution = conflict_resolution; prompt_webhook_setup_styled(config, theme)?; println!( @@ -736,6 +760,102 @@ fn prompt_conflict_resolution_styled( Ok(conflict_resolution_from_index(index)) } +fn prompt_sync_visibility_styled( + theme: &ColorfulTheme, + existing: Option<&SyncVisibility>, +) -> Result { + let options = [ + "All repositories", + "Only private repositories", + "Only public repositories", + ]; + let default = existing.map(sync_visibility_index).unwrap_or(0); + let index = Select::with_theme(theme) + .with_prompt("Which repositories should this sync group include?") + .items(options) + .default(default) + .interact()?; + Ok(sync_visibility_from_index(index)) +} + +fn prompt_repo_filters_styled( + theme: &ColorfulTheme, + existing: Option<&RepoFilterInput>, +) -> Result { + let existing = existing.cloned().unwrap_or_default(); + let has_existing = !existing.whitelist.is_empty() || !existing.blacklist.is_empty(); + if !Confirm::with_theme(theme) + .with_prompt("Configure repository name whitelist/blacklist?") + .default(has_existing) + .interact()? + { + return Ok(RepoFilterInput::default()); + } + + Ok(RepoFilterInput { + whitelist: prompt_repo_pattern_list_styled( + theme, + "Whitelist regexes (comma-separated, empty means all repo names)", + &existing.whitelist, + )?, + blacklist: prompt_repo_pattern_list_styled( + theme, + "Blacklist regexes (comma-separated)", + &existing.blacklist, + )?, + }) +} + +fn prompt_repo_pattern_list_styled( + theme: &ColorfulTheme, + prompt: &str, + existing: &[String], +) -> Result> { + let input = Input::::with_theme(theme) + .with_prompt(prompt) + .allow_empty(true) + .validate_with(|value: &String| validate_repo_pattern_list(value)); + let input = if existing.is_empty() { + input + } else { + input.default(existing.join(", ")) + }; + let value = input.interact_text()?; + Ok(parse_repo_pattern_list(&value)) +} + +fn validate_repo_pattern_list(value: &str) -> std::result::Result<(), String> { + for pattern in parse_repo_pattern_list(value) { + Regex::new(&pattern).map_err(|error| format!("invalid regex '{pattern}': {error}"))?; + } + Ok(()) +} + +fn parse_repo_pattern_list(value: &str) -> Vec { + value + .split(',') + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(ToOwned::to_owned) + .collect() +} + +fn sync_visibility_index(sync_visibility: &SyncVisibility) -> usize { + match sync_visibility { + SyncVisibility::All => 0, + SyncVisibility::Private => 1, + SyncVisibility::Public => 2, + } +} + +fn sync_visibility_from_index(index: usize) -> SyncVisibility { + match index { + 1 => SyncVisibility::Private, + 2 => SyncVisibility::Public, + _ => SyncVisibility::All, + } +} + fn conflict_resolution_index(strategy: &ConflictResolutionStrategy) -> usize { match strategy { ConflictResolutionStrategy::Fail => 0, @@ -803,12 +923,33 @@ fn sync_group_summary(config: &Config, mirror: &MirrorConfig) -> String { .collect::>() .join(" <-> "); format!( - "{} ({})", + "{} ({}, {}, {})", endpoints, + sync_visibility_label(&mirror.sync_visibility), + repo_filter_label(mirror), conflict_resolution_label(&mirror.conflict_resolution) ) } +fn sync_visibility_label(sync_visibility: &SyncVisibility) -> &'static str { + match sync_visibility { + SyncVisibility::All => "sync: all", + SyncVisibility::Private => "sync: private only", + SyncVisibility::Public => "sync: public only", + } +} + +fn repo_filter_label(mirror: &MirrorConfig) -> String { + match (mirror.repo_whitelist.len(), mirror.repo_blacklist.len()) { + (0, 0) => "repos: all names".to_string(), + (whitelist, 0) => format!("repos: whitelist {whitelist}"), + (0, blacklist) => format!("repos: blacklist {blacklist}"), + (whitelist, blacklist) => { + format!("repos: whitelist {whitelist}, blacklist {blacklist}") + } + } +} + fn conflict_resolution_label(strategy: &ConflictResolutionStrategy) -> &'static str { match strategy { ConflictResolutionStrategy::Fail => "conflicts: fail", diff --git a/src/provider.rs b/src/provider.rs index c68650c..4bd10e5 100644 --- a/src/provider.rs +++ b/src/provider.rs @@ -343,7 +343,11 @@ impl<'a> ProviderClient<'a> { "{pulls_url}?state=open&base={}&per_page=100", urlencoding(base_branch) ); - let pulls: Vec = self.paged_get(&url)?; + let pulls: Vec = match self.paged_get(&url) { + Ok(pulls) => pulls, + Err(error) if is_not_found_error(&error) => return Ok(0), + Err(error) => return Err(error), + }; let mut closed = 0; for pull in pulls.into_iter().filter(|pull| { pull.head_ref() @@ -681,7 +685,11 @@ impl<'a> ProviderClient<'a> { "{pulls_url}?state=open&base={}&limit=50", urlencoding(base_branch) ); - let pulls: Vec = self.paged_get(&url)?; + let pulls: Vec = match self.paged_get(&url) { + Ok(pulls) => pulls, + Err(error) if is_not_found_error(&error) => return Ok(0), + Err(error) => return Err(error), + }; let mut closed = 0; for pull in pulls.into_iter().filter(|pull| { pull.head_ref() @@ -931,6 +939,10 @@ fn check_response(method: &str, url: &str, response: Response) -> Result bool { + error.to_string().contains("404 Not Found") +} + fn next_link(headers: &HeaderMap) -> Option { let header = headers.get("link")?.to_str().ok()?; for part in header.split(',') { diff --git a/src/sync.rs b/src/sync.rs index c3b7061..280c642 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -9,12 +9,12 @@ use console::style; use regex::Regex; use crate::config::{ - Config, ConflictResolutionStrategy, EndpointConfig, MirrorConfig, default_work_dir, - validate_config, + Config, ConflictResolutionStrategy, EndpointConfig, MirrorConfig, RepoNameFilter, + SyncVisibility, default_work_dir, validate_config, }; use crate::git::{ - BranchConflict, BranchDeletion, BranchUpdate, GitMirror, Redactor, RemoteRefSnapshot, - RemoteSpec, is_disabled_repository_error, ls_remote_refs, safe_remote_name, + BranchConflict, BranchDeletion, BranchUpdate, GitMirror, Redactor, RemoteSpec, + is_disabled_repository_error, ls_remote_refs, safe_remote_name, }; use crate::logging; use crate::provider::{EndpointRepo, ProviderClient, PullRequestRequest, repos_by_name}; @@ -165,8 +165,9 @@ fn sync_group( .create_missing_override .unwrap_or(mirror.create_missing); let allow_force = context.options.force_override.unwrap_or(mirror.allow_force); + let repo_filter = mirror.repo_filter()?; - let all_endpoint_repos = list_group_repos(context.config, mirror)?; + let all_endpoint_repos = list_group_repos(context.config, mirror, &repo_filter)?; if !context.options.dry_run { webhook::ensure_configured_webhooks( context.config, @@ -181,12 +182,7 @@ fn sync_group( let retry_repo_names = context .retry_failed_repos .and_then(|repos| repos.get(&mirror.name)); - let tracked_repo_names = context.ref_state.repo_names(&mirror.name); - let all_repo_names = repos - .keys() - .cloned() - .chain(tracked_repo_names) - .collect::>(); + let all_repo_names = sync_candidate_repo_names(&repos, context.ref_state, mirror, &repo_filter); let all_repo_count = all_repo_names.len(); let repo_names = all_repo_names .into_iter() @@ -349,7 +345,7 @@ fn sync_group( }); if create_missing && !context.options.dry_run { - let repos = list_group_repos(context.config, mirror)?; + let repos = list_group_repos(context.config, mirror, &repo_filter)?; webhook::ensure_configured_webhooks( context.config, mirror, @@ -362,7 +358,11 @@ fn sync_group( Ok(failures) } -fn list_group_repos(config: &Config, mirror: &MirrorConfig) -> Result> { +fn list_group_repos( + config: &Config, + mirror: &MirrorConfig, + repo_filter: &RepoNameFilter, +) -> Result> { let mut all_endpoint_repos = Vec::new(); for endpoint in &mirror.endpoints { let site = config.site(&endpoint.site).unwrap(); @@ -375,7 +375,11 @@ fn list_group_repos(config: &Config, mirror: &MirrorConfig) -> Result Result>, + ref_state: &RefState, + mirror: &MirrorConfig, + repo_filter: &RepoNameFilter, +) -> BTreeSet { + let mut names = repos.keys().cloned().collect::>(); + if mirror.sync_visibility == SyncVisibility::All { + names.extend( + ref_state + .repo_names(&mirror.name) + .into_iter() + .filter(|name| repo_filter.matches(name)), + ); + } + names +} + fn pop_repo_job(queue: &Arc>>) -> Option { queue .lock() @@ -561,19 +583,6 @@ fn sync_repo( mirror_repo.configure_remotes(&initial_remotes)?; let cached_ref_state = cached_ref_state(&mirror_repo, &initial_remotes)?; - if !context.dry_run - && all_endpoints_present - && cached_refs_match(&mirror_repo, &initial_remotes, &initial_ref_state)? - { - crate::logln!( - " {} refs unchanged from local mirror cache", - style("up-to-date").green().bold() - ); - return Ok(RepoSyncOutcome { - state_update: Some(RepoStateUpdate::Set(initial_ref_state)), - }); - } - for remote in &initial_remotes { if let Err(error) = mirror_repo.fetch_remote(remote) { if is_disabled_repository_error(&error) { @@ -772,22 +781,6 @@ fn all_configured_endpoints_present(mirror: &MirrorConfig, repos: &[EndpointRepo .all(|endpoint| present.contains(endpoint)) } -fn cached_refs_match( - mirror_repo: &GitMirror, - remotes: &[RemoteSpec], - expected_refs: &BTreeMap, -) -> Result { - for remote in remotes { - let Some(expected) = expected_refs.get(&remote.name) else { - return Ok(false); - }; - if !mirror_repo.cached_remote_refs_match(remote, &RemoteRefSnapshot::from(expected))? { - return Ok(false); - } - } - Ok(true) -} - fn cached_ref_state( mirror_repo: &GitMirror, remotes: &[RemoteSpec], @@ -1144,6 +1137,9 @@ fn open_conflict_pull_requests( for (source_remote, source_sha) in conflict.tips.iter().filter(|(_, sha)| sha != target_sha) { + if mirror_repo.is_ancestor(source_sha, target_sha)? { + continue; + } let head_branch = conflict_pr_branch(&conflict.branch, source_remote, source_sha); let update = BranchUpdate { branch: head_branch.clone(), diff --git a/src/webhook.rs b/src/webhook.rs index 7bee10c..e1793ad 100644 --- a/src/webhook.rs +++ b/src/webhook.rs @@ -204,6 +204,7 @@ pub fn install_webhooks(config: &Config, options: WebhookInstallOptions) -> Resu style("Webhook group").cyan().bold(), style(&mirror.name).bold() ); + let repo_filter = mirror.repo_filter()?; let mut tasks = Vec::new(); for endpoint in &mirror.endpoints { let site = config.site(&endpoint.site).unwrap(); @@ -216,7 +217,11 @@ pub fn install_webhooks(config: &Config, options: WebhookInstallOptions) -> Resu let repos = client .list_repos(endpoint) .with_context(|| format!("failed to list repos for {}", endpoint.label()))?; - for repo in repos { + for repo in repos + .into_iter() + .filter(|repo| mirror.sync_visibility.matches_private(repo.private)) + .filter(|repo| repo_filter.matches(&repo.name)) + { if options.repo.as_ref().is_some_and(|name| name != &repo.name) { continue; } @@ -1009,19 +1014,22 @@ fn matching_jobs(config: &Config, event: &WebhookEvent) -> Vec { .mirrors .iter() .filter(|mirror| { - mirror.endpoints.iter().any(|endpoint| { - let Some(site) = config.site(&endpoint.site) else { - return false; - }; - event - .provider - .as_ref() - .is_none_or(|provider| &site.provider == provider) - && event - .namespace + mirror + .repo_filter() + .is_ok_and(|filter| filter.matches(&event.repo)) + && mirror.endpoints.iter().any(|endpoint| { + let Some(site) = config.site(&endpoint.site) else { + return false; + }; + event + .provider .as_ref() - .is_none_or(|namespace| namespace == &endpoint.namespace) - }) + .is_none_or(|provider| &site.provider == provider) + && event + .namespace + .as_ref() + .is_none_or(|namespace| namespace == &endpoint.namespace) + }) }) .map(|mirror| WebhookJob { group: mirror.name.clone(), diff --git a/tests/e2e/sequential.rs b/tests/e2e/sequential.rs new file mode 100644 index 0000000..5e1c42b --- /dev/null +++ b/tests/e2e/sequential.rs @@ -0,0 +1,1737 @@ +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::fs; +use std::net::TcpListener; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, Output}; +use std::thread; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use anyhow::{Context, Result, anyhow, bail}; +use hmac::{Hmac, Mac}; +use reqwest::blocking::{Client, RequestBuilder}; +use reqwest::header::{ACCEPT, AUTHORIZATION, HeaderMap, HeaderValue, USER_AGENT}; +use serde_json::{Value, json}; +use sha2::Sha256; +use tempfile::TempDir; +use url::Url; + +type HmacSha256 = Hmac; + +const REPO_PREFIX: &str = "refray-e2e-"; +const MAIN_BRANCH: &str = "main"; +const WEBHOOK_SECRET: &str = "refray-e2e-secret"; + +#[test] +#[ignore = "destructive live-provider e2e test; run explicitly with --ignored"] +fn sequential_live_e2e_all_supported_features() -> Result<()> { + let env = EnvFile::load(Path::new(".env"))?; + let settings = E2eSettings::from_env(&env)?; + settings.require_destructive_guard()?; + + let mut run = E2eRun::new(settings)?; + run.preflight()?; + run.clear_repositories()?; + run.write_config(ConflictMode::AutoRebasePullRequest, None, true)?; + + eprintln!("e2e phase: core sync"); + run.creation_branch_tag_and_read_write_sync()?; + eprintln!("e2e phase: dry-run/no-create"); + run.dry_run_and_no_create_do_not_write()?; + eprintln!("e2e phase: retry failed"); + run.failed_sync_can_retry_only_failed_repo()?; + eprintln!("e2e phase: auto rebase"); + run.auto_rebase_resolves_non_conflicting_divergence()?; + eprintln!("e2e phase: force sync"); + run.force_sync_chooses_newest_divergent_commit()?; + eprintln!("e2e phase: pull-request conflicts"); + run.pull_request_strategy_pushes_conflict_branches()?; + eprintln!("e2e phase: auto-rebase PR fallback"); + run.auto_rebase_pull_request_falls_back_to_conflict_branches()?; + eprintln!("e2e phase: branch deletion"); + run.branch_deletion_propagates()?; + eprintln!("e2e phase: repo deletion"); + run.repo_deletion_propagates()?; + eprintln!("e2e phase: webhooks"); + run.webhook_commands_and_receiver_work()?; + + run.clear_e2e_repositories()?; + Ok(()) +} + +struct EnvFile { + values: HashMap, +} + +impl EnvFile { + fn load(path: &Path) -> Result { + let mut values = std::env::vars().collect::>(); + if path.exists() { + let contents = fs::read_to_string(path) + .with_context(|| format!("failed to read {}", path.display()))?; + for (line_number, line) in contents.lines().enumerate() { + let line = line.trim(); + if line.is_empty() || line.starts_with('#') { + continue; + } + let Some((name, value)) = line.split_once('=') else { + bail!("invalid .env line {}", line_number + 1); + }; + values.insert(name.trim().to_string(), parse_env_value(value.trim())); + } + } + Ok(Self { values }) + } + + fn required(&self, names: &[&str]) -> Result { + names + .iter() + .find_map(|name| self.values.get(*name).filter(|value| !value.is_empty())) + .cloned() + .ok_or_else(|| anyhow!("missing required env var; tried {}", names.join(", "))) + } + + fn optional(&self, names: &[&str]) -> Option { + names + .iter() + .find_map(|name| self.values.get(*name).filter(|value| !value.is_empty())) + .cloned() + } + + fn flag(&self, names: &[&str]) -> bool { + self.optional(names) + .is_some_and(|value| matches!(value.as_str(), "1" | "true" | "yes" | "on")) + } +} + +fn parse_env_value(value: &str) -> String { + let value = value.trim(); + if value.len() >= 2 { + let quoted = (value.starts_with('"') && value.ends_with('"')) + || (value.starts_with('\'') && value.ends_with('\'')); + if quoted { + return value[1..value.len() - 1].to_string(); + } + } + value.to_string() +} + +struct E2eSettings { + providers: Vec, + destructive_guard: bool, + clear_all_repos: bool, + allow_partial: bool, +} + +impl E2eSettings { + fn from_env(env: &EnvFile) -> Result { + let mut providers = Vec::new(); + if !env.flag(&["REFRAY_E2E_SKIP_GITHUB"]) { + providers.push(ProviderAccount::new( + "github", + ProviderKind::Github, + "https://github.com".to_string(), + env.required(&["REFRAY_E2E_GITHUB_USERNAME", "GH_USER"])?, + env.required(&["REFRAY_E2E_GITHUB_TOKEN", "GH_TOKEN"])?, + )); + } + if !env.flag(&["REFRAY_E2E_SKIP_GITLAB"]) { + providers.push(ProviderAccount::new( + "gitlab", + ProviderKind::Gitlab, + env.optional(&["REFRAY_E2E_GITLAB_BASE_URL", "GL_BASE_URL"]) + .unwrap_or_else(|| "https://gitlab.com".to_string()), + env.required(&["REFRAY_E2E_GITLAB_USERNAME", "GL_USER"])?, + env.required(&["REFRAY_E2E_GITLAB_TOKEN", "GL_TOKEN"])?, + )); + } + if !env.flag(&["REFRAY_E2E_SKIP_GITEA"]) { + providers.push(ProviderAccount::new( + "gitea", + ProviderKind::Gitea, + env.optional(&["REFRAY_E2E_GITEA_BASE_URL", "GT_BASE_URL", "GITEA_BASE_URL"]) + .unwrap_or_else(|| "https://gitea.com".to_string()), + env.required(&["REFRAY_E2E_GITEA_USERNAME", "GT_USER"])?, + env.required(&["REFRAY_E2E_GITEA_TOKEN", "GT_TOKEN"])?, + )); + } + if !env.flag(&["REFRAY_E2E_SKIP_FORGEJO"]) { + providers.push(ProviderAccount::new( + "forgejo", + ProviderKind::Forgejo, + env.optional(&[ + "REFRAY_E2E_FORGEJO_BASE_URL", + "FO_BASE_URL", + "FORGEJO_BASE_URL", + ]) + .unwrap_or_else(|| "https://codeberg.org".to_string()), + env.required(&["REFRAY_E2E_FORGEJO_USERNAME", "FO_USER"])?, + env.required(&["REFRAY_E2E_FORGEJO_TOKEN", "FO_TOKEN"])?, + )); + } + Ok(Self { + providers, + destructive_guard: env.flag(&["REFRAY_E2E_ALLOW_DESTRUCTIVE"]), + clear_all_repos: clear_all_repos_enabled(env)?, + allow_partial: env.flag(&["REFRAY_E2E_ALLOW_PARTIAL"]), + }) + } + + fn require_destructive_guard(&self) -> Result<()> { + if self.destructive_guard { + return Ok(()); + } + bail!("refusing to run destructive e2e test without REFRAY_E2E_ALLOW_DESTRUCTIVE=1") + } + + fn secrets(&self) -> Vec { + self.providers + .iter() + .map(|provider| provider.token.clone()) + .collect() + } +} + +fn clear_all_repos_enabled(env: &EnvFile) -> Result { + let Some(value) = env.optional(&["REFRAY_E2E_CLEAR_ALL_REPOS"]) else { + return Ok(false); + }; + if value == "DELETE_ALL_OWNED_REPOS" { + return Ok(true); + } + bail!( + "REFRAY_E2E_CLEAR_ALL_REPOS is destructive; set it to DELETE_ALL_OWNED_REPOS to delete every owned repo on the configured accounts" + ) +} + +struct E2eRun { + temp: TempDir, + config_path: PathBuf, + work_dir: PathBuf, + settings: E2eSettings, + redactor: Redactor, + run_id: String, +} + +impl E2eRun { + fn new(settings: E2eSettings) -> Result { + let temp = tempfile::tempdir().context("failed to create e2e temp dir")?; + let config_path = temp.path().join("config.toml"); + let work_dir = temp.path().join("work"); + let redactor = Redactor::new(settings.secrets()); + let run_id = SystemTime::now() + .duration_since(UNIX_EPOCH)? + .as_secs() + .to_string(); + Ok(Self { + temp, + config_path, + work_dir, + settings, + redactor, + run_id, + }) + } + + fn preflight(&mut self) -> Result<()> { + let mut valid = Vec::new(); + let mut failures = Vec::new(); + for mut provider in self.settings.providers.drain(..) { + match provider.validate_token() { + Ok(authenticated_username) => { + if provider.username != authenticated_username { + eprintln!( + "using authenticated {} username '{}' instead of configured username '{}'", + provider.site_name, authenticated_username, provider.username + ); + provider.username = authenticated_username; + } + valid.push(provider); + } + Err(error) => failures.push((provider.site_name.clone(), error)), + } + } + if !failures.is_empty() && !self.settings.allow_partial { + let details = failures + .into_iter() + .map(|(site, error)| format!("{site}: {error:#}")) + .collect::>() + .join("\n"); + bail!("provider preflight failed:\n{details}"); + } + for (site, error) in failures { + eprintln!("skipping {site} e2e provider after preflight failure: {error:#}"); + } + if valid.len() < 2 { + bail!( + "need at least two valid providers for e2e sync, got {}", + valid.len() + ); + } + self.settings.providers = valid; + Ok(()) + } + + fn clear_repositories(&self) -> Result<()> { + self.clear_repositories_matching(self.settings.clear_all_repos) + } + + fn clear_e2e_repositories(&self) -> Result<()> { + self.clear_repositories_matching(false) + } + + fn clear_repositories_matching(&self, clear_all: bool) -> Result<()> { + for provider in &self.settings.providers { + let repos = provider.list_repos()?; + for repo in repos { + if clear_all || repo.name.starts_with(REPO_PREFIX) { + provider.delete_repo(&repo.name)?; + } + } + } + Ok(()) + } + + fn repo_name(&self, label: &str) -> String { + format!("{REPO_PREFIX}{}-{label}", self.run_id) + } + + fn write_config( + &self, + conflict_mode: ConflictMode, + repo_pattern: Option<&str>, + create_missing: bool, + ) -> Result<()> { + self.write_config_for_sites(conflict_mode, repo_pattern, create_missing, None) + } + + fn write_config_for_sites( + &self, + conflict_mode: ConflictMode, + repo_pattern: Option<&str>, + create_missing: bool, + endpoint_sites: Option<&[&str]>, + ) -> Result<()> { + let default_whitelist = format!("^{REPO_PREFIX}{}-", self.run_id); + let whitelist = repo_pattern.unwrap_or(&default_whitelist); + let mut contents = String::new(); + for provider in &self.settings.providers { + contents.push_str(&format!( + r#"[[sites]] +name = "{}" +provider = "{}" +base_url = "{}" +token = {{ value = "{}" }} + +"#, + provider.site_name, + provider.kind.config_name(), + provider.base_url, + provider.token.replace('"', "\\\"") + )); + } + contents.push_str(&format!( + r#"[webhook] +install = false +url = "http://127.0.0.1:1/webhook" +secret = {{ value = "{WEBHOOK_SECRET}" }} + +[[mirrors]] +name = "all" +sync_visibility = "all" +repo_whitelist = ['{}'] +create_missing = {} +visibility = "public" +allow_force = false +conflict_resolution = "{}" + +"#, + whitelist.replace('\'', "''"), + create_missing, + conflict_mode.config_name() + )); + for provider in &self.settings.providers { + if endpoint_sites.is_some_and(|sites| !sites.contains(&provider.site_name.as_str())) { + continue; + } + contents.push_str(&format!( + r#"[[mirrors.endpoints]] +site = "{}" +kind = "user" +namespace = "{}" + +"#, + provider.site_name, + provider.username.replace('"', "\\\"") + )); + } + fs::write(&self.config_path, contents) + .with_context(|| format!("failed to write {}", self.config_path.display())) + } + + fn creation_branch_tag_and_read_write_sync(&self) -> Result<()> { + let repo = self.repo_name("core"); + let (source, peer) = self.provider_pair(); + source.create_repo(&repo)?; + let work = self.git_worktree("core-seed")?; + git_init(&work)?; + write_commit(&work, "README.md", "seed\n", "seed", 1_700_000_001)?; + git(&work, ["branch", "feature/github"])?; + git(&work, ["tag", "v1.0.0"])?; + let remote_url = source.authenticated_repo_url(&repo)?; + self.git(&work, ["remote", "add", "origin", &remote_url])?; + self.git( + &work, + ["push", "origin", "HEAD:main", "feature/github", "v1.0.0"], + )?; + source.wait_branch( + &repo, + MAIN_BRANCH, + &git_output(&work, ["rev-parse", "HEAD"])?, + )?; + + source.wait_repo_listed(&repo)?; + self.sync(["--repo-pattern", &exact_pattern(&repo)])?; + self.assert_branch_all_equal_after_optional_resync(&repo, MAIN_BRANCH)?; + self.assert_branch_all_equal(&repo, "feature/github")?; + self.assert_tag_all_equal(&repo, "v1.0.0")?; + + let work = self.clone_repo(peer, &repo, "core-peer")?; + write_commit( + &work, + "peer.txt", + "peer update\n", + "peer update", + 1_700_000_101, + )?; + self.git(&work, ["push", "origin", "HEAD:main"])?; + peer.wait_branch( + &repo, + MAIN_BRANCH, + &git_output(&work, ["rev-parse", "HEAD"])?, + )?; + peer.wait_repo_listed(&repo)?; + self.sync(["--repo-pattern", &exact_pattern(&repo)])?; + self.assert_branch_all_equal_after_optional_resync(&repo, MAIN_BRANCH)?; + Ok(()) + } + + fn dry_run_and_no_create_do_not_write(&self) -> Result<()> { + let repo = self.repo_name("dry-run"); + let source = self.primary_provider(); + source.create_repo(&repo)?; + self.seed_main(source, &repo, "dry run", 1_700_000_201)?; + + self.sync(["--repo-pattern", &exact_pattern(&repo), "--dry-run"])?; + self.assert_only_provider_has_repo(&repo, &source.site_name)?; + + self.sync(["--repo-pattern", &exact_pattern(&repo), "--no-create"])?; + self.assert_only_provider_has_repo(&repo, &source.site_name)?; + Ok(()) + } + + fn failed_sync_can_retry_only_failed_repo(&self) -> Result<()> { + let repo = self.repo_name("retry"); + let (source, peer) = self.provider_pair(); + self.seed_all_main(&repo, "retry base", 1_700_000_301)?; + self.sync(["--repo-pattern", &exact_pattern(&repo)])?; + self.unprotect_main_all(&repo)?; + + self.commit_to_provider( + source, + &repo, + "retry.txt", + "source\n", + "source retry", + 1_700_000_401, + )?; + self.commit_to_provider( + peer, + &repo, + "retry.txt", + "peer\n", + "peer retry", + 1_700_000_402, + )?; + self.write_config(ConflictMode::Fail, Some(&exact_pattern(&repo)), true)?; + self.sync_expect_failure(["--repo-pattern", &exact_pattern(&repo)])?; + self.sync(["--retry-failed", "--force"])?; + self.assert_branch_all_equal(&repo, MAIN_BRANCH)?; + self.write_config(ConflictMode::AutoRebasePullRequest, None, true)?; + Ok(()) + } + + fn auto_rebase_resolves_non_conflicting_divergence(&self) -> Result<()> { + let repo = self.repo_name("rebase"); + let (source, peer) = self.provider_pair(); + self.seed_all_main(&repo, "rebase base", 1_700_000_501)?; + self.sync(["--repo-pattern", &exact_pattern(&repo)])?; + self.unprotect_main_all(&repo)?; + + self.commit_to_provider( + source, + &repo, + "source-rebase.txt", + "source\n", + "source rebase", + 1_700_000_601, + )?; + self.commit_to_provider( + peer, + &repo, + "peer-rebase.txt", + "peer\n", + "peer rebase", + 1_700_000_602, + )?; + self.write_config(ConflictMode::AutoRebase, Some(&exact_pattern(&repo)), true)?; + self.sync([])?; + self.assert_branch_all_equal(&repo, MAIN_BRANCH)?; + let clone = self.clone_repo(source, &repo, "rebase-verify")?; + assert!(clone.join("source-rebase.txt").exists()); + assert!(clone.join("peer-rebase.txt").exists()); + self.write_config(ConflictMode::AutoRebasePullRequest, None, true)?; + Ok(()) + } + + fn force_sync_chooses_newest_divergent_commit(&self) -> Result<()> { + let repo = self.repo_name("force"); + let (source, peer) = self.provider_pair(); + self.seed_all_main(&repo, "force base", 1_700_000_701)?; + self.sync(["--repo-pattern", &exact_pattern(&repo)])?; + self.unprotect_main_all(&repo)?; + + self.commit_to_provider( + source, + &repo, + "force.txt", + "old\n", + "old force", + 1_700_000_801, + )?; + self.commit_to_provider( + peer, + &repo, + "force.txt", + "new\n", + "new force", + 1_700_000_901, + )?; + self.sync(["--repo-pattern", &exact_pattern(&repo), "--force"])?; + self.assert_branch_all_equal(&repo, MAIN_BRANCH)?; + let clone = self.clone_repo(source, &repo, "force-verify")?; + let contents = fs::read_to_string(clone.join("force.txt"))?; + assert_eq!(contents, "new\n"); + Ok(()) + } + + fn pull_request_strategy_pushes_conflict_branches(&self) -> Result<()> { + let repo = self.repo_name("pull-request"); + let (source, peer) = self.provider_pair(); + self.seed_all_main(&repo, "pr base", 1_700_001_001)?; + self.sync(["--repo-pattern", &exact_pattern(&repo)])?; + self.unprotect_main_all(&repo)?; + + self.commit_to_provider( + source, + &repo, + "conflict.txt", + "source\n", + "source pr", + 1_700_001_101, + )?; + self.commit_to_provider( + peer, + &repo, + "conflict.txt", + "peer\n", + "peer pr", + 1_700_001_102, + )?; + let endpoint_sites = [source.site_name.as_str(), peer.site_name.as_str()]; + self.write_config_for_sites( + ConflictMode::PullRequest, + Some(&exact_pattern(&repo)), + true, + Some(&endpoint_sites), + )?; + self.sync([])?; + self.assert_conflict_branch_exists(&repo)?; + self.write_config(ConflictMode::AutoRebasePullRequest, None, true)?; + Ok(()) + } + + fn auto_rebase_pull_request_falls_back_to_conflict_branches(&self) -> Result<()> { + let repo = self.repo_name("fallback"); + let (source, peer) = self.provider_pair(); + self.seed_all_main(&repo, "fallback base", 1_700_001_201)?; + self.sync(["--repo-pattern", &exact_pattern(&repo)])?; + self.unprotect_main_all(&repo)?; + + self.commit_to_provider( + source, + &repo, + "fallback.txt", + "source\n", + "source fallback", + 1_700_001_301, + )?; + self.commit_to_provider( + peer, + &repo, + "fallback.txt", + "peer\n", + "peer fallback", + 1_700_001_302, + )?; + let endpoint_sites = [source.site_name.as_str(), peer.site_name.as_str()]; + self.write_config_for_sites( + ConflictMode::AutoRebasePullRequest, + Some(&exact_pattern(&repo)), + true, + Some(&endpoint_sites), + )?; + self.sync([])?; + self.assert_conflict_branch_exists(&repo)?; + self.write_config(ConflictMode::AutoRebasePullRequest, None, true)?; + Ok(()) + } + + fn branch_deletion_propagates(&self) -> Result<()> { + let repo = self.repo_name("branch-delete"); + let source = self.primary_provider(); + self.seed_all_main(&repo, "delete branch base", 1_700_001_401)?; + let work = self.clone_repo(source, &repo, "branch-delete")?; + git(&work, ["checkout", "-b", "delete-me"])?; + write_commit( + &work, + "delete-me.txt", + "delete me\n", + "delete branch", + 1_700_001_402, + )?; + self.git(&work, ["push", "origin", "HEAD:delete-me"])?; + source.wait_branch( + &repo, + "delete-me", + &git_output(&work, ["rev-parse", "HEAD"])?, + )?; + self.sync(["--repo-pattern", &exact_pattern(&repo)])?; + source.wait_repo_listed(&repo)?; + self.assert_branch_all_equal(&repo, "delete-me")?; + + self.git(&work, ["push", "origin", ":refs/heads/delete-me"])?; + source.wait_branch_absent(&repo, "delete-me")?; + self.sync(["--repo-pattern", &exact_pattern(&repo)])?; + self.assert_branch_absent_everywhere(&repo, "delete-me")?; + Ok(()) + } + + fn repo_deletion_propagates(&self) -> Result<()> { + let repo = self.repo_name("repo-delete"); + let source = self.primary_provider(); + self.seed_all_main(&repo, "repo delete base", 1_700_001_501)?; + self.sync(["--repo-pattern", &exact_pattern(&repo)])?; + self.assert_repo_exists_everywhere(&repo)?; + + source.delete_repo(&repo)?; + source.wait_repo_absent(&repo)?; + self.sync(["--repo-pattern", &exact_pattern(&repo)])?; + self.assert_repo_absent_everywhere(&repo)?; + Ok(()) + } + + fn webhook_commands_and_receiver_work(&self) -> Result<()> { + let repo = self.repo_name("webhook"); + let source = self.primary_provider(); + self.seed_all_main(&repo, "webhook base", 1_700_001_601)?; + self.sync(["--repo-pattern", &exact_pattern(&repo)])?; + + self.refray([ + "webhook", + "install", + "--dry-run", + "--repo-pattern", + &exact_pattern(&repo), + "--url", + "https://example.invalid/webhook", + "--secret", + WEBHOOK_SECRET, + ])?; + self.refray([ + "webhook", + "uninstall", + &repo, + "--dry-run", + "--url", + "https://example.invalid/webhook", + ])?; + self.refray([ + "webhook", + "update", + "--dry-run", + "--url", + "https://example.invalid/new-webhook", + "--secret", + WEBHOOK_SECRET, + ])?; + + let listener = TcpListener::bind("127.0.0.1:0")?; + let addr = listener.local_addr()?; + drop(listener); + let mut server = self.spawn_refray([ + "serve", + "--listen", + &addr.to_string(), + "--secret", + WEBHOOK_SECRET, + "--jobs", + "1", + ])?; + thread::sleep(Duration::from_millis(750)); + + let (body, headers) = source.webhook_payload(&repo, WEBHOOK_SECRET); + let mut request = Client::new().post(format!("http://{addr}/webhook")); + for (name, value) in headers { + request = request.header(name, value); + } + let response = request + .body(body) + .send() + .context("failed to post webhook payload")?; + let status = response.status(); + let text = response.text().unwrap_or_default(); + stop_child(&mut server); + if !status.is_success() || !text.starts_with("queued ") { + bail!("webhook receiver returned {status}: {text}"); + } + Ok(()) + } + + fn seed_main( + &self, + provider: &ProviderAccount, + repo: &str, + contents: &str, + timestamp: i64, + ) -> Result<()> { + let work = self.git_worktree(&format!("seed-{repo}"))?; + git_init(&work)?; + write_commit( + &work, + "README.md", + &format!("{contents}\n"), + contents, + timestamp, + )?; + let remote_url = provider.authenticated_repo_url(repo)?; + self.git(&work, ["remote", "add", "origin", &remote_url])?; + self.git(&work, ["push", "origin", "HEAD:main"])?; + provider.wait_branch( + repo, + MAIN_BRANCH, + &git_output(&work, ["rev-parse", "HEAD"])?, + )?; + provider.wait_repo_listed(repo)?; + Ok(()) + } + + fn seed_all_main(&self, repo: &str, contents: &str, timestamp: i64) -> Result<()> { + for provider in &self.settings.providers { + provider.create_repo(repo)?; + } + let work = self.git_worktree(&format!("seed-all-{repo}"))?; + git_init(&work)?; + write_commit( + &work, + "README.md", + &format!("{contents}\n"), + contents, + timestamp, + )?; + let sha = git_output(&work, ["rev-parse", "HEAD"])?; + for provider in &self.settings.providers { + let remote_url = provider.authenticated_repo_url(repo)?; + self.git(&work, ["remote", "add", &provider.site_name, &remote_url])?; + self.git(&work, ["push", &provider.site_name, "HEAD:main"])?; + provider.wait_branch(repo, MAIN_BRANCH, &sha)?; + provider.wait_repo_listed(repo)?; + provider.unprotect_branch(repo, MAIN_BRANCH)?; + } + Ok(()) + } + + fn commit_to_provider( + &self, + provider: &ProviderAccount, + repo: &str, + path: &str, + contents: &str, + message: &str, + timestamp: i64, + ) -> Result<()> { + let work = self.clone_repo( + provider, + repo, + &format!("commit-{}-{repo}", provider.site_name), + )?; + write_commit(&work, path, contents, message, timestamp)?; + self.git(&work, ["push", "origin", "HEAD:main"])?; + provider.wait_branch( + repo, + MAIN_BRANCH, + &git_output(&work, ["rev-parse", "HEAD"])?, + )?; + provider.wait_repo_listed(repo)?; + Ok(()) + } + + fn clone_repo(&self, provider: &ProviderAccount, repo: &str, label: &str) -> Result { + let path = self.git_worktree(label)?; + let remote_url = provider.authenticated_repo_url(repo)?; + let output = Command::new("git") + .args(["clone", &remote_url, path.to_str().unwrap()]) + .env("GIT_TERMINAL_PROMPT", "0") + .output() + .context("failed to run git clone")?; + assert_output_success(output, "git clone", &self.redactor)?; + Ok(path) + } + + fn git_worktree(&self, label: &str) -> Result { + let path = self.temp.path().join("git").join(sanitize_path(label)); + fs::create_dir_all(path.parent().unwrap())?; + Ok(path) + } + + fn git(&self, path: &Path, args: [&str; N]) -> Result<()> { + let output = Command::new("git") + .args(args) + .current_dir(path) + .env("GIT_TERMINAL_PROMPT", "0") + .output() + .context("failed to run git")?; + assert_output_success(output, "git", &self.redactor) + } + + fn sync(&self, args: [&str; N]) -> Result<()> { + let mut command = vec![ + "sync", + "--work-dir", + self.work_dir.to_str().unwrap(), + "--jobs", + "1", + ]; + command.extend(args); + self.refray(command) + } + + fn sync_expect_failure(&self, args: [&str; N]) -> Result<()> { + let mut command = vec![ + "sync", + "--work-dir", + self.work_dir.to_str().unwrap(), + "--jobs", + "1", + ]; + command.extend(args); + let output = self.refray_output(command)?; + if output.status.success() { + bail!("expected refray sync to fail, but it succeeded"); + } + Ok(()) + } + + fn refray(&self, args: I) -> Result<()> + where + I: IntoIterator, + S: AsRef, + { + let args = args + .into_iter() + .map(|arg| arg.as_ref().to_string()) + .collect::>(); + let mut output = self.refray_output(args.iter().map(String::as_str))?; + if output.status.code() == Some(124) { + output = self.refray_output(args.iter().map(String::as_str))?; + } + assert_output_success(output, "refray", &self.redactor) + } + + fn refray_output(&self, args: I) -> Result + where + I: IntoIterator, + S: AsRef, + { + let mut command = Command::new("timeout"); + command + .arg("--kill-after=5s") + .arg("240s") + .arg(env!("CARGO_BIN_EXE_refray")) + .arg("--config") + .arg(&self.config_path); + for arg in args { + command.arg(arg.as_ref()); + } + command + .env("GIT_TERMINAL_PROMPT", "0") + .output() + .context("failed to run refray") + } + + fn spawn_refray(&self, args: [&str; N]) -> Result { + let mut command = Command::new(env!("CARGO_BIN_EXE_refray")); + command.arg("--config").arg(&self.config_path); + command.args(args); + command + .env("GIT_TERMINAL_PROMPT", "0") + .spawn() + .context("failed to spawn refray") + } + + fn primary_provider(&self) -> &ProviderAccount { + &self.settings.providers[self.primary_provider_index()] + } + + fn provider_pair(&self) -> (&ProviderAccount, &ProviderAccount) { + let first = self.primary_provider_index(); + let second = self + .settings + .providers + .iter() + .enumerate() + .find(|(index, provider)| *index != first && provider.site_name == "gitlab") + .map(|(index, _)| index) + .or_else(|| { + self.settings + .providers + .iter() + .enumerate() + .find(|(index, _)| *index != first) + .map(|(index, _)| index) + }) + .expect("preflight ensures at least two providers"); + ( + &self.settings.providers[first], + &self.settings.providers[second], + ) + } + + fn primary_provider_index(&self) -> usize { + self.settings + .providers + .iter() + .position(|provider| provider.site_name == "github") + .unwrap_or(0) + } + + fn assert_repo_exists_everywhere(&self, repo: &str) -> Result<()> { + for provider in &self.settings.providers { + provider.wait_repo_present(repo)?; + } + Ok(()) + } + + fn assert_repo_absent_everywhere(&self, repo: &str) -> Result<()> { + for provider in &self.settings.providers { + provider.wait_repo_absent(repo)?; + } + Ok(()) + } + + fn assert_only_provider_has_repo(&self, repo: &str, site_name: &str) -> Result<()> { + for provider in &self.settings.providers { + let exists = provider.repo_exists(repo)?; + if provider.site_name == site_name && !exists { + bail!("expected {repo} to exist on {}", provider.site_name); + } + if provider.site_name != site_name && exists { + bail!("expected {repo} to be absent on {}", provider.site_name); + } + } + Ok(()) + } + + fn assert_branch_all_equal(&self, repo: &str, branch: &str) -> Result<()> { + retry("branch convergence", || { + let refs = self.refs_by_provider(repo)?; + let values = refs + .values() + .map(|refs| refs.branches.get(branch).cloned()) + .collect::>(); + if values.iter().any(Option::is_none) { + bail!("branch {branch} missing on at least one provider: {values:?}"); + } + let unique = values.into_iter().flatten().collect::>(); + if unique.len() != 1 { + bail!("branch {branch} differs across providers: {unique:?}"); + } + Ok(()) + }) + } + + fn assert_branch_all_equal_after_optional_resync( + &self, + repo: &str, + branch: &str, + ) -> Result<()> { + match self.assert_branch_all_equal(repo, branch) { + Ok(()) => Ok(()), + Err(first_error) => { + thread::sleep(Duration::from_secs(2)); + self.sync(["--repo-pattern", &exact_pattern(repo)]) + .with_context(|| format!("initial convergence failed: {first_error:#}"))?; + self.assert_branch_all_equal(repo, branch) + } + } + } + + fn assert_tag_all_equal(&self, repo: &str, tag: &str) -> Result<()> { + retry("tag convergence", || { + let refs = self.refs_by_provider(repo)?; + let values = refs + .values() + .map(|refs| refs.tags.get(tag).cloned()) + .collect::>(); + if values.iter().any(Option::is_none) { + bail!("tag {tag} missing on at least one provider: {values:?}"); + } + let unique = values.into_iter().flatten().collect::>(); + if unique.len() != 1 { + bail!("tag {tag} differs across providers: {unique:?}"); + } + Ok(()) + }) + } + + fn assert_branch_absent_everywhere(&self, repo: &str, branch: &str) -> Result<()> { + retry("branch deletion", || { + for (provider, refs) in self.refs_by_provider(repo)? { + if refs.branches.contains_key(branch) { + bail!("branch {branch} still exists on {provider}"); + } + } + Ok(()) + }) + } + + fn assert_conflict_branch_exists(&self, repo: &str) -> Result<()> { + retry("conflict branch", || { + for refs in self.refs_by_provider(repo)?.values() { + if refs + .branches + .keys() + .any(|branch| branch.starts_with("refray/conflicts/")) + { + return Ok(()); + } + } + bail!("no refray/conflicts branch found for {repo}") + }) + } + + fn refs_by_provider(&self, repo: &str) -> Result> { + let mut output = BTreeMap::new(); + for provider in &self.settings.providers { + output.insert(provider.site_name.clone(), provider.ls_remote(repo)?); + } + Ok(output) + } + + fn unprotect_main_all(&self, repo: &str) -> Result<()> { + for provider in &self.settings.providers { + provider.unprotect_branch(repo, MAIN_BRANCH)?; + } + Ok(()) + } +} + +impl Drop for E2eRun { + fn drop(&mut self) { + if !self.settings.destructive_guard { + return; + } + for provider in &self.settings.providers { + let repos = match provider.list_repos() { + Ok(repos) => repos, + Err(error) => { + eprintln!( + "best-effort e2e cleanup could not list {} repos: {error:#}", + provider.site_name + ); + continue; + } + }; + for repo in repos + .into_iter() + .filter(|repo| repo.name.starts_with(REPO_PREFIX)) + { + if let Err(error) = provider.delete_repo(&repo.name) { + eprintln!( + "best-effort e2e cleanup could not delete {} on {}: {error:#}", + repo.name, provider.site_name + ); + } + } + } + } +} + +#[derive(Clone)] +struct ProviderAccount { + site_name: String, + kind: ProviderKind, + base_url: String, + username: String, + token: String, + http: Client, +} + +impl ProviderAccount { + fn new( + site_name: impl Into, + kind: ProviderKind, + base_url: String, + username: String, + token: String, + ) -> Self { + Self { + site_name: site_name.into(), + kind, + base_url: trim_url(&base_url).to_string(), + username, + token, + http: Client::builder() + .timeout(Duration::from_secs(30)) + .build() + .unwrap(), + } + } + + fn api_base(&self) -> String { + match self.kind { + ProviderKind::Github if self.base_url == "https://github.com" => { + "https://api.github.com".to_string() + } + ProviderKind::Github => format!("{}/api/v3", self.base_url), + ProviderKind::Gitlab => format!("{}/api/v4", self.base_url), + ProviderKind::Gitea | ProviderKind::Forgejo => format!("{}/api/v1", self.base_url), + } + } + + fn validate_token(&self) -> Result { + let value = self + .get_json::(&format!("{}/user", self.api_base())) + .with_context(|| format!("failed to validate {} token", self.site_name))?; + value + .get(match self.kind { + ProviderKind::Github | ProviderKind::Gitea | ProviderKind::Forgejo => "login", + ProviderKind::Gitlab => "username", + }) + .and_then(Value::as_str) + .map(ToOwned::to_owned) + .ok_or_else(|| anyhow!("{} /user response did not include username", self.site_name)) + } + + fn list_repos(&self) -> Result> { + match self.kind { + ProviderKind::Github => { + let url = format!( + "{}/user/repos?affiliation=owner&visibility=all&per_page=100", + self.api_base() + ); + let repos = self.paged_get(&url)?; + Ok(repos + .into_iter() + .filter(|repo| repo.owner_login.as_deref() == Some(self.username.as_str())) + .collect()) + } + ProviderKind::Gitlab => { + let url = format!( + "{}/projects?owned=true&simple=true&per_page=100", + self.api_base() + ); + let repos = self.paged_get(&url)?; + Ok(repos + .into_iter() + .filter(|repo| repo.owner_login.as_deref() == Some(self.username.as_str())) + .collect()) + } + ProviderKind::Gitea | ProviderKind::Forgejo => { + let url = format!("{}/user/repos?limit=50", self.api_base()); + let repos = self.paged_get(&url)?; + Ok(repos + .into_iter() + .filter(|repo| repo.owner_login.as_deref() == Some(self.username.as_str())) + .collect()) + } + } + } + + fn repo_exists(&self, repo: &str) -> Result { + let url = self.repo_api_url(repo); + let response = self.auth(self.http.get(url)).send()?; + if response.status().is_success() { + if matches!(self.kind, ProviderKind::Gitlab) { + let value: Value = response.json()?; + if value + .get("marked_for_deletion_on") + .is_some_and(|value| !value.is_null()) + || value + .get("pending_delete") + .and_then(Value::as_bool) + .unwrap_or(false) + { + return Ok(false); + } + } + Ok(true) + } else if response.status().as_u16() == 404 { + Ok(false) + } else { + bail!( + "{} repo lookup returned {}", + self.site_name, + response.status() + ) + } + } + + fn wait_repo_present(&self, repo: &str) -> Result<()> { + retry("repo present", || { + if self.repo_exists(repo)? { + Ok(()) + } else { + bail!("{repo} is not visible on {} yet", self.site_name) + } + }) + } + + fn wait_repo_listed(&self, repo: &str) -> Result<()> { + retry("repo listed", || { + if self.list_repos()?.iter().any(|item| item.name == repo) { + Ok(()) + } else { + bail!("{repo} is not listed on {} yet", self.site_name) + } + }) + } + + fn wait_repo_absent(&self, repo: &str) -> Result<()> { + retry("repo absent", || { + if self.repo_exists(repo)? { + bail!("{repo} still exists on {}", self.site_name) + } else { + Ok(()) + } + }) + } + + fn wait_branch(&self, repo: &str, branch: &str, expected: &str) -> Result<()> { + retry("branch visible", || { + let refs = self.ls_remote(repo)?; + match refs.branches.get(branch) { + Some(actual) if actual == expected => Ok(()), + Some(actual) => bail!( + "{} branch {branch} is at {actual}, expected {expected}", + self.site_name + ), + None => bail!("{} branch {branch} is not visible yet", self.site_name), + } + }) + } + + fn wait_branch_absent(&self, repo: &str, branch: &str) -> Result<()> { + retry("branch absent", || { + let refs = self.ls_remote(repo)?; + if refs.branches.contains_key(branch) { + bail!("{} branch {branch} is still visible", self.site_name) + } + Ok(()) + }) + } + + fn create_repo(&self, name: &str) -> Result<()> { + if self.repo_exists(name)? { + self.wait_repo_listed(name)?; + return Ok(()); + } + let body = match self.kind { + ProviderKind::Github => json!({ "name": name, "private": false, "auto_init": false }), + ProviderKind::Gitlab => json!({ "name": name, "path": name, "visibility": "public" }), + ProviderKind::Gitea | ProviderKind::Forgejo => { + json!({ "name": name, "private": false, "auto_init": false }) + } + }; + let url = match self.kind { + ProviderKind::Github => format!("{}/user/repos", self.api_base()), + ProviderKind::Gitlab => format!("{}/projects", self.api_base()), + ProviderKind::Gitea | ProviderKind::Forgejo => { + format!("{}/user/repos", self.api_base()) + } + }; + let response = self.auth(self.http.post(url.clone()).json(&body)).send()?; + check_response("POST", &url, response)?; + self.wait_repo_present(name)?; + self.wait_repo_listed(name) + } + + fn delete_repo(&self, name: &str) -> Result<()> { + let url = self.repo_api_url(name); + let response = self.auth(self.http.delete(url.clone())).send()?; + if response.status().as_u16() == 404 { + return Ok(()); + } + if matches!(self.kind, ProviderKind::Gitlab) && response.status().as_u16() == 400 { + let body = response.text().unwrap_or_default(); + if body.contains("already been marked for deletion") { + return Ok(()); + } + bail!("DELETE {url} returned 400 Bad Request: {body}"); + } + check_response("DELETE", &url, response).map(|_| ()) + } + + fn unprotect_branch(&self, repo: &str, branch: &str) -> Result<()> { + if !matches!(self.kind, ProviderKind::Gitlab) { + return Ok(()); + } + let url = format!( + "{}/protected_branches/{}", + self.repo_api_url(repo), + urlencoding(branch) + ); + let response = self.auth(self.http.delete(url.clone())).send()?; + if response.status().is_success() || response.status().as_u16() == 404 { + return Ok(()); + } + check_response("DELETE", &url, response).map(|_| ()) + } + + fn repo_api_url(&self, repo: &str) -> String { + match self.kind { + ProviderKind::Github => format!("{}/repos/{}/{}", self.api_base(), self.username, repo), + ProviderKind::Gitlab => format!( + "{}/projects/{}", + self.api_base(), + urlencoding(&format!("{}/{}", self.username, repo)) + ), + ProviderKind::Gitea | ProviderKind::Forgejo => { + format!("{}/repos/{}/{}", self.api_base(), self.username, repo) + } + } + } + + fn authenticated_repo_url(&self, repo: &str) -> Result { + let username = match self.kind { + ProviderKind::Github => "x-access-token", + ProviderKind::Gitlab | ProviderKind::Gitea | ProviderKind::Forgejo => "oauth2", + }; + let mut url = Url::parse(&self.base_url) + .with_context(|| format!("invalid {} base URL", self.site_name))?; + url.set_username(username) + .map_err(|_| anyhow!("failed to set username on {} clone URL", self.site_name))?; + url.set_password(Some(&self.token)) + .map_err(|_| anyhow!("failed to set token on {} clone URL", self.site_name))?; + let base_path = url.path().trim_end_matches('/'); + let repo_path = format!("{}/{}.git", self.username, repo); + if base_path.is_empty() { + url.set_path(&repo_path); + } else { + url.set_path(&format!("{base_path}/{repo_path}")); + } + Ok(url.to_string()) + } + + fn webhook_payload(&self, repo: &str, secret: &str) -> (String, Vec<(&'static str, String)>) { + match self.kind { + ProviderKind::Gitlab => { + let body = json!({ + "project": { + "path": repo, + "path_with_namespace": format!("{}/{}", self.username, repo), + "namespace": self.username, + } + }) + .to_string(); + ( + body, + vec![ + ("X-Gitlab-Event", "Push Hook".to_string()), + ("X-Gitlab-Token", secret.to_string()), + ], + ) + } + ProviderKind::Github | ProviderKind::Gitea | ProviderKind::Forgejo => { + let body = json!({ + "repository": { + "name": repo, + "full_name": format!("{}/{}", self.username, repo), + "owner": { "login": self.username }, + } + }) + .to_string(); + let signature = format!( + "sha256={}", + hmac_sha256_hex(secret.as_bytes(), body.as_bytes()) + ); + let event_header = match self.kind { + ProviderKind::Github => "X-GitHub-Event", + ProviderKind::Gitea => "X-Gitea-Event", + ProviderKind::Forgejo => "X-Forgejo-Event", + ProviderKind::Gitlab => unreachable!(), + }; + let signature_header = match self.kind { + ProviderKind::Github => "X-Hub-Signature-256", + ProviderKind::Gitea => "X-Gitea-Signature", + ProviderKind::Forgejo => "X-Forgejo-Signature", + ProviderKind::Gitlab => unreachable!(), + }; + ( + body, + vec![ + (event_header, "push".to_string()), + (signature_header, signature), + ], + ) + } + } + } + + fn ls_remote(&self, repo: &str) -> Result { + let remote_url = self.authenticated_repo_url(repo)?; + let output = Command::new("git") + .args(["ls-remote", "--heads", "--tags", "--refs", &remote_url]) + .env("GIT_TERMINAL_PROMPT", "0") + .output() + .context("failed to run git ls-remote")?; + if !output.status.success() { + let redactor = Redactor::new(vec![self.token.clone()]); + let stdout = redactor.redact(&String::from_utf8_lossy(&output.stdout)); + let stderr = redactor.redact(&String::from_utf8_lossy(&output.stderr)); + bail!( + "failed to inspect refs for {} on {}: {}\nstdout:\n{}\nstderr:\n{}", + repo, + self.site_name, + output.status, + stdout, + stderr + ); + } + GitRefs::from_output(&output.stdout) + } + + fn paged_get(&self, first_url: &str) -> Result> { + let mut url = first_url.to_string(); + let mut output = Vec::new(); + loop { + let response = self.auth(self.http.get(url.clone())).send()?; + let headers = response.headers().clone(); + let response = check_response("GET", &url, response)?; + let value: Value = response.json()?; + let items = value + .as_array() + .ok_or_else(|| anyhow!("{} did not return a repository array", self.site_name))?; + output.extend( + items + .iter() + .map(|value| ProviderRepo::from_json(self.kind, value)), + ); + let Some(next) = next_link(&headers) else { + break; + }; + url = next; + } + Ok(output) + } + + fn get_json(&self, url: &str) -> Result { + let response = self.auth(self.http.get(url.to_string())).send()?; + check_response("GET", url, response)? + .json() + .map_err(Into::into) + } + + fn auth(&self, request: RequestBuilder) -> RequestBuilder { + let mut headers = HeaderMap::new(); + headers.insert(USER_AGENT, HeaderValue::from_static("refray-e2e/0.1")); + headers.insert(ACCEPT, HeaderValue::from_static("application/json")); + match self.kind { + ProviderKind::Github => { + headers.insert( + AUTHORIZATION, + HeaderValue::from_str(&format!("Bearer {}", self.token)).unwrap(), + ); + headers.insert( + "X-GitHub-Api-Version", + HeaderValue::from_static("2022-11-28"), + ); + } + ProviderKind::Gitlab => { + headers.insert("PRIVATE-TOKEN", HeaderValue::from_str(&self.token).unwrap()); + } + ProviderKind::Gitea | ProviderKind::Forgejo => { + headers.insert( + AUTHORIZATION, + HeaderValue::from_str(&format!("token {}", self.token)).unwrap(), + ); + } + } + request.headers(headers) + } +} + +#[derive(Clone, Copy)] +enum ProviderKind { + Github, + Gitlab, + Gitea, + Forgejo, +} + +impl ProviderKind { + fn config_name(self) -> &'static str { + match self { + ProviderKind::Github => "github", + ProviderKind::Gitlab => "gitlab", + ProviderKind::Gitea => "gitea", + ProviderKind::Forgejo => "forgejo", + } + } +} + +enum ConflictMode { + Fail, + AutoRebase, + PullRequest, + AutoRebasePullRequest, +} + +impl ConflictMode { + fn config_name(&self) -> &'static str { + match self { + ConflictMode::Fail => "fail", + ConflictMode::AutoRebase => "auto_rebase", + ConflictMode::PullRequest => "pull_request", + ConflictMode::AutoRebasePullRequest => "auto_rebase_pull_request", + } + } +} + +struct ProviderRepo { + name: String, + owner_login: Option, +} + +impl ProviderRepo { + fn from_json(kind: ProviderKind, value: &Value) -> Self { + let name = match kind { + ProviderKind::Github | ProviderKind::Gitea | ProviderKind::Forgejo => value + .get("name") + .and_then(Value::as_str) + .unwrap_or_default(), + ProviderKind::Gitlab => value + .get("path") + .or_else(|| value.get("name")) + .and_then(Value::as_str) + .unwrap_or_default(), + } + .to_string(); + let owner_login = match kind { + ProviderKind::Github | ProviderKind::Gitea | ProviderKind::Forgejo => value + .pointer("/owner/login") + .or_else(|| value.pointer("/owner/username")) + .and_then(Value::as_str) + .map(ToOwned::to_owned), + ProviderKind::Gitlab => value + .pointer("/namespace/path") + .and_then(Value::as_str) + .map(ToOwned::to_owned), + }; + Self { name, owner_login } + } +} + +struct GitRefs { + branches: BTreeMap, + tags: BTreeMap, +} + +impl GitRefs { + fn from_output(output: &[u8]) -> Result { + let mut branches = BTreeMap::new(); + let mut tags = BTreeMap::new(); + for line in String::from_utf8_lossy(output).lines() { + let Some((sha, reference)) = line.split_once('\t') else { + continue; + }; + if let Some(branch) = reference.strip_prefix("refs/heads/") { + branches.insert(branch.to_string(), sha.to_string()); + } else if let Some(tag) = reference.strip_prefix("refs/tags/") { + tags.insert(tag.to_string(), sha.to_string()); + } + } + Ok(Self { branches, tags }) + } +} + +#[derive(Clone)] +struct Redactor { + secrets: Vec, +} + +impl Redactor { + fn new(secrets: Vec) -> Self { + Self { secrets } + } + + fn redact(&self, value: &str) -> String { + self.secrets + .iter() + .fold(value.to_string(), |mut value, secret| { + if !secret.is_empty() { + value = value.replace(secret, ""); + } + value + }) + } +} + +fn git_init(path: &Path) -> Result<()> { + fs::create_dir_all(path)?; + git(path, ["init", "-b", MAIN_BRANCH])?; + git(path, ["config", "user.email", "refray-e2e@example.invalid"])?; + git(path, ["config", "user.name", "refray e2e"]) +} + +fn write_commit( + path: &Path, + file: &str, + contents: &str, + message: &str, + timestamp: i64, +) -> Result<()> { + let file_path = path.join(file); + if let Some(parent) = file_path.parent() { + fs::create_dir_all(parent)?; + } + fs::write(file_path, contents)?; + git(path, ["add", file])?; + let timestamp = format!("{timestamp} +0000"); + let output = Command::new("git") + .args(["commit", "-m", message]) + .current_dir(path) + .env("GIT_AUTHOR_DATE", ×tamp) + .env("GIT_COMMITTER_DATE", ×tamp) + .output() + .context("failed to run git commit")?; + assert_output_success(output, "git commit", &Redactor::new(Vec::new())) +} + +fn git(path: &Path, args: [&str; N]) -> Result<()> { + let output = Command::new("git") + .args(args) + .current_dir(path) + .env("GIT_TERMINAL_PROMPT", "0") + .output() + .context("failed to run git")?; + assert_output_success(output, "git", &Redactor::new(Vec::new())) +} + +fn git_output(path: &Path, args: [&str; N]) -> Result { + let output = Command::new("git") + .args(args) + .current_dir(path) + .env("GIT_TERMINAL_PROMPT", "0") + .output() + .context("failed to run git")?; + if !output.status.success() { + return assert_output_success(output, "git", &Redactor::new(Vec::new())) + .map(|_| String::new()); + } + Ok(String::from_utf8_lossy(&output.stdout).trim().to_string()) +} + +fn assert_output_success(output: Output, label: &str, redactor: &Redactor) -> Result<()> { + if output.status.success() { + return Ok(()); + } + let stdout = redactor.redact(&String::from_utf8_lossy(&output.stdout)); + let stderr = redactor.redact(&String::from_utf8_lossy(&output.stderr)); + bail!( + "{label} failed with {}\nstdout:\n{stdout}\nstderr:\n{stderr}", + output.status + ) +} + +fn retry(label: &str, mut action: impl FnMut() -> Result<()>) -> Result<()> { + let mut last_error = None; + for _ in 0..30 { + match action() { + Ok(()) => return Ok(()), + Err(error) => last_error = Some(error), + } + thread::sleep(Duration::from_secs(2)); + } + Err(last_error.unwrap_or_else(|| anyhow!("{label} did not complete"))) + .with_context(|| format!("timed out waiting for {label}")) +} + +fn check_response( + method: &str, + url: &str, + response: reqwest::blocking::Response, +) -> Result { + if response.status().is_success() { + return Ok(response); + } + let status = response.status(); + let body = response.text().unwrap_or_default(); + bail!("{method} {url} returned {status}: {body}") +} + +fn next_link(headers: &HeaderMap) -> Option { + let header = headers.get("link")?.to_str().ok()?; + for part in header.split(',') { + let mut sections = part.trim().split(';'); + let url = sections.next()?.trim(); + let rel = sections.any(|section| section.trim() == "rel=\"next\""); + if rel { + return url + .strip_prefix('<') + .and_then(|value| value.strip_suffix('>')) + .map(ToString::to_string); + } + } + None +} + +fn hmac_sha256_hex(secret: &[u8], body: &[u8]) -> String { + let mut mac = HmacSha256::new_from_slice(secret).unwrap(); + mac.update(body); + let bytes = mac.finalize().into_bytes(); + bytes.iter().map(|byte| format!("{byte:02x}")).collect() +} + +fn exact_pattern(repo: &str) -> String { + format!("^{}$", regex::escape(repo)) +} + +fn trim_url(value: &str) -> &str { + value.trim_end_matches('/') +} + +fn urlencoding(value: &str) -> String { + url::form_urlencoded::byte_serialize(value.as_bytes()).collect() +} + +fn sanitize_path(value: &str) -> String { + value + .chars() + .map(|ch| if ch.is_ascii_alphanumeric() { ch } else { '-' }) + .collect() +} + +fn stop_child(child: &mut Child) { + let _ = child.kill(); + let _ = child.wait(); +} diff --git a/tests/unit/config.rs b/tests/unit/config.rs index b26f90b..3c5b0e9 100644 --- a/tests/unit/config.rs +++ b/tests/unit/config.rs @@ -19,6 +19,9 @@ fn parses_token_forms() { [[mirrors]] name = "personal" + sync_visibility = "public" + repo_whitelist = ["^important-", "-mirror$"] + repo_blacklist = ["-archive$"] create_missing = true visibility = "private" allow_force = false @@ -43,6 +46,15 @@ fn parses_token_forms() { config.mirrors[0].conflict_resolution, ConflictResolutionStrategy::AutoRebasePullRequest ); + assert_eq!(config.mirrors[0].sync_visibility, SyncVisibility::Public); + assert_eq!( + config.mirrors[0].repo_whitelist, + vec!["^important-".to_string(), "-mirror$".to_string()] + ); + assert_eq!( + config.mirrors[0].repo_blacklist, + vec!["-archive$".to_string()] + ); let webhook = config.webhook.unwrap(); assert!(webhook.install); assert_eq!(webhook.url, "https://mirror.example.test/webhook"); @@ -64,6 +76,9 @@ fn validation_rejects_unknown_sites_and_single_endpoint_groups() { kind: NamespaceKind::User, namespace: "alice".to_string(), }], + sync_visibility: SyncVisibility::All, + repo_whitelist: Vec::new(), + repo_blacklist: Vec::new(), create_missing: true, visibility: Visibility::Private, allow_force: false, @@ -90,6 +105,9 @@ fn validation_rejects_unknown_sites_and_single_endpoint_groups() { namespace: "alice".to_string(), }, ], + sync_visibility: SyncVisibility::All, + repo_whitelist: Vec::new(), + repo_blacklist: Vec::new(), create_missing: true, visibility: Visibility::Private, allow_force: false, @@ -141,6 +159,68 @@ fn api_base_defaults_match_providers() { ); } +#[test] +fn sync_visibility_matches_repo_privacy() { + assert!(SyncVisibility::All.matches_private(true)); + assert!(SyncVisibility::All.matches_private(false)); + assert!(SyncVisibility::Private.matches_private(true)); + assert!(!SyncVisibility::Private.matches_private(false)); + assert!(!SyncVisibility::Public.matches_private(true)); + assert!(SyncVisibility::Public.matches_private(false)); +} + +#[test] +fn repo_name_filter_applies_whitelist_then_blacklist() { + let mut mirror = mirror_config(); + mirror.repo_whitelist = vec!["^important-".to_string(), "-mirror$".to_string()]; + mirror.repo_blacklist = vec!["-archive$".to_string()]; + let filter = mirror.repo_filter().unwrap(); + + assert!(filter.matches("important-api")); + assert!(filter.matches("user-mirror")); + assert!(!filter.matches("important-archive")); + assert!(!filter.matches("random")); +} + +#[test] +fn validation_rejects_invalid_repo_filter_regex() { + let mut config = Config { + sites: vec![site("github", ProviderKind::Github)], + mirrors: vec![mirror_config()], + webhook: None, + }; + config.mirrors[0].repo_whitelist = vec!["(".to_string()]; + + let err = validate_config(&config).unwrap_err().to_string(); + + assert!(err.contains("invalid repo_whitelist regex")); +} + +fn mirror_config() -> MirrorConfig { + MirrorConfig { + name: "personal".to_string(), + endpoints: vec![ + EndpointConfig { + site: "github".to_string(), + kind: NamespaceKind::User, + namespace: "alice".to_string(), + }, + EndpointConfig { + site: "github".to_string(), + kind: NamespaceKind::Org, + namespace: "example".to_string(), + }, + ], + sync_visibility: SyncVisibility::All, + repo_whitelist: Vec::new(), + repo_blacklist: Vec::new(), + create_missing: true, + visibility: Visibility::Private, + allow_force: false, + conflict_resolution: ConflictResolutionStrategy::Fail, + } +} + fn site(name: &str, provider: ProviderKind) -> SiteConfig { SiteConfig { name: name.to_string(), diff --git a/tests/unit/interactive.rs b/tests/unit/interactive.rs index a8eeb84..9e3109a 100644 --- a/tests/unit/interactive.rs +++ b/tests/unit/interactive.rs @@ -12,6 +12,8 @@ fn wizard_builds_sync_group_from_profile_urls() { "", "n", "", + "", + "", "n", "4", ] @@ -42,6 +44,7 @@ fn wizard_builds_sync_group_from_profile_urls() { assert_eq!(config.mirrors[0].endpoints[0].namespace, "hykilpikonna"); assert_eq!(config.mirrors[0].endpoints[1].site, "gitea-example-test"); assert_eq!(config.mirrors[0].endpoints[1].namespace, "azalea"); + assert_eq!(config.mirrors[0].sync_visibility, SyncVisibility::All); assert!(config.mirrors[0].create_missing); assert_eq!(config.mirrors[0].visibility, Visibility::Private); assert!(!config.mirrors[0].allow_force); @@ -73,6 +76,8 @@ fn wizard_can_build_three_way_sync() { "", "n", "", + "", + "", "n", "4", ] @@ -99,6 +104,8 @@ fn wizard_can_enable_webhooks() { "", "n", "", + "", + "", "y", "https://mirror.example.test/webhook", "y", @@ -150,6 +157,8 @@ fn wizard_reuses_existing_credentials_for_same_instance() { "", "n", "", + "", + "", "n", "4", ] @@ -200,6 +209,9 @@ fn wizard_starts_existing_config_at_sync_group_menu() { namespace: "alice".to_string(), }, ], + sync_visibility: SyncVisibility::All, + repo_whitelist: Vec::new(), + repo_blacklist: Vec::new(), create_missing: true, visibility: Visibility::Private, allow_force: false, @@ -226,6 +238,9 @@ fn wizard_can_ask_to_run_full_sync_after_config() { mirrors: vec![MirrorConfig { name: "sync-1".to_string(), endpoints: Vec::new(), + sync_visibility: SyncVisibility::All, + repo_whitelist: Vec::new(), + repo_blacklist: Vec::new(), create_missing: true, visibility: Visibility::Private, allow_force: false, @@ -299,6 +314,9 @@ fn wizard_edits_existing_sync_group_from_menu() { namespace: "alice".to_string(), }, ], + sync_visibility: SyncVisibility::Private, + repo_whitelist: vec!["^important-".to_string()], + repo_blacklist: vec!["-archive$".to_string()], create_missing: false, visibility: Visibility::Public, allow_force: true, @@ -314,6 +332,10 @@ fn wizard_edits_existing_sync_group_from_menu() { "https://gitlab.com/bob", "", "n", + "public", + "y", + "^public-", + "-skip$", "", "n", "4", @@ -334,6 +356,9 @@ fn wizard_edits_existing_sync_group_from_menu() { assert_eq!(mirror.endpoints[1].site, "gitlab"); assert_eq!(mirror.endpoints[1].namespace, "bob"); assert!(!mirror.create_missing); + assert_eq!(mirror.sync_visibility, SyncVisibility::Public); + assert_eq!(mirror.repo_whitelist, vec!["^public-".to_string()]); + assert_eq!(mirror.repo_blacklist, vec!["-skip$".to_string()]); assert_eq!(mirror.visibility, Visibility::Public); assert!(mirror.allow_force); @@ -378,6 +403,9 @@ fn wizard_prefills_existing_sync_group_when_editing() { namespace: "alice".to_string(), }, ], + sync_visibility: SyncVisibility::All, + repo_whitelist: Vec::new(), + repo_blacklist: Vec::new(), create_missing: true, visibility: Visibility::Private, allow_force: false, @@ -385,7 +413,7 @@ fn wizard_prefills_existing_sync_group_when_editing() { }], webhook: None, }; - let input = ["2", "1", "", "", "", "", "n", "", "n", "4"].join("\n") + "\n"; + let input = ["2", "1", "", "", "", "", "n", "", "", "", "n", "4"].join("\n") + "\n"; let mut reader = Cursor::new(input.as_bytes()); let mut output = Vec::new(); @@ -439,6 +467,9 @@ fn wizard_deletes_existing_sync_group_from_menu() { namespace: "alice".to_string(), }, ], + sync_visibility: SyncVisibility::All, + repo_whitelist: Vec::new(), + repo_blacklist: Vec::new(), create_missing: true, visibility: Visibility::Private, allow_force: false, @@ -495,6 +526,9 @@ fn wizard_can_go_back_from_delete_menu() { namespace: "alice".to_string(), }, ], + sync_visibility: SyncVisibility::All, + repo_whitelist: Vec::new(), + repo_blacklist: Vec::new(), create_missing: true, visibility: Visibility::Private, allow_force: false, diff --git a/tests/unit/interactive_test_io.rs b/tests/unit/interactive_test_io.rs index bbf564b..4cbcf07 100644 --- a/tests/unit/interactive_test_io.rs +++ b/tests/unit/interactive_test_io.rs @@ -63,10 +63,15 @@ where W: Write, { let endpoints = prompt_sync_group_endpoints(reader, writer, config, &[])?; + let sync_visibility = prompt_sync_visibility(reader, writer, None)?; + let repo_filters = prompt_repo_filters(reader, writer, None)?; let conflict_resolution = prompt_conflict_resolution(reader, writer, None)?; config.upsert_mirror(MirrorConfig { name: next_mirror_name(config), endpoints, + sync_visibility, + repo_whitelist: repo_filters.whitelist, + repo_blacklist: repo_filters.blacklist, create_missing: true, visibility: Visibility::Private, allow_force: false, @@ -267,15 +272,27 @@ where match value.parse::() { Ok(index) if (1..=config.mirrors.len()).contains(&index) => { let existing = config.mirrors[index - 1].endpoints.clone(); + let existing_sync_visibility = config.mirrors[index - 1].sync_visibility.clone(); + let existing_repo_filters = RepoFilterInput { + whitelist: config.mirrors[index - 1].repo_whitelist.clone(), + blacklist: config.mirrors[index - 1].repo_blacklist.clone(), + }; let existing_conflict_resolution = config.mirrors[index - 1].conflict_resolution.clone(); let endpoints = prompt_sync_group_endpoints(reader, writer, config, &existing)?; + let sync_visibility = + prompt_sync_visibility(reader, writer, Some(&existing_sync_visibility))?; + let repo_filters = + prompt_repo_filters(reader, writer, Some(&existing_repo_filters))?; let conflict_resolution = prompt_conflict_resolution( reader, writer, Some(&existing_conflict_resolution), )?; config.mirrors[index - 1].endpoints = endpoints; + config.mirrors[index - 1].sync_visibility = sync_visibility; + config.mirrors[index - 1].repo_whitelist = repo_filters.whitelist; + config.mirrors[index - 1].repo_blacklist = repo_filters.blacklist; config.mirrors[index - 1].conflict_resolution = conflict_resolution; prompt_webhook_setup(reader, writer, config)?; writeln!(writer, "updated sync group {index}")?; @@ -476,6 +493,100 @@ where } } +fn prompt_sync_visibility( + reader: &mut R, + writer: &mut W, + existing: Option<&SyncVisibility>, +) -> Result +where + R: BufRead, + W: Write, +{ + let default = existing.map(sync_visibility_value).unwrap_or("all"); + loop { + writeln!(writer, "Which repositories should this sync group include?")?; + writeln!(writer, " 1. all")?; + writeln!(writer, " 2. private only")?; + writeln!(writer, " 3. public only")?; + let value = prompt_with_default(reader, writer, "Sync visibility", default)?; + match value.trim().to_ascii_lowercase().as_str() { + "1" | "all" => return Ok(SyncVisibility::All), + "2" | "private" | "private only" | "private-only" => { + return Ok(SyncVisibility::Private); + } + "3" | "public" | "public only" | "public-only" => { + return Ok(SyncVisibility::Public); + } + _ => writeln!(writer, "Enter 1, 2, 3, all, private, or public.")?, + } + } +} + +fn prompt_repo_filters( + reader: &mut R, + writer: &mut W, + existing: Option<&RepoFilterInput>, +) -> Result +where + R: BufRead, + W: Write, +{ + let existing = existing.cloned().unwrap_or_default(); + let has_existing = !existing.whitelist.is_empty() || !existing.blacklist.is_empty(); + if !prompt_bool( + reader, + writer, + "Configure repository name whitelist/blacklist?", + has_existing, + )? { + return Ok(RepoFilterInput::default()); + } + + Ok(RepoFilterInput { + whitelist: prompt_repo_pattern_list( + reader, + writer, + "Whitelist regexes (comma-separated, empty means all repo names)", + &existing.whitelist, + )?, + blacklist: prompt_repo_pattern_list( + reader, + writer, + "Blacklist regexes (comma-separated)", + &existing.blacklist, + )?, + }) +} + +fn prompt_repo_pattern_list( + reader: &mut R, + writer: &mut W, + label: &str, + existing: &[String], +) -> Result> +where + R: BufRead, + W: Write, +{ + let value = if existing.is_empty() { + prompt_optional(reader, writer, label)? + } else { + prompt_with_default(reader, writer, label, &existing.join(", "))? + }; + if let Err(error) = validate_repo_pattern_list(&value) { + bail!(error); + } + Ok(parse_repo_pattern_list(&value)) +} + +fn sync_visibility_value(sync_visibility: &SyncVisibility) -> &'static str { + match sync_visibility { + SyncVisibility::All => "all", + SyncVisibility::Private => "private", + SyncVisibility::Public => "public", + } +} + fn conflict_resolution_value(strategy: &ConflictResolutionStrategy) -> &'static str { match strategy { ConflictResolutionStrategy::Fail => "fail", @@ -540,6 +651,16 @@ where } } +fn prompt_optional(reader: &mut R, writer: &mut W, label: &str) -> Result +where + R: BufRead, + W: Write, +{ + write!(writer, "{label}: ")?; + writer.flush()?; + Ok(read_line(reader)?.trim().to_string()) +} + fn prompt_with_default( reader: &mut R, writer: &mut W, diff --git a/tests/unit/sync.rs b/tests/unit/sync.rs index fe702d6..aba8f79 100644 --- a/tests/unit/sync.rs +++ b/tests/unit/sync.rs @@ -308,6 +308,56 @@ fn repo_deletion_decision_ignores_repos_not_previously_synced_everywhere() { assert_eq!(decision, RepoDeletionDecision::None); } +#[test] +fn filtered_sync_visibility_does_not_treat_state_only_repos_as_deleted() { + let mut mirror = test_mirror(); + mirror.sync_visibility = crate::config::SyncVisibility::Public; + let mut ref_state = RefState::default(); + ref_state.set_repo( + &mirror.name, + "private-repo", + BTreeMap::from([("github_alice".to_string(), remote_ref_state("a", &[]))]), + ); + + let repo_filter = mirror.repo_filter().unwrap(); + let names = sync_candidate_repo_names(&HashMap::new(), &ref_state, &mirror, &repo_filter); + + assert!(names.is_empty()); +} + +#[test] +fn all_visibility_keeps_state_only_repos_for_deletion_detection() { + let mirror = test_mirror(); + let mut ref_state = RefState::default(); + ref_state.set_repo( + &mirror.name, + "deleted-repo", + BTreeMap::from([("github_alice".to_string(), remote_ref_state("a", &[]))]), + ); + + let repo_filter = mirror.repo_filter().unwrap(); + let names = sync_candidate_repo_names(&HashMap::new(), &ref_state, &mirror, &repo_filter); + + assert_eq!(names, BTreeSet::from(["deleted-repo".to_string()])); +} + +#[test] +fn repo_name_filters_do_not_treat_state_only_repos_as_deleted() { + let mut mirror = test_mirror(); + mirror.repo_whitelist = vec!["^public-".to_string()]; + let repo_filter = mirror.repo_filter().unwrap(); + let mut ref_state = RefState::default(); + ref_state.set_repo( + &mirror.name, + "private-repo", + BTreeMap::from([("github_alice".to_string(), remote_ref_state("a", &[]))]), + ); + + let names = sync_candidate_repo_names(&HashMap::new(), &ref_state, &mirror, &repo_filter); + + assert!(names.is_empty()); +} + #[test] fn conflict_branch_prefixes_are_reversible_not_slug_collisions() { let slash_branch = conflict_pr_branch_prefix("release/foo"); @@ -358,6 +408,9 @@ fn test_mirror() -> MirrorConfig { MirrorConfig { name: "sync-1".to_string(), endpoints: vec![endpoint("github"), endpoint("gitea")], + sync_visibility: crate::config::SyncVisibility::All, + repo_whitelist: Vec::new(), + repo_blacklist: Vec::new(), create_missing: true, visibility: crate::config::Visibility::Private, allow_force: false, diff --git a/tests/unit/webhook.rs b/tests/unit/webhook.rs index a2259e2..8c44042 100644 --- a/tests/unit/webhook.rs +++ b/tests/unit/webhook.rs @@ -1,4 +1,5 @@ use super::*; +use crate::config::SyncVisibility; use crate::config::{ ConflictResolutionStrategy, EndpointConfig, MirrorConfig, NamespaceKind, SiteConfig, TokenConfig, Visibility, @@ -109,6 +110,9 @@ fn matches_jobs_by_provider_and_namespace() { endpoint("github", NamespaceKind::User, "alice"), endpoint("gitea", NamespaceKind::User, "azalea"), ], + sync_visibility: SyncVisibility::All, + repo_whitelist: Vec::new(), + repo_blacklist: Vec::new(), create_missing: true, visibility: Visibility::Private, allow_force: false, @@ -129,6 +133,41 @@ fn matches_jobs_by_provider_and_namespace() { assert_eq!(jobs[0].repo, "repo"); } +#[test] +fn matching_jobs_respects_repo_name_filters() { + let mut mirror = MirrorConfig { + name: "sync-1".to_string(), + endpoints: vec![endpoint("github", NamespaceKind::User, "alice")], + sync_visibility: SyncVisibility::All, + repo_whitelist: vec!["^important-".to_string()], + repo_blacklist: vec!["-archive$".to_string()], + create_missing: true, + visibility: Visibility::Private, + allow_force: false, + conflict_resolution: ConflictResolutionStrategy::Fail, + }; + let config = Config { + sites: vec![site("github", ProviderKind::Github)], + mirrors: vec![mirror.clone()], + webhook: None, + }; + + assert_eq!( + matching_jobs(&config, &webhook_event("important-api")).len(), + 1 + ); + assert!(matching_jobs(&config, &webhook_event("important-archive")).is_empty()); + assert!(matching_jobs(&config, &webhook_event("random")).is_empty()); + + mirror.repo_whitelist.clear(); + let config = Config { + sites: vec![site("github", ProviderKind::Github)], + mirrors: vec![mirror], + webhook: None, + }; + assert_eq!(matching_jobs(&config, &webhook_event("random")).len(), 1); +} + #[test] fn webhook_state_persists_installations() { let temp = tempfile::TempDir::new().unwrap(); @@ -330,6 +369,14 @@ fn site(name: &str, provider: ProviderKind) -> SiteConfig { } } +fn webhook_event(repo: &str) -> WebhookEvent { + WebhookEvent { + provider: Some(ProviderKind::Github), + repo: repo.to_string(), + namespace: Some("alice".to_string()), + } +} + fn endpoint(site: &str, kind: NamespaceKind, namespace: &str) -> EndpointConfig { EndpointConfig { site: site.to_string(),