[+] End-to-end testing

This commit is contained in:
2026-05-08 15:32:33 +00:00
parent e43e555b37
commit 3d73f20c1f
15 changed files with 2405 additions and 62 deletions
+1
View File
@@ -1 +1,2 @@
/target
.env
+4
View File
@@ -19,3 +19,7 @@ tempfile = "3.13"
tiny_http = "0.12"
toml = "0.8"
url = "2.5"
[[test]]
name = "sequential"
path = "tests/e2e/sequential.rs"
+41
View File
@@ -72,6 +72,8 @@ Sync only repositories whose names match a regex:
refray sync --repo-pattern '^(foo|bar)-'
```
For persistent per-group filters, set `repo_whitelist` and/or `repo_blacklist` in the config instead. `--repo-pattern` is an extra one-off filter applied on top of the group config.
Retry only repositories that failed during the previous non-dry-run sync:
```sh
@@ -148,6 +150,10 @@ If `[webhook].reachability_check_interval_minutes` is configured, `serve` period
Each mirror group is treated as a set of equivalent namespaces. Repositories are matched by repository name across all endpoints.
Set `sync_visibility = "all"`, `"private"`, or `"public"` on a mirror group to choose which repository visibility is included in that group. `visibility` still controls the visibility used when `refray` creates a missing repository.
Set `repo_whitelist = ["..."]` and/or `repo_blacklist = ["..."]` on a mirror group to filter repository names with regular expressions. An empty whitelist includes all repository names, and blacklist matches are excluded after whitelist matches. These name filters are independent from `sync_visibility`; both must match for a repository to be synced.
For every repository name found in any endpoint, `refray` will:
1. Create missing repositories on the other endpoints when `create_missing = true`.
@@ -194,6 +200,9 @@ token = { env = "GITEA_TOKEN" }
[[mirrors]]
name = "personal"
sync_visibility = "all"
repo_whitelist = ["^important-"]
repo_blacklist = ["-archive$"]
create_missing = true
visibility = "private"
allow_force = false
@@ -210,6 +219,38 @@ kind = "user"
namespace = "azalea"
```
## Testing
Run the normal, non-destructive test suite:
```sh
cargo test
```
The sequential live e2e test is ignored by default because it creates and deletes repositories on real provider accounts. Configure four token/username pairs in `.env` or the process environment:
```sh
GH_USER=...
GH_TOKEN=...
GL_USER=...
GL_TOKEN=...
GT_USER=...
GT_TOKEN=...
FO_USER=...
FO_TOKEN=...
```
Optional base URL overrides are `GL_BASE_URL`, `GT_BASE_URL` or `GITEA_BASE_URL`, and `FO_BASE_URL` or `FORGEJO_BASE_URL`. The Gitea and Forgejo defaults are `https://gitea.com` and `https://codeberg.org`.
Run the destructive e2e test explicitly:
```sh
REFRAY_E2E_ALLOW_DESTRUCTIVE=1 \
cargo test --test sequential -- --ignored --test-threads=1 --nocapture
```
By default cleanup only deletes repositories named `refray-e2e-*`. To start by deleting every owned repository visible to the configured accounts, set `REFRAY_E2E_CLEAR_ALL_REPOS=DELETE_ALL_OWNED_REPOS`. Provider skips (`REFRAY_E2E_SKIP_GITHUB`, `REFRAY_E2E_SKIP_GITLAB`, `REFRAY_E2E_SKIP_GITEA`, `REFRAY_E2E_SKIP_FORGEJO`) and `REFRAY_E2E_ALLOW_PARTIAL=1` are available for local debugging, but the full support check should run with all four providers.
## Issues and Pull Requests
Issues and pull requests are not mirrored.
+67
View File
@@ -5,6 +5,7 @@ use std::path::{Path, PathBuf};
use anyhow::{Context, Result, anyhow, bail};
use directories::ProjectDirs;
use regex::Regex;
use serde::{Deserialize, Serialize};
const APP_NAME: &str = "refray";
@@ -51,6 +52,12 @@ pub enum TokenConfig {
pub struct MirrorConfig {
pub name: String,
pub endpoints: Vec<EndpointConfig>,
#[serde(default)]
pub sync_visibility: SyncVisibility,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub repo_whitelist: Vec<String>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub repo_blacklist: Vec<String>,
#[serde(default = "default_true")]
pub create_missing: bool,
#[serde(default)]
@@ -106,6 +113,65 @@ pub enum Visibility {
Public,
}
#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum SyncVisibility {
#[default]
All,
Private,
Public,
}
#[derive(Clone, Debug)]
pub struct RepoNameFilter {
whitelist: Vec<Regex>,
blacklist: Vec<Regex>,
}
impl SyncVisibility {
pub fn matches_private(&self, private: bool) -> bool {
match self {
SyncVisibility::All => true,
SyncVisibility::Private => private,
SyncVisibility::Public => !private,
}
}
}
impl MirrorConfig {
pub fn repo_filter(&self) -> Result<RepoNameFilter> {
Ok(RepoNameFilter {
whitelist: compile_repo_patterns(&self.name, "repo_whitelist", &self.repo_whitelist)?,
blacklist: compile_repo_patterns(&self.name, "repo_blacklist", &self.repo_blacklist)?,
})
}
}
impl RepoNameFilter {
pub fn matches(&self, repo_name: &str) -> bool {
let whitelisted = self.whitelist.is_empty()
|| self
.whitelist
.iter()
.any(|pattern| pattern.is_match(repo_name));
let blacklisted = self
.blacklist
.iter()
.any(|pattern| pattern.is_match(repo_name));
whitelisted && !blacklisted
}
}
fn compile_repo_patterns(mirror: &str, field: &str, patterns: &[String]) -> Result<Vec<Regex>> {
patterns
.iter()
.map(|pattern| {
Regex::new(pattern)
.with_context(|| format!("mirror '{mirror}' has invalid {field} regex '{pattern}'"))
})
.collect()
}
fn default_true() -> bool {
true
}
@@ -262,6 +328,7 @@ pub fn validate_config(config: &Config) -> Result<()> {
bail!("no mirror groups configured");
}
for mirror in &config.mirrors {
mirror.repo_filter()?;
if mirror.endpoints.len() < 2 {
bail!(
"mirror '{}' must contain at least two endpoints",
+2 -1
View File
@@ -133,6 +133,7 @@ impl GitMirror {
self.run(["fetch", "--prune", &remote.name, &tag_refspec])
}
#[cfg(test)]
pub fn cached_remote_refs_match(
&self,
remote: &RemoteSpec,
@@ -589,7 +590,7 @@ impl GitMirror {
Ok(rebased.trim().to_string())
}
fn is_ancestor(&self, ancestor: &str, descendant: &str) -> Result<bool> {
pub fn is_ancestor(&self, ancestor: &str, descendant: &str) -> Result<bool> {
let status = self
.command()
.args(["merge-base", "--is-ancestor", ancestor, descendant])
+143 -2
View File
@@ -9,13 +9,14 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH};
use anyhow::{Context, Result, anyhow};
use console::{Term, style};
use dialoguer::{Confirm, Input, Password, Select, theme::ColorfulTheme};
use regex::Regex;
use reqwest::blocking::Client;
use tiny_http::{Request, Response, Server, StatusCode};
use url::Url;
use crate::config::{
Config, ConflictResolutionStrategy, EndpointConfig, MirrorConfig, NamespaceKind, ProviderKind,
SiteConfig, TokenConfig, Visibility, WebhookConfig,
SiteConfig, SyncVisibility, TokenConfig, Visibility, WebhookConfig,
};
use crate::provider::ProviderClient;
use crate::webhook::check_webhook_url_reachable;
@@ -37,6 +38,12 @@ struct ParsedProfileUrl {
namespace: String,
}
#[derive(Clone, Debug, Default)]
struct RepoFilterInput {
whitelist: Vec<String>,
blacklist: Vec<String>,
}
pub fn run_config_wizard(path: &Path) -> Result<ConfigWizardOutcome> {
let existing_config = path.exists();
let mut config = Config::load_or_default(path)?;
@@ -108,10 +115,15 @@ enum WizardAction {
fn add_sync_group_styled(config: &mut Config, theme: &ColorfulTheme) -> Result<()> {
let endpoints = prompt_sync_group_endpoints_styled(config, theme, &[])?;
let sync_visibility = prompt_sync_visibility_styled(theme, None)?;
let repo_filters = prompt_repo_filters_styled(theme, None)?;
let conflict_resolution = prompt_conflict_resolution_styled(theme, None)?;
config.upsert_mirror(MirrorConfig {
name: next_mirror_name(config),
endpoints,
sync_visibility,
repo_whitelist: repo_filters.whitelist,
repo_blacklist: repo_filters.blacklist,
create_missing: true,
visibility: Visibility::Private,
allow_force: false,
@@ -435,11 +447,23 @@ fn edit_sync_group_styled(config: &mut Config, theme: &ColorfulTheme) -> Result<
style(format!("sync group {}", index + 1)).cyan()
);
let existing = config.mirrors[index].endpoints.clone();
let existing_sync_visibility = config.mirrors[index].sync_visibility.clone();
let existing_repo_whitelist = config.mirrors[index].repo_whitelist.clone();
let existing_repo_blacklist = config.mirrors[index].repo_blacklist.clone();
let existing_conflict_resolution = config.mirrors[index].conflict_resolution.clone();
let endpoints = prompt_sync_group_endpoints_styled(config, theme, &existing)?;
let sync_visibility = prompt_sync_visibility_styled(theme, Some(&existing_sync_visibility))?;
let existing_repo_filters = RepoFilterInput {
whitelist: existing_repo_whitelist,
blacklist: existing_repo_blacklist,
};
let repo_filters = prompt_repo_filters_styled(theme, Some(&existing_repo_filters))?;
let conflict_resolution =
prompt_conflict_resolution_styled(theme, Some(&existing_conflict_resolution))?;
config.mirrors[index].endpoints = endpoints;
config.mirrors[index].sync_visibility = sync_visibility;
config.mirrors[index].repo_whitelist = repo_filters.whitelist;
config.mirrors[index].repo_blacklist = repo_filters.blacklist;
config.mirrors[index].conflict_resolution = conflict_resolution;
prompt_webhook_setup_styled(config, theme)?;
println!(
@@ -736,6 +760,102 @@ fn prompt_conflict_resolution_styled(
Ok(conflict_resolution_from_index(index))
}
fn prompt_sync_visibility_styled(
theme: &ColorfulTheme,
existing: Option<&SyncVisibility>,
) -> Result<SyncVisibility> {
let options = [
"All repositories",
"Only private repositories",
"Only public repositories",
];
let default = existing.map(sync_visibility_index).unwrap_or(0);
let index = Select::with_theme(theme)
.with_prompt("Which repositories should this sync group include?")
.items(options)
.default(default)
.interact()?;
Ok(sync_visibility_from_index(index))
}
fn prompt_repo_filters_styled(
theme: &ColorfulTheme,
existing: Option<&RepoFilterInput>,
) -> Result<RepoFilterInput> {
let existing = existing.cloned().unwrap_or_default();
let has_existing = !existing.whitelist.is_empty() || !existing.blacklist.is_empty();
if !Confirm::with_theme(theme)
.with_prompt("Configure repository name whitelist/blacklist?")
.default(has_existing)
.interact()?
{
return Ok(RepoFilterInput::default());
}
Ok(RepoFilterInput {
whitelist: prompt_repo_pattern_list_styled(
theme,
"Whitelist regexes (comma-separated, empty means all repo names)",
&existing.whitelist,
)?,
blacklist: prompt_repo_pattern_list_styled(
theme,
"Blacklist regexes (comma-separated)",
&existing.blacklist,
)?,
})
}
fn prompt_repo_pattern_list_styled(
theme: &ColorfulTheme,
prompt: &str,
existing: &[String],
) -> Result<Vec<String>> {
let input = Input::<String>::with_theme(theme)
.with_prompt(prompt)
.allow_empty(true)
.validate_with(|value: &String| validate_repo_pattern_list(value));
let input = if existing.is_empty() {
input
} else {
input.default(existing.join(", "))
};
let value = input.interact_text()?;
Ok(parse_repo_pattern_list(&value))
}
fn validate_repo_pattern_list(value: &str) -> std::result::Result<(), String> {
for pattern in parse_repo_pattern_list(value) {
Regex::new(&pattern).map_err(|error| format!("invalid regex '{pattern}': {error}"))?;
}
Ok(())
}
fn parse_repo_pattern_list(value: &str) -> Vec<String> {
value
.split(',')
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned)
.collect()
}
fn sync_visibility_index(sync_visibility: &SyncVisibility) -> usize {
match sync_visibility {
SyncVisibility::All => 0,
SyncVisibility::Private => 1,
SyncVisibility::Public => 2,
}
}
fn sync_visibility_from_index(index: usize) -> SyncVisibility {
match index {
1 => SyncVisibility::Private,
2 => SyncVisibility::Public,
_ => SyncVisibility::All,
}
}
fn conflict_resolution_index(strategy: &ConflictResolutionStrategy) -> usize {
match strategy {
ConflictResolutionStrategy::Fail => 0,
@@ -803,12 +923,33 @@ fn sync_group_summary(config: &Config, mirror: &MirrorConfig) -> String {
.collect::<Vec<_>>()
.join(" <-> ");
format!(
"{} ({})",
"{} ({}, {}, {})",
endpoints,
sync_visibility_label(&mirror.sync_visibility),
repo_filter_label(mirror),
conflict_resolution_label(&mirror.conflict_resolution)
)
}
fn sync_visibility_label(sync_visibility: &SyncVisibility) -> &'static str {
match sync_visibility {
SyncVisibility::All => "sync: all",
SyncVisibility::Private => "sync: private only",
SyncVisibility::Public => "sync: public only",
}
}
fn repo_filter_label(mirror: &MirrorConfig) -> String {
match (mirror.repo_whitelist.len(), mirror.repo_blacklist.len()) {
(0, 0) => "repos: all names".to_string(),
(whitelist, 0) => format!("repos: whitelist {whitelist}"),
(0, blacklist) => format!("repos: blacklist {blacklist}"),
(whitelist, blacklist) => {
format!("repos: whitelist {whitelist}, blacklist {blacklist}")
}
}
}
fn conflict_resolution_label(strategy: &ConflictResolutionStrategy) -> &'static str {
match strategy {
ConflictResolutionStrategy::Fail => "conflicts: fail",
+14 -2
View File
@@ -343,7 +343,11 @@ impl<'a> ProviderClient<'a> {
"{pulls_url}?state=open&base={}&per_page=100",
urlencoding(base_branch)
);
let pulls: Vec<ProviderPullRequest> = self.paged_get(&url)?;
let pulls: Vec<ProviderPullRequest> = match self.paged_get(&url) {
Ok(pulls) => pulls,
Err(error) if is_not_found_error(&error) => return Ok(0),
Err(error) => return Err(error),
};
let mut closed = 0;
for pull in pulls.into_iter().filter(|pull| {
pull.head_ref()
@@ -681,7 +685,11 @@ impl<'a> ProviderClient<'a> {
"{pulls_url}?state=open&base={}&limit=50",
urlencoding(base_branch)
);
let pulls: Vec<ProviderPullRequest> = self.paged_get(&url)?;
let pulls: Vec<ProviderPullRequest> = match self.paged_get(&url) {
Ok(pulls) => pulls,
Err(error) if is_not_found_error(&error) => return Ok(0),
Err(error) => return Err(error),
};
let mut closed = 0;
for pull in pulls.into_iter().filter(|pull| {
pull.head_ref()
@@ -931,6 +939,10 @@ fn check_response(method: &str, url: &str, response: Response) -> Result<Respons
bail!("{method} {url} returned {status}: {body}");
}
fn is_not_found_error(error: &anyhow::Error) -> bool {
error.to_string().contains("404 Not Found")
}
fn next_link(headers: &HeaderMap) -> Option<String> {
let header = headers.get("link")?.to_str().ok()?;
for part in header.split(',') {
+39 -43
View File
@@ -9,12 +9,12 @@ use console::style;
use regex::Regex;
use crate::config::{
Config, ConflictResolutionStrategy, EndpointConfig, MirrorConfig, default_work_dir,
validate_config,
Config, ConflictResolutionStrategy, EndpointConfig, MirrorConfig, RepoNameFilter,
SyncVisibility, default_work_dir, validate_config,
};
use crate::git::{
BranchConflict, BranchDeletion, BranchUpdate, GitMirror, Redactor, RemoteRefSnapshot,
RemoteSpec, is_disabled_repository_error, ls_remote_refs, safe_remote_name,
BranchConflict, BranchDeletion, BranchUpdate, GitMirror, Redactor, RemoteSpec,
is_disabled_repository_error, ls_remote_refs, safe_remote_name,
};
use crate::logging;
use crate::provider::{EndpointRepo, ProviderClient, PullRequestRequest, repos_by_name};
@@ -165,8 +165,9 @@ fn sync_group(
.create_missing_override
.unwrap_or(mirror.create_missing);
let allow_force = context.options.force_override.unwrap_or(mirror.allow_force);
let repo_filter = mirror.repo_filter()?;
let all_endpoint_repos = list_group_repos(context.config, mirror)?;
let all_endpoint_repos = list_group_repos(context.config, mirror, &repo_filter)?;
if !context.options.dry_run {
webhook::ensure_configured_webhooks(
context.config,
@@ -181,12 +182,7 @@ fn sync_group(
let retry_repo_names = context
.retry_failed_repos
.and_then(|repos| repos.get(&mirror.name));
let tracked_repo_names = context.ref_state.repo_names(&mirror.name);
let all_repo_names = repos
.keys()
.cloned()
.chain(tracked_repo_names)
.collect::<BTreeSet<_>>();
let all_repo_names = sync_candidate_repo_names(&repos, context.ref_state, mirror, &repo_filter);
let all_repo_count = all_repo_names.len();
let repo_names = all_repo_names
.into_iter()
@@ -349,7 +345,7 @@ fn sync_group(
});
if create_missing && !context.options.dry_run {
let repos = list_group_repos(context.config, mirror)?;
let repos = list_group_repos(context.config, mirror, &repo_filter)?;
webhook::ensure_configured_webhooks(
context.config,
mirror,
@@ -362,7 +358,11 @@ fn sync_group(
Ok(failures)
}
fn list_group_repos(config: &Config, mirror: &MirrorConfig) -> Result<Vec<EndpointRepo>> {
fn list_group_repos(
config: &Config,
mirror: &MirrorConfig,
repo_filter: &RepoNameFilter,
) -> Result<Vec<EndpointRepo>> {
let mut all_endpoint_repos = Vec::new();
for endpoint in &mirror.endpoints {
let site = config.site(&endpoint.site).unwrap();
@@ -375,7 +375,11 @@ fn list_group_repos(config: &Config, mirror: &MirrorConfig) -> Result<Vec<Endpoi
let repos = client
.list_repos(endpoint)
.with_context(|| format!("failed to list repos for {}", endpoint.label()))?;
for repo in repos {
for repo in repos
.into_iter()
.filter(|repo| mirror.sync_visibility.matches_private(repo.private))
.filter(|repo| repo_filter.matches(&repo.name))
{
all_endpoint_repos.push(EndpointRepo {
endpoint: endpoint.clone(),
repo,
@@ -385,6 +389,24 @@ fn list_group_repos(config: &Config, mirror: &MirrorConfig) -> Result<Vec<Endpoi
Ok(all_endpoint_repos)
}
fn sync_candidate_repo_names(
repos: &HashMap<String, Vec<EndpointRepo>>,
ref_state: &RefState,
mirror: &MirrorConfig,
repo_filter: &RepoNameFilter,
) -> BTreeSet<String> {
let mut names = repos.keys().cloned().collect::<BTreeSet<_>>();
if mirror.sync_visibility == SyncVisibility::All {
names.extend(
ref_state
.repo_names(&mirror.name)
.into_iter()
.filter(|name| repo_filter.matches(name)),
);
}
names
}
fn pop_repo_job(queue: &Arc<Mutex<VecDeque<RepoSyncJob>>>) -> Option<RepoSyncJob> {
queue
.lock()
@@ -561,19 +583,6 @@ fn sync_repo(
mirror_repo.configure_remotes(&initial_remotes)?;
let cached_ref_state = cached_ref_state(&mirror_repo, &initial_remotes)?;
if !context.dry_run
&& all_endpoints_present
&& cached_refs_match(&mirror_repo, &initial_remotes, &initial_ref_state)?
{
crate::logln!(
" {} refs unchanged from local mirror cache",
style("up-to-date").green().bold()
);
return Ok(RepoSyncOutcome {
state_update: Some(RepoStateUpdate::Set(initial_ref_state)),
});
}
for remote in &initial_remotes {
if let Err(error) = mirror_repo.fetch_remote(remote) {
if is_disabled_repository_error(&error) {
@@ -772,22 +781,6 @@ fn all_configured_endpoints_present(mirror: &MirrorConfig, repos: &[EndpointRepo
.all(|endpoint| present.contains(endpoint))
}
fn cached_refs_match(
mirror_repo: &GitMirror,
remotes: &[RemoteSpec],
expected_refs: &BTreeMap<String, RemoteRefState>,
) -> Result<bool> {
for remote in remotes {
let Some(expected) = expected_refs.get(&remote.name) else {
return Ok(false);
};
if !mirror_repo.cached_remote_refs_match(remote, &RemoteRefSnapshot::from(expected))? {
return Ok(false);
}
}
Ok(true)
}
fn cached_ref_state(
mirror_repo: &GitMirror,
remotes: &[RemoteSpec],
@@ -1144,6 +1137,9 @@ fn open_conflict_pull_requests(
for (source_remote, source_sha) in
conflict.tips.iter().filter(|(_, sha)| sha != target_sha)
{
if mirror_repo.is_ancestor(source_sha, target_sha)? {
continue;
}
let head_branch = conflict_pr_branch(&conflict.branch, source_remote, source_sha);
let update = BranchUpdate {
branch: head_branch.clone(),
+21 -13
View File
@@ -204,6 +204,7 @@ pub fn install_webhooks(config: &Config, options: WebhookInstallOptions) -> Resu
style("Webhook group").cyan().bold(),
style(&mirror.name).bold()
);
let repo_filter = mirror.repo_filter()?;
let mut tasks = Vec::new();
for endpoint in &mirror.endpoints {
let site = config.site(&endpoint.site).unwrap();
@@ -216,7 +217,11 @@ pub fn install_webhooks(config: &Config, options: WebhookInstallOptions) -> Resu
let repos = client
.list_repos(endpoint)
.with_context(|| format!("failed to list repos for {}", endpoint.label()))?;
for repo in repos {
for repo in repos
.into_iter()
.filter(|repo| mirror.sync_visibility.matches_private(repo.private))
.filter(|repo| repo_filter.matches(&repo.name))
{
if options.repo.as_ref().is_some_and(|name| name != &repo.name) {
continue;
}
@@ -1009,19 +1014,22 @@ fn matching_jobs(config: &Config, event: &WebhookEvent) -> Vec<WebhookJob> {
.mirrors
.iter()
.filter(|mirror| {
mirror.endpoints.iter().any(|endpoint| {
let Some(site) = config.site(&endpoint.site) else {
return false;
};
event
.provider
.as_ref()
.is_none_or(|provider| &site.provider == provider)
&& event
.namespace
mirror
.repo_filter()
.is_ok_and(|filter| filter.matches(&event.repo))
&& mirror.endpoints.iter().any(|endpoint| {
let Some(site) = config.site(&endpoint.site) else {
return false;
};
event
.provider
.as_ref()
.is_none_or(|namespace| namespace == &endpoint.namespace)
})
.is_none_or(|provider| &site.provider == provider)
&& event
.namespace
.as_ref()
.is_none_or(|namespace| namespace == &endpoint.namespace)
})
})
.map(|mirror| WebhookJob {
group: mirror.name.clone(),
File diff suppressed because it is too large Load Diff
+80
View File
@@ -19,6 +19,9 @@ fn parses_token_forms() {
[[mirrors]]
name = "personal"
sync_visibility = "public"
repo_whitelist = ["^important-", "-mirror$"]
repo_blacklist = ["-archive$"]
create_missing = true
visibility = "private"
allow_force = false
@@ -43,6 +46,15 @@ fn parses_token_forms() {
config.mirrors[0].conflict_resolution,
ConflictResolutionStrategy::AutoRebasePullRequest
);
assert_eq!(config.mirrors[0].sync_visibility, SyncVisibility::Public);
assert_eq!(
config.mirrors[0].repo_whitelist,
vec!["^important-".to_string(), "-mirror$".to_string()]
);
assert_eq!(
config.mirrors[0].repo_blacklist,
vec!["-archive$".to_string()]
);
let webhook = config.webhook.unwrap();
assert!(webhook.install);
assert_eq!(webhook.url, "https://mirror.example.test/webhook");
@@ -64,6 +76,9 @@ fn validation_rejects_unknown_sites_and_single_endpoint_groups() {
kind: NamespaceKind::User,
namespace: "alice".to_string(),
}],
sync_visibility: SyncVisibility::All,
repo_whitelist: Vec::new(),
repo_blacklist: Vec::new(),
create_missing: true,
visibility: Visibility::Private,
allow_force: false,
@@ -90,6 +105,9 @@ fn validation_rejects_unknown_sites_and_single_endpoint_groups() {
namespace: "alice".to_string(),
},
],
sync_visibility: SyncVisibility::All,
repo_whitelist: Vec::new(),
repo_blacklist: Vec::new(),
create_missing: true,
visibility: Visibility::Private,
allow_force: false,
@@ -141,6 +159,68 @@ fn api_base_defaults_match_providers() {
);
}
#[test]
fn sync_visibility_matches_repo_privacy() {
assert!(SyncVisibility::All.matches_private(true));
assert!(SyncVisibility::All.matches_private(false));
assert!(SyncVisibility::Private.matches_private(true));
assert!(!SyncVisibility::Private.matches_private(false));
assert!(!SyncVisibility::Public.matches_private(true));
assert!(SyncVisibility::Public.matches_private(false));
}
#[test]
fn repo_name_filter_applies_whitelist_then_blacklist() {
let mut mirror = mirror_config();
mirror.repo_whitelist = vec!["^important-".to_string(), "-mirror$".to_string()];
mirror.repo_blacklist = vec!["-archive$".to_string()];
let filter = mirror.repo_filter().unwrap();
assert!(filter.matches("important-api"));
assert!(filter.matches("user-mirror"));
assert!(!filter.matches("important-archive"));
assert!(!filter.matches("random"));
}
#[test]
fn validation_rejects_invalid_repo_filter_regex() {
let mut config = Config {
sites: vec![site("github", ProviderKind::Github)],
mirrors: vec![mirror_config()],
webhook: None,
};
config.mirrors[0].repo_whitelist = vec!["(".to_string()];
let err = validate_config(&config).unwrap_err().to_string();
assert!(err.contains("invalid repo_whitelist regex"));
}
fn mirror_config() -> MirrorConfig {
MirrorConfig {
name: "personal".to_string(),
endpoints: vec![
EndpointConfig {
site: "github".to_string(),
kind: NamespaceKind::User,
namespace: "alice".to_string(),
},
EndpointConfig {
site: "github".to_string(),
kind: NamespaceKind::Org,
namespace: "example".to_string(),
},
],
sync_visibility: SyncVisibility::All,
repo_whitelist: Vec::new(),
repo_blacklist: Vec::new(),
create_missing: true,
visibility: Visibility::Private,
allow_force: false,
conflict_resolution: ConflictResolutionStrategy::Fail,
}
}
fn site(name: &str, provider: ProviderKind) -> SiteConfig {
SiteConfig {
name: name.to_string(),
+35 -1
View File
@@ -12,6 +12,8 @@ fn wizard_builds_sync_group_from_profile_urls() {
"",
"n",
"",
"",
"",
"n",
"4",
]
@@ -42,6 +44,7 @@ fn wizard_builds_sync_group_from_profile_urls() {
assert_eq!(config.mirrors[0].endpoints[0].namespace, "hykilpikonna");
assert_eq!(config.mirrors[0].endpoints[1].site, "gitea-example-test");
assert_eq!(config.mirrors[0].endpoints[1].namespace, "azalea");
assert_eq!(config.mirrors[0].sync_visibility, SyncVisibility::All);
assert!(config.mirrors[0].create_missing);
assert_eq!(config.mirrors[0].visibility, Visibility::Private);
assert!(!config.mirrors[0].allow_force);
@@ -73,6 +76,8 @@ fn wizard_can_build_three_way_sync() {
"",
"n",
"",
"",
"",
"n",
"4",
]
@@ -99,6 +104,8 @@ fn wizard_can_enable_webhooks() {
"",
"n",
"",
"",
"",
"y",
"https://mirror.example.test/webhook",
"y",
@@ -150,6 +157,8 @@ fn wizard_reuses_existing_credentials_for_same_instance() {
"",
"n",
"",
"",
"",
"n",
"4",
]
@@ -200,6 +209,9 @@ fn wizard_starts_existing_config_at_sync_group_menu() {
namespace: "alice".to_string(),
},
],
sync_visibility: SyncVisibility::All,
repo_whitelist: Vec::new(),
repo_blacklist: Vec::new(),
create_missing: true,
visibility: Visibility::Private,
allow_force: false,
@@ -226,6 +238,9 @@ fn wizard_can_ask_to_run_full_sync_after_config() {
mirrors: vec![MirrorConfig {
name: "sync-1".to_string(),
endpoints: Vec::new(),
sync_visibility: SyncVisibility::All,
repo_whitelist: Vec::new(),
repo_blacklist: Vec::new(),
create_missing: true,
visibility: Visibility::Private,
allow_force: false,
@@ -299,6 +314,9 @@ fn wizard_edits_existing_sync_group_from_menu() {
namespace: "alice".to_string(),
},
],
sync_visibility: SyncVisibility::Private,
repo_whitelist: vec!["^important-".to_string()],
repo_blacklist: vec!["-archive$".to_string()],
create_missing: false,
visibility: Visibility::Public,
allow_force: true,
@@ -314,6 +332,10 @@ fn wizard_edits_existing_sync_group_from_menu() {
"https://gitlab.com/bob",
"",
"n",
"public",
"y",
"^public-",
"-skip$",
"",
"n",
"4",
@@ -334,6 +356,9 @@ fn wizard_edits_existing_sync_group_from_menu() {
assert_eq!(mirror.endpoints[1].site, "gitlab");
assert_eq!(mirror.endpoints[1].namespace, "bob");
assert!(!mirror.create_missing);
assert_eq!(mirror.sync_visibility, SyncVisibility::Public);
assert_eq!(mirror.repo_whitelist, vec!["^public-".to_string()]);
assert_eq!(mirror.repo_blacklist, vec!["-skip$".to_string()]);
assert_eq!(mirror.visibility, Visibility::Public);
assert!(mirror.allow_force);
@@ -378,6 +403,9 @@ fn wizard_prefills_existing_sync_group_when_editing() {
namespace: "alice".to_string(),
},
],
sync_visibility: SyncVisibility::All,
repo_whitelist: Vec::new(),
repo_blacklist: Vec::new(),
create_missing: true,
visibility: Visibility::Private,
allow_force: false,
@@ -385,7 +413,7 @@ fn wizard_prefills_existing_sync_group_when_editing() {
}],
webhook: None,
};
let input = ["2", "1", "", "", "", "", "n", "", "n", "4"].join("\n") + "\n";
let input = ["2", "1", "", "", "", "", "n", "", "", "", "n", "4"].join("\n") + "\n";
let mut reader = Cursor::new(input.as_bytes());
let mut output = Vec::new();
@@ -439,6 +467,9 @@ fn wizard_deletes_existing_sync_group_from_menu() {
namespace: "alice".to_string(),
},
],
sync_visibility: SyncVisibility::All,
repo_whitelist: Vec::new(),
repo_blacklist: Vec::new(),
create_missing: true,
visibility: Visibility::Private,
allow_force: false,
@@ -495,6 +526,9 @@ fn wizard_can_go_back_from_delete_menu() {
namespace: "alice".to_string(),
},
],
sync_visibility: SyncVisibility::All,
repo_whitelist: Vec::new(),
repo_blacklist: Vec::new(),
create_missing: true,
visibility: Visibility::Private,
allow_force: false,
+121
View File
@@ -63,10 +63,15 @@ where
W: Write,
{
let endpoints = prompt_sync_group_endpoints(reader, writer, config, &[])?;
let sync_visibility = prompt_sync_visibility(reader, writer, None)?;
let repo_filters = prompt_repo_filters(reader, writer, None)?;
let conflict_resolution = prompt_conflict_resolution(reader, writer, None)?;
config.upsert_mirror(MirrorConfig {
name: next_mirror_name(config),
endpoints,
sync_visibility,
repo_whitelist: repo_filters.whitelist,
repo_blacklist: repo_filters.blacklist,
create_missing: true,
visibility: Visibility::Private,
allow_force: false,
@@ -267,15 +272,27 @@ where
match value.parse::<usize>() {
Ok(index) if (1..=config.mirrors.len()).contains(&index) => {
let existing = config.mirrors[index - 1].endpoints.clone();
let existing_sync_visibility = config.mirrors[index - 1].sync_visibility.clone();
let existing_repo_filters = RepoFilterInput {
whitelist: config.mirrors[index - 1].repo_whitelist.clone(),
blacklist: config.mirrors[index - 1].repo_blacklist.clone(),
};
let existing_conflict_resolution =
config.mirrors[index - 1].conflict_resolution.clone();
let endpoints = prompt_sync_group_endpoints(reader, writer, config, &existing)?;
let sync_visibility =
prompt_sync_visibility(reader, writer, Some(&existing_sync_visibility))?;
let repo_filters =
prompt_repo_filters(reader, writer, Some(&existing_repo_filters))?;
let conflict_resolution = prompt_conflict_resolution(
reader,
writer,
Some(&existing_conflict_resolution),
)?;
config.mirrors[index - 1].endpoints = endpoints;
config.mirrors[index - 1].sync_visibility = sync_visibility;
config.mirrors[index - 1].repo_whitelist = repo_filters.whitelist;
config.mirrors[index - 1].repo_blacklist = repo_filters.blacklist;
config.mirrors[index - 1].conflict_resolution = conflict_resolution;
prompt_webhook_setup(reader, writer, config)?;
writeln!(writer, "updated sync group {index}")?;
@@ -476,6 +493,100 @@ where
}
}
fn prompt_sync_visibility<R, W>(
reader: &mut R,
writer: &mut W,
existing: Option<&SyncVisibility>,
) -> Result<SyncVisibility>
where
R: BufRead,
W: Write,
{
let default = existing.map(sync_visibility_value).unwrap_or("all");
loop {
writeln!(writer, "Which repositories should this sync group include?")?;
writeln!(writer, " 1. all")?;
writeln!(writer, " 2. private only")?;
writeln!(writer, " 3. public only")?;
let value = prompt_with_default(reader, writer, "Sync visibility", default)?;
match value.trim().to_ascii_lowercase().as_str() {
"1" | "all" => return Ok(SyncVisibility::All),
"2" | "private" | "private only" | "private-only" => {
return Ok(SyncVisibility::Private);
}
"3" | "public" | "public only" | "public-only" => {
return Ok(SyncVisibility::Public);
}
_ => writeln!(writer, "Enter 1, 2, 3, all, private, or public.")?,
}
}
}
fn prompt_repo_filters<R, W>(
reader: &mut R,
writer: &mut W,
existing: Option<&RepoFilterInput>,
) -> Result<RepoFilterInput>
where
R: BufRead,
W: Write,
{
let existing = existing.cloned().unwrap_or_default();
let has_existing = !existing.whitelist.is_empty() || !existing.blacklist.is_empty();
if !prompt_bool(
reader,
writer,
"Configure repository name whitelist/blacklist?",
has_existing,
)? {
return Ok(RepoFilterInput::default());
}
Ok(RepoFilterInput {
whitelist: prompt_repo_pattern_list(
reader,
writer,
"Whitelist regexes (comma-separated, empty means all repo names)",
&existing.whitelist,
)?,
blacklist: prompt_repo_pattern_list(
reader,
writer,
"Blacklist regexes (comma-separated)",
&existing.blacklist,
)?,
})
}
fn prompt_repo_pattern_list<R, W>(
reader: &mut R,
writer: &mut W,
label: &str,
existing: &[String],
) -> Result<Vec<String>>
where
R: BufRead,
W: Write,
{
let value = if existing.is_empty() {
prompt_optional(reader, writer, label)?
} else {
prompt_with_default(reader, writer, label, &existing.join(", "))?
};
if let Err(error) = validate_repo_pattern_list(&value) {
bail!(error);
}
Ok(parse_repo_pattern_list(&value))
}
fn sync_visibility_value(sync_visibility: &SyncVisibility) -> &'static str {
match sync_visibility {
SyncVisibility::All => "all",
SyncVisibility::Private => "private",
SyncVisibility::Public => "public",
}
}
fn conflict_resolution_value(strategy: &ConflictResolutionStrategy) -> &'static str {
match strategy {
ConflictResolutionStrategy::Fail => "fail",
@@ -540,6 +651,16 @@ where
}
}
fn prompt_optional<R, W>(reader: &mut R, writer: &mut W, label: &str) -> Result<String>
where
R: BufRead,
W: Write,
{
write!(writer, "{label}: ")?;
writer.flush()?;
Ok(read_line(reader)?.trim().to_string())
}
fn prompt_with_default<R, W>(
reader: &mut R,
writer: &mut W,
+53
View File
@@ -308,6 +308,56 @@ fn repo_deletion_decision_ignores_repos_not_previously_synced_everywhere() {
assert_eq!(decision, RepoDeletionDecision::None);
}
#[test]
fn filtered_sync_visibility_does_not_treat_state_only_repos_as_deleted() {
let mut mirror = test_mirror();
mirror.sync_visibility = crate::config::SyncVisibility::Public;
let mut ref_state = RefState::default();
ref_state.set_repo(
&mirror.name,
"private-repo",
BTreeMap::from([("github_alice".to_string(), remote_ref_state("a", &[]))]),
);
let repo_filter = mirror.repo_filter().unwrap();
let names = sync_candidate_repo_names(&HashMap::new(), &ref_state, &mirror, &repo_filter);
assert!(names.is_empty());
}
#[test]
fn all_visibility_keeps_state_only_repos_for_deletion_detection() {
let mirror = test_mirror();
let mut ref_state = RefState::default();
ref_state.set_repo(
&mirror.name,
"deleted-repo",
BTreeMap::from([("github_alice".to_string(), remote_ref_state("a", &[]))]),
);
let repo_filter = mirror.repo_filter().unwrap();
let names = sync_candidate_repo_names(&HashMap::new(), &ref_state, &mirror, &repo_filter);
assert_eq!(names, BTreeSet::from(["deleted-repo".to_string()]));
}
#[test]
fn repo_name_filters_do_not_treat_state_only_repos_as_deleted() {
let mut mirror = test_mirror();
mirror.repo_whitelist = vec!["^public-".to_string()];
let repo_filter = mirror.repo_filter().unwrap();
let mut ref_state = RefState::default();
ref_state.set_repo(
&mirror.name,
"private-repo",
BTreeMap::from([("github_alice".to_string(), remote_ref_state("a", &[]))]),
);
let names = sync_candidate_repo_names(&HashMap::new(), &ref_state, &mirror, &repo_filter);
assert!(names.is_empty());
}
#[test]
fn conflict_branch_prefixes_are_reversible_not_slug_collisions() {
let slash_branch = conflict_pr_branch_prefix("release/foo");
@@ -358,6 +408,9 @@ fn test_mirror() -> MirrorConfig {
MirrorConfig {
name: "sync-1".to_string(),
endpoints: vec![endpoint("github"), endpoint("gitea")],
sync_visibility: crate::config::SyncVisibility::All,
repo_whitelist: Vec::new(),
repo_blacklist: Vec::new(),
create_missing: true,
visibility: crate::config::Visibility::Private,
allow_force: false,
+47
View File
@@ -1,4 +1,5 @@
use super::*;
use crate::config::SyncVisibility;
use crate::config::{
ConflictResolutionStrategy, EndpointConfig, MirrorConfig, NamespaceKind, SiteConfig,
TokenConfig, Visibility,
@@ -109,6 +110,9 @@ fn matches_jobs_by_provider_and_namespace() {
endpoint("github", NamespaceKind::User, "alice"),
endpoint("gitea", NamespaceKind::User, "azalea"),
],
sync_visibility: SyncVisibility::All,
repo_whitelist: Vec::new(),
repo_blacklist: Vec::new(),
create_missing: true,
visibility: Visibility::Private,
allow_force: false,
@@ -129,6 +133,41 @@ fn matches_jobs_by_provider_and_namespace() {
assert_eq!(jobs[0].repo, "repo");
}
#[test]
fn matching_jobs_respects_repo_name_filters() {
let mut mirror = MirrorConfig {
name: "sync-1".to_string(),
endpoints: vec![endpoint("github", NamespaceKind::User, "alice")],
sync_visibility: SyncVisibility::All,
repo_whitelist: vec!["^important-".to_string()],
repo_blacklist: vec!["-archive$".to_string()],
create_missing: true,
visibility: Visibility::Private,
allow_force: false,
conflict_resolution: ConflictResolutionStrategy::Fail,
};
let config = Config {
sites: vec![site("github", ProviderKind::Github)],
mirrors: vec![mirror.clone()],
webhook: None,
};
assert_eq!(
matching_jobs(&config, &webhook_event("important-api")).len(),
1
);
assert!(matching_jobs(&config, &webhook_event("important-archive")).is_empty());
assert!(matching_jobs(&config, &webhook_event("random")).is_empty());
mirror.repo_whitelist.clear();
let config = Config {
sites: vec![site("github", ProviderKind::Github)],
mirrors: vec![mirror],
webhook: None,
};
assert_eq!(matching_jobs(&config, &webhook_event("random")).len(), 1);
}
#[test]
fn webhook_state_persists_installations() {
let temp = tempfile::TempDir::new().unwrap();
@@ -330,6 +369,14 @@ fn site(name: &str, provider: ProviderKind) -> SiteConfig {
}
}
fn webhook_event(repo: &str) -> WebhookEvent {
WebhookEvent {
provider: Some(ProviderKind::Github),
repo: repo.to_string(),
namespace: Some("alice".to_string()),
}
}
fn endpoint(site: &str, kind: NamespaceKind, namespace: &str) -> EndpointConfig {
EndpointConfig {
site: site.to_string(),