[+] End-to-end testing

src/config.rs

@@ -5,6 +5,7 @@ use std::path::{Path, PathBuf};
 
 use anyhow::{Context, Result, anyhow, bail};
 use directories::ProjectDirs;
+use regex::Regex;
 use serde::{Deserialize, Serialize};
 
 const APP_NAME: &str = "refray";
@@ -51,6 +52,12 @@ pub enum TokenConfig {
 pub struct MirrorConfig {
     pub name: String,
     pub endpoints: Vec<EndpointConfig>,
+    #[serde(default)]
+    pub sync_visibility: SyncVisibility,
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub repo_whitelist: Vec<String>,
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub repo_blacklist: Vec<String>,
     #[serde(default = "default_true")]
     pub create_missing: bool,
     #[serde(default)]
@@ -106,6 +113,65 @@ pub enum Visibility {
     Public,
 }
 
+#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub enum SyncVisibility {
+    #[default]
+    All,
+    Private,
+    Public,
+}
+
+#[derive(Clone, Debug)]
+pub struct RepoNameFilter {
+    whitelist: Vec<Regex>,
+    blacklist: Vec<Regex>,
+}
+
+impl SyncVisibility {
+    pub fn matches_private(&self, private: bool) -> bool {
+        match self {
+            SyncVisibility::All => true,
+            SyncVisibility::Private => private,
+            SyncVisibility::Public => !private,
+        }
+    }
+}
+
+impl MirrorConfig {
+    pub fn repo_filter(&self) -> Result<RepoNameFilter> {
+        Ok(RepoNameFilter {
+            whitelist: compile_repo_patterns(&self.name, "repo_whitelist", &self.repo_whitelist)?,
+            blacklist: compile_repo_patterns(&self.name, "repo_blacklist", &self.repo_blacklist)?,
+        })
+    }
+}
+
+impl RepoNameFilter {
+    pub fn matches(&self, repo_name: &str) -> bool {
+        let whitelisted = self.whitelist.is_empty()
+            || self
+                .whitelist
+                .iter()
+                .any(|pattern| pattern.is_match(repo_name));
+        let blacklisted = self
+            .blacklist
+            .iter()
+            .any(|pattern| pattern.is_match(repo_name));
+        whitelisted && !blacklisted
+    }
+}
+
+fn compile_repo_patterns(mirror: &str, field: &str, patterns: &[String]) -> Result<Vec<Regex>> {
+    patterns
+        .iter()
+        .map(|pattern| {
+            Regex::new(pattern)
+                .with_context(|| format!("mirror '{mirror}' has invalid {field} regex '{pattern}'"))
+        })
+        .collect()
+}
+
 fn default_true() -> bool {
     true
 }
@@ -262,6 +328,7 @@ pub fn validate_config(config: &Config) -> Result<()> {
         bail!("no mirror groups configured");
     }
     for mirror in &config.mirrors {
+        mirror.repo_filter()?;
         if mirror.endpoints.len() < 2 {
             bail!(
                 "mirror '{}' must contain at least two endpoints",

src/git.rs

@@ -133,6 +133,7 @@ impl GitMirror {
         self.run(["fetch", "--prune", &remote.name, &tag_refspec])
     }
 
+    #[cfg(test)]
     pub fn cached_remote_refs_match(
         &self,
         remote: &RemoteSpec,
@@ -589,7 +590,7 @@ impl GitMirror {
         Ok(rebased.trim().to_string())
     }
 
-    fn is_ancestor(&self, ancestor: &str, descendant: &str) -> Result<bool> {
+    pub fn is_ancestor(&self, ancestor: &str, descendant: &str) -> Result<bool> {
         let status = self
             .command()
             .args(["merge-base", "--is-ancestor", ancestor, descendant])

src/interactive.rs

@@ -9,13 +9,14 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH};
 use anyhow::{Context, Result, anyhow};
 use console::{Term, style};
 use dialoguer::{Confirm, Input, Password, Select, theme::ColorfulTheme};
+use regex::Regex;
 use reqwest::blocking::Client;
 use tiny_http::{Request, Response, Server, StatusCode};
 use url::Url;
 
 use crate::config::{
     Config, ConflictResolutionStrategy, EndpointConfig, MirrorConfig, NamespaceKind, ProviderKind,
-    SiteConfig, TokenConfig, Visibility, WebhookConfig,
+    SiteConfig, SyncVisibility, TokenConfig, Visibility, WebhookConfig,
 };
 use crate::provider::ProviderClient;
 use crate::webhook::check_webhook_url_reachable;
@@ -37,6 +38,12 @@ struct ParsedProfileUrl {
     namespace: String,
 }
 
+#[derive(Clone, Debug, Default)]
+struct RepoFilterInput {
+    whitelist: Vec<String>,
+    blacklist: Vec<String>,
+}
+
 pub fn run_config_wizard(path: &Path) -> Result<ConfigWizardOutcome> {
     let existing_config = path.exists();
     let mut config = Config::load_or_default(path)?;
@@ -108,10 +115,15 @@ enum WizardAction {
 
 fn add_sync_group_styled(config: &mut Config, theme: &ColorfulTheme) -> Result<()> {
     let endpoints = prompt_sync_group_endpoints_styled(config, theme, &[])?;
+    let sync_visibility = prompt_sync_visibility_styled(theme, None)?;
+    let repo_filters = prompt_repo_filters_styled(theme, None)?;
     let conflict_resolution = prompt_conflict_resolution_styled(theme, None)?;
     config.upsert_mirror(MirrorConfig {
         name: next_mirror_name(config),
         endpoints,
+        sync_visibility,
+        repo_whitelist: repo_filters.whitelist,
+        repo_blacklist: repo_filters.blacklist,
         create_missing: true,
         visibility: Visibility::Private,
         allow_force: false,
@@ -435,11 +447,23 @@ fn edit_sync_group_styled(config: &mut Config, theme: &ColorfulTheme) -> Result<
         style(format!("sync group {}", index + 1)).cyan()
     );
     let existing = config.mirrors[index].endpoints.clone();
+    let existing_sync_visibility = config.mirrors[index].sync_visibility.clone();
+    let existing_repo_whitelist = config.mirrors[index].repo_whitelist.clone();
+    let existing_repo_blacklist = config.mirrors[index].repo_blacklist.clone();
     let existing_conflict_resolution = config.mirrors[index].conflict_resolution.clone();
     let endpoints = prompt_sync_group_endpoints_styled(config, theme, &existing)?;
+    let sync_visibility = prompt_sync_visibility_styled(theme, Some(&existing_sync_visibility))?;
+    let existing_repo_filters = RepoFilterInput {
+        whitelist: existing_repo_whitelist,
+        blacklist: existing_repo_blacklist,
+    };
+    let repo_filters = prompt_repo_filters_styled(theme, Some(&existing_repo_filters))?;
     let conflict_resolution =
         prompt_conflict_resolution_styled(theme, Some(&existing_conflict_resolution))?;
     config.mirrors[index].endpoints = endpoints;
+    config.mirrors[index].sync_visibility = sync_visibility;
+    config.mirrors[index].repo_whitelist = repo_filters.whitelist;
+    config.mirrors[index].repo_blacklist = repo_filters.blacklist;
     config.mirrors[index].conflict_resolution = conflict_resolution;
     prompt_webhook_setup_styled(config, theme)?;
     println!(
@@ -736,6 +760,102 @@ fn prompt_conflict_resolution_styled(
     Ok(conflict_resolution_from_index(index))
 }
 
+fn prompt_sync_visibility_styled(
+    theme: &ColorfulTheme,
+    existing: Option<&SyncVisibility>,
+) -> Result<SyncVisibility> {
+    let options = [
+        "All repositories",
+        "Only private repositories",
+        "Only public repositories",
+    ];
+    let default = existing.map(sync_visibility_index).unwrap_or(0);
+    let index = Select::with_theme(theme)
+        .with_prompt("Which repositories should this sync group include?")
+        .items(options)
+        .default(default)
+        .interact()?;
+    Ok(sync_visibility_from_index(index))
+}
+
+fn prompt_repo_filters_styled(
+    theme: &ColorfulTheme,
+    existing: Option<&RepoFilterInput>,
+) -> Result<RepoFilterInput> {
+    let existing = existing.cloned().unwrap_or_default();
+    let has_existing = !existing.whitelist.is_empty() || !existing.blacklist.is_empty();
+    if !Confirm::with_theme(theme)
+        .with_prompt("Configure repository name whitelist/blacklist?")
+        .default(has_existing)
+        .interact()?
+    {
+        return Ok(RepoFilterInput::default());
+    }
+
+    Ok(RepoFilterInput {
+        whitelist: prompt_repo_pattern_list_styled(
+            theme,
+            "Whitelist regexes (comma-separated, empty means all repo names)",
+            &existing.whitelist,
+        )?,
+        blacklist: prompt_repo_pattern_list_styled(
+            theme,
+            "Blacklist regexes (comma-separated)",
+            &existing.blacklist,
+        )?,
+    })
+}
+
+fn prompt_repo_pattern_list_styled(
+    theme: &ColorfulTheme,
+    prompt: &str,
+    existing: &[String],
+) -> Result<Vec<String>> {
+    let input = Input::<String>::with_theme(theme)
+        .with_prompt(prompt)
+        .allow_empty(true)
+        .validate_with(|value: &String| validate_repo_pattern_list(value));
+    let input = if existing.is_empty() {
+        input
+    } else {
+        input.default(existing.join(", "))
+    };
+    let value = input.interact_text()?;
+    Ok(parse_repo_pattern_list(&value))
+}
+
+fn validate_repo_pattern_list(value: &str) -> std::result::Result<(), String> {
+    for pattern in parse_repo_pattern_list(value) {
+        Regex::new(&pattern).map_err(|error| format!("invalid regex '{pattern}': {error}"))?;
+    }
+    Ok(())
+}
+
+fn parse_repo_pattern_list(value: &str) -> Vec<String> {
+    value
+        .split(',')
+        .map(str::trim)
+        .filter(|value| !value.is_empty())
+        .map(ToOwned::to_owned)
+        .collect()
+}
+
+fn sync_visibility_index(sync_visibility: &SyncVisibility) -> usize {
+    match sync_visibility {
+        SyncVisibility::All => 0,
+        SyncVisibility::Private => 1,
+        SyncVisibility::Public => 2,
+    }
+}
+
+fn sync_visibility_from_index(index: usize) -> SyncVisibility {
+    match index {
+        1 => SyncVisibility::Private,
+        2 => SyncVisibility::Public,
+        _ => SyncVisibility::All,
+    }
+}
+
 fn conflict_resolution_index(strategy: &ConflictResolutionStrategy) -> usize {
     match strategy {
         ConflictResolutionStrategy::Fail => 0,
@@ -803,12 +923,33 @@ fn sync_group_summary(config: &Config, mirror: &MirrorConfig) -> String {
         .collect::<Vec<_>>()
         .join(" <-> ");
     format!(
-        "{} ({})",
+        "{} ({}, {}, {})",
         endpoints,
+        sync_visibility_label(&mirror.sync_visibility),
+        repo_filter_label(mirror),
         conflict_resolution_label(&mirror.conflict_resolution)
     )
 }
 
+fn sync_visibility_label(sync_visibility: &SyncVisibility) -> &'static str {
+    match sync_visibility {
+        SyncVisibility::All => "sync: all",
+        SyncVisibility::Private => "sync: private only",
+        SyncVisibility::Public => "sync: public only",
+    }
+}
+
+fn repo_filter_label(mirror: &MirrorConfig) -> String {
+    match (mirror.repo_whitelist.len(), mirror.repo_blacklist.len()) {
+        (0, 0) => "repos: all names".to_string(),
+        (whitelist, 0) => format!("repos: whitelist {whitelist}"),
+        (0, blacklist) => format!("repos: blacklist {blacklist}"),
+        (whitelist, blacklist) => {
+            format!("repos: whitelist {whitelist}, blacklist {blacklist}")
+        }
+    }
+}
+
 fn conflict_resolution_label(strategy: &ConflictResolutionStrategy) -> &'static str {
     match strategy {
         ConflictResolutionStrategy::Fail => "conflicts: fail",

src/provider.rs

@@ -343,7 +343,11 @@ impl<'a> ProviderClient<'a> {
             "{pulls_url}?state=open&base={}&per_page=100",
             urlencoding(base_branch)
         );
-        let pulls: Vec<ProviderPullRequest> = self.paged_get(&url)?;
+        let pulls: Vec<ProviderPullRequest> = match self.paged_get(&url) {
+            Ok(pulls) => pulls,
+            Err(error) if is_not_found_error(&error) => return Ok(0),
+            Err(error) => return Err(error),
+        };
         let mut closed = 0;
         for pull in pulls.into_iter().filter(|pull| {
             pull.head_ref()
@@ -681,7 +685,11 @@ impl<'a> ProviderClient<'a> {
             "{pulls_url}?state=open&base={}&limit=50",
             urlencoding(base_branch)
         );
-        let pulls: Vec<ProviderPullRequest> = self.paged_get(&url)?;
+        let pulls: Vec<ProviderPullRequest> = match self.paged_get(&url) {
+            Ok(pulls) => pulls,
+            Err(error) if is_not_found_error(&error) => return Ok(0),
+            Err(error) => return Err(error),
+        };
         let mut closed = 0;
         for pull in pulls.into_iter().filter(|pull| {
             pull.head_ref()
@@ -931,6 +939,10 @@ fn check_response(method: &str, url: &str, response: Response) -> Result<Respons
     bail!("{method} {url} returned {status}: {body}");
 }
 
+fn is_not_found_error(error: &anyhow::Error) -> bool {
+    error.to_string().contains("404 Not Found")
+}
+
 fn next_link(headers: &HeaderMap) -> Option<String> {
     let header = headers.get("link")?.to_str().ok()?;
     for part in header.split(',') {

src/sync.rs

@@ -9,12 +9,12 @@ use console::style;
 use regex::Regex;
 
 use crate::config::{
-    Config, ConflictResolutionStrategy, EndpointConfig, MirrorConfig, default_work_dir,
-    validate_config,
+    Config, ConflictResolutionStrategy, EndpointConfig, MirrorConfig, RepoNameFilter,
+    SyncVisibility, default_work_dir, validate_config,
 };
 use crate::git::{
-    BranchConflict, BranchDeletion, BranchUpdate, GitMirror, Redactor, RemoteRefSnapshot,
-    RemoteSpec, is_disabled_repository_error, ls_remote_refs, safe_remote_name,
+    BranchConflict, BranchDeletion, BranchUpdate, GitMirror, Redactor, RemoteSpec,
+    is_disabled_repository_error, ls_remote_refs, safe_remote_name,
 };
 use crate::logging;
 use crate::provider::{EndpointRepo, ProviderClient, PullRequestRequest, repos_by_name};
@@ -165,8 +165,9 @@ fn sync_group(
         .create_missing_override
         .unwrap_or(mirror.create_missing);
     let allow_force = context.options.force_override.unwrap_or(mirror.allow_force);
+    let repo_filter = mirror.repo_filter()?;
 
-    let all_endpoint_repos = list_group_repos(context.config, mirror)?;
+    let all_endpoint_repos = list_group_repos(context.config, mirror, &repo_filter)?;
     if !context.options.dry_run {
         webhook::ensure_configured_webhooks(
             context.config,
@@ -181,12 +182,7 @@ fn sync_group(
     let retry_repo_names = context
         .retry_failed_repos
         .and_then(|repos| repos.get(&mirror.name));
-    let tracked_repo_names = context.ref_state.repo_names(&mirror.name);
-    let all_repo_names = repos
-        .keys()
-        .cloned()
-        .chain(tracked_repo_names)
-        .collect::<BTreeSet<_>>();
+    let all_repo_names = sync_candidate_repo_names(&repos, context.ref_state, mirror, &repo_filter);
     let all_repo_count = all_repo_names.len();
     let repo_names = all_repo_names
         .into_iter()
@@ -349,7 +345,7 @@ fn sync_group(
     });
 
     if create_missing && !context.options.dry_run {
-        let repos = list_group_repos(context.config, mirror)?;
+        let repos = list_group_repos(context.config, mirror, &repo_filter)?;
         webhook::ensure_configured_webhooks(
             context.config,
             mirror,
@@ -362,7 +358,11 @@ fn sync_group(
     Ok(failures)
 }
 
-fn list_group_repos(config: &Config, mirror: &MirrorConfig) -> Result<Vec<EndpointRepo>> {
+fn list_group_repos(
+    config: &Config,
+    mirror: &MirrorConfig,
+    repo_filter: &RepoNameFilter,
+) -> Result<Vec<EndpointRepo>> {
     let mut all_endpoint_repos = Vec::new();
     for endpoint in &mirror.endpoints {
         let site = config.site(&endpoint.site).unwrap();
@@ -375,7 +375,11 @@ fn list_group_repos(config: &Config, mirror: &MirrorConfig) -> Result<Vec<Endpoi
         let repos = client
             .list_repos(endpoint)
             .with_context(|| format!("failed to list repos for {}", endpoint.label()))?;
-        for repo in repos {
+        for repo in repos
+            .into_iter()
+            .filter(|repo| mirror.sync_visibility.matches_private(repo.private))
+            .filter(|repo| repo_filter.matches(&repo.name))
+        {
             all_endpoint_repos.push(EndpointRepo {
                 endpoint: endpoint.clone(),
                 repo,
@@ -385,6 +389,24 @@ fn list_group_repos(config: &Config, mirror: &MirrorConfig) -> Result<Vec<Endpoi
     Ok(all_endpoint_repos)
 }
 
+fn sync_candidate_repo_names(
+    repos: &HashMap<String, Vec<EndpointRepo>>,
+    ref_state: &RefState,
+    mirror: &MirrorConfig,
+    repo_filter: &RepoNameFilter,
+) -> BTreeSet<String> {
+    let mut names = repos.keys().cloned().collect::<BTreeSet<_>>();
+    if mirror.sync_visibility == SyncVisibility::All {
+        names.extend(
+            ref_state
+                .repo_names(&mirror.name)
+                .into_iter()
+                .filter(|name| repo_filter.matches(name)),
+        );
+    }
+    names
+}
+
 fn pop_repo_job(queue: &Arc<Mutex<VecDeque<RepoSyncJob>>>) -> Option<RepoSyncJob> {
     queue
         .lock()
@@ -561,19 +583,6 @@ fn sync_repo(
 
     mirror_repo.configure_remotes(&initial_remotes)?;
     let cached_ref_state = cached_ref_state(&mirror_repo, &initial_remotes)?;
-    if !context.dry_run
-        && all_endpoints_present
-        && cached_refs_match(&mirror_repo, &initial_remotes, &initial_ref_state)?
-    {
-        crate::logln!(
-            "  {} refs unchanged from local mirror cache",
-            style("up-to-date").green().bold()
-        );
-        return Ok(RepoSyncOutcome {
-            state_update: Some(RepoStateUpdate::Set(initial_ref_state)),
-        });
-    }
-
     for remote in &initial_remotes {
         if let Err(error) = mirror_repo.fetch_remote(remote) {
             if is_disabled_repository_error(&error) {
@@ -772,22 +781,6 @@ fn all_configured_endpoints_present(mirror: &MirrorConfig, repos: &[EndpointRepo
         .all(|endpoint| present.contains(endpoint))
 }
 
-fn cached_refs_match(
-    mirror_repo: &GitMirror,
-    remotes: &[RemoteSpec],
-    expected_refs: &BTreeMap<String, RemoteRefState>,
-) -> Result<bool> {
-    for remote in remotes {
-        let Some(expected) = expected_refs.get(&remote.name) else {
-            return Ok(false);
-        };
-        if !mirror_repo.cached_remote_refs_match(remote, &RemoteRefSnapshot::from(expected))? {
-            return Ok(false);
-        }
-    }
-    Ok(true)
-}
-
 fn cached_ref_state(
     mirror_repo: &GitMirror,
     remotes: &[RemoteSpec],
@@ -1144,6 +1137,9 @@ fn open_conflict_pull_requests(
             for (source_remote, source_sha) in
                 conflict.tips.iter().filter(|(_, sha)| sha != target_sha)
             {
+                if mirror_repo.is_ancestor(source_sha, target_sha)? {
+                    continue;
+                }
                 let head_branch = conflict_pr_branch(&conflict.branch, source_remote, source_sha);
                 let update = BranchUpdate {
                     branch: head_branch.clone(),

src/webhook.rs

@@ -204,6 +204,7 @@ pub fn install_webhooks(config: &Config, options: WebhookInstallOptions) -> Resu
             style("Webhook group").cyan().bold(),
             style(&mirror.name).bold()
         );
+        let repo_filter = mirror.repo_filter()?;
         let mut tasks = Vec::new();
         for endpoint in &mirror.endpoints {
             let site = config.site(&endpoint.site).unwrap();
@@ -216,7 +217,11 @@ pub fn install_webhooks(config: &Config, options: WebhookInstallOptions) -> Resu
             let repos = client
                 .list_repos(endpoint)
                 .with_context(|| format!("failed to list repos for {}", endpoint.label()))?;
-            for repo in repos {
+            for repo in repos
+                .into_iter()
+                .filter(|repo| mirror.sync_visibility.matches_private(repo.private))
+                .filter(|repo| repo_filter.matches(&repo.name))
+            {
                 if options.repo.as_ref().is_some_and(|name| name != &repo.name) {
                     continue;
                 }
@@ -1009,19 +1014,22 @@ fn matching_jobs(config: &Config, event: &WebhookEvent) -> Vec<WebhookJob> {
         .mirrors
         .iter()
         .filter(|mirror| {
-            mirror.endpoints.iter().any(|endpoint| {
-                let Some(site) = config.site(&endpoint.site) else {
-                    return false;
-                };
-                event
-                    .provider
-                    .as_ref()
-                    .is_none_or(|provider| &site.provider == provider)
-                    && event
-                        .namespace
+            mirror
+                .repo_filter()
+                .is_ok_and(|filter| filter.matches(&event.repo))
+                && mirror.endpoints.iter().any(|endpoint| {
+                    let Some(site) = config.site(&endpoint.site) else {
+                        return false;
+                    };
+                    event
+                        .provider
                         .as_ref()
-                        .is_none_or(|namespace| namespace == &endpoint.namespace)
-            })
+                        .is_none_or(|provider| &site.provider == provider)
+                        && event
+                            .namespace
+                            .as_ref()
+                            .is_none_or(|namespace| namespace == &endpoint.namespace)
+                })
         })
         .map(|mirror| WebhookJob {
             group: mirror.name.clone(),