MaigoLabs/refray
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[+] Create repo by codex

Browse Source
This commit is contained in:
Azalea Gui
2026-05-03 17:24:24 +00:00
commit 947e71d1fd
9 changed files with 4542 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/config.rs
+415
View File
@@ -0,0 +1,415 @@
use std::env;
use std::fs;
use std::io::Write;
use std::path::{Path, PathBuf};
use anyhow::{Context, Result, anyhow, bail};
use directories::ProjectDirs;
use serde::{Deserialize, Serialize};
#[derive(Clone, Debug, Default, Deserialize, Serialize)]
pub struct Config {
#[serde(default)]
pub sites: Vec<SiteConfig>,
#[serde(default)]
pub mirrors: Vec<MirrorConfig>,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
pub struct SiteConfig {
pub name: String,
pub provider: ProviderKind,
pub base_url: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub api_url: Option<String>,
pub token: TokenConfig,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub git_username: Option<String>,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum ProviderKind {
Github,
Gitlab,
Gitea,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum TokenConfig {
Value(String),
Env(String),
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
pub struct MirrorConfig {
pub name: String,
pub endpoints: Vec<EndpointConfig>,
#[serde(default = "default_true")]
pub create_missing: bool,
#[serde(default)]
pub visibility: Visibility,
#[serde(default)]
pub allow_force: bool,
}
#[derive(Clone, Debug, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)]
pub struct EndpointConfig {
pub site: String,
pub kind: NamespaceKind,
pub namespace: String,
}
#[derive(Clone, Debug, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum NamespaceKind {
User,
Org,
Group,
}
#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum Visibility {
#[default]
Private,
Public,
}
fn default_true() -> bool {
true
}
impl Config {
pub fn load(path: &Path) -> Result<Self> {
let contents = fs::read_to_string(path)
.with_context(|| format!("failed to read {}", path.display()))?;
toml::from_str(&contents).with_context(|| format!("failed to parse {}", path.display()))
}
pub fn load_or_default(path: &Path) -> Result<Self> {
if path.exists() {
Self::load(path)
} else {
Ok(Self::default())
}
}
pub fn save(&self, path: &Path) -> Result<()> {
if let Some(parent) = path.parent() {
fs::create_dir_all(parent)
.with_context(|| format!("failed to create {}", parent.display()))?;
}
let contents = toml::to_string_pretty(self)?;
let mut file = fs::File::create(path)
.with_context(|| format!("failed to create {}", path.display()))?;
file.write_all(contents.as_bytes())
.with_context(|| format!("failed to write {}", path.display()))?;
protect_file(path)?;
Ok(())
}
pub fn site(&self, name: &str) -> Option<&SiteConfig> {
self.sites.iter().find(|site| site.name == name)
}
pub fn upsert_site(&mut self, site: SiteConfig) {
if let Some(existing) = self
.sites
.iter_mut()
.find(|existing| existing.name == site.name)
{
*existing = site;
} else {
self.sites.push(site);
}
}
pub fn remove_site(&mut self, name: &str) -> Result<()> {
if !self.sites.iter().any(|site| site.name == name) {
bail!("site '{name}' does not exist");
}
for mirror in &self.mirrors {
if mirror
.endpoints
.iter()
.any(|endpoint| endpoint.site == name)
{
bail!("site '{name}' is still used by mirror '{}'", mirror.name);
}
}
self.sites.retain(|site| site.name != name);
Ok(())
}
pub fn upsert_mirror(&mut self, mirror: MirrorConfig) {
if let Some(existing) = self
.mirrors
.iter_mut()
.find(|existing| existing.name == mirror.name)
{
*existing = mirror;
} else {
self.mirrors.push(mirror);
}
}
pub fn remove_mirror(&mut self, name: &str) -> Result<()> {
let old_len = self.mirrors.len();
self.mirrors.retain(|mirror| mirror.name != name);
if self.mirrors.len() == old_len {
bail!("mirror '{name}' does not exist");
}
Ok(())
}
}
impl SiteConfig {
pub fn token(&self) -> Result<String> {
match &self.token {
TokenConfig::Value(value) => Ok(value.clone()),
TokenConfig::Env(name) => {
env::var(name).with_context(|| format!("environment variable {name} is not set"))
}
}
}
pub fn api_base(&self) -> String {
if let Some(api_url) = &self.api_url {
return trim_end(api_url).to_string();
}
match self.provider {
ProviderKind::Github => {
if self.base_url.trim_end_matches('/') == "https://github.com" {
"https://api.github.com".to_string()
} else {
format!("{}/api/v3", trim_end(&self.base_url))
}
}
ProviderKind::Gitlab => format!("{}/api/v4", trim_end(&self.base_url)),
ProviderKind::Gitea => format!("{}/api/v1", trim_end(&self.base_url)),
}
}
}
impl EndpointConfig {
pub fn label(&self) -> String {
format!("{}:{}:{:?}", self.site, self.namespace, self.kind)
}
}
pub fn default_config_path() -> PathBuf {
ProjectDirs::from("dev", "git-sync", "git-sync")
.map(|dirs| dirs.config_dir().join("config.toml"))
.unwrap_or_else(|| PathBuf::from("git-sync.toml"))
}
pub fn default_work_dir() -> PathBuf {
ProjectDirs::from("dev", "git-sync", "git-sync")
.map(|dirs| dirs.cache_dir().join("mirrors"))
.unwrap_or_else(|| PathBuf::from(".git-sync-cache"))
}
fn trim_end(value: &str) -> &str {
value.trim_end_matches('/')
}
#[cfg(unix)]
fn protect_file(path: &Path) -> Result<()> {
use std::os::unix::fs::PermissionsExt;
let permissions = fs::Permissions::from_mode(0o600);
fs::set_permissions(path, permissions)
.with_context(|| format!("failed to set permissions on {}", path.display()))
}
#[cfg(not(unix))]
fn protect_file(_path: &Path) -> Result<()> {
Ok(())
}
pub fn validate_config(config: &Config) -> Result<()> {
if config.sites.is_empty() {
bail!("no sites configured");
}
if config.mirrors.is_empty() {
bail!("no mirror groups configured");
}
for mirror in &config.mirrors {
if mirror.endpoints.len() < 2 {
bail!(
"mirror '{}' must contain at least two endpoints",
mirror.name
);
}
for endpoint in &mirror.endpoints {
config.site(&endpoint.site).ok_or_else(|| {
anyhow!(
"mirror '{}' references unknown site '{}'",
mirror.name,
endpoint.site
)
})?;
}
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn parses_token_forms() {
let config: Config = toml::from_str(
r#"
[[sites]]
name = "github"
provider = "github"
base_url = "https://github.com"
token = { env = "GITHUB_TOKEN" }
[[mirrors]]
name = "personal"
create_missing = true
visibility = "private"
allow_force = false
[[mirrors.endpoints]]
site = "github"
kind = "user"
namespace = "alice"
[[mirrors.endpoints]]
site = "github"
kind = "org"
namespace = "example"
"#,
)
.unwrap();
assert_eq!(config.sites.len(), 1);
assert_eq!(config.mirrors[0].endpoints.len(), 2);
}
#[test]
fn validation_rejects_unknown_sites_and_single_endpoint_groups() {
let config = Config {
sites: vec![site("github", ProviderKind::Github)],
mirrors: vec![MirrorConfig {
name: "broken".to_string(),
endpoints: vec![EndpointConfig {
site: "github".to_string(),
kind: NamespaceKind::User,
namespace: "alice".to_string(),
}],
create_missing: true,
visibility: Visibility::Private,
allow_force: false,
}],
};
let err = validate_config(&config).unwrap_err().to_string();
assert!(err.contains("at least two endpoints"));
let config = Config {
sites: vec![site("github", ProviderKind::Github)],
mirrors: vec![MirrorConfig {
name: "broken".to_string(),
endpoints: vec![
EndpointConfig {
site: "github".to_string(),
kind: NamespaceKind::User,
namespace: "alice".to_string(),
},
EndpointConfig {
site: "missing".to_string(),
kind: NamespaceKind::User,
namespace: "alice".to_string(),
},
],
create_missing: true,
visibility: Visibility::Private,
allow_force: false,
}],
};
let err = validate_config(&config).unwrap_err().to_string();
assert!(err.contains("unknown site 'missing'"));
}
#[test]
fn removing_referenced_site_is_rejected() {
let mut config = Config {
sites: vec![
site("github", ProviderKind::Github),
site("gitea", ProviderKind::Gitea),
],
mirrors: vec![MirrorConfig {
name: "personal".to_string(),
endpoints: vec![
EndpointConfig {
site: "github".to_string(),
kind: NamespaceKind::User,
namespace: "alice".to_string(),
},
EndpointConfig {
site: "gitea".to_string(),
kind: NamespaceKind::User,
namespace: "alice".to_string(),
},
],
create_missing: true,
visibility: Visibility::Private,
allow_force: false,
}],
};
let err = config.remove_site("github").unwrap_err().to_string();
assert!(err.contains("still used by mirror 'personal'"));
assert!(config.site("github").is_some());
}
#[test]
fn api_base_defaults_match_providers() {
assert_eq!(
site("github", ProviderKind::Github).api_base(),
"https://api.github.com"
);
assert_eq!(
SiteConfig {
base_url: "https://github.example.test/".to_string(),
..site("github-enterprise", ProviderKind::Github)
}
.api_base(),
"https://github.example.test/api/v3"
);
assert_eq!(
SiteConfig {
base_url: "https://gitlab.example.test".to_string(),
..site("gitlab", ProviderKind::Gitlab)
}
.api_base(),
"https://gitlab.example.test/api/v4"
);
assert_eq!(
SiteConfig {
base_url: "https://gitea.example.test".to_string(),
..site("gitea", ProviderKind::Gitea)
}
.api_base(),
"https://gitea.example.test/api/v1"
);
}
fn site(name: &str, provider: ProviderKind) -> SiteConfig {
SiteConfig {
name: name.to_string(),
provider,
base_url: "https://github.com".to_string(),
api_url: None,
token: TokenConfig::Value("token".to_string()),
git_username: None,
}
}
}
src/git.rs
+758
View File
@@ -0,0 +1,758 @@
use std::collections::{BTreeMap, BTreeSet};
use std::fs;
use std::path::{Path, PathBuf};
use std::process::Command;
use anyhow::{Context, Result, bail};
#[derive(Clone, Debug)]
pub struct RemoteSpec {
pub name: String,
pub url: String,
pub display: String,
}
#[derive(Clone, Debug)]
pub struct BranchDecision {
pub branch: String,
pub sha: String,
pub source_remotes: Vec<String>,
}
#[derive(Clone, Debug)]
pub struct BranchConflict {
pub branch: String,
pub tips: Vec<(String, String)>,
}
#[derive(Clone, Debug)]
pub struct TagDecision {
pub tag: String,
pub sha: String,
pub source_remotes: Vec<String>,
}
#[derive(Clone, Debug)]
pub struct TagConflict {
pub tag: String,
pub tips: Vec<(String, String)>,
}
pub struct GitMirror {
path: PathBuf,
redactor: Redactor,
dry_run: bool,
}
#[derive(Clone, Debug)]
pub struct Redactor {
secrets: Vec<String>,
}
impl GitMirror {
pub fn open(path: PathBuf, redactor: Redactor, dry_run: bool) -> Result<Self> {
if !path.exists() {
if dry_run {
println!("dry-run: git init --bare {}", path.display());
return Ok(Self {
path,
redactor,
dry_run,
});
}
fs::create_dir_all(&path)?;
run_plain("git", ["init", "--bare"], Some(&path), &redactor, dry_run)
.with_context(|| format!("failed to initialize {}", path.display()))?;
}
Ok(Self {
path,
redactor,
dry_run,
})
}
pub fn configure_remotes(&self, remotes: &[RemoteSpec]) -> Result<()> {
for remote in remotes {
let existing = self.remote_url(&remote.name)?;
match existing {
Some(_) => self.run(["remote", "set-url", &remote.name, &remote.url])?,
None => self.run(["remote", "add", &remote.name, &remote.url])?,
}
}
Ok(())
}
pub fn fetch_remote(&self, remote: &RemoteSpec) -> Result<()> {
let branch_refspec = format!("+refs/heads/*:refs/remotes/{}/*", remote.name);
let tag_refspec = format!("+refs/tags/*:refs/remote-tags/{}/*", remote.name);
println!("fetching {}", remote.display);
self.run(["fetch", "--prune", &remote.name, &branch_refspec])?;
self.run(["fetch", "--prune", &remote.name, &tag_refspec])
}
pub fn branch_decisions(
&self,
remotes: &[RemoteSpec],
allow_force: bool,
) -> Result<(Vec<BranchDecision>, Vec<BranchConflict>)> {
let mut by_branch: BTreeMap<String, Vec<(String, String)>> = BTreeMap::new();
for remote in remotes {
for (branch, sha) in self.remote_branches(&remote.name)? {
by_branch
.entry(branch)
.or_default()
.push((remote.name.clone(), sha));
}
}
let mut decisions = Vec::new();
let mut conflicts = Vec::new();
for (branch, tips) in by_branch {
let unique = tips
.iter()
.map(|(_, sha)| sha.clone())
.collect::<BTreeSet<_>>();
if unique.len() == 1 {
decisions.push(BranchDecision {
branch,
sha: unique.into_iter().next().unwrap(),
source_remotes: tips.into_iter().map(|(remote, _)| remote).collect(),
});
continue;
}
if let Some(winner) = self.fast_forward_winner(unique.iter())? {
let source_remotes = tips
.into_iter()
.filter_map(|(remote, sha)| (sha == winner).then_some(remote))
.collect();
decisions.push(BranchDecision {
branch,
sha: winner,
source_remotes,
});
} else if allow_force {
let winner = self.newest_commit(unique.iter())?;
let source_remotes = tips
.into_iter()
.filter_map(|(remote, sha)| (sha == winner).then_some(remote))
.collect();
decisions.push(BranchDecision {
branch,
sha: winner,
source_remotes,
});
} else {
conflicts.push(BranchConflict { branch, tips });
}
}
Ok((decisions, conflicts))
}
pub fn tag_decisions(
&self,
remotes: &[RemoteSpec],
) -> Result<(Vec<TagDecision>, Vec<TagConflict>)> {
let mut by_tag: BTreeMap<String, Vec<(String, String)>> = BTreeMap::new();
for remote in remotes {
for (tag, sha) in self.remote_tags(&remote.name)? {
by_tag
.entry(tag)
.or_default()
.push((remote.name.clone(), sha));
}
}
let mut decisions = Vec::new();
let mut conflicts = Vec::new();
for (tag, tips) in by_tag {
let unique = tips
.iter()
.map(|(_, sha)| sha.clone())
.collect::<BTreeSet<_>>();
if unique.len() == 1 {
decisions.push(TagDecision {
tag,
sha: unique.into_iter().next().unwrap(),
source_remotes: tips.into_iter().map(|(remote, _)| remote).collect(),
});
} else {
conflicts.push(TagConflict { tag, tips });
}
}
Ok((decisions, conflicts))
}
pub fn push_branches(
&self,
remotes: &[RemoteSpec],
branches: &[BranchDecision],
force: bool,
) -> Result<()> {
for remote in remotes {
for branch in branches {
let refspec = if force {
format!("+{}:refs/heads/{}", branch.sha, branch.branch)
} else {
format!("{}:refs/heads/{}", branch.sha, branch.branch)
};
println!("pushing {} to {}", branch.branch, remote.display);
self.run(["push", &remote.name, &refspec])?;
}
}
Ok(())
}
pub fn push_tags(&self, remotes: &[RemoteSpec], tags: &[TagDecision]) -> Result<()> {
for remote in remotes {
for tag in tags {
let refspec = format!("{}:refs/tags/{}", tag.sha, tag.tag);
println!("pushing tag {} to {}", tag.tag, remote.display);
self.run(["push", &remote.name, &refspec])?;
}
}
Ok(())
}
fn remote_url(&self, name: &str) -> Result<Option<String>> {
let output = Command::new("git")
.arg("--git-dir")
.arg(&self.path)
.args(["remote", "get-url", name])
.output()
.with_context(|| "failed to run git remote get-url")?;
if output.status.success() {
Ok(Some(
String::from_utf8_lossy(&output.stdout).trim().to_string(),
))
} else {
Ok(None)
}
}
fn remote_branches(&self, remote: &str) -> Result<Vec<(String, String)>> {
let prefix = format!("refs/remotes/{remote}/");
let output = self.output(["for-each-ref", "--format=%(refname) %(objectname)", &prefix])?;
let mut branches = Vec::new();
for line in output.lines() {
let Some((refname, sha)) = line.split_once(' ') else {
continue;
};
let Some(branch) = refname.strip_prefix(&prefix) else {
continue;
};
if branch == "HEAD" {
continue;
}
branches.push((branch.to_string(), sha.to_string()));
}
Ok(branches)
}
fn remote_tags(&self, remote: &str) -> Result<Vec<(String, String)>> {
let prefix = format!("refs/remote-tags/{remote}/");
let output = self.output(["for-each-ref", "--format=%(refname) %(objectname)", &prefix])?;
let mut tags = Vec::new();
for line in output.lines() {
let Some((refname, sha)) = line.split_once(' ') else {
continue;
};
let Some(tag) = refname.strip_prefix(&prefix) else {
continue;
};
tags.push((tag.to_string(), sha.to_string()));
}
Ok(tags)
}
fn fast_forward_winner<'a>(
&self,
shas: impl Iterator<Item = &'a String> + Clone,
) -> Result<Option<String>> {
for candidate in shas.clone() {
let mut is_descendant = true;
for other in shas.clone() {
if candidate == other {
continue;
}
if !self.is_ancestor(other, candidate)? {
is_descendant = false;
break;
}
}
if is_descendant {
return Ok(Some(candidate.clone()));
}
}
Ok(None)
}
fn newest_commit<'a>(&self, shas: impl Iterator<Item = &'a String>) -> Result<String> {
let mut newest: Option<(i64, String)> = None;
for sha in shas {
let timestamp = self
.output(["show", "-s", "--format=%ct", sha])?
.trim()
.parse::<i64>()?;
match &newest {
Some((old, _)) if *old >= timestamp => {}
_ => newest = Some((timestamp, sha.clone())),
}
}
newest
.map(|(_, sha)| sha)
.context("no commits found while choosing force winner")
}
fn is_ancestor(&self, ancestor: &str, descendant: &str) -> Result<bool> {
let status = Command::new("git")
.arg("--git-dir")
.arg(&self.path)
.args(["merge-base", "--is-ancestor", ancestor, descendant])
.status()
.with_context(|| "failed to run git merge-base")?;
match status.code() {
Some(0) => Ok(true),
Some(1) => Ok(false),
_ => bail!("git merge-base failed for {ancestor} and {descendant}"),
}
}
fn run<const N: usize>(&self, args: [&str; N]) -> Result<()> {
run_plain(
"git",
std::iter::once("--git-dir")
.chain(std::iter::once(self.path.to_str().unwrap()))
.chain(args),
None,
&self.redactor,
self.dry_run,
)
}
fn output<const N: usize>(&self, args: [&str; N]) -> Result<String> {
if self.dry_run {
return Ok(String::new());
}
let output = Command::new("git")
.arg("--git-dir")
.arg(&self.path)
.args(args)
.output()
.with_context(|| "failed to run git")?;
if output.status.success() {
Ok(String::from_utf8_lossy(&output.stdout).to_string())
} else {
bail!(
"git failed: {}",
self.redactor
.redact(&String::from_utf8_lossy(&output.stderr))
);
}
}
}
impl Redactor {
pub fn new(secrets: Vec<String>) -> Self {
let secrets = secrets
.into_iter()
.filter(|secret| !secret.is_empty())
.collect();
Self { secrets }
}
pub fn redact(&self, value: &str) -> String {
let mut redacted = value.to_string();
for secret in &self.secrets {
redacted = redacted.replace(secret, "<redacted>");
}
redacted
}
}
fn run_plain<I, S>(
program: &str,
args: I,
current_dir: Option<&Path>,
redactor: &Redactor,
dry_run: bool,
) -> Result<()>
where
I: IntoIterator<Item = S>,
S: AsRef<str>,
{
let args = args
.into_iter()
.map(|arg| arg.as_ref().to_string())
.collect::<Vec<_>>();
if dry_run {
println!("dry-run: {} {}", program, redactor.redact(&args.join(" ")));
return Ok(());
}
let mut command = Command::new(program);
command.args(&args);
if let Some(current_dir) = current_dir {
command.current_dir(current_dir);
}
let output = command
.output()
.with_context(|| format!("failed to run {program}"))?;
if output.status.success() {
Ok(())
} else {
let stdout = redactor.redact(&String::from_utf8_lossy(&output.stdout));
let stderr = redactor.redact(&String::from_utf8_lossy(&output.stderr));
bail!("{program} failed\nstdout: {stdout}\nstderr: {stderr}");
}
}
pub fn safe_remote_name(value: &str) -> String {
value
.chars()
.map(|ch| {
if ch.is_ascii_alphanumeric() || ch == '-' || ch == '_' {
ch
} else {
'_'
}
})
.collect()
}
#[cfg(test)]
mod tests {
use super::*;
use std::io::Write;
use tempfile::TempDir;
#[test]
fn remote_names_are_git_friendly() {
assert_eq!(
safe_remote_name("github:alice/project"),
"github_alice_project"
);
}
#[test]
fn redacts_all_secrets() {
let redactor = Redactor::new(vec!["secret".to_string()]);
assert_eq!(
redactor.redact("https://secret@example.test"),
"https://<redacted>@example.test"
);
}
#[test]
fn branch_decisions_choose_fast_forward_tip() {
let fixture = GitFixture::new();
let base = fixture.commit("base", "base", 1_700_000_000);
fixture.push_head(&fixture.remote_a, "main");
fixture.push_head(&fixture.remote_b, "main");
let newer = fixture.commit("newer", "newer", 1_700_000_100);
fixture.push_head(&fixture.remote_a, "main");
let mirror = fixture.mirror();
fixture.fetch_all(&mirror);
let (decisions, conflicts) = mirror.branch_decisions(&fixture.remotes(), false).unwrap();
assert!(conflicts.is_empty());
let main = find_branch(&decisions, "main");
assert_eq!(main.sha, newer);
assert_eq!(main.source_remotes, vec!["a".to_string()]);
assert_ne!(main.sha, base);
}
#[test]
fn branch_decisions_report_divergent_tips_without_force() {
let fixture = GitFixture::new();
let base = fixture.commit("base", "base", 1_700_000_000);
fixture.push_head(&fixture.remote_a, "main");
fixture.push_head(&fixture.remote_b, "main");
let a_tip = fixture.commit("a", "a", 1_700_000_100);
fixture.push_head(&fixture.remote_a, "main");
fixture.reset_hard(&base);
let b_tip = fixture.commit("b", "b", 1_700_000_200);
fixture.push_head(&fixture.remote_b, "main");
let mirror = fixture.mirror();
fixture.fetch_all(&mirror);
let (decisions, conflicts) = mirror.branch_decisions(&fixture.remotes(), false).unwrap();
assert!(decisions.is_empty());
assert_eq!(conflicts.len(), 1);
assert_eq!(conflicts[0].branch, "main");
assert!(conflicts[0].tips.iter().any(|(_, sha)| sha == &a_tip));
assert!(conflicts[0].tips.iter().any(|(_, sha)| sha == &b_tip));
}
#[test]
fn branch_decisions_force_selects_newest_divergent_tip() {
let fixture = GitFixture::new();
let base = fixture.commit("base", "base", 1_700_000_000);
fixture.push_head(&fixture.remote_a, "main");
fixture.push_head(&fixture.remote_b, "main");
let older = fixture.commit("older", "older", 1_700_000_100);
fixture.push_head(&fixture.remote_a, "main");
fixture.reset_hard(&base);
let newer = fixture.commit("newer", "newer", 1_700_000_200);
fixture.push_head(&fixture.remote_b, "main");
let mirror = fixture.mirror();
fixture.fetch_all(&mirror);
let (decisions, conflicts) = mirror.branch_decisions(&fixture.remotes(), true).unwrap();
assert!(conflicts.is_empty());
let main = find_branch(&decisions, "main");
assert_eq!(main.sha, newer);
assert_ne!(main.sha, older);
assert_eq!(main.source_remotes, vec!["b".to_string()]);
}
#[test]
fn push_branches_creates_missing_branch_on_other_remotes() {
let fixture = GitFixture::new();
let expected = fixture.commit("base", "base", 1_700_000_000);
fixture.push_head(&fixture.remote_a, "main");
let mirror = fixture.mirror();
fixture.fetch_all(&mirror);
let (decisions, conflicts) = mirror.branch_decisions(&fixture.remotes(), false).unwrap();
assert!(conflicts.is_empty());
mirror
.push_branches(&fixture.remotes(), &decisions, false)
.unwrap();
assert_eq!(
fixture.remote_ref(&fixture.remote_b, "refs/heads/main"),
expected
);
}
#[test]
fn tag_decisions_mirror_matching_or_missing_tags_and_skip_divergent_tags() {
let fixture = GitFixture::new();
let base = fixture.commit("base", "base", 1_700_000_000);
fixture.tag("v1");
fixture.push_head(&fixture.remote_a, "main");
fixture.push_head(&fixture.remote_b, "main");
fixture.push_tag(&fixture.remote_a, "v1");
fixture.push_tag(&fixture.remote_b, "v1");
let a_tip = fixture.commit("a", "a", 1_700_000_100);
fixture.tag("release");
fixture.push_head(&fixture.remote_a, "main");
fixture.push_tag(&fixture.remote_a, "release");
fixture.delete_tag("release");
fixture.reset_hard(&base);
let b_tip = fixture.commit("b", "b", 1_700_000_200);
fixture.tag("release");
fixture.push_head(&fixture.remote_b, "main");
fixture.push_tag(&fixture.remote_b, "release");
fixture.delete_tag("missing-on-b");
fixture.reset_hard(&a_tip);
fixture.tag("missing-on-b");
fixture.push_tag(&fixture.remote_a, "missing-on-b");
let mirror = fixture.mirror();
fixture.fetch_all(&mirror);
let (tags, conflicts) = mirror.tag_decisions(&fixture.remotes()).unwrap();
assert_eq!(find_tag(&tags, "v1").sha, base);
assert_eq!(find_tag(&tags, "missing-on-b").sha, a_tip);
assert_eq!(conflicts.len(), 1);
assert_eq!(conflicts[0].tag, "release");
assert!(conflicts[0].tips.iter().any(|(_, sha)| sha == &a_tip));
assert!(conflicts[0].tips.iter().any(|(_, sha)| sha == &b_tip));
mirror.push_tags(&fixture.remotes(), &tags).unwrap();
assert_eq!(
fixture.remote_ref(&fixture.remote_b, "refs/tags/missing-on-b"),
a_tip
);
}
fn find_branch<'a>(decisions: &'a [BranchDecision], name: &str) -> &'a BranchDecision {
decisions
.iter()
.find(|decision| decision.branch == name)
.unwrap_or_else(|| panic!("missing branch decision for {name}"))
}
fn find_tag<'a>(decisions: &'a [TagDecision], name: &str) -> &'a TagDecision {
decisions
.iter()
.find(|decision| decision.tag == name)
.unwrap_or_else(|| panic!("missing tag decision for {name}"))
}
struct GitFixture {
_temp: TempDir,
work: PathBuf,
mirror_path: PathBuf,
remote_a: PathBuf,
remote_b: PathBuf,
}
impl GitFixture {
fn new() -> Self {
let temp = TempDir::new().unwrap();
let work = temp.path().join("work");
let mirror_path = temp.path().join("mirror.git");
let remote_a = temp.path().join("a.git");
let remote_b = temp.path().join("b.git");
git(None, ["init", "--bare", remote_a.to_str().unwrap()]);
git(None, ["init", "--bare", remote_b.to_str().unwrap()]);
fs::create_dir_all(&work).unwrap();
git(Some(&work), ["init"]);
git(Some(&work), ["config", "user.email", "test@example.test"]);
git(Some(&work), ["config", "user.name", "Test User"]);
git(Some(&work), ["checkout", "-b", "main"]);
Self {
_temp: temp,
work,
mirror_path,
remote_a,
remote_b,
}
}
fn mirror(&self) -> GitMirror {
let mirror =
GitMirror::open(self.mirror_path.clone(), Redactor::new(Vec::new()), false)
.unwrap();
mirror.configure_remotes(&self.remotes()).unwrap();
mirror
}
fn remotes(&self) -> Vec<RemoteSpec> {
vec![
RemoteSpec {
name: "a".to_string(),
url: self.remote_a.to_string_lossy().to_string(),
display: "remote a".to_string(),
},
RemoteSpec {
name: "b".to_string(),
url: self.remote_b.to_string_lossy().to_string(),
display: "remote b".to_string(),
},
]
}
fn fetch_all(&self, mirror: &GitMirror) {
for remote in self.remotes() {
mirror.fetch_remote(&remote).unwrap();
}
}
fn commit(&self, message: &str, contents: &str, timestamp: i64) -> String {
let path = self.work.join("file.txt");
let mut file = fs::OpenOptions::new()
.create(true)
.append(true)
.open(path)
.unwrap();
writeln!(file, "{contents}").unwrap();
git(Some(&self.work), ["add", "file.txt"]);
let date = format!("@{timestamp} +0000");
let output = Command::new("git")
.current_dir(&self.work)
.env("GIT_AUTHOR_DATE", &date)
.env("GIT_COMMITTER_DATE", &date)
.args(["commit", "-m", message])
.output()
.unwrap();
assert_success(&output, "git commit");
self.head()
}
fn head(&self) -> String {
git_output(Some(&self.work), ["rev-parse", "HEAD"])
}
fn reset_hard(&self, sha: &str) {
git(Some(&self.work), ["reset", "--hard", sha]);
}
fn push_head(&self, remote: &Path, branch: &str) {
let refspec = format!("HEAD:refs/heads/{branch}");
git(
Some(&self.work),
["push", remote.to_str().unwrap(), &refspec],
);
}
fn tag(&self, name: &str) {
git(Some(&self.work), ["tag", name]);
}
fn delete_tag(&self, name: &str) {
let _ = Command::new("git")
.current_dir(&self.work)
.args(["tag", "-d", name])
.output()
.unwrap();
}
fn push_tag(&self, remote: &Path, tag: &str) {
let refspec = format!("refs/tags/{tag}:refs/tags/{tag}");
git(
Some(&self.work),
["push", remote.to_str().unwrap(), &refspec],
);
}
fn remote_ref(&self, remote: &Path, reference: &str) -> String {
git_output(
None,
[
"--git-dir",
remote.to_str().unwrap(),
"rev-parse",
reference,
],
)
}
}
fn git<const N: usize>(current_dir: Option<&Path>, args: [&str; N]) {
let output = git_command(current_dir, args).output().unwrap();
assert_success(&output, "git");
}
fn git_output<const N: usize>(current_dir: Option<&Path>, args: [&str; N]) -> String {
let output = git_command(current_dir, args).output().unwrap();
assert_success(&output, "git output");
String::from_utf8_lossy(&output.stdout).trim().to_string()
}
fn git_command<const N: usize>(current_dir: Option<&Path>, args: [&str; N]) -> Command {
let mut command = Command::new("git");
command.args(args);
if let Some(current_dir) = current_dir {
command.current_dir(current_dir);
}
command
}
fn assert_success(output: &std::process::Output, label: &str) {
assert!(
output.status.success(),
"{label} failed\nstdout: {}\nstderr: {}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
}
}
src/main.rs
+399
View File
@@ -0,0 +1,399 @@
mod config;
mod git;
mod provider;
mod sync;
use std::path::PathBuf;
use anyhow::{Context, Result, bail};
use clap::{Args, Parser, Subcommand, ValueEnum};
use crate::config::{
Config, EndpointConfig, NamespaceKind, ProviderKind, SiteConfig, TokenConfig, Visibility,
default_config_path,
};
use crate::sync::{SyncOptions, sync_all};
#[derive(Parser, Debug)]
#[command(name = "git-sync")]
#[command(about = "Mirror repositories between Git hosting providers")]
struct Cli {
#[arg(long, global = true, value_name = "PATH")]
config: Option<PathBuf>,
#[command(subcommand)]
command: Command,
}
#[derive(Subcommand, Debug)]
enum Command {
#[command(subcommand)]
Config(ConfigCommand),
Sync(SyncCommand),
}
#[derive(Subcommand, Debug)]
enum ConfigCommand {
Init,
#[command(subcommand)]
Site(SiteCommand),
#[command(subcommand)]
Mirror(MirrorCommand),
Show,
}
#[derive(Subcommand, Debug)]
enum SiteCommand {
Add(SiteAddCommand),
Remove(NameCommand),
List,
}
#[derive(Subcommand, Debug)]
enum MirrorCommand {
Add(MirrorAddCommand),
Remove(NameCommand),
List,
}
#[derive(Args, Debug)]
struct NameCommand {
name: String,
}
#[derive(Args, Debug)]
struct SiteAddCommand {
#[arg(long)]
name: String,
#[arg(long)]
provider: ProviderArg,
#[arg(long, value_name = "URL")]
base_url: String,
#[arg(long, value_name = "URL")]
api_url: Option<String>,
#[arg(long, conflicts_with = "token_env")]
token: Option<String>,
#[arg(long, value_name = "ENV", conflicts_with = "token")]
token_env: Option<String>,
#[arg(long)]
git_username: Option<String>,
}
#[derive(Args, Debug)]
struct MirrorAddCommand {
#[arg(long)]
name: String,
#[arg(long = "endpoint", required = true, action = clap::ArgAction::Append, value_name = "SITE:KIND:NAMESPACE")]
endpoints: Vec<String>,
#[arg(long, default_value_t = true, action = clap::ArgAction::Set)]
create_missing: bool,
#[arg(long, default_value_t = VisibilityArg::Private)]
visibility: VisibilityArg,
#[arg(long, default_value_t = false)]
allow_force: bool,
}
#[derive(Args, Debug)]
struct SyncCommand {
#[arg(long, value_name = "NAME")]
group: Option<String>,
#[arg(long)]
dry_run: bool,
#[arg(long)]
no_create: bool,
#[arg(long)]
force: bool,
#[arg(long, value_name = "PATH")]
work_dir: Option<PathBuf>,
}
#[derive(Clone, Debug, ValueEnum)]
enum ProviderArg {
Github,
Gitlab,
Gitea,
}
#[derive(Clone, Debug, ValueEnum)]
enum VisibilityArg {
Private,
Public,
}
impl std::fmt::Display for VisibilityArg {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Private => write!(f, "private"),
Self::Public => write!(f, "public"),
}
}
}
fn main() -> Result<()> {
let cli = Cli::parse();
let config_path = cli.config.unwrap_or_else(default_config_path);
match cli.command {
Command::Config(command) => handle_config(command, config_path),
Command::Sync(command) => {
let config = Config::load(&config_path)
.with_context(|| format!("failed to load config at {}", config_path.display()))?;
sync_all(
&config,
SyncOptions {
group: command.group,
dry_run: command.dry_run,
create_missing_override: command.no_create.then_some(false),
force_override: command.force.then_some(true),
work_dir: command.work_dir,
},
)
}
}
}
fn handle_config(command: ConfigCommand, path: PathBuf) -> Result<()> {
match command {
ConfigCommand::Init => {
if path.exists() {
bail!("config already exists at {}", path.display());
}
let config = Config::default();
config.save(&path)?;
println!("created {}", path.display());
Ok(())
}
ConfigCommand::Site(command) => handle_site(command, path),
ConfigCommand::Mirror(command) => handle_mirror(command, path),
ConfigCommand::Show => {
let config = Config::load(&path)?;
println!("{}", toml::to_string_pretty(&config)?);
Ok(())
}
}
}
fn handle_site(command: SiteCommand, path: PathBuf) -> Result<()> {
let mut config = Config::load_or_default(&path)?;
match command {
SiteCommand::Add(args) => {
let token = match (args.token, args.token_env) {
(Some(value), None) => TokenConfig::Value(value),
(None, Some(env)) => TokenConfig::Env(env),
(None, None) => bail!("pass either --token or --token-env"),
(Some(_), Some(_)) => unreachable!("clap enforces token conflicts"),
};
config.upsert_site(SiteConfig {
name: args.name,
provider: args.provider.into(),
base_url: args.base_url,
api_url: args.api_url,
token,
git_username: args.git_username,
});
config.save(&path)?;
println!("updated {}", path.display());
Ok(())
}
SiteCommand::Remove(args) => {
config.remove_site(&args.name)?;
config.save(&path)?;
println!("removed site {}", args.name);
Ok(())
}
SiteCommand::List => {
for site in &config.sites {
println!("{}\t{:?}\t{}", site.name, site.provider, site.base_url);
}
Ok(())
}
}
}
fn handle_mirror(command: MirrorCommand, path: PathBuf) -> Result<()> {
let mut config = Config::load_or_default(&path)?;
match command {
MirrorCommand::Add(args) => {
if args.endpoints.len() < 2 {
bail!("mirror groups need at least two --endpoint values");
}
let endpoints = args
.endpoints
.iter()
.map(|value| parse_endpoint(value))
.collect::<Result<Vec<_>>>()?;
for endpoint in &endpoints {
config
.site(&endpoint.site)
.with_context(|| format!("unknown site '{}'", endpoint.site))?;
}
config.upsert_mirror(config::MirrorConfig {
name: args.name,
endpoints,
create_missing: args.create_missing,
visibility: args.visibility.into(),
allow_force: args.allow_force,
});
config.save(&path)?;
println!("updated {}", path.display());
Ok(())
}
MirrorCommand::Remove(args) => {
config.remove_mirror(&args.name)?;
config.save(&path)?;
println!("removed mirror {}", args.name);
Ok(())
}
MirrorCommand::List => {
for mirror in &config.mirrors {
let endpoints = mirror
.endpoints
.iter()
.map(|endpoint| {
format!(
"{}:{:?}:{}",
endpoint.site, endpoint.kind, endpoint.namespace
)
})
.collect::<Vec<_>>()
.join(", ");
println!("{}\t{}", mirror.name, endpoints);
}
Ok(())
}
}
}
fn parse_endpoint(value: &str) -> Result<EndpointConfig> {
let parts = value.splitn(3, ':').collect::<Vec<_>>();
if parts.len() != 3 {
bail!("endpoint must be SITE:KIND:NAMESPACE, got '{value}'");
}
let kind = match parts[1].to_ascii_lowercase().as_str() {
"user" => NamespaceKind::User,
"org" | "organization" => NamespaceKind::Org,
"group" => NamespaceKind::Group,
other => bail!("unsupported namespace kind '{other}'"),
};
Ok(EndpointConfig {
site: parts[0].to_string(),
kind,
namespace: parts[2].to_string(),
})
}
impl From<ProviderArg> for ProviderKind {
fn from(value: ProviderArg) -> Self {
match value {
ProviderArg::Github => Self::Github,
ProviderArg::Gitlab => Self::Gitlab,
ProviderArg::Gitea => Self::Gitea,
}
}
}
impl From<VisibilityArg> for Visibility {
fn from(value: VisibilityArg) -> Self {
match value {
VisibilityArg::Private => Self::Private,
VisibilityArg::Public => Self::Public,
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn cli_accepts_repeated_mirror_endpoints() {
let cli = Cli::try_parse_from([
"git-sync",
"config",
"mirror",
"add",
"--name",
"personal",
"--endpoint",
"github:user:hykilpikonna",
"--endpoint",
"gitea:user:azalea",
])
.unwrap();
let Command::Config(ConfigCommand::Mirror(MirrorCommand::Add(args))) = cli.command else {
panic!("parsed unexpected command");
};
assert_eq!(args.name, "personal");
assert_eq!(
args.endpoints,
vec![
"github:user:hykilpikonna".to_string(),
"gitea:user:azalea".to_string()
]
);
}
#[test]
fn endpoint_parser_supports_aliases_and_rejects_bad_kinds() {
let endpoint = parse_endpoint("github:organization:MewoLab").unwrap();
assert_eq!(endpoint.site, "github");
assert_eq!(endpoint.kind, NamespaceKind::Org);
assert_eq!(endpoint.namespace, "MewoLab");
let endpoint = parse_endpoint("gitlab:group:parent/child").unwrap();
assert_eq!(endpoint.kind, NamespaceKind::Group);
let err = parse_endpoint("github:team:alice").unwrap_err().to_string();
assert!(err.contains("unsupported namespace kind 'team'"));
let err = parse_endpoint("github:user").unwrap_err().to_string();
assert!(err.contains("SITE:KIND:NAMESPACE"));
}
#[test]
fn site_add_requires_one_token_source() {
let missing = Cli::try_parse_from([
"git-sync",
"config",
"site",
"add",
"--name",
"github",
"--provider",
"github",
"--base-url",
"https://github.com",
])
.unwrap();
let Command::Config(ConfigCommand::Site(SiteCommand::Add(args))) = missing.command else {
panic!("parsed unexpected command");
};
let temp = tempfile::TempDir::new().unwrap();
let err = handle_site(SiteCommand::Add(args), temp.path().join("config.toml"))
.unwrap_err()
.to_string();
assert!(err.contains("pass either --token or --token-env"));
let conflict = Cli::try_parse_from([
"git-sync",
"config",
"site",
"add",
"--name",
"github",
"--provider",
"github",
"--base-url",
"https://github.com",
"--token",
"a",
"--token-env",
"GITHUB_TOKEN",
]);
assert!(conflict.is_err());
}
}
src/provider.rs
+512
View File
@@ -0,0 +1,512 @@
use std::collections::HashMap;
use anyhow::{Context, Result, anyhow, bail};
use reqwest::blocking::{Client, Response};
use reqwest::header::{ACCEPT, AUTHORIZATION, HeaderMap, HeaderValue, USER_AGENT};
use serde::Deserialize;
use serde_json::json;
use url::Url;
use crate::config::{EndpointConfig, NamespaceKind, ProviderKind, SiteConfig, Visibility};
#[derive(Clone, Debug)]
pub struct RemoteRepo {
pub name: String,
pub clone_url: String,
pub private: bool,
pub description: Option<String>,
}
#[derive(Clone, Debug)]
pub struct EndpointRepo {
pub endpoint: EndpointConfig,
pub repo: RemoteRepo,
}
pub struct ProviderClient<'a> {
site: &'a SiteConfig,
token: String,
http: Client,
}
impl<'a> ProviderClient<'a> {
pub fn new(site: &'a SiteConfig) -> Result<Self> {
let token = site.token()?;
Ok(Self {
site,
token,
http: Client::builder().build()?,
})
}
pub fn list_repos(&self, endpoint: &EndpointConfig) -> Result<Vec<RemoteRepo>> {
match self.site.provider {
ProviderKind::Github => self.github_list_repos(endpoint),
ProviderKind::Gitlab => self.gitlab_list_repos(endpoint),
ProviderKind::Gitea => self.gitea_list_repos(endpoint),
}
}
pub fn create_repo(
&self,
endpoint: &EndpointConfig,
name: &str,
visibility: &Visibility,
description: Option<&str>,
) -> Result<RemoteRepo> {
match self.site.provider {
ProviderKind::Github => {
self.github_create_repo(endpoint, name, visibility, description)
}
ProviderKind::Gitlab => {
self.gitlab_create_repo(endpoint, name, visibility, description)
}
ProviderKind::Gitea => self.gitea_create_repo(endpoint, name, visibility, description),
}
}
pub fn authenticated_clone_url(&self, clone_url: &str) -> Result<String> {
let mut url = Url::parse(clone_url)
.or_else(|_| Url::parse(&format!("{}/{}", self.site.base_url, clone_url)))
.with_context(|| format!("failed to parse clone URL '{clone_url}'"))?;
if url.scheme() != "https" && url.scheme() != "http" {
bail!("only HTTP(S) clone URLs are supported, got '{clone_url}'");
}
let username = self
.site
.git_username
.clone()
.unwrap_or_else(|| match self.site.provider {
ProviderKind::Github => "x-access-token".to_string(),
ProviderKind::Gitlab | ProviderKind::Gitea => "oauth2".to_string(),
});
url.set_username(&username)
.map_err(|_| anyhow!("failed to set username on clone URL"))?;
url.set_password(Some(&self.token))
.map_err(|_| anyhow!("failed to set token on clone URL"))?;
Ok(url.to_string())
}
fn github_list_repos(&self, endpoint: &EndpointConfig) -> Result<Vec<RemoteRepo>> {
match endpoint.kind {
NamespaceKind::User => {
let url = format!(
"{}/user/repos?affiliation=owner&visibility=all&per_page=100",
self.site.api_base()
);
let repos: Vec<GithubRepo> = self
.paged_get(&url)?
.into_iter()
.filter(|repo: &GithubRepo| {
repo.owner.login.eq_ignore_ascii_case(&endpoint.namespace)
})
.collect();
Ok(repos.into_iter().map(Into::into).collect())
}
NamespaceKind::Org => {
let url = format!(
"{}/orgs/{}/repos?type=all&per_page=100",
self.site.api_base(),
endpoint.namespace
);
let repos: Vec<GithubRepo> = self.paged_get(&url)?;
Ok(repos.into_iter().map(Into::into).collect())
}
NamespaceKind::Group => bail!("GitHub endpoints use kind 'user' or 'org'"),
}
}
fn github_create_repo(
&self,
endpoint: &EndpointConfig,
name: &str,
visibility: &Visibility,
description: Option<&str>,
) -> Result<RemoteRepo> {
let url = match endpoint.kind {
NamespaceKind::User => format!("{}/user/repos", self.site.api_base()),
NamespaceKind::Org => {
format!("{}/orgs/{}/repos", self.site.api_base(), endpoint.namespace)
}
NamespaceKind::Group => bail!("GitHub endpoints use kind 'user' or 'org'"),
};
let body = json!({
"name": name,
"private": matches!(visibility, Visibility::Private),
"description": description.unwrap_or(""),
});
self.post_json::<GithubRepo>(&url, &body).map(Into::into)
}
fn gitlab_list_repos(&self, endpoint: &EndpointConfig) -> Result<Vec<RemoteRepo>> {
match endpoint.kind {
NamespaceKind::User => {
let url = format!(
"{}/users/{}/projects?simple=true&per_page=100&owned=true",
self.site.api_base(),
endpoint.namespace
);
let repos: Vec<GitlabProject> = self.paged_get(&url)?;
Ok(repos.into_iter().map(Into::into).collect())
}
NamespaceKind::Org | NamespaceKind::Group => {
let encoded = urlencoding(&endpoint.namespace);
let url = format!(
"{}/groups/{}/projects?simple=true&include_subgroups=false&per_page=100",
self.site.api_base(),
encoded
);
let repos: Vec<GitlabProject> = self.paged_get(&url)?;
Ok(repos.into_iter().map(Into::into).collect())
}
}
}
fn gitlab_create_repo(
&self,
endpoint: &EndpointConfig,
name: &str,
visibility: &Visibility,
description: Option<&str>,
) -> Result<RemoteRepo> {
let mut body = serde_json::Map::from_iter([
("name".to_string(), json!(name)),
("path".to_string(), json!(name)),
(
"visibility".to_string(),
json!(match visibility {
Visibility::Private => "private",
Visibility::Public => "public",
}),
),
("description".to_string(), json!(description.unwrap_or(""))),
]);
if matches!(endpoint.kind, NamespaceKind::Org | NamespaceKind::Group) {
let group = self.gitlab_group(&endpoint.namespace)?;
body.insert("namespace_id".to_string(), json!(group.id));
}
let url = format!("{}/projects", self.site.api_base());
self.post_json::<GitlabProject>(&url, &serde_json::Value::Object(body))
.map(Into::into)
}
fn gitlab_group(&self, namespace: &str) -> Result<GitlabGroup> {
let url = format!("{}/groups/{}", self.site.api_base(), urlencoding(namespace));
self.get_json(&url)
}
fn gitea_list_repos(&self, endpoint: &EndpointConfig) -> Result<Vec<RemoteRepo>> {
match endpoint.kind {
NamespaceKind::User => {
let url = format!("{}/user/repos?limit=50", self.site.api_base());
let repos: Vec<GiteaRepo> = self
.paged_get(&url)?
.into_iter()
.filter(|repo: &GiteaRepo| {
repo.owner.login.eq_ignore_ascii_case(&endpoint.namespace)
})
.collect();
Ok(repos.into_iter().map(Into::into).collect())
}
NamespaceKind::Org => {
let url = format!(
"{}/orgs/{}/repos?limit=50",
self.site.api_base(),
endpoint.namespace
);
let repos: Vec<GiteaRepo> = self.paged_get(&url)?;
Ok(repos.into_iter().map(Into::into).collect())
}
NamespaceKind::Group => bail!("Gitea endpoints use kind 'user' or 'org'"),
}
}
fn gitea_create_repo(
&self,
endpoint: &EndpointConfig,
name: &str,
visibility: &Visibility,
description: Option<&str>,
) -> Result<RemoteRepo> {
let url = match endpoint.kind {
NamespaceKind::User => format!("{}/user/repos", self.site.api_base()),
NamespaceKind::Org => {
format!("{}/orgs/{}/repos", self.site.api_base(), endpoint.namespace)
}
NamespaceKind::Group => bail!("Gitea endpoints use kind 'user' or 'org'"),
};
let body = json!({
"name": name,
"private": matches!(visibility, Visibility::Private),
"description": description.unwrap_or(""),
"auto_init": false,
});
self.post_json::<GiteaRepo>(&url, &body).map(Into::into)
}
fn paged_get<T>(&self, first_url: &str) -> Result<Vec<T>>
where
T: for<'de> Deserialize<'de>,
{
let mut output = Vec::new();
let mut next_url = Some(first_url.to_string());
while let Some(url) = next_url.take() {
let response = self.get(&url)?;
next_url = next_link(response.headers());
let mut page: Vec<T> = response
.json()
.with_context(|| format!("invalid JSON from {url}"))?;
output.append(&mut page);
}
Ok(output)
}
fn get_json<T>(&self, url: &str) -> Result<T>
where
T: for<'de> Deserialize<'de>,
{
self.get(url)?
.json()
.with_context(|| format!("invalid JSON from {url}"))
}
fn post_json<T>(&self, url: &str, body: &serde_json::Value) -> Result<T>
where
T: for<'de> Deserialize<'de>,
{
self.request_headers(self.http.post(url))?
.json(body)
.send()
.with_context(|| format!("POST {url} failed"))
.and_then(|response| check_response("POST", url, response))?
.json()
.with_context(|| format!("invalid JSON from {url}"))
}
fn get(&self, url: &str) -> Result<Response> {
self.request_headers(self.http.get(url))?
.send()
.with_context(|| format!("GET {url} failed"))
.and_then(|response| check_response("GET", url, response))
}
fn request_headers(
&self,
request: reqwest::blocking::RequestBuilder,
) -> Result<reqwest::blocking::RequestBuilder> {
let mut headers = HeaderMap::new();
headers.insert(USER_AGENT, HeaderValue::from_static("git-sync/0.1"));
headers.insert(ACCEPT, HeaderValue::from_static("application/json"));
match self.site.provider {
ProviderKind::Github => {
headers.insert(
AUTHORIZATION,
HeaderValue::from_str(&format!("Bearer {}", self.token))
.context("PAT contains invalid header characters")?,
);
headers.insert(
"X-GitHub-Api-Version",
HeaderValue::from_static("2022-11-28"),
);
}
ProviderKind::Gitlab => {
headers.insert(
"PRIVATE-TOKEN",
HeaderValue::from_str(&self.token)
.context("PAT contains invalid header characters")?,
);
}
ProviderKind::Gitea => {
headers.insert(
AUTHORIZATION,
HeaderValue::from_str(&format!("token {}", self.token))
.context("PAT contains invalid header characters")?,
);
}
}
Ok(request.headers(headers))
}
}
fn check_response(method: &str, url: &str, response: Response) -> Result<Response> {
if response.status().is_success() {
return Ok(response);
}
let status = response.status();
let body = response.text().unwrap_or_default();
bail!("{method} {url} returned {status}: {body}");
}
fn next_link(headers: &HeaderMap) -> Option<String> {
let header = headers.get("link")?.to_str().ok()?;
for part in header.split(',') {
let mut sections = part.trim().split(';');
let url = sections.next()?.trim();
let rel = sections.any(|section| section.trim() == "rel=\"next\"");
if rel {
return url
.strip_prefix('<')
.and_then(|value| value.strip_suffix('>'))
.map(ToString::to_string);
}
}
None
}
fn urlencoding(value: &str) -> String {
url::form_urlencoded::byte_serialize(value.as_bytes()).collect()
}
#[derive(Deserialize)]
struct GithubRepo {
name: String,
clone_url: String,
private: bool,
description: Option<String>,
owner: GithubOwner,
}
#[derive(Deserialize)]
struct GithubOwner {
login: String,
}
impl From<GithubRepo> for RemoteRepo {
fn from(value: GithubRepo) -> Self {
Self {
name: value.name,
clone_url: value.clone_url,
private: value.private,
description: value.description,
}
}
}
#[derive(Deserialize)]
struct GitlabProject {
name: String,
path: Option<String>,
http_url_to_repo: String,
visibility: String,
description: Option<String>,
}
impl From<GitlabProject> for RemoteRepo {
fn from(value: GitlabProject) -> Self {
Self {
name: value.path.unwrap_or(value.name),
clone_url: value.http_url_to_repo,
private: value.visibility != "public",
description: value.description,
}
}
}
#[derive(Deserialize)]
struct GitlabGroup {
id: u64,
}
#[derive(Deserialize)]
struct GiteaRepo {
name: String,
clone_url: String,
private: bool,
description: Option<String>,
owner: GiteaOwner,
}
#[derive(Deserialize)]
struct GiteaOwner {
login: String,
}
impl From<GiteaRepo> for RemoteRepo {
fn from(value: GiteaRepo) -> Self {
Self {
name: value.name,
clone_url: value.clone_url,
private: value.private,
description: value.description,
}
}
}
pub fn repos_by_name(repos: Vec<EndpointRepo>) -> HashMap<String, Vec<EndpointRepo>> {
let mut output: HashMap<String, Vec<EndpointRepo>> = HashMap::new();
for repo in repos {
output.entry(repo.repo.name.clone()).or_default().push(repo);
}
output
}
#[cfg(test)]
mod tests {
use super::*;
use crate::config::TokenConfig;
#[test]
fn extracts_next_link() {
let mut headers = HeaderMap::new();
headers.insert(
"link",
HeaderValue::from_static("<https://example.test?page=2>; rel=\"next\", <https://example.test?page=5>; rel=\"last\""),
);
assert_eq!(next_link(&headers).unwrap(), "https://example.test?page=2");
}
#[test]
fn authenticated_clone_urls_use_provider_defaults() {
let github_site = site(ProviderKind::Github, None);
let github = ProviderClient::new(&github_site).unwrap();
assert_eq!(
github
.authenticated_clone_url("https://github.com/alice/repo.git")
.unwrap(),
"https://x-access-token:secret@github.com/alice/repo.git"
);
let gitlab_site = site(ProviderKind::Gitlab, None);
let gitlab = ProviderClient::new(&gitlab_site).unwrap();
assert_eq!(
gitlab
.authenticated_clone_url("https://gitlab.example.test/alice/repo.git")
.unwrap(),
"https://oauth2:secret@gitlab.example.test/alice/repo.git"
);
}
#[test]
fn authenticated_clone_urls_can_override_git_username() {
let gitea_site = site(ProviderKind::Gitea, Some("mirror-user".to_string()));
let client = ProviderClient::new(&gitea_site).unwrap();
assert_eq!(
client
.authenticated_clone_url("https://gitea.example.test/alice/repo.git")
.unwrap(),
"https://mirror-user:secret@gitea.example.test/alice/repo.git"
);
}
#[test]
fn group_paths_are_url_encoded_for_gitlab() {
assert_eq!(urlencoding("parent/child group"), "parent%2Fchild+group");
}
fn site(provider: ProviderKind, git_username: Option<String>) -> SiteConfig {
SiteConfig {
name: "site".to_string(),
provider,
base_url: "https://example.test".to_string(),
api_url: None,
token: TokenConfig::Value("secret".to_string()),
git_username,
}
}
}
src/sync.rs
+296
View File
@@ -0,0 +1,296 @@
use std::collections::{BTreeSet, HashMap};
use std::fs;
use std::path::{Path, PathBuf};
use anyhow::{Context, Result, bail};
use crate::config::{Config, EndpointConfig, MirrorConfig, default_work_dir, validate_config};
use crate::git::{GitMirror, Redactor, RemoteSpec, safe_remote_name};
use crate::provider::{EndpointRepo, ProviderClient, repos_by_name};
#[derive(Clone, Debug, Default)]
pub struct SyncOptions {
pub group: Option<String>,
pub dry_run: bool,
pub create_missing_override: Option<bool>,
pub force_override: Option<bool>,
pub work_dir: Option<PathBuf>,
}
pub fn sync_all(config: &Config, options: SyncOptions) -> Result<()> {
validate_config(config)?;
let work_dir = options.work_dir.clone().unwrap_or_else(default_work_dir);
fs::create_dir_all(&work_dir)
.with_context(|| format!("failed to create {}", work_dir.display()))?;
let mirrors = config
.mirrors
.iter()
.filter(|mirror| {
options
.group
.as_ref()
.is_none_or(|name| mirror.name == *name)
})
.collect::<Vec<_>>();
if mirrors.is_empty() {
bail!("no mirror group matched");
}
let tokens = config
.sites
.iter()
.map(|site| site.token())
.collect::<Result<Vec<_>>>()?;
let redactor = Redactor::new(tokens);
for mirror in mirrors {
sync_group(config, mirror, &options, &work_dir, redactor.clone())?;
}
Ok(())
}
fn sync_group(
config: &Config,
mirror: &MirrorConfig,
options: &SyncOptions,
work_dir: &Path,
redactor: Redactor,
) -> Result<()> {
println!("syncing mirror group {}", mirror.name);
let create_missing = options
.create_missing_override
.unwrap_or(mirror.create_missing);
let allow_force = options.force_override.unwrap_or(mirror.allow_force);
let mut all_endpoint_repos = Vec::new();
for endpoint in &mirror.endpoints {
let site = config.site(&endpoint.site).unwrap();
let client = ProviderClient::new(site)?;
println!("listing {}", endpoint.label());
let repos = client
.list_repos(endpoint)
.with_context(|| format!("failed to list repos for {}", endpoint.label()))?;
for repo in repos {
all_endpoint_repos.push(EndpointRepo {
endpoint: endpoint.clone(),
repo,
});
}
}
let mut repos = repos_by_name(all_endpoint_repos);
let repo_names = repos.keys().cloned().collect::<BTreeSet<_>>();
if repo_names.is_empty() {
println!("mirror group {} has no repositories", mirror.name);
return Ok(());
}
for repo_name in repo_names {
let mut existing = repos.remove(&repo_name).unwrap_or_default();
ensure_missing_repos(
config,
mirror,
&repo_name,
&mut existing,
create_missing,
options.dry_run,
)?;
if existing.len() < 2 {
println!(
"skipping {}: fewer than two endpoints have this repository",
repo_name
);
continue;
}
let context = RepoSyncContext {
config,
mirror,
work_dir,
redactor: redactor.clone(),
dry_run: options.dry_run,
allow_force,
};
sync_repo(&context, &repo_name, &existing)?;
}
Ok(())
}
fn ensure_missing_repos(
config: &Config,
mirror: &MirrorConfig,
repo_name: &str,
existing: &mut Vec<EndpointRepo>,
create_missing: bool,
dry_run: bool,
) -> Result<()> {
let present = existing
.iter()
.map(|repo| repo.endpoint.clone())
.collect::<BTreeSet<_>>();
let template = existing.first().map(|repo| repo.repo.clone());
for endpoint in &mirror.endpoints {
if present.contains(endpoint) {
continue;
}
if !create_missing {
println!(
"{} is missing on {}; creation disabled",
repo_name,
endpoint.label()
);
continue;
}
println!("creating {} on {}", repo_name, endpoint.label());
if dry_run {
continue;
}
let site = config.site(&endpoint.site).unwrap();
let client = ProviderClient::new(site)?;
let created = client
.create_repo(
endpoint,
repo_name,
&mirror.visibility,
template
.as_ref()
.and_then(|repo| repo.description.as_deref()),
)
.with_context(|| format!("failed to create {} on {}", repo_name, endpoint.label()))?;
if created.private != matches!(mirror.visibility, crate::config::Visibility::Private) {
println!(
"created {} on {}, but provider reported a different visibility than requested",
repo_name,
endpoint.label()
);
}
existing.push(EndpointRepo {
endpoint: endpoint.clone(),
repo: created,
});
}
Ok(())
}
struct RepoSyncContext<'a> {
config: &'a Config,
mirror: &'a MirrorConfig,
work_dir: &'a Path,
redactor: Redactor,
dry_run: bool,
allow_force: bool,
}
fn sync_repo(context: &RepoSyncContext<'_>, repo_name: &str, repos: &[EndpointRepo]) -> Result<()> {
println!("syncing repo {}", repo_name);
let endpoint_map = context
.mirror
.endpoints
.iter()
.map(|endpoint| (endpoint.clone(), endpoint))
.collect::<HashMap<EndpointConfig, &EndpointConfig>>();
let mut remotes = Vec::new();
for endpoint_repo in repos {
if !endpoint_map.contains_key(&endpoint_repo.endpoint) {
continue;
}
let site = context.config.site(&endpoint_repo.endpoint.site).unwrap();
let client = ProviderClient::new(site)?;
let remote_name = safe_remote_name(&format!(
"{}_{}",
endpoint_repo.endpoint.site, endpoint_repo.endpoint.namespace
));
remotes.push(RemoteSpec {
name: remote_name,
url: client.authenticated_clone_url(&endpoint_repo.repo.clone_url)?,
display: endpoint_repo.endpoint.label(),
});
}
let path = context
.work_dir
.join(safe_remote_name(&context.mirror.name))
.join(format!("{}.git", safe_remote_name(repo_name)));
let mirror_repo = GitMirror::open(path, context.redactor.clone(), context.dry_run)?;
mirror_repo.configure_remotes(&remotes)?;
for remote in &remotes {
mirror_repo.fetch_remote(remote)?;
}
let (branches, conflicts) = mirror_repo.branch_decisions(&remotes, context.allow_force)?;
for conflict in conflicts {
let details = conflict
.tips
.iter()
.map(|(remote, sha)| format!("{remote}@{}", short_sha(sha)))
.collect::<Vec<_>>()
.join(", ");
println!(
"conflict in {}/{}: branch {} diverged across {}. Skipping that branch.",
context.mirror.name, repo_name, conflict.branch, details
);
}
let (tags, tag_conflicts) = mirror_repo.tag_decisions(&remotes)?;
for conflict in tag_conflicts {
let details = conflict
.tips
.iter()
.map(|(remote, sha)| format!("{remote}@{}", short_sha(sha)))
.collect::<Vec<_>>()
.join(", ");
println!(
"conflict in {}/{}: tag {} differs across {}. Skipping that tag.",
context.mirror.name, repo_name, conflict.tag, details
);
}
if branches.is_empty() && tags.is_empty() {
println!("{} has no branches or tags to push", repo_name);
return Ok(());
}
if !branches.is_empty() {
let branch_summary = branches
.iter()
.map(|branch| {
format!(
"{}@{} from {}",
branch.branch,
short_sha(&branch.sha),
branch.source_remotes.join("+")
)
})
.collect::<Vec<_>>()
.join(", ");
println!("resolved branches for {}: {}", repo_name, branch_summary);
mirror_repo.push_branches(&remotes, &branches, context.allow_force)?;
}
if !tags.is_empty() {
let tag_summary = tags
.iter()
.map(|tag| {
format!(
"{}@{} from {}",
tag.tag,
short_sha(&tag.sha),
tag.source_remotes.join("+")
)
})
.collect::<Vec<_>>()
.join(", ");
println!("resolved tags for {}: {}", repo_name, tag_summary);
mirror_repo.push_tags(&remotes, &tags)?;
}
Ok(())
}
fn short_sha(sha: &str) -> &str {
sha.get(..12).unwrap_or(sha)
}