diff --git a/README.md b/README.md
index c3cd33e..67a6748 100644
--- a/README.md
+++ b/README.md
@@ -52,6 +52,8 @@ refray config
Example Config
```toml
+jobs = 8
+
[[sites]]
name = "github"
provider = "github"
@@ -123,10 +125,10 @@ Retry only repositories that failed during the previous non-dry-run sync:
refray sync --retry-failed
```
-Control repo-level parallelism:
+Control parallelism for sync, serve, and webhook commands in config:
-```sh
-refray sync --jobs 8
+```toml
+jobs = 8
```
@@ -139,18 +141,17 @@ You can run `refray` as a service that listens for webhook events and runs full
> If you want to use webhooks, you need to expose port 8787 to a public URL that can be accessed by the git provider.
> This can be done using e.g. port forwarding, reverse proxy, cloudflare tunnel, or tailscale funnel.
-Start the service:
+Start the service (to sync on push and also do full sync periodically):
```sh
refray serve
```
-Install webhooks on all repos:
+Install webhooks on all repos (with the URL in config):
```sh
refray webhook install
```
-Webhook install uses `[webhook].url` and `[webhook].secret` from your config.
To uninstall webhooks previously installed by `refray`:
@@ -161,20 +162,18 @@ To uninstall webhooks previously installed by `refray`:
refray webhook uninstall
```
+By default, uninstall uses `[webhook].url` from your config. To remove hooks for a previous URL, pass it explicitly:
+
+```sh
+refray webhook uninstall https://old.example.com/webhook
+```
+
To move installed hooks to a new public URL, use `webhook update`. It removes hooks matching the current configured `[webhook].url`, installs the new URL, updates `[webhook].url` in the config, and refreshes local webhook state:
```sh
refray webhook update https://new.example.com/webhook
```
-Serve can also run periodic full syncs. The interval can be configured in `[webhook].full_sync_interval_minutes` or overridden at startup:
-
-```sh
-refray serve --full-sync-interval-minutes 30
-```
-
-If `[webhook].reachability_check_interval_minutes` is configured, `serve` periodically checks that the public webhook URL is still reachable and logs a warning when it is not.
-
## Sync Semantics
Each mirror group is treated as a set of equivalent namespaces. Repositories are matched by repository name across all endpoints.
diff --git a/src/config.rs b/src/config.rs
index d3a6f03..8c6c012 100644
--- a/src/config.rs
+++ b/src/config.rs
@@ -9,9 +9,12 @@ use regex::Regex;
use serde::{Deserialize, Serialize};
const APP_NAME: &str = "refray";
+pub const DEFAULT_JOBS: usize = 4;
-#[derive(Clone, Debug, Default, Deserialize, Serialize)]
+#[derive(Clone, Debug, Deserialize, Serialize)]
pub struct Config {
+ #[serde(default = "default_jobs", skip_serializing_if = "is_default_jobs")]
+ pub jobs: usize,
#[serde(default)]
pub sites: Vec,
#[serde(default)]
@@ -90,6 +93,17 @@ pub struct WebhookConfig {
pub reachability_check_interval_minutes: Option,
}
+impl Default for Config {
+ fn default() -> Self {
+ Self {
+ jobs: DEFAULT_JOBS,
+ sites: Vec::new(),
+ mirrors: Vec::new(),
+ webhook: None,
+ }
+ }
+}
+
#[derive(Clone, Debug, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)]
pub struct EndpointConfig {
pub site: String,
@@ -176,6 +190,14 @@ fn default_true() -> bool {
true
}
+fn default_jobs() -> usize {
+ DEFAULT_JOBS
+}
+
+fn is_default_jobs(jobs: &usize) -> bool {
+ *jobs == DEFAULT_JOBS
+}
+
impl Config {
pub fn load(path: &Path) -> Result {
let contents = fs::read_to_string(path)
@@ -321,6 +343,9 @@ fn protect_file(_path: &Path) -> Result<()> {
}
pub fn validate_config(config: &Config) -> Result<()> {
+ if config.jobs == 0 {
+ bail!("jobs must be at least 1");
+ }
if config.sites.is_empty() {
bail!("no sites configured");
}
diff --git a/src/main.rs b/src/main.rs
index 06c3ca3..643334d 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -7,14 +7,13 @@ mod state;
mod sync;
mod webhook;
-use std::env;
use std::path::{Path, PathBuf};
use anyhow::{Context, Result};
use clap::{Args, Parser, Subcommand};
use crate::config::{Config, default_config_path};
-use crate::sync::{DEFAULT_JOBS, SyncOptions, sync_all};
+use crate::sync::{SyncOptions, sync_all};
use crate::webhook::{
ServeOptions, WebhookInstallOptions, WebhookUninstallOptions, WebhookUpdateOptions,
install_webhooks, serve, uninstall_webhooks, update_webhooks,
@@ -60,24 +59,14 @@ struct SyncCommand {
retry_failed: bool,
#[arg(long, value_name = "PATH")]
work_dir: Option,
- #[arg(long, default_value_t = DEFAULT_JOBS, value_name = "N")]
- jobs: usize,
}
#[derive(Args, Debug)]
struct ServeCommand {
#[arg(long, default_value = "127.0.0.1:8787", value_name = "HOST:PORT")]
listen: String,
- #[arg(long, conflicts_with = "secret_env")]
- secret: Option,
- #[arg(long, value_name = "ENV", conflicts_with = "secret")]
- secret_env: Option,
- #[arg(long, default_value_t = DEFAULT_JOBS, value_name = "N")]
- jobs: usize,
#[arg(long, value_name = "PATH")]
work_dir: Option,
- #[arg(long, value_name = "MINUTES")]
- full_sync_interval_minutes: Option,
}
#[derive(Subcommand, Debug)]
@@ -91,16 +80,14 @@ enum WebhookCommand {
struct WebhookInstallCommand {
#[arg(long)]
dry_run: bool,
- #[arg(long, default_value_t = DEFAULT_JOBS, value_name = "N")]
- jobs: usize,
}
#[derive(Args, Debug)]
struct WebhookUninstallCommand {
+ #[arg(value_name = "URL", value_parser = parse_webhook_url)]
+ url: Option,
#[arg(long)]
dry_run: bool,
- #[arg(long, default_value_t = DEFAULT_JOBS, value_name = "N")]
- jobs: usize,
}
#[derive(Args, Debug)]
@@ -111,8 +98,6 @@ struct WebhookUpdateCommand {
dry_run: bool,
#[arg(long, value_name = "PATH")]
work_dir: Option,
- #[arg(long, default_value_t = DEFAULT_JOBS, value_name = "N")]
- jobs: usize,
}
fn main() -> Result<()> {
@@ -123,7 +108,13 @@ fn main() -> Result<()> {
Command::Config => {
let outcome = interactive::run_config_wizard(&config_path)?;
if outcome.run_full_sync_now {
- sync_all(&outcome.config, SyncOptions::default())
+ sync_all(
+ &outcome.config,
+ SyncOptions {
+ jobs: outcome.config.jobs,
+ ..SyncOptions::default()
+ },
+ )
} else {
Ok(())
}
@@ -140,30 +131,29 @@ fn main() -> Result<()> {
repo_pattern: command.repo_pattern,
retry_failed: command.retry_failed,
work_dir: command.work_dir,
- jobs: command.jobs,
+ jobs: config.jobs,
},
)
}
Command::Serve(command) => {
let config = load_config(&config_path)?;
- let full_sync_interval_minutes = command.full_sync_interval_minutes.or_else(|| {
- config
- .webhook
- .as_ref()
- .and_then(|webhook| webhook.full_sync_interval_minutes)
- });
+ let full_sync_interval_minutes = config
+ .webhook
+ .as_ref()
+ .and_then(|webhook| webhook.full_sync_interval_minutes);
let reachability_url = config.webhook.as_ref().map(|webhook| webhook.url.clone());
let reachability_check_interval_minutes = config
.webhook
.as_ref()
.and_then(|webhook| webhook.reachability_check_interval_minutes);
- let secret = resolve_webhook_secret(&config, command.secret, command.secret_env)?;
+ let secret = resolve_config_webhook_secret(&config)?;
+ let workers = config.jobs;
serve(
config,
ServeOptions {
listen: command.listen,
secret,
- workers: command.jobs,
+ workers,
work_dir: command.work_dir,
full_sync_interval_minutes,
reachability_url,
@@ -182,20 +172,20 @@ fn main() -> Result<()> {
secret,
dry_run: command.dry_run,
work_dir: None,
- jobs: command.jobs,
+ jobs: config.jobs,
},
)
}
Command::Webhook(WebhookCommand::Uninstall(command)) => {
let config = load_config(&config_path)?;
- let url = resolve_config_webhook_url(&config)?;
+ let url = resolve_uninstall_webhook_url(&config, command.url)?;
uninstall_webhooks(
&config,
WebhookUninstallOptions {
url,
dry_run: command.dry_run,
work_dir: None,
- jobs: command.jobs,
+ jobs: config.jobs,
},
)
}
@@ -217,7 +207,7 @@ fn main() -> Result<()> {
secret,
dry_run: command.dry_run,
work_dir: command.work_dir,
- jobs: command.jobs,
+ jobs: config.jobs,
},
)?;
if !command.dry_run {
@@ -246,29 +236,10 @@ fn resolve_config_webhook_secret(config: &Config) -> Result {
.map(|webhook| webhook.secret())
.transpose()?
.ok_or_else(|| {
- anyhow::anyhow!("configure [webhook].secret before running webhook commands")
+ anyhow::anyhow!("configure [webhook].secret before running serve or webhook commands")
})
}
-fn resolve_webhook_secret(
- config: &Config,
- value: Option,
- env_name: Option,
-) -> Result {
- match (value, env_name) {
- (Some(value), None) => Ok(value),
- (None, Some(env_name)) => env::var(&env_name)
- .with_context(|| format!("environment variable {env_name} is not set")),
- (None, None) => config
- .webhook
- .as_ref()
- .map(|webhook| webhook.secret())
- .transpose()?
- .ok_or_else(|| anyhow::anyhow!("pass either --secret or --secret-env")),
- (Some(_), Some(_)) => unreachable!("clap enforces secret conflicts"),
- }
-}
-
fn resolve_config_webhook_url(config: &Config) -> Result {
config
.webhook
@@ -277,6 +248,24 @@ fn resolve_config_webhook_url(config: &Config) -> Result {
.ok_or_else(|| anyhow::anyhow!("configure [webhook].url before running webhook commands"))
}
+fn resolve_uninstall_webhook_url(config: &Config, url: Option) -> Result {
+ match url {
+ Some(url) => Ok(url),
+ None => resolve_config_webhook_url(config),
+ }
+}
+
+fn parse_webhook_url(value: &str) -> std::result::Result {
+ if value.trim().is_empty() {
+ return Err("A value is required".to_string());
+ }
+ let url = url::Url::parse(value).map_err(|error| format!("Invalid URL: {error}"))?;
+ match url.scheme() {
+ "http" | "https" => Ok(value.to_string()),
+ _ => Err("URL must start with http:// or https://".to_string()),
+ }
+}
+
fn set_config_webhook_url(config: &mut Config, url: String) {
let Some(webhook) = &mut config.webhook else {
unreachable!("caller verifies webhook config exists before saving update")
diff --git a/src/sync.rs b/src/sync.rs
index 280c642..002accf 100644
--- a/src/sync.rs
+++ b/src/sync.rs
@@ -9,7 +9,7 @@ use console::style;
use regex::Regex;
use crate::config::{
- Config, ConflictResolutionStrategy, EndpointConfig, MirrorConfig, RepoNameFilter,
+ Config, ConflictResolutionStrategy, DEFAULT_JOBS, EndpointConfig, MirrorConfig, RepoNameFilter,
SyncVisibility, default_work_dir, validate_config,
};
use crate::git::{
@@ -34,7 +34,6 @@ use self::state::{
save_failure_state, save_ref_state,
};
-pub const DEFAULT_JOBS: usize = 4;
const CONFLICT_BRANCH_ROOT: &str = "refray/conflicts/";
#[derive(Clone, Debug)]
@@ -67,7 +66,7 @@ impl Default for SyncOptions {
pub fn sync_all(config: &Config, options: SyncOptions) -> Result<()> {
validate_config(config)?;
if options.jobs == 0 {
- bail!("--jobs must be at least 1");
+ bail!("jobs must be at least 1");
}
let work_dir = options.work_dir.clone().unwrap_or_else(default_work_dir);
fs::create_dir_all(&work_dir)
diff --git a/src/webhook.rs b/src/webhook.rs
index 1034687..ec91baa 100644
--- a/src/webhook.rs
+++ b/src/webhook.rs
@@ -77,7 +77,7 @@ struct JobQueue {
pub fn serve(config: Config, options: ServeOptions) -> Result<()> {
validate_config(&config)?;
if options.workers == 0 {
- bail!("--jobs must be at least 1");
+ bail!("jobs must be at least 1");
}
let server = Server::http(&options.listen)
.map_err(|error| anyhow::anyhow!("failed to listen on {}: {error}", options.listen))?;
@@ -174,7 +174,7 @@ fn reachability_timer_loop(url: String, minutes: u64) {
pub fn install_webhooks(config: &Config, options: WebhookInstallOptions) -> Result<()> {
validate_config(config)?;
if options.jobs == 0 {
- bail!("--jobs must be at least 1");
+ bail!("jobs must be at least 1");
}
let work_dir = options.work_dir.clone().unwrap_or_else(default_work_dir);
let state = Arc::new(Mutex::new(load_webhook_state(&work_dir)?));
@@ -229,7 +229,7 @@ pub fn install_webhooks(config: &Config, options: WebhookInstallOptions) -> Resu
pub fn uninstall_webhooks(config: &Config, options: WebhookUninstallOptions) -> Result<()> {
validate_config(config)?;
if options.jobs == 0 {
- bail!("--jobs must be at least 1");
+ bail!("jobs must be at least 1");
}
let work_dir = options.work_dir.clone().unwrap_or_else(default_work_dir);
let mut state = load_webhook_state(&work_dir)?;
@@ -267,10 +267,7 @@ pub fn uninstall_webhooks(config: &Config, options: WebhookUninstallOptions) ->
let removed_keys = run_uninstall_tasks(tasks, options.jobs)?;
if !options.dry_run {
- for key in removed_keys {
- state.installations.remove(&key);
- state.skipped.remove(&key);
- }
+ remove_webhook_state_keys(&mut state, removed_keys, &options.url);
save_webhook_state(&work_dir, &state)?;
}
Ok(())
@@ -279,7 +276,7 @@ pub fn uninstall_webhooks(config: &Config, options: WebhookUninstallOptions) ->
pub fn update_webhooks(config: &Config, options: WebhookUpdateOptions) -> Result<()> {
validate_config(config)?;
if options.jobs == 0 {
- bail!("--jobs must be at least 1");
+ bail!("jobs must be at least 1");
}
if options.old_url != options.new_url {
crate::logln!(
@@ -325,7 +322,7 @@ pub fn ensure_configured_webhooks(
return Ok(());
}
if jobs == 0 {
- bail!("--jobs must be at least 1");
+ bail!("jobs must be at least 1");
}
let secret = webhook.secret()?;
let state = Arc::new(Mutex::new(load_webhook_state(work_dir)?));
@@ -696,6 +693,25 @@ fn record_webhook_installation(
);
}
+fn remove_webhook_state_keys(state: &mut WebhookState, keys: Vec, url: &str) {
+ for key in keys {
+ if state
+ .installations
+ .get(&key)
+ .is_some_and(|installation| installation.url == url)
+ {
+ state.installations.remove(&key);
+ }
+ if state
+ .skipped
+ .get(&key)
+ .is_some_and(|skipped| skipped.url == url)
+ {
+ state.skipped.remove(&key);
+ }
+ }
+}
+
fn uninstall_webhook_task(task: WebhookUninstallTask) -> Result