diff --git a/README.md b/README.md index 320a737..253aff0 100644 --- a/README.md +++ b/README.md @@ -506,6 +506,8 @@ cooldown: default: "3d" ``` +See the [configuration reference](docs/configuration.md#upstream-registries) for every upstream key, environment variable, and default URL. + Run with config file: ```bash diff --git a/config.example.yaml b/config.example.yaml index 1df95b3..f0a08f2 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -82,26 +82,74 @@ log: access_log: path: "" -# Upstream registry URLs and authentication +# Upstream URLs for built-in routes and authentication upstream: # npm registry URL npm: "https://registry.npmjs.org" + # Cargo sparse index URL + cargo: "https://index.crates.io" + + # Cargo crate download URL + cargo_download: "https://static.crates.io/crates" + + # RubyGems registry URL + gem: "https://rubygems.org" + + # Go module proxy URL + go: "https://proxy.golang.org" + + # Hex repository URL + hex: "https://repo.hex.pm" + + # Hex API URL used for package timestamps + hex_api: "https://hex.pm" + + # pub registry URL + pub: "https://pub.dev" + + # PyPI index and API URL + pypi: "https://pypi.org" + + # PyPI package download URL + pypi_download: "https://files.pythonhosted.org" + # Maven repository URL (used by /maven endpoint) maven: "https://repo1.maven.org/maven2" # Gradle Plugin Portal Maven URL (fallback for plugin marker artifacts) gradle_plugin_portal: "https://plugins.gradle.org/m2" - # Cargo sparse index URL - cargo: "https://index.crates.io" + # NuGet API URL + nuget: "https://api.nuget.org" - # Cargo crate download URL - cargo_download: "https://static.crates.io/crates" + # NuGet search API URL + nuget_search: "https://azuresearch-usnc.nuget.org" + + # Packagist API URL + composer: "https://packagist.org" + + # Packagist repository URL + composer_repository: "https://repo.packagist.org" + + # Conan registry URL + conan: "https://center.conan.io" + + # Conda channel base URL + conda: "https://conda.anaconda.org" + + # CRAN mirror URL + cran: "https://cloud.r-project.org" + + # Julia package server URL + julia: "https://pkg.julialang.org" # Debian/APT repository URL (used by /debian endpoint) debian: "http://deb.debian.org/debian" + # RPM repository URL (used by /rpm endpoint) + rpm: "https://dl.fedoraproject.org/pub/fedora/linux" + # Named HTTP Helm chart repositories (used by /helm/{name}/) # helm: # bitnami: "https://charts.bitnami.com/bitnami" diff --git a/docs/configuration.md b/docs/configuration.md index 3b8b935..e3fcdd7 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -134,16 +134,51 @@ Upstream retries and OCI authentication calls are separate `upstream` records, s ## Upstream Registries -Override default upstream registry URLs: +Each upstream used by a built-in package route can be set in YAML or JSON under `upstream`, or with its matching environment variable. Existing installations keep the same public upstreams by default. Trailing slashes are ignored. + +| Config | Environment | Default | +|--------|-------------|---------| +| `upstream.npm` | `PROXY_UPSTREAM_NPM` | `https://registry.npmjs.org` | +| `upstream.cargo` | `PROXY_UPSTREAM_CARGO` | `https://index.crates.io` | +| `upstream.cargo_download` | `PROXY_UPSTREAM_CARGO_DOWNLOAD` | `https://static.crates.io/crates` | +| `upstream.gem` | `PROXY_UPSTREAM_GEM` | `https://rubygems.org` | +| `upstream.go` | `PROXY_UPSTREAM_GO` | `https://proxy.golang.org` | +| `upstream.hex` | `PROXY_UPSTREAM_HEX` | `https://repo.hex.pm` | +| `upstream.hex_api` | `PROXY_UPSTREAM_HEX_API` | `https://hex.pm` | +| `upstream.pub` | `PROXY_UPSTREAM_PUB` | `https://pub.dev` | +| `upstream.pypi` | `PROXY_UPSTREAM_PYPI` | `https://pypi.org` | +| `upstream.pypi_download` | `PROXY_UPSTREAM_PYPI_DOWNLOAD` | `https://files.pythonhosted.org` | +| `upstream.maven` | `PROXY_UPSTREAM_MAVEN` | `https://repo1.maven.org/maven2` | +| `upstream.gradle_plugin_portal` | `PROXY_UPSTREAM_GRADLE_PLUGIN_PORTAL` | `https://plugins.gradle.org/m2` | +| `upstream.nuget` | `PROXY_UPSTREAM_NUGET` | `https://api.nuget.org` | +| `upstream.nuget_search` | `PROXY_UPSTREAM_NUGET_SEARCH` | `https://azuresearch-usnc.nuget.org` | +| `upstream.composer` | `PROXY_UPSTREAM_COMPOSER` | `https://packagist.org` | +| `upstream.composer_repository` | `PROXY_UPSTREAM_COMPOSER_REPOSITORY` | `https://repo.packagist.org` | +| `upstream.conan` | `PROXY_UPSTREAM_CONAN` | `https://center.conan.io` | +| `upstream.conda` | `PROXY_UPSTREAM_CONDA` | `https://conda.anaconda.org` | +| `upstream.cran` | `PROXY_UPSTREAM_CRAN` | `https://cloud.r-project.org` | +| `upstream.julia` | `PROXY_UPSTREAM_JULIA` | `https://pkg.julialang.org` | +| `upstream.debian` | `PROXY_UPSTREAM_DEBIAN` | `http://deb.debian.org/debian` | +| `upstream.rpm` | `PROXY_UPSTREAM_RPM` | `https://dl.fedoraproject.org/pub/fedora/linux` | + +For protocols that use separate metadata and download services, configure both values. They may point to the same endpoint when chaining proxies: ```yaml upstream: - npm: "https://registry.npmjs.org" - maven: "https://repo1.maven.org/maven2" - gradle_plugin_portal: "https://plugins.gradle.org/m2" - cargo: "https://index.crates.io" - cargo_download: "https://static.crates.io/crates" + pypi: "https://upstream-proxy.example.com/pypi" + pypi_download: "https://upstream-proxy.example.com/pypi" + nuget: "https://upstream-proxy.example.com/nuget" + nuget_search: "https://upstream-proxy.example.com/nuget" + composer: "https://upstream-proxy.example.com/composer" + composer_repository: "https://upstream-proxy.example.com/composer" +``` + +`upstream.hex_api` is used for cooldown timestamps and must expose Hex's `/api/packages/{name}` JSON endpoint. + +Helm HTTP repositories and additional OCI registries are configured as named maps rather than single URLs: +```yaml +upstream: # Named HTTP Helm chart repositories, served at /helm/{name}/. helm: bitnami: "https://charts.bitnami.com/bitnami" diff --git a/internal/config/config.go b/internal/config/config.go index fdf890d..ca6fdb6 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -289,13 +289,49 @@ type AccessLogConfig struct { Path string `json:"path" yaml:"path"` } -// UpstreamConfig configures upstream registry URLs and authentication. +// UpstreamConfig configures upstream URLs for built-in routes and authentication. // Leave empty to use defaults. type UpstreamConfig struct { // NPM is the upstream npm registry URL. // Default: https://registry.npmjs.org NPM string `json:"npm" yaml:"npm"` + // Cargo is the upstream cargo index URL. + // Default: https://index.crates.io + Cargo string `json:"cargo" yaml:"cargo"` + + // CargoDownload is the upstream cargo download URL. + // Default: https://static.crates.io/crates + CargoDownload string `json:"cargo_download" yaml:"cargo_download"` + + // Gem is the upstream RubyGems registry URL. + // Default: https://rubygems.org + Gem string `json:"gem" yaml:"gem"` + + // Go is the upstream Go module proxy URL. + // Default: https://proxy.golang.org + Go string `json:"go" yaml:"go"` + + // Hex is the upstream Hex repository URL. + // Default: https://repo.hex.pm + Hex string `json:"hex" yaml:"hex"` + + // HexAPI is the upstream Hex API URL used for package timestamps. + // Default: https://hex.pm + HexAPI string `json:"hex_api" yaml:"hex_api"` + + // Pub is the upstream pub registry URL. + // Default: https://pub.dev + Pub string `json:"pub" yaml:"pub"` + + // PyPI is the upstream PyPI index and API URL. + // Default: https://pypi.org + PyPI string `json:"pypi" yaml:"pypi"` + + // PyPIDownload is the upstream PyPI package download URL. + // Default: https://files.pythonhosted.org + PyPIDownload string `json:"pypi_download" yaml:"pypi_download"` + // Maven is the upstream Maven repository URL. // Default: https://repo1.maven.org/maven2 Maven string `json:"maven" yaml:"maven"` @@ -305,19 +341,47 @@ type UpstreamConfig struct { // Default: https://plugins.gradle.org/m2 GradlePluginPortal string `json:"gradle_plugin_portal" yaml:"gradle_plugin_portal"` - // Cargo is the upstream cargo index URL. - // Default: https://index.crates.io - Cargo string `json:"cargo" yaml:"cargo"` + // NuGet is the upstream NuGet API URL. + // Default: https://api.nuget.org + NuGet string `json:"nuget" yaml:"nuget"` - // CargoDownload is the upstream cargo download URL. - // Default: https://static.crates.io/crates - CargoDownload string `json:"cargo_download" yaml:"cargo_download"` + // NuGetSearch is the upstream NuGet search API URL. + // Default: https://azuresearch-usnc.nuget.org + NuGetSearch string `json:"nuget_search" yaml:"nuget_search"` + + // Composer is the upstream Packagist API URL. + // Default: https://packagist.org + Composer string `json:"composer" yaml:"composer"` + + // ComposerRepository is the upstream Packagist repository URL. + // Default: https://repo.packagist.org + ComposerRepository string `json:"composer_repository" yaml:"composer_repository"` + + // Conan is the upstream Conan registry URL. + // Default: https://center.conan.io + Conan string `json:"conan" yaml:"conan"` + + // Conda is the upstream Conda channel base URL. + // Default: https://conda.anaconda.org + Conda string `json:"conda" yaml:"conda"` + + // CRAN is the upstream CRAN mirror URL. + // Default: https://cloud.r-project.org + CRAN string `json:"cran" yaml:"cran"` + + // Julia is the upstream Julia package server URL. + // Default: https://pkg.julialang.org + Julia string `json:"julia" yaml:"julia"` // Debian is the upstream APT repository base URL. // Example: http://archive.ubuntu.com/ubuntu would get Ubuntu. // Default: http://deb.debian.org/debian Debian string `json:"debian" yaml:"debian"` + // RPM is the upstream RPM repository base URL. + // Default: https://dl.fedoraproject.org/pub/fedora/linux + RPM string `json:"rpm" yaml:"rpm"` + // Helm maps repository names to HTTP Helm chart repository URLs. // Requests use /helm/{name}/index.yaml and chart URLs in the index are // rewritten to the same named proxy endpoint. @@ -471,11 +535,27 @@ func Default() *Config { }, Upstream: UpstreamConfig{ NPM: "https://registry.npmjs.org", - Maven: "https://repo1.maven.org/maven2", - GradlePluginPortal: "https://plugins.gradle.org/m2", Cargo: "https://index.crates.io", CargoDownload: "https://static.crates.io/crates", + Gem: "https://rubygems.org", + Go: "https://proxy.golang.org", + Hex: "https://repo.hex.pm", + HexAPI: "https://hex.pm", + Pub: "https://pub.dev", + PyPI: "https://pypi.org", + PyPIDownload: "https://files.pythonhosted.org", + Maven: "https://repo1.maven.org/maven2", + GradlePluginPortal: "https://plugins.gradle.org/m2", + NuGet: "https://api.nuget.org", + NuGetSearch: "https://azuresearch-usnc.nuget.org", + Composer: "https://packagist.org", + ComposerRepository: "https://repo.packagist.org", + Conan: "https://center.conan.io", + Conda: "https://conda.anaconda.org", + CRAN: "https://cloud.r-project.org", + Julia: "https://pkg.julialang.org", Debian: "http://deb.debian.org/debian", + RPM: "https://dl.fedoraproject.org/pub/fedora/linux", }, Gradle: GradleConfig{ BuildCache: GradleBuildCacheConfig{ @@ -563,9 +643,28 @@ func (c *Config) LoadFromEnv() { setEnvString(&c.Log.Level, "PROXY_LOG_LEVEL") setEnvString(&c.Log.Format, "PROXY_LOG_FORMAT") setEnvString(&c.AccessLog.Path, "PROXY_ACCESS_LOG_PATH") + setEnvString(&c.Upstream.NPM, "PROXY_UPSTREAM_NPM") + setEnvString(&c.Upstream.Cargo, "PROXY_UPSTREAM_CARGO") + setEnvString(&c.Upstream.CargoDownload, "PROXY_UPSTREAM_CARGO_DOWNLOAD") + setEnvString(&c.Upstream.Gem, "PROXY_UPSTREAM_GEM") + setEnvString(&c.Upstream.Go, "PROXY_UPSTREAM_GO") + setEnvString(&c.Upstream.Hex, "PROXY_UPSTREAM_HEX") + setEnvString(&c.Upstream.HexAPI, "PROXY_UPSTREAM_HEX_API") + setEnvString(&c.Upstream.Pub, "PROXY_UPSTREAM_PUB") + setEnvString(&c.Upstream.PyPI, "PROXY_UPSTREAM_PYPI") + setEnvString(&c.Upstream.PyPIDownload, "PROXY_UPSTREAM_PYPI_DOWNLOAD") setEnvString(&c.Upstream.Maven, "PROXY_UPSTREAM_MAVEN") setEnvString(&c.Upstream.GradlePluginPortal, "PROXY_UPSTREAM_GRADLE_PLUGIN_PORTAL") + setEnvString(&c.Upstream.NuGet, "PROXY_UPSTREAM_NUGET") + setEnvString(&c.Upstream.NuGetSearch, "PROXY_UPSTREAM_NUGET_SEARCH") + setEnvString(&c.Upstream.Composer, "PROXY_UPSTREAM_COMPOSER") + setEnvString(&c.Upstream.ComposerRepository, "PROXY_UPSTREAM_COMPOSER_REPOSITORY") + setEnvString(&c.Upstream.Conan, "PROXY_UPSTREAM_CONAN") + setEnvString(&c.Upstream.Conda, "PROXY_UPSTREAM_CONDA") + setEnvString(&c.Upstream.CRAN, "PROXY_UPSTREAM_CRAN") + setEnvString(&c.Upstream.Julia, "PROXY_UPSTREAM_JULIA") setEnvString(&c.Upstream.Debian, "PROXY_UPSTREAM_DEBIAN") + setEnvString(&c.Upstream.RPM, "PROXY_UPSTREAM_RPM") setEnvString(&c.Cooldown.Default, "PROXY_COOLDOWN_DEFAULT") setEnvBool(&c.CacheMetadata, "PROXY_CACHE_METADATA") setEnvBool(&c.MirrorAPI, "PROXY_MIRROR_API") diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 0ccc308..52f381d 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -14,6 +14,97 @@ const ( testLevelDebug = "debug" ) +func upstreamConfigValues(upstream UpstreamConfig) map[string]string { + return map[string]string{ + "npm": upstream.NPM, + "cargo": upstream.Cargo, + "cargo_download": upstream.CargoDownload, + "gem": upstream.Gem, + "go": upstream.Go, + "hex": upstream.Hex, + "hex_api": upstream.HexAPI, + "pub": upstream.Pub, + "pypi": upstream.PyPI, + "pypi_download": upstream.PyPIDownload, + "maven": upstream.Maven, + "gradle_plugin_portal": upstream.GradlePluginPortal, + "nuget": upstream.NuGet, + "nuget_search": upstream.NuGetSearch, + "composer": upstream.Composer, + "composer_repository": upstream.ComposerRepository, + "conan": upstream.Conan, + "conda": upstream.Conda, + "cran": upstream.CRAN, + "julia": upstream.Julia, + "debian": upstream.Debian, + "rpm": upstream.RPM, + } +} + +func defaultUpstreamValues() map[string]string { + return map[string]string{ + "npm": "https://registry.npmjs.org", + "cargo": "https://index.crates.io", + "cargo_download": "https://static.crates.io/crates", + "gem": "https://rubygems.org", + "go": "https://proxy.golang.org", + "hex": "https://repo.hex.pm", + "hex_api": "https://hex.pm", + "pub": "https://pub.dev", + "pypi": "https://pypi.org", + "pypi_download": "https://files.pythonhosted.org", + "maven": "https://repo1.maven.org/maven2", + "gradle_plugin_portal": "https://plugins.gradle.org/m2", + "nuget": "https://api.nuget.org", + "nuget_search": "https://azuresearch-usnc.nuget.org", + "composer": "https://packagist.org", + "composer_repository": "https://repo.packagist.org", + "conan": "https://center.conan.io", + "conda": "https://conda.anaconda.org", + "cran": "https://cloud.r-project.org", + "julia": "https://pkg.julialang.org", + "debian": "http://deb.debian.org/debian", + "rpm": "https://dl.fedoraproject.org/pub/fedora/linux", + } +} + +func upstreamEnvironmentVariables() map[string]string { + return map[string]string{ + "npm": "PROXY_UPSTREAM_NPM", + "cargo": "PROXY_UPSTREAM_CARGO", + "cargo_download": "PROXY_UPSTREAM_CARGO_DOWNLOAD", + "gem": "PROXY_UPSTREAM_GEM", + "go": "PROXY_UPSTREAM_GO", + "hex": "PROXY_UPSTREAM_HEX", + "hex_api": "PROXY_UPSTREAM_HEX_API", + "pub": "PROXY_UPSTREAM_PUB", + "pypi": "PROXY_UPSTREAM_PYPI", + "pypi_download": "PROXY_UPSTREAM_PYPI_DOWNLOAD", + "maven": "PROXY_UPSTREAM_MAVEN", + "gradle_plugin_portal": "PROXY_UPSTREAM_GRADLE_PLUGIN_PORTAL", + "nuget": "PROXY_UPSTREAM_NUGET", + "nuget_search": "PROXY_UPSTREAM_NUGET_SEARCH", + "composer": "PROXY_UPSTREAM_COMPOSER", + "composer_repository": "PROXY_UPSTREAM_COMPOSER_REPOSITORY", + "conan": "PROXY_UPSTREAM_CONAN", + "conda": "PROXY_UPSTREAM_CONDA", + "cran": "PROXY_UPSTREAM_CRAN", + "julia": "PROXY_UPSTREAM_JULIA", + "debian": "PROXY_UPSTREAM_DEBIAN", + "rpm": "PROXY_UPSTREAM_RPM", + } +} + +func assertUpstreamValues(t *testing.T, cfg *Config, want map[string]string) { + t.Helper() + got := upstreamConfigValues(cfg.Upstream) + for name, wantValue := range want { + if gotValue := got[name]; gotValue != wantValue { + t.Errorf("Upstream %s = %q, want %q", name, gotValue, wantValue) + } + } +} + func TestDefault(t *testing.T) { cfg := Default() @@ -35,15 +126,7 @@ func TestDefault(t *testing.T) { if cfg.Gradle.BuildCache.MaxAge != "168h" { t.Errorf("Gradle.BuildCache.MaxAge = %q, want %q", cfg.Gradle.BuildCache.MaxAge, "168h") } - if cfg.Upstream.Maven != "https://repo1.maven.org/maven2" { - t.Errorf("Upstream.Maven = %q, want %q", cfg.Upstream.Maven, "https://repo1.maven.org/maven2") - } - if cfg.Upstream.GradlePluginPortal != "https://plugins.gradle.org/m2" { - t.Errorf("Upstream.GradlePluginPortal = %q, want %q", cfg.Upstream.GradlePluginPortal, "https://plugins.gradle.org/m2") - } - if cfg.Upstream.Debian != "http://deb.debian.org/debian" { - t.Errorf("Upstream.Debian = %q, want %q", cfg.Upstream.Debian, "http://deb.debian.org/debian") - } + assertUpstreamValues(t, cfg, defaultUpstreamValues()) } func TestValidate(t *testing.T) { @@ -250,13 +333,60 @@ access_log: } } +func TestLoadYAMLUpstreams(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.yaml") + content := ` +upstream: + npm: "https://upstream.example.com/npm" + cargo: "https://upstream.example.com/cargo" + cargo_download: "https://upstream.example.com/cargo_download" + gem: "https://upstream.example.com/gem" + go: "https://upstream.example.com/go" + hex: "https://upstream.example.com/hex" + hex_api: "https://upstream.example.com/hex_api" + pub: "https://upstream.example.com/pub" + pypi: "https://upstream.example.com/pypi" + pypi_download: "https://upstream.example.com/pypi_download" + maven: "https://upstream.example.com/maven" + gradle_plugin_portal: "https://upstream.example.com/gradle_plugin_portal" + nuget: "https://upstream.example.com/nuget" + nuget_search: "https://upstream.example.com/nuget_search" + composer: "https://upstream.example.com/composer" + composer_repository: "https://upstream.example.com/composer_repository" + conan: "https://upstream.example.com/conan" + conda: "https://upstream.example.com/conda" + cran: "https://upstream.example.com/cran" + julia: "https://upstream.example.com/julia" + debian: "https://upstream.example.com/debian" + rpm: "https://upstream.example.com/rpm" +` + if err := os.WriteFile(path, []byte(content), 0644); err != nil { + t.Fatalf("writing config file: %v", err) + } + + cfg, err := Load(path) + if err != nil { + t.Fatalf("Load failed: %v", err) + } + + want := make(map[string]string) + for name := range defaultUpstreamValues() { + want[name] = "https://upstream.example.com/" + name + } + assertUpstreamValues(t, cfg, want) +} + func TestLoadJSON(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, "config.json") content := `{ "listen": ":4000", - "base_url": "https://json.example.com" + "base_url": "https://json.example.com", + "upstream": { + "gem": "https://json.example.com/gem", + "rpm": "https://json.example.com/rpm" + } }` if err := os.WriteFile(path, []byte(content), 0644); err != nil { t.Fatalf("writing config file: %v", err) @@ -273,6 +403,12 @@ func TestLoadJSON(t *testing.T) { if cfg.BaseURL != "https://json.example.com" { t.Errorf("BaseURL = %q, want %q", cfg.BaseURL, "https://json.example.com") } + if cfg.Upstream.Gem != "https://json.example.com/gem" { + t.Errorf("Upstream.Gem = %q, want %q", cfg.Upstream.Gem, "https://json.example.com/gem") + } + if cfg.Upstream.RPM != "https://json.example.com/rpm" { + t.Errorf("Upstream.RPM = %q, want %q", cfg.Upstream.RPM, "https://json.example.com/rpm") + } } func TestLoadFromEnv(t *testing.T) { @@ -284,9 +420,6 @@ func TestLoadFromEnv(t *testing.T) { t.Setenv("PROXY_STORAGE_PATH", "/env/cache") t.Setenv("PROXY_LOG_LEVEL", testLevelDebug) t.Setenv("PROXY_ACCESS_LOG_PATH", "/tmp/proxy-access.jsonl") - t.Setenv("PROXY_UPSTREAM_MAVEN", "https://maven.example.com/repository/maven-public") - t.Setenv("PROXY_UPSTREAM_GRADLE_PLUGIN_PORTAL", "https://plugins.example.com/m2") - t.Setenv("PROXY_UPSTREAM_DEBIAN", "http://archive.ubuntu.com/ubuntu") t.Setenv("PROXY_GRADLE_BUILD_CACHE_READ_ONLY", "true") t.Setenv("PROXY_GRADLE_BUILD_CACHE_MAX_UPLOAD_SIZE", "32MB") t.Setenv("PROXY_GRADLE_BUILD_CACHE_MAX_AGE", "12h") @@ -313,15 +446,6 @@ func TestLoadFromEnv(t *testing.T) { if cfg.AccessLog.Path != "/tmp/proxy-access.jsonl" { t.Errorf("AccessLog.Path = %q, want %q", cfg.AccessLog.Path, "/tmp/proxy-access.jsonl") } - if cfg.Upstream.Maven != "https://maven.example.com/repository/maven-public" { - t.Errorf("Upstream.Maven = %q, want %q", cfg.Upstream.Maven, "https://maven.example.com/repository/maven-public") - } - if cfg.Upstream.GradlePluginPortal != "https://plugins.example.com/m2" { - t.Errorf("Upstream.GradlePluginPortal = %q, want %q", cfg.Upstream.GradlePluginPortal, "https://plugins.example.com/m2") - } - if cfg.Upstream.Debian != "http://archive.ubuntu.com/ubuntu" { - t.Errorf("Upstream.Debian = %q, want %q", cfg.Upstream.Debian, "http://archive.ubuntu.com/ubuntu") - } if !cfg.Gradle.BuildCache.ReadOnly { t.Error("Gradle.BuildCache.ReadOnly = false, want true") } @@ -339,6 +463,19 @@ func TestLoadFromEnv(t *testing.T) { } } +func TestLoadFromEnvUpstreams(t *testing.T) { + cfg := Default() + want := make(map[string]string) + for name, envName := range upstreamEnvironmentVariables() { + value := "https://env.example.com/" + name + t.Setenv(envName, value) + want[name] = value + } + + cfg.LoadFromEnv() + assertUpstreamValues(t, cfg, want) +} + func TestLoadCooldownConfig(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, "config.yaml") diff --git a/internal/handler/composer.go b/internal/handler/composer.go index 45935b7..9b2de80 100644 --- a/internal/handler/composer.go +++ b/internal/handler/composer.go @@ -37,6 +37,15 @@ func NewComposerHandler(proxy *Proxy, proxyURL string) *ComposerHandler { } } +// NewComposerHandlerWithUpstreams creates a Composer handler with custom API +// and repository upstreams. +func NewComposerHandlerWithUpstreams(proxy *Proxy, proxyURL, upstreamURL, repoURL string) *ComposerHandler { + h := NewComposerHandler(proxy, proxyURL) + h.upstreamURL = configuredUpstreamURL(upstreamURL, composerUpstream) + h.repoURL = configuredUpstreamURL(repoURL, composerRepo) + return h +} + // Routes returns the HTTP handler for Composer requests. func (h *ComposerHandler) Routes() http.Handler { mux := http.NewServeMux() diff --git a/internal/handler/conan.go b/internal/handler/conan.go index c0476f9..7142f0d 100644 --- a/internal/handler/conan.go +++ b/internal/handler/conan.go @@ -27,6 +27,13 @@ func NewConanHandler(proxy *Proxy, proxyURL string) *ConanHandler { } } +// NewConanHandlerWithUpstream creates a Conan handler with a custom upstream. +func NewConanHandlerWithUpstream(proxy *Proxy, proxyURL, upstreamURL string) *ConanHandler { + h := NewConanHandler(proxy, proxyURL) + h.upstreamURL = configuredUpstreamURL(upstreamURL, conanUpstream) + return h +} + // Routes returns the HTTP handler for Conan requests. func (h *ConanHandler) Routes() http.Handler { mux := http.NewServeMux() diff --git a/internal/handler/conda.go b/internal/handler/conda.go index 224c25f..bee5694 100644 --- a/internal/handler/conda.go +++ b/internal/handler/conda.go @@ -29,6 +29,13 @@ func NewCondaHandler(proxy *Proxy, proxyURL string) *CondaHandler { } } +// NewCondaHandlerWithUpstream creates a Conda handler with a custom upstream. +func NewCondaHandlerWithUpstream(proxy *Proxy, proxyURL, upstreamURL string) *CondaHandler { + h := NewCondaHandler(proxy, proxyURL) + h.upstreamURL = configuredUpstreamURL(upstreamURL, condaUpstream) + return h +} + // Routes returns the HTTP handler for Conda requests. func (h *CondaHandler) Routes() http.Handler { mux := http.NewServeMux() diff --git a/internal/handler/cran.go b/internal/handler/cran.go index 2fc4fab..4a6ded8 100644 --- a/internal/handler/cran.go +++ b/internal/handler/cran.go @@ -25,6 +25,13 @@ func NewCRANHandler(proxy *Proxy, proxyURL string) *CRANHandler { } } +// NewCRANHandlerWithUpstream creates a CRAN handler with a custom upstream. +func NewCRANHandlerWithUpstream(proxy *Proxy, proxyURL, upstreamURL string) *CRANHandler { + h := NewCRANHandler(proxy, proxyURL) + h.upstreamURL = configuredUpstreamURL(upstreamURL, cranUpstream) + return h +} + // Routes returns the HTTP handler for CRAN requests. func (h *CRANHandler) Routes() http.Handler { mux := http.NewServeMux() diff --git a/internal/handler/gem.go b/internal/handler/gem.go index 260568a..080bcdd 100644 --- a/internal/handler/gem.go +++ b/internal/handler/gem.go @@ -30,6 +30,13 @@ func NewGemHandler(proxy *Proxy, proxyURL string) *GemHandler { } } +// NewGemHandlerWithUpstream creates a RubyGems handler with a custom upstream. +func NewGemHandlerWithUpstream(proxy *Proxy, proxyURL, upstreamURL string) *GemHandler { + h := NewGemHandler(proxy, proxyURL) + h.upstreamURL = configuredUpstreamURL(upstreamURL, gemUpstream) + return h +} + // Routes returns the HTTP handler for RubyGems requests. func (h *GemHandler) Routes() http.Handler { mux := http.NewServeMux() diff --git a/internal/handler/go.go b/internal/handler/go.go index cf40aa1..c2690f9 100644 --- a/internal/handler/go.go +++ b/internal/handler/go.go @@ -30,6 +30,13 @@ func NewGoHandler(proxy *Proxy, proxyURL string) *GoHandler { } } +// NewGoHandlerWithUpstream creates a Go module handler with a custom upstream. +func NewGoHandlerWithUpstream(proxy *Proxy, proxyURL, upstreamURL string) *GoHandler { + h := NewGoHandler(proxy, proxyURL) + h.upstreamURL = configuredUpstreamURL(upstreamURL, goUpstream) + return h +} + // Routes returns the HTTP handler for Go proxy requests. func (h *GoHandler) Routes() http.Handler { // Go module paths can contain slashes, so just use the handler directly diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 8dcc9c9..2b88f98 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -47,6 +47,13 @@ func hasDotDotSegment(path string) bool { return false } +func configuredUpstreamURL(value, defaultValue string) string { + if value == "" { + value = defaultValue + } + return strings.TrimRight(value, "/") +} + const defaultHTTPTimeout = 30 * time.Second const artifactCopyBufferSize = 32 << 10 diff --git a/internal/handler/hex.go b/internal/handler/hex.go index 2ff4f0f..05416c6 100644 --- a/internal/handler/hex.go +++ b/internal/handler/hex.go @@ -21,6 +21,7 @@ const ( type HexHandler struct { proxy *Proxy upstreamURL string + apiURL string proxyURL string } @@ -29,10 +30,20 @@ func NewHexHandler(proxy *Proxy, proxyURL string) *HexHandler { return &HexHandler{ proxy: proxy, upstreamURL: hexUpstream, + apiURL: hexAPIURL, proxyURL: strings.TrimSuffix(proxyURL, "/"), } } +// NewHexHandlerWithUpstreams creates a Hex handler with custom repository and +// API upstreams. +func NewHexHandlerWithUpstreams(proxy *Proxy, proxyURL, upstreamURL, apiURL string) *HexHandler { + h := NewHexHandler(proxy, proxyURL) + h.upstreamURL = configuredUpstreamURL(upstreamURL, hexUpstream) + h.apiURL = configuredUpstreamURL(apiURL, hexAPIURL) + return h +} + // Routes returns the HTTP handler for Hex requests. func (h *HexHandler) Routes() http.Handler { mux := http.NewServeMux() @@ -197,7 +208,7 @@ type hexPackageAPI struct { // fetchFilteredVersions fetches the Hex API and returns a set of version // strings that should be filtered out by cooldown. func (h *HexHandler) fetchFilteredVersions(r *http.Request, name string) (map[string]bool, error) { - apiURL := fmt.Sprintf("%s/api/packages/%s", hexAPIURL, name) + apiURL := fmt.Sprintf("%s/api/packages/%s", h.apiURL, name) req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, apiURL, nil) if err != nil { return nil, err diff --git a/internal/handler/julia.go b/internal/handler/julia.go index 0fed8c9..211d092 100644 --- a/internal/handler/julia.go +++ b/internal/handler/julia.go @@ -53,6 +53,13 @@ func NewJuliaHandler(proxy *Proxy, _ string) *JuliaHandler { } } +// NewJuliaHandlerWithUpstream creates a Julia handler with a custom upstream. +func NewJuliaHandlerWithUpstream(proxy *Proxy, upstreamURL string) *JuliaHandler { + h := NewJuliaHandler(proxy, "") + h.upstreamURL = configuredUpstreamURL(upstreamURL, juliaUpstream) + return h +} + // Routes returns the HTTP handler for Julia requests. func (h *JuliaHandler) Routes() http.Handler { mux := http.NewServeMux() diff --git a/internal/handler/nuget.go b/internal/handler/nuget.go index 4785e40..0fb139b 100644 --- a/internal/handler/nuget.go +++ b/internal/handler/nuget.go @@ -11,13 +11,15 @@ import ( ) const ( - nugetUpstream = "https://api.nuget.org" + nugetUpstream = "https://api.nuget.org" + nugetSearchUpstream = "https://azuresearch-usnc.nuget.org" ) // NuGetHandler handles NuGet V3 API protocol requests. type NuGetHandler struct { proxy *Proxy upstreamURL string + searchURL string proxyURL string } @@ -26,10 +28,20 @@ func NewNuGetHandler(proxy *Proxy, proxyURL string) *NuGetHandler { return &NuGetHandler{ proxy: proxy, upstreamURL: nugetUpstream, + searchURL: nugetSearchUpstream, proxyURL: strings.TrimSuffix(proxyURL, "/"), } } +// NewNuGetHandlerWithUpstreams creates a NuGet handler with custom API and +// search upstreams. +func NewNuGetHandlerWithUpstreams(proxy *Proxy, proxyURL, upstreamURL, searchURL string) *NuGetHandler { + h := NewNuGetHandler(proxy, proxyURL) + h.upstreamURL = configuredUpstreamURL(upstreamURL, nugetUpstream) + h.searchURL = configuredUpstreamURL(searchURL, nugetSearchUpstream) + return h +} + // Routes returns the HTTP handler for NuGet requests. func (h *NuGetHandler) Routes() http.Handler { mux := http.NewServeMux() @@ -103,55 +115,31 @@ func (h *NuGetHandler) rewriteServiceIndex(body []byte) ([]byte, error) { id, _ := rmap["@id"].(string) rtype, _ := rmap["@type"].(string) - // Rewrite URLs for services we proxy - if id != "" && h.shouldRewriteService(rtype) { - newURL := h.rewriteNuGetURL(id) - rmap["@id"] = newURL + // Rewrite URLs for services we proxy. The service type determines the + // local route because an upstream index may advertise a different host. + if id != "" { + rmap["@id"] = h.rewriteNuGetURL(id, rtype) } } return json.Marshal(index) } -// shouldRewriteService returns true if the service type should be rewritten. -func (h *NuGetHandler) shouldRewriteService(serviceType string) bool { - // Rewrite package content and registration services - rewriteTypes := []string{ - "PackageBaseAddress/3.0.0", - "RegistrationsBaseUrl/3.6.0", - "RegistrationsBaseUrl/Versioned", - "SearchQueryService", - "SearchQueryService/3.0.0-rc", - "SearchQueryService/3.5.0", - "SearchAutocompleteService", - "SearchAutocompleteService/3.5.0", - } - - for _, t := range rewriteTypes { - if serviceType == t { - return true - } - } - return false -} - -// rewriteNuGetURL rewrites a NuGet API URL to point at this proxy. -func (h *NuGetHandler) rewriteNuGetURL(origURL string) string { - // Map known NuGet API endpoints to our proxy paths - replacements := map[string]string{ - "https://api.nuget.org/v3-flatcontainer/": h.proxyURL + "/nuget/v3-flatcontainer/", - "https://api.nuget.org/v3/registration5-gz-semver2/": h.proxyURL + "/nuget/v3/registration5-gz-semver2/", - "https://azuresearch-usnc.nuget.org/query": h.proxyURL + "/nuget/query", - "https://azuresearch-usnc.nuget.org/autocomplete": h.proxyURL + "/nuget/autocomplete", - } - - for old, new := range replacements { - if strings.HasPrefix(origURL, old) { - return strings.Replace(origURL, old, new, 1) - } +// rewriteNuGetURL rewrites a NuGet service URL based on its advertised type. +// Service types the proxy does not handle are returned unchanged. +func (h *NuGetHandler) rewriteNuGetURL(origURL, serviceType string) string { + switch serviceType { + case "PackageBaseAddress/3.0.0": + return h.proxyURL + "/nuget/v3-flatcontainer/" + case "RegistrationsBaseUrl/3.6.0", "RegistrationsBaseUrl/Versioned": + return h.proxyURL + "/nuget/v3/registration5-gz-semver2/" + case "SearchQueryService", "SearchQueryService/3.0.0-rc", "SearchQueryService/3.5.0": + return h.proxyURL + "/nuget/query" + case "SearchAutocompleteService", "SearchAutocompleteService/3.5.0": + return h.proxyURL + "/nuget/autocomplete" + default: + return origURL } - - return origURL } // handleRegistration proxies NuGet registration pages, applying cooldown filtering. @@ -363,7 +351,7 @@ func (h *NuGetHandler) buildUpstreamURL(r *http.Request) string { // Handle query and autocomplete which go to azuresearch if strings.HasPrefix(path, "/query") || strings.HasPrefix(path, "/autocomplete") { - return "https://azuresearch-usnc.nuget.org" + path + "?" + r.URL.RawQuery + return h.searchURL + path + "?" + r.URL.RawQuery } return h.upstreamURL + path diff --git a/internal/handler/nuget_test.go b/internal/handler/nuget_test.go index b2164e5..09efe95 100644 --- a/internal/handler/nuget_test.go +++ b/internal/handler/nuget_test.go @@ -91,75 +91,82 @@ func TestNuGetRewriteServiceIndex(t *testing.T) { } } -func TestNuGetShouldRewriteService(t *testing.T) { - h := &NuGetHandler{} - - rewriteTypes := []string{ - "PackageBaseAddress/3.0.0", - "RegistrationsBaseUrl/3.6.0", - "RegistrationsBaseUrl/Versioned", - "SearchQueryService", - "SearchQueryService/3.0.0-rc", - "SearchQueryService/3.5.0", - "SearchAutocompleteService", - "SearchAutocompleteService/3.5.0", - } - - for _, stype := range rewriteTypes { - if !h.shouldRewriteService(stype) { - t.Errorf("shouldRewriteService(%q) = false, want true", stype) - } - } - - noRewriteTypes := []string{ - "SomeOtherService/1.0.0", - "PackagePublish/2.0.0", - "", - "SearchQueryService/99.0.0", - } - - for _, stype := range noRewriteTypes { - if h.shouldRewriteService(stype) { - t.Errorf("shouldRewriteService(%q) = true, want false", stype) - } - } -} - func TestNuGetRewriteURL(t *testing.T) { h := &NuGetHandler{ proxyURL: "http://localhost:8080", } tests := []struct { - input string - want string + input string + serviceType string + want string }{ { "https://api.nuget.org/v3-flatcontainer/", + "PackageBaseAddress/3.0.0", "http://localhost:8080/nuget/v3-flatcontainer/", }, { "https://api.nuget.org/v3/registration5-gz-semver2/", + "RegistrationsBaseUrl/3.6.0", + "http://localhost:8080/nuget/v3/registration5-gz-semver2/", + }, + { + "https://api.nuget.org/v3/registration5-gz-semver2/", + "RegistrationsBaseUrl/Versioned", "http://localhost:8080/nuget/v3/registration5-gz-semver2/", }, { "https://azuresearch-usnc.nuget.org/query", + "SearchQueryService", + "http://localhost:8080/nuget/query", + }, + { + "https://azuresearch-usnc.nuget.org/query", + "SearchQueryService/3.0.0-rc", + "http://localhost:8080/nuget/query", + }, + { + "https://azuresearch-usnc.nuget.org/query", + "SearchQueryService/3.5.0", "http://localhost:8080/nuget/query", }, { "https://azuresearch-usnc.nuget.org/autocomplete", + "SearchAutocompleteService", + "http://localhost:8080/nuget/autocomplete", + }, + { + "https://azuresearch-usnc.nuget.org/autocomplete", + "SearchAutocompleteService/3.5.0", "http://localhost:8080/nuget/autocomplete", }, { "https://example.com/unknown", + "SomeOtherService/1.0.0", "https://example.com/unknown", }, + { + "https://api.nuget.org/v2/package", + "PackagePublish/2.0.0", + "https://api.nuget.org/v2/package", + }, + { + "https://azuresearch-usnc.nuget.org/query", + "SearchQueryService/99.0.0", + "https://azuresearch-usnc.nuget.org/query", + }, + { + "https://example.com/resource", + "", + "https://example.com/resource", + }, } for _, tt := range tests { - got := h.rewriteNuGetURL(tt.input) + got := h.rewriteNuGetURL(tt.input, tt.serviceType) if got != tt.want { - t.Errorf("rewriteNuGetURL(%q) = %q, want %q", tt.input, got, tt.want) + t.Errorf("rewriteNuGetURL(%q, %q) = %q, want %q", tt.input, tt.serviceType, got, tt.want) } } } @@ -458,6 +465,7 @@ func TestNuGetProxyUpstreamForwardsAcceptEncoding(t *testing.T) { func TestNuGetBuildUpstreamURL(t *testing.T) { h := &NuGetHandler{ upstreamURL: "https://api.nuget.org", + searchURL: "https://azuresearch-usnc.nuget.org", } tests := []struct { @@ -736,6 +744,7 @@ func TestNuGetHandleDownloadMissingFilename(t *testing.T) { func TestNuGetBuildUpstreamURLQueryPath(t *testing.T) { h := &NuGetHandler{ upstreamURL: "https://api.nuget.org", + searchURL: "https://azuresearch-usnc.nuget.org", } // Query endpoint should go to azuresearch @@ -750,6 +759,7 @@ func TestNuGetBuildUpstreamURLQueryPath(t *testing.T) { func TestNuGetBuildUpstreamURLAutocompletePath(t *testing.T) { h := &NuGetHandler{ upstreamURL: "https://api.nuget.org", + searchURL: "https://azuresearch-usnc.nuget.org", } req := httptest.NewRequest(http.MethodGet, "/autocomplete?q=new&take=10", nil) diff --git a/internal/handler/pub.go b/internal/handler/pub.go index e5ca199..fda1b1e 100644 --- a/internal/handler/pub.go +++ b/internal/handler/pub.go @@ -30,6 +30,13 @@ func NewPubHandler(proxy *Proxy, proxyURL string) *PubHandler { } } +// NewPubHandlerWithUpstream creates a pub handler with a custom upstream. +func NewPubHandlerWithUpstream(proxy *Proxy, proxyURL, upstreamURL string) *PubHandler { + h := NewPubHandler(proxy, proxyURL) + h.upstreamURL = configuredUpstreamURL(upstreamURL, pubUpstream) + return h +} + // Routes returns the HTTP handler for pub requests. func (h *PubHandler) Routes() http.Handler { mux := http.NewServeMux() diff --git a/internal/handler/pypi.go b/internal/handler/pypi.go index 10a3a13..08ab852 100644 --- a/internal/handler/pypi.go +++ b/internal/handler/pypi.go @@ -8,18 +8,18 @@ import ( "fmt" "io" "net/http" - "net/url" "regexp" "strings" "time" ) const ( - pypiUpstream = "https://pypi.org" - minWheelParts = 5 // name + version + python + abi + platform - minSubmatchParts = 2 // full match + first capture group - minPyPIPathParts = 3 // hash_prefix + hash + filename - minEggParts = 3 // name + version + python tag + pypiUpstream = "https://pypi.org" + pypiDownloadUpstream = "https://files.pythonhosted.org" + minWheelParts = 5 // name + version + python + abi + platform + minSubmatchParts = 2 // full match + first capture group + minPyPIPathParts = 3 // hash_prefix + hash + filename + minEggParts = 3 // name + version + python tag // PyPIMetadataSuffix is the PEP 658 core-metadata sidecar suffix that pip // appends to a distribution URL when the index advertises core metadata. @@ -31,18 +31,29 @@ const ( // PyPIHandler handles PyPI registry protocol requests. type PyPIHandler struct { - proxy *Proxy - upstreamURL string - proxyURL string + proxy *Proxy + upstreamURL string + downloadURL string + downloadHrefRe *regexp.Regexp + proxyURL string } // NewPyPIHandler creates a new PyPI protocol handler. func NewPyPIHandler(proxy *Proxy, proxyURL string) *PyPIHandler { - return &PyPIHandler{ + return NewPyPIHandlerWithUpstreams(proxy, proxyURL, "", "") +} + +// NewPyPIHandlerWithUpstreams creates a PyPI handler with custom API and +// package download upstreams. +func NewPyPIHandlerWithUpstreams(proxy *Proxy, proxyURL, upstreamURL, downloadURL string) *PyPIHandler { + h := &PyPIHandler{ proxy: proxy, - upstreamURL: pypiUpstream, + upstreamURL: configuredUpstreamURL(upstreamURL, pypiUpstream), + downloadURL: configuredUpstreamURL(downloadURL, pypiDownloadUpstream), proxyURL: strings.TrimSuffix(proxyURL, "/"), } + h.downloadHrefRe = regexp.MustCompile(`href="(` + regexp.QuoteMeta(h.downloadURL) + `/packages/[^"]+)"`) + return h } // Routes returns the HTTP handler for PyPI requests. @@ -169,24 +180,16 @@ func (h *PyPIHandler) rewriteSimpleHTML(body []byte, filteredVersions map[string }) } - // Match href attributes pointing to packages - // PyPI URLs look like: https://files.pythonhosted.org/packages/... - re := regexp.MustCompile(`href="(https://files\.pythonhosted\.org/packages/[^"]+)"`) - - return re.ReplaceAllFunc(body, func(match []byte) []byte { - submatch := re.FindSubmatch(match) + // Match href attributes pointing to packages on the configured download host. + return h.downloadHrefRe.ReplaceAllFunc(body, func(match []byte) []byte { + submatch := h.downloadHrefRe.FindSubmatch(match) if len(submatch) < minSubmatchParts { return match } origURL := string(submatch[1]) - u, err := url.Parse(origURL) - if err != nil { - return match - } - - newURL := fmt.Sprintf("%s/pypi/packages%s", h.proxyURL, u.Path) + newURL := h.proxyURL + "/pypi/packages" + strings.TrimPrefix(origURL, h.downloadURL) return []byte(fmt.Sprintf(`href="%s"`, newURL)) }) } @@ -391,15 +394,8 @@ func (h *PyPIHandler) rewriteURLEntry(entry map[string]any) { return } - u, err := url.Parse(urlStr) - if err != nil { - return - } - - // Only rewrite pythonhosted.org URLs - if u.Host == "files.pythonhosted.org" { - newURL := fmt.Sprintf("%s/pypi/packages%s", h.proxyURL, u.Path) - entry["url"] = newURL + if strings.HasPrefix(urlStr, h.downloadURL+"/packages/") { + entry["url"] = h.proxyURL + "/pypi/packages" + strings.TrimPrefix(urlStr, h.downloadURL) } } @@ -441,7 +437,7 @@ func (h *PyPIHandler) handleDownload(w http.ResponseWriter, r *http.Request) { // Construct upstream URL; the incoming path starts with // '/packages' so there is no need to include it in the format // string - upstreamURL := fmt.Sprintf("https://files.pythonhosted.org/%s", path) + upstreamURL := fmt.Sprintf("%s/%s", h.downloadURL, path) result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "pypi", name, version, filename, upstreamURL) if err != nil { diff --git a/internal/handler/rpm.go b/internal/handler/rpm.go index a5752ec..f4cf9e7 100644 --- a/internal/handler/rpm.go +++ b/internal/handler/rpm.go @@ -30,6 +30,13 @@ func NewRPMHandler(proxy *Proxy, proxyURL string) *RPMHandler { } } +// NewRPMHandlerWithUpstream creates an RPM handler with a custom upstream. +func NewRPMHandlerWithUpstream(proxy *Proxy, proxyURL, upstreamURL string) *RPMHandler { + h := NewRPMHandler(proxy, proxyURL) + h.upstreamURL = configuredUpstreamURL(upstreamURL, defaultRPMUpstream) + return h +} + // Routes returns the HTTP handler for RPM requests. // Mount this at /rpm on your router. func (h *RPMHandler) Routes() http.Handler { diff --git a/internal/handler/upstream_test.go b/internal/handler/upstream_test.go new file mode 100644 index 0000000..86ffb0a --- /dev/null +++ b/internal/handler/upstream_test.go @@ -0,0 +1,128 @@ +package handler + +import ( + "io" + "net/http" + "net/http/httptest" + "testing" +) + +func TestHandlerUpstreamConfiguration(t *testing.T) { + const ( + proxyURL = "https://proxy.example.com/" + baseURL = "https://upstream.example.com" + ) + hex := NewHexHandlerWithUpstreams(nil, proxyURL, baseURL+"/hex/", baseURL+"/hex-api/") + pypi := NewPyPIHandlerWithUpstreams(nil, proxyURL, baseURL+"/pypi/", baseURL+"/pypi-download/") + nuget := NewNuGetHandlerWithUpstreams(nil, proxyURL, baseURL+"/nuget/", baseURL+"/nuget-search/") + composer := NewComposerHandlerWithUpstreams( + nil, proxyURL, baseURL+"/composer/", baseURL+"/composer-repository/", + ) + + got := map[string]string{ + "gem": NewGemHandlerWithUpstream(nil, proxyURL, baseURL+"/gem/").upstreamURL, + "go": NewGoHandlerWithUpstream(nil, proxyURL, baseURL+"/go/").upstreamURL, + "hex": hex.upstreamURL, + "hex_api": hex.apiURL, + "pub": NewPubHandlerWithUpstream(nil, proxyURL, baseURL+"/pub/").upstreamURL, + "pypi": pypi.upstreamURL, + "pypi_download": pypi.downloadURL, + "nuget": nuget.upstreamURL, + "nuget_search": nuget.searchURL, + "composer": composer.upstreamURL, + "composer_repository": composer.repoURL, + "conan": NewConanHandlerWithUpstream(nil, proxyURL, baseURL+"/conan/").upstreamURL, + "conda": NewCondaHandlerWithUpstream(nil, proxyURL, baseURL+"/conda/").upstreamURL, + "cran": NewCRANHandlerWithUpstream(nil, proxyURL, baseURL+"/cran/").upstreamURL, + "julia": NewJuliaHandlerWithUpstream(nil, baseURL+"/julia/").upstreamURL, + "rpm": NewRPMHandlerWithUpstream(nil, proxyURL, baseURL+"/rpm/").upstreamURL, + } + + want := map[string]string{ + "gem": baseURL + "/gem", + "go": baseURL + "/go", + "hex": baseURL + "/hex", + "hex_api": baseURL + "/hex-api", + "pub": baseURL + "/pub", + "pypi": baseURL + "/pypi", + "pypi_download": baseURL + "/pypi-download", + "nuget": baseURL + "/nuget", + "nuget_search": baseURL + "/nuget-search", + "composer": baseURL + "/composer", + "composer_repository": baseURL + "/composer-repository", + "conan": baseURL + "/conan", + "conda": baseURL + "/conda", + "cran": baseURL + "/cran", + "julia": baseURL + "/julia", + "rpm": baseURL + "/rpm", + } + + for name, wantURL := range want { + if gotURL := got[name]; gotURL != wantURL { + t.Errorf("%s upstream = %q, want %q", name, gotURL, wantURL) + } + } +} + +func TestConfiguredUpstreamURL(t *testing.T) { + if got := configuredUpstreamURL("", "https://default.example.com/"); got != "https://default.example.com" { + t.Errorf("empty configured URL = %q, want default", got) + } + if got := configuredUpstreamURL("https://custom.example.com/", "https://default.example.com"); got != "https://custom.example.com" { + t.Errorf("configured URL = %q, want trimmed custom URL", got) + } + if got := configuredUpstreamURL("https://custom.example.com///", "https://default.example.com"); got != "https://custom.example.com" { + t.Errorf("configured URL with trailing slashes = %q, want trimmed custom URL", got) + } +} + +func TestHexHandlerUsesConfiguredAPIUpstream(t *testing.T) { + var requestedPath string + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestedPath = r.URL.Path + _, _ = io.WriteString(w, `{"releases":[]}`) + })) + defer upstream.Close() + + h := NewHexHandlerWithUpstreams( + &Proxy{HTTPClient: upstream.Client()}, + "https://proxy.example.com", + upstream.URL+"/hex", + upstream.URL+"/hex-api", + ) + _, err := h.fetchFilteredVersions(httptest.NewRequest(http.MethodGet, "/", nil), "demo") + if err != nil { + t.Fatalf("fetchFilteredVersions failed: %v", err) + } + if requestedPath != "/hex-api/api/packages/demo" { + t.Errorf("API path = %q, want %q", requestedPath, "/hex-api/api/packages/demo") + } +} + +func TestPyPIHandlerRewritesConfiguredDownloadUpstream(t *testing.T) { + h := NewPyPIHandlerWithUpstreams( + nil, + "https://proxy.example.com", + "https://upstream.example.com/pypi", + "https://upstream.example.com/pypi", + ) + body := []byte(`demo`) + want := `demo` + if got := string(h.rewriteSimpleHTML(body, nil)); got != want { + t.Errorf("rewritten HTML = %q, want %q", got, want) + } +} + +func TestNuGetHandlerUsesConfiguredSearchUpstream(t *testing.T) { + h := NewNuGetHandlerWithUpstreams( + nil, + "https://proxy.example.com", + "https://upstream.example.com/nuget", + "https://upstream.example.com/nuget-search", + ) + req := httptest.NewRequest(http.MethodGet, "/query?q=demo", nil) + want := "https://upstream.example.com/nuget-search/query?q=demo" + if got := h.buildUpstreamURL(req); got != want { + t.Errorf("search URL = %q, want %q", got, want) + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 71af1af..fac5c9f 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -241,11 +241,21 @@ func (s *Server) Start() error { s.cfg.Upstream.Cargo, s.cfg.Upstream.CargoDownload, ) - gemHandler := handler.NewGemHandler(proxy, s.cfg.BaseURL) - goHandler := handler.NewGoHandler(proxy, s.cfg.BaseURL) - hexHandler := handler.NewHexHandler(proxy, s.cfg.BaseURL) - pubHandler := handler.NewPubHandler(proxy, s.cfg.BaseURL) - pypiHandler := handler.NewPyPIHandler(proxy, s.cfg.BaseURL) + gemHandler := handler.NewGemHandlerWithUpstream(proxy, s.cfg.BaseURL, s.cfg.Upstream.Gem) + goHandler := handler.NewGoHandlerWithUpstream(proxy, s.cfg.BaseURL, s.cfg.Upstream.Go) + hexHandler := handler.NewHexHandlerWithUpstreams( + proxy, + s.cfg.BaseURL, + s.cfg.Upstream.Hex, + s.cfg.Upstream.HexAPI, + ) + pubHandler := handler.NewPubHandlerWithUpstream(proxy, s.cfg.BaseURL, s.cfg.Upstream.Pub) + pypiHandler := handler.NewPyPIHandlerWithUpstreams( + proxy, + s.cfg.BaseURL, + s.cfg.Upstream.PyPI, + s.cfg.Upstream.PyPIDownload, + ) mavenHandler := handler.NewMavenHandler( proxy, s.cfg.BaseURL, @@ -253,16 +263,26 @@ func (s *Server) Start() error { s.cfg.Upstream.GradlePluginPortal, ) gradleHandler := handler.NewGradleBuildCacheHandler(proxy) - nugetHandler := handler.NewNuGetHandler(proxy, s.cfg.BaseURL) - composerHandler := handler.NewComposerHandler(proxy, s.cfg.BaseURL) - conanHandler := handler.NewConanHandler(proxy, s.cfg.BaseURL) - condaHandler := handler.NewCondaHandler(proxy, s.cfg.BaseURL) - cranHandler := handler.NewCRANHandler(proxy, s.cfg.BaseURL) - juliaHandler := handler.NewJuliaHandler(proxy, s.cfg.BaseURL) + nugetHandler := handler.NewNuGetHandlerWithUpstreams( + proxy, + s.cfg.BaseURL, + s.cfg.Upstream.NuGet, + s.cfg.Upstream.NuGetSearch, + ) + composerHandler := handler.NewComposerHandlerWithUpstreams( + proxy, + s.cfg.BaseURL, + s.cfg.Upstream.Composer, + s.cfg.Upstream.ComposerRepository, + ) + conanHandler := handler.NewConanHandlerWithUpstream(proxy, s.cfg.BaseURL, s.cfg.Upstream.Conan) + condaHandler := handler.NewCondaHandlerWithUpstream(proxy, s.cfg.BaseURL, s.cfg.Upstream.Conda) + cranHandler := handler.NewCRANHandlerWithUpstream(proxy, s.cfg.BaseURL, s.cfg.Upstream.CRAN) + juliaHandler := handler.NewJuliaHandlerWithUpstream(proxy, s.cfg.Upstream.Julia) containerHandler := handler.NewContainerHandler(proxy, s.cfg.BaseURL, s.cfg.Upstream.OCI) helmHandler := handler.NewHelmHandler(proxy, s.cfg.BaseURL, s.cfg.Upstream.Helm) debianHandler := handler.NewDebianHandler(proxy, s.cfg.BaseURL, s.cfg.Upstream.Debian) - rpmHandler := handler.NewRPMHandler(proxy, s.cfg.BaseURL) + rpmHandler := handler.NewRPMHandlerWithUpstream(proxy, s.cfg.BaseURL, s.cfg.Upstream.RPM) r.Mount("/npm", http.StripPrefix("/npm", npmHandler.Routes())) r.Mount("/cargo", http.StripPrefix("/cargo", cargoHandler.Routes()))