diff --git a/controllers/dependencyfirewall/composer.go b/controllers/dependencyfirewall/composer.go
index 6d8e0a86f..739eb8b41 100644
--- a/controllers/dependencyfirewall/composer.go
+++ b/controllers/dependencyfirewall/composer.go
@@ -250,14 +250,19 @@ func (d *ComposerDependencyProxyController) proxyComposerDist(c shared.Context)
}
}
- distURL, releaseTime, err := d.resolveComposerDist(ctx, packageName, version)
+ registry := configs.getRegistry(composer, composerMetadataRegistry)
+ distURL, releaseTime, err := d.resolveComposerDist(ctx, packageName, version, registry)
if err != nil {
span.RecordError(err)
slog.Warn("Could not resolve dist url", "proxy", "composer", "package", packageName, "version", version, "error", err)
return echo.NewHTTPError(http.StatusNotFound, fmt.Sprintf("could not resolve %s@%s", packageName, version))
}
- if err := checkComposerDistHost(distURL); err != nil {
+ registryURL, err := url.Parse(registry)
+ if err != nil {
+ return echo.NewHTTPError(http.StatusInternalServerError, "invalid composer registry")
+ }
+ if err := checkDistHost(distURL, append([]string{registryURL.Hostname()}, composerDistHosts...)...); err != nil {
span.RecordError(err)
slog.Warn("Blocked dist url", "proxy", "composer", "package", packageName, "version", version, "url", distURL, "error", err)
return echo.NewHTTPError(http.StatusForbidden, fmt.Sprintf("blocked dist url: %v", err))
@@ -344,7 +349,7 @@ func (d *ComposerDependencyProxyController) proxyComposerMetadata(c shared.Conte
}
data, headers, statusCode, err := d.fetchFromUpstream(
- ctx, composer, composerMetadataRegistry, requestPath, c.Request().Header, nil,
+ ctx, composer, configs.getRegistry(composer, composerMetadataRegistry), requestPath, c.Request().Header, nil,
)
if err != nil {
@@ -494,10 +499,10 @@ func filterComposerMetadata(data []byte, proxyBaseURL string, minAge time.Durati
return out, removed, nil
}
-func (d *ComposerDependencyProxyController) resolveComposerDist(ctx context.Context, packageName, version string) (string, time.Time, error) {
+func (d *ComposerDependencyProxyController) resolveComposerDist(ctx context.Context, packageName, version, registry string) (string, time.Time, error) {
metadataPath := fmt.Sprintf("p2/%s.json", packageName)
- data, _, statusCode, err := d.fetchFromUpstream(ctx, composer, composerMetadataRegistry, metadataPath, http.Header{}, nil)
+ data, _, statusCode, err := d.fetchFromUpstream(ctx, composer, registry, metadataPath, http.Header{}, nil)
if err != nil {
return "", time.Time{}, err
}
@@ -543,24 +548,6 @@ func (d *ComposerDependencyProxyController) resolveComposerDist(ctx context.Cont
return "", time.Time{}, fmt.Errorf("%s@%s not found upstream", packageName, version)
}
-// checkComposerDistHost keeps a tampered or compromised upstream entry from turning this
-// proxy into a fetcher for arbitrary hosts.
-func checkComposerDistHost(distURL string) error {
- parsed, err := url.Parse(distURL)
- if err != nil {
- return fmt.Errorf("invalid dist url")
- }
- if parsed.Scheme != "https" {
- return fmt.Errorf("dist url is not https")
- }
- for _, host := range composerDistHosts {
- if parsed.Hostname() == host {
- return nil
- }
- }
- return fmt.Errorf("dist host %s is not allowed", parsed.Hostname())
-}
-
func composerCacheTTL(requestPath string) time.Duration {
if strings.Contains(requestPath, ".zip") {
return 168 * time.Hour // 7 days
diff --git a/controllers/dependencyfirewall/controller.go b/controllers/dependencyfirewall/controller.go
index 9493fddac..f99b8d5e7 100644
--- a/controllers/dependencyfirewall/controller.go
+++ b/controllers/dependencyfirewall/controller.go
@@ -25,6 +25,7 @@ import (
"net/url"
"os"
"regexp"
+ "slices"
"strings"
"time"
@@ -59,8 +60,17 @@ type DependencyProxyCache struct {
}
type DependencyProxyConfigs struct {
- Rules []string `json:"rules"`
- MinReleaseAge int `json:"minReleaseAge"` // in hours
+ Rules []string `json:"rules"`
+ MinReleaseAge int `json:"minReleaseAge"` // in hours
+ Registries map[string]string `json:"registries"`
+}
+
+func (c DependencyProxyConfigs) getRegistry(eco ecosystem, defaultRegistry string) string {
+ individualRegistry := c.Registries[eco.name()]
+ if individualRegistry == "" {
+ return defaultRegistry
+ }
+ return individualRegistry
}
type DependencyProxyController struct {
@@ -293,8 +303,9 @@ func (d *DependencyProxyController) LoadConfigsBySecret(c shared.Context, secret
return configs, fmt.Errorf("unexpected config file json type: %T", configFilesJSON)
}
var raw struct {
- Rules string `json:"rules"`
- MinReleaseAge int `json:"minReleaseAge"`
+ Rules string `json:"rules"`
+ MinReleaseAge int `json:"minReleaseAge"`
+ Registries map[string]string `json:"registries"`
}
if err := json.Unmarshal([]byte(s), &raw); err != nil {
return configs, fmt.Errorf("failed to unmarshal config file json into configs: %w", err)
@@ -306,6 +317,7 @@ func (d *DependencyProxyController) LoadConfigsBySecret(c shared.Context, secret
configs.Rules = append(configs.Rules, line)
}
}
+ configs.Registries = raw.Registries
}
return configs, nil
@@ -317,6 +329,20 @@ func (d *DependencyProxyController) GetDependencyProxyConfigs(c shared.Context)
return d.LoadConfigsBySecret(c, c.Param("secret"))
}
+func checkDistHost(distURL string, allowedHosts ...string) error {
+ parsed, err := url.Parse(distURL)
+ if err != nil {
+ return fmt.Errorf("invalid dist url")
+ }
+ if parsed.Scheme != "https" {
+ return fmt.Errorf("dist url is not https")
+ }
+ if !slices.Contains(allowedHosts, parsed.Hostname()) {
+ return fmt.Errorf("dist host %s is not allowed", parsed.Hostname())
+ }
+ return nil
+}
+
// matchPattern matches a packagePurl against a pattern that may contain '*' wildcards.
func matchPattern(pattern, str string) bool {
parts := strings.Split(pattern, "*")
diff --git a/controllers/dependencyfirewall/debian.go b/controllers/dependencyfirewall/debian.go
index d79726c45..3b27c1c92 100644
--- a/controllers/dependencyfirewall/debian.go
+++ b/controllers/dependencyfirewall/debian.go
@@ -174,7 +174,16 @@ func (d *DebDependencyProxyController) proxyDebMetadata(c shared.Context) error
slog.Info("Proxy request", "proxy", "deb", "type", "metadata", "method", c.Request().Method, "path", requestPath)
- data, headers, statusCode, err := d.fetchFromUpstream(ctx, deb, debRegistry, requestPath, c.Request().Header, nil)
+ configs, err := d.GetDependencyProxyConfigs(c)
+ if err != nil {
+ slog.Error("Error getting dependency proxy configs", "error", err)
+ if strings.Contains(err.Error(), "invalid dependency proxy secret") {
+ return echo.NewHTTPError(http.StatusUnauthorized, "dependency proxy secret is required or invalid")
+ }
+ return echo.NewHTTPError(http.StatusInternalServerError, "failed to load dependency proxy configuration")
+ }
+
+ data, headers, statusCode, err := d.fetchFromUpstream(ctx, deb, configs.getRegistry(deb, debRegistry), requestPath, c.Request().Header, nil)
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
@@ -263,7 +272,7 @@ func (d *DebDependencyProxyController) proxyDebPackage(c shared.Context) error {
span.SetAttributes(attribute.Bool("proxy.cache_hit", false))
- data, headers, statusCode, err := d.fetchFromUpstream(ctx, deb, debRegistry, requestPath, c.Request().Header, nil)
+ data, headers, statusCode, err := d.fetchFromUpstream(ctx, deb, configs.getRegistry(deb, debRegistry), requestPath, c.Request().Header, nil)
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
diff --git a/controllers/dependencyfirewall/golang.go b/controllers/dependencyfirewall/golang.go
index 3fcc6127d..6ea5ac42f 100644
--- a/controllers/dependencyfirewall/golang.go
+++ b/controllers/dependencyfirewall/golang.go
@@ -207,7 +207,7 @@ func (d *GoDependencyProxyController) proxyGoExplicitVersion(c shared.Context, c
span.SetAttributes(attribute.Bool("proxy.cache_hit", false))
- data, headers, statusCode, err := d.fetchFromUpstream(ctx, eco, goProxyURL, requestPath, c.Request().Header, nil)
+ data, headers, statusCode, err := d.fetchFromUpstream(ctx, eco, configs.getRegistry(golang, goProxyURL), requestPath, c.Request().Header, nil)
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
@@ -230,7 +230,7 @@ func (d *GoDependencyProxyController) proxyGoExplicitVersion(c shared.Context, c
releaseTimeErr = fmt.Errorf("no release time in .info response")
}
} else {
- releaseTime, releaseTimeErr = d.fetchGoReleaseTime(ctx, moduleName, version)
+ releaseTime, releaseTimeErr = d.fetchGoReleaseTime(ctx, moduleName, version, configs.getRegistry(golang, goProxyURL))
}
if releaseTimeErr != nil {
slog.Warn("Could not determine release time", "proxy", "go", "module", moduleName, "version", version, "error", releaseTimeErr)
@@ -273,8 +273,10 @@ func (d *GoDependencyProxyController) proxyGoLatest(c shared.Context, ctx contex
return d.blockMaliciousPackage(c, eco, requestPath, reason, status)
}
+ registry := configs.getRegistry(golang, goProxyURL)
+
// Fetch from upstream — we need the response to resolve the version before we can check rules.
- data, headers, statusCode, err := d.fetchFromUpstream(ctx, eco, goProxyURL, requestPath, c.Request().Header, nil)
+ data, headers, statusCode, err := d.fetchFromUpstream(ctx, eco, registry, requestPath, c.Request().Header, nil)
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
@@ -292,7 +294,9 @@ func (d *GoDependencyProxyController) proxyGoLatest(c shared.Context, ctx contex
if strings.HasSuffix(requestPath, "/@v/list") && (configs.MinReleaseAge > 0 || len(configs.Rules) > 0) {
filtered, removed := filterGoVersionList(data,
time.Duration(configs.MinReleaseAge)*time.Hour,
- func(version string) (time.Time, error) { return d.fetchGoReleaseTime(ctx, packageName, version) },
+ func(version string) (time.Time, error) {
+ return d.fetchGoReleaseTime(ctx, packageName, version, registry)
+ },
func(version string) bool {
blocked, _ := matchRules(eco.packageIdentifier(packageName, version), configs.Rules)
return !blocked
@@ -360,14 +364,14 @@ func (d *GoDependencyProxyController) proxyGoSumDB(c shared.Context, ctx context
// fetchGoReleaseTime returns the release time of module@version from the version's .info
// file, preferring the cached copy. A fetched .info is cached as well, so the explicit
// .info request and the version list filtering share it.
-func (d *GoDependencyProxyController) fetchGoReleaseTime(ctx context.Context, moduleName, version string) (time.Time, error) {
+func (d *GoDependencyProxyController) fetchGoReleaseTime(ctx context.Context, moduleName, version, registry string) (time.Time, error) {
infoPath := moduleName + "/@v/" + version + ".info"
cacheKey := "go/" + infoPath
if entry, ok := d.cache.Get(cacheKey); ok && !entry.releaseTime.IsZero() {
return entry.releaseTime, nil
}
- data, _, statusCode, err := d.fetchFromUpstream(ctx, golang, goProxyURL, infoPath, http.Header{}, nil)
+ data, _, statusCode, err := d.fetchFromUpstream(ctx, golang, registry, infoPath, http.Header{}, nil)
if err != nil {
return time.Time{}, err
}
diff --git a/controllers/dependencyfirewall/maven.go b/controllers/dependencyfirewall/maven.go
index 005ac7b60..858a181e6 100644
--- a/controllers/dependencyfirewall/maven.go
+++ b/controllers/dependencyfirewall/maven.go
@@ -231,7 +231,7 @@ func (d *MavenDependencyProxyController) proxyMavenPackage(c shared.Context) err
span.SetAttributes(attribute.Bool("proxy.cache_hit", false))
- data, headers, statusCode, err := d.fetchFromUpstream(ctx, maven, mavenRegistry, requestPath, c.Request().Header, nil)
+ data, headers, statusCode, err := d.fetchFromUpstream(ctx, maven, configs.getRegistry(maven, mavenRegistry), requestPath, c.Request().Header, nil)
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
@@ -334,7 +334,9 @@ func (d *MavenDependencyProxyController) proxyMavenMetadata(c shared.Context) er
span.SetAttributes(attribute.Bool("proxy.cache_hit", false))
- data, headers, statusCode, err := d.fetchFromUpstream(ctx, maven, mavenRegistry, requestPath, c.Request().Header, nil)
+ registry := configs.getRegistry(maven, mavenRegistry)
+
+ data, headers, statusCode, err := d.fetchFromUpstream(ctx, maven, registry, requestPath, c.Request().Header, nil)
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
@@ -356,7 +358,9 @@ func (d *MavenDependencyProxyController) proxyMavenMetadata(c shared.Context) er
if version == "" && (configs.MinReleaseAge > 0 || len(configs.Rules) > 0) {
filtered, removed := filterMavenMetadata(data,
time.Duration(configs.MinReleaseAge)*time.Hour,
- func(v string) (time.Time, error) { return d.fetchMavenReleaseTime(ctx, packageName, v) },
+ func(v string) (time.Time, error) {
+ return d.fetchMavenReleaseTime(ctx, packageName, v, registry)
+ },
func(v string) bool {
blocked, _ := matchRules(maven.packageIdentifier(packageName, v), configs.Rules)
return !blocked
@@ -377,7 +381,7 @@ func (d *MavenDependencyProxyController) proxyMavenMetadata(c shared.Context) er
// the version's POM is the only source. The POM is used rather than the JAR because it is
// small and always exists, even for pom-packaging artifacts. The cache key matches the one
// proxyMavenPackage uses, so a POM that was proxied normally is reused here.
-func (d *MavenDependencyProxyController) fetchMavenReleaseTime(ctx context.Context, packageName, version string) (time.Time, error) {
+func (d *MavenDependencyProxyController) fetchMavenReleaseTime(ctx context.Context, packageName, version, registry string) (time.Time, error) {
groupID, artifactID, ok := strings.Cut(packageName, "/")
if !ok {
return time.Time{}, fmt.Errorf("invalid maven coordinates %q", packageName)
@@ -390,7 +394,7 @@ func (d *MavenDependencyProxyController) fetchMavenReleaseTime(ctx context.Conte
return entry.releaseTime, nil
}
- data, headers, statusCode, err := d.fetchFromUpstream(ctx, maven, mavenRegistry, pomPath, http.Header{}, nil)
+ data, headers, statusCode, err := d.fetchFromUpstream(ctx, maven, registry, pomPath, http.Header{}, nil)
if err != nil {
return time.Time{}, err
}
diff --git a/controllers/dependencyfirewall/npm.go b/controllers/dependencyfirewall/npm.go
index 2ac12f842..9cc391810 100644
--- a/controllers/dependencyfirewall/npm.go
+++ b/controllers/dependencyfirewall/npm.go
@@ -187,7 +187,7 @@ func (d *NPMDependencyProxyController) ProxyNPMTarball(c shared.Context) error {
// The tarball itself carries no publish date - it lives in the package metadata.
// Always resolve it, even without MinReleaseAge: the cache is shared across proxy
// secrets, so an entry stored for one config must be checkable under another.
- releaseTime, err := d.fetchNPMReleaseTime(ctx, packageName, version)
+ releaseTime, err := d.fetchNPMReleaseTime(ctx, packageName, version, configs.getRegistry(npm, npmRegistry))
if err != nil {
slog.Warn("Could not determine release time", "proxy", "npm", "package", packageName, "version", version, "error", err)
if configs.MinReleaseAge > 0 {
@@ -200,7 +200,7 @@ func (d *NPMDependencyProxyController) ProxyNPMTarball(c shared.Context) error {
}
}
- data, headers, statusCode, err := d.fetchFromUpstream(ctx, npm, npmRegistry, requestPath, c.Request().Header, nil)
+ data, headers, statusCode, err := d.fetchFromUpstream(ctx, npm, configs.getRegistry(npm, npmRegistry), requestPath, c.Request().Header, nil)
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
@@ -240,7 +240,7 @@ var npmMetadata = utils.NewTTLCache[string](5*time.Minute, 256*1024*1024, func(e
// fetchPackageMetadata returns the full (non-abbreviated) npm package document.
// Only successful responses are cached.
-func (d *NPMDependencyProxyController) fetchPackageMetadata(ctx context.Context, packageName string) ([]byte, http.Header, int, error) {
+func (d *NPMDependencyProxyController) fetchPackageMetadata(ctx context.Context, packageName, registry string) ([]byte, http.Header, int, error) {
cacheKey := "npm/metadata/" + packageName
if entry, ok := npmMetadata.Get(cacheKey); ok {
slog.Debug("Cache hit for metadata", "proxy", "npm", "package", packageName)
@@ -249,7 +249,7 @@ func (d *NPMDependencyProxyController) fetchPackageMetadata(ctx context.Context,
return entry.data, headers, http.StatusOK, nil
}
- data, headers, status, err := d.fetchFromUpstream(ctx, npm, npmRegistry, "/"+packageName, nil, nil)
+ data, headers, status, err := d.fetchFromUpstream(ctx, npm, registry, "/"+packageName, nil, nil)
if err != nil {
return nil, nil, status, fmt.Errorf("failed to fetch metadata from upstream: %w", err)
}
@@ -260,8 +260,8 @@ func (d *NPMDependencyProxyController) fetchPackageMetadata(ctx context.Context,
}
// fetchNPMReleaseTime returns the publish time of packageName@version from the package metadata.
-func (d *NPMDependencyProxyController) fetchNPMReleaseTime(ctx context.Context, packageName, version string) (time.Time, error) {
- metadata, _, status, err := d.fetchPackageMetadata(ctx, packageName)
+func (d *NPMDependencyProxyController) fetchNPMReleaseTime(ctx context.Context, packageName, version, registry string) (time.Time, error) {
+ metadata, _, status, err := d.fetchPackageMetadata(ctx, packageName, registry)
if err != nil {
return time.Time{}, err
}
@@ -336,7 +336,7 @@ func (d *NPMDependencyProxyController) ProxyNPMMetadata(c shared.Context) error
return d.blockMaliciousPackage(c, npm, requestPath, reason, status)
}
- data, headers, statusCode, err := d.fetchPackageMetadata(ctx, packageName)
+ data, headers, statusCode, err := d.fetchPackageMetadata(ctx, packageName, configs.getRegistry(npm, npmRegistry))
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
@@ -396,6 +396,15 @@ func (d *NPMDependencyProxyController) ProxyNPMRegistryAPI(c shared.Context) err
requestPath := npm.trimPrefix(c.Request().URL.Path)
method := c.Request().Method
+ configs, err := d.GetDependencyProxyConfigs(c)
+ if err != nil {
+ slog.Error("Error getting dependency proxy configs", "error", err)
+ if strings.Contains(err.Error(), "invalid dependency proxy secret") {
+ return echo.NewHTTPError(http.StatusUnauthorized, "dependency proxy secret is required or invalid")
+ }
+ return echo.NewHTTPError(http.StatusInternalServerError, "failed to load dependency proxy configuration")
+ }
+
ctx, span := depProxyTracer.Start(c.Request().Context(), "dependency-proxy.npm-registry-api",
trace.WithAttributes(
attribute.String("proxy.ecosystem", "npm"),
@@ -423,7 +432,7 @@ func (d *NPMDependencyProxyController) ProxyNPMRegistryAPI(c shared.Context) err
slog.Info("Proxy request", "proxy", "npm", "type", "registry-api", "method", method, "path", requestPath, "bodySize", len(body))
- data, headers, statusCode, err := d.fetchNPMRegistryAPIFromUpstream(ctx, method, requestPath, c.Request().URL.RawQuery, c.Request().Header, body)
+ data, headers, statusCode, err := d.fetchNPMRegistryAPIFromUpstream(ctx, method, requestPath, c.Request().URL.RawQuery, c.Request().Header, body, configs.getRegistry(npm, npmRegistry))
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
@@ -436,9 +445,9 @@ func (d *NPMDependencyProxyController) ProxyNPMRegistryAPI(c shared.Context) err
// fetchNPMRegistryAPIFromUpstream forwards a registry API request including the
// headers npm relies on (body encoding for audits, Accept for content negotiation).
-func (d *NPMDependencyProxyController) fetchNPMRegistryAPIFromUpstream(ctx context.Context, method, requestPath, rawQuery string, headers http.Header, body []byte) ([]byte, http.Header, int, error) {
+func (d *NPMDependencyProxyController) fetchNPMRegistryAPIFromUpstream(ctx context.Context, method, requestPath, rawQuery string, headers http.Header, body []byte, registry string) ([]byte, http.Header, int, error) {
requestPath = strings.TrimRight(requestPath, "/")
- upstreamURL, err := url.JoinPath(npmRegistry, requestPath)
+ upstreamURL, err := url.JoinPath(registry, requestPath)
if err != nil {
return nil, nil, 0, fmt.Errorf("failed to join URL: %w", err)
}
diff --git a/controllers/dependencyfirewall/python.go b/controllers/dependencyfirewall/python.go
index 1c347683e..69b7e36b9 100644
--- a/controllers/dependencyfirewall/python.go
+++ b/controllers/dependencyfirewall/python.go
@@ -38,15 +38,15 @@ import (
)
const (
- pypiRegistry = "https://pypi.org"
- pypiFilesURL = "https://files.pythonhosted.org"
+ pypiRegistry = "https://pypi.org"
+ pypiFilesHost = "files.pythonhosted.org"
)
var (
pypiProxyPrefixRe = regexp.MustCompile(`^/api/v1/dependency-proxy/(?:[^/]+/)?pypi(?:/|$)`)
pypiFilenameRe = regexp.MustCompile(`^(.+?)-(\d[^-]*?)(?:-.+\.whl|\.zip|\.tar(?:\.gz|\.bz2|\.xz|\.lz|\.lzma)?|\.t[bgx]z|\.tlz)$`)
pypiNameSeparatorRe = regexp.MustCompile(`[-_.]+`)
- pypiAbsoluteURLRe = regexp.MustCompile(`(?:https?:)?//[^/"'\s]+/`)
+ pypiAbsoluteURLRe = regexp.MustCompile(`(?:https?:)?//[^"'\s#]*/`)
)
// PythonDependencyProxyController handles PyPI dependency proxy requests.
@@ -238,12 +238,14 @@ func (d *PythonDependencyProxyController) ProxyPyPIPackage(c shared.Context) err
// Always resolve the release time, even without MinReleaseAge: the cache is shared
// across proxy secrets, so an entry stored for one config must be checkable under
// another. Files are only cached together with a known release time.
- releaseTime, releaseTimeErr := time.Time{}, fmt.Errorf("could not determine package and version from path")
- if packageName != "" {
- releaseTime, releaseTimeErr = d.fetchPyPIReleaseTime(ctx, packageName, version)
+ fileURL, releaseTime, err := d.resolvePyPIFile(ctx, configs.getRegistry(pypi, pypiRegistry), packageName, filepath.Base(requestPath))
+ if err != nil {
+ span.RecordError(err)
+ slog.Warn("Could not resolve file url", "proxy", "pypi", "package", packageName, "version", version, "error", err)
+ return echo.NewHTTPError(http.StatusNotFound, fmt.Sprintf("could not resolve %s", requestPath))
}
- if releaseTimeErr != nil {
- slog.Warn("Could not determine release time", "proxy", "pypi", "package", packageName, "version", version, "error", releaseTimeErr)
+ if releaseTime.IsZero() {
+ slog.Warn("Could not determine release time", "proxy", "pypi", "package", packageName, "version", version)
if configs.MinReleaseAge > 0 {
// Fail closed: without a publish date the MinReleaseAge policy cannot be verified.
return d.blockNotAllowedPackage(c, pypi, requestPath, fmt.Sprintf("Release time of package %s@%s could not be determined", packageName, version))
@@ -252,7 +254,7 @@ func (d *PythonDependencyProxyController) ProxyPyPIPackage(c shared.Context) err
return d.blockTooNewPackage(c, pypi, requestPath, releaseTime, configs.MinReleaseAge)
}
- data, headers, statusCode, err := d.fetchFromUpstream(ctx, pypi, pypiFilesURL, requestPath, c.Request().Header, nil)
+ data, headers, statusCode, err := d.fetchFromUpstream(ctx, pypi, fileURL, "", c.Request().Header, nil)
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
@@ -265,7 +267,7 @@ func (d *PythonDependencyProxyController) ProxyPyPIPackage(c shared.Context) err
return d.passthroughUpstreamResponse(c, headers, statusCode, data)
}
- if releaseTimeErr == nil {
+ if !releaseTime.IsZero() {
if err := d.cache.Set(cacheKey, cacheValue{data: data, releaseTime: releaseTime}); err != nil {
slog.Warn("Failed to cache response", "proxy", "pypi", "error", err)
}
@@ -327,7 +329,6 @@ func (d *PythonDependencyProxyController) ProxyPyPISimple(c shared.Context) erro
return echo.NewHTTPError(http.StatusInternalServerError, "failed to load dependency proxy configuration")
}
-
pkgName := normalizePyPIName(c.Param("package"))
requestPath := pypi.trimPrefix(c.Request().URL.Path)
@@ -364,7 +365,7 @@ func (d *PythonDependencyProxyController) ProxyPyPISimple(c shared.Context) erro
headers := c.Request().Header
headers.Set("Accept", "application/vnd.pypi.simple.v1+json")
- data, headers, statusCode, err := d.fetchPyPIFromUpstream(ctx, requestPath, headers)
+ data, headers, statusCode, err := d.fetchPyPIFromUpstream(ctx, config.getRegistry(pypi, pypiRegistry), requestPath, headers)
if err != nil {
span.RecordError(err)
@@ -397,7 +398,7 @@ func (d *PythonDependencyProxyController) ProxyPyPISimple(c shared.Context) erro
c.Response().Header().Set("Content-Type", contentType)
}
- data = pypiAbsoluteURLRe.ReplaceAllLiteral(data, []byte(pypiProxyPrefixRe.FindString(c.Request().URL.Path)))
+ data = pypiAbsoluteURLRe.ReplaceAllLiteral(data, []byte(pypiProxyPrefixRe.FindString(c.Request().URL.Path)+"packages/"))
return pypi.writeResponse(c, data, requestPath, false)
}
@@ -441,9 +442,9 @@ func filterPyPiSimpleIndex(data []byte, keep func(version string, published time
return filteredData, nil
}
-func (d *PythonDependencyProxyController) fetchPyPIFromUpstream(ctx context.Context, requestPath string, headers http.Header) ([]byte, http.Header, int, error) {
+func (d *PythonDependencyProxyController) fetchPyPIFromUpstream(ctx context.Context, registry, requestPath string, headers http.Header) ([]byte, http.Header, int, error) {
requestPath = strings.TrimRight(requestPath, "/")
- url, err := url.JoinPath(pypiRegistry, requestPath)
+ url, err := url.JoinPath(registry, requestPath)
if err != nil {
return nil, nil, 0, fmt.Errorf("failed to join URL: %w", err)
}
@@ -475,50 +476,59 @@ func (d *PythonDependencyProxyController) fetchPyPIFromUpstream(ctx context.Cont
return data, resp.Header, resp.StatusCode, nil
}
-// ExtractPyPIReleaseTime parses a PyPI JSON API response and returns the resolved version and its upload time.
-// If version is empty, it uses info.version (the current release).
-func extractPyPIReleaseTime(data []byte, version string) (time.Time, error) {
- var metadata struct {
- Info struct {
- Version string `json:"version"`
- } `json:"info"`
- Releases map[string][]struct {
- UploadTime string `json:"upload_time_iso_8601"`
- } `json:"releases"`
- }
- if err := json.Unmarshal(data, &metadata); err != nil {
- return time.Time{}, err
- }
- if version == "" {
- version = metadata.Info.Version
- }
- files, ok := metadata.Releases[version]
- if !ok || len(files) == 0 {
- return time.Time{}, fmt.Errorf("version %s not found in PyPI metadata", version)
- }
- // A version's release time is its earliest upload - wheels are often uploaded later
- // than the sdist. The simple index filter uses the same definition per file.
- var earliest time.Time
- for _, file := range files {
- t, err := time.Parse(time.RFC3339Nano, file.UploadTime)
- if err != nil {
- return time.Time{}, fmt.Errorf("failed to parse upload time for version %s: %w", version, err)
+func (d *PythonDependencyProxyController) resolvePyPIFile(ctx context.Context, registry, packageName, filename string) (string, time.Time, error) {
+ if packageName == "" {
+ return "", time.Time{}, fmt.Errorf("could not determine package from %s", filename)
+ }
+
+ indexURL, err := url.JoinPath(registry, "simple", packageName, "/")
+ if err != nil {
+ return "", time.Time{}, err
+ }
+ data, _, statusCode, err := d.fetchPyPIFromUpstream(ctx, registry, "simple/"+packageName, http.Header{"Accept": {"application/vnd.pypi.simple.v1+json"}})
+ if err != nil {
+ return "", time.Time{}, err
+ }
+ if statusCode != http.StatusOK {
+ return "", time.Time{}, fmt.Errorf("upstream returned status %d for simple index of %s", statusCode, packageName)
+ }
+
+ var index pySimple
+ if err := json.Unmarshal(data, &index); err != nil {
+ return "", time.Time{}, fmt.Errorf("could not decode simple index of %s: %w", packageName, err)
+ }
+
+ name, isMetadata := strings.CutSuffix(filename, ".metadata")
+ _, version := pypi.parsePackage("packages/" + name)
+ var fileURL string
+ var released time.Time
+ for _, file := range index.Files {
+ if file.Filename == name {
+ fileURL = file.URL
}
- if earliest.IsZero() || t.Before(earliest) {
- earliest = t
+ if _, v := pypi.parsePackage("packages/" + file.Filename); v == version && !file.UploadTime.IsZero() && (released.IsZero() || file.UploadTime.Before(released)) {
+ released = file.UploadTime
}
}
- return earliest, nil
-}
+ if fileURL == "" {
+ return "", time.Time{}, fmt.Errorf("%s not found in simple index of %s", name, packageName)
+ }
-// fetchPyPIReleaseTime fetches the PyPI JSON API and returns the resolved version and its release time.
-func (d *PythonDependencyProxyController) fetchPyPIReleaseTime(ctx context.Context, pkgName, version string) (time.Time, error) {
- data, _, statusCode, err := d.fetchPyPIFromUpstream(ctx, "/pypi/"+pkgName+"/json", http.Header{})
+ base, err := url.Parse(indexURL)
if err != nil {
- return time.Time{}, err
+ return "", time.Time{}, err
}
- if statusCode != http.StatusOK {
- return time.Time{}, fmt.Errorf("upstream returned status %d for release metadata", statusCode)
+ resolved, err := base.Parse(fileURL)
+ if err != nil {
+ return "", time.Time{}, err
+ }
+ resolved.Fragment = ""
+ if isMetadata {
+ resolved.Path += ".metadata"
+ }
+
+ if err := checkDistHost(resolved.String(), pypiFilesHost, base.Hostname()); err != nil {
+ return "", time.Time{}, err
}
- return extractPyPIReleaseTime(data, version)
+ return resolved.String(), released, nil
}
diff --git a/controllers/dependencyfirewall/python_test.go b/controllers/dependencyfirewall/python_test.go
index c0ea044d4..c6e06aadd 100644
--- a/controllers/dependencyfirewall/python_test.go
+++ b/controllers/dependencyfirewall/python_test.go
@@ -123,21 +123,21 @@ func TestProxyPyPISimpleRewritesFileURLs(t *testing.T) {
path: "/api/v1/dependency-proxy/pypi/simple/requests/",
contentType: "text/html",
upstream: `requests-2.31.0-py3-none-any.whl`,
- expected: `requests-2.31.0-py3-none-any.whl`,
+ expected: `requests-2.31.0-py3-none-any.whl`,
},
{
name: "json with secret",
path: "/api/v1/dependency-proxy/550e8400-e29b-41d4-a716-446655440000/pypi/simple/requests/",
contentType: "application/vnd.pypi.simple.v1+json",
upstream: `{"files":[{"url":"https://files.pythonhosted.org/packages/ab/cd/requests-2.31.0-py3-none-any.whl"}]}`,
- expected: `{"files":[{"url":"/api/v1/dependency-proxy/550e8400-e29b-41d4-a716-446655440000/pypi/packages/ab/cd/requests-2.31.0-py3-none-any.whl"}]}`,
+ expected: `{"files":[{"url":"/api/v1/dependency-proxy/550e8400-e29b-41d4-a716-446655440000/pypi/packages/requests-2.31.0-py3-none-any.whl"}]}`,
},
{
name: "scheme-relative and foreign host",
path: "/api/v1/dependency-proxy/pypi/simple/requests/",
contentType: "text/html",
upstream: `ab`,
- expected: `ab`,
+ expected: `ab`,
},
}
@@ -253,11 +253,12 @@ type pypiFileTransport struct {
func (p pypiFileTransport) RoundTrip(req *http.Request) (*http.Response, error) {
body := "sdist"
- if req.URL.Path == "/pypi/requests/json" {
- if p.uploadTime.IsZero() {
- return &http.Response{StatusCode: http.StatusNotFound, Body: http.NoBody}, nil
+ if req.URL.Path == "/simple/requests" {
+ uploadTime := ""
+ if !p.uploadTime.IsZero() {
+ uploadTime = fmt.Sprintf(`,"upload-time":%q`, p.uploadTime.Format(time.RFC3339Nano))
}
- body = fmt.Sprintf(`{"releases":{"2.32.3":[{"upload_time_iso_8601":%q}]}}`, p.uploadTime.Format(time.RFC3339Nano))
+ body = fmt.Sprintf(`{"files":[{"filename":"requests-2.32.3.tar.gz","url":"https://files.pythonhosted.org/packages/ab/cd/requests-2.32.3.tar.gz"%s}]}`, uploadTime)
}
return &http.Response{StatusCode: http.StatusOK, Header: http.Header{}, Body: io.NopCloser(strings.NewReader(body))}, nil
}
diff --git a/dtos/dependency_proxy_dto.go b/dtos/dependency_proxy_dto.go
index d91cd69bf..147819b48 100644
--- a/dtos/dependency_proxy_dto.go
+++ b/dtos/dependency_proxy_dto.go
@@ -21,8 +21,9 @@ import (
// DependencyProxyConfig is the config-file payload persisted for the dependency proxy.
type DependencyProxyConfig struct {
- Rules string `json:"rules"`
- MinReleaseAge int `json:"minReleaseAge" validate:"gte=0,lte=87600"` // in hours, capped at 10 years
+ Rules string `json:"rules"`
+ MinReleaseAge int `json:"minReleaseAge" validate:"gte=0,lte=87600"` // in hours, capped at 10 years
+ Registries map[string]string `json:"registries" validate:"omitempty,dive,keys,oneof=npm go pypi maven composer deb,endkeys,required,url"`
}
// dependencyProxyConfigFileID is the key under which dependency-proxy settings