From c0287bdb428e0231867c20317f9b0ed0952311a9 Mon Sep 17 00:00:00 2001 From: Julian Kepka Date: Wed, 7 Oct 2026 15:59:10 +0200 Subject: [PATCH] feat: added possibility for own registry urls for proxy --- controllers/dependencyfirewall/composer.go | 33 ++--- controllers/dependencyfirewall/controller.go | 34 ++++- controllers/dependencyfirewall/debian.go | 13 +- controllers/dependencyfirewall/golang.go | 16 ++- controllers/dependencyfirewall/maven.go | 14 ++- controllers/dependencyfirewall/npm.go | 29 +++-- controllers/dependencyfirewall/python.go | 118 ++++++++++-------- controllers/dependencyfirewall/python_test.go | 15 +-- dtos/dependency_proxy_dto.go | 5 +- 9 files changed, 164 insertions(+), 113 deletions(-) diff --git a/controllers/dependencyfirewall/composer.go b/controllers/dependencyfirewall/composer.go index 6d8e0a86f..739eb8b41 100644 --- a/controllers/dependencyfirewall/composer.go +++ b/controllers/dependencyfirewall/composer.go @@ -250,14 +250,19 @@ func (d *ComposerDependencyProxyController) proxyComposerDist(c shared.Context) } } - distURL, releaseTime, err := d.resolveComposerDist(ctx, packageName, version) + registry := configs.getRegistry(composer, composerMetadataRegistry) + distURL, releaseTime, err := d.resolveComposerDist(ctx, packageName, version, registry) if err != nil { span.RecordError(err) slog.Warn("Could not resolve dist url", "proxy", "composer", "package", packageName, "version", version, "error", err) return echo.NewHTTPError(http.StatusNotFound, fmt.Sprintf("could not resolve %s@%s", packageName, version)) } - if err := checkComposerDistHost(distURL); err != nil { + registryURL, err := url.Parse(registry) + if err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, "invalid composer registry") + } + if err := checkDistHost(distURL, append([]string{registryURL.Hostname()}, composerDistHosts...)...); err != nil { span.RecordError(err) slog.Warn("Blocked dist url", "proxy", "composer", "package", packageName, "version", version, "url", distURL, "error", err) return echo.NewHTTPError(http.StatusForbidden, fmt.Sprintf("blocked dist url: %v", err)) @@ -344,7 +349,7 @@ func (d *ComposerDependencyProxyController) proxyComposerMetadata(c shared.Conte } data, headers, statusCode, err := d.fetchFromUpstream( - ctx, composer, composerMetadataRegistry, requestPath, c.Request().Header, nil, + ctx, composer, configs.getRegistry(composer, composerMetadataRegistry), requestPath, c.Request().Header, nil, ) if err != nil { @@ -494,10 +499,10 @@ func filterComposerMetadata(data []byte, proxyBaseURL string, minAge time.Durati return out, removed, nil } -func (d *ComposerDependencyProxyController) resolveComposerDist(ctx context.Context, packageName, version string) (string, time.Time, error) { +func (d *ComposerDependencyProxyController) resolveComposerDist(ctx context.Context, packageName, version, registry string) (string, time.Time, error) { metadataPath := fmt.Sprintf("p2/%s.json", packageName) - data, _, statusCode, err := d.fetchFromUpstream(ctx, composer, composerMetadataRegistry, metadataPath, http.Header{}, nil) + data, _, statusCode, err := d.fetchFromUpstream(ctx, composer, registry, metadataPath, http.Header{}, nil) if err != nil { return "", time.Time{}, err } @@ -543,24 +548,6 @@ func (d *ComposerDependencyProxyController) resolveComposerDist(ctx context.Cont return "", time.Time{}, fmt.Errorf("%s@%s not found upstream", packageName, version) } -// checkComposerDistHost keeps a tampered or compromised upstream entry from turning this -// proxy into a fetcher for arbitrary hosts. -func checkComposerDistHost(distURL string) error { - parsed, err := url.Parse(distURL) - if err != nil { - return fmt.Errorf("invalid dist url") - } - if parsed.Scheme != "https" { - return fmt.Errorf("dist url is not https") - } - for _, host := range composerDistHosts { - if parsed.Hostname() == host { - return nil - } - } - return fmt.Errorf("dist host %s is not allowed", parsed.Hostname()) -} - func composerCacheTTL(requestPath string) time.Duration { if strings.Contains(requestPath, ".zip") { return 168 * time.Hour // 7 days diff --git a/controllers/dependencyfirewall/controller.go b/controllers/dependencyfirewall/controller.go index 9493fddac..f99b8d5e7 100644 --- a/controllers/dependencyfirewall/controller.go +++ b/controllers/dependencyfirewall/controller.go @@ -25,6 +25,7 @@ import ( "net/url" "os" "regexp" + "slices" "strings" "time" @@ -59,8 +60,17 @@ type DependencyProxyCache struct { } type DependencyProxyConfigs struct { - Rules []string `json:"rules"` - MinReleaseAge int `json:"minReleaseAge"` // in hours + Rules []string `json:"rules"` + MinReleaseAge int `json:"minReleaseAge"` // in hours + Registries map[string]string `json:"registries"` +} + +func (c DependencyProxyConfigs) getRegistry(eco ecosystem, defaultRegistry string) string { + individualRegistry := c.Registries[eco.name()] + if individualRegistry == "" { + return defaultRegistry + } + return individualRegistry } type DependencyProxyController struct { @@ -293,8 +303,9 @@ func (d *DependencyProxyController) LoadConfigsBySecret(c shared.Context, secret return configs, fmt.Errorf("unexpected config file json type: %T", configFilesJSON) } var raw struct { - Rules string `json:"rules"` - MinReleaseAge int `json:"minReleaseAge"` + Rules string `json:"rules"` + MinReleaseAge int `json:"minReleaseAge"` + Registries map[string]string `json:"registries"` } if err := json.Unmarshal([]byte(s), &raw); err != nil { return configs, fmt.Errorf("failed to unmarshal config file json into configs: %w", err) @@ -306,6 +317,7 @@ func (d *DependencyProxyController) LoadConfigsBySecret(c shared.Context, secret configs.Rules = append(configs.Rules, line) } } + configs.Registries = raw.Registries } return configs, nil @@ -317,6 +329,20 @@ func (d *DependencyProxyController) GetDependencyProxyConfigs(c shared.Context) return d.LoadConfigsBySecret(c, c.Param("secret")) } +func checkDistHost(distURL string, allowedHosts ...string) error { + parsed, err := url.Parse(distURL) + if err != nil { + return fmt.Errorf("invalid dist url") + } + if parsed.Scheme != "https" { + return fmt.Errorf("dist url is not https") + } + if !slices.Contains(allowedHosts, parsed.Hostname()) { + return fmt.Errorf("dist host %s is not allowed", parsed.Hostname()) + } + return nil +} + // matchPattern matches a packagePurl against a pattern that may contain '*' wildcards. func matchPattern(pattern, str string) bool { parts := strings.Split(pattern, "*") diff --git a/controllers/dependencyfirewall/debian.go b/controllers/dependencyfirewall/debian.go index d79726c45..3b27c1c92 100644 --- a/controllers/dependencyfirewall/debian.go +++ b/controllers/dependencyfirewall/debian.go @@ -174,7 +174,16 @@ func (d *DebDependencyProxyController) proxyDebMetadata(c shared.Context) error slog.Info("Proxy request", "proxy", "deb", "type", "metadata", "method", c.Request().Method, "path", requestPath) - data, headers, statusCode, err := d.fetchFromUpstream(ctx, deb, debRegistry, requestPath, c.Request().Header, nil) + configs, err := d.GetDependencyProxyConfigs(c) + if err != nil { + slog.Error("Error getting dependency proxy configs", "error", err) + if strings.Contains(err.Error(), "invalid dependency proxy secret") { + return echo.NewHTTPError(http.StatusUnauthorized, "dependency proxy secret is required or invalid") + } + return echo.NewHTTPError(http.StatusInternalServerError, "failed to load dependency proxy configuration") + } + + data, headers, statusCode, err := d.fetchFromUpstream(ctx, deb, configs.getRegistry(deb, debRegistry), requestPath, c.Request().Header, nil) if err != nil { span.RecordError(err) span.SetStatus(codes.Error, err.Error()) @@ -263,7 +272,7 @@ func (d *DebDependencyProxyController) proxyDebPackage(c shared.Context) error { span.SetAttributes(attribute.Bool("proxy.cache_hit", false)) - data, headers, statusCode, err := d.fetchFromUpstream(ctx, deb, debRegistry, requestPath, c.Request().Header, nil) + data, headers, statusCode, err := d.fetchFromUpstream(ctx, deb, configs.getRegistry(deb, debRegistry), requestPath, c.Request().Header, nil) if err != nil { span.RecordError(err) span.SetStatus(codes.Error, err.Error()) diff --git a/controllers/dependencyfirewall/golang.go b/controllers/dependencyfirewall/golang.go index 3fcc6127d..6ea5ac42f 100644 --- a/controllers/dependencyfirewall/golang.go +++ b/controllers/dependencyfirewall/golang.go @@ -207,7 +207,7 @@ func (d *GoDependencyProxyController) proxyGoExplicitVersion(c shared.Context, c span.SetAttributes(attribute.Bool("proxy.cache_hit", false)) - data, headers, statusCode, err := d.fetchFromUpstream(ctx, eco, goProxyURL, requestPath, c.Request().Header, nil) + data, headers, statusCode, err := d.fetchFromUpstream(ctx, eco, configs.getRegistry(golang, goProxyURL), requestPath, c.Request().Header, nil) if err != nil { span.RecordError(err) span.SetStatus(codes.Error, err.Error()) @@ -230,7 +230,7 @@ func (d *GoDependencyProxyController) proxyGoExplicitVersion(c shared.Context, c releaseTimeErr = fmt.Errorf("no release time in .info response") } } else { - releaseTime, releaseTimeErr = d.fetchGoReleaseTime(ctx, moduleName, version) + releaseTime, releaseTimeErr = d.fetchGoReleaseTime(ctx, moduleName, version, configs.getRegistry(golang, goProxyURL)) } if releaseTimeErr != nil { slog.Warn("Could not determine release time", "proxy", "go", "module", moduleName, "version", version, "error", releaseTimeErr) @@ -273,8 +273,10 @@ func (d *GoDependencyProxyController) proxyGoLatest(c shared.Context, ctx contex return d.blockMaliciousPackage(c, eco, requestPath, reason, status) } + registry := configs.getRegistry(golang, goProxyURL) + // Fetch from upstream — we need the response to resolve the version before we can check rules. - data, headers, statusCode, err := d.fetchFromUpstream(ctx, eco, goProxyURL, requestPath, c.Request().Header, nil) + data, headers, statusCode, err := d.fetchFromUpstream(ctx, eco, registry, requestPath, c.Request().Header, nil) if err != nil { span.RecordError(err) span.SetStatus(codes.Error, err.Error()) @@ -292,7 +294,9 @@ func (d *GoDependencyProxyController) proxyGoLatest(c shared.Context, ctx contex if strings.HasSuffix(requestPath, "/@v/list") && (configs.MinReleaseAge > 0 || len(configs.Rules) > 0) { filtered, removed := filterGoVersionList(data, time.Duration(configs.MinReleaseAge)*time.Hour, - func(version string) (time.Time, error) { return d.fetchGoReleaseTime(ctx, packageName, version) }, + func(version string) (time.Time, error) { + return d.fetchGoReleaseTime(ctx, packageName, version, registry) + }, func(version string) bool { blocked, _ := matchRules(eco.packageIdentifier(packageName, version), configs.Rules) return !blocked @@ -360,14 +364,14 @@ func (d *GoDependencyProxyController) proxyGoSumDB(c shared.Context, ctx context // fetchGoReleaseTime returns the release time of module@version from the version's .info // file, preferring the cached copy. A fetched .info is cached as well, so the explicit // .info request and the version list filtering share it. -func (d *GoDependencyProxyController) fetchGoReleaseTime(ctx context.Context, moduleName, version string) (time.Time, error) { +func (d *GoDependencyProxyController) fetchGoReleaseTime(ctx context.Context, moduleName, version, registry string) (time.Time, error) { infoPath := moduleName + "/@v/" + version + ".info" cacheKey := "go/" + infoPath if entry, ok := d.cache.Get(cacheKey); ok && !entry.releaseTime.IsZero() { return entry.releaseTime, nil } - data, _, statusCode, err := d.fetchFromUpstream(ctx, golang, goProxyURL, infoPath, http.Header{}, nil) + data, _, statusCode, err := d.fetchFromUpstream(ctx, golang, registry, infoPath, http.Header{}, nil) if err != nil { return time.Time{}, err } diff --git a/controllers/dependencyfirewall/maven.go b/controllers/dependencyfirewall/maven.go index 005ac7b60..858a181e6 100644 --- a/controllers/dependencyfirewall/maven.go +++ b/controllers/dependencyfirewall/maven.go @@ -231,7 +231,7 @@ func (d *MavenDependencyProxyController) proxyMavenPackage(c shared.Context) err span.SetAttributes(attribute.Bool("proxy.cache_hit", false)) - data, headers, statusCode, err := d.fetchFromUpstream(ctx, maven, mavenRegistry, requestPath, c.Request().Header, nil) + data, headers, statusCode, err := d.fetchFromUpstream(ctx, maven, configs.getRegistry(maven, mavenRegistry), requestPath, c.Request().Header, nil) if err != nil { span.RecordError(err) span.SetStatus(codes.Error, err.Error()) @@ -334,7 +334,9 @@ func (d *MavenDependencyProxyController) proxyMavenMetadata(c shared.Context) er span.SetAttributes(attribute.Bool("proxy.cache_hit", false)) - data, headers, statusCode, err := d.fetchFromUpstream(ctx, maven, mavenRegistry, requestPath, c.Request().Header, nil) + registry := configs.getRegistry(maven, mavenRegistry) + + data, headers, statusCode, err := d.fetchFromUpstream(ctx, maven, registry, requestPath, c.Request().Header, nil) if err != nil { span.RecordError(err) span.SetStatus(codes.Error, err.Error()) @@ -356,7 +358,9 @@ func (d *MavenDependencyProxyController) proxyMavenMetadata(c shared.Context) er if version == "" && (configs.MinReleaseAge > 0 || len(configs.Rules) > 0) { filtered, removed := filterMavenMetadata(data, time.Duration(configs.MinReleaseAge)*time.Hour, - func(v string) (time.Time, error) { return d.fetchMavenReleaseTime(ctx, packageName, v) }, + func(v string) (time.Time, error) { + return d.fetchMavenReleaseTime(ctx, packageName, v, registry) + }, func(v string) bool { blocked, _ := matchRules(maven.packageIdentifier(packageName, v), configs.Rules) return !blocked @@ -377,7 +381,7 @@ func (d *MavenDependencyProxyController) proxyMavenMetadata(c shared.Context) er // the version's POM is the only source. The POM is used rather than the JAR because it is // small and always exists, even for pom-packaging artifacts. The cache key matches the one // proxyMavenPackage uses, so a POM that was proxied normally is reused here. -func (d *MavenDependencyProxyController) fetchMavenReleaseTime(ctx context.Context, packageName, version string) (time.Time, error) { +func (d *MavenDependencyProxyController) fetchMavenReleaseTime(ctx context.Context, packageName, version, registry string) (time.Time, error) { groupID, artifactID, ok := strings.Cut(packageName, "/") if !ok { return time.Time{}, fmt.Errorf("invalid maven coordinates %q", packageName) @@ -390,7 +394,7 @@ func (d *MavenDependencyProxyController) fetchMavenReleaseTime(ctx context.Conte return entry.releaseTime, nil } - data, headers, statusCode, err := d.fetchFromUpstream(ctx, maven, mavenRegistry, pomPath, http.Header{}, nil) + data, headers, statusCode, err := d.fetchFromUpstream(ctx, maven, registry, pomPath, http.Header{}, nil) if err != nil { return time.Time{}, err } diff --git a/controllers/dependencyfirewall/npm.go b/controllers/dependencyfirewall/npm.go index 2ac12f842..9cc391810 100644 --- a/controllers/dependencyfirewall/npm.go +++ b/controllers/dependencyfirewall/npm.go @@ -187,7 +187,7 @@ func (d *NPMDependencyProxyController) ProxyNPMTarball(c shared.Context) error { // The tarball itself carries no publish date - it lives in the package metadata. // Always resolve it, even without MinReleaseAge: the cache is shared across proxy // secrets, so an entry stored for one config must be checkable under another. - releaseTime, err := d.fetchNPMReleaseTime(ctx, packageName, version) + releaseTime, err := d.fetchNPMReleaseTime(ctx, packageName, version, configs.getRegistry(npm, npmRegistry)) if err != nil { slog.Warn("Could not determine release time", "proxy", "npm", "package", packageName, "version", version, "error", err) if configs.MinReleaseAge > 0 { @@ -200,7 +200,7 @@ func (d *NPMDependencyProxyController) ProxyNPMTarball(c shared.Context) error { } } - data, headers, statusCode, err := d.fetchFromUpstream(ctx, npm, npmRegistry, requestPath, c.Request().Header, nil) + data, headers, statusCode, err := d.fetchFromUpstream(ctx, npm, configs.getRegistry(npm, npmRegistry), requestPath, c.Request().Header, nil) if err != nil { span.RecordError(err) span.SetStatus(codes.Error, err.Error()) @@ -240,7 +240,7 @@ var npmMetadata = utils.NewTTLCache[string](5*time.Minute, 256*1024*1024, func(e // fetchPackageMetadata returns the full (non-abbreviated) npm package document. // Only successful responses are cached. -func (d *NPMDependencyProxyController) fetchPackageMetadata(ctx context.Context, packageName string) ([]byte, http.Header, int, error) { +func (d *NPMDependencyProxyController) fetchPackageMetadata(ctx context.Context, packageName, registry string) ([]byte, http.Header, int, error) { cacheKey := "npm/metadata/" + packageName if entry, ok := npmMetadata.Get(cacheKey); ok { slog.Debug("Cache hit for metadata", "proxy", "npm", "package", packageName) @@ -249,7 +249,7 @@ func (d *NPMDependencyProxyController) fetchPackageMetadata(ctx context.Context, return entry.data, headers, http.StatusOK, nil } - data, headers, status, err := d.fetchFromUpstream(ctx, npm, npmRegistry, "/"+packageName, nil, nil) + data, headers, status, err := d.fetchFromUpstream(ctx, npm, registry, "/"+packageName, nil, nil) if err != nil { return nil, nil, status, fmt.Errorf("failed to fetch metadata from upstream: %w", err) } @@ -260,8 +260,8 @@ func (d *NPMDependencyProxyController) fetchPackageMetadata(ctx context.Context, } // fetchNPMReleaseTime returns the publish time of packageName@version from the package metadata. -func (d *NPMDependencyProxyController) fetchNPMReleaseTime(ctx context.Context, packageName, version string) (time.Time, error) { - metadata, _, status, err := d.fetchPackageMetadata(ctx, packageName) +func (d *NPMDependencyProxyController) fetchNPMReleaseTime(ctx context.Context, packageName, version, registry string) (time.Time, error) { + metadata, _, status, err := d.fetchPackageMetadata(ctx, packageName, registry) if err != nil { return time.Time{}, err } @@ -336,7 +336,7 @@ func (d *NPMDependencyProxyController) ProxyNPMMetadata(c shared.Context) error return d.blockMaliciousPackage(c, npm, requestPath, reason, status) } - data, headers, statusCode, err := d.fetchPackageMetadata(ctx, packageName) + data, headers, statusCode, err := d.fetchPackageMetadata(ctx, packageName, configs.getRegistry(npm, npmRegistry)) if err != nil { span.RecordError(err) span.SetStatus(codes.Error, err.Error()) @@ -396,6 +396,15 @@ func (d *NPMDependencyProxyController) ProxyNPMRegistryAPI(c shared.Context) err requestPath := npm.trimPrefix(c.Request().URL.Path) method := c.Request().Method + configs, err := d.GetDependencyProxyConfigs(c) + if err != nil { + slog.Error("Error getting dependency proxy configs", "error", err) + if strings.Contains(err.Error(), "invalid dependency proxy secret") { + return echo.NewHTTPError(http.StatusUnauthorized, "dependency proxy secret is required or invalid") + } + return echo.NewHTTPError(http.StatusInternalServerError, "failed to load dependency proxy configuration") + } + ctx, span := depProxyTracer.Start(c.Request().Context(), "dependency-proxy.npm-registry-api", trace.WithAttributes( attribute.String("proxy.ecosystem", "npm"), @@ -423,7 +432,7 @@ func (d *NPMDependencyProxyController) ProxyNPMRegistryAPI(c shared.Context) err slog.Info("Proxy request", "proxy", "npm", "type", "registry-api", "method", method, "path", requestPath, "bodySize", len(body)) - data, headers, statusCode, err := d.fetchNPMRegistryAPIFromUpstream(ctx, method, requestPath, c.Request().URL.RawQuery, c.Request().Header, body) + data, headers, statusCode, err := d.fetchNPMRegistryAPIFromUpstream(ctx, method, requestPath, c.Request().URL.RawQuery, c.Request().Header, body, configs.getRegistry(npm, npmRegistry)) if err != nil { span.RecordError(err) span.SetStatus(codes.Error, err.Error()) @@ -436,9 +445,9 @@ func (d *NPMDependencyProxyController) ProxyNPMRegistryAPI(c shared.Context) err // fetchNPMRegistryAPIFromUpstream forwards a registry API request including the // headers npm relies on (body encoding for audits, Accept for content negotiation). -func (d *NPMDependencyProxyController) fetchNPMRegistryAPIFromUpstream(ctx context.Context, method, requestPath, rawQuery string, headers http.Header, body []byte) ([]byte, http.Header, int, error) { +func (d *NPMDependencyProxyController) fetchNPMRegistryAPIFromUpstream(ctx context.Context, method, requestPath, rawQuery string, headers http.Header, body []byte, registry string) ([]byte, http.Header, int, error) { requestPath = strings.TrimRight(requestPath, "/") - upstreamURL, err := url.JoinPath(npmRegistry, requestPath) + upstreamURL, err := url.JoinPath(registry, requestPath) if err != nil { return nil, nil, 0, fmt.Errorf("failed to join URL: %w", err) } diff --git a/controllers/dependencyfirewall/python.go b/controllers/dependencyfirewall/python.go index 1c347683e..69b7e36b9 100644 --- a/controllers/dependencyfirewall/python.go +++ b/controllers/dependencyfirewall/python.go @@ -38,15 +38,15 @@ import ( ) const ( - pypiRegistry = "https://pypi.org" - pypiFilesURL = "https://files.pythonhosted.org" + pypiRegistry = "https://pypi.org" + pypiFilesHost = "files.pythonhosted.org" ) var ( pypiProxyPrefixRe = regexp.MustCompile(`^/api/v1/dependency-proxy/(?:[^/]+/)?pypi(?:/|$)`) pypiFilenameRe = regexp.MustCompile(`^(.+?)-(\d[^-]*?)(?:-.+\.whl|\.zip|\.tar(?:\.gz|\.bz2|\.xz|\.lz|\.lzma)?|\.t[bgx]z|\.tlz)$`) pypiNameSeparatorRe = regexp.MustCompile(`[-_.]+`) - pypiAbsoluteURLRe = regexp.MustCompile(`(?:https?:)?//[^/"'\s]+/`) + pypiAbsoluteURLRe = regexp.MustCompile(`(?:https?:)?//[^"'\s#]*/`) ) // PythonDependencyProxyController handles PyPI dependency proxy requests. @@ -238,12 +238,14 @@ func (d *PythonDependencyProxyController) ProxyPyPIPackage(c shared.Context) err // Always resolve the release time, even without MinReleaseAge: the cache is shared // across proxy secrets, so an entry stored for one config must be checkable under // another. Files are only cached together with a known release time. - releaseTime, releaseTimeErr := time.Time{}, fmt.Errorf("could not determine package and version from path") - if packageName != "" { - releaseTime, releaseTimeErr = d.fetchPyPIReleaseTime(ctx, packageName, version) + fileURL, releaseTime, err := d.resolvePyPIFile(ctx, configs.getRegistry(pypi, pypiRegistry), packageName, filepath.Base(requestPath)) + if err != nil { + span.RecordError(err) + slog.Warn("Could not resolve file url", "proxy", "pypi", "package", packageName, "version", version, "error", err) + return echo.NewHTTPError(http.StatusNotFound, fmt.Sprintf("could not resolve %s", requestPath)) } - if releaseTimeErr != nil { - slog.Warn("Could not determine release time", "proxy", "pypi", "package", packageName, "version", version, "error", releaseTimeErr) + if releaseTime.IsZero() { + slog.Warn("Could not determine release time", "proxy", "pypi", "package", packageName, "version", version) if configs.MinReleaseAge > 0 { // Fail closed: without a publish date the MinReleaseAge policy cannot be verified. return d.blockNotAllowedPackage(c, pypi, requestPath, fmt.Sprintf("Release time of package %s@%s could not be determined", packageName, version)) @@ -252,7 +254,7 @@ func (d *PythonDependencyProxyController) ProxyPyPIPackage(c shared.Context) err return d.blockTooNewPackage(c, pypi, requestPath, releaseTime, configs.MinReleaseAge) } - data, headers, statusCode, err := d.fetchFromUpstream(ctx, pypi, pypiFilesURL, requestPath, c.Request().Header, nil) + data, headers, statusCode, err := d.fetchFromUpstream(ctx, pypi, fileURL, "", c.Request().Header, nil) if err != nil { span.RecordError(err) span.SetStatus(codes.Error, err.Error()) @@ -265,7 +267,7 @@ func (d *PythonDependencyProxyController) ProxyPyPIPackage(c shared.Context) err return d.passthroughUpstreamResponse(c, headers, statusCode, data) } - if releaseTimeErr == nil { + if !releaseTime.IsZero() { if err := d.cache.Set(cacheKey, cacheValue{data: data, releaseTime: releaseTime}); err != nil { slog.Warn("Failed to cache response", "proxy", "pypi", "error", err) } @@ -327,7 +329,6 @@ func (d *PythonDependencyProxyController) ProxyPyPISimple(c shared.Context) erro return echo.NewHTTPError(http.StatusInternalServerError, "failed to load dependency proxy configuration") } - pkgName := normalizePyPIName(c.Param("package")) requestPath := pypi.trimPrefix(c.Request().URL.Path) @@ -364,7 +365,7 @@ func (d *PythonDependencyProxyController) ProxyPyPISimple(c shared.Context) erro headers := c.Request().Header headers.Set("Accept", "application/vnd.pypi.simple.v1+json") - data, headers, statusCode, err := d.fetchPyPIFromUpstream(ctx, requestPath, headers) + data, headers, statusCode, err := d.fetchPyPIFromUpstream(ctx, config.getRegistry(pypi, pypiRegistry), requestPath, headers) if err != nil { span.RecordError(err) @@ -397,7 +398,7 @@ func (d *PythonDependencyProxyController) ProxyPyPISimple(c shared.Context) erro c.Response().Header().Set("Content-Type", contentType) } - data = pypiAbsoluteURLRe.ReplaceAllLiteral(data, []byte(pypiProxyPrefixRe.FindString(c.Request().URL.Path))) + data = pypiAbsoluteURLRe.ReplaceAllLiteral(data, []byte(pypiProxyPrefixRe.FindString(c.Request().URL.Path)+"packages/")) return pypi.writeResponse(c, data, requestPath, false) } @@ -441,9 +442,9 @@ func filterPyPiSimpleIndex(data []byte, keep func(version string, published time return filteredData, nil } -func (d *PythonDependencyProxyController) fetchPyPIFromUpstream(ctx context.Context, requestPath string, headers http.Header) ([]byte, http.Header, int, error) { +func (d *PythonDependencyProxyController) fetchPyPIFromUpstream(ctx context.Context, registry, requestPath string, headers http.Header) ([]byte, http.Header, int, error) { requestPath = strings.TrimRight(requestPath, "/") - url, err := url.JoinPath(pypiRegistry, requestPath) + url, err := url.JoinPath(registry, requestPath) if err != nil { return nil, nil, 0, fmt.Errorf("failed to join URL: %w", err) } @@ -475,50 +476,59 @@ func (d *PythonDependencyProxyController) fetchPyPIFromUpstream(ctx context.Cont return data, resp.Header, resp.StatusCode, nil } -// ExtractPyPIReleaseTime parses a PyPI JSON API response and returns the resolved version and its upload time. -// If version is empty, it uses info.version (the current release). -func extractPyPIReleaseTime(data []byte, version string) (time.Time, error) { - var metadata struct { - Info struct { - Version string `json:"version"` - } `json:"info"` - Releases map[string][]struct { - UploadTime string `json:"upload_time_iso_8601"` - } `json:"releases"` - } - if err := json.Unmarshal(data, &metadata); err != nil { - return time.Time{}, err - } - if version == "" { - version = metadata.Info.Version - } - files, ok := metadata.Releases[version] - if !ok || len(files) == 0 { - return time.Time{}, fmt.Errorf("version %s not found in PyPI metadata", version) - } - // A version's release time is its earliest upload - wheels are often uploaded later - // than the sdist. The simple index filter uses the same definition per file. - var earliest time.Time - for _, file := range files { - t, err := time.Parse(time.RFC3339Nano, file.UploadTime) - if err != nil { - return time.Time{}, fmt.Errorf("failed to parse upload time for version %s: %w", version, err) +func (d *PythonDependencyProxyController) resolvePyPIFile(ctx context.Context, registry, packageName, filename string) (string, time.Time, error) { + if packageName == "" { + return "", time.Time{}, fmt.Errorf("could not determine package from %s", filename) + } + + indexURL, err := url.JoinPath(registry, "simple", packageName, "/") + if err != nil { + return "", time.Time{}, err + } + data, _, statusCode, err := d.fetchPyPIFromUpstream(ctx, registry, "simple/"+packageName, http.Header{"Accept": {"application/vnd.pypi.simple.v1+json"}}) + if err != nil { + return "", time.Time{}, err + } + if statusCode != http.StatusOK { + return "", time.Time{}, fmt.Errorf("upstream returned status %d for simple index of %s", statusCode, packageName) + } + + var index pySimple + if err := json.Unmarshal(data, &index); err != nil { + return "", time.Time{}, fmt.Errorf("could not decode simple index of %s: %w", packageName, err) + } + + name, isMetadata := strings.CutSuffix(filename, ".metadata") + _, version := pypi.parsePackage("packages/" + name) + var fileURL string + var released time.Time + for _, file := range index.Files { + if file.Filename == name { + fileURL = file.URL } - if earliest.IsZero() || t.Before(earliest) { - earliest = t + if _, v := pypi.parsePackage("packages/" + file.Filename); v == version && !file.UploadTime.IsZero() && (released.IsZero() || file.UploadTime.Before(released)) { + released = file.UploadTime } } - return earliest, nil -} + if fileURL == "" { + return "", time.Time{}, fmt.Errorf("%s not found in simple index of %s", name, packageName) + } -// fetchPyPIReleaseTime fetches the PyPI JSON API and returns the resolved version and its release time. -func (d *PythonDependencyProxyController) fetchPyPIReleaseTime(ctx context.Context, pkgName, version string) (time.Time, error) { - data, _, statusCode, err := d.fetchPyPIFromUpstream(ctx, "/pypi/"+pkgName+"/json", http.Header{}) + base, err := url.Parse(indexURL) if err != nil { - return time.Time{}, err + return "", time.Time{}, err } - if statusCode != http.StatusOK { - return time.Time{}, fmt.Errorf("upstream returned status %d for release metadata", statusCode) + resolved, err := base.Parse(fileURL) + if err != nil { + return "", time.Time{}, err + } + resolved.Fragment = "" + if isMetadata { + resolved.Path += ".metadata" + } + + if err := checkDistHost(resolved.String(), pypiFilesHost, base.Hostname()); err != nil { + return "", time.Time{}, err } - return extractPyPIReleaseTime(data, version) + return resolved.String(), released, nil } diff --git a/controllers/dependencyfirewall/python_test.go b/controllers/dependencyfirewall/python_test.go index c0ea044d4..c6e06aadd 100644 --- a/controllers/dependencyfirewall/python_test.go +++ b/controllers/dependencyfirewall/python_test.go @@ -123,21 +123,21 @@ func TestProxyPyPISimpleRewritesFileURLs(t *testing.T) { path: "/api/v1/dependency-proxy/pypi/simple/requests/", contentType: "text/html", upstream: `requests-2.31.0-py3-none-any.whl`, - expected: `requests-2.31.0-py3-none-any.whl`, + expected: `requests-2.31.0-py3-none-any.whl`, }, { name: "json with secret", path: "/api/v1/dependency-proxy/550e8400-e29b-41d4-a716-446655440000/pypi/simple/requests/", contentType: "application/vnd.pypi.simple.v1+json", upstream: `{"files":[{"url":"https://files.pythonhosted.org/packages/ab/cd/requests-2.31.0-py3-none-any.whl"}]}`, - expected: `{"files":[{"url":"/api/v1/dependency-proxy/550e8400-e29b-41d4-a716-446655440000/pypi/packages/ab/cd/requests-2.31.0-py3-none-any.whl"}]}`, + expected: `{"files":[{"url":"/api/v1/dependency-proxy/550e8400-e29b-41d4-a716-446655440000/pypi/packages/requests-2.31.0-py3-none-any.whl"}]}`, }, { name: "scheme-relative and foreign host", path: "/api/v1/dependency-proxy/pypi/simple/requests/", contentType: "text/html", upstream: `ab`, - expected: `ab`, + expected: `ab`, }, } @@ -253,11 +253,12 @@ type pypiFileTransport struct { func (p pypiFileTransport) RoundTrip(req *http.Request) (*http.Response, error) { body := "sdist" - if req.URL.Path == "/pypi/requests/json" { - if p.uploadTime.IsZero() { - return &http.Response{StatusCode: http.StatusNotFound, Body: http.NoBody}, nil + if req.URL.Path == "/simple/requests" { + uploadTime := "" + if !p.uploadTime.IsZero() { + uploadTime = fmt.Sprintf(`,"upload-time":%q`, p.uploadTime.Format(time.RFC3339Nano)) } - body = fmt.Sprintf(`{"releases":{"2.32.3":[{"upload_time_iso_8601":%q}]}}`, p.uploadTime.Format(time.RFC3339Nano)) + body = fmt.Sprintf(`{"files":[{"filename":"requests-2.32.3.tar.gz","url":"https://files.pythonhosted.org/packages/ab/cd/requests-2.32.3.tar.gz"%s}]}`, uploadTime) } return &http.Response{StatusCode: http.StatusOK, Header: http.Header{}, Body: io.NopCloser(strings.NewReader(body))}, nil } diff --git a/dtos/dependency_proxy_dto.go b/dtos/dependency_proxy_dto.go index d91cd69bf..147819b48 100644 --- a/dtos/dependency_proxy_dto.go +++ b/dtos/dependency_proxy_dto.go @@ -21,8 +21,9 @@ import ( // DependencyProxyConfig is the config-file payload persisted for the dependency proxy. type DependencyProxyConfig struct { - Rules string `json:"rules"` - MinReleaseAge int `json:"minReleaseAge" validate:"gte=0,lte=87600"` // in hours, capped at 10 years + Rules string `json:"rules"` + MinReleaseAge int `json:"minReleaseAge" validate:"gte=0,lte=87600"` // in hours, capped at 10 years + Registries map[string]string `json:"registries" validate:"omitempty,dive,keys,oneof=npm go pypi maven composer deb,endkeys,required,url"` } // dependencyProxyConfigFileID is the key under which dependency-proxy settings