diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..f50fc47 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +.git +.github +charts +docs +*.md +config.yaml diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml new file mode 100644 index 0000000..7351921 --- /dev/null +++ b/.github/workflows/verify.yml @@ -0,0 +1,64 @@ +name: Verify + +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + +jobs: + go: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-go@v7 + with: + go-version-file: go.mod + - run: go test ./... + - run: go vet ./... + - run: go build ./cmd/minigate + - uses: azure/setup-helm@v4 + - run: helm lint charts/minigate --set existingSecret=minigate-config + - run: helm template minigate charts/minigate --set existingSecret=minigate-config --set replicaCount=3 --set ingress.enabled=true --set ingress.host=files.example.com --set ingress.tlsSecretName=files-tls --set serviceMonitor.enabled=true > /dev/null + - if: github.event_name == 'pull_request' + run: docker build -t minigate:ci . + + release: + needs: go + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + permissions: + contents: write + packages: write + concurrency: + group: minigate-release + cancel-in-progress: false + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + - uses: actions/setup-node@v4 + with: + node-version: '24' + - uses: azure/setup-helm@v4 + - uses: docker/setup-buildx-action@v3 + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Publish release + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + IMAGE_NAME: ghcr.io/ehsanmsb/minigate + run: | + npx --yes \ + -p semantic-release@25 \ + -p @semantic-release/commit-analyzer@13 \ + -p @semantic-release/release-notes-generator@14 \ + -p @semantic-release/exec@7 \ + -p @semantic-release/github@12 \ + -p conventional-changelog-conventionalcommits@8 \ + semantic-release diff --git a/.releaserc.yml b/.releaserc.yml new file mode 100644 index 0000000..d1da3e3 --- /dev/null +++ b/.releaserc.yml @@ -0,0 +1,31 @@ +branches: + - main +tagFormat: v${version} +plugins: + - - '@semantic-release/commit-analyzer' + - preset: conventionalcommits + releaseRules: + - breaking: true + release: major + - type: feat + release: minor + - type: '*' + release: patch + - - '@semantic-release/release-notes-generator' + - preset: conventionalcommits + - - '@semantic-release/exec' + - analyzeCommitsCmd: "printf patch" + prepareCmd: | + set -eu + version="${nextRelease.version}" + mkdir -p dist + helm package charts/minigate --destination dist --version "$version" --app-version "$version" + docker buildx build --platform linux/amd64,linux/arm64 --push \ + --tag "$IMAGE_NAME:$version" \ + --tag "$IMAGE_NAME:latest" \ + --label "org.opencontainers.image.source=https://github.com/$GITHUB_REPOSITORY" \ + --label "org.opencontainers.image.version=$version" \ + . + - - '@semantic-release/github' + - assets: + - dist/*.tgz diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..31386a4 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,15 @@ +FROM golang:1.25.3-alpine AS build + +WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download +COPY cmd ./cmd +COPY internal ./internal +RUN CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o /minigate ./cmd/minigate + +FROM alpine:3.22 +RUN apk add --no-cache ca-certificates && adduser -D -u 10001 minigate +USER minigate +COPY --from=build /minigate /usr/local/bin/minigate +EXPOSE 8080 9090 +ENTRYPOINT ["/usr/local/bin/minigate"] diff --git a/README.md b/README.md index c8a7d9c..67ce98d 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # MiniGate +[Documentation](docs/README.md) · [Kubernetes deployment](docs/deployment.md) · [Releases](docs/releases.md) + MiniGate exposes selected private S3-compatible buckets through public, read-only URLs. Each configured name has its own storage endpoint and credentials. The only object operation is `GetObject`. ```text @@ -40,7 +42,11 @@ Give each configured credential `GetObject` permission only for the intended pre go run ./cmd/minigate -config config.yaml -addr :8080 ``` -The service accepts `GET /{name}/{object-key}` and `GET /health`. Other methods return 405. A missing object returns 404; storage failures return 502. `Range` requests are forwarded to storage for partial downloads. S3 content metadata is forwarded in the response. +Prometheus metrics are served separately at `127.0.0.1:9090/metrics` by default. Set `-metrics-addr` to change the listener. Metrics include `minigate_configured_buckets`, request counts and durations by HTTP status and method, plus Go runtime and process metrics. Health probes are excluded from request counts. Object keys and bucket names are not metric labels. Prometheus itself supplies `up` for each scrape target; `GET /health` remains the Kubernetes health check. + +The service accepts `GET /{name}/{object-key}` and `GET /health`. Other methods return 405. A missing or inaccessible object returns 404; storage failures return 502. S3 can return `AccessDenied` for a missing object when credentials lack `ListBucket`, so MiniGate maps that error to 404. `Range` requests are forwarded to storage for partial downloads. S3 content metadata is forwarded in the response. + +Logs are JSON on standard output. `MINIGATE_LOG_LEVEL` accepts `DEBUG`, `INFO`, `WARN`, or `ERROR` (default `WARN`). Unexpected storage and server failures are logged by default. Set `INFO` to also see access denials and invalid ranges, or `DEBUG` for one compact record per object request, including successful reads and expected 404s. Health probes are not logged. Routine request logs omit object keys and credentials. HTTP routing uses [chi](https://github.com/go-chi/chi); storage access uses the official AWS SDK for Go v2. The HTTP server itself uses Go's standard `net/http` package. diff --git a/charts/minigate/Chart.yaml b/charts/minigate/Chart.yaml new file mode 100644 index 0000000..065793b --- /dev/null +++ b/charts/minigate/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: minigate +description: Read-only gateway for private S3-compatible buckets +type: application +version: 1.0.0 +appVersion: "1.0.0" diff --git a/charts/minigate/templates/_helpers.tpl b/charts/minigate/templates/_helpers.tpl new file mode 100644 index 0000000..8d8b2c8 --- /dev/null +++ b/charts/minigate/templates/_helpers.tpl @@ -0,0 +1,9 @@ +{{- define "minigate.fullname" -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- define "minigate.labels" -}} +app.kubernetes.io/name: minigate +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end -}} diff --git a/charts/minigate/templates/deployment.yaml b/charts/minigate/templates/deployment.yaml new file mode 100644 index 0000000..539f076 --- /dev/null +++ b/charts/minigate/templates/deployment.yaml @@ -0,0 +1,64 @@ +{{- if not .Values.existingSecret }} +{{- fail "existingSecret must name a Secret containing config.yaml" }} +{{- end }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "minigate.fullname" . }} + labels: + {{- include "minigate.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + app.kubernetes.io/name: minigate + app.kubernetes.io/instance: {{ .Release.Name }} + template: + metadata: + labels: + {{- include "minigate.labels" . | nindent 8 }} + spec: + terminationGracePeriodSeconds: 35 + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: minigate + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: ["-config", "/etc/minigate/config.yaml", "-metrics-addr", ":9090"] + env: + - name: MINIGATE_LOG_LEVEL + value: {{ .Values.logLevel | quote }} + ports: + - name: http + containerPort: 8080 + - name: metrics + containerPort: 9090 + readinessProbe: + httpGet: + path: /health + port: http + livenessProbe: + httpGet: + path: /health + port: http + resources: + {{- toYaml .Values.resources | nindent 12 }} + volumeMounts: + - name: config + mountPath: /etc/minigate + readOnly: true + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumes: + - name: config + secret: + secretName: {{ .Values.existingSecret | quote }} + items: + - key: config.yaml + path: config.yaml diff --git a/charts/minigate/templates/ingress.yaml b/charts/minigate/templates/ingress.yaml new file mode 100644 index 0000000..fd9229d --- /dev/null +++ b/charts/minigate/templates/ingress.yaml @@ -0,0 +1,31 @@ +{{- if .Values.ingress.enabled }} +{{- if not .Values.ingress.host }} +{{- fail "ingress.host is required when ingress.enabled=true" }} +{{- end }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ include "minigate.fullname" . }} + labels: + {{- include "minigate.labels" . | nindent 4 }} +spec: + {{- with .Values.ingress.className }} + ingressClassName: {{ . | quote }} + {{- end }} + {{- with .Values.ingress.tlsSecretName }} + tls: + - hosts: [{{ $.Values.ingress.host | quote }}] + secretName: {{ . | quote }} + {{- end }} + rules: + - host: {{ .Values.ingress.host | quote }} + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: {{ include "minigate.fullname" . }} + port: + name: http +{{- end }} diff --git a/charts/minigate/templates/service.yaml b/charts/minigate/templates/service.yaml new file mode 100644 index 0000000..9edd302 --- /dev/null +++ b/charts/minigate/templates/service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "minigate.fullname" . }} + labels: + {{- include "minigate.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - name: http + port: {{ .Values.service.port }} + targetPort: http + - name: metrics + port: {{ .Values.service.metricsPort }} + targetPort: metrics + selector: + app.kubernetes.io/name: minigate + app.kubernetes.io/instance: {{ .Release.Name }} diff --git a/charts/minigate/templates/servicemonitor.yaml b/charts/minigate/templates/servicemonitor.yaml new file mode 100644 index 0000000..372baad --- /dev/null +++ b/charts/minigate/templates/servicemonitor.yaml @@ -0,0 +1,19 @@ +{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "minigate.fullname" . }} + labels: + {{- include "minigate.labels" . | nindent 4 }} + {{- with .Values.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + app.kubernetes.io/name: minigate + app.kubernetes.io/instance: {{ .Release.Name }} + endpoints: + - port: metrics + path: /metrics +{{- end }} diff --git a/charts/minigate/values.yaml b/charts/minigate/values.yaml new file mode 100644 index 0000000..26a30f5 --- /dev/null +++ b/charts/minigate/values.yaml @@ -0,0 +1,29 @@ +replicaCount: 1 +logLevel: WARN + +image: + repository: ghcr.io/ehsanmsb/minigate + tag: "" + pullPolicy: IfNotPresent + +imagePullSecrets: [] + +# Create this Secret separately. Its config.yaml key contains S3 credentials. +existingSecret: "" + +service: + type: ClusterIP + port: 80 + metricsPort: 9090 + +serviceMonitor: + enabled: false + additionalLabels: {} + +ingress: + enabled: false + className: "" + host: "" + tlsSecretName: "" + +resources: {} diff --git a/cmd/minigate/main.go b/cmd/minigate/main.go index a0464cf..9b0ab71 100644 --- a/cmd/minigate/main.go +++ b/cmd/minigate/main.go @@ -4,49 +4,109 @@ import ( "context" "errors" "flag" + "fmt" "log" + "log/slog" "net/http" + "os" "os/signal" "syscall" "time" "github.com/ehsanmsb/minigate/internal/config" "github.com/ehsanmsb/minigate/internal/gateway" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" + "github.com/prometheus/client_golang/prometheus/promhttp" ) func main() { configPath := flag.String("config", "config.yaml", "path to config file") addr := flag.String("addr", ":8080", "http listen address") + metricsAddr := flag.String("metrics-addr", "127.0.0.1:9090", "metrics listen address") flag.Parse() + level := slog.LevelWarn + if value := os.Getenv("MINIGATE_LOG_LEVEL"); value != "" { + if err := level.UnmarshalText([]byte(value)); err != nil { + log.Fatalf("invalid MINIGATE_LOG_LEVEL: %v", err) + } + } + slog.SetDefault(slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: level}))) cfg, err := config.Load(*configPath) if err != nil { - log.Fatal(err) + slog.Error("load config", "error", err) + os.Exit(1) } + appHandler, metricsHandler := instrument(gateway.New(cfg), len(cfg.Buckets)) server := &http.Server{ Addr: *addr, - Handler: gateway.New(cfg), + Handler: appHandler, ReadHeaderTimeout: 5 * time.Second, IdleTimeout: 60 * time.Second, } + metricsMux := http.NewServeMux() + metricsMux.Handle("GET /metrics", metricsHandler) + metricsServer := &http.Server{Addr: *metricsAddr, Handler: metricsMux, ReadHeaderTimeout: 5 * time.Second} ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) defer stop() - shutdownDone := make(chan struct{}) + errorsCh := make(chan error, 2) go func() { - <-ctx.Done() - shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - if err := server.Shutdown(shutdownCtx); err != nil { - log.Printf("shutdown: %v", err) + if err := server.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { + errorsCh <- fmt.Errorf("http server: %w", err) } - close(shutdownDone) }() - - log.Printf("minigate listening on %s", *addr) - if err := server.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { - log.Fatal(err) + go func() { + if err := metricsServer.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { + errorsCh <- fmt.Errorf("metrics server: %w", err) + } + }() + slog.Info("listening", "addr", *addr, "metrics_addr", *metricsAddr) + failed := false + select { + case err := <-errorsCh: + slog.Error("serve", "error", err) + failed = true + case <-ctx.Done(): } - if ctx.Err() != nil { - <-shutdownDone + shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + for _, s := range []*http.Server{server, metricsServer} { + if err := s.Shutdown(shutdownCtx); err != nil { + slog.Error("shutdown", "error", err) + failed = true + } + } + if failed { + os.Exit(1) } } + +func instrument(handler http.Handler, bucketCount int) (http.Handler, http.Handler) { + registry := prometheus.NewRegistry() + registry.MustRegister(collectors.NewGoCollector(), collectors.NewProcessCollector(collectors.ProcessCollectorOpts{})) + buckets := prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "minigate_configured_buckets", + Help: "Number of configured public bucket routes in this replica.", + }) + buckets.Set(float64(bucketCount)) + requests := prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "minigate_http_requests_total", + Help: "Total HTTP requests by status code and method, excluding health probes.", + }, []string{"code", "method"}) + duration := prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Name: "minigate_http_request_duration_seconds", + Help: "HTTP request duration in seconds, excluding health probes.", + Buckets: []float64{0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30}, + }, []string{"code", "method"}) + registry.MustRegister(buckets, requests, duration) + instrumented := promhttp.InstrumentHandlerCounter(requests, promhttp.InstrumentHandlerDuration(duration, handler)) + app := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/health" { + handler.ServeHTTP(w, r) + return + } + instrumented.ServeHTTP(w, r) + }) + return app, promhttp.HandlerFor(registry, promhttp.HandlerOpts{}) +} diff --git a/cmd/minigate/main_test.go b/cmd/minigate/main_test.go new file mode 100644 index 0000000..4e59acd --- /dev/null +++ b/cmd/minigate/main_test.go @@ -0,0 +1,42 @@ +package main + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestMetricsCountRequestsWithoutHealthProbes(t *testing.T) { + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/missing" { + http.NotFound(w, r) + return + } + w.WriteHeader(http.StatusOK) + }) + app, metrics := instrument(handler, 2) + for _, path := range []string{"/health", "/file", "/missing"} { + app.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, path, nil)) + } + w := httptest.NewRecorder() + metrics.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/metrics", nil)) + if w.Code != http.StatusOK { + t.Fatalf("metrics status=%d", w.Code) + } + for _, sample := range []string{ + `minigate_configured_buckets 2`, + `minigate_http_requests_total{code="200",method="get"} 1`, + `minigate_http_requests_total{code="404",method="get"} 1`, + `minigate_http_request_duration_seconds_count{code="200",method="get"} 1`, + `minigate_http_request_duration_seconds_count{code="404",method="get"} 1`, + `go_goroutines`, + } { + if !strings.Contains(w.Body.String(), sample) { + t.Errorf("missing metric sample %q", sample) + } + } + if strings.Contains(w.Body.String(), `path=`) { + t.Fatal("object paths must not be metric labels") + } +} diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..c748d0b --- /dev/null +++ b/docs/README.md @@ -0,0 +1,5 @@ +# MiniGate documentation + +- [Configuration and routes](configuration.md) +- [Kubernetes deployment](deployment.md) +- [Releases](releases.md) diff --git a/docs/configuration.md b/docs/configuration.md new file mode 100644 index 0000000..cb77f56 --- /dev/null +++ b/docs/configuration.md @@ -0,0 +1,20 @@ +# Configuration and routes + +MiniGate serves `GET /{name}/{object-key}`. The URL keeps the complete S3 object key. Each `name` may point to a different bucket, endpoint and credential. + +```yaml +buckets: + - name: avatars + bucket: private-images + endpoint: https://s3.example.com + region: us-east-1 + access_key: ACCESS_KEY + secret_key: SECRET_KEY + allowed_paths: + - public/photos + - public/icons +``` + +Here `/avatars/public/photos/profile.png` reads `public/photos/profile.png` from `private-images`. Keys outside `allowed_paths` return 404. Omit `allowed_paths` to expose the entire bucket through that name. Give the credential only `GetObject` permission for the intended objects. Anyone with a public URL can read a reachable object. + +See the root [README](../README.md) for local execution and HTTP behavior. diff --git a/docs/deployment.md b/docs/deployment.md new file mode 100644 index 0000000..c43e9e4 --- /dev/null +++ b/docs/deployment.md @@ -0,0 +1,30 @@ +# Kubernetes deployment + +The chart is in [`charts/minigate`](../charts/minigate). Create a Secret from a local configuration file; keep real credentials out of Git: + +```sh +kubectl create secret generic minigate-config --from-file=config.yaml=./private-config.yaml +helm upgrade --install minigate ./charts/minigate \ + --set existingSecret=minigate-config \ + --set replicaCount=3 +``` + +The Secret needs a `config.yaml` key. Every replica mounts the same Secret read-only and starts with the same routes. Change the Secret and restart the Deployment to load a new configuration. The application has no shared state or database. + +The chart defaults to `logLevel: WARN`, so routine requests do not fill the logs. Use `--set logLevel=INFO` to include access denials and invalid ranges, or `--set logLevel=DEBUG` to record every object request. Health probes stay silent. + +For a public hostname, configure your ingress controller and TLS Secret, then enable the optional Ingress: + +```sh +helm upgrade --install minigate ./charts/minigate \ + --set existingSecret=minigate-config \ + --set replicaCount=3 \ + --set ingress.enabled=true \ + --set ingress.className=nginx \ + --set ingress.host=files.example.com \ + --set ingress.tlsSecretName=files-tls +``` + +Use a released chart archive from a GitHub release for a matching image tag. If deploying directly from source before a release, set `image.tag` to an existing GHCR tag. The packaged chart defaults to its matching released image version. The service listens internally on port 8080; Kubernetes Service exposes port 80. `/health` is used for readiness and liveness. Ensure the cluster allows outbound HTTPS to each configured storage endpoint and can pull from GHCR. + +Prometheus metrics are available on the internal Service port `9090` at `/metrics`. The Ingress routes only to the file-serving port, so it does not expose metrics publicly. If your cluster has Prometheus Operator and its ServiceMonitor CRD, enable `--set serviceMonitor.enabled=true`; set `serviceMonitor.additionalLabels` as needed by your Prometheus selector. Each replica exports its own metrics, including `minigate_configured_buckets`; scrape every pod through service discovery. Prometheus generates `up` per scrape target, while `/health` is the Kubernetes readiness and liveness check. diff --git a/docs/releases.md b/docs/releases.md new file mode 100644 index 0000000..8ce6b02 --- /dev/null +++ b/docs/releases.md @@ -0,0 +1,7 @@ +# Releases + +The [Verify workflow](../.github/workflows/verify.yml) checks Go code, the Helm chart and the container build on pull requests. A successful push to `main` runs the same checks, then semantic-release creates a `vX.Y.Z` tag and GitHub release, packages the chart, and publishes `ghcr.io/ehsanmsb/minigate:X.Y.Z` plus `:latest`. + +Release versions follow [Conventional Commits](https://www.conventionalcommits.org/): `feat:` increments minor, `fix:` and other recognized types increment patch, and `!` or a `BREAKING CHANGE:` footer increments major. Every successful push to `main` gets at least a patch release, even if its commit title does not follow that format. The first automatic release is `v1.0.0`. Use a Conventional Commit title when squashing a pull request so feature and breaking changes get the right version. Release tags are generated by the workflow. + +The chart archive attached to each GitHub release has matching `version` and `appVersion`; its default image tag is that release version. The source chart metadata stays at `1.0.0` because packaging sets the release version without committing generated files. diff --git a/go.mod b/go.mod index ae3e5b5..02a0f54 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0 github.com/aws/smithy-go v1.24.0 github.com/go-chi/chi/v5 v5.2.5 + github.com/prometheus/client_golang v1.24.1 go.yaml.in/yaml/v3 v3.0.4 ) @@ -20,4 +21,12 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.70.1 // indirect + github.com/prometheus/procfs v0.21.1 // indirect + golang.org/x/sys v0.47.0 // indirect + google.golang.org/protobuf v1.36.11 // indirect ) diff --git a/go.sum b/go.sum index b07e9a2..620f9f4 100644 --- a/go.sum +++ b/go.sum @@ -2,18 +2,12 @@ github.com/aws/aws-sdk-go-v2 v1.41.1 h1:ABlyEARCDLN034NhxlRUSZr4l71mh+T5KAeGh6ce github.com/aws/aws-sdk-go-v2 v1.41.1/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4= -github.com/aws/aws-sdk-go-v2/config v1.32.7 h1:vxUyWGUwmkQ2g19n7JY/9YL8MfAIl7bTesIUykECXmY= -github.com/aws/aws-sdk-go-v2/config v1.32.7/go.mod h1:2/Qm5vKUU/r7Y+zUk/Ptt2MDAEKAfUtKc1+3U1Mo3oY= github.com/aws/aws-sdk-go-v2/credentials v1.19.7 h1:tHK47VqqtJxOymRrNtUXN5SP/zUTvZKeLx4tH6PGQc8= github.com/aws/aws-sdk-go-v2/credentials v1.19.7/go.mod h1:qOZk8sPDrxhf+4Wf4oT2urYJrYt3RejHSzgAquYeppw= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17 h1:I0GyV8wiYrP8XpA70g1HBcQO1JlQxCMTW9npl5UbDHY= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17/go.mod h1:tyw7BOl5bBe/oqvoIeECFJjMdzXoa/dfVz3QQ5lgHGA= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 h1:xOLELNKGp2vsiteLsvLPwxC+mYmO6OZ8PYgiuPJzF8U= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17/go.mod h1:5M5CI3D12dNOtH3/mk6minaRwI2/37ifCURZISxA/IQ= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 h1:WWLqlh79iO48yLkj1v3ISRNiv+3KdQoZ6JWyfcsyQik= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17/go.mod h1:EhG22vHRrvF8oXSTYStZhJc1aUgKtnJe+aOiFEV90cM= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc= github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 h1:JqcdRG//czea7Ppjb+g/n4o8i/R50aTBHkA7vu0lK+k= github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17/go.mod h1:CO+WeGmIdj/MlPel2KwID9Gt7CNq4M65HUfBW97liM0= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 h1:0ryTNEdJbzUCEWkVXEXoqlXV72J5keC1GvILMOuD00E= @@ -26,53 +20,47 @@ github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 h1:bGeHBsGZx0Dvu github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17/go.mod h1:dcW24lbU0CzHusTE8LLHhRLI42ejmINN8Lcr22bwh/g= github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0 h1:oeu8VPlOre74lBA/PMhxa5vewaMIMmILM+RraSyB8KA= github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0/go.mod h1:5jggDlZ2CLQhwJBiZJb4vfk4f0GxWdEDruWKEJ1xOdo= -github.com/aws/aws-sdk-go-v2/service/signin v1.0.5 h1:VrhDvQib/i0lxvr3zqlUwLwJP4fpmpyD9wYG1vfSu+Y= -github.com/aws/aws-sdk-go-v2/service/signin v1.0.5/go.mod h1:k029+U8SY30/3/ras4G/Fnv/b88N4mAfliNn08Dem4M= -github.com/aws/aws-sdk-go-v2/service/sso v1.30.9 h1:v6EiMvhEYBoHABfbGB4alOYmCIrcgyPPiBE1wZAEbqk= -github.com/aws/aws-sdk-go-v2/service/sso v1.30.9/go.mod h1:yifAsgBxgJWn3ggx70A3urX2AN49Y5sJTD1UQFlfqBw= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.13 h1:gd84Omyu9JLriJVCbGApcLzVR3XtmC4ZDPcAI6Ftvds= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.13/go.mod h1:sTGThjphYE4Ohw8vJiRStAcu3rbjtXRsdNB0TvZ5wwo= -github.com/aws/aws-sdk-go-v2/service/sts v1.41.6 h1:5fFjR/ToSOzB2OQ/XqWpZBmNvmP/pJ1jOWYlFDJTjRQ= -github.com/aws/aws-sdk-go-v2/service/sts v1.41.6/go.mod h1:qgFDZQSD/Kys7nJnVqYlWKnh0SSdMjAi0uSwON4wgYQ= github.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk= github.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0= -github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be h1:J5BL2kskAlV9ckgEsNQXscjIaLiOYiZ75d4e94E6dcQ= -github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be/go.mod h1:mk5IQ+Y0ZeO87b858TlA645sVcEcbiX6YqP98kt+7+w= -github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= -github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-chi/chi/v5 v5.2.5 h1:Eg4myHZBjyvJmAFjFvWgrqDTXFyOzjj7YIm3L3mu6Ug= github.com/go-chi/chi/v5 v5.2.5/go.mod h1:X7Gx4mteadT3eDOMTsXzmI4/rwUpOwBHLpAfupzFJP0= -github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= -github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= -github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= -github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU= -github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= -github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= -github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= -github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI= -github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= -github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= -github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.98 h1:MeAVKjLVz+XJ28zFcuYyImNSAh8Mq725uNW4beRisi0= -github.com/minio/minio-go/v7 v7.0.98/go.mod h1:cY0Y+W7yozf0mdIclrttzo1Iiu7mEf9y7nk2uXqMOvM= -github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= -github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= -github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= -github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= -github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY= -github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk= +github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59uiuyR2bPlnU= +github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.70.1 h1:1HvjP4D5oL3t8RsPlwxA9onvvStjtIHYE5XuuwOi/PY= +github.com/prometheus/common v0.70.1/go.mod h1:VdFUQDMZK3VLkurFUVhia6uys/0suUp86TJz5qbJRhc= +github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI= +github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= +go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= -golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= -golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= -golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= -golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= -golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= -golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= -golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/gateway/gateway.go b/internal/gateway/gateway.go index 1b0dd8a..ed76267 100644 --- a/internal/gateway/gateway.go +++ b/internal/gateway/gateway.go @@ -4,7 +4,7 @@ import ( "context" "errors" "io" - "log" + "log/slog" "net/http" "net/url" "strconv" @@ -55,6 +55,14 @@ func New(cfg *config.Config) *Handler { _, _ = io.WriteString(w, "ok") }) r.Get("/{name}/*", h.getObject) + r.NotFound(func(w http.ResponseWriter, r *http.Request) { + slog.Debug("request", "status", http.StatusNotFound, "reason", "unknown_route") + http.NotFound(w, r) + }) + r.MethodNotAllowed(func(w http.ResponseWriter, r *http.Request) { + slog.Debug("request", "status", http.StatusMethodNotAllowed, "reason", "method_not_allowed") + http.Error(w, http.StatusText(http.StatusMethodNotAllowed), http.StatusMethodNotAllowed) + }) h.router = r return h } @@ -67,11 +75,13 @@ func (h *Handler) getObject(w http.ResponseWriter, r *http.Request) { name, nameErr := routeParam(r, "name") key, keyErr := routeParam(r, "*") if nameErr != nil || keyErr != nil || name == "" || key == "" || hasDotSegment(key) { + slog.Debug("request", "status", http.StatusNotFound, "reason", "invalid_path") http.NotFound(w, r) return } selected, ok := h.routes[name] if !ok { + slog.Debug("request", "status", http.StatusNotFound, "reason", "unknown_route", "route", name) http.NotFound(w, r) return } @@ -84,12 +94,14 @@ func (h *Handler) getObject(w http.ResponseWriter, r *http.Request) { } } if !allowed { + slog.Debug("request", "status", http.StatusNotFound, "reason", "path_not_allowed", "route", name) http.NotFound(w, r) return } } input := &s3.GetObjectInput{Bucket: aws.String(selected.bucket), Key: aws.String(key)} - if value := r.Header.Get("Range"); value != "" { + // A full response is safe when we cannot check the client's If-Range validator. + if value := r.Header.Get("Range"); strings.HasPrefix(value, "bytes=") && r.Header.Get("If-Range") == "" { input.Range = aws.String(value) } obj, err := selected.client.GetObject(r.Context(), input) @@ -98,19 +110,26 @@ func (h *Handler) getObject(w http.ResponseWriter, r *http.Request) { if errors.As(err, &apiErr) { switch apiErr.ErrorCode() { case "NoSuchKey", "NotFound": + slog.Debug("request", "status", http.StatusNotFound, "reason", "missing_object", "route", name) + http.NotFound(w, r) + return + case "AccessDenied": + // Without ListBucket, S3 also returns AccessDenied for missing keys. + slog.Info("request", "status", http.StatusNotFound, "reason", "access_denied", "route", name) http.NotFound(w, r) return case "InvalidRange": + slog.Info("request", "status", http.StatusRequestedRangeNotSatisfiable, "reason", "invalid_range", "route", name) http.Error(w, "invalid range", http.StatusRequestedRangeNotSatisfiable) return } } - log.Printf("S3 GetObject failed for route %q: %v", name, err) + slog.Error("request", "status", http.StatusBadGateway, "reason", "storage_error", "route", name, "error", err) http.Error(w, "storage unavailable", http.StatusBadGateway) return } if obj.Body == nil { - log.Printf("S3 GetObject returned no body for route %q", name) + slog.Error("request", "status", http.StatusBadGateway, "reason", "empty_storage_body", "route", name) http.Error(w, "storage unavailable", http.StatusBadGateway) return } @@ -134,9 +153,16 @@ func (h *Handler) getObject(w http.ResponseWriter, r *http.Request) { status = http.StatusPartialContent } w.WriteHeader(status) - if _, err := io.Copy(w, obj.Body); err != nil { - log.Printf("stream S3 object from route %q: %v", name, err) + bytes, err := io.Copy(w, obj.Body) + if err != nil { + if r.Context().Err() != nil { + slog.Debug("request", "status", status, "reason", "client_disconnected", "route", name, "bytes", bytes) + } else { + slog.Error("request", "status", status, "reason", "stream_error", "route", name, "bytes", bytes, "error", err) + } + return } + slog.Debug("request", "status", status, "reason", "ok", "route", name, "bytes", bytes) } func routeParam(r *http.Request, name string) (string, error) { diff --git a/internal/gateway/gateway_test.go b/internal/gateway/gateway_test.go index ecf962a..6539206 100644 --- a/internal/gateway/gateway_test.go +++ b/internal/gateway/gateway_test.go @@ -1,9 +1,12 @@ package gateway import ( + "bytes" "context" + "encoding/json" "errors" "io" + "log/slog" "net/http" "net/http/httptest" "strings" @@ -154,6 +157,29 @@ func TestGetObjectRange(t *testing.T) { } } +func TestIfRangeDoesNotReturnUncheckedPartialObject(t *testing.T) { + f := &fakeStorage{out: &s3.GetObjectOutput{Body: io.NopCloser(strings.NewReader("whole file"))}} + request := httptest.NewRequest(http.MethodGet, "/avatars/public/video.mp4", nil) + request.Header.Set("Range", "bytes=5-") + request.Header.Set("If-Range", "\"old-version\"") + w := httptest.NewRecorder() + testHandler(f).ServeHTTP(w, request) + if w.Code != http.StatusOK || f.input.Range != nil { + t.Errorf("status=%d upstream range=%q", w.Code, aws.ToString(f.input.Range)) + } +} + +func TestUnknownRangeUnitIsIgnored(t *testing.T) { + f := &fakeStorage{out: &s3.GetObjectOutput{Body: io.NopCloser(strings.NewReader("whole file"))}} + request := httptest.NewRequest(http.MethodGet, "/avatars/public/video.mp4", nil) + request.Header.Set("Range", "items=0-3") + w := httptest.NewRecorder() + testHandler(f).ServeHTTP(w, request) + if w.Code != http.StatusOK || f.input.Range != nil { + t.Errorf("status=%d upstream range=%q", w.Code, aws.ToString(f.input.Range)) + } +} + func TestGetObjectErrorStatus(t *testing.T) { for _, tc := range []struct { name string @@ -161,6 +187,8 @@ func TestGetObjectErrorStatus(t *testing.T) { want int }{ {"missing", &smithy.GenericAPIError{Code: "NoSuchKey"}, http.StatusNotFound}, + {"missing without list permission", &smithy.GenericAPIError{Code: "AccessDenied"}, http.StatusNotFound}, + {"invalid credential", &smithy.GenericAPIError{Code: "InvalidAccessKeyId"}, http.StatusBadGateway}, {"storage failure", errors.New("connection failed"), http.StatusBadGateway}, } { t.Run(tc.name, func(t *testing.T) { @@ -173,6 +201,45 @@ func TestGetObjectErrorStatus(t *testing.T) { } } +func TestRequestLogs(t *testing.T) { + var output bytes.Buffer + previous := slog.Default() + slog.SetDefault(slog.New(slog.NewJSONHandler(&output, &slog.HandlerOptions{Level: slog.LevelDebug}))) + t.Cleanup(func() { slog.SetDefault(previous) }) + + h := testHandler(&fakeStorage{err: &smithy.GenericAPIError{Code: "AccessDenied"}}) + h.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/health", nil)) + if output.Len() != 0 { + t.Fatal("health probe should not be logged") + } + h.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/avatars/public/private-file.png", nil)) + if bytes.Count(output.Bytes(), []byte{'\n'}) != 1 || bytes.Contains(output.Bytes(), []byte("private-file.png")) { + t.Fatalf("expected one log without object key: %s", output.String()) + } + var record struct { + Status int `json:"status"` + Reason string `json:"reason"` + Route string `json:"route"` + } + if err := json.Unmarshal(bytes.TrimSpace(output.Bytes()), &record); err != nil { + t.Fatal(err) + } + if record.Status != http.StatusNotFound || record.Reason != "access_denied" || record.Route != "avatars" { + t.Fatalf("unexpected log: %+v", record) + } + + output.Reset() + slog.SetDefault(slog.New(slog.NewJSONHandler(&output, &slog.HandlerOptions{Level: slog.LevelWarn}))) + h.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/avatars/public/private-file.png", nil)) + if output.Len() != 0 { + t.Fatal("expected denied request to be quiet at WARN level") + } + testHandler(&fakeStorage{err: errors.New("connection failed")}).ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/avatars/public/file.png", nil)) + if bytes.Count(output.Bytes(), []byte{'\n'}) != 1 || !bytes.Contains(output.Bytes(), []byte(`"reason":"storage_error"`)) { + t.Fatalf("expected one storage error log: %s", output.String()) + } +} + func TestConfiguredEndpointUsesPathStyle(t *testing.T) { var upstreamPath string upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -192,3 +259,19 @@ func TestConfiguredEndpointUsesPathStyle(t *testing.T) { t.Fatalf("status=%d body=%q upstream path=%q", w.Code, w.Body.String(), upstreamPath) } } + +func TestGenericStorage404ReturnsNotFound(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer upstream.Close() + h := New(&config.Config{Buckets: []config.Bucket{{ + Name: "public", Bucket: "private-files", Endpoint: upstream.URL, + Region: "us-east-1", AccessKey: "access", SecretKey: "secret", + }}}) + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/public/missing.txt", nil)) + if w.Code != http.StatusNotFound { + t.Errorf("status=%d want=404", w.Code) + } +}