fix: bound storage response and proxy idle times

.github/workflows/verify.yml

@@ -30,6 +30,8 @@ jobs:
           grep -q '^kind: Route$' /tmp/minigate-route.yaml
           helm template minigate charts/minigate --set existingSecret=minigate-config --set httpProxy.enabled=true --set httpProxy.fqdn=files.example.com --set httpProxy.tlsSecretName=files-tls > /tmp/minigate-httpproxy.yaml
           grep -q '^kind: HTTPProxy$' /tmp/minigate-httpproxy.yaml
+          grep -q 'response: "infinity"' /tmp/minigate-httpproxy.yaml
+          grep -q 'idle: "60s"' /tmp/minigate-httpproxy.yaml
       - if: github.event_name == 'pull_request'
         run: docker build -t minigate:ci .
 

README.md

@@ -46,6 +46,8 @@ Prometheus metrics are served separately at `127.0.0.1:9090/metrics` by default.
 
 The service accepts `GET /{name}/{object-key}` and `GET /health`. Other methods return 405. A missing or inaccessible object returns 404; storage failures return 502. S3 can return `AccessDenied` for a missing object when credentials lack `ListBucket`, so MiniGate maps that error to 404. `Range` requests are forwarded to storage for partial downloads. S3 content metadata is forwarded in the response.
 
+S3 response headers must arrive within 5 seconds per attempt. The entire object transfer, including retries and streaming, is limited to 10 minutes by default; set `-s3-timeout` to change that limit. This is a transfer ceiling, not a 10-minute wait before the first byte. Video players can use `Range` requests to fetch parts of a file. A timeout before response headers returns 504. If streaming stalls after headers were sent, the connection ends and the failure is logged because its HTTP status cannot be changed.
+
 Logs are JSON on standard output. `MINIGATE_LOG_LEVEL` accepts `DEBUG`, `INFO`, `WARN`, or `ERROR` (default `WARN`). Unexpected storage and server failures are logged by default. Set `INFO` to also see access denials and invalid ranges, or `DEBUG` for one compact record per object request, including successful reads and expected 404s. Health probes are not logged. Routine request logs omit object keys and credentials.
 
 HTTP routing uses [chi](https://github.com/go-chi/chi); storage access uses the official AWS SDK for Go v2. The HTTP server itself uses Go's standard `net/http` package.

charts/minigate/README.md

@@ -34,6 +34,7 @@ To let Kubernetes adjust replica count, enable `autoscaling.enabled` and set `mi
 | `image.tag` | `""` | Uses the chart's `appVersion` when empty |
 | `imagePullSecrets` | `[]` | Credentials for a private registry |
 | `logLevel` | `WARN` | Application log level |
+| `s3Timeout` | `10m` | Maximum transfer time for one object request |
 | `service.port` | `80` | Internal HTTP Service port |
 | `service.metricsPort` | `9090` | Internal Prometheus Service port |
 | `ingress.enabled` | `false` | Expose file URLs through an Ingress |
@@ -43,6 +44,8 @@ To let Kubernetes adjust replica count, enable `autoscaling.enabled` and set `mi
 | `httpProxy.enabled` | `false` | Create a Contour HTTPProxy |
 | `httpProxy.fqdn` | `""` | Required hostname when HTTPProxy is enabled |
 | `httpProxy.tlsSecretName` | `""` | Optional TLS Secret for HTTPProxy |
+| `httpProxy.timeoutPolicy.response` | `infinity` | Let the app control total transfer time for videos |
+| `httpProxy.timeoutPolicy.idle` | `60s` | Close streams with no traffic for 60 seconds |
 | `serviceMonitor.enabled` | `false` | Create a Prometheus Operator ServiceMonitor |
 | `autoscaling.enabled` | `false` | Create a CPU-based HPA |
 | `podDisruptionBudget.enabled` | `false` | Create a PDB with `maxUnavailable: 1` |
@@ -55,3 +58,5 @@ The chart also supports `nameOverride`, `fullnameOverride`, `podAnnotations`, `p
 Ingress sends traffic only to the file-serving port. Metrics stay on the internal Service. Enable ServiceMonitor only when its CRD is installed. The chart does not create or store S3 credentials.
 
 For OpenShift, see the [deployment guide](../../docs/deployment.md#openshift). HTTPProxy requires Contour and its `projectcontour.io/v1` CRD; it is not an OpenShift built-in. Enable only the exposure template your cluster uses. Route and HTTPProxy also send traffic only to the file-serving Service port.
+
+The app limits each S3 response to `s3Timeout`, including active streaming. This does not delay the first byte; S3 response headers have a separate 5-second timeout per attempt. HTTPProxy allows long responses but closes streams idle for 60 seconds. Video players can use `Range` requests for partial downloads.

charts/minigate/templates/deployment.yaml

@@ -36,7 +36,7 @@ spec:
         - name: minigate
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
           imagePullPolicy: {{ .Values.image.pullPolicy }}
-          args: ["-config", "/etc/minigate/config.yaml", "-metrics-addr", ":9090"]
+          args: ["-config", "/etc/minigate/config.yaml", "-metrics-addr", ":9090", "-s3-timeout", {{ .Values.s3Timeout | quote }}]
           env:
             - name: MINIGATE_LOG_LEVEL
               value: {{ .Values.logLevel | quote }}

charts/minigate/templates/httpproxy.yaml

@@ -18,6 +18,9 @@ spec:
   routes:
     - conditions:
         - prefix: /
+      timeoutPolicy:
+        response: {{ .Values.httpProxy.timeoutPolicy.response | quote }}
+        idle: {{ .Values.httpProxy.timeoutPolicy.idle | quote }}
       services:
         - name: {{ include "minigate.fullname" . }}
           port: {{ .Values.service.port }}

charts/minigate/values.yaml

@@ -12,6 +12,7 @@ imagePullSecrets: []
 # Create this Secret separately with a config.yaml key containing S3 credentials.
 existingSecret: ""
 logLevel: WARN
+s3Timeout: 10m # Maximum transfer time per object request; not time to first byte.
 
 serviceAccount:
   create: false
@@ -56,6 +57,9 @@ httpProxy:
   enabled: false
   fqdn: ""
   tlsSecretName: "" # Optional TLS Secret in the release namespace.
+  timeoutPolicy:
+    response: infinity # Let the app control total transfer time for videos.
+    idle: 60s # Close streams that stop transferring data.
 
 resources:
   requests:

cmd/minigate/main.go

@@ -24,7 +24,11 @@ func main() {
 	configPath := flag.String("config", "config.yaml", "path to config file")
 	addr := flag.String("addr", ":8080", "http listen address")
 	metricsAddr := flag.String("metrics-addr", "127.0.0.1:9090", "metrics listen address")
+	s3Timeout := flag.Duration("s3-timeout", 10*time.Minute, "maximum time for one object response, including download")
 	flag.Parse()
+	if *s3Timeout <= 0 {
+		log.Fatal("s3-timeout must be positive")
+	}
 	level := slog.LevelWarn
 	if value := os.Getenv("MINIGATE_LOG_LEVEL"); value != "" {
 		if err := level.UnmarshalText([]byte(value)); err != nil {
@@ -38,7 +42,7 @@ func main() {
 		slog.Error("load config", "error", err)
 		os.Exit(1)
 	}
-	appHandler, metricsHandler := instrument(gateway.New(cfg), len(cfg.Buckets))
+	appHandler, metricsHandler := instrument(gateway.New(cfg, *s3Timeout), len(cfg.Buckets))
 	server := &http.Server{
 		Addr:              *addr,
 		Handler:           appHandler,

docs/deployment.md

@@ -17,6 +17,8 @@ The chart requests `100m` CPU and `64Mi` memory and limits each container to `50
 
 The chart defaults to `logLevel: WARN`, so routine requests do not fill the logs. Use `--set logLevel=INFO` to include access denials and invalid ranges, or `--set logLevel=DEBUG` to record every object request. Health probes stay silent.
 
+The application waits at most 5 seconds per attempt for S3 response headers and allows up to 10 minutes for the complete object transfer. The 10-minute limit is for transfer, not for waiting before playback starts. Set `s3Timeout` in chart values if needed. A timeout before response headers returns 504; a stalled stream after headers is terminated and logged.
+
 For a public hostname, configure your ingress controller and TLS Secret, then enable the optional Ingress:
 
 ```sh
@@ -63,3 +65,5 @@ helm upgrade --install minigate ./charts/minigate \
 ```
 
 The `files-tls` Secret must contain a certificate for `files.example.com`. Leave `httpProxy.tlsSecretName` empty only if HTTP is intentionally served behind another TLS edge. Enable only the exposure template used by your cluster; Ingress, Route, and HTTPProxy are all disabled by default.
+
+When HTTPProxy is enabled, its route uses `timeoutPolicy.response: infinity` so a video transfer is not cut off by Contour's 15-second total response default. `timeoutPolicy.idle: 60s` closes a connection that stops transferring data. The application's `s3Timeout` still caps each complete object request.

internal/gateway/gateway.go

@@ -5,12 +5,15 @@ import (
 	"errors"
 	"io"
 	"log/slog"
+	"net"
 	"net/http"
 	"net/url"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
+	awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http"
 	"github.com/aws/aws-sdk-go-v2/credentials"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/smithy-go"
@@ -30,24 +33,29 @@ type route struct {
 }
 
 type Handler struct {
-	routes map[string]route
-	router http.Handler
+	routes  map[string]route
+	router  http.Handler
+	timeout time.Duration
 }
 
-func New(cfg *config.Config) *Handler {
+func New(cfg *config.Config, timeout time.Duration) *Handler {
 	routes := make(map[string]route, len(cfg.Buckets))
+	httpClient := awshttp.NewBuildableClient().WithTransportOptions(func(t *http.Transport) {
+		t.ResponseHeaderTimeout = min(timeout, 5*time.Second)
+	})
 	for _, b := range cfg.Buckets {
 		awsCfg := aws.Config{
 			Region:      b.Region,
 			Credentials: aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider(b.AccessKey, b.SecretKey, "")),
+			HTTPClient:  httpClient,
 		}
 		client := s3.NewFromConfig(awsCfg, func(o *s3.Options) {
 			o.BaseEndpoint = aws.String(b.Endpoint)
 			o.UsePathStyle = true
 		})
 		routes[b.Name] = route{bucket: b.Bucket, allowedPaths: b.AllowedPaths, client: client}
 	}
-	h := &Handler{routes: routes}
+	h := &Handler{routes: routes, timeout: timeout}
 	r := chi.NewRouter()
 	r.Use(middleware.SetHeader("Access-Control-Allow-Origin", "*"))
 	r.Use(middleware.Recoverer)
@@ -104,7 +112,9 @@ func (h *Handler) getObject(w http.ResponseWriter, r *http.Request) {
 	if value := r.Header.Get("Range"); strings.HasPrefix(value, "bytes=") && r.Header.Get("If-Range") == "" {
 		input.Range = aws.String(value)
 	}
-	obj, err := selected.client.GetObject(r.Context(), input)
+	ctx, cancel := context.WithTimeout(r.Context(), h.timeout)
+	defer cancel()
+	obj, err := selected.client.GetObject(ctx, input)
 	if err != nil {
 		var apiErr smithy.APIError
 		if errors.As(err, &apiErr) {
@@ -124,6 +134,12 @@ func (h *Handler) getObject(w http.ResponseWriter, r *http.Request) {
 				return
 			}
 		}
+		var netErr net.Error
+		if errors.Is(err, context.DeadlineExceeded) || (errors.As(err, &netErr) && netErr.Timeout()) {
+			slog.Error("request", "status", http.StatusGatewayTimeout, "reason", "storage_timeout", "route", name, "error", err)
+			http.Error(w, "storage timeout", http.StatusGatewayTimeout)
+			return
+		}
 		slog.Error("request", "status", http.StatusBadGateway, "reason", "storage_error", "route", name, "error", err)
 		http.Error(w, "storage unavailable", http.StatusBadGateway)
 		return
@@ -157,6 +173,8 @@ func (h *Handler) getObject(w http.ResponseWriter, r *http.Request) {
 	if err != nil {
 		if r.Context().Err() != nil {
 			slog.Debug("request", "status", status, "reason", "client_disconnected", "route", name, "bytes", bytes)
+		} else if ctx.Err() == context.DeadlineExceeded {
+			slog.Error("request", "status", status, "reason", "stream_timeout", "route", name, "bytes", bytes)
 		} else {
 			slog.Error("request", "status", status, "reason", "stream_error", "route", name, "bytes", bytes, "error", err)
 		}

internal/gateway/gateway_test.go

@@ -11,6 +11,7 @@ import (
 	"net/http/httptest"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
@@ -35,7 +36,7 @@ func testHandler(f *fakeStorage) *Handler {
 	h := New(&config.Config{Buckets: []config.Bucket{{
 		Name: "avatars", Bucket: "private-images", Endpoint: "https://storage.example.com",
 		Region: "us-east-1", AccessKey: "access", SecretKey: "secret", AllowedPaths: []string{"public"},
-	}}})
+	}}}, 10*time.Minute)
 	r := h.routes["avatars"]
 	r.client = f
 	h.routes["avatars"] = r
@@ -96,7 +97,7 @@ func TestAllowedPathsExposeOnlySelectedDirectories(t *testing.T) {
 			Name: "avatars", Bucket: "private-images", Endpoint: "https://storage.example.com",
 			Region: "us-east-1", AccessKey: "access", SecretKey: "secret",
 			AllowedPaths: []string{"photos/public", "docs/shared"},
-		}}})
+		}}}, 10*time.Minute)
 		r := h.routes["avatars"]
 		r.client = f
 		h.routes["avatars"] = r
@@ -252,7 +253,7 @@ func TestConfiguredEndpointUsesPathStyle(t *testing.T) {
 	h := New(&config.Config{Buckets: []config.Bucket{{
 		Name: "public", Bucket: "private-files", Endpoint: upstream.URL,
 		Region: "us-east-1", AccessKey: "access", SecretKey: "secret",
-	}}})
+	}}}, 10*time.Minute)
 	w := httptest.NewRecorder()
 	h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/public/folder/file.txt", nil))
 	if w.Code != http.StatusOK || w.Body.String() != "file" || upstreamPath != "/private-files/folder/file.txt" {
@@ -268,10 +269,54 @@ func TestGenericStorage404ReturnsNotFound(t *testing.T) {
 	h := New(&config.Config{Buckets: []config.Bucket{{
 		Name: "public", Bucket: "private-files", Endpoint: upstream.URL,
 		Region: "us-east-1", AccessKey: "access", SecretKey: "secret",
-	}}})
+	}}}, 10*time.Minute)
 	w := httptest.NewRecorder()
 	h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/public/missing.txt", nil))
 	if w.Code != http.StatusNotFound {
 		t.Errorf("status=%d want=404", w.Code)
 	}
 }
+
+func TestStorageTimeout(t *testing.T) {
+	for _, tc := range []struct {
+		name       string
+		upstream   http.HandlerFunc
+		wantStatus int
+		wantBody   string
+	}{
+		{
+			name: "before headers",
+			upstream: func(w http.ResponseWriter, r *http.Request) {
+				time.Sleep(time.Second)
+				_, _ = io.WriteString(w, "late")
+			},
+			wantStatus: http.StatusGatewayTimeout,
+		},
+		{
+			name: "during body",
+			upstream: func(w http.ResponseWriter, r *http.Request) {
+				w.Header().Set("Content-Length", "2")
+				_, _ = io.WriteString(w, "a")
+				w.(http.Flusher).Flush()
+				time.Sleep(time.Second)
+				_, _ = io.WriteString(w, "b")
+			},
+			wantStatus: http.StatusOK,
+			wantBody:   "a",
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			upstream := httptest.NewServer(tc.upstream)
+			defer upstream.Close()
+			h := New(&config.Config{Buckets: []config.Bucket{{
+				Name: "public", Bucket: "private-files", Endpoint: upstream.URL,
+				Region: "us-east-1", AccessKey: "access", SecretKey: "secret",
+			}}}, 250*time.Millisecond)
+			w := httptest.NewRecorder()
+			h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/public/file.txt", nil))
+			if w.Code != tc.wantStatus || (tc.wantBody != "" && w.Body.String() != tc.wantBody) {
+				t.Fatalf("status=%d body=%q", w.Code, w.Body.String())
+			}
+		})
+	}
+}