Add option for not verifying SSL during S3 reading and writing

CHANGELOG.md

@@ -11,13 +11,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Added `{second}` and `{minute}` as a filename formatting variable [!15](https://github.com/dmidk/sunflow/pull/15), @JoachimKoenigslieb
 - Added a `clearsky` config object which can load clear-sky data from files via `clearsky.path` or generate it with `pvlib` via the `simplified_solis` method [!15](https://github.com/dmidk/sunflow/pull/15), @JoachimKoenigslieb
 - Added `MIN_SOLAR_ELEVATION_DEGREES` (defaulting to 6 degrees) to configure the minimum maximum-corner solar elevation required to run [!15](https://github.com/dmidk/sunflow/pull/15), @JoachimKoenigslieb
-- Added configurations to run with MTG data with either calculated or pvlib clearsky [!23](https://github.com/dmidk/sunflow/pull/15), @KristianHMoller
-- Added the ability to provide more configuration parameters to config.yaml [!23](https://github.com/dmidk/sunflow/pull/15), @KristianHMoller
+- Added configurations to run with MTG data with either calculated or pvlib clearsky [!24](https://github.com/dmidk/sunflow/pull/24), @KristianHMoller
+- Added the ability to provide more configuration parameters to config.yaml [!24](https://github.com/dmidk/sunflow/pull/24), @KristianHMoller
+- Added to option to bypass SSL verification in S3 reading and writing [!27](https://github.com/dmidk/sunflow/pull/27), @KristianHMoller
 
 ### Changed
 - Use `.expand_dims` instead of `.assign_coords` to make sure we have both time dimension and time coordinates when loading from files [!15](https://github.com/dmidk/sunflow/pull/15), @JoachimKoenigslieb
 - `check_solar_elevation` now does not assume location is in Copenhagen by default [!15](https://github.com/dmidk/sunflow/pull/15), @JoachimKoenigslieb
-- Renamed MTG and MTG_pvlib to specMAGIC_MTG and specMAGIC_MTG_pvlib, respectively [!24](https://github.com/dmidk/sunflow/pull/24), @KristianHMoller
+- Renamed MTG and MTG_pvlib to specMAGIC_MTG and specMAGIC_MTG_pvlib, respectively [!25](https://github.com/dmidk/sunflow/pull/25), @KristianHMoller
 
 ## [v1.2.0]
 

README.md

@@ -126,6 +126,7 @@ podman run -it --rm --entrypoint="" sunflow bash
 | `S3_BUCKET` | None | S3 bucket name |
 | `S3_INPUT_PREFIX` | `satellite_data` | S3 prefix for input data |
 | `S3_OUTPUT_PREFIX` | `nowcasts` | S3 prefix for forecast output |
+| `AWS_VERIFY_SSL` | `true` (when unset) | Verify TLS/SSL certificates for S3 connections. Set to `0`, `false`, `no`, or `off` to disable verification. |
 
 ### Volume Mounts
 

sunflow/config.py

@@ -46,6 +46,7 @@ class S3Config:
     bucket: str
     input_prefix: str
     output_prefix: str
+    verify_ssl: bool = True
 
     @classmethod
     def from_env(cls) -> Self:
@@ -57,12 +58,15 @@ def from_env(cls) -> Self:
         - S3_BUCKET (default: None)
         - S3_INPUT_PREFIX (default: satellite_data)
         - S3_OUTPUT_PREFIX (default: nowcasts)
+        - AWS_VERIFY_SSL (default: true; set to 0/false/no/off to disable)
         """
         return cls(
             endpoint_url=os.getenv("S3_ENDPOINT_URL", None),
             bucket=os.getenv("S3_BUCKET", None),
             input_prefix=os.getenv("S3_INPUT_PREFIX", "satellite_data"),
             output_prefix=os.getenv("S3_OUTPUT_PREFIX", "nowcasts"),
+            verify_ssl=os.getenv("AWS_VERIFY_SSL", "true").strip().lower()
+            in {"1", "true", "yes", "on"},
         )
 
 

sunflow/data_io.py

@@ -15,6 +15,19 @@
 from .validation import DataNotAvailableError
 
 
+def _build_s3_filesystem(s3_config: S3Config) -> fsspec.AbstractFileSystem:
+    """Build an S3 filesystem using SSL verification setting from config."""
+    client_kwargs: dict[str, Any] = {"endpoint_url": s3_config.endpoint_url}
+
+    if not s3_config.verify_ssl:
+        client_kwargs["verify"] = False
+        logger.warning(
+            "S3 SSL certificate verification disabled via S3Config.verify_ssl=False"
+        )
+
+    return fsspec.filesystem("s3", client_kwargs=client_kwargs)
+
+
 def fetch_current_data_with_retry(
     time_step: datetime,
     run_mode: str,
@@ -290,10 +303,7 @@ def check_current_data_existence_s3(
     logger.info(f"Checking existence of data at {s3_path}")
 
     try:
-        fs = fsspec.filesystem(
-            "s3",
-            client_kwargs={"endpoint_url": s3_config.endpoint_url},
-        )
+        fs = _build_s3_filesystem(s3_config)
         if not fs.exists(s3_path):
             raise FileNotFoundError(f"Input file not yet found in S3: {s3_path}")
 
@@ -339,10 +349,7 @@ def load_data_from_s3(
         FileNotFoundError: If any expected file does not exist in S3.
     """
     collected: list[xr.Dataset] = []
-    fs = fsspec.filesystem(
-        "s3",
-        client_kwargs={"endpoint_url": s3_config.endpoint_url},
-    )
+    fs = _build_s3_filesystem(s3_config)
 
     for time_step in time_steps:
         filename = generate_input_filename(
@@ -664,10 +671,7 @@ def save_forecast_to_s3(
     logger.info(f"Saving forecast to {s3_path}")
 
     try:
-        fs = fsspec.filesystem(
-            "s3",
-            client_kwargs={"endpoint_url": s3_config.endpoint_url},
-        )
+        fs = _build_s3_filesystem(s3_config)
         # Write to BytesIO buffer first
         buffer = BytesIO()
         dataset.to_netcdf(buffer, encoding={"time": time_encoding})