From eb2d0fbffc9b4ba4be9a4ee71ac70484befb0b3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristian=20H=2E=20M=C3=B8ller?= Date: Fri, 23 Oct 2026 10:22:02 +0200 Subject: [PATCH 1/7] Add option for not verifying ssl --- sunflow/config.py | 8 ++++++++ sunflow/data_io.py | 23 +++++++++++++++-------- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/sunflow/config.py b/sunflow/config.py index cec970f..a6147d2 100644 --- a/sunflow/config.py +++ b/sunflow/config.py @@ -25,6 +25,11 @@ } +def _env_var_is_truthy(var_name: str) -> bool: + """Return True when env var is set to a truthy value.""" + return os.getenv(var_name, "").strip().lower() in {"1", "true", "yes", "on"} + + # Predefined domain options # Format: lon_min,lat_min,lon_max,lat_max DOMAIN_OPTIONS: dict[str, str | None] = { @@ -46,6 +51,7 @@ class S3Config: bucket: str input_prefix: str output_prefix: str + verify_ssl: bool = True @classmethod def from_env(cls) -> Self: @@ -57,12 +63,14 @@ def from_env(cls) -> Self: - S3_BUCKET (default: None) - S3_INPUT_PREFIX (default: satellite_data) - S3_OUTPUT_PREFIX (default: nowcasts) + - AWS_NO_VERIFY_SSL (default: unset/false; when truthy, verify_ssl=False) """ return cls( endpoint_url=os.getenv("S3_ENDPOINT_URL", None), bucket=os.getenv("S3_BUCKET", None), input_prefix=os.getenv("S3_INPUT_PREFIX", "satellite_data"), output_prefix=os.getenv("S3_OUTPUT_PREFIX", "nowcasts"), + verify_ssl=not _env_var_is_truthy("AWS_NO_VERIFY_SSL"), ) diff --git a/sunflow/data_io.py b/sunflow/data_io.py index b487bde..6255b15 100644 --- a/sunflow/data_io.py +++ b/sunflow/data_io.py @@ -15,6 +15,19 @@ from .validation import DataNotAvailableError +def _build_s3_filesystem(s3_config: S3Config) -> fsspec.AbstractFileSystem: + """Build an S3 filesystem using SSL verification setting from config.""" + client_kwargs: dict[str, Any] = {"endpoint_url": s3_config.endpoint_url} + + if not s3_config.verify_ssl: + client_kwargs["verify"] = False + logger.warning( + "S3 SSL certificate verification disabled via S3Config.verify_ssl=False" + ) + + return fsspec.filesystem("s3", client_kwargs=client_kwargs) + + def fetch_current_data_with_retry( time_step: datetime, run_mode: str, @@ -290,10 +303,7 @@ def check_current_data_existence_s3( logger.info(f"Checking existence of data at {s3_path}") try: - fs = fsspec.filesystem( - "s3", - client_kwargs={"endpoint_url": s3_config.endpoint_url}, - ) + fs = _build_s3_filesystem(s3_config) if not fs.exists(s3_path): raise FileNotFoundError(f"Input file not yet found in S3: {s3_path}") @@ -339,10 +349,7 @@ def load_data_from_s3( FileNotFoundError: If any expected file does not exist in S3. """ collected: list[xr.Dataset] = [] - fs = fsspec.filesystem( - "s3", - client_kwargs={"endpoint_url": s3_config.endpoint_url}, - ) + fs = _build_s3_filesystem(s3_config) for time_step in time_steps: filename = generate_input_filename( From aa3e30780a83e0eac0476edb2d9d960c74a338a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristian=20H=2E=20M=C3=B8ller?= Date: Fri, 23 Oct 2026 12:08:20 +0200 Subject: [PATCH 2/7] Update changelog --- CHANGELOG.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8483d79..10ae1c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,13 +11,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added `{second}` and `{minute}` as a filename formatting variable [!15](https://github.com/dmidk/sunflow/pull/15), @JoachimKoenigslieb - Added a `clearsky` config object which can load clear-sky data from files via `clearsky.path` or generate it with `pvlib` via the `simplified_solis` method [!15](https://github.com/dmidk/sunflow/pull/15), @JoachimKoenigslieb - Added `MIN_SOLAR_ELEVATION_DEGREES` (defaulting to 6 degrees) to configure the minimum maximum-corner solar elevation required to run [!15](https://github.com/dmidk/sunflow/pull/15), @JoachimKoenigslieb -- Added configurations to run with MTG data with either calculated or pvlib clearsky [!23](https://github.com/dmidk/sunflow/pull/15), @KristianHMoller -- Added the ability to provide more configuration parameters to config.yaml [!23](https://github.com/dmidk/sunflow/pull/15), @KristianHMoller +- Added configurations to run with MTG data with either calculated or pvlib clearsky [!24](https://github.com/dmidk/sunflow/pull/24), @KristianHMoller +- Added the ability to provide more configuration parameters to config.yaml [!24](https://github.com/dmidk/sunflow/pull/24), @KristianHMoller +- Added to option to bypass SSL verification in S3 reading [!27](https://github.com/dmidk/sunflow/pull/27), @KristianHMoller ### Changed - Use `.expand_dims` instead of `.assign_coords` to make sure we have both time dimension and time coordinates when loading from files [!15](https://github.com/dmidk/sunflow/pull/15), @JoachimKoenigslieb - `check_solar_elevation` now does not assume location is in Copenhagen by default [!15](https://github.com/dmidk/sunflow/pull/15), @JoachimKoenigslieb -- Renamed MTG and MTG_pvlib to specMAGIC_MTG and specMAGIC_MTG_pvlib, respectively [!24](https://github.com/dmidk/sunflow/pull/24), @KristianHMoller +- Renamed MTG and MTG_pvlib to specMAGIC_MTG and specMAGIC_MTG_pvlib, respectively [!25](https://github.com/dmidk/sunflow/pull/25), @KristianHMoller ## [v1.2.0] From 575ef434a35dd7ce9d3072435837bf80430e184e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristian=20H=2E=20M=C3=B8ller?= Date: Fri, 23 Oct 2026 12:12:58 +0200 Subject: [PATCH 3/7] Invert logic and update readme --- README.md | 1 + sunflow/config.py | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 55b8096..81116dc 100644 --- a/README.md +++ b/README.md @@ -126,6 +126,7 @@ podman run -it --rm --entrypoint="" sunflow bash | `S3_BUCKET` | None | S3 bucket name | | `S3_INPUT_PREFIX` | `satellite_data` | S3 prefix for input data | | `S3_OUTPUT_PREFIX` | `nowcasts` | S3 prefix for forecast output | +| `AWS_VERIFY_SSL` | `true` (when unset) | Verify TLS/SSL certificates for S3 connections. Set to `0`, `false`, `no`, or `off` to disable verification. | ### Volume Mounts diff --git a/sunflow/config.py b/sunflow/config.py index a6147d2..6c2e74f 100644 --- a/sunflow/config.py +++ b/sunflow/config.py @@ -63,14 +63,18 @@ def from_env(cls) -> Self: - S3_BUCKET (default: None) - S3_INPUT_PREFIX (default: satellite_data) - S3_OUTPUT_PREFIX (default: nowcasts) - - AWS_NO_VERIFY_SSL (default: unset/false; when truthy, verify_ssl=False) + - AWS_VERIFY_SSL (default: true; set to 0/false/no/off to disable) """ return cls( endpoint_url=os.getenv("S3_ENDPOINT_URL", None), bucket=os.getenv("S3_BUCKET", None), input_prefix=os.getenv("S3_INPUT_PREFIX", "satellite_data"), output_prefix=os.getenv("S3_OUTPUT_PREFIX", "nowcasts"), - verify_ssl=not _env_var_is_truthy("AWS_NO_VERIFY_SSL"), + verify_ssl=( + True + if os.getenv("AWS_VERIFY_SSL") is None + else _env_var_is_truthy("AWS_VERIFY_SSL") + ), ) From 88b1e353652b67040debf1f4d43d2ace9c2a98b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristian=20H=2E=20M=C3=B8ller?= Date: Fri, 23 Oct 2026 12:21:06 +0200 Subject: [PATCH 4/7] Simplify --- sunflow/config.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/sunflow/config.py b/sunflow/config.py index 6c2e74f..e89d72a 100644 --- a/sunflow/config.py +++ b/sunflow/config.py @@ -70,11 +70,8 @@ def from_env(cls) -> Self: bucket=os.getenv("S3_BUCKET", None), input_prefix=os.getenv("S3_INPUT_PREFIX", "satellite_data"), output_prefix=os.getenv("S3_OUTPUT_PREFIX", "nowcasts"), - verify_ssl=( - True - if os.getenv("AWS_VERIFY_SSL") is None - else _env_var_is_truthy("AWS_VERIFY_SSL") - ), + verify_ssl=os.getenv("AWS_VERIFY_SSL", "true").strip().lower() + in {"1", "true", "yes", "on"}, ) From 318efda15a0c7f17c0d46608da1b8db916861143 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristian=20H=2E=20M=C3=B8ller?= Date: Fri, 23 Oct 2026 12:28:24 +0200 Subject: [PATCH 5/7] Remove unused function --- sunflow/config.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/sunflow/config.py b/sunflow/config.py index e89d72a..7173cbc 100644 --- a/sunflow/config.py +++ b/sunflow/config.py @@ -25,11 +25,6 @@ } -def _env_var_is_truthy(var_name: str) -> bool: - """Return True when env var is set to a truthy value.""" - return os.getenv(var_name, "").strip().lower() in {"1", "true", "yes", "on"} - - # Predefined domain options # Format: lon_min,lat_min,lon_max,lat_max DOMAIN_OPTIONS: dict[str, str | None] = { From d89e88f326b55dc0fa0803eb262faf6252c97c6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristian=20H=2E=20M=C3=B8ller?= Date: Fri, 23 Oct 2026 12:48:52 +0200 Subject: [PATCH 6/7] Update also S3 writing --- sunflow/data_io.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/sunflow/data_io.py b/sunflow/data_io.py index 6255b15..d0a810d 100644 --- a/sunflow/data_io.py +++ b/sunflow/data_io.py @@ -671,10 +671,7 @@ def save_forecast_to_s3( logger.info(f"Saving forecast to {s3_path}") try: - fs = fsspec.filesystem( - "s3", - client_kwargs={"endpoint_url": s3_config.endpoint_url}, - ) + fs = _build_s3_filesystem(s3_config) # Write to BytesIO buffer first buffer = BytesIO() dataset.to_netcdf(buffer, encoding={"time": time_encoding}) From 050b594824e7b5aff3eb513c6eb1bfcbb6ec2d78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristian=20H=2E=20M=C3=B8ller?= Date: Fri, 23 Oct 2026 12:54:15 +0200 Subject: [PATCH 7/7] Update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 10ae1c6..de1dbfc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added `MIN_SOLAR_ELEVATION_DEGREES` (defaulting to 6 degrees) to configure the minimum maximum-corner solar elevation required to run [!15](https://github.com/dmidk/sunflow/pull/15), @JoachimKoenigslieb - Added configurations to run with MTG data with either calculated or pvlib clearsky [!24](https://github.com/dmidk/sunflow/pull/24), @KristianHMoller - Added the ability to provide more configuration parameters to config.yaml [!24](https://github.com/dmidk/sunflow/pull/24), @KristianHMoller -- Added to option to bypass SSL verification in S3 reading [!27](https://github.com/dmidk/sunflow/pull/27), @KristianHMoller +- Added to option to bypass SSL verification in S3 reading and writing [!27](https://github.com/dmidk/sunflow/pull/27), @KristianHMoller ### Changed - Use `.expand_dims` instead of `.assign_coords` to make sure we have both time dimension and time coordinates when loading from files [!15](https://github.com/dmidk/sunflow/pull/15), @JoachimKoenigslieb