diff --git a/CHANGELOG.md b/CHANGELOG.md index 8483d79..de1dbfc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,13 +11,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added `{second}` and `{minute}` as a filename formatting variable [!15](https://github.com/dmidk/sunflow/pull/15), @JoachimKoenigslieb - Added a `clearsky` config object which can load clear-sky data from files via `clearsky.path` or generate it with `pvlib` via the `simplified_solis` method [!15](https://github.com/dmidk/sunflow/pull/15), @JoachimKoenigslieb - Added `MIN_SOLAR_ELEVATION_DEGREES` (defaulting to 6 degrees) to configure the minimum maximum-corner solar elevation required to run [!15](https://github.com/dmidk/sunflow/pull/15), @JoachimKoenigslieb -- Added configurations to run with MTG data with either calculated or pvlib clearsky [!23](https://github.com/dmidk/sunflow/pull/15), @KristianHMoller -- Added the ability to provide more configuration parameters to config.yaml [!23](https://github.com/dmidk/sunflow/pull/15), @KristianHMoller +- Added configurations to run with MTG data with either calculated or pvlib clearsky [!24](https://github.com/dmidk/sunflow/pull/24), @KristianHMoller +- Added the ability to provide more configuration parameters to config.yaml [!24](https://github.com/dmidk/sunflow/pull/24), @KristianHMoller +- Added to option to bypass SSL verification in S3 reading and writing [!27](https://github.com/dmidk/sunflow/pull/27), @KristianHMoller ### Changed - Use `.expand_dims` instead of `.assign_coords` to make sure we have both time dimension and time coordinates when loading from files [!15](https://github.com/dmidk/sunflow/pull/15), @JoachimKoenigslieb - `check_solar_elevation` now does not assume location is in Copenhagen by default [!15](https://github.com/dmidk/sunflow/pull/15), @JoachimKoenigslieb -- Renamed MTG and MTG_pvlib to specMAGIC_MTG and specMAGIC_MTG_pvlib, respectively [!24](https://github.com/dmidk/sunflow/pull/24), @KristianHMoller +- Renamed MTG and MTG_pvlib to specMAGIC_MTG and specMAGIC_MTG_pvlib, respectively [!25](https://github.com/dmidk/sunflow/pull/25), @KristianHMoller ## [v1.2.0] diff --git a/README.md b/README.md index 55b8096..81116dc 100644 --- a/README.md +++ b/README.md @@ -126,6 +126,7 @@ podman run -it --rm --entrypoint="" sunflow bash | `S3_BUCKET` | None | S3 bucket name | | `S3_INPUT_PREFIX` | `satellite_data` | S3 prefix for input data | | `S3_OUTPUT_PREFIX` | `nowcasts` | S3 prefix for forecast output | +| `AWS_VERIFY_SSL` | `true` (when unset) | Verify TLS/SSL certificates for S3 connections. Set to `0`, `false`, `no`, or `off` to disable verification. | ### Volume Mounts diff --git a/sunflow/config.py b/sunflow/config.py index cec970f..7173cbc 100644 --- a/sunflow/config.py +++ b/sunflow/config.py @@ -46,6 +46,7 @@ class S3Config: bucket: str input_prefix: str output_prefix: str + verify_ssl: bool = True @classmethod def from_env(cls) -> Self: @@ -57,12 +58,15 @@ def from_env(cls) -> Self: - S3_BUCKET (default: None) - S3_INPUT_PREFIX (default: satellite_data) - S3_OUTPUT_PREFIX (default: nowcasts) + - AWS_VERIFY_SSL (default: true; set to 0/false/no/off to disable) """ return cls( endpoint_url=os.getenv("S3_ENDPOINT_URL", None), bucket=os.getenv("S3_BUCKET", None), input_prefix=os.getenv("S3_INPUT_PREFIX", "satellite_data"), output_prefix=os.getenv("S3_OUTPUT_PREFIX", "nowcasts"), + verify_ssl=os.getenv("AWS_VERIFY_SSL", "true").strip().lower() + in {"1", "true", "yes", "on"}, ) diff --git a/sunflow/data_io.py b/sunflow/data_io.py index b487bde..d0a810d 100644 --- a/sunflow/data_io.py +++ b/sunflow/data_io.py @@ -15,6 +15,19 @@ from .validation import DataNotAvailableError +def _build_s3_filesystem(s3_config: S3Config) -> fsspec.AbstractFileSystem: + """Build an S3 filesystem using SSL verification setting from config.""" + client_kwargs: dict[str, Any] = {"endpoint_url": s3_config.endpoint_url} + + if not s3_config.verify_ssl: + client_kwargs["verify"] = False + logger.warning( + "S3 SSL certificate verification disabled via S3Config.verify_ssl=False" + ) + + return fsspec.filesystem("s3", client_kwargs=client_kwargs) + + def fetch_current_data_with_retry( time_step: datetime, run_mode: str, @@ -290,10 +303,7 @@ def check_current_data_existence_s3( logger.info(f"Checking existence of data at {s3_path}") try: - fs = fsspec.filesystem( - "s3", - client_kwargs={"endpoint_url": s3_config.endpoint_url}, - ) + fs = _build_s3_filesystem(s3_config) if not fs.exists(s3_path): raise FileNotFoundError(f"Input file not yet found in S3: {s3_path}") @@ -339,10 +349,7 @@ def load_data_from_s3( FileNotFoundError: If any expected file does not exist in S3. """ collected: list[xr.Dataset] = [] - fs = fsspec.filesystem( - "s3", - client_kwargs={"endpoint_url": s3_config.endpoint_url}, - ) + fs = _build_s3_filesystem(s3_config) for time_step in time_steps: filename = generate_input_filename( @@ -664,10 +671,7 @@ def save_forecast_to_s3( logger.info(f"Saving forecast to {s3_path}") try: - fs = fsspec.filesystem( - "s3", - client_kwargs={"endpoint_url": s3_config.endpoint_url}, - ) + fs = _build_s3_filesystem(s3_config) # Write to BytesIO buffer first buffer = BytesIO() dataset.to_netcdf(buffer, encoding={"time": time_encoding})