Make the output filename flexible and change default to ISO 8601 datetime

CHANGELOG.md

@@ -8,9 +8,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 
 ### Added
+- Added optional `output_filename_format` to config.yaml to configure the output file name [!30](https://github.com/dmidk/sunflow/pull/30), @KristianHMoller
+- Added `INPUT_FILENAME_FORMAT` and `OUTPUT_FILENAME_FORMAT` environment variables, which take priority over the corresponding config.yaml values [!30](https://github.com/dmidk/sunflow/pull/30), @KristianHMoller
+- Added the dataset used for the run (e.g. `KNMI_MTG`) as the `satellite_data_source` global attribute in the output file [!30](https://github.com/dmidk/sunflow/pull/30), @KristianHMoller
 - Added configuration for KNMI_MTG [!26](https://github.com/dmidk/sunflow/pull/26), @KristianHMoller
 - Added `satellite` as a required field in config.yaml to be printed in the outpt file [!26](https://github.com/dmidk/sunflow/pull/26), @KristianHMoller
 
+### Changed
+- Renamed `filename_format` to `input_filename_format` in config.yaml. It has no default and must be set in config.yaml or via `INPUT_FILENAME_FORMAT` [!30](https://github.com/dmidk/sunflow/pull/30), @KristianHMoller
+- Changed the default output filename to include the dataset name and an ISO 8601-compliant datetime (`SolarNowcast_{dataset_name}_{iso8601_compact_timestamp}.nc`, e.g. `SolarNowcast_DWD_2026-03-09T1200Z.nc`). KNMI_MSG and KNMI_MTG keep their own `output_filename_format` in config.yaml [!30](https://github.com/dmidk/sunflow/pull/30), @KristianHMoller
+
 ## [v1.3.0]
 
 ### Added

README.md

@@ -109,6 +109,8 @@ podman run -it --rm --entrypoint="" sunflow bash
 | `MAX_CLEARSKY_FALLBACK_DAYS` | `3` | Days back to search for fallback clear-sky data |
 | `MIN_SOLAR_ELEVATION_DEGREES` | `6` | Minimum maximum-corner solar elevation required to run |
 | `ENSEMBLE_STATISTICS` | `median,mean,p10,p25,p75,p90` | Comma-separated list of statistics for ensemble output. Allowed: `median`, `mean`, `p10`, `p25`, `p75`, `p90` (aliases `10th_percentile`, `25th_percentile`, `75th_percentile`, `90th_percentile` are accepted). |
+| `INPUT_FILENAME_FORMAT` | None (required) | Template for input data filenames, e.g. `NetCDF4_sds_{pds_timestamp}.nc`. Must be set here or as `input_filename_format` in `config.yaml`; the environment variable takes priority. See `config.yaml` for the available template variables. |
+| `OUTPUT_FILENAME_FORMAT` | `SolarNowcast_{dataset_name}_{iso8601_compact_timestamp}.nc` | Template for forecast output filenames. Can also be set as `output_filename_format` in `config.yaml`; the environment variable takes priority. |
 
 #### Data Source Configuration
 
@@ -130,7 +132,7 @@ podman run -it --rm --entrypoint="" sunflow bash
 
 ### Volume Mounts
 
-- `/data/nowcasts` - Forecast output files (NetCDF format)
+- `/data/nowcasts` - Forecast output files (NetCDF format), named `SolarNowcast_<dataset>_<YYYY-MM-DDTHHMMZ>.nc` by default (configurable via `output_filename_format` in `config.yaml`)
 - `/data/satellite_data` - Input satellite data archive
 
 ### Arguments

config.yaml

@@ -9,21 +9,32 @@ KNMI_MSG:
   nc_variable_names:
     sds: sds
     sds_cs: sds_cs
-  # Required: Filename format for input data files
-  # filename_format: "{dataset_name}_{timestamp}_{domain_satellite_name}.nc"
+  # Required here or via the INPUT_FILENAME_FORMAT environment variable
+  # (the environment variable takes priority): Filename format for input data files
+  # input_filename_format: "{dataset_name}_{timestamp}_{domain_satellite_name}.nc"
   # Format for PDS data:
-  filename_format: "NetCDF4_sds_{pds_timestamp}.nc"
+  input_filename_format: "NetCDF4_sds_{pds_timestamp}.nc"
   # Available template variables:
   #   {dataset_name}: Dataset name (e.g., KNMI)
   #   {domain_satellite_name}: Satellite domain identifier (e.g., NW_EUROPE_SATELLITE)
   #   {timestamp}: Compact format YYYYMMDDHHMM
   #   {pds_timestamp}: PDS format YYYY-MM-DDTHH_MM_SSZ
+  #   {iso8601_compact_timestamp}: ISO 8601 format timestamp (e.g., YYYY-MM-DDTHHMMZ)
   #   {year}: Four-digit year (e.g., 2026)
   #   {month}: Two-digit month (e.g., 03)
   #   {day}: Two-digit day of month (e.g., 09)
   #   {hour}: Two-digit hour (e.g., 12)
   #   {second}: Two-digit second (e.g., 00)
   # Path separators are supported, e.g.: "{year}/{month}/{day}/{dataset_name}_{timestamp}.nc"
+  #
+  output_filename_format: "SolarNowcast_{timestamp}.nc"
+  # Optional: output_filename_format (or the OUTPUT_FILENAME_FORMAT environment
+  # variable, which takes priority)
+  # Filename format for output forecast files.
+  # Defaults to "SolarNowcast_{dataset_name}_{iso8601_compact_timestamp}.nc" if omitted.
+  # Supports the same template variables as input_filename_format, except
+  # {domain_satellite_name}, which is not available for output files.
+  # output_filename_format: "SolarNowcast_{dataset_name}_{iso8601_compact_timestamp}.nc"
   satellite: Meteosat Second Generation (MSG)
 
 KNMI_MTG:
@@ -36,7 +47,8 @@ KNMI_MTG:
   nc_variable_names:
     sds: sds
     sds_cs: sds_cs
-  filename_format: "NetCDF4_sds_{pds_timestamp}.nc"
+  input_filename_format: "NetCDF4_sds_{pds_timestamp}.nc"
+  output_filename_format: "SolarNowcast_MTG-CPP_{iso8601_compact_timestamp}.nc"
   input_data_frequency_minutes: 10
   satellite: Meteosat Third Generation (MTG)
 
@@ -48,16 +60,16 @@ DWD:
   nc_variable_names:
     sds: SIS
     sds_cs: SISc
-  # Required: Filename format for input data files
+  # Required here or via the INPUT_FILENAME_FORMAT environment variable: Filename format for input data files
   # Supports subdirectories via path separators, e.g.: "{year}/{month}/{day}/{dataset_name}_{timestamp}_{domain_satellite_name}.nc"
-  filename_format: "{dataset_name}_{timestamp}_{domain_satellite_name}.nc"
+  input_filename_format: "{dataset_name}_{timestamp}_{domain_satellite_name}.nc"
   satellite: Meteosat Second Generation (MSG)
 
 specMAGIC_MTG:
   nc_variable_names:
     sds: GHI
     sds_cs: CSR
-  filename_format: "GHI/GHI{year}{month}{day}{hour}{minute}.nc"
+  input_filename_format: "GHI/GHI{year}{month}{day}{hour}{minute}.nc"
   input_data_frequency_minutes: 10
   clearsky:
     method: file
@@ -68,7 +80,7 @@ specMAGIC_MTG_pvlib:
   nc_variable_names:
     sds: GHI
     sds_cs: GHI_clearsky
-  filename_format: "GHI/GHI{year}{month}{day}{hour}{minute}.nc"
+  input_filename_format: "GHI/GHI{year}{month}{day}{hour}{minute}.nc"
   input_data_frequency_minutes: 10
   clearsky:
     method: pvlib

sunflow/config.py

@@ -5,6 +5,9 @@
 from typing import Self
 
 DEFAULT_ENSEMBLE_STATISTICS = "median,mean,p10,p25,p75,p90"
+DEFAULT_OUTPUT_FILENAME_FORMAT = (
+    "SolarNowcast_{dataset_name}_{iso8601_compact_timestamp}.nc"
+)
 _ALLOWED_STATISTICS = {
     "median",
     "mean",
@@ -87,6 +90,8 @@ class NowcastConfig:
     max_clearsky_fallback_days: int
     min_solar_elevation_degrees: float
     ensemble_statistics: list[str]
+    input_filename_format: str | None
+    output_filename_format: str
 
     @classmethod
     def from_env(
@@ -113,18 +118,21 @@ def from_env(
         - MAX_CLEARSKY_FALLBACK_DAYS (default: 3)
         - MIN_SOLAR_ELEVATION_DEGREES (default: 6)
         - ENSEMBLE_STATISTICS (default: median,mean,p10,p25,p75,p90)
+        - INPUT_FILENAME_FORMAT (no default; must be set here or in config.yaml)
+        - OUTPUT_FILENAME_FORMAT
+            (default: SolarNowcast_{dataset_name}_{iso8601_compact_timestamp}.nc)
         """
 
         overrides = overrides or {}
 
-        def _get_value(name: str, default: str) -> str:
+        def _get_optional_value(name: str) -> str | None:
             env_value = os.getenv(name)
             if env_value is not None:
                 return env_value
 
             override_value = overrides.get(name.lower())
             if override_value is None:
-                return default
+                return None
 
             if name == "ENSEMBLE_STATISTICS":
                 if isinstance(override_value, str):
@@ -134,6 +142,10 @@ def _get_value(name: str, default: str) -> str:
 
             return str(override_value)
 
+        def _get_value(name: str, default: str) -> str:
+            value = _get_optional_value(name)
+            return default if value is None else value
+
         ens_members = ensemble_members
         # Reference for default noise values:
         # A. Carpentieri, D. Folini, D. Nerini, S. Pulkkinen, M. Wild, A. Meyer,
@@ -166,6 +178,10 @@ def _get_value(name: str, default: str) -> str:
                 _get_value("MIN_SOLAR_ELEVATION_DEGREES", "6")
             ),
             ensemble_statistics=statistics,
+            input_filename_format=_get_optional_value("INPUT_FILENAME_FORMAT"),
+            output_filename_format=_get_value(
+                "OUTPUT_FILENAME_FORMAT", DEFAULT_OUTPUT_FILENAME_FORMAT
+            ),
         )