diff --git a/CHANGELOG.md b/CHANGELOG.md index 5521bab..1b72dcb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added +- Added configuration for KNMI_MTG [!26](https://github.com/dmidk/sunflow/pull/26), @KristianHMoller +- Added `satellite` as a required field in config.yaml to be printed in the outpt file [!26](https://github.com/dmidk/sunflow/pull/26), @KristianHMoller + ## [v1.3.0] ### Added diff --git a/README.md b/README.md index 81116dc..239db59 100644 --- a/README.md +++ b/README.md @@ -5,9 +5,9 @@ Framework for real-time satellite-based solar irradiance nowcasting at the Danis ## Overview This application performs short-term solar irradiance forecasting using: -- Real-time gridded surface solar irradiance products from processed geostationary satellite-data: [KNMI MSG-CPP](https://dataplatform.knmi.nl/dataset/msg-cpp-products-1-0) and [DWD SIS](https://opendata.dwd.de/weather/satellite/radiation/sis/) +- Real-time gridded surface solar irradiance products from processed geostationary satellite-data: [KNMI MSG-CPP](https://dataplatform.knmi.nl/dataset/msg-cpp-products-1-0), [KNMI MTG-CPP](https://dataplatform.knmi.nl/dataset/mtg-cpp-products-1-0) and [DWD SIS](https://opendata.dwd.de/weather/satellite/radiation/sis/) - Optical flow motion field computation with the Lucas-Kanade method applied to computed clear-sky indices -- Probabilistic advection for forecast generation, based on [SolarSTEPS](https://github.com/EnergyWeatherAI/SolarSTEPS) in deterministic mode +- Probabilistic advection for forecast generation, based on [SolarSTEPS](https://github.com/EnergyWeatherAI/SolarSTEPS) - Solar irradiance nowcasts calculated using clear-sky data from the previous day ## Fetching the source code @@ -65,7 +65,7 @@ To start a basic run, simply type the command below. This will start the default podman run --rm sunflow ``` -To run for a specific time, specify this when calling the container in the ISO8601 format. Only multiples of 15 minutes are available. Input data from KNMI is available going back one week. Other input can be specified in the same way. +To run for a specific time, specify this when calling the container in the ISO8601 format. Input data from KNMI is available going back one week. Other input can be specified in the same way. ```shell podman run --rm sunflow --time 2025-10-27T10:00Z @@ -135,7 +135,7 @@ podman run -it --rm --entrypoint="" sunflow bash ### Arguments -- `--dataset` - Choose between KNMI or DWD data sources +- `--dataset` - Choose between input data sources - `--domain_satellite` - Domain for required satellite input coverage (DENMARK, NW_EUROPE, NW_EUROPE_SATELLITE, CUSTOM) - `--custom_domain_satellite` - Custom `domain_satellite` (lon_min,lat_min,lon_max,lat_max) - `--domain_nowcast` - Domain written to output (DENMARK, NW_EUROPE, NW_EUROPE_SATELLITE, CUSTOM, defaults to `--domain_satellite`) @@ -153,8 +153,9 @@ all members instead. ## Data Sources -- **KNMI**: MSG-CPP products — requires a free API key from the [KNMI Data Platform](https://dataplatform.knmi.nl/dataset/access/msg-cpp-products-1-0) +- **KNMI**: MSG-CPP and MTG-CPP products — requires a free API key from the [KNMI Data Platform](https://dataplatform.knmi.nl/dataset/access/msg-cpp-products-1-0) - **DWD**: Surface Incoming Shortwave radiation (SIS) — publicly available, no API key required +- **specMAGIC_\***: GHI produced using the open-source [specMAGIC](https://github.com/dmidk/specMAGIC) code ## Installing locally Building pysteps can sometimes fail if a pre-built wheel is not available for your Python version. If the environment variables `CC` and `CXX` are not set, building pysteps falls back to clang, which may only work on Intel-based macOS. To use GNU compilers instead: diff --git a/config.yaml b/config.yaml index 70fa4c6..2f842aa 100644 --- a/config.yaml +++ b/config.yaml @@ -1,5 +1,5 @@ --- -KNMI: +KNMI_MSG: crs: EPSG:4326 format: NetCDF4 base_url: https://api.dataplatform.knmi.nl/wms/adaguc-server?dataset=msg_cpp_products&service=wcs&request=getcoverage& @@ -24,6 +24,21 @@ KNMI: # {hour}: Two-digit hour (e.g., 12) # {second}: Two-digit second (e.g., 00) # Path separators are supported, e.g.: "{year}/{month}/{day}/{dataset_name}_{timestamp}.nc" + satellite: Meteosat Second Generation (MSG) + +KNMI_MTG: + crs: EPSG:4326 + format: NetCDF4 + base_url: https://api.dataplatform.knmi.nl/wms/adaguc-server?dataset=mtg_cpp_products&service=wcs&request=getcoverage& + variables: + - surface_downwelling_shortwave_flux_in_air + - surface_downwelling_shortwave_flux_in_air_assuming_clear_sky + nc_variable_names: + sds: sds + sds_cs: sds_cs + filename_format: "NetCDF4_sds_{pds_timestamp}.nc" + input_data_frequency_minutes: 10 + satellite: Meteosat Third Generation (MTG) DWD: base_url: https://opendata.dwd.de/weather/satellite/radiation/sis/ @@ -36,6 +51,7 @@ DWD: # Required: Filename format for input data files # Supports subdirectories via path separators, e.g.: "{year}/{month}/{day}/{dataset_name}_{timestamp}_{domain_satellite_name}.nc" filename_format: "{dataset_name}_{timestamp}_{domain_satellite_name}.nc" + satellite: Meteosat Second Generation (MSG) specMAGIC_MTG: nc_variable_names: @@ -46,6 +62,7 @@ specMAGIC_MTG: clearsky: method: file path: "CSR/CSR{year}{month}{day}{hour}{minute}.nc" + satellite: Meteosat Third Generation (MTG) specMAGIC_MTG_pvlib: nc_variable_names: @@ -55,3 +72,4 @@ specMAGIC_MTG_pvlib: input_data_frequency_minutes: 10 clearsky: method: pvlib + satellite: Meteosat Third Generation (MTG) diff --git a/pyproject.toml b/pyproject.toml index 42aa2cf..0bb2caa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "sunflow" version = "1.3.0" -description = "Solar nowcasting using probabilistic advection with KNMI/DWD satellite data" +description = "Optical flow solar nowcasting based on satellite data" authors = [ {name = "Irene Livia Kruse", email = "ikr@dmi.dk"}, {name = "Kristian Holten Møller", email = "khm@dmi.dk"}, diff --git a/sunflow/data_io.py b/sunflow/data_io.py index d0a810d..e96e2cb 100644 --- a/sunflow/data_io.py +++ b/sunflow/data_io.py @@ -46,7 +46,7 @@ def fetch_current_data_with_retry( run_mode: One of 'download', 'files', or 's3' config: Dataset configuration dict domain_satellite: Domain string for required satellite input coverage - dataset_name: Name of dataset (options: KNMI, DWD) + dataset_name: Name of the dataset source. domain_satellite_name: Domain choice used in input filenames nowcast_config: NowcastConfig object s3_config: S3Config object @@ -132,7 +132,7 @@ def generate_input_filename( Args: time_step: Datetime of the data timestep. - dataset_name: Name of dataset (options: KNMI, DWD). + dataset_name: Name of the dataset source. domain_satellite_name: Satellite domain identifier. filename_format: Template string from config['filename_format']. @@ -140,7 +140,7 @@ def generate_input_filename( Filename string with all template variables substituted. Template variables supported: - {dataset_name}: Name of the dataset + {dataset_name}: Name of the dataset source {domain_satellite_name}: Satellite domain identifier {timestamp}: Compact format YYYYMMDDHHMM {pds_timestamp}: PDS format YYYY-MM-DDTHH_MM_SSZ @@ -187,7 +187,7 @@ def check_current_data_existence_file( Args: request_time: Python datetime object. - dataset_name: Name of dataset (options: KNMI, DWD). + dataset_name: Name of the dataset source. domain_satellite_name: Satellite domain identifier. satellite_data_directory: Directory containing data files. filename_format: Template string from config['filename_format']. @@ -227,7 +227,7 @@ def load_data_from_files( Args: time_steps: List of timesteps to load. - dataset_name: Name of dataset (options: KNMI, DWD). + dataset_name: Name of the dataset source. domain_satellite_name: Satellite domain identifier. satellite_data_directory: Directory containing data files. data_type: Type of data for logging (options: past data, clearsky data). @@ -286,7 +286,7 @@ def check_current_data_existence_s3( Args: request_time: Python datetime object. - dataset_name: Name of dataset (options: KNMI, DWD). + dataset_name: Name of the dataset source. domain_satellite_name: Satellite domain identifier. s3_config: S3 configuration object. filename_format: Template string from config['filename_format']. @@ -336,7 +336,7 @@ def load_data_from_s3( Args: time_steps: List of timesteps to load. - dataset_name: Name of dataset (options: KNMI, DWD). + dataset_name: Name of the dataset source. domain_satellite_name: Satellite domain identifier. s3_config: S3 configuration object. data_type: Type of data for logging (options: past data, clearsky data). @@ -402,7 +402,7 @@ def fetch_clearsky_with_fallback( filename_format: Template string for clear-sky files. config: Dataset configuration dict. bbox: Bounding box string. - dataset_name: Name of dataset (options: KNMI, DWD). + dataset_name: Name of the dataset source. domain_satellite_name: Satellite domain identifier. nowcast_config: NowcastConfig object. s3_config: S3Config object. @@ -486,6 +486,7 @@ def save_forecast( latitudes: np.ndarray, longitudes: np.ndarray, dataset_name: str, + satellite: str, nowcast_config: NowcastConfig, model_version: str, output_mode: str, @@ -505,7 +506,8 @@ def save_forecast( n_steps: Number of forecast time steps to write. latitudes: 1-D array of latitude values (degrees). longitudes: 1-D array of longitude values (degrees). - dataset_name: Name of the source dataset (options: KNMI, DWD). + dataset_name: Name of the dataset source. + satellite: Satellite platform name written as a global attribute. nowcast_config: NowcastConfig object supplying output directory and input data frequency. model_version: Model version string written as a global attribute. @@ -616,6 +618,7 @@ def save_forecast( f"using {dataset_name} data" ), "output_mode": output_mode, + "satellite": satellite, "history": ( f"Created " f"{datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S')} UTC" diff --git a/sunflow/downloaders.py b/sunflow/downloaders.py index 528776d..6a3f8e3 100644 --- a/sunflow/downloaders.py +++ b/sunflow/downloaders.py @@ -91,11 +91,11 @@ def download_current_data( config: Dataset configuration dict from config.yaml, including base_url, variables, format, crs, and filename_format. bbox: Bounding box string lon_min,lat_min,lon_max,lat_max. - dataset_name: Name of the dataset source (options: KNMI, DWD). + dataset_name: Name of the dataset source. domain_satellite_name: Satellite domain identifier used in the output filename. satellite_data_directory: Directory where the NetCDF file is saved. """ - if dataset_name == "KNMI": + if dataset_name in ["KNMI_MSG", "KNMI_MTG"]: datasets = [] for variable in config["variables"]: url = ( @@ -149,7 +149,7 @@ def download_past_data( time_steps: Ordered list of datetimes to download. config: Dataset configuration dict from config.yaml. bbox: Bounding box string lon_min,lat_min,lon_max,lat_max. - dataset_name: Name of the dataset source (options: KNMI, DWD). + dataset_name: Name of the dataset source. Returns: xarray Dataset concatenated along a 'time' dimension, with one @@ -160,7 +160,7 @@ def download_past_data( for time_step in time_steps: time_str = time_step.strftime("%Y-%m-%dT%H:%M:%SZ") - if dataset_name == "KNMI": + if dataset_name in ["KNMI_MSG", "KNMI_MTG"]: # Download each variable separately, then merge var_datasets: list[xr.Dataset] = [] failed_vars = [] @@ -170,6 +170,7 @@ def download_past_data( f"&CRS={config['crs']}&BBOX={bbox}&time={time_str}" ) url = f"{base_url}&coverage={var}" + logger.info(f"Downloading {var} for {time_str} from {url}") try: netcdf_file = download_netcdf_knmi(url, os.getenv("KNMI_API_KEY")) ds = xr.open_dataset(netcdf_file, engine="h5netcdf") @@ -231,7 +232,7 @@ def download_clearsky_data( (typically forecast lead times offset by one day). config: Dataset configuration dict from config.yaml. bbox: Bounding box string lon_min,lat_min,lon_max,lat_max. - dataset_name: Name of the dataset source (options: KNMI, DWD). + dataset_name: Name of the dataset source. Returns: xarray Dataset concatenated along a 'time' dimension, with one @@ -239,7 +240,7 @@ def download_clearsky_data( """ collected: list[xr.Dataset] = [] - if dataset_name == "KNMI": + if dataset_name in ["KNMI_MSG", "KNMI_MTG"]: var = config["variables"][1] # clearsky variable for time_step in time_steps: clearsky_time_str = time_step.strftime("%Y-%m-%dT%H:%M:%SZ") diff --git a/sunflow/main.py b/sunflow/main.py index 1be43f0..fb9feaa 100644 --- a/sunflow/main.py +++ b/sunflow/main.py @@ -105,7 +105,7 @@ def parse_datetime_with_timezone(datetime_str: str) -> datetime: "--run_mode", choices=["download", "files", "s3"], default="download", - help="Run mode (default: download): \n" + help="Run mode (default: %(default)s): \n" "download: fetch from API. For KNMI data, this requires " "KNMI_API_KEY environment variable\n" "files: read from local files\n" @@ -114,10 +114,11 @@ def parse_datetime_with_timezone(datetime_str: str) -> datetime: ) parser.add_argument( "--dataset", - choices=["KNMI", "DWD", "specMAGIC_MTG", "specMAGIC_MTG_pvlib"], - default="KNMI", - help="Dataset (default: KNMI): \n" - "KNMI: MSG-CPP data from KNMI\n" + choices=["KNMI_MSG", "KNMI_MTG", "DWD", "specMAGIC_MTG", "specMAGIC_MTG_pvlib"], + default="KNMI_MSG", + help="Dataset (default: %(default)s): \n" + "KNMI_MSG: MSG-CPP data from KNMI\n" + "KNMI_MTG: MTG-CPP data from KNMI\n" "DWD: MSG-based data from DWD\n" "specMAGIC_MTG: MTG-based GHI data from specMAGIC\n" "specMAGIC_MTG_pvlib: MTG-based GHI data from specMAGIC with pvlib clearsky", @@ -126,7 +127,7 @@ def parse_datetime_with_timezone(datetime_str: str) -> datetime: "--domain_satellite", choices=DOMAIN_CHOICES, default="NW_EUROPE", - help="Domain required for satellite input coverage (default: NW_EUROPE)", + help="Domain required for satellite input coverage (default: %(default)s)", ) parser.add_argument( "--custom_domain_satellite", @@ -171,7 +172,7 @@ def parse_datetime_with_timezone(datetime_str: str) -> datetime: "--ensemble_members", type=int, default=1, - help="Number of ensemble members (default: 1)", + help="Number of ensemble members (default: %(default)s)", ) parser.add_argument( "--full_ensemble", @@ -235,7 +236,7 @@ def run_nowcast( config: Dataset configuration dict. domain_satellite: Domain string used for satellite input coverage. domain_nowcast: Domain string used for output cropping. - dataset_name: Name of dataset. + dataset_name: Name of the dataset source. domain_satellite_name: Domain identifier used for input filenames. nowcast_config: NowcastConfig object. s3_config: S3Config object. @@ -244,8 +245,7 @@ def run_nowcast( custom_time: If True, skip the retry wait loop on missing data. Returns: - Output filename on success, or None if the run was skipped - (e.g. sun too low or missing data in a time-span run). + RunResult object indicating the outcome of the nowcast. """ time_step_str = time_step.strftime("%Y-%m-%dT%H:%M:%SZ") logger.info(f"--- Running nowcast for {time_step_str} ---") @@ -509,6 +509,7 @@ def run_nowcast( latitudes, longitudes, dataset_name, + config["satellite"], nowcast_config, model_version, output_mode, diff --git a/sunflow/validation.py b/sunflow/validation.py index b32e475..297950d 100644 --- a/sunflow/validation.py +++ b/sunflow/validation.py @@ -53,12 +53,12 @@ def validate_config(config: dict[str, Any], dataset_name: str) -> None: Args: config: Dataset configuration dict loaded from config.yaml. Requires the full config for the dataset to check for all required keys. - dataset_name: Name of the dataset (used in error messages). + dataset_name: Name of the dataset source. Raises: SystemExit: If any required key is absent. """ - required_keys = ["filename_format"] + required_keys = ["filename_format", "satellite"] missing = [k for k in required_keys if k not in config] if missing: logger.error( @@ -99,7 +99,7 @@ def validate_run_mode(run_mode: str, dataset_name: str) -> None: Args: run_mode: The requested run mode ('download', 'files', or 's3'). - dataset_name: Name of the dataset. + dataset_name: Name of the dataset source. Raises: SystemExit: If the run mode is incompatible with the dataset. @@ -119,12 +119,12 @@ def verify_environment_variables(run_mode: str, dataset_name: str) -> None: Args: run_mode: The mode in which the application is running ('download', 'files', 's3'). - dataset_name: Name of the dataset being used. + dataset_name: Name of the dataset source. Raises: SystemExit: If any required environment variable is missing. """ - if run_mode == "download" and dataset_name == "KNMI": + if run_mode == "download" and dataset_name in ["KNMI_MSG", "KNMI_MTG"]: if not os.getenv("KNMI_API_KEY"): logger.error("KNMI_API_KEY environment variable not set. Exiting.\n") sys.exit(1)