API Reference¶
Generated from the source. For task-oriented documentation see Usage; for provider specifics see Datasets.
Workflow¶
ClimData is the main entry point — an alias for ClimateExtractor. It
composes the configuration, drives each provider, and carries a dataset between
steps.
climdata.utils.wrapper_workflow.ClimateExtractor
¶
Climate data extraction and extreme-index workflow manager.
Provides a high-level API for: - loading/configuring dataset providers via Hydra config - uploading NetCDF/CSV content into xarray Datasets - extracting data from supported providers (CMIP, DWD, MSWX, HYRAS, POWER) - computing extreme indices using configured xclim indices - converting datasets to long-form DataFrames and saving results
Attributes:
| Name | Type | Description |
|---|---|---|
cfg |
DictConfig |
Hydra configuration object describing dataset, region/time/variables, outputs. |
current_ds |
xr.Dataset |
The most recently loaded or extracted dataset. |
current_df |
pd.DataFrame |
The most recently produced long-form DataFrame. |
filename_csv/filename_nc/filename_zarr |
str |
Generated output filename templates/paths. |
Examples:
extractor = ClimateExtractor(overrides=['dataset=cmip', 'region=europe']) extractor.extract() idx_ds = extractor.calc_index() df = extractor.to_dataframe(idx_ds) extractor.to_csv(df)
Source code in climdata/utils/wrapper_workflow.py
class ClimateExtractor:
"""Climate data extraction and extreme-index workflow manager.
Provides a high-level API for:
- loading/configuring dataset providers via Hydra config
- uploading NetCDF/CSV content into xarray Datasets
- extracting data from supported providers (CMIP, DWD, MSWX, HYRAS, POWER)
- computing extreme indices using configured xclim indices
- converting datasets to long-form DataFrames and saving results
Attributes:
cfg (DictConfig): Hydra configuration object describing dataset, region/time/variables, outputs.
current_ds (xr.Dataset): The most recently loaded or extracted dataset.
current_df (pd.DataFrame): The most recently produced long-form DataFrame.
filename_csv/filename_nc/filename_zarr (str): Generated output filename templates/paths.
Example:
extractor = ClimateExtractor(overrides=['dataset=cmip', 'region=europe'])
extractor.extract()
idx_ds = extractor.calc_index()
df = extractor.to_dataframe(idx_ds)
extractor.to_csv(df)
"""
def __init__(self, cfg_name="config", conf_path=None, overrides: Optional[List[str]] = None):
"""Initialize the workflow manager and load configuration.
Args:
cfg_name (str): Name of the Hydra configuration (default: "config").
conf_path (str, optional): Optional config path override.
overrides (list[str], optional): Hydra overrides to apply to the configuration.
"""
self.cfg_name = cfg_name
self.conf_path = conf_path
self.cfg: Optional[DictConfig] = None
# Stage datasets
self.ds = None
self.current_ds = None
self.index_ds = None
self.impute_ds = None
self.reprojected_ds = None
self.bias_corrected_ds = None
# Stage DataFrames
self.raw_df = None
self.current_df = None
self.index_df = None
self.impute_df = None
self.bias_corrected_df = None
self.df = None # alias for current_df
# filenames
self.filename = None
self.filetype = None
# Optional shared Dask distributed client (created lazily by extract())
self._dask_client = None
# Automatically load config on init
self.load_config(overrides)
self.cfg = self.preprocess_aoi(self.cfg)
# instance logger for this extractor
self.logger = logging.getLogger(f"{__name__}.{self.__class__.__name__}")
def _gen_fn(self, *, ds: xr.Dataset = None, df: pd.DataFrame = None):
"""Build output filenames from *data* metadata rather than from config.
The counterpart to :meth:`_gen_fn_cfg`, used after uploading a file: the
provider, variables and extent are read from the data itself, because an
uploaded dataset need not match what the configuration describes.
Args:
ds (xr.Dataset, optional): Dataset to describe. Keyword-only.
df (pd.DataFrame, optional): Long-form frame with ``lat``/``lon``,
``time`` or ``date``, ``variable``, ``value`` and optionally
``source`` columns. Keyword-only.
Returns:
ClimateExtractor: ``self``, with ``filename_csv``, ``filename_nc``
and ``filename_zarr`` set.
Raises:
ValueError: If both or neither of ``ds`` and ``df`` are given.
"""
# ------------------------
# Validate inputs
# ------------------------
if (ds is None) == (df is None):
raise ValueError("Provide exactly one of `ds` or `df` as a keyword argument.")
# ------------------------
# Helper: find coord alias (xarray)
# ------------------------
def find_coord(ds, names):
for name in names:
if name in ds.coords:
return ds[name]
return None
# ------------------------
# Case 1: xarray.Dataset
# ------------------------
if ds is not None:
lat = find_coord(ds, ["lat", "latitude"])
lon = find_coord(ds, ["lon", "longitude"])
time = find_coord(ds, ["time", "date"])
provider = ds.attrs.get("source", "unknown")
vars_list = list(ds.data_vars)
parameter = vars_list[0] if len(vars_list) == 1 else "_".join(vars_list)
# Latitude range
if lat is not None:
lat_vals = lat.values.reshape(-1)
lat_min, lat_max = float(lat_vals.min()), float(lat_vals.max())
else:
lat_min = lat_max = None
# Longitude range
if lon is not None:
lon_vals = lon.values.reshape(-1)
lon_min, lon_max = float(lon_vals.min()), float(lon_vals.max())
else:
lon_min = lon_max = None
# Time range
if time is not None:
tvals = pd.to_datetime(time.values)
start, end = tvals.min().strftime("%Y-%m-%d"), tvals.max().strftime("%Y-%m-%d")
else:
start = end = "unknown"
# ------------------------
# Case 2: pandas.DataFrame (long form)
# ------------------------
else:
cols = df.columns.astype(str)
# Identify coordinate columns
lat_cols = [c for c in cols if "lat" in c.lower()]
lon_cols = [c for c in cols if "lon" in c.lower()]
time_cols = [c for c in cols if "time" in c.lower() or "date" in c.lower()]
# Provider from 'source' column
if "source" in df.columns:
unique_sources = df["source"].dropna().unique()
provider = unique_sources[0] if len(unique_sources) == 1 else "_".join(map(str, unique_sources))
else:
provider = "unknown"
# Unique parameters from 'variable' column
if "variable" in df.columns:
unique_parameters = sorted(df["variable"].dropna().unique())
parameter = unique_parameters[0] if len(unique_parameters) == 1 else "_".join(unique_parameters)
else:
parameter = "unknown"
# Latitude range
if lat_cols:
lat_vals = pd.to_numeric(df[lat_cols[0]], errors="coerce")
lat_min, lat_max = float(lat_vals.min()), float(lat_vals.max())
else:
lat_min = lat_max = None
# Longitude range
if lon_cols:
lon_vals = pd.to_numeric(df[lon_cols[0]], errors="coerce")
lon_min, lon_max = float(lon_vals.min()), float(lon_vals.max())
else:
lon_min = lon_max = None
# Time range
if time_cols:
tvals = pd.to_datetime(df[time_cols[0]], errors="coerce")
start = tvals.min().strftime("%Y-%m-%d")
end = tvals.max().strftime("%Y-%m-%d")
else:
start = end = "unknown"
# ------------------------
# Format lat/lon strings
# ------------------------
if lat_min is None:
lat_str = lat_range = "unknown"
elif lat_min == lat_max:
lat_str = lat_range = f"{lat_min}"
else:
lat_str = f"{lat_min}_{lat_max}"
lat_range = f"{lat_min}-{lat_max}"
if lon_min is None:
lon_str = lon_range = "unknown"
elif lon_min == lon_max:
lon_str = lon_range = f"{lon_min}"
else:
lon_str = f"{lon_min}_{lon_max}"
lon_range = f"{lon_min}-{lon_max}"
# ------------------------
# Build filenames
# ------------------------
outdir = Path(self.cfg.output.out_dir)
outdir.mkdir(parents=True, exist_ok=True)
def build(fn_template):
# The same placeholder set as _gen_fn_cfg, so one template works for
# both the configured and the data-derived path. `lat_or_lat_range`
# is what conf/config.yaml actually uses.
return fn_template.format(
provider=provider,
parameter=parameter,
lat=lat_str,
lon=lon_str,
start=start,
end=end,
lat_range=lat_range,
lon_range=lon_range,
lat_or_lat_range=lat_range,
lon_or_lon_range=lon_range,
)
self.filename_csv = str(outdir / build(self.cfg.output.filename_csv))
self.filename_nc = str(outdir / build(self.cfg.output.filename_nc))
self.filename_zarr = str(outdir / build(self.cfg.output.filename_zarr))
return self
def _gen_fn_cfg(self):
"""Build output filenames from the configuration and the current dataset.
The usual path, called automatically by :func:`update_ds` after every
stage. The provider name, region and period come from ``cfg``; the
variable part comes from ``current_ds`` when one exists, so the filename
follows the data through the workflow — after ``calc_index()`` it names
the index rather than the input variables.
Returns:
None: ``filename_csv``, ``filename_nc`` and ``filename_zarr`` are set
on the instance, and ``cfg.output.out_dir`` is created.
"""
cfg = self.cfg
out = cfg.output
provider = cfg.dataset.lower()
if self.current_ds:
if len(self.current_ds.data_vars) == 0:
parameter = "unknown"
elif len(self.current_ds.data_vars) == 1:
parameter = next(iter(self.current_ds.data_vars))
else:
parameter = "_".join(self.current_ds.data_vars)
else:
parameter = "_".join(self.cfg.variables)
# --------------------------------
# Determine lat/lon values
# --------------------------------
if cfg.lat is not None and cfg.lon is not None:
lat_range = lon_range = None # single point
lat_str = str(cfg.lat)
lon_str = str(cfg.lon)
elif cfg.region is not None:
b = cfg.bounds[cfg.region]
lat_min, lat_max = b["lat_min"], b["lat_max"]
lon_min, lon_max = b["lon_min"], b["lon_max"]
lat_str = f"{lat_min}_{lat_max}"
lon_str = f"{lon_min}_{lon_max}"
lat_range = f"{lat_min}-{lat_max}"
lon_range = f"{lon_min}-{lon_max}"
else:
# Fallback for undefined regions
lat_str = "unknown"
lon_str = "unknown"
lat_range = None
lon_range = None
# --------------------------------
# Time range from cfg
# --------------------------------
start = pd.to_datetime(cfg.time_range.start_date).strftime("%Y-%m-%d")
end = pd.to_datetime(cfg.time_range.end_date).strftime("%Y-%m-%d")
# --------------------------------
# Format filenames
# --------------------------------
def format_template(template):
return template.format(
provider=provider,
parameter=parameter,
lat=lat_str,
lon=lon_str,
lat_or_lat_range=lat_range or lat_str,
lon_or_lon_range=lon_range or lon_str,
start=start,
end=end,
)
out_dir = Path(self.cfg.output.out_dir)
out_dir.mkdir(parents=True, exist_ok=True)
self.filename_csv = str(out_dir / format_template(out.filename_csv))
self.filename_nc = str(out_dir / format_template(out.filename_nc))
self.filename_zarr = str(out_dir / format_template(out.filename_zarr))
# ----------------------------
# Hydra config
# ----------------------------
def load_config(self, overrides: Optional[List[str]] = None) -> DictConfig:
"""Load and compose the Hydra configuration.
Args:
overrides (list[str], optional): Hydra overrides to apply when composing the configuration.
Returns:
DictConfig: Composed Hydra configuration object stored on ``self.cfg``.
"""
overrides = overrides or []
conf_dir_abs = os.path.abspath(_ensure_local_conf())
GlobalHydra.instance().clear()
with initialize_config_dir(config_dir=conf_dir_abs, version_base=None):
self.cfg = compose(config_name=self.cfg_name, overrides=overrides)
return self.cfg
# ----------------------------
# AOI preprocessing
# ----------------------------
def preprocess_aoi(self, cfg: DictConfig) -> DictConfig:
"""Process an 'aoi' specification in the configuration.
Supports GeoJSON strings or dictionaries for FeatureCollection, Feature, or simple geometry objects (Point/Polygon).
Args:
cfg (DictConfig): Configuration object with optional ``aoi`` entry.
Returns:
DictConfig: The modified configuration. When a Point is provided, ``cfg.lat`` and ``cfg.lon`` are set; when a Polygon is provided, ``cfg.bounds`` is set and ``cfg.region`` is set to "custom".
"""
if not hasattr(cfg, "aoi") or cfg.aoi is None:
return cfg
if isinstance(cfg.aoi, str):
try:
cfg.aoi = json.loads(cfg.aoi)
except json.JSONDecodeError:
raise ValueError("Invalid AOI JSON string")
aoi = cfg.aoi
if aoi.get("type") == "FeatureCollection":
geom = shape(aoi["features"][0]["geometry"])
elif aoi.get("type") == "Feature":
geom = shape(aoi["geometry"])
elif "type" in aoi:
geom = shape(aoi)
else:
raise ValueError(f"Unsupported AOI format: {aoi}")
if isinstance(geom, Point):
cfg.lat = geom.y
cfg.lon = geom.x
cfg.bounds = None
elif isinstance(geom, Polygon):
minx, miny, maxx, maxy = geom.bounds
cfg.bounds = {"custom": {"lat_min": miny, "lat_max": maxy,
"lon_min": minx, "lon_max": maxx}}
cfg.region = "custom"
cfg.lat = None
cfg.lon = None
else:
raise ValueError(f"Unknown geometry type {geom.geom_type}")
return cfg
# ----------------------------
# Upload NetCDF
# ----------------------------
@update_ds(attr_name='ds')
def upload_netcdf(self, nc_file: str) -> xr.Dataset:
"""Load a NetCDF file into an xarray.Dataset and update file metadata.
Args:
nc_file (str): Path to the NetCDF file to open.
Returns:
xr.Dataset: The loaded dataset (also sets ``self.current_ds``).
"""
if not os.path.exists(nc_file):
raise FileNotFoundError(f"{nc_file} does not exist")
ds = xr.open_dataset(nc_file)
# Update cfg variables & varinfo
if not hasattr(self.cfg, "variables") or not self.cfg.variables:
self.cfg.variables = list(ds.data_vars)
if not hasattr(self.cfg, "varinfo") or not self.cfg.varinfo:
self.cfg.varinfo = {v: {"units": ds[v].attrs.get("units", "unknown")}
for v in ds.data_vars}
self._gen_fn(ds=ds)
return ds
# ----------------------------
# Upload CSV → xarray.Dataset
# ----------------------------
@update_ds(attr_name='ds')
def upload_csv(self, csv_file: str) -> xr.Dataset:
"""Load a long-form CSV into an xarray.Dataset.
The CSV must contain ``time`` and ``lat``/``latitude``, ``lon``/``longitude``, ``variable``, ``value``. Units may be supplied in a ``units`` column and an optional ``source`` column is recognized.
Args:
csv_file (str): Path to the CSV file to load.
Returns:
xr.Dataset: The converted dataset (also sets ``self.current_ds``).
"""
if not os.path.exists(csv_file):
raise FileNotFoundError(f"{csv_file} does not exist")
df = pd.read_csv(csv_file, parse_dates=["time"])
lat_col = next((c for c in ["lat", "latitude"] if c in df.columns), None)
lon_col = next((c for c in ["lon", "longitude"] if c in df.columns), None)
if lat_col is None or lon_col is None:
raise ValueError("CSV must have 'lat'/'latitude' and 'lon'/'longitude' columns")
id_vars = ["time", lat_col, lon_col]
df_wide = df.pivot_table(index=id_vars, columns="variable", values="value").reset_index()
ds = df_wide.set_index(id_vars).to_xarray()
# Attach units from CSV. A blank cell reads back as NaN, which is not a
# units string — it must become "unknown" like an absent column, or
# xclim fails downstream on a float where it expects a unit.
for var in ds.data_vars:
units_series = (
df[df["variable"] == var]["units"].dropna()
if "units" in df.columns
else pd.Series(dtype=object)
)
ds[var].attrs["units"] = (
str(units_series.iloc[0]) if not units_series.empty else "unknown"
)
# Global source attribute
if "source" in df.columns:
source_series = df["source"].dropna().unique()
if len(source_series) > 0:
ds.attrs["source"] = source_series[0]
# Update cfg variables & varinfo
if not hasattr(self.cfg, "variables") or not self.cfg.variables:
self.cfg.variables = list(ds.data_vars)
if not hasattr(self.cfg, "varinfo") or not self.cfg.varinfo:
self.cfg.varinfo = {v: {"units": ds[v].attrs.get("units", "unknown")} for v in ds.data_vars}
self._gen_fn(ds=ds)
return ds
# ----------------------------
# Extract data from datasets like CMIP, DWD, etc.
# ----------------------------
def _ensure_dask_client(self):
"""Optionally start a shared Dask distributed cluster for parallel opens/compute.
Controlled by ``cfg.load.dask``. Opt-in (``enabled: false`` by default) so
other entry points are unaffected. Idempotent: reuses a client this
extractor already created, or one the user started elsewhere in the
process, instead of spinning up a second cluster.
Once a distributed client exists in the process, ``open_mfdataset(parallel=True)``
routes its per-file opens through it automatically — no other code changes.
Returns:
distributed.Client | None: The active client, or None when disabled/unavailable.
"""
from omegaconf import OmegaConf
if not OmegaConf.select(self.cfg, "load.dask.enabled", default=False):
return None
# Reuse a client we already created for this extractor.
if self._dask_client is not None:
return self._dask_client
try:
from dask.distributed import Client, LocalCluster, get_client
except ImportError:
self.logger.warning(
"cfg.load.dask.enabled=true but dask.distributed is not installed; "
"falling back to the default scheduler."
)
return None
# Reuse an externally-created client if the user already started one.
try:
self._dask_client = get_client()
self.logger.info("Reusing existing Dask client (dashboard: %s)", self._dask_client.dashboard_link)
return self._dask_client
except ValueError:
pass # no active client -> create our own below
cluster = LocalCluster(
n_workers=int(OmegaConf.select(self.cfg, "load.dask.n_workers", default=4)),
threads_per_worker=int(OmegaConf.select(self.cfg, "load.dask.threads_per_worker", default=2)),
memory_limit=OmegaConf.select(self.cfg, "load.dask.memory_limit", default="auto"),
)
self._dask_client = Client(cluster)
self.logger.info("Started Dask cluster (dashboard: %s)", self._dask_client.dashboard_link)
return self._dask_client
def close_dask(self):
"""Shut down the Dask cluster this extractor started, if any.
Safe to call unconditionally: it is a no-op when no cluster was started.
A cluster the extractor merely reused, rather than created, is left
running for whoever owns it.
Returns:
None
"""
client = self._dask_client
if client is None:
return
cluster = getattr(client, "cluster", None)
try:
client.close()
if cluster is not None:
cluster.close()
finally:
self._dask_client = None
@update_ds(attr_name='ds')
def extract(self) -> xr.Dataset:
"""Extract data from the configured provider using ``self.cfg``.
Uses provider-specific classes (e.g., ``CMIP``, ``DWD``, ``MSWX``, ``HYRAS``, ``POWER``)
to fetch, load and extract datasets. When extraction completes, units are converted to those declared in ``cfg.varinfo``, the dataset is computed, and filenames are generated from the configuration.
Returns:
xr.Dataset: The extracted and computed dataset (also sets ``self.current_ds``).
"""
cfg = self.cfg
extract_kwargs = {}
# Optionally start a shared Dask cluster so per-file opens (and later
# compute) run in parallel across worker processes.
self._ensure_dask_client()
if cfg.lat is not None and cfg.lon is not None:
extract_kwargs["point"] = (cfg.lon, cfg.lat)
if cfg.dataset.upper() == "DWD":
extract_kwargs["buffer_km"] = 30
elif cfg.box.lat_min is not None:
extract_kwargs["box"] = {
"lat_min": cfg.box.lat_min,
"lat_max": cfg.box.lat_max,
"lon_min": cfg.box.lon_min,
"lon_max": cfg.box.lon_max,
}
elif cfg.region is not None:
extract_kwargs["box"] = cfg.bounds[cfg.region]
elif cfg.shapefile is not None:
extract_kwargs["shapefile"] = cfg.shapefile
ds = None
dataset_upper = cfg.dataset.upper()
if dataset_upper == "MSWX":
ds_vars = []
for var in cfg.variables:
mswx = climdata.MSWX(cfg)
mswx.extract(**extract_kwargs)
mswx.load(var)
ds_vars.append(mswx.dataset)
ds = xr.merge(ds_vars)
self.dataset_class = mswx
elif dataset_upper == "CMIP":
cmip = climdata.CMIP(cfg)
cmip.fetch()
cmip.load()
cmip.extract(**extract_kwargs)
ds = cmip.ds
self.dataset_class = cmip
elif dataset_upper == "POWER":
power = climdata.POWER(cfg)
power.fetch()
power.load()
ds = power.ds
self.dataset_class = power
elif dataset_upper == "DWD":
ds_vars = []
for var in cfg.variables:
dwd = climdata.DWD(cfg)
ds_var = dwd.extract(variable=var, **extract_kwargs)
ds_vars.append(ds_var)
ds = xr.merge(ds_vars)
self.dataset_class = dwd
elif dataset_upper == "HYRAS":
hyras = climdata.HYRAS(cfg)
ds_vars = []
for var in cfg.variables:
hyras.extract(**extract_kwargs)
ds_vars.append(hyras.load(var, chunking={'time':"auto"})[[var]])
ds = xr.merge(ds_vars, compat="override")
self.dataset_class = hyras
elif dataset_upper == "HOSTRADA":
hostrada = climdata.HOSTRADA(cfg)
hostrada.extract(**extract_kwargs)
ds_vars = []
for var in cfg.variables:
ds_vars.append(hostrada.load(var, chunking={'time': 'auto'})[[var]])
ds = xr.merge(ds_vars, compat="override")
self.dataset_class = hostrada
elif dataset_upper == "W5E5":
w5e5 = climdata.W5E5(cfg)
w5e5.fetch() # Download from ISIMIP
w5e5.load() # Load into xarray
w5e5.extract(**extract_kwargs)
ds = w5e5.ds
self.dataset_class = w5e5
elif dataset_upper == "CMIP_W5E5":
cmip_w5e5 = climdata.CMIPW5E5(cfg)
cmip_w5e5.fetch() # Download CMIP6 data from ISIMIP
cmip_w5e5.load() # Load into xarray
cmip_w5e5.extract(**extract_kwargs)
ds = cmip_w5e5.ds
self.dataset_class = cmip_w5e5
elif dataset_upper == "NEXGDDP":
nexgddp = climdata.NEXGDDP(cfg)
nexgddp.fetch()
nexgddp.extract(**extract_kwargs) # set params BEFORE load so preprocess clips per-file
nexgddp.load() # spatial clip applied inside open_mfdataset preprocess
ds = nexgddp.ds
self.dataset_class = nexgddp
elif dataset_upper == "AGRI_ISIMIP":
agri = climdata.AGRI_ISIMIP(cfg)
agri.fetch() # Download agricultural data from ISIMIP
ds_vars = []
for var in cfg.variables:
ds_vars.append(agri.load(var, chunking={'time': 'auto'})[[var]])
ds = xr.merge(ds_vars, compat="override")
agri.extract(**extract_kwargs)
self.dataset_class = agri
else:
# Without this the unhandled provider leaves ds as None and fails
# below with a bare `TypeError: 'NoneType' is not subscriptable`,
# which says nothing about the actual mistake.
raise ValueError(
f"Dataset provider '{cfg.dataset}' has no extraction path in "
f"ClimateExtractor.extract(). Supported: {', '.join(_EXTRACTABLE)}.\n"
f" ERA5 is available as climdata.ERA5, but it mirrors whole "
f"months to Zarr rather than extracting, so it is driven directly "
f"rather than through this workflow."
)
for var in cfg.variables:
ds[var] = xclim.core.units.convert_units_to(ds[var], cfg.varinfo[var].units)
# ds = ds.compute()
return ds
# ----------------------------
# Compute extreme index
# ----------------------------
@update_ds(attr_name='index_ds')
def calc_index(self, ds: xr.Dataset = None) -> xr.Dataset:
"""Calculate the configured extreme index using xclim indices.
Args:
ds (xr.Dataset, optional): Dataset to operate on. If ``None``, ``self.current_ds`` is used.
Returns:
xr.Dataset: The computed index as an xarray Dataset (also sets ``self.index_ds``).
"""
cfg = self.cfg
# Use provided ds or fallback
ds = ds or self.current_ds
if ds is None:
raise ValueError("No dataset provided and no current_ds is available.")
if cfg.index is None:
self.logger.info("No index selected.")
return None
if "time" in ds.coords:
# Handle both numpy datetime64 and cftime datetime objects
time_values = ds.time.values
try:
# Try pandas conversion first (works for datetime64)
years = pd.to_datetime(time_values).year
except (TypeError, ValueError):
# Fall back to cftime handling
years = np.array([t.year for t in time_values])
n_years = len(pd.unique(years))
if n_years < 30:
warnings.warn(f"Index {cfg.index} usually requires ≥30 years, got {n_years}", UserWarning)
indices = extreme_index(cfg, ds)
index_ds = indices.calculate(cfg.index).compute()
index_ds = index_ds.to_dataset(name=cfg.index)
return index_ds
# ----------------------------
# Dataset → Long-form DataFrame
# ----------------------------
def plot(self, variable=None, ds: xr.Dataset = None, **kwargs):
"""Plot a variable on a Cartopy map with coastlines and country borders.
Thin wrapper around :func:`climdata.viz.plot_map` operating on
``self.current_ds`` by default.
Args:
variable (str, optional): Variable to plot. Defaults to the sole
data variable if the dataset has exactly one.
ds (xr.Dataset, optional): Dataset to plot. If ``None``, uses ``self.current_ds``.
**kwargs: Forwarded to :func:`climdata.viz.plot_map`
(e.g. ``time``, ``reduce``, ``cmap``, ``ax``).
Returns:
The Cartopy ``GeoAxes`` the field was drawn on.
"""
from climdata.viz import plot_map
ds = ds if ds is not None else self.current_ds
if ds is None:
raise ValueError("No dataset provided and no current_ds is available.")
return plot_map(ds, variable=variable, **kwargs)
@update_df()
def to_dataframe(self, ds: xr.Dataset = None) -> pd.DataFrame:
"""Convert a dataset to a long-form pandas DataFrame.
The output contains columns: time, lat, lon (or latitude/longitude), variable, value, units, source.
Args:
ds (xr.Dataset, optional): Dataset to convert. If ``None``, uses ``self.current_ds``.
Returns:
pd.DataFrame: Long-form DataFrame (also sets ``self.current_df``).
"""
ds = ds or self.current_ds
if ds is None:
raise ValueError("No dataset provided and no current_ds is available.")
df = ds.to_dataframe().reset_index()
id_vars = [c for c in ("time", "lat", "lon", "latitude", "longitude") if c in df]
value_vars = [v for v in ds.data_vars if v in df.columns]
if not value_vars:
raise ValueError("No variables in dataset available to melt into long format")
df_long = df.melt(
id_vars=id_vars,
value_vars=value_vars,
var_name="variable",
value_name="value"
)
df_long["units"] = df_long["variable"].apply(
lambda v: ds[v].attrs.get("units", "unknown")
)
if getattr(self.cfg, "dataset") == 'cmip':
df_long["source_id"] = getattr(self.cfg, "source_id")
df_long["source"] = getattr(self.cfg, "dataset", ds.attrs.get("source", "unknown"))
df_long = df_long.drop_duplicates()
self._gen_fn_cfg()
return df_long
# ----------------------------
# Save CSV
# ----------------------------
def to_csv(self, df: Optional[pd.DataFrame] = None, filename: Optional[str] = None, format: str = "default") -> str:
"""Save a DataFrame to CSV with optional format specification.
Args:
df (pd.DataFrame, optional): DataFrame to save. Defaults to ``self.current_df``.
filename (str, optional): Output filename/directory. Defaults to ``self.filename_csv``.
format (str): Output format. Options:
- 'default': Long-form (single file)
- 'simplace': SIMPLACE format (splits by location, tab-separated)
- 'monica': MONICA format (splits by location, tab-separated)
Defaults to 'default'.
Returns:
str: The path of the written CSV file(s). For SIMPLACE/MONICA, returns base directory.
Raises:
ValueError: If format is not supported or required data is missing.
"""
df = df if df is not None else self.current_df
format_lower = format.lower()
# Apply format conversion if requested
if format_lower in ("simplace", "monica"):
return self._convert_to_gridded_format(df, filename, format=format_lower)
elif format_lower == "default":
filename = filename or getattr(self, "filename_csv", None)
if filename is None:
raise ValueError("No filename provided and filename_csv is not set")
path = Path(filename)
path.parent.mkdir(parents=True, exist_ok=True)
df.to_csv(filename, index=False, sep='\t')
self.filename_csv = str(path)
self.current_filename = str(path)
self.logger.info(f"DataFrame saved to CSV file ({format} format): {self.current_filename}")
return filename
else:
raise ValueError(f"Unsupported format: {format}. Supported formats: 'default', 'simplace', 'monica'")
def _convert_to_gridded_format(self, df: pd.DataFrame, filename: Optional[str] = None, format: str = "simplace") -> str:
"""Convert long-form DataFrame to gridded SIMPLACE/MONICA format by row/column indices.
Creates a folder structure: <base_dir>/col_<col_number>/<format>_variables_row_<r>_col_<c>.csv
Args:
df (pd.DataFrame): Long-form DataFrame with columns: date/time, lat/x, lon/y, variable, value
filename (str, optional): Base output directory. If None, creates 'simplace_output' or 'monica_output'
format (str): Format type - 'simplace' or 'monica'
Returns:
str: Base directory path where all files were created
Raises:
ValueError: If DataFrame format is incompatible or required columns missing.
"""
# Validate input DataFrame structure
time_col = None
for col in df.columns:
if col.lower() in ('date', 'time'):
time_col = col
break
if time_col is None:
raise ValueError("DataFrame must contain 'date' or 'time' column")
var_col = None
for col in df.columns:
if col.lower() == 'variable':
var_col = col
break
if var_col is None:
raise ValueError("DataFrame must contain 'variable' column")
val_col = None
for col in df.columns:
if col.lower() in ('value', 'data'):
val_col = col
break
if val_col is None:
raise ValueError("DataFrame must contain 'value' or 'data' column")
# Handle both lat/lon and x/y dimensions
lat_col = next((c for c in df.columns if c.lower() in ('lat', 'latitude', 'y')), None)
lon_col = next((c for c in df.columns if c.lower() in ('lon', 'longitude', 'x')), None)
if lat_col is None or lon_col is None:
raise ValueError("DataFrame must contain 'lat'/'latitude'/'y' and 'lon'/'longitude'/'x' columns")
# Get dimensions from xarray Dataset if available
if hasattr(self, 'ds') and self.ds is not None:
ds = self.ds
# Determine dimension names (handle lat/lon or y/x)
lat_dim = next((d for d in ds.dims if d.lower() in ('lat', 'latitude', 'y')), None)
lon_dim = next((d for d in ds.dims if d.lower() in ('lon', 'longitude', 'x')), None)
if lat_dim and lon_dim:
n_rows = ds.sizes[lat_dim]
n_cols = ds.sizes[lon_dim]
self.logger.info(f"Dataset dimensions: {n_rows} rows (lat/y) × {n_cols} columns (lon/x)")
else:
n_rows = df[lat_col].nunique()
n_cols = df[lon_col].nunique()
self.logger.info(f"Inferred dimensions: {n_rows} rows × {n_cols} columns")
else:
n_rows = df[lat_col].nunique()
n_cols = df[lon_col].nunique()
self.logger.info(f"DataFrame dimensions: {n_rows} rows × {n_cols} columns")
# Create base output directory
if filename is None:
filename = f"{format}_{self.cfg.dataset}"
base_dir = Path(filename)
base_dir.mkdir(parents=True, exist_ok=True)
# Get unique lat/lon pairs and assign row/column indices
unique_locations = df.drop_duplicates(subset=[lat_col, lon_col]).sort_values([lat_col, lon_col]).reset_index(drop=True)
# Create mapping from lat/lon to row/col indices
lat_sorted = sorted(df[lat_col].unique(), reverse=True) # rows decrease from north to south
lon_sorted = sorted(df[lon_col].unique()) # columns increase from west to east
lat_to_row = {lat: i for i, lat in enumerate(lat_sorted)}
lon_to_col = {lon: j for j, lon in enumerate(lon_sorted)}
n_locations = len(unique_locations)
self.logger.info(f"Converting to {format.upper()} format with {n_locations} unique locations")
created_files = []
# Process each location
for idx, (_, loc_row) in enumerate(unique_locations.iterrows(), 1):
lat = loc_row[lat_col]
lon = loc_row[lon_col]
row_num = lat_to_row[lat]
col_num = lon_to_col[lon]
# Filter data for this location
df_loc = df[(df[lat_col] == lat) & (df[lon_col] == lon)].copy()
# Pivot: time x variables
loc_df = df_loc.pivot_table(
index=time_col,
columns=var_col,
values=val_col,
aggfunc='first'
).reset_index()
# Rename Date column
loc_df.rename(columns={time_col: 'Date'}, inplace=True)
# Rename variables from CF names to DWD/standard names
rename_mapping = {}
for col in loc_df.columns:
if col in CF_TO_DWD_NAMES:
rename_mapping[col] = CF_TO_DWD_NAMES[col]
loc_df.rename(columns=rename_mapping, inplace=True)
# Generate filename
# Get dataset name from config
dataset_name = getattr(self.cfg, 'dataset', 'unknown')
# Create column-based folder: col_<col_number> (no zero-padding)
col_folder = base_dir / f"{col_num}"
col_folder.mkdir(parents=True, exist_ok=True)
# Save file: <format>_<variables>_<dataset>.csv
output_filename = col_folder / f"{dataset_name.upper()}_Daily_C{col_num}R{row_num}.csv"
loc_df.to_csv(output_filename, index=False, sep='\t')
created_files.append(str(output_filename))
self.logger.debug(f"[{idx}/{n_locations}] Saved col_{col_num} (row {row_num}) -> {output_filename}")
self.logger.info(f"Successfully created {len(created_files)} {format.upper()} format files in {base_dir}")
self.logger.info(f"Grid dimensions: {n_rows} rows × {n_cols} columns")
self.filename_csv = str(base_dir)
self.current_filename = str(base_dir)
return str(base_dir)
def to_nc(self, ds: Optional[xr.Dataset] = None, filename: Optional[str] = None) -> str:
"""Save an xarray Dataset to NetCDF.
Notes:
- If ``ds`` is ``None``: save ``current_ds``.
- If ``filename`` is ``None``: use ``self.filename_nc``.
- Creates directories if needed and updates ``self.filename_nc`` and ``self.current_filename``.
Args:
ds (xr.Dataset, optional): Dataset to save. If ``None``, uses ``self.current_ds``.
filename (str, optional): Output filename. Defaults to ``self.filename_nc``.
Returns:
str: The path of the written NetCDF file.
"""
# -------------------------------
# 1. Determine dataset to save
# -------------------------------
ds = ds or getattr(self, "current_ds", None)
if ds is None:
raise ValueError("No dataset available to save")
# -------------------------------
# 2. Determine filename
# -------------------------------
filename = filename or getattr(self, "filename_nc", None)
if filename is None:
raise ValueError("No filename provided and filename_nc is not set")
path = Path(filename)
path.parent.mkdir(parents=True, exist_ok=True)
# -------------------------------
# 3. Save to NetCDF
# -------------------------------
ds.to_netcdf(path)
# -------------------------------
# 4. Track filenames
# -------------------------------
self.filename_nc = str(path)
self.current_filename = str(path)
# print(f"Dataset saved to NetCDF file: {self.current_filename}")
self.logger.info(f"Dataset saved to NetCDF file: {self.current_filename}")
return str(path)
# ----------------------------
# FAIR data object (RO-Crate)
# ----------------------------
def to_fair(
self,
ds: Optional[xr.Dataset] = None,
output_dir: Optional[str] = None,
title: Optional[str] = None,
description: Optional[str] = None,
license_url: str = "https://creativecommons.org/licenses/by/4.0/",
creator: Optional[str] = None,
zip_crate: bool = False,
) -> str:
"""Export the dataset as a FAIR Research Object Crate (RO-Crate).
Packages the dataset into a self-describing directory that satisfies
the FAIR principles (Findable, Accessible, Interoperable, Reusable):
* **Findable** – unique dataset identifier & rich JSON-LD metadata
* **Accessible** – standard CF-compliant NetCDF + open JSON-LD manifest
* **Interoperable** – CF conventions in NetCDF; Schema.org / RO-Crate vocab
* **Reusable** – license, provenance, variable descriptions embedded
The output folder contains:
.. code-block:: text
<output_dir>/
data.nc # CF-compliant NetCDF dataset
ro-crate-metadata.json # JSON-LD manifest (RO-Crate 1.1)
ro-crate-preview.html # human-readable summary (optional)
Uses the ``rocrate`` Python package when available; falls back to a
hand-written JSON-LD manifest so the method always works.
Args:
ds (xr.Dataset, optional): Dataset to package. Defaults to
``self.current_ds``.
output_dir (str, optional): Destination folder. Defaults to
``<filename_nc_stem>_fair_crate``.
title (str, optional): Human-readable dataset title.
description (str, optional): Free-text description of the dataset.
license_url (str): SPDX or Creative-Commons URL for the data
licence. Defaults to CC-BY-4.0.
creator (str, optional): Name of the person / organisation that
produced the data.
zip_crate (bool): If *True*, zip the crate directory and return
the ``.zip`` path. Defaults to *False*.
Returns:
str: Absolute path to the RO-Crate directory (or ``.zip`` file).
Raises:
ValueError: If no dataset is available.
Example:
>>> extractor = ClimData(overrides=["dataset=mswx", ...])
>>> ds = extractor.extract()
>>> crate_path = extractor.to_fair(title="MSWX Germany 2014")
>>> print(crate_path) # …/mswx_fair_crate/
"""
import datetime
import zipfile
import uuid
ds = ds if ds is not None else self.current_ds
if ds is None:
raise ValueError(
"No dataset available. Run extract() or upload_netcdf() first."
)
cfg = self.cfg
# ── 1. Resolve output directory ─────────────────────────────────────
if output_dir is None:
base = getattr(self, "filename_nc", None)
stem = Path(base).stem if base else getattr(cfg, "dataset", "climdata")
output_dir = str(Path(base).parent / f"{stem}_fair_crate") if base else f"{stem}_fair_crate"
crate_dir = Path(output_dir)
crate_dir.mkdir(parents=True, exist_ok=True)
# ── 2. Write the NetCDF payload ──────────────────────────────────────
nc_path = crate_dir / "data.nc"
# Embed CF global attributes before saving
ds_out = ds.copy()
ds_out.attrs.setdefault("Conventions", "CF-1.8")
ds_out.attrs.setdefault("institution", creator or "climdata")
ds_out.attrs.setdefault(
"source",
getattr(cfg, "dataset", "unknown"),
)
ds_out.attrs.setdefault(
"history",
f"{datetime.datetime.utcnow().isoformat()}Z climdata to_fair()",
)
ds_out.to_netcdf(nc_path)
self.logger.info("FAIR payload written to %s", nc_path)
# ── 3. Collect metadata from cfg ────────────────────────────────────
dataset_name = getattr(cfg, "dataset", "unknown")
variables = list(getattr(cfg, "variables", list(ds.data_vars)))
t_start = getattr(getattr(cfg, "time_range", None), "start_date", None)
t_end = getattr(getattr(cfg, "time_range", None), "end_date", None)
if t_start is None and "time" in ds.coords:
t_start = str(ds.time.values[0])[:10]
if t_end is None and "time" in ds.coords:
t_end = str(ds.time.values[-1])[:10]
# spatial extent
lat_min = lat_max = lon_min = lon_max = None
try:
box = getattr(cfg, "box", None)
if box and getattr(box, "lat_min", None) is not None:
lat_min = float(box.lat_min)
lat_max = float(box.lat_max)
lon_min = float(box.lon_min)
lon_max = float(box.lon_max)
elif getattr(cfg, "lat", None) is not None:
lat_min = lat_max = float(cfg.lat)
lon_min = lon_max = float(cfg.lon)
elif "lat" in ds.coords:
lat_min = float(ds.lat.values.min())
lat_max = float(ds.lat.values.max())
lon_min = float(ds.lon.values.min())
lon_max = float(ds.lon.values.max())
except Exception:
pass
# variable-level metadata
var_descriptions = []
varinfo = getattr(cfg, "varinfo", {})
for v in variables:
info = varinfo.get(v, {}) if varinfo else {}
units = getattr(info, "units", None) or ds[v].attrs.get("units", "1") if v in ds else "1"
long_name = ds[v].attrs.get("long_name", v) if v in ds else v
var_descriptions.append({"@type": "PropertyValue", "name": v,
"description": long_name, "unitText": units})
title = title or f"{dataset_name.upper()} climate data extract"
description = description or (
f"Climate dataset extracted by climdata v{climdata.__version__} "
f"from {dataset_name.upper()}. "
f"Variables: {', '.join(variables)}. "
f"Period: {t_start} to {t_end}."
)
crate_id = str(uuid.uuid4())
now_iso = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")
# ── 4. Try rocrate library; fall back to hand-crafted JSON-LD ────────
metadata_path = crate_dir / "ro-crate-metadata.json"
try:
from rocrate.rocrate import ROCrate # type: ignore
from rocrate.model import ContextEntity # type: ignore
crate = ROCrate()
crate.name = title
crate.description = description
crate.add_file(
str(nc_path),
dest_path="data.nc",
properties={
"name": "data.nc",
"description": "CF-compliant NetCDF dataset",
"encodingFormat": "application/x-netcdf",
"variableMeasured": var_descriptions,
},
)
if t_start:
crate.root_dataset["temporalCoverage"] = f"{t_start}/{t_end}"
if lat_min is not None:
crate.root_dataset["spatialCoverage"] = {
"@type": "Place",
"geo": {
"@type": "GeoShape",
"box": f"{lat_min} {lon_min} {lat_max} {lon_max}",
},
}
if creator:
person = crate.add(
ContextEntity(crate, f"#{creator.replace(' ', '_')}",
properties={"@type": "Person", "name": creator})
)
crate.root_dataset["creator"] = person
crate.root_dataset["license"] = license_url
crate.root_dataset["datePublished"] = now_iso
crate.root_dataset["identifier"] = crate_id
crate.root_dataset["keywords"] = variables
crate.root_dataset["producer"] = {
"@type": "SoftwareApplication",
"name": "climdata",
"version": climdata.__version__,
"url": "https://github.com/climdata/climdata",
}
crate.write(str(crate_dir))
self.logger.info("RO-Crate written via rocrate library.")
except ImportError:
# ── fallback: hand-craft the JSON-LD ────────────────────────────
self.logger.warning(
"'rocrate' package not found. Writing plain JSON-LD metadata. "
"Install with: pip install rocrate"
)
geo_shape: Optional[Dict] = None
if lat_min is not None:
geo_shape = {
"@type": "GeoShape",
"box": f"{lat_min} {lon_min} {lat_max} {lon_max}",
}
metadata = {
"@context": [
"https://w3id.org/ro/crate/1.1/context",
{"climdata": "https://climdata.readthedocs.io/"},
],
"@graph": [
# ── RO-Crate metadata descriptor ────────────────────
{
"@id": "ro-crate-metadata.json",
"@type": "CreativeWork",
"conformsTo": {"@id": "https://w3id.org/ro/crate/1.1"},
"about": {"@id": "./"},
},
# ── Root data entity (the crate itself) ─────────────
{
"@id": "./",
"@type": "Dataset",
"identifier": crate_id,
"name": title,
"description": description,
"datePublished": now_iso,
"license": {"@id": license_url},
"keywords": variables,
**({
"creator": {
"@type": "Person",
"name": creator,
}
} if creator else {}),
**({
"temporalCoverage": f"{t_start}/{t_end}"
} if t_start else {}),
**({
"spatialCoverage": {
"@type": "Place",
"geo": geo_shape,
}
} if geo_shape else {}),
"producer": {
"@type": "SoftwareApplication",
"name": "climdata",
"version": climdata.__version__,
"url": "https://github.com/Kaushikreddym/climdata",
},
"hasPart": [{"@id": "data.nc"}],
},
# ── File entity (the NetCDF) ─────────────────────────
{
"@id": "data.nc",
"@type": "File",
"name": "data.nc",
"description": "CF-compliant NetCDF dataset",
"encodingFormat": "application/x-netcdf",
"variableMeasured": var_descriptions,
**({
"temporalCoverage": f"{t_start}/{t_end}"
} if t_start else {}),
},
# ── License entity ───────────────────────────────────
{
"@id": license_url,
"@type": "CreativeWork",
"name": "CC-BY 4.0" if "creativecommons" in license_url else license_url,
"url": license_url,
},
],
}
with open(metadata_path, "w", encoding="utf-8") as f:
json.dump(metadata, f, indent=2, ensure_ascii=False)
# ── 5. Write a simple HTML preview ──────────────────────────────────
html_path = crate_dir / "ro-crate-preview.html"
var_rows = "".join(
f"<tr><td><code>{v['name']}</code></td>"
f"<td>{v['description']}</td>"
f"<td>{v['unitText']}</td></tr>"
for v in var_descriptions
)
spatial_str = (
f"{lat_min:.3f}°N – {lat_max:.3f}°N, {lon_min:.3f}°E – {lon_max:.3f}°E"
if lat_min is not None
else "(not specified)"
)
html_content = f"""<!DOCTYPE html>
<html lang="en">
<head><meta charset="UTF-8">
<title>{title}</title>
<style>
body{{font-family:sans-serif;max-width:900px;margin:2em auto;padding:0 1em}}
table{{border-collapse:collapse;width:100%}}
th,td{{border:1px solid #ccc;padding:.4em .8em;text-align:left}}
th{{background:#f0f0f0}}
.badge{{background:#0078d4;color:#fff;padding:.2em .6em;border-radius:4px;font-size:.85em}}
</style>
</head>
<body>
<h1>{title} <span class="badge">RO-Crate</span></h1>
<p>{description}</p>
<table>
<tr><th>Field</th><th>Value</th></tr>
<tr><td>Dataset</td><td>{dataset_name.upper()}</td></tr>
<tr><td>Period</td><td>{t_start} → {t_end}</td></tr>
<tr><td>Spatial coverage</td><td>{spatial_str}</td></tr>
<tr><td>License</td><td><a href="{license_url}">{license_url}</a></td></tr>
<tr><td>Produced by</td><td>climdata v{climdata.__version__}</td></tr>
<tr><td>Crate ID</td><td><code>{crate_id}</code></td></tr>
</table>
<h2>Variables</h2>
<table>
<tr><th>CF name</th><th>Long name</th><th>Units</th></tr>
{var_rows}
</table>
<h2>Files</h2>
<ul>
<li><a href="data.nc">data.nc</a> – CF-compliant NetCDF</li>
<li><a href="ro-crate-metadata.json">ro-crate-metadata.json</a> – JSON-LD metadata</li>
</ul>
</body>
</html>
"""
html_path.write_text(html_content, encoding="utf-8")
# ── 6. Optionally zip ────────────────────────────────────────────────
if zip_crate:
zip_path = crate_dir.with_suffix(".zip")
with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf:
for file in crate_dir.rglob("*"):
zf.write(file, file.relative_to(crate_dir.parent))
self.logger.info("RO-Crate zipped to %s", zip_path)
return str(zip_path)
self.logger.info("FAIR RO-Crate ready at %s", crate_dir)
return str(crate_dir)
# ----------------------------
# Unified workflow
# ----------------------------
def run_workflow(self, overrides: Optional[List[str]] = None,
actions: Optional[List[str]] = None,
file: Optional[str] = None) -> WorkflowResult:
"""Execute a sequence of workflow actions.
Args:
overrides (list[str], optional): Hydra overrides to apply (not all actions will use these).
actions (list[str], optional): Ordered list of actions to perform. Supported actions include: 'upload_netcdf', 'upload_csv', 'extract', 'calc_index', 'to_dataframe', 'to_csv', 'to_nc', 'to_fair'.
file (str, optional): File path used for upload actions when required.
Returns:
WorkflowResult: Named result container with populated fields for dataframe/dataset/filenames.
"""
actions = actions or ["extract", "calc_index", "to_csv", "to_nc"]
result = WorkflowResult(cfg=self.cfg)
for action in actions:
self.logger.info("Starting action: %s", action)
try:
if action == "upload_netcdf":
if file is None:
raise ValueError(
"Action 'upload_netcdf' requires argument 'netcdf_file', "
"but none was provided."
)
# Validate extension
valid_nc_ext = (".nc", ".nc4", ".nc.gz")
if not any(str(file).lower().endswith(ext) for ext in valid_nc_ext):
raise ValueError(
f"Invalid file format for upload_netcdf: '{file}'. "
f"Expected one of: {valid_nc_ext}"
)
self.upload_netcdf(file)
result.dataset = self.current_ds
elif action == "upload_csv":
if file is None:
raise ValueError(
"Action 'upload_csv' requires argument 'csv_file', "
"but none was provided."
)
# Validate CSV extension
valid_csv_ext = (".csv", ".csv.gz")
if not any(str(file).lower().endswith(ext) for ext in valid_csv_ext):
raise ValueError(
f"Invalid file format for upload_csv: '{file}'. "
f"Expected one of: {valid_csv_ext}"
)
self.upload_csv(file)
result.dataset = self.current_ds
elif action == "extract":
if self.cfg.dataset is None:
raise ValueError(
"Action 'extract' cannot run because no dataset provider is set "
"(cfg.dataset is None)."
)
self.extract()
result.dataset = self.current_ds
elif action == "calc_index":
if self.current_ds is None:
raise ValueError(
"Action 'calc_index' requires a dataset, but no dataset is available. "
"Upload or extract a dataset before computing an index."
)
self.calc_index()
result.index_ds = self.current_ds
elif action == "to_csv":
if self.current_ds is None:
raise ValueError(
"Action 'to_dataframe' requires a dataset, but no dataset is available. "
"Upload or extract a dataset before converting to a DataFrame."
)
self.to_dataframe()
result.dataframe = self.current_df
result.filename = self.to_csv()
elif action == "to_nc":
if self.current_ds is None:
raise ValueError(
"Action 'to_nc' requires a dataset, but no dataset is available. "
"Upload or extract a dataset before saving to NetCDF."
)
result.filename = self.to_nc()
elif action == "impute":
if self.current_ds is None:
raise ValueError("Action 'impute' requires a dataset, but no dataset is available.")
self.impute()
result.dataset = self.current_ds
result.impute_ds = getattr(self, "impute_ds", None)
elif action == "reproject":
if self.current_ds is None:
raise ValueError(
"Action 'reproject' requires a dataset, but no dataset is available. "
"Upload or extract a dataset before regridding."
)
self.reproject()
result.dataset = self.current_ds
result.reprojected_ds = getattr(self, "reprojected_ds", None)
elif action == "to_fair":
if self.current_ds is None:
raise ValueError(
"Action 'to_fair' requires a dataset, but no dataset is available. "
"Upload or extract a dataset before exporting a FAIR crate."
)
result.filename = self.to_fair()
else:
raise ValueError(
f"Unknown action '{action}'. Supported: "
+ ", ".join(sorted(_WORKFLOW_ACTIONS))
)
self.logger.info("Completed action: %s", action)
except Exception:
self.logger.exception("Action '%s' failed", action)
raise
return result
# ----------------------------
# Exploration helpers using cfg.dsinfo
# ----------------------------
def get_datasets(self) -> List[str]:
"""Return the list of dataset provider names available in configuration.
Returns:
List[str]: Names of available dataset providers from ``cfg.dsinfo``.
"""
if not self.cfg or not hasattr(self.cfg, "dsinfo"):
raise ValueError("Configuration or dsinfo not loaded")
return list(self.cfg.dsinfo.keys())
def get_variables(self, dataset: Optional[str] = None) -> List[str]:
"""Return the list of variables available for a dataset.
Args:
dataset (str, optional): Dataset name to query. Defaults to ``cfg.dataset``.
Returns:
List[str]: List of variable names.
"""
if not self.cfg or not hasattr(self.cfg, "dsinfo"):
raise ValueError("Configuration or dsinfo not loaded")
dataset_name = dataset or getattr(self.cfg, "dataset", None)
if dataset_name is None:
raise ValueError("Dataset not specified and cfg.dataset is None")
dsinfo = self.cfg.dsinfo.get(dataset_name)
if not dsinfo or "variables" not in dsinfo:
raise ValueError(f"No variable info available for dataset '{dataset_name}'")
return list(dsinfo["variables"].keys())
def get_varinfo(self, var: str) -> dict:
"""Get metadata for a variable from varinfo.
Args:
var (str): Name of the variable, e.g., 'tas', 'tasmax', 'pr'.
Returns:
dict: Metadata dictionary containing cf_name, long_name, units, etc.
Raises:
ValueError: If varinfo is not loaded or variable not found.
"""
if not self.cfg or not hasattr(self.cfg, "varinfo") or not self.cfg.varinfo:
raise ValueError("Configuration or varinfo not loaded")
if var not in self.cfg.varinfo:
raise ValueError(f"Variable '{var}' not found in varinfo")
return self.cfg.varinfo[var]
def get_actions(self) -> dict:
"""Return a dictionary of workflow actions with their outputs and descriptions.
Supports ``actionsinfo`` in mapping style or list style and returns a consistent mapping of action name to description/output.
Returns:
dict: Mapping action name -> {'output': ..., 'description': ...}
"""
if not self.cfg or not hasattr(self.cfg, "actionsinfo"):
raise ValueError("Configuration or actionsinfo not loaded")
actions_map = getattr(self.cfg, "actionsinfo")
# If 'actions' key exists, fallback to list style
if "actions" in actions_map:
actions_map = {a["name"]: {"output": a["output"], "description": a["description"]}
for a in actions_map["actions"]}
return actions_map
def get_indices(self, variables: List[str], require_all: bool = True) -> Dict[str, dict]:
"""Fetch climate extreme indices from ``cfg.extinfo`` that involve the given variables.
Args:
variables (list[str]): Variables to filter indices by (if ``None``, uses ``cfg.variables``).
require_all (bool): If True, return indices that require all provided variables; otherwise return indices if any variable matches.
Returns:
dict: Mapping index_name -> index_definition.
"""
cfg = self.cfg
variables = variables or cfg.variables
if not hasattr(cfg, "extinfo") or not cfg.extinfo:
raise ValueError("cfg.extinfo is not defined or empty")
indices_def = cfg.extinfo.get("indices", {})
if not indices_def:
return {}
matched_indices = {}
for idx_name, idx_info in indices_def.items():
idx_vars = idx_info.get("variables", [])
if require_all:
if all(var in variables for var in idx_vars):
matched_indices[idx_name] = idx_info
else:
if any(var in variables for var in idx_vars):
matched_indices[idx_name] = idx_info
return matched_indices
# ----------------------------
# Imputation
# ----------------------------
@update_ds(attr_name='impute_ds')
def impute(self, ds: xr.Dataset = None) -> xr.Dataset:
"""Impute missing values using the configured imputation method.
Args:
ds (xr.Dataset, optional): Dataset to impute. If None, uses
``self.current_ds``.
Returns:
xr.Dataset | None: The imputed dataset (also sets
``self.current_ds`` and ``self.impute_ds``). Returns ``None``
if no imputation method is configured.
Raises:
ValueError: If ``ds`` is ``None`` and ``self.current_ds`` is not set.
"""
cfg = self.cfg
impute_cfg = cfg.imputeinfo
ds = ds or self.current_ds
if ds is None:
raise ValueError("No dataset provided and no current_ds is available.")
if cfg.impute is None:
self.logger.warning("No imputation method selected.")
return None
# select variables (optional)
# variables = cfg.get("variables", None)
# if variables:
# missing = [v for v in variables if v not in self.current_ds.data_vars]
# if missing:
# raise ValueError(f"Variables not present in dataset: {missing}")
# ds_in = self.current_ds[variables]
# else:
# ds_in = self.current_ds
method = cfg.impute
normalize = impute_cfg[method].get("normalize", True)
time_dim = cfg.dsinfo[cfg.dataset.upper()].get("time_dim", "time")
lat_dim = cfg.dsinfo[cfg.dataset.upper()].get("lat_dim", "lat")
lon_dim = cfg.dsinfo[cfg.dataset.upper()].get("lon_dim", "lon")
# epochs = impute_cfg[method].get("epochs", 300)
# run imputer (Imputer expects dims (time, lat, lon))
imputer = Imputer(
ds,
time_dim=time_dim,
lat_dim=lat_dim,
lon_dim=lon_dim,
method=method,
normalize=normalize,
)
recovered = imputer.impute()
# merge imputed variables back into original dataset if we operated on a subset
ds_out = recovered
# Return dataset (decorator will set current_ds and impute_ds and generate filenames)
return ds_out
@update_ds("reprojected_ds")
def reproject(self, ds: xr.Dataset = None, **overrides) -> xr.Dataset:
"""Reproject / resample the dataset onto the configured target grid.
Reads ``target_projection`` and ``target_resolution`` from the config, plus
the secondary knobs under ``regrid``. Returns ``None`` (leaving the current
dataset untouched) when neither target is configured.
The units of ``target_resolution`` must match the axis units of
``target_projection``: a metric resolution such as ``"10 km"`` has no fixed
angular size, so combining it with a geographic CRS raises
:class:`~climdata.grid.units.ResolutionCRSMismatch` rather than silently
approximating. The check runs before any data is touched.
Args:
ds (xr.Dataset, optional): Dataset to transform. If ``None``, uses
``self.current_ds``.
**overrides: Any argument of :func:`climdata.grid.reproject`, overriding
the configured value for this call (e.g. ``method="average"``).
Returns:
xr.Dataset | None: The reprojected dataset (also sets ``self.current_ds``
and ``self.reprojected_ds``), or ``None`` if no target grid is configured.
Raises:
ValueError: If ``ds`` is ``None`` and ``self.current_ds`` is not set.
ResolutionCRSMismatch: If the resolution units contradict the target CRS.
Example:
extractor = ClimData(overrides=[
"target_projection=EPSG:3035", "target_resolution=10 km",
])
extractor.extract()
grid_ds = extractor.reproject()
"""
from climdata.grid import reproject as _reproject
cfg = self.cfg
ds = ds if ds is not None else self.current_ds
if ds is None:
raise ValueError("No dataset provided and no current_ds is available.")
target_projection = overrides.pop("target_projection", cfg.get("target_projection"))
target_resolution = overrides.pop("target_resolution", cfg.get("target_resolution"))
if target_projection is None and target_resolution is None and "like" not in overrides:
self.logger.info(
"No target_projection or target_resolution configured; skipping reprojection."
)
return None
regrid_cfg = cfg.get("regrid") or {}
kwargs = {
"method": regrid_cfg.get("method", "bilinear"),
"align": regrid_cfg.get("align", True),
"bounds": regrid_cfg.get("bounds"),
"engine": regrid_cfg.get("engine", "rasterio"),
}
kwargs.update(overrides)
# Dimension names are already declared per dataset in parameters.yaml.
try:
dsinfo = cfg.dsinfo[cfg.dataset.upper()]
kwargs.setdefault(
"dsinfo_dims", (dsinfo.get("lon_dim", "lon"), dsinfo.get("lat_dim", "lat"))
)
except Exception:
pass
if kwargs.get("bounds") is not None:
kwargs["bounds"] = [float(v) for v in kwargs["bounds"]]
if isinstance(kwargs.get("method"), DictConfig):
kwargs["method"] = dict(kwargs["method"])
self.logger.info(
"Reprojecting to %s at %s", target_projection or "source CRS", target_resolution
)
return _reproject(ds, target_projection, target_resolution, **kwargs)
def get_impute_methods(self) -> Dict[str, dict]:
"""Return mapping of available imputation methods from config.
Returns:
Dict[str, dict]: Mapping of method name -> config (empty dict if none configured).
"""
if not hasattr(self.cfg, "imputeinfo") or not self.cfg.imputeinfo:
return {}
return dict(self.cfg.imputeinfo)
def configure_logging(self, level=logging.INFO, handler: logging.Handler = None):
"""Configure logging for this extractor instance.
Args:
level (int, optional): Logging level (default: ``logging.INFO``).
handler (logging.Handler, optional): Handler to add; if ``None``, a default StreamHandler is created.
"""
if handler is None:
handler = logging.StreamHandler()
handler.setFormatter(logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s"))
# Avoid adding duplicate handlers
if not any(isinstance(h, handler.__class__) for h in self.logger.handlers):
self.logger.addHandler(handler)
self.logger.setLevel(level)
# also set module logger level
logger.setLevel(level)
__init__(self, cfg_name='config', conf_path=None, overrides=None)
special
¶
Initialize the workflow manager and load configuration.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
cfg_name |
str |
Name of the Hydra configuration (default: "config"). |
'config' |
conf_path |
str |
Optional config path override. |
None |
overrides |
list[str] |
Hydra overrides to apply to the configuration. |
None |
Source code in climdata/utils/wrapper_workflow.py
def __init__(self, cfg_name="config", conf_path=None, overrides: Optional[List[str]] = None):
"""Initialize the workflow manager and load configuration.
Args:
cfg_name (str): Name of the Hydra configuration (default: "config").
conf_path (str, optional): Optional config path override.
overrides (list[str], optional): Hydra overrides to apply to the configuration.
"""
self.cfg_name = cfg_name
self.conf_path = conf_path
self.cfg: Optional[DictConfig] = None
# Stage datasets
self.ds = None
self.current_ds = None
self.index_ds = None
self.impute_ds = None
self.reprojected_ds = None
self.bias_corrected_ds = None
# Stage DataFrames
self.raw_df = None
self.current_df = None
self.index_df = None
self.impute_df = None
self.bias_corrected_df = None
self.df = None # alias for current_df
# filenames
self.filename = None
self.filetype = None
# Optional shared Dask distributed client (created lazily by extract())
self._dask_client = None
# Automatically load config on init
self.load_config(overrides)
self.cfg = self.preprocess_aoi(self.cfg)
# instance logger for this extractor
self.logger = logging.getLogger(f"{__name__}.{self.__class__.__name__}")
calc_index(self, ds=None)
¶
Calculate the configured extreme index using xclim indices.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
ds |
xr.Dataset |
Dataset to operate on. If |
None |
Returns:
| Type | Description |
|---|---|
xr.Dataset |
The computed index as an xarray Dataset (also sets |
Source code in climdata/utils/wrapper_workflow.py
@update_ds(attr_name='index_ds')
def calc_index(self, ds: xr.Dataset = None) -> xr.Dataset:
"""Calculate the configured extreme index using xclim indices.
Args:
ds (xr.Dataset, optional): Dataset to operate on. If ``None``, ``self.current_ds`` is used.
Returns:
xr.Dataset: The computed index as an xarray Dataset (also sets ``self.index_ds``).
"""
cfg = self.cfg
# Use provided ds or fallback
ds = ds or self.current_ds
if ds is None:
raise ValueError("No dataset provided and no current_ds is available.")
if cfg.index is None:
self.logger.info("No index selected.")
return None
if "time" in ds.coords:
# Handle both numpy datetime64 and cftime datetime objects
time_values = ds.time.values
try:
# Try pandas conversion first (works for datetime64)
years = pd.to_datetime(time_values).year
except (TypeError, ValueError):
# Fall back to cftime handling
years = np.array([t.year for t in time_values])
n_years = len(pd.unique(years))
if n_years < 30:
warnings.warn(f"Index {cfg.index} usually requires ≥30 years, got {n_years}", UserWarning)
indices = extreme_index(cfg, ds)
index_ds = indices.calculate(cfg.index).compute()
index_ds = index_ds.to_dataset(name=cfg.index)
return index_ds
close_dask(self)
¶
Shut down the Dask cluster this extractor started, if any.
Safe to call unconditionally: it is a no-op when no cluster was started. A cluster the extractor merely reused, rather than created, is left running for whoever owns it.
Returns:
| Type | Description |
|---|---|
None |
Source code in climdata/utils/wrapper_workflow.py
def close_dask(self):
"""Shut down the Dask cluster this extractor started, if any.
Safe to call unconditionally: it is a no-op when no cluster was started.
A cluster the extractor merely reused, rather than created, is left
running for whoever owns it.
Returns:
None
"""
client = self._dask_client
if client is None:
return
cluster = getattr(client, "cluster", None)
try:
client.close()
if cluster is not None:
cluster.close()
finally:
self._dask_client = None
configure_logging(self, level=20, handler=None)
¶
Configure logging for this extractor instance.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
level |
int |
Logging level (default: |
20 |
handler |
logging.Handler |
Handler to add; if |
None |
Source code in climdata/utils/wrapper_workflow.py
def configure_logging(self, level=logging.INFO, handler: logging.Handler = None):
"""Configure logging for this extractor instance.
Args:
level (int, optional): Logging level (default: ``logging.INFO``).
handler (logging.Handler, optional): Handler to add; if ``None``, a default StreamHandler is created.
"""
if handler is None:
handler = logging.StreamHandler()
handler.setFormatter(logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s"))
# Avoid adding duplicate handlers
if not any(isinstance(h, handler.__class__) for h in self.logger.handlers):
self.logger.addHandler(handler)
self.logger.setLevel(level)
# also set module logger level
logger.setLevel(level)
extract(self)
¶
Extract data from the configured provider using self.cfg.
Uses provider-specific classes (e.g., CMIP, DWD, MSWX, HYRAS, POWER)
to fetch, load and extract datasets. When extraction completes, units are converted to those declared in cfg.varinfo, the dataset is computed, and filenames are generated from the configuration.
Returns:
| Type | Description |
|---|---|
xr.Dataset |
The extracted and computed dataset (also sets |
Source code in climdata/utils/wrapper_workflow.py
@update_ds(attr_name='ds')
def extract(self) -> xr.Dataset:
"""Extract data from the configured provider using ``self.cfg``.
Uses provider-specific classes (e.g., ``CMIP``, ``DWD``, ``MSWX``, ``HYRAS``, ``POWER``)
to fetch, load and extract datasets. When extraction completes, units are converted to those declared in ``cfg.varinfo``, the dataset is computed, and filenames are generated from the configuration.
Returns:
xr.Dataset: The extracted and computed dataset (also sets ``self.current_ds``).
"""
cfg = self.cfg
extract_kwargs = {}
# Optionally start a shared Dask cluster so per-file opens (and later
# compute) run in parallel across worker processes.
self._ensure_dask_client()
if cfg.lat is not None and cfg.lon is not None:
extract_kwargs["point"] = (cfg.lon, cfg.lat)
if cfg.dataset.upper() == "DWD":
extract_kwargs["buffer_km"] = 30
elif cfg.box.lat_min is not None:
extract_kwargs["box"] = {
"lat_min": cfg.box.lat_min,
"lat_max": cfg.box.lat_max,
"lon_min": cfg.box.lon_min,
"lon_max": cfg.box.lon_max,
}
elif cfg.region is not None:
extract_kwargs["box"] = cfg.bounds[cfg.region]
elif cfg.shapefile is not None:
extract_kwargs["shapefile"] = cfg.shapefile
ds = None
dataset_upper = cfg.dataset.upper()
if dataset_upper == "MSWX":
ds_vars = []
for var in cfg.variables:
mswx = climdata.MSWX(cfg)
mswx.extract(**extract_kwargs)
mswx.load(var)
ds_vars.append(mswx.dataset)
ds = xr.merge(ds_vars)
self.dataset_class = mswx
elif dataset_upper == "CMIP":
cmip = climdata.CMIP(cfg)
cmip.fetch()
cmip.load()
cmip.extract(**extract_kwargs)
ds = cmip.ds
self.dataset_class = cmip
elif dataset_upper == "POWER":
power = climdata.POWER(cfg)
power.fetch()
power.load()
ds = power.ds
self.dataset_class = power
elif dataset_upper == "DWD":
ds_vars = []
for var in cfg.variables:
dwd = climdata.DWD(cfg)
ds_var = dwd.extract(variable=var, **extract_kwargs)
ds_vars.append(ds_var)
ds = xr.merge(ds_vars)
self.dataset_class = dwd
elif dataset_upper == "HYRAS":
hyras = climdata.HYRAS(cfg)
ds_vars = []
for var in cfg.variables:
hyras.extract(**extract_kwargs)
ds_vars.append(hyras.load(var, chunking={'time':"auto"})[[var]])
ds = xr.merge(ds_vars, compat="override")
self.dataset_class = hyras
elif dataset_upper == "HOSTRADA":
hostrada = climdata.HOSTRADA(cfg)
hostrada.extract(**extract_kwargs)
ds_vars = []
for var in cfg.variables:
ds_vars.append(hostrada.load(var, chunking={'time': 'auto'})[[var]])
ds = xr.merge(ds_vars, compat="override")
self.dataset_class = hostrada
elif dataset_upper == "W5E5":
w5e5 = climdata.W5E5(cfg)
w5e5.fetch() # Download from ISIMIP
w5e5.load() # Load into xarray
w5e5.extract(**extract_kwargs)
ds = w5e5.ds
self.dataset_class = w5e5
elif dataset_upper == "CMIP_W5E5":
cmip_w5e5 = climdata.CMIPW5E5(cfg)
cmip_w5e5.fetch() # Download CMIP6 data from ISIMIP
cmip_w5e5.load() # Load into xarray
cmip_w5e5.extract(**extract_kwargs)
ds = cmip_w5e5.ds
self.dataset_class = cmip_w5e5
elif dataset_upper == "NEXGDDP":
nexgddp = climdata.NEXGDDP(cfg)
nexgddp.fetch()
nexgddp.extract(**extract_kwargs) # set params BEFORE load so preprocess clips per-file
nexgddp.load() # spatial clip applied inside open_mfdataset preprocess
ds = nexgddp.ds
self.dataset_class = nexgddp
elif dataset_upper == "AGRI_ISIMIP":
agri = climdata.AGRI_ISIMIP(cfg)
agri.fetch() # Download agricultural data from ISIMIP
ds_vars = []
for var in cfg.variables:
ds_vars.append(agri.load(var, chunking={'time': 'auto'})[[var]])
ds = xr.merge(ds_vars, compat="override")
agri.extract(**extract_kwargs)
self.dataset_class = agri
else:
# Without this the unhandled provider leaves ds as None and fails
# below with a bare `TypeError: 'NoneType' is not subscriptable`,
# which says nothing about the actual mistake.
raise ValueError(
f"Dataset provider '{cfg.dataset}' has no extraction path in "
f"ClimateExtractor.extract(). Supported: {', '.join(_EXTRACTABLE)}.\n"
f" ERA5 is available as climdata.ERA5, but it mirrors whole "
f"months to Zarr rather than extracting, so it is driven directly "
f"rather than through this workflow."
)
for var in cfg.variables:
ds[var] = xclim.core.units.convert_units_to(ds[var], cfg.varinfo[var].units)
# ds = ds.compute()
return ds
get_actions(self)
¶
Return a dictionary of workflow actions with their outputs and descriptions.
Supports actionsinfo in mapping style or list style and returns a consistent mapping of action name to description/output.
Returns:
| Type | Description |
|---|---|
dict |
Mapping action name -> {'output': ..., 'description': ...} |
Source code in climdata/utils/wrapper_workflow.py
def get_actions(self) -> dict:
"""Return a dictionary of workflow actions with their outputs and descriptions.
Supports ``actionsinfo`` in mapping style or list style and returns a consistent mapping of action name to description/output.
Returns:
dict: Mapping action name -> {'output': ..., 'description': ...}
"""
if not self.cfg or not hasattr(self.cfg, "actionsinfo"):
raise ValueError("Configuration or actionsinfo not loaded")
actions_map = getattr(self.cfg, "actionsinfo")
# If 'actions' key exists, fallback to list style
if "actions" in actions_map:
actions_map = {a["name"]: {"output": a["output"], "description": a["description"]}
for a in actions_map["actions"]}
return actions_map
get_datasets(self)
¶
Return the list of dataset provider names available in configuration.
Returns:
| Type | Description |
|---|---|
List[str] |
Names of available dataset providers from |
Source code in climdata/utils/wrapper_workflow.py
def get_datasets(self) -> List[str]:
"""Return the list of dataset provider names available in configuration.
Returns:
List[str]: Names of available dataset providers from ``cfg.dsinfo``.
"""
if not self.cfg or not hasattr(self.cfg, "dsinfo"):
raise ValueError("Configuration or dsinfo not loaded")
return list(self.cfg.dsinfo.keys())
get_impute_methods(self)
¶
Return mapping of available imputation methods from config.
Returns:
| Type | Description |
|---|---|
Dict[str, dict] |
Mapping of method name -> config (empty dict if none configured). |
Source code in climdata/utils/wrapper_workflow.py
def get_impute_methods(self) -> Dict[str, dict]:
"""Return mapping of available imputation methods from config.
Returns:
Dict[str, dict]: Mapping of method name -> config (empty dict if none configured).
"""
if not hasattr(self.cfg, "imputeinfo") or not self.cfg.imputeinfo:
return {}
return dict(self.cfg.imputeinfo)
get_indices(self, variables, require_all=True)
¶
Fetch climate extreme indices from cfg.extinfo that involve the given variables.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
variables |
list[str] |
Variables to filter indices by (if |
required |
require_all |
bool |
If True, return indices that require all provided variables; otherwise return indices if any variable matches. |
True |
Returns:
| Type | Description |
|---|---|
dict |
Mapping index_name -> index_definition. |
Source code in climdata/utils/wrapper_workflow.py
def get_indices(self, variables: List[str], require_all: bool = True) -> Dict[str, dict]:
"""Fetch climate extreme indices from ``cfg.extinfo`` that involve the given variables.
Args:
variables (list[str]): Variables to filter indices by (if ``None``, uses ``cfg.variables``).
require_all (bool): If True, return indices that require all provided variables; otherwise return indices if any variable matches.
Returns:
dict: Mapping index_name -> index_definition.
"""
cfg = self.cfg
variables = variables or cfg.variables
if not hasattr(cfg, "extinfo") or not cfg.extinfo:
raise ValueError("cfg.extinfo is not defined or empty")
indices_def = cfg.extinfo.get("indices", {})
if not indices_def:
return {}
matched_indices = {}
for idx_name, idx_info in indices_def.items():
idx_vars = idx_info.get("variables", [])
if require_all:
if all(var in variables for var in idx_vars):
matched_indices[idx_name] = idx_info
else:
if any(var in variables for var in idx_vars):
matched_indices[idx_name] = idx_info
return matched_indices
get_variables(self, dataset=None)
¶
Return the list of variables available for a dataset.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
dataset |
str |
Dataset name to query. Defaults to |
None |
Returns:
| Type | Description |
|---|---|
List[str] |
List of variable names. |
Source code in climdata/utils/wrapper_workflow.py
def get_variables(self, dataset: Optional[str] = None) -> List[str]:
"""Return the list of variables available for a dataset.
Args:
dataset (str, optional): Dataset name to query. Defaults to ``cfg.dataset``.
Returns:
List[str]: List of variable names.
"""
if not self.cfg or not hasattr(self.cfg, "dsinfo"):
raise ValueError("Configuration or dsinfo not loaded")
dataset_name = dataset or getattr(self.cfg, "dataset", None)
if dataset_name is None:
raise ValueError("Dataset not specified and cfg.dataset is None")
dsinfo = self.cfg.dsinfo.get(dataset_name)
if not dsinfo or "variables" not in dsinfo:
raise ValueError(f"No variable info available for dataset '{dataset_name}'")
return list(dsinfo["variables"].keys())
get_varinfo(self, var)
¶
Get metadata for a variable from varinfo.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
var |
str |
Name of the variable, e.g., 'tas', 'tasmax', 'pr'. |
required |
Returns:
| Type | Description |
|---|---|
dict |
Metadata dictionary containing cf_name, long_name, units, etc. |
Exceptions:
| Type | Description |
|---|---|
ValueError |
If varinfo is not loaded or variable not found. |
Source code in climdata/utils/wrapper_workflow.py
def get_varinfo(self, var: str) -> dict:
"""Get metadata for a variable from varinfo.
Args:
var (str): Name of the variable, e.g., 'tas', 'tasmax', 'pr'.
Returns:
dict: Metadata dictionary containing cf_name, long_name, units, etc.
Raises:
ValueError: If varinfo is not loaded or variable not found.
"""
if not self.cfg or not hasattr(self.cfg, "varinfo") or not self.cfg.varinfo:
raise ValueError("Configuration or varinfo not loaded")
if var not in self.cfg.varinfo:
raise ValueError(f"Variable '{var}' not found in varinfo")
return self.cfg.varinfo[var]
impute(self, ds=None)
¶
Impute missing values using the configured imputation method.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
ds |
xr.Dataset |
Dataset to impute. If None, uses
|
None |
Returns:
| Type | Description |
|---|---|
xr.Dataset | None |
The imputed dataset (also sets
|
Exceptions:
| Type | Description |
|---|---|
ValueError |
If |
Source code in climdata/utils/wrapper_workflow.py
@update_ds(attr_name='impute_ds')
def impute(self, ds: xr.Dataset = None) -> xr.Dataset:
"""Impute missing values using the configured imputation method.
Args:
ds (xr.Dataset, optional): Dataset to impute. If None, uses
``self.current_ds``.
Returns:
xr.Dataset | None: The imputed dataset (also sets
``self.current_ds`` and ``self.impute_ds``). Returns ``None``
if no imputation method is configured.
Raises:
ValueError: If ``ds`` is ``None`` and ``self.current_ds`` is not set.
"""
cfg = self.cfg
impute_cfg = cfg.imputeinfo
ds = ds or self.current_ds
if ds is None:
raise ValueError("No dataset provided and no current_ds is available.")
if cfg.impute is None:
self.logger.warning("No imputation method selected.")
return None
# select variables (optional)
# variables = cfg.get("variables", None)
# if variables:
# missing = [v for v in variables if v not in self.current_ds.data_vars]
# if missing:
# raise ValueError(f"Variables not present in dataset: {missing}")
# ds_in = self.current_ds[variables]
# else:
# ds_in = self.current_ds
method = cfg.impute
normalize = impute_cfg[method].get("normalize", True)
time_dim = cfg.dsinfo[cfg.dataset.upper()].get("time_dim", "time")
lat_dim = cfg.dsinfo[cfg.dataset.upper()].get("lat_dim", "lat")
lon_dim = cfg.dsinfo[cfg.dataset.upper()].get("lon_dim", "lon")
# epochs = impute_cfg[method].get("epochs", 300)
# run imputer (Imputer expects dims (time, lat, lon))
imputer = Imputer(
ds,
time_dim=time_dim,
lat_dim=lat_dim,
lon_dim=lon_dim,
method=method,
normalize=normalize,
)
recovered = imputer.impute()
# merge imputed variables back into original dataset if we operated on a subset
ds_out = recovered
# Return dataset (decorator will set current_ds and impute_ds and generate filenames)
return ds_out
load_config(self, overrides=None)
¶
Load and compose the Hydra configuration.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
overrides |
list[str] |
Hydra overrides to apply when composing the configuration. |
None |
Returns:
| Type | Description |
|---|---|
DictConfig |
Composed Hydra configuration object stored on |
Source code in climdata/utils/wrapper_workflow.py
def load_config(self, overrides: Optional[List[str]] = None) -> DictConfig:
"""Load and compose the Hydra configuration.
Args:
overrides (list[str], optional): Hydra overrides to apply when composing the configuration.
Returns:
DictConfig: Composed Hydra configuration object stored on ``self.cfg``.
"""
overrides = overrides or []
conf_dir_abs = os.path.abspath(_ensure_local_conf())
GlobalHydra.instance().clear()
with initialize_config_dir(config_dir=conf_dir_abs, version_base=None):
self.cfg = compose(config_name=self.cfg_name, overrides=overrides)
return self.cfg
plot(self, variable=None, ds=None, **kwargs)
¶
Plot a variable on a Cartopy map with coastlines and country borders.
Thin wrapper around :func:climdata.viz.plot_map operating on
self.current_ds by default.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
variable |
str |
Variable to plot. Defaults to the sole data variable if the dataset has exactly one. |
None |
ds |
xr.Dataset |
Dataset to plot. If |
None |
**kwargs |
Forwarded to :func: |
{} |
Returns:
| Type | Description |
|---|---|
The Cartopy |
Source code in climdata/utils/wrapper_workflow.py
def plot(self, variable=None, ds: xr.Dataset = None, **kwargs):
"""Plot a variable on a Cartopy map with coastlines and country borders.
Thin wrapper around :func:`climdata.viz.plot_map` operating on
``self.current_ds`` by default.
Args:
variable (str, optional): Variable to plot. Defaults to the sole
data variable if the dataset has exactly one.
ds (xr.Dataset, optional): Dataset to plot. If ``None``, uses ``self.current_ds``.
**kwargs: Forwarded to :func:`climdata.viz.plot_map`
(e.g. ``time``, ``reduce``, ``cmap``, ``ax``).
Returns:
The Cartopy ``GeoAxes`` the field was drawn on.
"""
from climdata.viz import plot_map
ds = ds if ds is not None else self.current_ds
if ds is None:
raise ValueError("No dataset provided and no current_ds is available.")
return plot_map(ds, variable=variable, **kwargs)
preprocess_aoi(self, cfg)
¶
Process an 'aoi' specification in the configuration.
Supports GeoJSON strings or dictionaries for FeatureCollection, Feature, or simple geometry objects (Point/Polygon).
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
cfg |
DictConfig |
Configuration object with optional |
required |
Returns:
| Type | Description |
|---|---|
DictConfig |
The modified configuration. When a Point is provided, |
Source code in climdata/utils/wrapper_workflow.py
def preprocess_aoi(self, cfg: DictConfig) -> DictConfig:
"""Process an 'aoi' specification in the configuration.
Supports GeoJSON strings or dictionaries for FeatureCollection, Feature, or simple geometry objects (Point/Polygon).
Args:
cfg (DictConfig): Configuration object with optional ``aoi`` entry.
Returns:
DictConfig: The modified configuration. When a Point is provided, ``cfg.lat`` and ``cfg.lon`` are set; when a Polygon is provided, ``cfg.bounds`` is set and ``cfg.region`` is set to "custom".
"""
if not hasattr(cfg, "aoi") or cfg.aoi is None:
return cfg
if isinstance(cfg.aoi, str):
try:
cfg.aoi = json.loads(cfg.aoi)
except json.JSONDecodeError:
raise ValueError("Invalid AOI JSON string")
aoi = cfg.aoi
if aoi.get("type") == "FeatureCollection":
geom = shape(aoi["features"][0]["geometry"])
elif aoi.get("type") == "Feature":
geom = shape(aoi["geometry"])
elif "type" in aoi:
geom = shape(aoi)
else:
raise ValueError(f"Unsupported AOI format: {aoi}")
if isinstance(geom, Point):
cfg.lat = geom.y
cfg.lon = geom.x
cfg.bounds = None
elif isinstance(geom, Polygon):
minx, miny, maxx, maxy = geom.bounds
cfg.bounds = {"custom": {"lat_min": miny, "lat_max": maxy,
"lon_min": minx, "lon_max": maxx}}
cfg.region = "custom"
cfg.lat = None
cfg.lon = None
else:
raise ValueError(f"Unknown geometry type {geom.geom_type}")
return cfg
reproject(self, ds=None, **overrides)
¶
Reproject / resample the dataset onto the configured target grid.
Reads target_projection and target_resolution from the config, plus
the secondary knobs under regrid. Returns None (leaving the current
dataset untouched) when neither target is configured.
The units of target_resolution must match the axis units of
target_projection: a metric resolution such as "10 km" has no fixed
angular size, so combining it with a geographic CRS raises
:class:~climdata.grid.units.ResolutionCRSMismatch rather than silently
approximating. The check runs before any data is touched.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
ds |
xr.Dataset |
Dataset to transform. If |
None |
**overrides |
Any argument of :func: |
{} |
Returns:
| Type | Description |
|---|---|
xr.Dataset | None |
The reprojected dataset (also sets |
Exceptions:
| Type | Description |
|---|---|
ValueError |
If |
ResolutionCRSMismatch |
If the resolution units contradict the target CRS. |
Examples:
extractor = ClimData(overrides=[ "target_projection=EPSG:3035", "target_resolution=10 km", ]) extractor.extract() grid_ds = extractor.reproject()
Source code in climdata/utils/wrapper_workflow.py
@update_ds("reprojected_ds")
def reproject(self, ds: xr.Dataset = None, **overrides) -> xr.Dataset:
"""Reproject / resample the dataset onto the configured target grid.
Reads ``target_projection`` and ``target_resolution`` from the config, plus
the secondary knobs under ``regrid``. Returns ``None`` (leaving the current
dataset untouched) when neither target is configured.
The units of ``target_resolution`` must match the axis units of
``target_projection``: a metric resolution such as ``"10 km"`` has no fixed
angular size, so combining it with a geographic CRS raises
:class:`~climdata.grid.units.ResolutionCRSMismatch` rather than silently
approximating. The check runs before any data is touched.
Args:
ds (xr.Dataset, optional): Dataset to transform. If ``None``, uses
``self.current_ds``.
**overrides: Any argument of :func:`climdata.grid.reproject`, overriding
the configured value for this call (e.g. ``method="average"``).
Returns:
xr.Dataset | None: The reprojected dataset (also sets ``self.current_ds``
and ``self.reprojected_ds``), or ``None`` if no target grid is configured.
Raises:
ValueError: If ``ds`` is ``None`` and ``self.current_ds`` is not set.
ResolutionCRSMismatch: If the resolution units contradict the target CRS.
Example:
extractor = ClimData(overrides=[
"target_projection=EPSG:3035", "target_resolution=10 km",
])
extractor.extract()
grid_ds = extractor.reproject()
"""
from climdata.grid import reproject as _reproject
cfg = self.cfg
ds = ds if ds is not None else self.current_ds
if ds is None:
raise ValueError("No dataset provided and no current_ds is available.")
target_projection = overrides.pop("target_projection", cfg.get("target_projection"))
target_resolution = overrides.pop("target_resolution", cfg.get("target_resolution"))
if target_projection is None and target_resolution is None and "like" not in overrides:
self.logger.info(
"No target_projection or target_resolution configured; skipping reprojection."
)
return None
regrid_cfg = cfg.get("regrid") or {}
kwargs = {
"method": regrid_cfg.get("method", "bilinear"),
"align": regrid_cfg.get("align", True),
"bounds": regrid_cfg.get("bounds"),
"engine": regrid_cfg.get("engine", "rasterio"),
}
kwargs.update(overrides)
# Dimension names are already declared per dataset in parameters.yaml.
try:
dsinfo = cfg.dsinfo[cfg.dataset.upper()]
kwargs.setdefault(
"dsinfo_dims", (dsinfo.get("lon_dim", "lon"), dsinfo.get("lat_dim", "lat"))
)
except Exception:
pass
if kwargs.get("bounds") is not None:
kwargs["bounds"] = [float(v) for v in kwargs["bounds"]]
if isinstance(kwargs.get("method"), DictConfig):
kwargs["method"] = dict(kwargs["method"])
self.logger.info(
"Reprojecting to %s at %s", target_projection or "source CRS", target_resolution
)
return _reproject(ds, target_projection, target_resolution, **kwargs)
run_workflow(self, overrides=None, actions=None, file=None)
¶
Execute a sequence of workflow actions.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
overrides |
list[str] |
Hydra overrides to apply (not all actions will use these). |
None |
actions |
list[str] |
Ordered list of actions to perform. Supported actions include: 'upload_netcdf', 'upload_csv', 'extract', 'calc_index', 'to_dataframe', 'to_csv', 'to_nc', 'to_fair'. |
None |
file |
str |
File path used for upload actions when required. |
None |
Returns:
| Type | Description |
|---|---|
WorkflowResult |
Named result container with populated fields for dataframe/dataset/filenames. |
Source code in climdata/utils/wrapper_workflow.py
def run_workflow(self, overrides: Optional[List[str]] = None,
actions: Optional[List[str]] = None,
file: Optional[str] = None) -> WorkflowResult:
"""Execute a sequence of workflow actions.
Args:
overrides (list[str], optional): Hydra overrides to apply (not all actions will use these).
actions (list[str], optional): Ordered list of actions to perform. Supported actions include: 'upload_netcdf', 'upload_csv', 'extract', 'calc_index', 'to_dataframe', 'to_csv', 'to_nc', 'to_fair'.
file (str, optional): File path used for upload actions when required.
Returns:
WorkflowResult: Named result container with populated fields for dataframe/dataset/filenames.
"""
actions = actions or ["extract", "calc_index", "to_csv", "to_nc"]
result = WorkflowResult(cfg=self.cfg)
for action in actions:
self.logger.info("Starting action: %s", action)
try:
if action == "upload_netcdf":
if file is None:
raise ValueError(
"Action 'upload_netcdf' requires argument 'netcdf_file', "
"but none was provided."
)
# Validate extension
valid_nc_ext = (".nc", ".nc4", ".nc.gz")
if not any(str(file).lower().endswith(ext) for ext in valid_nc_ext):
raise ValueError(
f"Invalid file format for upload_netcdf: '{file}'. "
f"Expected one of: {valid_nc_ext}"
)
self.upload_netcdf(file)
result.dataset = self.current_ds
elif action == "upload_csv":
if file is None:
raise ValueError(
"Action 'upload_csv' requires argument 'csv_file', "
"but none was provided."
)
# Validate CSV extension
valid_csv_ext = (".csv", ".csv.gz")
if not any(str(file).lower().endswith(ext) for ext in valid_csv_ext):
raise ValueError(
f"Invalid file format for upload_csv: '{file}'. "
f"Expected one of: {valid_csv_ext}"
)
self.upload_csv(file)
result.dataset = self.current_ds
elif action == "extract":
if self.cfg.dataset is None:
raise ValueError(
"Action 'extract' cannot run because no dataset provider is set "
"(cfg.dataset is None)."
)
self.extract()
result.dataset = self.current_ds
elif action == "calc_index":
if self.current_ds is None:
raise ValueError(
"Action 'calc_index' requires a dataset, but no dataset is available. "
"Upload or extract a dataset before computing an index."
)
self.calc_index()
result.index_ds = self.current_ds
elif action == "to_csv":
if self.current_ds is None:
raise ValueError(
"Action 'to_dataframe' requires a dataset, but no dataset is available. "
"Upload or extract a dataset before converting to a DataFrame."
)
self.to_dataframe()
result.dataframe = self.current_df
result.filename = self.to_csv()
elif action == "to_nc":
if self.current_ds is None:
raise ValueError(
"Action 'to_nc' requires a dataset, but no dataset is available. "
"Upload or extract a dataset before saving to NetCDF."
)
result.filename = self.to_nc()
elif action == "impute":
if self.current_ds is None:
raise ValueError("Action 'impute' requires a dataset, but no dataset is available.")
self.impute()
result.dataset = self.current_ds
result.impute_ds = getattr(self, "impute_ds", None)
elif action == "reproject":
if self.current_ds is None:
raise ValueError(
"Action 'reproject' requires a dataset, but no dataset is available. "
"Upload or extract a dataset before regridding."
)
self.reproject()
result.dataset = self.current_ds
result.reprojected_ds = getattr(self, "reprojected_ds", None)
elif action == "to_fair":
if self.current_ds is None:
raise ValueError(
"Action 'to_fair' requires a dataset, but no dataset is available. "
"Upload or extract a dataset before exporting a FAIR crate."
)
result.filename = self.to_fair()
else:
raise ValueError(
f"Unknown action '{action}'. Supported: "
+ ", ".join(sorted(_WORKFLOW_ACTIONS))
)
self.logger.info("Completed action: %s", action)
except Exception:
self.logger.exception("Action '%s' failed", action)
raise
return result
to_csv(self, df=None, filename=None, format='default')
¶
Save a DataFrame to CSV with optional format specification.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
df |
pd.DataFrame |
DataFrame to save. Defaults to |
None |
filename |
str |
Output filename/directory. Defaults to |
None |
format |
str |
Output format. Options: - 'default': Long-form (single file) - 'simplace': SIMPLACE format (splits by location, tab-separated) - 'monica': MONICA format (splits by location, tab-separated) Defaults to 'default'. |
'default' |
Returns:
| Type | Description |
|---|---|
str |
The path of the written CSV file(s). For SIMPLACE/MONICA, returns base directory. |
Exceptions:
| Type | Description |
|---|---|
ValueError |
If format is not supported or required data is missing. |
Source code in climdata/utils/wrapper_workflow.py
def to_csv(self, df: Optional[pd.DataFrame] = None, filename: Optional[str] = None, format: str = "default") -> str:
"""Save a DataFrame to CSV with optional format specification.
Args:
df (pd.DataFrame, optional): DataFrame to save. Defaults to ``self.current_df``.
filename (str, optional): Output filename/directory. Defaults to ``self.filename_csv``.
format (str): Output format. Options:
- 'default': Long-form (single file)
- 'simplace': SIMPLACE format (splits by location, tab-separated)
- 'monica': MONICA format (splits by location, tab-separated)
Defaults to 'default'.
Returns:
str: The path of the written CSV file(s). For SIMPLACE/MONICA, returns base directory.
Raises:
ValueError: If format is not supported or required data is missing.
"""
df = df if df is not None else self.current_df
format_lower = format.lower()
# Apply format conversion if requested
if format_lower in ("simplace", "monica"):
return self._convert_to_gridded_format(df, filename, format=format_lower)
elif format_lower == "default":
filename = filename or getattr(self, "filename_csv", None)
if filename is None:
raise ValueError("No filename provided and filename_csv is not set")
path = Path(filename)
path.parent.mkdir(parents=True, exist_ok=True)
df.to_csv(filename, index=False, sep='\t')
self.filename_csv = str(path)
self.current_filename = str(path)
self.logger.info(f"DataFrame saved to CSV file ({format} format): {self.current_filename}")
return filename
else:
raise ValueError(f"Unsupported format: {format}. Supported formats: 'default', 'simplace', 'monica'")
to_dataframe(self, ds=None)
¶
Convert a dataset to a long-form pandas DataFrame.
The output contains columns: time, lat, lon (or latitude/longitude), variable, value, units, source.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
ds |
xr.Dataset |
Dataset to convert. If |
None |
Returns:
| Type | Description |
|---|---|
pd.DataFrame |
Long-form DataFrame (also sets |
Source code in climdata/utils/wrapper_workflow.py
@update_df()
def to_dataframe(self, ds: xr.Dataset = None) -> pd.DataFrame:
"""Convert a dataset to a long-form pandas DataFrame.
The output contains columns: time, lat, lon (or latitude/longitude), variable, value, units, source.
Args:
ds (xr.Dataset, optional): Dataset to convert. If ``None``, uses ``self.current_ds``.
Returns:
pd.DataFrame: Long-form DataFrame (also sets ``self.current_df``).
"""
ds = ds or self.current_ds
if ds is None:
raise ValueError("No dataset provided and no current_ds is available.")
df = ds.to_dataframe().reset_index()
id_vars = [c for c in ("time", "lat", "lon", "latitude", "longitude") if c in df]
value_vars = [v for v in ds.data_vars if v in df.columns]
if not value_vars:
raise ValueError("No variables in dataset available to melt into long format")
df_long = df.melt(
id_vars=id_vars,
value_vars=value_vars,
var_name="variable",
value_name="value"
)
df_long["units"] = df_long["variable"].apply(
lambda v: ds[v].attrs.get("units", "unknown")
)
if getattr(self.cfg, "dataset") == 'cmip':
df_long["source_id"] = getattr(self.cfg, "source_id")
df_long["source"] = getattr(self.cfg, "dataset", ds.attrs.get("source", "unknown"))
df_long = df_long.drop_duplicates()
self._gen_fn_cfg()
return df_long
to_fair(self, ds=None, output_dir=None, title=None, description=None, license_url='https://creativecommons.org/licenses/by/4.0/', creator=None, zip_crate=False)
¶
Export the dataset as a FAIR Research Object Crate (RO-Crate).
Packages the dataset into a self-describing directory that satisfies the FAIR principles (Findable, Accessible, Interoperable, Reusable):
- Findable – unique dataset identifier & rich JSON-LD metadata
- Accessible – standard CF-compliant NetCDF + open JSON-LD manifest
- Interoperable – CF conventions in NetCDF; Schema.org / RO-Crate vocab
- Reusable – license, provenance, variable descriptions embedded
The output folder contains:
.. code-block:: text
1 2 3 4 | |
Uses the rocrate Python package when available; falls back to a
hand-written JSON-LD manifest so the method always works.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
ds |
xr.Dataset |
Dataset to package. Defaults to
|
None |
output_dir |
str |
Destination folder. Defaults to
|
None |
title |
str |
Human-readable dataset title. |
None |
description |
str |
Free-text description of the dataset. |
None |
license_url |
str |
SPDX or Creative-Commons URL for the data licence. Defaults to CC-BY-4.0. |
'https://creativecommons.org/licenses/by/4.0/' |
creator |
str |
Name of the person / organisation that produced the data. |
None |
zip_crate |
bool |
If True, zip the crate directory and return
the |
False |
Returns:
| Type | Description |
|---|---|
str |
Absolute path to the RO-Crate directory (or |
Exceptions:
| Type | Description |
|---|---|
ValueError |
If no dataset is available. |
Examples:
>>> extractor = ClimData(overrides=["dataset=mswx", ...])
>>> ds = extractor.extract()
>>> crate_path = extractor.to_fair(title="MSWX Germany 2014")
>>> print(crate_path) # …/mswx_fair_crate/
Source code in climdata/utils/wrapper_workflow.py
def to_fair(
self,
ds: Optional[xr.Dataset] = None,
output_dir: Optional[str] = None,
title: Optional[str] = None,
description: Optional[str] = None,
license_url: str = "https://creativecommons.org/licenses/by/4.0/",
creator: Optional[str] = None,
zip_crate: bool = False,
) -> str:
"""Export the dataset as a FAIR Research Object Crate (RO-Crate).
Packages the dataset into a self-describing directory that satisfies
the FAIR principles (Findable, Accessible, Interoperable, Reusable):
* **Findable** – unique dataset identifier & rich JSON-LD metadata
* **Accessible** – standard CF-compliant NetCDF + open JSON-LD manifest
* **Interoperable** – CF conventions in NetCDF; Schema.org / RO-Crate vocab
* **Reusable** – license, provenance, variable descriptions embedded
The output folder contains:
.. code-block:: text
<output_dir>/
data.nc # CF-compliant NetCDF dataset
ro-crate-metadata.json # JSON-LD manifest (RO-Crate 1.1)
ro-crate-preview.html # human-readable summary (optional)
Uses the ``rocrate`` Python package when available; falls back to a
hand-written JSON-LD manifest so the method always works.
Args:
ds (xr.Dataset, optional): Dataset to package. Defaults to
``self.current_ds``.
output_dir (str, optional): Destination folder. Defaults to
``<filename_nc_stem>_fair_crate``.
title (str, optional): Human-readable dataset title.
description (str, optional): Free-text description of the dataset.
license_url (str): SPDX or Creative-Commons URL for the data
licence. Defaults to CC-BY-4.0.
creator (str, optional): Name of the person / organisation that
produced the data.
zip_crate (bool): If *True*, zip the crate directory and return
the ``.zip`` path. Defaults to *False*.
Returns:
str: Absolute path to the RO-Crate directory (or ``.zip`` file).
Raises:
ValueError: If no dataset is available.
Example:
>>> extractor = ClimData(overrides=["dataset=mswx", ...])
>>> ds = extractor.extract()
>>> crate_path = extractor.to_fair(title="MSWX Germany 2014")
>>> print(crate_path) # …/mswx_fair_crate/
"""
import datetime
import zipfile
import uuid
ds = ds if ds is not None else self.current_ds
if ds is None:
raise ValueError(
"No dataset available. Run extract() or upload_netcdf() first."
)
cfg = self.cfg
# ── 1. Resolve output directory ─────────────────────────────────────
if output_dir is None:
base = getattr(self, "filename_nc", None)
stem = Path(base).stem if base else getattr(cfg, "dataset", "climdata")
output_dir = str(Path(base).parent / f"{stem}_fair_crate") if base else f"{stem}_fair_crate"
crate_dir = Path(output_dir)
crate_dir.mkdir(parents=True, exist_ok=True)
# ── 2. Write the NetCDF payload ──────────────────────────────────────
nc_path = crate_dir / "data.nc"
# Embed CF global attributes before saving
ds_out = ds.copy()
ds_out.attrs.setdefault("Conventions", "CF-1.8")
ds_out.attrs.setdefault("institution", creator or "climdata")
ds_out.attrs.setdefault(
"source",
getattr(cfg, "dataset", "unknown"),
)
ds_out.attrs.setdefault(
"history",
f"{datetime.datetime.utcnow().isoformat()}Z climdata to_fair()",
)
ds_out.to_netcdf(nc_path)
self.logger.info("FAIR payload written to %s", nc_path)
# ── 3. Collect metadata from cfg ────────────────────────────────────
dataset_name = getattr(cfg, "dataset", "unknown")
variables = list(getattr(cfg, "variables", list(ds.data_vars)))
t_start = getattr(getattr(cfg, "time_range", None), "start_date", None)
t_end = getattr(getattr(cfg, "time_range", None), "end_date", None)
if t_start is None and "time" in ds.coords:
t_start = str(ds.time.values[0])[:10]
if t_end is None and "time" in ds.coords:
t_end = str(ds.time.values[-1])[:10]
# spatial extent
lat_min = lat_max = lon_min = lon_max = None
try:
box = getattr(cfg, "box", None)
if box and getattr(box, "lat_min", None) is not None:
lat_min = float(box.lat_min)
lat_max = float(box.lat_max)
lon_min = float(box.lon_min)
lon_max = float(box.lon_max)
elif getattr(cfg, "lat", None) is not None:
lat_min = lat_max = float(cfg.lat)
lon_min = lon_max = float(cfg.lon)
elif "lat" in ds.coords:
lat_min = float(ds.lat.values.min())
lat_max = float(ds.lat.values.max())
lon_min = float(ds.lon.values.min())
lon_max = float(ds.lon.values.max())
except Exception:
pass
# variable-level metadata
var_descriptions = []
varinfo = getattr(cfg, "varinfo", {})
for v in variables:
info = varinfo.get(v, {}) if varinfo else {}
units = getattr(info, "units", None) or ds[v].attrs.get("units", "1") if v in ds else "1"
long_name = ds[v].attrs.get("long_name", v) if v in ds else v
var_descriptions.append({"@type": "PropertyValue", "name": v,
"description": long_name, "unitText": units})
title = title or f"{dataset_name.upper()} climate data extract"
description = description or (
f"Climate dataset extracted by climdata v{climdata.__version__} "
f"from {dataset_name.upper()}. "
f"Variables: {', '.join(variables)}. "
f"Period: {t_start} to {t_end}."
)
crate_id = str(uuid.uuid4())
now_iso = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")
# ── 4. Try rocrate library; fall back to hand-crafted JSON-LD ────────
metadata_path = crate_dir / "ro-crate-metadata.json"
try:
from rocrate.rocrate import ROCrate # type: ignore
from rocrate.model import ContextEntity # type: ignore
crate = ROCrate()
crate.name = title
crate.description = description
crate.add_file(
str(nc_path),
dest_path="data.nc",
properties={
"name": "data.nc",
"description": "CF-compliant NetCDF dataset",
"encodingFormat": "application/x-netcdf",
"variableMeasured": var_descriptions,
},
)
if t_start:
crate.root_dataset["temporalCoverage"] = f"{t_start}/{t_end}"
if lat_min is not None:
crate.root_dataset["spatialCoverage"] = {
"@type": "Place",
"geo": {
"@type": "GeoShape",
"box": f"{lat_min} {lon_min} {lat_max} {lon_max}",
},
}
if creator:
person = crate.add(
ContextEntity(crate, f"#{creator.replace(' ', '_')}",
properties={"@type": "Person", "name": creator})
)
crate.root_dataset["creator"] = person
crate.root_dataset["license"] = license_url
crate.root_dataset["datePublished"] = now_iso
crate.root_dataset["identifier"] = crate_id
crate.root_dataset["keywords"] = variables
crate.root_dataset["producer"] = {
"@type": "SoftwareApplication",
"name": "climdata",
"version": climdata.__version__,
"url": "https://github.com/climdata/climdata",
}
crate.write(str(crate_dir))
self.logger.info("RO-Crate written via rocrate library.")
except ImportError:
# ── fallback: hand-craft the JSON-LD ────────────────────────────
self.logger.warning(
"'rocrate' package not found. Writing plain JSON-LD metadata. "
"Install with: pip install rocrate"
)
geo_shape: Optional[Dict] = None
if lat_min is not None:
geo_shape = {
"@type": "GeoShape",
"box": f"{lat_min} {lon_min} {lat_max} {lon_max}",
}
metadata = {
"@context": [
"https://w3id.org/ro/crate/1.1/context",
{"climdata": "https://climdata.readthedocs.io/"},
],
"@graph": [
# ── RO-Crate metadata descriptor ────────────────────
{
"@id": "ro-crate-metadata.json",
"@type": "CreativeWork",
"conformsTo": {"@id": "https://w3id.org/ro/crate/1.1"},
"about": {"@id": "./"},
},
# ── Root data entity (the crate itself) ─────────────
{
"@id": "./",
"@type": "Dataset",
"identifier": crate_id,
"name": title,
"description": description,
"datePublished": now_iso,
"license": {"@id": license_url},
"keywords": variables,
**({
"creator": {
"@type": "Person",
"name": creator,
}
} if creator else {}),
**({
"temporalCoverage": f"{t_start}/{t_end}"
} if t_start else {}),
**({
"spatialCoverage": {
"@type": "Place",
"geo": geo_shape,
}
} if geo_shape else {}),
"producer": {
"@type": "SoftwareApplication",
"name": "climdata",
"version": climdata.__version__,
"url": "https://github.com/Kaushikreddym/climdata",
},
"hasPart": [{"@id": "data.nc"}],
},
# ── File entity (the NetCDF) ─────────────────────────
{
"@id": "data.nc",
"@type": "File",
"name": "data.nc",
"description": "CF-compliant NetCDF dataset",
"encodingFormat": "application/x-netcdf",
"variableMeasured": var_descriptions,
**({
"temporalCoverage": f"{t_start}/{t_end}"
} if t_start else {}),
},
# ── License entity ───────────────────────────────────
{
"@id": license_url,
"@type": "CreativeWork",
"name": "CC-BY 4.0" if "creativecommons" in license_url else license_url,
"url": license_url,
},
],
}
with open(metadata_path, "w", encoding="utf-8") as f:
json.dump(metadata, f, indent=2, ensure_ascii=False)
# ── 5. Write a simple HTML preview ──────────────────────────────────
html_path = crate_dir / "ro-crate-preview.html"
var_rows = "".join(
f"<tr><td><code>{v['name']}</code></td>"
f"<td>{v['description']}</td>"
f"<td>{v['unitText']}</td></tr>"
for v in var_descriptions
)
spatial_str = (
f"{lat_min:.3f}°N – {lat_max:.3f}°N, {lon_min:.3f}°E – {lon_max:.3f}°E"
if lat_min is not None
else "(not specified)"
)
html_content = f"""<!DOCTYPE html>
<html lang="en">
<head><meta charset="UTF-8">
<title>{title}</title>
<style>
body{{font-family:sans-serif;max-width:900px;margin:2em auto;padding:0 1em}}
table{{border-collapse:collapse;width:100%}}
th,td{{border:1px solid #ccc;padding:.4em .8em;text-align:left}}
th{{background:#f0f0f0}}
.badge{{background:#0078d4;color:#fff;padding:.2em .6em;border-radius:4px;font-size:.85em}}
</style>
</head>
<body>
<h1>{title} <span class="badge">RO-Crate</span></h1>
<p>{description}</p>
<table>
<tr><th>Field</th><th>Value</th></tr>
<tr><td>Dataset</td><td>{dataset_name.upper()}</td></tr>
<tr><td>Period</td><td>{t_start} → {t_end}</td></tr>
<tr><td>Spatial coverage</td><td>{spatial_str}</td></tr>
<tr><td>License</td><td><a href="{license_url}">{license_url}</a></td></tr>
<tr><td>Produced by</td><td>climdata v{climdata.__version__}</td></tr>
<tr><td>Crate ID</td><td><code>{crate_id}</code></td></tr>
</table>
<h2>Variables</h2>
<table>
<tr><th>CF name</th><th>Long name</th><th>Units</th></tr>
{var_rows}
</table>
<h2>Files</h2>
<ul>
<li><a href="data.nc">data.nc</a> – CF-compliant NetCDF</li>
<li><a href="ro-crate-metadata.json">ro-crate-metadata.json</a> – JSON-LD metadata</li>
</ul>
</body>
</html>
"""
html_path.write_text(html_content, encoding="utf-8")
# ── 6. Optionally zip ────────────────────────────────────────────────
if zip_crate:
zip_path = crate_dir.with_suffix(".zip")
with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf:
for file in crate_dir.rglob("*"):
zf.write(file, file.relative_to(crate_dir.parent))
self.logger.info("RO-Crate zipped to %s", zip_path)
return str(zip_path)
self.logger.info("FAIR RO-Crate ready at %s", crate_dir)
return str(crate_dir)
to_nc(self, ds=None, filename=None)
¶
Save an xarray Dataset to NetCDF.
Notes
- If
dsisNone: savecurrent_ds. - If
filenameisNone: useself.filename_nc. - Creates directories if needed and updates
self.filename_ncandself.current_filename.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
ds |
xr.Dataset |
Dataset to save. If |
None |
filename |
str |
Output filename. Defaults to |
None |
Returns:
| Type | Description |
|---|---|
str |
The path of the written NetCDF file. |
Source code in climdata/utils/wrapper_workflow.py
def to_nc(self, ds: Optional[xr.Dataset] = None, filename: Optional[str] = None) -> str:
"""Save an xarray Dataset to NetCDF.
Notes:
- If ``ds`` is ``None``: save ``current_ds``.
- If ``filename`` is ``None``: use ``self.filename_nc``.
- Creates directories if needed and updates ``self.filename_nc`` and ``self.current_filename``.
Args:
ds (xr.Dataset, optional): Dataset to save. If ``None``, uses ``self.current_ds``.
filename (str, optional): Output filename. Defaults to ``self.filename_nc``.
Returns:
str: The path of the written NetCDF file.
"""
# -------------------------------
# 1. Determine dataset to save
# -------------------------------
ds = ds or getattr(self, "current_ds", None)
if ds is None:
raise ValueError("No dataset available to save")
# -------------------------------
# 2. Determine filename
# -------------------------------
filename = filename or getattr(self, "filename_nc", None)
if filename is None:
raise ValueError("No filename provided and filename_nc is not set")
path = Path(filename)
path.parent.mkdir(parents=True, exist_ok=True)
# -------------------------------
# 3. Save to NetCDF
# -------------------------------
ds.to_netcdf(path)
# -------------------------------
# 4. Track filenames
# -------------------------------
self.filename_nc = str(path)
self.current_filename = str(path)
# print(f"Dataset saved to NetCDF file: {self.current_filename}")
self.logger.info(f"Dataset saved to NetCDF file: {self.current_filename}")
return str(path)
upload_csv(self, csv_file)
¶
Load a long-form CSV into an xarray.Dataset.
The CSV must contain time and lat/latitude, lon/longitude, variable, value. Units may be supplied in a units column and an optional source column is recognized.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
csv_file |
str |
Path to the CSV file to load. |
required |
Returns:
| Type | Description |
|---|---|
xr.Dataset |
The converted dataset (also sets |
Source code in climdata/utils/wrapper_workflow.py
@update_ds(attr_name='ds')
def upload_csv(self, csv_file: str) -> xr.Dataset:
"""Load a long-form CSV into an xarray.Dataset.
The CSV must contain ``time`` and ``lat``/``latitude``, ``lon``/``longitude``, ``variable``, ``value``. Units may be supplied in a ``units`` column and an optional ``source`` column is recognized.
Args:
csv_file (str): Path to the CSV file to load.
Returns:
xr.Dataset: The converted dataset (also sets ``self.current_ds``).
"""
if not os.path.exists(csv_file):
raise FileNotFoundError(f"{csv_file} does not exist")
df = pd.read_csv(csv_file, parse_dates=["time"])
lat_col = next((c for c in ["lat", "latitude"] if c in df.columns), None)
lon_col = next((c for c in ["lon", "longitude"] if c in df.columns), None)
if lat_col is None or lon_col is None:
raise ValueError("CSV must have 'lat'/'latitude' and 'lon'/'longitude' columns")
id_vars = ["time", lat_col, lon_col]
df_wide = df.pivot_table(index=id_vars, columns="variable", values="value").reset_index()
ds = df_wide.set_index(id_vars).to_xarray()
# Attach units from CSV. A blank cell reads back as NaN, which is not a
# units string — it must become "unknown" like an absent column, or
# xclim fails downstream on a float where it expects a unit.
for var in ds.data_vars:
units_series = (
df[df["variable"] == var]["units"].dropna()
if "units" in df.columns
else pd.Series(dtype=object)
)
ds[var].attrs["units"] = (
str(units_series.iloc[0]) if not units_series.empty else "unknown"
)
# Global source attribute
if "source" in df.columns:
source_series = df["source"].dropna().unique()
if len(source_series) > 0:
ds.attrs["source"] = source_series[0]
# Update cfg variables & varinfo
if not hasattr(self.cfg, "variables") or not self.cfg.variables:
self.cfg.variables = list(ds.data_vars)
if not hasattr(self.cfg, "varinfo") or not self.cfg.varinfo:
self.cfg.varinfo = {v: {"units": ds[v].attrs.get("units", "unknown")} for v in ds.data_vars}
self._gen_fn(ds=ds)
return ds
upload_netcdf(self, nc_file)
¶
Load a NetCDF file into an xarray.Dataset and update file metadata.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
nc_file |
str |
Path to the NetCDF file to open. |
required |
Returns:
| Type | Description |
|---|---|
xr.Dataset |
The loaded dataset (also sets |
Source code in climdata/utils/wrapper_workflow.py
@update_ds(attr_name='ds')
def upload_netcdf(self, nc_file: str) -> xr.Dataset:
"""Load a NetCDF file into an xarray.Dataset and update file metadata.
Args:
nc_file (str): Path to the NetCDF file to open.
Returns:
xr.Dataset: The loaded dataset (also sets ``self.current_ds``).
"""
if not os.path.exists(nc_file):
raise FileNotFoundError(f"{nc_file} does not exist")
ds = xr.open_dataset(nc_file)
# Update cfg variables & varinfo
if not hasattr(self.cfg, "variables") or not self.cfg.variables:
self.cfg.variables = list(ds.data_vars)
if not hasattr(self.cfg, "varinfo") or not self.cfg.varinfo:
self.cfg.varinfo = {v: {"units": ds[v].attrs.get("units", "unknown")}
for v in ds.data_vars}
self._gen_fn(ds=ds)
return ds
climdata.utils.wrapper_workflow.WorkflowResult
dataclass
¶
What :meth:ClimateExtractor.run_workflow produces.
Every field but cfg is optional and stays None unless an action in
the sequence populated it, so which attributes are set records which actions
ran. :meth:keys names them.
Attributes:
| Name | Type | Description |
|---|---|---|
cfg |
DictConfig |
The configuration the workflow ran with. |
dataset |
xr.Dataset | None |
Extracted or uploaded data. |
dataframe |
pd.DataFrame | None |
Long-form conversion of the dataset. |
filename |
str | None |
Path written by the last file-writing action. |
index_ds |
xr.Dataset | None |
Computed extreme index. |
index_filename |
str | None |
Path of the written index. |
impute_ds |
xr.Dataset | None |
Gap-filled dataset. |
impute_filename |
str | None |
Path of the written imputed dataset. |
reprojected_ds |
xr.Dataset | None |
Regridded dataset. |
reprojected_filename |
str | None |
Path of the written regridded dataset. |
Source code in climdata/utils/wrapper_workflow.py
@dataclass
class WorkflowResult:
"""What :meth:`ClimateExtractor.run_workflow` produces.
Every field but ``cfg`` is optional and stays ``None`` unless an action in
the sequence populated it, so which attributes are set records which actions
ran. :meth:`keys` names them.
Attributes:
cfg (DictConfig): The configuration the workflow ran with.
dataset (xr.Dataset | None): Extracted or uploaded data.
dataframe (pd.DataFrame | None): Long-form conversion of the dataset.
filename (str | None): Path written by the last file-writing action.
index_ds (xr.Dataset | None): Computed extreme index.
index_filename (str | None): Path of the written index.
impute_ds (xr.Dataset | None): Gap-filled dataset.
impute_filename (str | None): Path of the written imputed dataset.
reprojected_ds (xr.Dataset | None): Regridded dataset.
reprojected_filename (str | None): Path of the written regridded dataset.
"""
cfg: DictConfig
dataset: Optional[xr.Dataset] = None
dataframe: Optional[pd.DataFrame] = None
filename: Optional[str] = None
index_ds: Optional[xr.Dataset] = None
index_filename: Optional[str] = None
impute_ds: Optional[xr.Dataset] = None
impute_filename: Optional[str] = None
reprojected_ds: Optional[xr.Dataset] = None
reprojected_filename: Optional[str] = None
def keys(self):
"""Name the fields that were actually populated.
Returns:
list[str]: Attribute names whose value is not ``None`` — in effect,
a record of which workflow actions produced output.
"""
return [k for k, v in self.__dict__.items() if v is not None]
keys(self)
¶
Name the fields that were actually populated.
Returns:
| Type | Description |
|---|---|
list[str] |
Attribute names whose value is not |
Source code in climdata/utils/wrapper_workflow.py
def keys(self):
"""Name the fields that were actually populated.
Returns:
list[str]: Attribute names whose value is not ``None`` — in effect,
a record of which workflow actions produced output.
"""
return [k for k, v in self.__dict__.items() if v is not None]
climdata.utils.config.load_config(config_name='config', overrides=None, verbose=False)
¶
Load Hydra config from ./conf directory.
Uses initialize_config_dir with absolute path to ensure Hydra finds the config.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
config_name |
str |
Name of the config file (default: "config") |
'config' |
overrides |
list[str] |
Hydra overrides to apply |
None |
verbose |
bool |
Print the loaded config to stdout |
False |
Returns:
| Type | Description |
|---|---|
DictConfig |
Hydra configuration object |
Exceptions:
| Type | Description |
|---|---|
FileNotFoundError |
If conf directory cannot be found |
RuntimeError |
If Hydra initialization or config composition fails |
Source code in climdata/utils/config.py
def load_config(config_name="config", overrides=None, verbose=False):
"""
Load Hydra config from ./conf directory.
Uses initialize_config_dir with absolute path to ensure Hydra finds the config.
Args:
config_name (str): Name of the config file (default: "config")
overrides (list[str], optional): Hydra overrides to apply
verbose (bool): Print the loaded config to stdout
Returns:
DictConfig: Hydra configuration object
Raises:
FileNotFoundError: If conf directory cannot be found
RuntimeError: If Hydra initialization or config composition fails
"""
# Ensure conf directory exists locally
conf_dir_path = _ensure_local_conf()
conf_dir_abs = os.path.abspath(conf_dir_path)
# Reset Hydra if it was already initialized
GlobalHydra.instance().clear()
# Initialize and compose config using absolute path
with initialize_config_dir(config_dir=conf_dir_abs, version_base=None):
cfg = compose(config_name=config_name, overrides=overrides or [])
if verbose:
print(OmegaConf.to_yaml(cfg))
return cfg
Discovery¶
Browsing the catalogue without downloading anything.
climdata.explore.queries.list_available_data()
¶
Print a summary table of every dataset in the registry.
One row per provider, with its abbreviation, long name, type, spatial
coverage and native resolution. The abbreviation in the first column is the
key accepted by :func:explore, :func:inspect and the dataset= Hydra
override.
Returns:
| Type | Description |
|---|---|
None |
The table is written to stdout. |
Examples:
>>> import climdata as cd
>>> cd.list_available_data()
Source code in climdata/explore/queries.py
def list_available_data() -> None:
"""Print a summary table of every dataset in the registry.
One row per provider, with its abbreviation, long name, type, spatial
coverage and native resolution. The abbreviation in the first column is the
key accepted by :func:`explore`, :func:`inspect` and the ``dataset=`` Hydra
override.
Returns:
None: The table is written to stdout.
Example:
>>> import climdata as cd
>>> cd.list_available_data() # doctest: +SKIP
"""
col_abbr = 12
col_name = 44
col_type = 26
col_cov = 10
col_res = 14
sep = "-" * (col_abbr + col_name + col_type + col_cov + col_res + 12)
header = (
f"{'Abbr.':<{col_abbr}} | "
f"{'Long Name':<{col_name}} | "
f"{'Type':<{col_type}} | "
f"{'Coverage':<{col_cov}} | "
f"{'Resolution':<{col_res}}"
)
print("\nAVAILABLE CLIMATE DATASETS")
print(sep)
print(header)
print(sep)
for key, meta in REGISTRY.items():
print(
f"{key:<{col_abbr}} | "
f"{meta['full_name']:<{col_name}} | "
f"{meta['type']:<{col_type}} | "
f"{meta['coverage']:<{col_cov}} | "
f"{meta['resolution']:<{col_res}}"
)
print(sep)
print(f" Total: {len(REGISTRY)} datasets | Use cd.explore(dataset=...) for details.\n")
climdata.explore.queries.explore(dataset)
¶
Print a detailed profile for one dataset.
Covers type, coverage, resolution, frequency, temporal extent and source, followed by the experiments, models and variables the registry knows about. An unknown name prints the list of valid ones rather than raising.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
dataset |
str |
Dataset abbreviation as shown by :func: |
required |
Returns:
| Type | Description |
|---|---|
None |
The profile is written to stdout. |
Examples:
>>> import climdata as cd
>>> cd.explore(dataset="NEXGDDP")
Source code in climdata/explore/queries.py
def explore(dataset: str) -> None:
"""Print a detailed profile for one dataset.
Covers type, coverage, resolution, frequency, temporal extent and source,
followed by the experiments, models and variables the registry knows about.
An unknown name prints the list of valid ones rather than raising.
Args:
dataset (str): Dataset abbreviation as shown by :func:`list_available_data`
(e.g. ``"ERA5"``, ``"NEXGDDP"``, ``"CMIP"``). Matched case-insensitively.
Returns:
None: The profile is written to stdout.
Example:
>>> import climdata as cd
>>> cd.explore(dataset="NEXGDDP") # doctest: +SKIP
"""
key = resolve_dataset_key(dataset)
if key is None:
_unknown_dataset_hint(dataset)
return
meta = REGISTRY[key]
sep = "=" * 68
sep2 = "-" * 68
print(f"\n{sep}")
print(f" EXPLORING: {key} — {meta['full_name']}")
print(sep)
print(f" Type : {meta['type']}")
print(f" Coverage : {meta['coverage']}")
print(f" Resolution : {meta['resolution']}")
print(f" Frequency : {meta['frequency']}")
print(f" Time range : {meta['time_range']}")
print(f" Data source : {meta['source']}")
if meta.get("experiments"):
exps = ", ".join(meta["experiments"])
print(f"\n Experiments ({len(meta['experiments'])}): [{exps}]")
if meta.get("models"):
print(f"\n Models available ({len(meta['models'])}):")
for m in meta["models"]:
print(f" • {m}")
if meta.get("variables"):
print(f"\n Variables stored:")
print(sep2)
print(f" {'Variable':<12} {'Long Name':<46} {'Units'}")
print(sep2)
for v in meta["variables"]:
long = v # Variable short name as fallback
units = "—"
print(f" {v:<12} {long:<46} {units}")
print(sep2)
if meta.get("notes"):
print(f"\n ℹ Note: {meta['notes']}")
print(f"\n Tip: cd.inspect('{key}', variable='<var>') for unit/BASD details.\n")
climdata.explore.queries.find(variable=None, frequency=None, type_filter=None, coverage=None)
¶
Search the registry for datasets matching the given criteria.
Criteria combine with AND: a dataset is listed only if it satisfies every
argument that was supplied. variable requires an exact CF-name match;
the other three match as case-insensitive substrings. Passing no arguments
lists everything.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
variable |
str |
CF variable name that the dataset must provide
(e.g. |
None |
frequency |
str |
Temporal resolution keyword, e.g. |
None |
type_filter |
str |
Dataset type, e.g. |
None |
coverage |
str |
Coverage region, e.g. |
None |
Returns:
| Type | Description |
|---|---|
None |
Matches are written to stdout. |
Examples:
>>> import climdata as cd
>>> cd.find(variable="pr", frequency="daily")
>>> cd.find(type_filter="ESM")
Source code in climdata/explore/queries.py
def find(
variable: Optional[str] = None,
frequency: Optional[str] = None,
type_filter: Optional[str] = None,
coverage: Optional[str] = None,
) -> None:
"""Search the registry for datasets matching the given criteria.
Criteria combine with AND: a dataset is listed only if it satisfies every
argument that was supplied. ``variable`` requires an exact CF-name match;
the other three match as case-insensitive substrings. Passing no arguments
lists everything.
Args:
variable (str, optional): CF variable name that the dataset must provide
(e.g. ``"pr"``, ``"tas"``). Exact match.
frequency (str, optional): Temporal resolution keyword, e.g. ``"daily"``,
``"hourly"``. Substring match.
type_filter (str, optional): Dataset type, e.g. ``"Observation"``,
``"ESM"``. Substring match.
coverage (str, optional): Coverage region, e.g. ``"Global"``,
``"Germany"``. Substring match.
Returns:
None: Matches are written to stdout.
Example:
>>> import climdata as cd
>>> cd.find(variable="pr", frequency="daily") # doctest: +SKIP
>>> cd.find(type_filter="ESM") # doctest: +SKIP
"""
matches = []
for key, meta in REGISTRY.items():
if variable and variable not in meta.get("variables", []):
continue
if frequency and frequency.lower() not in meta["frequency"].lower():
continue
if type_filter and type_filter.lower() not in meta["type"].lower():
continue
if coverage and coverage.lower() not in meta["coverage"].lower():
continue
matches.append((key, meta))
if not matches:
print("\n No datasets matched the given criteria.\n")
return
# Build a compact query description for the header
criteria = []
if variable: criteria.append(f"variable='{variable}'")
if frequency: criteria.append(f"frequency='{frequency}'")
if type_filter: criteria.append(f"type='{type_filter}'")
if coverage: criteria.append(f"coverage='{coverage}'")
query_str = ", ".join(criteria) if criteria else "all"
print(f"\nMATCHING DATASETS (query: {query_str})")
print("-" * 68)
for i, (key, meta) in enumerate(matches, 1):
models_info = f" ({len(meta['models'])} models)" if meta.get("models") else ""
print(
f" {i:>2}. {key:<12} ({meta['type']:<26}) "
f"{meta['coverage']}, {meta['resolution']}{models_info}"
)
print("-" * 68)
print(f" {len(matches)} result(s). Use cd.explore(dataset=...) for full details.\n")
climdata.explore.queries.inspect(dataset, variable)
¶
Print variable-level metadata for one variable of one dataset.
Reports the long name, the unit BASD expects for the variable, and any
conversion note attached to it — the detail that matters before feeding data
to :mod:climdata.sdba, since a unit mismatch there fails silently rather
than loudly. Metadata comes from conf/mappings/parameters.yaml.
Unknown datasets, and variables the dataset does not carry, print the valid alternatives rather than raising.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
dataset |
str |
Dataset abbreviation (e.g. |
required |
variable |
str |
CF variable name (e.g. |
required |
Returns:
| Type | Description |
|---|---|
None |
The report is written to stdout. |
Examples:
>>> import climdata as cd
>>> cd.inspect("ERA5", variable="tp")
Source code in climdata/explore/queries.py
def inspect(dataset: str, variable: str) -> None:
"""Print variable-level metadata for one variable of one dataset.
Reports the long name, the unit BASD expects for the variable, and any
conversion note attached to it — the detail that matters before feeding data
to :mod:`climdata.sdba`, since a unit mismatch there fails silently rather
than loudly. Metadata comes from ``conf/mappings/parameters.yaml``.
Unknown datasets, and variables the dataset does not carry, print the valid
alternatives rather than raising.
Args:
dataset (str): Dataset abbreviation (e.g. ``"ERA5"``). Case-insensitive.
variable (str): CF variable name (e.g. ``"tp"``, ``"pr"``).
Returns:
None: The report is written to stdout.
Example:
>>> import climdata as cd
>>> cd.inspect("ERA5", variable="tp") # doctest: +SKIP
"""
key = resolve_dataset_key(dataset)
if key is None:
_unknown_dataset_hint(dataset)
return
meta = REGISTRY[key]
if variable not in meta.get("variables", []):
available = ", ".join(meta.get("variables", []))
print(
f"\n ✗ Variable '{variable}' is not available in {key}.\n"
f" Available variables: [{available}]\n"
)
return
# Try to load metadata from YAML config first
vmeta = _get_variable_metadata(variable)
if vmeta is None:
print(f"\n Variable '{variable}' found in {key} but has no detailed metadata yet.\n")
return
sep = "-" * 68
print(f"\n{sep}")
print(f" VARIABLE INSPECTION: {variable} in {key} ({meta['full_name']})")
print(sep)
print(f" Long name : {vmeta['long_name']}")
print(f" BASD unit : {vmeta['basd_unit']}")
if vmeta.get("basd_note"):
print(f" Conversion : ⚠ {vmeta['basd_note']}")
else:
print(" Conversion : ✓ No special conversion needed for BASD.")
print(sep + "\n")
climdata.explore.queries.list_esm_experiments(dataset)
¶
Print the experiments (scenarios) available for an ESM dataset.
Reads the experiment list straight from the registry, so it costs nothing and
needs no credentials. Non-ESM datasets, and ESM datasets whose experiments
are not declared in the configuration, print a hint pointing at the dataset
class's own get_experiment_ids().
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
dataset |
str |
Dataset abbreviation (e.g. |
required |
Returns:
| Type | Description |
|---|---|
None |
The list is written to stdout. |
Examples:
>>> import climdata as cd
>>> cd.list_esm_experiments("CMIP")
Source code in climdata/explore/queries.py
def list_esm_experiments(dataset: str) -> None:
"""Print the experiments (scenarios) available for an ESM dataset.
Reads the experiment list straight from the registry, so it costs nothing and
needs no credentials. Non-ESM datasets, and ESM datasets whose experiments
are not declared in the configuration, print a hint pointing at the dataset
class's own ``get_experiment_ids()``.
Args:
dataset (str): Dataset abbreviation (e.g. ``"CMIP"``, ``"NEXGDDP"``,
``"CMIP_W5E5"``). Case-insensitive.
Returns:
None: The list is written to stdout.
Example:
>>> import climdata as cd
>>> cd.list_esm_experiments("CMIP") # doctest: +SKIP
"""
key = resolve_dataset_key(dataset)
if key is None:
_unknown_dataset_hint(dataset)
return
meta = REGISTRY[key]
# Check if dataset is ESM-based
if "ESM" not in meta.get("type", ""):
print(f"\n ℹ {key} is not an ESM dataset (Type: {meta.get('type')}).\n")
return
experiments = meta.get("experiments", [])
if not experiments:
print(f"\n ℹ No experiment metadata found for {key}.\n"
f" Try accessing the dataset directly:\n"
f" from climdata.datasets import {key.upper()}\n"
f" ds = {key.upper()}(cfg)\n"
f" experiments = ds.get_experiment_ids()\n")
return
sep = "-" * 60
print(f"\n{sep}")
print(f" EXPERIMENTS AVAILABLE IN: {key}")
print(sep)
for i, exp in enumerate(experiments, 1):
print(f" {i}. {exp}")
print(sep + "\n")
climdata.explore.queries.list_esm_models(dataset, experiment=None)
¶
Print the ESM models available for a dataset, optionally for one experiment.
Unlike :func:list_esm_experiments, this queries the provider's live
catalogue through the dataset class, so it needs network access and can be
slow. Failures at any step — unimportable class, unreachable catalogue,
unknown experiment — are reported and swallowed rather than raised, so an
exploratory call never aborts a session.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
dataset |
str |
Dataset abbreviation (e.g. |
required |
experiment |
str |
Experiment ID to filter by, e.g.
|
None |
Returns:
| Type | Description |
|---|---|
None |
The list is written to stdout. |
Examples:
>>> import climdata as cd
>>> cd.list_esm_models("NEXGDDP")
>>> cd.list_esm_models("CMIP", experiment="ssp585")
Source code in climdata/explore/queries.py
def list_esm_models(dataset: str, experiment: Optional[str] = None) -> None:
"""Print the ESM models available for a dataset, optionally for one experiment.
Unlike :func:`list_esm_experiments`, this queries the provider's live
catalogue through the dataset class, so it needs network access and can be
slow. Failures at any step — unimportable class, unreachable catalogue,
unknown experiment — are reported and swallowed rather than raised, so an
exploratory call never aborts a session.
Args:
dataset (str): Dataset abbreviation (e.g. ``"CMIP"``, ``"NEXGDDP"``,
``"CMIP_W5E5"``). Case-insensitive.
experiment (str, optional): Experiment ID to filter by, e.g.
``"historical"`` or ``"ssp585"``. Defaults to the provider's first
experiment.
Returns:
None: The list is written to stdout.
Example:
>>> import climdata as cd
>>> cd.list_esm_models("NEXGDDP") # doctest: +SKIP
>>> cd.list_esm_models("CMIP", experiment="ssp585") # doctest: +SKIP
"""
key = resolve_dataset_key(dataset)
if key is None:
_unknown_dataset_hint(dataset)
return
meta = REGISTRY[key]
# Check if dataset is ESM-based
if "ESM" not in meta.get("type", ""):
print(f"\n ℹ {key} is not an ESM dataset (Type: {meta.get('type')}).\n")
return
# Import the dataset class dynamically
try:
if key == "CMIP":
from climdata.datasets.CMIPCloud import CMIPCloud as DatasetClass
elif key == "NEXGDDP":
from climdata.datasets.NEXGDDP import NEXGDDP as DatasetClass
elif key == "CMIP_W5E5":
from climdata.datasets.CMIP_W5E5 import CMIPW5E5 as DatasetClass
else:
print(f"\n ✗ {key} has no direct model access method yet.\n")
return
except ImportError as e:
print(f"\n ✗ Could not import {key} dataset class: {e}\n")
return
# Instantiate the dataset class to access methods
try:
from omegaconf import DictConfig
dummy_cfg = DictConfig({"experiment_id": "historical", "source_id": "dummy"})
ds_instance = DatasetClass(dummy_cfg)
except Exception as e:
print(f"\n ✗ Could not initialize {key}: {e}\n")
return
# Get experiment IDs if not provided
if experiment is None:
try:
all_experiments = ds_instance.get_experiment_ids()
experiment = all_experiments[0] if all_experiments else "historical"
except Exception as e:
print(f"\n ✗ Could not retrieve experiments: {e}\n")
return
# Get models for the specified experiment
try:
models = ds_instance.get_source_ids(experiment)
sep = "-" * 60
print(f"\n{sep}")
print(f" ESM MODELS IN: {key} (experiment: {experiment})")
print(sep)
for i, model in enumerate(models, 1):
print(f" {i}. {model}")
print(sep)
print(f" Total: {len(models)} models\n")
except Exception as e:
print(f"\n ✗ Could not retrieve models for {experiment}: {e}\n")
climdata.explore.registry.get_registry()
¶
Return the dataset registry for programmatic use.
Returns:
| Type | Description |
|---|---|
dict[str, dict] |
A shallow copy of :data: |
Examples:
>>> from climdata.explore import get_registry
>>> sorted(get_registry())[:3]
['AGRI_ISIMIP', 'CMIP', 'CMIP_W5E5']
Source code in climdata/explore/registry.py
def get_registry() -> dict[str, dict]:
"""Return the dataset registry for programmatic use.
Returns:
dict[str, dict]: A shallow copy of :data:`REGISTRY`, keyed by dataset
abbreviation. The copy protects the top level only — the per-dataset
metadata dicts are shared, so do not mutate them.
Example:
>>> from climdata.explore import get_registry
>>> sorted(get_registry())[:3] # doctest: +SKIP
['AGRI_ISIMIP', 'CMIP', 'CMIP_W5E5']
"""
return REGISTRY.copy()
climdata.explore.registry.resolve_dataset_key(name)
¶
Resolve a dataset name to its canonical registry key, case-insensitively.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
name |
str |
Dataset name in any casing, e.g. |
required |
Returns:
| Type | Description |
|---|---|
str | None |
The canonical key ( |
Examples:
>>> from climdata.explore import resolve_dataset_key
>>> resolve_dataset_key("nexgddp")
'NEXGDDP'
>>> resolve_dataset_key("not-a-dataset") is None
True
Source code in climdata/explore/registry.py
def resolve_dataset_key(name: str) -> Optional[str]:
"""Resolve a dataset name to its canonical registry key, case-insensitively.
Args:
name (str): Dataset name in any casing, e.g. ``"era5"`` or ``"ERA5"``.
Returns:
str | None: The canonical key (``"ERA5"``), or ``None`` if no dataset
matches. Callers report the miss themselves rather than catching an
exception.
Example:
>>> from climdata.explore import resolve_dataset_key
>>> resolve_dataset_key("nexgddp")
'NEXGDDP'
>>> resolve_dataset_key("not-a-dataset") is None
True
"""
name_up = name.upper()
for k in REGISTRY:
if k.upper() == name_up:
return k
return None
climdata.explore.catalog.DatasetRegistry
¶
A browsable handle on the climdata dataset catalogue.
Wraps the module-level functions of :mod:climdata.explore.queries so the
catalogue can be held in a variable, and adds dict-style lookup for callers
who want the metadata rather than a printed report. Echoing the object at a
REPL prints the summary table.
Attributes:
| Name | Type | Description |
|---|---|---|
keys |
Bound method returning the canonical dataset abbreviations. |
Examples:
>>> from climdata.explore import DatasetRegistry
>>> reg = DatasetRegistry()
>>> reg
AVAILABLE CLIMATE DATASETS ...
>>> reg.explore("ERA5")
>>> reg["era5"]["resolution"] # dict-style, case-insensitive
'0.25°'
Source code in climdata/explore/catalog.py
class DatasetRegistry:
"""A browsable handle on the climdata dataset catalogue.
Wraps the module-level functions of :mod:`climdata.explore.queries` so the
catalogue can be held in a variable, and adds dict-style lookup for callers
who want the metadata rather than a printed report. Echoing the object at a
REPL prints the summary table.
Attributes:
keys: Bound method returning the canonical dataset abbreviations.
Example:
>>> from climdata.explore import DatasetRegistry
>>> reg = DatasetRegistry()
>>> reg # doctest: +SKIP
AVAILABLE CLIMATE DATASETS ...
>>> reg.explore("ERA5") # doctest: +SKIP
>>> reg["era5"]["resolution"] # dict-style, case-insensitive
'0.25°'
"""
def __init__(self) -> None:
"""Bind the process-wide registry snapshot."""
self._registry = REGISTRY
# ------------------------------------------------------------------
def __repr__(self) -> str:
"""Render the catalogue as a summary table.
Returns:
str: Abbreviation, long name and type for every dataset.
"""
col_abbr = 12
col_name = 44
col_type = 26
sep = "-" * (col_abbr + col_name + col_type + 6)
header = f"{'Abbr.':<{col_abbr}} | {'Long Name':<{col_name}} | {'Type':<{col_type}}"
rows = [
f"{k:<{col_abbr}} | {v['full_name']:<{col_name}} | {v['type']:<{col_type}}"
for k, v in self._registry.items()
]
return (
"\nAVAILABLE CLIMATE DATASETS\n"
+ sep + "\n"
+ header + "\n"
+ sep + "\n"
+ "\n".join(rows) + "\n"
+ sep
)
# Delegate to module-level functions
def list_available_data(self) -> None:
"""Print the summary table. See :func:`climdata.explore.list_available_data`."""
list_available_data()
def explore(self, dataset: str) -> None:
"""Print a dataset profile. See :func:`climdata.explore.explore`.
Args:
dataset (str): Dataset abbreviation, case-insensitive.
"""
explore(dataset)
def find(self, **kwargs) -> None:
"""Search the catalogue. See :func:`climdata.explore.find`.
Args:
**kwargs: Any of ``variable``, ``frequency``, ``type_filter`` or
``coverage``.
"""
find(**kwargs)
def inspect(self, dataset: str, variable: str) -> None:
"""Print variable metadata. See :func:`climdata.explore.inspect`.
Args:
dataset (str): Dataset abbreviation, case-insensitive.
variable (str): CF variable name.
"""
inspect(dataset, variable)
# Allow dict-like access for power users
def __getitem__(self, key: str) -> dict:
"""Return the raw metadata dict for one dataset.
Args:
key (str): Dataset abbreviation, case-insensitive.
Returns:
dict: The registry entry, as described in
:func:`climdata.explore.get_registry`.
Raises:
KeyError: If no dataset matches ``key``.
"""
resolved = resolve_dataset_key(key)
if resolved is None:
raise KeyError(f"Dataset '{key}' not found in registry.")
return self._registry[resolved]
def keys(self):
"""Return the canonical dataset abbreviations.
Returns:
KeysView[str]: The registry keys, e.g. ``"ERA5"``, ``"NEXGDDP"``.
"""
return self._registry.keys()
__getitem__(self, key)
special
¶
Return the raw metadata dict for one dataset.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
key |
str |
Dataset abbreviation, case-insensitive. |
required |
Returns:
| Type | Description |
|---|---|
dict |
The registry entry, as described in
:func: |
Exceptions:
| Type | Description |
|---|---|
KeyError |
If no dataset matches |
Source code in climdata/explore/catalog.py
def __getitem__(self, key: str) -> dict:
"""Return the raw metadata dict for one dataset.
Args:
key (str): Dataset abbreviation, case-insensitive.
Returns:
dict: The registry entry, as described in
:func:`climdata.explore.get_registry`.
Raises:
KeyError: If no dataset matches ``key``.
"""
resolved = resolve_dataset_key(key)
if resolved is None:
raise KeyError(f"Dataset '{key}' not found in registry.")
return self._registry[resolved]
__init__(self)
special
¶
Bind the process-wide registry snapshot.
Source code in climdata/explore/catalog.py
def __init__(self) -> None:
"""Bind the process-wide registry snapshot."""
self._registry = REGISTRY
__repr__(self)
special
¶
Render the catalogue as a summary table.
Returns:
| Type | Description |
|---|---|
str |
Abbreviation, long name and type for every dataset. |
Source code in climdata/explore/catalog.py
def __repr__(self) -> str:
"""Render the catalogue as a summary table.
Returns:
str: Abbreviation, long name and type for every dataset.
"""
col_abbr = 12
col_name = 44
col_type = 26
sep = "-" * (col_abbr + col_name + col_type + 6)
header = f"{'Abbr.':<{col_abbr}} | {'Long Name':<{col_name}} | {'Type':<{col_type}}"
rows = [
f"{k:<{col_abbr}} | {v['full_name']:<{col_name}} | {v['type']:<{col_type}}"
for k, v in self._registry.items()
]
return (
"\nAVAILABLE CLIMATE DATASETS\n"
+ sep + "\n"
+ header + "\n"
+ sep + "\n"
+ "\n".join(rows) + "\n"
+ sep
)
explore(self, dataset)
¶
Print a dataset profile. See :func:climdata.explore.explore.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
dataset |
str |
Dataset abbreviation, case-insensitive. |
required |
Source code in climdata/explore/catalog.py
def explore(self, dataset: str) -> None:
"""Print a dataset profile. See :func:`climdata.explore.explore`.
Args:
dataset (str): Dataset abbreviation, case-insensitive.
"""
explore(dataset)
find(self, **kwargs)
¶
Search the catalogue. See :func:climdata.explore.find.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
**kwargs |
Any of |
{} |
Source code in climdata/explore/catalog.py
def find(self, **kwargs) -> None:
"""Search the catalogue. See :func:`climdata.explore.find`.
Args:
**kwargs: Any of ``variable``, ``frequency``, ``type_filter`` or
``coverage``.
"""
find(**kwargs)
inspect(self, dataset, variable)
¶
Print variable metadata. See :func:climdata.explore.inspect.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
dataset |
str |
Dataset abbreviation, case-insensitive. |
required |
variable |
str |
CF variable name. |
required |
Source code in climdata/explore/catalog.py
def inspect(self, dataset: str, variable: str) -> None:
"""Print variable metadata. See :func:`climdata.explore.inspect`.
Args:
dataset (str): Dataset abbreviation, case-insensitive.
variable (str): CF variable name.
"""
inspect(dataset, variable)
keys(self)
¶
Return the canonical dataset abbreviations.
Returns:
| Type | Description |
|---|---|
KeysView[str] |
The registry keys, e.g. |
Source code in climdata/explore/catalog.py
def keys(self):
"""Return the canonical dataset abbreviations.
Returns:
KeysView[str]: The registry keys, e.g. ``"ERA5"``, ``"NEXGDDP"``.
"""
return self._registry.keys()
list_available_data(self)
¶
Print the summary table. See :func:climdata.explore.list_available_data.
Source code in climdata/explore/catalog.py
def list_available_data(self) -> None:
"""Print the summary table. See :func:`climdata.explore.list_available_data`."""
list_available_data()
Providers¶
Each class can be used directly when the workflow wrapper is more than you need.
All follow a fetch → load → extract lifecycle, with the per-provider
deviations noted in each docstring.
Observations, global¶
climdata.datasets.MSWX.MSWXmirror
¶
Download and load MSWX daily data, one variable at a time.
The order of operations is inverted relative to the other providers: call
:meth:extract first to record the region of interest, then
:meth:load, which downloads what is missing and applies the subset
per-file during opening. Loading before extracting works but pulls whole
global frames into the graph.
One call handles one variable, because each variable lives in its own Drive
folder. :class:~climdata.utils.wrapper_workflow.ClimateExtractor loops over
cfg.variables and merges the results.
Attributes:
| Name | Type | Description |
|---|---|---|
cfg |
DictConfig |
Hydra configuration. |
variables |
list[str] |
|
dataset |
xr.Dataset | None |
The loaded dataset, set by :meth: |
Examples:
>>> mswx = MSWXmirror(cfg)
>>> mswx.extract(box={"lat_min": 47, "lat_max": 55,
... "lon_min": 5, "lon_max": 15})
>>> ds = mswx.load("pr")
Source code in climdata/datasets/MSWX.py
class MSWXmirror:
"""Download and load MSWX daily data, one variable at a time.
The order of operations is inverted relative to the other providers: call
:meth:`extract` *first* to record the region of interest, then
:meth:`load`, which downloads what is missing and applies the subset
per-file during opening. Loading before extracting works but pulls whole
global frames into the graph.
One call handles one variable, because each variable lives in its own Drive
folder. :class:`~climdata.utils.wrapper_workflow.ClimateExtractor` loops over
``cfg.variables`` and merges the results.
Attributes:
cfg (DictConfig): Hydra configuration.
variables (list[str]): ``cfg.variables``, used to shape long-form output.
dataset (xr.Dataset | None): The loaded dataset, set by :meth:`load`.
Example:
>>> mswx = MSWXmirror(cfg) # doctest: +SKIP
>>> mswx.extract(box={"lat_min": 47, "lat_max": 55, # doctest: +SKIP
... "lon_min": 5, "lon_max": 15})
>>> ds = mswx.load("pr") # doctest: +SKIP
"""
def __init__(self, cfg: DictConfig):
"""Bind a configuration.
Args:
cfg (DictConfig): Configuration with ``variables``, ``data_dir``,
``dataset``, ``time_range`` and ``dsinfo.MSWX``.
Raises:
ImportError: If the Google API client libraries are not installed.
"""
if not _GOOGLE_AVAILABLE:
raise ImportError(
"MSWX requires google-api-python-client and google-auth. "
"Install with: pip install google-api-python-client google-auth"
)
self.cfg = cfg
self.dataset = None
self.variables = cfg.variables
self.files = []
self._extract_mode = None
self._extract_params = None
def _load_opts(self):
"""Resolve the optional ``cfg.load`` block into concrete open options.
Every key is optional, so a config written before ``load:`` existed keeps
working. The default chunking pins ``lat`` and ``lon`` to their full
extent: MSWX files are stored in roughly 8x32 blocks, which would give
about 25 000 tiny chunks per global frame and stall Dask's graph
optimisation long before any data moves. Spatial dimensions the caller
leaves unspecified are pinned the same way.
Returns:
dict: Keys ``engine``, ``parallel``, ``chunks``, ``fix_coords`` and
``eager``. ``eager`` is the inverse of ``load.dask.enabled``, so
without a Dask cluster the dataset is materialised into memory at the
end of :meth:`load` rather than left lazy.
"""
defaults = {
"engine": "h5netcdf",
"parallel": True,
# One chunk per global frame. Leaving lat/lon unset makes xarray
# inherit the file's native ~(8, 32) on-disk chunking, i.e. ~25k
# tiny chunks per frame (~46M tasks/global-year) that stall dask's
# graph optimisation. Pin the spatial dims to the whole extent.
"chunks": {"time": 1, "lat": -1, "lon": -1},
"fix_coords": True,
}
opts = dict(defaults)
chunks = OmegaConf.select(self.cfg, "load.chunks", default="__missing__")
if chunks != "__missing__":
# Allow either a mapping of dim -> size, or the string "auto".
if isinstance(chunks, str):
opts["chunks"] = chunks
else:
chunks = OmegaConf.to_container(chunks, resolve=True)
# Any spatial dim the caller didn't pin defaults to the whole
# extent (-1) rather than the tiny native on-disk chunking.
for dim in ("lat", "lon"):
chunks.setdefault(dim, -1)
opts["chunks"] = chunks
for key in ("engine", "parallel", "fix_coords"):
val = OmegaConf.select(self.cfg, f"load.{key}", default="__missing__")
if val != "__missing__":
opts[key] = val
# When the Dask cluster is disabled, treat extraction as eager: the
# dataset is materialised into memory (NumPy) at the end of load() so
# nothing downstream stays lazily dask-backed. `load.dask.enabled`
# defaults to False (opt-in cluster), matching the wrapper's default.
opts["eager"] = not bool(
OmegaConf.select(self.cfg, "load.dask.enabled", default=False)
)
return opts
def _fix_coords(self, ds):
"""Put latitude in ascending order and longitude on -180..180.
A descending latitude axis is reversed by slicing rather than
``sortby``. ``sortby`` adds an argsort plus a fancy-index reindex layer
*per file*; across the ~1800 daily files a multi-year request opens, that
alone stalls the Dask client in graph optimisation. A ``[::-1]`` reversal
is exact for a monotone axis and adds essentially no graph. Genuinely
non-monotone axes still fall back to ``sortby``.
Args:
ds (xr.Dataset | xr.DataArray): Object with CF-identifiable latitude
and longitude coordinates.
Returns:
xr.Dataset | xr.DataArray: Same type as the input, reoriented.
"""
lat_name = ds.cf["latitude"].name
lat = ds[lat_name]
# A single .values read of the (tiny) 1-D coord; cheaper than argsort.
lat_vals = lat.values
if lat_vals[0] > lat_vals[-1]: # descending -> flip
ds = ds.isel({lat_name: slice(None, None, -1)})
elif not (lat_vals[1:] >= lat_vals[:-1]).all():
ds = ds.sortby(lat_name) # non-monotone: fall back
lon_name = ds.cf["longitude"].name
lon = ds[lon_name]
# Only convert if longitudes are in the [0, 360] convention
if lon.max().item() > 180:
ds = ds.assign_coords(
{lon_name: ((lon + 180) % 360) - 180}
)
ds = ds.sortby(lon_name)
return ds
def fetch(self, folder_id: str, variable: str):
"""Download the daily MSWX files for one variable, skipping what exists.
Expands the configured date range into one expected filename per day
(``YYYYDDD.nc``), checks which are already under
``<data_dir>/<DATASET>/<VARIABLE>/``, and downloads only the rest — so an
interrupted run resumes rather than restarting. Google credentials are
only required when something is actually missing.
Args:
folder_id (str): Google Drive folder ID for this variable, from
``cfg.dsinfo.MSWX.variables[<var>].folder_id``.
variable (str): CF variable name, e.g. ``"tasmax"``.
Returns:
list[str]: Basenames now present locally. May be shorter than the
requested range if Drive does not hold every day.
Raises:
ValueError: If files are missing and no service-account key is
configured.
FileNotFoundError: If the configured key file does not exist.
"""
start = datetime.fromisoformat(self.cfg.time_range.start_date)
end = datetime.fromisoformat(self.cfg.time_range.end_date)
expected_files = []
current = start
while current <= end:
doy = current.timetuple().tm_yday
basename = f"{current.year}{doy:03d}.nc"
expected_files.append(basename)
current += timedelta(days=1)
output_dir = self.cfg.data_dir
local_files, missing_files = [], []
for basename in expected_files:
local_path = os.path.join(output_dir, self.cfg.dataset.upper(), variable.upper(), basename)
if os.path.exists(local_path):
local_files.append(basename)
else:
missing_files.append(basename)
if not missing_files:
print(f"✅ All {len(expected_files)} {variable} files already exist locally.")
return local_files
print(f"📂 {len(local_files)} exist, {len(missing_files)} missing — fetching {variable} from Drive...")
SCOPES = ['https://www.googleapis.com/auth/drive.readonly']
key_file = self.cfg.dsinfo.MSWX.params.google_service_account
# Hydra/YAML can hand back the literal string "None" as well as a real null.
if key_file in (None, "None", ""):
raise ValueError(
"MSWX downloads need a Google service-account key. Add the override "
"'dsinfo.MSWX.params.google_service_account=/path/to/service.json' "
"(note the key is case-sensitive and must not be prefixed with '+')."
)
key_file = os.path.expanduser(str(key_file))
if not os.path.exists(key_file):
raise FileNotFoundError(f"MSWX service-account key not found: {key_file}")
creds = service_account.Credentials.from_service_account_file(
key_file, scopes=SCOPES
)
service = build('drive', 'v3', credentials=creds)
drive_files = list_drive_files(folder_id, service)
valid_filenames = set(missing_files)
files_to_download = [f for f in drive_files if f['name'] in valid_filenames]
if not files_to_download:
print(f"⚠️ No {variable} files found in Drive for requested dates.")
return local_files
for file in files_to_download:
filename = file['name']
local_path = os.path.join(output_dir, self.cfg.dataset.upper(), variable.upper(), filename)
os.makedirs(os.path.dirname(local_path), exist_ok=True)
print(f"⬇️ Downloading {filename} ...")
download_drive_file(file['id'], local_path, service)
local_files.append(filename)
return local_files
def _extract_preprocess(self, ds):
"""Subset one daily file as it is opened by ``open_mfdataset``.
Runs inside the ``preprocess`` hook, so each global frame is cut to the
region of interest before it joins the concatenation — the difference
between a graph over global arrays and one over the requested box.
Args:
ds (xr.Dataset): One day's global data, as just opened.
Returns:
xr.Dataset: The subset recorded by :meth:`extract`, or ``ds``
unchanged if no extraction mode was set.
"""
# Fix coords first (can be disabled via cfg.load.fix_coords for speed
# when the source grid is already ascending-lat / [0,360]-lon).
if getattr(self, "_fix_coords_enabled", True):
ds = self._fix_coords(ds)
# ---- Point extraction ----
if self._extract_mode == "point":
lon, lat, buffer_deg = self._extract_params
if buffer_deg > 0:
ds = ds.sel(
lon=slice(lon-buffer_deg, lon+buffer_deg),
lat=slice(lat-buffer_deg, lat+buffer_deg),
).mean(["lat", "lon"])
else:
ds = ds.sel(lon=lon, lat=lat, method="nearest")
# ---- Box extraction ----
elif self._extract_mode == "box":
box = self._extract_params
ds = ds.sel(
lon=slice(box["lon_min"], box["lon_max"]),
lat=slice(box["lat_min"], box["lat_max"]),
)
# ---- Shapefile extraction ----
elif self._extract_mode == "shapefile":
gdf = self._extract_params
# Suppose your dataset uses 'lon' and 'lat' as coordinates
ds = ds.rio.set_spatial_dims(x_dim="lon", y_dim="lat")
# Also ensure CRS is set
ds = ds.rio.write_crs("EPSG:4326", inplace=True)
clipped_list = []
for geom in gdf.geometry:
clipped = ds.rio.clip([mapping(geom)], gdf.crs, drop=True)
clipped_list.append(clipped)
ds = xr.concat(clipped_list, dim="geom_id")
return ds
def extract(self, *, point=None, box=None, shapefile=None, buffer_km=0.0):
"""Record the region of interest for :meth:`load` to apply.
Nothing is read here — the instruction is stored and executed per-file
inside :meth:`load`, which is why this should be called first. The three
modes are mutually exclusive; the first supplied wins, in the order
``point``, ``box``, ``shapefile``.
Args:
point (tuple[float, float], optional): ``(lon, lat)`` in degrees,
longitude first.
box (dict, optional): Bounding box with keys ``lon_min``, ``lon_max``,
``lat_min``, ``lat_max``.
shapefile (str | geopandas.GeoDataFrame, optional): Polygon(s) to clip
to, as a path or an in-memory frame.
buffer_km (float): For ``point``, half-width of a box that is
area-averaged instead of taking the nearest cell, converted at a
flat 111 km per degree; for ``shapefile``, a dilation applied in
Web Mercator. Defaults to ``0.0``.
Returns:
MSWXmirror: ``self``, so the call can be chained into :meth:`load`.
Raises:
ValueError: If none of ``point``, ``box`` or ``shapefile`` was given.
"""
if point is not None:
lon, lat = point
buffer_deg = buffer_km / 111.0
self._extract_mode = "point"
self._extract_params = (lon, lat, buffer_deg)
elif box is not None:
self._extract_mode = "box"
self._extract_params = box
elif shapefile is not None:
if isinstance(shapefile, str):
gdf = gpd.read_file(shapefile)
else:
gdf = shapefile
if buffer_km > 0:
gdf = gdf.to_crs(epsg=3857)
gdf["geometry"] = gdf.buffer(buffer_km * 1000)
gdf = gdf.to_crs(epsg=4326)
self._extract_mode = "shapefile"
self._extract_params = gdf
else:
raise ValueError("Must provide point, box, or shapefile.")
return self
def load(self, variable: str):
"""Download what is missing, then open every daily file as one dataset.
Calls :meth:`fetch` for this variable, then opens the files with
``open_mfdataset``, concatenating along ``time``. The per-file
``preprocess`` hook fixes coordinates, applies the subset recorded by
:meth:`extract`, and renames the MSWX-internal variable to its CF name.
Whether the result is lazy depends on ``cfg.load.dask.enabled``: with a
Dask cluster the dataset stays lazy, without one it is materialised into
memory before returning.
Args:
variable (str): CF variable name, as declared in ``cfg.variables``.
Returns:
xr.Dataset: The concatenated dataset, also stored on :attr:`dataset`.
Raises:
RuntimeError: If no files could be found for the variable, or if
opening them failed.
KeyError: If the variable has no entry in ``cfg.dsinfo.MSWX.variables``.
"""
# Get folder ID and list of files
folder_id = self.cfg.dsinfo[self.cfg.dataset.upper()]["variables"][variable]["folder_id"]
files = self.fetch(folder_id, variable)
if not files:
raise RuntimeError(f"No files found for variable '{variable}' in Drive or local directory.")
# Full paths
file_paths = [
os.path.join(self.cfg.data_dir, self.cfg.dataset.upper(), variable.upper(), f)
for f in files
]
# MSWX internal variable name
varname = self.cfg.dsinfo[self.cfg.dataset.upper()].variables[variable].name
# Resolve open/preprocess options from cfg.load (with defaults).
opts = self._load_opts()
self._fix_coords_enabled = opts["fix_coords"]
# Optional: preprocess each file (e.g., rename variable)
def preprocess(ds):
ds = self._extract_preprocess(ds)
return ds[[varname]].rename({varname: variable})
# Open all files as a single, lazily-loaded (Dask-backed) dataset.
# `chunks` triggers xarray's Dask backend so data stays on disk until
# explicitly computed; `parallel=True` opens files concurrently.
try:
dset = xr.open_mfdataset(
file_paths,
combine="nested",
concat_dim="time",
parallel=opts["parallel"], # uses Dask for parallel file opening
engine=opts["engine"], # e.g. h5netcdf, faster than netcdf4
chunks=opts["chunks"], # Dask chunks -> lazy access
preprocess=preprocess,
)
except Exception as e:
raise RuntimeError(f"Failed to load dataset for variable '{variable}': {e}")
# Ensure consistent dimension order
if self._extract_mode != "point":
dset = dset.transpose("time", "lat", "lon")
# Eager mode (Dask cluster disabled): pull the data into memory now so
# the returned dataset is NumPy-backed, not lazily Dask-backed. The
# bounded chunking above keeps this a straightforward read rather than a
# graph-optimisation stall.
if opts["eager"]:
dset = dset.load()
# Store in the class
self.dataset = dset
return dset
def to_zarr(self, zarr_filename: str):
"""Write the loaded dataset to a Zarr store under ``data/MSWX/``.
Args:
zarr_filename (str): Store name, resolved relative to ``data/MSWX``.
Returns:
None
Raises:
ValueError: If :meth:`load` has not been called.
"""
if self.dataset is None:
raise ValueError("No dataset loaded. Call `load()` first.")
var_name = self.dataset.name
if var_name == 'pr':
self.dataset.attrs['units'] = 'mm/day'
elif var_name in ['tas', 'tasmax', 'tasmin']:
self.dataset.attrs['units'] = 'degC'
zarr_path = os.path.join("data/MSWX", zarr_filename)
os.makedirs(os.path.dirname(zarr_path), exist_ok=True)
print(f"💾 Saving {var_name} to Zarr: {zarr_path}")
self.dataset.to_zarr(zarr_path, mode="w")
def _format(self, df):
"""Reshape a wide frame into climdata's long output form.
Melts the variable columns into ``variable``/``value`` pairs, attaches
each variable's units and stamps the provider name.
Args:
df (pd.DataFrame): Wide frame from ``ds.to_dataframe().reset_index()``.
Returns:
pd.DataFrame: Long frame with columns ``source``, ``time``, ``lat``,
``lon``, ``variable``, ``value``, ``units`` — those of them present.
"""
value_vars = [v for v in self.variables if v in df.columns]
id_vars = [c for c in df.columns if c not in value_vars]
df_long = df.melt(
id_vars=id_vars,
value_vars=value_vars,
var_name="variable",
value_name="value",
)
df_long["units"] = df_long["variable"].map(
lambda v: self.dataset[v].attrs.get("units", "unknown")
if v in self.dataset.data_vars
else "unknown"
)
df_long["source"] = self.cfg.dataset
cols = [
"source",
"table",
"time",
"lat",
"lon",
"variable",
"value",
"units",
]
df_long = df_long[[c for c in cols if c in df_long.columns]]
return df_long
def save_csv(self, filename):
"""Write the loaded dataset to CSV in long form.
Does nothing if no dataset is loaded.
Args:
filename (str): Destination path. The parent directory must exist.
Returns:
None
"""
if self.dataset is not None:
df = self.dataset.to_dataframe().reset_index()
df = self._format(df)
df.to_csv(filename, index=False)
# print(f"Saved CSV to {filename}")
__init__(self, cfg)
special
¶
Bind a configuration.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
cfg |
DictConfig |
Configuration with |
required |
Exceptions:
| Type | Description |
|---|---|
ImportError |
If the Google API client libraries are not installed. |
Source code in climdata/datasets/MSWX.py
def __init__(self, cfg: DictConfig):
"""Bind a configuration.
Args:
cfg (DictConfig): Configuration with ``variables``, ``data_dir``,
``dataset``, ``time_range`` and ``dsinfo.MSWX``.
Raises:
ImportError: If the Google API client libraries are not installed.
"""
if not _GOOGLE_AVAILABLE:
raise ImportError(
"MSWX requires google-api-python-client and google-auth. "
"Install with: pip install google-api-python-client google-auth"
)
self.cfg = cfg
self.dataset = None
self.variables = cfg.variables
self.files = []
self._extract_mode = None
self._extract_params = None
extract(self, *, point=None, box=None, shapefile=None, buffer_km=0.0)
¶
Record the region of interest for :meth:load to apply.
Nothing is read here — the instruction is stored and executed per-file
inside :meth:load, which is why this should be called first. The three
modes are mutually exclusive; the first supplied wins, in the order
point, box, shapefile.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
point |
tuple[float, float] |
|
None |
box |
dict |
Bounding box with keys |
None |
shapefile |
str | geopandas.GeoDataFrame |
Polygon(s) to clip to, as a path or an in-memory frame. |
None |
buffer_km |
float |
For |
0.0 |
Returns:
| Type | Description |
|---|---|
MSWXmirror |
|
Exceptions:
| Type | Description |
|---|---|
ValueError |
If none of |
Source code in climdata/datasets/MSWX.py
def extract(self, *, point=None, box=None, shapefile=None, buffer_km=0.0):
"""Record the region of interest for :meth:`load` to apply.
Nothing is read here — the instruction is stored and executed per-file
inside :meth:`load`, which is why this should be called first. The three
modes are mutually exclusive; the first supplied wins, in the order
``point``, ``box``, ``shapefile``.
Args:
point (tuple[float, float], optional): ``(lon, lat)`` in degrees,
longitude first.
box (dict, optional): Bounding box with keys ``lon_min``, ``lon_max``,
``lat_min``, ``lat_max``.
shapefile (str | geopandas.GeoDataFrame, optional): Polygon(s) to clip
to, as a path or an in-memory frame.
buffer_km (float): For ``point``, half-width of a box that is
area-averaged instead of taking the nearest cell, converted at a
flat 111 km per degree; for ``shapefile``, a dilation applied in
Web Mercator. Defaults to ``0.0``.
Returns:
MSWXmirror: ``self``, so the call can be chained into :meth:`load`.
Raises:
ValueError: If none of ``point``, ``box`` or ``shapefile`` was given.
"""
if point is not None:
lon, lat = point
buffer_deg = buffer_km / 111.0
self._extract_mode = "point"
self._extract_params = (lon, lat, buffer_deg)
elif box is not None:
self._extract_mode = "box"
self._extract_params = box
elif shapefile is not None:
if isinstance(shapefile, str):
gdf = gpd.read_file(shapefile)
else:
gdf = shapefile
if buffer_km > 0:
gdf = gdf.to_crs(epsg=3857)
gdf["geometry"] = gdf.buffer(buffer_km * 1000)
gdf = gdf.to_crs(epsg=4326)
self._extract_mode = "shapefile"
self._extract_params = gdf
else:
raise ValueError("Must provide point, box, or shapefile.")
return self
fetch(self, folder_id, variable)
¶
Download the daily MSWX files for one variable, skipping what exists.
Expands the configured date range into one expected filename per day
(YYYYDDD.nc), checks which are already under
<data_dir>/<DATASET>/<VARIABLE>/, and downloads only the rest — so an
interrupted run resumes rather than restarting. Google credentials are
only required when something is actually missing.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
folder_id |
str |
Google Drive folder ID for this variable, from
|
required |
variable |
str |
CF variable name, e.g. |
required |
Returns:
| Type | Description |
|---|---|
list[str] |
Basenames now present locally. May be shorter than the requested range if Drive does not hold every day. |
Exceptions:
| Type | Description |
|---|---|
ValueError |
If files are missing and no service-account key is configured. |
FileNotFoundError |
If the configured key file does not exist. |
Source code in climdata/datasets/MSWX.py
def fetch(self, folder_id: str, variable: str):
"""Download the daily MSWX files for one variable, skipping what exists.
Expands the configured date range into one expected filename per day
(``YYYYDDD.nc``), checks which are already under
``<data_dir>/<DATASET>/<VARIABLE>/``, and downloads only the rest — so an
interrupted run resumes rather than restarting. Google credentials are
only required when something is actually missing.
Args:
folder_id (str): Google Drive folder ID for this variable, from
``cfg.dsinfo.MSWX.variables[<var>].folder_id``.
variable (str): CF variable name, e.g. ``"tasmax"``.
Returns:
list[str]: Basenames now present locally. May be shorter than the
requested range if Drive does not hold every day.
Raises:
ValueError: If files are missing and no service-account key is
configured.
FileNotFoundError: If the configured key file does not exist.
"""
start = datetime.fromisoformat(self.cfg.time_range.start_date)
end = datetime.fromisoformat(self.cfg.time_range.end_date)
expected_files = []
current = start
while current <= end:
doy = current.timetuple().tm_yday
basename = f"{current.year}{doy:03d}.nc"
expected_files.append(basename)
current += timedelta(days=1)
output_dir = self.cfg.data_dir
local_files, missing_files = [], []
for basename in expected_files:
local_path = os.path.join(output_dir, self.cfg.dataset.upper(), variable.upper(), basename)
if os.path.exists(local_path):
local_files.append(basename)
else:
missing_files.append(basename)
if not missing_files:
print(f"✅ All {len(expected_files)} {variable} files already exist locally.")
return local_files
print(f"📂 {len(local_files)} exist, {len(missing_files)} missing — fetching {variable} from Drive...")
SCOPES = ['https://www.googleapis.com/auth/drive.readonly']
key_file = self.cfg.dsinfo.MSWX.params.google_service_account
# Hydra/YAML can hand back the literal string "None" as well as a real null.
if key_file in (None, "None", ""):
raise ValueError(
"MSWX downloads need a Google service-account key. Add the override "
"'dsinfo.MSWX.params.google_service_account=/path/to/service.json' "
"(note the key is case-sensitive and must not be prefixed with '+')."
)
key_file = os.path.expanduser(str(key_file))
if not os.path.exists(key_file):
raise FileNotFoundError(f"MSWX service-account key not found: {key_file}")
creds = service_account.Credentials.from_service_account_file(
key_file, scopes=SCOPES
)
service = build('drive', 'v3', credentials=creds)
drive_files = list_drive_files(folder_id, service)
valid_filenames = set(missing_files)
files_to_download = [f for f in drive_files if f['name'] in valid_filenames]
if not files_to_download:
print(f"⚠️ No {variable} files found in Drive for requested dates.")
return local_files
for file in files_to_download:
filename = file['name']
local_path = os.path.join(output_dir, self.cfg.dataset.upper(), variable.upper(), filename)
os.makedirs(os.path.dirname(local_path), exist_ok=True)
print(f"⬇️ Downloading {filename} ...")
download_drive_file(file['id'], local_path, service)
local_files.append(filename)
return local_files
load(self, variable)
¶
Download what is missing, then open every daily file as one dataset.
Calls :meth:fetch for this variable, then opens the files with
open_mfdataset, concatenating along time. The per-file
preprocess hook fixes coordinates, applies the subset recorded by
:meth:extract, and renames the MSWX-internal variable to its CF name.
Whether the result is lazy depends on cfg.load.dask.enabled: with a
Dask cluster the dataset stays lazy, without one it is materialised into
memory before returning.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
variable |
str |
CF variable name, as declared in |
required |
Returns:
| Type | Description |
|---|---|
xr.Dataset |
The concatenated dataset, also stored on :attr: |
Exceptions:
| Type | Description |
|---|---|
RuntimeError |
If no files could be found for the variable, or if opening them failed. |
KeyError |
If the variable has no entry in |
Source code in climdata/datasets/MSWX.py
def load(self, variable: str):
"""Download what is missing, then open every daily file as one dataset.
Calls :meth:`fetch` for this variable, then opens the files with
``open_mfdataset``, concatenating along ``time``. The per-file
``preprocess`` hook fixes coordinates, applies the subset recorded by
:meth:`extract`, and renames the MSWX-internal variable to its CF name.
Whether the result is lazy depends on ``cfg.load.dask.enabled``: with a
Dask cluster the dataset stays lazy, without one it is materialised into
memory before returning.
Args:
variable (str): CF variable name, as declared in ``cfg.variables``.
Returns:
xr.Dataset: The concatenated dataset, also stored on :attr:`dataset`.
Raises:
RuntimeError: If no files could be found for the variable, or if
opening them failed.
KeyError: If the variable has no entry in ``cfg.dsinfo.MSWX.variables``.
"""
# Get folder ID and list of files
folder_id = self.cfg.dsinfo[self.cfg.dataset.upper()]["variables"][variable]["folder_id"]
files = self.fetch(folder_id, variable)
if not files:
raise RuntimeError(f"No files found for variable '{variable}' in Drive or local directory.")
# Full paths
file_paths = [
os.path.join(self.cfg.data_dir, self.cfg.dataset.upper(), variable.upper(), f)
for f in files
]
# MSWX internal variable name
varname = self.cfg.dsinfo[self.cfg.dataset.upper()].variables[variable].name
# Resolve open/preprocess options from cfg.load (with defaults).
opts = self._load_opts()
self._fix_coords_enabled = opts["fix_coords"]
# Optional: preprocess each file (e.g., rename variable)
def preprocess(ds):
ds = self._extract_preprocess(ds)
return ds[[varname]].rename({varname: variable})
# Open all files as a single, lazily-loaded (Dask-backed) dataset.
# `chunks` triggers xarray's Dask backend so data stays on disk until
# explicitly computed; `parallel=True` opens files concurrently.
try:
dset = xr.open_mfdataset(
file_paths,
combine="nested",
concat_dim="time",
parallel=opts["parallel"], # uses Dask for parallel file opening
engine=opts["engine"], # e.g. h5netcdf, faster than netcdf4
chunks=opts["chunks"], # Dask chunks -> lazy access
preprocess=preprocess,
)
except Exception as e:
raise RuntimeError(f"Failed to load dataset for variable '{variable}': {e}")
# Ensure consistent dimension order
if self._extract_mode != "point":
dset = dset.transpose("time", "lat", "lon")
# Eager mode (Dask cluster disabled): pull the data into memory now so
# the returned dataset is NumPy-backed, not lazily Dask-backed. The
# bounded chunking above keeps this a straightforward read rather than a
# graph-optimisation stall.
if opts["eager"]:
dset = dset.load()
# Store in the class
self.dataset = dset
return dset
save_csv(self, filename)
¶
Write the loaded dataset to CSV in long form.
Does nothing if no dataset is loaded.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
filename |
str |
Destination path. The parent directory must exist. |
required |
Returns:
| Type | Description |
|---|---|
None |
Source code in climdata/datasets/MSWX.py
def save_csv(self, filename):
"""Write the loaded dataset to CSV in long form.
Does nothing if no dataset is loaded.
Args:
filename (str): Destination path. The parent directory must exist.
Returns:
None
"""
if self.dataset is not None:
df = self.dataset.to_dataframe().reset_index()
df = self._format(df)
df.to_csv(filename, index=False)
# print(f"Saved CSV to {filename}")
to_zarr(self, zarr_filename)
¶
Write the loaded dataset to a Zarr store under data/MSWX/.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
zarr_filename |
str |
Store name, resolved relative to |
required |
Returns:
| Type | Description |
|---|---|
None |
Exceptions:
| Type | Description |
|---|---|
ValueError |
If :meth: |
Source code in climdata/datasets/MSWX.py
def to_zarr(self, zarr_filename: str):
"""Write the loaded dataset to a Zarr store under ``data/MSWX/``.
Args:
zarr_filename (str): Store name, resolved relative to ``data/MSWX``.
Returns:
None
Raises:
ValueError: If :meth:`load` has not been called.
"""
if self.dataset is None:
raise ValueError("No dataset loaded. Call `load()` first.")
var_name = self.dataset.name
if var_name == 'pr':
self.dataset.attrs['units'] = 'mm/day'
elif var_name in ['tas', 'tasmax', 'tasmin']:
self.dataset.attrs['units'] = 'degC'
zarr_path = os.path.join("data/MSWX", zarr_filename)
os.makedirs(os.path.dirname(zarr_path), exist_ok=True)
print(f"💾 Saving {var_name} to Zarr: {zarr_path}")
self.dataset.to_zarr(zarr_path, mode="w")
climdata.datasets.ERA5.ERA5Mirror
¶
Build and maintain a local Zarr mirror of ERA5 from the Copernicus CDS.
Unlike the other providers this is a mirroring tool rather than an
extractor: it downloads whole global months and appends them to one Zarr
store per variable, to be read afterwards with xarray.open_zarr. It also
takes a storage path rather than a Hydra config, so it is not driven by
:class:~climdata.utils.wrapper_workflow.ClimateExtractor.
Progress is journalled in a metadata.json beside the stores, so an
interrupted mirror resumes at the first month it has not yet written instead
of re-downloading. Months are fetched concurrently through Dask.
Requires a Copernicus CDS account and a ~/.cdsapirc key file.
Attributes:
| Name | Type | Description |
|---|---|---|
base_path |
str |
Directory holding the Zarr stores and |
fs |
fsspec.AbstractFileSystem |
Filesystem the stores live on — local by default, but any fsspec target (S3, GCS) works. |
metadata |
dict |
The journal of downloaded chunks. |
Examples:
>>> import datetime
>>> mirror = ERA5Mirror("./era5")
>>> paths = mirror.download(
... ["2m_temperature", ("temperature", 500)],
... (datetime.date(2020, 1, 1), datetime.date(2020, 3, 1)),
... )
Source code in climdata/datasets/ERA5.py
class ERA5Mirror:
"""Build and maintain a local Zarr mirror of ERA5 from the Copernicus CDS.
Unlike the other providers this is a *mirroring* tool rather than an
extractor: it downloads whole global months and appends them to one Zarr
store per variable, to be read afterwards with ``xarray.open_zarr``. It also
takes a storage path rather than a Hydra config, so it is not driven by
:class:`~climdata.utils.wrapper_workflow.ClimateExtractor`.
Progress is journalled in a ``metadata.json`` beside the stores, so an
interrupted mirror resumes at the first month it has not yet written instead
of re-downloading. Months are fetched concurrently through Dask.
Requires a Copernicus CDS account and a ``~/.cdsapirc`` key file.
Attributes:
base_path (str): Directory holding the Zarr stores and ``metadata.json``.
fs (fsspec.AbstractFileSystem): Filesystem the stores live on — local by
default, but any fsspec target (S3, GCS) works.
metadata (dict): The journal of downloaded chunks.
Example:
>>> import datetime
>>> mirror = ERA5Mirror("./era5") # doctest: +SKIP
>>> paths = mirror.download( # doctest: +SKIP
... ["2m_temperature", ("temperature", 500)],
... (datetime.date(2020, 1, 1), datetime.date(2020, 3, 1)),
... )
"""
def __init__(self, base_path: str, fs: fsspec.AbstractFileSystem = None):
"""Open (or create) a mirror directory and read its journal.
Args:
base_path (str): Directory for the Zarr stores. Created if absent.
fs (fsspec.AbstractFileSystem, optional): Filesystem to use.
Defaults to the local filesystem.
Raises:
ImportError: If ``cdsapi`` is not installed.
"""
if not _CDSAPI_AVAILABLE:
raise ImportError(
"ERA5 requires cdsapi. "
"Install with: pip install cdsapi"
)
# Get parameters
self.base_path = base_path
if fs is None:
fs = fsspec.filesystem("file")
self.fs = fs
# Create the base path if it doesn't exist
if not self.fs.exists(self.base_path):
self.fs.makedirs(self.base_path)
# Create metadata that will be used to track which chunks have been downloaded
self.metadata_file = os.path.join(self.base_path, "metadata.json")
self.metadata = self.get_metadata()
def get_metadata(self):
"""Read the download journal from ``metadata.json``.
Returns:
dict: ``{"chunks": [...]}``. An absent or corrupt file yields an
empty journal rather than raising, so a truncated write from an
interrupted run costs a re-download rather than a crash.
"""
if self.fs.exists(self.metadata_file):
with self.fs.open(self.metadata_file, "r") as f:
try:
metadata = json.load(f)
except json.decoder.JSONDecodeError:
metadata = {"chunks": []}
else:
metadata = {"chunks": []}
return metadata
def save_metadata(self):
"""Write the download journal back to ``metadata.json``.
Returns:
None
"""
with self.fs.open(self.metadata_file, "w") as f:
json.dump(self.metadata, f)
def chunk_exists(self, variable, year, month, pressure_level):
"""Test whether a month has already been mirrored.
Args:
variable (str): CDS variable name.
year (int): Calendar year.
month (int): Calendar month, 1-12.
pressure_level (int | None): Pressure level in hPa, or ``None`` for
single-level data.
Returns:
bool: ``True`` if the journal records this chunk.
"""
for chunk in self.metadata["chunks"]:
if (
chunk["variable"] == variable
and chunk["year"] == year
and chunk["month"] == month
and chunk["pressure_level"] == pressure_level
):
return True
return False
def download_chunk(
self,
variable: str,
year: int,
month: int,
pressure_level: int = None,
):
"""Download one global month of daily-mean ERA5 data.
Requests the CDS daily-statistics product, so what arrives is already
aggregated to daily means from 6-hourly steps rather than raw hourly
data. The NetCDF lands in a temporary directory and is opened before
that directory is removed, so the returned dataset must be consumed or
loaded before it goes out of scope.
Args:
variable (str): CDS variable name, e.g. ``"2m_temperature"`` or
``"total_precipitation"``.
year (int): Calendar year.
month (int): Calendar month, 1-12.
pressure_level (int, optional): Pressure level in hPa. ``None``, the
default, requests the single-level product.
Returns:
xr.Dataset: The downloaded month.
Raises:
Exception: Whatever ``cdsapi`` raises for a rejected request — an
unknown variable name, or a missing/invalid ``~/.cdsapirc``.
"""
with tempfile.TemporaryDirectory() as tmpdir:
# Get all days in the month
days_in_month = calendar.monthrange(year, month)[1]
# Make tmpfile to store the data
output_file = os.path.join(
tmpdir,
f"{variable}_{year}_{month:02d}_{str(pressure_level)}.nc",
)
# start the CDS API client (maybe need to move this outside the loop?)
c = cdsapi.Client(quiet=True)
# Setup the request parameters
request_params = {
"product_type": "reanalysis",
"variable": [variable],
"year": str(year),
"month": str(month),
"day": [f"{day:02d}" for day in range(1, days_in_month + 1)],
"time_zone": "utc+00:00",
"frequency": "6_hourly",
"daily_statistic": "daily_mean",
"data_format": "netcdf"
}
if pressure_level:
request_params["pressure_level"] = [str(pressure_level)]
dataset_name = "derived-era5-pressure-levels-daily-statistics"
else:
dataset_name = "derived-era5-single-levels-daily-statistics"
# Download the data
c.retrieve(
dataset_name,
request_params,
output_file,
)
# Open the downloaded data
ds = xr.open_dataset(output_file)
return ds
def variable_to_zarr_name(self, variable: str, pressure_level: int = None):
"""Build the Zarr store path for a variable.
Args:
variable (str): CDS variable name.
pressure_level (int, optional): Pressure level in hPa. When given it
is folded into the name, so each level gets its own store.
Returns:
str: Path of the form ``<base_path>/<variable>.zarr`` or
``<base_path>/<variable>_pressure_level_<n>.zarr``.
"""
# create zarr path for variable
zarr_path = f"{self.base_path}/{variable}"
if pressure_level:
zarr_path += f"_pressure_level_{pressure_level}"
zarr_path += ".zarr"
return zarr_path
def download_and_upload_chunk(
self,
variable: str,
year: int,
month: int,
pressure_level: int = None,
):
"""Download one month and append it to the variable's Zarr store.
Creates the store on the first month and appends along ``time``
thereafter. Chunking is one timestep by the full global grid, which suits
reading a few dates over a wide area — the common ERA5 access pattern —
rather than long time series at one point.
The journal is updated only after a successful write, so a crash
mid-append leaves the month marked as missing and it is retried.
Args:
variable (str): CDS variable name.
year (int): Calendar year.
month (int): Calendar month, 1-12.
pressure_level (int, optional): Pressure level in hPa.
Returns:
None
"""
# Download the data
ds = self.download_chunk(variable, year, month, pressure_level)
if "valid_time" in ds.dims:
ds = ds.rename({"valid_time": "time"})
# Create the Zarr path
zarr_path = self.variable_to_zarr_name(variable, pressure_level)
# Specify the chunking options
chunking = {"time": 1, "latitude": 721, "longitude": 1440}
if "level" in ds.dims:
chunking["level"] = 1
# Re-chunk the dataset
ds = ds.chunk(chunking)
# Check if the Zarr dataset exists
if self.fs.exists(zarr_path):
mode = "a"
append_dim = "time"
create = False
else:
mode = "w"
append_dim = None
create = True
# Upload the data to the Zarr dataset
mapper = self.fs.get_mapper(zarr_path, create=create)
ds.to_zarr(mapper, mode=mode, consolidated=True, append_dim=append_dim)
# Update the metadata
self.metadata["chunks"].append(
{
"variable": variable,
"year": year,
"month": month,
"pressure_level": pressure_level,
}
)
self.save_metadata()
def download(
self,
variables: List[Union[str, Tuple[str, int]]],
date_range: Tuple[datetime.date, datetime.date],
):
"""Mirror a set of variables over a date range, month by month.
Walks the range one calendar month at a time, skipping months the journal
already records and submitting the rest to Dask so several download
concurrently. Both endpoints are rounded down to the first of the month,
so any month the range touches is fetched whole.
Every resulting store is verified afterwards to have an evenly spaced
time axis — the symptom of a download that failed partway through.
Args:
variables (list[str | tuple[str, int]]): Variables to mirror. A bare
string requests single-level data; a ``(variable, level)`` tuple
requests one pressure level.
date_range (tuple[datetime.date, datetime.date]): Inclusive start and
end dates.
Returns:
list[str]: Path of the Zarr store for each requested variable.
Raises:
AssertionError: If a store ends with an unevenly spaced time axis.
Delete that store and re-run — the message names the path.
"""
start_date, end_date = date_range
# Reformat the variables list so all elements are tuples
reformated_variables = []
for variable in variables:
if isinstance(variable, str):
reformated_variables.append(tuple([variable, None]))
else:
reformated_variables.append(variable)
# Start Downloading
with ProgressBar():
# Round dates to months
current_date = start_date.replace(day=1)
end_date = end_date.replace(day=1)
while current_date <= end_date:
# Create a list of tasks to download the data
tasks = []
for variable, pressure_level in reformated_variables:
if not self.chunk_exists(
variable,
current_date.year,
current_date.month,
pressure_level,
):
task = dask.delayed(self.download_and_upload_chunk)(
variable,
current_date.year,
current_date.month,
pressure_level,
)
tasks.append(task)
else:
print(
f"Chunk for {variable} {pressure_level} {current_date.year}-{current_date.month} already exists. Skipping."
)
# Execute the tasks with Dask
print(f"Downloading data for {current_date.year}-{current_date.month}")
if tasks:
dask.compute(*tasks)
# Update the metadata
self.save_metadata()
# Update the current date
days_in_month = calendar.monthrange(
year=current_date.year, month=current_date.month
)[1]
current_date += datetime.timedelta(days=days_in_month)
# Return the Zarr paths
zarr_paths = []
for variable, pressure_level in reformated_variables:
zarr_path = self.variable_to_zarr_name(variable, pressure_level)
zarr_paths.append(zarr_path)
# Check that Zarr arrays have correct dt for time dimension
for zarr_path in zarr_paths:
ds = xr.open_zarr(zarr_path)
time_stamps = ds.time.values
dt = time_stamps[1:] - time_stamps[:-1]
assert np.all(dt == dt[0]), (
f"Zarr array {zarr_path} has incorrect dt for time dimension. An error may have occurred during download. Please delete the Zarr array and try again."
)
return zarr_paths
__init__(self, base_path, fs=None)
special
¶
Open (or create) a mirror directory and read its journal.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
base_path |
str |
Directory for the Zarr stores. Created if absent. |
required |
fs |
fsspec.AbstractFileSystem |
Filesystem to use. Defaults to the local filesystem. |
None |
Exceptions:
| Type | Description |
|---|---|
ImportError |
If |
Source code in climdata/datasets/ERA5.py
def __init__(self, base_path: str, fs: fsspec.AbstractFileSystem = None):
"""Open (or create) a mirror directory and read its journal.
Args:
base_path (str): Directory for the Zarr stores. Created if absent.
fs (fsspec.AbstractFileSystem, optional): Filesystem to use.
Defaults to the local filesystem.
Raises:
ImportError: If ``cdsapi`` is not installed.
"""
if not _CDSAPI_AVAILABLE:
raise ImportError(
"ERA5 requires cdsapi. "
"Install with: pip install cdsapi"
)
# Get parameters
self.base_path = base_path
if fs is None:
fs = fsspec.filesystem("file")
self.fs = fs
# Create the base path if it doesn't exist
if not self.fs.exists(self.base_path):
self.fs.makedirs(self.base_path)
# Create metadata that will be used to track which chunks have been downloaded
self.metadata_file = os.path.join(self.base_path, "metadata.json")
self.metadata = self.get_metadata()
chunk_exists(self, variable, year, month, pressure_level)
¶
Test whether a month has already been mirrored.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
variable |
str |
CDS variable name. |
required |
year |
int |
Calendar year. |
required |
month |
int |
Calendar month, 1-12. |
required |
pressure_level |
int | None |
Pressure level in hPa, or |
required |
Returns:
| Type | Description |
|---|---|
bool |
|
Source code in climdata/datasets/ERA5.py
def chunk_exists(self, variable, year, month, pressure_level):
"""Test whether a month has already been mirrored.
Args:
variable (str): CDS variable name.
year (int): Calendar year.
month (int): Calendar month, 1-12.
pressure_level (int | None): Pressure level in hPa, or ``None`` for
single-level data.
Returns:
bool: ``True`` if the journal records this chunk.
"""
for chunk in self.metadata["chunks"]:
if (
chunk["variable"] == variable
and chunk["year"] == year
and chunk["month"] == month
and chunk["pressure_level"] == pressure_level
):
return True
return False
download(self, variables, date_range)
¶
Mirror a set of variables over a date range, month by month.
Walks the range one calendar month at a time, skipping months the journal already records and submitting the rest to Dask so several download concurrently. Both endpoints are rounded down to the first of the month, so any month the range touches is fetched whole.
Every resulting store is verified afterwards to have an evenly spaced time axis — the symptom of a download that failed partway through.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
variables |
list[str | tuple[str, int]] |
Variables to mirror. A bare
string requests single-level data; a |
required |
date_range |
tuple[datetime.date, datetime.date] |
Inclusive start and end dates. |
required |
Returns:
| Type | Description |
|---|---|
list[str] |
Path of the Zarr store for each requested variable. |
Exceptions:
| Type | Description |
|---|---|
AssertionError |
If a store ends with an unevenly spaced time axis. Delete that store and re-run — the message names the path. |
Source code in climdata/datasets/ERA5.py
def download(
self,
variables: List[Union[str, Tuple[str, int]]],
date_range: Tuple[datetime.date, datetime.date],
):
"""Mirror a set of variables over a date range, month by month.
Walks the range one calendar month at a time, skipping months the journal
already records and submitting the rest to Dask so several download
concurrently. Both endpoints are rounded down to the first of the month,
so any month the range touches is fetched whole.
Every resulting store is verified afterwards to have an evenly spaced
time axis — the symptom of a download that failed partway through.
Args:
variables (list[str | tuple[str, int]]): Variables to mirror. A bare
string requests single-level data; a ``(variable, level)`` tuple
requests one pressure level.
date_range (tuple[datetime.date, datetime.date]): Inclusive start and
end dates.
Returns:
list[str]: Path of the Zarr store for each requested variable.
Raises:
AssertionError: If a store ends with an unevenly spaced time axis.
Delete that store and re-run — the message names the path.
"""
start_date, end_date = date_range
# Reformat the variables list so all elements are tuples
reformated_variables = []
for variable in variables:
if isinstance(variable, str):
reformated_variables.append(tuple([variable, None]))
else:
reformated_variables.append(variable)
# Start Downloading
with ProgressBar():
# Round dates to months
current_date = start_date.replace(day=1)
end_date = end_date.replace(day=1)
while current_date <= end_date:
# Create a list of tasks to download the data
tasks = []
for variable, pressure_level in reformated_variables:
if not self.chunk_exists(
variable,
current_date.year,
current_date.month,
pressure_level,
):
task = dask.delayed(self.download_and_upload_chunk)(
variable,
current_date.year,
current_date.month,
pressure_level,
)
tasks.append(task)
else:
print(
f"Chunk for {variable} {pressure_level} {current_date.year}-{current_date.month} already exists. Skipping."
)
# Execute the tasks with Dask
print(f"Downloading data for {current_date.year}-{current_date.month}")
if tasks:
dask.compute(*tasks)
# Update the metadata
self.save_metadata()
# Update the current date
days_in_month = calendar.monthrange(
year=current_date.year, month=current_date.month
)[1]
current_date += datetime.timedelta(days=days_in_month)
# Return the Zarr paths
zarr_paths = []
for variable, pressure_level in reformated_variables:
zarr_path = self.variable_to_zarr_name(variable, pressure_level)
zarr_paths.append(zarr_path)
# Check that Zarr arrays have correct dt for time dimension
for zarr_path in zarr_paths:
ds = xr.open_zarr(zarr_path)
time_stamps = ds.time.values
dt = time_stamps[1:] - time_stamps[:-1]
assert np.all(dt == dt[0]), (
f"Zarr array {zarr_path} has incorrect dt for time dimension. An error may have occurred during download. Please delete the Zarr array and try again."
)
return zarr_paths
download_and_upload_chunk(self, variable, year, month, pressure_level=None)
¶
Download one month and append it to the variable's Zarr store.
Creates the store on the first month and appends along time
thereafter. Chunking is one timestep by the full global grid, which suits
reading a few dates over a wide area — the common ERA5 access pattern —
rather than long time series at one point.
The journal is updated only after a successful write, so a crash mid-append leaves the month marked as missing and it is retried.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
variable |
str |
CDS variable name. |
required |
year |
int |
Calendar year. |
required |
month |
int |
Calendar month, 1-12. |
required |
pressure_level |
int |
Pressure level in hPa. |
None |
Returns:
| Type | Description |
|---|---|
None |
Source code in climdata/datasets/ERA5.py
def download_and_upload_chunk(
self,
variable: str,
year: int,
month: int,
pressure_level: int = None,
):
"""Download one month and append it to the variable's Zarr store.
Creates the store on the first month and appends along ``time``
thereafter. Chunking is one timestep by the full global grid, which suits
reading a few dates over a wide area — the common ERA5 access pattern —
rather than long time series at one point.
The journal is updated only after a successful write, so a crash
mid-append leaves the month marked as missing and it is retried.
Args:
variable (str): CDS variable name.
year (int): Calendar year.
month (int): Calendar month, 1-12.
pressure_level (int, optional): Pressure level in hPa.
Returns:
None
"""
# Download the data
ds = self.download_chunk(variable, year, month, pressure_level)
if "valid_time" in ds.dims:
ds = ds.rename({"valid_time": "time"})
# Create the Zarr path
zarr_path = self.variable_to_zarr_name(variable, pressure_level)
# Specify the chunking options
chunking = {"time": 1, "latitude": 721, "longitude": 1440}
if "level" in ds.dims:
chunking["level"] = 1
# Re-chunk the dataset
ds = ds.chunk(chunking)
# Check if the Zarr dataset exists
if self.fs.exists(zarr_path):
mode = "a"
append_dim = "time"
create = False
else:
mode = "w"
append_dim = None
create = True
# Upload the data to the Zarr dataset
mapper = self.fs.get_mapper(zarr_path, create=create)
ds.to_zarr(mapper, mode=mode, consolidated=True, append_dim=append_dim)
# Update the metadata
self.metadata["chunks"].append(
{
"variable": variable,
"year": year,
"month": month,
"pressure_level": pressure_level,
}
)
self.save_metadata()
download_chunk(self, variable, year, month, pressure_level=None)
¶
Download one global month of daily-mean ERA5 data.
Requests the CDS daily-statistics product, so what arrives is already aggregated to daily means from 6-hourly steps rather than raw hourly data. The NetCDF lands in a temporary directory and is opened before that directory is removed, so the returned dataset must be consumed or loaded before it goes out of scope.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
variable |
str |
CDS variable name, e.g. |
required |
year |
int |
Calendar year. |
required |
month |
int |
Calendar month, 1-12. |
required |
pressure_level |
int |
Pressure level in hPa. |
None |
Returns:
| Type | Description |
|---|---|
xr.Dataset |
The downloaded month. |
Exceptions:
| Type | Description |
|---|---|
Exception |
Whatever |
Source code in climdata/datasets/ERA5.py
def download_chunk(
self,
variable: str,
year: int,
month: int,
pressure_level: int = None,
):
"""Download one global month of daily-mean ERA5 data.
Requests the CDS daily-statistics product, so what arrives is already
aggregated to daily means from 6-hourly steps rather than raw hourly
data. The NetCDF lands in a temporary directory and is opened before
that directory is removed, so the returned dataset must be consumed or
loaded before it goes out of scope.
Args:
variable (str): CDS variable name, e.g. ``"2m_temperature"`` or
``"total_precipitation"``.
year (int): Calendar year.
month (int): Calendar month, 1-12.
pressure_level (int, optional): Pressure level in hPa. ``None``, the
default, requests the single-level product.
Returns:
xr.Dataset: The downloaded month.
Raises:
Exception: Whatever ``cdsapi`` raises for a rejected request — an
unknown variable name, or a missing/invalid ``~/.cdsapirc``.
"""
with tempfile.TemporaryDirectory() as tmpdir:
# Get all days in the month
days_in_month = calendar.monthrange(year, month)[1]
# Make tmpfile to store the data
output_file = os.path.join(
tmpdir,
f"{variable}_{year}_{month:02d}_{str(pressure_level)}.nc",
)
# start the CDS API client (maybe need to move this outside the loop?)
c = cdsapi.Client(quiet=True)
# Setup the request parameters
request_params = {
"product_type": "reanalysis",
"variable": [variable],
"year": str(year),
"month": str(month),
"day": [f"{day:02d}" for day in range(1, days_in_month + 1)],
"time_zone": "utc+00:00",
"frequency": "6_hourly",
"daily_statistic": "daily_mean",
"data_format": "netcdf"
}
if pressure_level:
request_params["pressure_level"] = [str(pressure_level)]
dataset_name = "derived-era5-pressure-levels-daily-statistics"
else:
dataset_name = "derived-era5-single-levels-daily-statistics"
# Download the data
c.retrieve(
dataset_name,
request_params,
output_file,
)
# Open the downloaded data
ds = xr.open_dataset(output_file)
return ds
get_metadata(self)
¶
Read the download journal from metadata.json.
Returns:
| Type | Description |
|---|---|
dict |
|
Source code in climdata/datasets/ERA5.py
def get_metadata(self):
"""Read the download journal from ``metadata.json``.
Returns:
dict: ``{"chunks": [...]}``. An absent or corrupt file yields an
empty journal rather than raising, so a truncated write from an
interrupted run costs a re-download rather than a crash.
"""
if self.fs.exists(self.metadata_file):
with self.fs.open(self.metadata_file, "r") as f:
try:
metadata = json.load(f)
except json.decoder.JSONDecodeError:
metadata = {"chunks": []}
else:
metadata = {"chunks": []}
return metadata
save_metadata(self)
¶
Write the download journal back to metadata.json.
Returns:
| Type | Description |
|---|---|
None |
Source code in climdata/datasets/ERA5.py
def save_metadata(self):
"""Write the download journal back to ``metadata.json``.
Returns:
None
"""
with self.fs.open(self.metadata_file, "w") as f:
json.dump(self.metadata, f)
variable_to_zarr_name(self, variable, pressure_level=None)
¶
Build the Zarr store path for a variable.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
variable |
str |
CDS variable name. |
required |
pressure_level |
int |
Pressure level in hPa. When given it is folded into the name, so each level gets its own store. |
None |
Returns:
| Type | Description |
|---|---|
str |
Path of the form |
Source code in climdata/datasets/ERA5.py
def variable_to_zarr_name(self, variable: str, pressure_level: int = None):
"""Build the Zarr store path for a variable.
Args:
variable (str): CDS variable name.
pressure_level (int, optional): Pressure level in hPa. When given it
is folded into the name, so each level gets its own store.
Returns:
str: Path of the form ``<base_path>/<variable>.zarr`` or
``<base_path>/<variable>_pressure_level_<n>.zarr``.
"""
# create zarr path for variable
zarr_path = f"{self.base_path}/{variable}"
if pressure_level:
zarr_path += f"_pressure_level_{pressure_level}"
zarr_path += ".zarr"
return zarr_path
climdata.datasets.W5E5.W5E5
¶
Download and assemble W5E5 climate data from the ISIMIP repository.
W5E5 is distributed through ISIMIP3a as observational input data and serves
as the bias-adjustment reference for ISIMIP3b projections — pair it with
:class:~climdata.datasets.CMIP_W5E5.CMIPW5E5 to compare a projection
against the reference it was adjusted to.
The lifecycle is fetch → load → extract. :meth:extract may be
called before :meth:load, in which case the subsetting instructions are
stored and applied as soon as data arrives.
Attributes:
| Name | Type | Description |
|---|---|---|
cfg |
DictConfig |
Configuration supplying |
ds |
xr.Dataset | None |
The loaded dataset, set by :meth: |
client |
ISIMIPClient |
ISIMIP API client, created in :meth: |
downloaded_files |
list[str] |
Local paths accumulated by :meth: |
Examples:
>>> w5e5 = W5E5(cfg)
>>> w5e5.fetch()
>>> w5e5.load()
>>> w5e5.extract(point=(13.4, 52.5))
Source code in climdata/datasets/W5E5.py
class W5E5:
"""Download and assemble W5E5 climate data from the ISIMIP repository.
W5E5 is distributed through ISIMIP3a as observational input data and serves
as the bias-adjustment reference for ISIMIP3b projections — pair it with
:class:`~climdata.datasets.CMIP_W5E5.CMIPW5E5` to compare a projection
against the reference it was adjusted to.
The lifecycle is ``fetch`` → ``load`` → ``extract``. :meth:`extract` may be
called before :meth:`load`, in which case the subsetting instructions are
stored and applied as soon as data arrives.
Attributes:
cfg (DictConfig): Configuration supplying ``variables``, ``time_range``,
``data_dir`` and ``dataset``.
ds (xr.Dataset | None): The loaded dataset, set by :meth:`load`.
client (ISIMIPClient): ISIMIP API client, created in :meth:`__init__`.
downloaded_files (list[str]): Local paths accumulated by :meth:`fetch`.
Example:
>>> w5e5 = W5E5(cfg) # doctest: +SKIP
>>> w5e5.fetch() # doctest: +SKIP
>>> w5e5.load() # doctest: +SKIP
>>> w5e5.extract(point=(13.4, 52.5)) # doctest: +SKIP
"""
def __init__(self, cfg: DictConfig):
"""Bind a configuration and open an ISIMIP client.
Args:
cfg (DictConfig): Configuration with ``variables``,
``time_range.start_date`` / ``.end_date``, ``data_dir`` and
``dataset``.
Raises:
ImportError: If ``isimip-client`` is not installed.
"""
self.cfg = cfg
self.ds = None
self.client = None
self.downloaded_files = []
self._extract_mode = None
self._extract_params = None
# Initialize ISIMIP client
try:
from isimip_client.client import ISIMIPClient
self.client = ISIMIPClient()
except ImportError:
raise ImportError(
"isimip-client is required for W5E5 data access. "
"Install it with: pip install isimip-client"
)
def fetch(self):
"""Search ISIMIP and download the W5E5 files covering the request.
Queries the ISIMIP3a ``obsclim`` catalogue once per configured variable,
keeps the files whose filename year-range overlaps the requested period
(see :meth:`_is_file_in_date_range`), and downloads any that are not
already on disk under ``<data_dir>/<DATASET>/<variable>/``.
Per-variable failures are reported and skipped rather than raised, so one
unavailable variable does not abort a multi-variable request — check
:attr:`downloaded_files` to see what actually arrived.
Returns:
None: Paths are appended to :attr:`downloaded_files`.
"""
print("🔍 Searching for W5E5 datasets in ISIMIP repository...")
start_date = datetime.fromisoformat(self.cfg.time_range.start_date)
end_date = datetime.fromisoformat(self.cfg.time_range.end_date)
output_dir = Path(self.cfg.data_dir) / self.cfg.dataset.upper()
output_dir.mkdir(parents=True, exist_ok=True)
# Search for each variable separately
for var in self.cfg.variables:
print(f"\n📥 Fetching {var}...")
# Map variable names to W5E5 names if needed
w5e5_var = self._map_variable_name(var)
# Search ISIMIP repository for W5E5 data
# W5E5 is available in ISIMIP3a as observational input data
try:
response = self.client.datasets(
simulation_round='ISIMIP3a',
product='InputData',
climate_forcing='20crv3-w5e5', # W5E5 version 2.0
climate_scenario='obsclim', # Observed climate
climate_variable=w5e5_var
)
results = self._normalize_dataset_results(response)
if not results:
print(f"⚠️ No W5E5 datasets found for {var}")
continue
# Get the first matching dataset
dataset = results[0]
print(f"✅ Found dataset: {dataset.get('name', 'unnamed')}")
# Filter files by date range
for file_info in self._extract_files_list(dataset):
file_name = file_info['name']
# Parse date from filename (W5E5 files typically contain year ranges)
# Example: w5e5v2.0_obsclim_tas_global_daily_1979_1989.nc
if self._is_file_in_date_range(file_name, start_date, end_date):
local_path = output_dir / var / file_name
local_path.parent.mkdir(parents=True, exist_ok=True)
if local_path.exists():
print(f" ✓ Already exists: {file_name}")
self.downloaded_files.append(str(local_path))
else:
print(f" ⬇️ Downloading: {file_name}")
# Download directly using the file URL
self.client.download(
file_info['file_url'],
path=str(local_path.parent),
validate=False
)
self.downloaded_files.append(str(local_path))
except Exception as e:
print(f"❌ Error fetching {var}: {str(e)}")
continue
print(f"\n✅ Downloaded {len(self.downloaded_files)} files")
def _normalize_dataset_results(self, response) -> List[Dict]:
"""Coerce an isimip-client response into a list of dataset records.
The client returns a bare list from some endpoints and a paginated
``{"results": [...]}`` envelope from others.
Args:
response: The value returned by ``client.datasets(...)``.
Returns:
list[dict]: Dataset records, empty if the response has neither shape.
"""
if response is None:
return []
if isinstance(response, list):
return response
if isinstance(response, dict):
results = response.get('results')
if isinstance(results, list):
return results
return []
def _extract_files_list(self, dataset: Dict) -> List[Dict]:
"""Pull the file records out of one ISIMIP dataset record.
Args:
dataset (dict): A single dataset record.
Returns:
list[dict]: File records, each with ``path``, ``name`` and
``file_url``. Empty if the record carries neither ``files`` nor a
single ``file``.
"""
files = dataset.get('files')
if isinstance(files, list):
return files
single_file = dataset.get('file')
if isinstance(single_file, dict):
return [single_file]
return []
def load(self):
"""Open the downloaded files and merge them into one dataset.
Files are grouped by variable, concatenated along time where a variable
spans several decades, merged across variables, and finally clipped to
the configured time range.
Returns:
None: The dataset is stored on :attr:`ds`.
Raises:
ValueError: If :meth:`fetch` has not run, or produced no files.
"""
if not self.downloaded_files:
raise ValueError("No files to load. Run fetch() first.")
print(f"📂 Loading {len(self.downloaded_files)} W5E5 files...")
# Group files by variable
files_by_var = {}
for fpath in self.downloaded_files:
# Determine which variable this file contains
for var in self.cfg.variables:
if f"/{var}/" in fpath or f"_{self._map_variable_name(var)}_" in fpath:
if var not in files_by_var:
files_by_var[var] = []
files_by_var[var].append(fpath)
break
# Load each variable separately and merge
datasets = []
for var, file_list in files_by_var.items():
print(f" Loading {var} from {len(file_list)} file(s)...")
if len(file_list) == 1:
ds_var = xr.open_dataset(file_list[0])
else:
# Multiple files - open as multi-file dataset
ds_var = xr.open_mfdataset(
file_list,
combine='by_coords',
parallel=True
)
datasets.append(ds_var)
# Merge all variables into one dataset
if len(datasets) == 1:
self.ds = datasets[0]
else:
self.ds = xr.merge(datasets)
# Subset to requested time range
start = self.cfg.time_range.start_date
end = self.cfg.time_range.end_date
self.ds = self.ds.sel(time=slice(start, end))
# Add metadata
self.ds.attrs['source'] = 'W5E5 via ISIMIP'
self.ds.attrs['dataset'] = 'W5E5v2.0'
self.ds.attrs['description'] = 'WFDE5 over land merged with ERA5 over ocean'
print(f"✅ Loaded dataset with {len(self.ds.data_vars)} variables")
def extract(self, *, point: Optional[Tuple[float, float]] = None,
box: Optional[Dict] = None,
shapefile: Optional[str] = None,
buffer_km: float = 0.0):
"""Record a spatial subset, and apply it if data is already loaded.
The instruction is stored rather than executed, so this may be called
before :meth:`load`; :meth:`_apply_extraction` runs it once :attr:`ds`
exists. The three modes are mutually exclusive — the first one supplied
wins, in the order ``point``, ``box``, ``shapefile``. Calling with no
argument clears nothing and does nothing.
Args:
point (tuple[float, float], optional): ``(lon, lat)`` in degrees.
Note the order: longitude first.
box (dict, optional): Bounding box with keys ``lon_min``, ``lon_max``,
``lat_min``, ``lat_max``.
shapefile (str | geopandas.GeoDataFrame, optional): Polygon(s) to clip
to, as a path or an in-memory frame.
buffer_km (float): Half-width of a box averaged around ``point``,
converted at a flat 111 km per degree. ``0.0``, the default,
selects the single nearest grid cell instead.
Returns:
None
"""
if point is not None:
lon, lat = point
buffer_deg = buffer_km / 111.0
self._extract_mode = "point"
self._extract_params = (lon, lat, buffer_deg)
# Apply extraction if dataset is already loaded
if self.ds is not None:
self._apply_extraction()
elif box is not None:
self._extract_mode = "box"
self._extract_params = box
if self.ds is not None:
self._apply_extraction()
elif shapefile is not None:
import geopandas as gpd
if isinstance(shapefile, str):
gdf = gpd.read_file(shapefile)
else:
gdf = shapefile
self._extract_mode = "shapefile"
self._extract_params = gdf
if self.ds is not None:
self._apply_extraction()
def _apply_extraction(self):
"""Run the subset recorded by :meth:`extract` against :attr:`ds`.
Point requests either take the nearest cell or area-average a buffered
box; box requests slice; shapefile requests clip each geometry through
rioxarray and stack the results along a new ``geom_id`` dimension.
Returns:
None: :attr:`ds` is replaced in place.
"""
if self._extract_mode == "point":
lon, lat, buffer_deg = self._extract_params
if buffer_deg > 0:
self.ds = self.ds.sel(
lon=slice(lon - buffer_deg, lon + buffer_deg),
lat=slice(lat - buffer_deg, lat + buffer_deg)
).mean(["lat", "lon"])
else:
self.ds = self.ds.sel(lon=lon, lat=lat, method="nearest")
elif self._extract_mode == "box":
box = self._extract_params
self.ds = self.ds.sel(
lon=slice(box["lon_min"], box["lon_max"]),
lat=slice(box["lat_min"], box["lat_max"])
)
elif self._extract_mode == "shapefile":
import rioxarray # noqa: F401 — registers the .rio accessor
from shapely.geometry import mapping
gdf = self._extract_params
self.ds = self.ds.rio.set_spatial_dims(x_dim="lon", y_dim="lat")
self.ds = self.ds.rio.write_crs("EPSG:4326", inplace=True)
clipped_list = []
for geom in gdf.geometry:
clipped = self.ds.rio.clip([mapping(geom)], gdf.crs, drop=True)
clipped_list.append(clipped)
self.ds = xr.concat(clipped_list, dim="geom_id")
def save_netcdf(self, filename: str):
"""Write the dataset to NetCDF, creating parent directories.
Args:
filename (str): Destination path.
Returns:
None
Raises:
ValueError: If no dataset has been loaded.
"""
if self.ds is None:
raise ValueError("No dataset loaded. Run load() first.")
output_path = Path(filename)
output_path.parent.mkdir(parents=True, exist_ok=True)
self.ds.to_netcdf(output_path)
print(f"💾 Saved to: {output_path}")
def save_csv(self, filename: str):
"""Write the dataset to CSV, creating parent directories.
The frame is written in xarray's wide form — one column per variable,
indexed by the dataset's dimensions — not the long form produced by
:meth:`~climdata.utils.wrapper_workflow.ClimateExtractor.to_dataframe`.
Args:
filename (str): Destination path.
Returns:
None
Raises:
ValueError: If no dataset has been loaded.
"""
if self.ds is None:
raise ValueError("No dataset loaded. Run load() first.")
output_path = Path(filename)
output_path.parent.mkdir(parents=True, exist_ok=True)
df = self.ds.to_dataframe()
df.to_csv(output_path)
print(f"💾 Saved to: {output_path}")
def _map_variable_name(self, var: str) -> str:
"""Translate a CF variable name to the name W5E5 files use.
W5E5 follows CMIP naming, so nearly all names pass through unchanged;
``sfcWind`` is the exception, stored lowercase as ``sfcwind``.
Args:
var (str): CF variable name, e.g. ``"tasmax"``.
Returns:
str: The W5E5 name, or ``var`` unchanged if it is not in the table.
"""
# W5E5 uses standard names, so most map directly
variable_map = {
'tas': 'tas', # Near-surface air temperature
'tasmax': 'tasmax', # Maximum temperature
'tasmin': 'tasmin', # Minimum temperature
'pr': 'pr', # Precipitation
'rsds': 'rsds', # Surface downwelling shortwave radiation
'hurs': 'hurs', # Near-surface relative humidity
'sfcWind': 'sfcwind', # Near-surface wind speed (note: lowercase 'w')
'ps': 'ps', # Surface air pressure
'rlds': 'rlds', # Surface downwelling longwave radiation
}
return variable_map.get(var, var)
def _is_file_in_date_range(self, filename: str, start_date: datetime, end_date: datetime) -> bool:
"""Test whether a file's year range overlaps the requested period.
W5E5 encodes coverage in the filename, e.g.
``w5e5v2.0_obsclim_tas_global_daily_1979_1989.nc``. Comparison is at
year granularity, so a file may be kept for a request that only touches
part of it.
Args:
filename (str): Basename of the candidate file.
start_date (datetime): Start of the requested period.
end_date (datetime): End of the requested period.
Returns:
bool: ``True`` if the ranges overlap. Also ``True`` when the filename
carries no parseable year range — an unrecognised name is downloaded
rather than silently dropped.
"""
import re
# Extract year range from filename
match = re.search(r'_(\d{4})_(\d{4})\.nc', filename)
if match:
file_start_year = int(match.group(1))
file_end_year = int(match.group(2))
# Check if there's any overlap
return not (file_end_year < start_date.year or file_start_year > end_date.year)
# If we can't parse the date, include the file to be safe
return True
__init__(self, cfg)
special
¶
Bind a configuration and open an ISIMIP client.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
cfg |
DictConfig |
Configuration with |
required |
Exceptions:
| Type | Description |
|---|---|
ImportError |
If |
Source code in climdata/datasets/W5E5.py
def __init__(self, cfg: DictConfig):
"""Bind a configuration and open an ISIMIP client.
Args:
cfg (DictConfig): Configuration with ``variables``,
``time_range.start_date`` / ``.end_date``, ``data_dir`` and
``dataset``.
Raises:
ImportError: If ``isimip-client`` is not installed.
"""
self.cfg = cfg
self.ds = None
self.client = None
self.downloaded_files = []
self._extract_mode = None
self._extract_params = None
# Initialize ISIMIP client
try:
from isimip_client.client import ISIMIPClient
self.client = ISIMIPClient()
except ImportError:
raise ImportError(
"isimip-client is required for W5E5 data access. "
"Install it with: pip install isimip-client"
)
extract(self, *, point=None, box=None, shapefile=None, buffer_km=0.0)
¶
Record a spatial subset, and apply it if data is already loaded.
The instruction is stored rather than executed, so this may be called
before :meth:load; :meth:_apply_extraction runs it once :attr:ds
exists. The three modes are mutually exclusive — the first one supplied
wins, in the order point, box, shapefile. Calling with no
argument clears nothing and does nothing.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
point |
tuple[float, float] |
|
None |
box |
dict |
Bounding box with keys |
None |
shapefile |
str | geopandas.GeoDataFrame |
Polygon(s) to clip to, as a path or an in-memory frame. |
None |
buffer_km |
float |
Half-width of a box averaged around |
0.0 |
Returns:
| Type | Description |
|---|---|
None |
Source code in climdata/datasets/W5E5.py
def extract(self, *, point: Optional[Tuple[float, float]] = None,
box: Optional[Dict] = None,
shapefile: Optional[str] = None,
buffer_km: float = 0.0):
"""Record a spatial subset, and apply it if data is already loaded.
The instruction is stored rather than executed, so this may be called
before :meth:`load`; :meth:`_apply_extraction` runs it once :attr:`ds`
exists. The three modes are mutually exclusive — the first one supplied
wins, in the order ``point``, ``box``, ``shapefile``. Calling with no
argument clears nothing and does nothing.
Args:
point (tuple[float, float], optional): ``(lon, lat)`` in degrees.
Note the order: longitude first.
box (dict, optional): Bounding box with keys ``lon_min``, ``lon_max``,
``lat_min``, ``lat_max``.
shapefile (str | geopandas.GeoDataFrame, optional): Polygon(s) to clip
to, as a path or an in-memory frame.
buffer_km (float): Half-width of a box averaged around ``point``,
converted at a flat 111 km per degree. ``0.0``, the default,
selects the single nearest grid cell instead.
Returns:
None
"""
if point is not None:
lon, lat = point
buffer_deg = buffer_km / 111.0
self._extract_mode = "point"
self._extract_params = (lon, lat, buffer_deg)
# Apply extraction if dataset is already loaded
if self.ds is not None:
self._apply_extraction()
elif box is not None:
self._extract_mode = "box"
self._extract_params = box
if self.ds is not None:
self._apply_extraction()
elif shapefile is not None:
import geopandas as gpd
if isinstance(shapefile, str):
gdf = gpd.read_file(shapefile)
else:
gdf = shapefile
self._extract_mode = "shapefile"
self._extract_params = gdf
if self.ds is not None:
self._apply_extraction()
fetch(self)
¶
Search ISIMIP and download the W5E5 files covering the request.
Queries the ISIMIP3a obsclim catalogue once per configured variable,
keeps the files whose filename year-range overlaps the requested period
(see :meth:_is_file_in_date_range), and downloads any that are not
already on disk under <data_dir>/<DATASET>/<variable>/.
Per-variable failures are reported and skipped rather than raised, so one
unavailable variable does not abort a multi-variable request — check
:attr:downloaded_files to see what actually arrived.
Returns:
| Type | Description |
|---|---|
None |
Paths are appended to :attr: |
Source code in climdata/datasets/W5E5.py
def fetch(self):
"""Search ISIMIP and download the W5E5 files covering the request.
Queries the ISIMIP3a ``obsclim`` catalogue once per configured variable,
keeps the files whose filename year-range overlaps the requested period
(see :meth:`_is_file_in_date_range`), and downloads any that are not
already on disk under ``<data_dir>/<DATASET>/<variable>/``.
Per-variable failures are reported and skipped rather than raised, so one
unavailable variable does not abort a multi-variable request — check
:attr:`downloaded_files` to see what actually arrived.
Returns:
None: Paths are appended to :attr:`downloaded_files`.
"""
print("🔍 Searching for W5E5 datasets in ISIMIP repository...")
start_date = datetime.fromisoformat(self.cfg.time_range.start_date)
end_date = datetime.fromisoformat(self.cfg.time_range.end_date)
output_dir = Path(self.cfg.data_dir) / self.cfg.dataset.upper()
output_dir.mkdir(parents=True, exist_ok=True)
# Search for each variable separately
for var in self.cfg.variables:
print(f"\n📥 Fetching {var}...")
# Map variable names to W5E5 names if needed
w5e5_var = self._map_variable_name(var)
# Search ISIMIP repository for W5E5 data
# W5E5 is available in ISIMIP3a as observational input data
try:
response = self.client.datasets(
simulation_round='ISIMIP3a',
product='InputData',
climate_forcing='20crv3-w5e5', # W5E5 version 2.0
climate_scenario='obsclim', # Observed climate
climate_variable=w5e5_var
)
results = self._normalize_dataset_results(response)
if not results:
print(f"⚠️ No W5E5 datasets found for {var}")
continue
# Get the first matching dataset
dataset = results[0]
print(f"✅ Found dataset: {dataset.get('name', 'unnamed')}")
# Filter files by date range
for file_info in self._extract_files_list(dataset):
file_name = file_info['name']
# Parse date from filename (W5E5 files typically contain year ranges)
# Example: w5e5v2.0_obsclim_tas_global_daily_1979_1989.nc
if self._is_file_in_date_range(file_name, start_date, end_date):
local_path = output_dir / var / file_name
local_path.parent.mkdir(parents=True, exist_ok=True)
if local_path.exists():
print(f" ✓ Already exists: {file_name}")
self.downloaded_files.append(str(local_path))
else:
print(f" ⬇️ Downloading: {file_name}")
# Download directly using the file URL
self.client.download(
file_info['file_url'],
path=str(local_path.parent),
validate=False
)
self.downloaded_files.append(str(local_path))
except Exception as e:
print(f"❌ Error fetching {var}: {str(e)}")
continue
print(f"\n✅ Downloaded {len(self.downloaded_files)} files")
load(self)
¶
Open the downloaded files and merge them into one dataset.
Files are grouped by variable, concatenated along time where a variable spans several decades, merged across variables, and finally clipped to the configured time range.
Returns:
| Type | Description |
|---|---|
None |
The dataset is stored on :attr: |
Exceptions:
| Type | Description |
|---|---|
ValueError |
If :meth: |
Source code in climdata/datasets/W5E5.py
def load(self):
"""Open the downloaded files and merge them into one dataset.
Files are grouped by variable, concatenated along time where a variable
spans several decades, merged across variables, and finally clipped to
the configured time range.
Returns:
None: The dataset is stored on :attr:`ds`.
Raises:
ValueError: If :meth:`fetch` has not run, or produced no files.
"""
if not self.downloaded_files:
raise ValueError("No files to load. Run fetch() first.")
print(f"📂 Loading {len(self.downloaded_files)} W5E5 files...")
# Group files by variable
files_by_var = {}
for fpath in self.downloaded_files:
# Determine which variable this file contains
for var in self.cfg.variables:
if f"/{var}/" in fpath or f"_{self._map_variable_name(var)}_" in fpath:
if var not in files_by_var:
files_by_var[var] = []
files_by_var[var].append(fpath)
break
# Load each variable separately and merge
datasets = []
for var, file_list in files_by_var.items():
print(f" Loading {var} from {len(file_list)} file(s)...")
if len(file_list) == 1:
ds_var = xr.open_dataset(file_list[0])
else:
# Multiple files - open as multi-file dataset
ds_var = xr.open_mfdataset(
file_list,
combine='by_coords',
parallel=True
)
datasets.append(ds_var)
# Merge all variables into one dataset
if len(datasets) == 1:
self.ds = datasets[0]
else:
self.ds = xr.merge(datasets)
# Subset to requested time range
start = self.cfg.time_range.start_date
end = self.cfg.time_range.end_date
self.ds = self.ds.sel(time=slice(start, end))
# Add metadata
self.ds.attrs['source'] = 'W5E5 via ISIMIP'
self.ds.attrs['dataset'] = 'W5E5v2.0'
self.ds.attrs['description'] = 'WFDE5 over land merged with ERA5 over ocean'
print(f"✅ Loaded dataset with {len(self.ds.data_vars)} variables")
save_csv(self, filename)
¶
Write the dataset to CSV, creating parent directories.
The frame is written in xarray's wide form — one column per variable,
indexed by the dataset's dimensions — not the long form produced by
:meth:~climdata.utils.wrapper_workflow.ClimateExtractor.to_dataframe.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
filename |
str |
Destination path. |
required |
Returns:
| Type | Description |
|---|---|
None |
Exceptions:
| Type | Description |
|---|---|
ValueError |
If no dataset has been loaded. |
Source code in climdata/datasets/W5E5.py
def save_csv(self, filename: str):
"""Write the dataset to CSV, creating parent directories.
The frame is written in xarray's wide form — one column per variable,
indexed by the dataset's dimensions — not the long form produced by
:meth:`~climdata.utils.wrapper_workflow.ClimateExtractor.to_dataframe`.
Args:
filename (str): Destination path.
Returns:
None
Raises:
ValueError: If no dataset has been loaded.
"""
if self.ds is None:
raise ValueError("No dataset loaded. Run load() first.")
output_path = Path(filename)
output_path.parent.mkdir(parents=True, exist_ok=True)
df = self.ds.to_dataframe()
df.to_csv(output_path)
print(f"💾 Saved to: {output_path}")
save_netcdf(self, filename)
¶
Write the dataset to NetCDF, creating parent directories.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
filename |
str |
Destination path. |
required |
Returns:
| Type | Description |
|---|---|
None |
Exceptions:
| Type | Description |
|---|---|
ValueError |
If no dataset has been loaded. |
Source code in climdata/datasets/W5E5.py
def save_netcdf(self, filename: str):
"""Write the dataset to NetCDF, creating parent directories.
Args:
filename (str): Destination path.
Returns:
None
Raises:
ValueError: If no dataset has been loaded.
"""
if self.ds is None:
raise ValueError("No dataset loaded. Run load() first.")
output_path = Path(filename)
output_path.parent.mkdir(parents=True, exist_ok=True)
self.ds.to_netcdf(output_path)
print(f"💾 Saved to: {output_path}")
climdata.datasets.NASAPOWER.POWER
¶
Fetch and assemble a NASA POWER daily time series for one point.
Follows the same fetch → load → extract lifecycle as the gridded
providers, so :class:~climdata.utils.wrapper_workflow.ClimateExtractor can
drive it identically. Because the API is point-based, extract only
narrows the time axis; there is no spatial subsetting to do.
Variable names are translated through cfg.dsinfo.POWER.variables: CF names
(tasmax, pr) on the climdata side, POWER api_id codes
(T2M_MAX, PRECTOTCORR) on the wire.
Attributes:
| Name | Type | Description |
|---|---|---|
cfg |
DictConfig |
Hydra configuration supplying |
raw |
dict | None |
The decoded JSON response, set by :meth: |
ds |
xr.Dataset | None |
The assembled dataset, set by :meth: |
Source code in climdata/datasets/NASAPOWER.py
class POWER:
"""Fetch and assemble a NASA POWER daily time series for one point.
Follows the same ``fetch`` → ``load`` → ``extract`` lifecycle as the gridded
providers, so :class:`~climdata.utils.wrapper_workflow.ClimateExtractor` can
drive it identically. Because the API is point-based, ``extract`` only
narrows the time axis; there is no spatial subsetting to do.
Variable names are translated through ``cfg.dsinfo.POWER.variables``: CF names
(``tasmax``, ``pr``) on the climdata side, POWER ``api_id`` codes
(``T2M_MAX``, ``PRECTOTCORR``) on the wire.
Attributes:
cfg (DictConfig): Hydra configuration supplying ``lat``, ``lon``,
``variables``, ``time_range`` and ``dsinfo.POWER``.
raw (dict | None): The decoded JSON response, set by :meth:`fetch`.
ds (xr.Dataset | None): The assembled dataset, set by :meth:`load`.
"""
def __init__(self, cfg):
"""Bind a configuration.
No network call is made here.
Args:
cfg (DictConfig): Configuration with ``lat``, ``lon``, ``variables``,
``time_range.start_date`` / ``.end_date``, and a
``dsinfo.POWER.variables`` mapping.
"""
self.cfg = cfg
self.raw = None
self.ds = None
# ---------------------------------------------------------
# Step 1: Fetch data from NASA POWER
# ---------------------------------------------------------
def fetch(self):
"""Request the daily point series from the POWER API.
Builds a single query for all configured variables over the full
configured date range and stores the decoded JSON on :attr:`raw`.
Returns:
None: The response is stored on :attr:`raw`.
Raises:
requests.HTTPError: If the API returns a non-2xx status — most often
an unsupported variable code or an out-of-range date.
KeyError: If a configured variable has no ``api_id`` in
``cfg.dsinfo.POWER.variables``.
"""
lat = self.cfg.lat
lon = self.cfg.lon
params = self.cfg.variables
api_vars = ",".join([
self.cfg.dsinfo.POWER.variables[v].api_id
for v in params
])
start = self.cfg.time_range.start_date.replace("-", "")
end = self.cfg.time_range.end_date.replace("-", "")
url = (
"https://power.larc.nasa.gov/api/temporal/daily/point"
f"?parameters={api_vars}"
f"&community=AG"
f"&latitude={lat}"
f"&longitude={lon}"
f"&start={start}"
f"&end={end}"
f"&format=JSON"
)
r = requests.get(url)
r.raise_for_status()
self.raw = r.json()
# ---------------------------------------------------------
# Step 2: Load JSON into xarray Dataset
# ---------------------------------------------------------
def load(self):
"""Convert the fetched JSON into an :class:`xarray.Dataset`.
Renames POWER ``api_id`` columns back to CF names, parses the
``YYYYMMDD`` keys into a ``time`` coordinate, attaches the request's
latitude and longitude as scalar coordinates, and copies ``long_name``
and ``units`` from the configuration onto each variable.
POWER encodes gaps as the sentinel ``-999``; those values are passed
through unchanged, so screen for them before computing statistics.
Returns:
None: The dataset is stored on :attr:`ds`.
Raises:
TypeError: If :meth:`fetch` has not been called (``raw`` is ``None``).
KeyError: If the response carries no ``properties.parameter`` block,
which is how the API reports a rejected request.
"""
data = self.raw["properties"]["parameter"]
df = pd.DataFrame(data)
df.index = pd.to_datetime(df.index, format="%Y%m%d")
var_map = {
v.api_id: cmip_id
for cmip_id, v in self.cfg.dsinfo.POWER.variables.items()
}
df = df.rename(columns=var_map)
self.ds = xr.Dataset.from_dataframe(df)
# Add coords
self.ds = self.ds.assign_coords(
latitude=self.cfg.lat,
longitude=self.cfg.lon
)
self.ds = self.ds.rename({"index":"time"})
self.ds.latitude.attrs["units"] = "degrees_north"
self.ds.longitude.attrs["units"] = "degrees_east"
for cmip_id in self.ds.data_vars:
if cmip_id in self.cfg.dsinfo.POWER.variables:
vinfo = self.cfg.dsinfo.POWER.variables[cmip_id]
self.ds[cmip_id].attrs.update({
"long_name": vinfo.long_name,
"units": vinfo.units,
"source": "NASA POWER",
"api_id": vinfo.api_id,
})
# ---------------------------------------------------------
# Step 3: Extract (temporal subsetting etc.)
# ---------------------------------------------------------
def extract(self, start=None, end=None):
"""Narrow the loaded dataset to a time window.
There is no spatial argument: the API already returned exactly one point.
Calling with neither bound is a no-op, which is what lets the generic
workflow invoke ``extract`` unconditionally.
Args:
start (str | datetime, optional): Inclusive start of the window.
Open-ended if ``None``.
end (str | datetime, optional): Inclusive end of the window.
Open-ended if ``None``.
Returns:
None: :attr:`ds` is replaced in place.
"""
if start or end:
self.ds = self.ds.sel(time=slice(start, end))
# ---------------------------------------------------------
# Step 4: Save methods (same API as CMIP)
# ---------------------------------------------------------
def save_netcdf(self, filename):
"""Write the loaded dataset to NetCDF.
Args:
filename (str): Destination path. The parent directory must exist.
Returns:
None
"""
self.ds.to_netcdf(filename)
def save_csv(self, filename):
"""Write the loaded dataset to CSV, one row per timestep.
Args:
filename (str): Destination path. The parent directory must exist.
Returns:
None
"""
self.ds.to_dataframe().to_csv(filename)
__init__(self, cfg)
special
¶
Bind a configuration.
No network call is made here.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
cfg |
DictConfig |
Configuration with |
required |
Source code in climdata/datasets/NASAPOWER.py
def __init__(self, cfg):
"""Bind a configuration.
No network call is made here.
Args:
cfg (DictConfig): Configuration with ``lat``, ``lon``, ``variables``,
``time_range.start_date`` / ``.end_date``, and a
``dsinfo.POWER.variables`` mapping.
"""
self.cfg = cfg
self.raw = None
self.ds = None
extract(self, start=None, end=None)
¶
Narrow the loaded dataset to a time window.
There is no spatial argument: the API already returned exactly one point.
Calling with neither bound is a no-op, which is what lets the generic
workflow invoke extract unconditionally.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
start |
str | datetime |
Inclusive start of the window.
Open-ended if |
None |
end |
str | datetime |
Inclusive end of the window.
Open-ended if |
None |
Returns:
| Type | Description |
|---|---|
None |
:attr: |
Source code in climdata/datasets/NASAPOWER.py
def extract(self, start=None, end=None):
"""Narrow the loaded dataset to a time window.
There is no spatial argument: the API already returned exactly one point.
Calling with neither bound is a no-op, which is what lets the generic
workflow invoke ``extract`` unconditionally.
Args:
start (str | datetime, optional): Inclusive start of the window.
Open-ended if ``None``.
end (str | datetime, optional): Inclusive end of the window.
Open-ended if ``None``.
Returns:
None: :attr:`ds` is replaced in place.
"""
if start or end:
self.ds = self.ds.sel(time=slice(start, end))
fetch(self)
¶
Request the daily point series from the POWER API.
Builds a single query for all configured variables over the full
configured date range and stores the decoded JSON on :attr:raw.
Returns:
| Type | Description |
|---|---|
None |
The response is stored on :attr: |
Exceptions:
| Type | Description |
|---|---|
requests.HTTPError |
If the API returns a non-2xx status — most often an unsupported variable code or an out-of-range date. |
KeyError |
If a configured variable has no |
Source code in climdata/datasets/NASAPOWER.py
def fetch(self):
"""Request the daily point series from the POWER API.
Builds a single query for all configured variables over the full
configured date range and stores the decoded JSON on :attr:`raw`.
Returns:
None: The response is stored on :attr:`raw`.
Raises:
requests.HTTPError: If the API returns a non-2xx status — most often
an unsupported variable code or an out-of-range date.
KeyError: If a configured variable has no ``api_id`` in
``cfg.dsinfo.POWER.variables``.
"""
lat = self.cfg.lat
lon = self.cfg.lon
params = self.cfg.variables
api_vars = ",".join([
self.cfg.dsinfo.POWER.variables[v].api_id
for v in params
])
start = self.cfg.time_range.start_date.replace("-", "")
end = self.cfg.time_range.end_date.replace("-", "")
url = (
"https://power.larc.nasa.gov/api/temporal/daily/point"
f"?parameters={api_vars}"
f"&community=AG"
f"&latitude={lat}"
f"&longitude={lon}"
f"&start={start}"
f"&end={end}"
f"&format=JSON"
)
r = requests.get(url)
r.raise_for_status()
self.raw = r.json()
load(self)
¶
Convert the fetched JSON into an :class:xarray.Dataset.
Renames POWER api_id columns back to CF names, parses the
YYYYMMDD keys into a time coordinate, attaches the request's
latitude and longitude as scalar coordinates, and copies long_name
and units from the configuration onto each variable.
POWER encodes gaps as the sentinel -999; those values are passed
through unchanged, so screen for them before computing statistics.
Returns:
| Type | Description |
|---|---|
None |
The dataset is stored on :attr: |
Exceptions:
| Type | Description |
|---|---|
TypeError |
If :meth: |
KeyError |
If the response carries no |
Source code in climdata/datasets/NASAPOWER.py
def load(self):
"""Convert the fetched JSON into an :class:`xarray.Dataset`.
Renames POWER ``api_id`` columns back to CF names, parses the
``YYYYMMDD`` keys into a ``time`` coordinate, attaches the request's
latitude and longitude as scalar coordinates, and copies ``long_name``
and ``units`` from the configuration onto each variable.
POWER encodes gaps as the sentinel ``-999``; those values are passed
through unchanged, so screen for them before computing statistics.
Returns:
None: The dataset is stored on :attr:`ds`.
Raises:
TypeError: If :meth:`fetch` has not been called (``raw`` is ``None``).
KeyError: If the response carries no ``properties.parameter`` block,
which is how the API reports a rejected request.
"""
data = self.raw["properties"]["parameter"]
df = pd.DataFrame(data)
df.index = pd.to_datetime(df.index, format="%Y%m%d")
var_map = {
v.api_id: cmip_id
for cmip_id, v in self.cfg.dsinfo.POWER.variables.items()
}
df = df.rename(columns=var_map)
self.ds = xr.Dataset.from_dataframe(df)
# Add coords
self.ds = self.ds.assign_coords(
latitude=self.cfg.lat,
longitude=self.cfg.lon
)
self.ds = self.ds.rename({"index":"time"})
self.ds.latitude.attrs["units"] = "degrees_north"
self.ds.longitude.attrs["units"] = "degrees_east"
for cmip_id in self.ds.data_vars:
if cmip_id in self.cfg.dsinfo.POWER.variables:
vinfo = self.cfg.dsinfo.POWER.variables[cmip_id]
self.ds[cmip_id].attrs.update({
"long_name": vinfo.long_name,
"units": vinfo.units,
"source": "NASA POWER",
"api_id": vinfo.api_id,
})
save_csv(self, filename)
¶
Write the loaded dataset to CSV, one row per timestep.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
filename |
str |
Destination path. The parent directory must exist. |
required |
Returns:
| Type | Description |
|---|---|
None |
Source code in climdata/datasets/NASAPOWER.py
def save_csv(self, filename):
"""Write the loaded dataset to CSV, one row per timestep.
Args:
filename (str): Destination path. The parent directory must exist.
Returns:
None
"""
self.ds.to_dataframe().to_csv(filename)
save_netcdf(self, filename)
¶
Write the loaded dataset to NetCDF.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
filename |
str |
Destination path. The parent directory must exist. |
required |
Returns:
| Type | Description |
|---|---|
None |
Source code in climdata/datasets/NASAPOWER.py
def save_netcdf(self, filename):
"""Write the loaded dataset to NetCDF.
Args:
filename (str): Destination path. The parent directory must exist.
Returns:
None
"""
self.ds.to_netcdf(filename)
Observations, Germany¶
climdata.datasets.HYRAS.HYRASmirror
¶
Download, open and subset HYRAS grids across both of its file layouts.
Call :meth:extract to record the region of interest, then :meth:load,
which downloads what is missing and applies the subset. Each of the three
subsetting modes takes a different route, chosen for cost:
- point — resolved per file inside
open_mfdataset'spreprocesshook, so only one cell of each file is ever materialised. - box — index bounds computed once from a sample file, then applied to
each file opened individually and concatenated. Files are opened one at a
time rather than through
open_mfdatasetbecause the index bounds must be known before the first file is read. - shapefile — a mask rasterised once, then applied with
where. Unlike the box path this keeps the full grid extent and blanks cells outside the geometry, so the result has the source's spatial shape.
Loading also normalises two unit strings DWD writes in a form xclim does not
accept: pr in "mm" becomes "mm/day", and hurs in
"Percent" becomes "%".
Attributes:
| Name | Type | Description |
|---|---|---|
cfg |
DictConfig |
Hydra configuration. |
variables |
list[str] |
|
files |
list[str] |
Local paths resolved by the last :meth: |
dataset |
xr.Dataset | None |
The loaded dataset, set by :meth: |
Examples:
>>> hyras = HYRASmirror(cfg)
>>> hyras.extract(box={"lat_min": 47, "lat_max": 55,
... "lon_min": 6, "lon_max": 15})
>>> ds = hyras.load("pr")
Source code in climdata/datasets/HYRAS.py
class HYRASmirror:
"""Download, open and subset HYRAS grids across both of its file layouts.
Call :meth:`extract` to record the region of interest, then :meth:`load`,
which downloads what is missing and applies the subset. Each of the three
subsetting modes takes a different route, chosen for cost:
* **point** — resolved per file inside ``open_mfdataset``'s ``preprocess``
hook, so only one cell of each file is ever materialised.
* **box** — index bounds computed once from a sample file, then applied to
each file opened individually and concatenated. Files are opened one at a
time rather than through ``open_mfdataset`` because the index bounds must
be known before the first file is read.
* **shapefile** — a mask rasterised once, then applied with ``where``. Unlike
the box path this keeps the full grid extent and blanks cells outside the
geometry, so the result has the source's spatial shape.
Loading also normalises two unit strings DWD writes in a form xclim does not
accept: ``pr`` in ``"mm"`` becomes ``"mm/day"``, and ``hurs`` in
``"Percent"`` becomes ``"%"``.
Attributes:
cfg (DictConfig): Hydra configuration.
variables (list[str]): ``cfg.variables``.
files (list[str]): Local paths resolved by the last :meth:`fetch`.
dataset (xr.Dataset | None): The loaded dataset, set by :meth:`load`.
Example:
>>> hyras = HYRASmirror(cfg) # doctest: +SKIP
>>> hyras.extract(box={"lat_min": 47, "lat_max": 55, # doctest: +SKIP
... "lon_min": 6, "lon_max": 15})
>>> ds = hyras.load("pr") # doctest: +SKIP
"""
def __init__(self, cfg: DictConfig):
"""Bind a configuration.
Args:
cfg (DictConfig): Configuration with ``dataset``, ``data_dir``,
``variables``, ``time_range`` and ``dsinfo.HYRAS``.
"""
self.cfg = cfg
self.dataset: Optional[xr.Dataset] = None
self.variables = cfg.variables
self.files: List[str] = []
# extraction state
self._extract_mode: Optional[str] = None
self._extract_params = None
# cached grid helpers (computed from first file when needed)
self._cached_box_idx = None # (y0,y1,x0,x1)
self._cached_mask = None # 2D boolean mask for shapefile
self._cached_lonlat_info = None # (lon_min, lon_max, lat_min, lat_max, nx, ny)
# --------------------------
# File discovery / fetch
# --------------------------
def fetch(self, variable: str) -> List[str]:
"""Download a variable's files and list the local paths now available.
Args:
variable (str): CF variable name, e.g. ``"pr"``.
Returns:
list[str]: Local paths in chronological order — monthly ``.tgz``
archives or yearly ``.nc`` files depending on the variable. Also
stored on :attr:`files`.
"""
# keep your fetch behavior (calls fetch_dwd)
fetch_dwd(self.cfg, variable)
provider = self.cfg.dataset.upper()
param_info = self.cfg.dsinfo[provider]['variables'][variable]
prefix = param_info["prefix"]
version = param_info["version"]
start_date = datetime.fromisoformat(self.cfg.time_range.start_date)
end_date = datetime.fromisoformat(self.cfg.time_range.end_date)
start_year = start_date.year
start_month = start_date.month
end_year = end_date.year
end_month = end_date.month
files = []
var_dir = os.path.join(self.cfg.data_dir, provider.upper(), variable.upper())
# Determine if this is a newer dataset (uses .tgz/.asc with YYYYMM naming)
is_new_format = prefix in ["grids_germany_daily_evapo_p", "grids_germany_daily_soil_moist", "grids_germany_daily_soil_temperature_5cm"]
if is_new_format:
# Return .tgz paths directly — members are read in-memory at load time
for year in range(start_year, end_year + 1):
start_m = start_month if year == start_year else 1
end_m = end_month if year == end_year else 12
for month in range(start_m, end_m + 1):
tgz_name = f"{prefix}_{year}{month:02d}.tgz"
tgz_path = os.path.join(var_dir, tgz_name)
if os.path.exists(tgz_path):
files.append(tgz_path)
else:
# Handle old format: *_{year}_{version}_de.nc
for year in range(start_year, end_year + 1):
file_name = f"{prefix}_{year}_{version}_de.nc"
file_path = os.path.join(var_dir, file_name)
if os.path.exists(file_path):
files.append(file_path)
self.files = files
return files
# --------------------------
# Public extract setter
# --------------------------
def extract(self, *, point: Tuple[float, float] = None, box: Dict[str, float] = None,
shapefile: str = None, buffer_km: float = 0.0):
"""Record the region of interest for :meth:`load` to apply.
Nothing is read here, and any cached grid indices from a previous call
are discarded. The three modes are mutually exclusive; the first supplied
wins, in the order ``point``, ``box``, ``shapefile``.
Args:
point (tuple[float, float], optional): ``(lon, lat)`` in degrees,
longitude first. Selects the single nearest grid cell.
box (dict, optional): Bounding box; all four of ``lat_min``,
``lat_max``, ``lon_min``, ``lon_max`` are required.
shapefile (str | geopandas.GeoDataFrame, optional): Polygon(s) to
mask to, as a path or an in-memory frame.
buffer_km (float): Dilation applied to ``shapefile`` geometries in
Web Mercator. Ignored for ``point`` and ``box``. Defaults to ``0.0``.
Returns:
HYRASmirror: ``self``, so the call can be chained into :meth:`load`.
Raises:
ValueError: If no geometry was given, or if ``box`` is missing a key.
"""
if point is not None:
lon, lat = point
self._extract_mode = "point"
self._extract_params = (lon, lat)
elif box is not None:
# expect keys lat_min, lat_max, lon_min, lon_max
for k in ("lat_min", "lat_max", "lon_min", "lon_max"):
if k not in box:
raise ValueError(f"box missing key {k}")
self._extract_mode = "box"
self._extract_params = box
elif shapefile is not None:
gdf = gpd.read_file(shapefile) if isinstance(shapefile, str) else shapefile
if buffer_km > 0:
gdf = gdf.to_crs(epsg=3857)
gdf["geometry"] = gdf.buffer(buffer_km * 1000)
gdf = gdf.to_crs(epsg=4326)
self._extract_mode = "shapefile"
self._extract_params = gdf
else:
raise ValueError("Must provide point, box, or shapefile.")
# Clear cached helpers when extraction changes
self._cached_box_idx = None
self._cached_mask = None
self._cached_lonlat_info = None
return self
# --------------------------
# Helpers to compute indices/mask from a sample file
# --------------------------
def _load_sample_grid(self, sample_file: str, varname: Optional[str] = None):
"""Read just the coordinate arrays from one file.
Opens with ``decode_times=False`` and touches only the coordinates, so
no data array is materialised.
Args:
sample_file (str): Path to a HYRAS NetCDF file.
varname (str, optional): Unused; accepted for call-site symmetry.
Returns:
tuple[numpy.ndarray, numpy.ndarray]: ``(lat, lon)``, each 1-D or 2-D
depending on the file.
"""
ds = xr.open_dataset(sample_file, engine="netcdf4", decode_times=False)
# Try to access coordinates in common names; adapt if your files differ.
# Accept either 1D 'lat','lon' or 2D 'lat','lon' on dims (y,x).
if ("lat" in ds.coords) and ("lon" in ds.coords):
lat = ds["lat"].values
lon = ds["lon"].values
else:
# fallback: if coordinates stored as variables
lat = ds["lat"].values
lon = ds["lon"].values
ds.close()
return lat, lon
def _compute_box_indices(self, sample_file: str):
"""Resolve the requested box to cached integer index bounds.
Maps the box's two opposite corners to grid indices through
:func:`find_nearest_xy` and takes the enclosing rectangle. Because the
grid is curvilinear, that rectangle is an approximation of the
geographic box, and a rotated domain yields slightly more area than
asked for. The result is cached until :meth:`extract` is called again.
Args:
sample_file (str): Path to a file whose grid represents the request.
Returns:
tuple[int, int, int, int]: ``(y0, y1, x0, x1)`` as Python slice
endpoints, so ``y1`` and ``x1`` are exclusive.
"""
if self._cached_box_idx is not None:
return self._cached_box_idx
lat, lon = self._load_sample_grid(sample_file)
box = self._extract_params
# find_nearest_xy expects ds-like input; we'll open a tiny ds for indices
ds_sample = xr.open_dataset(sample_file, engine="netcdf4", decode_times=False)
iy_min, ix_min = find_nearest_xy(ds_sample, box["lat_min"], box["lon_min"])
iy_max, ix_max = find_nearest_xy(ds_sample, box["lat_max"], box["lon_max"])
ds_sample.close()
y0, y1 = sorted([iy_min, iy_max])
x0, x1 = sorted([ix_min, ix_max])
self._cached_box_idx = (y0, y1 + 1, x0, x1 + 1) # python slice endpoints
return self._cached_box_idx
def _compute_shapefile_mask(self, sample_file: str):
"""Rasterise the requested geometry onto the grid, and cache the mask.
The transform is derived from the coordinate bounding box and the grid
shape, which treats the curvilinear grid as regular in lat/lon. Over
Germany the distortion is under a cell; over a larger or more rotated
domain it would not be.
Args:
sample_file (str): Path to a file whose grid represents the request.
Returns:
numpy.ndarray: Boolean mask of shape ``(ny, nx)``, ``True`` inside
the geometry.
Raises:
RuntimeError: If the extraction mode is not ``"shapefile"``, or the
coordinate arrays are neither 1-D nor 2-D.
"""
if self._cached_mask is not None:
return self._cached_mask
if self._extract_mode != "shapefile":
raise RuntimeError("shapefile mask requested but extract mode is not 'shapefile'")
gdf = self._extract_params
lat, lon = self._load_sample_grid(sample_file)
# handle 1D or 2D lat/lon
if lat.ndim == 1 and lon.ndim == 1:
ny = lat.size
nx = lon.size
lon_min, lon_max = float(lon.min()), float(lon.max())
lat_min, lat_max = float(lat.min()), float(lat.max())
elif lat.ndim == 2 and lon.ndim == 2:
ny, nx = lat.shape
lon_min, lon_max = float(lon.min()), float(lon.max())
lat_min, lat_max = float(lat.min()), float(lat.max())
else:
raise RuntimeError("Unsupported lat/lon shapes for rasterization")
transform = from_bounds(lon_min, lat_min, lon_max, lat_max, nx, ny)
shapes = ((mapping(geom), 1) for geom in gdf.geometry)
mask = rfeatures.rasterize(
shapes=shapes,
out_shape=(ny, nx),
transform=transform,
fill=0,
default_value=1,
dtype="uint8",
).astype(bool)
self._cached_mask = mask
# also cache lon/lat info (useful if needed)
self._cached_lonlat_info = (lon_min, lon_max, lat_min, lat_max, nx, ny)
return mask
# --------------------------
# Core loading logic
# --------------------------
def _apply_time_subset(self, ds):
"""Clip a dataset to the configured time range.
Yearly and monthly files always overrun the request at both ends. If the
time coordinate has not been decoded, ``decode_cf`` is applied and the
slice retried; if that also fails the dataset is returned unclipped, so a
surprising time encoding costs extra timesteps rather than the whole load.
Args:
ds (xr.Dataset): Dataset with a ``time`` dimension.
Returns:
xr.Dataset: The clipped dataset, or ``ds`` unchanged on failure.
"""
start = getattr(self.cfg.time_range, "start_date", None)
end = getattr(self.cfg.time_range, "end_date", None)
if start or end:
try:
ds = ds.sel(time=slice(start, end))
except Exception:
# If time coord not decoded yet, try forcing decode then slice
try:
ds = xr.decode_cf(ds)
ds = ds.sel(time=slice(start, end))
except Exception:
# fallback: return ds unchanged
pass
return ds
def load(self, variable: str, use_dask: bool = True, chunking: dict = None):
"""Download what is missing, then open the variable with the subset applied.
Dispatches on file layout first (``.tgz`` archives versus yearly NetCDF)
and then on extraction mode; see the class docstring for why each mode
takes the route it does. Unit strings are normalised and the result is
clipped to the configured time range.
Args:
variable (str): CF variable name, e.g. ``"pr"``.
use_dask (bool): Whether ``chunking`` is applied. Defaults to ``True``.
chunking (dict, optional): Chunk sizes, e.g. ``{"time": "auto"}``.
Only used when ``use_dask`` is true.
Returns:
xr.Dataset: The subset dataset, also stored on :attr:`dataset`.
Dimensions are ``(time, y, x)``, or ``(time,)`` in point mode.
Raises:
FileNotFoundError: If no files exist for the variable and period.
"""
files = self.fetch(variable)
if not files:
raise FileNotFoundError(f"No files found for variable {variable}")
mode = self._extract_mode
# Check if files are .tgz archives (read members in-memory)
is_tgz_format = any(f.endswith('.tgz') for f in files)
if is_tgz_format:
tgz_files = [f for f in files if f.endswith('.tgz')]
provider = self.cfg.dataset.upper()
units = None
try:
param_info = self.cfg.dsinfo[provider]['variables'].get(variable, {})
units = param_info.get('units')
except (AttributeError, KeyError, TypeError):
pass
dset = read_asc_timeseries_from_tgz(tgz_files, varname=variable, units=units)
# Apply same spatial extraction as the .asc branch below
original_attrs = dset.attrs.copy()
if mode == "point":
lon_pt, lat_pt = self._extract_params
ix = np.abs(dset['x'].values - lon_pt).argmin()
iy = np.abs(dset['y'].values - lat_pt).argmin()
dset = dset.isel(x=ix, y=iy)
elif mode == "box":
box = self._extract_params
lat_2d = dset['lat'].values
lon_2d = dset['lon'].values
mask = (
(lat_2d >= box['lat_min']) & (lat_2d <= box['lat_max']) &
(lon_2d >= box['lon_min']) & (lon_2d <= box['lon_max'])
)
mask_da = xr.DataArray(mask, dims=('y', 'x'),
coords={'y': dset['y'], 'x': dset['x']})
dset = dset.where(mask_da, drop=True)
dset.attrs.update(original_attrs)
start_date = getattr(self.cfg.time_range, "start_date", None)
end_date = getattr(self.cfg.time_range, "end_date", None)
if start_date or end_date:
try:
dset = dset.sel(time=slice(start_date, end_date))
except Exception as e:
print(f"⚠️ Time subsetting failed: {e}")
self.dataset = dset
return dset
# -------------------------
# POINT mode: preprocess per-file and use open_mfdataset
# -------------------------
if mode == "point":
lon, lat = self._extract_params
def preprocess_point(ds):
iy, ix = find_nearest_xy(ds, lat, lon)
# ensure dimension order if present
if variable in ds:
try:
ds[variable] = ds[variable].transpose("time", "y", "x")
ds["time"] = ds["time"].dt.floor("D")
except Exception:
pass
# point selection via nearest index (fast)
return ds.isel(y=iy, x=ix)
dset = xr.open_mfdataset(
files,
combine="nested",
concat_dim="time",
preprocess=preprocess_point,
engine="netcdf4",
parallel=False,
)
if use_dask and chunking:
dset = dset.chunk(chunking)
# normalize pr units
if "pr" in dset:
if dset["pr"].attrs.get("units") == "mm":
dset["pr"].attrs["units"] = "mm/day"
if "hurs" in dset:
if dset["hurs"].attrs.get("units") == "Percent":
dset["hurs"].attrs["units"] = "%"
self.dataset = dset
return dset
# -------------------------
# BOX or SHAPEFILE mode: compute indices/mask once and apply per-file
# -------------------------
elif mode in ("box", "shapefile"):
sample_file = files[0]
datasets = []
if mode == "box":
y0, y1, x0, x1 = self._compute_box_indices(sample_file)
else: # shapefile
mask = self._compute_shapefile_mask(sample_file)
for f in files:
# open each file lightly
ds = xr.open_dataset(f, engine="netcdf4", decode_times=True)
# ensure dims and variable layout
if variable in ds:
try:
ds[variable] = ds[variable].transpose("time", "y", "x")
ds["time"] = ds["time"].dt.floor("D")
except Exception:
pass
# apply slice or mask
if mode == "box":
sub = ds.isel(y=slice(y0, y1), x=slice(x0, x1))
else: # shapefile
mask_da = xr.DataArray(mask, dims=("y", "x"))
sub = ds.where(mask_da, drop=False)
# optionally chunk lazily
if use_dask and chunking:
sub = sub.chunk(chunking)
datasets.append(sub)
# concatenate along time
dset = xr.concat(datasets, dim="time", combine_attrs="override")
# normalize pr units
if "pr" in dset:
if dset["pr"].attrs.get("units") == "mm":
dset["pr"].attrs["units"] = "mm/day"
if "hurs" in dset:
if dset["hurs"].attrs.get("units") == "Percent":
dset["hurs"].attrs["units"] = "%"
dset = self._apply_time_subset(dset)
self.dataset = dset
return dset
else:
# no extraction mode -> just open normally (light transpose)
def preprocess_identity(ds):
if variable in ds:
try:
ds[variable] = ds[variable].transpose("time", "y", "x")
ds["time"] = ds["time"].dt.floor("D")
except Exception:
pass
return ds
dset = xr.open_mfdataset(
files,
combine="nested",
concat_dim="time",
preprocess=preprocess_identity,
engine="netcdf4",
parallel=False,
)
if use_dask and chunking:
dset = dset.chunk(chunking)
if "pr" in dset:
if dset["pr"].attrs.get("units") == "mm":
dset["pr"].attrs["units"] = "mm/day"
if "hurs" in dset:
if dset["hurs"].attrs.get("units") == "Percent":
dset["hurs"].attrs["units"] = "%"
self.dataset = dset
return dset
# --------------------------
# Utility: save current dataset to CSV
# --------------------------
def save_csv(self, filename: str, df: pd.DataFrame = None):
"""Write a frame, or the loaded dataset, to CSV.
A gridded HYRAS dataset flattens to one row per cell per timestep, so
this can be very large for anything but a point or a small box.
Args:
filename (str): Destination path. The parent directory must exist.
df (pd.DataFrame, optional): Frame to write. Defaults to the loaded
dataset converted with ``to_dataframe()``.
Returns:
None
Raises:
ValueError: If no frame is given and no dataset is loaded.
"""
if df is None:
if self.dataset is None:
raise ValueError("No dataset loaded")
# convert to dataframe (may be large)
df = self.dataset.to_dataframe().reset_index()
df.to_csv(filename, index=False)
print(f"Saved CSV to {filename}")
__init__(self, cfg)
special
¶
Bind a configuration.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
cfg |
DictConfig |
Configuration with |
required |
Source code in climdata/datasets/HYRAS.py
def __init__(self, cfg: DictConfig):
"""Bind a configuration.
Args:
cfg (DictConfig): Configuration with ``dataset``, ``data_dir``,
``variables``, ``time_range`` and ``dsinfo.HYRAS``.
"""
self.cfg = cfg
self.dataset: Optional[xr.Dataset] = None
self.variables = cfg.variables
self.files: List[str] = []
# extraction state
self._extract_mode: Optional[str] = None
self._extract_params = None
# cached grid helpers (computed from first file when needed)
self._cached_box_idx = None # (y0,y1,x0,x1)
self._cached_mask = None # 2D boolean mask for shapefile
self._cached_lonlat_info = None # (lon_min, lon_max, lat_min, lat_max, nx, ny)
extract(self, *, point=None, box=None, shapefile=None, buffer_km=0.0)
¶
Record the region of interest for :meth:load to apply.
Nothing is read here, and any cached grid indices from a previous call
are discarded. The three modes are mutually exclusive; the first supplied
wins, in the order point, box, shapefile.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
point |
tuple[float, float] |
|
None |
box |
dict |
Bounding box; all four of |
None |
shapefile |
str | geopandas.GeoDataFrame |
Polygon(s) to mask to, as a path or an in-memory frame. |
None |
buffer_km |
float |
Dilation applied to |
0.0 |
Returns:
| Type | Description |
|---|---|
HYRASmirror |
|
Exceptions:
| Type | Description |
|---|---|
ValueError |
If no geometry was given, or if |
Source code in climdata/datasets/HYRAS.py
def extract(self, *, point: Tuple[float, float] = None, box: Dict[str, float] = None,
shapefile: str = None, buffer_km: float = 0.0):
"""Record the region of interest for :meth:`load` to apply.
Nothing is read here, and any cached grid indices from a previous call
are discarded. The three modes are mutually exclusive; the first supplied
wins, in the order ``point``, ``box``, ``shapefile``.
Args:
point (tuple[float, float], optional): ``(lon, lat)`` in degrees,
longitude first. Selects the single nearest grid cell.
box (dict, optional): Bounding box; all four of ``lat_min``,
``lat_max``, ``lon_min``, ``lon_max`` are required.
shapefile (str | geopandas.GeoDataFrame, optional): Polygon(s) to
mask to, as a path or an in-memory frame.
buffer_km (float): Dilation applied to ``shapefile`` geometries in
Web Mercator. Ignored for ``point`` and ``box``. Defaults to ``0.0``.
Returns:
HYRASmirror: ``self``, so the call can be chained into :meth:`load`.
Raises:
ValueError: If no geometry was given, or if ``box`` is missing a key.
"""
if point is not None:
lon, lat = point
self._extract_mode = "point"
self._extract_params = (lon, lat)
elif box is not None:
# expect keys lat_min, lat_max, lon_min, lon_max
for k in ("lat_min", "lat_max", "lon_min", "lon_max"):
if k not in box:
raise ValueError(f"box missing key {k}")
self._extract_mode = "box"
self._extract_params = box
elif shapefile is not None:
gdf = gpd.read_file(shapefile) if isinstance(shapefile, str) else shapefile
if buffer_km > 0:
gdf = gdf.to_crs(epsg=3857)
gdf["geometry"] = gdf.buffer(buffer_km * 1000)
gdf = gdf.to_crs(epsg=4326)
self._extract_mode = "shapefile"
self._extract_params = gdf
else:
raise ValueError("Must provide point, box, or shapefile.")
# Clear cached helpers when extraction changes
self._cached_box_idx = None
self._cached_mask = None
self._cached_lonlat_info = None
return self
fetch(self, variable)
¶
Download a variable's files and list the local paths now available.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
variable |
str |
CF variable name, e.g. |
required |
Returns:
| Type | Description |
|---|---|
list[str] |
Local paths in chronological order — monthly |
Source code in climdata/datasets/HYRAS.py
def fetch(self, variable: str) -> List[str]:
"""Download a variable's files and list the local paths now available.
Args:
variable (str): CF variable name, e.g. ``"pr"``.
Returns:
list[str]: Local paths in chronological order — monthly ``.tgz``
archives or yearly ``.nc`` files depending on the variable. Also
stored on :attr:`files`.
"""
# keep your fetch behavior (calls fetch_dwd)
fetch_dwd(self.cfg, variable)
provider = self.cfg.dataset.upper()
param_info = self.cfg.dsinfo[provider]['variables'][variable]
prefix = param_info["prefix"]
version = param_info["version"]
start_date = datetime.fromisoformat(self.cfg.time_range.start_date)
end_date = datetime.fromisoformat(self.cfg.time_range.end_date)
start_year = start_date.year
start_month = start_date.month
end_year = end_date.year
end_month = end_date.month
files = []
var_dir = os.path.join(self.cfg.data_dir, provider.upper(), variable.upper())
# Determine if this is a newer dataset (uses .tgz/.asc with YYYYMM naming)
is_new_format = prefix in ["grids_germany_daily_evapo_p", "grids_germany_daily_soil_moist", "grids_germany_daily_soil_temperature_5cm"]
if is_new_format:
# Return .tgz paths directly — members are read in-memory at load time
for year in range(start_year, end_year + 1):
start_m = start_month if year == start_year else 1
end_m = end_month if year == end_year else 12
for month in range(start_m, end_m + 1):
tgz_name = f"{prefix}_{year}{month:02d}.tgz"
tgz_path = os.path.join(var_dir, tgz_name)
if os.path.exists(tgz_path):
files.append(tgz_path)
else:
# Handle old format: *_{year}_{version}_de.nc
for year in range(start_year, end_year + 1):
file_name = f"{prefix}_{year}_{version}_de.nc"
file_path = os.path.join(var_dir, file_name)
if os.path.exists(file_path):
files.append(file_path)
self.files = files
return files
load(self, variable, use_dask=True, chunking=None)
¶
Download what is missing, then open the variable with the subset applied.
Dispatches on file layout first (.tgz archives versus yearly NetCDF)
and then on extraction mode; see the class docstring for why each mode
takes the route it does. Unit strings are normalised and the result is
clipped to the configured time range.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
variable |
str |
CF variable name, e.g. |
required |
use_dask |
bool |
Whether |
True |
chunking |
dict |
Chunk sizes, e.g. |
None |
Returns:
| Type | Description |
|---|---|
xr.Dataset |
The subset dataset, also stored on :attr: |
Exceptions:
| Type | Description |
|---|---|
FileNotFoundError |
If no files exist for the variable and period. |
Source code in climdata/datasets/HYRAS.py
def load(self, variable: str, use_dask: bool = True, chunking: dict = None):
"""Download what is missing, then open the variable with the subset applied.
Dispatches on file layout first (``.tgz`` archives versus yearly NetCDF)
and then on extraction mode; see the class docstring for why each mode
takes the route it does. Unit strings are normalised and the result is
clipped to the configured time range.
Args:
variable (str): CF variable name, e.g. ``"pr"``.
use_dask (bool): Whether ``chunking`` is applied. Defaults to ``True``.
chunking (dict, optional): Chunk sizes, e.g. ``{"time": "auto"}``.
Only used when ``use_dask`` is true.
Returns:
xr.Dataset: The subset dataset, also stored on :attr:`dataset`.
Dimensions are ``(time, y, x)``, or ``(time,)`` in point mode.
Raises:
FileNotFoundError: If no files exist for the variable and period.
"""
files = self.fetch(variable)
if not files:
raise FileNotFoundError(f"No files found for variable {variable}")
mode = self._extract_mode
# Check if files are .tgz archives (read members in-memory)
is_tgz_format = any(f.endswith('.tgz') for f in files)
if is_tgz_format:
tgz_files = [f for f in files if f.endswith('.tgz')]
provider = self.cfg.dataset.upper()
units = None
try:
param_info = self.cfg.dsinfo[provider]['variables'].get(variable, {})
units = param_info.get('units')
except (AttributeError, KeyError, TypeError):
pass
dset = read_asc_timeseries_from_tgz(tgz_files, varname=variable, units=units)
# Apply same spatial extraction as the .asc branch below
original_attrs = dset.attrs.copy()
if mode == "point":
lon_pt, lat_pt = self._extract_params
ix = np.abs(dset['x'].values - lon_pt).argmin()
iy = np.abs(dset['y'].values - lat_pt).argmin()
dset = dset.isel(x=ix, y=iy)
elif mode == "box":
box = self._extract_params
lat_2d = dset['lat'].values
lon_2d = dset['lon'].values
mask = (
(lat_2d >= box['lat_min']) & (lat_2d <= box['lat_max']) &
(lon_2d >= box['lon_min']) & (lon_2d <= box['lon_max'])
)
mask_da = xr.DataArray(mask, dims=('y', 'x'),
coords={'y': dset['y'], 'x': dset['x']})
dset = dset.where(mask_da, drop=True)
dset.attrs.update(original_attrs)
start_date = getattr(self.cfg.time_range, "start_date", None)
end_date = getattr(self.cfg.time_range, "end_date", None)
if start_date or end_date:
try:
dset = dset.sel(time=slice(start_date, end_date))
except Exception as e:
print(f"⚠️ Time subsetting failed: {e}")
self.dataset = dset
return dset
# -------------------------
# POINT mode: preprocess per-file and use open_mfdataset
# -------------------------
if mode == "point":
lon, lat = self._extract_params
def preprocess_point(ds):
iy, ix = find_nearest_xy(ds, lat, lon)
# ensure dimension order if present
if variable in ds:
try:
ds[variable] = ds[variable].transpose("time", "y", "x")
ds["time"] = ds["time"].dt.floor("D")
except Exception:
pass
# point selection via nearest index (fast)
return ds.isel(y=iy, x=ix)
dset = xr.open_mfdataset(
files,
combine="nested",
concat_dim="time",
preprocess=preprocess_point,
engine="netcdf4",
parallel=False,
)
if use_dask and chunking:
dset = dset.chunk(chunking)
# normalize pr units
if "pr" in dset:
if dset["pr"].attrs.get("units") == "mm":
dset["pr"].attrs["units"] = "mm/day"
if "hurs" in dset:
if dset["hurs"].attrs.get("units") == "Percent":
dset["hurs"].attrs["units"] = "%"
self.dataset = dset
return dset
# -------------------------
# BOX or SHAPEFILE mode: compute indices/mask once and apply per-file
# -------------------------
elif mode in ("box", "shapefile"):
sample_file = files[0]
datasets = []
if mode == "box":
y0, y1, x0, x1 = self._compute_box_indices(sample_file)
else: # shapefile
mask = self._compute_shapefile_mask(sample_file)
for f in files:
# open each file lightly
ds = xr.open_dataset(f, engine="netcdf4", decode_times=True)
# ensure dims and variable layout
if variable in ds:
try:
ds[variable] = ds[variable].transpose("time", "y", "x")
ds["time"] = ds["time"].dt.floor("D")
except Exception:
pass
# apply slice or mask
if mode == "box":
sub = ds.isel(y=slice(y0, y1), x=slice(x0, x1))
else: # shapefile
mask_da = xr.DataArray(mask, dims=("y", "x"))
sub = ds.where(mask_da, drop=False)
# optionally chunk lazily
if use_dask and chunking:
sub = sub.chunk(chunking)
datasets.append(sub)
# concatenate along time
dset = xr.concat(datasets, dim="time", combine_attrs="override")
# normalize pr units
if "pr" in dset:
if dset["pr"].attrs.get("units") == "mm":
dset["pr"].attrs["units"] = "mm/day"
if "hurs" in dset:
if dset["hurs"].attrs.get("units") == "Percent":
dset["hurs"].attrs["units"] = "%"
dset = self._apply_time_subset(dset)
self.dataset = dset
return dset
else:
# no extraction mode -> just open normally (light transpose)
def preprocess_identity(ds):
if variable in ds:
try:
ds[variable] = ds[variable].transpose("time", "y", "x")
ds["time"] = ds["time"].dt.floor("D")
except Exception:
pass
return ds
dset = xr.open_mfdataset(
files,
combine="nested",
concat_dim="time",
preprocess=preprocess_identity,
engine="netcdf4",
parallel=False,
)
if use_dask and chunking:
dset = dset.chunk(chunking)
if "pr" in dset:
if dset["pr"].attrs.get("units") == "mm":
dset["pr"].attrs["units"] = "mm/day"
if "hurs" in dset:
if dset["hurs"].attrs.get("units") == "Percent":
dset["hurs"].attrs["units"] = "%"
self.dataset = dset
return dset
save_csv(self, filename, df=None)
¶
Write a frame, or the loaded dataset, to CSV.
A gridded HYRAS dataset flattens to one row per cell per timestep, so this can be very large for anything but a point or a small box.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
filename |
str |
Destination path. The parent directory must exist. |
required |
df |
pd.DataFrame |
Frame to write. Defaults to the loaded
dataset converted with |
None |
Returns:
| Type | Description |
|---|---|
None |
Exceptions:
| Type | Description |
|---|---|
ValueError |
If no frame is given and no dataset is loaded. |
Source code in climdata/datasets/HYRAS.py
def save_csv(self, filename: str, df: pd.DataFrame = None):
"""Write a frame, or the loaded dataset, to CSV.
A gridded HYRAS dataset flattens to one row per cell per timestep, so
this can be very large for anything but a point or a small box.
Args:
filename (str): Destination path. The parent directory must exist.
df (pd.DataFrame, optional): Frame to write. Defaults to the loaded
dataset converted with ``to_dataframe()``.
Returns:
None
Raises:
ValueError: If no frame is given and no dataset is loaded.
"""
if df is None:
if self.dataset is None:
raise ValueError("No dataset loaded")
# convert to dataframe (may be large)
df = self.dataset.to_dataframe().reset_index()
df.to_csv(filename, index=False)
print(f"Saved CSV to {filename}")
climdata.datasets.HOSTRADA.HOSTRADAmirror
¶
Download, open and subset HOSTRADA hourly grids.
Follows the same call order as :class:~climdata.datasets.MSWX.MSWXmirror:
:meth:extract records the region of interest, then :meth:load downloads
what is missing and applies the subset per file as it opens.
The grid is projected with 2-D latitude/longitude coordinates, so the subset
is an integer Y/X rectangle rather than a coordinate slice — see the
module docstring for what that implies. Output keeps the projected Y/X
dimensions; pass it through :func:climdata.reproject for a geographic grid.
Attributes:
| Name | Type | Description |
|---|---|---|
cfg |
DictConfig |
Hydra configuration. |
variables |
list[str] |
|
dataset |
xr.Dataset | None |
The loaded dataset, set by :meth: |
Examples:
>>> mirror = HOSTRADAmirror(cfg)
>>> mirror.extract(box={"lat_min": 47, "lat_max": 55,
... "lon_min": 6, "lon_max": 15})
>>> ds = mirror.load("rsds", chunking={"time": "auto"})
Source code in climdata/datasets/HOSTRADA.py
class HOSTRADAmirror:
"""Download, open and subset HOSTRADA hourly grids.
Follows the same call order as :class:`~climdata.datasets.MSWX.MSWXmirror`:
:meth:`extract` records the region of interest, then :meth:`load` downloads
what is missing and applies the subset per file as it opens.
The grid is projected with 2-D latitude/longitude coordinates, so the subset
is an integer ``Y``/``X`` rectangle rather than a coordinate slice — see the
module docstring for what that implies. Output keeps the projected ``Y``/``X``
dimensions; pass it through :func:`climdata.reproject` for a geographic grid.
Attributes:
cfg (DictConfig): Hydra configuration.
variables (list[str]): ``cfg.variables``.
dataset (xr.Dataset | None): The loaded dataset, set by :meth:`load`.
Example:
>>> mirror = HOSTRADAmirror(cfg) # doctest: +SKIP
>>> mirror.extract(box={"lat_min": 47, "lat_max": 55, # doctest: +SKIP
... "lon_min": 6, "lon_max": 15})
>>> ds = mirror.load("rsds", chunking={"time": "auto"}) # doctest: +SKIP
"""
def __init__(self, cfg: DictConfig) -> None:
"""Bind a configuration.
Args:
cfg (DictConfig): Configuration with ``dataset``, ``data_dir``,
``variables``, ``time_range`` and ``dsinfo.HOSTRADA``.
"""
self.cfg = cfg
self.dataset: Optional[xr.Dataset] = None
self.variables: List[str] = list(cfg.variables)
self._extract_mode: Optional[str] = None
self._extract_params = None
# ------------------------------------------------------------------
# File discovery / download
# ------------------------------------------------------------------
def fetch(self, variable: str) -> List[str]:
"""Download the files for a variable and list what is now on disk.
Args:
variable (str): CF variable name, e.g. ``"rsds"``.
Returns:
list[str]: Sorted local paths covering the configured period. Shorter
than the period if the server was missing months.
"""
fetch_hostrada(self.cfg, variable)
return self._find_files(variable)
def _find_files(self, variable: str) -> List[str]:
"""List the local files for a variable across the configured period.
Globs one month at a time so the result is in chronological order, which
matters because :meth:`load` concatenates along ``time`` positionally.
Args:
variable (str): CF variable name.
Returns:
list[str]: Existing local paths, in month order.
"""
import glob
provider = self.cfg.dataset.upper()
var_dir = os.path.join(self.cfg.data_dir, provider.upper(), variable.upper())
start_dt = datetime.fromisoformat(self.cfg.time_range.start_date)
end_dt = datetime.fromisoformat(self.cfg.time_range.end_date)
files: List[str] = []
year, month = start_dt.year, start_dt.month
while (year, month) <= (end_dt.year, end_dt.month):
pattern = os.path.join(
var_dir,
f"{variable}_1hr_HOSTRADA-*_{year}{month:02d}*.nc",
)
found = sorted(glob.glob(pattern))
files.extend(found)
if month == 12:
year += 1
month = 1
else:
month += 1
return files
# ------------------------------------------------------------------
# Extraction intent setter (mirrors HYRAS API)
# ------------------------------------------------------------------
def extract(
self,
*,
point: Optional[Tuple[float, float]] = None,
box: Optional[Dict[str, float]] = None,
shapefile=None,
buffer_km: float = 0.0,
) -> "HOSTRADAmirror":
"""Record the region of interest for :meth:`load` to apply.
Nothing is read here. The three modes are mutually exclusive; the first
supplied wins, in the order ``point``, ``box``, ``shapefile``. Calling
with no geometry clears any previous request, so the next :meth:`load`
returns the full German domain.
A ``shapefile`` is reduced to its bounding box, not clipped to the
polygon — cells outside the geometry but inside its envelope are kept.
Args:
point (tuple[float, float], optional): ``(lon, lat)`` in degrees,
longitude first. Selects the single nearest grid cell.
box (dict, optional): Bounding box with keys ``lat_min``, ``lat_max``,
``lon_min``, ``lon_max``.
shapefile (str | geopandas.GeoDataFrame, optional): Geometry whose
bounding box is used.
buffer_km (float): Accepted for signature compatibility with the
other providers; ignored.
Returns:
HOSTRADAmirror: ``self``, so the call can be chained into :meth:`load`.
"""
if point is not None:
self._extract_mode = "point"
self._extract_params = point # (lon, lat)
elif box is not None:
self._extract_mode = "box"
self._extract_params = box
elif shapefile is not None:
self._extract_mode = "shapefile"
self._extract_params = shapefile
else:
self._extract_mode = None
self._extract_params = None
return self
# ------------------------------------------------------------------
# Load
# ------------------------------------------------------------------
def load(
self,
variable: str,
chunking: Optional[Dict] = None,
use_dask: bool = True,
) -> xr.Dataset:
"""Download what is missing, then open every monthly file as one dataset.
The ``Y``/``X`` index rectangle is computed once from the first file and
reused for all of them, so the per-file ``preprocess`` hook is a cheap
``isel`` rather than a repeated mask evaluation. Files are opened
serially (``parallel=False``): HOSTRADA's NetCDF-4 files are not reliably
thread-safe to open concurrently. Finally the result is clipped to the
configured time range.
Args:
variable (str): CF variable name, e.g. ``"rsds"``.
chunking (dict, optional): Chunk sizes applied after opening, e.g.
``{"time": "auto"}``. Only used when ``use_dask`` is true.
use_dask (bool): Whether ``chunking`` is applied. Defaults to ``True``.
Returns:
xr.Dataset: The concatenated, subset dataset, also stored on
:attr:`dataset`. Dimensions are ``(time, Y, X)``.
Raises:
FileNotFoundError: If no files exist for the variable and period.
ValueError: If the requested box does not intersect the German domain.
"""
files = self.fetch(variable)
if not files:
raise FileNotFoundError(
f"No HOSTRADA files found for variable '{variable}' "
f"in the requested time range "
f"({self.cfg.time_range.start_date} – {self.cfg.time_range.end_date}). "
f"Check data_dir and that the files have been downloaded."
)
mode = self._extract_mode
# Compute Y/X index bounds once from the first file so we can use
# fast isel in the preprocess callbacks.
y_slice, x_slice = self._compute_yx_slices(files[0], mode)
def _preprocess(ds: xr.Dataset) -> xr.Dataset:
# Apply Y/X rectangular slice (works whether mode is box/shapefile/point)
if y_slice is not None and x_slice is not None:
ds = ds.isel(Y=y_slice, X=x_slice)
return ds
dset = xr.open_mfdataset(
files,
combine="nested",
concat_dim="time",
preprocess=_preprocess,
engine="netcdf4",
parallel=False,
)
if use_dask and chunking:
dset = dset.chunk(chunking)
dset = self._apply_time_subset(dset)
self.dataset = dset
return dset
# ------------------------------------------------------------------
# Internal helpers
# ------------------------------------------------------------------
def _compute_yx_slices(self, sample_file: str, mode: Optional[str]):
"""Reduce the requested geometry to integer ``Y``/``X`` index slices.
Reads the 2-D ``lat``/``lon`` auxiliary coordinates from one file and
masks them against the request. For ``point`` this is the cell minimising
squared distance in degrees — an approximation, since a degree of
longitude is shorter than one of latitude, but harmless at HOSTRADA's
1 km spacing. For ``box`` and ``shapefile`` it is the bounding rectangle
of every matching cell, so the result is a superset of the request.
Args:
sample_file (str): Path to one HOSTRADA file, used for its grid.
mode (str | None): ``"point"``, ``"box"``, ``"shapefile"``, or
``None`` for no subsetting.
Returns:
tuple[slice, slice] | tuple[None, None]: Index slices for ``Y`` and
``X``, or ``(None, None)`` when ``mode`` is ``None``.
Raises:
ValueError: If the requested area does not intersect the grid.
"""
if mode is None:
return None, None
ds_sample = xr.open_dataset(sample_file, engine="netcdf4")
lat2d = ds_sample["lat"].values # shape (Y, X)
lon2d = ds_sample["lon"].values # shape (Y, X)
ds_sample.close()
if mode == "point":
lon_pt, lat_pt = self._extract_params
dist = (lat2d - lat_pt) ** 2 + (lon2d - lon_pt) ** 2
iy, ix = np.unravel_index(int(np.argmin(dist)), dist.shape)
return slice(iy, iy + 1), slice(ix, ix + 1)
elif mode in ("box", "shapefile"):
if mode == "box":
box = self._extract_params
lat_min, lat_max = box["lat_min"], box["lat_max"]
lon_min, lon_max = box["lon_min"], box["lon_max"]
else:
import geopandas as gpd
if isinstance(self._extract_params, str):
gdf = gpd.read_file(self._extract_params)
else:
gdf = self._extract_params
lon_min, lat_min, lon_max, lat_max = gdf.total_bounds
mask = (
(lat2d >= lat_min) & (lat2d <= lat_max) &
(lon2d >= lon_min) & (lon2d <= lon_max)
)
if not mask.any():
raise ValueError(
f"Bounding box lat=[{lat_min},{lat_max}] "
f"lon=[{lon_min},{lon_max}] does not intersect the HOSTRADA grid."
)
y_idx, x_idx = np.where(mask)
y_slice = slice(int(y_idx.min()), int(y_idx.max()) + 1)
x_slice = slice(int(x_idx.min()), int(x_idx.max()) + 1)
return y_slice, x_slice
return None, None
def _apply_time_subset(self, ds: xr.Dataset) -> xr.Dataset:
"""Clip a dataset to the configured time range.
Monthly files always overrun the requested range at both ends. A failure
here is reported and swallowed, returning the unclipped dataset, so a
surprising time encoding costs extra timesteps rather than the whole load.
Args:
ds (xr.Dataset): Dataset with a ``time`` dimension.
Returns:
xr.Dataset: The clipped dataset, or ``ds`` unchanged on failure.
"""
start = getattr(self.cfg.time_range, "start_date", None)
end = getattr(self.cfg.time_range, "end_date", None)
if start or end:
try:
ds = ds.sel(time=slice(start, end))
except Exception as exc:
print(f"⚠️ Time subsetting failed: {exc}")
return ds
__init__(self, cfg)
special
¶
Bind a configuration.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
cfg |
DictConfig |
Configuration with |
required |
Source code in climdata/datasets/HOSTRADA.py
def __init__(self, cfg: DictConfig) -> None:
"""Bind a configuration.
Args:
cfg (DictConfig): Configuration with ``dataset``, ``data_dir``,
``variables``, ``time_range`` and ``dsinfo.HOSTRADA``.
"""
self.cfg = cfg
self.dataset: Optional[xr.Dataset] = None
self.variables: List[str] = list(cfg.variables)
self._extract_mode: Optional[str] = None
self._extract_params = None
extract(self, *, point=None, box=None, shapefile=None, buffer_km=0.0)
¶
Record the region of interest for :meth:load to apply.
Nothing is read here. The three modes are mutually exclusive; the first
supplied wins, in the order point, box, shapefile. Calling
with no geometry clears any previous request, so the next :meth:load
returns the full German domain.
A shapefile is reduced to its bounding box, not clipped to the
polygon — cells outside the geometry but inside its envelope are kept.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
point |
tuple[float, float] |
|
None |
box |
dict |
Bounding box with keys |
None |
shapefile |
str | geopandas.GeoDataFrame |
Geometry whose bounding box is used. |
None |
buffer_km |
float |
Accepted for signature compatibility with the other providers; ignored. |
0.0 |
Returns:
| Type | Description |
|---|---|
HOSTRADAmirror |
|
Source code in climdata/datasets/HOSTRADA.py
def extract(
self,
*,
point: Optional[Tuple[float, float]] = None,
box: Optional[Dict[str, float]] = None,
shapefile=None,
buffer_km: float = 0.0,
) -> "HOSTRADAmirror":
"""Record the region of interest for :meth:`load` to apply.
Nothing is read here. The three modes are mutually exclusive; the first
supplied wins, in the order ``point``, ``box``, ``shapefile``. Calling
with no geometry clears any previous request, so the next :meth:`load`
returns the full German domain.
A ``shapefile`` is reduced to its bounding box, not clipped to the
polygon — cells outside the geometry but inside its envelope are kept.
Args:
point (tuple[float, float], optional): ``(lon, lat)`` in degrees,
longitude first. Selects the single nearest grid cell.
box (dict, optional): Bounding box with keys ``lat_min``, ``lat_max``,
``lon_min``, ``lon_max``.
shapefile (str | geopandas.GeoDataFrame, optional): Geometry whose
bounding box is used.
buffer_km (float): Accepted for signature compatibility with the
other providers; ignored.
Returns:
HOSTRADAmirror: ``self``, so the call can be chained into :meth:`load`.
"""
if point is not None:
self._extract_mode = "point"
self._extract_params = point # (lon, lat)
elif box is not None:
self._extract_mode = "box"
self._extract_params = box
elif shapefile is not None:
self._extract_mode = "shapefile"
self._extract_params = shapefile
else:
self._extract_mode = None
self._extract_params = None
return self
fetch(self, variable)
¶
Download the files for a variable and list what is now on disk.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
variable |
str |
CF variable name, e.g. |
required |
Returns:
| Type | Description |
|---|---|
list[str] |
Sorted local paths covering the configured period. Shorter than the period if the server was missing months. |
Source code in climdata/datasets/HOSTRADA.py
def fetch(self, variable: str) -> List[str]:
"""Download the files for a variable and list what is now on disk.
Args:
variable (str): CF variable name, e.g. ``"rsds"``.
Returns:
list[str]: Sorted local paths covering the configured period. Shorter
than the period if the server was missing months.
"""
fetch_hostrada(self.cfg, variable)
return self._find_files(variable)
load(self, variable, chunking=None, use_dask=True)
¶
Download what is missing, then open every monthly file as one dataset.
The Y/X index rectangle is computed once from the first file and
reused for all of them, so the per-file preprocess hook is a cheap
isel rather than a repeated mask evaluation. Files are opened
serially (parallel=False): HOSTRADA's NetCDF-4 files are not reliably
thread-safe to open concurrently. Finally the result is clipped to the
configured time range.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
variable |
str |
CF variable name, e.g. |
required |
chunking |
dict |
Chunk sizes applied after opening, e.g.
|
None |
use_dask |
bool |
Whether |
True |
Returns:
| Type | Description |
|---|---|
xr.Dataset |
The concatenated, subset dataset, also stored on
:attr: |
Exceptions:
| Type | Description |
|---|---|
FileNotFoundError |
If no files exist for the variable and period. |
ValueError |
If the requested box does not intersect the German domain. |
Source code in climdata/datasets/HOSTRADA.py
def load(
self,
variable: str,
chunking: Optional[Dict] = None,
use_dask: bool = True,
) -> xr.Dataset:
"""Download what is missing, then open every monthly file as one dataset.
The ``Y``/``X`` index rectangle is computed once from the first file and
reused for all of them, so the per-file ``preprocess`` hook is a cheap
``isel`` rather than a repeated mask evaluation. Files are opened
serially (``parallel=False``): HOSTRADA's NetCDF-4 files are not reliably
thread-safe to open concurrently. Finally the result is clipped to the
configured time range.
Args:
variable (str): CF variable name, e.g. ``"rsds"``.
chunking (dict, optional): Chunk sizes applied after opening, e.g.
``{"time": "auto"}``. Only used when ``use_dask`` is true.
use_dask (bool): Whether ``chunking`` is applied. Defaults to ``True``.
Returns:
xr.Dataset: The concatenated, subset dataset, also stored on
:attr:`dataset`. Dimensions are ``(time, Y, X)``.
Raises:
FileNotFoundError: If no files exist for the variable and period.
ValueError: If the requested box does not intersect the German domain.
"""
files = self.fetch(variable)
if not files:
raise FileNotFoundError(
f"No HOSTRADA files found for variable '{variable}' "
f"in the requested time range "
f"({self.cfg.time_range.start_date} – {self.cfg.time_range.end_date}). "
f"Check data_dir and that the files have been downloaded."
)
mode = self._extract_mode
# Compute Y/X index bounds once from the first file so we can use
# fast isel in the preprocess callbacks.
y_slice, x_slice = self._compute_yx_slices(files[0], mode)
def _preprocess(ds: xr.Dataset) -> xr.Dataset:
# Apply Y/X rectangular slice (works whether mode is box/shapefile/point)
if y_slice is not None and x_slice is not None:
ds = ds.isel(Y=y_slice, X=x_slice)
return ds
dset = xr.open_mfdataset(
files,
combine="nested",
concat_dim="time",
preprocess=_preprocess,
engine="netcdf4",
parallel=False,
)
if use_dask and chunking:
dset = dset.chunk(chunking)
dset = self._apply_time_subset(dset)
self.dataset = dset
return dset
climdata.datasets.DWD.DWDmirror
¶
Query DWD station observations for one variable at a time.
Unlike the gridded providers there is no fetch/load split:
:meth:extract selects the stations and downloads their records in one
step. One call handles one variable, because DWD groups variables into
datasets (climate_summary, solar, …) with different station lists;
:class:~climdata.utils.wrapper_workflow.ClimateExtractor loops over
cfg.variables and merges the results.
Attributes:
| Name | Type | Description |
|---|---|---|
cfg |
DictConfig |
Hydra configuration. |
param_mapping |
DictConfig |
|
start_date |
str |
Start of the requested period. |
end_date |
str |
End of the requested period. |
df |
pd.DataFrame | None |
Long-form frame, set by :meth: |
dataset |
xr.Dataset | None |
The extracted dataset, set by :meth: |
Examples:
>>> dwd = DWDmirror(cfg)
>>> ds = dwd.extract(variable="pr", point=(13.4, 52.5),
... buffer_km=30)
Source code in climdata/datasets/DWD.py
class DWDmirror:
"""Query DWD station observations for one variable at a time.
Unlike the gridded providers there is no ``fetch``/``load`` split:
:meth:`extract` selects the stations and downloads their records in one
step. One call handles one variable, because DWD groups variables into
datasets (``climate_summary``, ``solar``, …) with different station lists;
:class:`~climdata.utils.wrapper_workflow.ClimateExtractor` loops over
``cfg.variables`` and merges the results.
Attributes:
cfg (DictConfig): Hydra configuration.
param_mapping (DictConfig): ``cfg.dsinfo``, holding the CF-name to DWD
resolution/dataset/parameter mapping.
start_date (str): Start of the requested period.
end_date (str): End of the requested period.
df (pd.DataFrame | None): Long-form frame, set by :meth:`load` or :meth:`format`.
dataset (xr.Dataset | None): The extracted dataset, set by :meth:`extract`.
Example:
>>> dwd = DWDmirror(cfg) # doctest: +SKIP
>>> ds = dwd.extract(variable="pr", point=(13.4, 52.5), # doctest: +SKIP
... buffer_km=30)
"""
def __init__(self, cfg):
"""Bind a configuration.
Args:
cfg (DictConfig): Configuration with ``dsinfo.DWD.variables`` and
``time_range.start_date`` / ``.end_date``.
Raises:
ImportError: If ``wetterdienst`` is not installed.
"""
if not _WETTERDIENST_AVAILABLE:
raise ImportError(
"DWD requires wetterdienst. "
"Install with: pip install wetterdienst"
)
self.cfg = cfg
self.param_mapping = cfg.dsinfo
self.start_date = cfg.time_range.start_date
self.end_date = cfg.time_range.end_date
self.df = None
def get_stations(self, variable='pr'):
"""Download and cache the station list for a variable's DWD dataset.
Caching is keyed by DWD *dataset*, not by variable, since several
variables share one dataset but different datasets have genuinely
different station networks — a cache keyed by variable would hand a
solar request the climate-summary stations.
Args:
variable (str): CF variable name, e.g. ``"pr"``. Defaults to ``"pr"``.
Returns:
pd.DataFrame: Station metadata — ``latitude``, ``longitude``,
``height``, ``name`` and the station id as the index.
Raises:
KeyError: If ``variable`` is not declared in ``cfg.dsinfo.DWD.variables``.
"""
# Lookup info from your mapping
param_info = self.param_mapping.DWD.variables[variable]
resolution = param_info["resolution"] # e.g., "daily"
dataset = param_info["dataset"] # e.g., "climate_summary"
# Create request with updated API
request = DwdObservationRequest(
parameters=[(resolution, dataset)]
)
# Get station metadata as pandas DataFrame
stations_df = request.all().df.to_pandas()
# Cache per dataset so different variables with different datasets
# (e.g. climate_summary vs solar) don't share the same station list
if not hasattr(self, "_stations_cache"):
self._stations_cache = {}
self._stations_cache[dataset] = stations_df
self.stations = stations_df
return stations_df
def load(self, variable, lat_loc, lon_loc, buffer_km=50):
"""Download raw observations from every station within a radius.
A true great-circle radius query, in contrast to :meth:`extract`, which
approximates with a lat/lon box. The result is the unaggregated long
frame straight from wetterdienst, one row per station, timestamp and
parameter — pass it through :meth:`format` to collapse it to a single
series.
Args:
variable (str): CF variable name, e.g. ``"pr"``.
lat_loc (float): Latitude of the search centre, in degrees.
lon_loc (float): Longitude of the search centre, in degrees.
buffer_km (float): Search radius in kilometres. Defaults to ``50``.
Returns:
pd.DataFrame: Long-form observations, also stored on :attr:`df`.
Raises:
KeyError: If ``variable`` is not declared in ``cfg.dsinfo.DWD.variables``.
"""
param_info = self.param_mapping.DWD.variables[variable]
resolution = param_info["resolution"]
dataset = param_info["dataset"]
variable_name = param_info["name"]
settings = Settings(ts_shape="long", ts_humanize=True)
request = DwdObservationRequest(
parameters=(resolution, dataset, variable_name),
start_date=self.start_date,
end_date=self.end_date,
settings=settings
).filter_by_distance(
latlon=(lat_loc, lon_loc),
distance=buffer_km,
unit="km"
)
df = request.values.all().df.to_pandas()
self.df = df
return self.df
def extract(self, *, variable, point=None, box=None, shapefile=None, buffer_km=25.0):
"""Select stations by geometry and download their records as a dataset.
Station selection differs by mode. ``point`` with ``buffer_km > 0`` takes
every station inside a lat/lon box of that half-width — an approximation,
since a degree of longitude shortens with latitude; ``buffer_km=0`` takes
the single nearest station by planar lat/lon distance. ``box`` takes
every station inside it. ``shapefile`` takes every station falling
strictly within a geometry, after an optional buffer applied in Web
Mercator.
The returned dataset is indexed by ``station_id`` and ``time``, carries a
``quality_<variable>`` companion variable, and has ``units`` set from the
configuration so xclim can convert it.
Args:
variable (str): CF variable name, e.g. ``"pr"``. Keyword-only.
point (tuple[float, float], optional): ``(lon, lat)`` in degrees,
longitude first.
box (dict, optional): Bounding box with keys ``lon_min``, ``lon_max``,
``lat_min``, ``lat_max``.
shapefile (str | geopandas.GeoDataFrame, optional): Polygon(s) to
select within, as a path or an in-memory frame.
buffer_km (float): Half-width for ``point``, or dilation for
``shapefile``. Defaults to ``25.0``.
Returns:
xr.Dataset: Observations dimensioned ``(station_id, time)``, also
stored on :attr:`dataset`.
Raises:
ValueError: If none of ``point``, ``box`` or ``shapefile`` was given,
or if the geometry contains no stations — a common outcome for a
small ``buffer_km`` in a sparsely instrumented area.
KeyError: If ``variable`` is not declared in ``cfg.dsinfo.DWD.variables``.
"""
param_info = self.param_mapping.DWD.variables[variable]
resolution = param_info["resolution"]
dataset = param_info["dataset"]
variable_name = param_info["name"]
# Fetch station list for this variable's dataset (cached per dataset)
cache = getattr(self, "_stations_cache", {})
if dataset not in cache:
self.get_stations(variable=variable)
stations_df = self._stations_cache[dataset]
# ---- Point extraction ----
if point is not None:
lon, lat = point
if buffer_km > 0:
buffer_deg = buffer_km / 111
subset = stations_df[
(stations_df.longitude.between(lon - buffer_deg, lon + buffer_deg)) &
(stations_df.latitude.between(lat - buffer_deg, lat + buffer_deg))
]
else:
# Find nearest station
subset = stations_df.copy()
subset["distance"] = ((subset.longitude - lon)**2 + (subset.latitude - lat)**2)**0.5
subset = subset.nsmallest(1, "distance")
# ---- Box extraction ----
elif box is not None:
subset = stations_df[
(stations_df.longitude.between(box["lon_min"], box["lon_max"])) &
(stations_df.latitude.between(box["lat_min"], box["lat_max"]))
]
# ---- Shapefile extraction ----
elif shapefile is not None:
if isinstance(shapefile, str):
gdf = gpd.read_file(shapefile)
else:
gdf = shapefile
# Optional buffer
if buffer_km > 0:
gdf = gdf.to_crs(epsg=3857)
gdf["geometry"] = gdf.buffer(buffer_km * 1000)
gdf = gdf.to_crs(epsg=4326)
points = gpd.GeoDataFrame(
stations_df, geometry=gpd.points_from_xy(stations_df.longitude, stations_df.latitude), crs="EPSG:4326"
)
# Keep stations inside any of the geometries
mask = points.geometry.apply(lambda p: any(g.contains(p) for g in gdf.geometry))
subset = stations_df[mask]
else:
raise ValueError("Must provide either point, box, or shapefile.")
# ---- Download data from selected stations ----
station_ids = subset.index.tolist()
if not station_ids:
raise ValueError("No stations found in selection.")
request = DwdObservationRequest(
parameters=(resolution, dataset, variable_name),
start_date=self.start_date,
end_date=self.end_date,
).filter_by_station_id(station_id=station_ids)
print(point)
data = request.values.all().df.to_pandas() # pandas DataFrame
# Normalise column names across wetterdienst versions
col_map = {}
for old, new in [("datetime", "date"), ("stationid", "station_id"), ("station", "station_id")]:
if old in data.columns and new not in data.columns:
col_map[old] = new
if col_map:
data = data.rename(columns=col_map)
# If station_id is the index (some versions), promote it to a column
if data.index.name == "station_id":
data = data.reset_index()
# Strip timezone and cast to nanosecond precision so xarray won't warn
import pandas as pd
if isinstance(data["date"].dtype, pd.DatetimeTZDtype):
data["date"] = data["date"].dt.tz_localize(None)
data["date"] = data["date"].astype("datetime64[ns]")
# Convert to xarray
ds = data.set_index(["station_id", "date"]).to_xarray()
ds = ds.rename(
{
"date": "time",
"value": variable,
"quality": f"quality_{variable}"
}
)
attrs = {}
for attr,n in zip(["resolution", "dataset", "parameter"],["resolution", "dataset", "long_name"]):
attrs[n] = ds[attr].values[0, 0]
ds = ds.assign_attrs(attrs)
vars_to_drop = [v for v in ["index", "resolution", "dataset", "parameter"] if v in ds]
ds = ds.drop_vars(vars_to_drop)
# Assign 'units' attribute so xclim can read/convert units
unit = self.param_mapping.DWD.variables[variable].unit
ds[variable].attrs["units"] = unit
self.dataset = ds
return ds
def format(self, variable, lat_loc, lon_loc):
"""Collapse the raw frame from :meth:`load` into one long-form series.
Averages across every selected station per timestamp, producing a single
series stamped with the *requested* coordinates rather than any station's
own — so the output describes the location asked about, not the network
that answered. The other columns are reduced by mode.
Args:
variable (str): CF variable name to label the rows with.
lat_loc (float): Latitude to stamp on every row, in degrees.
lon_loc (float): Longitude to stamp on every row, in degrees.
Returns:
pd.DataFrame: Columns ``lat``, ``lon``, ``time``, ``source``,
``variable``, ``value``, ``units``. Also stored on :attr:`df`.
Raises:
TypeError: If :meth:`load` has not been called (:attr:`df` is ``None``).
"""
self.df['date'] = pd.to_datetime(self.df['date'])
self.df = self.df.groupby(['date']).agg({
'value': 'mean',
'station_id': lambda x: x.mode().iloc[0] if not x.mode().empty else None,
'resolution': lambda x: x.mode().iloc[0] if not x.mode().empty else None,
'dataset': lambda x: x.mode().iloc[0] if not x.mode().empty else None,
'parameter': lambda x: x.mode().iloc[0] if not x.mode().empty else None,
'quality': lambda x: x.mode().iloc[0] if not x.mode().empty else None,
}).reset_index()
self.df = self.df.rename(columns={
"date": "time",
"value": "value",
"station_id": "frequent_station",
})
self.df["variable"] = variable
self.df["lat"] = lat_loc
self.df["lon"] = lon_loc
self.df['source'] = 'DWD'
self.df['units'] = self.param_mapping.DWD.variables[variable].unit
self.df = self.df[["lat", "lon", "time", "source", "variable", "value", "units"]]
# self.df = df
return self.df
def save_csv(self, filename):
"""Write :attr:`df` to CSV.
Args:
filename (str): Destination path. The parent directory must exist.
Returns:
str: The path written.
Raises:
AttributeError: If neither :meth:`load` nor :meth:`format` has run.
"""
self.df.to_csv(filename, index=False)
print(f"✅ Saved time series to: {filename}")
return filename
__init__(self, cfg)
special
¶
Bind a configuration.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
cfg |
DictConfig |
Configuration with |
required |
Exceptions:
| Type | Description |
|---|---|
ImportError |
If |
Source code in climdata/datasets/DWD.py
def __init__(self, cfg):
"""Bind a configuration.
Args:
cfg (DictConfig): Configuration with ``dsinfo.DWD.variables`` and
``time_range.start_date`` / ``.end_date``.
Raises:
ImportError: If ``wetterdienst`` is not installed.
"""
if not _WETTERDIENST_AVAILABLE:
raise ImportError(
"DWD requires wetterdienst. "
"Install with: pip install wetterdienst"
)
self.cfg = cfg
self.param_mapping = cfg.dsinfo
self.start_date = cfg.time_range.start_date
self.end_date = cfg.time_range.end_date
self.df = None
extract(self, *, variable, point=None, box=None, shapefile=None, buffer_km=25.0)
¶
Select stations by geometry and download their records as a dataset.
Station selection differs by mode. point with buffer_km > 0 takes
every station inside a lat/lon box of that half-width — an approximation,
since a degree of longitude shortens with latitude; buffer_km=0 takes
the single nearest station by planar lat/lon distance. box takes
every station inside it. shapefile takes every station falling
strictly within a geometry, after an optional buffer applied in Web
Mercator.
The returned dataset is indexed by station_id and time, carries a
quality_<variable> companion variable, and has units set from the
configuration so xclim can convert it.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
variable |
str |
CF variable name, e.g. |
required |
point |
tuple[float, float] |
|
None |
box |
dict |
Bounding box with keys |
None |
shapefile |
str | geopandas.GeoDataFrame |
Polygon(s) to select within, as a path or an in-memory frame. |
None |
buffer_km |
float |
Half-width for |
25.0 |
Returns:
| Type | Description |
|---|---|
xr.Dataset |
Observations dimensioned |
Exceptions:
| Type | Description |
|---|---|
ValueError |
If none of |
KeyError |
If |
Source code in climdata/datasets/DWD.py
def extract(self, *, variable, point=None, box=None, shapefile=None, buffer_km=25.0):
"""Select stations by geometry and download their records as a dataset.
Station selection differs by mode. ``point`` with ``buffer_km > 0`` takes
every station inside a lat/lon box of that half-width — an approximation,
since a degree of longitude shortens with latitude; ``buffer_km=0`` takes
the single nearest station by planar lat/lon distance. ``box`` takes
every station inside it. ``shapefile`` takes every station falling
strictly within a geometry, after an optional buffer applied in Web
Mercator.
The returned dataset is indexed by ``station_id`` and ``time``, carries a
``quality_<variable>`` companion variable, and has ``units`` set from the
configuration so xclim can convert it.
Args:
variable (str): CF variable name, e.g. ``"pr"``. Keyword-only.
point (tuple[float, float], optional): ``(lon, lat)`` in degrees,
longitude first.
box (dict, optional): Bounding box with keys ``lon_min``, ``lon_max``,
``lat_min``, ``lat_max``.
shapefile (str | geopandas.GeoDataFrame, optional): Polygon(s) to
select within, as a path or an in-memory frame.
buffer_km (float): Half-width for ``point``, or dilation for
``shapefile``. Defaults to ``25.0``.
Returns:
xr.Dataset: Observations dimensioned ``(station_id, time)``, also
stored on :attr:`dataset`.
Raises:
ValueError: If none of ``point``, ``box`` or ``shapefile`` was given,
or if the geometry contains no stations — a common outcome for a
small ``buffer_km`` in a sparsely instrumented area.
KeyError: If ``variable`` is not declared in ``cfg.dsinfo.DWD.variables``.
"""
param_info = self.param_mapping.DWD.variables[variable]
resolution = param_info["resolution"]
dataset = param_info["dataset"]
variable_name = param_info["name"]
# Fetch station list for this variable's dataset (cached per dataset)
cache = getattr(self, "_stations_cache", {})
if dataset not in cache:
self.get_stations(variable=variable)
stations_df = self._stations_cache[dataset]
# ---- Point extraction ----
if point is not None:
lon, lat = point
if buffer_km > 0:
buffer_deg = buffer_km / 111
subset = stations_df[
(stations_df.longitude.between(lon - buffer_deg, lon + buffer_deg)) &
(stations_df.latitude.between(lat - buffer_deg, lat + buffer_deg))
]
else:
# Find nearest station
subset = stations_df.copy()
subset["distance"] = ((subset.longitude - lon)**2 + (subset.latitude - lat)**2)**0.5
subset = subset.nsmallest(1, "distance")
# ---- Box extraction ----
elif box is not None:
subset = stations_df[
(stations_df.longitude.between(box["lon_min"], box["lon_max"])) &
(stations_df.latitude.between(box["lat_min"], box["lat_max"]))
]
# ---- Shapefile extraction ----
elif shapefile is not None:
if isinstance(shapefile, str):
gdf = gpd.read_file(shapefile)
else:
gdf = shapefile
# Optional buffer
if buffer_km > 0:
gdf = gdf.to_crs(epsg=3857)
gdf["geometry"] = gdf.buffer(buffer_km * 1000)
gdf = gdf.to_crs(epsg=4326)
points = gpd.GeoDataFrame(
stations_df, geometry=gpd.points_from_xy(stations_df.longitude, stations_df.latitude), crs="EPSG:4326"
)
# Keep stations inside any of the geometries
mask = points.geometry.apply(lambda p: any(g.contains(p) for g in gdf.geometry))
subset = stations_df[mask]
else:
raise ValueError("Must provide either point, box, or shapefile.")
# ---- Download data from selected stations ----
station_ids = subset.index.tolist()
if not station_ids:
raise ValueError("No stations found in selection.")
request = DwdObservationRequest(
parameters=(resolution, dataset, variable_name),
start_date=self.start_date,
end_date=self.end_date,
).filter_by_station_id(station_id=station_ids)
print(point)
data = request.values.all().df.to_pandas() # pandas DataFrame
# Normalise column names across wetterdienst versions
col_map = {}
for old, new in [("datetime", "date"), ("stationid", "station_id"), ("station", "station_id")]:
if old in data.columns and new not in data.columns:
col_map[old] = new
if col_map:
data = data.rename(columns=col_map)
# If station_id is the index (some versions), promote it to a column
if data.index.name == "station_id":
data = data.reset_index()
# Strip timezone and cast to nanosecond precision so xarray won't warn
import pandas as pd
if isinstance(data["date"].dtype, pd.DatetimeTZDtype):
data["date"] = data["date"].dt.tz_localize(None)
data["date"] = data["date"].astype("datetime64[ns]")
# Convert to xarray
ds = data.set_index(["station_id", "date"]).to_xarray()
ds = ds.rename(
{
"date": "time",
"value": variable,
"quality": f"quality_{variable}"
}
)
attrs = {}
for attr,n in zip(["resolution", "dataset", "parameter"],["resolution", "dataset", "long_name"]):
attrs[n] = ds[attr].values[0, 0]
ds = ds.assign_attrs(attrs)
vars_to_drop = [v for v in ["index", "resolution", "dataset", "parameter"] if v in ds]
ds = ds.drop_vars(vars_to_drop)
# Assign 'units' attribute so xclim can read/convert units
unit = self.param_mapping.DWD.variables[variable].unit
ds[variable].attrs["units"] = unit
self.dataset = ds
return ds
format(self, variable, lat_loc, lon_loc)
¶
Collapse the raw frame from :meth:load into one long-form series.
Averages across every selected station per timestamp, producing a single series stamped with the requested coordinates rather than any station's own — so the output describes the location asked about, not the network that answered. The other columns are reduced by mode.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
variable |
str |
CF variable name to label the rows with. |
required |
lat_loc |
float |
Latitude to stamp on every row, in degrees. |
required |
lon_loc |
float |
Longitude to stamp on every row, in degrees. |
required |
Returns:
| Type | Description |
|---|---|
pd.DataFrame |
Columns |
Exceptions:
| Type | Description |
|---|---|
TypeError |
If :meth: |
Source code in climdata/datasets/DWD.py
def format(self, variable, lat_loc, lon_loc):
"""Collapse the raw frame from :meth:`load` into one long-form series.
Averages across every selected station per timestamp, producing a single
series stamped with the *requested* coordinates rather than any station's
own — so the output describes the location asked about, not the network
that answered. The other columns are reduced by mode.
Args:
variable (str): CF variable name to label the rows with.
lat_loc (float): Latitude to stamp on every row, in degrees.
lon_loc (float): Longitude to stamp on every row, in degrees.
Returns:
pd.DataFrame: Columns ``lat``, ``lon``, ``time``, ``source``,
``variable``, ``value``, ``units``. Also stored on :attr:`df`.
Raises:
TypeError: If :meth:`load` has not been called (:attr:`df` is ``None``).
"""
self.df['date'] = pd.to_datetime(self.df['date'])
self.df = self.df.groupby(['date']).agg({
'value': 'mean',
'station_id': lambda x: x.mode().iloc[0] if not x.mode().empty else None,
'resolution': lambda x: x.mode().iloc[0] if not x.mode().empty else None,
'dataset': lambda x: x.mode().iloc[0] if not x.mode().empty else None,
'parameter': lambda x: x.mode().iloc[0] if not x.mode().empty else None,
'quality': lambda x: x.mode().iloc[0] if not x.mode().empty else None,
}).reset_index()
self.df = self.df.rename(columns={
"date": "time",
"value": "value",
"station_id": "frequent_station",
})
self.df["variable"] = variable
self.df["lat"] = lat_loc
self.df["lon"] = lon_loc
self.df['source'] = 'DWD'
self.df['units'] = self.param_mapping.DWD.variables[variable].unit
self.df = self.df[["lat", "lon", "time", "source", "variable", "value", "units"]]
# self.df = df
return self.df
get_stations(self, variable='pr')
¶
Download and cache the station list for a variable's DWD dataset.
Caching is keyed by DWD dataset, not by variable, since several variables share one dataset but different datasets have genuinely different station networks — a cache keyed by variable would hand a solar request the climate-summary stations.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
variable |
str |
CF variable name, e.g. |
'pr' |
Returns:
| Type | Description |
|---|---|
pd.DataFrame |
Station metadata — |
Exceptions:
| Type | Description |
|---|---|
KeyError |
If |
Source code in climdata/datasets/DWD.py
def get_stations(self, variable='pr'):
"""Download and cache the station list for a variable's DWD dataset.
Caching is keyed by DWD *dataset*, not by variable, since several
variables share one dataset but different datasets have genuinely
different station networks — a cache keyed by variable would hand a
solar request the climate-summary stations.
Args:
variable (str): CF variable name, e.g. ``"pr"``. Defaults to ``"pr"``.
Returns:
pd.DataFrame: Station metadata — ``latitude``, ``longitude``,
``height``, ``name`` and the station id as the index.
Raises:
KeyError: If ``variable`` is not declared in ``cfg.dsinfo.DWD.variables``.
"""
# Lookup info from your mapping
param_info = self.param_mapping.DWD.variables[variable]
resolution = param_info["resolution"] # e.g., "daily"
dataset = param_info["dataset"] # e.g., "climate_summary"
# Create request with updated API
request = DwdObservationRequest(
parameters=[(resolution, dataset)]
)
# Get station metadata as pandas DataFrame
stations_df = request.all().df.to_pandas()
# Cache per dataset so different variables with different datasets
# (e.g. climate_summary vs solar) don't share the same station list
if not hasattr(self, "_stations_cache"):
self._stations_cache = {}
self._stations_cache[dataset] = stations_df
self.stations = stations_df
return stations_df
load(self, variable, lat_loc, lon_loc, buffer_km=50)
¶
Download raw observations from every station within a radius.
A true great-circle radius query, in contrast to :meth:extract, which
approximates with a lat/lon box. The result is the unaggregated long
frame straight from wetterdienst, one row per station, timestamp and
parameter — pass it through :meth:format to collapse it to a single
series.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
variable |
str |
CF variable name, e.g. |
required |
lat_loc |
float |
Latitude of the search centre, in degrees. |
required |
lon_loc |
float |
Longitude of the search centre, in degrees. |
required |
buffer_km |
float |
Search radius in kilometres. Defaults to |
50 |
Returns:
| Type | Description |
|---|---|
pd.DataFrame |
Long-form observations, also stored on :attr: |
Exceptions:
| Type | Description |
|---|---|
KeyError |
If |
Source code in climdata/datasets/DWD.py
def load(self, variable, lat_loc, lon_loc, buffer_km=50):
"""Download raw observations from every station within a radius.
A true great-circle radius query, in contrast to :meth:`extract`, which
approximates with a lat/lon box. The result is the unaggregated long
frame straight from wetterdienst, one row per station, timestamp and
parameter — pass it through :meth:`format` to collapse it to a single
series.
Args:
variable (str): CF variable name, e.g. ``"pr"``.
lat_loc (float): Latitude of the search centre, in degrees.
lon_loc (float): Longitude of the search centre, in degrees.
buffer_km (float): Search radius in kilometres. Defaults to ``50``.
Returns:
pd.DataFrame: Long-form observations, also stored on :attr:`df`.
Raises:
KeyError: If ``variable`` is not declared in ``cfg.dsinfo.DWD.variables``.
"""
param_info = self.param_mapping.DWD.variables[variable]
resolution = param_info["resolution"]
dataset = param_info["dataset"]
variable_name = param_info["name"]
settings = Settings(ts_shape="long", ts_humanize=True)
request = DwdObservationRequest(
parameters=(resolution, dataset, variable_name),
start_date=self.start_date,
end_date=self.end_date,
settings=settings
).filter_by_distance(
latlon=(lat_loc, lon_loc),
distance=buffer_km,
unit="km"
)
df = request.values.all().df.to_pandas()
self.df = df
return self.df
save_csv(self, filename)
¶
Write :attr:df to CSV.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
filename |
str |
Destination path. The parent directory must exist. |
required |
Returns:
| Type | Description |
|---|---|
str |
The path written. |
Exceptions:
| Type | Description |
|---|---|
AttributeError |
If neither :meth: |
Source code in climdata/datasets/DWD.py
def save_csv(self, filename):
"""Write :attr:`df` to CSV.
Args:
filename (str): Destination path. The parent directory must exist.
Returns:
str: The path written.
Raises:
AttributeError: If neither :meth:`load` nor :meth:`format` has run.
"""
self.df.to_csv(filename, index=False)
print(f"✅ Saved time series to: {filename}")
return filename
Projections¶
climdata.datasets.CMIPCloud.CMIPCloud
¶
Stream CMIP6 model output from the Pangeo cloud catalogue.
Reads analysis-ready Zarr stores from Google Cloud through intake-esm,
so nothing is downloaded up front — only the requested subset is transferred
when the data is finally computed.
The lifecycle is fetch → load → extract, and unlike the
file-based providers the order is strict: :meth:extract needs a loaded
dataset and raises without one.
The catalogue lookups (:meth:open_cmip6_catalog, :meth:get_experiment_ids,
:meth:get_source_ids) are classmethods, so a caller browsing what is
available — a model picker, say — can query them without building an
extraction config first.
Attributes:
| Name | Type | Description |
|---|---|---|
experiment_id |
str |
CMIP6 experiment, e.g. |
source_id |
str |
Model name, e.g. |
table_id |
str |
MIP table setting the frequency, e.g. |
variables |
list[str] |
Requested CF variable names. |
ds |
xr.Dataset | None |
The loaded dataset, set by :meth: |
col_subsets |
list |
Per-variable catalogue subsets, set by :meth: |
Examples:
>>> cmip = CMIPCloud(cfg)
>>> cmip.fetch()
>>> cmip.load()
>>> ds = cmip.extract(point=(13.4, 52.5))
Source code in climdata/datasets/CMIPCloud.py
class CMIPCloud:
"""Stream CMIP6 model output from the Pangeo cloud catalogue.
Reads analysis-ready Zarr stores from Google Cloud through ``intake-esm``,
so nothing is downloaded up front — only the requested subset is transferred
when the data is finally computed.
The lifecycle is ``fetch`` → ``load`` → ``extract``, and unlike the
file-based providers the order is strict: :meth:`extract` needs a loaded
dataset and raises without one.
The catalogue lookups (:meth:`open_cmip6_catalog`, :meth:`get_experiment_ids`,
:meth:`get_source_ids`) are classmethods, so a caller browsing what is
available — a model picker, say — can query them without building an
extraction config first.
Attributes:
experiment_id (str): CMIP6 experiment, e.g. ``"historical"``, ``"ssp585"``.
source_id (str): Model name, e.g. ``"GFDL-ESM4"``.
table_id (str): MIP table setting the frequency, e.g. ``"day"``.
variables (list[str]): Requested CF variable names.
ds (xr.Dataset | None): The loaded dataset, set by :meth:`load`.
col_subsets (list): Per-variable catalogue subsets, set by :meth:`fetch`.
Example:
>>> cmip = CMIPCloud(cfg) # doctest: +SKIP
>>> cmip.fetch() # doctest: +SKIP
>>> cmip.load() # doctest: +SKIP
>>> ds = cmip.extract(point=(13.4, 52.5)) # doctest: +SKIP
"""
# Catalogue lookups are classmethods so callers that only want to browse
# the Pangeo catalogue (e.g. the GUI's model/experiment pickers) can query
# them without building a full extraction config.
_catalog = None # process-wide cache; the catalogue is a ~10 s download
@classmethod
def open_cmip6_catalog(cls, refresh=False):
"""Open the Pangeo CMIP6 ESM datastore, caching it for the process.
The catalogue is a multi-megabyte download taking roughly ten seconds,
so the first call pays for it and every later call reuses the result.
Args:
refresh (bool): Re-download even if a cached catalogue exists. Use
this to pick up newly published models within a long session.
Returns:
intake_esm.esm_datastore: The catalogue, whose ``.df`` is a pandas
frame with one row per published Zarr store.
"""
if cls._catalog is None or refresh:
cls._catalog = intake.open_esm_datastore(
"https://storage.googleapis.com/cmip6/pangeo-cmip6.json"
)
return cls._catalog
@classmethod
def get_experiment_ids(cls):
"""List the experiments climdata supports, from the live catalogue.
Filtered deliberately to ``historical`` plus the canonical ``sspNNN``
scenarios. The raw catalogue carries hundreds of DECK, CFMIP and
diagnostic experiments whose time axes and forcings do not fit the
historical/projection workflow this class is built for.
Returns:
list[str]: Sorted experiment IDs, e.g.
``["historical", "ssp119", "ssp126", ...]``.
"""
import re
col = cls.open_cmip6_catalog()
experiments = sorted(col.df["experiment_id"].unique())
pattern = re.compile(r"^ssp\d{3}$") # ssp + exactly 3 digits
experiments = [
e for e in experiments
if e == "historical" or pattern.match(e)
]
return experiments
@classmethod
def get_source_ids(cls, experiment_id, table_id=None, variables=None):
"""List the CMIP6 models available for *experiment_id*.
Args:
experiment_id: CMIP6 experiment, e.g. ``"historical"`` or ``"ssp585"``.
table_id: Restrict to a MIP table, e.g. ``"day"``. ``None`` keeps
every frequency.
variables: Restrict to models that publish **all** of these
``variable_id`` values (within *table_id* when given).
``None`` keeps every model.
Returns:
list[str]: Sorted model names.
Notes:
Filtering matters for callers that go on to extract: a model listed
for an experiment does not necessarily publish the requested
variables at the requested frequency, and ``fetch()`` would then
fail. Passing the ``table_id``/``variables`` that will be used for
the extraction yields only models that can actually serve it.
"""
col = cls.open_cmip6_catalog()
df = col.df[col.df["experiment_id"] == experiment_id]
if len(df) == 0:
raise ValueError(f"No data found for experiment_id={experiment_id}")
if table_id:
df = df[df["table_id"] == table_id]
if variables:
# Keep only models that carry every requested variable — the
# intersection, not the union, since extract() needs them all.
per_var = [
set(df[df["variable_id"] == v]["source_id"].unique())
for v in variables
]
usable = set.intersection(*per_var) if per_var else set()
df = df[df["source_id"].isin(usable)]
sources = sorted(df["source_id"].unique())
if not sources:
raise ValueError(
f"No models found for experiment_id={experiment_id}"
+ (f", table_id={table_id}" if table_id else "")
+ (f", variables={list(variables)}" if variables else "")
)
logger.info(f"{len(sources)} models found for experiment '{experiment_id}'")
return sources
def get_variables(self, *, experiment_id, source_id, table_id="day"):
"""List the climdata variables common to every requested model and experiment.
Returns the *intersection*, not the union: only variables that every
combination of ``experiment_id`` and ``source_id`` publishes. That is
what a multi-model extraction needs, since one model missing one variable
breaks the merge. The result is further narrowed to the six variables
climdata's downstream index and bias-correction code understands
(``tas``, ``tasmin``, ``tasmax``, ``pr``, ``hurs``, ``sfcWind``).
Args:
experiment_id (str | list[str]): One experiment or several.
source_id (str | list[str]): One model or several.
table_id (str | None): MIP table, e.g. ``"day"``. ``None`` searches
every frequency. Defaults to ``"day"``.
Returns:
list[str]: Sorted CF variable names available across all combinations.
Raises:
ValueError: If no variable is common to every combination — including
the case where a model/experiment pair publishes nothing at all.
Example:
>>> CMIPCloud(cfg).get_variables( # doctest: +SKIP
... experiment_id="historical", source_id="GFDL-ESM4")
['hurs', 'pr', 'sfcWind', 'tas', 'tasmax', 'tasmin']
"""
TARGET_VARS = {"tas", "tasmin", "tasmax", "pr", "hurs", "sfcWind"}
col = self.open_cmip6_catalog()
# Normalize to lists
if isinstance(experiment_id, str):
experiment_ids = [experiment_id]
else:
experiment_ids = list(experiment_id)
if isinstance(source_id, str):
source_ids = [source_id]
else:
source_ids = list(source_id)
common_vars = None
for exp in experiment_ids:
for src in source_ids:
query = dict(
experiment_id=[exp],
source_id=[src],
)
if table_id is not None:
query["table_id"] = [table_id]
subset = col.search(**query)
if len(subset.df) == 0:
continue
available = set(subset.df["variable_id"].unique())
selected = available & TARGET_VARS
if common_vars is None:
common_vars = selected
else:
common_vars = common_vars & selected
if not common_vars:
raise ValueError(
f"No common variables found for "
f"experiment_id={experiment_ids}, "
f"source_id={source_ids}, "
f"table_id={table_id}"
)
return sorted(common_vars)
def __init__(self, cfg: DictConfig):
"""Bind a configuration and validate the requested period.
Args:
cfg (DictConfig): Configuration with ``experiment_id``, ``source_id``,
``table_id``, ``variables`` and ``time_range.start_date`` /
``.end_date``.
Raises:
ImportError: If ``intake`` and ``intake-esm`` are not installed.
ValueError: If the time range cannot belong to the experiment — see
:meth:`_validate_time_range`.
"""
if not _INTAKE_AVAILABLE:
raise ImportError(
"CMIPCloud requires intake and intake-esm. "
"Install with: pip install intake intake-esm"
)
# Directly read from flat config
self.experiment_id = cfg.experiment_id
self.source_id = cfg.source_id
self.table_id = cfg.table_id
self.variables = cfg.variables
self.start_date = cfg.time_range.start_date
self.end_date = cfg.time_range.end_date
self.cfg = cfg
self.col_subsets = []
self.ds = None
self.col = None
self._validate_time_range()
def _validate_time_range(self):
"""Check the requested period against the experiment's simulated period.
CMIP6 historical runs cover 1850-2014 and SSP scenarios 2015-2100, so
asking an SSP for 1990 returns nothing — a failure worth catching at
construction rather than after a catalogue query. A range that merely
overruns the period at one end warns and proceeds; one that misses it
entirely raises. ``picontrol`` and unrecognised experiments are skipped.
Returns:
None
Raises:
ValueError: If the requested range lies wholly outside the
experiment's period.
"""
start_date = datetime.fromisoformat(self.cfg.time_range.start_date)
end_date = datetime.fromisoformat(self.cfg.time_range.end_date)
start_year = start_date.year
end_year = end_date.year
# Define valid periods for each experiment type
if self.experiment_id == 'historical':
valid_start = 1850
valid_end = 2014
period_name = "Historical"
elif self.experiment_id.startswith('ssp'):
valid_start = 2015
valid_end = 2100
period_name = f"SSP scenario ({self.experiment_id})"
elif self.experiment_id == 'picontrol':
# Pre-industrial control - typically long runs, less strict
return
else:
# Unknown experiment, skip validation
return
# Check if requested period is outside valid range
if end_year < valid_start or start_year > valid_end:
raise ValueError(
f"❌ Time range mismatch for experiment '{self.experiment_id}'!\n"
f" Requested: {start_year}-{end_year}\n"
f" Valid period for {period_name}: {valid_start}-{valid_end}\n"
f" \n"
f" Hint: Use 'historical' for years 1850-2014, and SSP scenarios (ssp126, ssp370, ssp585) for 2015-2100."
)
# Warn if requested period extends beyond valid range
if start_year < valid_start or end_year > valid_end:
print(f"⚠️ Warning: Requested time range {start_year}-{end_year} extends beyond")
print(f" the typical {period_name} period ({valid_start}-{valid_end}).")
print(f" Data availability may be limited.")
def fetch(self):
"""Collect intake catalog subsets for each variable.
Raises:
ValueError: If the model/experiment/table combination does not
publish every requested variable. Partial coverage is treated
as an error rather than silently dropped, because downstream
extraction indexes the dataset by every name in
``cfg.variables`` and would otherwise fail with a bare
``KeyError`` far from the cause.
"""
col = self.open_cmip6_catalog()
self.col_subsets = []
missing = []
for var in self.variables:
query = dict(
experiment_id=[self.experiment_id],
source_id=self.source_id,
table_id=self.table_id,
variable_id=var,
)
col_subset = col.search(require_all_on=["source_id"], **query)
if len(col_subset.df) == 0:
missing.append(var)
continue
self.col_subsets.append(col_subset)
self.col = col
if missing:
# Report what this model *does* offer at this frequency so the
# caller can pick a workable variable set or a different model.
available = sorted(
col.df[
(col.df["experiment_id"] == self.experiment_id)
& (col.df["source_id"] == self.source_id)
& (col.df["table_id"] == self.table_id)
]["variable_id"].unique()
)
raise ValueError(
f"CMIP6 model '{self.source_id}' has no "
f"{self.table_id}-frequency data for {missing} "
f"under experiment '{self.experiment_id}'.\n"
f" Requested variables : {list(self.variables)}\n"
f" Available for this model/table: {available or 'none'}\n"
f" Hint: choose a different model, or reduce the requested "
f"variables to the available ones."
)
if not self.col_subsets:
raise ValueError(
f"No matching CMIP6 data found for: "
f"experiment_id={self.experiment_id}, "
f"source_id={self.source_id}, "
f"table_id={self.table_id}, "
f"variables={self.variables}"
)
return self.col_subsets
def convert_to_noleap(self, ds):
"""Rewrite a CMIP time axis as day-floored pandas timestamps.
CMIP6 models use whichever calendar their modelling centre chose —
``noleap``, ``360_day``, ``proleptic_gregorian`` — and stamp daily data
at 12:00. Both properties block merging across models and joining against
observations. Converting to ``pandas.Timestamp`` at midnight makes the
axes comparable.
This is lossy where the source calendar is not Gregorian: 360-day
timestamps become real dates, which can produce duplicate or missing days
across a long record. It is applied at the end of :meth:`extract`, once
the data has been subset.
Args:
ds (xr.Dataset): Dataset with a CMIP time coordinate.
Returns:
xr.Dataset: The dataset with a ``pandas``-backed daily time axis, or
unchanged if it has no ``time`` coordinate.
"""
if "time" not in ds.coords:
return ds
t = ds.indexes["time"] # can be pandas.DatetimeIndex or CFTimeIndex
new_times = []
for ti in t:
year, month, day = ti.year, ti.month, ti.day
# Floor to day: ignore hour, minute, second
# Convert all time types to pandas Timestamp for compatibility
new_times.append(pd.Timestamp(year=year, month=month, day=day))
ds = ds.assign_coords(time=("time", new_times))
return ds
def load(self):
"""Open the Zarr store behind each catalogue subset and merge them.
Stores are opened lazily over HTTPS, so this is fast and transfers almost
nothing; the data moves when the result is computed. The merge uses
``compat="override"``, taking coordinate values from the first dataset
where models disagree in the last bits of their grid coordinates.
Only the first store per variable is opened, so a variable split across
several ensemble members contributes one member.
Returns:
xr.Dataset | None: The merged dataset, also stored on :attr:`ds`.
``None`` if :meth:`fetch` collected nothing.
"""
datasets = []
for col_subset in self.col_subsets:
zstore_path = col_subset.df.zstore.values[0].replace(
"gs:/", "https://storage.googleapis.com"
)
ds_var = xr.open_zarr(zstore_path)
datasets.append(ds_var)
if datasets:
self.ds = xr.merge(datasets,compat='override')
else:
self.ds = None
return self.ds
def extract(self, *, point=None, box=None, shapefile=None, buffer_km=0.0):
"""Subset the loaded dataset in time and space.
Always clips to the configured time range first, then applies exactly one
spatial selector. Afterwards the model name is added as a ``source_id``
dimension — so several models can be concatenated — and the time axis is
normalised by :meth:`convert_to_noleap`.
CMIP6 longitudes run 0-360, so ``point`` and ``box`` coordinates must be
given in that convention, not as negative western longitudes.
Args:
point (tuple[float, float], optional): ``(lon, lat)`` in degrees,
longitude first.
box (dict, optional): Bounding box with keys ``lon_min``, ``lon_max``,
``lat_min``, ``lat_max``.
shapefile (str | geopandas.GeoDataFrame, optional): Polygon(s) to clip
to, as a path or an in-memory frame.
buffer_km (float): For ``point``, half-width of a box selected around
it instead of the nearest cell; for ``shapefile``, a dilation
applied to each geometry. Converted at a flat 111 km per degree.
Defaults to ``0.0``.
Returns:
xr.Dataset: The subset, also stored on :attr:`ds`.
Raises:
ValueError: If :meth:`load` has not run, or if none of ``point``,
``box`` or ``shapefile`` was given.
"""
import geopandas as gpd
from shapely.geometry import mapping
if self.ds is None:
raise ValueError("No dataset loaded. Call `load()` first.")
self._subset_time(self.start_date, self.end_date)
ds = self.ds
if point is not None:
lon, lat = point
if buffer_km > 0:
buffer_deg = buffer_km / 111
ds_subset = ds.sel(
lon=slice(lon - buffer_deg, lon + buffer_deg),
lat=slice(lat - buffer_deg, lat + buffer_deg),
)
else:
ds_subset = ds.sel(lon=lon, lat=lat, method="nearest")
elif box is not None:
ds_subset = ds.sel(
lon=slice(box["lon_min"], box["lon_max"]),
lat=slice(box["lat_min"], box["lat_max"]),
)
elif shapefile is not None:
if isinstance(shapefile, str):
gdf = gpd.read_file(shapefile)
else:
gdf = shapefile
if buffer_km > 0:
gdf = gdf.to_crs(epsg=3857)
gdf["geometry"] = gdf.buffer(buffer_km * 1000)
gdf = gdf.to_crs(epsg=4326)
geom = [mapping(g) for g in gdf.geometry]
import rioxarray # noqa: F401 — registers the .rio accessor
ds = ds.rio.write_crs("EPSG:4326", inplace=False)
ds_subset = ds.rio.clip(geom, gdf.crs, drop=True)
else:
raise ValueError("Must provide either point, box, or shapefile.")
self.ds = ds_subset
self.ds = self.ds.assign_coords(source_id=self.source_id)
self.ds = self.ds.expand_dims("source_id")
self.ds = self.convert_to_noleap(self.ds)
return ds_subset
def _subset_time(self, start_date, end_date):
"""Clip :attr:`ds` to a time range, in place.
Args:
start_date (str): Inclusive start, ISO format.
end_date (str): Inclusive end, ISO format.
Returns:
xr.Dataset | None: The clipped dataset, or ``None`` if nothing is loaded.
"""
if self.ds is None:
return None
ds_time = self.ds.sel(time=slice(start_date, end_date))
self.ds = ds_time
return ds_time
def save_netcdf(self, filename):
"""Write the dataset to NetCDF, doing nothing if none is loaded.
Clears the time encoding inherited from the Zarr store first: it names a
calendar that no longer matches the axis rewritten by
:meth:`convert_to_noleap`, and the NetCDF writer would otherwise fail on
the contradiction.
Args:
filename (str): Destination path. The parent directory must exist.
Returns:
None
"""
if self.ds is not None:
if "time" in self.ds.variables:
self.ds["time"].encoding.clear()
self.ds.to_netcdf(filename)
# print(f"Saved NetCDF to {filename}")
def save_zarr(self, store_path):
"""Write the dataset to a Zarr store, overwriting any existing one.
Args:
store_path (str): Destination store path.
Returns:
None
"""
if self.ds is not None:
self.ds.to_zarr(store_path, mode="w")
print(f"Saved Zarr to {store_path}")
def _format(self, df):
"""Reshape a wide frame into climdata's long output form.
Melts the variable columns into ``variable``/``value`` pairs, attaches
each variable's units, and stamps the model, experiment and MIP table so
rows stay identifiable once several models are concatenated.
Args:
df (pd.DataFrame): Wide frame from ``ds.to_dataframe().reset_index()``.
Returns:
pd.DataFrame: Long frame with columns ``source``, ``experiment``,
``table``, ``time``, ``lat``, ``lon``, ``variable``, ``value``,
``units`` — those of them that are present.
"""
value_vars = [v for v in self.variables if v in df.columns]
id_vars = [c for c in df.columns if c not in value_vars]
df_long = df.melt(
id_vars=id_vars,
value_vars=value_vars,
var_name="variable",
value_name="value",
)
df_long["units"] = df_long["variable"].map(
lambda v: self.ds[v].attrs.get("units", "unknown")
if v in self.ds.data_vars
else "unknown"
)
df_long["source"] = self.source_id
df_long["experiment"] = self.experiment_id
df_long["table"] = self.table_id
cols = [
"source",
"experiment",
"table",
"time",
"lat",
"lon",
"variable",
"value",
"units",
]
df_long = df_long[[c for c in cols if c in df_long.columns]]
return df_long
def save_csv(self, filename):
"""Write the dataset to CSV in long form, doing nothing if none is loaded.
Args:
filename (str): Destination path. The parent directory must exist.
Returns:
None
"""
if self.ds is not None:
df = self.ds.to_dataframe().reset_index()
df = self._format(df)
df.to_csv(filename, index=False)
# print(f"Saved CSV to {filename}")
__init__(self, cfg)
special
¶
Bind a configuration and validate the requested period.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
cfg |
DictConfig |
Configuration with |
required |
Exceptions:
| Type | Description |
|---|---|
ImportError |
If |
ValueError |
If the time range cannot belong to the experiment — see
:meth: |
Source code in climdata/datasets/CMIPCloud.py
def __init__(self, cfg: DictConfig):
"""Bind a configuration and validate the requested period.
Args:
cfg (DictConfig): Configuration with ``experiment_id``, ``source_id``,
``table_id``, ``variables`` and ``time_range.start_date`` /
``.end_date``.
Raises:
ImportError: If ``intake`` and ``intake-esm`` are not installed.
ValueError: If the time range cannot belong to the experiment — see
:meth:`_validate_time_range`.
"""
if not _INTAKE_AVAILABLE:
raise ImportError(
"CMIPCloud requires intake and intake-esm. "
"Install with: pip install intake intake-esm"
)
# Directly read from flat config
self.experiment_id = cfg.experiment_id
self.source_id = cfg.source_id
self.table_id = cfg.table_id
self.variables = cfg.variables
self.start_date = cfg.time_range.start_date
self.end_date = cfg.time_range.end_date
self.cfg = cfg
self.col_subsets = []
self.ds = None
self.col = None
self._validate_time_range()
convert_to_noleap(self, ds)
¶
Rewrite a CMIP time axis as day-floored pandas timestamps.
CMIP6 models use whichever calendar their modelling centre chose —
noleap, 360_day, proleptic_gregorian — and stamp daily data
at 12:00. Both properties block merging across models and joining against
observations. Converting to pandas.Timestamp at midnight makes the
axes comparable.
This is lossy where the source calendar is not Gregorian: 360-day
timestamps become real dates, which can produce duplicate or missing days
across a long record. It is applied at the end of :meth:extract, once
the data has been subset.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
ds |
xr.Dataset |
Dataset with a CMIP time coordinate. |
required |
Returns:
| Type | Description |
|---|---|
xr.Dataset |
The dataset with a |
Source code in climdata/datasets/CMIPCloud.py
def convert_to_noleap(self, ds):
"""Rewrite a CMIP time axis as day-floored pandas timestamps.
CMIP6 models use whichever calendar their modelling centre chose —
``noleap``, ``360_day``, ``proleptic_gregorian`` — and stamp daily data
at 12:00. Both properties block merging across models and joining against
observations. Converting to ``pandas.Timestamp`` at midnight makes the
axes comparable.
This is lossy where the source calendar is not Gregorian: 360-day
timestamps become real dates, which can produce duplicate or missing days
across a long record. It is applied at the end of :meth:`extract`, once
the data has been subset.
Args:
ds (xr.Dataset): Dataset with a CMIP time coordinate.
Returns:
xr.Dataset: The dataset with a ``pandas``-backed daily time axis, or
unchanged if it has no ``time`` coordinate.
"""
if "time" not in ds.coords:
return ds
t = ds.indexes["time"] # can be pandas.DatetimeIndex or CFTimeIndex
new_times = []
for ti in t:
year, month, day = ti.year, ti.month, ti.day
# Floor to day: ignore hour, minute, second
# Convert all time types to pandas Timestamp for compatibility
new_times.append(pd.Timestamp(year=year, month=month, day=day))
ds = ds.assign_coords(time=("time", new_times))
return ds
extract(self, *, point=None, box=None, shapefile=None, buffer_km=0.0)
¶
Subset the loaded dataset in time and space.
Always clips to the configured time range first, then applies exactly one
spatial selector. Afterwards the model name is added as a source_id
dimension — so several models can be concatenated — and the time axis is
normalised by :meth:convert_to_noleap.
CMIP6 longitudes run 0-360, so point and box coordinates must be
given in that convention, not as negative western longitudes.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
point |
tuple[float, float] |
|
None |
box |
dict |
Bounding box with keys |
None |
shapefile |
str | geopandas.GeoDataFrame |
Polygon(s) to clip to, as a path or an in-memory frame. |
None |
buffer_km |
float |
For |
0.0 |
Returns:
| Type | Description |
|---|---|
xr.Dataset |
The subset, also stored on :attr: |
Exceptions:
| Type | Description |
|---|---|
ValueError |
If :meth: |
Source code in climdata/datasets/CMIPCloud.py
def extract(self, *, point=None, box=None, shapefile=None, buffer_km=0.0):
"""Subset the loaded dataset in time and space.
Always clips to the configured time range first, then applies exactly one
spatial selector. Afterwards the model name is added as a ``source_id``
dimension — so several models can be concatenated — and the time axis is
normalised by :meth:`convert_to_noleap`.
CMIP6 longitudes run 0-360, so ``point`` and ``box`` coordinates must be
given in that convention, not as negative western longitudes.
Args:
point (tuple[float, float], optional): ``(lon, lat)`` in degrees,
longitude first.
box (dict, optional): Bounding box with keys ``lon_min``, ``lon_max``,
``lat_min``, ``lat_max``.
shapefile (str | geopandas.GeoDataFrame, optional): Polygon(s) to clip
to, as a path or an in-memory frame.
buffer_km (float): For ``point``, half-width of a box selected around
it instead of the nearest cell; for ``shapefile``, a dilation
applied to each geometry. Converted at a flat 111 km per degree.
Defaults to ``0.0``.
Returns:
xr.Dataset: The subset, also stored on :attr:`ds`.
Raises:
ValueError: If :meth:`load` has not run, or if none of ``point``,
``box`` or ``shapefile`` was given.
"""
import geopandas as gpd
from shapely.geometry import mapping
if self.ds is None:
raise ValueError("No dataset loaded. Call `load()` first.")
self._subset_time(self.start_date, self.end_date)
ds = self.ds
if point is not None:
lon, lat = point
if buffer_km > 0:
buffer_deg = buffer_km / 111
ds_subset = ds.sel(
lon=slice(lon - buffer_deg, lon + buffer_deg),
lat=slice(lat - buffer_deg, lat + buffer_deg),
)
else:
ds_subset = ds.sel(lon=lon, lat=lat, method="nearest")
elif box is not None:
ds_subset = ds.sel(
lon=slice(box["lon_min"], box["lon_max"]),
lat=slice(box["lat_min"], box["lat_max"]),
)
elif shapefile is not None:
if isinstance(shapefile, str):
gdf = gpd.read_file(shapefile)
else:
gdf = shapefile
if buffer_km > 0:
gdf = gdf.to_crs(epsg=3857)
gdf["geometry"] = gdf.buffer(buffer_km * 1000)
gdf = gdf.to_crs(epsg=4326)
geom = [mapping(g) for g in gdf.geometry]
import rioxarray # noqa: F401 — registers the .rio accessor
ds = ds.rio.write_crs("EPSG:4326", inplace=False)
ds_subset = ds.rio.clip(geom, gdf.crs, drop=True)
else:
raise ValueError("Must provide either point, box, or shapefile.")
self.ds = ds_subset
self.ds = self.ds.assign_coords(source_id=self.source_id)
self.ds = self.ds.expand_dims("source_id")
self.ds = self.convert_to_noleap(self.ds)
return ds_subset
fetch(self)
¶
Collect intake catalog subsets for each variable.
Exceptions:
| Type | Description |
|---|---|
ValueError |
If the model/experiment/table combination does not
publish every requested variable. Partial coverage is treated
as an error rather than silently dropped, because downstream
extraction indexes the dataset by every name in
|
Source code in climdata/datasets/CMIPCloud.py
def fetch(self):
"""Collect intake catalog subsets for each variable.
Raises:
ValueError: If the model/experiment/table combination does not
publish every requested variable. Partial coverage is treated
as an error rather than silently dropped, because downstream
extraction indexes the dataset by every name in
``cfg.variables`` and would otherwise fail with a bare
``KeyError`` far from the cause.
"""
col = self.open_cmip6_catalog()
self.col_subsets = []
missing = []
for var in self.variables:
query = dict(
experiment_id=[self.experiment_id],
source_id=self.source_id,
table_id=self.table_id,
variable_id=var,
)
col_subset = col.search(require_all_on=["source_id"], **query)
if len(col_subset.df) == 0:
missing.append(var)
continue
self.col_subsets.append(col_subset)
self.col = col
if missing:
# Report what this model *does* offer at this frequency so the
# caller can pick a workable variable set or a different model.
available = sorted(
col.df[
(col.df["experiment_id"] == self.experiment_id)
& (col.df["source_id"] == self.source_id)
& (col.df["table_id"] == self.table_id)
]["variable_id"].unique()
)
raise ValueError(
f"CMIP6 model '{self.source_id}' has no "
f"{self.table_id}-frequency data for {missing} "
f"under experiment '{self.experiment_id}'.\n"
f" Requested variables : {list(self.variables)}\n"
f" Available for this model/table: {available or 'none'}\n"
f" Hint: choose a different model, or reduce the requested "
f"variables to the available ones."
)
if not self.col_subsets:
raise ValueError(
f"No matching CMIP6 data found for: "
f"experiment_id={self.experiment_id}, "
f"source_id={self.source_id}, "
f"table_id={self.table_id}, "
f"variables={self.variables}"
)
return self.col_subsets
get_experiment_ids()
classmethod
¶
List the experiments climdata supports, from the live catalogue.
Filtered deliberately to historical plus the canonical sspNNN
scenarios. The raw catalogue carries hundreds of DECK, CFMIP and
diagnostic experiments whose time axes and forcings do not fit the
historical/projection workflow this class is built for.
Returns:
| Type | Description |
|---|---|
list[str] |
Sorted experiment IDs, e.g.
|
Source code in climdata/datasets/CMIPCloud.py
@classmethod
def get_experiment_ids(cls):
"""List the experiments climdata supports, from the live catalogue.
Filtered deliberately to ``historical`` plus the canonical ``sspNNN``
scenarios. The raw catalogue carries hundreds of DECK, CFMIP and
diagnostic experiments whose time axes and forcings do not fit the
historical/projection workflow this class is built for.
Returns:
list[str]: Sorted experiment IDs, e.g.
``["historical", "ssp119", "ssp126", ...]``.
"""
import re
col = cls.open_cmip6_catalog()
experiments = sorted(col.df["experiment_id"].unique())
pattern = re.compile(r"^ssp\d{3}$") # ssp + exactly 3 digits
experiments = [
e for e in experiments
if e == "historical" or pattern.match(e)
]
return experiments
get_source_ids(experiment_id, table_id=None, variables=None)
classmethod
¶
List the CMIP6 models available for experiment_id.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
experiment_id |
CMIP6 experiment, e.g. |
required | |
table_id |
Restrict to a MIP table, e.g. |
None |
|
variables |
Restrict to models that publish all of these
|
None |
Returns:
| Type | Description |
|---|---|
list[str] |
Sorted model names. |
Notes
Filtering matters for callers that go on to extract: a model listed
for an experiment does not necessarily publish the requested
variables at the requested frequency, and fetch() would then
fail. Passing the table_id/variables that will be used for
the extraction yields only models that can actually serve it.
Source code in climdata/datasets/CMIPCloud.py
@classmethod
def get_source_ids(cls, experiment_id, table_id=None, variables=None):
"""List the CMIP6 models available for *experiment_id*.
Args:
experiment_id: CMIP6 experiment, e.g. ``"historical"`` or ``"ssp585"``.
table_id: Restrict to a MIP table, e.g. ``"day"``. ``None`` keeps
every frequency.
variables: Restrict to models that publish **all** of these
``variable_id`` values (within *table_id* when given).
``None`` keeps every model.
Returns:
list[str]: Sorted model names.
Notes:
Filtering matters for callers that go on to extract: a model listed
for an experiment does not necessarily publish the requested
variables at the requested frequency, and ``fetch()`` would then
fail. Passing the ``table_id``/``variables`` that will be used for
the extraction yields only models that can actually serve it.
"""
col = cls.open_cmip6_catalog()
df = col.df[col.df["experiment_id"] == experiment_id]
if len(df) == 0:
raise ValueError(f"No data found for experiment_id={experiment_id}")
if table_id:
df = df[df["table_id"] == table_id]
if variables:
# Keep only models that carry every requested variable — the
# intersection, not the union, since extract() needs them all.
per_var = [
set(df[df["variable_id"] == v]["source_id"].unique())
for v in variables
]
usable = set.intersection(*per_var) if per_var else set()
df = df[df["source_id"].isin(usable)]
sources = sorted(df["source_id"].unique())
if not sources:
raise ValueError(
f"No models found for experiment_id={experiment_id}"
+ (f", table_id={table_id}" if table_id else "")
+ (f", variables={list(variables)}" if variables else "")
)
logger.info(f"{len(sources)} models found for experiment '{experiment_id}'")
return sources
get_variables(self, *, experiment_id, source_id, table_id='day')
¶
List the climdata variables common to every requested model and experiment.
Returns the intersection, not the union: only variables that every
combination of experiment_id and source_id publishes. That is
what a multi-model extraction needs, since one model missing one variable
breaks the merge. The result is further narrowed to the six variables
climdata's downstream index and bias-correction code understands
(tas, tasmin, tasmax, pr, hurs, sfcWind).
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
experiment_id |
str | list[str] |
One experiment or several. |
required |
source_id |
str | list[str] |
One model or several. |
required |
table_id |
str | None |
MIP table, e.g. |
'day' |
Returns:
| Type | Description |
|---|---|
list[str] |
Sorted CF variable names available across all combinations. |
Exceptions:
| Type | Description |
|---|---|
ValueError |
If no variable is common to every combination — including the case where a model/experiment pair publishes nothing at all. |
Examples:
>>> CMIPCloud(cfg).get_variables(
... experiment_id="historical", source_id="GFDL-ESM4")
['hurs', 'pr', 'sfcWind', 'tas', 'tasmax', 'tasmin']
Source code in climdata/datasets/CMIPCloud.py
def get_variables(self, *, experiment_id, source_id, table_id="day"):
"""List the climdata variables common to every requested model and experiment.
Returns the *intersection*, not the union: only variables that every
combination of ``experiment_id`` and ``source_id`` publishes. That is
what a multi-model extraction needs, since one model missing one variable
breaks the merge. The result is further narrowed to the six variables
climdata's downstream index and bias-correction code understands
(``tas``, ``tasmin``, ``tasmax``, ``pr``, ``hurs``, ``sfcWind``).
Args:
experiment_id (str | list[str]): One experiment or several.
source_id (str | list[str]): One model or several.
table_id (str | None): MIP table, e.g. ``"day"``. ``None`` searches
every frequency. Defaults to ``"day"``.
Returns:
list[str]: Sorted CF variable names available across all combinations.
Raises:
ValueError: If no variable is common to every combination — including
the case where a model/experiment pair publishes nothing at all.
Example:
>>> CMIPCloud(cfg).get_variables( # doctest: +SKIP
... experiment_id="historical", source_id="GFDL-ESM4")
['hurs', 'pr', 'sfcWind', 'tas', 'tasmax', 'tasmin']
"""
TARGET_VARS = {"tas", "tasmin", "tasmax", "pr", "hurs", "sfcWind"}
col = self.open_cmip6_catalog()
# Normalize to lists
if isinstance(experiment_id, str):
experiment_ids = [experiment_id]
else:
experiment_ids = list(experiment_id)
if isinstance(source_id, str):
source_ids = [source_id]
else:
source_ids = list(source_id)
common_vars = None
for exp in experiment_ids:
for src in source_ids:
query = dict(
experiment_id=[exp],
source_id=[src],
)
if table_id is not None:
query["table_id"] = [table_id]
subset = col.search(**query)
if len(subset.df) == 0:
continue
available = set(subset.df["variable_id"].unique())
selected = available & TARGET_VARS
if common_vars is None:
common_vars = selected
else:
common_vars = common_vars & selected
if not common_vars:
raise ValueError(
f"No common variables found for "
f"experiment_id={experiment_ids}, "
f"source_id={source_ids}, "
f"table_id={table_id}"
)
return sorted(common_vars)
load(self)
¶
Open the Zarr store behind each catalogue subset and merge them.
Stores are opened lazily over HTTPS, so this is fast and transfers almost
nothing; the data moves when the result is computed. The merge uses
compat="override", taking coordinate values from the first dataset
where models disagree in the last bits of their grid coordinates.
Only the first store per variable is opened, so a variable split across several ensemble members contributes one member.
Returns:
| Type | Description |
|---|---|
xr.Dataset | None |
The merged dataset, also stored on :attr: |
Source code in climdata/datasets/CMIPCloud.py
def load(self):
"""Open the Zarr store behind each catalogue subset and merge them.
Stores are opened lazily over HTTPS, so this is fast and transfers almost
nothing; the data moves when the result is computed. The merge uses
``compat="override"``, taking coordinate values from the first dataset
where models disagree in the last bits of their grid coordinates.
Only the first store per variable is opened, so a variable split across
several ensemble members contributes one member.
Returns:
xr.Dataset | None: The merged dataset, also stored on :attr:`ds`.
``None`` if :meth:`fetch` collected nothing.
"""
datasets = []
for col_subset in self.col_subsets:
zstore_path = col_subset.df.zstore.values[0].replace(
"gs:/", "https://storage.googleapis.com"
)
ds_var = xr.open_zarr(zstore_path)
datasets.append(ds_var)
if datasets:
self.ds = xr.merge(datasets,compat='override')
else:
self.ds = None
return self.ds
open_cmip6_catalog(refresh=False)
classmethod
¶
Open the Pangeo CMIP6 ESM datastore, caching it for the process.
The catalogue is a multi-megabyte download taking roughly ten seconds, so the first call pays for it and every later call reuses the result.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
refresh |
bool |
Re-download even if a cached catalogue exists. Use this to pick up newly published models within a long session. |
False |
Returns:
| Type | Description |
|---|---|
intake_esm.esm_datastore |
The catalogue, whose |
Source code in climdata/datasets/CMIPCloud.py
@classmethod
def open_cmip6_catalog(cls, refresh=False):
"""Open the Pangeo CMIP6 ESM datastore, caching it for the process.
The catalogue is a multi-megabyte download taking roughly ten seconds,
so the first call pays for it and every later call reuses the result.
Args:
refresh (bool): Re-download even if a cached catalogue exists. Use
this to pick up newly published models within a long session.
Returns:
intake_esm.esm_datastore: The catalogue, whose ``.df`` is a pandas
frame with one row per published Zarr store.
"""
if cls._catalog is None or refresh:
cls._catalog = intake.open_esm_datastore(
"https://storage.googleapis.com/cmip6/pangeo-cmip6.json"
)
return cls._catalog
save_csv(self, filename)
¶
Write the dataset to CSV in long form, doing nothing if none is loaded.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
filename |
str |
Destination path. The parent directory must exist. |
required |
Returns:
| Type | Description |
|---|---|
None |
Source code in climdata/datasets/CMIPCloud.py
def save_csv(self, filename):
"""Write the dataset to CSV in long form, doing nothing if none is loaded.
Args:
filename (str): Destination path. The parent directory must exist.
Returns:
None
"""
if self.ds is not None:
df = self.ds.to_dataframe().reset_index()
df = self._format(df)
df.to_csv(filename, index=False)
# print(f"Saved CSV to {filename}")
save_netcdf(self, filename)
¶
Write the dataset to NetCDF, doing nothing if none is loaded.
Clears the time encoding inherited from the Zarr store first: it names a
calendar that no longer matches the axis rewritten by
:meth:convert_to_noleap, and the NetCDF writer would otherwise fail on
the contradiction.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
filename |
str |
Destination path. The parent directory must exist. |
required |
Returns:
| Type | Description |
|---|---|
None |
Source code in climdata/datasets/CMIPCloud.py
def save_netcdf(self, filename):
"""Write the dataset to NetCDF, doing nothing if none is loaded.
Clears the time encoding inherited from the Zarr store first: it names a
calendar that no longer matches the axis rewritten by
:meth:`convert_to_noleap`, and the NetCDF writer would otherwise fail on
the contradiction.
Args:
filename (str): Destination path. The parent directory must exist.
Returns:
None
"""
if self.ds is not None:
if "time" in self.ds.variables:
self.ds["time"].encoding.clear()
self.ds.to_netcdf(filename)
# print(f"Saved NetCDF to {filename}")
save_zarr(self, store_path)
¶
Write the dataset to a Zarr store, overwriting any existing one.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
store_path |
str |
Destination store path. |
required |
Returns:
| Type | Description |
|---|---|
None |
Source code in climdata/datasets/CMIPCloud.py
def save_zarr(self, store_path):
"""Write the dataset to a Zarr store, overwriting any existing one.
Args:
store_path (str): Destination store path.
Returns:
None
"""
if self.ds is not None:
self.ds.to_zarr(store_path, mode="w")
print(f"Saved Zarr to {store_path}")
climdata.datasets.NEXGDDP.NEXGDDP
¶
Download and assemble NEX-GDDP-CMIP6 downscaled projections.
Call order matters here: :meth:extract should run before :meth:load, so
the region of interest is known while each global annual file is being
opened. Calling it afterwards still works but materialises whole global
frames first.
Unrecognised models, experiments and variables are warned about rather than
rejected, because NASA adds to the collection faster than the hardcoded
lists here are updated; a genuinely wrong name surfaces as a download
failure in :meth:fetch.
Attributes:
| Name | Type | Description |
|---|---|---|
cfg |
DictConfig |
Configuration supplying |
ds |
xr.Dataset | None |
The loaded dataset, set by :meth: |
experiment_id |
str |
Experiment, e.g. |
source_id |
str |
Model, uppercase, e.g. |
member_id |
str |
Realisation, e.g. |
grid_label |
str |
Grid label, e.g. |
base_url |
str |
NASA THREDDS HTTP file server root. |
downloaded_files |
list[str] |
Local paths accumulated by :meth: |
Examples:
>>> nex = NEXGDDP(cfg)
>>> nex.get_source_ids()[:3]
['ACCESS-CM2', 'ACCESS-ESM1-5', 'BCC-CSM2-MR']
Source code in climdata/datasets/NEXGDDP.py
class NEXGDDP:
"""Download and assemble NEX-GDDP-CMIP6 downscaled projections.
Call order matters here: :meth:`extract` should run *before* :meth:`load`, so
the region of interest is known while each global annual file is being
opened. Calling it afterwards still works but materialises whole global
frames first.
Unrecognised models, experiments and variables are warned about rather than
rejected, because NASA adds to the collection faster than the hardcoded
lists here are updated; a genuinely wrong name surfaces as a download
failure in :meth:`fetch`.
Attributes:
cfg (DictConfig): Configuration supplying ``variables``, ``time_range``
and ``data_dir``.
ds (xr.Dataset | None): The loaded dataset, set by :meth:`load`.
experiment_id (str): Experiment, e.g. ``"historical"``, ``"ssp585"``.
source_id (str): Model, uppercase, e.g. ``"GFDL-ESM4"``.
member_id (str): Realisation, e.g. ``"r1i1p1f1"``. Discovered by probing
when not configured.
grid_label (str): Grid label, e.g. ``"gn"`` or ``"gr1"``. Also discovered.
base_url (str): NASA THREDDS HTTP file server root.
downloaded_files (list[str]): Local paths accumulated by :meth:`fetch`.
Example:
>>> nex = NEXGDDP(cfg) # doctest: +SKIP
>>> nex.get_source_ids()[:3] # doctest: +SKIP
['ACCESS-CM2', 'ACCESS-ESM1-5', 'BCC-CSM2-MR']
"""
# Available models in NEX-GDDP-CMIP6
AVAILABLE_MODELS = [
"ACCESS-CM2",
"ACCESS-ESM1-5",
"BCC-CSM2-MR",
"CESM2",
"CESM2-WACCM",
"CMCC-CM2-SR5",
"CMCC-ESM2",
"CNRM-CM6-1",
"CNRM-ESM2-1",
"CanESM5",
"EC-Earth3",
"EC-Earth3-Veg-LR",
"FGOALS-g3",
"GFDL-CM4",
"GFDL-ESM4",
"GISS-E2-1-G",
"HadGEM3-GC31-LL",
"HadGEM3-GC31-MM",
"IITM-ESM",
"INM-CM4-8",
"INM-CM5-0",
"IPSL-CM6A-LR",
"KACE-1-0-G",
"KIOST-ESM",
"MIROC-ES2L",
"MIROC6",
"MPI-ESM1-2-HR",
"MPI-ESM1-2-LR",
"MRI-ESM2-0",
"NESM3",
"NorESM2-LM",
"NorESM2-MM",
"TaiESM1",
"UKESM1-0-LL",
]
# Available experiments
AVAILABLE_EXPERIMENTS = [
"historical",
"ssp126",
"ssp245",
"ssp370",
"ssp585",
]
# Available variables
AVAILABLE_VARIABLES = {
'tas': 'Near-Surface Air Temperature',
'tasmax': 'Daily Maximum Near-Surface Air Temperature',
'tasmin': 'Daily Minimum Near-Surface Air Temperature',
'pr': 'Precipitation',
'hurs': 'Near-Surface Relative Humidity',
'huss': 'Near-Surface Specific Humidity',
'sfcWind': 'Near-Surface Wind Speed',
'rsds': 'Surface Downwelling Shortwave Radiation',
'rlds': 'Surface Downwelling Longwave Radiation',
}
def __init__(self, cfg: DictConfig):
"""Initialize NEX-GDDP data accessor.
Args:
cfg (DictConfig):
Configuration with required fields:
- variables: list of variable names
- time_range: dict with start_date and end_date
- data_dir: directory to save downloaded files
Optional fields:
- experiment_id: experiment/scenario (default: 'historical')
- source_id: model name (default: 'GFDL-ESM4')
- member_id: realization (default: 'r1i1p1f1')
"""
self.cfg = cfg
self.ds = None
self.downloaded_files = []
self._extract_mode = None
self._extract_params = None
# Extract NEX-GDDP-specific parameters
self.experiment_id = cfg.get('experiment_id', 'historical')
self.source_id = cfg.get('source_id', 'GFDL-ESM4')
self.member_id = cfg.get('member_id', 'r1i1p1f1')
# Base URL for NASA THREDDS HTTP File Server
self.base_url = "https://ds.nccs.nasa.gov/thredds/fileServer/AMES/NEX/GDDP-CMIP6"
self.catalog_base_url = "https://ds.nccs.nasa.gov/thredds/catalog/AMES/NEX/GDDP-CMIP6"
# Validate inputs
self._validate_inputs()
self._validate_time_range()
# Auto-discover member_id and grid_label if not explicitly set
self.grid_label = cfg.get('grid_label', 'gn')
self._auto_discover_metadata()
def _validate_inputs(self):
"""Warn about model, experiment or variable names that look unfamiliar.
Warns rather than raises: the hardcoded lists lag what NASA publishes, so
rejecting an unknown name would block access to newly added data. A name
that is genuinely wrong fails later, in :meth:`fetch`.
Returns:
None: Messages are printed.
"""
# Normalize model name to uppercase with hyphens
# self.source_id = self.source_id.upper().replace('_', '-')
if self.source_id not in self.AVAILABLE_MODELS:
print(f"⚠️ Warning: Model '{self.source_id}' may not be available.")
print(f" Available models: {', '.join(self.AVAILABLE_MODELS[:5])}...")
if self.experiment_id not in self.AVAILABLE_EXPERIMENTS:
print(f"⚠️ Warning: Experiment '{self.experiment_id}' may not be available.")
print(f" Available experiments: {', '.join(self.AVAILABLE_EXPERIMENTS)}")
for var in self.cfg.variables:
if var not in self.AVAILABLE_VARIABLES:
print(f"⚠️ Warning: Variable '{var}' may not be available.")
print(f" Available variables: {', '.join(self.AVAILABLE_VARIABLES.keys())}")
def _validate_time_range(self):
"""Validate that the requested time range is appropriate for the experiment.
Historical runs: 1950-2014
SSP scenarios: 2015-2100
Raises:
ValueError: If the time range doesn't match the experiment period
"""
start_date = datetime.fromisoformat(self.cfg.time_range.start_date)
end_date = datetime.fromisoformat(self.cfg.time_range.end_date)
start_year = start_date.year
end_year = end_date.year
# Define valid periods for each experiment type
if self.experiment_id == 'historical':
valid_start = 1950
valid_end = 2014
period_name = "Historical"
elif self.experiment_id.startswith('ssp'):
valid_start = 2015
valid_end = 2100
period_name = f"SSP scenario ({self.experiment_id})"
else:
# Unknown experiment, skip validation
return
# Check if requested period is outside valid range
if end_year < valid_start or start_year > valid_end:
raise ValueError(
f"❌ Time range mismatch for experiment '{self.experiment_id}'!\n"
f" Requested: {start_year}-{end_year}\n"
f" Valid period for {period_name}: {valid_start}-{valid_end}\n"
f" \n"
f" Hint: Use 'historical' for years 1950-2014, and SSP scenarios for 2015-2100."
)
# Warn if requested period extends beyond valid range
if start_year < valid_start or end_year > valid_end:
print(f"⚠️ Warning: Requested time range {start_year}-{end_year} extends beyond")
print(f" the typical {period_name} period ({valid_start}-{valid_end}).")
print(f" Data availability may be limited.")
def _auto_discover_metadata(self):
"""Probe the THREDDS server for this model's realisation and grid label.
NEX-GDDP has no catalogue API, and models differ in which realisation
they publish (``r1i1p1f1`` for most, ``r1i1p1f2`` for the UKESM family)
and on which grid (``gn`` or ``gr1``). Both are found by issuing HEAD
requests against candidate URLs and keeping the first that answers 200.
Values given explicitly in the config are used as-is and not probed.
Returns:
None: Sets :attr:`member_id` and :attr:`grid_label`.
"""
# Only auto-discover if member_id was not explicitly set
if 'member_id' in self.cfg:
print(f"ℹ️ Using configured member_id: {self.member_id}")
return
print(f"🔍 Auto-discovering metadata for {self.source_id}/{self.experiment_id}...")
try:
# Get available member_ids from THREDDS catalog
member_ids = self._get_available_member_ids()
if not member_ids:
print(f"⚠️ Could not auto-discover member_ids, using default: {self.member_id}")
return
# Use the first available member_id
discovered_member_id = member_ids[0]
# Get grid label from a sample file
grid_label = self._get_grid_label(discovered_member_id)
if grid_label:
self.grid_label = grid_label
print(f"✓ Discovered grid_label: {self.grid_label}")
if discovered_member_id != self.member_id:
self.member_id = discovered_member_id
print(f"✓ Discovered member_id: {self.member_id}")
except Exception as e:
print(f"⚠️ Auto-discovery failed: {str(e)}")
print(f" Using defaults: member_id={self.member_id}, grid_label={self.grid_label}")
def _get_available_member_ids(self) -> List[str]:
"""Get available member_ids (realizations) for the current model and experiment
by probing the server with common member_id patterns.
Returns:
List[str]: List of available member_ids (e.g., ['r1i1p1f1', 'r2i1p1f1'])
"""
# Common member_id patterns to check
common_members = [
'r1i1p1f1', 'r1i1p1f2', 'r1i1p1f3',
'r2i1p1f1', 'r2i1p1f2', 'r3i1p1f1',
'r4i1p1f1', 'r5i1p1f1'
]
available_members = []
# Use the first variable to check availability
if not self.cfg.variables:
return []
test_var = self.cfg.variables[0]
# Determine a test year based on experiment
if self.experiment_id == 'historical':
test_year = 2000 # Use a year that's more likely to exist
else:
test_year = 2050
print(f" Checking available realizations...")
for member_id in common_members:
# Try common grid labels (gr1 is used by many models)
for grid_label in ['gr1', 'gn', 'gr']:
url = self._construct_test_url(test_var, test_year, member_id, grid_label)
try:
# Use HEAD request to check if file exists without downloading
response = requests.head(url, timeout=10, allow_redirects=True)
if response.status_code == 200:
available_members.append(member_id)
print(f" ✓ Found: {member_id} (grid: {grid_label})")
break # Found this member_id, move to next
except requests.exceptions.RequestException:
continue
return sorted(set(available_members))
def _construct_test_url(self, variable: str, year: int, member_id: str, grid_label: str) -> str:
"""Construct a test URL for probing file availability.
Args:
variable (str): Variable name
year (int): Year
member_id (str): Member/realization ID
grid_label (str): Grid label (e.g., 'gn', 'gr')
Returns:
str: Complete URL for testing
"""
filename = f"{variable}_day_{self.source_id}_{self.experiment_id}_{member_id}_{grid_label}_{year}_v2.0.nc"
url = f"{self.base_url}/{self.source_id}/{self.experiment_id}/{member_id}/{variable}/{filename}"
return url
def _get_grid_label(self, member_id: str) -> Optional[str]:
"""Get grid label by probing with common grid labels.
Args:
member_id (str): The member_id to check
Returns:
str or None: Grid label (e.g., 'gn', 'gr') or None if not found
"""
# Use the first available variable
if not self.cfg.variables:
return None
sample_var = self.cfg.variables[0]
# Determine a test year based on experiment
if self.experiment_id == 'historical':
test_year = 2000 # Use a year that's more likely to exist
else:
test_year = 2050
# Try common grid labels (gr1 is common for NEX-GDDP)
for grid_label in ['gr1', 'gn', 'gr']:
url = self._construct_test_url(sample_var, test_year, member_id, grid_label)
try:
response = requests.head(url, timeout=10, allow_redirects=True)
if response.status_code == 200:
return grid_label
except requests.exceptions.RequestException:
continue
return None
def get_experiment_ids(self) -> List[str]:
"""Get available NEX-GDDP-CMIP6 experiment IDs.
Returns:
List[str]: List of available experiment IDs
"""
return self.AVAILABLE_EXPERIMENTS.copy()
def get_source_ids(self, experiment_id: Optional[str] = None) -> List[str]:
"""Get available NEX-GDDP-CMIP6 model (source) IDs.
Args:
experiment_id (str, optional): Experiment ID (not used for NEX-GDDP as
models are consistent across experiments)
Returns:
List[str]: List of available model IDs
"""
return self.AVAILABLE_MODELS.copy()
def get_variables(self) -> Dict[str, str]:
"""Get available variables with descriptions.
Returns:
Dict[str, str]: Dictionary mapping variable names to descriptions
"""
return self.AVAILABLE_VARIABLES.copy()
def get_member_ids(self) -> List[str]:
"""Get available member IDs (realizations) for the current model and experiment
by querying the THREDDS server.
Returns:
List[str]: List of available member_ids (e.g., ['r1i1p1f1', 'r2i1p1f1'])
"""
return self._get_available_member_ids()
def _construct_download_url(self, variable: str, year: int) -> str:
"""Construct THREDDS HTTP file server URL for downloading NEX-GDDP data.
Args:
variable (str): Variable name (e.g., 'tasmax', 'pr')
year (int): Year to download
Returns:
tuple: (url, filename) - Complete HTTP URL and filename
"""
# Filename format: {var}_day_{model}_{experiment}_{member}_{grid}_{year}_v2.0.nc
filename = f"{variable}_day_{self.source_id}_{self.experiment_id}_{self.member_id}_{self.grid_label}_{year}_v2.0.nc"
# Construct full URL using HTTP file server
url = f"{self.base_url}/{self.source_id}/{self.experiment_id}/{self.member_id}/{variable}/{filename}"
return url, filename
def fetch(self, max_download_workers: int = 8):
"""Download NEX-GDDP-CMIP6 files from NASA THREDDS server using
``dask.delayed`` for fully parallel file-existence checks and downloads.
Both phases (check + download) use the ``"threads"`` scheduler so that
the GIL is released during I/O and multiple HTTP connections run
concurrently without spawning heavyweight processes.
Args:
max_download_workers (int, default 8): Maximum concurrent HTTP download
connections. Keep this ≤ 8 to avoid triggering NASA THREDDS rate-
limiting / 503 errors.
"""
print(f"🔍 Downloading NEX-GDDP-CMIP6 data from NASA THREDDS...")
print(f" Model: {self.source_id}, Experiment: {self.experiment_id}")
start_date = datetime.fromisoformat(self.cfg.time_range.start_date)
end_date = datetime.fromisoformat(self.cfg.time_range.end_date)
base_dir = (
Path(self.cfg.data_dir)
/ "NEXGDDP"
/ self.source_id
/ self.experiment_id
)
base_dir.mkdir(parents=True, exist_ok=True)
years = range(start_date.year, end_date.year + 1)
# ── Build full (url, local_path) task list ────────────────────────────
all_tasks = []
for var in self.cfg.variables:
var_dir = base_dir / var
var_dir.mkdir(parents=True, exist_ok=True)
for year in years:
url, filename = self._construct_download_url(var, year)
all_tasks.append({"url": url, "local_path": var_dir / filename,
"filename": filename})
n_total = len(all_tasks)
print(f" Total files to check: {n_total} "
f"({len(self.cfg.variables)} var(s) × {len(years)} year(s))")
# ── Phase 1: parallel existence / size checks (threads, no data I/O) ──
@dask.delayed(pure=False)
def _check(task):
lp = task["local_path"]
if lp.exists() and self._verify_file_complete(lp, task["url"]):
return {"skip": True, "local_path": str(lp)}
return {"skip": False, **task}
print(f"\n🔎 Checking {n_total} files for completeness (parallel HEAD requests)...")
with ProgressBar():
check_results = dask.compute(
[_check(t) for t in all_tasks],
scheduler="threads",
num_workers=16, # HEAD requests are cheap
)[0]
skipped = [r for r in check_results if r["skip"]]
to_download = [r for r in check_results if not r["skip"]]
for r in skipped:
self.downloaded_files.append(r["local_path"])
print(f" ✓ {len(skipped):>4d} files already complete — skipping")
print(f" ↓ {len(to_download):>4d} files need downloading")
if not to_download:
print(f"\n✅ All {len(self.downloaded_files)} files already present")
return
# ── Phase 2: parallel downloads (threads, capped to avoid throttling) ─
@dask.delayed(pure=False)
def _download(task):
success = self._download_with_retry(
task["url"], task["local_path"], max_retries=5
)
return {"success": success,
"local_path": str(task["local_path"]),
"filename": task["filename"]}
print(f"\n📥 Downloading {len(to_download)} files "
f"(max {max_download_workers} concurrent connections)...")
with ProgressBar():
dl_results = dask.compute(
[_download(t) for t in to_download],
scheduler="threads",
num_workers=min(max_download_workers, len(to_download)),
)[0]
failed = []
for r in dl_results:
if r["success"]:
self.downloaded_files.append(r["local_path"])
else:
failed.append(r["filename"])
print(f"\n✅ {len(self.downloaded_files)} files ready "
f"({len(skipped)} cached + {len(to_download) - len(failed)} downloaded)")
if failed:
print(f"⚠️ {len(failed)} file(s) failed after all retries:")
for fn in failed:
print(f" • {fn}")
def _verify_file_complete(self, local_path: Path, url: str) -> bool:
"""Verify if a local file is complete by comparing size with server.
Args:
local_path (Path): Path to local file
url (str): URL of the file on server
Returns:
bool: True if file is complete, False otherwise
"""
try:
local_size = local_path.stat().st_size
# Get expected size from server
response = requests.head(url, timeout=30, allow_redirects=True)
if response.status_code == 200:
expected_size = int(response.headers.get('Content-Length', 0))
if expected_size > 0 and local_size == expected_size:
return True
return False
except Exception:
return False
def _download_with_retry(self, url: str, local_path: Path, max_retries: int = 5) -> bool:
"""Download a file with retry logic and resume capability.
Args:
url (str): URL to download from
local_path (Path): Local path to save to
max_retries (int): Maximum number of retry attempts
Returns:
bool: True if download succeeded, False otherwise
"""
for attempt in range(max_retries):
try:
# Get expected file size
head_response = requests.head(url, timeout=30, allow_redirects=True)
head_response.raise_for_status()
expected_size = int(head_response.headers.get('Content-Length', 0))
# Check if we have a partial download
existing_size = 0
if local_path.exists():
existing_size = local_path.stat().st_size
# If file is complete, we're done
if existing_size == expected_size:
return True
# If partial, try to resume
if existing_size > 0 and existing_size < expected_size:
headers = {'Range': f'bytes={existing_size}-'}
mode = 'ab' # Append mode
else:
# File exists but wrong size, start over
local_path.unlink()
headers = {}
mode = 'wb'
existing_size = 0
else:
headers = {}
mode = 'wb'
# Download with streaming
response = requests.get(
url,
headers=headers,
stream=True,
timeout=120,
allow_redirects=True
)
response.raise_for_status()
# Save file
with open(local_path, mode) as f:
for chunk in response.iter_content(chunk_size=8192):
if chunk:
f.write(chunk)
# Verify download completed
final_size = local_path.stat().st_size
if final_size == expected_size or (existing_size > 0 and final_size == expected_size):
return True
else:
if attempt < max_retries - 1:
print(f" ↻ Incomplete download (attempt {attempt + 1}/{max_retries}), retrying...")
time.sleep(2 ** attempt) # Exponential backoff
continue
else:
return False
except (requests.exceptions.RequestException, IOError) as e:
if attempt < max_retries - 1:
wait_time = 2 ** attempt
print(f" ↻ Download error (attempt {attempt + 1}/{max_retries}): {type(e).__name__}")
print(f" Retrying in {wait_time} seconds...")
time.sleep(wait_time)
continue
else:
print(f" ❌ Failed after {max_retries} attempts: {str(e)}")
# Clean up partial file on final failure
if local_path.exists():
local_path.unlink()
return False
return False
def _extract_preprocess(self, ds: xr.Dataset, var: str) -> xr.Dataset:
"""Clip one annual file to the region of interest as it is opened.
Applied per file during loading, so each global frame is cut down before
it joins the concatenation. For a global file of roughly 1440x600 cells
reduced to a German box of about 36x32, that is around 750 times less
data read per file.
Everything but the target variable is dropped first, for the same reason.
Args:
ds (xr.Dataset): One year of global data, as just opened.
var (str): CF variable name to keep.
Returns:
xr.Dataset: The clipped single-variable dataset, or ``ds`` reduced to
``var`` if no extraction mode was set.
"""
# Drop all variables except the target to avoid memory bloat
drop = [v for v in ds.data_vars if v != var]
if drop:
ds = ds.drop_vars(drop)
if self._extract_mode == "point":
lon, lat, buffer_deg = self._extract_params
if buffer_deg > 0:
ds = ds.sel(
lat=slice(lat - buffer_deg, lat + buffer_deg),
lon=slice(lon - buffer_deg, lon + buffer_deg),
).mean(["lat", "lon"])
else:
ds = ds.sel(lat=lat, lon=lon, method="nearest")
elif self._extract_mode == "box":
box = self._extract_params
ds = ds.sel(
lat=slice(box["lat_min"], box["lat_max"]),
lon=slice(box["lon_min"], box["lon_max"]),
)
elif self._extract_mode == "shapefile":
import rioxarray
from shapely.geometry import mapping as shape_mapping
gdf = self._extract_params
ds = ds.rio.set_spatial_dims(x_dim="lon", y_dim="lat")
ds = ds.rio.write_crs("EPSG:4326", inplace=True)
clipped = [ds.rio.clip([shape_mapping(g)], gdf.crs, drop=True)
for g in gdf.geometry]
ds = xr.concat(clipped, dim="geom_id")
return ds
def load(self):
"""
Load NEX-GDDP files using a ThreadPoolExecutor — one thread per file.
Each thread calls ``xr.open_dataset(engine="h5netcdf")`` directly
(no dask graph overhead), applies ``_extract_preprocess`` to clip the
global grid (~1440×600) to the target region (~36×32 for Germany),
and loads the result into RAM as plain numpy arrays.
All annual datasets for a variable are then concatenated along ``time``
with ``xr.concat`` and merged across variables. The result is a fully
in-memory xarray Dataset — no dask, no lazy evaluation, no scheduler.
This is deliberately not ``open_mfdataset``. Because each file is clipped
to a small region and read immediately into NumPy, there is nothing for a
Dask graph to defer, and building one costs more than it saves:
``h5netcdf`` releases the GIL during reads, so plain threads already run
in parallel, the concatenation is a straight O(n) copy, and no chunk
unification is needed.
The consequence is that the returned dataset is fully in memory. Clip
before loading — see :meth:`extract` — or a multi-decade global request
will not fit.
Returns:
xr.Dataset: The merged dataset, also stored on :attr:`ds`.
Raises:
ValueError: If :meth:`fetch` has not run, or produced no files.
"""
from concurrent.futures import ThreadPoolExecutor, as_completed
if not self.downloaded_files:
raise ValueError("No files to load. Run fetch() first.")
# Group files by variable
files_by_var: Dict[str, List[str]] = {}
for fpath in self.downloaded_files:
for var in self.cfg.variables:
if f"/{var}/" in fpath or f"_{var}_day_" in fpath:
files_by_var.setdefault(var, []).append(fpath)
break
start = self.cfg.time_range.start_date
end = self.cfg.time_range.end_date
n_total = sum(len(v) for v in files_by_var.values())
n_workers = min(n_total, (os.cpu_count()/4 or 8))
print(f"📂 Loading {n_total} NEX-GDDP files "
f"({n_workers} threads, engine=h5netcdf, clip-per-file) ...")
if self._extract_mode:
print(f" mode='{self._extract_mode}' → each file clipped before concat")
def _open_one(fpath: str, var: str) -> xr.Dataset:
"""Open one annual file, clip to region, load to numpy, return Dataset."""
ds = xr.open_dataset(fpath, engine="h5netcdf", mask_and_scale=True)
ds = self._extract_preprocess(ds, var)
# .load() materialises to numpy inside the thread — no dask involved
return ds.load()
datasets = []
for var, file_list in files_by_var.items():
file_list = sorted(file_list)
print(f" {var}: opening {len(file_list)} files with {n_workers} threads ...")
annual: Dict[str, xr.Dataset] = {}
with ThreadPoolExecutor(max_workers=n_workers) as pool:
futures = {pool.submit(_open_one, fp, var): fp for fp in file_list}
for fut in as_completed(futures):
fp = futures[fut]
try:
annual[fp] = fut.result()
except Exception as exc:
print(f" ⚠️ {fp} failed: {exc}")
# Concatenate in sorted order (as_completed is unordered)
sorted_ds = [annual[fp] for fp in file_list if fp in annual]
ds_var = xr.concat(sorted_ds, dim="time",
data_vars="minimal", coords="minimal", compat="override")
ds_var = ds_var.sel(time=slice(start, end))
datasets.append(ds_var)
if len(datasets) == 1:
self.ds = datasets[0]
else:
self.ds = xr.merge(datasets, compat="override", join="override")
# Add metadata
self.ds.attrs['source'] = 'NEX-GDDP-CMIP6'
self.ds.attrs['source_url'] = 'https://www.nccs.nasa.gov/services/data-collections/land-based-products/nex-gddp-cmip6'
self.ds.attrs['experiment_id'] = self.experiment_id
self.ds.attrs['source_id'] = self.source_id
self.ds.attrs['member_id'] = self.member_id
self.ds.attrs['resolution'] = '0.25 degrees'
self.ds.attrs['description'] = f'NEX-GDDP-CMIP6 downscaled {self.experiment_id} from {self.source_id}'
print(f"✅ Loaded dataset with {len(self.ds.data_vars)} variable(s) backed by Dask arrays")
print(f" Chunks: {dict(self.ds.chunks)}")
# Apply extraction if it was set before loading
if self._extract_mode is not None:
self._apply_extraction()
def extract(self, *, point: Optional[Tuple[float, float]] = None,
box: Optional[Dict] = None,
shapefile: Optional[str] = None,
buffer_km: float = 0.0):
"""Store extraction instructions to be applied during or after load.
Args:
point (tuple of (lon, lat), optional): Extract data for a specific point
location
box (dict, optional): Extract data for a bounding box with keys: lon_min,
lon_max, lat_min, lat_max
shapefile (str or GeoDataFrame, optional): Extract data for a shapefile
region
buffer_km (float, default=0.0): Buffer distance in kilometers around point
(converted to degrees)
"""
if point is not None:
lon, lat = point
buffer_deg = buffer_km / 111.0
self._extract_mode = "point"
self._extract_params = (lon, lat, buffer_deg)
# Apply extraction if dataset is already loaded
if self.ds is not None:
self._apply_extraction()
elif box is not None:
self._extract_mode = "box"
self._extract_params = box
if self.ds is not None:
self._apply_extraction()
elif shapefile is not None:
import geopandas as gpd
if isinstance(shapefile, str):
gdf = gpd.read_file(shapefile)
else:
gdf = shapefile
self._extract_mode = "shapefile"
self._extract_params = gdf
if self.ds is not None:
self._apply_extraction()
def _apply_extraction(self):
"""Run the subset recorded by :meth:`extract` against :attr:`ds`.
The fallback path for an :meth:`extract` call made after :meth:`load`; the
efficient path is :meth:`_extract_preprocess`, which runs per file during
loading. Coordinate names are detected rather than assumed, since
NEX-GDDP files use ``lat``/``lon`` and the clipped output may not.
Returns:
None: :attr:`ds` is replaced in place.
"""
if self._extract_mode == "point":
lon, lat, buffer_deg = self._extract_params
# Find coordinate names
lat_name = None
lon_name = None
for coord in self.ds.coords:
if coord.lower() in ['lat', 'latitude']:
lat_name = coord
elif coord.lower() in ['lon', 'longitude']:
lon_name = coord
if lat_name is None or lon_name is None:
raise ValueError("Could not find latitude/longitude coordinates in dataset")
# Preprocess already reduced lat/lon to scalars — nothing left to select
if lon_name not in self.ds.dims:
return
if buffer_deg > 0:
self.ds = self.ds.sel(
{lon_name: slice(lon - buffer_deg, lon + buffer_deg),
lat_name: slice(lat - buffer_deg, lat + buffer_deg)}
).mean([lat_name, lon_name])
else:
self.ds = self.ds.sel({lon_name: lon, lat_name: lat}, method="nearest")
elif self._extract_mode == "box":
box = self._extract_params
# Find coordinate names
lat_name = None
lon_name = None
for coord in self.ds.coords:
if coord.lower() in ['lat', 'latitude']:
lat_name = coord
elif coord.lower() in ['lon', 'longitude']:
lon_name = coord
if lat_name is None or lon_name is None:
raise ValueError("Could not find latitude/longitude coordinates in dataset")
if lon_name not in self.ds.dims:
return
self.ds = self.ds.sel(
{lon_name: slice(box["lon_min"], box["lon_max"]),
lat_name: slice(box["lat_min"], box["lat_max"])}
)
elif self._extract_mode == "shapefile":
import rioxarray
from shapely.geometry import mapping
gdf = self._extract_params
# Find coordinate names
lat_name = None
lon_name = None
for coord in self.ds.coords:
if coord.lower() in ['lat', 'latitude']:
lat_name = coord
elif coord.lower() in ['lon', 'longitude']:
lon_name = coord
if lat_name is None or lon_name is None:
raise ValueError("Could not find latitude/longitude coordinates in dataset")
self.ds = self.ds.rio.set_spatial_dims(x_dim=lon_name, y_dim=lat_name)
self.ds = self.ds.rio.write_crs("EPSG:4326", inplace=True)
clipped_list = []
for geom in gdf.geometry:
clipped = self.ds.rio.clip([mapping(geom)], gdf.crs, drop=True)
clipped_list.append(clipped)
self.ds = xr.concat(clipped_list, dim="geom_id")
def save_netcdf(self, filename: str):
"""Save the dataset to a NetCDF file.
Args:
filename (str): Output filename or path
"""
if self.ds is None:
raise ValueError("No dataset loaded. Run load() first.")
output_path = Path(filename)
output_path.parent.mkdir(parents=True, exist_ok=True)
self.ds.to_netcdf(output_path)
print(f"💾 Saved to: {output_path}")
def save_csv(self, filename: str):
"""Save the dataset to a CSV file.
Args:
filename (str): Output filename or path
"""
if self.ds is None:
raise ValueError("No dataset loaded. Run load() first.")
output_path = Path(filename)
output_path.parent.mkdir(parents=True, exist_ok=True)
df = self.ds.to_dataframe()
df.to_csv(output_path)
print(f"💾 Saved to: {output_path}")
def to_dataframe(self) -> pd.DataFrame:
"""Convert the dataset to a pandas DataFrame.
Returns:
pd.DataFrame: Dataset as DataFrame
"""
if self.ds is None:
raise ValueError("No dataset loaded. Run load() first.")
return self.ds.to_dataframe()
__init__(self, cfg)
special
¶
Initialize NEX-GDDP data accessor.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
cfg |
DictConfig |
Configuration with required fields: - variables: list of variable names - time_range: dict with start_date and end_date - data_dir: directory to save downloaded files Optional fields: - experiment_id: experiment/scenario (default: 'historical') - source_id: model name (default: 'GFDL-ESM4') - member_id: realization (default: 'r1i1p1f1') |
required |
Source code in climdata/datasets/NEXGDDP.py
def __init__(self, cfg: DictConfig):
"""Initialize NEX-GDDP data accessor.
Args:
cfg (DictConfig):
Configuration with required fields:
- variables: list of variable names
- time_range: dict with start_date and end_date
- data_dir: directory to save downloaded files
Optional fields:
- experiment_id: experiment/scenario (default: 'historical')
- source_id: model name (default: 'GFDL-ESM4')
- member_id: realization (default: 'r1i1p1f1')
"""
self.cfg = cfg
self.ds = None
self.downloaded_files = []
self._extract_mode = None
self._extract_params = None
# Extract NEX-GDDP-specific parameters
self.experiment_id = cfg.get('experiment_id', 'historical')
self.source_id = cfg.get('source_id', 'GFDL-ESM4')
self.member_id = cfg.get('member_id', 'r1i1p1f1')
# Base URL for NASA THREDDS HTTP File Server
self.base_url = "https://ds.nccs.nasa.gov/thredds/fileServer/AMES/NEX/GDDP-CMIP6"
self.catalog_base_url = "https://ds.nccs.nasa.gov/thredds/catalog/AMES/NEX/GDDP-CMIP6"
# Validate inputs
self._validate_inputs()
self._validate_time_range()
# Auto-discover member_id and grid_label if not explicitly set
self.grid_label = cfg.get('grid_label', 'gn')
self._auto_discover_metadata()
extract(self, *, point=None, box=None, shapefile=None, buffer_km=0.0)
¶
Store extraction instructions to be applied during or after load.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
point |
tuple of (lon, lat) |
Extract data for a specific point location |
None |
box |
dict |
Extract data for a bounding box with keys: lon_min, lon_max, lat_min, lat_max |
None |
shapefile |
str or GeoDataFrame |
Extract data for a shapefile region |
None |
buffer_km |
float, default=0.0 |
Buffer distance in kilometers around point (converted to degrees) |
0.0 |
Source code in climdata/datasets/NEXGDDP.py
def extract(self, *, point: Optional[Tuple[float, float]] = None,
box: Optional[Dict] = None,
shapefile: Optional[str] = None,
buffer_km: float = 0.0):
"""Store extraction instructions to be applied during or after load.
Args:
point (tuple of (lon, lat), optional): Extract data for a specific point
location
box (dict, optional): Extract data for a bounding box with keys: lon_min,
lon_max, lat_min, lat_max
shapefile (str or GeoDataFrame, optional): Extract data for a shapefile
region
buffer_km (float, default=0.0): Buffer distance in kilometers around point
(converted to degrees)
"""
if point is not None:
lon, lat = point
buffer_deg = buffer_km / 111.0
self._extract_mode = "point"
self._extract_params = (lon, lat, buffer_deg)
# Apply extraction if dataset is already loaded
if self.ds is not None:
self._apply_extraction()
elif box is not None:
self._extract_mode = "box"
self._extract_params = box
if self.ds is not None:
self._apply_extraction()
elif shapefile is not None:
import geopandas as gpd
if isinstance(shapefile, str):
gdf = gpd.read_file(shapefile)
else:
gdf = shapefile
self._extract_mode = "shapefile"
self._extract_params = gdf
if self.ds is not None:
self._apply_extraction()
fetch(self, max_download_workers=8)
¶
Download NEX-GDDP-CMIP6 files from NASA THREDDS server using
dask.delayed for fully parallel file-existence checks and downloads.
Both phases (check + download) use the "threads" scheduler so that
the GIL is released during I/O and multiple HTTP connections run
concurrently without spawning heavyweight processes.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
max_download_workers |
int, default 8 |
Maximum concurrent HTTP download connections. Keep this ≤ 8 to avoid triggering NASA THREDDS rate- limiting / 503 errors. |
8 |
Source code in climdata/datasets/NEXGDDP.py
def fetch(self, max_download_workers: int = 8):
"""Download NEX-GDDP-CMIP6 files from NASA THREDDS server using
``dask.delayed`` for fully parallel file-existence checks and downloads.
Both phases (check + download) use the ``"threads"`` scheduler so that
the GIL is released during I/O and multiple HTTP connections run
concurrently without spawning heavyweight processes.
Args:
max_download_workers (int, default 8): Maximum concurrent HTTP download
connections. Keep this ≤ 8 to avoid triggering NASA THREDDS rate-
limiting / 503 errors.
"""
print(f"🔍 Downloading NEX-GDDP-CMIP6 data from NASA THREDDS...")
print(f" Model: {self.source_id}, Experiment: {self.experiment_id}")
start_date = datetime.fromisoformat(self.cfg.time_range.start_date)
end_date = datetime.fromisoformat(self.cfg.time_range.end_date)
base_dir = (
Path(self.cfg.data_dir)
/ "NEXGDDP"
/ self.source_id
/ self.experiment_id
)
base_dir.mkdir(parents=True, exist_ok=True)
years = range(start_date.year, end_date.year + 1)
# ── Build full (url, local_path) task list ────────────────────────────
all_tasks = []
for var in self.cfg.variables:
var_dir = base_dir / var
var_dir.mkdir(parents=True, exist_ok=True)
for year in years:
url, filename = self._construct_download_url(var, year)
all_tasks.append({"url": url, "local_path": var_dir / filename,
"filename": filename})
n_total = len(all_tasks)
print(f" Total files to check: {n_total} "
f"({len(self.cfg.variables)} var(s) × {len(years)} year(s))")
# ── Phase 1: parallel existence / size checks (threads, no data I/O) ──
@dask.delayed(pure=False)
def _check(task):
lp = task["local_path"]
if lp.exists() and self._verify_file_complete(lp, task["url"]):
return {"skip": True, "local_path": str(lp)}
return {"skip": False, **task}
print(f"\n🔎 Checking {n_total} files for completeness (parallel HEAD requests)...")
with ProgressBar():
check_results = dask.compute(
[_check(t) for t in all_tasks],
scheduler="threads",
num_workers=16, # HEAD requests are cheap
)[0]
skipped = [r for r in check_results if r["skip"]]
to_download = [r for r in check_results if not r["skip"]]
for r in skipped:
self.downloaded_files.append(r["local_path"])
print(f" ✓ {len(skipped):>4d} files already complete — skipping")
print(f" ↓ {len(to_download):>4d} files need downloading")
if not to_download:
print(f"\n✅ All {len(self.downloaded_files)} files already present")
return
# ── Phase 2: parallel downloads (threads, capped to avoid throttling) ─
@dask.delayed(pure=False)
def _download(task):
success = self._download_with_retry(
task["url"], task["local_path"], max_retries=5
)
return {"success": success,
"local_path": str(task["local_path"]),
"filename": task["filename"]}
print(f"\n📥 Downloading {len(to_download)} files "
f"(max {max_download_workers} concurrent connections)...")
with ProgressBar():
dl_results = dask.compute(
[_download(t) for t in to_download],
scheduler="threads",
num_workers=min(max_download_workers, len(to_download)),
)[0]
failed = []
for r in dl_results:
if r["success"]:
self.downloaded_files.append(r["local_path"])
else:
failed.append(r["filename"])
print(f"\n✅ {len(self.downloaded_files)} files ready "
f"({len(skipped)} cached + {len(to_download) - len(failed)} downloaded)")
if failed:
print(f"⚠️ {len(failed)} file(s) failed after all retries:")
for fn in failed:
print(f" • {fn}")
get_experiment_ids(self)
¶
Get available NEX-GDDP-CMIP6 experiment IDs.
Returns:
| Type | Description |
|---|---|
List[str] |
List of available experiment IDs |
Source code in climdata/datasets/NEXGDDP.py
def get_experiment_ids(self) -> List[str]:
"""Get available NEX-GDDP-CMIP6 experiment IDs.
Returns:
List[str]: List of available experiment IDs
"""
return self.AVAILABLE_EXPERIMENTS.copy()
get_member_ids(self)
¶
Get available member IDs (realizations) for the current model and experiment by querying the THREDDS server.
Returns:
| Type | Description |
|---|---|
List[str] |
List of available member_ids (e.g., ['r1i1p1f1', 'r2i1p1f1']) |
Source code in climdata/datasets/NEXGDDP.py
def get_member_ids(self) -> List[str]:
"""Get available member IDs (realizations) for the current model and experiment
by querying the THREDDS server.
Returns:
List[str]: List of available member_ids (e.g., ['r1i1p1f1', 'r2i1p1f1'])
"""
return self._get_available_member_ids()
get_source_ids(self, experiment_id=None)
¶
Get available NEX-GDDP-CMIP6 model (source) IDs.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
experiment_id |
str |
Experiment ID (not used for NEX-GDDP as models are consistent across experiments) |
None |
Returns:
| Type | Description |
|---|---|
List[str] |
List of available model IDs |
Source code in climdata/datasets/NEXGDDP.py
def get_source_ids(self, experiment_id: Optional[str] = None) -> List[str]:
"""Get available NEX-GDDP-CMIP6 model (source) IDs.
Args:
experiment_id (str, optional): Experiment ID (not used for NEX-GDDP as
models are consistent across experiments)
Returns:
List[str]: List of available model IDs
"""
return self.AVAILABLE_MODELS.copy()
get_variables(self)
¶
Get available variables with descriptions.
Returns:
| Type | Description |
|---|---|
Dict[str, str] |
Dictionary mapping variable names to descriptions |
Source code in climdata/datasets/NEXGDDP.py
def get_variables(self) -> Dict[str, str]:
"""Get available variables with descriptions.
Returns:
Dict[str, str]: Dictionary mapping variable names to descriptions
"""
return self.AVAILABLE_VARIABLES.copy()
load(self)
¶
Load NEX-GDDP files using a ThreadPoolExecutor — one thread per file.
Each thread calls xr.open_dataset(engine="h5netcdf") directly
(no dask graph overhead), applies _extract_preprocess to clip the
global grid (~1440×600) to the target region (~36×32 for Germany),
and loads the result into RAM as plain numpy arrays.
All annual datasets for a variable are then concatenated along time
with xr.concat and merged across variables. The result is a fully
in-memory xarray Dataset — no dask, no lazy evaluation, no scheduler.
This is deliberately not open_mfdataset. Because each file is clipped
to a small region and read immediately into NumPy, there is nothing for a
Dask graph to defer, and building one costs more than it saves:
h5netcdf releases the GIL during reads, so plain threads already run
in parallel, the concatenation is a straight O(n) copy, and no chunk
unification is needed.
The consequence is that the returned dataset is fully in memory. Clip
before loading — see :meth:extract — or a multi-decade global request
will not fit.
Returns:
| Type | Description |
|---|---|
xr.Dataset |
The merged dataset, also stored on :attr: |
Exceptions:
| Type | Description |
|---|---|
ValueError |
If :meth: |
Source code in climdata/datasets/NEXGDDP.py
def load(self):
"""
Load NEX-GDDP files using a ThreadPoolExecutor — one thread per file.
Each thread calls ``xr.open_dataset(engine="h5netcdf")`` directly
(no dask graph overhead), applies ``_extract_preprocess`` to clip the
global grid (~1440×600) to the target region (~36×32 for Germany),
and loads the result into RAM as plain numpy arrays.
All annual datasets for a variable are then concatenated along ``time``
with ``xr.concat`` and merged across variables. The result is a fully
in-memory xarray Dataset — no dask, no lazy evaluation, no scheduler.
This is deliberately not ``open_mfdataset``. Because each file is clipped
to a small region and read immediately into NumPy, there is nothing for a
Dask graph to defer, and building one costs more than it saves:
``h5netcdf`` releases the GIL during reads, so plain threads already run
in parallel, the concatenation is a straight O(n) copy, and no chunk
unification is needed.
The consequence is that the returned dataset is fully in memory. Clip
before loading — see :meth:`extract` — or a multi-decade global request
will not fit.
Returns:
xr.Dataset: The merged dataset, also stored on :attr:`ds`.
Raises:
ValueError: If :meth:`fetch` has not run, or produced no files.
"""
from concurrent.futures import ThreadPoolExecutor, as_completed
if not self.downloaded_files:
raise ValueError("No files to load. Run fetch() first.")
# Group files by variable
files_by_var: Dict[str, List[str]] = {}
for fpath in self.downloaded_files:
for var in self.cfg.variables:
if f"/{var}/" in fpath or f"_{var}_day_" in fpath:
files_by_var.setdefault(var, []).append(fpath)
break
start = self.cfg.time_range.start_date
end = self.cfg.time_range.end_date
n_total = sum(len(v) for v in files_by_var.values())
n_workers = min(n_total, (os.cpu_count()/4 or 8))
print(f"📂 Loading {n_total} NEX-GDDP files "
f"({n_workers} threads, engine=h5netcdf, clip-per-file) ...")
if self._extract_mode:
print(f" mode='{self._extract_mode}' → each file clipped before concat")
def _open_one(fpath: str, var: str) -> xr.Dataset:
"""Open one annual file, clip to region, load to numpy, return Dataset."""
ds = xr.open_dataset(fpath, engine="h5netcdf", mask_and_scale=True)
ds = self._extract_preprocess(ds, var)
# .load() materialises to numpy inside the thread — no dask involved
return ds.load()
datasets = []
for var, file_list in files_by_var.items():
file_list = sorted(file_list)
print(f" {var}: opening {len(file_list)} files with {n_workers} threads ...")
annual: Dict[str, xr.Dataset] = {}
with ThreadPoolExecutor(max_workers=n_workers) as pool:
futures = {pool.submit(_open_one, fp, var): fp for fp in file_list}
for fut in as_completed(futures):
fp = futures[fut]
try:
annual[fp] = fut.result()
except Exception as exc:
print(f" ⚠️ {fp} failed: {exc}")
# Concatenate in sorted order (as_completed is unordered)
sorted_ds = [annual[fp] for fp in file_list if fp in annual]
ds_var = xr.concat(sorted_ds, dim="time",
data_vars="minimal", coords="minimal", compat="override")
ds_var = ds_var.sel(time=slice(start, end))
datasets.append(ds_var)
if len(datasets) == 1:
self.ds = datasets[0]
else:
self.ds = xr.merge(datasets, compat="override", join="override")
# Add metadata
self.ds.attrs['source'] = 'NEX-GDDP-CMIP6'
self.ds.attrs['source_url'] = 'https://www.nccs.nasa.gov/services/data-collections/land-based-products/nex-gddp-cmip6'
self.ds.attrs['experiment_id'] = self.experiment_id
self.ds.attrs['source_id'] = self.source_id
self.ds.attrs['member_id'] = self.member_id
self.ds.attrs['resolution'] = '0.25 degrees'
self.ds.attrs['description'] = f'NEX-GDDP-CMIP6 downscaled {self.experiment_id} from {self.source_id}'
print(f"✅ Loaded dataset with {len(self.ds.data_vars)} variable(s) backed by Dask arrays")
print(f" Chunks: {dict(self.ds.chunks)}")
# Apply extraction if it was set before loading
if self._extract_mode is not None:
self._apply_extraction()
save_csv(self, filename)
¶
Save the dataset to a CSV file.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
filename |
str |
Output filename or path |
required |
Source code in climdata/datasets/NEXGDDP.py
def save_csv(self, filename: str):
"""Save the dataset to a CSV file.
Args:
filename (str): Output filename or path
"""
if self.ds is None:
raise ValueError("No dataset loaded. Run load() first.")
output_path = Path(filename)
output_path.parent.mkdir(parents=True, exist_ok=True)
df = self.ds.to_dataframe()
df.to_csv(output_path)
print(f"💾 Saved to: {output_path}")
save_netcdf(self, filename)
¶
Save the dataset to a NetCDF file.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
filename |
str |
Output filename or path |
required |
Source code in climdata/datasets/NEXGDDP.py
def save_netcdf(self, filename: str):
"""Save the dataset to a NetCDF file.
Args:
filename (str): Output filename or path
"""
if self.ds is None:
raise ValueError("No dataset loaded. Run load() first.")
output_path = Path(filename)
output_path.parent.mkdir(parents=True, exist_ok=True)
self.ds.to_netcdf(output_path)
print(f"💾 Saved to: {output_path}")
to_dataframe(self)
¶
Convert the dataset to a pandas DataFrame.
Returns:
| Type | Description |
|---|---|
pd.DataFrame |
Dataset as DataFrame |
Source code in climdata/datasets/NEXGDDP.py
def to_dataframe(self) -> pd.DataFrame:
"""Convert the dataset to a pandas DataFrame.
Returns:
pd.DataFrame: Dataset as DataFrame
"""
if self.ds is None:
raise ValueError("No dataset loaded. Run load() first.")
return self.ds.to_dataframe()
climdata.datasets.CMIP_W5E5.CMIPW5E5
¶
Download and assemble ISIMIP3b bias-adjusted CMIP6 projections.
The lifecycle is fetch -> load -> extract. :meth:extract may be
called before :meth:load, in which case the subset is stored and applied as
soon as data arrives.
Attributes:
| Name | Type | Description |
|---|---|---|
cfg |
DictConfig |
Configuration supplying |
ds |
xr.Dataset | None |
The loaded dataset, set by :meth: |
client |
ISIMIPClient |
ISIMIP API client, created in :meth: |
experiment_id |
str |
Experiment, e.g. |
source_id |
str |
Model, lowercase, e.g. |
member_id |
str |
Realisation, e.g. |
downloaded_files |
list[str] |
Local paths accumulated by :meth: |
Examples:
>>> cmip = CMIPW5E5(cfg)
>>> cmip.get_source_ids("ssp585")
['gfdl-esm4', 'ipsl-cm6a-lr', 'mpi-esm1-2-hr', ...]
Source code in climdata/datasets/CMIP_W5E5.py
class CMIPW5E5:
"""Download and assemble ISIMIP3b bias-adjusted CMIP6 projections.
The lifecycle is ``fetch`` -> ``load`` -> ``extract``. :meth:`extract` may be
called before :meth:`load`, in which case the subset is stored and applied as
soon as data arrives.
Attributes:
cfg (DictConfig): Configuration supplying ``variables``, ``time_range``,
``data_dir`` and ``dataset``.
ds (xr.Dataset | None): The loaded dataset, set by :meth:`load`.
client (ISIMIPClient): ISIMIP API client, created in :meth:`__init__`.
experiment_id (str): Experiment, e.g. ``"historical"``, ``"ssp585"``.
source_id (str): Model, lowercase, e.g. ``"gfdl-esm4"``.
member_id (str): Realisation, e.g. ``"r1i1p1f1"``.
downloaded_files (list[str]): Local paths accumulated by :meth:`fetch`.
Example:
>>> cmip = CMIPW5E5(cfg) # doctest: +SKIP
>>> cmip.get_source_ids("ssp585") # doctest: +SKIP
['gfdl-esm4', 'ipsl-cm6a-lr', 'mpi-esm1-2-hr', ...]
"""
def __init__(self, cfg: DictConfig):
"""Bind a configuration, open an ISIMIP client and validate the period.
Args:
cfg (DictConfig): Configuration with ``variables``, ``time_range``,
``data_dir``, ``dataset``, and optionally ``experiment_id``
(default ``"historical"``), ``source_id`` (default
``"gfdl-esm4"``) and ``member_id`` (default ``"r1i1p1f1"``).
Raises:
ImportError: If ``isimip-client`` is not installed.
ValueError: If the time range cannot belong to the experiment - see
:meth:`_validate_time_range`.
"""
self.cfg = cfg
self.ds = None
self.client = None
self.downloaded_files = []
self._extract_mode = None
self._extract_params = None
# Extract CMIP6-specific parameters
self.experiment_id = cfg.get('experiment_id', 'historical')
self.source_id = cfg.get('source_id', 'gfdl-esm4')
self.member_id = cfg.get('member_id', 'r1i1p1f1')
# Initialize ISIMIP client
try:
from isimip_client.client import ISIMIPClient
self.client = ISIMIPClient()
except ImportError:
raise ImportError(
"isimip-client is required for CMIP-W5E5 data access. "
"Install it with: pip install isimip-client"
)
# Validate time range for experiment
self._validate_time_range()
def _fix_coords(self, ds):
"""Put latitude in ascending order and longitude on the 0-360 convention.
Args:
ds (xr.Dataset | xr.DataArray): Object with CF-identifiable latitude
and longitude coordinates.
Returns:
xr.Dataset | xr.DataArray: Same type as the input, reoriented.
"""
ds = ds.cf.sortby("latitude")
lon_name = ds.cf["longitude"].name
ds = ds.assign_coords({lon_name: ds.cf["longitude"] % 360})
return ds.sortby(lon_name)
def _validate_time_range(self):
"""Validate that the requested time range is appropriate for the experiment.
Historical runs: 1850-2014
SSP scenarios: 2015-2100
Raises:
ValueError: If the time range doesn't match the experiment period
"""
start_date = datetime.fromisoformat(self.cfg.time_range.start_date)
end_date = datetime.fromisoformat(self.cfg.time_range.end_date)
start_year = start_date.year
end_year = end_date.year
# Define valid periods for each experiment type
if self.experiment_id == 'historical':
valid_start = 1850
valid_end = 2014
period_name = "Historical"
elif self.experiment_id.startswith('ssp'):
valid_start = 2015
valid_end = 2100
period_name = f"SSP scenario ({self.experiment_id})"
elif self.experiment_id == 'picontrol':
# Pre-industrial control - typically long runs, less strict
return
else:
# Unknown experiment, skip validation
return
# Check if requested period is outside valid range
if end_year < valid_start or start_year > valid_end:
raise ValueError(
f"❌ Time range mismatch for experiment '{self.experiment_id}'!\n"
f" Requested: {start_year}-{end_year}\n"
f" Valid period for {period_name}: {valid_start}-{valid_end}\n"
f" \n"
f" Hint: Use 'historical' for years 1850-2014, and SSP scenarios (ssp126, ssp370, ssp585) for 2015-2100."
)
# Warn if requested period extends beyond valid range
if start_year < valid_start or end_year > valid_end:
print(f"⚠️ Warning: Requested time range {start_year}-{end_year} extends beyond")
print(f" the typical {period_name} period ({valid_start}-{valid_end}).")
print(f" Data availability may be limited.")
def get_experiment_ids(self) -> List[str]:
"""Get available CMIP6 experiment IDs from ISIMIP repository.
Returns:
List[str]: List of available experiment IDs (e.g., ['historical', 'ssp126',
'ssp245', 'ssp370', 'ssp585'])
"""
print("🔍 Fetching available experiment IDs from ISIMIP...")
try:
# Query ISIMIP3b for CMIP6 scenarios
response = self.client.datasets(
simulation_round='ISIMIP3b',
product='InputData'
)
results = self._normalize_dataset_results(response)
if not results:
print("⚠️ No datasets found")
return []
# Extract unique experiment IDs from results
experiment_ids = set()
for dataset in results:
# Parse experiment ID from dataset metadata or path
climate_scenario = dataset.get('climate_scenario', '')
if climate_scenario and climate_scenario not in ['obsclim', 'counterclim']:
experiment_ids.add(climate_scenario)
# Common CMIP6 experiments in ISIMIP3b
common_experiments = ['historical', 'ssp126', 'ssp370', 'ssp585']
available = sorted([exp for exp in common_experiments if exp in experiment_ids or exp == 'historical'])
print(f"✅ Found {len(available)} experiment IDs: {available}")
return available
except Exception as e:
print(f"❌ Error fetching experiment IDs: {str(e)}")
# Return common CMIP6 experiments as fallback
return ['historical', 'ssp126', 'ssp370', 'ssp585']
def get_source_ids(self, experiment_id: Optional[str] = None) -> List[str]:
"""Get available CMIP6 model (source) IDs for a given experiment from ISIMIP repository.
Args:
experiment_id (str, optional): CMIP6 experiment ID. If None, uses
self.experiment_id
Returns:
List[str]: List of available model IDs (e.g., ['gfdl-esm4', 'ipsl-cm6a-lr',
'mpi-esm1-2-hr', 'mri-esm2-0', 'ukesm1-0-ll'])
"""
if experiment_id is None:
experiment_id = self.experiment_id
print(f"🔍 Fetching available source IDs for experiment '{experiment_id}'...")
try:
# Map experiment_id to climate_scenario for ISIMIP
climate_scenario = self._map_experiment_to_scenario(experiment_id)
# Query ISIMIP3b for available models
response = self.client.datasets(
simulation_round='ISIMIP3b',
product='InputData',
climate_scenario=climate_scenario
)
results = self._normalize_dataset_results(response)
if not results:
print(f"⚠️ No datasets found for experiment '{experiment_id}'")
return []
# Extract unique source IDs from results
source_ids = set()
for dataset in results:
# Parse source ID from dataset metadata or filename
climate_forcing = dataset.get('climate_forcing', '')
if climate_forcing:
# ISIMIP format typically: gfdl-esm4, ipsl-cm6a-lr, etc.
# Extract model name from climate forcing string
model_match = re.search(r'(gfdl-esm4|ipsl-cm6a-lr|mpi-esm1-2-hr|mri-esm2-0|ukesm1-0-ll)',
climate_forcing, re.IGNORECASE)
if model_match:
source_ids.add(model_match.group(1).lower())
# Common ISIMIP3b CMIP6 models
common_models = ['gfdl-esm4', 'ipsl-cm6a-lr', 'mpi-esm1-2-hr', 'mri-esm2-0', 'ukesm1-0-ll']
available = sorted([model for model in common_models if model in source_ids])
if not available:
available = common_models # Fallback to common models
print(f"✅ Found {len(available)} source IDs: {available}")
return available
except Exception as e:
print(f"❌ Error fetching source IDs: {str(e)}")
# Return common ISIMIP3b models as fallback
return ['gfdl-esm4', 'ipsl-cm6a-lr', 'mpi-esm1-2-hr', 'mri-esm2-0', 'ukesm1-0-ll']
def _map_experiment_to_scenario(self, experiment_id: str) -> str:
"""Map CMIP6 experiment ID to ISIMIP climate scenario name.
Args:
experiment_id (str): CMIP6 experiment ID
Returns:
str: ISIMIP climate scenario name
"""
scenario_map = {
'historical': 'historical',
'ssp126': 'ssp126',
'ssp245': 'ssp245',
'ssp370': 'ssp370',
'ssp585': 'ssp585',
'picontrol': 'picontrol',
'ssp119': 'ssp119',
'ssp434': 'ssp434',
'ssp460': 'ssp460',
}
return scenario_map.get(experiment_id, experiment_id)
def fetch(self):
"""Search ISIMIP3b and download the files covering the request.
Queries once per configured variable for the configured model and
experiment, keeps the files whose filename year-range overlaps the period
(see :meth:`_is_file_in_date_range`), and downloads any not already under
``<data_dir>/<DATASET>/<variable>/``.
Per-variable failures are reported and skipped rather than raised, so one
unavailable variable does not abort a multi-variable request - check
:attr:`downloaded_files` to see what actually arrived.
Returns:
None: Paths are appended to :attr:`downloaded_files`.
"""
print(f"🔍 Searching for CMIP6 datasets in ISIMIP repository...")
print(f" Model: {self.source_id}, Experiment: {self.experiment_id}")
start_date = datetime.fromisoformat(self.cfg.time_range.start_date)
end_date = datetime.fromisoformat(self.cfg.time_range.end_date)
# Create directory structure: cmip_w5e5/global/daily/{experiment_id}/{MODEL}/
base_dir = Path(self.cfg.data_dir) / "cmip_w5e5" / "global" / "daily" / self.experiment_id / self.source_id.upper()
base_dir.mkdir(parents=True, exist_ok=True)
# Map experiment_id to ISIMIP climate scenario
climate_scenario = self._map_experiment_to_scenario(self.experiment_id)
# Search for each variable separately
for var in self.cfg.variables:
print(f"\n📥 Fetching {var}...")
# Map variable names to CMIP6/W5E5 names if needed
cmip_var = self._map_variable_name(var)
# Search ISIMIP repository for CMIP6 data
# CMIP6 is available in ISIMIP3b as secondary input data
try:
response = self.client.datasets(
simulation_round='ISIMIP3b',
product='InputData',
climate_forcing=self.source_id.lower(), # CMIP6 model
climate_scenario=climate_scenario,
climate_variable=cmip_var
)
results = self._normalize_dataset_results(response)
if not results:
print(f"⚠️ No CMIP6 datasets found for {var} with model {self.source_id}")
continue
# Get the first matching dataset
dataset = results[0]
print(f"✅ Found dataset: {dataset.get('name', 'unnamed')}")
# Filter files by date range
for file_info in self._extract_files_list(dataset):
file_name = file_info['name']
# Parse date from filename
# Example: gfdl-esm4_r1i1p1f1_ssp585_tas_global_daily_2015_2024.nc
if self._is_file_in_date_range(file_name, start_date, end_date):
# Add variable to path: cmip_w5e5/global/daily/{experiment_id}/{MODEL}/{variable}/
var_dir = base_dir / var
var_dir.mkdir(parents=True, exist_ok=True)
local_path = var_dir / file_name
if local_path.exists():
print(f" ✓ Already exists: {file_name}")
self.downloaded_files.append(str(local_path))
else:
print(f" ⬇️ Downloading: {file_name}")
# Download directly using the file URL
self.client.download(
file_info['file_url'],
path=str(local_path.parent),
validate=False
)
self.downloaded_files.append(str(local_path))
except Exception as e:
print(f"❌ Error fetching {var}: {str(e)}")
continue
print(f"\n✅ Downloaded {len(self.downloaded_files)} files")
def _normalize_dataset_results(self, response) -> List[Dict]:
"""Coerce an isimip-client response into a list of dataset records.
The client returns a bare list from some endpoints and a paginated
``{"results": [...]}`` envelope from others.
Args:
response: The value returned by ``client.datasets(...)``.
Returns:
list[dict]: Dataset records, empty if the response has neither shape.
"""
if response is None:
return []
if isinstance(response, list):
return response
if isinstance(response, dict):
results = response.get('results')
if isinstance(results, list):
return results
return []
def _extract_files_list(self, dataset: Dict) -> List[Dict]:
"""Pull the file records out of one ISIMIP dataset record.
Args:
dataset (dict): A single dataset record.
Returns:
list[dict]: File records, each with ``path``, ``name`` and
``file_url``. Empty if the record carries neither ``files`` nor a
single ``file``.
"""
files = dataset.get('files')
if isinstance(files, list):
return files
single_file = dataset.get('file')
if isinstance(single_file, dict):
return [single_file]
return []
def load(self):
"""Open the downloaded files and merge them into one dataset.
Files are grouped by variable, concatenated along time where a variable
spans several decades, merged across variables, and clipped to the
configured time range.
Returns:
None: The dataset is stored on :attr:`ds`.
Raises:
ValueError: If :meth:`fetch` has not run, or produced no files.
"""
if not self.downloaded_files:
raise ValueError("No files to load. Run fetch() first.")
print(f"📂 Loading {len(self.downloaded_files)} CMIP6 files...")
# Group files by variable
files_by_var = {}
for fpath in self.downloaded_files:
# Determine which variable this file contains
for var in self.cfg.variables:
cmip_var = self._map_variable_name(var)
if f"/{var}/" in fpath or f"_{cmip_var}_" in fpath:
if var not in files_by_var:
files_by_var[var] = []
files_by_var[var].append(fpath)
break
# Load each variable separately and merge
datasets = []
for var, file_list in files_by_var.items():
print(f" Loading {var} from {len(file_list)} file(s)...")
if len(file_list) == 1:
ds_var = xr.open_dataset(file_list[0])
else:
# Multiple files - open as multi-file dataset
ds_var = xr.open_mfdataset(
file_list,
combine='by_coords',
parallel=True
)
datasets.append(ds_var)
# Merge all variables into one dataset
if len(datasets) == 1:
self.ds = self._fix_coords(datasets[0])
else:
self.ds = self._fix_coords(xr.merge(datasets))
# Subset to requested time range
start = self.cfg.time_range.start_date
end = self.cfg.time_range.end_date
self.ds = self.ds.sel(time=slice(start, end))
# Add metadata
self.ds.attrs['source'] = f'CMIP6 {self.source_id} via ISIMIP'
self.ds.attrs['dataset'] = f'CMIP6-W5E5'
self.ds.attrs['experiment_id'] = self.experiment_id
self.ds.attrs['source_id'] = self.source_id
self.ds.attrs['member_id'] = self.member_id
self.ds.attrs['description'] = f'CMIP6 {self.experiment_id} scenario from {self.source_id} in W5E5 format'
print(f"✅ Loaded dataset with {len(self.ds.data_vars)} variables")
def extract(self, *, point: Optional[Tuple[float, float]] = None,
box: Optional[Dict] = None,
shapefile: Optional[str] = None,
buffer_km: float = 0.0):
"""Store extraction instructions to be applied during or after load.
Args:
point (tuple of (lon, lat), optional): Extract data for a specific point
location
box (dict, optional): Extract data for a bounding box with keys: lon_min,
lon_max, lat_min, lat_max
shapefile (str or GeoDataFrame, optional): Extract data for a shapefile
region
buffer_km (float, default=0.0): Buffer distance in kilometers around point
(converted to degrees)
"""
if point is not None:
lon, lat = point
buffer_deg = buffer_km / 111.0
self._extract_mode = "point"
self._extract_params = (lon, lat, buffer_deg)
# Apply extraction if dataset is already loaded
if self.ds is not None:
self._apply_extraction()
elif box is not None:
self._extract_mode = "box"
self._extract_params = box
if self.ds is not None:
self._apply_extraction()
elif shapefile is not None:
import geopandas as gpd
if isinstance(shapefile, str):
gdf = gpd.read_file(shapefile)
else:
gdf = shapefile
self._extract_mode = "shapefile"
self._extract_params = gdf
if self.ds is not None:
self._apply_extraction()
def _apply_extraction(self):
"""Run the subset recorded by :meth:`extract` against :attr:`ds`.
Point requests either take the nearest cell or area-average a buffered
box; box requests slice; shapefile requests clip each geometry through
rioxarray and stack the results along a new ``geom_id`` dimension.
Returns:
None: :attr:`ds` is replaced in place.
"""
if self._extract_mode == "point":
lon, lat, buffer_deg = self._extract_params
if buffer_deg > 0:
self.ds = self.ds.sel(
lon=slice(lon - buffer_deg, lon + buffer_deg),
lat=slice(lat - buffer_deg, lat + buffer_deg)
).mean(["lat", "lon"])
else:
self.ds = self.ds.sel(lon=lon, lat=lat, method="nearest")
elif self._extract_mode == "box":
box = self._extract_params
self.ds = self.ds.sel(
lon=slice(box["lon_min"], box["lon_max"]),
lat=slice(box["lat_min"], box["lat_max"])
)
elif self._extract_mode == "shapefile":
import rioxarray # noqa: F401 — registers the .rio accessor
from shapely.geometry import mapping
gdf = self._extract_params
self.ds = self.ds.rio.set_spatial_dims(x_dim="lon", y_dim="lat")
self.ds = self.ds.rio.write_crs("EPSG:4326", inplace=True)
clipped_list = []
for geom in gdf.geometry:
clipped = self.ds.rio.clip([mapping(geom)], gdf.crs, drop=True)
clipped_list.append(clipped)
self.ds = xr.concat(clipped_list, dim="geom_id")
def save_netcdf(self, filename: str):
"""Write the dataset to NetCDF, creating parent directories.
Args:
filename (str): Destination path.
Returns:
None
Raises:
ValueError: If no dataset has been loaded.
"""
if self.ds is None:
raise ValueError("No dataset loaded. Run load() first.")
output_path = Path(filename)
output_path.parent.mkdir(parents=True, exist_ok=True)
self.ds.to_netcdf(output_path)
print(f"💾 Saved to: {output_path}")
def save_csv(self, filename: str):
"""Write the dataset to CSV, creating parent directories.
The frame is written in xarray's wide form - one column per variable,
indexed by the dataset's dimensions - not the long form produced by
:meth:`~climdata.utils.wrapper_workflow.ClimateExtractor.to_dataframe`.
Args:
filename (str): Destination path.
Returns:
None
Raises:
ValueError: If no dataset has been loaded.
"""
if self.ds is None:
raise ValueError("No dataset loaded. Run load() first.")
output_path = Path(filename)
output_path.parent.mkdir(parents=True, exist_ok=True)
df = self.ds.to_dataframe()
df.to_csv(output_path)
print(f"💾 Saved to: {output_path}")
def _map_variable_name(self, var: str) -> str:
"""Translate a CF variable name to the name ISIMIP3b files use.
ISIMIP follows CMIP naming, so nearly all names pass through unchanged;
``sfcWind`` is the exception, stored lowercase as ``sfcwind``.
Args:
var (str): CF variable name, e.g. ``"tasmax"``.
Returns:
str: The ISIMIP name, or ``var`` unchanged if it is not in the table.
"""
# CMIP6 uses standard names
variable_map = {
'tas': 'tas', # Near-surface air temperature
'tasmax': 'tasmax', # Maximum temperature
'tasmin': 'tasmin', # Minimum temperature
'pr': 'pr', # Precipitation
'rsds': 'rsds', # Surface downwelling shortwave radiation
'hurs': 'hurs', # Near-surface relative humidity
'sfcWind': 'sfcwind', # Near-surface wind speed (note: lowercase 'w')
'ps': 'ps', # Surface air pressure
'rlds': 'rlds', # Surface downwelling longwave radiation
'psl': 'psl', # Sea level pressure
}
return variable_map.get(var, var)
def _is_file_in_date_range(self, filename: str, start_date: datetime, end_date: datetime) -> bool:
"""Test whether a file's year range overlaps the requested period.
ISIMIP encodes coverage in the filename, e.g.
``gfdl-esm4_r1i1p1f1_ssp585_tas_global_daily_2015_2024.nc``. Comparison
is at year granularity, so a file may be kept for a request that only
touches part of it.
Args:
filename (str): Basename of the candidate file.
start_date (datetime): Start of the requested period.
end_date (datetime): End of the requested period.
Returns:
bool: ``True`` if the ranges overlap. Also ``True`` when the filename
carries no parseable year range - an unrecognised name is downloaded
rather than silently dropped.
"""
import re
# Extract year range from filename
match = re.search(r'_(\d{4})_(\d{4})\.nc', filename)
if match:
file_start_year = int(match.group(1))
file_end_year = int(match.group(2))
# Check if there's any overlap
return not (file_end_year < start_date.year or file_start_year > end_date.year)
# If we can't parse the date, include the file to be safe
return True
__init__(self, cfg)
special
¶
Bind a configuration, open an ISIMIP client and validate the period.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
cfg |
DictConfig |
Configuration with |
required |
Exceptions:
| Type | Description |
|---|---|
ImportError |
If |
ValueError |
If the time range cannot belong to the experiment - see
:meth: |
Source code in climdata/datasets/CMIP_W5E5.py
def __init__(self, cfg: DictConfig):
"""Bind a configuration, open an ISIMIP client and validate the period.
Args:
cfg (DictConfig): Configuration with ``variables``, ``time_range``,
``data_dir``, ``dataset``, and optionally ``experiment_id``
(default ``"historical"``), ``source_id`` (default
``"gfdl-esm4"``) and ``member_id`` (default ``"r1i1p1f1"``).
Raises:
ImportError: If ``isimip-client`` is not installed.
ValueError: If the time range cannot belong to the experiment - see
:meth:`_validate_time_range`.
"""
self.cfg = cfg
self.ds = None
self.client = None
self.downloaded_files = []
self._extract_mode = None
self._extract_params = None
# Extract CMIP6-specific parameters
self.experiment_id = cfg.get('experiment_id', 'historical')
self.source_id = cfg.get('source_id', 'gfdl-esm4')
self.member_id = cfg.get('member_id', 'r1i1p1f1')
# Initialize ISIMIP client
try:
from isimip_client.client import ISIMIPClient
self.client = ISIMIPClient()
except ImportError:
raise ImportError(
"isimip-client is required for CMIP-W5E5 data access. "
"Install it with: pip install isimip-client"
)
# Validate time range for experiment
self._validate_time_range()
extract(self, *, point=None, box=None, shapefile=None, buffer_km=0.0)
¶
Store extraction instructions to be applied during or after load.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
point |
tuple of (lon, lat) |
Extract data for a specific point location |
None |
box |
dict |
Extract data for a bounding box with keys: lon_min, lon_max, lat_min, lat_max |
None |
shapefile |
str or GeoDataFrame |
Extract data for a shapefile region |
None |
buffer_km |
float, default=0.0 |
Buffer distance in kilometers around point (converted to degrees) |
0.0 |
Source code in climdata/datasets/CMIP_W5E5.py
def extract(self, *, point: Optional[Tuple[float, float]] = None,
box: Optional[Dict] = None,
shapefile: Optional[str] = None,
buffer_km: float = 0.0):
"""Store extraction instructions to be applied during or after load.
Args:
point (tuple of (lon, lat), optional): Extract data for a specific point
location
box (dict, optional): Extract data for a bounding box with keys: lon_min,
lon_max, lat_min, lat_max
shapefile (str or GeoDataFrame, optional): Extract data for a shapefile
region
buffer_km (float, default=0.0): Buffer distance in kilometers around point
(converted to degrees)
"""
if point is not None:
lon, lat = point
buffer_deg = buffer_km / 111.0
self._extract_mode = "point"
self._extract_params = (lon, lat, buffer_deg)
# Apply extraction if dataset is already loaded
if self.ds is not None:
self._apply_extraction()
elif box is not None:
self._extract_mode = "box"
self._extract_params = box
if self.ds is not None:
self._apply_extraction()
elif shapefile is not None:
import geopandas as gpd
if isinstance(shapefile, str):
gdf = gpd.read_file(shapefile)
else:
gdf = shapefile
self._extract_mode = "shapefile"
self._extract_params = gdf
if self.ds is not None:
self._apply_extraction()
fetch(self)
¶
Search ISIMIP3b and download the files covering the request.
Queries once per configured variable for the configured model and
experiment, keeps the files whose filename year-range overlaps the period
(see :meth:_is_file_in_date_range), and downloads any not already under
<data_dir>/<DATASET>/<variable>/.
Per-variable failures are reported and skipped rather than raised, so one
unavailable variable does not abort a multi-variable request - check
:attr:downloaded_files to see what actually arrived.
Returns:
| Type | Description |
|---|---|
None |
Paths are appended to :attr: |
Source code in climdata/datasets/CMIP_W5E5.py
def fetch(self):
"""Search ISIMIP3b and download the files covering the request.
Queries once per configured variable for the configured model and
experiment, keeps the files whose filename year-range overlaps the period
(see :meth:`_is_file_in_date_range`), and downloads any not already under
``<data_dir>/<DATASET>/<variable>/``.
Per-variable failures are reported and skipped rather than raised, so one
unavailable variable does not abort a multi-variable request - check
:attr:`downloaded_files` to see what actually arrived.
Returns:
None: Paths are appended to :attr:`downloaded_files`.
"""
print(f"🔍 Searching for CMIP6 datasets in ISIMIP repository...")
print(f" Model: {self.source_id}, Experiment: {self.experiment_id}")
start_date = datetime.fromisoformat(self.cfg.time_range.start_date)
end_date = datetime.fromisoformat(self.cfg.time_range.end_date)
# Create directory structure: cmip_w5e5/global/daily/{experiment_id}/{MODEL}/
base_dir = Path(self.cfg.data_dir) / "cmip_w5e5" / "global" / "daily" / self.experiment_id / self.source_id.upper()
base_dir.mkdir(parents=True, exist_ok=True)
# Map experiment_id to ISIMIP climate scenario
climate_scenario = self._map_experiment_to_scenario(self.experiment_id)
# Search for each variable separately
for var in self.cfg.variables:
print(f"\n📥 Fetching {var}...")
# Map variable names to CMIP6/W5E5 names if needed
cmip_var = self._map_variable_name(var)
# Search ISIMIP repository for CMIP6 data
# CMIP6 is available in ISIMIP3b as secondary input data
try:
response = self.client.datasets(
simulation_round='ISIMIP3b',
product='InputData',
climate_forcing=self.source_id.lower(), # CMIP6 model
climate_scenario=climate_scenario,
climate_variable=cmip_var
)
results = self._normalize_dataset_results(response)
if not results:
print(f"⚠️ No CMIP6 datasets found for {var} with model {self.source_id}")
continue
# Get the first matching dataset
dataset = results[0]
print(f"✅ Found dataset: {dataset.get('name', 'unnamed')}")
# Filter files by date range
for file_info in self._extract_files_list(dataset):
file_name = file_info['name']
# Parse date from filename
# Example: gfdl-esm4_r1i1p1f1_ssp585_tas_global_daily_2015_2024.nc
if self._is_file_in_date_range(file_name, start_date, end_date):
# Add variable to path: cmip_w5e5/global/daily/{experiment_id}/{MODEL}/{variable}/
var_dir = base_dir / var
var_dir.mkdir(parents=True, exist_ok=True)
local_path = var_dir / file_name
if local_path.exists():
print(f" ✓ Already exists: {file_name}")
self.downloaded_files.append(str(local_path))
else:
print(f" ⬇️ Downloading: {file_name}")
# Download directly using the file URL
self.client.download(
file_info['file_url'],
path=str(local_path.parent),
validate=False
)
self.downloaded_files.append(str(local_path))
except Exception as e:
print(f"❌ Error fetching {var}: {str(e)}")
continue
print(f"\n✅ Downloaded {len(self.downloaded_files)} files")
get_experiment_ids(self)
¶
Get available CMIP6 experiment IDs from ISIMIP repository.
Returns:
| Type | Description |
|---|---|
List[str] |
List of available experiment IDs (e.g., ['historical', 'ssp126', 'ssp245', 'ssp370', 'ssp585']) |
Source code in climdata/datasets/CMIP_W5E5.py
def get_experiment_ids(self) -> List[str]:
"""Get available CMIP6 experiment IDs from ISIMIP repository.
Returns:
List[str]: List of available experiment IDs (e.g., ['historical', 'ssp126',
'ssp245', 'ssp370', 'ssp585'])
"""
print("🔍 Fetching available experiment IDs from ISIMIP...")
try:
# Query ISIMIP3b for CMIP6 scenarios
response = self.client.datasets(
simulation_round='ISIMIP3b',
product='InputData'
)
results = self._normalize_dataset_results(response)
if not results:
print("⚠️ No datasets found")
return []
# Extract unique experiment IDs from results
experiment_ids = set()
for dataset in results:
# Parse experiment ID from dataset metadata or path
climate_scenario = dataset.get('climate_scenario', '')
if climate_scenario and climate_scenario not in ['obsclim', 'counterclim']:
experiment_ids.add(climate_scenario)
# Common CMIP6 experiments in ISIMIP3b
common_experiments = ['historical', 'ssp126', 'ssp370', 'ssp585']
available = sorted([exp for exp in common_experiments if exp in experiment_ids or exp == 'historical'])
print(f"✅ Found {len(available)} experiment IDs: {available}")
return available
except Exception as e:
print(f"❌ Error fetching experiment IDs: {str(e)}")
# Return common CMIP6 experiments as fallback
return ['historical', 'ssp126', 'ssp370', 'ssp585']
get_source_ids(self, experiment_id=None)
¶
Get available CMIP6 model (source) IDs for a given experiment from ISIMIP repository.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
experiment_id |
str |
CMIP6 experiment ID. If None, uses self.experiment_id |
None |
Returns:
| Type | Description |
|---|---|
List[str] |
List of available model IDs (e.g., ['gfdl-esm4', 'ipsl-cm6a-lr', 'mpi-esm1-2-hr', 'mri-esm2-0', 'ukesm1-0-ll']) |
Source code in climdata/datasets/CMIP_W5E5.py
def get_source_ids(self, experiment_id: Optional[str] = None) -> List[str]:
"""Get available CMIP6 model (source) IDs for a given experiment from ISIMIP repository.
Args:
experiment_id (str, optional): CMIP6 experiment ID. If None, uses
self.experiment_id
Returns:
List[str]: List of available model IDs (e.g., ['gfdl-esm4', 'ipsl-cm6a-lr',
'mpi-esm1-2-hr', 'mri-esm2-0', 'ukesm1-0-ll'])
"""
if experiment_id is None:
experiment_id = self.experiment_id
print(f"🔍 Fetching available source IDs for experiment '{experiment_id}'...")
try:
# Map experiment_id to climate_scenario for ISIMIP
climate_scenario = self._map_experiment_to_scenario(experiment_id)
# Query ISIMIP3b for available models
response = self.client.datasets(
simulation_round='ISIMIP3b',
product='InputData',
climate_scenario=climate_scenario
)
results = self._normalize_dataset_results(response)
if not results:
print(f"⚠️ No datasets found for experiment '{experiment_id}'")
return []
# Extract unique source IDs from results
source_ids = set()
for dataset in results:
# Parse source ID from dataset metadata or filename
climate_forcing = dataset.get('climate_forcing', '')
if climate_forcing:
# ISIMIP format typically: gfdl-esm4, ipsl-cm6a-lr, etc.
# Extract model name from climate forcing string
model_match = re.search(r'(gfdl-esm4|ipsl-cm6a-lr|mpi-esm1-2-hr|mri-esm2-0|ukesm1-0-ll)',
climate_forcing, re.IGNORECASE)
if model_match:
source_ids.add(model_match.group(1).lower())
# Common ISIMIP3b CMIP6 models
common_models = ['gfdl-esm4', 'ipsl-cm6a-lr', 'mpi-esm1-2-hr', 'mri-esm2-0', 'ukesm1-0-ll']
available = sorted([model for model in common_models if model in source_ids])
if not available:
available = common_models # Fallback to common models
print(f"✅ Found {len(available)} source IDs: {available}")
return available
except Exception as e:
print(f"❌ Error fetching source IDs: {str(e)}")
# Return common ISIMIP3b models as fallback
return ['gfdl-esm4', 'ipsl-cm6a-lr', 'mpi-esm1-2-hr', 'mri-esm2-0', 'ukesm1-0-ll']
load(self)
¶
Open the downloaded files and merge them into one dataset.
Files are grouped by variable, concatenated along time where a variable spans several decades, merged across variables, and clipped to the configured time range.
Returns:
| Type | Description |
|---|---|
None |
The dataset is stored on :attr: |
Exceptions:
| Type | Description |
|---|---|
ValueError |
If :meth: |
Source code in climdata/datasets/CMIP_W5E5.py
def load(self):
"""Open the downloaded files and merge them into one dataset.
Files are grouped by variable, concatenated along time where a variable
spans several decades, merged across variables, and clipped to the
configured time range.
Returns:
None: The dataset is stored on :attr:`ds`.
Raises:
ValueError: If :meth:`fetch` has not run, or produced no files.
"""
if not self.downloaded_files:
raise ValueError("No files to load. Run fetch() first.")
print(f"📂 Loading {len(self.downloaded_files)} CMIP6 files...")
# Group files by variable
files_by_var = {}
for fpath in self.downloaded_files:
# Determine which variable this file contains
for var in self.cfg.variables:
cmip_var = self._map_variable_name(var)
if f"/{var}/" in fpath or f"_{cmip_var}_" in fpath:
if var not in files_by_var:
files_by_var[var] = []
files_by_var[var].append(fpath)
break
# Load each variable separately and merge
datasets = []
for var, file_list in files_by_var.items():
print(f" Loading {var} from {len(file_list)} file(s)...")
if len(file_list) == 1:
ds_var = xr.open_dataset(file_list[0])
else:
# Multiple files - open as multi-file dataset
ds_var = xr.open_mfdataset(
file_list,
combine='by_coords',
parallel=True
)
datasets.append(ds_var)
# Merge all variables into one dataset
if len(datasets) == 1:
self.ds = self._fix_coords(datasets[0])
else:
self.ds = self._fix_coords(xr.merge(datasets))
# Subset to requested time range
start = self.cfg.time_range.start_date
end = self.cfg.time_range.end_date
self.ds = self.ds.sel(time=slice(start, end))
# Add metadata
self.ds.attrs['source'] = f'CMIP6 {self.source_id} via ISIMIP'
self.ds.attrs['dataset'] = f'CMIP6-W5E5'
self.ds.attrs['experiment_id'] = self.experiment_id
self.ds.attrs['source_id'] = self.source_id
self.ds.attrs['member_id'] = self.member_id
self.ds.attrs['description'] = f'CMIP6 {self.experiment_id} scenario from {self.source_id} in W5E5 format'
print(f"✅ Loaded dataset with {len(self.ds.data_vars)} variables")
save_csv(self, filename)
¶
Write the dataset to CSV, creating parent directories.
The frame is written in xarray's wide form - one column per variable,
indexed by the dataset's dimensions - not the long form produced by
:meth:~climdata.utils.wrapper_workflow.ClimateExtractor.to_dataframe.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
filename |
str |
Destination path. |
required |
Returns:
| Type | Description |
|---|---|
None |
Exceptions:
| Type | Description |
|---|---|
ValueError |
If no dataset has been loaded. |
Source code in climdata/datasets/CMIP_W5E5.py
def save_csv(self, filename: str):
"""Write the dataset to CSV, creating parent directories.
The frame is written in xarray's wide form - one column per variable,
indexed by the dataset's dimensions - not the long form produced by
:meth:`~climdata.utils.wrapper_workflow.ClimateExtractor.to_dataframe`.
Args:
filename (str): Destination path.
Returns:
None
Raises:
ValueError: If no dataset has been loaded.
"""
if self.ds is None:
raise ValueError("No dataset loaded. Run load() first.")
output_path = Path(filename)
output_path.parent.mkdir(parents=True, exist_ok=True)
df = self.ds.to_dataframe()
df.to_csv(output_path)
print(f"💾 Saved to: {output_path}")
save_netcdf(self, filename)
¶
Write the dataset to NetCDF, creating parent directories.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
filename |
str |
Destination path. |
required |
Returns:
| Type | Description |
|---|---|
None |
Exceptions:
| Type | Description |
|---|---|
ValueError |
If no dataset has been loaded. |
Source code in climdata/datasets/CMIP_W5E5.py
def save_netcdf(self, filename: str):
"""Write the dataset to NetCDF, creating parent directories.
Args:
filename (str): Destination path.
Returns:
None
Raises:
ValueError: If no dataset has been loaded.
"""
if self.ds is None:
raise ValueError("No dataset loaded. Run load() first.")
output_path = Path(filename)
output_path.parent.mkdir(parents=True, exist_ok=True)
self.ds.to_netcdf(output_path)
print(f"💾 Saved to: {output_path}")
climdata.datasets.AGRI_ISIMIP.AGRI_ISIMIP
¶
Discover, download and subset ISIMIP agricultural model output.
The lifecycle has an extra first step compared to the climate providers:
:meth:query_available_metadata (or :meth:explore, which loops it over
every configured combination) must run before the AVAILABLE_* properties
return anything, because what a given impact model published for a given
forcing and period can only be learned from the API.
After that the usual fetch → load → extract applies. Requests are
described by cfg.query_configs, a list of impact_model /
climate_forcing / period combinations.
Attributes:
| Name | Type | Description |
|---|---|---|
cfg |
DictConfig |
Configuration with |
client |
ISIMIPClient |
ISIMIP API client, created in :meth: |
ds |
xr.Dataset | None |
The loaded dataset, set by :meth: |
downloaded_files |
list[str] |
Local paths accumulated by :meth: |
Examples:
>>> agri = AGRI_ISIMIP(cfg)
>>> agri.query_available_metadata("LPJmL", "gfdl-esm4", "future")
>>> sorted(agri.AVAILABLE_CROPS)[:3]
['mai', 'ri1', 'soy']
Source code in climdata/datasets/AGRI_ISIMIP.py
class AGRI_ISIMIP:
"""Discover, download and subset ISIMIP agricultural model output.
The lifecycle has an extra first step compared to the climate providers:
:meth:`query_available_metadata` (or :meth:`explore`, which loops it over
every configured combination) must run before the ``AVAILABLE_*`` properties
return anything, because what a given impact model published for a given
forcing and period can only be learned from the API.
After that the usual ``fetch`` → ``load`` → ``extract`` applies. Requests are
described by ``cfg.query_configs``, a list of ``impact_model`` /
``climate_forcing`` / ``period`` combinations.
Attributes:
cfg (DictConfig): Configuration with ``query_configs`` and ``data_dir``.
client (ISIMIPClient): ISIMIP API client, created in :meth:`__init__`.
ds (xr.Dataset | None): The loaded dataset, set by :meth:`load`.
downloaded_files (list[str]): Local paths accumulated by :meth:`fetch`.
Example:
>>> agri = AGRI_ISIMIP(cfg) # doctest: +SKIP
>>> agri.query_available_metadata("LPJmL", "gfdl-esm4", "future")
>>> sorted(agri.AVAILABLE_CROPS)[:3] # doctest: +SKIP
['mai', 'ri1', 'soy']
"""
def __init__(self, cfg: DictConfig):
"""Bind a configuration and open an ISIMIP client.
No metadata is queried here; the ``AVAILABLE_*`` caches start empty.
Args:
cfg (DictConfig): Configuration with ``query_configs``, ``data_dir``
and ``variables``.
Raises:
ImportError: If ``isimip-client`` is not installed.
"""
self.cfg = cfg
self.ds = None
self.downloaded_files = []
self._extract_mode = None
self._extract_params = None
# Initialize ISIMIP client
try:
from isimip_client.client import ISIMIPClient
self.client = ISIMIPClient()
except ImportError:
raise ImportError(
"isimip-client is required for AGRI-ISIMIP data access. "
"Install it with: pip install isimip-client"
)
# Cached metadata from API queries
self._available_crops: Optional[Set[str]] = None
self._available_variables: Optional[Set[str]] = None
self._available_irrigations: Optional[Set[str]] = None
self._available_scenarios: Optional[Set[str]] = None
@property
def AVAILABLE_CROPS(self) -> Set[str]:
"""Crops the API reported for the queried configuration.
Returns:
set[str]: Crop codes, e.g. ``{"mai", "ri1", "soy", "swh", "wwh"}``.
Raises:
ValueError: If :meth:`query_available_metadata` has not run.
"""
if self._available_crops is None:
raise ValueError("No crops loaded. Call query_available_metadata() first.")
return self._available_crops
@property
def AVAILABLE_VARIABLES(self) -> Set[str]:
"""Variables the API reported for the queried configuration.
Returns:
set[str]: Variable names, e.g. ``{"yield", "biom", "plantyear"}``.
Raises:
ValueError: If :meth:`query_available_metadata` has not run.
"""
if self._available_variables is None:
raise ValueError("No variables loaded. Call query_available_metadata() first.")
return self._available_variables
@property
def AVAILABLE_IRRIGATIONS(self) -> Set[str]:
"""Irrigation regimes the API reported for the queried configuration.
Returns:
set[str]: Regime codes — typically ``{"noirr", "firr"}`` for rainfed
and fully irrigated.
Raises:
ValueError: If :meth:`query_available_metadata` has not run.
"""
if self._available_irrigations is None:
raise ValueError("No irrigations loaded. Call query_available_metadata() first.")
return self._available_irrigations
@property
def AVAILABLE_SCENARIOS(self) -> Set[str]:
"""Climate scenarios the API reported for the queried configuration.
Returns:
set[str]: Scenario names, e.g. ``{"historical", "ssp126", "ssp585"}``.
Raises:
ValueError: If :meth:`query_available_metadata` has not run.
"""
if self._available_scenarios is None:
raise ValueError("No scenarios loaded. Call query_available_metadata() first.")
return self._available_scenarios
def query_available_metadata(self, impact_model: str, climate_forcing: str, period: str) -> None:
"""Query ISIMIP API to get all available metadata for a specific configuration.
Extracts crops, variables, irrigation types, and scenarios.
Args:
impact_model (str): Impact model name (e.g., 'SIMPLACE-LINTUL5')
climate_forcing (str): Climate forcing/model (e.g., 'mpi-esm1-2-hr')
period (str): Time period (e.g., 'historical')
Raises:
Exception: If API query fails or no datasets found
"""
path = f'ISIMIP3b/OutputData/agriculture/{impact_model}/{climate_forcing}/{period}'
crops: Set[str] = set()
variables: Set[str] = set()
irrigation: Set[str] = set()
scenarios: Set[str] = set()
page = 1
total_fetched = 0
while True:
response = self.client.datasets(
path=path,
page=page,
page_size=100
)
results = response.get('results', [])
if not results:
break
total_fetched += len(results)
for ds in results:
spec = ds.get('specifiers', {})
crop = spec.get('crop')
var = spec.get('variable')
irr = spec.get('irrigation')
scen = spec.get('climate_scenario')
if crop:
crops.add(crop)
if var:
variables.add(var)
if irr:
irrigation.add(irr)
if scen:
scenarios.add(scen)
# Check pagination
if not response.get('next'):
break
page += 1
self._available_crops = crops
self._available_variables = variables
self._available_irrigations = irrigation
self._available_scenarios = scenarios
print(f"✅ Fetched {total_fetched} datasets from ISIMIP API")
print(f" Crops: {sorted(crops)}")
print(f" Variables: {sorted(variables)}")
print(f" Irrigation: {sorted(irrigation)}")
print(f" Scenarios: {sorted(scenarios)}")
def explore(self):
"""Print what ISIMIP holds for every configured query combination.
Runs :meth:`query_available_metadata` over each entry of
``cfg.query_configs`` and prints the crops, variables, irrigation
regimes and scenarios found. The intended first call on a new
configuration, before committing to a download.
Note that the ``AVAILABLE_*`` caches are overwritten by each combination
in turn, so after this they reflect the last one only.
Returns:
None: The report is written to stdout.
"""
print("\n" + "=" * 70)
print("🌾 AGRI-ISIMIP Dataset Explorer")
print("=" * 70)
query_configs = self.cfg.get('query_configs', [])
if not query_configs:
print("\n⚠️ No query_configs found in configuration")
return
for i, config in enumerate(query_configs, 1):
print(f"\n[{i}/{len(query_configs)}] Configuration:")
print(f" Impact Model: {config['impact_model']}")
print(f" Climate Forcing: {config['climate_forcing']}")
print(f" Period: {config['period']}")
print("-" * 70)
try:
self.query_available_metadata(
impact_model=config['impact_model'],
climate_forcing=config['climate_forcing'],
period=config['period']
)
except Exception as e:
print(f"❌ Error: {str(e)}")
def list_files(self, impact_model: str, climate_forcing: str, period: str,
crop: Optional[str] = None, irrigation: Optional[str] = None,
variable: Optional[str] = None) -> List[Dict]:
"""List available files matching the given criteria.
Args:
impact_model (str): Impact model name (e.g., 'SIMPLACE-LINTUL5')
climate_forcing (str): Climate forcing (e.g., 'mpi-esm1-2-hr')
period (str): Time period (e.g., 'historical')
crop (str, optional): Filter by crop name
irrigation (str, optional): Filter by irrigation type ('firr' or 'noirr')
variable (str, optional): Filter by variable name
Returns:
List[Dict]: List of file metadata dictionaries
"""
path = f'ISIMIP3b/OutputData/agriculture/{impact_model}/{climate_forcing}/{period}'
print(f"\n🔍 Listing AGRI-ISIMIP files:")
print(f" Path: {path}")
if crop:
print(f" Crop: {crop}")
if irrigation:
print(f" Irrigation: {irrigation}")
if variable:
print(f" Variable: {variable}")
files_list = []
page = 1
total_fetched = 0
try:
while True:
response = self.client.datasets(
path=path,
page=page,
page_size=100
)
results = response.get('results', [])
if not results:
break
total_fetched += len(results)
for dataset in results:
spec = dataset.get('specifiers', {})
file_crop = spec.get('crop')
file_irr = spec.get('irrigation')
file_var = spec.get('variable')
# Apply filters
if crop and file_crop != crop:
continue
if irrigation and file_irr != irrigation:
continue
if variable and file_var != variable:
continue
files_list.append({
'name': dataset.get('name', ''),
'path': dataset.get('path', ''),
'size': dataset.get('size', 'N/A'),
'crop': file_crop,
'irrigation': file_irr,
'variable': file_var,
'dataset': dataset,
})
# Check pagination
if not response.get('next'):
break
page += 1
print(f"✅ Found {len(files_list)} matching files (from {total_fetched} total)\n")
# Display first 20 results
if files_list:
print("Files found:")
print("-" * 100)
for i, f in enumerate(files_list[:20], 1):
print(f"{i:3d}. {f['name']}")
print(f" Path: {f['path']}")
print(f" Crop: {f['crop']}, Irrigation: {f['irrigation']}, Variable: {f['variable']}")
if f['size'] != 'N/A':
size_gb = f['size'] / (1024**3) if isinstance(f['size'], (int, float)) else 'N/A'
if size_gb != 'N/A':
print(f" Size: {size_gb:.2f} GB")
if len(files_list) > 20:
print(f"\n ... and {len(files_list) - 20} more files")
return files_list
except Exception as e:
print(f"❌ Error listing files: {str(e)}")
return []
def fetch(self, impact_model: str, climate_forcing: str, period: str,
crop: Optional[str] = None, irrigation: Optional[str] = None,
variable: Optional[str] = None):
"""Download AGRI-ISIMIP files matching the criteria using isimip_client.download().
Files are stored as::
<data_dir>/AGRI_ISIMIP/<impact_model>/<climate_forcing>/<period>/<crop>/<variable>/<irrigation>/
Args:
impact_model (str): Impact model name
climate_forcing (str): Climate forcing
period (str): Time period
crop (str, optional): Specific crop to download
irrigation (str, optional): Specific irrigation type to download
variable (str, optional): Specific variable to download
"""
files_list = self.list_files(
impact_model=impact_model,
climate_forcing=climate_forcing,
period=period,
crop=crop,
irrigation=irrigation,
variable=variable
)
if not files_list:
print("⚠️ No files found to download")
return
# Create directory structure
base_dir = (
Path(self.cfg.data_dir) /
"AGRI_ISIMIP" /
impact_model /
climate_forcing /
period
)
base_dir.mkdir(parents=True, exist_ok=True)
print(f"\n📥 Downloading {len(files_list)} files to {base_dir}...\n")
for i, file_info in enumerate(files_list, 1):
try:
dataset_info = file_info['dataset']
# Create subdirectory based on crop/variable/irrigation
crop_dir = base_dir / (file_info['crop'] or 'unknown')
var_dir = crop_dir / (file_info['variable'] or 'unknown')
irr_dir = var_dir / (file_info['irrigation'] or 'unknown')
irr_dir.mkdir(parents=True, exist_ok=True)
# Get files from dataset
files = dataset_info.get('files', [])
if not files:
print(f"[{i}/{len(files_list)}] ⚠️ No files in dataset")
continue
# Download each file using client.download()
for file_dict in files:
file_url = file_dict.get('file_url')
file_name = file_dict.get('path', '').split('/')[-1] or 'file'
if not file_url:
print(f"[{i}/{len(files_list)}] ⚠️ No file_url for {file_name}")
continue
out_file = irr_dir / file_name
# NEW: Check if file already exists
local_file = irr_dir / file_dict['name']
if local_file.exists():
# Check file size matches
local_size = local_file.stat().st_size
remote_size = file_dict.get('size', 0)
if local_size == remote_size:
print(f" ⏭️ Skipping (exists): {file_dict['name']}")
self.downloaded_files.append(str(local_file))
continue
else:
print(f" ⚠️ Incomplete file, re-downloading: {file_dict['name']}")
# NEW: Only download if file doesn't exist or is incomplete
print(f" ⬇️ Downloading: {file_dict['name']}")
self.client.download(
file_url,
path=str(irr_dir),
validate=True
)
print(f" ✅ Downloaded {file_name}")
self.downloaded_files.append(str(out_file))
except Exception as e:
print(f" ❌ Error: {str(e)}")
print(f"\n✅ Downloaded {len(self.downloaded_files)} files")
def load(self, file_path: str, use_dask: bool = True, decode_times: bool = False) -> xr.Dataset:
"""Load a NetCDF file into an xarray Dataset with custom time handling for ISIMIP data.
Args:
file_path (str): Path to the NetCDF file
use_dask (bool, optional): Whether to use dask for lazy loading (default:
True)
decode_times (bool, optional): Whether to decode times (default: False for
ISIMIP data with custom calendars) Set to True if your data uses
standard time encoding
Returns:
xr.Dataset: Loaded xarray dataset with properly decoded time coordinates
Note:
ISIMIP data often uses custom time encoding like "growing seasons since 1601-01-01"
with custom calendars. This function automatically detects and converts such times
to proper datetime coordinates by:
1. Loading with decode_times=False
2. Parsing the time unit string (e.g., "since 1601-01-01")
3. Interpreting time values as years offset from the reference year
4. Creating proper datetime coordinates
"""
print(f"📂 Loading {file_path}...")
try:
# Load with decode_times=False to avoid calendar errors
if use_dask:
self.ds = xr.open_dataset(file_path, chunks='auto', decode_times=False)
else:
self.ds = xr.open_dataset(file_path, decode_times=False)
# Check if time dimension exists and needs custom decoding
if 'time' in self.ds.dims and 'time' in self.ds.data_vars or 'time' in self.ds.coords:
try:
# Try to decode ISIMIP custom time format
time_attrs = self.ds.time.attrs.get('units', '')
# Check if it's a custom ISIMIP time format (e.g., "growing seasons since ...")
if 'since' in time_attrs:
print(f" 📅 Detected custom time format: {time_attrs}")
# Extract start year from units string
match = re.search(r"since\s(.+)", time_attrs)
if match:
start_date_str = match.group(1).strip()
start = pd.Timestamp(start_date_str)
# Interpret time values as years offset from start year
years = start.year + self.ds['time'].values.astype(int)
# Assign proper datetime coordinates
self.ds = self.ds.assign_coords(
time=pd.to_datetime(years, format="%Y")
)
print(f" ✅ Converted time coordinates (reference year: {start.year})")
except Exception as time_error:
print(f" ⚠️ Could not decode custom time format: {str(time_error)}")
print(f" Keeping original time values")
print(f"✅ Loaded dataset with dimensions: {dict(self.ds.dims)}")
return self.ds
except Exception as e:
print(f"❌ Error loading file: {str(e)}")
if 'decode_times' in str(e) or 'calendar' in str(e):
print("💡 Hint: ISIMIP data uses custom calendars. Load is called with decode_times=False")
return None
def extract(self, *, point: Tuple[float, float] = None, box: Dict[str, float] = None,
shapefile: str = None, buffer_km: float = 0.0):
"""Specify spatial extraction intent.
Args:
point (tuple, optional): (lon, lat) for point extraction
box (dict, optional): dict(lat_min, lat_max, lon_min, lon_max) for box
extraction
shapefile (str, optional): Path to shapefile for shapefile extraction
buffer_km (float, optional): Buffer in km for shapefile extraction
"""
if point is not None:
lon, lat = point
self._extract_mode = "point"
self._extract_params = (lon, lat)
print(f"📍 Set point extraction mode: ({lon}, {lat})")
elif box is not None:
for k in ("lat_min", "lat_max", "lon_min", "lon_max"):
if k not in box:
raise ValueError(f"box missing key {k}")
self._extract_mode = "box"
self._extract_params = box
print(f"📦 Set box extraction mode: {box}")
elif shapefile is not None:
import geopandas as gpd
gdf = gpd.read_file(shapefile) if isinstance(shapefile, str) else shapefile
if buffer_km > 0:
gdf = gdf.to_crs(epsg=3857)
gdf["geometry"] = gdf.buffer(buffer_km * 1000)
gdf = gdf.to_crs(epsg=4326)
self._extract_mode = "shapefile"
self._extract_params = gdf
print(f"🗺️ Set shapefile extraction mode with buffer {buffer_km} km")
else:
raise ValueError("Must provide point, box, or shapefile.")
return self
def get_extracted_data(self) -> xr.Dataset:
"""Extract data based on spatial mode set via extract().
Returns:
xr.Dataset: Extracted dataset at specified location(s)
"""
if self.ds is None:
raise ValueError("No dataset loaded. Call load() first.")
if self._extract_mode == "point":
lon, lat = self._extract_params
extracted = self.ds.sel(lon=lon, lat=lat, method='nearest')
print(f"✅ Extracted point data at ({lon}, {lat})")
return extracted
elif self._extract_mode == "box":
box = self._extract_params
extracted = self.ds.sel(
lon=slice(box['lon_min'], box['lon_max']),
lat=slice(box['lat_min'], box['lat_max'])
)
print(f"✅ Extracted box data")
return extracted
elif self._extract_mode == "shapefile":
# TODO: Implement shapefile extraction
raise NotImplementedError("Shapefile extraction not yet implemented")
else:
raise ValueError("No extraction mode set. Call extract() first.")
def save_csv(self, filename: str):
"""Write the loaded dataset to CSV, one row per cell per timestep.
Args:
filename (str): Destination path. The parent directory must exist.
Returns:
None
Raises:
ValueError: If no dataset has been loaded.
"""
if self.ds is None:
raise ValueError("No dataset loaded. Call load() first.")
df = self.ds.to_dataframe().reset_index()
df.to_csv(filename, index=False)
print(f"✅ Saved CSV to {filename}")
def save_netcdf(self, filename: str):
"""Write the loaded dataset to NetCDF.
Args:
filename (str): Destination path. The parent directory must exist.
Returns:
None
Raises:
ValueError: If no dataset has been loaded.
"""
if self.ds is None:
raise ValueError("No dataset loaded. Call load() first.")
self.ds.to_netcdf(filename)
print(f"✅ Saved NetCDF to {filename}")
AVAILABLE_CROPS: Set[str]
property
readonly
¶
Crops the API reported for the queried configuration.
Returns:
| Type | Description |
|---|---|
set[str] |
Crop codes, e.g. |
Exceptions:
| Type | Description |
|---|---|
ValueError |
If :meth: |
AVAILABLE_IRRIGATIONS: Set[str]
property
readonly
¶
Irrigation regimes the API reported for the queried configuration.
Returns:
| Type | Description |
|---|---|
set[str] |
Regime codes — typically |
Exceptions:
| Type | Description |
|---|---|
ValueError |
If :meth: |
AVAILABLE_SCENARIOS: Set[str]
property
readonly
¶
Climate scenarios the API reported for the queried configuration.
Returns:
| Type | Description |
|---|---|
set[str] |
Scenario names, e.g. |
Exceptions:
| Type | Description |
|---|---|
ValueError |
If :meth: |
AVAILABLE_VARIABLES: Set[str]
property
readonly
¶
Variables the API reported for the queried configuration.
Returns:
| Type | Description |
|---|---|
set[str] |
Variable names, e.g. |
Exceptions:
| Type | Description |
|---|---|
ValueError |
If :meth: |
__init__(self, cfg)
special
¶
Bind a configuration and open an ISIMIP client.
No metadata is queried here; the AVAILABLE_* caches start empty.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
cfg |
DictConfig |
Configuration with |
required |
Exceptions:
| Type | Description |
|---|---|
ImportError |
If |
Source code in climdata/datasets/AGRI_ISIMIP.py
def __init__(self, cfg: DictConfig):
"""Bind a configuration and open an ISIMIP client.
No metadata is queried here; the ``AVAILABLE_*`` caches start empty.
Args:
cfg (DictConfig): Configuration with ``query_configs``, ``data_dir``
and ``variables``.
Raises:
ImportError: If ``isimip-client`` is not installed.
"""
self.cfg = cfg
self.ds = None
self.downloaded_files = []
self._extract_mode = None
self._extract_params = None
# Initialize ISIMIP client
try:
from isimip_client.client import ISIMIPClient
self.client = ISIMIPClient()
except ImportError:
raise ImportError(
"isimip-client is required for AGRI-ISIMIP data access. "
"Install it with: pip install isimip-client"
)
# Cached metadata from API queries
self._available_crops: Optional[Set[str]] = None
self._available_variables: Optional[Set[str]] = None
self._available_irrigations: Optional[Set[str]] = None
self._available_scenarios: Optional[Set[str]] = None
explore(self)
¶
Print what ISIMIP holds for every configured query combination.
Runs :meth:query_available_metadata over each entry of
cfg.query_configs and prints the crops, variables, irrigation
regimes and scenarios found. The intended first call on a new
configuration, before committing to a download.
Note that the AVAILABLE_* caches are overwritten by each combination
in turn, so after this they reflect the last one only.
Returns:
| Type | Description |
|---|---|
None |
The report is written to stdout. |
Source code in climdata/datasets/AGRI_ISIMIP.py
def explore(self):
"""Print what ISIMIP holds for every configured query combination.
Runs :meth:`query_available_metadata` over each entry of
``cfg.query_configs`` and prints the crops, variables, irrigation
regimes and scenarios found. The intended first call on a new
configuration, before committing to a download.
Note that the ``AVAILABLE_*`` caches are overwritten by each combination
in turn, so after this they reflect the last one only.
Returns:
None: The report is written to stdout.
"""
print("\n" + "=" * 70)
print("🌾 AGRI-ISIMIP Dataset Explorer")
print("=" * 70)
query_configs = self.cfg.get('query_configs', [])
if not query_configs:
print("\n⚠️ No query_configs found in configuration")
return
for i, config in enumerate(query_configs, 1):
print(f"\n[{i}/{len(query_configs)}] Configuration:")
print(f" Impact Model: {config['impact_model']}")
print(f" Climate Forcing: {config['climate_forcing']}")
print(f" Period: {config['period']}")
print("-" * 70)
try:
self.query_available_metadata(
impact_model=config['impact_model'],
climate_forcing=config['climate_forcing'],
period=config['period']
)
except Exception as e:
print(f"❌ Error: {str(e)}")
extract(self, *, point=None, box=None, shapefile=None, buffer_km=0.0)
¶
Specify spatial extraction intent.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
point |
tuple |
(lon, lat) for point extraction |
None |
box |
dict |
dict(lat_min, lat_max, lon_min, lon_max) for box extraction |
None |
shapefile |
str |
Path to shapefile for shapefile extraction |
None |
buffer_km |
float |
Buffer in km for shapefile extraction |
0.0 |
Source code in climdata/datasets/AGRI_ISIMIP.py
def extract(self, *, point: Tuple[float, float] = None, box: Dict[str, float] = None,
shapefile: str = None, buffer_km: float = 0.0):
"""Specify spatial extraction intent.
Args:
point (tuple, optional): (lon, lat) for point extraction
box (dict, optional): dict(lat_min, lat_max, lon_min, lon_max) for box
extraction
shapefile (str, optional): Path to shapefile for shapefile extraction
buffer_km (float, optional): Buffer in km for shapefile extraction
"""
if point is not None:
lon, lat = point
self._extract_mode = "point"
self._extract_params = (lon, lat)
print(f"📍 Set point extraction mode: ({lon}, {lat})")
elif box is not None:
for k in ("lat_min", "lat_max", "lon_min", "lon_max"):
if k not in box:
raise ValueError(f"box missing key {k}")
self._extract_mode = "box"
self._extract_params = box
print(f"📦 Set box extraction mode: {box}")
elif shapefile is not None:
import geopandas as gpd
gdf = gpd.read_file(shapefile) if isinstance(shapefile, str) else shapefile
if buffer_km > 0:
gdf = gdf.to_crs(epsg=3857)
gdf["geometry"] = gdf.buffer(buffer_km * 1000)
gdf = gdf.to_crs(epsg=4326)
self._extract_mode = "shapefile"
self._extract_params = gdf
print(f"🗺️ Set shapefile extraction mode with buffer {buffer_km} km")
else:
raise ValueError("Must provide point, box, or shapefile.")
return self
fetch(self, impact_model, climate_forcing, period, crop=None, irrigation=None, variable=None)
¶
Download AGRI-ISIMIP files matching the criteria using isimip_client.download().
Files are stored as::
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
impact_model |
str |
Impact model name |
required |
climate_forcing |
str |
Climate forcing |
required |
period |
str |
Time period |
required |
crop |
str |
Specific crop to download |
None |
irrigation |
str |
Specific irrigation type to download |
None |
variable |
str |
Specific variable to download |
None |
Source code in climdata/datasets/AGRI_ISIMIP.py
def fetch(self, impact_model: str, climate_forcing: str, period: str,
crop: Optional[str] = None, irrigation: Optional[str] = None,
variable: Optional[str] = None):
"""Download AGRI-ISIMIP files matching the criteria using isimip_client.download().
Files are stored as::
<data_dir>/AGRI_ISIMIP/<impact_model>/<climate_forcing>/<period>/<crop>/<variable>/<irrigation>/
Args:
impact_model (str): Impact model name
climate_forcing (str): Climate forcing
period (str): Time period
crop (str, optional): Specific crop to download
irrigation (str, optional): Specific irrigation type to download
variable (str, optional): Specific variable to download
"""
files_list = self.list_files(
impact_model=impact_model,
climate_forcing=climate_forcing,
period=period,
crop=crop,
irrigation=irrigation,
variable=variable
)
if not files_list:
print("⚠️ No files found to download")
return
# Create directory structure
base_dir = (
Path(self.cfg.data_dir) /
"AGRI_ISIMIP" /
impact_model /
climate_forcing /
period
)
base_dir.mkdir(parents=True, exist_ok=True)
print(f"\n📥 Downloading {len(files_list)} files to {base_dir}...\n")
for i, file_info in enumerate(files_list, 1):
try:
dataset_info = file_info['dataset']
# Create subdirectory based on crop/variable/irrigation
crop_dir = base_dir / (file_info['crop'] or 'unknown')
var_dir = crop_dir / (file_info['variable'] or 'unknown')
irr_dir = var_dir / (file_info['irrigation'] or 'unknown')
irr_dir.mkdir(parents=True, exist_ok=True)
# Get files from dataset
files = dataset_info.get('files', [])
if not files:
print(f"[{i}/{len(files_list)}] ⚠️ No files in dataset")
continue
# Download each file using client.download()
for file_dict in files:
file_url = file_dict.get('file_url')
file_name = file_dict.get('path', '').split('/')[-1] or 'file'
if not file_url:
print(f"[{i}/{len(files_list)}] ⚠️ No file_url for {file_name}")
continue
out_file = irr_dir / file_name
# NEW: Check if file already exists
local_file = irr_dir / file_dict['name']
if local_file.exists():
# Check file size matches
local_size = local_file.stat().st_size
remote_size = file_dict.get('size', 0)
if local_size == remote_size:
print(f" ⏭️ Skipping (exists): {file_dict['name']}")
self.downloaded_files.append(str(local_file))
continue
else:
print(f" ⚠️ Incomplete file, re-downloading: {file_dict['name']}")
# NEW: Only download if file doesn't exist or is incomplete
print(f" ⬇️ Downloading: {file_dict['name']}")
self.client.download(
file_url,
path=str(irr_dir),
validate=True
)
print(f" ✅ Downloaded {file_name}")
self.downloaded_files.append(str(out_file))
except Exception as e:
print(f" ❌ Error: {str(e)}")
print(f"\n✅ Downloaded {len(self.downloaded_files)} files")
get_extracted_data(self)
¶
Extract data based on spatial mode set via extract().
Returns:
| Type | Description |
|---|---|
xr.Dataset |
Extracted dataset at specified location(s) |
Source code in climdata/datasets/AGRI_ISIMIP.py
def get_extracted_data(self) -> xr.Dataset:
"""Extract data based on spatial mode set via extract().
Returns:
xr.Dataset: Extracted dataset at specified location(s)
"""
if self.ds is None:
raise ValueError("No dataset loaded. Call load() first.")
if self._extract_mode == "point":
lon, lat = self._extract_params
extracted = self.ds.sel(lon=lon, lat=lat, method='nearest')
print(f"✅ Extracted point data at ({lon}, {lat})")
return extracted
elif self._extract_mode == "box":
box = self._extract_params
extracted = self.ds.sel(
lon=slice(box['lon_min'], box['lon_max']),
lat=slice(box['lat_min'], box['lat_max'])
)
print(f"✅ Extracted box data")
return extracted
elif self._extract_mode == "shapefile":
# TODO: Implement shapefile extraction
raise NotImplementedError("Shapefile extraction not yet implemented")
else:
raise ValueError("No extraction mode set. Call extract() first.")
list_files(self, impact_model, climate_forcing, period, crop=None, irrigation=None, variable=None)
¶
List available files matching the given criteria.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
impact_model |
str |
Impact model name (e.g., 'SIMPLACE-LINTUL5') |
required |
climate_forcing |
str |
Climate forcing (e.g., 'mpi-esm1-2-hr') |
required |
period |
str |
Time period (e.g., 'historical') |
required |
crop |
str |
Filter by crop name |
None |
irrigation |
str |
Filter by irrigation type ('firr' or 'noirr') |
None |
variable |
str |
Filter by variable name |
None |
Returns:
| Type | Description |
|---|---|
List[Dict] |
List of file metadata dictionaries |
Source code in climdata/datasets/AGRI_ISIMIP.py
def list_files(self, impact_model: str, climate_forcing: str, period: str,
crop: Optional[str] = None, irrigation: Optional[str] = None,
variable: Optional[str] = None) -> List[Dict]:
"""List available files matching the given criteria.
Args:
impact_model (str): Impact model name (e.g., 'SIMPLACE-LINTUL5')
climate_forcing (str): Climate forcing (e.g., 'mpi-esm1-2-hr')
period (str): Time period (e.g., 'historical')
crop (str, optional): Filter by crop name
irrigation (str, optional): Filter by irrigation type ('firr' or 'noirr')
variable (str, optional): Filter by variable name
Returns:
List[Dict]: List of file metadata dictionaries
"""
path = f'ISIMIP3b/OutputData/agriculture/{impact_model}/{climate_forcing}/{period}'
print(f"\n🔍 Listing AGRI-ISIMIP files:")
print(f" Path: {path}")
if crop:
print(f" Crop: {crop}")
if irrigation:
print(f" Irrigation: {irrigation}")
if variable:
print(f" Variable: {variable}")
files_list = []
page = 1
total_fetched = 0
try:
while True:
response = self.client.datasets(
path=path,
page=page,
page_size=100
)
results = response.get('results', [])
if not results:
break
total_fetched += len(results)
for dataset in results:
spec = dataset.get('specifiers', {})
file_crop = spec.get('crop')
file_irr = spec.get('irrigation')
file_var = spec.get('variable')
# Apply filters
if crop and file_crop != crop:
continue
if irrigation and file_irr != irrigation:
continue
if variable and file_var != variable:
continue
files_list.append({
'name': dataset.get('name', ''),
'path': dataset.get('path', ''),
'size': dataset.get('size', 'N/A'),
'crop': file_crop,
'irrigation': file_irr,
'variable': file_var,
'dataset': dataset,
})
# Check pagination
if not response.get('next'):
break
page += 1
print(f"✅ Found {len(files_list)} matching files (from {total_fetched} total)\n")
# Display first 20 results
if files_list:
print("Files found:")
print("-" * 100)
for i, f in enumerate(files_list[:20], 1):
print(f"{i:3d}. {f['name']}")
print(f" Path: {f['path']}")
print(f" Crop: {f['crop']}, Irrigation: {f['irrigation']}, Variable: {f['variable']}")
if f['size'] != 'N/A':
size_gb = f['size'] / (1024**3) if isinstance(f['size'], (int, float)) else 'N/A'
if size_gb != 'N/A':
print(f" Size: {size_gb:.2f} GB")
if len(files_list) > 20:
print(f"\n ... and {len(files_list) - 20} more files")
return files_list
except Exception as e:
print(f"❌ Error listing files: {str(e)}")
return []
load(self, file_path, use_dask=True, decode_times=False)
¶
Load a NetCDF file into an xarray Dataset with custom time handling for ISIMIP data.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
file_path |
str |
Path to the NetCDF file |
required |
use_dask |
bool |
Whether to use dask for lazy loading (default: True) |
True |
decode_times |
bool |
Whether to decode times (default: False for ISIMIP data with custom calendars) Set to True if your data uses standard time encoding |
False |
Returns:
| Type | Description |
|---|---|
xr.Dataset |
Loaded xarray dataset with properly decoded time coordinates |
Note
ISIMIP data often uses custom time encoding like "growing seasons since 1601-01-01" with custom calendars. This function automatically detects and converts such times to proper datetime coordinates by: 1. Loading with decode_times=False 2. Parsing the time unit string (e.g., "since 1601-01-01") 3. Interpreting time values as years offset from the reference year 4. Creating proper datetime coordinates
Source code in climdata/datasets/AGRI_ISIMIP.py
def load(self, file_path: str, use_dask: bool = True, decode_times: bool = False) -> xr.Dataset:
"""Load a NetCDF file into an xarray Dataset with custom time handling for ISIMIP data.
Args:
file_path (str): Path to the NetCDF file
use_dask (bool, optional): Whether to use dask for lazy loading (default:
True)
decode_times (bool, optional): Whether to decode times (default: False for
ISIMIP data with custom calendars) Set to True if your data uses
standard time encoding
Returns:
xr.Dataset: Loaded xarray dataset with properly decoded time coordinates
Note:
ISIMIP data often uses custom time encoding like "growing seasons since 1601-01-01"
with custom calendars. This function automatically detects and converts such times
to proper datetime coordinates by:
1. Loading with decode_times=False
2. Parsing the time unit string (e.g., "since 1601-01-01")
3. Interpreting time values as years offset from the reference year
4. Creating proper datetime coordinates
"""
print(f"📂 Loading {file_path}...")
try:
# Load with decode_times=False to avoid calendar errors
if use_dask:
self.ds = xr.open_dataset(file_path, chunks='auto', decode_times=False)
else:
self.ds = xr.open_dataset(file_path, decode_times=False)
# Check if time dimension exists and needs custom decoding
if 'time' in self.ds.dims and 'time' in self.ds.data_vars or 'time' in self.ds.coords:
try:
# Try to decode ISIMIP custom time format
time_attrs = self.ds.time.attrs.get('units', '')
# Check if it's a custom ISIMIP time format (e.g., "growing seasons since ...")
if 'since' in time_attrs:
print(f" 📅 Detected custom time format: {time_attrs}")
# Extract start year from units string
match = re.search(r"since\s(.+)", time_attrs)
if match:
start_date_str = match.group(1).strip()
start = pd.Timestamp(start_date_str)
# Interpret time values as years offset from start year
years = start.year + self.ds['time'].values.astype(int)
# Assign proper datetime coordinates
self.ds = self.ds.assign_coords(
time=pd.to_datetime(years, format="%Y")
)
print(f" ✅ Converted time coordinates (reference year: {start.year})")
except Exception as time_error:
print(f" ⚠️ Could not decode custom time format: {str(time_error)}")
print(f" Keeping original time values")
print(f"✅ Loaded dataset with dimensions: {dict(self.ds.dims)}")
return self.ds
except Exception as e:
print(f"❌ Error loading file: {str(e)}")
if 'decode_times' in str(e) or 'calendar' in str(e):
print("💡 Hint: ISIMIP data uses custom calendars. Load is called with decode_times=False")
return None
query_available_metadata(self, impact_model, climate_forcing, period)
¶
Query ISIMIP API to get all available metadata for a specific configuration. Extracts crops, variables, irrigation types, and scenarios.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
impact_model |
str |
Impact model name (e.g., 'SIMPLACE-LINTUL5') |
required |
climate_forcing |
str |
Climate forcing/model (e.g., 'mpi-esm1-2-hr') |
required |
period |
str |
Time period (e.g., 'historical') |
required |
Exceptions:
| Type | Description |
|---|---|
Exception |
If API query fails or no datasets found |
Source code in climdata/datasets/AGRI_ISIMIP.py
def query_available_metadata(self, impact_model: str, climate_forcing: str, period: str) -> None:
"""Query ISIMIP API to get all available metadata for a specific configuration.
Extracts crops, variables, irrigation types, and scenarios.
Args:
impact_model (str): Impact model name (e.g., 'SIMPLACE-LINTUL5')
climate_forcing (str): Climate forcing/model (e.g., 'mpi-esm1-2-hr')
period (str): Time period (e.g., 'historical')
Raises:
Exception: If API query fails or no datasets found
"""
path = f'ISIMIP3b/OutputData/agriculture/{impact_model}/{climate_forcing}/{period}'
crops: Set[str] = set()
variables: Set[str] = set()
irrigation: Set[str] = set()
scenarios: Set[str] = set()
page = 1
total_fetched = 0
while True:
response = self.client.datasets(
path=path,
page=page,
page_size=100
)
results = response.get('results', [])
if not results:
break
total_fetched += len(results)
for ds in results:
spec = ds.get('specifiers', {})
crop = spec.get('crop')
var = spec.get('variable')
irr = spec.get('irrigation')
scen = spec.get('climate_scenario')
if crop:
crops.add(crop)
if var:
variables.add(var)
if irr:
irrigation.add(irr)
if scen:
scenarios.add(scen)
# Check pagination
if not response.get('next'):
break
page += 1
self._available_crops = crops
self._available_variables = variables
self._available_irrigations = irrigation
self._available_scenarios = scenarios
print(f"✅ Fetched {total_fetched} datasets from ISIMIP API")
print(f" Crops: {sorted(crops)}")
print(f" Variables: {sorted(variables)}")
print(f" Irrigation: {sorted(irrigation)}")
print(f" Scenarios: {sorted(scenarios)}")
save_csv(self, filename)
¶
Write the loaded dataset to CSV, one row per cell per timestep.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
filename |
str |
Destination path. The parent directory must exist. |
required |
Returns:
| Type | Description |
|---|---|
None |
Exceptions:
| Type | Description |
|---|---|
ValueError |
If no dataset has been loaded. |
Source code in climdata/datasets/AGRI_ISIMIP.py
def save_csv(self, filename: str):
"""Write the loaded dataset to CSV, one row per cell per timestep.
Args:
filename (str): Destination path. The parent directory must exist.
Returns:
None
Raises:
ValueError: If no dataset has been loaded.
"""
if self.ds is None:
raise ValueError("No dataset loaded. Call load() first.")
df = self.ds.to_dataframe().reset_index()
df.to_csv(filename, index=False)
print(f"✅ Saved CSV to {filename}")
save_netcdf(self, filename)
¶
Write the loaded dataset to NetCDF.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
filename |
str |
Destination path. The parent directory must exist. |
required |
Returns:
| Type | Description |
|---|---|
None |
Exceptions:
| Type | Description |
|---|---|
ValueError |
If no dataset has been loaded. |
Source code in climdata/datasets/AGRI_ISIMIP.py
def save_netcdf(self, filename: str):
"""Write the loaded dataset to NetCDF.
Args:
filename (str): Destination path. The parent directory must exist.
Returns:
None
Raises:
ValueError: If no dataset has been loaded.
"""
if self.ds is None:
raise ValueError("No dataset loaded. Call load() first.")
self.ds.to_netcdf(filename)
print(f"✅ Saved NetCDF to {filename}")
Extreme indices¶
Indices are declared in conf/mappings/indices.yaml and evaluated through
xclim.
climdata.extremes.indices.extreme_index
¶
Evaluate a configured extreme index against a loaded dataset.
Attributes:
| Name | Type | Description |
|---|---|---|
cfg |
DictConfig |
Hydra configuration; |
climate_data |
xr.Dataset |
Dataset supplying the input variables. Linked intermediate variables are written back into it as they are built. |
Examples:
>>> indices = extreme_index(cfg, ds)
>>> da = indices.calculate("tx90p")
Source code in climdata/extremes/indices.py
class extreme_index:
"""Evaluate a configured extreme index against a loaded dataset.
Attributes:
cfg (DictConfig): Hydra configuration; ``cfg.extinfo.indices`` holds the
index definitions.
climate_data (xr.Dataset): Dataset supplying the input variables. Linked
intermediate variables are written back into it as they are built.
Example:
>>> indices = extreme_index(cfg, ds) # doctest: +SKIP
>>> da = indices.calculate("tx90p") # doctest: +SKIP
"""
def __init__(self, cfg, climate_data):
"""Bind a configuration and a dataset.
Args:
cfg (DictConfig): Hydra configuration containing ``extinfo.indices``.
climate_data (xr.Dataset): Dataset holding the required input variables.
"""
self.cfg = cfg
self.climate_data = climate_data
def calculate(self, index):
"""Compute a single index by name.
Resolution proceeds in three steps:
1. **Linked variables** — each entry under ``index_cfg.link`` produces an
intermediate DataArray, either by calling an importable function
(``function_call``) or by invoking a method on one input variable
(``operation``). An optional ``postprocess`` block may then subset the
result. Each is stored back into :attr:`climate_data` under its key so
later links and the index itself can reference it.
2. **Argument resolution** — any argument written as an OmegaConf-style
reference (``"${...}"``) is replaced by the matching entry of
:attr:`climate_data`.
3. **Evaluation** — ``index_cfg.function`` is imported and called with the
variables listed in ``index_cfg.variables`` (falling back to ``pr``).
Args:
index (str): Index name, matching a key of ``cfg.extinfo.indices``
(e.g. ``"tn10p"``, ``"tx90p"``, ``"rx1day"``).
Returns:
xr.DataArray: The computed index, renamed to ``index``. The result is
lazy if the inputs are lazy — call ``.compute()`` to materialise it.
Raises:
KeyError: If ``index`` is not defined in ``cfg.extinfo.indices``, or a
referenced input variable is absent from :attr:`climate_data`.
ValueError: If a ``link`` entry defines neither ``function_call`` nor
``operation``.
NotImplementedError: If a ``postprocess`` block names an unsupported
operation.
Example:
>>> indices = extreme_index(cfg, ds) # doctest: +SKIP
>>> indices.calculate("tn10p").compute() # doctest: +SKIP
"""
index_cfg = self.cfg.extinfo.indices[index]
args = dict(index_cfg.args)
# Handle linked intermediate variables
if "link" in index_cfg:
for var_name, link_cfg in index_cfg.link.items():
# --- External function call ---
if "function_call" in link_cfg:
inputs = [self.climate_data[name] for name in link_cfg["inputs"]]
module_path, func_name = link_cfg["function_call"].rsplit(".", 1)
func = getattr(importlib.import_module(module_path), func_name)
result = func(*inputs, **link_cfg.get("kwargs", {}))
# --- Method call on single input ---
elif "operation" in link_cfg:
input_var = self.climate_data[link_cfg["input"]]
method = getattr(input_var, link_cfg["operation"])
result = method(**link_cfg.get("kwargs", {}))
else:
raise ValueError(f"Link for '{var_name}' must define either 'function_call' or 'operation'.")
# Optional postprocessing
if "postprocess" in link_cfg:
for op_name, op_args in link_cfg["postprocess"].items():
if op_name == "sel":
result = result.sel(**op_args)
else:
raise NotImplementedError(f"Postprocess operation '{op_name}' not supported.")
self.climate_data[var_name] = result
# Resolve references in args
for key, val in args.items():
if isinstance(val, str) and val.startswith("${"):
ref = val.strip("${}").split(".")[-1]
args[key] = self.climate_data[ref]
# Load and call final function
module_name, func_name = index_cfg.function.rsplit(".", 1)
func = getattr(importlib.import_module(module_name), func_name)
if hasattr(index_cfg, "variables"):
inputs = [self.climate_data[v] for v in index_cfg.variables]
result = func(*inputs, **args)
else:
result = func(self.climate_data["pr"], **args)
if isinstance(result, xr.DataArray):
result.name = index
return result
__init__(self, cfg, climate_data)
special
¶
Bind a configuration and a dataset.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
cfg |
DictConfig |
Hydra configuration containing |
required |
climate_data |
xr.Dataset |
Dataset holding the required input variables. |
required |
Source code in climdata/extremes/indices.py
def __init__(self, cfg, climate_data):
"""Bind a configuration and a dataset.
Args:
cfg (DictConfig): Hydra configuration containing ``extinfo.indices``.
climate_data (xr.Dataset): Dataset holding the required input variables.
"""
self.cfg = cfg
self.climate_data = climate_data
calculate(self, index)
¶
Compute a single index by name.
Resolution proceeds in three steps:
- Linked variables — each entry under
index_cfg.linkproduces an intermediate DataArray, either by calling an importable function (function_call) or by invoking a method on one input variable (operation). An optionalpostprocessblock may then subset the result. Each is stored back into :attr:climate_dataunder its key so later links and the index itself can reference it. - Argument resolution — any argument written as an OmegaConf-style
reference (
"${...}") is replaced by the matching entry of :attr:climate_data. - Evaluation —
index_cfg.functionis imported and called with the variables listed inindex_cfg.variables(falling back topr).
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
index |
str |
Index name, matching a key of |
required |
Returns:
| Type | Description |
|---|---|
xr.DataArray |
The computed index, renamed to |
Exceptions:
| Type | Description |
|---|---|
KeyError |
If |
ValueError |
If a |
NotImplementedError |
If a |
Examples:
>>> indices = extreme_index(cfg, ds)
>>> indices.calculate("tn10p").compute()
Source code in climdata/extremes/indices.py
def calculate(self, index):
"""Compute a single index by name.
Resolution proceeds in three steps:
1. **Linked variables** — each entry under ``index_cfg.link`` produces an
intermediate DataArray, either by calling an importable function
(``function_call``) or by invoking a method on one input variable
(``operation``). An optional ``postprocess`` block may then subset the
result. Each is stored back into :attr:`climate_data` under its key so
later links and the index itself can reference it.
2. **Argument resolution** — any argument written as an OmegaConf-style
reference (``"${...}"``) is replaced by the matching entry of
:attr:`climate_data`.
3. **Evaluation** — ``index_cfg.function`` is imported and called with the
variables listed in ``index_cfg.variables`` (falling back to ``pr``).
Args:
index (str): Index name, matching a key of ``cfg.extinfo.indices``
(e.g. ``"tn10p"``, ``"tx90p"``, ``"rx1day"``).
Returns:
xr.DataArray: The computed index, renamed to ``index``. The result is
lazy if the inputs are lazy — call ``.compute()`` to materialise it.
Raises:
KeyError: If ``index`` is not defined in ``cfg.extinfo.indices``, or a
referenced input variable is absent from :attr:`climate_data`.
ValueError: If a ``link`` entry defines neither ``function_call`` nor
``operation``.
NotImplementedError: If a ``postprocess`` block names an unsupported
operation.
Example:
>>> indices = extreme_index(cfg, ds) # doctest: +SKIP
>>> indices.calculate("tn10p").compute() # doctest: +SKIP
"""
index_cfg = self.cfg.extinfo.indices[index]
args = dict(index_cfg.args)
# Handle linked intermediate variables
if "link" in index_cfg:
for var_name, link_cfg in index_cfg.link.items():
# --- External function call ---
if "function_call" in link_cfg:
inputs = [self.climate_data[name] for name in link_cfg["inputs"]]
module_path, func_name = link_cfg["function_call"].rsplit(".", 1)
func = getattr(importlib.import_module(module_path), func_name)
result = func(*inputs, **link_cfg.get("kwargs", {}))
# --- Method call on single input ---
elif "operation" in link_cfg:
input_var = self.climate_data[link_cfg["input"]]
method = getattr(input_var, link_cfg["operation"])
result = method(**link_cfg.get("kwargs", {}))
else:
raise ValueError(f"Link for '{var_name}' must define either 'function_call' or 'operation'.")
# Optional postprocessing
if "postprocess" in link_cfg:
for op_name, op_args in link_cfg["postprocess"].items():
if op_name == "sel":
result = result.sel(**op_args)
else:
raise NotImplementedError(f"Postprocess operation '{op_name}' not supported.")
self.climate_data[var_name] = result
# Resolve references in args
for key, val in args.items():
if isinstance(val, str) and val.startswith("${"):
ref = val.strip("${}").split(".")[-1]
args[key] = self.climate_data[ref]
# Load and call final function
module_name, func_name = index_cfg.function.rsplit(".", 1)
func = getattr(importlib.import_module(module_name), func_name)
if hasattr(index_cfg, "variables"):
inputs = [self.climate_data[v] for v in index_cfg.variables]
result = func(*inputs, **args)
else:
result = func(self.climate_data["pr"], **args)
if isinstance(result, xr.DataArray):
result.name = index
return result
Grid transformation¶
Reprojection and resampling, plus the resolution/CRS compatibility gate. See the worked example.
climdata.grid.reproject.reproject(obj, target_projection=None, target_resolution=None, *, like=None, method='bilinear', bounds=None, align=True, src_crs=None, x_dim=None, y_dim=None, nodata=nan, engine='rasterio', dsinfo_dims=None)
¶
Reproject and/or resample a dataset onto a new grid.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
obj |
Union[xr.Dataset, xr.DataArray] |
The :class: |
required |
target_projection |
Any |
Destination CRS -- |
None |
target_resolution |
Any |
Destination cell size -- |
None |
like |
Optional[Union[xr.Dataset, xr.DataArray]] |
A template whose grid the output should match exactly. Cannot be
combined with |
None |
method |
Union[str, Mapping[str, str]] |
Resampling method name, or a |
'bilinear' |
bounds |
Optional[Sequence[float]] |
Optional |
None |
align |
bool |
Snap the grid origin to whole multiples of the resolution so that
separately-processed domains stay co-registered. Default |
True |
src_crs |
Any |
Source CRS, when it cannot be inferred from the data. |
None |
x_dim |
Optional[str] |
Source x/longitude dimension name. |
None |
y_dim |
Optional[str] |
Source y/latitude dimension name. |
None |
nodata |
Optional[float] |
Fill value written before warping. Keep the |
nan |
engine |
str |
|
'rasterio' |
dsinfo_dims |
Optional[Sequence[str]] |
Optional |
None |
Returns:
| Type | Description |
|---|---|
Union[xr.Dataset, xr.DataArray] |
The reprojected object, of the same type as |
Exceptions:
| Type | Description |
|---|---|
ResolutionCRSMismatch |
If the resolution units contradict the target CRS. |
ValueError |
If |
Examples:
>>> out = reproject(ds, "EPSG:3035", "10 km")
>>> out = reproject(ds, "EPSG:4326", "0.1 deg",
... method={"pr": "average", "tas": "bilinear"})
Source code in climdata/grid/reproject.py
def reproject(
obj: Union[xr.Dataset, xr.DataArray],
target_projection: Any = None,
target_resolution: Any = None,
*,
like: Optional[Union[xr.Dataset, xr.DataArray]] = None,
method: Union[str, Mapping[str, str]] = "bilinear",
bounds: Optional[Sequence[float]] = None,
align: bool = True,
src_crs: Any = None,
x_dim: Optional[str] = None,
y_dim: Optional[str] = None,
nodata: Optional[float] = np.nan,
engine: str = "rasterio",
dsinfo_dims: Optional[Sequence[str]] = None,
) -> Union[xr.Dataset, xr.DataArray]:
"""Reproject and/or resample a dataset onto a new grid.
Args:
obj: The :class:`xarray.Dataset` or :class:`xarray.DataArray` to transform.
target_projection: Destination CRS -- ``"EPSG:4326"``, an integer EPSG code,
OGC WKT, a PROJ string, or a ``pyproj``/``rasterio`` CRS object. Defaults
to the source CRS (resample without reprojecting).
target_resolution: Destination cell size -- ``"10 km"``, ``"0.1 deg"``,
``"30 arcsec"``, a bare number (interpreted in the target CRS axis units),
or a 2-tuple for an anisotropic grid. Its units must match the target CRS:
a metric resolution with a geographic CRS raises
:class:`~climdata.grid.units.ResolutionCRSMismatch`.
like: A template whose grid the output should match exactly. Cannot be
combined with ``target_projection``/``target_resolution``.
method: Resampling method name, or a ``{variable: method}`` mapping. Use
``"bilinear"`` for state variables, ``"average"`` for fluxes,
``"nearest"`` for categorical fields.
bounds: Optional ``(left, bottom, right, top)`` target extent, expressed in
the units of the target CRS.
align: Snap the grid origin to whole multiples of the resolution so that
separately-processed domains stay co-registered. Default ``True``.
src_crs: Source CRS, when it cannot be inferred from the data.
x_dim: Source x/longitude dimension name.
y_dim: Source y/latitude dimension name.
nodata: Fill value written before warping. Keep the ``np.nan`` default for
float data; passing ``None`` lets rasterio fill gaps with zeros.
engine: ``"rasterio"`` (default) or ``"xesmf"`` for true area-weighted
conservative remapping.
dsinfo_dims: Optional ``(x_dim, y_dim)`` fallback, typically from
``cfg.dsinfo[DATASET]``.
Returns:
The reprojected object, of the same type as ``obj``, with coordinates named
``lon``/``lat`` for a geographic target and ``x``/``y`` for a projected one.
Raises:
ResolutionCRSMismatch: If the resolution units contradict the target CRS.
ValueError: If ``like`` is combined with an explicit target, or if inputs
cannot be interpreted.
Example:
>>> out = reproject(ds, "EPSG:3035", "10 km") # doctest: +SKIP
>>> out = reproject(ds, "EPSG:4326", "0.1 deg", # doctest: +SKIP
... method={"pr": "average", "tas": "bilinear"})
"""
if like is not None and (target_projection is not None or target_resolution is not None):
raise ValueError(
"`like` already fixes both the projection and the resolution; passing "
"target_projection or target_resolution alongside it is contradictory. "
"Drop one."
)
prepared, source_crs = _prepare(obj, src_crs, x_dim, y_dim, dsinfo_dims, nodata)
if like is not None:
template, _ = _prepare(like, None, None, None, None, nodata)
result = prepared.rio.reproject_match(
template, resampling=resampling_from_name(method if isinstance(method, str) else "bilinear")
)
return _rename_output(_restore_attrs(result, obj), infer_src_crs(template))
dst_crs = parse_crs(target_projection) if target_projection is not None else source_crs
# The gate runs on the *effective* target CRS, after defaulting: passing no
# projection at all must fail exactly as naming the source CRS would.
if target_resolution is not None:
try:
mid_lat = _mid_latitude(prepared, source_crs)
except Exception:
mid_lat = None
resolution = resolution_in_crs_units(target_resolution, dst_crs, latitude=mid_lat)
else:
resolution = None
_warn_if_mercator(dst_crs, prepared, source_crs)
if resolution is None:
result = _reproject_dispatch(prepared, dst_crs, None, None, None, method, nodata, engine)
else:
transform, width, height = _target_grid(
prepared, source_crs, dst_crs, resolution, bounds, align
)
result = _reproject_dispatch(
prepared, dst_crs, transform, width, height, method, nodata, engine
)
return _rename_output(_restore_attrs(result, obj), dst_crs)
climdata.grid.reproject.resampling_from_name(name)
¶
Look up a :class:rasterio.enums.Resampling member by name.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
name |
Any |
A method name such as |
required |
Returns:
| Type | Description |
|---|---|
rasterio.enums.Resampling |
The matching member. |
Exceptions:
| Type | Description |
|---|---|
ValueError |
If |
Source code in climdata/grid/reproject.py
def resampling_from_name(name: Any):
"""Look up a :class:`rasterio.enums.Resampling` member by name.
Args:
name: A method name such as ``"bilinear"`` or ``"average"``. A ``Resampling``
member passes through unchanged.
Returns:
rasterio.enums.Resampling: The matching member.
Raises:
ValueError: If ``name`` is not a valid resampling method.
"""
from rasterio.enums import Resampling
if isinstance(name, Resampling):
return name
try:
return Resampling[str(name).lower()]
except KeyError as err:
raise ValueError(
f"Unknown resampling method {name!r}. Valid options: "
f"{', '.join(Resampling.__members__)}."
) from err
Resolution parsing¶
climdata.grid.units.parse_resolution(value)
¶
Parse a resolution into canonical units.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
value |
Any |
A string ( |
required |
Returns:
| Type | Description |
|---|---|
Resolution |
Linear values in metres, angular values in degrees, bare numbers left alone. |
Exceptions:
| Type | Description |
|---|---|
ValueError |
If the string cannot be parsed, or if the two components of a tuple are of different kinds. |
TypeError |
If |
Examples:
>>> parse_resolution("10 km")
Resolution(x=10000.0, y=10000.0, kind='linear', unit='m')
>>> parse_resolution("0.1 deg").as_tuple()
(0.1, 0.1)
Source code in climdata/grid/units.py
def parse_resolution(value: Any) -> Resolution:
"""Parse a resolution into canonical units.
Args:
value: A string (``"10 km"``, ``"0.1 deg"``, ``"30 arcsec"``, ``"10km"``,
``"0.1°"``), a bare number, a pint ``Quantity``, a
:class:`Resolution`, or a 2-tuple ``(x, y)`` of any of those for an
anisotropic grid.
Returns:
Resolution: Linear values in metres, angular values in degrees, bare numbers
left alone.
Raises:
ValueError: If the string cannot be parsed, or if the two components of a
tuple are of different kinds.
TypeError: If ``value`` is of an unsupported type.
Example:
>>> parse_resolution("10 km")
Resolution(x=10000.0, y=10000.0, kind='linear', unit='m')
>>> parse_resolution("0.1 deg").as_tuple()
(0.1, 0.1)
"""
if isinstance(value, Resolution):
return value
if isinstance(value, (tuple, list)):
if len(value) != 2:
raise ValueError(
f"An anisotropic resolution needs exactly 2 components, got {len(value)}."
)
x_mag, x_kind = _parse_scalar(value[0])
y_mag, y_kind = _parse_scalar(value[1])
if x_kind != y_kind:
raise ValueError(
f"Mixed resolution kinds: {value[0]!r} is {x_kind} but {value[1]!r} "
f"is {y_kind}. Both axes must use the same kind of unit."
)
kind = x_kind
else:
x_mag, kind = _parse_scalar(value)
y_mag = x_mag
unit = {"linear": "m", "angular": "deg", "unitless": ""}[kind]
return Resolution(x=x_mag, y=y_mag, kind=kind, unit=unit)
climdata.grid.units.resolution_in_crs_units(value, crs, *, latitude=None)
¶
Express a resolution in the axis units of crs, or refuse to.
This is the single enforcement point for the projection/resolution compatibility
rules, so every entry point -- direct call, ClimateExtractor.reproject, a
Hydra override, an LLM tool call -- inherits the check. It runs before any data is
read, so a bad configuration fails immediately rather than after a download.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
value |
Any |
Anything :func: |
required |
crs |
Any |
Anything :func: |
required |
latitude |
Optional[float] |
Reference latitude used only to make the error message concrete. |
None |
Returns:
| Type | Description |
|---|---|
tuple[float, float] |
|
Exceptions:
| Type | Description |
|---|---|
ResolutionCRSMismatch |
For a linear resolution with a geographic CRS, or an angular resolution with a projected CRS. |
Examples:
>>> resolution_in_crs_units("10 km", "EPSG:3035")
(10000.0, 10000.0)
>>> resolution_in_crs_units("10 km", "EPSG:4326")
Traceback (most recent call last):
...
climdata.grid.units.ResolutionCRSMismatch: ...
Source code in climdata/grid/units.py
def resolution_in_crs_units(
value: Any, crs: Any, *, latitude: Optional[float] = None
) -> Tuple[float, float]:
"""Express a resolution in the axis units of ``crs``, or refuse to.
This is the single enforcement point for the projection/resolution compatibility
rules, so every entry point -- direct call, ``ClimateExtractor.reproject``, a
Hydra override, an LLM tool call -- inherits the check. It runs before any data is
read, so a bad configuration fails immediately rather than after a download.
Args:
value: Anything :func:`parse_resolution` accepts.
crs: Anything :func:`~climdata.grid.crs.parse_crs` accepts.
latitude: Reference latitude used only to make the error message concrete.
Returns:
tuple[float, float]: ``(x, y)`` in the axis units of ``crs``.
Raises:
ResolutionCRSMismatch: For a linear resolution with a geographic CRS, or an
angular resolution with a projected CRS.
Example:
>>> resolution_in_crs_units("10 km", "EPSG:3035")
(10000.0, 10000.0)
>>> resolution_in_crs_units("10 km", "EPSG:4326")
Traceback (most recent call last):
...
climdata.grid.units.ResolutionCRSMismatch: ...
"""
resolution = parse_resolution(value)
crs = parse_crs(crs)
crs_kind, factor = crs_axis_unit(crs)
if resolution.kind == "unitless":
return resolution.as_tuple() # adopt the CRS axis units verbatim
if resolution.kind != crs_kind:
raise ResolutionCRSMismatch(
_mismatch_message(resolution, crs, crs_kind, latitude, original=value)
)
if crs_kind == "linear":
# `factor` is metres per CRS unit (1.0 for metre, 0.3048006096 for US survey foot).
return (resolution.x / factor, resolution.y / factor)
# `factor` is radians per CRS unit; for a degree axis that is pi/180.
deg_per_unit = factor * _DEG_PER_RAD
return (resolution.x / deg_per_unit, resolution.y / deg_per_unit)
climdata.grid.units.to_angular(value, latitude)
¶
Convert a linear resolution to degrees at an explicit latitude.
The deliberate escape hatch from :class:ResolutionCRSMismatch, for a caller who
wants the approximation and is prepared to own it. latitude has no default, so
the approximation cannot be made by accident, and there is intentionally no config
flag that reaches this -- a YAML toggle is how such an approximation creeps back
into production runs unnoticed.
The result is exact at latitude and only there.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
value |
Any |
A linear resolution, e.g. |
required |
latitude |
float |
Reference latitude in degrees. Required. |
required |
Returns:
| Type | Description |
|---|---|
Resolution |
An angular resolution in degrees. |
Exceptions:
| Type | Description |
|---|---|
ValueError |
If |
Examples:
>>> res = to_angular("10 km", latitude=50.0)
>>> round(res.x, 4), round(res.y, 4)
(0.1395, 0.0899)
Source code in climdata/grid/units.py
def to_angular(value: Any, latitude: float) -> Resolution:
"""Convert a linear resolution to degrees at an explicit latitude.
The deliberate escape hatch from :class:`ResolutionCRSMismatch`, for a caller who
wants the approximation and is prepared to own it. ``latitude`` has no default, so
the approximation cannot be made by accident, and there is intentionally no config
flag that reaches this -- a YAML toggle is how such an approximation creeps back
into production runs unnoticed.
The result is exact at ``latitude`` and only there.
Args:
value: A linear resolution, e.g. ``"10 km"``.
latitude: Reference latitude in degrees. Required.
Returns:
Resolution: An angular resolution in degrees.
Raises:
ValueError: If ``value`` is not a linear resolution.
Example:
>>> res = to_angular("10 km", latitude=50.0)
>>> round(res.x, 4), round(res.y, 4)
(0.1395, 0.0899)
"""
resolution = parse_resolution(value)
if resolution.kind != "linear":
raise ValueError(
f"to_angular() converts a linear resolution to degrees, but {value!r} is "
f"{resolution.kind}."
)
d_lon, d_lat = _geod_deltas(resolution.x, resolution.y, float(latitude))
return Resolution(x=d_lon, y=d_lat, kind="angular", unit="deg")
climdata.grid.units.Resolution
dataclass
¶
A parsed grid resolution.
Attributes:
| Name | Type | Description |
|---|---|---|
x |
float |
Magnitude along the x/longitude axis, in :attr: |
y |
float |
Magnitude along the y/latitude axis, in :attr: |
kind |
str |
|
unit |
str |
Canonical unit -- |
Source code in climdata/grid/units.py
@dataclass(frozen=True)
class Resolution:
"""A parsed grid resolution.
Attributes:
x: Magnitude along the x/longitude axis, in :attr:`unit`.
y: Magnitude along the y/latitude axis, in :attr:`unit`.
kind: ``"linear"``, ``"angular"`` or ``"unitless"``.
unit: Canonical unit -- ``"m"`` for linear, ``"deg"`` for angular,
``""`` for unitless (adopts the target CRS axis unit).
"""
x: float
y: float
kind: str
unit: str
def as_tuple(self) -> Tuple[float, float]:
"""Return ``(x, y)`` magnitudes."""
return (self.x, self.y)
def __str__(self) -> str:
if self.x == self.y:
return f"{self.x:g} {self.unit}".strip()
return f"{self.x:g} x {self.y:g} {self.unit}".strip()
as_tuple(self)
¶
Return (x, y) magnitudes.
Source code in climdata/grid/units.py
def as_tuple(self) -> Tuple[float, float]:
"""Return ``(x, y)`` magnitudes."""
return (self.x, self.y)
climdata.grid.units.ResolutionCRSMismatch (ValueError)
¶
target_resolution units are incompatible with target_projection.
Subclasses :class:ValueError so existing except ValueError handlers --
including the LLM execution layer -- keep working.
Source code in climdata/grid/units.py
class ResolutionCRSMismatch(ValueError):
"""``target_resolution`` units are incompatible with ``target_projection``.
Subclasses :class:`ValueError` so existing ``except ValueError`` handlers --
including the LLM execution layer -- keep working.
"""
CRS handling¶
climdata.grid.crs.parse_crs(value)
¶
Coerce value to a :class:pyproj.CRS.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
value |
Any |
Anything :func: |
required |
Returns:
| Type | Description |
|---|---|
pyproj.CRS | None |
The parsed CRS. |
Exceptions:
| Type | Description |
|---|---|
ValueError |
If |
Source code in climdata/grid/crs.py
def parse_crs(value: Any):
"""Coerce ``value`` to a :class:`pyproj.CRS`.
Args:
value: Anything :func:`pyproj.CRS.from_user_input` understands --
``"EPSG:4326"``, ``4326``, an OGC WKT string, a PROJ string,
a CF ``grid_mapping`` dict, a ``rasterio.crs.CRS`` or a ``pyproj.CRS``.
``None`` passes through as ``None``.
Returns:
pyproj.CRS | None: The parsed CRS.
Raises:
ValueError: If ``value`` cannot be interpreted as a CRS.
"""
if value is None:
return None
import pyproj
if isinstance(value, pyproj.CRS):
return value
try:
return pyproj.CRS.from_user_input(value)
except Exception as err: # pyproj raises CRSError, a subclass of RuntimeError
raise ValueError(
f"Could not interpret {value!r} as a coordinate reference system. "
"Expected an EPSG code (e.g. 'EPSG:4326' or 4326), an OGC WKT string, "
"a PROJ string (e.g. '+proj=laea +lat_0=52 ...'), or a pyproj/rasterio "
"CRS object. Look codes up at https://epsg.io."
) from err
climdata.grid.crs.crs_axis_unit(crs)
¶
Return the axis unit kind and conversion factor of crs.
The kind is derived from :attr:pyproj.CRS.is_geographic -- never from the
position of an axis. EPSG:4326 lists latitude first, so positional indexing
silently mis-assigns anisotropic resolutions.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
crs |
Anything :func: |
required |
Returns:
| Type | Description |
|---|---|
tuple[str, float] |
|
Exceptions:
| Type | Description |
|---|---|
ValueError |
If the two horizontal axes carry different units, or if the CRS is neither geographic nor projected. |
Source code in climdata/grid/crs.py
def crs_axis_unit(crs) -> Tuple[str, float]:
"""Return the axis unit kind and conversion factor of ``crs``.
The kind is derived from :attr:`pyproj.CRS.is_geographic` -- never from the
position of an axis. EPSG:4326 lists *latitude* first, so positional indexing
silently mis-assigns anisotropic resolutions.
Args:
crs: Anything :func:`parse_crs` accepts.
Returns:
tuple[str, float]: ``("angular", radians_per_unit)`` for a geographic CRS
or ``("linear", metres_per_unit)`` for a projected one. For EPSG:4326 the
factor is ``pi/180``; for EPSG:3035 it is ``1.0``; for EPSG:2263
(US survey foot) it is ``0.3048006096``.
Raises:
ValueError: If the two horizontal axes carry different units, or if the CRS
is neither geographic nor projected.
"""
crs = _horizontal(parse_crs(crs))
axes = crs.axis_info[:2]
if len(axes) < 2:
raise ValueError(f"CRS {crs.name!r} does not expose two horizontal axes.")
if axes[0].unit_name != axes[1].unit_name:
raise ValueError(
f"CRS {crs.name!r} has mixed horizontal axis units "
f"({axes[0].unit_name!r} and {axes[1].unit_name!r}); a single "
"resolution cannot describe such a grid."
)
factor = float(axes[0].unit_conversion_factor)
if crs.is_geographic:
return "angular", factor
if crs.is_projected:
return "linear", factor
raise ValueError(
f"CRS {crs.name!r} is neither geographic nor projected, so it has no "
"well-defined grid resolution."
)
climdata.grid.crs.infer_spatial_dims(obj, x_dim=None, y_dim=None, dsinfo_dims=None)
¶
Infer the horizontal dimension names of obj.
Resolution order: explicit arguments, then whatever rioxarray already knows,
then a table of conventional names, then dsinfo_dims.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
obj |
An :class: |
required | |
x_dim |
Optional[str] |
Explicit x/longitude dimension name. |
None |
y_dim |
Optional[str] |
Explicit y/latitude dimension name. |
None |
dsinfo_dims |
Optional[Sequence[str]] |
Optional |
None |
Returns:
| Type | Description |
|---|---|
tuple[str, str] |
|
Exceptions:
| Type | Description |
|---|---|
ValueError |
If the dimensions cannot be determined. |
Source code in climdata/grid/crs.py
def infer_spatial_dims(
obj,
x_dim: Optional[str] = None,
y_dim: Optional[str] = None,
dsinfo_dims: Optional[Sequence[str]] = None,
) -> Tuple[str, str]:
"""Infer the horizontal dimension names of ``obj``.
Resolution order: explicit arguments, then whatever ``rioxarray`` already knows,
then a table of conventional names, then ``dsinfo_dims``.
Args:
obj: An :class:`xarray.Dataset` or :class:`xarray.DataArray`.
x_dim: Explicit x/longitude dimension name.
y_dim: Explicit y/latitude dimension name.
dsinfo_dims: Optional ``(x_dim, y_dim)`` fallback, typically read from
``cfg.dsinfo[DATASET].lon_dim`` / ``.lat_dim``.
Returns:
tuple[str, str]: ``(x_dim, y_dim)``.
Raises:
ValueError: If the dimensions cannot be determined.
"""
if x_dim and y_dim:
return x_dim, y_dim
try: # rioxarray may already have them set
import rioxarray # noqa: F401 — registers the .rio accessor
return obj.rio.x_dim, obj.rio.y_dim
except Exception:
pass
dims = set(obj.dims) | set(obj.coords)
found_x = x_dim or next((n for n in _X_NAMES if n in dims), None)
found_y = y_dim or next((n for n in _Y_NAMES if n in dims), None)
if (not found_x or not found_y) and dsinfo_dims:
cand_x, cand_y = dsinfo_dims
found_x = found_x or (cand_x if cand_x in dims else None)
found_y = found_y or (cand_y if cand_y in dims else None)
if not found_x or not found_y:
raise ValueError(
f"Could not infer spatial dimensions from {sorted(map(str, obj.dims))}. "
"Pass x_dim= and y_dim= explicitly."
)
return found_x, found_y
climdata.grid.crs.infer_src_crs(obj, default='EPSG:4326')
¶
Infer the CRS of obj, following the conventions found in real files.
Order: an existing rio.crs; dataset-level attributes; a CF grid_mapping
variable, when it survived subsetting; CRS attributes carried on the data variables
themselves (real HYRAS keeps its WKT in esri_pe_string there, and the crs
variable its grid_mapping points at is dropped); a spatial_ref coordinate;
then a geographic heuristic on the coordinate ranges.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
obj |
An :class: |
required | |
default |
Any |
CRS assumed when the coordinates look geographic. Pass |
'EPSG:4326' |
Returns:
| Type | Description |
|---|---|
pyproj.CRS |
The inferred CRS. |
Exceptions:
| Type | Description |
|---|---|
ValueError |
If no CRS can be determined. |
Source code in climdata/grid/crs.py
def infer_src_crs(obj, default: Any = "EPSG:4326"):
"""Infer the CRS of ``obj``, following the conventions found in real files.
Order: an existing ``rio.crs``; dataset-level attributes; a CF ``grid_mapping``
variable, when it survived subsetting; CRS attributes carried on the data variables
themselves (real HYRAS keeps its WKT in ``esri_pe_string`` there, and the ``crs``
variable its ``grid_mapping`` points at is dropped); a ``spatial_ref`` coordinate;
then a geographic heuristic on the coordinate ranges.
Args:
obj: An :class:`xarray.Dataset` or :class:`xarray.DataArray`.
default: CRS assumed when the coordinates look geographic. Pass ``None``
to make that case raise instead.
Returns:
pyproj.CRS: The inferred CRS.
Raises:
ValueError: If no CRS can be determined.
"""
try:
import rioxarray # noqa: F401
if obj.rio.crs is not None:
return parse_crs(obj.rio.crs)
except Exception:
pass
found = _crs_from_attrs(obj.attrs)
if found is not None:
return found
# A CF grid_mapping pointer, when the variable it names still exists.
variables = getattr(obj, "variables", {})
candidates = list(getattr(obj, "data_vars", {}).values()) or [obj]
for var in candidates:
target = var.attrs.get("grid_mapping")
if target and target in variables:
found = _crs_from_attrs(variables[target].attrs)
if found is not None:
return found
# CRS attributes carried directly on the data variables.
for var in candidates:
found = _crs_from_attrs(var.attrs)
if found is not None:
return found
if "spatial_ref" in getattr(obj, "coords", {}):
found = _crs_from_attrs(obj.coords["spatial_ref"].attrs)
if found is not None:
return found
if default is not None:
try:
x_dim, y_dim = infer_spatial_dims(obj)
x, y = obj[x_dim], obj[y_dim]
if float(abs(y).max()) <= 90.0 and float(abs(x).max()) <= 360.0:
warnings.warn(
f"No CRS found on the dataset; assuming {default} because the "
f"{y_dim!r}/{x_dim!r} coordinates fall in geographic ranges. "
"Pass src_crs= to be explicit.",
UserWarning,
stacklevel=2,
)
return parse_crs(default)
except (ValueError, KeyError):
pass
raise ValueError(
"Could not determine the source CRS of the dataset. Set it with "
"ds.rio.write_crs(...) or pass src_crs= explicitly."
)
climdata.grid.crs.normalize_longitude(obj, x_dim)
¶
Roll a 0-360 longitude axis onto -180-180 and re-sort.
CMIP and NEX-GDDP data arrive on 0-360. rasterio will happily warp such an
axis without raising and produce nonsense, so this runs unconditionally for
geographic sources whose longitudes exceed 180.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
obj |
An :class: |
required | |
x_dim |
str |
Name of the longitude dimension. |
required |
Returns:
| Type | Description |
|---|---|
The object with longitudes in -180..180, sorted ascending. Returned unchanged when no rolling is needed. |
Source code in climdata/grid/crs.py
def normalize_longitude(obj, x_dim: str):
"""Roll a 0-360 longitude axis onto -180-180 and re-sort.
CMIP and NEX-GDDP data arrive on 0-360. ``rasterio`` will happily warp such an
axis without raising and produce nonsense, so this runs unconditionally for
geographic sources whose longitudes exceed 180.
Args:
obj: An :class:`xarray.Dataset` or :class:`xarray.DataArray`.
x_dim: Name of the longitude dimension.
Returns:
The object with longitudes in -180..180, sorted ascending. Returned
unchanged when no rolling is needed.
"""
if x_dim not in obj.coords:
return obj
x = obj[x_dim]
if float(x.max()) <= 180.0:
return obj
rolled = ((x + 180.0) % 360.0) - 180.0
return obj.assign_coords({x_dim: rolled}).sortby(x_dim)
Bias correction and downscaling¶
The ISIMIP3BASD method, wrapped for xarray. See the BCSD guide.
climdata.sdba.bcsd.BCSD
¶
Complete Bias Correction and Statistical Downscaling (BCSD) workflow.
Combines BiasCorrection and StatisticalDownscaling in a single pipeline following the ISIMIP3BASD methodology.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
variable |
str |
Climate variable name. |
required |
bias_correction_kwargs |
dict |
Forwarded to BiasCorrection. |
None |
downscaling_kwargs |
dict |
Forwarded to StatisticalDownscaling. |
None |
regridding_method |
str |
xESMF method for obs_fine→coarse regridding (default 'conservative'). |
'conservative' |
regridding_tool |
str |
'xesmf' (default) or 'cdo'. |
'xesmf' |
cdo_method |
str |
CDO operator when regridding_tool='cdo'. |
'remapcon' |
cdo_env |
str |
Conda environment with CDO. |
'cdo_stable' |
weights_dir |
str |
Directory to cache regridding weights. |
None |
Examples:
>>> bcsd = BCSD(variable='tas',
... bias_correction_kwargs={'detrend': True},
... downscaling_kwargs={'n_iterations': 20})
>>> paths = bcsd.run(obs_fine=obs_025deg,
... sim_hist_coarse=gcm_hist_1deg,
... sim_fut_coarse=gcm_fut_1deg)
Source code in climdata/sdba/bcsd.py
class BCSD:
"""Complete Bias Correction and Statistical Downscaling (BCSD) workflow.
Combines BiasCorrection and StatisticalDownscaling in a single pipeline
following the ISIMIP3BASD methodology.
Args:
variable (str): Climate variable name.
bias_correction_kwargs (dict, optional): Forwarded to BiasCorrection.
downscaling_kwargs (dict, optional): Forwarded to StatisticalDownscaling.
regridding_method (str): xESMF method for obs_fine→coarse regridding (default
'conservative').
regridding_tool (str): 'xesmf' (default) or 'cdo'.
cdo_method (str): CDO operator when regridding_tool='cdo'.
cdo_env (str): Conda environment with CDO.
weights_dir (str, optional): Directory to cache regridding weights.
Example:
>>> bcsd = BCSD(variable='tas',
... bias_correction_kwargs={'detrend': True},
... downscaling_kwargs={'n_iterations': 20})
>>> paths = bcsd.run(obs_fine=obs_025deg,
... sim_hist_coarse=gcm_hist_1deg,
... sim_fut_coarse=gcm_fut_1deg)
"""
def __init__(
self,
variable: str,
bias_correction_kwargs: Optional[Dict] = None,
downscaling_kwargs: Optional[Dict] = None,
regridding_method: str = 'conservative',
regridding_tool: str = 'xesmf',
cdo_method: str = 'remapcon',
cdo_env: str = 'cdo_stable',
weights_dir: Optional[str] = None
):
"""Configure the two stages of a BCSD run.
Every argument is documented on the class. The stage objects are built
here, so an invalid ``bias_correction_kwargs`` or ``downscaling_kwargs``
is rejected now rather than partway through :meth:`run`.
Raises:
ImportError: If ``iris`` is not installed.
"""
self.variable = variable
self.regridding_method = regridding_method
self.regridding_tool = regridding_tool
self.cdo_method = cdo_method
self.cdo_env = cdo_env
self.weights_dir = weights_dir
self.bias_correction = BiasCorrection(variable=variable, **(bias_correction_kwargs or {}))
self.downscaling = StatisticalDownscaling(variable=variable, **(downscaling_kwargs or {}))
print(f"\n{'='*60}")
print(f"BCSD Pipeline initialized for {variable}")
print(f"Regridding: {regridding_tool} ({regridding_method})")
print(f"{'='*60}")
def run(
self,
obs_fine: xr.Dataset,
sim_hist_coarse: xr.Dataset,
sim_fut_coarse: xr.Dataset,
obs_hist_coarse: Optional[xr.Dataset] = None,
output_path: Optional[Union[str, Path]] = None,
save_intermediate: bool = False
) -> list:
"""Run the complete BCSD workflow.
Three steps, of which the first is conditional:
0. If ``obs_hist_coarse`` is not supplied, regrid ``obs_fine`` onto the
simulation's coarse grid by conservative remapping.
1. Bias-correct the coarse simulation against the coarse observations
using ISIMIP3BASD.
2. Downscale the corrected coarse field onto the fine grid, again with
ISIMIP3BASD, writing one NetCDF per year.
Args:
obs_fine (xr.Dataset): Historical observations at fine resolution.
sim_hist_coarse (xr.Dataset): Historical GCM simulations at coarse
resolution.
sim_fut_coarse (xr.Dataset): Future GCM simulations at coarse resolution.
obs_hist_coarse (xr.Dataset, optional): Historical observations already at
coarse resolution. When None, derived automatically via conservative
regridding.
output_path (str or Path, optional): Directory for per-year downscaled
NetCDF files.
save_intermediate (bool): Save intermediate bias-corrected output alongside
final results.
Returns:
list[Path]: Path of the downscaled NetCDF written for each year.
"""
print(f"\n{'='*60}\nStarting BCSD Workflow\n{'='*60}\n")
# Step 0: Derive coarse observations if not supplied
if obs_hist_coarse is None:
print("STEP 0: Regridding obs_fine → coarse resolution")
print("-" * 60)
obs_hist_coarse = regrid_to_coarse(
obs_fine, sim_hist_coarse,
method=self.regridding_method,
regridding_tool=self.regridding_tool,
cdo_method=self.cdo_method,
cdo_env=self.cdo_env,
weights_dir=self.weights_dir,
)
if save_intermediate and output_path:
p = Path(output_path) / f"{self.variable}_obs_coarse.nc"
obs_hist_coarse.to_netcdf(p)
print(f" Saved coarse obs to: {p}")
else:
print("Using provided coarse observations.\n")
# Step 1: Bias correction
print("\nSTEP 1: BIAS CORRECTION\n" + "-" * 60)
bc_output = None
if save_intermediate and output_path:
bc_output = Path(output_path) / f"{self.variable}_bias_corrected.nc"
sim_fut_corrected = self.bias_correction.correct(
obs_hist=obs_hist_coarse,
sim_hist=sim_hist_coarse,
sim_fut=sim_fut_coarse,
output_path=bc_output,
)
# Step 2: Statistical downscaling
print("\nSTEP 2: STATISTICAL DOWNSCALING\n" + "-" * 60)
result = self.downscaling.downscale(
obs_fine=obs_fine,
sim_coarse=sim_fut_corrected,
output_path=output_path,
)
print(f"\n{'='*60}\nBCSD Workflow Complete!\n{'='*60}\n")
return result
__init__(self, variable, bias_correction_kwargs=None, downscaling_kwargs=None, regridding_method='conservative', regridding_tool='xesmf', cdo_method='remapcon', cdo_env='cdo_stable', weights_dir=None)
special
¶
Configure the two stages of a BCSD run.
Every argument is documented on the class. The stage objects are built
here, so an invalid bias_correction_kwargs or downscaling_kwargs
is rejected now rather than partway through :meth:run.
Exceptions:
| Type | Description |
|---|---|
ImportError |
If |
Source code in climdata/sdba/bcsd.py
def __init__(
self,
variable: str,
bias_correction_kwargs: Optional[Dict] = None,
downscaling_kwargs: Optional[Dict] = None,
regridding_method: str = 'conservative',
regridding_tool: str = 'xesmf',
cdo_method: str = 'remapcon',
cdo_env: str = 'cdo_stable',
weights_dir: Optional[str] = None
):
"""Configure the two stages of a BCSD run.
Every argument is documented on the class. The stage objects are built
here, so an invalid ``bias_correction_kwargs`` or ``downscaling_kwargs``
is rejected now rather than partway through :meth:`run`.
Raises:
ImportError: If ``iris`` is not installed.
"""
self.variable = variable
self.regridding_method = regridding_method
self.regridding_tool = regridding_tool
self.cdo_method = cdo_method
self.cdo_env = cdo_env
self.weights_dir = weights_dir
self.bias_correction = BiasCorrection(variable=variable, **(bias_correction_kwargs or {}))
self.downscaling = StatisticalDownscaling(variable=variable, **(downscaling_kwargs or {}))
print(f"\n{'='*60}")
print(f"BCSD Pipeline initialized for {variable}")
print(f"Regridding: {regridding_tool} ({regridding_method})")
print(f"{'='*60}")
run(self, obs_fine, sim_hist_coarse, sim_fut_coarse, obs_hist_coarse=None, output_path=None, save_intermediate=False)
¶
Run the complete BCSD workflow.
Three steps, of which the first is conditional:
- If
obs_hist_coarseis not supplied, regridobs_fineonto the simulation's coarse grid by conservative remapping. - Bias-correct the coarse simulation against the coarse observations using ISIMIP3BASD.
- Downscale the corrected coarse field onto the fine grid, again with ISIMIP3BASD, writing one NetCDF per year.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
obs_fine |
xr.Dataset |
Historical observations at fine resolution. |
required |
sim_hist_coarse |
xr.Dataset |
Historical GCM simulations at coarse resolution. |
required |
sim_fut_coarse |
xr.Dataset |
Future GCM simulations at coarse resolution. |
required |
obs_hist_coarse |
xr.Dataset |
Historical observations already at coarse resolution. When None, derived automatically via conservative regridding. |
None |
output_path |
str or Path |
Directory for per-year downscaled NetCDF files. |
None |
save_intermediate |
bool |
Save intermediate bias-corrected output alongside final results. |
False |
Returns:
| Type | Description |
|---|---|
list[Path] |
Path of the downscaled NetCDF written for each year. |
Source code in climdata/sdba/bcsd.py
def run(
self,
obs_fine: xr.Dataset,
sim_hist_coarse: xr.Dataset,
sim_fut_coarse: xr.Dataset,
obs_hist_coarse: Optional[xr.Dataset] = None,
output_path: Optional[Union[str, Path]] = None,
save_intermediate: bool = False
) -> list:
"""Run the complete BCSD workflow.
Three steps, of which the first is conditional:
0. If ``obs_hist_coarse`` is not supplied, regrid ``obs_fine`` onto the
simulation's coarse grid by conservative remapping.
1. Bias-correct the coarse simulation against the coarse observations
using ISIMIP3BASD.
2. Downscale the corrected coarse field onto the fine grid, again with
ISIMIP3BASD, writing one NetCDF per year.
Args:
obs_fine (xr.Dataset): Historical observations at fine resolution.
sim_hist_coarse (xr.Dataset): Historical GCM simulations at coarse
resolution.
sim_fut_coarse (xr.Dataset): Future GCM simulations at coarse resolution.
obs_hist_coarse (xr.Dataset, optional): Historical observations already at
coarse resolution. When None, derived automatically via conservative
regridding.
output_path (str or Path, optional): Directory for per-year downscaled
NetCDF files.
save_intermediate (bool): Save intermediate bias-corrected output alongside
final results.
Returns:
list[Path]: Path of the downscaled NetCDF written for each year.
"""
print(f"\n{'='*60}\nStarting BCSD Workflow\n{'='*60}\n")
# Step 0: Derive coarse observations if not supplied
if obs_hist_coarse is None:
print("STEP 0: Regridding obs_fine → coarse resolution")
print("-" * 60)
obs_hist_coarse = regrid_to_coarse(
obs_fine, sim_hist_coarse,
method=self.regridding_method,
regridding_tool=self.regridding_tool,
cdo_method=self.cdo_method,
cdo_env=self.cdo_env,
weights_dir=self.weights_dir,
)
if save_intermediate and output_path:
p = Path(output_path) / f"{self.variable}_obs_coarse.nc"
obs_hist_coarse.to_netcdf(p)
print(f" Saved coarse obs to: {p}")
else:
print("Using provided coarse observations.\n")
# Step 1: Bias correction
print("\nSTEP 1: BIAS CORRECTION\n" + "-" * 60)
bc_output = None
if save_intermediate and output_path:
bc_output = Path(output_path) / f"{self.variable}_bias_corrected.nc"
sim_fut_corrected = self.bias_correction.correct(
obs_hist=obs_hist_coarse,
sim_hist=sim_hist_coarse,
sim_fut=sim_fut_coarse,
output_path=bc_output,
)
# Step 2: Statistical downscaling
print("\nSTEP 2: STATISTICAL DOWNSCALING\n" + "-" * 60)
result = self.downscaling.downscale(
obs_fine=obs_fine,
sim_coarse=sim_fut_corrected,
output_path=output_path,
)
print(f"\n{'='*60}\nBCSD Workflow Complete!\n{'='*60}\n")
return result
climdata.sdba.bcsd.BiasCorrection
¶
Bias correction using ISIMIP3BASD trend-preserving quantile mapping.
Applies bias correction at the coarse GCM resolution before any spatial downscaling. Supports single-variable and multivariate (MBCn) correction.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
variable |
str or list of str |
Climate variable name(s). When a list is given MBCn is used. |
required |
method |
str |
'parametric' (default). |
'parametric' |
distribution |
str |
Parametric distribution override. Defaults come from DEFAULT_CONFIGS. |
None |
trend_preservation |
str |
Override for trend preservation ('additive', 'multiplicative', 'mixed', 'bounded'). |
None |
detrend |
bool |
Detrend before correction. |
False |
adjust_p_values |
bool |
Adjust p-values for perfect match in the reference period. |
False |
lower_bound, |
lower_threshold, upper_bound, upper_threshold (float |
Physical bounds / thresholds. |
required |
halfwin_upper_bound_climatology |
int |
Half-window for upper bound climatology (0 = disabled). |
0 |
n_processes |
int |
Parallel worker count. Defaults to all CPUs. |
None |
n_quantiles |
int |
Quantiles for non-parametric QM (default 50). |
50 |
n_iterations |
int |
MBCn iterations (default 20; 0 = univariate, no MBCn). |
20 |
randomization_seed |
int |
RNG seed for reproducibility. |
None |
Examples:
>>> bc = BiasCorrection(variable='tas', distribution='normal',
... trend_preservation='additive', detrend=True)
>>> tas_corrected = bc.correct(obs_hist=obs_ds, sim_hist=gcm_hist_ds,
... sim_fut=gcm_fut_ds)
Source code in climdata/sdba/bcsd.py
class BiasCorrection:
"""Bias correction using ISIMIP3BASD trend-preserving quantile mapping.
Applies bias correction at the coarse GCM resolution before any spatial
downscaling. Supports single-variable and multivariate (MBCn) correction.
Args:
variable (str or list of str): Climate variable name(s). When a list is given
MBCn is used.
method (str): 'parametric' (default).
distribution (str, optional): Parametric distribution override. Defaults come
from DEFAULT_CONFIGS.
trend_preservation (str, optional): Override for trend preservation ('additive',
'multiplicative', 'mixed', 'bounded').
detrend (bool): Detrend before correction.
adjust_p_values (bool): Adjust p-values for perfect match in the reference
period.
lower_bound, lower_threshold, upper_bound, upper_threshold (float, optional):
Physical bounds / thresholds.
halfwin_upper_bound_climatology (int): Half-window for upper bound climatology
(0 = disabled).
n_processes (int, optional): Parallel worker count. Defaults to all CPUs.
n_quantiles (int): Quantiles for non-parametric QM (default 50).
n_iterations (int): MBCn iterations (default 20; 0 = univariate, no MBCn).
randomization_seed (int, optional): RNG seed for reproducibility.
Example:
>>> bc = BiasCorrection(variable='tas', distribution='normal',
... trend_preservation='additive', detrend=True)
>>> tas_corrected = bc.correct(obs_hist=obs_ds, sim_hist=gcm_hist_ds,
... sim_fut=gcm_fut_ds)
"""
DEFAULT_CONFIGS = {
'tas': {
'distribution': 'normal', 'trend_preservation': 'additive',
'detrend': True, 'adjust_p_values': False,
},
'tasmax': {
'distribution': 'normal', 'trend_preservation': 'additive',
'detrend': True, 'adjust_p_values': False,
},
'tasmin': {
'distribution': 'normal', 'trend_preservation': 'additive',
'detrend': True, 'adjust_p_values': False,
},
'pr': {
'distribution': 'gamma', 'trend_preservation': 'mixed',
'lower_bound': 0, 'lower_threshold': 0.1, 'adjust_p_values': True,
},
'rsds': {
'distribution': 'beta', 'trend_preservation': 'bounded',
'lower_bound': 0, 'lower_threshold': 0.01,
'upper_bound': 1, 'upper_threshold': 0.9999,
'adjust_p_values': True, 'halfwin_upper_bound_climatology': 15,
},
'hurs': {
'distribution': 'beta', 'trend_preservation': 'bounded',
'lower_bound': 0, 'lower_threshold': 0.01,
'upper_bound': 100, 'upper_threshold': 99.99,
'adjust_p_values': True,
},
'sfcWind': {
'distribution': 'weibull', 'trend_preservation': 'mixed',
'lower_bound': 0, 'lower_threshold': 0.01, 'adjust_p_values': True,
},
'psl': {
'distribution': 'normal', 'trend_preservation': 'additive',
'adjust_p_values': True, 'detrend': True,
},
'rlds': {
'distribution': 'normal', 'trend_preservation': 'additive',
'adjust_p_values': True, 'detrend': True,
},
}
def __init__(
self,
variable: Union[str, List[str]],
method: str = 'parametric',
distribution: Optional[str] = None,
trend_preservation: Optional[str] = None,
detrend: bool = False,
adjust_p_values: bool = False,
lower_bound: Optional[float] = None,
lower_threshold: Optional[float] = None,
upper_bound: Optional[float] = None,
upper_threshold: Optional[float] = None,
halfwin_upper_bound_climatology: int = 0,
n_processes: Optional[int] = None,
n_quantiles: int = 50,
n_iterations: int = 20,
randomization_seed: Optional[int] = None,
**kwargs
):
"""Configure a bias-correction run.
Every argument is documented on the class. Where an argument is left as
``None``, the per-variable entry in :attr:`DEFAULT_CONFIGS` supplies the
value, so a bare ``BiasCorrection("pr")`` already uses the distribution,
bounds and trend-preservation mode appropriate to precipitation.
Raises:
ImportError: If ``iris`` is not installed.
"""
self.variables = [variable] if isinstance(variable, str) else list(variable)
self.variable = self.variables[0] if len(self.variables) == 1 else None
self.method = method
self.n_processes = n_processes if n_processes is not None else os.cpu_count()
self.n_quantiles = n_quantiles
self.n_iterations = n_iterations
self.randomization_seed = randomization_seed
self.kwargs = kwargs
self._var_configs = []
for v in self.variables:
cfg = self.DEFAULT_CONFIGS.get(v, {})
self._var_configs.append({
'distribution': distribution if distribution is not None else cfg.get('distribution'),
'trend_preservation': trend_preservation if trend_preservation is not None else cfg.get('trend_preservation', 'additive'),
'detrend': detrend if detrend is not False else cfg.get('detrend', False),
'adjust_p_values': adjust_p_values if adjust_p_values is not False else cfg.get('adjust_p_values', False),
'lower_bound': lower_bound if lower_bound is not None else cfg.get('lower_bound'),
'lower_threshold': lower_threshold if lower_threshold is not None else cfg.get('lower_threshold'),
'upper_bound': upper_bound if upper_bound is not None else cfg.get('upper_bound'),
'upper_threshold': upper_threshold if upper_threshold is not None else cfg.get('upper_threshold'),
'halfwin_upper_bound_climatology': halfwin_upper_bound_climatology if halfwin_upper_bound_climatology != 0
else cfg.get('halfwin_upper_bound_climatology', 0),
})
# Scalar attrs for single-variable backward compatibility
if len(self.variables) == 1:
c = self._var_configs[0]
self.distribution = c['distribution']
self.trend_preservation = c['trend_preservation']
self.detrend = c['detrend']
self.adjust_p_values = c['adjust_p_values']
self.lower_bound = c['lower_bound']
self.lower_threshold = c['lower_threshold']
self.upper_bound = c['upper_bound']
self.upper_threshold = c['upper_threshold']
self.halfwin_upper_bound_climatology = c['halfwin_upper_bound_climatology']
mv_label = ('(MBCn multivariate)' if self.n_iterations > 0 and len(self.variables) > 1
else '(MBCn univariate)' if self.n_iterations > 0
else '(univariate, no MBCn)')
print(f"BiasCorrection initialized for {self.variables}")
print(f" n_iterations={self.n_iterations} {mv_label} n_processes={self.n_processes} seed={self.randomization_seed}")
for v, c in zip(self.variables, self._var_configs):
print(f" [{v}] dist={c['distribution']} trend={c['trend_preservation']} "
f"detrend={c['detrend']} adjust_p={c['adjust_p_values']}")
# ------------------------------------------------------------------ helpers
def _to_iris_cubes(self, obs_hist, sim_hist, sim_fut):
"""Convert three xarray Datasets to per-variable iris cube lists."""
obs_cubes, sh_cubes, sf_cubes = [], [], []
for var in self.variables:
for ds, out in [(obs_hist, obs_cubes), (sim_hist, sh_cubes), (sim_fut, sf_cubes)]:
out.append(_xda_to_cube(ds[var]))
return obs_cubes, sh_cubes, sf_cubes
def _ba_vendor_kwargs(self) -> dict:
"""Build per-variable kwarg lists consumed by ba.adjust_bias."""
return {
'distribution': [c['distribution'] for c in self._var_configs],
'trend_preservation': [c['trend_preservation'] for c in self._var_configs],
'detrend': [c['detrend'] for c in self._var_configs],
'adjust_p_values': [c['adjust_p_values'] for c in self._var_configs],
'halfwin_upper_bound_climatology': [c['halfwin_upper_bound_climatology'] for c in self._var_configs],
'lower_bound': [c['lower_bound'] for c in self._var_configs],
'lower_threshold': [c['lower_threshold'] for c in self._var_configs],
'upper_bound': [c['upper_bound'] for c in self._var_configs],
'upper_threshold': [c['upper_threshold'] for c in self._var_configs],
'if_all_invalid_use': [np.nan for _ in self._var_configs],
'unconditional_ccs_transfer': [False for _ in self._var_configs],
'trendless_bound_frequency': [False for _ in self._var_configs],
}
# --------------------------------------------------- low-level hooks for
# custom per-location loops (e.g. nexgddp_hyras_bc_zarr.py)
def _prepare_ba_state(self, obs_cubes, sh_cubes, sf_cubes, step_size=0):
"""
Set up global vendor BA state needed by adjust_bias_one_location.
Returns (lazy_data, month_numbers, years, doys, space_shape, rotation_matrices).
Call ba.initializer internally, then iterate locations with correct_one_location.
"""
from climdata._vendor.isimip3basd import bias_adjustment as ba
from climdata._vendor.isimip3basd import utility_functions as uf
detrend = [c['detrend'] for c in self._var_configs]
halfwin = [c['halfwin_upper_bound_climatology'] for c in self._var_configs]
cubes_map = {'obs_hist': obs_cubes, 'sim_hist': sh_cubes, 'sim_fut': sf_cubes}
lazy_data = {}
space_shape = None
month_numbers = {k: None for k in cubes_map}
years = {k: None for k in cubes_map}
doys = {k: None for k in cubes_map}
for key, cube_list in cubes_map.items():
lazy_data[key] = [c.core_data() for c in cube_list]
for i, cube in enumerate(cube_list):
if space_shape is None:
space_shape = cube.shape[1:]
uf.assert_calendar(cube, 'proleptic_gregorian')
uf.assert_coord_axis(cube, 'time', 0)
datetimes = cube.coord('time').units.num2date(cube.coord('time').points)
if not step_size:
j = uf.convert_datetimes(datetimes, 'month_number')
if month_numbers[key] is None:
month_numbers[key] = j
if step_size or detrend[i]:
j = uf.convert_datetimes(datetimes, 'year')
if years[key] is None:
years[key] = j
if step_size or halfwin[i]:
j = uf.convert_datetimes(datetimes, 'day_of_year')
if doys[key] is None:
doys[key] = j
if step_size:
uf.assert_full_period_coverage(years[key], doys[key], key)
if self.randomization_seed is not None:
np.random.seed(self.randomization_seed)
rotation_matrices = [
uf.generateCREmatrix(len(self.variables))
for _ in range(self.n_iterations)
]
ba.initializer(lazy_data, month_numbers, years, doys)
return lazy_data, month_numbers, years, doys, space_shape, rotation_matrices
def correct_one_location(self, i_loc, space_shape, npy_paths, rotation_matrices=None, step_size=0):
"""
Bias-correct one grid cell via ba.adjust_bias_one_location.
Requires _prepare_ba_state to have been called first (sets vendor globals).
"""
from climdata._vendor.isimip3basd import bias_adjustment as ba
vkw = self._ba_vendor_kwargs()
ba.adjust_bias_one_location(
i_loc=i_loc,
sim_fut_ba_path=[str(p) for p in npy_paths],
space_shape=space_shape,
halfwin_upper_bound_climatology=vkw.pop('halfwin_upper_bound_climatology'),
lower_bound=vkw.pop('lower_bound'),
lower_threshold=vkw.pop('lower_threshold'),
upper_bound=vkw.pop('upper_bound'),
upper_threshold=vkw.pop('upper_threshold'),
if_all_invalid_use=vkw.pop('if_all_invalid_use'),
step_size=step_size,
rotation_matrices=rotation_matrices or [],
randomization_seed=self.randomization_seed,
n_quantiles=self.n_quantiles,
**vkw,
)
# ------------------------------------------------------------------ public
def correct(
self,
obs_hist: xr.Dataset,
sim_hist: xr.Dataset,
sim_fut: xr.Dataset,
output_path: Optional[Union[str, Path]] = None,
**kwargs,
) -> xr.Dataset:
"""Apply bias correction to climate model data.
Args:
obs_hist (xr.Dataset): Historical observations (e.g. 1980–2014).
sim_hist (xr.Dataset): Historical model simulations matching the obs_hist
period.
sim_fut (xr.Dataset): Future model simulations to be corrected.
output_path (str or Path, optional): Save the merged result to this NetCDF
path.
Returns:
xr.Dataset: Bias-corrected future simulations.
"""
if not IRIS_AVAILABLE:
raise ImportError("iris is required: pip install scitools-iris")
try:
from climdata._vendor.isimip3basd import bias_adjustment as ba
from climdata._vendor.isimip3basd import utility_functions as uf
except ImportError:
raise ImportError(
"ISIMIP3BASD vendor code not found in climdata/_vendor/isimip3basd/."
)
mv_tag = (f"{len(self.variables)} variables (MBCn multivariate)"
if len(self.variables) > 1 else self.variables[0])
print(f"\nBias correction: {mv_tag}")
print(f" obs_hist : {obs_hist.time.values[0]} → {obs_hist.time.values[-1]}")
print(f" sim_hist : {sim_hist.time.values[0]} → {sim_hist.time.values[-1]}")
print(f" sim_fut : {sim_fut.time.values[0]} → {sim_fut.time.values[-1]}")
tmpdir = Path.cwd() / "tmp_bcsd"
tmpdir.mkdir(parents=True, exist_ok=True)
try:
print(" Converting to iris cubes...")
obs_cubes, sh_cubes, sf_cubes = self._to_iris_cubes(obs_hist, sim_hist, sim_fut)
# Resolve one output path per variable
if output_path is not None and len(self.variables) == 1:
ba_paths = [Path(output_path).resolve()]
else:
ba_paths = [
(Path(output_path).parent / f"{Path(output_path).stem}_{v}{Path(output_path).suffix}").resolve()
if output_path else (tmpdir / f"sim_fut_ba_{v}.nc").resolve()
for v in self.variables
]
vkw = self._ba_vendor_kwargs()
print(f" Running ba.adjust_bias (n_iterations={self.n_iterations}) ...")
ba.adjust_bias(
obs_hist=obs_cubes,
sim_hist=sh_cubes,
sim_fut=sf_cubes,
sim_fut_ba_path=[str(p) for p in ba_paths],
n_processes=self.n_processes,
n_quantiles=self.n_quantiles,
n_iterations=self.n_iterations,
randomization_seed=self.randomization_seed,
detrend=vkw.pop('detrend'),
halfwin_upper_bound_climatology=vkw.pop('halfwin_upper_bound_climatology'),
**vkw, **kwargs, **self.kwargs,
)
print(" Collecting results from npy_stack...")
result_datasets = []
for var, ba_path, sf_cube in zip(self.variables, ba_paths, sf_cubes):
npy_dir = uf.npy_stack_dir(str(ba_path))
# Assemble a contiguous array from the per-location npy stack. Each
# {flat}.npy holds one location's (n_time, 1) series; flat is the
# row-major index over the spatial dims. This replaces
# da.from_npy_stack(...).reshape(...), whose lazy graph yielded
# all-NaN with this dask/iris stack. Building the xarray EAGERLY
# from the numpy array (no iris.save -> lazy load_cube roundtrip)
# also avoids returning data bound to the temp file that the
# `finally` block below deletes.
arr = np.full(sf_cube.shape, np.nan, dtype=np.float32)
flat_view = arr.reshape(sf_cube.shape[0], -1)
for flat in range(flat_view.shape[1]):
fp = f"{npy_dir}{flat}.npy"
if os.path.exists(fp):
flat_view[:, flat] = np.load(fp).squeeze()
out_cube = sf_cube.copy()
out_cube.data = np.ma.masked_invalid(arr)
result_datasets.append(
xr.DataArray.from_iris(out_cube).to_dataset(name=var))
result = xr.merge(result_datasets)
result.attrs['bias_correction_method'] = (
'ISIMIP3BASD-MBCn' if self.n_iterations > 0 else 'ISIMIP3BASD')
result.attrs['n_iterations'] = self.n_iterations
result.attrs['variables'] = str(self.variables)
if output_path is not None:
result.to_netcdf(output_path)
print(f" Saved to: {output_path}")
finally:
shutil.rmtree(tmpdir, ignore_errors=True)
return result
__init__(self, variable, method='parametric', distribution=None, trend_preservation=None, detrend=False, adjust_p_values=False, lower_bound=None, lower_threshold=None, upper_bound=None, upper_threshold=None, halfwin_upper_bound_climatology=0, n_processes=None, n_quantiles=50, n_iterations=20, randomization_seed=None, **kwargs)
special
¶
Configure a bias-correction run.
Every argument is documented on the class. Where an argument is left as
None, the per-variable entry in :attr:DEFAULT_CONFIGS supplies the
value, so a bare BiasCorrection("pr") already uses the distribution,
bounds and trend-preservation mode appropriate to precipitation.
Exceptions:
| Type | Description |
|---|---|
ImportError |
If |
Source code in climdata/sdba/bcsd.py
def __init__(
self,
variable: Union[str, List[str]],
method: str = 'parametric',
distribution: Optional[str] = None,
trend_preservation: Optional[str] = None,
detrend: bool = False,
adjust_p_values: bool = False,
lower_bound: Optional[float] = None,
lower_threshold: Optional[float] = None,
upper_bound: Optional[float] = None,
upper_threshold: Optional[float] = None,
halfwin_upper_bound_climatology: int = 0,
n_processes: Optional[int] = None,
n_quantiles: int = 50,
n_iterations: int = 20,
randomization_seed: Optional[int] = None,
**kwargs
):
"""Configure a bias-correction run.
Every argument is documented on the class. Where an argument is left as
``None``, the per-variable entry in :attr:`DEFAULT_CONFIGS` supplies the
value, so a bare ``BiasCorrection("pr")`` already uses the distribution,
bounds and trend-preservation mode appropriate to precipitation.
Raises:
ImportError: If ``iris`` is not installed.
"""
self.variables = [variable] if isinstance(variable, str) else list(variable)
self.variable = self.variables[0] if len(self.variables) == 1 else None
self.method = method
self.n_processes = n_processes if n_processes is not None else os.cpu_count()
self.n_quantiles = n_quantiles
self.n_iterations = n_iterations
self.randomization_seed = randomization_seed
self.kwargs = kwargs
self._var_configs = []
for v in self.variables:
cfg = self.DEFAULT_CONFIGS.get(v, {})
self._var_configs.append({
'distribution': distribution if distribution is not None else cfg.get('distribution'),
'trend_preservation': trend_preservation if trend_preservation is not None else cfg.get('trend_preservation', 'additive'),
'detrend': detrend if detrend is not False else cfg.get('detrend', False),
'adjust_p_values': adjust_p_values if adjust_p_values is not False else cfg.get('adjust_p_values', False),
'lower_bound': lower_bound if lower_bound is not None else cfg.get('lower_bound'),
'lower_threshold': lower_threshold if lower_threshold is not None else cfg.get('lower_threshold'),
'upper_bound': upper_bound if upper_bound is not None else cfg.get('upper_bound'),
'upper_threshold': upper_threshold if upper_threshold is not None else cfg.get('upper_threshold'),
'halfwin_upper_bound_climatology': halfwin_upper_bound_climatology if halfwin_upper_bound_climatology != 0
else cfg.get('halfwin_upper_bound_climatology', 0),
})
# Scalar attrs for single-variable backward compatibility
if len(self.variables) == 1:
c = self._var_configs[0]
self.distribution = c['distribution']
self.trend_preservation = c['trend_preservation']
self.detrend = c['detrend']
self.adjust_p_values = c['adjust_p_values']
self.lower_bound = c['lower_bound']
self.lower_threshold = c['lower_threshold']
self.upper_bound = c['upper_bound']
self.upper_threshold = c['upper_threshold']
self.halfwin_upper_bound_climatology = c['halfwin_upper_bound_climatology']
mv_label = ('(MBCn multivariate)' if self.n_iterations > 0 and len(self.variables) > 1
else '(MBCn univariate)' if self.n_iterations > 0
else '(univariate, no MBCn)')
print(f"BiasCorrection initialized for {self.variables}")
print(f" n_iterations={self.n_iterations} {mv_label} n_processes={self.n_processes} seed={self.randomization_seed}")
for v, c in zip(self.variables, self._var_configs):
print(f" [{v}] dist={c['distribution']} trend={c['trend_preservation']} "
f"detrend={c['detrend']} adjust_p={c['adjust_p_values']}")
correct(self, obs_hist, sim_hist, sim_fut, output_path=None, **kwargs)
¶
Apply bias correction to climate model data.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
obs_hist |
xr.Dataset |
Historical observations (e.g. 1980–2014). |
required |
sim_hist |
xr.Dataset |
Historical model simulations matching the obs_hist period. |
required |
sim_fut |
xr.Dataset |
Future model simulations to be corrected. |
required |
output_path |
str or Path |
Save the merged result to this NetCDF path. |
None |
Returns:
| Type | Description |
|---|---|
xr.Dataset |
Bias-corrected future simulations. |
Source code in climdata/sdba/bcsd.py
def correct(
self,
obs_hist: xr.Dataset,
sim_hist: xr.Dataset,
sim_fut: xr.Dataset,
output_path: Optional[Union[str, Path]] = None,
**kwargs,
) -> xr.Dataset:
"""Apply bias correction to climate model data.
Args:
obs_hist (xr.Dataset): Historical observations (e.g. 1980–2014).
sim_hist (xr.Dataset): Historical model simulations matching the obs_hist
period.
sim_fut (xr.Dataset): Future model simulations to be corrected.
output_path (str or Path, optional): Save the merged result to this NetCDF
path.
Returns:
xr.Dataset: Bias-corrected future simulations.
"""
if not IRIS_AVAILABLE:
raise ImportError("iris is required: pip install scitools-iris")
try:
from climdata._vendor.isimip3basd import bias_adjustment as ba
from climdata._vendor.isimip3basd import utility_functions as uf
except ImportError:
raise ImportError(
"ISIMIP3BASD vendor code not found in climdata/_vendor/isimip3basd/."
)
mv_tag = (f"{len(self.variables)} variables (MBCn multivariate)"
if len(self.variables) > 1 else self.variables[0])
print(f"\nBias correction: {mv_tag}")
print(f" obs_hist : {obs_hist.time.values[0]} → {obs_hist.time.values[-1]}")
print(f" sim_hist : {sim_hist.time.values[0]} → {sim_hist.time.values[-1]}")
print(f" sim_fut : {sim_fut.time.values[0]} → {sim_fut.time.values[-1]}")
tmpdir = Path.cwd() / "tmp_bcsd"
tmpdir.mkdir(parents=True, exist_ok=True)
try:
print(" Converting to iris cubes...")
obs_cubes, sh_cubes, sf_cubes = self._to_iris_cubes(obs_hist, sim_hist, sim_fut)
# Resolve one output path per variable
if output_path is not None and len(self.variables) == 1:
ba_paths = [Path(output_path).resolve()]
else:
ba_paths = [
(Path(output_path).parent / f"{Path(output_path).stem}_{v}{Path(output_path).suffix}").resolve()
if output_path else (tmpdir / f"sim_fut_ba_{v}.nc").resolve()
for v in self.variables
]
vkw = self._ba_vendor_kwargs()
print(f" Running ba.adjust_bias (n_iterations={self.n_iterations}) ...")
ba.adjust_bias(
obs_hist=obs_cubes,
sim_hist=sh_cubes,
sim_fut=sf_cubes,
sim_fut_ba_path=[str(p) for p in ba_paths],
n_processes=self.n_processes,
n_quantiles=self.n_quantiles,
n_iterations=self.n_iterations,
randomization_seed=self.randomization_seed,
detrend=vkw.pop('detrend'),
halfwin_upper_bound_climatology=vkw.pop('halfwin_upper_bound_climatology'),
**vkw, **kwargs, **self.kwargs,
)
print(" Collecting results from npy_stack...")
result_datasets = []
for var, ba_path, sf_cube in zip(self.variables, ba_paths, sf_cubes):
npy_dir = uf.npy_stack_dir(str(ba_path))
# Assemble a contiguous array from the per-location npy stack. Each
# {flat}.npy holds one location's (n_time, 1) series; flat is the
# row-major index over the spatial dims. This replaces
# da.from_npy_stack(...).reshape(...), whose lazy graph yielded
# all-NaN with this dask/iris stack. Building the xarray EAGERLY
# from the numpy array (no iris.save -> lazy load_cube roundtrip)
# also avoids returning data bound to the temp file that the
# `finally` block below deletes.
arr = np.full(sf_cube.shape, np.nan, dtype=np.float32)
flat_view = arr.reshape(sf_cube.shape[0], -1)
for flat in range(flat_view.shape[1]):
fp = f"{npy_dir}{flat}.npy"
if os.path.exists(fp):
flat_view[:, flat] = np.load(fp).squeeze()
out_cube = sf_cube.copy()
out_cube.data = np.ma.masked_invalid(arr)
result_datasets.append(
xr.DataArray.from_iris(out_cube).to_dataset(name=var))
result = xr.merge(result_datasets)
result.attrs['bias_correction_method'] = (
'ISIMIP3BASD-MBCn' if self.n_iterations > 0 else 'ISIMIP3BASD')
result.attrs['n_iterations'] = self.n_iterations
result.attrs['variables'] = str(self.variables)
if output_path is not None:
result.to_netcdf(output_path)
print(f" Saved to: {output_path}")
finally:
shutil.rmtree(tmpdir, ignore_errors=True)
return result
correct_one_location(self, i_loc, space_shape, npy_paths, rotation_matrices=None, step_size=0)
¶
Bias-correct one grid cell via ba.adjust_bias_one_location.
Requires _prepare_ba_state to have been called first (sets vendor globals).
Source code in climdata/sdba/bcsd.py
def correct_one_location(self, i_loc, space_shape, npy_paths, rotation_matrices=None, step_size=0):
"""
Bias-correct one grid cell via ba.adjust_bias_one_location.
Requires _prepare_ba_state to have been called first (sets vendor globals).
"""
from climdata._vendor.isimip3basd import bias_adjustment as ba
vkw = self._ba_vendor_kwargs()
ba.adjust_bias_one_location(
i_loc=i_loc,
sim_fut_ba_path=[str(p) for p in npy_paths],
space_shape=space_shape,
halfwin_upper_bound_climatology=vkw.pop('halfwin_upper_bound_climatology'),
lower_bound=vkw.pop('lower_bound'),
lower_threshold=vkw.pop('lower_threshold'),
upper_bound=vkw.pop('upper_bound'),
upper_threshold=vkw.pop('upper_threshold'),
if_all_invalid_use=vkw.pop('if_all_invalid_use'),
step_size=step_size,
rotation_matrices=rotation_matrices or [],
randomization_seed=self.randomization_seed,
n_quantiles=self.n_quantiles,
**vkw,
)
climdata.sdba.bcsd.StatisticalDownscaling
¶
Statistical downscaling using the ISIMIP3BASD modified MBCn algorithm.
Downscales bias-corrected coarse-resolution data to the fine resolution of the reference observations, processing one year at a time to bound memory use. Already-completed years are skipped automatically.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
variable |
str |
Climate variable name. |
required |
n_processes |
int |
Parallel worker count. |
None |
n_iterations |
int |
MBCn iterations (default 20). |
20 |
randomization_seed |
int |
RNG seed. |
None |
lower_bound, |
lower_threshold, upper_bound, upper_threshold (float |
Physical bounds / thresholds (defaults from BiasCorrection.DEFAULT_CONFIGS). |
required |
Examples:
>>> sd = StatisticalDownscaling(variable='tas', n_iterations=20)
>>> paths = sd.downscale(obs_fine=obs_fine_ds, sim_coarse=gcm_corrected_ds,
... output_path='./downscaled/', model='ACCESS', scenario='ssp585')
Source code in climdata/sdba/bcsd.py
class StatisticalDownscaling:
"""Statistical downscaling using the ISIMIP3BASD modified MBCn algorithm.
Downscales bias-corrected coarse-resolution data to the fine resolution
of the reference observations, processing one year at a time to bound
memory use. Already-completed years are skipped automatically.
Args:
variable (str): Climate variable name.
n_processes (int, optional): Parallel worker count.
n_iterations (int): MBCn iterations (default 20).
randomization_seed (int, optional): RNG seed.
lower_bound, lower_threshold, upper_bound, upper_threshold (float, optional):
Physical bounds / thresholds (defaults from BiasCorrection.DEFAULT_CONFIGS).
Example:
>>> sd = StatisticalDownscaling(variable='tas', n_iterations=20)
>>> paths = sd.downscale(obs_fine=obs_fine_ds, sim_coarse=gcm_corrected_ds,
... output_path='./downscaled/', model='ACCESS', scenario='ssp585')
"""
DEFAULT_CONFIGS = BiasCorrection.DEFAULT_CONFIGS
def __init__(
self,
variable: str,
n_processes: Optional[int] = None,
n_iterations: int = 20,
randomization_seed: Optional[int] = None,
lower_bound: Optional[float] = None,
lower_threshold: Optional[float] = None,
upper_bound: Optional[float] = None,
upper_threshold: Optional[float] = None,
**kwargs
):
"""Configure a downscaling run.
Every argument is documented on the class. Arguments left as ``None``
fall back to the variable's entry in :attr:`DEFAULT_CONFIGS`.
Raises:
ImportError: If ``iris`` is not installed.
"""
self.variable = variable
self.n_processes = n_processes if n_processes is not None else os.cpu_count()
self.n_iterations = n_iterations
self.randomization_seed = randomization_seed
self.kwargs = kwargs
cfg = self.DEFAULT_CONFIGS.get(variable, {})
self.lower_bound = lower_bound if lower_bound is not None else cfg.get('lower_bound')
self.lower_threshold = lower_threshold if lower_threshold is not None else cfg.get('lower_threshold')
self.upper_bound = upper_bound if upper_bound is not None else cfg.get('upper_bound')
self.upper_threshold = upper_threshold if upper_threshold is not None else cfg.get('upper_threshold')
print(f"StatisticalDownscaling initialized for {variable}")
print(f" n_iterations={n_iterations} n_processes={self.n_processes} seed={self.randomization_seed}")
def downscale(
self,
obs_fine: xr.Dataset,
sim_coarse: xr.Dataset,
output_path: Optional[Union[str, Path]] = None,
model: Optional[str] = None,
scenario: Optional[str] = None,
**kwargs
) -> list:
"""Apply statistical downscaling to coarse-resolution data.
Args:
obs_fine (xr.Dataset or DataArray): Fine-resolution observations.
sim_coarse (xr.Dataset or DataArray): Coarse-resolution bias-corrected
simulations.
output_path (str or Path, optional): Directory for per-year output NetCDF
files.
model (str, optional): Model tag used in output filenames.
scenario (str, optional): Scenario tag used in output filenames.
Returns:
list of Path: Per-year NetCDF paths
(BCSD_{var}_{model}_{scenario}_{year}.nc).
"""
if not IRIS_AVAILABLE:
raise ImportError("iris is required: pip install scitools-iris")
try:
from climdata._vendor.isimip3basd import statistical_downscaling as sd
from climdata._vendor.isimip3basd import utility_functions as uf
except ImportError:
raise ImportError("ISIMIP3BASD vendor code not found in climdata/_vendor/isimip3basd/.")
import dask.array as da
if isinstance(obs_fine, xr.DataArray):
obs_fine = obs_fine.to_dataset(name=self.variable)
if isinstance(sim_coarse, xr.DataArray):
sim_coarse = sim_coarse.to_dataset(name=self.variable)
print(f"\nStatistical downscaling: {self.variable}")
tmpdir = Path.cwd() / "tmp_bcsd_downscale"
tmpdir.mkdir(parents=True, exist_ok=True)
try:
# Convert to iris cubes
obs_fine_cube = _xda_to_cube(obs_fine[self.variable])
sim_coarse_cube = _xda_to_cube(sim_coarse[self.variable])
_stamp_geogcs(obs_fine_cube)
_stamp_geogcs(sim_coarse_cube)
obs_fine_cube = _ensure_rectilinear_iris_cube(obs_fine_cube)
print(f" obs_fine {obs_fine_cube.shape} sim_coarse {sim_coarse_cube.shape}")
# Crop obs_fine to integer multiple of the coarse grid
lat_name = 'lat' if 'lat' in sim_coarse.coords else 'latitude'
lon_name = 'lon' if 'lon' in sim_coarse.coords else 'longitude'
nlat_c, nlon_c = sim_coarse.sizes[lat_name], sim_coarse.sizes[lon_name]
nlat_f, nlon_f = obs_fine_cube.shape[1], obs_fine_cube.shape[2]
nlat_crop = (nlat_f // nlat_c) * nlat_c
nlon_crop = (nlon_f // nlon_c) * nlon_c
if nlat_crop != nlat_f or nlon_crop != nlon_f:
r = (nlat_f - nlat_crop) // 2
c = (nlon_f - nlon_crop) // 2
obs_fine_cube = obs_fine_cube[:, r:r + nlat_crop, c:c + nlon_crop]
print(f" obs_fine cropped to ({nlat_crop}×{nlon_crop}) "
f"[was {nlat_f}×{nlon_f}, coarse {nlat_c}×{nlon_c}]")
# Output directory and filename template
out_dir = Path(output_path).resolve() if output_path else tmpdir
out_dir.mkdir(parents=True, exist_ok=True)
_model_tag = model or "model"
_scenario_tag = scenario or "scenario"
# Determine year indices for the year-by-year loop
tc = sim_coarse_cube.coord("time")
years = np.array([d.year for d in tc.units.num2date(tc.points)])
unique_years = np.unique(years)
print(f" Processing {len(unique_years)} years: {unique_years[0]}–{unique_years[-1]}")
try:
from tqdm import tqdm as _tqdm
yr_iter = _tqdm(unique_years, desc=" Downscaling", unit="yr", dynamic_ncols=True)
except Exception:
yr_iter = unique_years
yearly_result_paths = []
for yr in yr_iter:
idx = np.where(years == yr)[0]
yr_path = (out_dir / f"BCSD_{self.variable}_{_model_tag}_{_scenario_tag}_{yr}.nc").resolve()
if yr_path.exists():
yearly_result_paths.append(yr_path)
continue
# Per-year npy_stack
yr_npy_root = tmpdir / f"npy_{yr}"
yr_npy_root.mkdir(parents=True, exist_ok=True)
yr_npy_sentinel = str(yr_npy_root / "sim_fine.nc")
uf.setup_npy_stack(yr_npy_sentinel, (len(idx),) + obs_fine_cube.shape[1:])
# Slice and prepare per-year cubes
sc_yr = _ensure_rectilinear_iris_cube(sim_coarse_cube[idx])
# Vendor uf.remapbil bilinearly interpolates coarse → fine grid.
# iris Linear regrid realizes the data to a numpy MaskedArray, but
# the vendor sd.downscale calls .compute() on every cube's
# core_data(), so re-wrap as a dask masked array (as _xda_to_cube does).
remap_yr = uf.remapbil(sc_yr, obs_fine_cube)
remap_yr.data = da.ma.masked_invalid(remap_yr.core_data())
_stamp_geogcs(remap_yr)
t0 = __import__("time").time()
sd.downscale(
obs_fine=obs_fine_cube,
sim_coarse=sc_yr,
sim_coarse_remapbil=remap_yr,
sim_fine_path=yr_npy_sentinel,
n_processes=self.n_processes,
n_iterations=self.n_iterations,
randomization_seed=self.randomization_seed,
lower_bound=self.lower_bound,
lower_threshold=self.lower_threshold,
upper_bound=self.upper_bound,
upper_threshold=self.upper_threshold,
**kwargs, **self.kwargs,
)
elapsed = __import__("time").time() - t0
# npy_stack → contiguous array → yearly NetCDF. Assemble each
# {flat}.npy (one fine location's (n_time, 1) series; flat is the
# row-major spatial index) into a contiguous numpy array. This
# replaces da.from_npy_stack(...).reshape(...) fed straight to
# iris.save, which produced thousands of tiny HDF5 chunks (~1 GB,
# very slow). A contiguous array writes a compact ~MB file quickly.
sim_fine_yr = obs_fine_cube[:len(idx)].copy()
npy_dir = uf.npy_stack_dir(yr_npy_sentinel)
arr = np.full(sim_fine_yr.shape, np.nan, dtype=np.float32)
flat_view = arr.reshape(sim_fine_yr.shape[0], -1)
for flat in range(flat_view.shape[1]):
fp = f"{npy_dir}{flat}.npy"
if os.path.exists(fp):
flat_view[:, flat] = np.load(fp).squeeze()
sim_fine_yr.data = np.ma.masked_invalid(arr)
# Output carries the (application) sim_coarse times, not obs_fine's.
_t = sim_fine_yr.coord("time")
_t.points, _t.units = sc_yr.coord("time").points, sc_yr.coord("time").units
iris.save(sim_fine_yr, str(yr_path),
saver=iris.fileformats.netcdf.save,
unlimited_dimensions=["time"], zlib=True, complevel=1)
yearly_result_paths.append(yr_path)
shutil.rmtree(str(yr_npy_root), ignore_errors=True)
print(f" {yr} → {yr_path.name} ({elapsed:.1f}s)")
print(" Statistical downscaling complete.")
finally:
shutil.rmtree(tmpdir, ignore_errors=True)
return yearly_result_paths
__init__(self, variable, n_processes=None, n_iterations=20, randomization_seed=None, lower_bound=None, lower_threshold=None, upper_bound=None, upper_threshold=None, **kwargs)
special
¶
Configure a downscaling run.
Every argument is documented on the class. Arguments left as None
fall back to the variable's entry in :attr:DEFAULT_CONFIGS.
Exceptions:
| Type | Description |
|---|---|
ImportError |
If |
Source code in climdata/sdba/bcsd.py
def __init__(
self,
variable: str,
n_processes: Optional[int] = None,
n_iterations: int = 20,
randomization_seed: Optional[int] = None,
lower_bound: Optional[float] = None,
lower_threshold: Optional[float] = None,
upper_bound: Optional[float] = None,
upper_threshold: Optional[float] = None,
**kwargs
):
"""Configure a downscaling run.
Every argument is documented on the class. Arguments left as ``None``
fall back to the variable's entry in :attr:`DEFAULT_CONFIGS`.
Raises:
ImportError: If ``iris`` is not installed.
"""
self.variable = variable
self.n_processes = n_processes if n_processes is not None else os.cpu_count()
self.n_iterations = n_iterations
self.randomization_seed = randomization_seed
self.kwargs = kwargs
cfg = self.DEFAULT_CONFIGS.get(variable, {})
self.lower_bound = lower_bound if lower_bound is not None else cfg.get('lower_bound')
self.lower_threshold = lower_threshold if lower_threshold is not None else cfg.get('lower_threshold')
self.upper_bound = upper_bound if upper_bound is not None else cfg.get('upper_bound')
self.upper_threshold = upper_threshold if upper_threshold is not None else cfg.get('upper_threshold')
print(f"StatisticalDownscaling initialized for {variable}")
print(f" n_iterations={n_iterations} n_processes={self.n_processes} seed={self.randomization_seed}")
downscale(self, obs_fine, sim_coarse, output_path=None, model=None, scenario=None, **kwargs)
¶
Apply statistical downscaling to coarse-resolution data.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
obs_fine |
xr.Dataset or DataArray |
Fine-resolution observations. |
required |
sim_coarse |
xr.Dataset or DataArray |
Coarse-resolution bias-corrected simulations. |
required |
output_path |
str or Path |
Directory for per-year output NetCDF files. |
None |
model |
str |
Model tag used in output filenames. |
None |
scenario |
str |
Scenario tag used in output filenames. |
None |
Returns:
| Type | Description |
|---|---|
list of Path |
Per-year NetCDF paths (BCSD_{var}{model}.nc).}_{year |
Source code in climdata/sdba/bcsd.py
def downscale(
self,
obs_fine: xr.Dataset,
sim_coarse: xr.Dataset,
output_path: Optional[Union[str, Path]] = None,
model: Optional[str] = None,
scenario: Optional[str] = None,
**kwargs
) -> list:
"""Apply statistical downscaling to coarse-resolution data.
Args:
obs_fine (xr.Dataset or DataArray): Fine-resolution observations.
sim_coarse (xr.Dataset or DataArray): Coarse-resolution bias-corrected
simulations.
output_path (str or Path, optional): Directory for per-year output NetCDF
files.
model (str, optional): Model tag used in output filenames.
scenario (str, optional): Scenario tag used in output filenames.
Returns:
list of Path: Per-year NetCDF paths
(BCSD_{var}_{model}_{scenario}_{year}.nc).
"""
if not IRIS_AVAILABLE:
raise ImportError("iris is required: pip install scitools-iris")
try:
from climdata._vendor.isimip3basd import statistical_downscaling as sd
from climdata._vendor.isimip3basd import utility_functions as uf
except ImportError:
raise ImportError("ISIMIP3BASD vendor code not found in climdata/_vendor/isimip3basd/.")
import dask.array as da
if isinstance(obs_fine, xr.DataArray):
obs_fine = obs_fine.to_dataset(name=self.variable)
if isinstance(sim_coarse, xr.DataArray):
sim_coarse = sim_coarse.to_dataset(name=self.variable)
print(f"\nStatistical downscaling: {self.variable}")
tmpdir = Path.cwd() / "tmp_bcsd_downscale"
tmpdir.mkdir(parents=True, exist_ok=True)
try:
# Convert to iris cubes
obs_fine_cube = _xda_to_cube(obs_fine[self.variable])
sim_coarse_cube = _xda_to_cube(sim_coarse[self.variable])
_stamp_geogcs(obs_fine_cube)
_stamp_geogcs(sim_coarse_cube)
obs_fine_cube = _ensure_rectilinear_iris_cube(obs_fine_cube)
print(f" obs_fine {obs_fine_cube.shape} sim_coarse {sim_coarse_cube.shape}")
# Crop obs_fine to integer multiple of the coarse grid
lat_name = 'lat' if 'lat' in sim_coarse.coords else 'latitude'
lon_name = 'lon' if 'lon' in sim_coarse.coords else 'longitude'
nlat_c, nlon_c = sim_coarse.sizes[lat_name], sim_coarse.sizes[lon_name]
nlat_f, nlon_f = obs_fine_cube.shape[1], obs_fine_cube.shape[2]
nlat_crop = (nlat_f // nlat_c) * nlat_c
nlon_crop = (nlon_f // nlon_c) * nlon_c
if nlat_crop != nlat_f or nlon_crop != nlon_f:
r = (nlat_f - nlat_crop) // 2
c = (nlon_f - nlon_crop) // 2
obs_fine_cube = obs_fine_cube[:, r:r + nlat_crop, c:c + nlon_crop]
print(f" obs_fine cropped to ({nlat_crop}×{nlon_crop}) "
f"[was {nlat_f}×{nlon_f}, coarse {nlat_c}×{nlon_c}]")
# Output directory and filename template
out_dir = Path(output_path).resolve() if output_path else tmpdir
out_dir.mkdir(parents=True, exist_ok=True)
_model_tag = model or "model"
_scenario_tag = scenario or "scenario"
# Determine year indices for the year-by-year loop
tc = sim_coarse_cube.coord("time")
years = np.array([d.year for d in tc.units.num2date(tc.points)])
unique_years = np.unique(years)
print(f" Processing {len(unique_years)} years: {unique_years[0]}–{unique_years[-1]}")
try:
from tqdm import tqdm as _tqdm
yr_iter = _tqdm(unique_years, desc=" Downscaling", unit="yr", dynamic_ncols=True)
except Exception:
yr_iter = unique_years
yearly_result_paths = []
for yr in yr_iter:
idx = np.where(years == yr)[0]
yr_path = (out_dir / f"BCSD_{self.variable}_{_model_tag}_{_scenario_tag}_{yr}.nc").resolve()
if yr_path.exists():
yearly_result_paths.append(yr_path)
continue
# Per-year npy_stack
yr_npy_root = tmpdir / f"npy_{yr}"
yr_npy_root.mkdir(parents=True, exist_ok=True)
yr_npy_sentinel = str(yr_npy_root / "sim_fine.nc")
uf.setup_npy_stack(yr_npy_sentinel, (len(idx),) + obs_fine_cube.shape[1:])
# Slice and prepare per-year cubes
sc_yr = _ensure_rectilinear_iris_cube(sim_coarse_cube[idx])
# Vendor uf.remapbil bilinearly interpolates coarse → fine grid.
# iris Linear regrid realizes the data to a numpy MaskedArray, but
# the vendor sd.downscale calls .compute() on every cube's
# core_data(), so re-wrap as a dask masked array (as _xda_to_cube does).
remap_yr = uf.remapbil(sc_yr, obs_fine_cube)
remap_yr.data = da.ma.masked_invalid(remap_yr.core_data())
_stamp_geogcs(remap_yr)
t0 = __import__("time").time()
sd.downscale(
obs_fine=obs_fine_cube,
sim_coarse=sc_yr,
sim_coarse_remapbil=remap_yr,
sim_fine_path=yr_npy_sentinel,
n_processes=self.n_processes,
n_iterations=self.n_iterations,
randomization_seed=self.randomization_seed,
lower_bound=self.lower_bound,
lower_threshold=self.lower_threshold,
upper_bound=self.upper_bound,
upper_threshold=self.upper_threshold,
**kwargs, **self.kwargs,
)
elapsed = __import__("time").time() - t0
# npy_stack → contiguous array → yearly NetCDF. Assemble each
# {flat}.npy (one fine location's (n_time, 1) series; flat is the
# row-major spatial index) into a contiguous numpy array. This
# replaces da.from_npy_stack(...).reshape(...) fed straight to
# iris.save, which produced thousands of tiny HDF5 chunks (~1 GB,
# very slow). A contiguous array writes a compact ~MB file quickly.
sim_fine_yr = obs_fine_cube[:len(idx)].copy()
npy_dir = uf.npy_stack_dir(yr_npy_sentinel)
arr = np.full(sim_fine_yr.shape, np.nan, dtype=np.float32)
flat_view = arr.reshape(sim_fine_yr.shape[0], -1)
for flat in range(flat_view.shape[1]):
fp = f"{npy_dir}{flat}.npy"
if os.path.exists(fp):
flat_view[:, flat] = np.load(fp).squeeze()
sim_fine_yr.data = np.ma.masked_invalid(arr)
# Output carries the (application) sim_coarse times, not obs_fine's.
_t = sim_fine_yr.coord("time")
_t.points, _t.units = sc_yr.coord("time").points, sc_yr.coord("time").units
iris.save(sim_fine_yr, str(yr_path),
saver=iris.fileformats.netcdf.save,
unlimited_dimensions=["time"], zlib=True, complevel=1)
yearly_result_paths.append(yr_path)
shutil.rmtree(str(yr_npy_root), ignore_errors=True)
print(f" {yr} → {yr_path.name} ({elapsed:.1f}s)")
print(" Statistical downscaling complete.")
finally:
shutil.rmtree(tmpdir, ignore_errors=True)
return yearly_result_paths
climdata.sdba.bcsd.regrid_to_coarse(fine_data, coarse_template, method='conservative', regridding_tool='xesmf', cdo_method='remapcon', cdo_env='cdo_stable', weights_dir=None)
¶
Regrid fine-resolution data to coarse resolution.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
fine_data |
xr.Dataset |
Fine-resolution dataset to regrid. |
required |
coarse_template |
xr.Dataset |
Coarse-resolution dataset used as the target grid. |
required |
method |
str |
xESMF method: 'conservative' (default), 'bilinear', or 'nearest'. |
'conservative' |
regridding_tool |
str |
'xesmf' (default) or 'cdo'. |
'xesmf' |
cdo_method |
str |
CDO operator string when regridding_tool='cdo'. |
'remapcon' |
cdo_env |
str |
Conda environment that has CDO installed. |
'cdo_stable' |
weights_dir |
str |
Directory to cache regridding weight files. |
None |
Returns:
| Type | Description |
|---|---|
xr.Dataset |
Regridded dataset on the coarse grid. |
Source code in climdata/sdba/bcsd.py
def regrid_to_coarse(
fine_data: xr.Dataset,
coarse_template: xr.Dataset,
method: str = 'conservative',
regridding_tool: str = 'xesmf',
cdo_method: str = 'remapcon',
cdo_env: str = 'cdo_stable',
weights_dir: Optional[str] = None
) -> xr.Dataset:
"""Regrid fine-resolution data to coarse resolution.
Args:
fine_data (xr.Dataset): Fine-resolution dataset to regrid.
coarse_template (xr.Dataset): Coarse-resolution dataset used as the target grid.
method (str): xESMF method: 'conservative' (default), 'bilinear', or 'nearest'.
regridding_tool (str): 'xesmf' (default) or 'cdo'.
cdo_method (str): CDO operator string when regridding_tool='cdo'.
cdo_env (str): Conda environment that has CDO installed.
weights_dir (str, optional): Directory to cache regridding weight files.
Returns:
xr.Dataset: Regridded dataset on the coarse grid.
"""
print(f"Regridding fine→coarse using {regridding_tool}...")
if regridding_tool == 'xesmf':
if not XESMF_AVAILABLE:
raise ImportError("xESMF is not installed. Use: pip install xesmf")
weight_file = None
if weights_dir:
os.makedirs(weights_dir, exist_ok=True)
weight_file = os.path.join(weights_dir, f'weights_fine_to_coarse_{method}.nc')
regridder = xe.Regridder(
fine_data, coarse_template, method=method,
periodic=False, ignore_degenerate=True,
filename=weight_file,
reuse_weights=bool(weight_file and os.path.exists(weight_file))
)
# Build a valid-source mask so border cells with no valid source → NaN
first_var = next(iter(fine_data.data_vars))
fine_valid = xr.where(fine_data[first_var].isel(time=0).notnull(), 1.0, 0.0)
coarse_valid_frac = regridder(fine_valid)
valid_mask = coarse_valid_frac > 0
result_vars = {}
for var in fine_data.data_vars:
regridded = regridder(fine_data[var].where(fine_data[var].notnull(), other=0.0))
regridded = (regridded / coarse_valid_frac).where(valid_mask)
regridded.attrs = fine_data[var].attrs
result_vars[var] = regridded
result = xr.Dataset(result_vars, attrs=fine_data.attrs)
elif regridding_tool == 'cdo':
import tempfile
with tempfile.TemporaryDirectory() as tmpdir:
fine_file = os.path.join(tmpdir, 'fine_input.nc')
template_file = os.path.join(tmpdir, 'coarse_template.nc')
output_file = os.path.join(tmpdir, 'regridded_output.nc')
fine_data.to_netcdf(fine_file)
coarse_template.to_netcdf(template_file)
cmd = (f"conda run -n {cdo_env} "
f"cdo {cdo_method},{template_file} {fine_file} {output_file}")
print(f" Running: {cmd}")
try:
subprocess.run(cmd, shell=True, check=True, capture_output=True, text=True)
result = xr.open_dataset(output_file).load()
except subprocess.CalledProcessError as e:
raise RuntimeError(f"CDO regridding failed: {e.stderr}") from e
else:
raise ValueError(f"Unknown regridding_tool: {regridding_tool!r}. Use 'xesmf' or 'cdo'.")
print(" Regridding complete.")
return result
climdata.sdba.utils.compute_daily_climatology(data, n_harmonics=3)
¶
Build a smoothed day-of-year climatology.
Groups data by day of year, averages over all years, then applies
:func:smooth_fft along the dayofyear axis. The computation is lazy and
Dask-friendly: chunked inputs stay chunked.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
data |
xr.DataArray |
Input with a |
required |
n_harmonics |
int |
Harmonics retained by :func: |
3 |
Returns:
| Type | Description |
|---|---|
xr.DataArray |
The climatology, with |
Examples:
>>> clim = compute_daily_climatology(ds.tasmax)
>>> clim.dims
('dayofyear', 'lat', 'lon')
Source code in climdata/sdba/utils.py
def compute_daily_climatology(data: xr.DataArray, n_harmonics: int = 3) -> xr.DataArray:
"""Build a smoothed day-of-year climatology.
Groups ``data`` by day of year, averages over all years, then applies
:func:`smooth_fft` along the ``dayofyear`` axis. The computation is lazy and
Dask-friendly: chunked inputs stay chunked.
Args:
data (xr.DataArray): Input with a ``time`` dimension spanning at least one
full year. Any additional dimensions (``lat``, ``lon``, …) are
broadcast over.
n_harmonics (int): Harmonics retained by :func:`smooth_fft`. Defaults to ``3``.
Returns:
xr.DataArray: The climatology, with ``time`` replaced by ``dayofyear``.
Example:
>>> clim = compute_daily_climatology(ds.tasmax) # doctest: +SKIP
>>> clim.dims # doctest: +SKIP
('dayofyear', 'lat', 'lon')
"""
doy_clim = data.groupby("time.dayofyear").mean("time")
return xr.apply_ufunc(
smooth_fft,
doy_clim,
kwargs={"n_harmonics": n_harmonics},
input_core_dims=[["dayofyear"]],
output_core_dims=[["dayofyear"]],
vectorize=True,
dask="parallelized",
output_dtypes=[float],
dask_gufunc_kwargs={"allow_rechunk": True},
)
climdata.sdba.utils.smooth_fft(x, n_harmonics=3)
¶
Smooth a periodic series by truncating its Fourier spectrum.
All harmonics above n_harmonics are zeroed and the series transformed
back, leaving the annual cycle and its first overtones. The input is treated
as exactly one period, so it is meant for a full day-of-year axis (365 or 366
values), not for an arbitrary time slice.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
x |
numpy.ndarray |
One-dimensional periodic series, e.g. a day-of-year climatology. |
required |
n_harmonics |
int |
Number of harmonics to retain. |
3 |
Returns:
| Type | Description |
|---|---|
numpy.ndarray |
Smoothed series, the same shape and dtype-kind as |
Examples:
>>> import numpy as np
>>> doy = np.arange(365)
>>> raw = 10 * np.sin(2 * np.pi * doy / 365) + np.random.default_rng(0).normal(size=365)
>>> smooth = smooth_fft(raw, n_harmonics=2)
>>> smooth.std() < raw.std()
np.True_
Source code in climdata/sdba/utils.py
def smooth_fft(x, n_harmonics: int = 3):
"""Smooth a periodic series by truncating its Fourier spectrum.
All harmonics above ``n_harmonics`` are zeroed and the series transformed
back, leaving the annual cycle and its first overtones. The input is treated
as exactly one period, so it is meant for a full day-of-year axis (365 or 366
values), not for an arbitrary time slice.
Args:
x (numpy.ndarray): One-dimensional periodic series, e.g. a day-of-year
climatology.
n_harmonics (int): Number of harmonics to retain. ``1`` keeps only the
annual cycle; larger values follow the seasonal shape more closely.
Defaults to ``3``.
Returns:
numpy.ndarray: Smoothed series, the same shape and dtype-kind as ``x``.
Example:
>>> import numpy as np
>>> doy = np.arange(365)
>>> raw = 10 * np.sin(2 * np.pi * doy / 365) + np.random.default_rng(0).normal(size=365)
>>> smooth = smooth_fft(raw, n_harmonics=2)
>>> smooth.std() < raw.std()
np.True_
"""
f = fft(x)
f[n_harmonics + 1: -n_harmonics] = 0
return np.real(ifft(f))
Imputation¶
climdata.impute.impute_xarray.Imputer
¶
Fill missing values in a (time, lat, lon) dataset, cell by cell.
Reshapes the dataset into one time series per variable and grid cell, runs an imputegap algorithm over the resulting matrix, and reassembles the output. Optional z-score normalisation is applied before imputation and inverted afterwards, which matters for the neural methods and is harmless otherwise.
Attributes:
| Name | Type | Description |
|---|---|---|
ds |
xr.Dataset |
The input dataset. |
method |
str |
Algorithm name — |
normalize |
bool |
Whether series are z-scored before imputation. |
variables |
list[str] |
Data variables that will be imputed. |
recovered_ds |
xr.Dataset | None |
Result of the last :meth: |
Examples:
>>> imputer = Imputer(ds, method="CDRec")
>>> imputer.missing_fraction()["global"]
0.043
>>> filled = imputer.impute()
Source code in climdata/impute/impute_xarray.py
class Imputer:
"""Fill missing values in a ``(time, lat, lon)`` dataset, cell by cell.
Reshapes the dataset into one time series per variable and grid cell, runs an
imputegap algorithm over the resulting matrix, and reassembles the output.
Optional z-score normalisation is applied before imputation and inverted
afterwards, which matters for the neural methods and is harmless otherwise.
Attributes:
ds (xr.Dataset): The input dataset.
method (str): Algorithm name — ``"BRITS"``, ``"SoftImpute"``,
``"CDRec"`` or ``"XGBOOST"``.
normalize (bool): Whether series are z-scored before imputation.
variables (list[str]): Data variables that will be imputed.
recovered_ds (xr.Dataset | None): Result of the last :meth:`impute`.
Example:
>>> imputer = Imputer(ds, method="CDRec") # doctest: +SKIP
>>> imputer.missing_fraction()["global"] # doctest: +SKIP
0.043
>>> filled = imputer.impute() # doctest: +SKIP
"""
# Methods that require PyTorch (BRITS, GRIN, BayOTIDE, DeepMVI)
_TORCH_METHODS = {"BRITS", "GRIN", "BayOTIDE", "DeepMVI"}
def __init__(
self,
ds: xr.Dataset,
time_dim: str = "time",
lat_dim: str = "lat",
lon_dim: str = "lon",
method: str = "BRITS",
normalize: bool = True,
):
"""Bind a dataset and check that the chosen method can actually run.
The PyTorch check happens here rather than at :meth:`impute`, so a
missing backend is reported before any reshaping work is done.
Args:
ds (xr.Dataset): Dataset with time, latitude and longitude dimensions.
time_dim (str): Name of the time dimension. Defaults to ``"time"``.
lat_dim (str): Name of the latitude dimension. Defaults to ``"lat"``.
lon_dim (str): Name of the longitude dimension. Defaults to ``"lon"``.
method (str): Algorithm to use. Defaults to ``"BRITS"``.
normalize (bool): Z-score each series before imputing, and invert
afterwards. Defaults to ``True``.
Raises:
ImportError: If the vendored imputegap is unavailable, or if
``method`` needs PyTorch and PyTorch is not installed.
"""
if not _IMPUTEGAP_AVAILABLE:
raise ImportError(
"[climdata.impute] imputegap is not available. "
"It is bundled with climdata — please reinstall the package."
)
if method in self._TORCH_METHODS:
try:
import torch # noqa: F401
except ImportError:
raise ImportError(
f"[climdata.impute] Method '{method}' requires PyTorch, which is not installed. "
f"Install it with:\n"
f" pip install torch torchvision torchaudio "
f"--index-url https://download.pytorch.org/whl/cpu\n"
f"See docs/installation.md for GPU and full dependency instructions."
)
self.ds = ds
self.time_dim = time_dim
self.lat_dim = lat_dim
self.lon_dim = lon_dim
self.method = method
self.normalize = normalize
self.variables = list(ds.data_vars)
self.coords = ds.coords
self.attrs = ds.attrs
self.ts = None
self.mask = None
self.shape_info = None
self.recovered_ds = None
def missing_fraction(self):
"""Report how much data is missing, per variable and overall.
Returns:
dict[str, float]: One entry per data variable plus a ``"global"``
key. The global figure is the unweighted mean of the per-variable
fractions, so variables of different sizes count equally.
"""
frac = {
v: float(self.ds[v].isnull().mean())
for v in self.variables
}
frac["global"] = float(
np.mean(list(frac.values()))
)
return frac
def _to_timeseries(self):
"""Flatten the dataset into the ``(n_series, seq_len)`` matrix imputegap wants.
Series are ordered ``(variable, lat, lon)``, and the shape and coordinate
values are recorded on ``self.shape_info`` so :meth:`_from_timeseries`
can invert the operation. The NaN mask is captured before any
normalisation.
Returns:
None: Sets ``self.ts``, ``self.mask`` and ``self.shape_info``.
"""
da = self.ds[self.variables].to_array("variable")
# dims: (variable, time, lat, lon)
da = da.transpose(
"variable",
self.time_dim,
self.lat_dim,
self.lon_dim,
)
data = da.values
n_var, t, n_lat, n_lon = data.shape
self.shape_info = {
"variables": np.array(self.variables, dtype=object),
self.time_dim: self.ds[self.time_dim].values,
self.lat_dim: self.ds[self.lat_dim].values,
self.lon_dim: self.ds[self.lon_dim].values,
"n_var": n_var,
"n_lat": n_lat,
"n_lon": n_lon,
}
data_2d = data.reshape(
n_var * n_lat * n_lon,
t
)
self.mask = np.isnan(data_2d)
ts = TimeSeries()
ts.data = data_2d
ts.n_series, ts.seq_len = ts.data.shape
if self.normalize:
ts.normalize(normalizer="z_score")
self.ts = ts
def _from_timeseries(self, data_2d):
"""Rebuild a dataset from the imputed matrix.
Coordinates are restored from ``self.shape_info``. Any coordinate that
does not match the expected length is replaced by a plain integer range
rather than raising, so a shape surprise costs coordinate labels rather
than the whole result.
Args:
data_2d (numpy.ndarray): Imputed matrix of shape
``(n_series, seq_len)``.
Returns:
xr.Dataset: One variable per input variable, dimensioned
``(time, lat, lon)``, carrying the original global attributes.
"""
n_var = self.shape_info["n_var"]
n_lat = self.shape_info["n_lat"]
n_lon = self.shape_info["n_lon"]
# rebuild 4D array to (variable, time, lat, lon)
data_4d = data_2d.reshape(
n_var,
n_lat,
n_lon,
-1
).transpose(0, 3, 1, 2)
# (variable, time, lat, lon)
# --- sanitize coords (ensure 1-D arrays and correct lengths) ---
# helper to coerce to 1D and validate length
def _safe_coord(key, expected_len, fallback_range=True):
val = self.shape_info.get(key, None)
val = np.asarray(val) if val is not None else None
if val is None:
return np.arange(expected_len) if fallback_range else None
# flatten to 1D if possible
if val.ndim != 1 or len(val) != expected_len:
# try ravel() then trim / pad if needed
val = val.ravel()
if len(val) >= expected_len:
return val[:expected_len]
else:
# fallback to integer index
return np.arange(expected_len)
return val
variable_coord = _safe_coord("variables", n_var)
time_coord = _safe_coord(self.time_dim, data_4d.shape[1])
lat_coord = _safe_coord(self.lat_dim, n_lat)
lon_coord = _safe_coord(self.lon_dim, n_lon)
da = xr.DataArray(
data_4d,
dims=("variable", self.time_dim, self.lat_dim, self.lon_dim),
coords=(
("variable", np.asarray(variable_coord).astype(str)),
(self.time_dim, time_coord),
(self.lat_dim, lat_coord),
(self.lon_dim, lon_coord),
),
attrs=self.attrs,
)
return da.to_dataset(dim="variable")
def impute(self, epochs: int = 300):
"""Fill the missing values and return the completed dataset.
Returns a copy unchanged when there is nothing missing, so calling this
unconditionally in a workflow is safe and cheap.
Note that every value is reconstructed, not just the gaps: the returned
dataset carries the algorithm's estimate everywhere. Use :meth:`metrics`
to see how closely it reproduces the values that were present.
Args:
epochs (int): Training epochs, used only by ``BRITS``. Defaults to ``300``.
Returns:
xr.Dataset: The completed dataset, also stored on
:attr:`recovered_ds`.
Raises:
ValueError: If :attr:`method` is not one of the supported algorithms.
"""
if self.missing_fraction()["global"] == 0.0:
logger.info(f"{_COLOR_YELLOW}No missing data found. Imputation not required.{_COLOR_RESET}")
self.recovered_ds = self.ds.copy(deep=True)
return self.recovered_ds
self._to_timeseries()
data = self.ts.data
# print(data.shape)
if self.method == "BRITS":
imputer = Imputation.DeepLearning.BRITS(data)
imputer.epochs = epochs
elif self.method == "SoftImpute":
imputer = Imputation.MatrixCompletion.SoftImpute(data)
elif self.method == "CDRec":
imputer = Imputation.MatrixCompletion.CDRec(data)
elif self.method == "XGBOOST":
imputer = Imputation.MachineLearning.XGBOOST(data)
else:
raise ValueError(f"Unknown method: {self.method}")
imputer.impute()
rec = imputer.recov_data
if self.normalize:
mean = self.ts.data.mean(axis=1, keepdims=True)
std = self.ts.data.std(axis=1, keepdims=True)
rec = rec * std + mean
self.recovered_ds = self._from_timeseries(rec)
return self.recovered_ds
def metrics(self):
"""Score the reconstruction against the values that were actually present.
Compares imputed against original on the cells that were *not* missing —
the only cells where ground truth exists. This measures how faithfully
the algorithm reproduces known data, which is an upper bound on, not a
measure of, its accuracy in the gaps. For a real estimate, mask out known
values yourself and score the reconstruction there.
Variables with no missing values are omitted entirely.
Returns:
dict[str, dict]: Per variable, ``rmse``, ``mae`` and
``missing_fraction``.
Raises:
RuntimeError: If :meth:`impute` has not been called.
"""
if self.recovered_ds is None:
raise RuntimeError("Call impute() first")
scores = {}
for v in self.variables:
orig = self.ds[v].values
rec = self.recovered_ds[v].values
mask = np.isnan(orig)
if mask.sum() == 0:
continue
scores[v] = {
"rmse": mean_squared_error(
orig[~mask],
rec[~mask],
squared=False,
),
"mae": mean_absolute_error(
orig[~mask],
rec[~mask],
),
"missing_fraction": float(mask.mean()),
}
return scores
def summary(self):
"""Describe the imputation setup and the gaps it faces.
Returns:
dict: ``method``, ``normalize``, ``variables``, ``missing_fraction``
and ``dims``. Cheap to call before :meth:`impute` to check what will
be run.
"""
return {
"method": self.method,
"normalize": self.normalize,
"variables": self.variables,
"missing_fraction": self.missing_fraction(),
"dims": dict(self.ds.dims),
}
__init__(self, ds, time_dim='time', lat_dim='lat', lon_dim='lon', method='BRITS', normalize=True)
special
¶
Bind a dataset and check that the chosen method can actually run.
The PyTorch check happens here rather than at :meth:impute, so a
missing backend is reported before any reshaping work is done.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
ds |
xr.Dataset |
Dataset with time, latitude and longitude dimensions. |
required |
time_dim |
str |
Name of the time dimension. Defaults to |
'time' |
lat_dim |
str |
Name of the latitude dimension. Defaults to |
'lat' |
lon_dim |
str |
Name of the longitude dimension. Defaults to |
'lon' |
method |
str |
Algorithm to use. Defaults to |
'BRITS' |
normalize |
bool |
Z-score each series before imputing, and invert
afterwards. Defaults to |
True |
Exceptions:
| Type | Description |
|---|---|
ImportError |
If the vendored imputegap is unavailable, or if
|
Source code in climdata/impute/impute_xarray.py
def __init__(
self,
ds: xr.Dataset,
time_dim: str = "time",
lat_dim: str = "lat",
lon_dim: str = "lon",
method: str = "BRITS",
normalize: bool = True,
):
"""Bind a dataset and check that the chosen method can actually run.
The PyTorch check happens here rather than at :meth:`impute`, so a
missing backend is reported before any reshaping work is done.
Args:
ds (xr.Dataset): Dataset with time, latitude and longitude dimensions.
time_dim (str): Name of the time dimension. Defaults to ``"time"``.
lat_dim (str): Name of the latitude dimension. Defaults to ``"lat"``.
lon_dim (str): Name of the longitude dimension. Defaults to ``"lon"``.
method (str): Algorithm to use. Defaults to ``"BRITS"``.
normalize (bool): Z-score each series before imputing, and invert
afterwards. Defaults to ``True``.
Raises:
ImportError: If the vendored imputegap is unavailable, or if
``method`` needs PyTorch and PyTorch is not installed.
"""
if not _IMPUTEGAP_AVAILABLE:
raise ImportError(
"[climdata.impute] imputegap is not available. "
"It is bundled with climdata — please reinstall the package."
)
if method in self._TORCH_METHODS:
try:
import torch # noqa: F401
except ImportError:
raise ImportError(
f"[climdata.impute] Method '{method}' requires PyTorch, which is not installed. "
f"Install it with:\n"
f" pip install torch torchvision torchaudio "
f"--index-url https://download.pytorch.org/whl/cpu\n"
f"See docs/installation.md for GPU and full dependency instructions."
)
self.ds = ds
self.time_dim = time_dim
self.lat_dim = lat_dim
self.lon_dim = lon_dim
self.method = method
self.normalize = normalize
self.variables = list(ds.data_vars)
self.coords = ds.coords
self.attrs = ds.attrs
self.ts = None
self.mask = None
self.shape_info = None
self.recovered_ds = None
impute(self, epochs=300)
¶
Fill the missing values and return the completed dataset.
Returns a copy unchanged when there is nothing missing, so calling this unconditionally in a workflow is safe and cheap.
Note that every value is reconstructed, not just the gaps: the returned
dataset carries the algorithm's estimate everywhere. Use :meth:metrics
to see how closely it reproduces the values that were present.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
epochs |
int |
Training epochs, used only by |
300 |
Returns:
| Type | Description |
|---|---|
xr.Dataset |
The completed dataset, also stored on
:attr: |
Exceptions:
| Type | Description |
|---|---|
ValueError |
If :attr: |
Source code in climdata/impute/impute_xarray.py
def impute(self, epochs: int = 300):
"""Fill the missing values and return the completed dataset.
Returns a copy unchanged when there is nothing missing, so calling this
unconditionally in a workflow is safe and cheap.
Note that every value is reconstructed, not just the gaps: the returned
dataset carries the algorithm's estimate everywhere. Use :meth:`metrics`
to see how closely it reproduces the values that were present.
Args:
epochs (int): Training epochs, used only by ``BRITS``. Defaults to ``300``.
Returns:
xr.Dataset: The completed dataset, also stored on
:attr:`recovered_ds`.
Raises:
ValueError: If :attr:`method` is not one of the supported algorithms.
"""
if self.missing_fraction()["global"] == 0.0:
logger.info(f"{_COLOR_YELLOW}No missing data found. Imputation not required.{_COLOR_RESET}")
self.recovered_ds = self.ds.copy(deep=True)
return self.recovered_ds
self._to_timeseries()
data = self.ts.data
# print(data.shape)
if self.method == "BRITS":
imputer = Imputation.DeepLearning.BRITS(data)
imputer.epochs = epochs
elif self.method == "SoftImpute":
imputer = Imputation.MatrixCompletion.SoftImpute(data)
elif self.method == "CDRec":
imputer = Imputation.MatrixCompletion.CDRec(data)
elif self.method == "XGBOOST":
imputer = Imputation.MachineLearning.XGBOOST(data)
else:
raise ValueError(f"Unknown method: {self.method}")
imputer.impute()
rec = imputer.recov_data
if self.normalize:
mean = self.ts.data.mean(axis=1, keepdims=True)
std = self.ts.data.std(axis=1, keepdims=True)
rec = rec * std + mean
self.recovered_ds = self._from_timeseries(rec)
return self.recovered_ds
metrics(self)
¶
Score the reconstruction against the values that were actually present.
Compares imputed against original on the cells that were not missing — the only cells where ground truth exists. This measures how faithfully the algorithm reproduces known data, which is an upper bound on, not a measure of, its accuracy in the gaps. For a real estimate, mask out known values yourself and score the reconstruction there.
Variables with no missing values are omitted entirely.
Returns:
| Type | Description |
|---|---|
dict[str, dict] |
Per variable, |
Exceptions:
| Type | Description |
|---|---|
RuntimeError |
If :meth: |
Source code in climdata/impute/impute_xarray.py
def metrics(self):
"""Score the reconstruction against the values that were actually present.
Compares imputed against original on the cells that were *not* missing —
the only cells where ground truth exists. This measures how faithfully
the algorithm reproduces known data, which is an upper bound on, not a
measure of, its accuracy in the gaps. For a real estimate, mask out known
values yourself and score the reconstruction there.
Variables with no missing values are omitted entirely.
Returns:
dict[str, dict]: Per variable, ``rmse``, ``mae`` and
``missing_fraction``.
Raises:
RuntimeError: If :meth:`impute` has not been called.
"""
if self.recovered_ds is None:
raise RuntimeError("Call impute() first")
scores = {}
for v in self.variables:
orig = self.ds[v].values
rec = self.recovered_ds[v].values
mask = np.isnan(orig)
if mask.sum() == 0:
continue
scores[v] = {
"rmse": mean_squared_error(
orig[~mask],
rec[~mask],
squared=False,
),
"mae": mean_absolute_error(
orig[~mask],
rec[~mask],
),
"missing_fraction": float(mask.mean()),
}
return scores
missing_fraction(self)
¶
Report how much data is missing, per variable and overall.
Returns:
| Type | Description |
|---|---|
dict[str, float] |
One entry per data variable plus a |
Source code in climdata/impute/impute_xarray.py
def missing_fraction(self):
"""Report how much data is missing, per variable and overall.
Returns:
dict[str, float]: One entry per data variable plus a ``"global"``
key. The global figure is the unweighted mean of the per-variable
fractions, so variables of different sizes count equally.
"""
frac = {
v: float(self.ds[v].isnull().mean())
for v in self.variables
}
frac["global"] = float(
np.mean(list(frac.values()))
)
return frac
summary(self)
¶
Describe the imputation setup and the gaps it faces.
Returns:
| Type | Description |
|---|---|
dict |
|
Source code in climdata/impute/impute_xarray.py
def summary(self):
"""Describe the imputation setup and the gaps it faces.
Returns:
dict: ``method``, ``normalize``, ``variables``, ``missing_fraction``
and ``dims``. Cheap to call before :meth:`impute` to check what will
be run.
"""
return {
"method": self.method,
"normalize": self.normalize,
"variables": self.variables,
"missing_fraction": self.missing_fraction(),
"dims": dict(self.ds.dims),
}
Visualisation¶
climdata.viz.plot_map(data, *, variable=None, time=0, reduce=None, ax=None, projection=None, figsize=(10, 5), coastlines=True, borders=True, gridlines=True, land=False, ocean=False, add_colorbar=True, robust=True, cmap=None, title=None, **plot_kwargs)
¶
Plot a 2D geographic field with coastlines and country boundaries.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
data |
An |
required | |
variable |
str |
Variable name when |
None |
time |
int | str | datetime |
Time slice to plot when a |
0 |
reduce |
str |
Reduce the time dimension instead of slicing it —
one of |
None |
ax |
Existing Cartopy |
None |
|
projection |
A |
None |
|
figsize |
tuple |
Figure size when creating a new figure. |
(10, 5) |
coastlines, |
borders, gridlines, land, ocean (bool |
Map features to add. |
required |
add_colorbar |
bool |
Whether to draw a colorbar. |
True |
robust |
bool |
Use 2nd/98th percentiles for color limits (passed to xarray). |
True |
cmap |
str |
Matplotlib colormap name. |
None |
title |
str |
Axis title. Defaults to the variable name + time. |
None |
**plot_kwargs |
Forwarded to |
{} |
Returns:
| Type | Description |
|---|---|
The Cartopy |
Source code in climdata/viz.py
def plot_map(
data,
*,
variable=None,
time=0,
reduce=None,
ax=None,
projection=None,
figsize=(10, 5),
coastlines=True,
borders=True,
gridlines=True,
land=False,
ocean=False,
add_colorbar=True,
robust=True,
cmap=None,
title=None,
**plot_kwargs,
):
"""Plot a 2D geographic field with coastlines and country boundaries.
Args:
data: An ``xr.DataArray`` or ``xr.Dataset``. For a Dataset pass ``variable``
(or it must contain exactly one data variable).
variable (str, optional): Variable name when ``data`` is a Dataset.
time (int | str | datetime, optional): Time slice to plot when a ``time``
dimension is present. An ``int`` selects by position (``isel``);
anything else selects by label (nearest). Ignored if ``reduce`` is set.
reduce (str, optional): Reduce the time dimension instead of slicing it —
one of ``"mean"``, ``"sum"``, ``"min"``, ``"max"``, ``"median"``, ``"std"``.
ax: Existing Cartopy ``GeoAxes`` to draw on. If ``None``, one is created.
projection: A ``cartopy.crs`` projection for a new axis (default ``PlateCarree``).
figsize (tuple): Figure size when creating a new figure.
coastlines, borders, gridlines, land, ocean (bool): Map features to add.
add_colorbar (bool): Whether to draw a colorbar.
robust (bool): Use 2nd/98th percentiles for color limits (passed to xarray).
cmap (str, optional): Matplotlib colormap name.
title (str, optional): Axis title. Defaults to the variable name + time.
**plot_kwargs: Forwarded to ``xarray.DataArray.plot`` (e.g. ``vmin``, ``vmax``).
Returns:
The Cartopy ``GeoAxes`` the field was drawn on.
"""
plt, ccrs, cfeature = _require_plotting()
da = _as_dataarray(data, variable)
x, y = _spatial_dim_names(da)
da = _reduce_to_2d(da, x, y, time, reduce)
if da.ndim != 2:
raise ValueError(
f"Expected a 2D field after reduction but got dims {da.dims}. "
"This can happen for point/station extractions that have no spatial grid."
)
data_crs = ccrs.PlateCarree()
if ax is None:
if projection is None:
projection = ccrs.PlateCarree()
fig, ax = plt.subplots(figsize=figsize, subplot_kw={"projection": projection})
da.plot(
ax=ax,
x=x,
y=y,
transform=data_crs,
robust=robust,
cmap=cmap,
add_colorbar=add_colorbar,
**plot_kwargs,
)
if land:
ax.add_feature(cfeature.LAND, facecolor="none", zorder=0)
if ocean:
ax.add_feature(cfeature.OCEAN, zorder=0)
if coastlines:
ax.coastlines(resolution="50m", linewidth=0.6)
if borders:
ax.add_feature(cfeature.BORDERS, linewidth=0.4, edgecolor="gray")
if gridlines:
gl = ax.gridlines(draw_labels=True, linewidth=0.3, color="gray", alpha=0.5)
gl.top_labels = False
gl.right_labels = False
if title is None:
name = da.name or (variable if variable else "")
title = str(name)
if "time" in da.coords and da["time"].ndim == 0:
try:
import pandas as pd
title = f"{title} @ {pd.to_datetime(da['time'].values):%Y-%m-%d}"
except Exception:
title = f"{title} @ {da['time'].values}"
elif reduce:
title = f"{title} ({reduce} over time)"
ax.set_title(title)
return ax