In [1]:
Copied!
# Uncomment to install climdata in Google Colab or other environments
# !pip install climdata
# Uncomment to install climdata in Google Colab or other environments
# !pip install climdata
MSWX Dataset — climdata Example¶
MSWX (Multi-Source Weather) is a global, daily, 0.1° meteorological dataset based on the ERA5 reanalysis, bias-corrected against in-situ station observations. It is distributed via Google Drive and accessed in climdata through a Google Service Account.
Available variables:
| Variable | Description | Units |
|---|---|---|
tasmin |
Daily minimum air temperature | °C |
tasmax |
Daily maximum air temperature | °C |
tas |
Daily mean air temperature | °C |
pr |
Daily precipitation | mm/day |
rsds |
Downward shortwave radiation | W/m² |
hurs |
Relative humidity | % |
sfcWind |
Wind speed | m/s |
Prerequisites: A Google Service Account JSON key file with read access to the MSWX Google Drive folders is required for downloading data.
1. Setup & Imports¶
In [2]:
Copied!
from climdata import ClimData
from climdata.datasets.MSWX import MSWXmirror
import xarray as xr
import pandas as pd
import logging
logging.basicConfig(
level=logging.INFO,
format="%(levelname)s | %(message)s",
force=True,
)
from climdata import ClimData
from climdata.datasets.MSWX import MSWXmirror
import xarray as xr
import pandas as pd
import logging
logging.basicConfig(
level=logging.INFO,
format="%(levelname)s | %(message)s",
force=True,
)
2. Configuration¶
Set up the configuration overrides. The google_service_account path must point to your Google Service Account JSON key.
In [ ]:
Copied!
overrides = [
"dataset=mswx", # Select the MSWX dataset
"lat=52", # Target latitude
"lon=13", # Target longitude (Berlin, Germany)
"time_range.start_date=1989-01-01", # Start date for data extraction
"time_range.end_date=2024-12-31", # End date for data extraction
"variables=[tasmin,tasmax,pr]", # Variables to extract
"data_dir=./data", # Local directory to store downloaded files
"dsinfo.mswx.params.google_service_account=./.climdata_conf/service.json", # Path to service account key
]
extractor = ClimData(overrides=overrides)
overrides = [
"dataset=mswx", # Select the MSWX dataset
"lat=52", # Target latitude
"lon=13", # Target longitude (Berlin, Germany)
"time_range.start_date=1989-01-01", # Start date for data extraction
"time_range.end_date=2024-12-31", # End date for data extraction
"variables=[tasmin,tasmax,pr]", # Variables to extract
"data_dir=./data", # Local directory to store downloaded files
"dsinfo.mswx.params.google_service_account=./.climdata_conf/service.json", # Path to service account key
]
extractor = ClimData(overrides=overrides)
3. Download & Load Data¶
MSWX data is stored as daily NetCDF files on Google Drive (one file per day per variable). The fetch() method checks for existing local files and only downloads missing ones. The load() method reads all files into a single xarray.Dataset.
In [ ]:
Copied!
mswx = MSWXmirror(extractor.cfg)
# Extract a single point (nearest grid cell)
mswx.extract(point=(extractor.cfg.lon, extractor.cfg.lat))
# Load each variable
#ds_tasmin = mswx.load(variable="tasmin")
#ds_tasmax = mswx.load(variable="tasmax")
ds_hurs = mswx.load(variable="hurs")
print(ds_tasmin)
mswx = MSWXmirror(extractor.cfg)
# Extract a single point (nearest grid cell)
mswx.extract(point=(extractor.cfg.lon, extractor.cfg.lat))
# Load each variable
#ds_tasmin = mswx.load(variable="tasmin")
#ds_tasmax = mswx.load(variable="tasmax")
ds_hurs = mswx.load(variable="hurs")
print(ds_tasmin)
INFO | file_cache is only supported with oauth2client<4.0.0
📂 10327 exist, 2822 missing — fetching hurs from Drive... ⬇️ Downloading 1996266.nc ... → Download 100% complete ⬇️ Downloading 1996265.nc ... → Download 100% complete ⬇️ Downloading 1996264.nc ... → Download 100% complete ⬇️ Downloading 1996263.nc ... → Download 100% complete ⬇️ Downloading 1996262.nc ... → Download 100% complete ⬇️ Downloading 1996261.nc ... → Download 100% complete ⬇️ Downloading 1996260.nc ... → Download 100% complete ⬇️ Downloading 1996259.nc ... → Download 100% complete ⬇️ Downloading 1996258.nc ... → Download 100% complete ⬇️ Downloading 1996257.nc ... → Download 100% complete ⬇️ Downloading 1996256.nc ... → Download 100% complete ⬇️ Downloading 1996255.nc ... → Download 100% complete ⬇️ Downloading 1996254.nc ... → Download 100% complete ⬇️ Downloading 1996253.nc ... → Download 100% complete ⬇️ Downloading 1996252.nc ... → Download 100% complete ⬇️ Downloading 1996251.nc ... → Download 100% complete ⬇️ Downloading 1996250.nc ... → Download 100% complete ⬇️ Downloading 1996249.nc ... → Download 100% complete ⬇️ Downloading 1996248.nc ... → Download 100% complete ⬇️ Downloading 1996247.nc ... → Download 100% complete ⬇️ Downloading 1996246.nc ... → Download 100% complete ⬇️ Downloading 1996245.nc ... → Download 100% complete ⬇️ Downloading 1996244.nc ... → Download 100% complete ⬇️ Downloading 1996243.nc ... → Download 100% complete ⬇️ Downloading 1996242.nc ... → Download 100% complete ⬇️ Downloading 1996241.nc ... → Download 100% complete ⬇️ Downloading 1996240.nc ... → Download 100% complete ⬇️ Downloading 1996239.nc ... → Download 100% complete ⬇️ Downloading 1996238.nc ... → Download 100% complete ⬇️ Downloading 1996237.nc ... → Download 100% complete ⬇️ Downloading 1996236.nc ... → Download 100% complete ⬇️ Downloading 1996235.nc ... → Download 100% complete ⬇️ Downloading 1996234.nc ... → Download 100% complete ⬇️ Downloading 1996233.nc ... → Download 100% complete ⬇️ Downloading 1996232.nc ... → Download 100% complete ⬇️ Downloading 1996231.nc ... → Download 100% complete ⬇️ Downloading 1996230.nc ... → Download 100% complete ⬇️ Downloading 1996229.nc ... → Download 100% complete ⬇️ Downloading 1996228.nc ... → Download 100% complete ⬇️ Downloading 1996227.nc ... → Download 100% complete ⬇️ Downloading 1996226.nc ... → Download 100% complete ⬇️ Downloading 1996225.nc ... → Download 100% complete ⬇️ Downloading 1996224.nc ... → Download 100% complete ⬇️ Downloading 1996223.nc ... → Download 100% complete ⬇️ Downloading 1996222.nc ... → Download 100% complete ⬇️ Downloading 1996221.nc ... → Download 100% complete ⬇️ Downloading 1996220.nc ... → Download 100% complete ⬇️ Downloading 1996219.nc ... → Download 100% complete ⬇️ Downloading 1996218.nc ... → Download 100% complete ⬇️ Downloading 1996217.nc ... → Download 100% complete ⬇️ Downloading 1996216.nc ... → Download 100% complete ⬇️ Downloading 1996215.nc ... → Download 100% complete ⬇️ Downloading 1996214.nc ... → Download 100% complete ⬇️ Downloading 1996213.nc ... → Download 100% complete ⬇️ Downloading 1996212.nc ... → Download 100% complete ⬇️ Downloading 1996211.nc ... → Download 100% complete ⬇️ Downloading 1996210.nc ... → Download 100% complete ⬇️ Downloading 1996209.nc ... → Download 100% complete ⬇️ Downloading 1996208.nc ... → Download 100% complete ⬇️ Downloading 1996207.nc ... → Download 100% complete ⬇️ Downloading 1996206.nc ... → Download 100% complete ⬇️ Downloading 1996205.nc ... → Download 100% complete ⬇️ Downloading 1996204.nc ... → Download 100% complete ⬇️ Downloading 1996203.nc ... → Download 100% complete ⬇️ Downloading 1996202.nc ... → Download 100% complete ⬇️ Downloading 1996201.nc ... → Download 100% complete ⬇️ Downloading 1996200.nc ... → Download 100% complete ⬇️ Downloading 1996199.nc ... → Download 100% complete ⬇️ Downloading 1996198.nc ... → Download 100% complete ⬇️ Downloading 1996197.nc ... → Download 100% complete ⬇️ Downloading 1996196.nc ... → Download 100% complete ⬇️ Downloading 1996195.nc ... → Download 100% complete ⬇️ Downloading 1996194.nc ... → Download 100% complete ⬇️ Downloading 1996193.nc ... → Download 100% complete ⬇️ Downloading 1996192.nc ... → Download 100% complete ⬇️ Downloading 1996191.nc ... → Download 100% complete ⬇️ Downloading 1996190.nc ... → Download 100% complete ⬇️ Downloading 1996189.nc ... → Download 100% complete ⬇️ Downloading 1996188.nc ... → Download 100% complete ⬇️ Downloading 1996187.nc ... → Download 100% complete ⬇️ Downloading 1996186.nc ... → Download 100% complete ⬇️ Downloading 1996185.nc ... → Download 100% complete ⬇️ Downloading 1996184.nc ... → Download 100% complete ⬇️ Downloading 1996183.nc ... → Download 100% complete ⬇️ Downloading 1996182.nc ... → Download 100% complete ⬇️ Downloading 1996181.nc ... → Download 100% complete ⬇️ Downloading 1996180.nc ... → Download 100% complete ⬇️ Downloading 1996179.nc ... → Download 100% complete ⬇️ Downloading 1996178.nc ... → Download 100% complete ⬇️ Downloading 1996177.nc ... → Download 100% complete ⬇️ Downloading 1996176.nc ... → Download 100% complete ⬇️ Downloading 1996175.nc ... → Download 100% complete ⬇️ Downloading 1996174.nc ... → Download 100% complete ⬇️ Downloading 1996173.nc ... → Download 100% complete ⬇️ Downloading 1996172.nc ... → Download 100% complete ⬇️ Downloading 1996171.nc ... → Download 100% complete ⬇️ Downloading 1996170.nc ... → Download 100% complete ⬇️ Downloading 1996169.nc ... → Download 100% complete ⬇️ Downloading 1996168.nc ... → Download 100% complete ⬇️ Downloading 1996167.nc ... → Download 100% complete ⬇️ Downloading 1996166.nc ... → Download 100% complete ⬇️ Downloading 1996165.nc ... → Download 100% complete ⬇️ Downloading 1996164.nc ... → Download 100% complete ⬇️ Downloading 1996163.nc ... → Download 100% complete ⬇️ Downloading 1996162.nc ... → Download 100% complete ⬇️ Downloading 1996161.nc ... → Download 100% complete ⬇️ Downloading 1996160.nc ... → Download 100% complete ⬇️ Downloading 1996159.nc ... → Download 100% complete ⬇️ Downloading 1996158.nc ... → Download 100% complete ⬇️ Downloading 1996157.nc ... → Download 100% complete ⬇️ Downloading 1996156.nc ... → Download 100% complete ⬇️ Downloading 1996155.nc ... → Download 100% complete ⬇️ Downloading 1996154.nc ... → Download 100% complete ⬇️ Downloading 1996153.nc ... → Download 100% complete ⬇️ Downloading 1996152.nc ... → Download 100% complete ⬇️ Downloading 1996151.nc ... → Download 100% complete ⬇️ Downloading 1996150.nc ... → Download 100% complete ⬇️ Downloading 1996149.nc ... → Download 100% complete ⬇️ Downloading 1996148.nc ... → Download 100% complete ⬇️ Downloading 1996147.nc ... → Download 100% complete ⬇️ Downloading 1996146.nc ... → Download 100% complete ⬇️ Downloading 1996145.nc ... → Download 100% complete ⬇️ Downloading 1996144.nc ... → Download 100% complete ⬇️ Downloading 1996143.nc ... → Download 100% complete ⬇️ Downloading 1996142.nc ... → Download 100% complete ⬇️ Downloading 1996141.nc ... → Download 100% complete ⬇️ Downloading 1996140.nc ... → Download 100% complete ⬇️ Downloading 1996139.nc ... → Download 100% complete ⬇️ Downloading 1996138.nc ... → Download 100% complete ⬇️ Downloading 1996137.nc ... → Download 100% complete ⬇️ Downloading 1996136.nc ... → Download 100% complete ⬇️ Downloading 1996135.nc ... → Download 100% complete ⬇️ Downloading 1996134.nc ...
4. Inspect the Dataset¶
In [ ]:
Copied!
# Inspect one variable
ds_tasmin
# Inspect one variable
ds_tasmin
In [ ]:
Copied!
# Convert to pandas DataFrame for quick inspection
df_tasmin = ds_tasmin.to_dataframe().reset_index()
df_tasmin.head()
# Convert to pandas DataFrame for quick inspection
df_tasmin = ds_tasmin.to_dataframe().reset_index()
df_tasmin.head()
5. Spatial Extraction — Bounding Box¶
In [ ]:
Copied!
overrides_box = [
"dataset=mswx",
"time_range.start_date=2010-01-01",
"time_range.end_date=2010-03-31",
"variables=[tasmax,pr]",
"data_dir=./data",
"dsinfo.mswx.params.google_service_account=./.climdata_conf/service.json",
]
extractor_box = ClimData(overrides=overrides_box)
mswx_box = MSWXmirror(extractor_box.cfg)
# Extract a bounding box (Germany)
mswx_box.extract(box={
"lon_min": 5.8,
"lon_max": 15.0,
"lat_min": 47.3,
"lat_max": 55.1
})
ds_box = mswx_box.load(variable="tasmax")
print(ds_box)
overrides_box = [
"dataset=mswx",
"time_range.start_date=2010-01-01",
"time_range.end_date=2010-03-31",
"variables=[tasmax,pr]",
"data_dir=./data",
"dsinfo.mswx.params.google_service_account=./.climdata_conf/service.json",
]
extractor_box = ClimData(overrides=overrides_box)
mswx_box = MSWXmirror(extractor_box.cfg)
# Extract a bounding box (Germany)
mswx_box.extract(box={
"lon_min": 5.8,
"lon_max": 15.0,
"lat_min": 47.3,
"lat_max": 55.1
})
ds_box = mswx_box.load(variable="tasmax")
print(ds_box)
6. Run the Full Workflow via ClimData¶
Use ClimData.run_workflow() to chain extraction, optional imputation, climate index calculation, and saving to NetCDF in one call.
In [ ]:
Copied!
overrides_wf = [
"dataset=mswx",
"lat=52",
"lon=13",
"time_range.start_date=2010-01-01",
"time_range.end_date=2010-12-31",
"variables=[tasmin,tasmax,pr]",
"data_dir=./data",
"dsinfo.mswx.params.google_service_account=./.climdata_conf/service.json",
"index=tn10p", # Cold nights climate extreme index
"impute=BRITS", # Deep-learning gap-filling (optional)
]
seq = ["extract", "impute", "calc_index", "to_nc"]
extractor_wf = ClimData(overrides=overrides_wf)
result = extractor_wf.run_workflow(actions=seq)
result
overrides_wf = [
"dataset=mswx",
"lat=52",
"lon=13",
"time_range.start_date=2010-01-01",
"time_range.end_date=2010-12-31",
"variables=[tasmin,tasmax,pr]",
"data_dir=./data",
"dsinfo.mswx.params.google_service_account=./.climdata_conf/service.json",
"index=tn10p", # Cold nights climate extreme index
"impute=BRITS", # Deep-learning gap-filling (optional)
]
seq = ["extract", "impute", "calc_index", "to_nc"]
extractor_wf = ClimData(overrides=overrides_wf)
result = extractor_wf.run_workflow(actions=seq)
result
7. Save to CSV¶
In [ ]:
Copied!
mswx.save_csv("mswx_point_2010.csv")
print("Saved to mswx_point_2010.csv")
mswx.save_csv("mswx_point_2010.csv")
print("Saved to mswx_point_2010.csv")