In [1]:

# Uncomment to install climdata in Google Colab or other environments
# !pip install climdata

# Uncomment to install climdata in Google Colab or other environments # !pip install climdata

MSWX Dataset — climdata Example¶

MSWX (Multi-Source Weather) is a global, daily, 0.1° meteorological dataset based on the ERA5 reanalysis, bias-corrected against in-situ station observations. It is distributed via Google Drive and accessed in climdata through a Google Service Account.

Available variables:

Variable	Description	Units
tasmin	Daily minimum air temperature	°C
tasmax	Daily maximum air temperature	°C
tas	Daily mean air temperature	°C
pr	Daily precipitation	mm/day
rsds	Downward shortwave radiation	W/m²
hurs	Relative humidity	%
sfcWind	Wind speed	m/s

Prerequisites: A Google Service Account JSON key file with read access to the MSWX Google Drive folders is required for downloading data.

1. Setup & Imports¶

In [1]:

from climdata import ClimData
from climdata.datasets.MSWX import MSWXmirror
import xarray as xr
import pandas as pd
import logging

logging.basicConfig(
    level=logging.INFO,
    format="%(levelname)s | %(message)s",
    force=True,
)

from climdata import ClimData from climdata.datasets.MSWX import MSWXmirror import xarray as xr import pandas as pd import logging logging.basicConfig( level=logging.INFO, format="%(levelname)s | %(message)s", force=True, )

2. Configuration¶

Set up the configuration overrides. The google_service_account path must point to your Google Service Account JSON key.

In [2]:

overrides = [
    "dataset=mswx",                             # Select the MSWX dataset
    "region=germany",
    "time_range.start_date=1989-01-01",         # Start date for data extraction
    "time_range.end_date=2024-12-31",           # End date for data extraction
    "variables=[tasmin, tasmax, tas, pr]",             # Variables to extract
    "data_dir=/data01/FDS/muduchuru/Atmos/",                          # Local directory to store downloaded files
    "+dsinfo.mswx.params.google_service_account=/home/muduchuru/.climdata_conf/service.json",  # Path to service account key
]

extractor = ClimData(overrides=overrides)
ds = extractor.extract()

overrides = [ "dataset=mswx", # Select the MSWX dataset "region=germany", "time_range.start_date=1989-01-01", # Start date for data extraction "time_range.end_date=2024-12-31", # End date for data extraction "variables=[tasmin, tasmax, tas, pr]", # Variables to extract "data_dir=/data01/FDS/muduchuru/Atmos/", # Local directory to store downloaded files "+dsinfo.mswx.params.google_service_account=/home/muduchuru/.climdata_conf/service.json", # Path to service account key ] extractor = ClimData(overrides=overrides) ds = extractor.extract()

✅ All 13149 tasmin files already exist locally.

<frozen importlib._bootstrap>:241: RuntimeWarning: numpy.ndarray size changed, may indicate binary incompatibility. Expected 16 from C header, got 96 from PyObject

---------------------------------------------------------------------------
KeyboardInterrupt                         Traceback (most recent call last)
Cell In[2], line 12
      1 overrides = [
      2     "dataset=mswx",                             # Select the MSWX dataset
      3     "region=germany",
   (...)
      8     "+dsinfo.mswx.params.google_service_account=/home/muduchuru/.climdata_conf/service.json",  # Path to service account key
      9 ]
     11 extractor = ClimData(overrides=overrides)
---> 12 ds = extractor.extract()

File /beegfs/muduchuru/pkgs_fnl/climdata/climdata/utils/wrapper_workflow.py:74, in update_ds.<locals>.decorator.<locals>.wrapper(self, *args, **kwargs)
     72 @wraps(func)
     73 def wrapper(self, *args, **kwargs):
---> 74     ds = func(self, *args, **kwargs)
     75     if ds is not None:
     76         self.current_ds = ds

File /beegfs/muduchuru/pkgs_fnl/climdata/climdata/utils/wrapper_workflow.py:576, in ClimateExtractor.extract(self)
    574     mswx = climdata.MSWX(cfg)
    575     mswx.extract(**extract_kwargs)
--> 576     mswx.load(var)
    577     ds_vars.append(mswx.dataset)
    578 ds = xr.merge(ds_vars)

File /beegfs/muduchuru/pkgs_fnl/climdata/climdata/datasets/MSWX.py:207, in MSWXmirror.load(self, variable)
    205     batch_files = file_paths[i:i+batch_size]
    206     delayed_batch = [dask.delayed(open_point)(f) for f in batch_files]
--> 207     batch_ds = list(dask.compute(*delayed_batch))
    208     dsets.extend(batch_ds)
    210 dset = xr.concat(dsets, dim="time")

File ~/miniforge3/envs/sdba/lib/python3.10/site-packages/dask/base.py:664, in compute(traverse, optimize_graph, scheduler, get, *args, **kwargs)
    661     postcomputes.append(x.__dask_postcompute__())
    663 with shorten_traceback():
--> 664     results = schedule(dsk, keys, **kwargs)
    666 return repack([f(r, *a) for r, (f, a) in zip(results, postcomputes)])

File ~/miniforge3/envs/sdba/lib/python3.10/queue.py:171, in Queue.get(self, block, timeout)
    169 elif timeout is None:
    170     while not self._qsize():
--> 171         self.not_empty.wait()
    172 elif timeout < 0:
    173     raise ValueError("'timeout' must be a non-negative number")

File ~/miniforge3/envs/sdba/lib/python3.10/threading.py:320, in Condition.wait(self, timeout)
    318 try:    # restore state no matter what (e.g., KeyboardInterrupt)
    319     if timeout is None:
--> 320         waiter.acquire()
    321         gotit = True
    322     else:

KeyboardInterrupt: 

3. Download & Load Data¶

MSWX data is stored as daily NetCDF files on Google Drive (one file per day per variable). The fetch() method checks for existing local files and only downloads missing ones. The load() method reads all files into a single xarray.Dataset.

In [ ]:

print(ds)

print(ds)

INFO | file_cache is only supported with oauth2client<4.0.0

📂 0 exist, 13149 missing — fetching tasmin from Drive...

---------------------------------------------------------------------------
KeyboardInterrupt                         Traceback (most recent call last)
Cell In[3], line 1
----> 1 ds = extractor.extract()
      3 print(ds)

File /beegfs/muduchuru/pkgs_fnl/climdata/climdata/utils/wrapper_workflow.py:74, in update_ds.<locals>.decorator.<locals>.wrapper(self, *args, **kwargs)
     72 @wraps(func)
     73 def wrapper(self, *args, **kwargs):
---> 74     ds = func(self, *args, **kwargs)
     75     if ds is not None:
     76         self.current_ds = ds

File /beegfs/muduchuru/pkgs_fnl/climdata/climdata/utils/wrapper_workflow.py:576, in ClimateExtractor.extract(self)
    574     mswx = climdata.MSWX(cfg)
    575     mswx.extract(**extract_kwargs)
--> 576     mswx.load(var)
    577     ds_vars.append(mswx.dataset)
    578 ds = xr.merge(ds_vars)

File /beegfs/muduchuru/pkgs_fnl/climdata/climdata/datasets/MSWX.py:177, in MSWXmirror.load(self, variable)
    175 # Get folder ID and list of files
    176 folder_id = self.cfg.dsinfo[self.cfg.dataset.upper()]["variables"][variable]["folder_id"]
--> 177 files = self.fetch(folder_id, variable)
    178 if not files:
    179     raise RuntimeError(f"No files found for variable '{variable}' in Drive or local directory.")

File /beegfs/muduchuru/pkgs_fnl/climdata/climdata/datasets/MSWX.py:72, in MSWXmirror.fetch(self, folder_id, variable)
     67 creds = service_account.Credentials.from_service_account_file(
     68     self.cfg.dsinfo.mswx.params.google_service_account, scopes=SCOPES
     69 )
     70 service = build('drive', 'v3', credentials=creds)
---> 72 drive_files = list_drive_files(folder_id, service)
     73 valid_filenames = set(missing_files)
     74 files_to_download = [f for f in drive_files if f['name'] in valid_filenames]

File /beegfs/muduchuru/pkgs_fnl/climdata/climdata/utils/utils_download.py:28, in list_drive_files(folder_id, service)
     21 page_token = None
     23 while True:
     24     results = service.files().list(
     25         q=f"'{folder_id}' in parents and trashed = false",
     26         fields="files(id, name), nextPageToken",
     27         pageToken=page_token
---> 28     ).execute()
     30     files.extend(results.get("files", []))
     31     page_token = results.get("nextPageToken", None)

File ~/miniforge3/envs/sdba/lib/python3.10/site-packages/googleapiclient/_helpers.py:130, in positional.<locals>.positional_decorator.<locals>.positional_wrapper(*args, **kwargs)
    128     elif positional_parameters_enforcement == POSITIONAL_WARNING:
    129         logger.warning(message)
--> 130 return wrapped(*args, **kwargs)

File ~/miniforge3/envs/sdba/lib/python3.10/site-packages/googleapiclient/http.py:923, in HttpRequest.execute(self, http, num_retries)
    920     self.headers["content-length"] = str(len(self.body))
    922 # Handle retries for server-side errors.
--> 923 resp, content = _retry_request(
    924     http,
    925     num_retries,
    926     "request",
    927     self._sleep,
    928     self._rand,
    929     str(self.uri),
    930     method=str(self.method),
    931     body=self.body,
    932     headers=self.headers,
    933 )
    935 for callback in self.response_callbacks:
    936     callback(resp)

File ~/miniforge3/envs/sdba/lib/python3.10/site-packages/googleapiclient/http.py:191, in _retry_request(http, num_retries, req_type, sleep, rand, uri, method, *args, **kwargs)
    189 try:
    190     exception = None
--> 191     resp, content = http.request(uri, method, *args, **kwargs)
    192 # Retry on SSL errors and socket timeout errors.
    193 except _ssl_SSLError as ssl_error:

File ~/miniforge3/envs/sdba/lib/python3.10/site-packages/google_auth_httplib2.py:218, in AuthorizedHttp.request(self, uri, method, body, headers, redirections, connection_type, **kwargs)
    215     body_stream_position = body.tell()
    217 # Make the request.
--> 218 response, content = self.http.request(
    219     uri,
    220     method,
    221     body=body,
    222     headers=request_headers,
    223     redirections=redirections,
    224     connection_type=connection_type,
    225     **kwargs
    226 )
    228 # If the response indicated that the credentials needed to be
    229 # refreshed, then refresh the credentials and re-attempt the
    230 # request.
    231 # A stored token may expire between the time it is retrieved and
    232 # the time the request is made, so we may need to try twice.
    233 if (
    234     response.status in self._refresh_status_codes
    235     and _credential_refresh_attempt < self._max_refresh_attempts
    236 ):

File ~/miniforge3/envs/sdba/lib/python3.10/site-packages/httplib2/__init__.py:1724, in Http.request(self, uri, method, body, headers, redirections, connection_type)
   1722             content = b""
   1723         else:
-> 1724             (response, content) = self._request(
   1725                 conn, authority, uri, request_uri, method, body, headers, redirections, cachekey,
   1726             )
   1727 except Exception as e:
   1728     is_timeout = isinstance(e, socket.timeout)

File ~/miniforge3/envs/sdba/lib/python3.10/site-packages/httplib2/__init__.py:1444, in Http._request(self, conn, host, absolute_uri, request_uri, method, body, headers, redirections, cachekey)
   1441 if auth:
   1442     auth.request(method, request_uri, headers, body)
-> 1444 (response, content) = self._conn_request(conn, request_uri, method, body, headers)
   1446 if auth:
   1447     if auth.response(response, body):

File ~/miniforge3/envs/sdba/lib/python3.10/site-packages/httplib2/__init__.py:1396, in Http._conn_request(self, conn, request_uri, method, body, headers)
   1394     pass
   1395 try:
-> 1396     response = conn.getresponse()
   1397 except (http.client.BadStatusLine, http.client.ResponseNotReady):
   1398     # If we get a BadStatusLine on the first try then that means
   1399     # the connection just went stale, so retry regardless of the
   1400     # number of RETRIES set.
   1401     if not seen_bad_status_line and i == 1:

File ~/miniforge3/envs/sdba/lib/python3.10/http/client.py:1375, in HTTPConnection.getresponse(self)
   1373 try:
   1374     try:
-> 1375         response.begin()
   1376     except ConnectionError:
   1377         self.close()

File ~/miniforge3/envs/sdba/lib/python3.10/http/client.py:318, in HTTPResponse.begin(self)
    316 # read until we get a non-100 response
    317 while True:
--> 318     version, status, reason = self._read_status()
    319     if status != CONTINUE:
    320         break

File ~/miniforge3/envs/sdba/lib/python3.10/http/client.py:279, in HTTPResponse._read_status(self)
    278 def _read_status(self):
--> 279     line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
    280     if len(line) > _MAXLINE:
    281         raise LineTooLong("status line")

File ~/miniforge3/envs/sdba/lib/python3.10/socket.py:717, in SocketIO.readinto(self, b)
    715 while True:
    716     try:
--> 717         return self._sock.recv_into(b)
    718     except timeout:
    719         self._timeout_occurred = True

File ~/miniforge3/envs/sdba/lib/python3.10/ssl.py:1307, in SSLSocket.recv_into(self, buffer, nbytes, flags)
   1303     if flags != 0:
   1304         raise ValueError(
   1305           "non-zero flags not allowed in calls to recv_into() on %s" %
   1306           self.__class__)
-> 1307     return self.read(nbytes, buffer)
   1308 else:
   1309     return super().recv_into(buffer, nbytes, flags)

File ~/miniforge3/envs/sdba/lib/python3.10/ssl.py:1163, in SSLSocket.read(self, len, buffer)
   1161 try:
   1162     if buffer is not None:
-> 1163         return self._sslobj.read(len, buffer)
   1164     else:
   1165         return self._sslobj.read(len)

KeyboardInterrupt: 

4. Inspect the Dataset¶

In [ ]:

# Inspect one variable
ds_tasmin

# Inspect one variable ds_tasmin

In [ ]:

# Convert to pandas DataFrame for quick inspection
df_tasmin = ds_tasmin.to_dataframe().reset_index()
df_tasmin.head()

# Convert to pandas DataFrame for quick inspection df_tasmin = ds_tasmin.to_dataframe().reset_index() df_tasmin.head()

5. Spatial Extraction — Bounding Box¶

In [ ]:

overrides_box = [
    "dataset=mswx",
    "time_range.start_date=2010-01-01",
    "time_range.end_date=2010-03-31",
    "variables=[tasmax,pr]",
    "data_dir=./data",
    "dsinfo.mswx.params.google_service_account=./.climdata_conf/service.json",
]

extractor_box = ClimData(overrides=overrides_box)
mswx_box = MSWXmirror(extractor_box.cfg)

# Extract a bounding box (Germany)
mswx_box.extract(box={
    "lon_min": 5.8,
    "lon_max": 15.0,
    "lat_min": 47.3,
    "lat_max": 55.1
})

ds_box = mswx_box.load(variable="tasmax")
print(ds_box)

overrides_box = [ "dataset=mswx", "time_range.start_date=2010-01-01", "time_range.end_date=2010-03-31", "variables=[tasmax,pr]", "data_dir=./data", "dsinfo.mswx.params.google_service_account=./.climdata_conf/service.json", ] extractor_box = ClimData(overrides=overrides_box) mswx_box = MSWXmirror(extractor_box.cfg) # Extract a bounding box (Germany) mswx_box.extract(box={ "lon_min": 5.8, "lon_max": 15.0, "lat_min": 47.3, "lat_max": 55.1 }) ds_box = mswx_box.load(variable="tasmax") print(ds_box)

6. Run the Full Workflow via ClimData¶

Use ClimData.run_workflow() to chain extraction, optional imputation, climate index calculation, and saving to NetCDF in one call.

In [ ]:

overrides_wf = [
    "dataset=mswx",
    "lat=52",
    "lon=13",
    "time_range.start_date=2010-01-01",
    "time_range.end_date=2010-12-31",
    "variables=[tasmin,tasmax,pr]",
    "data_dir=./data",
    "dsinfo.mswx.params.google_service_account=./.climdata_conf/service.json",
    "index=tn10p",      # Cold nights climate extreme index
    "impute=BRITS",     # Deep-learning gap-filling (optional)
]

seq = ["extract", "impute", "calc_index", "to_nc"]

extractor_wf = ClimData(overrides=overrides_wf)
result = extractor_wf.run_workflow(actions=seq)
result

overrides_wf = [ "dataset=mswx", "lat=52", "lon=13", "time_range.start_date=2010-01-01", "time_range.end_date=2010-12-31", "variables=[tasmin,tasmax,pr]", "data_dir=./data", "dsinfo.mswx.params.google_service_account=./.climdata_conf/service.json", "index=tn10p", # Cold nights climate extreme index "impute=BRITS", # Deep-learning gap-filling (optional) ] seq = ["extract", "impute", "calc_index", "to_nc"] extractor_wf = ClimData(overrides=overrides_wf) result = extractor_wf.run_workflow(actions=seq) result

7. Save to CSV¶

In [ ]:

mswx.save_csv("mswx_point_2010.csv")
print("Saved to mswx_point_2010.csv")

mswx.save_csv("mswx_point_2010.csv") print("Saved to mswx_point_2010.csv")