From 508b00e39f6483abbdfc482e1c2d2c4228ecd17a Mon Sep 17 00:00:00 2001 From: EC2 Default User Date: Mon, 13 May 2024 10:11:53 +0000 Subject: [PATCH] add degree days to CLI options and lint --- README.md | 19 ++++++++- src/hazard/cli.py | 32 +++++++++++++- src/hazard/models/degree_days.py | 13 +++--- src/hazard/protocols.py | 8 +++- src/hazard/services.py | 71 +++++++++++++++++++++++++++++++- src/hazard/sources/osc_zarr.py | 15 +++---- workflow.cwl | 12 +++--- 7 files changed, 145 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 6862781..bb9612e 100644 --- a/README.md +++ b/README.md @@ -39,13 +39,13 @@ os_climate_hazard days_tas_above_indicator --store $HOME/hazard_example ### In a docker container -First, build the image. +First, build the image. ``` docker build -t os-hazard-indicator -f dockerfiles/Dockerfile . ``` -Then, you can run an example the following way. In the example, we save the data locally to /data/hazard-test-container in the container. To have access to the output once the container finished running, we are mounting `/data` from the container to `$HOME/data` locally. +Then, you can run an example the following way. In the example, we save the data locally to /data/hazard-test-container in the container. To have access to the output once the container finished running, we are mounting `/data` from the container to `$HOME/data` locally. ``` docker run -it -v $HOME/data:/data os-hazard-indicator os_climate_hazard days_tas_above_indicator --store /data/hazard-test-container @@ -53,6 +53,21 @@ docker run -it -v $HOME/data:/data os-hazard-indicator os_climate_hazard days_ta ### In a CWL (Common Workflow Language) workflow +## Arguments parsing in CLI + +The CLI for this package is built on top of google's `fire` package. Arguments passed to the command line are parsed not based on their declared type in python but their string value at runtime. For complex types such as lists, this can lead to confusion if internal list elements have special characters like hyphens. + +This is an example of command that _works_ for the argument `gcm_list` (note the single and double quotes in that argument value) + +``` +os_climate_hazard degree_days_indicator --store $HOME/data/hazard-test --scenario_list [ssp126,ssp585] --central_year_list [2070,2080] --window_years 1 --gcm_list "['ACCESS-CM2','NorESM2-MM']" +``` + +And this is an example that does not : + +``` +os_climate_hazard degree_days_indicator --store $HOME/data/hazard-test --scenario_list [ssp126,ssp585] --central_year_list [2070,2080] --window_years 1 --gcm_list [ACCESS-CM2,NorESM2-MM] +``` # Contributing diff --git a/src/hazard/cli.py b/src/hazard/cli.py index 010e8c4..2bd9798 100644 --- a/src/hazard/cli.py +++ b/src/hazard/cli.py @@ -15,7 +15,7 @@ def days_tas_above_indicator( prefix: Optional[str] = None, store: Optional[str] = None, inventory_format: Optional[str] = "osc", - extra_xarray_store: Optional[bool] = False + extra_xarray_store: Optional[bool] = False, ): hazard_services.days_tas_above_indicator( @@ -32,9 +32,39 @@ def days_tas_above_indicator( ) +def degree_days_indicator( + gcm_list: List[str] = ["NorESM2-MM"], + scenario_list: List[str] = ["ssp585"], + threshold_temperature: float = 32, + central_year_list: List[int] = [2090], + central_year_historical: int = 2005, + window_years: int = 1, + bucket: Optional[str] = None, + prefix: Optional[str] = None, + store: Optional[str] = None, + extra_xarray_store: Optional[bool] = False, + inventory_format: Optional[str] = "osc", +): + + hazard_services.degree_days_indicator( + gcm_list, + scenario_list, + threshold_temperature, + central_year_list, + central_year_historical, + window_years, + bucket, + prefix, + store, + extra_xarray_store, + inventory_format, + ) + + class Cli(object): def __init__(self) -> None: self.days_tas_above_indicator = days_tas_above_indicator + self.degree_days_indicator = degree_days_indicator def cli(): diff --git a/src/hazard/models/degree_days.py b/src/hazard/models/degree_days.py index 15b02c7..73879de 100644 --- a/src/hazard/models/degree_days.py +++ b/src/hazard/models/degree_days.py @@ -56,7 +56,8 @@ def __init__( Defaults to [2010, 2030, 2040, 2050]. """ - self.threshold: float = 273.15 + threshold # in Kelvin; degree days above 32C + self.threshold: float = 273.15 + threshold # in Kelvin; degree days above {threshold}C + self.threshold_c: float = threshold # 1995 to 2014 (2010), 2021 tp 2040 (2030), 2031 to 2050 (2040), 2041 to 2060 (2050) self.window_years = window_years self.gcms = gcms @@ -88,10 +89,10 @@ def _resource(self): description = f.read().replace("\u00c2\u00b0", "\u00b0") resource = HazardResource( hazard_type="ChronicHeat", - indicator_id="mean_degree_days/above/32c", + indicator_id=f"mean_degree_days/above/{self.threshold_c}c", indicator_model_gcm="{gcm}", - path="chronic_heat/osc/v2/mean_degree_days_v2_above_32c_{gcm}_{scenario}_{year}", - display_name="Mean degree days above 32°C/{gcm}", + path=f"chronic_heat/osc/v2/mean_degree_days_v2_above_{self.threshold_c}c" + "_{gcm}_{scenario}_{year}", + display_name=f"Mean degree days above {self.threshold_c}°C/" + "{gcm}", description=description, params={"gcm": list(self.gcms)}, group_id="", @@ -107,7 +108,7 @@ def _resource(self): units="degree days", ), bounds=[(-180.0, 85.0), (180.0, 85.0), (180.0, -60.0), (-180.0, -60.0)], - path="mean_degree_days_v2_above_32c_{gcm}_{scenario}_{year}_map", + path=f"mean_degree_days_v2_above_{self.threshold_c}c_" + "{gcm}_{scenario}_{year}_map", source="map_array", ), units="degree days", @@ -159,7 +160,7 @@ def _generate_map( if top > 85.05 or bottom < -85.05: raise ValueError("invalid range") logger.info(f"Writing map file {map_path}") - target.write(map_path, reprojected) + target.write(map_path, reprojected, spatial_coords=False) return def _average_degree_days( diff --git a/src/hazard/protocols.py b/src/hazard/protocols.py index 9cce227..d335071 100644 --- a/src/hazard/protocols.py +++ b/src/hazard/protocols.py @@ -30,7 +30,13 @@ def read(self, path: str) -> xr.DataArray: ... class WriteDataArray(Protocol): """Write DataArray.""" - def write(self, path: str, data_array: xr.DataArray, chunks: Optional[List[int]] = None, spatial_coords: Optional[bool]= True): ... + def write( + self, + path: str, + data_array: xr.DataArray, + chunks: Optional[List[int]] = None, + spatial_coords: Optional[bool] = True, + ): ... class ReadWriteDataArray(ReadDataArray, WriteDataArray): ... # noqa: E701 diff --git a/src/hazard/services.py b/src/hazard/services.py index 80124cf..1d1363e 100644 --- a/src/hazard/services.py +++ b/src/hazard/services.py @@ -1,11 +1,12 @@ import logging # noqa: E402 -from typing import List, Optional +from typing import List, Optional, Tuple, Union from dask.distributed import Client, LocalCluster # noqa: E402 from fsspec.implementations.local import LocalFileSystem from hazard.docs_store import DocStore # type: ignore # noqa: E402 from hazard.models.days_tas_above import DaysTasAboveIndicator # noqa: E402 +from hazard.models.degree_days import DegreeDays # noqa: E402 from hazard.sources.nex_gddp_cmip6 import NexGddpCmip6 # noqa: E402 from hazard.sources.osc_zarr import OscZarr # noqa: E402 @@ -25,7 +26,7 @@ def days_tas_above_indicator( prefix: Optional[str] = None, store: Optional[str] = None, extra_xarray_store: Optional[bool] = False, - inventory_format: Optional[str] = "osc" + inventory_format: Optional[str] = "osc", ): """ Run the days_tas_above indicator generation for a list of models,scenarios, thresholds, @@ -62,3 +63,69 @@ def days_tas_above_indicator( docs_store.update_inventory(model.inventory(), format=inventory_format) model.run_all(source, target, client=client) + + +def degree_days_indicator( + gcm_list: List[str] = ["NorESM2-MM"], + scenario_list: List[str] = ["ssp585"], + threshold_temperature: float = 32, + central_year_list: List[int] = [2090], + central_year_historical: int = 2005, + window_years: int = 1, + bucket: Optional[str] = None, + prefix: Optional[str] = None, + store: Optional[str] = None, + extra_xarray_store: Optional[bool] = False, + inventory_format: Optional[str] = "osc", +): + """ + Run the degree days indicator generation for a list of models,scenarios, a threshold temperature, + central years and a given size of years window over which to compute the average. + Store the result in a zarr store, locally if `store` is provided, else in an S3 + bucket if `bucket` and `prefix` are provided. + An inventory filed is stored at the root of the zarr directory. + """ + + docs_store, target, client = setup(bucket, prefix, store, extra_xarray_store) + + source = NexGddpCmip6() + + model = DegreeDays( + threshold=threshold_temperature, + window_years=window_years, + gcms=gcm_list, + scenarios=scenario_list, + central_years=central_year_list, + central_year_historical=central_year_historical, + ) + + docs_store.update_inventory(model.inventory(), format=inventory_format) + + model.run_all(source, target, client=client) + + +def setup( + bucket: Optional[str] = None, + prefix: Optional[str] = None, + store: Optional[str] = None, + extra_xarray_store: Optional[bool] = False, +) -> Tuple[Union[DocStore, OscZarr, Client]]: + """ + initialize output store, docs store and local dask client + """ + + if store is not None: + docs_store = DocStore(fs=LocalFileSystem(), local_path=store) + target = OscZarr(store=store, extra_xarray_store=extra_xarray_store) + else: + if bucket is None or prefix is None: + raise ValueError("either of `store`, or `bucket` and `prefix` together, must be provided") + else: + docs_store = DocStore(bucket=bucket, prefix=prefix) + target = OscZarr(bucket=bucket, prefix=prefix, extra_xarray_store=extra_xarray_store) + + cluster = LocalCluster(processes=False) + + client = Client(cluster) + + return docs_store, target, client diff --git a/src/hazard/sources/osc_zarr.py b/src/hazard/sources/osc_zarr.py index ba6cd3f..3e0cb3f 100644 --- a/src/hazard/sources/osc_zarr.py +++ b/src/hazard/sources/osc_zarr.py @@ -22,7 +22,7 @@ def __init__( prefix: str = "hazard", s3: Optional[s3fs.S3File] = None, store: Optional[Any] = None, - extra_xarray_store: Optional[bool] = False + extra_xarray_store: Optional[bool] = False, ): """For reading and writing to OSC Climate Zarr storage. If store is provided this is used, otherwise if S3File is provided, this is used. @@ -47,7 +47,7 @@ def __init__( self.root = zarr.group(store=store) self.extra_xarray_store = extra_xarray_store - + def create_empty( self, path: str, @@ -132,13 +132,14 @@ def if_exists_remove(self, path): if path in self.root: self.root.pop(path) - def write(self, path: str, da: xr.DataArray, chunks: Optional[List[int]] = None, spatial_coords: Optional[bool] = True): + def write( + self, path: str, da: xr.DataArray, chunks: Optional[List[int]] = None, spatial_coords: Optional[bool] = True + ): if self.extra_xarray_store and spatial_coords: - self.write_data_array(f'{path}-xarray', da) - + self.write_data_array(f"{path}-xarray", da) + self.write_zarr(path, da, chunks) - def write_zarr(self, path: str, da: xr.DataArray, chunks: Optional[List[int]] = None): """Write DataArray according to the standard OS-Climate conventions. @@ -158,7 +159,7 @@ def write_zarr(self, path: str, da: xr.DataArray, chunks: Optional[List[int]] = chunks=chunks, ) z[:, :, :] = data[:, :, :] - + def write_slice(self, path, z_slice: slice, y_slice: slice, x_slice: slice, da: np.ndarray): z = self.root[path] z[z_slice, y_slice, x_slice] = np.expand_dims(da, 0) diff --git a/workflow.cwl b/workflow.cwl index a011847..b638b26 100644 --- a/workflow.cwl +++ b/workflow.cwl @@ -16,8 +16,8 @@ $graph: ResourceRequirement: coresMax: 2 ramMax: 4096 - - inputs: + + inputs: gcm_list: type: string default: "[NorESM2-MM]" @@ -51,18 +51,18 @@ $graph: dockerPull: public.ecr.aws/c9k5s3u3/os-hazard-indicator baseCommand: ["os_climate_hazard", "days_tas_above_indicator", "--inventory_format", "stac", "--store", "./indicator", "--"] - + arguments: [] - + inputs: gcm_list: type: string inputBinding: prefix: --gcm_list separate: true - + outputs: indicator-results: type: Directory outputBinding: - glob: "./indicator" \ No newline at end of file + glob: "./indicator"