From 30637a39aa70e3f1e99fd368d52865e2b90fd3c5 Mon Sep 17 00:00:00 2001 From: bnb32 Date: Thu, 2 May 2024 14:05:50 -0600 Subject: [PATCH 01/61] init commit --- reV/bespoke/bespoke.py | 69 +++++++++++++++++++-------------------- reV/utilities/__init__.py | 38 +++++++++++++++++++++ 2 files changed, 71 insertions(+), 36 deletions(-) diff --git a/reV/bespoke/bespoke.py b/reV/bespoke/bespoke.py index e88a9db58..dffd76804 100644 --- a/reV/bespoke/bespoke.py +++ b/reV/bespoke/bespoke.py @@ -3,41 +3,40 @@ reV bespoke wind plant analysis tools """ # pylint: disable=anomalous-backslash-in-string -from inspect import signature -import time -import logging import copy -import pandas as pd -import numpy as np -import os import json -import psutil +import logging +import os +import time +from concurrent.futures import as_completed from importlib import import_module +from inspect import signature from numbers import Number -from concurrent.futures import as_completed from warnings import warn +import numpy as np +import pandas as pd +import psutil +from rex.joint_pd.joint_pd import JointPD +from rex.multi_year_resource import MultiYearWindResource +from rex.renewable_resource import WindResource +from rex.utilities.bc_parse_table import parse_bc_table +from rex.utilities.execution import SpawnProcessPool +from rex.utilities.loggers import create_dirs, log_mem +from rex.utilities.utilities import parse_year + from reV.config.output_request import SAMOutputRequest -from reV.generation.generation import Gen -from reV.SAM.generation import WindPower, WindPowerPD from reV.econ.utilities import lcoe_fcr -from reV.handlers.outputs import Outputs +from reV.generation.generation import Gen from reV.handlers.exclusions import ExclusionLayers +from reV.handlers.outputs import Outputs +from reV.SAM.generation import WindPower, WindPowerPD +from reV.supply_curve.aggregation import AggFileHandler, BaseAggregation from reV.supply_curve.extent import SupplyCurveExtent from reV.supply_curve.points import AggregationSupplyCurvePoint as AggSCPoint from reV.supply_curve.points import SupplyCurvePoint -from reV.supply_curve.aggregation import BaseAggregation, AggFileHandler -from reV.utilities.exceptions import (EmptySupplyCurvePointError, - FileInputError) -from reV.utilities import log_versions, ModuleName - -from rex.utilities.bc_parse_table import parse_bc_table -from rex.joint_pd.joint_pd import JointPD -from rex.renewable_resource import WindResource -from rex.multi_year_resource import MultiYearWindResource -from rex.utilities.loggers import log_mem, create_dirs -from rex.utilities.utilities import parse_year -from rex.utilities.execution import SpawnProcessPool +from reV.utilities import ModuleName, log_versions +from reV.utilities.exceptions import EmptySupplyCurvePointError, FileInputError logger = logging.getLogger(__name__) @@ -79,7 +78,7 @@ def __init__(self, res_fpath, sc_gid_to_hh, sc_gid_to_res_gid): self._pre_load_data() def _pre_load_data(self): - """Pre-load the resource data. """ + """Pre-load the resource data.""" for sc_gid, gids in self.sc_gid_to_res_gid.items(): hh = self.sc_gid_to_hh[sc_gid] @@ -186,7 +185,7 @@ def __getitem__(self, key): class BespokeSinglePlant: - """Framework for analyzing and optimized a wind plant layout specific to + """Framework for analyzing and optimizing a wind plant layout specific to the local wind resource and exclusions for a single reV supply curve point. """ @@ -530,7 +529,7 @@ def _parse_gid_map(gid_map): for i in gid_map['gid'].keys()} elif gid_map.endswith('.json'): - with open(gid_map, 'r') as f: + with open(gid_map) as f: gid_map = json.load(f) return gid_map @@ -892,7 +891,7 @@ def initialize_wind_plant_ts(self): @property def wind_plant_pd(self): - """reV WindPowerPD compute object for plant layout optimization based + """ReV WindPowerPD compute object for plant layout optimization based on wind joint probability distribution Returns @@ -909,7 +908,7 @@ def wind_plant_pd(self): @property def wind_plant_ts(self): - """reV WindPower compute object(s) based on wind resource timeseries + """ReV WindPower compute object(s) based on wind resource timeseries data keyed by year Returns @@ -1066,9 +1065,7 @@ def _check_sys_inputs(plant1, plant2, """ bad = [] for k, v in plant1.sam_sys_inputs.items(): - if k not in plant2.sam_sys_inputs: - bad.append(k) - elif str(v) != str(plant2.sam_sys_inputs[k]): + if k not in plant2.sam_sys_inputs or str(v) != str(plant2.sam_sys_inputs[k]): bad.append(k) bad = [b for b in bad if b not in ignore] if any(bad): @@ -1277,7 +1274,7 @@ def __init__(self, excl_fpath, res_fpath, tm_dset, objective_function, resolution=64, excl_area=None, data_layers=None, pre_extract_inclusions=False, prior_run=None, gid_map=None, bias_correct=None, pre_load_data=False): - """reV bespoke analysis class. + r"""ReV bespoke analysis class. Much like generation, ``reV`` bespoke analysis runs SAM simulations by piping in renewable energy resource data (usually @@ -1861,7 +1858,7 @@ def _check_files(self): assert any(f.dsets) def _pre_load_data(self, pre_load_data): - """Pre-load resource data, if requested. """ + """Pre-load resource data, if requested.""" if not pre_load_data: return @@ -1898,7 +1895,7 @@ def _hh_for_sc_gid(self, sc_gid): return int(config["wind_turbine_hub_ht"]) def _pre_loaded_data_for_sc_gid(self, sc_gid): - """Pre-load data for a given SC GID, if requested. """ + """Pre-load data for a given SC GID, if requested.""" if self._pre_loaded_data is None: return None @@ -1981,7 +1978,7 @@ def meta(self): @property def slice_lookup(self): - """dict | None: Lookup mapping sc_point_gid to exclusion slice. """ + """Dict | None: Lookup mapping sc_point_gid to exclusion slice.""" if self._slice_lookup is None and self._inclusion_mask is not None: with SupplyCurveExtent(self._excl_fpath, resolution=self._resolution) as sc: @@ -2146,7 +2143,7 @@ def save_outputs(self, out_fpath): except Exception as e: msg = 'Failed to write "{}" to disk.'.format(dset) logger.exception(msg) - raise IOError(msg) from e + raise OSError(msg) from e logger.info('Saved output data to: {}'.format(out_fpath)) return out_fpath diff --git a/reV/utilities/__init__.py b/reV/utilities/__init__.py index ae58cdb7f..9db866072 100644 --- a/reV/utilities/__init__.py +++ b/reV/utilities/__init__.py @@ -3,11 +3,49 @@ reV utilities. """ from enum import Enum + import PySAM from rex.utilities.loggers import log_versions as rex_log_versions + from reV.version import __version__ +class SupplyCurvePointSummaryKey(str, Enum): + """An enumerated map to Supply Curve Point summary keys. + + Each output name should match the name of a key in + meth:`AggregationSupplyCurvePoint.summary` or + meth:`GenerationSupplyCurvePoint.point_summary` + """ + + SC_POINT_GID = 'sc_point_gid' + SOURCE_GIDS = 'source_gids' + GID_COUNTS = 'gid_counts' + GID = 'gid' + N_GIDS = 'n_gids' + RES_GIDS = 'res_gids' + GEN_GIDS = 'gen_gids' + AREA_SQ_KM = 'area_sq_km' + LATITUDE = 'latitude' + LONGITUDE = 'longitude' + COUNTRY = 'country' + STATE = 'state' + COUNTY = 'county' + COUNTRY = 'country', + ELEVATION = 'elevation' + TIMEZONE = 'timezone' + MEAN_CF = 'mean_cf' + MEAN_LCOE = 'mean_lcoe' + MEAN_RES = 'mean_res' + CAPACITY = 'capacity' + + def __str__(self): + return self.value + + def __format__(self, format_spec): + return str.__format__(self.value, format_spec) + + class ModuleName(str, Enum): """A collection of the module names available in reV. From 08d091bb0b477217c2bdd1578287bf5d4910470e Mon Sep 17 00:00:00 2001 From: bnb32 Date: Fri, 3 May 2024 16:32:21 -0600 Subject: [PATCH 02/61] MetaKeyName class for replacing hardcoded strings in summaries and metas --- examples/aws_pcluster/make_project_points.py | 5 +- examples/bespoke_wind_plants/single_run.py | 28 +- examples/marine_energy/plot_lcoe.py | 5 +- reV/SAM/SAM.py | 42 +- reV/SAM/econ.py | 33 +- reV/SAM/generation.py | 192 ++++--- reV/SAM/windbos.py | 172 +++--- reV/bespoke/bespoke.py | 75 +-- reV/bespoke/place_turbines.py | 9 +- reV/config/project_points.py | 53 +- reV/econ/econ.py | 57 +- reV/econ/economies_of_scale.py | 30 +- reV/generation/base.py | 59 +- reV/generation/generation.py | 110 ++-- reV/handlers/exclusions.py | 30 +- reV/handlers/multi_year.py | 25 +- reV/hybrids/hybrids.py | 83 +-- reV/nrwal/nrwal.py | 42 +- reV/qa_qc/cli_qa_qc.py | 24 +- reV/qa_qc/qa_qc.py | 30 +- reV/qa_qc/summary.py | 85 +-- reV/rep_profiles/rep_profiles.py | 97 ++-- reV/supply_curve/aggregation.py | 539 +++++++++++------- reV/supply_curve/competitive_wind_farms.py | 43 +- reV/supply_curve/exclusions.py | 21 +- reV/supply_curve/extent.py | 15 +- reV/supply_curve/points.py | 248 ++++---- reV/supply_curve/sc_aggregation.py | 65 ++- reV/supply_curve/supply_curve.py | 121 ++-- reV/supply_curve/tech_mapping.py | 22 +- reV/utilities/__init__.py | 29 +- reV/utilities/pytest_utils.py | 9 +- tests/eagle.py | 24 +- tests/hsds.py | 5 +- tests/test_bespoke.py | 211 ++++--- tests/test_config.py | 32 +- tests/test_curtailment.py | 35 +- tests/test_econ_lcoe.py | 27 +- tests/test_econ_of_scale.py | 77 +-- tests/test_econ_windbos.py | 25 +- tests/test_gen_5min.py | 13 +- tests/test_gen_config.py | 26 +- tests/test_gen_csp.py | 11 +- tests/test_gen_forecast.py | 28 +- tests/test_gen_geothermal.py | 60 +- tests/test_gen_linear.py | 9 +- tests/test_gen_pv.py | 94 +-- tests/test_gen_swh.py | 12 +- tests/test_gen_time_scale.py | 15 +- tests/test_gen_trough.py | 9 +- tests/test_gen_wave.py | 14 +- tests/test_gen_wind.py | 89 +-- tests/test_gen_wind_losses.py | 12 +- tests/test_handlers_multiyear.py | 48 +- tests/test_handlers_outputs.py | 14 +- tests/test_handlers_transmission.py | 5 +- tests/test_hybrids.py | 109 ++-- tests/test_losses_power_curve.py | 94 +-- tests/test_losses_scheduled.py | 86 +-- tests/test_nrwal.py | 124 ++-- tests/test_qa_qc_summary.py | 9 +- tests/test_rep_profiles.py | 109 ++-- tests/test_sam.py | 33 +- tests/test_supply_curve_aggregation.py | 42 +- .../test_supply_curve_aggregation_friction.py | 25 +- tests/test_supply_curve_compute.py | 86 +-- tests/test_supply_curve_points.py | 30 +- tests/test_supply_curve_sc_aggregation.py | 102 ++-- tests/test_supply_curve_tech_mapping.py | 31 +- tests/test_supply_curve_vpd.py | 18 +- tests/test_supply_curve_wind_dirs.py | 23 +- 71 files changed, 2336 insertions(+), 1878 deletions(-) diff --git a/examples/aws_pcluster/make_project_points.py b/examples/aws_pcluster/make_project_points.py index 8b9d08514..43a69173a 100644 --- a/examples/aws_pcluster/make_project_points.py +++ b/examples/aws_pcluster/make_project_points.py @@ -3,7 +3,6 @@ """ from rex import Resource - if __name__ == '__main__': fp = '/nrel/nsrdb/v3/nsrdb_2019.h5' fp = '/nrel/wtk/conus/wtk_conus_2007.h5' @@ -22,7 +21,7 @@ print(meta[mask]) pp = meta[mask] - pp['gid'] = pp.index.values + pp[MetaKeyName.GID] = pp.index.values pp['config'] = 'def' - pp = pp[['gid', 'config']] + pp = pp[[MetaKeyName.GID, 'config']] pp.to_csv('./points_front_range.csv', index=False) diff --git a/examples/bespoke_wind_plants/single_run.py b/examples/bespoke_wind_plants/single_run.py index 0e0ae0342..4a205999f 100644 --- a/examples/bespoke_wind_plants/single_run.py +++ b/examples/bespoke_wind_plants/single_run.py @@ -2,24 +2,29 @@ """ An example single run to get bespoke wind plant layout """ -import numpy as np -import matplotlib.pyplot as plt -from reV.bespoke.bespoke import BespokeSinglePlant -from reV.bespoke.plotting_functions import plot_poly, plot_turbines,\ - plot_windrose -from reV import TESTDATADIR -from reV.supply_curve.tech_mapping import TechMapping - import json import os import shutil import tempfile +import matplotlib.pyplot as plt +import numpy as np + +from reV import TESTDATADIR +from reV.bespoke.bespoke import BespokeSinglePlant +from reV.bespoke.plotting_functions import ( + plot_poly, + plot_turbines, + plot_windrose, +) +from reV.supply_curve.tech_mapping import TechMapping +from reV.utilities import MetaKeyName + SAM = os.path.join(TESTDATADIR, 'SAM/i_windpower.json') EXCL = os.path.join(TESTDATADIR, 'ri_exclusions/ri_exclusions.h5') RES = os.path.join(TESTDATADIR, 'wtk/ri_100_wtk_{}.h5') TM_DSET = 'techmap_wtk_ri_100' -AGG_DSET = ('cf_mean', 'cf_profile') +AGG_DSET = (MetaKeyName.CF_MEAN, MetaKeyName.CF_PROFILE) # note that this differs from the EXCL_DICT = {'ri_srtm_slope': {'inclusion_range': (None, 5), @@ -29,7 +34,7 @@ 'ri_reeds_regions': {'inclusion_range': (None, 400), 'exclude_nodata': False}} -with open(SAM, 'r') as f: +with open(SAM) as f: SAM_SYS_INPUTS = json.load(f) SAM_SYS_INPUTS['wind_farm_wake_model'] = 2 @@ -56,7 +61,8 @@ 1E5 * 0.1 + (1 - 0.1))""" objective_function = "cost / aep" - output_request = ('system_capacity', 'cf_mean', 'cf_profile') + output_request = ('system_capacity', MetaKeyName.CF_MEAN, + MetaKeyName.CF_PROFILE) gid = 33 with tempfile.TemporaryDirectory() as td: excl_fp = os.path.join(td, 'ri_exclusions.h5') diff --git a/examples/marine_energy/plot_lcoe.py b/examples/marine_energy/plot_lcoe.py index 8842228d5..c77b038cf 100644 --- a/examples/marine_energy/plot_lcoe.py +++ b/examples/marine_energy/plot_lcoe.py @@ -2,8 +2,9 @@ Simple plot script for wave LCOE """ import os -import pandas as pd + import matplotlib.pyplot as plt +import pandas as pd fps = ['./atlantic_rm5/atlantic_rm5_agg.csv', './pacific_rm5/pacific_rm5_agg.csv', @@ -11,7 +12,7 @@ for fp in fps: df = pd.read_csv(fp) - a = plt.scatter(df.longitude, df.latitude, c=df['mean_lcoe'], + a = plt.scatter(df.longitude, df.latitude, c=df[MetaKeyName.MEAN_LCOE], s=0.5, vmin=0, vmax=1500) plt.colorbar(a, label='lcoe_fcr ($/MWh)') tag = os.path.basename(fp).replace('_agg.csv', '') diff --git a/reV/SAM/SAM.py b/reV/SAM/SAM.py index e8bb9438e..927af2d05 100644 --- a/reV/SAM/SAM.py +++ b/reV/SAM/SAM.py @@ -6,22 +6,34 @@ import copy import json import logging -import numpy as np import os -import pandas as pd from warnings import warn -import PySAM.GenericSystem as generic - -from reV.utilities.exceptions import (SAMInputWarning, SAMInputError, - SAMExecutionError, ResourceError) -from rex.multi_file_resource import (MultiFileResource, MultiFileNSRDB, - MultiFileWTK) -from rex.renewable_resource import (WindResource, SolarResource, NSRDB, - WaveResource, GeothermalResource) +import numpy as np +import pandas as pd +import PySAM.GenericSystem as generic +from rex.multi_file_resource import ( + MultiFileNSRDB, + MultiFileResource, + MultiFileWTK, +) from rex.multi_res_resource import MultiResolutionResource +from rex.renewable_resource import ( + NSRDB, + GeothermalResource, + SolarResource, + WaveResource, + WindResource, +) from rex.utilities.utilities import check_res_file +from reV.utilities import MetaKeyName +from reV.utilities.exceptions import ( + ResourceError, + SAMExecutionError, + SAMInputError, + SAMInputWarning, +) logger = logging.getLogger(__name__) @@ -169,10 +181,10 @@ def _make_res_kwargs(cls, res_handler, project_points, output_request, elif res_handler == WindResource: args += (project_points.h, ) kwargs['icing'] = project_points.sam_config_obj.icing - if project_points.curtailment is not None: - if project_points.curtailment.precipitation: - # make precip rate available for curtailment analysis - kwargs['precip_rate'] = True + if (project_points.curtailment is not None and + project_points.curtailment.precipitation): + # make precip rate available for curtailment analysis + kwargs['precip_rate'] = True elif res_handler == GeothermalResource: args += (project_points.d, ) @@ -567,7 +579,7 @@ def __init__(self, meta, sam_sys_inputs, output_request, Site-agnostic SAM system model inputs arguments. output_request : list Requested SAM outputs (e.g., 'cf_mean', 'annual_energy', - 'cf_profile', 'gen_profile', 'energy_yield', 'ppa_price', + , 'gen_profile', 'energy_yield', 'ppa_price', 'lcoe_fcr'). site_sys_inputs : dict Optional set of site-specific SAM system inputs to complement the diff --git a/reV/SAM/econ.py b/reV/SAM/econ.py index 648606812..5a33c3d76 100644 --- a/reV/SAM/econ.py +++ b/reV/SAM/econ.py @@ -4,17 +4,19 @@ Wraps the NREL-PySAM lcoefcr and singleowner modules with additional reV features. """ -from copy import deepcopy import logging -import numpy as np +from copy import deepcopy from warnings import warn + +import numpy as np import PySAM.Lcoefcr as PySamLCOE import PySAM.Singleowner as PySamSingleOwner -from reV.SAM.defaults import DefaultSingleOwner, DefaultLCOE from reV.handlers.outputs import Outputs -from reV.SAM.windbos import WindBos +from reV.SAM.defaults import DefaultLCOE, DefaultSingleOwner from reV.SAM.SAM import RevPySam +from reV.SAM.windbos import WindBos +from reV.utilities import MetaKeyName from reV.utilities.exceptions import SAMExecutionError logger = logging.getLogger(__name__) @@ -22,6 +24,7 @@ class Economic(RevPySam): """Base class for SAM economic models.""" + MODULE = None def __init__(self, sam_sys_inputs, site_sys_inputs=None, @@ -172,7 +175,7 @@ def _get_cf_profiles(sites, cf_file, year): with Outputs(cf_file) as cfh: # get the index location of the site in question - site_gids = list(cfh.get_meta_arr('gid')) + site_gids = list(cfh.get_meta_arr(MetaKeyName.GID)) isites = [site_gids.index(s) for s in sites] # look for the cf_profile dataset @@ -335,6 +338,7 @@ def reV_run(cls, site, site_df, inputs, output_request): class LCOE(Economic): """SAM LCOE model. """ + MODULE = 'lcoefcr' PYSAM = PySamLCOE @@ -375,7 +379,7 @@ def _parse_lcoe_inputs(site_df, cf_file, year): # get the cf_file meta data gid's to use as indexing tools with Outputs(cf_file) as cfh: - site_gids = list(cfh.meta['gid']) + site_gids = list(cfh.meta[MetaKeyName.GID]) calc_aey = False if 'annual_energy' not in site_df: @@ -463,6 +467,7 @@ def reV_run(cls, points_control, site_df, cf_file, year, class SingleOwner(Economic): """SAM single owner economic model. """ + MODULE = 'singleowner' PYSAM = PySamSingleOwner @@ -497,14 +502,14 @@ def _windbos(inputs): """ outputs = {} - if inputs is not None: - if 'total_installed_cost' in inputs: - if isinstance(inputs['total_installed_cost'], str): - if inputs['total_installed_cost'].lower() == 'windbos': - wb = WindBos(inputs) - inputs['total_installed_cost'] = \ - wb.total_installed_cost - outputs = wb.output + if (inputs is not None and + 'total_installed_cost' in inputs and + isinstance(inputs['total_installed_cost'], str) and + inputs['total_installed_cost'].lower() == 'windbos'): + wb = WindBos(inputs) + inputs['total_installed_cost'] = \ + wb.total_installed_cost + outputs = wb.output return inputs, outputs @staticmethod diff --git a/reV/SAM/generation.py b/reV/SAM/generation.py index 5ebfa8803..b4ec2c413 100644 --- a/reV/SAM/generation.py +++ b/reV/SAM/generation.py @@ -5,9 +5,8 @@ additional reV features. """ import copy -import os import logging - +import os from abc import ABC, abstractmethod from tempfile import TemporaryDirectory from warnings import warn @@ -26,22 +25,28 @@ import PySAM.TroughPhysicalProcessHeat as PySamTpph import PySAM.Windpower as PySamWindPower -from reV.losses import ScheduledLossesMixin, PowerCurveLossesMixin -from reV.SAM.defaults import (DefaultGeothermal, - DefaultPvWattsv5, - DefaultPvWattsv8, - DefaultPvSamv1, - DefaultWindPower, - DefaultTcsMoltenSalt, - DefaultSwh, - DefaultTroughPhysicalProcessHeat, - DefaultLinearFresnelDsgIph, - DefaultMhkWave) +from reV.losses import PowerCurveLossesMixin, ScheduledLossesMixin +from reV.SAM.defaults import ( + DefaultGeothermal, + DefaultLinearFresnelDsgIph, + DefaultMhkWave, + DefaultPvSamv1, + DefaultPvWattsv5, + DefaultPvWattsv8, + DefaultSwh, + DefaultTcsMoltenSalt, + DefaultTroughPhysicalProcessHeat, + DefaultWindPower, +) from reV.SAM.econ import LCOE, SingleOwner from reV.SAM.SAM import RevPySam +from reV.utilities import MetaKeyName from reV.utilities.curtailment import curtail -from reV.utilities.exceptions import (SAMInputWarning, SAMExecutionError, - InputError) +from reV.utilities.exceptions import ( + InputError, + SAMExecutionError, + SAMInputWarning, +) logger = logging.getLogger(__name__) @@ -240,19 +245,23 @@ def tz_elev_check(sam_sys_inputs, site_sys_inputs, meta): if meta is not None: if sam_sys_inputs is not None: - if 'elevation' in sam_sys_inputs: - meta['elevation'] = sam_sys_inputs['elevation'] - if 'timezone' in sam_sys_inputs: - meta['timezone'] = int(sam_sys_inputs['timezone']) + if MetaKeyName.ELEVATION in sam_sys_inputs: + meta[MetaKeyName.ELEVATION] = \ + sam_sys_inputs[MetaKeyName.ELEVATION] + if MetaKeyName.TIMEZONE in sam_sys_inputs: + meta[MetaKeyName.TIMEZONE] = \ + int(sam_sys_inputs[MetaKeyName.TIMEZONE]) # site-specific inputs take priority over generic system inputs if site_sys_inputs is not None: - if 'elevation' in site_sys_inputs: - meta['elevation'] = site_sys_inputs['elevation'] - if 'timezone' in site_sys_inputs: - meta['timezone'] = int(site_sys_inputs['timezone']) - - if 'timezone' not in meta: + if MetaKeyName.ELEVATION in site_sys_inputs: + meta[MetaKeyName.ELEVATION] = \ + site_sys_inputs[MetaKeyName.ELEVATION] + if MetaKeyName.TIMEZONE in site_sys_inputs: + meta[MetaKeyName.TIMEZONE] = \ + int(site_sys_inputs[MetaKeyName.TIMEZONE]) + + if MetaKeyName.TIMEZONE not in meta: msg = ('Need timezone input to run SAM gen. Not found in ' 'resource meta or technology json input config.') raise SAMExecutionError(msg) @@ -261,10 +270,9 @@ def tz_elev_check(sam_sys_inputs, site_sys_inputs, meta): @property def has_timezone(self): - """ Returns true if instance has a timezone set """ - if self._meta is not None: - if 'timezone' in self.meta: - return True + """Returns true if instance has a timezone set""" + if self._meta is not None and MetaKeyName.TIMEZONE in self.meta: + return True return False @@ -288,7 +296,8 @@ def cf_profile(self): 1D numpy array of capacity factor profile. Datatype is float32 and array length is 8760*time_interval. """ - return self.gen_profile() / self.sam_sys_inputs['system_capacity'] + return (self.gen_profile() / + self.sam_sys_inputs['system_capacity']) def annual_energy(self): """Get annual energy generation value in kWh from SAM. @@ -349,8 +358,10 @@ def run_gen_and_econ(self): lcoe_out_reqs = None so_out_reqs = None - lcoe_vars = ('lcoe_fcr', 'fixed_charge_rate', 'capital_cost', - 'fixed_operating_cost', 'variable_operating_cost') + lcoe_vars = ('lcoe_fcr', 'fixed_charge_rate', + 'capital_cost', + 'fixed_operating_cost', + 'variable_operating_cost') so_vars = ('ppa_price', 'lcoe_real', 'lcoe_nom', 'project_return_aftertax_npv', 'flip_actual_irr', 'gross_revenue') @@ -514,10 +525,13 @@ def reV_run(cls, points_control, res_file, site_df, class AbstractSamGenerationFromWeatherFile(AbstractSamGeneration, ABC): - """Base class for running sam generation with a weather file on disk. """ - WF_META_DROP_COLS = {'elevation', 'timezone', 'country', 'state', 'county', - 'urban', 'population', 'landcover', 'latitude', - 'longitude'} + """Base class for running sam generation with a weather file on disk.""" + + WF_META_DROP_COLS = {MetaKeyName.ELEVATION, MetaKeyName.TIMEZONE, + MetaKeyName.COUNTRY, MetaKeyName.STATE, + MetaKeyName.COUNTY, 'urban', 'population', + 'landcover', MetaKeyName.LATITUDE, + MetaKeyName.LONGITUDE} @property @abstractmethod @@ -587,16 +601,18 @@ def _create_pysam_wfile(self, resource, meta): # ------- Process metadata m = pd.DataFrame(meta).T - timezone = m['timezone'] + timezone = m[MetaKeyName.TIMEZONE] m['Source'] = 'NSRDB' m['Location ID'] = meta.name m['City'] = '-' - m['State'] = m['state'].apply(lambda x: '-' if x == 'None' else x) - m['Country'] = m['country'].apply(lambda x: '-' if x == 'None' else x) - m['Latitude'] = m['latitude'] - m['Longitude'] = m['longitude'] + m['State'] = m[MetaKeyName.STATE].apply( + lambda x: '-' if x == 'None' else x) + m['Country'] = m[MetaKeyName.COUNTRY].apply( + lambda x: '-' if x == 'None' else x) + m['Latitude'] = m[MetaKeyName.LATITUDE] + m['Longitude'] = m[MetaKeyName.LONGITUDE] m['Time Zone'] = timezone - m['Elevation'] = m['elevation'] + m['Elevation'] = m[MetaKeyName.ELEVATION] m['Local Time Zone'] = timezone m['Dew Point Units'] = 'c' m['DHI Units'] = 'w/m2' @@ -734,24 +750,24 @@ def set_resource_data(self, resource, meta): # ensure that resource array length is multiple of 8760 arr = self.ensure_res_len(arr, time_index) - n_roll = int(self._meta['timezone'] * self.time_interval) + n_roll = int(self._meta[MetaKeyName.TIMEZONE] * + self.time_interval) arr = np.roll(arr, n_roll) - if var in irrad_vars: - if np.min(arr) < 0: - warn('Solar irradiance variable "{}" has a minimum ' - 'value of {}. Truncating to zero.' - .format(var, np.min(arr)), SAMInputWarning) - arr = np.where(arr < 0, 0, arr) + if var in irrad_vars and np.min(arr) < 0: + warn('Solar irradiance variable "{}" has a minimum ' + 'value of {}. Truncating to zero.' + .format(var, np.min(arr)), SAMInputWarning) + arr = np.where(arr < 0, 0, arr) resource[var] = arr.tolist() - resource['lat'] = meta['latitude'] - resource['lon'] = meta['longitude'] - resource['tz'] = meta['timezone'] + resource['lat'] = meta[MetaKeyName.LATITUDE] + resource['lon'] = meta[MetaKeyName.LONGITUDE] + resource['tz'] = meta[MetaKeyName.TIMEZONE] - if 'elevation' in meta: - resource['elev'] = meta['elevation'] + if MetaKeyName.ELEVATION in meta: + resource['elev'] = meta[MetaKeyName.ELEVATION] else: resource['elev'] = 0.0 @@ -905,10 +921,10 @@ def set_resource_data(self, resource, meta): respectively. """ - bad_location_input = ((meta['latitude'] < -90) - | (meta['latitude'] > 90) - | (meta['longitude'] < -180) - | (meta['longitude'] > 180)) + bad_location_input = ((meta[MetaKeyName.LATITUDE] < -90) + | (meta[MetaKeyName.LATITUDE] > 90) + | (meta[MetaKeyName.LONGITUDE] < -180) + | (meta[MetaKeyName.LONGITUDE] > 180)) if bad_location_input.any(): raise ValueError("Detected latitude/longitude values outside of " "the range -90 to 90 and -180 to 180, " @@ -945,15 +961,14 @@ def set_latitude_tilt_az(sam_sys_inputs, meta): warn('No tilt specified, setting at latitude.', SAMInputWarning) set_tilt = True - else: - if (sam_sys_inputs['tilt'] == 'lat' - or sam_sys_inputs['tilt'] == 'latitude'): - set_tilt = True + elif (sam_sys_inputs['tilt'] == 'lat' + or sam_sys_inputs['tilt'] == MetaKeyName.LATITUDE): + set_tilt = True if set_tilt: # set tilt to abs(latitude) - sam_sys_inputs['tilt'] = np.abs(meta['latitude']) - if meta['latitude'] > 0: + sam_sys_inputs['tilt'] = np.abs(meta[MetaKeyName.LATITUDE]) + if meta[MetaKeyName.LATITUDE] > 0: # above the equator, az = 180 sam_sys_inputs['azimuth'] = 180 else: @@ -1018,7 +1033,8 @@ def cf_profile(self): Datatype is float32 and array length is 8760*time_interval. PV CF is calculated as AC power / DC nameplate. """ - return self.gen_profile() / self.sam_sys_inputs['system_capacity'] + return (self.gen_profile() / + self.sam_sys_inputs['system_capacity']) def cf_profile_ac(self): """Get hourly AC capacity factor (frac) profile in local timezone. @@ -1127,6 +1143,7 @@ def collect_outputs(self, output_lookup=None): class PvWattsv5(AbstractSamPv): """Photovoltaic (PV) generation with pvwattsv5. """ + MODULE = 'pvwattsv5' PYSAM = PySamPv5 @@ -1144,6 +1161,7 @@ def default(): class PvWattsv7(AbstractSamPv): """Photovoltaic (PV) generation with pvwattsv7. """ + MODULE = 'pvwattsv7' PYSAM = PySamPv7 @@ -1161,6 +1179,7 @@ def default(): class PvWattsv8(AbstractSamPv): """Photovoltaic (PV) generation with pvwattsv8. """ + MODULE = 'pvwattsv8' PYSAM = PySamPv8 @@ -1220,6 +1239,7 @@ def default(): class TcsMoltenSalt(AbstractSamSolar): """Concentrated Solar Power (CSP) generation with tower molten salt """ + MODULE = 'tcsmolten_salt' PYSAM = PySamCSP @@ -1234,7 +1254,8 @@ def cf_profile(self): 1D numpy array of capacity factor profile. Datatype is float32 and array length is 8760*time_interval. """ - x = np.abs(self.gen_profile() / self.sam_sys_inputs['system_capacity']) + x = np.abs(self.gen_profile() / + self.sam_sys_inputs['system_capacity']) return x @staticmethod @@ -1252,6 +1273,7 @@ class SolarWaterHeat(AbstractSamGenerationFromWeatherFile): """ Solar Water Heater generation """ + MODULE = 'solarwaterheat' PYSAM = PySamSwh PYSAM_WEATHER_TAG = 'solar_resource_file' @@ -1271,6 +1293,7 @@ class LinearDirectSteam(AbstractSamGenerationFromWeatherFile): """ Process heat linear Fresnel direct steam generation """ + MODULE = 'lineardirectsteam' PYSAM = PySamLds PYSAM_WEATHER_TAG = 'file_name' @@ -1305,6 +1328,7 @@ class TroughPhysicalHeat(AbstractSamGenerationFromWeatherFile): """ Trough Physical Process Heat generation """ + MODULE = 'troughphysicalheat' PYSAM = PySamTpph PYSAM_WEATHER_TAG = 'file_name' @@ -1508,6 +1532,7 @@ class Geothermal(AbstractSamGenerationFromWeatherFile): ``time_index_step=2`` yields hourly output, and so forth). """ + # Per Matt Prilliman on 2/22/24, it's unclear where this ratio originates, # but SAM errors out if it's exceeded. MAX_RT_TO_EGS_RATIO = 1.134324 @@ -1580,7 +1605,7 @@ def set_resource_data(self, resource, meta): self._set_costs() def _set_resource_temperature(self, resource): - """Set resource temp from data if user did not specify it. """ + """Set resource temp from data if user did not specify it.""" if "resource_temp" in self.sam_sys_inputs: logger.debug("Found 'resource_temp' value in SAM config: {:.2f}" @@ -1640,7 +1665,7 @@ def _set_egs_plant_design_temperature(self): self.sam_sys_inputs["design_temp"] = resource_temp def _set_nameplate_to_match_resource_potential(self, resource): - """Set the nameplate capacity to match the resource potential. """ + """Set the nameplate capacity to match the resource potential.""" if "nameplate" in self.sam_sys_inputs: msg = ('Found "nameplate" input in config! Resource potential ' @@ -1679,7 +1704,7 @@ def _set_resource_potential_to_match_gross_output(self): self.sam_sys_inputs["resource_potential"] = -1 return - gross_gen = (getattr(self.pysam.Outputs, "gross_output") + gross_gen = (self.pysam.Outputs.gross_output * self._RESOURCE_POTENTIAL_MULT) if "resource_potential" in self.sam_sys_inputs: msg = ('Setting "resource_potential" is not allowed! Updating ' @@ -1696,9 +1721,9 @@ def _set_resource_potential_to_match_gross_output(self): ncw = self.sam_sys_inputs.pop("num_confirmation_wells", self._DEFAULT_NUM_CONFIRMATION_WELLS) self.sam_sys_inputs["prod_and_inj_wells_to_drill"] = ( - getattr(self.pysam.Outputs, "num_wells_getem_output") + self.pysam.Outputs.num_wells_getem_output - ncw - + getattr(self.pysam.Outputs, "num_wells_getem_inj")) + + self.pysam.Outputs.num_wells_getem_inj) self["ui_calculations_only"] = 0 def _set_costs(self): @@ -1900,8 +1925,8 @@ def __init__(self, *args, **kwargs): class WindPower(AbstractSamWind): - """Class for Wind generation from SAM - """ + """Class for Wind generation from SAM""" + MODULE = 'windpower' PYSAM = PySamWindPower @@ -1952,7 +1977,7 @@ def set_resource_data(self, resource, meta): if 'rh' in resource: # set relative humidity for icing. rh = self.ensure_res_len(resource['rh'].values, time_index) - n_roll = int(meta['timezone'] * self.time_interval) + n_roll = int(meta[MetaKeyName.TIMEZONE] * self.time_interval) rh = np.roll(rh, n_roll, axis=0) data_dict['rh'] = rh.tolist() @@ -1960,14 +1985,14 @@ def set_resource_data(self, resource, meta): # ensure that resource array length is multiple of 8760 # roll the truncated resource array to local timezone temp = self.ensure_res_len(resource[var_list].values, time_index) - n_roll = int(meta['timezone'] * self.time_interval) + n_roll = int(meta[MetaKeyName.TIMEZONE] * self.time_interval) temp = np.roll(temp, n_roll, axis=0) data_dict['data'] = temp.tolist() - data_dict['lat'] = float(meta['latitude']) - data_dict['lon'] = float(meta['longitude']) - data_dict['tz'] = int(meta['timezone']) - data_dict['elev'] = float(meta['elevation']) + data_dict['lat'] = float(meta[MetaKeyName.LATITUDE]) + data_dict['lon'] = float(meta[MetaKeyName.LONGITUDE]) + data_dict['tz'] = int(meta[MetaKeyName.TIMEZONE]) + data_dict['elev'] = float(meta[MetaKeyName.ELEVATION]) time_index = self.ensure_res_len(time_index, time_index) data_dict['minute'] = time_index.minute.tolist() @@ -2086,6 +2111,7 @@ def set_resource_data(self, ws_edges, wd_edges, wind_dist): class MhkWave(AbstractSamGeneration): """Class for Wave generation from SAM """ + MODULE = 'mhkwave' PYSAM = PySamMhkWave @@ -2131,12 +2157,12 @@ def set_resource_data(self, resource, meta): # roll the truncated resource array to local timezone for var in ['significant_wave_height', 'energy_period']: arr = self.ensure_res_len(resource[var].values, time_index) - n_roll = int(meta['timezone'] * self.time_interval) + n_roll = int(meta[MetaKeyName.TIMEZONE] * self.time_interval) data_dict[var] = np.roll(arr, n_roll, axis=0).tolist() - data_dict['lat'] = meta['latitude'] - data_dict['lon'] = meta['longitude'] - data_dict['tz'] = meta['timezone'] + data_dict['lat'] = meta[MetaKeyName.LATITUDE] + data_dict['lon'] = meta[MetaKeyName.LONGITUDE] + data_dict['tz'] = meta[MetaKeyName.TIMEZONE] time_index = self.ensure_res_len(time_index, time_index) data_dict['minute'] = time_index.minute diff --git a/reV/SAM/windbos.py b/reV/SAM/windbos.py index 9acd93ffc..2ca125da8 100644 --- a/reV/SAM/windbos.py +++ b/reV/SAM/windbos.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- -""" -SAM Wind Balance of System Cost Model -""" +"""SAM Wind Balance of System Cost Model""" + from copy import deepcopy + import numpy as np from PySAM.PySSC import ssc_sim_from_dict @@ -12,45 +12,47 @@ class WindBos: """Wind Balance of System Cost Model.""" - MODULE = 'windbos' + MODULE = "windbos" # keys for the windbos input data dictionary. # Some keys may not be found explicitly in the SAM input. - KEYS = ('tech_model', - 'financial_model', - 'machine_rating', - 'rotor_diameter', - 'hub_height', - 'number_of_turbines', - 'interconnect_voltage', - 'distance_to_interconnect', - 'site_terrain', - 'turbine_layout', - 'soil_condition', - 'construction_time', - 'om_building_size', - 'quantity_test_met_towers', - 'quantity_permanent_met_towers', - 'weather_delay_days', - 'crane_breakdowns', - 'access_road_entrances', - 'turbine_capital_cost', - 'turbine_cost_per_kw', - 'tower_top_mass', - 'delivery_assist_required', - 'pad_mount_transformer_required', - 'new_switchyard_required', - 'rock_trenching_required', - 'mv_thermal_backfill', - 'mv_overhead_collector', - 'performance_bond', - 'contingency', - 'warranty_management', - 'sales_and_use_tax', - 'overhead', - 'profit_margin', - 'development_fee', - 'turbine_transportation') + KEYS = ( + "tech_model", + "financial_model", + "machine_rating", + "rotor_diameter", + "hub_height", + "number_of_turbines", + "interconnect_voltage", + "distance_to_interconnect", + "site_terrain", + "turbine_layout", + "soil_condition", + "construction_time", + "om_building_size", + "quantity_test_met_towers", + "quantity_permanent_met_towers", + "weather_delay_days", + "crane_breakdowns", + "access_road_entrances", + "turbine_capital_cost", + "turbine_cost_per_kw", + "tower_top_mass", + "delivery_assist_required", + "pad_mount_transformer_required", + "new_switchyard_required", + "rock_trenching_required", + "mv_thermal_backfill", + "mv_overhead_collector", + "performance_bond", + "contingency", + "warranty_management", + "sales_and_use_tax", + "overhead", + "profit_margin", + "development_fee", + "turbine_transportation", + ) def __init__(self, inputs): """ @@ -64,14 +66,15 @@ def __init__(self, inputs): self._datadict = {} self._inputs = inputs - self._special = {'tech_model': 'windbos', - 'financial_model': 'none', - 'machine_rating': self.machine_rating, - 'hub_height': self.hub_height, - 'rotor_diameter': self.rotor_diameter, - 'number_of_turbines': self.number_of_turbines, - 'turbine_capital_cost': self.turbine_capital_cost, - } + self._special = { + "tech_model": "windbos", + "financial_model": "none", + "machine_rating": self.machine_rating, + "hub_height": self.hub_height, + "rotor_diameter": self.rotor_diameter, + "number_of_turbines": self.number_of_turbines, + "turbine_capital_cost": self.turbine_capital_cost, + } self._parse_inputs() self._out = ssc_sim_from_dict(self._datadict) @@ -83,52 +86,55 @@ def _parse_inputs(self): if k in self._special: self._datadict[k] = self._special[k] elif k not in self._inputs: - raise SAMInputError('Windbos requires input key: "{}"' - .format(k)) + raise SAMInputError( + 'Windbos requires input key: "{}"'.format(k) + ) else: self._datadict[k] = self._inputs[k] @property def machine_rating(self): """Single turbine machine rating either from input or power curve.""" - if 'machine_rating' in self._inputs: - return self._inputs['machine_rating'] + if "machine_rating" in self._inputs: + return self._inputs["machine_rating"] else: - return np.max(self._inputs['wind_turbine_powercurve_powerout']) + return np.max(self._inputs["wind_turbine_powercurve_powerout"]) @property def hub_height(self): """Turbine hub height.""" - if 'wind_turbine_hub_ht' in self._inputs: - return self._inputs['wind_turbine_hub_ht'] + if "wind_turbine_hub_ht" in self._inputs: + return self._inputs["wind_turbine_hub_ht"] else: - return self._inputs['hub_height'] + return self._inputs["hub_height"] @property def rotor_diameter(self): """Turbine rotor diameter.""" - if 'wind_turbine_rotor_diameter' in self._inputs: - return self._inputs['wind_turbine_rotor_diameter'] + if "wind_turbine_rotor_diameter" in self._inputs: + return self._inputs["wind_turbine_rotor_diameter"] else: - return self._inputs['rotor_diameter'] + return self._inputs["rotor_diameter"] @property def number_of_turbines(self): """Number of turbines either based on input or system (farm) capacity and machine rating""" - if 'number_of_turbines' in self._inputs: - return self._inputs['number_of_turbines'] + if "number_of_turbines" in self._inputs: + return self._inputs["number_of_turbines"] else: - return self._inputs['system_capacity'] / self.machine_rating + return ( + self._inputs['system_capacity'] / self.machine_rating + ) @property def turbine_capital_cost(self): """Returns zero (no turbine capital cost for WindBOS input, and assigns any input turbine_capital_cost to an attr""" - if 'turbine_capital_cost' in self._inputs: - self._turbine_capital_cost = self._inputs['turbine_capital_cost'] + if "turbine_capital_cost" in self._inputs: + self._turbine_capital_cost = self._inputs["turbine_capital_cost"] else: self._turbine_capital_cost = 0.0 return 0.0 @@ -136,23 +142,23 @@ def turbine_capital_cost(self): @property def bos_cost(self): """Get the balance of system cost ($).""" - return self._out['project_total_budgeted_cost'] + return self._out["project_total_budgeted_cost"] @property def turbine_cost(self): """Get the turbine cost ($).""" - tcost = ((self._inputs['turbine_cost_per_kw'] - * self.machine_rating - * self.number_of_turbines) - + (self._turbine_capital_cost - * self.number_of_turbines)) + tcost = ( + self._inputs["turbine_cost_per_kw"] + * self.machine_rating + * self.number_of_turbines + ) + (self._turbine_capital_cost * self.number_of_turbines) return tcost @property def sales_tax_mult(self): """Get a sales tax multiplier (frac of the total installed cost).""" - basis = self._inputs.get('sales_tax_basis', 0) / 100 - tax = self._datadict.get('sales_and_use_tax', 0) / 100 + basis = self._inputs.get("sales_tax_basis", 0) / 100 + tax = self._datadict.get("sales_and_use_tax", 0) / 100 return basis * tax @property @@ -168,16 +174,23 @@ def total_installed_cost(self): @property def output(self): """Get a dictionary containing the cost breakdown.""" - output = {'total_installed_cost': self.total_installed_cost, - 'turbine_cost': self.turbine_cost, - 'sales_tax_cost': self.sales_tax_cost, - 'bos_cost': self.bos_cost} + output = { + "total_installed_cost": self.total_installed_cost, + "turbine_cost": self.turbine_cost, + "sales_tax_cost": self.sales_tax_cost, + "bos_cost": self.bos_cost, + } return output # pylint: disable-msg=W0613 @classmethod - def reV_run(cls, points_control, site_df, - output_request=('total_installed_cost',), **kwargs): + def reV_run( + cls, + points_control, + site_df, + output_request=("total_installed_cost",), + **kwargs, + ): """Execute SAM SingleOwner simulations based on reV points control. Parameters @@ -216,7 +229,8 @@ def reV_run(cls, points_control, site_df, wb = cls(site_inputs) - out[site] = {k: v for k, v in wb.output.items() - if k in output_request} + out[site] = { + k: v for k, v in wb.output.items() if k in output_request + } return out diff --git a/reV/bespoke/bespoke.py b/reV/bespoke/bespoke.py index dffd76804..de8d42c4c 100644 --- a/reV/bespoke/bespoke.py +++ b/reV/bespoke/bespoke.py @@ -35,7 +35,7 @@ from reV.supply_curve.extent import SupplyCurveExtent from reV.supply_curve.points import AggregationSupplyCurvePoint as AggSCPoint from reV.supply_curve.points import SupplyCurvePoint -from reV.utilities import ModuleName, log_versions +from reV.utilities import MetaKeyName, ModuleName, log_versions from reV.utilities.exceptions import EmptySupplyCurvePointError, FileInputError logger = logging.getLogger(__name__) @@ -197,7 +197,8 @@ def __init__(self, gid, excl, res, tm_dset, sam_sys_inputs, fixed_operating_cost_function, variable_operating_cost_function, min_spacing='5x', wake_loss_multiplier=1, ga_kwargs=None, - output_request=('system_capacity', 'cf_mean'), + output_request=('system_capacity', + 'cf_mean'), ws_bins=(0.0, 20.0, 5.0), wd_bins=(0.0, 360.0, 45.0), excl_dict=None, inclusion_mask=None, data_layers=None, resolution=64, excl_area=None, exclusion_shape=None, @@ -469,8 +470,9 @@ def _parse_output_req(self): dset = req.replace('_mean', '') self._outputs[req] = self.res_df[dset].mean() - if ('lcoe_fcr' in self._out_req - and 'fixed_charge_rate' not in self.original_sam_sys_inputs): + if ('lcoe_fcr' in self._out_req and + ('fixed_charge_rate' not in + self.original_sam_sys_inputs)): msg = ('User requested "lcoe_fcr" but did not input ' '"fixed_charge_rate" in the SAM system config.') logger.error(msg) @@ -481,9 +483,10 @@ def _parse_prior_run(self): sure the SAM system inputs are set accordingly.""" # {meta_column: sam_sys_input_key} - required = {'capacity': 'system_capacity', - 'turbine_x_coords': 'wind_farm_xCoordinates', - 'turbine_y_coords': 'wind_farm_yCoordinates'} + required = { + MetaKeyName.CAPACITY: 'system_capacity' + MetaKeyName.TURBINE_X_COORDS: 'wind_farrm_xCoordinates', + MetaKeyName.TURBINE_Y_COORDS: 'wind_farrm_yCoordinates'} if self._prior_meta: missing = [k for k in required if k not in self.meta] @@ -496,7 +499,7 @@ def _parse_prior_run(self): self._sam_sys_inputs[sam_sys_key] = prior_value # convert reV supply curve cap in MW to SAM capacity in kW - self._sam_sys_inputs['system_capacity'] *= 1e3 + self._sam_sys_inputs['system_capacity' @staticmethod def _parse_gid_map(gid_map): @@ -523,10 +526,10 @@ def _parse_gid_map(gid_map): if isinstance(gid_map, str): if gid_map.endswith('.csv'): gid_map = pd.read_csv(gid_map).to_dict() - assert 'gid' in gid_map, 'Need "gid" in gid_map column' + assert MetaKeyName.GID in gid_map, 'Need "gid" in gid_map column' assert 'gid_map' in gid_map, 'Need "gid_map" in gid_map column' - gid_map = {gid_map['gid'][i]: gid_map['gid_map'][i] - for i in gid_map['gid'].keys()} + gid_map = {gid_map[MetaKeyName.GID][i]: gid_map['gid_map'][i] + for i in gid_map[MetaKeyName.GID].keys()} elif gid_map.endswith('.json'): with open(gid_map) as f: @@ -743,22 +746,22 @@ def meta(self): row_ind, col_ind = sc.get_sc_row_col_ind(self.sc_point.gid) self._meta = pd.DataFrame( - {'sc_point_gid': self.sc_point.gid, - 'sc_row_ind': row_ind, - 'sc_col_ind': col_ind, - 'gid': self.sc_point.gid, - 'latitude': self.sc_point.latitude, - 'longitude': self.sc_point.longitude, - 'timezone': self.sc_point.timezone, - 'country': self.sc_point.country, - 'state': self.sc_point.state, - 'county': self.sc_point.county, - 'elevation': self.sc_point.elevation, - 'offshore': self.sc_point.offshore, - 'res_gids': res_gids, - 'gid_counts': gid_counts, - 'n_gids': self.sc_point.n_gids, - 'area_sq_km': self.sc_point.area, + {MetaKeyName.SC_POINT_GID: self.sc_point.gid, + MetaKeyName.SC_ROW_IND: row_ind, + MetaKeyName.SC_COL_IND: col_ind, + MetaKeyName.GID: self.sc_point.gid, + MetaKeyName.LATITUDE: self.sc_point.latitude, + MetaKeyName.LONGITUDE: self.sc_point.longitude, + MetaKeyName.TIMEZONE: self.sc_point.timezone, + MetaKeyName.COUNTRY: self.sc_point.country, + MetaKeyName.STATE: self.sc_point.state, + MetaKeyName.COUNTY: self.sc_point.county, + MetaKeyName.ELEVATION: self.sc_point.elevation, + MetaKeyName.OFFSHORE: self.sc_point.offshore, + MetaKeyName.RES_GIDS: res_gids, + MetaKeyName.GID_COUNTS: gid_counts, + MetaKeyName.N_GIDS: self.sc_point.n_gids, + MetaKeyName.AREA_SQ_KM: self.sc_point.area, }, index=[self.sc_point.gid]) return self._meta @@ -960,7 +963,7 @@ def recalc_lcoe(self): my_mean_lcoe = lcoe_fcr(fcr, cap_cost, foc, aep, voc) self._outputs['lcoe_fcr-means'] = my_mean_lcoe - self._meta['mean_lcoe'] = my_mean_lcoe + self._meta[MetaKeyName.MEAN_LCOE] = my_mean_lcoe def get_lcoe_kwargs(self): """Get a namespace of arguments for calculating LCOE based on the @@ -1065,7 +1068,8 @@ def _check_sys_inputs(plant1, plant2, """ bad = [] for k, v in plant1.sam_sys_inputs.items(): - if k not in plant2.sam_sys_inputs or str(v) != str(plant2.sam_sys_inputs[k]): + if (k not in plant2.sam_sys_inputs or + str(v) != str(plant2.sam_sys_inputs[k])): bad.append(k) bad = [b for b in bad if b not in ignore] if any(bad): @@ -1114,9 +1118,9 @@ def run_wind_plant_ts(self): # copy dataset outputs to meta data for supply curve table summary if 'cf_mean-means' in self.outputs: - self._meta.loc[:, 'mean_cf'] = self.outputs['cf_mean-means'] + self._meta.loc[:, MetaKeyName.MEAN_CF] = self.outputs['cf_mean-means'] if 'lcoe_fcr-means' in self.outputs: - self._meta.loc[:, 'mean_lcoe'] = self.outputs['lcoe_fcr-means'] + self._meta.loc[:, MetaKeyName.MEAN_LCOE] = self.outputs['lcoe_fcr-means'] self.recalc_lcoe() logger.debug('Timeseries analysis complete!') @@ -1192,7 +1196,7 @@ def run_plant_optimization(self): # copy dataset outputs to meta data for supply curve table summary # convert SAM system capacity in kW to reV supply curve cap in MW - self._meta['capacity'] = self.outputs['system_capacity'] / 1e3 + self._meta[MetaKeyName.CAPACITY] = self.outputs['system_capacity'] / 1e3 # add required ReEDS multipliers to meta baseline_cost = self.plant_optimizer.capital_cost_per_kw( @@ -1752,7 +1756,7 @@ def _parse_points(points, sam_configs): Slice or list specifying project points, string pointing to a project points csv, or a fully instantiated PointsControl object. Can also be a single site integer value. Points csv should have - 'gid' and 'config' column, the config maps to the sam_configs dict + MetaKeyName.GID and 'config' column, the config maps to the sam_configs dict keys. sam_configs : dict | str | SAMConfig SAM input configuration ID(s) and file path(s). Keys are the SAM @@ -1827,7 +1831,7 @@ def _get_prior_meta(self, gid): meta = None if self._prior_meta is not None: - mask = self._prior_meta['gid'] == gid + mask = self._prior_meta[MetaKeyName.GID] == gid if any(mask): meta = self._prior_meta[mask] @@ -2156,7 +2160,8 @@ def run_serial(cls, excl_fpath, res_fpath, tm_dset, fixed_operating_cost_function, variable_operating_cost_function, min_spacing='5x', wake_loss_multiplier=1, ga_kwargs=None, - output_request=('system_capacity', 'cf_mean'), + output_request=('system_capacity', + 'cf_mean'), ws_bins=(0.0, 20.0, 5.0), wd_bins=(0.0, 360.0, 45.0), excl_dict=None, inclusion_mask=None, area_filter_kernel='queen', min_area=None, diff --git a/reV/bespoke/place_turbines.py b/reV/bespoke/place_turbines.py index fd6bc84cb..432372aa1 100644 --- a/reV/bespoke/place_turbines.py +++ b/reV/bespoke/place_turbines.py @@ -4,11 +4,10 @@ place turbines for bespoke wind plants """ import numpy as np +from shapely.geometry import MultiPoint, MultiPolygon, Point, Polygon -from shapely.geometry import Point, Polygon, MultiPolygon, MultiPoint - -from reV.bespoke.pack_turbs import PackTurbines from reV.bespoke.gradient_free import GeneticAlgorithm +from reV.bespoke.pack_turbs import PackTurbines from reV.utilities.exceptions import WhileLoopPackingError @@ -190,7 +189,7 @@ def define_exclusions(self): self.packing_polygons = MultiPolygon([]) def initialize_packing(self): - """run the turbine packing algorithm (maximizing plant capacity) to + """Run the turbine packing algorithm (maximizing plant capacity) to define potential turbine locations that will be used as design variables in the gentic algorithm. """ @@ -364,7 +363,7 @@ def capital_cost_per_kw(self, capacity_mw): def fixed_charge_rate(self): """Fixed charge rate if input to the SAM WindPowerPD object, None if not found in inputs.""" - return self.wind_plant.sam_sys_inputs.get('fixed_charge_rate', None) + return self.wind_plant.sam_sys_inputs.get(MetaKeyName.FIXED_CHARGE_RATE, None) @property @none_until_optimized diff --git a/reV/config/project_points.py b/reV/config/project_points.py index c8bb7a8b1..587c7f718 100644 --- a/reV/config/project_points.py +++ b/reV/config/project_points.py @@ -4,26 +4,29 @@ """ import copy import logging -import numpy as np import os -import pandas as pd from warnings import warn +import numpy as np +import pandas as pd +from rex.multi_file_resource import MultiFileResource +from rex.resource import Resource +from rex.resource_extraction.resource_extraction import ( + MultiFileResourceX, + ResourceX, +) +from rex.utilities import check_res_file, parse_table + from reV.config.curtailment import Curtailment from reV.config.sam_config import SAMConfig from reV.utilities.exceptions import ConfigError, ConfigWarning -from rex.resource import Resource -from rex.multi_file_resource import MultiFileResource -from rex.resource_extraction.resource_extraction import (ResourceX, - MultiFileResourceX) -from rex.utilities import check_res_file, parse_table - logger = logging.getLogger(__name__) class PointsControl: """Class to manage and split ProjectPoints.""" + def __init__(self, project_points, sites_per_split=100): """ Parameters @@ -269,7 +272,7 @@ def __getitem__(self, site): names (keys) and values. """ - site_bool = (self.df['gid'] == site) + site_bool = (self.df[MetaKeyName.GID] == site) try: config_id = self.df.loc[site_bool, 'config'].values[0] except (KeyError, IndexError) as ex: @@ -299,7 +302,7 @@ def df(self): ------- _df : pd.DataFrame Table of sites and corresponding SAM configuration IDs. - Has columns 'gid' and 'config'. + Has columns MetaKeyName.GID and 'config'. """ return self._df @@ -384,7 +387,7 @@ def sites(self): List of integer sites (resource file gids) belonging to this instance of ProjectPoints. """ - return self.df['gid'].values.tolist() + return self.df[MetaKeyName.GID].values.tolist() @property def sites_as_slice(self): @@ -485,7 +488,7 @@ def _parse_csv(fname): Parameters ---------- fname : str - Project points .csv file (with path). Must have 'gid' and 'config' + Project points .csv file (with path). Must have MetaKeyName.GID and 'config' column names. Returns @@ -522,7 +525,7 @@ def _parse_sites(points, res_file=None): df : pd.DataFrame DataFrame mapping sites (gids) to SAM technology (config) """ - df = pd.DataFrame(columns=['gid', 'config']) + df = pd.DataFrame(columns=[MetaKeyName.GID, 'config']) if isinstance(points, int): points = [points] if isinstance(points, (list, tuple, np.ndarray)): @@ -532,7 +535,7 @@ def _parse_sites(points, res_file=None): logger.error(msg) raise RuntimeError(msg) - df['gid'] = points + df[MetaKeyName.GID] = points elif isinstance(points, slice): stop = points.stop if stop is None: @@ -547,7 +550,7 @@ def _parse_sites(points, res_file=None): else: stop = Resource(res_file).shape[1] - df['gid'] = list(range(*points.indices(stop))) + df[MetaKeyName.GID] = list(range(*points.indices(stop))) else: raise TypeError('Project Points sites needs to be set as a list, ' 'tuple, or slice, but was set as: {}' @@ -588,14 +591,14 @@ def _parse_points(cls, points, res_file=None): raise ValueError('Cannot parse Project points data from {}' .format(type(points))) - if 'gid' not in df.columns: + if MetaKeyName.GID not in df.columns: raise KeyError('Project points data must contain "gid" column.') # pylint: disable=no-member if 'config' not in df.columns: df = cls._parse_sites(points["gid"].values, res_file=res_file) - gids = df['gid'].values + gids = df[MetaKeyName.GID].values if not np.array_equal(np.sort(gids), gids): msg = ('WARNING: points are not in sequential order and will be ' 'sorted! The original order is being preserved under ' @@ -603,7 +606,7 @@ def _parse_points(cls, points, res_file=None): logger.warning(msg) warn(msg) df['points_order'] = df.index.values - df = df.sort_values('gid').reset_index(drop=True) + df = df.sort_values(MetaKeyName.GID).reset_index(drop=True) return df @@ -691,13 +694,13 @@ def index(self, gid): ind : int Row index of gid in the project points dataframe. """ - if gid not in self._df['gid'].values: + if gid not in self._df[MetaKeyName.GID].values: e = ('Requested resource gid {} is not present in the project ' 'points dataframe. Cannot return row index.'.format(gid)) logger.error(e) raise ConfigError(e) - ind = np.where(self._df['gid'] == gid)[0][0] + ind = np.where(self._df[MetaKeyName.GID] == gid)[0][0] return ind @@ -747,7 +750,7 @@ def _check_points_config_mapping(self): logger.error(msg) raise ConfigError(msg) - def join_df(self, df2, key='gid'): + def join_df(self, df2, key=MetaKeyName.GID): """Join new df2 to the _df attribute using the _df's gid as pkey. This can be used to add site-specific data to the project_points, @@ -767,7 +770,7 @@ def join_df(self, df2, key='gid'): """ # ensure df2 doesnt have any duplicate columns for suffix reasons. df2_cols = [c for c in df2.columns if c not in self._df or c == key] - self._df = pd.merge(self._df, df2[df2_cols], how='left', left_on='gid', + self._df = pd.merge(self._df, df2[df2_cols], how='left', left_on=MetaKeyName.GID, right_on=key, copy=False, validate='1:1') def get_sites_from_config(self, config): @@ -784,7 +787,7 @@ def get_sites_from_config(self, config): List of sites associated with the requested configuration ID. If the configuration ID is not recognized, an empty list is returned. """ - sites = self.df.loc[(self.df['config'] == config), 'gid'].values + sites = self.df.loc[(self.df['config'] == config), MetaKeyName.GID].values return list(sites) @@ -938,8 +941,8 @@ def lat_lon_coords(cls, lat_lons, res_file, sam_configs, tech=None, if 'points_order' in pp.df: lat_lons = lat_lons[pp.df['points_order'].values] - pp._df['latitude'] = lat_lons[:, 0] - pp._df['longitude'] = lat_lons[:, 1] + pp._df[MetaKeyName.LATITUDE] = lat_lons[:, 0] + pp._df[MetaKeyName.LONGITUDE] = lat_lons[:, 1] return pp diff --git a/reV/econ/econ.py b/reV/econ/econ.py index 662798dcb..8b0cd7403 100644 --- a/reV/econ/econ.py +++ b/reV/econ/econ.py @@ -1,26 +1,24 @@ # -*- coding: utf-8 -*- -""" -reV econ module (lcoe-fcr, single owner, etc...) -""" +"""reV econ module (lcoe-fcr, single owner, etc...)""" import logging -import numpy as np import os -import pandas as pd import pprint from warnings import warn +import numpy as np +import pandas as pd +from rex.multi_file_resource import MultiFileResource +from rex.resource import Resource +from rex.utilities.utilities import check_res_file + from reV.config.project_points import PointsControl from reV.generation.base import BaseGen from reV.handlers.outputs import Outputs from reV.SAM.econ import LCOE as SAM_LCOE from reV.SAM.econ import SingleOwner from reV.SAM.windbos import WindBos -from reV.utilities.exceptions import (ExecutionError, OffshoreWindInputWarning) -from reV.utilities import ModuleName - -from rex.resource import Resource -from rex.multi_file_resource import MultiFileResource -from rex.utilities.utilities import check_res_file +from reV.utilities import MetaKeyName, ModuleName +from reV.utilities.exceptions import ExecutionError, OffshoreWindInputWarning logger = logging.getLogger(__name__) @@ -54,7 +52,7 @@ class Econ(BaseGen): def __init__(self, project_points, sam_files, cf_file, site_data=None, output_request=('lcoe_fcr',), sites_per_worker=100, memory_utilization_limit=0.4, append=False): - """reV econ analysis class. + """ReV econ analysis class. ``reV`` econ analysis runs SAM econ calculations, typically to compute LCOE (using :py:class:`PySAM.Lcoefcr.Lcoefcr`), though @@ -212,19 +210,20 @@ def meta(self): with Outputs(self.cf_file) as cfh: # only take meta that belongs to this project's site list self._meta = cfh.meta[ - cfh.meta['gid'].isin(self.points_control.sites)] + cfh.meta[MetaKeyName.GID].isin(self.points_control.sites)] - if 'offshore' in self._meta: - if self._meta['offshore'].sum() > 1: - w = ('Found offshore sites in econ meta data. ' - 'This functionality has been deprecated. ' - 'Please run the reV offshore module to ' - 'calculate offshore wind lcoe.') - warn(w, OffshoreWindInputWarning) - logger.warning(w) + if (MetaKeyName.OFFSHORE in self._meta and + self._meta[MetaKeyName.OFFSHORE].sum() > 1): + w = ('Found offshore sites in econ meta data. ' + 'This functionality has been deprecated. ' + 'Please run the reV offshore module to ' + 'calculate offshore wind lcoe.') + warn(w, OffshoreWindInputWarning) + logger.warning(w) elif self._meta is None and self.cf_file is None: - self._meta = pd.DataFrame({'gid': self.points_control.sites}) + self._meta = pd.DataFrame({MetaKeyName.GID: + self.points_control.sites}) return self._meta @@ -267,8 +266,8 @@ def _econ_append_pc(pp, cf_file, sites_per_worker=None): res_kwargs = {'hsds': hsds} with res_cls(cf_file, **res_kwargs) as f: - gid0 = f.meta['gid'].values[0] - gid1 = f.meta['gid'].values[-1] + gid0 = f.meta[MetaKeyName.GID].values[0] + gid1 = f.meta[MetaKeyName.GID].values[-1] i0 = pp.index(gid0) i1 = pp.index(gid1) + 1 @@ -354,7 +353,7 @@ def _run_single_worker(pc, econ_fun, output_request, **kwargs): # Extract the site df from the project points df. site_df = pc.project_points.df - site_df = site_df.set_index('gid', drop=True) + site_df = site_df.set_index(MetaKeyName.GID, drop=True) # SAM execute econ analysis based on output request try: @@ -389,10 +388,8 @@ def _parse_output_request(self, req): 'Will attempt to extract from PySAM.'.format(request)) logger.debug(msg) - modules = [] - for request in output_request: - if request in self.OPTIONS: - modules.append(self.OPTIONS[request]) + modules = [self.OPTIONS[request] for request in output_request + if request in self.OPTIONS] if not any(modules): msg = ('None of the user output requests were recognized. ' @@ -498,7 +495,7 @@ def run(self, out_fpath=None, max_workers=1, timeout=1800, self._init_out_arrays() diff = list(set(self.points_control.sites) - - set(self.meta['gid'].values)) + - set(self.meta[MetaKeyName.GID].values)) if diff: raise Exception('The following analysis sites were requested ' 'through project points for econ but are not ' diff --git a/reV/econ/economies_of_scale.py b/reV/econ/economies_of_scale.py index 326c1d6d3..539193aa1 100644 --- a/reV/econ/economies_of_scale.py +++ b/reV/econ/economies_of_scale.py @@ -3,14 +3,15 @@ reV module for calculating economies of scale where larger power plants will have reduced capital cost. """ -import logging import copy +import logging import re -import numpy as np # pylint: disable=unused-import + import pandas as pd +from rex.utilities.utilities import check_eval_str from reV.econ.utilities import lcoe_fcr -from rex.utilities.utilities import check_eval_str +from reV.utilities import MetaKeyName logger = logging.getLogger(__name__) @@ -66,10 +67,7 @@ def _preflight(self): logger.error(e) raise TypeError(e) - missing = [] - for name in self.vars: - if name not in self._data: - missing.append(name) + missing = [name for name in self.vars if name not in self._data] if any(missing): e = ('Cannot evaluate EconomiesOfScale, missing data for variables' @@ -110,10 +108,9 @@ def vars(self): regex_pattern = '|'.join(map(re.escape, delimiters)) var_names = [] for sub in re.split(regex_pattern, str(self._eqn)): - if sub: - if not self.is_num(sub) and not self.is_method(sub): - var_names.append(sub) - var_names = sorted(list(set(var_names))) + if sub and not self.is_num(sub) and not self.is_method(sub): + var_names.append(sub) + var_names = sorted(set(var_names)) return var_names @@ -231,7 +228,8 @@ def fcr(self): out : float | np.ndarray Fixed charge rate from input data arg """ - key_list = ['fixed_charge_rate', 'mean_fixed_charge_rate', + key_list = ['fixed_charge_rate', + 'mean_fixed_charge_rate', 'fcr', 'mean_fcr'] return self._get_prioritized_keys(self._data, key_list) @@ -244,7 +242,8 @@ def foc(self): out : float | np.ndarray Fixed operating cost from input data arg """ - key_list = ['fixed_operating_cost', 'mean_fixed_operating_cost', + key_list = ['fixed_operating_cost', + 'mean_fixed_operating_cost', 'foc', 'mean_foc'] return self._get_prioritized_keys(self._data, key_list) @@ -257,7 +256,8 @@ def voc(self): out : float | np.ndarray Variable operating cost from input data arg """ - key_list = ['variable_operating_cost', 'mean_variable_operating_cost', + key_list = ['variable_operating_cost', + 'mean_variable_operating_cost', 'voc', 'mean_voc'] return self._get_prioritized_keys(self._data, key_list) @@ -284,7 +284,7 @@ def raw_lcoe(self): ------- lcoe : float | np.ndarray """ - key_list = ['raw_lcoe', 'mean_lcoe'] + key_list = [MetaKeyName.RAW_LCOE, MetaKeyName.MEAN_LCOE] return copy.deepcopy(self._get_prioritized_keys(self._data, key_list)) @property diff --git a/reV/generation/base.py b/reV/generation/base.py index 6f7955e58..6512f4a22 100644 --- a/reV/generation/base.py +++ b/reV/generation/base.py @@ -2,44 +2,47 @@ """ reV base gen and econ module. """ -from abc import ABC, abstractmethod import copy -from concurrent.futures import TimeoutError +import json import logging -import pandas as pd -import numpy as np import os -import psutil -import json import sys +from abc import ABC, abstractmethod +from concurrent.futures import TimeoutError from warnings import warn +import numpy as np +import pandas as pd +import psutil +from rex.resource import Resource +from rex.utilities.execution import SpawnProcessPool + from reV.config.output_request import SAMOutputRequest -from reV.config.project_points import ProjectPoints, PointsControl +from reV.config.project_points import PointsControl, ProjectPoints from reV.handlers.outputs import Outputs from reV.SAM.version_checker import PySamVersionChecker -from reV.utilities.exceptions import (OutputWarning, ExecutionError, - ParallelExecutionWarning, - OffshoreWindInputWarning) -from reV.utilities import log_versions, ModuleName - -from rex.resource import Resource -from rex.utilities.execution import SpawnProcessPool +from reV.utilities import MetaKeyName, ModuleName, log_versions +from reV.utilities.exceptions import ( + ExecutionError, + OffshoreWindInputWarning, + OutputWarning, + ParallelExecutionWarning, +) logger = logging.getLogger(__name__) ATTR_DIR = os.path.dirname(os.path.realpath(__file__)) ATTR_DIR = os.path.join(ATTR_DIR, 'output_attributes') -with open(os.path.join(ATTR_DIR, 'other.json'), 'r') as f: +with open(os.path.join(ATTR_DIR, 'other.json')) as f: OTHER_ATTRS = json.load(f) -with open(os.path.join(ATTR_DIR, 'lcoe_fcr.json'), 'r') as f: +with open(os.path.join(ATTR_DIR, 'lcoe_fcr.json')) as f: LCOE_ATTRS = json.load(f) -with open(os.path.join(ATTR_DIR, 'single_owner.json'), 'r') as f: +with open(os.path.join(ATTR_DIR, 'single_owner.json')) as f: SO_ATTRS = json.load(f) -with open(os.path.join(ATTR_DIR, 'windbos.json'), 'r') as f: +with open(os.path.join(ATTR_DIR, 'windbos.json')) as f: BOS_ATTRS = json.load(f) -with open(os.path.join(ATTR_DIR, 'lcoe_fcr_inputs.json'), 'r') as f: +with open(os.path.join(ATTR_DIR, 'lcoe_fcr_inputs.json')) as f: LCOE_IN_ATTRS = json.load(f) @@ -65,7 +68,8 @@ class BaseGen(ABC): # SAM argument names used to calculate LCOE # Note that system_capacity is not included here because it is never used # downstream and could be confused with the supply_curve point capacity - LCOE_ARGS = ('fixed_charge_rate', 'capital_cost', 'fixed_operating_cost', + LCOE_ARGS = ('fixed_charge_rate', 'capital_cost', + 'fixed_operating_cost', 'variable_operating_cost') def __init__(self, points_control, output_request, site_data=None, @@ -735,7 +739,7 @@ def _parse_site_data(self, inp): if inp is None or inp is False: # no input, just initialize dataframe with site gids as index site_data = pd.DataFrame(index=self.project_points.sites) - site_data.index.name = 'gid' + site_data.index.name = MetaKeyName.GID else: # explicit input, initialize df if isinstance(inp, str): @@ -748,18 +752,19 @@ def _parse_site_data(self, inp): raise Exception('Site data input must be .csv or ' 'dataframe, but received: {}'.format(inp)) - if 'gid' not in site_data and site_data.index.name != 'gid': + if (MetaKeyName.GID not in site_data and + site_data.index.name != MetaKeyName.GID): # require gid as column label or index raise KeyError('Site data input must have "gid" column ' 'to match reV site gid.') # pylint: disable=no-member - if site_data.index.name != 'gid': + if site_data.index.name != MetaKeyName.GID: # make gid the dataframe index if not already - site_data = site_data.set_index('gid', drop=True) + site_data = site_data.set_index(MetaKeyName.GID, drop=True) - if 'offshore' in site_data: - if site_data['offshore'].sum() > 1: + if MetaKeyName.OFFSHORE in site_data: + if site_data[MetaKeyName.OFFSHORE].sum() > 1: w = ('Found offshore sites in econ site data input. ' 'This functionality has been deprecated. ' 'Please run the reV offshore module to ' @@ -829,7 +834,7 @@ def _get_data_shape_from_out_attrs(self, dset, n_sites): return (n_sites,) def _get_data_shape_from_sam_config(self, dset, n_sites): - """Get data shape from SAM input config """ + """Get data shape from SAM input config""" data = list(self.project_points.sam_inputs.values())[0][dset] if isinstance(data, (list, tuple, np.ndarray)): return (*np.array(data).shape, n_sites) diff --git a/reV/generation/generation.py b/reV/generation/generation.py index f505fda71..b26ac0c89 100644 --- a/reV/generation/generation.py +++ b/reV/generation/generation.py @@ -10,42 +10,46 @@ import numpy as np import pandas as pd - -from reV.generation.base import BaseGen -from reV.SAM.generation import (Geothermal, - MhkWave, - LinearDirectSteam, - PvSamv1, - PvWattsv5, - PvWattsv7, - PvWattsv8, - SolarWaterHeat, - TcsMoltenSalt, - TroughPhysicalHeat, - WindPower) -from reV.utilities import ModuleName -from reV.utilities.exceptions import (ConfigError, - InputError, - ProjectPointsValueError) from rex.multi_file_resource import MultiFileResource from rex.multi_res_resource import MultiResolutionResource from rex.resource import Resource from rex.utilities.utilities import check_res_file +from reV.generation.base import BaseGen +from reV.SAM.generation import ( + Geothermal, + LinearDirectSteam, + MhkWave, + PvSamv1, + PvWattsv5, + PvWattsv7, + PvWattsv8, + SolarWaterHeat, + TcsMoltenSalt, + TroughPhysicalHeat, + WindPower, +) +from reV.utilities import MetaKeyName, ModuleName +from reV.utilities.exceptions import ( + ConfigError, + InputError, + ProjectPointsValueError, +) + logger = logging.getLogger(__name__) ATTR_DIR = os.path.dirname(os.path.realpath(__file__)) ATTR_DIR = os.path.join(ATTR_DIR, 'output_attributes') -with open(os.path.join(ATTR_DIR, 'other.json'), 'r') as f: +with open(os.path.join(ATTR_DIR, 'other.json')) as f: OTHER_ATTRS = json.load(f) -with open(os.path.join(ATTR_DIR, 'generation.json'), 'r') as f: +with open(os.path.join(ATTR_DIR, 'generation.json')) as f: GEN_ATTRS = json.load(f) -with open(os.path.join(ATTR_DIR, 'linear_fresnel.json'), 'r') as f: +with open(os.path.join(ATTR_DIR, 'linear_fresnel.json')) as f: LIN_ATTRS = json.load(f) -with open(os.path.join(ATTR_DIR, 'solar_water_heat.json'), 'r') as f: +with open(os.path.join(ATTR_DIR, 'solar_water_heat.json')) as f: SWH_ATTRS = json.load(f) -with open(os.path.join(ATTR_DIR, 'trough_heat.json'), 'r') as f: +with open(os.path.join(ATTR_DIR, 'trough_heat.json')) as f: TPPH_ATTRS = json.load(f) @@ -77,12 +81,13 @@ class Gen(BaseGen): OUT_ATTRS.update(BaseGen.ECON_ATTRS) def __init__(self, technology, project_points, sam_files, resource_file, - low_res_resource_file=None, output_request=('cf_mean',), + low_res_resource_file=None, + output_request=(MetaKeyName.CF_MEAN,), site_data=None, curtailment=None, gid_map=None, drop_leap=False, sites_per_worker=None, memory_utilization_limit=0.4, scale_outputs=True, write_mapped_gids=False, bias_correct=None): - """reV generation analysis class. + """ReV generation analysis class. ``reV`` generation analysis runs SAM simulations by piping in renewable energy resource data (usually from the NSRDB or WTK), @@ -118,17 +123,17 @@ def __init__(self, technology, project_points, sam_files, resource_file, >>> gen.run() >>> >>> gen.out - {'cf_mean': array([0.16966143], dtype=float32)} + {MetaKeyName.CF_MEAN: array([0.16966143], dtype=float32)} >>> >>> sites = [3, 4, 7, 9] - >>> req = ('cf_mean', 'cf_profile', 'lcoe_fcr') + >>> req = (MetaKeyName.CF_MEAN, , MetaKeyName.LCOE_FCR) >>> gen = Gen(sam_tech, sites, fp_sam, fp_res, output_request=req) >>> gen.run() >>> >>> gen.out {'lcoe_fcr': array([131.39166, 131.31221, 127.54539, 125.49656]), - 'cf_mean': array([0.17713654, 0.17724372, 0.1824783 , 0.1854574 ]), - 'cf_profile': array([[0., 0., 0., 0.], + 'cf_mean': array([0.17713654, 0.17724372, 0.1824783 , 0.1854574 ]), + : array([[0., 0., 0., 0.], [0., 0., 0., 0.], [0., 0., 0., 0.], ..., @@ -278,7 +283,7 @@ def __init__(self, technology, project_points, sam_files, resource_file, aggregation/supply curve step if the ``"dc_ac_ratio"`` dataset is detected in the generation file. - By default, ``('cf_mean',)``. + By default, ``(MetaKeyName.CF_MEAN,)``. site_data : str | pd.DataFrame, optional Site-specific input data for SAM calculation. If this input is a string, it should be a path that points to a CSV file. @@ -451,11 +456,11 @@ def meta(self): self._meta = res['meta', res_gids] - self._meta.loc[:, 'gid'] = res_gids + self._meta.loc[:, MetaKeyName.GID] = res_gids if self.write_mapped_gids: - self._meta.loc[:, 'gid'] = self.project_points.sites + self._meta.loc[:, MetaKeyName.GID] = self.project_points.sites self._meta.index = self.project_points.sites - self._meta.index.name = 'gid' + self._meta.index.name = MetaKeyName.GID self._meta.loc[:, 'reV_tech'] = self.project_points.tech return self._meta @@ -542,7 +547,8 @@ def handle_lifetime_index(self, ti): var for var, attrs in GEN_ATTRS.items() if attrs['type'] == 'array' ] - valid_vars = ['gen_profile', 'cf_profile', 'cf_profile_ac'] + valid_vars = [MetaKeyName.GEN_PROFILE, MetaKeyName.CF_PROFILE, + 'cf_profile_ac'] invalid_vars = set(array_vars) - set(valid_vars) invalid_requests = [var for var in self.output_request if var in invalid_vars] @@ -637,7 +643,7 @@ def _run_single_worker(cls, points_control, tech=None, res_file=None, # Extract the site df from the project points df. site_df = points_control.project_points.df - site_df = site_df.set_index('gid', drop=True) + site_df = site_df.set_index(MetaKeyName.GID, drop=True) # run generation method for specified technology try: @@ -703,13 +709,14 @@ def _parse_gid_map(self, gid_map): if isinstance(gid_map, str): if gid_map.endswith('.csv'): gid_map = pd.read_csv(gid_map).to_dict() - assert 'gid' in gid_map, 'Need "gid" in gid_map column' + msg = f'Need "{MetaKeyName.GID}" in gid_map column' + assert MetaKeyName.GID in gid_map, msg assert 'gid_map' in gid_map, 'Need "gid_map" in gid_map column' - gid_map = {gid_map['gid'][i]: gid_map['gid_map'][i] - for i in gid_map['gid'].keys()} + gid_map = {gid_map[MetaKeyName.GID][i]: gid_map['gid_map'][i] + for i in gid_map[MetaKeyName.GID].keys()} elif gid_map.endswith('.json'): - with open(gid_map, 'r') as f: + with open(gid_map) as f: gid_map = json.load(f) if isinstance(gid_map, dict): @@ -763,14 +770,14 @@ def _parse_nn_map(self): if '*' in self.res_file or '*' in self.lr_res_file: handler_class = MultiFileResource - with handler_class(self.res_file) as hr_res: - with handler_class(self.lr_res_file) as lr_res: - logger.info('Making nearest neighbor map for multi ' - 'resolution resource data...') - nn_d, nn_map = MultiResolutionResource.make_nn_map(hr_res, - lr_res) - logger.info('Done making nearest neighbor map for multi ' - 'resolution resource data!') + with (handler_class(self.res_file) as hr_res, + handler_class(self.lr_res_file) as lr_res): + logger.info('Making nearest neighbor map for multi ' + 'resolution resource data...') + nn_d, nn_map = MultiResolutionResource.make_nn_map(hr_res, + lr_res) + logger.info('Done making nearest neighbor map for multi ' + 'resolution resource data!') logger.info('Made nearest neighbor mapping between nominal-' 'resolution and low-resolution resource files. ' @@ -838,10 +845,11 @@ def _parse_bc(bias_correct): msg = ('Bias correction table must have "gid" column but only found: ' '{}'.format(list(bias_correct.columns))) - assert 'gid' in bias_correct or bias_correct.index.name == 'gid', msg + assert (MetaKeyName.GID in bias_correct or bias_correct.index.name == + MetaKeyName.GID), msg - if bias_correct.index.name != 'gid': - bias_correct = bias_correct.set_index('gid') + if bias_correct.index.name != MetaKeyName.GID: + bias_correct = bias_correct.set_index(MetaKeyName.GID) msg = ('Bias correction table must have "method" column but only ' 'found: {}'.format(list(bias_correct.columns))) @@ -866,8 +874,8 @@ def _parse_output_request(self, req): output_request = self._output_request_type_check(req) # ensure that cf_mean is requested from output - if 'cf_mean' not in output_request: - output_request.append('cf_mean') + if MetaKeyName.CF_MEAN not in output_request: + output_request.append(MetaKeyName.CF_MEAN) for request in output_request: if request not in self.OUT_ATTRS: diff --git a/reV/handlers/exclusions.py b/reV/handlers/exclusions.py index d7bde9b04..13d192ee2 100644 --- a/reV/handlers/exclusions.py +++ b/reV/handlers/exclusions.py @@ -2,16 +2,16 @@ """ Exclusion layers handler """ -import logging import json +import logging + import numpy as np +from rex.multi_file_resource import MultiFileResource +from rex.resource import Resource +from rex.utilities.parse_keys import parse_keys from reV.utilities.exceptions import HandlerKeyError, MultiFileExclusionError -from rex.utilities.parse_keys import parse_keys -from rex.resource import Resource -from rex.multi_file_resource import MultiFileResource - logger = logging.getLogger(__name__) @@ -81,8 +81,8 @@ def __contains__(self, layer): def _preflight_multi_file(self): """Run simple multi-file exclusion checks.""" - lat_shape = self.h5.shapes['latitude'] - lon_shape = self.h5.shapes['longitude'] + lat_shape = self.h5.shapes[MetaKeyName.LATITUDE] + lon_shape = self.h5.shapes[MetaKeyName.LONGITUDE] for layer in self.layers: lshape = self.h5.shapes[layer] lshape = lshape[1:] if len(lshape) > 2 else lshape @@ -231,7 +231,7 @@ def shape(self): """ shape = self.h5.attrs.get('shape', None) if shape is None: - shape = self.h5.shapes['latitude'] + shape = self.h5.shapes[MetaKeyName.LATITUDE] return tuple(shape) @@ -247,7 +247,7 @@ def chunks(self): """ chunks = self.h5.attrs.get('chunks', None) if chunks is None: - chunks = self.h5.chunks['latitude'] + chunks = self.h5.chunks[MetaKeyName.LATITUDE] return chunks @@ -260,7 +260,7 @@ def latitude(self): ------- ndarray """ - return self['latitude'] + return self[MetaKeyName.LATITUDE] @property def longitude(self): @@ -271,7 +271,7 @@ def longitude(self): ------- ndarray """ - return self['longitude'] + return self[MetaKeyName.LONGITUDE] def get_layer_profile(self, layer): """ @@ -384,13 +384,13 @@ def _get_latitude(self, *ds_slice): lat : ndarray Latitude coordinates """ - if 'latitude' not in self.h5: + if MetaKeyName.LATITUDE not in self.h5: msg = ('"latitude" is missing from {}' .format(self.h5_file)) logger.error(msg) raise HandlerKeyError(msg) - ds_slice = ('latitude', ) + ds_slice + ds_slice = (MetaKeyName.LATITUDE, ) + ds_slice lat = self.h5[ds_slice] @@ -410,13 +410,13 @@ def _get_longitude(self, *ds_slice): lon : ndarray Longitude coordinates """ - if 'longitude' not in self.h5: + if MetaKeyName.LONGITUDE not in self.h5: msg = ('"longitude" is missing from {}' .format(self.h5_file)) logger.error(msg) raise HandlerKeyError(msg) - ds_slice = ('longitude', ) + ds_slice + ds_slice = (MetaKeyName.LONGITUDE, ) + ds_slice lon = self.h5[ds_slice] diff --git a/reV/handlers/multi_year.py b/reV/handlers/multi_year.py index 494ff3215..dd2d3c9b2 100644 --- a/reV/handlers/multi_year.py +++ b/reV/handlers/multi_year.py @@ -3,22 +3,25 @@ Classes to collect reV outputs from multiple annual files. """ import glob -import time import logging -import numpy as np import os -import pandas as pd +import time from warnings import warn -from rex import Resource -from rex.utilities.utilities import (get_class_properties, parse_year, - get_lat_lon_cols) +import numpy as np +import pandas as pd from gaps.pipeline import parse_previous_status +from rex import Resource +from rex.utilities.utilities import ( + get_class_properties, + get_lat_lon_cols, + parse_year, +) -from reV.handlers.outputs import Outputs from reV.config.output_request import SAMOutputRequest -from reV.utilities.exceptions import HandlerRuntimeError, ConfigError -from reV.utilities import log_versions, ModuleName +from reV.handlers.outputs import Outputs +from reV.utilities import ModuleName, log_versions +from reV.utilities.exceptions import ConfigError, HandlerRuntimeError logger = logging.getLogger(__name__) @@ -31,7 +34,7 @@ class MultiYearGroup: def __init__(self, name, out_dir, source_files=None, source_dir=None, source_prefix=None, source_pattern=None, - dsets=('cf_mean',), pass_through_dsets=None): + dsets=(MetaKeyName.CF_MEAN,), pass_through_dsets=None): """ Parameters ---------- @@ -58,7 +61,7 @@ def __init__(self, name, out_dir, source_files=None, `source_prefix` but is not used if `source_files` are specified explicitly. By default, ``None``. dsets : list | tuple, optional - List of datasets to collect. By default, ``('cf_mean',)``. + List of datasets to collect. By default, ``(MetaKeyName.CF_MEAN,)``. pass_through_dsets : list | tuple, optional Optional list of datasets that are identical in the multi-year files (e.g. input datasets that don't vary from diff --git a/reV/hybrids/hybrids.py b/reV/hybrids/hybrids.py index 7a8f2d207..4d5c11708 100644 --- a/reV/hybrids/hybrids.py +++ b/reV/hybrids/hybrids.py @@ -4,32 +4,36 @@ @author: ppinchuk """ import logging -import numpy as np import re -import pandas as pd +from collections import namedtuple from string import ascii_letters from warnings import warn -from collections import namedtuple - -from reV.handlers.outputs import Outputs -from reV.utilities.exceptions import (FileInputError, InputError, - InputWarning, OutputWarning) -from reV.hybrids.hybrid_methods import HYBRID_METHODS +import numpy as np +import pandas as pd from rex.resource import Resource from rex.utilities.utilities import to_records_array +from reV.handlers.outputs import Outputs +from reV.hybrids.hybrid_methods import HYBRID_METHODS +from reV.utilities.exceptions import ( + FileInputError, + InputError, + InputWarning, + OutputWarning, +) + logger = logging.getLogger(__name__) -MERGE_COLUMN = 'sc_point_gid' +MERGE_COLUMN = MetaKeyName.SC_POINT_GID PROFILE_DSET_REGEX = 'rep_profiles_[0-9]+$' SOLAR_PREFIX = 'solar_' WIND_PREFIX = 'wind_' NON_DUPLICATE_COLS = { - 'latitude', 'longitude', 'country', 'state', 'county', 'elevation', - 'timezone', 'sc_point_gid', 'sc_row_ind', 'sc_col_ind' + MetaKeyName.LATITUDE, MetaKeyName.LONGITUDE, MetaKeyName.COUNTRY, 'state', 'county', 'elevation', + MetaKeyName.TIMEZONE, MetaKeyName.SC_POINT_GID, MetaKeyName.SC_ROW_IND, MetaKeyName.SC_COL_IND } -DROPPED_COLUMNS = ['gid'] +DROPPED_COLUMNS = [MetaKeyName.GID] DEFAULT_FILL_VALUES = {'solar_capacity': 0, 'wind_capacity': 0, 'solar_mean_cf': 0, 'wind_mean_cf': 0} OUTPUT_PROFILE_NAMES = ['hybrid_profile', @@ -40,7 +44,8 @@ class ColNameFormatter: - """Column name formatting helper class. """ + """Column name formatting helper class.""" + ALLOWED = set(ascii_letters) @classmethod @@ -65,7 +70,7 @@ def fmt(cls, n): class HybridsData: - """Hybrids input data container. """ + """Hybrids input data container.""" def __init__(self, solar_fpath, wind_fpath): """ @@ -431,7 +436,7 @@ def _validate_limits_cols_prefixed(self): @staticmethod def __validate_col_prefix(col, prefixes, input_name): - """Validate the the col starts with the correct prefix. """ + """Validate the the col starts with the correct prefix.""" missing = [not col.startswith(p) for p in prefixes] if all(missing): @@ -585,7 +590,7 @@ def _validate_ratio_cols_exist(self): @property def _ratio_cols(self): - """Get the ratio columns from the ratio input. """ + """Get the ratio columns from the ratio input.""" if self._ratio is None: return [] return self._ratio.strip().split('/') @@ -603,7 +608,7 @@ def hybridize(self): self._sort_hybrid_meta_cols() def _format_meta_pre_merge(self): - """Prepare solar and wind meta for merging. """ + """Prepare solar and wind meta for merging.""" self.__col_name_map = { ColNameFormatter.fmt(c): c for c in self.data.solar_meta.columns.values @@ -616,7 +621,7 @@ def _format_meta_pre_merge(self): @staticmethod def _rename_cols(df, prefix): - """Replace column names with the ColNameFormatter.fmt is needed. """ + """Replace column names with the ColNameFormatter.fmt is needed.""" df.columns = [ ColNameFormatter.fmt(col_name) if col_name in NON_DUPLICATE_COLS @@ -625,13 +630,13 @@ def _rename_cols(df, prefix): ] def _save_rep_prof_index_internally(self): - """Save rep profiles index in hybrid meta for access later. """ + """Save rep profiles index in hybrid meta for access later.""" self.data.solar_meta[self.__solar_rpi_n] = self.data.solar_meta.index self.data.wind_meta[self.__wind_rpi_n] = self.data.wind_meta.index def _merge_solar_wind_meta(self): - """Merge the wind and solar meta DataFrames. """ + """Merge the wind and solar meta DataFrames.""" self._hybrid_meta = self.data.solar_meta.merge( self.data.wind_meta, on=ColNameFormatter.fmt(MERGE_COLUMN), @@ -639,7 +644,7 @@ def _merge_solar_wind_meta(self): ) def _merge_type(self): - """Determine the type of merge to use for meta based on user input. """ + """Determine the type of merge to use for meta based on user input.""" if self._allow_solar_only and self._allow_wind_only: return 'outer' elif self._allow_solar_only and not self._allow_wind_only: @@ -649,16 +654,16 @@ def _merge_type(self): return 'inner' def _format_meta_post_merge(self): - """Format hybrid meta after merging. """ + """Format hybrid meta after merging.""" duplicate_cols = [n for n in self._hybrid_meta.columns if "_x" in n] self._propagate_duplicate_cols(duplicate_cols) self._drop_cols(duplicate_cols) self._hybrid_meta.rename(self.__col_name_map, inplace=True, axis=1) - self._hybrid_meta.index.name = 'gid' + self._hybrid_meta.index.name = MetaKeyName.GID def _propagate_duplicate_cols(self, duplicate_cols): - """Fill missing column values from outer merge. """ + """Fill missing column values from outer merge.""" for duplicate in duplicate_cols: no_suffix = "_".join(duplicate.split("_")[:-1]) null_idx = self._hybrid_meta[no_suffix].isnull() @@ -666,14 +671,14 @@ def _propagate_duplicate_cols(self, duplicate_cols): self._hybrid_meta.loc[null_idx, no_suffix] = non_null_vals def _drop_cols(self, duplicate_cols): - """Drop any remaning duplicate and 'DROPPED_COLUMNS' columns. """ + """Drop any remaning duplicate and 'DROPPED_COLUMNS' columns.""" self._hybrid_meta.drop( duplicate_cols + DROPPED_COLUMNS, axis=1, inplace=True, errors='ignore' ) def _sort_hybrid_meta_cols(self): - """Sort the columns of the hybrid meta. """ + """Sort the columns of the hybrid meta.""" self.__hybrid_meta_cols = sorted( [c for c in self._hybrid_meta.columns if not c.startswith(self._INTERNAL_COL_PREFIX)], @@ -681,7 +686,7 @@ def _sort_hybrid_meta_cols(self): ) def _column_sorting_key(self, c): - """Helper function to sort hybrid meta columns. """ + """Helper function to sort hybrid meta columns.""" first_index = 0 if c.startswith('hybrid'): first_index = 1 @@ -695,8 +700,8 @@ def _column_sorting_key(self, c): def _verify_lat_long_match_post_merge(self): """Verify that all the lat/lon values match post merge.""" - lat = self._verify_col_match_post_merge(col_name='latitude') - lon = self._verify_col_match_post_merge(col_name='longitude') + lat = self._verify_col_match_post_merge(col_name=MetaKeyName.LATITUDE) + lon = self._verify_col_match_post_merge(col_name=MetaKeyName.LONGITUDE) if not lat or not lon: msg = ("Detected mismatched coordinate values (latitude or " "longitude) post merge. Please ensure that all matching " @@ -708,7 +713,7 @@ def _verify_lat_long_match_post_merge(self): raise FileInputError(e) def _verify_col_match_post_merge(self, col_name): - """Verify that all (non-null) values in a column match post merge. """ + """Verify that all (non-null) values in a column match post merge.""" c1, c2 = col_name, '{}_x'.format(col_name) if c1 in self._hybrid_meta.columns and c2 in self._hybrid_meta.columns: compare_df = self._hybrid_meta[ @@ -720,7 +725,7 @@ def _verify_col_match_post_merge(self, col_name): return True def _fillna_meta_cols(self): - """Fill N/A values as specified by user (and internals). """ + """Fill N/A values as specified by user (and internals).""" for col_name, fill_value in self._fillna.items(): if col_name in self._hybrid_meta.columns: self._hybrid_meta[col_name].fillna(fill_value, inplace=True) @@ -732,7 +737,7 @@ def _fillna_meta_cols(self): @staticmethod def __warn_missing_col(col_name, action): - """Warn that a column the user request an action for is missing. """ + """Warn that a column the user request an action for is missing.""" msg = ("Skipping {} values for {!r}: Unable to find column " "in hybrid meta. Did you forget to prefix with " "{!r} or {!r}? ") @@ -741,7 +746,7 @@ def __warn_missing_col(col_name, action): warn(w, InputWarning) def _apply_limits(self): - """Clip column values as specified by user. """ + """Clip column values as specified by user.""" for col_name, max_value in self._limits.items(): if col_name in self._hybrid_meta.columns: self._hybrid_meta[col_name].clip(upper=max_value, inplace=True) @@ -749,7 +754,7 @@ def _apply_limits(self): self.__warn_missing_col(col_name, action='limit') def _limit_by_ratio(self): - """ Limit the given pair of ratio columns based on input ratio. """ + """Limit the given pair of ratio columns based on input ratio.""" if self._ratio_bounds is None: return @@ -784,7 +789,7 @@ def _limit_by_ratio(self): self._hybrid_meta[h_denom_name] = denominator_vals.values def _add_hybrid_cols(self): - """Add new hybrid columns using registered hybrid methods. """ + """Add new hybrid columns using registered hybrid methods.""" for new_col_name, method in HYBRID_METHODS.items(): out = method(self) if out is not None: @@ -936,7 +941,7 @@ def __init__(self, solar_fpath, wind_fpath, allow_solar_only=False, self._validate_input() def _validate_input(self): - """Validate the user input and input files. """ + """Validate the user input and input files.""" self.data.validate() self.meta_hybridizer.validate_input() @@ -1085,7 +1090,7 @@ def _init_profiles(self): for k in OUTPUT_PROFILE_NAMES} def _compute_hybridized_profile_components(self): - """Compute the resource components of the hybridized profiles. """ + """Compute the resource components of the hybridized profiles.""" for params in self.__rep_profile_hybridization_params: col, (hybrid_idxs, solar_idxs), fpath, p_name, dset_name = params @@ -1099,7 +1104,7 @@ def _compute_hybridized_profile_components(self): @property def __rep_profile_hybridization_params(self): - """Zip the rep profile hybridization parameters. """ + """Zip the rep profile hybridization parameters.""" cap_col_names = ['hybrid_solar_capacity', 'hybrid_wind_capacity'] idx_maps = [self.meta_hybridizer.solar_profile_indices_map, @@ -1110,7 +1115,7 @@ def __rep_profile_hybridization_params(self): return zipped def _compute_hybridized_profiles_from_components(self): - """Compute the hybridized profiles from the resource components. """ + """Compute the hybridized profiles from the resource components.""" hp_name, sp_name, wp_name = OUTPUT_PROFILE_NAMES self._profiles[hp_name] = (self._profiles[sp_name] diff --git a/reV/nrwal/nrwal.py b/reV/nrwal/nrwal.py index 182ce3d31..b0c23853f 100644 --- a/reV/nrwal/nrwal.py +++ b/reV/nrwal/nrwal.py @@ -11,18 +11,20 @@ generation output file. This is usually the wind or solar resource resolution but could be the supply curve resolution after representative profiles is run. """ -import numpy as np -import pandas as pd import logging from warnings import warn +import numpy as np +import pandas as pd + from reV.generation.generation import Gen from reV.handlers.outputs import Outputs -from reV.utilities.exceptions import (DataShapeError, - OffshoreWindInputWarning, - OffshoreWindInputError) from reV.utilities import log_versions - +from reV.utilities.exceptions import ( + DataShapeError, + OffshoreWindInputError, + OffshoreWindInputWarning, +) logger = logging.getLogger(__name__) @@ -34,7 +36,7 @@ class RevNrwal: """Columns from the `site_data` table to join to the output meta data""" def __init__(self, gen_fpath, site_data, sam_files, nrwal_configs, - output_request, save_raw=True, meta_gid_col='gid', + output_request, save_raw=True, meta_gid_col=MetaKeyName.GID, site_meta_cols=None): """Framework to handle reV-NRWAL analysis. @@ -178,7 +180,7 @@ def __init__(self, gen_fpath, site_data, sam_files, nrwal_configs, self._meta_source = self._parse_gen_data() self._analysis_gids, self._site_data = self._parse_analysis_gids() - pc = Gen.get_pc(self._site_data[['gid', 'config']], points_range=None, + pc = Gen.get_pc(self._site_data[[MetaKeyName.GID, 'config']], points_range=None, sam_configs=sam_files, tech='windpower') self._project_points = pc.project_points @@ -191,7 +193,7 @@ def __init__(self, gen_fpath, site_data, sam_files, nrwal_configs, meta_gids.max(), len(self.meta_source), len(self.analysis_gids))) - def _parse_site_data(self, required_columns=('gid', 'config')): + def _parse_site_data(self, required_columns=(MetaKeyName.GID, 'config')): """Parse the site-specific spatial input data file Parameters @@ -227,7 +229,7 @@ def _parse_site_data(self, required_columns=('gid', 'config')): logger.error(msg) raise KeyError(msg) - self._site_data = self._site_data.sort_values('gid') + self._site_data = self._site_data.sort_values(MetaKeyName.GID) return self._site_data @@ -272,7 +274,7 @@ def _parse_analysis_gids(self): meta_gids = self.meta_source[self._meta_gid_col].values - missing = ~np.isin(meta_gids, self._site_data['gid']) + missing = ~np.isin(meta_gids, self._site_data[MetaKeyName.GID]) if any(missing): msg = ('{} sites from the generation meta data input were ' 'missing from the "site_data" input and will not be ' @@ -280,19 +282,19 @@ def _parse_analysis_gids(self): .format(missing.sum(), meta_gids[missing])) logger.info(msg) - missing = ~np.isin(self._site_data['gid'], meta_gids) + missing = ~np.isin(self._site_data[MetaKeyName.GID], meta_gids) if any(missing): - missing = self._site_data['gid'].values[missing] + missing = self._site_data[MetaKeyName.GID].values[missing] msg = ('{} sites from the "site_data" input were missing from the ' 'generation meta data and will not be run through NRWAL: {}' .format(len(missing), missing)) logger.info(msg) - analysis_gids = set(meta_gids) & set(self._site_data['gid']) + analysis_gids = set(meta_gids) & set(self._site_data[MetaKeyName.GID]) analysis_gids = np.array(sorted(list(analysis_gids))) # reduce the site data table to only those sites being analyzed - mask = np.isin(self._site_data['gid'], meta_gids) + mask = np.isin(self._site_data[MetaKeyName.GID], meta_gids) self._site_data = self._site_data[mask] return analysis_gids, self._site_data @@ -315,9 +317,9 @@ def _parse_sam_sys_inputs(self): system_inputs = pd.DataFrame(system_inputs).T system_inputs = system_inputs.sort_index() - system_inputs['gid'] = system_inputs.index.values - system_inputs.index.name = 'gid' - mask = system_inputs['gid'].isin(self.analysis_gids) + system_inputs[MetaKeyName.GID] = system_inputs.index.values + system_inputs.index.name = MetaKeyName.GID + mask = system_inputs[MetaKeyName.GID].isin(self.analysis_gids) system_inputs = system_inputs[mask] return system_inputs @@ -390,8 +392,8 @@ def _preflight_checks(self): logger.error(msg) raise OffshoreWindInputError(msg) - check_gid_order = (self._site_data['gid'].values - == self._sam_sys_inputs['gid'].values) + check_gid_order = (self._site_data[MetaKeyName.GID].values + == self._sam_sys_inputs[MetaKeyName.GID].values) msg = 'NRWAL site_data and system input dataframe had bad order' assert (check_gid_order).all(), msg diff --git a/reV/qa_qc/cli_qa_qc.py b/reV/qa_qc/cli_qa_qc.py index 8ec411dad..2359b4738 100644 --- a/reV/qa_qc/cli_qa_qc.py +++ b/reV/qa_qc/cli_qa_qc.py @@ -2,20 +2,24 @@ """ QA/QC CLI utility functions. """ -import click import logging -import numpy as np import os -from rex.utilities.cli_dtypes import STR, STRLIST, INT +import click +import numpy as np +from gaps.cli import CLICommandFromFunction, as_click_command +from rex.utilities.cli_dtypes import INT, STR, STRLIST from rex.utilities.loggers import init_logger -from gaps.cli import as_click_command, CLICommandFromFunction -from reV.utilities import ModuleName -from reV.qa_qc.qa_qc import QaQc, QaQcModule -from reV.qa_qc.summary import (SummarizeH5, SummarizeSupplyCurve, - SupplyCurvePlot, ExclusionsMask) from reV import __version__ +from reV.qa_qc.qa_qc import QaQc, QaQcModule +from reV.qa_qc.summary import ( + ExclusionsMask, + SummarizeH5, + SummarizeSupplyCurve, + SupplyCurvePlot, +) +from reV.utilities import ModuleName logger = logging.getLogger(__name__) @@ -190,8 +194,8 @@ def supply_curve_table(ctx, sc_table, columns): show_default=True, help=(" plot_type of plot to create 'plot' or 'plotly', by " "default 'plot'")) -@click.option('--lcoe', '-lcoe', type=STR, default='mean_lcoe', - help="LCOE value to plot, by default 'mean_lcoe'") +@click.option('--lcoe', '-lcoe', type=STR, default=MetaKeyName.MEAN_LCOE, + help="LCOE value to plot, by default MetaKeyName.MEAN_LCOE") @click.pass_context def supply_curve_plot(ctx, sc_table, plot_type, lcoe): """ diff --git a/reV/qa_qc/qa_qc.py b/reV/qa_qc/qa_qc.py index 061b7cb04..793964bd7 100644 --- a/reV/qa_qc/qa_qc.py +++ b/reV/qa_qc/qa_qc.py @@ -3,19 +3,24 @@ reV quality assurance and control classes """ import logging -import numpy as np import os -import pandas as pd from warnings import warn -from reV.qa_qc.summary import (SummarizeH5, SummarizeSupplyCurve, SummaryPlots, - SupplyCurvePlot, ExclusionsMask) +import numpy as np +import pandas as pd +from gaps.status import Status + +from reV.qa_qc.summary import ( + ExclusionsMask, + SummarizeH5, + SummarizeSupplyCurve, + SummaryPlots, + SupplyCurvePlot, +) from reV.supply_curve.exclusions import ExclusionMaskFromDict -from reV.utilities import log_versions, ModuleName +from reV.utilities import ModuleName, log_versions from reV.utilities.exceptions import PipelineError -from gaps.status import Status - logger = logging.getLogger(__name__) @@ -23,6 +28,7 @@ class QaQc: """ reV QA/QC """ + def __init__(self, out_dir): """ Parameters @@ -94,8 +100,8 @@ def create_scatter_plots(self, plot_type='plotly', cmap='viridis', if file.endswith('.csv'): summary_csv = os.path.join(self.out_dir, file) summary = pd.read_csv(summary_csv) - if ('gid' in summary and 'latitude' in summary - and 'longitude' in summary): + if (MetaKeyName.GID in summary and MetaKeyName.LATITUDE in summary + and MetaKeyName.LONGITUDE in summary): self._scatter_plot(summary_csv, self.out_dir, plot_type=plot_type, cmap=cmap, **kwargs) @@ -145,7 +151,7 @@ def h5(cls, h5_file, out_dir, dsets=None, group=None, process_size=None, .format(os.path.basename(h5_file), out_dir)) @classmethod - def supply_curve(cls, sc_table, out_dir, columns=None, lcoe='mean_lcoe', + def supply_curve(cls, sc_table, out_dir, columns=None, lcoe=MetaKeyName.MEAN_LCOE, plot_type='plotly', cmap='viridis', sc_plot_kwargs=None, scatter_plot_kwargs=None): """ @@ -161,7 +167,7 @@ def supply_curve(cls, sc_table, out_dir, columns=None, lcoe='mean_lcoe', Column(s) to summarize, if None summarize all numeric columns, by default None lcoe : str, optional - LCOE value to plot, by default 'mean_lcoe' + LCOE value to plot, by default MetaKeyName.MEAN_LCOE plot_type : str, optional plot_type of plot to create 'plot' or 'plotly', by default 'plotly' cmap : str, optional @@ -266,7 +272,7 @@ def __init__(self, module_name, config, out_root): self._default_plot_type = 'plotly' self._default_cmap = 'viridis' self._default_plot_step = 100 - self._default_lcoe = 'mean_lcoe' + self._default_lcoe = MetaKeyName.MEAN_LCOE self._default_area_filter_kernel = 'queen' @property diff --git a/reV/qa_qc/summary.py b/reV/qa_qc/summary.py index fb194b47f..98d27e357 100644 --- a/reV/qa_qc/summary.py +++ b/reV/qa_qc/summary.py @@ -3,15 +3,17 @@ Compute and plot summary data """ import logging -import numpy as np import os + +import numpy as np import pandas as pd -import plotting as mplt import plotly.express as px - +import plotting as mplt from rex import Resource from rex.utilities import SpawnProcessPool, parse_table +from reV.utilities import MetaKeyName + logger = logging.getLogger(__name__) @@ -19,6 +21,7 @@ class SummarizeH5: """ reV Summary data for QA/QC """ + def __init__(self, h5_file, group=None): """ Parameters @@ -160,26 +163,25 @@ def summarize_dset(self, ds_name, process_size=None, max_workers=None, summary = [future.result() for future in futures] summary = pd.concat(summary) + elif process_size is None: + summary = self._compute_sites_summary(self.h5_file, + ds_name, + sites=sites, + group=self._group) else: - if process_size is None: - summary = self._compute_sites_summary(self.h5_file, - ds_name, - sites=sites, - group=self._group) - else: - sites = np.array_split( - sites, int(np.ceil(len(sites) / process_size))) - - summary = [] - for site_slice in sites: - summary.append(self._compute_sites_summary( - self.h5_file, ds_name, - sites=site_slice, - group=self._group)) + sites = np.array_split( + sites, int(np.ceil(len(sites) / process_size))) + + summary = [] + for site_slice in sites: + summary.append(self._compute_sites_summary( + self.h5_file, ds_name, + sites=site_slice, + group=self._group)) - summary = pd.concat(summary) + summary = pd.concat(summary) - summary.index.name = 'gid' + summary.index.name = MetaKeyName.GID else: summary = self._compute_ds_summary(self.h5_file, ds_name, @@ -206,9 +208,9 @@ def summarize_means(self, out_path=None): """ with Resource(self.h5_file, group=self._group) as f: meta = f.meta - if 'gid' not in meta: - if meta.index.name != 'gid': - meta.index.name = 'gid' + if MetaKeyName.GID not in meta: + if meta.index.name != MetaKeyName.GID: + meta.index.name = MetaKeyName.GID meta = meta.reset_index() @@ -270,6 +272,7 @@ class SummarizeSupplyCurve: """ Summarize Supply Curve table """ + def __init__(self, sc_table): self._sc_table = self._parse_summary(sc_table) @@ -401,6 +404,7 @@ class PlotBase: """ QA/QC Plotting base class """ + def __init__(self, data): """ Parameters @@ -461,7 +465,7 @@ def _check_value(df, values, scatter=True): values = [values] if scatter: - values += ['latitude', 'longitude'] + values += [MetaKeyName.LATITUDE, MetaKeyName.LONGITUDE] for value in values: if value not in df: @@ -475,6 +479,7 @@ class SummaryPlots(PlotBase): """ Plot summary data for QA/QC """ + def __init__(self, summary): """ Parameters @@ -523,7 +528,7 @@ def scatter_plot(self, value, cmap='viridis', out_path=None, **kwargs): Additional kwargs for plotting.dataframes.df_scatter """ self._check_value(self.summary, value) - mplt.df_scatter(self.summary, x='longitude', y='latitude', c=value, + mplt.df_scatter(self.summary, x=MetaKeyName.LONGITUDE, y=MetaKeyName.LATITUDE, c=value, colormap=cmap, filename=out_path, **kwargs) def scatter_plotly(self, value, cmap='Viridis', out_path=None, **kwargs): @@ -544,7 +549,7 @@ def scatter_plotly(self, value, cmap='Viridis', out_path=None, **kwargs): Additional kwargs for plotly.express.scatter """ self._check_value(self.summary, value) - fig = px.scatter(self.summary, x='longitude', y='latitude', + fig = px.scatter(self.summary, x=MetaKeyName.LONGITUDE, y=MetaKeyName.LATITUDE, color=value, color_continuous_scale=cmap, **kwargs) fig.update_layout(font=dict(family="Arial", size=18, color="black")) @@ -553,24 +558,24 @@ def scatter_plotly(self, value, cmap='Viridis', out_path=None, **kwargs): fig.show() - def _extract_sc_data(self, lcoe='mean_lcoe'): + def _extract_sc_data(self, lcoe=MetaKeyName.MEAN_LCOE): """ Extract supply curve data Parameters ---------- lcoe : str, optional - LCOE value to use for supply curve, by default 'mean_lcoe' + LCOE value to use for supply curve, by default MetaKeyName.MEAN_LCOE Returns ------- sc_df : pandas.DataFrame Supply curve data """ - values = ['capacity', lcoe] + values = [MetaKeyName.CAPACITY, lcoe] self._check_value(self.summary, values, scatter=False) sc_df = self.summary[values].sort_values(lcoe) - sc_df['cumulative_capacity'] = sc_df['capacity'].cumsum() + sc_df['cumulative_capacity'] = sc_df[MetaKeyName.CAPACITY].cumsum() return sc_df @@ -735,35 +740,35 @@ def columns(self): """ return list(self.sc_table.columns) - def _extract_sc_data(self, lcoe='mean_lcoe'): + def _extract_sc_data(self, lcoe=MetaKeyName.MEAN_LCOE): """ Extract supply curve data Parameters ---------- lcoe : str, optional - LCOE value to use for supply curve, by default 'mean_lcoe' + LCOE value to use for supply curve, by default MetaKeyName.MEAN_LCOE Returns ------- sc_df : pandas.DataFrame Supply curve data """ - values = ['capacity', lcoe] + values = [MetaKeyName.CAPACITY, lcoe] self._check_value(self.sc_table, values, scatter=False) sc_df = self.sc_table[values].sort_values(lcoe) - sc_df['cumulative_capacity'] = sc_df['capacity'].cumsum() + sc_df['cumulative_capacity'] = sc_df[MetaKeyName.CAPACITY].cumsum() return sc_df - def supply_curve_plot(self, lcoe='mean_lcoe', out_path=None, **kwargs): + def supply_curve_plot(self, lcoe=MetaKeyName.MEAN_LCOE, out_path=None, **kwargs): """ Plot supply curve (cumulative capacity vs lcoe) using seaborn.scatter Parameters ---------- lcoe : str, optional - LCOE value to plot, by default 'mean_lcoe' + LCOE value to plot, by default MetaKeyName.MEAN_LCOE out_path : str, optional File path to save plot to, by default None kwargs : dict @@ -773,14 +778,14 @@ def supply_curve_plot(self, lcoe='mean_lcoe', out_path=None, **kwargs): mplt.df_scatter(sc_df, x='cumulative_capacity', y=lcoe, filename=out_path, **kwargs) - def supply_curve_plotly(self, lcoe='mean_lcoe', out_path=None, **kwargs): + def supply_curve_plotly(self, lcoe=MetaKeyName.MEAN_LCOE, out_path=None, **kwargs): """ Plot supply curve (cumulative capacity vs lcoe) using plotly Parameters ---------- lcoe : str, optional - LCOE value to plot, by default 'mean_lcoe' + LCOE value to plot, by default MetaKeyName.MEAN_LCOE out_path : str, optional File path to save plot to, can be a .html or static image, by default None @@ -797,7 +802,7 @@ def supply_curve_plotly(self, lcoe='mean_lcoe', out_path=None, **kwargs): fig.show() @classmethod - def plot(cls, sc_table, out_dir, plot_type='plotly', lcoe='mean_lcoe', + def plot(cls, sc_table, out_dir, plot_type='plotly', lcoe=MetaKeyName.MEAN_LCOE, **kwargs): """ Create supply curve plot from supply curve table using lcoe value @@ -812,7 +817,7 @@ def plot(cls, sc_table, out_dir, plot_type='plotly', lcoe='mean_lcoe', plot_type : str, optional plot_type of plot to create 'plot' or 'plotly', by default 'plotly' lcoe : str, optional - LCOE value to plot, by default 'mean_lcoe' + LCOE value to plot, by default MetaKeyName.MEAN_LCOE kwargs : dict Additional plotting kwargs """ diff --git a/reV/rep_profiles/rep_profiles.py b/reV/rep_profiles/rep_profiles.py index 305cc4548..5e0195e86 100644 --- a/reV/rep_profiles/rep_profiles.py +++ b/reV/rep_profiles/rep_profiles.py @@ -5,26 +5,26 @@ @author: gbuster """ -from abc import ABC, abstractmethod -from concurrent.futures import as_completed -from copy import deepcopy +import contextlib import json import logging -import numpy as np import os -import pandas as pd -from scipy import stats +from abc import ABC, abstractmethod +from concurrent.futures import as_completed +from copy import deepcopy from warnings import warn - -from reV.handlers.outputs import Outputs -from reV.utilities.exceptions import FileInputError, DataShapeError -from reV.utilities import log_versions - +import numpy as np +import pandas as pd from rex.resource import Resource from rex.utilities.execution import SpawnProcessPool from rex.utilities.loggers import log_mem from rex.utilities.utilities import parse_year, to_records_array +from scipy import stats + +from reV.handlers.outputs import Outputs +from reV.utilities import MetaKeyName, log_versions +from reV.utilities.exceptions import DataShapeError, FileInputError logger = logging.getLogger(__name__) @@ -299,11 +299,12 @@ def run(cls, profiles, weights=None, rep_method='meanoid', class RegionRepProfile: """Framework to handle rep profile for one resource region""" - RES_GID_COL = 'res_gids' - GEN_GID_COL = 'gen_gids' + RES_GID_COL = MetaKeyName.RES_GIDS + GEN_GID_COL = MetaKeyName.GEN_GIDS - def __init__(self, gen_fpath, rev_summary, cf_dset='cf_profile', - rep_method='meanoid', err_method='rmse', weight='gid_counts', + def __init__(self, gen_fpath, rev_summary, cf_dset=MetaKeyName.CF_PROFILE, + rep_method='meanoid', err_method='rmse', + weight=MetaKeyName.GID_COUNTS, n_profiles=1): """ Parameters @@ -372,8 +373,8 @@ def _init_profiles_weights(self): with Resource(self._gen_fpath) as res: meta = res.meta - assert 'gid' in meta - source_res_gids = meta['gid'].values + assert MetaKeyName.GID in meta + source_res_gids = meta[MetaKeyName.GID].values msg = ('Resource gids from "gid" column in meta data from "{}" ' 'must be sorted! reV generation should always be run with ' 'sequential project points.'.format(self._gen_fpath)) @@ -441,7 +442,7 @@ def _get_region_attr(rev_summary, attr_name): if any(data): if isinstance(data[0], str): # pylint: disable=simplifiable-condition - if ('[' and ']' in data[0]) or ('(' and ')' in data[0]): + if ('[' and ']' in data[0]) or ('(' and ')' in data[0 data = [json.loads(s) for s in data] if isinstance(data[0], (list, tuple)): @@ -453,20 +454,20 @@ def _run_rep_methods(self): """Run the representative profile methods to find the meanoid/medianoid profile and find the profiles most similar.""" - if self.weights is not None: - if len(self.weights) != self.source_profiles.shape[1]: - e = ('Weights column "{}" resulted in {} weight scalars ' - 'which doesnt match gid column which yields ' - 'profiles with shape {}.' - .format(self._weight, len(self.weights), - self.source_profiles.shape)) - logger.debug('Gids from column "res_gids" with len {}: {}' - .format(len(self._res_gids), self._res_gids)) - logger.debug('Weights from column "{}" with len {}: {}' - .format(self._weight, len(self.weights), - self.weights)) - logger.error(e) - raise DataShapeError(e) + if (self.weights is not None and + (len(self.weights) != self.source_profiles.shape[1])): + e = ('Weights column "{}" resulted in {} weight scalars ' + 'which doesnt match gid column which yields ' + 'profiles with shape {}.' + .format(self._weight, len(self.weights), + self.source_profiles.shape)) + logger.debug('Gids from column "res_gids" with len {}: {}' + .format(len(self._res_gids), self._res_gids)) + logger.debug('Weights from column "{}" with len {}: {}' + .format(self._weight, len(self.weights), + self.weights)) + logger.error(e) + raise DataShapeError(e) self._profiles, self._i_reps = RepresentativeMethods.run( self.source_profiles, weights=self.weights, @@ -513,8 +514,10 @@ def rep_res_gids(self): @classmethod def get_region_rep_profile(cls, gen_fpath, rev_summary, - cf_dset='cf_profile', rep_method='meanoid', - err_method='rmse', weight='gid_counts', + cf_dset=MetaKeyName.CF_PROFILE, + rep_method='meanoid', + err_method='rmse', + weight=MetaKeyName.GID_COUNTS, n_profiles=1): """Class method for parallelization of rep profile calc. @@ -565,8 +568,9 @@ class RepProfilesBase(ABC): """Abstract utility framework for representative profile run classes.""" def __init__(self, gen_fpath, rev_summary, reg_cols=None, - cf_dset='cf_profile', rep_method='meanoid', err_method='rmse', - weight='gid_counts', n_profiles=1): + cf_dset=MetaKeyName.CF_PROFILE, rep_method='meanoid', + err_method='rmse', weight=MetaKeyName.GID_COUNTS, + n_profiles=1): """ Parameters ---------- @@ -681,10 +685,7 @@ def _check_req_cols(df, cols): if isinstance(cols, str): cols = [cols] - missing = [] - for c in cols: - if c not in df: - missing.append(c) + missing = [c for c in cols if c not in df] if any(missing): e = ('Column labels not found in rev_summary table: {}' @@ -813,10 +814,8 @@ def _init_h5_out(self, fout, save_rev_summary=True, meta = self.meta.copy() for c in meta.columns: - try: + with contextlib.suppress(ValueError): meta[c] = pd.to_numeric(meta[c]) - except ValueError: - pass Outputs.init_h5(fout, dsets, shapes, attrs, chunks, dtypes, meta, time_index=self.time_index) @@ -883,10 +882,12 @@ def run(self): class RepProfiles(RepProfilesBase): """RepProfiles""" - def __init__(self, gen_fpath, rev_summary, reg_cols, cf_dset='cf_profile', - rep_method='meanoid', err_method='rmse', weight='gid_counts', + def __init__(self, gen_fpath, rev_summary, reg_cols, + cf_dset=MetaKeyName.CF_PROFILE, + rep_method='meanoid', err_method='rmse', + weight=MetaKeyName.GID_COUNTS, n_profiles=1, aggregate_profiles=False): - """reV rep profiles class. + """ReV rep profiles class. ``reV`` rep profiles compute representative generation profiles for each supply curve point output by ``reV`` supply curve @@ -977,7 +978,7 @@ def __init__(self, gen_fpath, rev_summary, reg_cols, cf_dset='cf_profile', *equally* to the meanoid profile unless these weights are specified. - By default, ``'gid_counts'``. + By default, ``MetaKeyName.GID_COUNTS``. n_profiles : int, optional Number of representative profiles to save to the output file. By default, ``1``. @@ -1030,7 +1031,7 @@ def _set_meta(self): else: self._meta = self._rev_summary.groupby(self._reg_cols) self._meta = ( - self._meta['timezone'] + self._meta[MetaKeyName.TIMEZONE] .apply(lambda x: stats.mode(x, keepdims=True).mode[0]) ) self._meta = self._meta.reset_index() diff --git a/reV/supply_curve/aggregation.py b/reV/supply_curve/aggregation.py index 77cfeb4ca..23fd0e388 100644 --- a/reV/supply_curve/aggregation.py +++ b/reV/supply_curve/aggregation.py @@ -1,27 +1,30 @@ # -*- coding: utf-8 -*- -""" -reV aggregation framework. -""" +"""reV aggregation framework.""" + +import contextlib +import logging +import os from abc import ABC, abstractmethod + import h5py -import logging import numpy as np -import os import pandas as pd +from rex.resource import Resource +from rex.utilities.execution import SpawnProcessPool +from rex.utilities.loggers import log_mem -from reV.handlers.outputs import Outputs from reV.handlers.exclusions import ExclusionLayers +from reV.handlers.outputs import Outputs from reV.supply_curve.exclusions import ExclusionMaskFromDict from reV.supply_curve.extent import SupplyCurveExtent -from reV.supply_curve.tech_mapping import TechMapping from reV.supply_curve.points import AggregationSupplyCurvePoint -from reV.utilities.exceptions import (EmptySupplyCurvePointError, - FileInputError, SupplyCurveInputError) -from reV.utilities import log_versions - -from rex.resource import Resource -from rex.utilities.execution import SpawnProcessPool -from rex.utilities.loggers import log_mem +from reV.supply_curve.tech_mapping import TechMapping +from reV.utilities import MetaKeyName, log_versions +from reV.utilities.exceptions import ( + EmptySupplyCurvePointError, + FileInputError, + SupplyCurveInputError, +) logger = logging.getLogger(__name__) @@ -29,8 +32,13 @@ class AbstractAggFileHandler(ABC): """Simple framework to handle aggregation file context managers.""" - def __init__(self, excl_fpath, excl_dict=None, area_filter_kernel='queen', - min_area=None): + def __init__( + self, + excl_fpath, + excl_dict=None, + area_filter_kernel="queen", + min_area=None, + ): """ Parameters ---------- @@ -51,9 +59,12 @@ def __init__(self, excl_fpath, excl_dict=None, area_filter_kernel='queen', by default None """ self._excl_fpath = excl_fpath - self._excl = ExclusionMaskFromDict(excl_fpath, layers_dict=excl_dict, - min_area=min_area, - kernel=area_filter_kernel) + self._excl = ExclusionMaskFromDict( + excl_fpath, + layers_dict=excl_dict, + min_area=min_area, + kernel=area_filter_kernel, + ) def __enter__(self): return self @@ -95,9 +106,15 @@ class AggFileHandler(AbstractAggFileHandler): DEFAULT_H5_HANDLER = Resource - def __init__(self, excl_fpath, h5_fpath, excl_dict=None, - area_filter_kernel='queen', min_area=None, - h5_handler=None): + def __init__( + self, + excl_fpath, + h5_fpath, + excl_dict=None, + area_filter_kernel="queen", + min_area=None, + h5_handler=None, + ): """ Parameters ---------- @@ -121,9 +138,12 @@ def __init__(self, excl_fpath, h5_fpath, excl_dict=None, Optional special handler similar to the rex.Resource handler which is default. """ - super().__init__(excl_fpath, excl_dict=excl_dict, - area_filter_kernel=area_filter_kernel, - min_area=min_area) + super().__init__( + excl_fpath, + excl_dict=excl_dict, + area_filter_kernel=area_filter_kernel, + min_area=min_area, + ) if h5_handler is None: self._h5 = Resource(h5_fpath) @@ -152,10 +172,19 @@ class BaseAggregation(ABC): """Abstract supply curve points aggregation framework based on only an exclusion file and techmap.""" - def __init__(self, excl_fpath, tm_dset, excl_dict=None, - area_filter_kernel='queen', min_area=None, - resolution=64, excl_area=None, res_fpath=None, gids=None, - pre_extract_inclusions=False): + def __init__( + self, + excl_fpath, + tm_dset, + excl_dict=None, + area_filter_kernel="queen", + min_area=None, + resolution=64, + excl_area=None, + res_fpath=None, + gids=None, + pre_extract_inclusions=False, + ): """ Parameters ---------- @@ -208,12 +237,15 @@ def __init__(self, excl_fpath, tm_dset, excl_dict=None, self._validate_tech_mapping() if pre_extract_inclusions: - self._inclusion_mask = \ + self._inclusion_mask = ( ExclusionMaskFromDict.extract_inclusion_mask( - excl_fpath, tm_dset, + excl_fpath, + tm_dset, excl_dict=excl_dict, area_filter_kernel=area_filter_kernel, - min_area=min_area) + min_area=min_area, + ) + ) else: self._inclusion_mask = None @@ -228,20 +260,28 @@ def _validate_tech_mapping(self): if tm_in_excl: logger.info('Found techmap "{}".'.format(self._tm_dset)) elif not tm_in_excl and not excl_fp_is_str: - msg = ('Could not find techmap dataset "{}" and cannot run ' - 'techmap with arbitrary multiple exclusion filepaths ' - 'to write to: {}'.format(self._tm_dset, self._excl_fpath)) + msg = ( + 'Could not find techmap dataset "{}" and cannot run ' + "techmap with arbitrary multiple exclusion filepaths " + "to write to: {}".format(self._tm_dset, self._excl_fpath) + ) logger.error(msg) raise RuntimeError(msg) else: - logger.info('Could not find techmap "{}". Running techmap module.' - .format(self._tm_dset)) + logger.info( + 'Could not find techmap "{}". Running techmap module.'.format( + self._tm_dset + ) + ) try: - TechMapping.run(self._excl_fpath, self._res_fpath, - dset=self._tm_dset) + TechMapping.run( + self._excl_fpath, self._res_fpath, dset=self._tm_dset + ) except Exception as e: - msg = ('TechMapping process failed. Received the ' - 'following error:\n{}'.format(e)) + msg = ( + "TechMapping process failed. Received the " + "following error:\n{}".format(e) + ) logger.exception(msg) raise RuntimeError(msg) from e @@ -255,8 +295,9 @@ def gids(self): ndarray """ if self._gids is None: - with SupplyCurveExtent(self._excl_fpath, - resolution=self._resolution) as sc: + with SupplyCurveExtent( + self._excl_fpath, resolution=self._resolution + ) as sc: self._gids = sc.valid_sc_points(self._tm_dset) elif np.issubdtype(type(self._gids), np.number): self._gids = np.array([self._gids]) @@ -274,8 +315,9 @@ def shape(self): tuple """ if self._shape is None: - with SupplyCurveExtent(self._excl_fpath, - resolution=self._resolution) as sc: + with SupplyCurveExtent( + self._excl_fpath, resolution=self._resolution + ) as sc: self._shape = sc.exclusions.shape return self._shape @@ -301,14 +343,18 @@ def _get_excl_area(excl_fpath, excl_area=None): Area of an exclusion pixel in km2 """ if excl_area is None: - logger.debug('Setting the exclusion area from the area of a pixel ' - 'in {}'.format(excl_fpath)) + logger.debug( + "Setting the exclusion area from the area of a pixel " + "in {}".format(excl_fpath) + ) with ExclusionLayers(excl_fpath) as excl: excl_area = excl.pixel_area if excl_area is None: - e = ('No exclusion pixel area was input and could not parse ' - 'area from the exclusion file attributes!') + e = ( + "No exclusion pixel area was input and could not parse " + "area from the exclusion file attributes!" + ) logger.error(e) raise SupplyCurveInputError(e) @@ -337,14 +383,17 @@ def _check_inclusion_mask(inclusion_mask, gids, excl_shape): elif isinstance(inclusion_mask, np.ndarray): assert inclusion_mask.shape == excl_shape elif inclusion_mask is not None: - msg = ('Expected inclusion_mask to be dict or array but received ' - '{}'.format(type(inclusion_mask))) + msg = ( + "Expected inclusion_mask to be dict or array but received " + "{}".format(type(inclusion_mask)) + ) logger.error(msg) raise SupplyCurveInputError(msg) @staticmethod - def _get_gid_inclusion_mask(inclusion_mask, gid, slice_lookup, - resolution=64): + def _get_gid_inclusion_mask( + inclusion_mask, gid, slice_lookup, resolution=64 + ): """ Get inclusion mask for desired gid @@ -381,8 +430,10 @@ def _get_gid_inclusion_mask(inclusion_mask, gid, slice_lookup, row_slice, col_slice = slice_lookup[gid] gid_inclusions = inclusion_mask[row_slice, col_slice] elif inclusion_mask is not None: - msg = ('Expected inclusion_mask to be dict or array but received ' - '{}'.format(type(inclusion_mask))) + msg = ( + "Expected inclusion_mask to be dict or array but received " + "{}".format(type(inclusion_mask)) + ) logger.error(msg) raise SupplyCurveInputError(msg) @@ -407,26 +458,32 @@ def _parse_gen_index(gen_fpath): generation run. """ - if gen_fpath.endswith('.h5'): + if gen_fpath.endswith(".h5"): with Resource(gen_fpath) as f: gen_index = f.meta - elif gen_fpath.endswith('.csv'): + elif gen_fpath.endswith(".csv"): gen_index = pd.read_csv(gen_fpath) else: - msg = ('Could not recognize gen_fpath input, needs to be reV gen ' - 'output h5 or project points csv but received: {}' - .format(gen_fpath)) + msg = ( + "Could not recognize gen_fpath input, needs to be reV gen " + "output h5 or project points csv but received: {}".format( + gen_fpath + ) + ) logger.error(msg) raise FileInputError(msg) - if 'gid' in gen_index: - gen_index = gen_index.rename(columns={'gid': 'res_gids'}) - gen_index['gen_gids'] = gen_index.index - gen_index = gen_index[['res_gids', 'gen_gids']] - gen_index = gen_index.set_index(keys='res_gids') - gen_index = \ - gen_index.reindex(range(int(gen_index.index.max() + 1))) - gen_index = gen_index['gen_gids'].values + if MetaKeyName.GID in gen_index: + gen_index = gen_index.rename( + columns={MetaKeyName.GID: MetaKeyName.RES_GIDS} + ) + gen_index[MetaKeyName.GEN_GIDS] = gen_index.index + gen_index = gen_index[[MetaKeyName.RES_GIDS, MetaKeyName.GEN_GIDS]] + gen_index = gen_index.set_index(keys=MetaKeyName.RES_GIDS) + gen_index = gen_index.reindex( + range(int(gen_index.index.max() + 1)) + ) + gen_index = gen_index[MetaKeyName.GEN_GIDS].values gen_index[np.isnan(gen_index)] = -1 gen_index = gen_index.astype(np.int32) else: @@ -439,10 +496,19 @@ class Aggregation(BaseAggregation): """Concrete but generalized aggregation framework to aggregate ANY reV h5 file to a supply curve grid (based on an aggregated exclusion grid).""" - def __init__(self, excl_fpath, tm_dset, *agg_dset, - excl_dict=None, area_filter_kernel='queen', min_area=None, - resolution=64, excl_area=None, gids=None, - pre_extract_inclusions=False): + def __init__( + self, + excl_fpath, + tm_dset, + *agg_dset, + excl_dict=None, + area_filter_kernel="queen", + min_area=None, + resolution=64, + excl_area=None, + gids=None, + pre_extract_inclusions=False, + ): """ Parameters ---------- @@ -486,18 +552,24 @@ def __init__(self, excl_fpath, tm_dset, *agg_dset, the inclusion mask on the fly with parallel workers. """ log_versions(logger) - logger.info('Initializing Aggregation...') - logger.debug('Exclusion filepath: {}'.format(excl_fpath)) - logger.debug('Exclusion dict: {}'.format(excl_dict)) - - super().__init__(excl_fpath, tm_dset, excl_dict=excl_dict, - area_filter_kernel=area_filter_kernel, - min_area=min_area, resolution=resolution, - excl_area=excl_area, gids=gids, - pre_extract_inclusions=pre_extract_inclusions) + logger.info("Initializing Aggregation...") + logger.debug("Exclusion filepath: {}".format(excl_fpath)) + logger.debug("Exclusion dict: {}".format(excl_dict)) + + super().__init__( + excl_fpath, + tm_dset, + excl_dict=excl_dict, + area_filter_kernel=area_filter_kernel, + min_area=min_area, + resolution=resolution, + excl_area=excl_area, + gids=gids, + pre_extract_inclusions=pre_extract_inclusions, + ) if isinstance(agg_dset, str): - agg_dset = (agg_dset, ) + agg_dset = (agg_dset,) self._agg_dsets = agg_dset @@ -505,33 +577,51 @@ def _check_files(self, h5_fpath): """Do a preflight check on input files""" if not os.path.exists(self._excl_fpath): - raise FileNotFoundError('Could not find required exclusions file: ' - '{}'.format(self._excl_fpath)) + raise FileNotFoundError( + "Could not find required exclusions file: " "{}".format( + self._excl_fpath + ) + ) if not os.path.exists(h5_fpath): - raise FileNotFoundError('Could not find required h5 file: ' - '{}'.format(h5_fpath)) + raise FileNotFoundError( + "Could not find required h5 file: " "{}".format(h5_fpath) + ) - with h5py.File(self._excl_fpath, 'r') as f: + with h5py.File(self._excl_fpath, "r") as f: if self._tm_dset not in f: - raise FileInputError('Could not find techmap dataset "{}" ' - 'in exclusions file: {}' - .format(self._tm_dset, - self._excl_fpath)) + raise FileInputError( + 'Could not find techmap dataset "{}" ' + "in exclusions file: {}".format( + self._tm_dset, self._excl_fpath + ) + ) with Resource(h5_fpath) as f: for dset in self._agg_dsets: if dset not in f: - raise FileInputError('Could not find provided dataset "{}"' - ' in h5 file: {}' - .format(dset, h5_fpath)) + raise FileInputError( + 'Could not find provided dataset "{}"' + " in h5 file: {}".format(dset, h5_fpath) + ) @classmethod - def run_serial(cls, excl_fpath, h5_fpath, tm_dset, *agg_dset, - agg_method='mean', excl_dict=None, inclusion_mask=None, - area_filter_kernel='queen', min_area=None, - resolution=64, excl_area=0.0081, gids=None, - gen_index=None): + def run_serial( + cls, + excl_fpath, + h5_fpath, + tm_dset, + *agg_dset, + agg_method="mean", + excl_dict=None, + inclusion_mask=None, + area_filter_kernel="queen", + min_area=None, + resolution=64, + excl_area=0.0081, + gids=None, + gen_index=None, + ): """ Standalone method to aggregate - can be parallelized. @@ -602,17 +692,19 @@ def run_serial(cls, excl_fpath, h5_fpath, tm_dset, *agg_dset, cls._check_inclusion_mask(inclusion_mask, gids, exclusion_shape) # pre-extract handlers so they are not repeatedly initialized - file_kwargs = {'excl_dict': excl_dict, - 'area_filter_kernel': area_filter_kernel, - 'min_area': min_area} - dsets = agg_dset + ('meta', ) + file_kwargs = { + "excl_dict": excl_dict, + "area_filter_kernel": area_filter_kernel, + "min_area": min_area, + } + dsets = (*agg_dset, "meta",) agg_out = {ds: [] for ds in dsets} with AggFileHandler(excl_fpath, h5_fpath, **file_kwargs) as fh: n_finished = 0 for gid in gids: gid_inclusions = cls._get_gid_inclusion_mask( - inclusion_mask, gid, slice_lookup, - resolution=resolution) + inclusion_mask, gid, slice_lookup, resolution=resolution + ) try: gid_out = AggregationSupplyCurvePoint.run( gid, @@ -627,28 +719,40 @@ def run_serial(cls, excl_fpath, h5_fpath, tm_dset, *agg_dset, excl_area=excl_area, exclusion_shape=exclusion_shape, close=False, - gen_index=gen_index) + gen_index=gen_index, + ) except EmptySupplyCurvePointError: - logger.debug('SC gid {} is fully excluded or does not ' - 'have any valid source data!'.format(gid)) + logger.debug( + "SC gid {} is fully excluded or does not " + "have any valid source data!".format(gid) + ) except Exception as e: - msg = 'SC gid {} failed!'.format(gid) + msg = "SC gid {} failed!".format(gid) logger.exception(msg) raise RuntimeError(msg) from e else: n_finished += 1 - logger.debug('Serial aggregation: ' - '{} out of {} points complete' - .format(n_finished, len(gids))) + logger.debug( + "Serial aggregation: " + "{} out of {} points complete".format( + n_finished, len(gids) + ) + ) log_mem(logger) for k, v in gid_out.items(): agg_out[k].append(v) return agg_out - def run_parallel(self, h5_fpath, agg_method='mean', excl_area=None, - max_workers=None, sites_per_worker=100): + def run_parallel( + self, + h5_fpath, + agg_method="mean", + excl_area=None, + max_workers=None, + sites_per_worker=100, + ): """ Aggregate in parallel @@ -681,22 +785,29 @@ def run_parallel(self, h5_fpath, agg_method='mean', excl_area=None, chunks = np.array_split(self.gids, chunks) if self._inclusion_mask is not None: - with SupplyCurveExtent(self._excl_fpath, - resolution=self._resolution) as sc: + with SupplyCurveExtent( + self._excl_fpath, resolution=self._resolution + ) as sc: assert sc.exclusions.shape == self._inclusion_mask.shape slice_lookup = sc.get_slice_lookup(self.gids) - logger.info('Running supply curve point aggregation for ' - 'points {} through {} at a resolution of {} ' - 'on {} cores in {} chunks.' - .format(self.gids[0], self.gids[-1], self._resolution, - max_workers, len(chunks))) + logger.info( + "Running supply curve point aggregation for " + "points {} through {} at a resolution of {} " + "on {} cores in {} chunks.".format( + self.gids[0], + self.gids[-1], + self._resolution, + max_workers, + len(chunks), + ) + ) n_finished = 0 futures = [] - dsets = self._agg_dsets + ('meta', ) + dsets = self._agg_dsets + ("meta",) agg_out = {ds: [] for ds in dsets} - loggers = [__name__, 'reV.supply_curve.points', 'reV'] + loggers = [__name__, "reV.supply_curve.points", "reV"] with SpawnProcessPool(max_workers=max_workers, loggers=loggers) as exe: # iterate through split executions, submitting each to worker for gid_set in chunks: @@ -709,36 +820,45 @@ def run_parallel(self, h5_fpath, agg_method='mean', excl_area=None, chunk_incl_masks[gid] = self._inclusion_mask[rs, cs] # submit executions and append to futures list - futures.append(exe.submit( - self.run_serial, - self._excl_fpath, - h5_fpath, - self._tm_dset, - *self._agg_dsets, - agg_method=agg_method, - excl_dict=self._excl_dict, - inclusion_mask=chunk_incl_masks, - area_filter_kernel=self._area_filter_kernel, - min_area=self._min_area, - resolution=self._resolution, - excl_area=excl_area, - gids=gid_set, - gen_index=gen_index)) + futures.append( + exe.submit( + self.run_serial, + self._excl_fpath, + h5_fpath, + self._tm_dset, + *self._agg_dsets, + agg_method=agg_method, + excl_dict=self._excl_dict, + inclusion_mask=chunk_incl_masks, + area_filter_kernel=self._area_filter_kernel, + min_area=self._min_area, + resolution=self._resolution, + excl_area=excl_area, + gids=gid_set, + gen_index=gen_index, + ) + ) # gather results for future in futures: n_finished += 1 - logger.info('Parallel aggregation futures collected: ' - '{} out of {}' - .format(n_finished, len(chunks))) + logger.info( + "Parallel aggregation futures collected: " + "{} out of {}".format(n_finished, len(chunks)) + ) for k, v in future.result().items(): if v: agg_out[k].extend(v) return agg_out - def aggregate(self, h5_fpath, agg_method='mean', max_workers=None, - sites_per_worker=100): + def aggregate( + self, + h5_fpath, + agg_method="mean", + max_workers=None, + sites_per_worker=100, + ): """ Aggregate with given agg_method @@ -766,38 +886,44 @@ def aggregate(self, h5_fpath, agg_method='mean', max_workers=None, if max_workers == 1: self._check_files(h5_fpath) gen_index = self._parse_gen_index(h5_fpath) - agg = self.run_serial(self._excl_fpath, - h5_fpath, - self._tm_dset, - *self._agg_dsets, - agg_method=agg_method, - excl_dict=self._excl_dict, - gids=self.gids, - inclusion_mask=self._inclusion_mask, - area_filter_kernel=self._area_filter_kernel, - min_area=self._min_area, - resolution=self._resolution, - excl_area=self._excl_area, - gen_index=gen_index) + agg = self.run_serial( + self._excl_fpath, + h5_fpath, + self._tm_dset, + *self._agg_dsets, + agg_method=agg_method, + excl_dict=self._excl_dict, + gids=self.gids, + inclusion_mask=self._inclusion_mask, + area_filter_kernel=self._area_filter_kernel, + min_area=self._min_area, + resolution=self._resolution, + excl_area=self._excl_area, + gen_index=gen_index, + ) else: - agg = self.run_parallel(h5_fpath=h5_fpath, - agg_method=agg_method, - excl_area=self._excl_area, - max_workers=max_workers, - sites_per_worker=sites_per_worker) - - if not agg['meta']: - e = ('Supply curve aggregation found no non-excluded SC points. ' - 'Please check your exclusions or subset SC GID selection.') + agg = self.run_parallel( + h5_fpath=h5_fpath, + agg_method=agg_method, + excl_area=self._excl_area, + max_workers=max_workers, + sites_per_worker=sites_per_worker, + ) + + if not agg["meta"]: + e = ( + "Supply curve aggregation found no non-excluded SC points. " + "Please check your exclusions or subset SC GID selection." + ) logger.error(e) raise EmptySupplyCurvePointError(e) for k, v in agg.items(): - if k == 'meta': + if k == "meta": v = pd.concat(v, axis=1).T - v = v.sort_values('sc_point_gid') + v = v.sort_values(MetaKeyName.SC_POINT_GID) v = v.reset_index(drop=True) - v.index.name = 'sc_gid' + v.index.name = MetaKeyName.SC_GID agg[k] = v else: v = np.dstack(v)[0] @@ -821,12 +947,10 @@ def save_agg_to_h5(h5_fpath, out_fpath, aggregation): Aggregated values for each aggregation dataset """ agg_out = aggregation.copy() - meta = agg_out.pop('meta').reset_index() - for c in meta.columns: - try: + meta = agg_out.pop("meta").reset_index() + with contextlib.suppress(ValueError, TypeError): + for c in meta.columns: meta[c] = pd.to_numeric(meta[c]) - except (ValueError, TypeError): - pass dsets = [] shapes = {} @@ -840,7 +964,7 @@ def save_agg_to_h5(h5_fpath, out_fpath, aggregation): shape = data.shape shapes[dset] = shape if len(data.shape) == 2: - if ('time_index' in f) and (shape[0] == f.shape[0]): + if ("time_index" in f) and (shape[0] == f.shape[0]): if time_index is None: time_index = f.time_index @@ -849,19 +973,40 @@ def save_agg_to_h5(h5_fpath, out_fpath, aggregation): chunks[dset] = chunk dtypes[dset] = dtype - Outputs.init_h5(out_fpath, dsets, shapes, attrs, chunks, dtypes, - meta, time_index=time_index) - - with Outputs(out_fpath, mode='a') as out: + Outputs.init_h5( + out_fpath, + dsets, + shapes, + attrs, + chunks, + dtypes, + meta, + time_index=time_index, + ) + + with Outputs(out_fpath, mode="a") as out: for dset, data in agg_out.items(): out[dset] = data @classmethod - def run(cls, excl_fpath, h5_fpath, tm_dset, *agg_dset, - excl_dict=None, area_filter_kernel='queen', min_area=None, - resolution=64, excl_area=None, gids=None, - pre_extract_inclusions=False, agg_method='mean', max_workers=None, - sites_per_worker=100, out_fpath=None): + def run( + cls, + excl_fpath, + h5_fpath, + tm_dset, + *agg_dset, + excl_dict=None, + area_filter_kernel="queen", + min_area=None, + resolution=64, + excl_area=None, + gids=None, + pre_extract_inclusions=False, + agg_method="mean", + max_workers=None, + sites_per_worker=100, + out_fpath=None, + ): """Get the supply curve points aggregation summary. Parameters @@ -923,15 +1068,25 @@ def run(cls, excl_fpath, h5_fpath, tm_dset, *agg_dset, Aggregated values for each aggregation dataset """ - agg = cls(excl_fpath, tm_dset, *agg_dset, - excl_dict=excl_dict, area_filter_kernel=area_filter_kernel, - min_area=min_area, resolution=resolution, - excl_area=excl_area, gids=gids, - pre_extract_inclusions=pre_extract_inclusions) - - aggregation = agg.aggregate(h5_fpath=h5_fpath, agg_method=agg_method, - max_workers=max_workers, - sites_per_worker=sites_per_worker) + agg = cls( + excl_fpath, + tm_dset, + *agg_dset, + excl_dict=excl_dict, + area_filter_kernel=area_filter_kernel, + min_area=min_area, + resolution=resolution, + excl_area=excl_area, + gids=gids, + pre_extract_inclusions=pre_extract_inclusions, + ) + + aggregation = agg.aggregate( + h5_fpath=h5_fpath, + agg_method=agg_method, + max_workers=max_workers, + sites_per_worker=sites_per_worker, + ) if out_fpath is not None: agg.save_agg_to_h5(h5_fpath, out_fpath, aggregation) diff --git a/reV/supply_curve/competitive_wind_farms.py b/reV/supply_curve/competitive_wind_farms.py index 455368715..674009d3d 100644 --- a/reV/supply_curve/competitive_wind_farms.py +++ b/reV/supply_curve/competitive_wind_farms.py @@ -3,10 +3,12 @@ Competitive Wind Farms exclusion handler """ import logging -import numpy as np +import numpy as np from rex.utilities.utilities import parse_table +from reV.utilities import MetaKeyName + logger = logging.getLogger(__name__) @@ -76,23 +78,23 @@ def __getitem__(self, keys): """ if not isinstance(keys, tuple): msg = ("{} must be a tuple of form (source, gid) where source is: " - "'sc_gid', 'sc_point_gid', or 'upwind', 'downwind'" - .format(keys)) + "MetaKeyName.SC_GID, '{}', or 'upwind', 'downwind'" + .format(keys, MetaKeyName.SC_POINT_GID)) logger.error(msg) raise ValueError(msg) source, gid = keys - if source == 'sc_point_gid': + if source == MetaKeyName.SC_POINT_GID: out = self.map_sc_gid_to_sc_point_gid(gid) - elif source == 'sc_gid': + elif source == MetaKeyName.SC_GID: out = self.map_sc_point_gid_to_sc_gid(gid) elif source == 'upwind': out = self.map_upwind(gid) elif source == 'downwind': out = self.map_downwind(gid) else: - msg = ("{} must be: 'sc_gid', 'sc_point_gid', or 'upwind', " - "'downwind'".format(source)) + msg = ("{} must be: MetaKeyName.SC_GID, {}, or 'upwind', " + "'downwind'".format(source, MetaKeyName.SC_POINT_GID)) logger.error(msg) raise ValueError(msg) @@ -182,7 +184,7 @@ def _parse_wind_dirs(cls, wind_dirs): """ wind_dirs = cls._parse_table(wind_dirs) - wind_dirs = wind_dirs.set_index('sc_point_gid') + wind_dirs = wind_dirs.set_index(MetaKeyName.SC_POINT_GID) columns = [c for c in wind_dirs if c.endswith(('_gid', '_pr'))] wind_dirs = wind_dirs[columns] @@ -212,21 +214,22 @@ def _parse_sc_points(cls, sc_points, offshore=False): Mask array to mask excluded sc_point_gids """ sc_points = cls._parse_table(sc_points) - if 'offshore' in sc_points and not offshore: + if MetaKeyName.OFFSHORE in sc_points and not offshore: logger.debug('Not including offshore supply curve points in ' 'CompetitiveWindFarm') - mask = sc_points['offshore'] == 0 + mask = sc_points[MetaKeyName.OFFSHORE] == 0 sc_points = sc_points.loc[mask] - mask = np.ones(int(1 + sc_points['sc_point_gid'].max()), dtype=bool) + mask = np.ones(int(1 + sc_points[MetaKeyName.SC_POINT_GID].max()), + dtype=bool) - sc_points = sc_points[['sc_gid', 'sc_point_gid']] - sc_gids = sc_points.set_index('sc_gid') + sc_points = sc_points[[MetaKeyName.SC_GID, MetaKeyName.SC_POINT_GID]] + sc_gids = sc_points.set_index(MetaKeyName.SC_GID) sc_gids = {k: int(v[0]) for k, v in sc_gids.iterrows()} sc_point_gids = \ - sc_points.groupby('sc_point_gid')['sc_gid'].unique().to_frame() - sc_point_gids = {int(k): v['sc_gid'] + sc_points.groupby(MetaKeyName.SC_POINT_GID)[MetaKeyName.SC_GID].unique().to_frame() + sc_point_gids = {int(k): v[MetaKeyName.SC_GID] for k, v in sc_point_gids.iterrows()} return sc_gids, sc_point_gids, mask @@ -338,6 +341,7 @@ def map_upwind(self, sc_point_gid): ---------- sc_point_gid : int Supply point curve gid to get upwind neighbors + Returns ------- int | list @@ -353,6 +357,7 @@ def map_downwind(self, sc_point_gid): ---------- sc_point_gid : int Supply point curve gid to get downwind neighbors + Returns ------- int | list @@ -407,13 +412,13 @@ def remove_noncompetitive_farm(self, sc_points, sort_on='total_lcoe', wind farms """ sc_points = self._parse_table(sc_points) - if 'offshore' in sc_points and not self._offshore: - mask = sc_points['offshore'] == 0 + if MetaKeyName.OFFSHORE in sc_points and not self._offshore: + mask = sc_points[MetaKeyName.OFFSHORE] == 0 sc_points = sc_points.loc[mask] sc_points = sc_points.sort_values(sort_on) - sc_point_gids = sc_points['sc_point_gid'].values.astype(int) + sc_point_gids = sc_points[MetaKeyName.SC_POINT_GID].values.astype(int) for i in range(len(sc_points)): gid = sc_point_gids[i] @@ -428,7 +433,7 @@ def remove_noncompetitive_farm(self, sc_points, sort_on='total_lcoe', self.exclude_sc_point_gid(n) sc_gids = self.sc_gids - mask = sc_points['sc_gid'].isin(sc_gids) + mask = sc_points[MetaKeyName.SC_GID].isin(sc_gids) return sc_points.loc[mask].reset_index(drop=True) diff --git a/reV/supply_curve/exclusions.py b/reV/supply_curve/exclusions.py index 4ec2b9429..a1278de52 100644 --- a/reV/supply_curve/exclusions.py +++ b/reV/supply_curve/exclusions.py @@ -3,14 +3,15 @@ Generate reV inclusion mask from exclusion layers """ import logging -import numpy as np -from scipy import ndimage from warnings import warn +import numpy as np from rex.utilities.loggers import log_mem +from scipy import ndimage + from reV.handlers.exclusions import ExclusionLayers -from reV.utilities.exceptions import ExclusionLayerError -from reV.utilities.exceptions import SupplyCurveInputError +from reV.utilities import MetaKeyName +from reV.utilities.exceptions import ExclusionLayerError, SupplyCurveInputError logger = logging.getLogger(__name__) @@ -331,7 +332,7 @@ def _check_mask_type(self): contradictory Returns - ------ + ------- mask : str Mask type """ @@ -640,7 +641,7 @@ def excl_h5(self): Returns ------- _excl_h5 : ExclusionLayers - """ + """ return self._excl_h5 @property @@ -665,7 +666,7 @@ def layer_names(self): Returns ------- list - """ + """ return self._layers.keys() @property @@ -676,7 +677,7 @@ def layers(self): Returns ------- list - """ + """ return self._layers.values() @property @@ -700,7 +701,7 @@ def latitude(self): ------- ndarray """ - return self.excl_h5['latitude'] + return self.excl_h5[MetaKeyName.LATITUDE] @property def longitude(self): @@ -711,7 +712,7 @@ def longitude(self): ------- ndarray """ - return self.excl_h5['longitude'] + return self.excl_h5[MetaKeyName.LONGITUDE] def add_layer(self, layer, replace=False): """ diff --git a/reV/supply_curve/extent.py b/reV/supply_curve/extent.py index 1e02b1511..b63e25cac 100644 --- a/reV/supply_curve/extent.py +++ b/reV/supply_curve/extent.py @@ -3,14 +3,15 @@ reV supply curve extent """ import logging + import numpy as np import pandas as pd +from rex.utilities.utilities import get_chunk_ranges from reV.handlers.exclusions import ExclusionLayers +from reV.utilities import MetaKeyName from reV.utilities.exceptions import SupplyCurveError, SupplyCurveInputError -from rex.utilities.utilities import get_chunk_ranges - logger = logging.getLogger(__name__) @@ -287,8 +288,8 @@ def latitude(self): for r, c in zip(sc_rows.flatten(), sc_cols.flatten()): r = self.excl_row_slices[r] c = self.excl_col_slices[c] - lats.append(self.exclusions['latitude', r, c].mean()) - lons.append(self.exclusions['longitude', r, c].mean()) + lats.append(self.exclusions[MetaKeyName.LATITUDE, r, c].mean()) + lons.append(self.exclusions[MetaKeyName.LONGITUDE, r, c].mean()) self._latitude = np.array(lats, dtype='float32') self._longitude = np.array(lons, dtype='float32') @@ -313,8 +314,8 @@ def longitude(self): for r, c in zip(sc_rows.flatten(), sc_cols.flatten()): r = self.excl_row_slices[r] c = self.excl_col_slices[c] - lats.append(self.exclusions['latitude', r, c].mean()) - lons.append(self.exclusions['longitude', r, c].mean()) + lats.append(self.exclusions[MetaKeyName.LATITUDE, r, c].mean()) + lons.append(self.exclusions[MetaKeyName.LONGITUDE, r, c].mean()) self._latitude = np.array(lats, dtype='float32') self._longitude = np.array(lons, dtype='float32') @@ -370,7 +371,7 @@ def points(self): self._points = pd.DataFrame({'row_ind': self.row_indices.copy(), 'col_ind': self.col_indices.copy()}) - self._points.index.name = 'gid' # sc_point_gid + self._points.index.name = MetaKeyName.GID # sc_point_gid return self._points diff --git a/reV/supply_curve/points.py b/reV/supply_curve/points.py index 7f229b1e3..2210d379b 100644 --- a/reV/supply_curve/points.py +++ b/reV/supply_curve/points.py @@ -2,26 +2,29 @@ """ reV supply curve points frameworks. """ -from abc import ABC import logging +from abc import ABC +from warnings import warn + import numpy as np import pandas as pd -from warnings import warn +from rex.multi_time_resource import MultiTimeResource +from rex.resource import BaseResource, Resource +from rex.utilities.utilities import jsonify_dict from reV.econ.economies_of_scale import EconomiesOfScale from reV.econ.utilities import lcoe_fcr from reV.handlers.exclusions import ExclusionLayers from reV.supply_curve.exclusions import ExclusionMask, ExclusionMaskFromDict -from reV.utilities.exceptions import (SupplyCurveInputError, - EmptySupplyCurvePointError, - InputWarning, - FileInputError, - DataShapeError, - OutputWarning) - -from rex.resource import Resource, BaseResource -from rex.multi_time_resource import MultiTimeResource -from rex.utilities.utilities import jsonify_dict +from reV.utilities import MetaKeyName +from reV.utilities.exceptions import ( + DataShapeError, + EmptySupplyCurvePointError, + FileInputError, + InputWarning, + OutputWarning, + SupplyCurveInputError, +) logger = logging.getLogger(__name__) @@ -98,7 +101,7 @@ def _parse_slices(self, gid, resolution, exclusion_shape): @property def gid(self): - """supply curve point gid""" + """Supply curve point gid""" return self._gid @property @@ -359,8 +362,10 @@ def centroid(self): """ if self._centroid is None: - lats = self.exclusions.excl_h5['latitude', self.rows, self.cols] - lons = self.exclusions.excl_h5['longitude', self.rows, self.cols] + lats = self.exclusions.excl_h5[MetaKeyName.LATITUDE, self.rows, + self.cols] + lons = self.exclusions.excl_h5[MetaKeyName.LONGITUDE, self.rows, + self.cols] self._centroid = (lats.mean(), lons.mean()) return self._centroid @@ -1043,15 +1048,18 @@ def h5(self): def country(self): """Get the SC point country based on the resource meta data.""" country = None - if 'country' in self.h5.meta and self.county is not None: + if MetaKeyName.COUNTRY in self.h5.meta and self.county is not None: # make sure country and county are coincident - counties = self.h5.meta.loc[self.h5_gid_set, 'county'].values + counties = self.h5.meta.loc[self.h5_gid_set, + MetaKeyName.COUNTY].values iloc = np.where(counties == self.county)[0][0] - country = self.h5.meta.loc[self.h5_gid_set, 'country'].values + country = self.h5.meta.loc[self.h5_gid_set, + MetaKeyName.COUNTRY].values country = country[iloc] - elif 'country' in self.h5.meta: - country = self.h5.meta.loc[self.h5_gid_set, 'country'].mode() + elif MetaKeyName.COUNTRY in self.h5.meta: + country = self.h5.meta.loc[self.h5_gid_set, + MetaKeyName.COUNTRY].mode() country = country.values[0] return country @@ -1060,15 +1068,16 @@ def country(self): def state(self): """Get the SC point state based on the resource meta data.""" state = None - if 'state' in self.h5.meta and self.county is not None: + if MetaKeyName.STATE in self.h5.meta and self.county is not None: # make sure state and county are coincident - counties = self.h5.meta.loc[self.h5_gid_set, 'county'].values + counties = self.h5.meta.loc[self.h5_gid_set, + MetaKeyName.COUNTY].values iloc = np.where(counties == self.county)[0][0] - state = self.h5.meta.loc[self.h5_gid_set, 'state'].values + state = self.h5.meta.loc[self.h5_gid_set, MetaKeyName.STATE].values state = state[iloc] - elif 'state' in self.h5.meta: - state = self.h5.meta.loc[self.h5_gid_set, 'state'].mode() + elif MetaKeyName.STATE in self.h5.meta: + state = self.h5.meta.loc[self.h5_gid_set, MetaKeyName.STATE].mode() state = state.values[0] return state @@ -1077,8 +1086,9 @@ def state(self): def county(self): """Get the SC point county based on the resource meta data.""" county = None - if 'county' in self.h5.meta: - county = self.h5.meta.loc[self.h5_gid_set, 'county'].mode() + if MetaKeyName.COUNTY in self.h5.meta: + county = self.h5.meta.loc[self.h5_gid_set, + MetaKeyName.COUNTY].mode() county = county.values[0] return county @@ -1087,8 +1097,9 @@ def county(self): def elevation(self): """Get the SC point elevation based on the resource meta data.""" elevation = None - if 'elevation' in self.h5.meta: - elevation = self.h5.meta.loc[self.h5_gid_set, 'elevation'].mean() + if MetaKeyName.ELEVATION in self.h5.meta: + elevation = self.h5.meta.loc[self.h5_gid_set, + MetaKeyName.ELEVATION].mean() return elevation @@ -1096,15 +1107,18 @@ def elevation(self): def timezone(self): """Get the SC point timezone based on the resource meta data.""" timezone = None - if 'timezone' in self.h5.meta and self.county is not None: + if MetaKeyName.TIMEZONE in self.h5.meta and self.county is not None: # make sure timezone flag and county are coincident - counties = self.h5.meta.loc[self.h5_gid_set, 'county'].values + counties = self.h5.meta.loc[self.h5_gid_set, + MetaKeyName.COUNTY].values iloc = np.where(counties == self.county)[0][0] - timezone = self.h5.meta.loc[self.h5_gid_set, 'timezone'].values + timezone = self.h5.meta.loc[self.h5_gid_set, + MetaKeyName.TIMEZONE].values timezone = timezone[iloc] - elif 'timezone' in self.h5.meta: - timezone = self.h5.meta.loc[self.h5_gid_set, 'timezone'].mode() + elif MetaKeyName.TIMEZONE in self.h5.meta: + timezone = self.h5.meta.loc[self.h5_gid_set, + MetaKeyName.TIMEZONE].mode() timezone = timezone.values[0] return timezone @@ -1114,15 +1128,18 @@ def offshore(self): """Get the SC point offshore flag based on the resource meta data (if offshore column is present).""" offshore = None - if 'offshore' in self.h5.meta and self.county is not None: + if MetaKeyName.OFFSHORE in self.h5.meta and self.county is not None: # make sure offshore flag and county are coincident - counties = self.h5.meta.loc[self.h5_gid_set, 'county'].values + counties = self.h5.meta.loc[self.h5_gid_set, + MetaKeyName.COUNTY].values iloc = np.where(counties == self.county)[0][0] - offshore = self.h5.meta.loc[self.h5_gid_set, 'offshore'].values + offshore = self.h5.meta.loc[self.h5_gid_set, + MetaKeyName.OFFSHORE].values offshore = offshore[iloc] - elif 'offshore' in self.h5.meta: - offshore = self.h5.meta.loc[self.h5_gid_set, 'offshore'].mode() + elif MetaKeyName.OFFSHORE in self.h5.meta: + offshore = self.h5.meta.loc[self.h5_gid_set, + MetaKeyName.OFFSHORE].mode() offshore = offshore.values[0] return offshore @@ -1153,18 +1170,18 @@ def summary(self): pandas.Series List of supply curve point's meta data """ - meta = {'sc_point_gid': self.sc_point_gid, - 'source_gids': self.h5_gid_set, - 'gid_counts': self.gid_counts, - 'n_gids': self.n_gids, - 'area_sq_km': self.area, - 'latitude': self.latitude, - 'longitude': self.longitude, - 'country': self.country, - 'state': self.state, - 'county': self.county, - 'elevation': self.elevation, - 'timezone': self.timezone, + meta = {MetaKeyName.SC_POINT_GID: self.sc_point_gid, + MetaKeyName.SOURCE_GIDS: self.h5_gid_set, + MetaKeyName.GID_COUNTS: self.gid_counts, + MetaKeyName.N_GIDS: self.n_gids, + MetaKeyName.AREA_SQ_KM: self.area, + MetaKeyName.LATITUDE: self.latitude, + MetaKeyName.LONGITUDE: self.longitude, + MetaKeyName.COUNTRY: self.country, + MetaKeyName.STATE: self.state, + MetaKeyName.COUNTY: self.county, + MetaKeyName.ELEVATION: self.elevation, + MetaKeyName.TIMEZONE: self.timezone, } meta = pd.Series(meta) @@ -1495,10 +1512,9 @@ def res_data(self): if isinstance(self._res_class_dset, np.ndarray): return self._res_class_dset - else: - if self._res_data is None: - if self._res_class_dset in self.gen.datasets: - self._res_data = self.gen[self._res_class_dset] + elif self._res_data is None: + if self._res_class_dset in self.gen.datasets: + self._res_data = self.gen[self._res_class_dset] return self._res_data @@ -1516,10 +1532,9 @@ def gen_data(self): if isinstance(self._cf_dset, np.ndarray): return self._cf_dset - else: - if self._gen_data is None: - if self._cf_dset in self.gen.datasets: - self._gen_data = self.gen[self._cf_dset] + elif self._gen_data is None: + if self._cf_dset in self.gen.datasets: + self._gen_data = self.gen[self._cf_dset] return self._gen_data @@ -1537,10 +1552,9 @@ def lcoe_data(self): if isinstance(self._lcoe_dset, np.ndarray): return self._lcoe_dset - else: - if self._lcoe_data is None: - if self._lcoe_dset in self.gen.datasets: - self._lcoe_data = self.gen[self._lcoe_dset] + elif self._lcoe_data is None: + if self._lcoe_dset in self.gen.datasets: + self._lcoe_data = self.gen[self._lcoe_dset] return self._lcoe_data @@ -1578,22 +1592,24 @@ def mean_lcoe(self): # year CF, but the output should be identical to the original LCOE and # so is not consequential). if self._recalc_lcoe: - required = ('fixed_charge_rate', 'capital_cost', - 'fixed_operating_cost', 'variable_operating_cost', + required = ('fixed_charge_rate', + 'capital_cost', + 'fixed_operating_cost', + 'variable_operating_cost', 'system_capacity') - if self.mean_h5_dsets_data is not None: - if all(k in self.mean_h5_dsets_data for k in required): - aep = (self.mean_h5_dsets_data['system_capacity'] - * self.mean_cf * 8760) - # Note the AEP computation uses the SAM config - # `system_capacity`, so no need to scale `capital_cost` - # or `fixed_operating_cost` by anything - mean_lcoe = lcoe_fcr( - self.mean_h5_dsets_data['fixed_charge_rate'], - self.mean_h5_dsets_data['capital_cost'], - self.mean_h5_dsets_data['fixed_operating_cost'], - aep, - self.mean_h5_dsets_data['variable_operating_cost']) + if (self.mean_h5_dsets_data is not None and + all(k in self.mean_h5_dsets_data for k in required)): + aep = (self.mean_h5_dsets_data['system_capacity'] + * self.mean_cf * 8760) + # Note the AEP computation uses the SAM config + # `system_capacity`, so no need to scale `capital_cost` + # or `fixed_operating_cost` by anything + mean_lcoe = lcoe_fcr( + self.mean_h5_dsets_data['fixed_charge_rate'], + self.mean_h5_dsets_data['capital_cost'], + self.mean_h5_dsets_data['fixed_operating_cost'], + aep, self.mean_h5_dsets_data[ + 'variable_operating_cost']) # alternative if lcoe was not able to be re-calculated from # multi year mean CF @@ -1820,8 +1836,9 @@ def sc_point_capital_cost(self): if not all(k in self.mean_h5_dsets_data for k in required): return None - cap_cost_per_mw = (self.mean_h5_dsets_data['capital_cost'] - / self.mean_h5_dsets_data['system_capacity']) + cap_cost_per_mw = ( + self.mean_h5_dsets_data['capital_cost'] / + self.mean_h5_dsets_data['system_capacity']) return cap_cost_per_mw * self.capacity @property @@ -1842,12 +1859,14 @@ def sc_point_fixed_operating_cost(self): if self.mean_h5_dsets_data is None: return None - required = ('fixed_operating_cost', 'system_capacity') + required = ('fixed_operating_cost', + 'system_capacity') if not all(k in self.mean_h5_dsets_data for k in required): return None - fixed_cost_per_mw = (self.mean_h5_dsets_data['fixed_operating_cost'] - / self.mean_h5_dsets_data['system_capacity']) + fixed_cost_per_mw = ( + self.mean_h5_dsets_data['fixed_operating_cost'] / + self.mean_h5_dsets_data['system_capacity']) return fixed_cost_per_mw * self.capacity @property @@ -2001,35 +2020,38 @@ def point_summary(self, args=None): Dictionary of summary outputs for this sc point. """ - ARGS = {'res_gids': self.res_gid_set, - 'gen_gids': self.gen_gid_set, - 'gid_counts': self.gid_counts, - 'n_gids': self.n_gids, - 'mean_cf': self.mean_cf, - 'mean_lcoe': self.mean_lcoe, - 'mean_res': self.mean_res, - 'capacity': self.capacity, - 'area_sq_km': self.area, - 'latitude': self.latitude, - 'longitude': self.longitude, - 'country': self.country, - 'state': self.state, - 'county': self.county, - 'elevation': self.elevation, - 'timezone': self.timezone, - } - - extra_atts = ['capacity_ac', 'offshore', 'sc_point_capital_cost', - 'sc_point_fixed_operating_cost', - 'sc_point_annual_energy', 'sc_point_annual_energy_ac'] + ARGS = { + MetaKeyName.LATITUDE: self.sc_point.latitude, + MetaKeyName.LONGITUDE: self.sc_point.longitude, + MetaKeyName.TIMEZONE: self.sc_point.timezone, + MetaKeyName.COUNTRY: self.sc_point.country, + MetaKeyName.STATE: self.sc_point.state, + MetaKeyName.COUNTY: self.sc_point.county, + MetaKeyName.ELEVATION: self.sc_point.elevation, + MetaKeyName.RES_GIDS: self.res_gid_set, + MetaKeyName.GEN_GIDS: self.gen_gid_set, + MetaKeyName.GID_COUNTS: self.gid_counts, + MetaKeyName.N_GIDS: self.sc_point.n_gids, + MetaKeyName.MEAN_CF: self.mean_cf, + MetaKeyName.MEAN_LCOE: self.mean_lcoe, + MetaKeyName.MEAN_RES: self.mean_res, + MetaKeyName.CAPACITY: self.capacity, + MetaKeyName.AREA_SQ_KM: self.sc_point.area} + + extra_atts = [MetaKeyName.CAPACITY_AC, + MetaKeyName.OFFSHORE, + MetaKeyName.SC_POINT_CAPITAL_COST, + MetaKeyName.SC_POINT_FIXED_OPERATING_COST, + MetaKeyName.SC_POINT_ANNUAL_ENERGY, + MetaKeyName.SC_POINT_ANNUAL_ENERGY_AC] for attr in extra_atts: value = getattr(self, attr) if value is not None: ARGS[attr] = value if self._friction_layer is not None: - ARGS['mean_friction'] = self.mean_friction - ARGS['mean_lcoe_friction'] = self.mean_lcoe_friction + ARGS[MetaKeyName.MEAN_FRICTION] = self.mean_friction + ARGS[MetaKeyName.MEAN_LCOE_FRICTION] = self.mean_lcoe_friction if self._h5_dsets is not None: for dset, data in self.mean_h5_dsets_data.items(): @@ -2070,14 +2092,14 @@ def economies_of_scale(cap_cost_scale, summary): """ eos = EconomiesOfScale(cap_cost_scale, summary) - summary['raw_lcoe'] = eos.raw_lcoe - summary['mean_lcoe'] = eos.scaled_lcoe - summary['capital_cost_scalar'] = eos.capital_cost_scalar - summary['scaled_capital_cost'] = eos.scaled_capital_cost - if "sc_point_capital_cost" in summary: - scaled_costs = (summary["sc_point_capital_cost"] + summary[MetaKeyName.RAW_LCOE] = eos.raw_lcoe + summary[MetaKeyName.MEAN_LCOE] = eos.scaled_lcoe + summary[MetaKeyName.CAPITAL_COST_SCALAR] = eos.capital_cost_scalar + summary[MetaKeyName.SCALED_CAPITAL_COST] = eos.scaled_capital_cost + if MetaKeyName.SC_POINT_CAPITAL_COST in summary: + scaled_costs = (summary[MetaKeyName.SC_POINT_CAPITAL_COST] * eos.capital_cost_scalar) - summary['scaled_sc_point_capital_cost'] = scaled_costs + summary[MetaKeyName.SCALED_SC_POINT_CAPITAL_COST] = scaled_costs return summary diff --git a/reV/supply_curve/sc_aggregation.py b/reV/supply_curve/sc_aggregation.py index 387382dac..472dc1c8d 100644 --- a/reV/supply_curve/sc_aggregation.py +++ b/reV/supply_curve/sc_aggregation.py @@ -6,29 +6,35 @@ @author: gbuster """ -from concurrent.futures import as_completed import logging -import numpy as np -import psutil import os -import pandas as pd +from concurrent.futures import as_completed from warnings import warn +import numpy as np +import pandas as pd +import psutil +from rex.multi_file_resource import MultiFileResource +from rex.resource import Resource +from rex.utilities.execution import SpawnProcessPool + from reV.generation.base import BaseGen from reV.handlers.exclusions import ExclusionLayers -from reV.supply_curve.aggregation import (AbstractAggFileHandler, - BaseAggregation, Aggregation) +from reV.supply_curve.aggregation import ( + AbstractAggFileHandler, + Aggregation, + BaseAggregation, +) from reV.supply_curve.exclusions import FrictionMask from reV.supply_curve.extent import SupplyCurveExtent from reV.supply_curve.points import GenerationSupplyCurvePoint -from reV.utilities.exceptions import (EmptySupplyCurvePointError, - OutputWarning, FileInputError, - InputWarning) -from reV.utilities import log_versions - -from rex.resource import Resource -from rex.multi_file_resource import MultiFileResource -from rex.utilities.execution import SpawnProcessPool +from reV.utilities import MetaKeyName, log_versions +from reV.utilities.exceptions import ( + EmptySupplyCurvePointError, + FileInputError, + InputWarning, + OutputWarning, +) logger = logging.getLogger(__name__) @@ -151,13 +157,15 @@ def _parse_power_density(self): if self._pdf.endswith('.csv'): self._power_density = pd.read_csv(self._pdf) - if ('gid' in self._power_density + if (MetaKeyName.GID in self._power_density and 'power_density' in self._power_density): - self._power_density = self._power_density.set_index('gid') + self._power_density = \ + self._power_density.set_index(MetaKeyName.GID) else: - msg = ('Variable power density file must include "gid" ' + msg = ('Variable power density file must include "{}" ' 'and "power_density" columns, but received: {}' - .format(self._power_density.columns.values)) + .format(MetaKeyName.GID, + self._power_density.columns.values)) logger.error(msg) raise FileInputError(msg) else: @@ -231,7 +239,7 @@ def __init__(self, excl_fpath, tm_dset, econ_fpath=None, lcoe_dset='lcoe_fcr-means', h5_dsets=None, data_layers=None, power_density=None, friction_fpath=None, friction_dset=None, cap_cost_scale=None, recalc_lcoe=True): - """reV supply curve points aggregation framework. + r"""ReV supply curve points aggregation framework. ``reV`` supply curve aggregation combines a high-resolution (e.g. 90m) exclusion dataset with a (typically) lower resolution @@ -825,8 +833,10 @@ def _get_extra_dsets(gen, h5_dsets): # look for the datasets required by the LCOE re-calculation and make # lists of the missing datasets gen_dsets = [] if gen is None else gen.datasets - lcoe_recalc_req = ('fixed_charge_rate', 'capital_cost', - 'fixed_operating_cost', 'variable_operating_cost', + lcoe_recalc_req = ('fixed_charge_rate', + 'capital_cost', + 'fixed_operating_cost', + 'variable_operating_cost', 'system_capacity') missing_lcoe_source = [k for k in lcoe_recalc_req if k not in gen_dsets] @@ -1060,9 +1070,11 @@ def run_serial(cls, excl_fpath, gen_fpath, tm_dset, gen_index, except EmptySupplyCurvePointError: logger.debug('SC point {} is empty'.format(gid)) else: - pointsum['sc_point_gid'] = gid - pointsum['sc_row_ind'] = points.loc[gid, 'row_ind'] - pointsum['sc_col_ind'] = points.loc[gid, 'col_ind'] + pointsum[MetaKeyName.SC_POINT_GID] = gid + pointsum[MetaKeyName.SC_ROW_IND] = \ + points.loc[gid, 'row_ind'] + pointsum[MetaKeyName.SC_COL_IND] = \ + points.loc[gid, 'col_ind'] pointsum['res_class'] = ri summary.append(pointsum) @@ -1221,10 +1233,11 @@ def _summary_to_df(summary): Summary of the SC points. """ summary = pd.DataFrame(summary) - sort_by = [x for x in ('sc_point_gid', 'res_class') if x in summary] + sort_by = [x for x in (MetaKeyName.SC_POINT_GID, 'res_class') + if x in summary] summary = summary.sort_values(sort_by) summary = summary.reset_index(drop=True) - summary.index.name = 'sc_gid' + summary.index.name = MetaKeyName.SC_GID return summary diff --git a/reV/supply_curve/supply_curve.py b/reV/supply_curve/supply_curve.py index edf6a1c66..201c0a02b 100644 --- a/reV/supply_curve/supply_curve.py +++ b/reV/supply_curve/supply_curve.py @@ -4,23 +4,22 @@ - Calculation of LCOT - Supply Curve creation """ -from copy import deepcopy import json import logging -import numpy as np import os -import pandas as pd +from copy import deepcopy from warnings import warn +import numpy as np +import pandas as pd +from rex import Resource +from rex.utilities import SpawnProcessPool, parse_table + from reV.handlers.transmission import TransmissionCosts as TC from reV.handlers.transmission import TransmissionFeatures as TF from reV.supply_curve.competitive_wind_farms import CompetitiveWindFarms -from reV.utilities.exceptions import SupplyCurveInputError, SupplyCurveError -from reV.utilities import log_versions - -from rex import Resource -from rex.utilities import parse_table, SpawnProcessPool - +from reV.utilities import MetaKeyName, log_versions +from reV.utilities.exceptions import SupplyCurveError, SupplyCurveInputError logger = logging.getLogger(__name__) @@ -29,8 +28,8 @@ class SupplyCurve: """SupplyCurve""" def __init__(self, sc_points, trans_table, sc_features=None, - sc_capacity_col='capacity'): - """reV LCOT calculation and SupplyCurve sorting class. + sc_capacity_col=MetaKeyName.CAPACITY): + """ReV LCOT calculation and SupplyCurve sorting class. ``reV`` supply curve computes the transmission costs associated with each supply curve point output by ``reV`` supply curve @@ -180,7 +179,7 @@ def _parse_sc_points(sc_points, sc_features=None): if isinstance(sc_points, str) and sc_points.endswith('.h5'): with Resource(sc_points) as res: sc_points = res.meta - sc_points.index.name = 'sc_gid' + sc_points.index.name = MetaKeyName.SC_GID sc_points = sc_points.reset_index() else: sc_points = parse_table(sc_points) @@ -275,7 +274,7 @@ def _parse_trans_table(trans_table): trans_table = trans_table.rename(columns={'dist_mi': 'dist_km'}) trans_table['dist_km'] *= 1.60934 - drop_cols = ['sc_gid', 'cap_left', 'sc_point_gid'] + drop_cols = [MetaKeyName.SC_GID, 'cap_left', MetaKeyName.SC_POINT_GID] drop_cols = [c for c in drop_cols if c in trans_table] if drop_cols: trans_table = trans_table.drop(columns=drop_cols) @@ -283,7 +282,8 @@ def _parse_trans_table(trans_table): return trans_table @staticmethod - def _map_trans_capacity(trans_sc_table, sc_capacity_col='capacity'): + def _map_trans_capacity(trans_sc_table, + sc_capacity_col=MetaKeyName.CAPACITY): """ Map SC gids to transmission features based on capacity. For any SC gids with capacity > the maximum transmission feature capacity, map @@ -396,7 +396,7 @@ def _check_sub_trans_lines(cls, features): return line_gids[~test].tolist() @classmethod - def _check_substation_conns(cls, trans_table, sc_cols='sc_gid'): + def _check_substation_conns(cls, trans_table, sc_cols=MetaKeyName.SC_GID): """ Run checks on substation transmission features to make sure that every sc point connecting to a substation can also connect to its @@ -409,7 +409,7 @@ def _check_substation_conns(cls, trans_table, sc_cols='sc_gid'): (should already be merged with SC points). sc_cols : str | list, optional Column(s) in trans_table with unique supply curve id, - by default 'sc_gid' + by default MetaKeyName.SC_GID """ missing = {} for sc_point, sc_table in trans_table.groupby(sc_cols): @@ -437,8 +437,8 @@ def _check_sc_trans_table(cls, sc_points, trans_table): Table mapping supply curve points to transmission features (should already be merged with SC points). """ - sc_gids = set(sc_points['sc_gid'].unique()) - trans_sc_gids = set(trans_table['sc_gid'].unique()) + sc_gids = set(sc_points[MetaKeyName.SC_GID].unique()) + trans_sc_gids = set(trans_table[MetaKeyName.SC_GID].unique()) missing = sorted(list(sc_gids - trans_sc_gids)) if any(missing): msg = ("There are {} Supply Curve points with missing " @@ -465,9 +465,11 @@ def _check_sc_trans_table(cls, sc_points, trans_table): @classmethod def _merge_sc_trans_tables(cls, sc_points, trans_table, - sc_cols=('sc_gid', 'capacity', 'mean_cf', - 'mean_lcoe'), - sc_capacity_col='capacity'): + sc_cols=(MetaKeyName.SC_GID, + MetaKeyName.CAPACITY, + MetaKeyName.MEAN_CF, + MetaKeyName.MEAN_LCOE), + sc_capacity_col=MetaKeyName.CAPACITY): """ Merge the supply curve table with the transmission features table. @@ -482,7 +484,7 @@ def _merge_sc_trans_tables(cls, sc_points, trans_table, sc_cols : tuple | list, optional List of column from sc_points to transfer into the trans table, If the `sc_capacity_col` is not included, it will get added. - by default ('sc_gid', 'capacity', 'mean_cf', 'mean_lcoe') + by default (MetaKeyName.SC_GID, 'capacity', 'mean_cf', 'mean_lcoe') sc_capacity_col : str, optional Name of capacity column in `trans_sc_table`. The values in this column determine the size of transmission lines built. @@ -524,8 +526,8 @@ def _merge_sc_trans_tables(cls, sc_points, trans_table, if isinstance(sc_cols, tuple): sc_cols = list(sc_cols) - if 'mean_lcoe_friction' in sc_points: - sc_cols.append('mean_lcoe_friction') + if MetaKeyName.MEAN_LCOE_FRICTION in sc_points: + sc_cols.append(MetaKeyName.MEAN_LCOE_FRICTION) if 'transmission_multiplier' in sc_points: sc_cols.append('transmission_multiplier') @@ -539,8 +541,9 @@ def _merge_sc_trans_tables(cls, sc_points, trans_table, @classmethod def _map_tables(cls, sc_points, trans_table, - sc_cols=('sc_gid', 'capacity', 'mean_cf', 'mean_lcoe'), - sc_capacity_col='capacity'): + sc_cols=(MetaKeyName.SC_GID, MetaKeyName.CAPACITY, + MetaKeyName.MEAN_CF, MetaKeyName.MEAN_LCOE), + sc_capacity_col=MetaKeyName.CAPACITY): """ Map supply curve points to transmission features @@ -583,7 +586,7 @@ def _map_tables(cls, sc_points, trans_table, trans_sc_table = \ trans_sc_table.sort_values( - ['sc_gid', 'trans_gid']).reset_index(drop=True) + [MetaKeyName.SC_GID, 'trans_gid']).reset_index(drop=True) cls._check_sc_trans_table(sc_points, trans_sc_table) @@ -625,7 +628,7 @@ def _create_handler(trans_table, trans_costs=None, avail_cap_frac=1): return trans_features @staticmethod - def _parse_sc_gids(trans_table, gid_key='sc_gid'): + def _parse_sc_gids(trans_table, gid_key=MetaKeyName.SC_GID): """Extract unique sc gids, make bool mask from tranmission table Parameters @@ -652,7 +655,7 @@ def _parse_sc_gids(trans_table, gid_key='sc_gid'): @staticmethod def _get_capacity(sc_gid, sc_table, connectable=True, - sc_capacity_col='capacity'): + sc_capacity_col=MetaKeyName.CAPACITY): """ Get capacity of supply curve point @@ -700,7 +703,7 @@ def _get_capacity(sc_gid, sc_table, connectable=True, def _compute_trans_cap_cost(cls, trans_table, trans_costs=None, avail_cap_frac=1, max_workers=None, connectable=True, line_limited=False, - sc_capacity_col='capacity'): + sc_capacity_col=MetaKeyName.CAPACITY): """ Compute levelized cost of transmission for all combinations of supply curve points and tranmission features in trans_table @@ -762,7 +765,7 @@ def _compute_trans_cap_cost(cls, trans_table, trans_costs=None, max_workers = os.cpu_count() logger.info('Computing LCOT costs for all possible connections...') - groups = trans_table.groupby('sc_gid') + groups = trans_table.groupby(MetaKeyName.SC_GID) if max_workers > 1: loggers = [__name__, 'reV.handlers.transmission', 'reV'] with SpawnProcessPool(max_workers=max_workers, @@ -844,29 +847,31 @@ def compute_total_lcoe(self, fcr, transmission_costs=None, self._trans_table['trans_cap_cost_per_mw'] = cost cost *= self._trans_table[self._sc_capacity_col] - cost /= self._trans_table['capacity'] # align with "mean_cf" + cost /= self._trans_table[MetaKeyName.CAPACITY] # align with "mean_cf" if 'reinforcement_cost_per_mw' in self._trans_table: logger.info("'reinforcement_cost_per_mw' column found in " "transmission table. Adding reinforcement costs " "to total LCOE.") - cf_mean_arr = self._trans_table['mean_cf'].values + cf_mean_arr = self._trans_table[MetaKeyName.MEAN_CF].values lcot = (cost * fcr) / (cf_mean_arr * 8760) - lcoe = lcot + self._trans_table['mean_lcoe'] + lcoe = lcot + self._trans_table[MetaKeyName.MEAN_LCOE] self._trans_table['lcot_no_reinforcement'] = lcot self._trans_table['lcoe_no_reinforcement'] = lcoe r_cost = (self._trans_table['reinforcement_cost_per_mw'] .values.copy()) r_cost *= self._trans_table[self._sc_capacity_col] - r_cost /= self._trans_table['capacity'] # align with "mean_cf" + # align with "mean_cf" + r_cost /= self._trans_table[MetaKeyName.CAPACITY] cost += r_cost # $/MW - cf_mean_arr = self._trans_table['mean_cf'].values + cf_mean_arr = self._trans_table[MetaKeyName.MEAN_CF].values lcot = (cost * fcr) / (cf_mean_arr * 8760) self._trans_table['lcot'] = lcot - self._trans_table['total_lcoe'] = (self._trans_table['lcot'] - + self._trans_table['mean_lcoe']) + self._trans_table['total_lcoe'] = ( + self._trans_table['lcot'] + + self._trans_table[MetaKeyName.MEAN_LCOE]) if consider_friction: self._calculate_total_lcoe_friction() @@ -875,10 +880,11 @@ def _calculate_total_lcoe_friction(self): """Look for site mean LCOE with friction in the trans table and if found make a total LCOE column with friction.""" - if 'mean_lcoe_friction' in self._trans_table: - lcoe_friction = (self._trans_table['lcot'] - + self._trans_table['mean_lcoe_friction']) - self._trans_table['total_lcoe_friction'] = lcoe_friction + if MetaKeyName.MEAN_LCOE_FRICTION in self._trans_table: + lcoe_friction = ( + self._trans_table['lcot'] + + self._trans_table[MetaKeyName.MEAN_LCOE_FRICTION]) + self._trans_table[MetaKeyName.TOTAL_LCOE_FRICTION] = lcoe_friction logger.info('Found mean LCOE with friction. Adding key ' '"total_lcoe_friction" to trans table.') @@ -912,7 +918,7 @@ def _exclude_noncompetitive_wind_farms(self, comp_wind_dirs, sc_gid, for n in exclude_gids: check = comp_wind_dirs.exclude_sc_point_gid(n) if check: - sc_gids = comp_wind_dirs['sc_gid', n] + sc_gids = comp_wind_dirs[MetaKeyName.SC_GID, n] for sc_id in sc_gids: if self._mask[sc_id]: logger.debug('Excluding sc_gid {}' @@ -1012,7 +1018,7 @@ def _full_sort(self, trans_table, trans_costs=None, conn_lists = {k: deepcopy(init_list) for k in columns} - trans_sc_gids = trans_table['sc_gid'].values.astype(int) + trans_sc_gids = trans_table[MetaKeyName.SC_GID].values.astype(int) # syntax is final_key: source_key (source from trans_table) all_cols = {k: k for k in columns} @@ -1047,7 +1053,7 @@ def _full_sort(self, trans_table, trans_costs=None, conn_lists[col_name][sc_gid] = data_arr[i] if total_lcoe_fric is not None: - conn_lists['total_lcoe_friction'][sc_gid] = \ + conn_lists[MetaKeyName.TOTAL_LCOE_FRICTION][sc_gid] = \ total_lcoe_fric[i] current_prog = connected // (len(self) / 100) @@ -1063,12 +1069,12 @@ def _full_sort(self, trans_table, trans_costs=None, index = range(0, int(1 + np.max(self._sc_gids))) connections = pd.DataFrame(conn_lists, index=index) - connections.index.name = 'sc_gid' + connections.index.name = MetaKeyName.SC_GID connections = connections.dropna(subset=[sort_on]) connections = connections[columns].reset_index() - sc_gids = self._sc_points['sc_gid'].values - connected = connections['sc_gid'].values + sc_gids = self._sc_points[MetaKeyName.SC_GID].values + connected = connections[MetaKeyName.SC_GID].values logger.debug('Connected gids {} out of total supply curve gids {}' .format(len(connected), len(sc_gids))) unconnected = ~np.isin(sc_gids, connected) @@ -1081,7 +1087,8 @@ def _full_sort(self, trans_table, trans_costs=None, logger.warning(msg) warn(msg) - supply_curve = self._sc_points.merge(connections, on='sc_gid') + supply_curve = self._sc_points.merge( + connections, on=MetaKeyName.SC_GID) return supply_curve.reset_index(drop=True) @@ -1096,15 +1103,15 @@ def _check_feature_capacity(self, avail_cap_frac=1): self._trans_table = self._trans_table.merge(fc, on='trans_gid') def _adjust_output_columns(self, columns, consider_friction): - """Add extra output columns, if needed. """ + """Add extra output columns, if needed.""" # These are essentially should-be-defaults that are not # backwards-compatible, so have to explicitly check for them extra_cols = ['ba_str', 'poi_lat', 'poi_lon', 'reinforcement_poi_lat', 'reinforcement_poi_lon', 'eos_mult', 'reg_mult', 'reinforcement_cost_per_mw', 'reinforcement_dist_km', - 'n_parallel_trans', 'total_lcoe_friction'] + 'n_parallel_trans', MetaKeyName.TOTAL_LCOE_FRICTION] if not consider_friction: - extra_cols -= {'total_lcoe_friction'} + extra_cols -= {MetaKeyName.TOTAL_LCOE_FRICTION} extra_cols = [col for col in extra_cols if col in self._trans_table and col not in columns] @@ -1201,8 +1208,9 @@ def full_sort(self, fcr, transmission_costs=None, trans_table = trans_table.loc[~pos].sort_values([sort_on, 'trans_gid']) total_lcoe_fric = None - if consider_friction and 'mean_lcoe_friction' in trans_table: - total_lcoe_fric = trans_table['total_lcoe_friction'].values + if consider_friction and MetaKeyName.MEAN_LCOE_FRICTION in trans_table: + total_lcoe_fric = \ + trans_table[MetaKeyName.TOTAL_LCOE_FRICTION].values comp_wind_dirs = None if wind_dirs is not None: @@ -1307,13 +1315,14 @@ def simple_sort(self, fcr, transmission_costs=None, sort_on = self._determine_sort_on(sort_on) connections = trans_table.sort_values([sort_on, 'trans_gid']) - connections = connections.groupby('sc_gid').first() + connections = connections.groupby(MetaKeyName.SC_GID).first() rename = {'trans_gid': 'trans_gid', 'category': 'trans_type'} connections = connections.rename(columns=rename) connections = connections[columns].reset_index() - supply_curve = self._sc_points.merge(connections, on='sc_gid') + supply_curve = self._sc_points.merge(connections, + on=MetaKeyName.SC_GID) if wind_dirs is not None: supply_curve = \ CompetitiveWindFarms.run(wind_dirs, diff --git a/reV/supply_curve/tech_mapping.py b/reV/supply_curve/tech_mapping.py index 5dabb88c1..b5b9e67c3 100644 --- a/reV/supply_curve/tech_mapping.py +++ b/reV/supply_curve/tech_mapping.py @@ -8,21 +8,22 @@ @author: gbuster """ -from concurrent.futures import as_completed -import h5py import logging -from math import ceil -import numpy as np import os -from scipy.spatial import cKDTree +from concurrent.futures import as_completed +from math import ceil from warnings import warn -from reV.supply_curve.extent import SupplyCurveExtent -from reV.utilities.exceptions import FileInputWarning, FileInputError - +import h5py +import numpy as np from rex.resource import Resource from rex.utilities.execution import SpawnProcessPool from rex.utilities.utilities import res_dist_threshold +from scipy.spatial import cKDTree + +from reV.supply_curve.extent import SupplyCurveExtent +from reV.utilities import MetaKeyName +from reV.utilities.exceptions import FileInputError, FileInputWarning logger = logging.getLogger(__name__) @@ -175,7 +176,8 @@ def _get_excl_slices(gid, sc_row_indices, sc_col_indices, excl_row_slices, @classmethod def _get_excl_coords(cls, excl_fpath, gids, sc_row_indices, sc_col_indices, excl_row_slices, excl_col_slices, - coord_labels=('latitude', 'longitude')): + coord_labels=(MetaKeyName.LATITUDE, + MetaKeyName.LONGITUDE)): """ Extract the exclusion coordinates for teh desired gids for TechMapping. @@ -339,7 +341,7 @@ def save_tech_map(excl_fpath, dset, indices, distance_threshold=None, def _check_fout(self): """Check the TechMapping output file for cached data.""" with h5py.File(self._excl_fpath, 'r') as f: - if 'latitude' not in f or 'longitude' not in f: + if MetaKeyName.LATITUDE not in f or MetaKeyName.LONGITUDE not in f: emsg = ('Datasets "latitude" and/or "longitude" not in ' 'pre-existing Exclusions TechMapping file "{}". ' 'Cannot proceed.' diff --git a/reV/utilities/__init__.py b/reV/utilities/__init__.py index 9db866072..8d53e9fe9 100644 --- a/reV/utilities/__init__.py +++ b/reV/utilities/__init__.py @@ -1,7 +1,5 @@ # -*- coding: utf-8 -*- -""" -reV utilities. -""" +"""reV utilities.""" from enum import Enum import PySAM @@ -10,16 +8,18 @@ from reV.version import __version__ -class SupplyCurvePointSummaryKey(str, Enum): - """An enumerated map to Supply Curve Point summary keys. +class MetaKeyName(str, Enum): + """An enumerated map to summary/meta keys. Each output name should match the name of a key in meth:`AggregationSupplyCurvePoint.summary` or - meth:`GenerationSupplyCurvePoint.point_summary` + meth:`GenerationSupplyCurvePoint.point_summary` or + meth:`BespokeSinglePlant.meta` """ SC_POINT_GID = 'sc_point_gid' SOURCE_GIDS = 'source_gids' + SC_GID = 'sc_gid' GID_COUNTS = 'gid_counts' GID = 'gid' N_GIDS = 'n_gids' @@ -38,6 +38,23 @@ class SupplyCurvePointSummaryKey(str, Enum): MEAN_LCOE = 'mean_lcoe' MEAN_RES = 'mean_res' CAPACITY = 'capacity' + OFFSHORE = 'offshore' + SC_ROW_IND = 'sc_row_ind' + SC_COL_IND = 'sc_col_ind' + CAPACITY_AC = 'capacity_ac' + SC_POINT_CAPITAL_COST = 'sc_point_capital_cost' + SC_POINT_FIXED_OPERATING_COST = 'sc_point_fixed_operating_cost' + SC_POINT_ANNUAL_ENERGY = 'sc_point_annual_energy' + SC_POINT_ANNUAL_ENERGY_AC = 'sc_point_annual_energy_ac' + MEAN_FRICTION = 'mean_friction' + MEAN_LCOE_FRICTION = 'mean_lcoe_friction' + TOTAL_LCOE_FRICTION = 'total_lcoe_friction' + RAW_LCOE = 'raw_lcoe' + CAPITAL_COST_SCALAR = 'capital_cost_scalar' + SCALED_CAPITAL_COST = 'scaled_capital_cost' + SCALED_SC_POINT_CAPITAL_COST = 'scaled_sc_point_capital_cost' + TURBINE_X_COORDS = 'turbine_x_coords' + TURBINE_Y_COORDS = 'turbine_y_coords' def __str__(self): return self.value diff --git a/reV/utilities/pytest_utils.py b/reV/utilities/pytest_utils.py index 3ba318a41..c95294944 100644 --- a/reV/utilities/pytest_utils.py +++ b/reV/utilities/pytest_utils.py @@ -1,8 +1,9 @@ # -*- coding: utf-8 -*- """Functions used for pytests""" -import numpy as np import os + +import numpy as np import pandas as pd from packaging import version from rex.outputs import Outputs as RexOutputs @@ -89,9 +90,9 @@ def make_fake_h5_chunks(td, features, shuffle=False): for i, s1 in enumerate(s_slices): for j, s2 in enumerate(s_slices): out_file = out_pattern.format(i=i, j=j) - meta = pd.DataFrame({'latitude': lat[s1, s2].flatten(), - 'longitude': lon[s1, s2].flatten(), - 'gid': gids[s1, s2].flatten()}) + meta = pd.DataFrame({MetaKeyName.LATITUDE: lat[s1, s2].flatten(), + MetaKeyName.LONGITUDE: lon[s1, s2].flatten(), + MetaKeyName.GID: gids[s1, s2].flatten()}) write_chunk(meta=meta, times=times, data=data[s1, s2], features=features, out_file=out_file) diff --git a/tests/eagle.py b/tests/eagle.py index 96583ac2e..a0c56e56c 100644 --- a/tests/eagle.py +++ b/tests/eagle.py @@ -12,17 +12,17 @@ """ import os -import h5py -import pytest -import numpy as np import time -from reV import TESTDATADIR -from reV.handlers.outputs import Outputs +import h5py +import numpy as np +import pytest from rex.utilities.hpc import SLURM from rex.utilities.loggers import init_logger -from reV.generation.cli_gen import get_node_cmd +from reV import TESTDATADIR +from reV.generation.cli_gen import get_node_cmd +from reV.handlers.outputs import Outputs RTOL = 0.0 ATOL = 0.04 @@ -54,7 +54,7 @@ def years(self): def get_cf_mean(self, site, year): """Get a cf mean based on site and year""" iy = self.years.index(year) - out = self._h5['pv']['cf_mean'][iy, site] + out = self._h5['pv'][MetaKeyName.CF_MEAN][iy, site] return out @@ -70,10 +70,10 @@ def is_num(n): def to_list(gen_out): """Generation output handler that converts to the rev 1.0 format.""" if isinstance(gen_out, list) and len(gen_out) == 1: - out = [c['cf_mean'] for c in gen_out[0].values()] + out = [c[MetaKeyName.CF_MEAN] for c in gen_out[0].values()] if isinstance(gen_out, dict): - out = [c['cf_mean'] for c in gen_out.values()] + out = [c[MetaKeyName.CF_MEAN] for c in gen_out.values()] return out @@ -104,7 +104,7 @@ def test_eagle(year): sam_files=sam_files, res_file=res_file, sites_per_worker=None, max_workers=None, fout=rev2_out, dirout=rev2_out_dir, logdir=rev2_out_dir, - output_request=('cf_profile', 'cf_mean'), + output_request=(, MetaKeyName.CF_MEAN), verbose=verbose) # create and submit the SLURM job @@ -125,7 +125,7 @@ def test_eagle(year): if rev2_out.strip('.h5') in fname: full_f = os.path.join(rev2_out_dir, fname) with Outputs(full_f, 'r') as cf: - rev2_profiles = cf['cf_profile'] + rev2_profiles = cf[] break # get reV 1.0 generation profiles @@ -147,7 +147,7 @@ def get_r1_profiles(year=2012): rev1 = os.path.join(TESTDATADIR, 'ri_pv', 'profile_outputs', 'pv_{}_0.h5'.format(year)) with Outputs(rev1) as cf: - data = cf['cf_profile'][...] / 10000 + data = cf[][...] / 10000 return data diff --git a/tests/hsds.py b/tests/hsds.py index 51474781d..64f2ada55 100644 --- a/tests/hsds.py +++ b/tests/hsds.py @@ -5,10 +5,10 @@ https://github.com/NREL/hsds-examples """ from datetime import datetime + import numpy as np import pandas as pd import pytest - from rex.renewable_resource import NSRDB @@ -54,7 +54,7 @@ def check_meta(res_cls): assert isinstance(meta, pd.DataFrame) assert meta.shape == (len(sites), meta_shape[1]) # select columns - meta = res_cls['meta', :, ['latitude', 'longitude']] + meta = res_cls['meta', :, [MetaKeyName.LATITUDE, MetaKeyName.LONGITUDE]] assert isinstance(meta, pd.DataFrame) assert meta.shape == (meta_shape[0], 2) @@ -160,6 +160,7 @@ class TestNSRDB: """ NSRDB Resource handler tests """ + @staticmethod def test_res(NSRDB_hsds): """ diff --git a/tests/test_bespoke.py b/tests/test_bespoke.py index 6c17a6698..9bc15ccbe 100644 --- a/tests/test_bespoke.py +++ b/tests/test_bespoke.py @@ -1,32 +1,31 @@ # -*- coding: utf-8 -*- -"""reV bespoke wind plant optimization tests -""" +"""reV bespoke wind plant optimization tests""" import copy -from glob import glob import json import os import shutil import tempfile +import traceback +from glob import glob + +import h5py import numpy as np import pandas as pd import pytest -import h5py -import traceback - from gaps.collection import Collector +from rex import Resource + from reV import TESTDATADIR -from reV.cli import main from reV.bespoke.bespoke import BespokeSinglePlant, BespokeWindPlants from reV.bespoke.place_turbines import PlaceTurbines +from reV.cli import main from reV.handlers.outputs import Outputs -from reV.supply_curve.tech_mapping import TechMapping -from reV.supply_curve.supply_curve import SupplyCurve -from reV.SAM.generation import WindPower from reV.losses.power_curve import PowerCurveLossesMixin from reV.losses.scheduled import ScheduledLossesMixin -from reV.utilities import ModuleName - -from rex import Resource +from reV.SAM.generation import WindPower +from reV.supply_curve.supply_curve import SupplyCurve +from reV.supply_curve.tech_mapping import TechMapping +from reV.utilities import MetaKeyName, ModuleName pytest.importorskip("shapely") @@ -54,7 +53,7 @@ 'ri_reeds_regions': {'inclusion_range': (None, 400), 'exclude_nodata': False}} -with open(SAM, 'r') as f: +with open(SAM) as f: SAM_SYS_INPUTS = json.load(f) SAM_SYS_INPUTS['wind_farm_wake_model'] = 2 @@ -74,8 +73,9 @@ def test_turbine_placement(gid=33): - """Test turbine placement with zero available area. """ - output_request = ('system_capacity', 'cf_mean', 'cf_profile') + """Test turbine placement with zero available area.""" + output_request = ('system_capacity'ame.CF_MEAN, + 'cf_profile') with tempfile.TemporaryDirectory() as td: res_fp = os.path.join(td, 'ri_100_wtk_{}.h5') excl_fp = os.path.join(td, 'ri_exclusions.h5') @@ -151,8 +151,9 @@ def test_turbine_placement(gid=33): def test_zero_area(gid=33): - """Test turbine placement with zero available area. """ - output_request = ('system_capacity', 'cf_mean', 'cf_profile') + """Test turbine placement with zero available area.""" + output_request = ('system_capacity', 'cf_mean', + 'cf_profile') objective_function = ( '(0.0975 * capital_cost + fixed_operating_cost) ' @@ -194,8 +195,9 @@ def test_zero_area(gid=33): def test_correct_turb_location(gid=33): - """Test turbine location is reported correctly. """ - output_request = ('system_capacity', 'cf_mean', 'cf_profile') + """Test turbine location is reported correctly.""" + output_request = ('system_capacity', 'cf_mean', + 'cf_profile') objective_function = ( '(0.0975 * capital_cost + fixed_operating_cost) ' @@ -237,7 +239,7 @@ def test_correct_turb_location(gid=33): def test_packing_algorithm(gid=33): - """Test turbine placement with zero available area. """ + """Test turbine placement with zero available area.""" output_request = () cap_cost_fun = "" foc_fun = "" @@ -277,13 +279,14 @@ def test_packing_algorithm(gid=33): def test_bespoke_points(): """Test the bespoke points input options""" # pylint: disable=W0612 - points = pd.DataFrame({'gid': [33, 34, 35], 'config': ['default'] * 3}) + points = pd.DataFrame({MetaKeyName.GID: [33, 34, 35], + 'config': ['default'] * 3}) pp = BespokeWindPlants._parse_points(points, {'default': SAM}) assert len(pp) == 3 for gid in pp.gids: assert pp[gid][0] == 'default' - points = pd.DataFrame({'gid': [33, 34, 35]}) + points = pd.DataFrame({MetaKeyName.GID: [33, 34, 35]}) pp = BespokeWindPlants._parse_points(points, {'default': SAM}) assert len(pp) == 3 assert 'config' in pp.df.columns @@ -293,7 +296,8 @@ def test_bespoke_points(): def test_single(gid=33): """Test a single wind plant bespoke optimization run""" - output_request = ('system_capacity', 'cf_mean', 'cf_profile') + output_request = ('system_capacity', 'cf_mean', + 'cf_profile') with tempfile.TemporaryDirectory() as td: res_fp = os.path.join(td, 'ri_100_wtk_{}.h5') excl_fp = os.path.join(td, 'ri_exclusions.h5') @@ -326,8 +330,8 @@ def test_single(gid=33): assert (TURB_RATING * bsp.meta['n_turbines'].values[0] == out['system_capacity']) - x_coords = json.loads(bsp.meta['turbine_x_coords'].values[0]) - y_coords = json.loads(bsp.meta['turbine_y_coords'].values[0]) + x_coords = json.loads(bsp.meta[MetaKeyName.TURBINE_X_COORDS].values[0]) + y_coords = json.loads(bsp.meta[MetaKeyName.TURBINE_Y_COORDS].values[0]) assert bsp.meta['n_turbines'].values[0] == len(x_coords) assert bsp.meta['n_turbines'].values[0] == len(y_coords) @@ -341,8 +345,8 @@ def test_single(gid=33): wp_sam_config = bsp.sam_sys_inputs wp_sam_config['wind_farm_wake_model'] = 0 wp_sam_config['wake_int_loss'] = 0 - wp_sam_config['wind_farm_xCoordinates'] = [0] - wp_sam_config['wind_farm_yCoordinates'] = [0] + wp_sam_config['wind_farrm_xCoordinates'] = [0] + wp_sam_config['wind_farrm_yCoordinates'] = [0] wp_sam_config['system_capacity'] = TURB_RATING res_df = bsp.res_df[(bsp.res_df.index.year == 2012)].copy() wp = WindPower(res_df, bsp.meta, wp_sam_config, @@ -369,7 +373,8 @@ def test_single(gid=33): def test_extra_outputs(gid=33): """Test running bespoke single farm optimization with lcoe requests""" - output_request = ('system_capacity', 'cf_mean', 'cf_profile', 'lcoe_fcr') + output_request = ('system_capacity', 'cf_mean', + 'cf_profile', 'lcoe_fcr') objective_function = ( '(fixed_charge_rate * capital_cost + fixed_operating_cost) ' @@ -417,9 +422,9 @@ def test_extra_outputs(gid=33): assert 'lcoe_fcr-2013' in out assert 'lcoe_fcr-means' in out - assert 'capacity' in bsp.meta - assert 'mean_cf' in bsp.meta - assert 'mean_lcoe' in bsp.meta + assert MetaKeyName.CAPACITY in bsp.meta + assert MetaKeyName.MEAN_CF in bsp.meta + assert MetaKeyName.MEAN_LCOE in bsp.meta assert 'pct_slope' in bsp.meta assert 'reeds_region' in bsp.meta @@ -451,9 +456,9 @@ def test_extra_outputs(gid=33): assert 'lcoe_fcr-2013' in out assert 'lcoe_fcr-means' in out - assert 'capacity' in bsp.meta - assert 'mean_cf' in bsp.meta - assert 'mean_lcoe' in bsp.meta + assert MetaKeyName.CAPACITY in bsp.meta + assert MetaKeyName.MEAN_CF in bsp.meta + assert MetaKeyName.MEAN_LCOE in bsp.meta assert 'pct_slope' in bsp.meta assert 'reeds_region' in bsp.meta @@ -478,7 +483,8 @@ def test_extra_outputs(gid=33): def test_bespoke(): """Test bespoke optimization with multiple plants, parallel processing, and file output. """ - output_request = ('system_capacity', 'cf_mean', 'cf_profile', + output_request = ('system_capacity', 'cf_mean', + 'cf_profile', 'extra_unused_data', 'winddirection', 'windspeed', 'ws_mean') @@ -492,9 +498,10 @@ def test_bespoke(): shutil.copy(RES.format(2013), res_fp.format(2013)) res_fp = res_fp.format('*') # both 33 and 35 are included, 37 is fully excluded - points = pd.DataFrame({'gid': [33, 35], 'config': ['default'] * 2, + points = pd.DataFrame({MetaKeyName.GID: [33, 35], + 'config': ['default'] * 2, 'extra_unused_data': [0, 42]}) - fully_excluded_points = pd.DataFrame({'gid': [37], + fully_excluded_points = pd.DataFrame({MetaKeyName.GID: [37], 'config': ['default'], 'extra_unused_data': [0]}) @@ -525,12 +532,12 @@ def test_bespoke(): with Resource(out_fpath_truth) as f: meta = f.meta assert len(meta) <= len(points) - assert 'sc_point_gid' in meta - assert 'turbine_x_coords' in meta - assert 'turbine_y_coords' in meta + assert MetaKeyName.SC_POINT_GID in meta + assert MetaKeyName.TURBINE_X_COORDS in meta + assert MetaKeyName.TURBINE_Y_COORDS in meta assert 'possible_x_coords' in meta assert 'possible_y_coords' in meta - assert 'res_gids' in meta + assert MetaKeyName.RES_GIDS in meta dsets_1d = ('system_capacity', 'cf_mean-2012', 'annual_energy-2012', 'cf_mean-means', @@ -568,7 +575,7 @@ def test_bespoke(): def test_collect_bespoke(): - """Test the collection of multiple chunked bespoke files. """ + """Test the collection of multiple chunked bespoke files.""" with tempfile.TemporaryDirectory() as td: source_dir = os.path.join(TESTDATADIR, 'bespoke/') source_pattern = source_dir + '/test_bespoke*.h5' @@ -581,7 +588,8 @@ def test_collect_bespoke(): with Resource(h5_file) as fout: meta = fout.meta - assert all(meta['gid'].values == sorted(meta['gid'].values)) + assert all(meta[MetaKeyName.GID].values == + sorted(meta[MetaKeyName.GID].values)) ti = fout.time_index assert len(ti) == 8760 assert 'time_index-2012' in fout @@ -590,10 +598,11 @@ def test_collect_bespoke(): for fp in source_fps: with Resource(fp) as source: - assert all(np.isin(source.meta['gid'].values, - meta['gid'].values)) - for isource, gid in enumerate(source.meta['gid'].values): - iout = np.where(meta['gid'].values == gid)[0] + assert all(np.isin(source.meta[MetaKeyName.GID].values, + meta[MetaKeyName.GID].values)) + for isource, gid in enumerate( + source.meta[MetaKeyName.GID].values): + iout = np.where(meta[MetaKeyName.GID].values == gid)[0] truth = source['cf_profile-2012', :, isource].flatten() test = data[:, iout].flatten() assert np.allclose(truth, test) @@ -601,7 +610,8 @@ def test_collect_bespoke(): def test_consistent_eval_namespace(gid=33): """Test that all the same variables are available for every eval.""" - output_request = ('system_capacity', 'cf_mean', 'cf_profile') + output_request = ('system_capacity', 'cf_mean', + 'cf_profile') cap_cost_fun = "2000" foc_fun = "0" voc_fun = "0" @@ -650,7 +660,7 @@ def test_bespoke_supply_curve(): del f['meta'] with Outputs(bespoke_sc_fp, mode='a') as f: bespoke_meta = normal_sc_points.copy() - bespoke_meta = bespoke_meta.drop('sc_gid', axis=1) + bespoke_meta = bespoke_meta.drop(MetaKeyName.SC_GID, axis=1) f.meta = bespoke_meta # this is basically copied from test_supply_curve_compute.py @@ -661,15 +671,16 @@ def test_bespoke_supply_curve(): sc = SupplyCurve(bespoke_sc_fp, trans_tables) sc_full = sc.full_sort(fcr=0.1, avail_cap_frac=0.1) - assert all(gid in sc_full['sc_gid'] - for gid in normal_sc_points['sc_gid']) + assert all(gid in sc_full[MetaKeyName.SC_GID] + for gid in normal_sc_points[MetaKeyName.SC_GID]) for _, inp_row in normal_sc_points.iterrows(): - sc_gid = inp_row['sc_gid'] - assert sc_gid in sc_full['sc_gid'] - test_ind = np.where(sc_full['sc_gid'] == sc_gid)[0] + sc_gid = inp_row[MetaKeyName.SC_GID] + assert sc_gid in sc_full[MetaKeyName.SC_GID] + test_ind = np.where(sc_full[MetaKeyName.SC_GID] == sc_gid)[0] assert len(test_ind) == 1 test_row = sc_full.iloc[test_ind] - assert test_row['total_lcoe'].values[0] > inp_row['mean_lcoe'] + assert (test_row['total_lcoe'].values[0] > + inp_row[MetaKeyName.MEAN_LCOE]) fpath_baseline = os.path.join(TESTDATADIR, 'sc_out/sc_full_lc.csv') sc_baseline = pd.read_csv(fpath_baseline) @@ -678,8 +689,9 @@ def test_bespoke_supply_curve(): @pytest.mark.parametrize('wlm', [2, 100]) def test_wake_loss_multiplier(wlm): - """Test wake loss multiplier. """ - output_request = ('system_capacity', 'cf_mean', 'cf_profile') + """Test wake loss multiplier.""" + output_request = ('system_capacity', 'cf_mean', + 'cf_profile') with tempfile.TemporaryDirectory() as td: res_fp = os.path.join(td, 'ri_100_wtk_{}.h5') excl_fp = os.path.join(td, 'ri_exclusions.h5') @@ -743,8 +755,9 @@ def test_wake_loss_multiplier(wlm): def test_bespoke_wind_plant_with_power_curve_losses(): - """Test bespoke ``wind_plant`` with power curve losses. """ - output_request = ('system_capacity', 'cf_mean', 'cf_profile') + """Test bespoke ``wind_plant`` with power curve losses.""" + output_request = ('system_capacity', 'cf_mean', + 'cf_profile') with tempfile.TemporaryDirectory() as td: res_fp = os.path.join(td, 'ri_100_wtk_{}.h5') excl_fp = os.path.join(td, 'ri_exclusions.h5') @@ -807,8 +820,9 @@ def test_bespoke_wind_plant_with_power_curve_losses(): def test_bespoke_run_with_power_curve_losses(): - """Test bespoke run with power curve losses. """ - output_request = ('system_capacity', 'cf_mean', 'cf_profile') + """Test bespoke run with power curve losses.""" + output_request = ('system_capacity', 'cf_mean', + 'cf_profile') with tempfile.TemporaryDirectory() as td: res_fp = os.path.join(td, 'ri_100_wtk_{}.h5') excl_fp = os.path.join(td, 'ri_exclusions.h5') @@ -858,8 +872,9 @@ def test_bespoke_run_with_power_curve_losses(): def test_bespoke_run_with_scheduled_losses(): - """Test bespoke run with scheduled losses. """ - output_request = ('system_capacity', 'cf_mean', 'cf_profile') + """Test bespoke run with scheduled losses.""" + output_request = ('system_capacity', 'cf_mean', + 'cf_profile') with tempfile.TemporaryDirectory() as td: res_fp = os.path.join(td, 'ri_100_wtk_{}.h5') excl_fp = os.path.join(td, 'ri_exclusions.h5') @@ -890,7 +905,8 @@ def test_bespoke_run_with_scheduled_losses(): 'allowed_months': ['April', 'May', 'June', 'July', 'August', 'September', 'October']}] sam_inputs['hourly'] = [0] * 8760 # only needed for testing - output_request = ('system_capacity', 'cf_mean', 'cf_profile', 'hourly') + output_request = ('system_capacity', 'cf_mean', + 'cf_profile', 'hourly') bsp = BespokeSinglePlant(33, excl_fp, res_fp, TM_DSET, sam_inputs, @@ -918,8 +934,9 @@ def test_bespoke_run_with_scheduled_losses(): def test_bespoke_aep_is_zero_if_no_turbines_placed(): - """Test that bespoke aep output is zero if no turbines placed. """ - output_request = ('system_capacity', 'cf_mean', 'cf_profile') + """Test that bespoke aep output is zero if no turbines placed.""" + output_request = ('system_capacity', 'cf_mean', + 'cf_profile') objective_function = 'aep' @@ -965,8 +982,9 @@ def test_bespoke_prior_run(): sam_sys_inputs = copy.deepcopy(SAM_SYS_INPUTS) sam_sys_inputs['fixed_charge_rate'] = 0.096 sam_configs = {'default': sam_sys_inputs} - output_request = ('system_capacity', 'cf_mean', 'cf_profile', - 'extra_unused_data', 'lcoe_fcr') + output_request = ('system_capacity', 'cf_mean', + 'cf_profile', 'extra_unused_data', + 'lcoe_fcr') with tempfile.TemporaryDirectory() as td: out_fpath1 = os.path.join(td, 'bespoke_out2.h5') out_fpath2 = os.path.join(td, 'bespoke_out1.h5') @@ -987,7 +1005,7 @@ def test_bespoke_prior_run(): res_fp_2013 = res_fp.format('2013') # gids 33 and 35 are included, 37 is fully excluded - points = pd.DataFrame({'gid': [33], 'config': ['default'], + points = pd.DataFrame({MetaKeyName.GID: [33], 'config': ['default'], 'extra_unused_data': [42]}) TechMapping.run(excl_fp, RES.format(2012), dset=TM_DSET, max_workers=1) @@ -1022,8 +1040,9 @@ def test_bespoke_prior_run(): meta2 = f2.meta data2 = {k: f2[k] for k in f2.dsets} - cols = ['turbine_x_coords', 'turbine_y_coords', 'capacity', - 'n_gids', 'gid_counts', 'res_gids'] + cols = [MetaKeyName.TURBINE_X_COORDS, MetaKeyName.TURBINE_Y_COORDS, + MetaKeyName.CAPACITY, MetaKeyName.N_GIDS, + MetaKeyName.GID_COUNTS, MetaKeyName.RES_GIDS] pd.testing.assert_frame_equal(meta1[cols], meta2[cols]) # multi-year means should not match the 2nd run with 2013 only. @@ -1042,8 +1061,9 @@ def test_gid_map(): new resource data files for example so you can run forecasted resource with the same spatial configuration.""" - output_request = ('system_capacity', 'cf_mean', 'cf_profile', - 'extra_unused_data', 'winddirection', 'ws_mean') + output_request = ('system_capacity', 'cf_mean', + 'cf_profile', 'extra_unused_data', + 'winddirection', 'ws_mean') with tempfile.TemporaryDirectory() as td: out_fpath1 = os.path.join(td, 'bespoke_out2.h5') out_fpath2 = os.path.join(td, 'bespoke_out1.h5') @@ -1055,10 +1075,10 @@ def test_gid_map(): res_fp_2013 = res_fp.format('2013') # gids 33 and 35 are included, 37 is fully excluded - points = pd.DataFrame({'gid': [33], 'config': ['default'], + points = pd.DataFrame({MetaKeyName.GID: [33], 'config': ['default'], 'extra_unused_data': [42]}) - gid_map = pd.DataFrame({'gid': [3, 4, 13, 12, 11, 10, 9]}) + gid_map = pd.DataFrame({MetaKeyName.GID: [3, 4, 13, 12, 11, 10, 9]}) new_gid = 50 gid_map['gid_map'] = new_gid fp_gid_map = os.path.join(td, 'gid_map.csv') @@ -1100,7 +1120,8 @@ def test_gid_map(): with Resource(res_fp_2013) as f3: ws = f3[f'windspeed_{hh}m', :, new_gid] - cols = ['n_gids', 'gid_counts', 'res_gids'] + cols = [MetaKeyName.N_GIDS, MetaKeyName.GID_COUNTS, + MetaKeyName.RES_GIDS] pd.testing.assert_frame_equal(meta1[cols], meta2[cols]) assert not np.allclose(data1['cf_mean-2013'], data2['cf_mean-2013']) @@ -1124,8 +1145,8 @@ def test_gid_map(): def test_bespoke_bias_correct(): """Test bespoke run with bias correction on windspeed data.""" - output_request = ('system_capacity', 'cf_mean', 'cf_profile', - 'extra_unused_data', 'ws_mean') + output_request = ('system_capacity', 'cf_mean', + 'cf_profile', 'extra_unused_data', 'ws_mean') with tempfile.TemporaryDirectory() as td: out_fpath1 = os.path.join(td, 'bespoke_out2.h5') out_fpath2 = os.path.join(td, 'bespoke_out1.h5') @@ -1137,12 +1158,12 @@ def test_bespoke_bias_correct(): res_fp_2013 = res_fp.format('2013') # gids 33 and 35 are included, 37 is fully excluded - points = pd.DataFrame({'gid': [33], 'config': ['default'], + points = pd.DataFrame({MetaKeyName.GID: [33], 'config': ['default'], 'extra_unused_data': [42]}) # intentionally leaving out WTK gid 13 which only has 5 included 90m # pixels in order to check that this is dynamically patched. - bias_correct = pd.DataFrame({'gid': [3, 4, 12, 11, 10, 9]}) + bias_correct = pd.DataFrame({MetaKeyName.GID: [3, 4, 12, 11, 10, 9]}) bias_correct['method'] = 'lin_ws' bias_correct['scalar'] = 0.5 fp_bc = os.path.join(td, 'bc.csv') @@ -1180,7 +1201,8 @@ def test_bespoke_bias_correct(): meta2 = f2.meta data2 = {k: f2[k] for k in f2.dsets} - cols = ['n_gids', 'gid_counts', 'res_gids'] + cols = [MetaKeyName.N_GIDS, MetaKeyName.GID_COUNTS, + MetaKeyName.RES_GIDS] pd.testing.assert_frame_equal(meta1[cols], meta2[cols]) assert data1['cf_mean-2013'] * 0.5 > data2['cf_mean-2013'] @@ -1189,8 +1211,9 @@ def test_bespoke_bias_correct(): def test_cli(runner, clear_loggers): """Test bespoke CLI""" - output_request = ('system_capacity', 'cf_mean', 'cf_profile', - 'winddirection', 'windspeed', 'ws_mean') + output_request = ('system_capacity', 'cf_mean', + 'cf_profile', 'winddirection', 'windspeed', + 'ws_mean') with tempfile.TemporaryDirectory() as td: dirname = os.path.basename(td) @@ -1256,12 +1279,12 @@ def test_cli(runner, clear_loggers): with Resource(out_fpath) as f: meta = f.meta assert len(meta) == 2 - assert 'sc_point_gid' in meta - assert 'turbine_x_coords' in meta - assert 'turbine_y_coords' in meta + assert MetaKeyName.SC_POINT_GID in meta + assert MetaKeyName.TURBINE_X_COORDS in meta + assert MetaKeyName.TURBINE_Y_COORDS in meta assert 'possible_x_coords' in meta assert 'possible_y_coords' in meta - assert 'res_gids' in meta + assert MetaKeyName.RES_GIDS in meta dsets_1d = ('system_capacity', 'cf_mean-2012', 'annual_energy-2012', 'cf_mean-means', 'ws_mean') @@ -1287,9 +1310,9 @@ def test_cli(runner, clear_loggers): def test_bespoke_5min_sample(): """Sample a 5min resource dataset for 60min outputs in bespoke""" - output_request = ('system_capacity', 'cf_mean', 'cf_profile', - 'extra_unused_data', 'winddirection', 'windspeed', - 'ws_mean') + output_request = ('system_capacity', 'cf_mean', + 'cf_profile', 'extra_unused_data', + 'winddirection', 'windspeed', 'ws_mean') tm_dset = 'test_wtk_5min' with tempfile.TemporaryDirectory() as td: @@ -1298,7 +1321,8 @@ def test_bespoke_5min_sample(): shutil.copy(EXCL, excl_fp) res_fp = os.path.join(TESTDATADIR, 'wtk/wtk_2010_*m.h5') - points = pd.DataFrame({'gid': [33, 35], 'config': ['default'] * 2, + points = pd.DataFrame({MetaKeyName.GID: [33, 35], + 'config': ['default'] * 2, 'extra_unused_data': [0, 42]}) sam_sys_inputs = copy.deepcopy(SAM_SYS_INPUTS) sam_sys_inputs['time_index_step'] = 12 @@ -1306,7 +1330,8 @@ def test_bespoke_5min_sample(): # hack techmap because 5min data only has 10 wind resource pixels with h5py.File(excl_fp, 'a') as excl_file: - arr = np.random.choice(10, size=excl_file['latitude'].shape) + arr = np.random.choice(10, + size=excl_file[MetaKeyName.LATITUDE].shape) excl_file.create_dataset(name=tm_dset, data=arr) bsp = BespokeWindPlants(excl_fp, res_fp, tm_dset, OBJECTIVE_FUNCTION, diff --git a/tests/test_config.py b/tests/test_config.py index eff40d846..73c045067 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -7,21 +7,21 @@ @author: gbuster """ -import numpy as np import os -import pandas as pd -import pytest import tempfile +import numpy as np +import pandas as pd +import pytest from rex import Resource from rex.utilities.exceptions import ResourceRuntimeError from rex.utilities.utilities import safe_json_load +from reV import TESTDATADIR from reV.config.base_analysis_config import AnalysisConfig -from reV.config.project_points import ProjectPoints, PointsControl +from reV.config.project_points import PointsControl, ProjectPoints from reV.generation.generation import Gen from reV.SAM.SAM import RevPySam -from reV import TESTDATADIR from reV.utilities.exceptions import ConfigError @@ -121,7 +121,7 @@ def test_config_mapping(): fpp = os.path.join(TESTDATADIR, 'project_points/pp_offshore.csv') sam_files = {'onshore': os.path.join( TESTDATADIR, 'SAM/wind_gen_standard_losses_0.json'), - 'offshore': os.path.join( + MetaKeyName.OFFSHORE: os.path.join( TESTDATADIR, 'SAM/wind_gen_standard_losses_1.json')} df = pd.read_csv(fpp, index_col=0) pp = ProjectPoints(fpp, sam_files, 'windpower') @@ -139,7 +139,7 @@ def test_sam_config_kw_replace(): fpp = os.path.join(TESTDATADIR, 'project_points/pp_offshore.csv') sam_files = {'onshore': os.path.join( TESTDATADIR, 'SAM/wind_gen_standard_losses_0.json'), - 'offshore': os.path.join( + MetaKeyName.OFFSHORE: os.path.join( TESTDATADIR, 'SAM/wind_gen_standard_losses_1.json')} res_file = os.path.join(TESTDATADIR, 'wtk/ri_100_wtk_2012.h5') pp = ProjectPoints(fpp, sam_files, 'windpower') @@ -147,19 +147,19 @@ def test_sam_config_kw_replace(): gen = Gen('windpower', pp, sam_files, resource_file=res_file, sites_per_worker=100) config_on = gen.project_points.sam_inputs['onshore'] - config_of = gen.project_points.sam_inputs['offshore'] + config_of = gen.project_points.sam_inputs[MetaKeyName.OFFSHORE] assert 'turb_generic_loss' in config_on assert 'turb_generic_loss' in config_of pp_split = ProjectPoints.split(0, 10000, gen.project_points) config_on = pp_split.sam_inputs['onshore'] - config_of = pp_split.sam_inputs['offshore'] + config_of = pp_split.sam_inputs[MetaKeyName.OFFSHORE] assert 'turb_generic_loss' in config_on assert 'turb_generic_loss' in config_of pc_split = PointsControl.split(0, 10000, gen.project_points) config_on = pc_split.project_points.sam_inputs['onshore'] - config_of = pc_split.project_points.sam_inputs['offshore'] + config_of = pc_split.project_points.sam_inputs[MetaKeyName.OFFSHORE] assert 'turb_generic_loss' in config_on assert 'turb_generic_loss' in config_of @@ -168,8 +168,8 @@ def test_sam_config_kw_replace(): config = ipc.project_points.sam_inputs['onshore'] assert 'turb_generic_loss' in config - if 'offshore' in ipc.project_points.sam_inputs: - config = ipc.project_points.sam_inputs['offshore'] + if MetaKeyName.OFFSHORE in ipc.project_points.sam_inputs: + config = ipc.project_points.sam_inputs[MetaKeyName.OFFSHORE] assert 'turb_generic_loss' in config @@ -207,7 +207,7 @@ def test_coords(sites): if not isinstance(gids, list): gids = [gids] - lat_lons = meta.loc[gids, ['latitude', 'longitude']].values + lat_lons = meta.loc[gids, [MetaKeyName.LATITUDE, MetaKeyName.LONGITUDE]].values pp = ProjectPoints.lat_lon_coords(lat_lons, res_file, sam_files) assert sorted(gids) == pp.sites @@ -226,7 +226,7 @@ def test_coords_from_file(): if not isinstance(gids, list): gids = [gids] - lat_lons = meta.loc[gids, ['latitude', 'longitude']] + lat_lons = meta.loc[gids, [MetaKeyName.LATITUDE, MetaKeyName.LONGITUDE]] with tempfile.TemporaryDirectory() as td: coords_fp = os.path.join(td, "lat_lon.csv") lat_lons.to_csv(coords_fp, index=False) @@ -245,7 +245,7 @@ def test_duplicate_coords(): with Resource(res_file) as f: meta = f.meta - duplicates = meta.loc[[2, 3, 3, 4], ['latitude', 'longitude']].values + duplicates = meta.loc[[2, 3, 3, 4], [MetaKeyName.LATITUDE, MetaKeyName.LONGITUDE]].values with pytest.raises(RuntimeError): ProjectPoints.lat_lon_coords(duplicates, res_file, sam_files) @@ -262,7 +262,7 @@ def test_sam_configs(): fpp = os.path.join(TESTDATADIR, 'project_points/pp_offshore.csv') sam_files = {'onshore': os.path.join( TESTDATADIR, 'SAM/wind_gen_standard_losses_0.json'), - 'offshore': os.path.join( + MetaKeyName.OFFSHORE: os.path.join( TESTDATADIR, 'SAM/wind_gen_standard_losses_1.json')} pp_json = ProjectPoints(fpp, sam_files, 'windpower') diff --git a/tests/test_curtailment.py b/tests/test_curtailment.py index aff02da18..893f4f051 100644 --- a/tests/test_curtailment.py +++ b/tests/test_curtailment.py @@ -5,19 +5,21 @@ @author: gbuster """ -from copy import deepcopy import os +from copy import deepcopy + import numpy as np import pandas as pd import pytest -from reV.SAM.SAM import RevPySam -from reV.config.project_points import ProjectPoints +from rex.utilities import safe_json_load +from rex.utilities.solar_position import SolarPosition + from reV import TESTDATADIR -from reV.utilities.curtailment import curtail +from reV.config.project_points import ProjectPoints from reV.generation.generation import Gen - -from rex.utilities.solar_position import SolarPosition -from rex.utilities import safe_json_load +from reV.SAM.SAM import RevPySam +from reV.utilities import MetaKeyName +from reV.utilities.curtailment import curtail def get_curtailment(year, curt_fn='curtailment.json'): @@ -61,14 +63,15 @@ def test_cf_curtailment(year, site): # run reV 2.0 generation gen = Gen('windpower', points, sam_files, res_file, - output_request=('cf_profile',), curtailment=curtailment, + output_request=(MetaKeyName.CF_PROFILE,), + curtailment=curtailment, sites_per_worker=50, scale_outputs=True) gen.run(max_workers=1) results, check_curtailment = test_res_curtailment(year, site=site) - results['cf_profile'] = gen.out['cf_profile'].flatten() + results[MetaKeyName.CF_PROFILE] = gen.out[MetaKeyName.CF_PROFILE].flatten() # was capacity factor NOT curtailed? - check_cf = (gen.out['cf_profile'].flatten() != 0) + check_cf = (gen.out[MetaKeyName.CF_PROFILE].flatten() != 0) # Were all thresholds met and windspeed NOT curtailed? check = check_curtailment & check_cf @@ -95,7 +98,7 @@ def test_curtailment_res_mean(year): curtailment = os.path.join(TESTDATADIR, 'config/', 'curtailment.json') points = slice(0, 100) - output_request = ('cf_mean', 'ws_mean') + output_request = (MetaKeyName.CF_MEAN, 'ws_mean') pc = Gen.get_pc(points, None, sam_files, 'windpower', sites_per_worker=50, res_file=res_file, curtailment=curtailment) @@ -143,11 +146,11 @@ def test_random(year, site): # run reV 2.0 generation and write to disk gen = Gen('windpower', points, sam_files, res_file, - output_request=('cf_profile',), curtailment=c, + output_request=(MetaKeyName.CF_PROFILE,), curtailment=c, sites_per_worker=50, scale_outputs=True) gen.run(max_workers=1) - results.append(gen.out['cf_mean']) + results.append(gen.out[MetaKeyName.CF_MEAN]) assert results[0] > results[1], 'Curtailment did not decrease cf_mean!' @@ -168,7 +171,8 @@ def test_res_curtailment(year, site): sza = SolarPosition( non_curtailed_res.time_index, - non_curtailed_res.meta[['latitude', 'longitude']].values).zenith + non_curtailed_res.meta[[MetaKeyName.LATITUDE, + MetaKeyName.LONGITUDE]].values).zenith ti = non_curtailed_res.time_index @@ -246,9 +250,6 @@ def test_eqn_curtailment(plot=False): nc_res = non_curtailed_res[0] c_mask = (c_res.windspeed == 0) & (nc_res.windspeed > 0) - temperature = nc_res['temperature'].values - wind_speed = nc_res['windspeed'].values - eval_mask = eval(c_eqn) # All curtailed windspeeds should satisfy the eqn eval but maybe not the diff --git a/tests/test_econ_lcoe.py b/tests/test_econ_lcoe.py index 8975ce453..722e624ab 100644 --- a/tests/test_econ_lcoe.py +++ b/tests/test_econ_lcoe.py @@ -7,20 +7,21 @@ @author: gbuster """ -import h5py import json -import numpy as np import os -import pandas as pd import shutil -from pandas.testing import assert_frame_equal -import pytest import tempfile import traceback +import h5py +import numpy as np +import pandas as pd +import pytest +from pandas.testing import assert_frame_equal + +from reV import TESTDATADIR from reV.cli import main from reV.econ.econ import Econ -from reV import TESTDATADIR from reV.handlers.outputs import Outputs from reV.utilities import ModuleName @@ -73,7 +74,8 @@ def test_fout(year): fout = 'lcoe_out_econ_{}.h5'.format(year) fpath = os.path.join(dirout, fout) points = slice(0, 100) - econ = Econ(points, sam_files, cf_file, output_request='lcoe_fcr', + econ = Econ(points, sam_files, cf_file, + output_request='lcoe_fcr', sites_per_worker=25) econ.run(max_workers=1, out_fpath=fpath) with Outputs(fpath) as f: @@ -103,7 +105,8 @@ def test_append_data(year): r1f = os.path.join(TESTDATADIR, 'ri_pv/scalar_outputs/project_outputs.h5') points = slice(0, 100) - econ = Econ(points, sam_files, cf_file, output_request='lcoe_fcr', + econ = Econ(points, sam_files, cf_file, + output_request='lcoe_fcr', sites_per_worker=25, append=True) econ.run(max_workers=1) @@ -143,7 +146,6 @@ def test_append_multi_node(node): shutil.copy(original_file, cf_file) sam_files = {'default': os.path.join( TESTDATADIR, 'SAM/pv_tracking_atb2020.json')} - year = 1998 points = os.path.join( TESTDATADIR, 'config/nsrdb_projpoints_atb2020_capcostmults_subset.csv') @@ -151,7 +153,8 @@ def test_append_multi_node(node): TESTDATADIR, 'config/nsrdb_sitedata_atb2020_capcostmults_subset.csv') econ = Econ(points, sam_files, cf_file, - output_request=('lcoe_fcr', 'capital_cost'), + output_request=('lcoe_fcr', + 'capital_cost'), sites_per_worker=25, append=True, site_data=site_data) econ.run(max_workers=1) @@ -165,8 +168,8 @@ def test_append_multi_node(node): assert np.allclose(data_baseline, data_test) site_data = pd.read_csv(site_data) - sd_cap_cost = \ - site_data.loc[site_data.gid.isin(meta.gid), 'capital_cost'] + sd_cap_cost = site_data.loc[site_data.gid.isin(meta.gid), + 'capital_cost'] assert np.allclose(test_cap_cost, sd_cap_cost) assert np.allclose(econ.out['capital_cost'], sd_cap_cost) diff --git a/tests/test_econ_of_scale.py b/tests/test_econ_of_scale.py index 272df05ed..d435c46e3 100644 --- a/tests/test_econ_of_scale.py +++ b/tests/test_econ_of_scale.py @@ -3,19 +3,21 @@ """ PyTest file for reV LCOE economies of scale """ -import h5py -import numpy as np -import pandas as pd -import pytest import os import shutil import tempfile +import h5py +import numpy as np +import pandas as pd +import pytest from rex import Resource -from reV.generation.generation import Gen + +from reV import TESTDATADIR from reV.econ.economies_of_scale import EconomiesOfScale +from reV.generation.generation import Gen from reV.supply_curve.sc_aggregation import SupplyCurveAggregation -from reV import TESTDATADIR +from reV.utilities import MetaKeyName EXCL = os.path.join(TESTDATADIR, 'ri_exclusions/ri_exclusions.h5') GEN = os.path.join(TESTDATADIR, 'gen_out/ri_my_pv_gen.h5') @@ -45,8 +47,8 @@ def test_pass_through_lcoe_args(): res_file = os.path.join(TESTDATADIR, 'wtk/ri_100_wtk_{}.h5'.format(year)) sam_files = os.path.join(TESTDATADIR, 'SAM/i_windpower_lcoe.json') - output_request = ('cf_mean', - 'lcoe_fcr', + output_request = (MetaKeyName.CF_MEAN, + MetaKeyName.LCOE_FCR, 'system_capacity', 'capital_cost', 'fixed_charge_rate', @@ -60,8 +62,8 @@ def test_pass_through_lcoe_args(): checks = [x in gen.out for x in Gen.LCOE_ARGS] assert all(checks) - assert 'lcoe_fcr' in gen.out - assert 'cf_mean' in gen.out + assert MetaKeyName.LCOE_FCR in gen.out + assert MetaKeyName.CF_MEAN in gen.out def test_lcoe_calc_simple(): @@ -76,7 +78,7 @@ def test_lcoe_calc_simple(): true_lcoe = ((data['fcr'] * data['capital_cost'] + data['foc']) / (data['aep'] / 1000)) - data['mean_lcoe'] = true_lcoe + data[MetaKeyName.MEAN_LCOE] = true_lcoe eos = EconomiesOfScale(eqn, data) assert eos.raw_capital_cost == eos.scaled_capital_cost @@ -92,7 +94,8 @@ def test_lcoe_calc_simple(): assert np.allclose(eos.scaled_lcoe, true_lcoe, rtol=0.001) eqn = 2 - true_scaled = ((data['fcr'] * eqn * data['capital_cost'] + data['foc']) + true_scaled = ((data['fcr'] * eqn * data['capital_cost'] + + data['foc']) / (data['aep'] / 1000)) eos = EconomiesOfScale(eqn, data) assert eqn * eos.raw_capital_cost == eos.scaled_capital_cost @@ -102,7 +105,8 @@ def test_lcoe_calc_simple(): data['system_capacity'] = 2 eqn = '1 / system_capacity' - true_scaled = ((data['fcr'] * 0.5 * data['capital_cost'] + data['foc']) + true_scaled = ((data['fcr'] * 0.5 * data['capital_cost'] + + data['foc']) / (data['aep'] / 1000)) eos = EconomiesOfScale(eqn, data) assert 0.5 * eos.raw_capital_cost == eos.scaled_capital_cost @@ -130,7 +134,8 @@ def test_econ_of_scale_baseline(): with Resource(GEN) as res: cf = res['cf_mean-means'] - lcoe = (1000 * (data['fixed_charge_rate'] * data['capital_cost'] + lcoe = (1000 * (data['fixed_charge_rate'] * + data['capital_cost'] + data['fixed_operating_cost']) / (cf * data['system_capacity'] * 8760)) @@ -160,10 +165,11 @@ def test_econ_of_scale_baseline(): base_df = pd.read_csv(out_fp_base + ".csv") sc_df = pd.read_csv(out_fp_sc + ".csv") - assert np.allclose(base_df['mean_lcoe'], sc_df['mean_lcoe']) - assert (sc_df['capital_cost_scalar'] == 1).all() + assert np.allclose(base_df[MetaKeyName.MEAN_LCOE], + sc_df[MetaKeyName.MEAN_LCOE]) + assert (sc_df[MetaKeyName.CAPITAL_COST_SCALAR] == 1).all() assert np.allclose(sc_df['mean_capital_cost'], - sc_df['scaled_capital_cost']) + sc_df[MetaKeyName.SCALED_CAPITAL_COST]) def test_sc_agg_econ_scale(): @@ -207,13 +213,18 @@ def test_sc_agg_econ_scale(): # check that econ of scale saved the raw lcoe and that it reduced all # of the mean lcoe values from baseline - assert np.allclose(sc_df['raw_lcoe'], base_df['mean_lcoe']) - assert all(sc_df['mean_lcoe'] < base_df['mean_lcoe']) - - aep = ((sc_df['mean_fixed_charge_rate'] * sc_df['mean_capital_cost'] - + sc_df['mean_fixed_operating_cost']) / sc_df['raw_lcoe']) - - true_raw_lcoe = ((data['fixed_charge_rate'] * data['capital_cost'] + assert np.allclose(sc_df[MetaKeyName.RAW_LCOE], + base_df[MetaKeyName.MEAN_LCOE]) + assert all(sc_df[MetaKeyName.MEAN_LCOE] < + base_df[MetaKeyName.MEAN_LCOE]) + + aep = ((sc_df['mean_fixed_charge_rate'] * + sc_df['mean_capital_cost'] + + sc_df['mean_fixed_operating_cost']) / + sc_df[MetaKeyName.RAW_LCOE]) + + true_raw_lcoe = ((data['fixed_charge_rate'] * + data['capital_cost'] + data['fixed_operating_cost']) / aep + data['variable_operating_cost']) @@ -226,19 +237,21 @@ def test_sc_agg_econ_scale(): + data['fixed_operating_cost']) / aep + data['variable_operating_cost']) - assert np.allclose(scalars, sc_df['capital_cost_scalar']) + assert np.allclose(scalars, sc_df[MetaKeyName.CAPITAL_COST_SCALAR]) assert np.allclose(scalars * sc_df['mean_capital_cost'], - sc_df['scaled_capital_cost']) + sc_df[MetaKeyName.SCALED_CAPITAL_COST]) - assert np.allclose(true_scaled_lcoe, sc_df['mean_lcoe']) - assert np.allclose(true_raw_lcoe, sc_df['raw_lcoe']) - sc_df = sc_df.sort_values('capacity') - assert all(sc_df['mean_lcoe'].diff()[1:] < 0) + assert np.allclose(true_scaled_lcoe, sc_df[MetaKeyName.MEAN_LCOE]) + assert np.allclose(true_raw_lcoe, sc_df[MetaKeyName.RAW_LCOE]) + sc_df = sc_df.sort_values(MetaKeyName.CAPACITY) + assert all(sc_df[MetaKeyName.MEAN_LCOE].diff()[1:] < 0) for i in sc_df.index.values: if sc_df.loc[i, 'scalars'] < 1: - assert sc_df.loc[i, 'mean_lcoe'] < sc_df.loc[i, 'raw_lcoe'] + assert (sc_df.loc[i, MetaKeyName.MEAN_LCOE] < + sc_df.loc[i, MetaKeyName.RAW_LCOE]) else: - assert sc_df.loc[i, 'mean_lcoe'] >= sc_df.loc[i, 'raw_lcoe'] + assert (sc_df.loc[i, MetaKeyName.MEAN_LCOE] >= + sc_df.loc[i, MetaKeyName.RAW_LCOE]) def execute_pytest(capture='all', flags='-rapP'): diff --git a/tests/test_econ_windbos.py b/tests/test_econ_windbos.py index a96cad8bb..4824be563 100644 --- a/tests/test_econ_windbos.py +++ b/tests/test_econ_windbos.py @@ -8,15 +8,16 @@ """ import json import os -import pytest +import tempfile + import numpy as np import pandas as pd -import tempfile +import pytest -from reV.generation.generation import Gen +from reV import TESTDATADIR from reV.econ.econ import Econ +from reV.generation.generation import Gen from reV.SAM.windbos import WindBos -from reV import TESTDATADIR RTOL = 0.000001 ATOL = 0.001 @@ -113,7 +114,7 @@ def test_sam_windbos(): def test_rev_windbos(): """Test baseline windbos calc with single owner defaults""" fpath = TESTDATADIR + '/SAM/i_singleowner_windbos.json' - with open(fpath, 'r') as f: + with open(fpath) as f: inputs = json.load(f) wb = WindBos(inputs) assert np.allclose(wb.turbine_cost, 52512000.00, atol=ATOL, rtol=RTOL) @@ -125,11 +126,11 @@ def test_rev_windbos(): def test_standalone_json(): """Test baseline windbos calc with standalone json file""" fpath = TESTDATADIR + '/SAM/i_singleowner_windbos.json' - with open(fpath, 'r') as f: + with open(fpath) as f: inputs = json.load(f) wb1 = WindBos(inputs) fpath = TESTDATADIR + '/SAM/i_windbos.json' - with open(fpath, 'r') as f: + with open(fpath) as f: inputs = json.load(f) wb2 = WindBos(inputs) @@ -140,7 +141,7 @@ def test_standalone_json(): def test_rev_windbos_perf_bond(): """Test windbos calc with performance bonds""" fpath = TESTDATADIR + '/SAM/i_singleowner_windbos.json' - with open(fpath, 'r') as f: + with open(fpath) as f: inputs = json.load(f) inputs['performance_bond'] = 10.0 wb = WindBos(inputs) @@ -153,7 +154,7 @@ def test_rev_windbos_perf_bond(): def test_rev_windbos_transport(): """Test windbos calc with turbine transport costs""" fpath = TESTDATADIR + '/SAM/i_singleowner_windbos.json' - with open(fpath, 'r') as f: + with open(fpath) as f: inputs = json.load(f) inputs['turbine_transportation'] = 100.0 wb = WindBos(inputs) @@ -166,7 +167,7 @@ def test_rev_windbos_transport(): def test_rev_windbos_sales(): """Test windbos calc with turbine transport costs""" fpath = TESTDATADIR + '/SAM/i_singleowner_windbos.json' - with open(fpath, 'r') as f: + with open(fpath) as f: inputs = json.load(f) inputs['sales_tax_basis'] = 5.0 wb = WindBos(inputs) @@ -189,7 +190,7 @@ def test_run_gen_econ(points=slice(0, 10), year=2012, max_workers=1): # run reV 2.0 generation gen = Gen('windpower', points, sam_files, res_file, - output_request=('cf_mean', 'cf_profile'), + output_request=(MetaKeyName.CF_MEAN, ), sites_per_worker=3) gen.run(max_workers=max_workers, out_fpath=cf_file) @@ -214,7 +215,7 @@ def test_run_bos(points=slice(0, 5), max_workers=1): # get full file paths. sam_files = TESTDATADIR + '/SAM/i_singleowner_windbos.json' - site_data = pd.DataFrame({'gid': range(5), + site_data = pd.DataFrame({MetaKeyName.GID: range(5), 'sales_tax_basis': range(5)}) econ_outs = ('total_installed_cost', 'turbine_cost', 'sales_tax_cost', diff --git a/tests/test_gen_5min.py b/tests/test_gen_5min.py index 91168880a..59239aa41 100644 --- a/tests/test_gen_5min.py +++ b/tests/test_gen_5min.py @@ -5,12 +5,13 @@ """ import os + import h5py -import pytest import numpy as np +import pytest -from reV.generation.generation import Gen from reV import TESTDATADIR +from reV.generation.generation import Gen pytest.importorskip("nsrdb") from nsrdb.utilities.statistics import mae_perc @@ -27,16 +28,16 @@ def test_gen_downscaling(): # run reV 2.0 generation gen = Gen('pvwattsv5', slice(0, None), sam_files, res_file, - output_request=('cf_mean', 'cf_profile'), sites_per_worker=100) + output_request=(MetaKeyName.CF_MEAN, ), sites_per_worker=100) gen.run(max_workers=1) - gen_outs = gen.out['cf_profile'].astype(np.int32) + gen_outs = gen.out[].astype(np.int32) if not os.path.exists(baseline): with h5py.File(baseline, 'w') as f: - f.create_dataset('cf_profile', data=gen_outs, dtype=gen_outs.dtype) + f.create_dataset(, data=gen_outs, dtype=gen_outs.dtype) else: with h5py.File(baseline, 'r') as f: - baseline = f['cf_profile'][...].astype(np.int32) + baseline = f[][...].astype(np.int32) x = mae_perc(gen_outs, baseline) msg = 'Mean absolute error is {}% from the baseline data'.format(x) diff --git a/tests/test_gen_config.py b/tests/test_gen_config.py index 277589a92..f5363a4f7 100644 --- a/tests/test_gen_config.py +++ b/tests/test_gen_config.py @@ -38,7 +38,7 @@ def get_r1_profiles(year=2012, tech='pv'): 'wind_{}_0.h5'.format(year)) with Outputs(rev1) as cf: - data = cf['cf_profile'][...] / 10000 + data = cf[][...] / 10000 return data @@ -98,10 +98,10 @@ def test_gen_from_config(runner, tech, clear_loggers): # noqa: C901 with Outputs(path, 'r') as cf: msg = 'cf_profile not written to disk' - assert 'cf_profile' in cf.datasets, msg - print(cf.scale_factors['cf_profile']) - print(cf.dtypes['cf_profile']) - rev2_profiles = cf['cf_profile'] + assert in cf.datasets, msg + print(cf.scale_factors[]) + print(cf.dtypes[]) + rev2_profiles = cf[] msg = 'monthly_energy not written to disk' assert 'monthly_energy' in cf.datasets, msg @@ -147,17 +147,17 @@ def test_sam_config(tech): "config": ['default'] * 100}) gen_json = Gen('pvwattsv5', points, sam_file, res_file, - output_request=('cf_profile',), sites_per_worker=50) + output_request=(,), sites_per_worker=50) gen_json.run(max_workers=2) gen_dict = Gen('pvwattsv5', points_config, sam_config, res_file, - output_request=('cf_profile',), sites_per_worker=50) + output_request=(,), sites_per_worker=50) gen_dict.run(max_workers=2) msg = ("reV {} generation run from JSON and SAM config dictionary do " "not match".format(tech)) - assert np.allclose(gen_json.out['cf_profile'], - gen_dict.out['cf_profile']), msg + assert np.allclose(gen_json.out[], + gen_dict.out[]), msg elif tech == 'wind': sam_file = TESTDATADIR + '/SAM/wind_gen_standard_losses_0.json' res_file = TESTDATADIR + '/wtk/ri_100_wtk_2012.h5' @@ -168,17 +168,17 @@ def test_sam_config(tech): "config": ['default'] * 10}) gen_json = Gen('windpower', points, sam_file, res_file, - output_request=('cf_profile',), sites_per_worker=3) + output_request=(,), sites_per_worker=3) gen_json.run(max_workers=2) gen_dict = Gen('windpower', points_config, sam_config, res_file, - output_request=('cf_profile',), sites_per_worker=3) + output_request=(,), sites_per_worker=3) gen_dict.run(max_workers=2) msg = ("reV {} generation run from JSON and SAM config dictionary do " "not match".format(tech)) - assert np.allclose(gen_json.out['cf_profile'], - gen_dict.out['cf_profile']), msg + assert np.allclose(gen_json.out[], + gen_dict.out[]), msg @pytest.mark.parametrize('expected_log_message', diff --git a/tests/test_gen_csp.py b/tests/test_gen_csp.py index 548b46e85..c75bedc0b 100644 --- a/tests/test_gen_csp.py +++ b/tests/test_gen_csp.py @@ -7,13 +7,15 @@ @author: gbuster """ -import numpy as np import os + +import numpy as np import pytest +from rex import Resource -from reV.generation.generation import Gen from reV import TESTDATADIR -from rex import Resource +from reV.generation.generation import Gen +from reV.utilities import MetaKeyName BASELINE = os.path.join(TESTDATADIR, 'gen_out', 'gen_ri_csp_2012.h5') RTOL = 0.1 @@ -27,7 +29,8 @@ def test_gen_csp(): res_file = TESTDATADIR + '/nsrdb/ri_100_nsrdb_{}.h5'.format(2012) # run reV 2.0 generation - output_request = ('cf_mean', 'cf_profile', 'gen_profile') + output_request = (MetaKeyName.CF_MEAN, MetaKeyName.CF_PROFILE, + MetaKeyName.GEN_PROFILE) gen = Gen('tcsmoltensalt', points, sam_files, res_file, output_request=output_request, sites_per_worker=1, scale_outputs=True) diff --git a/tests/test_gen_forecast.py b/tests/test_gen_forecast.py index a190aa02a..cd47101f3 100644 --- a/tests/test_gen_forecast.py +++ b/tests/test_gen_forecast.py @@ -8,21 +8,22 @@ """ import os -import pytest -import pandas as pd -import numpy as np import shutil import tempfile -from reV.handlers.outputs import Outputs -from reV.generation.generation import Gen -from reV.config.project_points import ProjectPoints -from reV import TESTDATADIR -from reV.utilities.exceptions import SAMExecutionError - +import numpy as np +import pandas as pd +import pytest from rex import Resource from rex.utilities.utilities import mean_irrad +from reV import TESTDATADIR +from reV.config.project_points import ProjectPoints +from reV.generation.generation import Gen +from reV.handlers.outputs import Outputs +from reV.utilities import MetaKeyName +from reV.utilities.exceptions import SAMExecutionError + def test_forecast(): """Test several forecast features implemented in reV gen including @@ -37,13 +38,13 @@ def test_forecast(): with Outputs(res_file, mode='a') as f: meta = f.meta - meta = meta.drop(['timezone', 'elevation'], axis=1) + meta = meta.drop([MetaKeyName.TIMEZONE, 'elevation'], axis=1) del f._h5['meta'] f._meta = None f.meta = meta with Outputs(res_file, mode='r') as f: - assert 'timezone' not in f.meta + assert MetaKeyName.TIMEZONE not in f.meta assert 'elevation' not in f.meta with Resource(res_file) as res: @@ -51,8 +52,9 @@ def test_forecast(): points = ProjectPoints(slice(0, 5), sam_files, 'pvwattsv7', res_file=res_file) - output_request = ('cf_mean', 'ghi_mean') - site_data = pd.DataFrame({'gid': np.arange(5), 'timezone': -5, + output_request = (MetaKeyName.CF_MEAN, 'ghi_mean') + site_data = pd.DataFrame({MetaKeyName.GID: np.arange(5), + MetaKeyName.TIMEZONE: -5, 'elevation': 0}) gid_map = {0: 20, 1: 20, 2: 50, 3: 51, 4: 51} diff --git a/tests/test_gen_geothermal.py b/tests/test_gen_geothermal.py index 91eb11b85..c1427c2ec 100644 --- a/tests/test_gen_geothermal.py +++ b/tests/test_gen_geothermal.py @@ -3,19 +3,20 @@ """ PyTest file for geothermal generation. """ -import os import json +import os import shutil from tempfile import TemporaryDirectory -import pytest -import pandas as pd import numpy as np +import pandas as pd +import pytest +from rex import Outputs +from reV import TESTDATADIR from reV.generation.generation import Gen from reV.SAM.generation import Geothermal -from reV import TESTDATADIR -from rex import Outputs +from reV.utilities import MetaKeyName DEFAULT_GEO_SAM_FILE = TESTDATADIR + '/SAM/geothermal_default.json' RTOL = 0.1 @@ -55,15 +56,18 @@ def test_gen_geothermal(depth, sample_resource_data): points = slice(0, 1) geo_sam_file, geo_res_file = sample_resource_data - with open(DEFAULT_GEO_SAM_FILE, "r") as fh: + with open(DEFAULT_GEO_SAM_FILE) as fh: geo_config = json.load(fh) geo_config["resource_depth"] = depth with open(geo_sam_file, "w") as fh: json.dump(geo_config, fh) - output_request = ('annual_energy', 'cf_mean', 'cf_profile', - 'gen_profile', 'lcoe_fcr', 'nameplate') + output_request = ('annual_energy', MetaKeyName.CF_MEAN, + MetaKeyName.CF_PROFILE, + MetaKeyName.GEN_PROFILE, + MetaKeyName.LCOE_FCR, + 'nameplate') gen = Gen('geothermal', points, geo_sam_file, geo_res_file, output_request=output_request, sites_per_worker=1, scale_outputs=True) @@ -93,8 +97,10 @@ def test_gen_geothermal_temp_too_low(sample_resource_data): geo_sam_file, geo_res_file = sample_resource_data shutil.copy(DEFAULT_GEO_SAM_FILE, geo_sam_file) - output_request = ('annual_energy', 'cf_mean', 'cf_profile', - 'gen_profile', 'lcoe_fcr', 'nameplate') + output_request = ('annual_energy', MetaKeyName.CF_MEAN, + MetaKeyName.CF_PROFILE, + MetaKeyName.GEN_PROFILE, + MetaKeyName.LCOE_FCR, 'nameplate') gen = Gen('geothermal', points, geo_sam_file, geo_res_file, output_request=output_request, sites_per_worker=1, scale_outputs=True) @@ -122,7 +128,7 @@ def test_per_kw_cost_inputs(sample_resource_data): points = slice(0, 1) geo_sam_file, geo_res_file = sample_resource_data - with open(DEFAULT_GEO_SAM_FILE, "r") as fh: + with open(DEFAULT_GEO_SAM_FILE) as fh: geo_config = json.load(fh) geo_config["resource_depth"] = 2000 @@ -133,7 +139,9 @@ def test_per_kw_cost_inputs(sample_resource_data): with open(geo_sam_file, "w") as fh: json.dump(geo_config, fh) - output_request = ('capital_cost', 'fixed_operating_cost', 'lcoe_fcr') + output_request = (MetaKeyName.CAPITAL_COST, + MetaKeyName.FIXED_OPERATING_COST, + MetaKeyName.LCOE_FCR) gen = Gen('geothermal', points, geo_sam_file, geo_res_file, output_request=output_request, sites_per_worker=1, scale_outputs=True) @@ -158,7 +166,7 @@ def test_drill_cost_inputs(sample_resource_data): points = slice(0, 1) geo_sam_file, geo_res_file = sample_resource_data - with open(DEFAULT_GEO_SAM_FILE, "r") as fh: + with open(DEFAULT_GEO_SAM_FILE) as fh: geo_config = json.load(fh) geo_config["resource_depth"] = 2000 @@ -170,7 +178,9 @@ def test_drill_cost_inputs(sample_resource_data): with open(geo_sam_file, "w") as fh: json.dump(geo_config, fh) - output_request = ('capital_cost', 'fixed_operating_cost', 'lcoe_fcr') + output_request = (MetaKeyName.CAPITAL_COST, + MetaKeyName.FIXED_OPERATING_COST, + MetaKeyName.LCOE_FCR) gen = Gen('geothermal', points, geo_sam_file, geo_res_file, output_request=output_request, sites_per_worker=1, scale_outputs=True) @@ -195,7 +205,7 @@ def test_gen_with_nameplate_input(sample_resource_data): points = slice(0, 1) geo_sam_file, geo_res_file = sample_resource_data - with open(DEFAULT_GEO_SAM_FILE, "r") as fh: + with open(DEFAULT_GEO_SAM_FILE) as fh: geo_config = json.load(fh) geo_config["resource_depth"] = 2000 @@ -203,8 +213,10 @@ def test_gen_with_nameplate_input(sample_resource_data): with open(geo_sam_file, "w") as fh: json.dump(geo_config, fh) - output_request = ('annual_energy', 'cf_mean', 'cf_profile', - 'gen_profile', 'lcoe_fcr', 'nameplate') + output_request = ('annual_energy', MetaKeyName.CF_MEAN, + MetaKeyName.CF_PROFILE, + MetaKeyName.GEN_PROFILE, + MetaKeyName.LCOE_FCR, 'nameplate') gen = Gen('geothermal', points, geo_sam_file, geo_res_file, output_request=output_request, sites_per_worker=1, scale_outputs=True) @@ -233,7 +245,7 @@ def test_gen_egs_too_high_egs_plant_design_temp(sample_resource_data): points = slice(0, 1) geo_sam_file, geo_res_file = sample_resource_data - with open(DEFAULT_GEO_SAM_FILE, "r") as fh: + with open(DEFAULT_GEO_SAM_FILE) as fh: geo_config = json.load(fh) geo_config["resource_depth"] = 2000 @@ -271,7 +283,7 @@ def test_gen_egs_too_low_egs_plant_design_temp(sample_resource_data): geo_sam_file, geo_res_file = sample_resource_data high_temp = 200 * Geothermal.MAX_RT_TO_EGS_RATIO + 10 - with open(DEFAULT_GEO_SAM_FILE, "r") as fh: + with open(DEFAULT_GEO_SAM_FILE) as fh: geo_config = json.load(fh) geo_config["resource_depth"] = 2000 @@ -309,7 +321,7 @@ def test_gen_egs_plant_design_temp_adjusted_from_user(sample_resource_data): geo_sam_file, geo_res_file = sample_resource_data not_too_high_temp = 200 * Geothermal.MAX_RT_TO_EGS_RATIO - 1 - with open(DEFAULT_GEO_SAM_FILE, "r") as fh: + with open(DEFAULT_GEO_SAM_FILE) as fh: geo_config = json.load(fh) geo_config["resource_depth"] = 2000 @@ -346,15 +358,17 @@ def test_gen_with_time_index_step_input(sample_resource_data): points = slice(0, 1) geo_sam_file, geo_res_file = sample_resource_data - with open(DEFAULT_GEO_SAM_FILE, "r") as fh: + with open(DEFAULT_GEO_SAM_FILE) as fh: geo_config = json.load(fh) geo_config["time_index_step"] = 2 with open(geo_sam_file, "w") as fh: json.dump(geo_config, fh) - output_request = ('annual_energy', 'cf_mean', 'cf_profile', - 'gen_profile', 'lcoe_fcr', 'nameplate') + output_request = ('annual_energy', MetaKeyName.CF_MEAN, + MetaKeyName.CF_PROFILE, + MetaKeyName.GEN_PROFILE, + MetaKeyName.LCOE_FCR, 'nameplate') gen = Gen('geothermal', points, geo_sam_file, geo_res_file, output_request=output_request, sites_per_worker=1, scale_outputs=True) diff --git a/tests/test_gen_linear.py b/tests/test_gen_linear.py index 63f3dbb6b..21fde7a89 100644 --- a/tests/test_gen_linear.py +++ b/tests/test_gen_linear.py @@ -6,12 +6,13 @@ @author: Mike Bannister """ import os -import pytest + import numpy as np +import pytest +from rex import Resource -from reV.generation.generation import Gen from reV import TESTDATADIR -from rex import Resource +from reV.generation.generation import Gen BASELINE = os.path.join(TESTDATADIR, 'gen_out', 'gen_ri_linear_2012.h5') RTOL = 0.01 @@ -39,7 +40,7 @@ def test_gen_linear(): # sequence: Receiver mass flow rate [kg/s] output_request = ('q_dot_to_heat_sink', 'gen', 'm_dot_field', 'q_dot_sf_out', 'W_dot_heat_sink_pump', 'm_dot_loop', - 'q_dot_rec_inc', 'cf_mean', 'gen_profile', + 'q_dot_rec_inc', MetaKeyName.CF_MEAN, 'gen_profile', 'annual_field_energy', 'annual_thermal_consumption',) # run reV 2.0 generation diff --git a/tests/test_gen_pv.py b/tests/test_gen_pv.py index f282eb5cf..1adac55b6 100644 --- a/tests/test_gen_pv.py +++ b/tests/test_gen_pv.py @@ -9,19 +9,20 @@ """ import os -import h5py -import pytest -import pandas as pd -import numpy as np -import tempfile import shutil +import tempfile +import h5py +import numpy as np +import pandas as pd +import pytest from rex.utilities.exceptions import ResourceRuntimeError -from reV.utilities.exceptions import ConfigError, ExecutionError -from reV.generation.generation import Gen -from reV.config.project_points import ProjectPoints + from reV import TESTDATADIR +from reV.config.project_points import ProjectPoints +from reV.generation.generation import Gen from reV.handlers.outputs import Outputs +from reV.utilities.exceptions import ConfigError, ExecutionError RTOL = 0.0 ATOL = 0.04 @@ -168,11 +169,11 @@ def test_pv_gen_profiles(year): # run reV 2.0 generation and write to disk gen = Gen('pvwattsv5', points, sam_files, res_file, - output_request=('cf_profile',), sites_per_worker=50) + output_request=(,), sites_per_worker=50) gen.run(max_workers=2, out_fpath=rev2_out) with Outputs(rev2_out, 'r') as cf: - rev2_profiles = cf['cf_profile'] + rev2_profiles = cf[] # get reV 1.0 generation profiles rev1_profiles = get_r1_profiles(year=year) @@ -194,11 +195,11 @@ def test_smart(year): # run reV 2.0 generation and write to disk gen = Gen('pvwattsv5', points, sam_files, res_file, - output_request=('cf_profile',), sites_per_worker=50) + output_request=(,), sites_per_worker=50) gen.run(max_workers=2, out_fpath=rev2_out) with Outputs(rev2_out, 'r') as cf: - rev2_profiles = cf['cf_profile'] + rev2_profiles = cf[] # get reV 1.0 generation profiles rev1_profiles = get_r1_profiles(year=year) @@ -216,7 +217,7 @@ def test_multi_file_nsrdb_2018(model): res_file = TESTDATADIR + '/nsrdb/nsrdb_*{}.h5'.format(2018) # run reV 2.0 generation gen = Gen(model, points, sam_files, res_file, - output_request=('cf_mean', 'cf_profile'), + output_request=('cf_mean', ), sites_per_worker=3) gen.run(max_workers=max_workers) @@ -224,7 +225,7 @@ def test_multi_file_nsrdb_2018(model): assert len(means_outs) == 10 assert np.mean(means_outs) > 0.14 - profiles_out = gen.out['cf_profile'] + profiles_out = gen.out[] assert profiles_out.shape == (105120, 10) assert np.mean(profiles_out) > 0.14 @@ -234,7 +235,7 @@ def get_r1_profiles(year=2012): rev1 = os.path.join(TESTDATADIR, 'ri_pv', 'profile_outputs', 'pv_{}_0.h5'.format(year)) with Outputs(rev1) as cf: - data = cf['cf_profile'][...] / 10000 + data = cf[][...] / 10000 return data @@ -260,7 +261,7 @@ def test_southern_hemisphere(): rev2_points = slice(0, 1) res_file = TESTDATADIR + '/nsrdb/brazil_solar.h5' sam_files = TESTDATADIR + '/SAM/i_pvwatts_fixed_lat_tilt.json' - output_request = ('cf_mean', 'cf_profile', 'dni_mean', 'dhi_mean', + output_request = ('cf_mean', , 'dni_mean', 'dhi_mean', 'ghi_mean', 'ac', 'dc', 'azimuth') gen = Gen('pvwattsv7', rev2_points, sam_files, res_file, @@ -285,7 +286,7 @@ def test_pvwattsv7_baseline(): res_file = TESTDATADIR + '/nsrdb/ri_100_nsrdb_{}.h5'.format(year) sam_files = TESTDATADIR + '/SAM/i_pvwattsv7.json' - output_request = ('cf_mean', 'cf_profile', 'dni_mean', 'dhi_mean', + output_request = ('cf_mean', , 'dni_mean', 'dhi_mean', 'ghi_mean', 'ac', 'dc') # run reV 2.0 generation @@ -321,7 +322,8 @@ def test_pvwatts_v5_v7(): gen5.run(max_workers=1) msg = 'PVwatts v5 and v7 did not match within test tolerance' - assert np.allclose(gen7.out['cf_mean'], gen5.out['cf_mean'], atol=3), msg + assert np.allclose(gen7.out['cf_mean'], + gen5.out['cf_mean'], atol=3), msg def test_pvwatts_v8_lifetime(): @@ -333,7 +335,7 @@ def test_pvwatts_v8_lifetime(): res_file = TESTDATADIR + '/nsrdb/ri_100_nsrdb_{}.h5'.format(year) sam_files = TESTDATADIR + '/SAM/i_pvwattsv8_degradation.json' - output_request = ('cf_mean', 'cf_profile', 'dni_mean', 'dhi_mean', + output_request = ('cf_mean', , 'dni_mean', 'dhi_mean', 'ghi_mean') # run reV 2.0 generation with valid output request @@ -342,7 +344,8 @@ def test_pvwatts_v8_lifetime(): gen.run(max_workers=1) msg = ('PVWATTSV8 cf_mean with system lifetime results {} did not match ' - 'baseline: {}'.format(gen.out['cf_mean'], baseline_cf_mean)) + 'baseline: {}'.format(gen.out['cf_mean'], + baseline_cf_mean)) assert np.allclose(gen.out['cf_mean'], baseline_cf_mean, rtol=0.005, atol=0.0), msg @@ -359,7 +362,7 @@ def test_pvwatts_v8_lifetime_invalid_request(): res_file = TESTDATADIR + '/nsrdb/ri_100_nsrdb_{}.h5'.format(year) sam_files = TESTDATADIR + '/SAM/i_pvwattsv8_degradation.json' - output_request_invalid = ('cf_mean', 'cf_profile', 'dni_mean', 'dhi_mean', + output_request_invalid = ('cf_mean', , 'dni_mean', 'dhi_mean', 'ghi_mean', 'ac', 'dc') # run reV 2.0 generation with invalid output request @@ -385,7 +388,7 @@ def test_bifacial(): sam_files = TESTDATADIR + '/SAM/i_pvwattsv7_bifacial.json' # run reV 2.0 generation - output_request = ('cf_mean', 'cf_profile', 'surface_albedo') + output_request = ('cf_mean', , 'surface_albedo') gen_bi = Gen('pvwattsv7', rev2_points, sam_files, res_file, sites_per_worker=1, output_request=output_request) gen_bi.run(max_workers=1) @@ -410,7 +413,7 @@ def test_gen_input_mods(): gen.run(max_workers=1) for i in range(5): inputs = gen.project_points[i][1] - assert inputs['tilt'] == 'latitude' + assert inputs['tilt'] == MetaKeyName.LATITUDE def test_gen_input_pass_through(): @@ -447,15 +450,17 @@ def test_gen_pv_site_data(): sites_per_worker=1, output_request=output_request) baseline.run(max_workers=1) - site_data = pd.DataFrame({'gid': np.arange(2), + site_data = pd.DataFrame({MetaKeyName.GID: np.arange(2), 'losses': np.ones(2)}) test = Gen('pvwattsv7', rev2_points, sam_files, res_file, sites_per_worker=1, output_request=output_request, site_data=site_data) test.run(max_workers=1) - assert all(test.out['cf_mean'][0:2] > baseline.out['cf_mean'][0:2]) - assert np.allclose(test.out['cf_mean'][2:], baseline.out['cf_mean'][2:]) + assert all(test.out['cf_mean'][0:2] > + baseline.out['cf_mean'][0:2]) + assert np.allclose(test.out['cf_mean'][2:], + baseline.out['cf_mean'][2:]) assert np.allclose(test.out['losses'][0:2], np.ones(2)) assert np.allclose(test.out['losses'][2:], 14.07566 * np.ones(3)) @@ -494,7 +499,7 @@ def test_detailed_pv_baseline(): res_file = TESTDATADIR + '/nsrdb/ri_100_nsrdb_{}.h5'.format(year) sam_files = TESTDATADIR + '/SAM/i_pvsamv1.json' - output_request = ('cf_mean', 'cf_profile', 'dni_mean', 'dhi_mean', + output_request = ('cf_mean', , 'dni_mean', 'dhi_mean', 'ghi_mean', 'ac', 'dc') # run reV 2.0 generation @@ -521,7 +526,7 @@ def test_detailed_pv_bifacial(): res_file = TESTDATADIR + '/nsrdb/ri_100_nsrdb_{}.h5'.format(year) sam_files = TESTDATADIR + '/SAM/i_pvsamv1_bifacial.json' - output_request = ('cf_mean', 'cf_profile', 'dni_mean', 'dhi_mean', + output_request = ('cf_mean', , 'dni_mean', 'dhi_mean', 'ghi_mean', 'ac', 'dc', 'surface_albedo') # run reV 2.0 generation @@ -540,16 +545,16 @@ def test_detailed_pv_bifacial(): def test_pv_clearsky(): - """test basic clearsky functionality""" + """Test basic clearsky functionality""" year = 2012 rev2_points = slice(0, 3) res_file = TESTDATADIR + '/nsrdb/ri_100_nsrdb_{}.h5'.format(year) sam_files = TESTDATADIR + '/SAM/naris_pv_1axis_inv13.json' sam_files_cs = TESTDATADIR + '/SAM/naris_pv_1axis_inv13_cs.json' - output_request = ('cf_mean', 'cf_profile', 'dni_mean', 'dhi_mean', + output_request = ('cf_mean', , 'dni_mean', 'dhi_mean', 'ghi_mean', 'ac', 'dc') - output_request_cs = ('cf_mean', 'cf_profile', 'clearsky_dni_mean', + output_request_cs = ('cf_mean', , 'clearsky_dni_mean', 'clearsky_dhi_mean', 'clearsky_ghi_mean', 'ac', 'dc') with tempfile.TemporaryDirectory() as td: @@ -585,31 +590,34 @@ def test_irrad_bias_correct(): res_file = TESTDATADIR + '/nsrdb/ri_100_nsrdb_{}.h5'.format(year) sam_files = TESTDATADIR + '/SAM/i_pvwattsv7.json' - output_request = ('cf_mean', 'cf_profile', 'dni_mean', 'dhi_mean', + output_request = ('cf_mean', , 'dni_mean', 'dhi_mean', 'ghi_mean', 'ac', 'dc') gen_base = Gen('pvwattsv7', points, sam_files, res_file, sites_per_worker=1, output_request=output_request) gen_base.run(max_workers=1) - bc_df = pd.DataFrame({'gid': np.arange(1, 10), 'method': 'lin_irrad', + bc_df = pd.DataFrame({MetaKeyName.GID: np.arange(1, 10), 'method': 'lin_irrad', 'scalar': 1, 'adder': 50}) gen = Gen('pvwattsv7', points, sam_files, res_file, sites_per_worker=1, output_request=output_request, bias_correct=bc_df) gen.run(max_workers=1) - assert (gen_base.out['cf_mean'][0] == gen.out['cf_mean'][0]).all() + assert (gen_base.out['cf_mean'][0] == + gen.out['cf_mean'][0]).all() assert (gen_base.out['ghi_mean'][0] == gen.out['ghi_mean'][0]).all() - assert np.allclose(gen_base.out['cf_profile'][:, 0], - gen.out['cf_profile'][:, 0]) + assert np.allclose(gen_base.out[][:, 0], + gen.out[][:, 0]) - assert (gen_base.out['cf_mean'][1:] < gen.out['cf_mean'][1:]).all() + assert (gen_base.out['cf_mean'][1:] < + gen.out['cf_mean'][1:]).all() assert (gen_base.out['ghi_mean'][1:] < gen.out['ghi_mean'][1:]).all() - mask = (gen_base.out['cf_profile'][:, 1:] <= gen.out['cf_profile'][:, 1:]) + mask = (gen_base.out[][:, 1:] <= gen.out[][:, 1:]) assert (mask.sum() / mask.size) > 0.99 - bc_df = pd.DataFrame({'gid': np.arange(100), 'method': 'lin_irrad', + bc_df = pd.DataFrame({MetaKeyName.GID: np.arange(100), + 'method': 'lin_irrad', 'scalar': 1, 'adder': -1500}) gen = Gen('pvwattsv7', points, sam_files, res_file, sites_per_worker=1, output_request=output_request, bias_correct=bc_df) @@ -627,7 +635,7 @@ def test_ac_outputs(): res_file = TESTDATADIR + '/nsrdb/ri_100_nsrdb_{}.h5'.format(year) sam_files = TESTDATADIR + '/SAM/i_pvwattsv8.json' - output_request = ('cf_mean', 'cf_mean_ac', 'cf_profile', 'cf_profile_ac', + output_request = ('cf_mean', 'cf_mean_ac', , 'cf_profile_ac', 'system_capacity', 'system_capacity_ac', 'ac', 'dc', 'dc_ac_ratio') @@ -641,7 +649,7 @@ def test_ac_outputs(): assert np.allclose(gen.out['cf_mean'], baseline_cf_mean, rtol=0.005, atol=0.0), msg - for req in ['cf_mean', 'cf_profile']: + for req in ['cf_mean', ]: ac_req = '{}_ac'.format(req) assert req in gen.out assert ac_req in gen.out @@ -651,7 +659,7 @@ def test_ac_outputs(): assert np.allclose(gen.out['system_capacity'] / gen.out['dc_ac_ratio'], gen.out['system_capacity_ac']) - assert not np.isclose(gen.out['cf_profile'], 1).any() + assert not np.isclose(gen.out[], 1).any() assert np.isclose(gen.out['cf_profile_ac'], 1).any() @@ -673,7 +681,7 @@ def test_bad_loc_inputs(bad_input): f.meta = meta gen = Gen('pvwattsv8', slice(0, 3), sam_files, res_file_bad, - output_request=('cf_profile',), sites_per_worker=50) + output_request=(,), sites_per_worker=50) with pytest.raises(ValueError): gen.run(max_workers=1) diff --git a/tests/test_gen_swh.py b/tests/test_gen_swh.py index efeddcc55..4c78bb48d 100644 --- a/tests/test_gen_swh.py +++ b/tests/test_gen_swh.py @@ -7,14 +7,14 @@ Created on 2/6/2020 @author: Mike Bannister """ -import numpy as np import os + +import numpy as np import pytest -import json +from rex import Resource -from reV.generation.generation import Gen from reV import TESTDATADIR -from rex import Resource +from reV.generation.generation import Gen RTOL = 0.01 ATOL = 0 @@ -29,7 +29,7 @@ def test_gen_swh_non_leap_year(): output_request = ('T_amb', 'T_cold', 'T_deliv', 'T_hot', 'draw', 'beam', 'diffuse', 'I_incident', 'I_transmitted', - 'annual_Q_deliv', 'Q_deliv', 'cf_mean', 'solar_fraction', + 'annual_Q_deliv', 'Q_deliv', MetaKeyName.CF_MEAN, 'solar_fraction', 'gen_profile') # run reV 2.0 generation @@ -61,7 +61,7 @@ def test_gen_swh_leap_year(): output_request = ('T_amb', 'T_cold', 'T_deliv', 'T_hot', 'draw', 'beam', 'diffuse', 'I_incident', 'I_transmitted', - 'annual_Q_deliv', 'Q_deliv', 'cf_mean', 'solar_fraction') + 'annual_Q_deliv', 'Q_deliv', MetaKeyName.CF_MEAN, 'solar_fraction') # run reV 2.0 generation gen = Gen('solarwaterheat', points, sam_files, res_file, diff --git a/tests/test_gen_time_scale.py b/tests/test_gen_time_scale.py index 47e93bf3d..d04ec6eae 100644 --- a/tests/test_gen_time_scale.py +++ b/tests/test_gen_time_scale.py @@ -4,14 +4,15 @@ Test resource up and down scaling """ -import os import json +import os + import h5py -import pytest import numpy as np +import pytest -from reV.generation.generation import Gen from reV import TESTDATADIR +from reV.generation.generation import Gen def test_time_index_step(): @@ -31,17 +32,17 @@ def test_time_index_step(): # run reV 2.0 generation gen = Gen('pvwattsv5', slice(0, None), sam_input, res_file, - output_request=('cf_mean', 'cf_profile'), + output_request=(MetaKeyName.CF_MEAN, ), sites_per_worker=100) gen.run(max_workers=1) - gen_outs = gen.out['cf_profile'].astype(np.int32) + gen_outs = gen.out[].astype(np.int32) if not os.path.exists(baseline): with h5py.File(baseline, 'w') as f: - f.create_dataset('cf_profile', data=gen_outs, dtype=gen_outs.dtype) + f.create_dataset(, data=gen_outs, dtype=gen_outs.dtype) else: with h5py.File(baseline, 'r') as f: - baseline = f['cf_profile'][...].astype(np.int32) + baseline = f[][...].astype(np.int32) assert np.allclose(gen_outs, baseline) diff --git a/tests/test_gen_trough.py b/tests/test_gen_trough.py index a1a01d909..244a49c9d 100644 --- a/tests/test_gen_trough.py +++ b/tests/test_gen_trough.py @@ -7,13 +7,14 @@ Created on 2/6/2020 @author: Mike Bannister """ -import numpy as np import os + +import numpy as np import pytest +from rex import Resource -from reV.generation.generation import Gen from reV import TESTDATADIR -from rex import Resource +from reV.generation.generation import Gen BASELINE = os.path.join(TESTDATADIR, 'gen_out', 'gen_ri_trough_2012.h5') RTOL = 0.1 @@ -38,7 +39,7 @@ def test_gen_tph(): 'm_dot_field_delivered', 'm_dot_field_recirc', 'q_dot_htf_sf_out', 'q_dot_to_heat_sink', 'q_dot_rec_inc', 'qinc_costh', 'dni_costh', 'beam', - 'cf_mean', 'annual_gross_energy', + MetaKeyName.CF_MEAN, 'annual_gross_energy', 'annual_thermal_consumption', 'annual_energy') # run reV 2.0 generation diff --git a/tests/test_gen_wave.py b/tests/test_gen_wave.py index a3b2f246d..e50a64a81 100644 --- a/tests/test_gen_wave.py +++ b/tests/test_gen_wave.py @@ -9,14 +9,14 @@ """ import os -import pytest + import numpy as np +import pytest +from rex import Resource, safe_json_load +from reV import TESTDATADIR from reV.generation.generation import Gen from reV.SAM.defaults import DefaultMhkWave -from reV import TESTDATADIR - -from rex import Resource, safe_json_load BASELINE = os.path.join(TESTDATADIR, 'gen_out', 'ri_wave_2010.h5') @@ -38,16 +38,16 @@ def test_mhkwave(): sam_files = TESTDATADIR + '/SAM/mhkwave_default.json' res_file = TESTDATADIR + '/wave/ri_wave_2010.h5' points = slice(0, 100) - output_request = ('cf_mean', 'cf_profile') + output_request = (MetaKeyName.CF_MEAN, ) test = Gen('mhkwave', points, sam_files, res_file, sites_per_worker=3, output_request=output_request) test.run(max_workers=1) with Resource(BASELINE) as f: - assert np.allclose(test.out['cf_mean'], f['cf_mean'], + assert np.allclose(test.out[MetaKeyName.CF_MEAN], f[MetaKeyName.CF_MEAN], atol=0.01, rtol=0.01) - assert np.allclose(test.out['cf_profile'], f['cf_profile'], + assert np.allclose(test.out[], f[], atol=0.01, rtol=0.01) diff --git a/tests/test_gen_wind.py b/tests/test_gen_wind.py index 47e986ca6..15ec42d99 100644 --- a/tests/test_gen_wind.py +++ b/tests/test_gen_wind.py @@ -10,17 +10,18 @@ import os import shutil +import tempfile + import h5py -import pytest -import pandas as pd import numpy as np -import tempfile +import pandas as pd +import pytest +from rex import Outputs, Resource, WindResource -from reV.generation.generation import Gen -from reV.config.project_points import ProjectPoints from reV import TESTDATADIR - -from rex import Resource, WindResource, Outputs +from reV.config.project_points import ProjectPoints +from reV.generation.generation import Gen +from reV.utilities import MetaKeyName RTOL = 0 ATOL = 0.001 @@ -52,7 +53,7 @@ def years(self): def get_cf_mean(self, site, year): """Get a cf mean based on site and year""" iy = self.years.index(year) - out = self._h5['wind']['cf_mean'][iy, site] + out = self._h5['wind'][MetaKeyName.CF_MEAN][iy, site] return out @@ -68,10 +69,10 @@ def is_num(n): def to_list(gen_out): """Generation output handler that converts to the rev 1.0 format.""" if isinstance(gen_out, list) and len(gen_out) == 1: - out = [c['cf_mean'] for c in gen_out[0].values()] + out = [c[MetaKeyName.CF_MEAN] for c in gen_out[0].values()] if isinstance(gen_out, dict): - out = [c['cf_mean'] for c in gen_out.values()] + out = [c[MetaKeyName.CF_MEAN] for c in gen_out.values()] return out @@ -93,7 +94,7 @@ def test_wind_gen_slice(f_rev1_out, rev2_points, year, max_workers): gen = Gen('windpower', rev2_points, sam_files, res_file, sites_per_worker=3) gen.run(max_workers=max_workers) - gen_outs = list(gen.out['cf_mean']) + gen_outs = list(gen.out[MetaKeyName.CF_MEAN]) # initialize the rev1 output hander with wind_results(rev1_outs) as wind: @@ -104,17 +105,17 @@ def test_wind_gen_slice(f_rev1_out, rev2_points, year, max_workers): msg = 'Wind cf_means results did not match reV 1.0 results!' assert np.allclose(gen_outs, cf_mean_list, rtol=RTOL, atol=ATOL), msg assert np.allclose(pp.sites, gen.meta.index.values), 'bad gen meta!' - assert np.allclose(pp.sites, gen.meta['gid'].values), 'bad gen meta!' + assert np.allclose(pp.sites, gen.meta[MetaKeyName.GID].values), 'bad gen meta!' - labels = ['latitude', 'longitude'] + labels = [MetaKeyName.LATITUDE, MetaKeyName.LONGITUDE] with Resource(res_file) as res: for i, (gen_gid, site_meta) in enumerate(gen.meta.iterrows()): - res_gid = site_meta['gid'] + res_gid = site_meta[MetaKeyName.GID] assert gen_gid == res_gid test_coords = site_meta[labels].values.astype(float) true_coords = res.meta.loc[res_gid, labels].values.astype(float) assert np.allclose(test_coords, true_coords) - assert site_meta['gid'] == res_gid + assert site_meta[MetaKeyName.GID] == res_gid @pytest.mark.parametrize('gid_map', @@ -137,7 +138,7 @@ def test_gid_map(gid_map): sam_files = TESTDATADIR + '/SAM/wind_gen_standard_losses_0.json' res_file = TESTDATADIR + '/wtk/ri_100_wtk_{}.h5'.format(year) - output_request = ('cf_mean', 'cf_profile', 'ws_mean', 'windspeed', + output_request = (MetaKeyName.CF_MEAN, , 'ws_mean', 'windspeed', 'monthly_energy') baseline = Gen('windpower', points_base, sam_files, res_file, @@ -157,13 +158,13 @@ def test_gid_map(gid_map): for key in output_request: assert np.allclose(map_test.out[key], write_gid_test.out[key]) - for map_test_gid, write_test_gid in zip(map_test.meta['gid'], - write_gid_test.meta['gid']): + for map_test_gid, write_test_gid in zip(map_test.meta[MetaKeyName.GID], + write_gid_test.meta[MetaKeyName.GID]): assert map_test_gid == gid_map[write_test_gid] - if len(baseline.out['cf_mean']) == len(map_test.out['cf_mean']): - assert not np.allclose(baseline.out['cf_mean'], - map_test.out['cf_mean']) + if len(baseline.out[MetaKeyName.CF_MEAN]) == len(map_test.out[MetaKeyName.CF_MEAN]): + assert not np.allclose(baseline.out[MetaKeyName.CF_MEAN], + map_test.out[MetaKeyName.CF_MEAN]) for gen_gid_test, res_gid in gid_map.items(): gen_gid_test = points_test.index(gen_gid_test) @@ -176,21 +177,21 @@ def test_gid_map(gid_map): assert np.allclose(baseline.out[key][gen_gid_base], map_test.out[key][gen_gid_test]) - labels = ['latitude', 'longitude'] + labels = [MetaKeyName.LATITUDE, MetaKeyName.LONGITUDE] with Resource(res_file) as res: for i, (gen_gid, site_meta) in enumerate(baseline.meta.iterrows()): - res_gid = site_meta['gid'] + res_gid = site_meta[MetaKeyName.GID] test_coords = site_meta[labels].values.astype(float) true_coords = res.meta.loc[res_gid, labels].values.astype(float) assert np.allclose(test_coords, true_coords) - assert site_meta['gid'] == res_gid + assert site_meta[MetaKeyName.GID] == res_gid for i, (gen_gid, site_meta) in enumerate(map_test.meta.iterrows()): res_gid = gid_map[gen_gid] test_coords = site_meta[labels].values.astype(float) true_coords = res.meta.loc[res_gid, labels].values.astype(float) assert np.allclose(test_coords, true_coords) - assert site_meta['gid'] == res_gid + assert site_meta[MetaKeyName.GID] == res_gid def test_wind_gen_new_outputs(points=slice(0, 10), year=2012, max_workers=1): @@ -199,19 +200,19 @@ def test_wind_gen_new_outputs(points=slice(0, 10), year=2012, max_workers=1): sam_files = TESTDATADIR + '/SAM/wind_gen_standard_losses_0.json' res_file = TESTDATADIR + '/wtk/ri_100_wtk_{}.h5'.format(year) - output_request = ('cf_mean', 'cf_profile', 'monthly_energy') + output_request = (MetaKeyName.CF_MEAN, , 'monthly_energy') # run reV 2.0 generation gen = Gen('windpower', points, sam_files, res_file, sites_per_worker=3, output_request=output_request) gen.run(max_workers=max_workers) - assert gen.out['cf_mean'].shape == (10, ) - assert gen.out['cf_profile'].shape == (8760, 10) + assert gen.out[MetaKeyName.CF_MEAN].shape == (10, ) + assert gen.out[].shape == (8760, 10) assert gen.out['monthly_energy'].shape == (12, 10) - assert gen._out['cf_mean'].dtype == np.float32 - assert gen._out['cf_profile'].dtype == np.uint16 + assert gen._out[MetaKeyName.CF_MEAN].dtype == np.float32 + assert gen._out[].dtype == np.uint16 assert gen._out['monthly_energy'].dtype == np.float32 @@ -223,7 +224,7 @@ def test_windspeed_pass_through(rev2_points=slice(0, 10), year=2012, sam_files = TESTDATADIR + '/SAM/wind_gen_standard_losses_0.json' res_file = TESTDATADIR + '/wtk/ri_100_wtk_{}.h5'.format(year) - output_requests = ('cf_mean', 'windspeed') + output_requests = (MetaKeyName.CF_MEAN, 'windspeed') # run reV 2.0 generation gen = Gen('windpower', rev2_points, sam_files, res_file, @@ -244,7 +245,7 @@ def test_multi_file_5min_wtk(): # run reV 2.0 generation gen = Gen('windpower', points, sam_files, res_file, sites_per_worker=3) gen.run(max_workers=max_workers) - gen_outs = list(gen._out['cf_mean']) + gen_outs = list(gen._out[MetaKeyName.CF_MEAN]) assert len(gen_outs) == 10 assert np.mean(gen_outs) > 0.55 @@ -254,20 +255,20 @@ def test_wind_gen_site_data(points=slice(0, 5), year=2012, max_workers=1): sam_files = TESTDATADIR + '/SAM/wind_gen_standard_losses_0.json' res_file = TESTDATADIR + '/wtk/ri_100_wtk_{}.h5'.format(year) - output_request = ('cf_mean', 'turb_generic_loss') + output_request = (MetaKeyName.CF_MEAN, 'turb_generic_loss') baseline = Gen('windpower', points, sam_files, res_file, sites_per_worker=3, output_request=output_request) baseline.run(max_workers=max_workers) - site_data = pd.DataFrame({'gid': np.arange(2), + site_data = pd.DataFrame({MetaKeyName.GID: np.arange(2), 'turb_generic_loss': np.zeros(2)}) test = Gen('windpower', points, sam_files, res_file, sites_per_worker=3, output_request=output_request, site_data=site_data) test.run(max_workers=max_workers) - assert all(test.out['cf_mean'][0:2] > baseline.out['cf_mean'][0:2]) - assert np.allclose(test.out['cf_mean'][2:], baseline.out['cf_mean'][2:]) + assert all(test.out[MetaKeyName.CF_MEAN][0:2] > baseline.out[MetaKeyName.CF_MEAN][0:2]) + assert np.allclose(test.out[MetaKeyName.CF_MEAN][2:], baseline.out[MetaKeyName.CF_MEAN][2:]) assert np.allclose(test.out['turb_generic_loss'][0:2], np.zeros(2)) assert np.allclose(test.out['turb_generic_loss'][2:], 16.7 * np.ones(3)) @@ -322,7 +323,7 @@ def test_multi_resolution_wtk(): low_res_resource_file=fp_lr, sites_per_worker=3) gen.run(max_workers=max_workers) - gen_outs = list(gen._out['cf_mean']) + gen_outs = list(gen._out[MetaKeyName.CF_MEAN]) assert len(gen_outs) == 2 assert np.mean(gen_outs) > 0.55 @@ -335,25 +336,25 @@ def test_wind_bias_correct(): # run reV 2.0 generation points = slice(0, 10) gen_base = Gen('windpower', points, sam_files, res_file, - output_request=('cf_mean', 'cf_profile', 'ws_mean'), + output_request=(MetaKeyName.CF_MEAN, , 'ws_mean'), sites_per_worker=3) gen_base.run(max_workers=1) - outs_base = np.array(list(gen_base.out['cf_mean'])) + outs_base = np.array(list(gen_base.out[MetaKeyName.CF_MEAN])) - bc_df = pd.DataFrame({'gid': np.arange(100), 'method': 'lin_ws', + bc_df = pd.DataFrame({MetaKeyName.GID: np.arange(100), 'method': 'lin_ws', 'scalar': 1, 'adder': 2}) gen = Gen('windpower', points, sam_files, res_file, - output_request=('cf_mean', 'cf_profile', 'ws_mean'), + output_request=(MetaKeyName.CF_MEAN, , 'ws_mean'), sites_per_worker=3, bias_correct=bc_df) gen.run(max_workers=1) - outs_bc = np.array(list(gen.out['cf_mean'])) + outs_bc = np.array(list(gen.out[MetaKeyName.CF_MEAN])) assert all(outs_bc > outs_base) assert np.allclose(gen_base.out['ws_mean'] + 2, gen.out['ws_mean']) - bc_df = pd.DataFrame({'gid': np.arange(100), 'method': 'lin_ws', + bc_df = pd.DataFrame({MetaKeyName.GID: np.arange(100), 'method': 'lin_ws', 'scalar': 1, 'adder': -100}) gen = Gen('windpower', points, sam_files, res_file, - output_request=('cf_mean', 'cf_profile', 'ws_mean'), + output_request=(MetaKeyName.CF_MEAN, , 'ws_mean'), sites_per_worker=3, bias_correct=bc_df) gen.run(max_workers=1) for k, arr in gen.out.items(): diff --git a/tests/test_gen_wind_losses.py b/tests/test_gen_wind_losses.py index c533520e8..fb83c320b 100644 --- a/tests/test_gen_wind_losses.py +++ b/tests/test_gen_wind_losses.py @@ -8,12 +8,12 @@ """ import os -import pytest + import numpy as np +import pytest -from reV.generation.generation import Gen from reV import TESTDATADIR - +from reV.generation.generation import Gen YEAR = 2012 REV2_POINTS = slice(0, 5) @@ -58,7 +58,7 @@ def test_wind_generic_losses(loss): gen = Gen('windpower', pc, SAM_FILE, RES_FILE, sites_per_worker=3) gen.run(max_workers=1) - gen_outs = list(gen.out['cf_mean']) + gen_outs = list(gen.out[MetaKeyName.CF_MEAN]) assert np.allclose(gen_outs, LOSS_BASELINE[loss], rtol=RTOL, atol=ATOL) @@ -77,7 +77,7 @@ def test_wind_icing_losses(i): gen = Gen('windpower', pc, SAM_FILE, RES_FILE, sites_per_worker=3) gen.run(max_workers=1) - gen_outs = list(gen.out['cf_mean']) + gen_outs = list(gen.out[MetaKeyName.CF_MEAN]) assert np.allclose(gen_outs, ICING_BASELINE[i]['output'], rtol=RTOL, atol=ATOL) @@ -95,7 +95,7 @@ def test_wind_low_temp_cutoff(i): gen = Gen('windpower', pc, SAM_FILE, RES_FILE, sites_per_worker=3) gen.run(max_workers=1) - gen_outs = list(gen.out['cf_mean']) + gen_outs = list(gen.out[MetaKeyName.CF_MEAN]) assert np.allclose(gen_outs, LOW_TEMP_BASELINE[i]['output'], rtol=RTOL, atol=ATOL) diff --git a/tests/test_handlers_multiyear.py b/tests/test_handlers_multiyear.py index 019adf8e6..e3b1a2a23 100644 --- a/tests/test_handlers_multiyear.py +++ b/tests/test_handlers_multiyear.py @@ -2,24 +2,24 @@ """ pytests for MultiYear collection and computation """ -import h5py -import numpy as np +import json import os import shutil -import pytest import tempfile -import json import traceback -from reV.cli import main -from reV.handlers.outputs import Outputs -from reV.handlers.multi_year import MultiYear -from reV import TESTDATADIR -from reV.utilities import ModuleName - +import h5py +import numpy as np +import pytest from rex import Resource from rex.utilities.loggers import init_logger +from reV import TESTDATADIR +from reV.cli import main +from reV.handlers.multi_year import MultiYear +from reV.handlers.outputs import Outputs +from reV.utilities import MetaKeyName, ModuleName + H5_DIR = os.path.join(TESTDATADIR, 'gen_out') YEARS = [2012, 2013] H5_FILES = [os.path.join(H5_DIR, 'gen_ri_pv_{}_x000.h5'.format(year)) @@ -96,14 +96,14 @@ def compare_arrays(my_arr, test_arr, desc): @pytest.mark.parametrize(('source', 'dset', 'group'), [ - (H5_FILES, 'cf_profile', None), - (H5_FILES, 'cf_mean', None), - (H5_FILES, 'cf_profile', 'pytest'), - (H5_FILES, 'cf_mean', 'pytest'), - (H5_PATTERN, 'cf_profile', None), - (H5_PATTERN, 'cf_mean', None), - (H5_PATTERN, 'cf_profile', 'pytest'), - (H5_PATTERN, 'cf_mean', 'pytest'), + (H5_FILES, , None), + (H5_FILES, MetaKeyName.CF_MEAN, None), + (H5_FILES, , 'pytest'), + (H5_FILES, MetaKeyName.CF_MEAN, 'pytest'), + (H5_PATTERN, , None), + (H5_PATTERN, MetaKeyName.CF_MEAN, None), + (H5_PATTERN, , 'pytest'), + (H5_PATTERN, MetaKeyName.CF_MEAN, 'pytest'), ]) def test_my_collection(source, dset, group): """ @@ -207,8 +207,8 @@ def test_cli(runner, clear_loggers): @pytest.mark.parametrize(('dset', 'group'), [ - ('cf_mean', None), - ('cf_mean', 'pytest')]) + (MetaKeyName.CF_MEAN, None), + (MetaKeyName.CF_MEAN, 'pytest')]) def test_my_means(dset, group): """ Test computation of multi-year means @@ -237,8 +237,8 @@ def test_my_means(dset, group): @pytest.mark.parametrize(('dset', 'group'), [ - ('cf_mean', None), - ('cf_mean', 'pytest')]) + (MetaKeyName.CF_MEAN, None), + (MetaKeyName.CF_MEAN, 'pytest')]) def test_update(dset, group): """ Test computation of multi-year means @@ -278,8 +278,8 @@ def test_update(dset, group): @pytest.mark.parametrize(('dset', 'group'), [ - ('cf_mean', None), - ('cf_mean', 'pytest')]) + (MetaKeyName.CF_MEAN, None), + (MetaKeyName.CF_MEAN, 'pytest')]) def test_my_stdev(dset, group): """ Test computation of multi-year means diff --git a/tests/test_handlers_outputs.py b/tests/test_handlers_outputs.py index 135674ef2..7f53efd14 100644 --- a/tests/test_handlers_outputs.py +++ b/tests/test_handlers_outputs.py @@ -2,23 +2,23 @@ """ PyTest file for reV LCOE economies of scale """ +import os +import tempfile + import h5py import numpy as np import pandas as pd import pytest -import os -import tempfile - -from reV.version import __version__ -from reV.handlers.outputs import Outputs from rex.utilities.utilities import pd_date_range +from reV.handlers.outputs import Outputs +from reV.version import __version__ arr1 = np.ones(100) arr2 = np.ones((8760, 100)) arr3 = np.ones((8760, 100), dtype=float) * 42.42 -meta = pd.DataFrame({'latitude': np.ones(100), - 'longitude': np.zeros(100)}) +meta = pd.DataFrame({MetaKeyName.LATITUDE: np.ones(100), + MetaKeyName.LONGITUDE: np.zeros(100)}) time_index = pd_date_range('20210101', '20220101', freq='1h', closed='right') diff --git a/tests/test_handlers_transmission.py b/tests/test_handlers_transmission.py index c421148e8..1ce11717d 100644 --- a/tests/test_handlers_transmission.py +++ b/tests/test_handlers_transmission.py @@ -3,6 +3,7 @@ Transmission Feature Tests """ import os + import pandas as pd import pytest @@ -45,7 +46,7 @@ def trans_table(): return trans_table -@pytest.mark.parametrize(('i', 'trans_costs', 'distance', 'gid'), +@pytest.mark.parametrize(('i', 'trans_costs', 'distance', MetaKeyName.GID), ((1, TRANS_COSTS_1, 0, 43300), (2, TRANS_COSTS_2, 0, 43300), (1, TRANS_COSTS_1, 100, 43300), @@ -75,7 +76,7 @@ def test_cost_calculation(i, trans_costs, distance, gid, trans_table): assert true_cost == trans_cost -@pytest.mark.parametrize(('trans_costs', 'capacity', 'gid'), +@pytest.mark.parametrize(('trans_costs', MetaKeyName.CAPACITY, MetaKeyName.GID), ((TRANS_COSTS_1, 350, 43300), (TRANS_COSTS_2, 350, 43300), (TRANS_COSTS_1, 100, 43300), diff --git a/tests/test_hybrids.py b/tests/test_hybrids.py index f2ad5a049..a31e959e0 100644 --- a/tests/test_hybrids.py +++ b/tests/test_hybrids.py @@ -1,21 +1,20 @@ # -*- coding: utf-8 -*- """reV hybrids tests. """ +import json import os -import pytest -import numpy as np import tempfile -import json - -from reV.hybrids import Hybridization, HYBRID_METHODS -from reV.hybrids.hybrids import HybridsData, MERGE_COLUMN, OUTPUT_PROFILE_NAMES -from reV.utilities.exceptions import FileInputError, InputError, OutputWarning -from reV.utilities import ModuleName -from reV.cli import main -from reV import Outputs, TESTDATADIR +import numpy as np +import pytest from rex.resource import Resource +from reV import TESTDATADIR, Outputs +from reV.cli import main +from reV.hybrids import HYBRID_METHODS, Hybridization +from reV.hybrids.hybrids import MERGE_COLUMN, OUTPUT_PROFILE_NAMES, HybridsData +from reV.utilities import ModuleName +from reV.utilities.exceptions import FileInputError, InputError, OutputWarning SOLAR_FPATH = os.path.join( TESTDATADIR, 'rep_profiles_out', 'rep_profiles_solar.h5') @@ -26,22 +25,22 @@ SOLAR_FPATH_MULT = os.path.join( TESTDATADIR, 'rep_profiles_out', 'rep_profiles_solar_multiple.h5') with Resource(SOLAR_FPATH) as res: - SOLAR_SCPGIDS = set(res.meta['sc_point_gid']) + SOLAR_SCPGIDS = set(res.meta[MetaKeyName.SC_POINT_GID]) with Resource(WIND_FPATH) as res: - WIND_SCPGIDS = set(res.meta['sc_point_gid']) + WIND_SCPGIDS = set(res.meta[MetaKeyName.SC_POINT_GID]) def test_hybridization_profile_output_single_resource(): - """Test that the hybridization calculation is correct (1 resource). """ + """Test that the hybridization calculation is correct (1 resource).""" sc_point_gid = 40005 with Resource(SOLAR_FPATH) as res: solar_idx = np.where( - res.meta['sc_point_gid'] == sc_point_gid + res.meta[MetaKeyName.SC_POINT_GID] == sc_point_gid )[0][0] - solar_cap = res.meta.loc[solar_idx, 'capacity'] + solar_cap = res.meta.loc[solar_idx, MetaKeyName.CAPACITY] solar_test_profile = res['rep_profiles_0', :, solar_idx] weighted_solar = solar_cap * solar_test_profile @@ -50,7 +49,7 @@ def test_hybridization_profile_output_single_resource(): h.run() hp, hsp, hwp, = h.profiles.values() h_meta = h.hybrid_meta - h_idx = np.where(h_meta['sc_point_gid'] == sc_point_gid)[0][0] + h_idx = np.where(h_meta[MetaKeyName.SC_POINT_GID] == sc_point_gid)[0][0] assert np.allclose(hp[:, h_idx], weighted_solar) assert np.allclose(hsp[:, h_idx], weighted_solar) @@ -59,16 +58,16 @@ def test_hybridization_profile_output_single_resource(): def test_hybridization_profile_output_with_ratio_none(): - """Test that the hybridization calculation is correct (1 resource). """ + """Test that the hybridization calculation is correct (1 resource).""" sc_point_gid = 40005 with Resource(SOLAR_FPATH) as res: solar_idx = np.where( - res.meta['sc_point_gid'] == sc_point_gid + res.meta[MetaKeyName.SC_POINT_GID] == sc_point_gid )[0][0] - solar_cap = res.meta.loc[solar_idx, 'capacity'] + solar_cap = res.meta.loc[solar_idx, MetaKeyName.CAPACITY] solar_test_profile = res['rep_profiles_0', :, solar_idx] weighted_solar = solar_cap * solar_test_profile @@ -80,7 +79,7 @@ def test_hybridization_profile_output_with_ratio_none(): h.run() hp, hsp, hwp, = h.profiles.values() h_meta = h.hybrid_meta - h_idx = np.where(h_meta['sc_point_gid'] == sc_point_gid)[0][0] + h_idx = np.where(h_meta[MetaKeyName.SC_POINT_GID] == sc_point_gid)[0][0] assert np.allclose(hp[:, h_idx], weighted_solar) assert np.allclose(hsp[:, h_idx], weighted_solar) @@ -89,21 +88,21 @@ def test_hybridization_profile_output_with_ratio_none(): def test_hybridization_profile_output(): - """Test that the hybridization calculation is correct. """ + """Test that the hybridization calculation is correct.""" common_sc_point_gid = 38883 with Resource(SOLAR_FPATH) as res: solar_idx = np.where( - res.meta['sc_point_gid'] == common_sc_point_gid + res.meta[MetaKeyName.SC_POINT_GID] == common_sc_point_gid )[0][0] - solar_cap = res.meta.loc[solar_idx, 'capacity'] + solar_cap = res.meta.loc[solar_idx, MetaKeyName.CAPACITY] solar_test_profile = res['rep_profiles_0', :, solar_idx] with Resource(WIND_FPATH) as res: wind_idx = np.where( - res.meta['sc_point_gid'] == common_sc_point_gid + res.meta[MetaKeyName.SC_POINT_GID] == common_sc_point_gid )[0][0] - wind_cap = res.meta.loc[wind_idx, 'capacity'] + wind_cap = res.meta.loc[wind_idx, MetaKeyName.CAPACITY] wind_test_profile = res['rep_profiles_0', :, wind_idx] weighted_solar = solar_cap * solar_test_profile @@ -113,7 +112,7 @@ def test_hybridization_profile_output(): h.run() hp, hsp, hwp, = h.profiles.values() h_meta = h.hybrid_meta - h_idx = np.where(h_meta['sc_point_gid'] == common_sc_point_gid)[0][0] + h_idx = np.where(h_meta[MetaKeyName.SC_POINT_GID] == common_sc_point_gid)[0][0] assert np.allclose(hp[:, h_idx], weighted_solar + weighted_wind) assert np.allclose(hsp[:, h_idx], weighted_solar) @@ -123,7 +122,7 @@ def test_hybridization_profile_output(): @pytest.mark.parametrize("input_files", [(SOLAR_FPATH, WIND_FPATH), (SOLAR_FPATH_30_MIN, WIND_FPATH)]) def test_hybridization_output_shapes(input_files): - """Test that the output shapes are as expected. """ + """Test that the output shapes are as expected.""" sfp, wfp = input_files h = Hybridization(sfp, wfp) @@ -169,11 +168,11 @@ def test_meta_hybridization(input_combination, expected_shape, overlap): ) h.run() assert h.hybrid_meta.shape == expected_shape - assert set(h.hybrid_meta['sc_point_gid']) == overlap + assert set(h.hybrid_meta[MetaKeyName.SC_POINT_GID]) == overlap def test_limits_and_ratios_output_values(): - """Test that limits and ratios are properly applied in succession. """ + """Test that limits and ratios are properly applied in succession.""" limits = {'solar_capacity': 50, 'wind_capacity': 0.5} ratio_numerator = 'solar_capacity' @@ -212,7 +211,7 @@ def test_limits_and_ratios_output_values(): ((0.3, 3.6), (0.3 - 1e6, 3.6 + 1e6)) ]) def test_ratios_input(ratio_cols, ratio_bounds, bounds): - """Test that the hybrid meta limits the ratio columns correctly. """ + """Test that the hybrid meta limits the ratio columns correctly.""" ratio_numerator, ratio_denominator = ratio_cols ratio = '{}/{}'.format(ratio_numerator, ratio_denominator) h = Hybridization( @@ -231,7 +230,7 @@ def test_ratios_input(ratio_cols, ratio_bounds, bounds): assert np.all(h.hybrid_meta['hybrid_{}'.format(ratio_denominator)] <= h.hybrid_meta[ratio_denominator]) - if 'capacity' in ratio: + if MetaKeyName.CAPACITY in ratio: max_solar_capacities = h.hybrid_meta['hybrid_solar_capacity'] max_solar_capacities = max_solar_capacities.values.reshape(1, -1) assert np.all(h.profiles['hybrid_solar_profile'] @@ -243,7 +242,7 @@ def test_ratios_input(ratio_cols, ratio_bounds, bounds): def test_rep_profile_idx_map(): - """Test that rep profile index mappings are correct shape. """ + """Test that rep profile index mappings are correct shape.""" h = Hybridization(SOLAR_FPATH, WIND_FPATH, allow_wind_only=True) for h_idxs, r_idxs in (h.meta_hybridizer.solar_profile_indices_map, @@ -262,7 +261,7 @@ def test_rep_profile_idx_map(): def test_limits_values(): - """Test that column values are properly limited on user input. """ + """Test that column values are properly limited on user input.""" limits = {'solar_capacity': 100, 'wind_capacity': 0.5} @@ -274,7 +273,7 @@ def test_limits_values(): def test_invalid_limits_column_name(): - """Test invalid inputs for limits columns. """ + """Test invalid inputs for limits columns.""" test_limits = {'un_prefixed_col': 0, 'wind_capacity': 10} with pytest.raises(InputError) as excinfo: @@ -285,7 +284,7 @@ def test_invalid_limits_column_name(): def test_fillna_values(): - """Test that N/A values are filled properly based on user input. """ + """Test that N/A values are filled properly based on user input.""" fill_vals = {'solar_n_gids': 0, 'wind_capacity': -1} @@ -302,7 +301,7 @@ def test_fillna_values(): def test_invalid_fillna_column_name(): - """Test invalid inputs for fillna columns. """ + """Test invalid inputs for fillna columns.""" test_fillna = {'un_prefixed_col': 0, 'wind_capacity': 10} with pytest.raises(InputError) as excinfo: @@ -318,7 +317,7 @@ def test_invalid_fillna_column_name(): ((False, True), (True, False)), ((True, True), (True, True))]) def test_all_allow_solar_allow_wind_combinations(input_combination, na_vals): - """Test that "allow_x_only" options perform the intended merges. """ + """Test that "allow_x_only" options perform the intended merges.""" allow_solar_only, allow_wind_only = input_combination h = Hybridization( @@ -337,7 +336,7 @@ def test_all_allow_solar_allow_wind_combinations(input_combination, na_vals): def test_warning_for_improper_data_output_from_hybrid_method(): - """Test that hybrid function with incorrect output throws warning. """ + """Test that hybrid function with incorrect output throws warning.""" def some_new_hybrid_func(__): return [0] @@ -355,7 +354,7 @@ def some_new_hybrid_func(__): def test_hybrid_col_additional_method(): - """Test that function decorated with 'hybrid_col' adds to hybrid meta. """ + """Test that function decorated with 'hybrid_col' adds to hybrid meta.""" def some_new_hybrid_func(h): return h.hybrid_meta['elevation'] * 1000 @@ -373,7 +372,7 @@ def some_new_hybrid_func(h): def test_duplicate_lat_long_values(): - """Test duplicate lat/long values corresponding to unique merge column. """ + """Test duplicate lat/long values corresponding to unique merge column.""" with tempfile.TemporaryDirectory() as td: fout_solar = os.path.join(td, 'rep_profiles_solar.h5') @@ -387,7 +386,7 @@ def test_duplicate_lat_long_values(): def test_invalid_ratio_bounds_length_input(): - """Test improper ratios input. """ + """Test improper ratios input.""" ratio = 'solar_capacity/wind_capacity' with pytest.raises(InputError) as excinfo: @@ -401,7 +400,7 @@ def test_invalid_ratio_bounds_length_input(): def test_ratio_column_missing(): - """Test missing ratio column. """ + """Test missing ratio column.""" ratio = 'solar_col_dne/wind_capacity' with pytest.raises(FileInputError) as excinfo: @@ -415,7 +414,7 @@ def test_ratio_column_missing(): @pytest.mark.parametrize("ratio", [None, ('solar_capacity', 'wind_capacity')]) def test_ratio_not_string(ratio): - """Test ratio input is not string. """ + """Test ratio input is not string.""" with pytest.raises(InputError) as excinfo: Hybridization( @@ -432,7 +431,7 @@ def test_ratio_not_string(ratio): 'solar_capacity/wind_capacity/solar_capacity'] ) def test_invalid_ratio_format(ratio): - """Test ratio input is not string. """ + """Test ratio input is not string.""" with pytest.raises(InputError) as excinfo: Hybridization( @@ -446,7 +445,7 @@ def test_invalid_ratio_format(ratio): def test_invalid_ratio_column_name(): - """Test invalid inputs for ratio columns. """ + """Test invalid inputs for ratio columns.""" ratio = 'un_prefixed_col/wind_capacity' with pytest.raises(InputError) as excinfo: @@ -459,7 +458,7 @@ def test_invalid_ratio_column_name(): def test_no_overlap_in_merge_column_values(): - """Test duplicate values in merge column. """ + """Test duplicate values in merge column.""" with tempfile.TemporaryDirectory() as td: fout_solar = os.path.join(td, 'rep_profiles_solar.h5') @@ -474,7 +473,7 @@ def test_no_overlap_in_merge_column_values(): def test_duplicate_merge_column_values(): - """Test duplicate values in merge column. """ + """Test duplicate values in merge column.""" with tempfile.TemporaryDirectory() as td: fout_solar = os.path.join(td, 'rep_profiles_solar.h5') @@ -487,7 +486,7 @@ def test_duplicate_merge_column_values(): def test_merge_columns_missing(): - """Test missing merge column. """ + """Test missing merge column.""" with tempfile.TemporaryDirectory() as td: fout_solar = os.path.join(td, 'rep_profiles_solar.h5') @@ -502,7 +501,7 @@ def test_merge_columns_missing(): def test_invalid_num_profiles(): - """Test input files with an invalid number of profiles (>1). """ + """Test input files with an invalid number of profiles (>1).""" with pytest.raises(FileInputError) as excinfo: Hybridization(SOLAR_FPATH_MULT, WIND_FPATH) @@ -514,7 +513,7 @@ def test_invalid_num_profiles(): def test_invalid_time_index_overlap(): - """Test input files with an invalid time index overlap. """ + """Test input files with an invalid time index overlap.""" with tempfile.TemporaryDirectory() as td: fout_solar = os.path.join(td, 'rep_profiles_solar.h5') @@ -531,7 +530,7 @@ def test_invalid_time_index_overlap(): def test_valid_time_index_overlap(): - """Test input files with a valid time index overlap. """ + """Test input files with a valid time index overlap.""" h = Hybridization(SOLAR_FPATH_30_MIN, WIND_FPATH) @@ -565,7 +564,7 @@ def test_write_to_file(): def test_hybrids_data_content(): - """Test HybridsData class content. """ + """Test HybridsData class content.""" fv = -999 h_data = HybridsData(SOLAR_FPATH, WIND_FPATH) @@ -669,7 +668,7 @@ def test_hybrids_cli_from_config(runner, input_files, ratio, ratio_bounds, os.path.join(TESTDATADIR, 'rep_profiles_out', 'rep_profiles_dne.h5'), ]) def test_hybrids_cli_bad_fpath_input(runner, bad_fpath, clear_loggers): - """Test cli when filepath input is ambiguous or invalid. """ + """Test cli when filepath input is ambiguous or invalid.""" with tempfile.TemporaryDirectory() as td: config = { @@ -740,8 +739,8 @@ def make_test_file(in_fp, out_fp, p_slice=slice(None), t_slice=slice(None), half_n_rows = n_rows // 2 meta.iloc[-half_n_rows:] = meta.iloc[:half_n_rows].values if duplicate_coord_values: - meta.loc[0, 'latitude'] = meta['latitude'].iloc[-1] - meta.loc[0, 'latitude'] = meta['latitude'].iloc[-1] + meta.loc[0, MetaKeyName.LATITUDE] = meta[MetaKeyName.LATITUDE].iloc[-1] + meta.loc[0, MetaKeyName.LATITUDE] = meta[MetaKeyName.LATITUDE].iloc[-1] shapes['meta'] = len(meta) for d in dset_names: shapes[d] = (len(res.time_index[t_slice]), len(meta)) diff --git a/tests/test_losses_power_curve.py b/tests/test_losses_power_curve.py index c62414710..1685e217f 100644 --- a/tests/test_losses_power_curve.py +++ b/tests/test_losses_power_curve.py @@ -7,27 +7,28 @@ @author: ppinchuk """ +import copy +import json import os -import pytest import tempfile -import json -import copy import numpy as np import pandas as pd +import pytest from reV import TESTDATADIR from reV.generation.generation import Gen -from reV.utilities.exceptions import reVLossesValueError, reVLossesWarning -from reV.losses.power_curve import (PowerCurve, PowerCurveLosses, - PowerCurveLossesMixin, - PowerCurveLossesInput, - TRANSFORMATIONS, - HorizontalTranslation, - AbstractPowerCurveTransformation, - ) +from reV.losses.power_curve import ( + TRANSFORMATIONS, + AbstractPowerCurveTransformation, + HorizontalTranslation, + PowerCurve, + PowerCurveLosses, + PowerCurveLossesInput, + PowerCurveLossesMixin, +) from reV.losses.scheduled import ScheduledLossesMixin - +from reV.utilities.exceptions import reVLossesValueError, reVLossesWarning REV_POINTS = list(range(3)) RES_FILE = TESTDATADIR + '/wtk/ri_100_wtk_2012.h5' @@ -55,7 +56,7 @@ def simple_power_curve(): @pytest.fixture def real_power_curve(): """Return a basic power curve.""" - with open(SAM_FILES[0], 'r') as fh: + with open(SAM_FILES[0]) as fh: sam_config = json.load(fh) wind_speed = sam_config['wind_turbine_powercurve_windspeeds'] @@ -67,7 +68,7 @@ def real_power_curve(): @pytest.mark.parametrize('target_losses', [0, 10, 50]) @pytest.mark.parametrize('transformation', TRANSFORMATIONS) def test_power_curve_losses(generic_losses, target_losses, transformation): - """Test full gen run with scheduled losses. """ + """Test full gen run with scheduled losses.""" gen_profiles, gen_profiles_with_losses = _run_gen_with_and_without_losses( generic_losses, target_losses=target_losses, @@ -88,7 +89,7 @@ def test_power_curve_losses(generic_losses, target_losses, transformation): @pytest.mark.parametrize('generic_losses', [0, 0.2]) def test_power_curve_losses_site_specific(generic_losses): - """Test full gen run with scheduled losses. """ + """Test full gen run with scheduled losses.""" gen_profiles, gen_profiles_with_losses = _run_gen_with_and_without_losses( generic_losses, target_losses=10, @@ -110,11 +111,11 @@ def _run_gen_with_and_without_losses( generic_losses, target_losses, transformation, include_outages=False, site_losses=None ): - """Run generation with and without losses for testing. """ + """Run generation with and without losses for testing.""" sam_file = SAM_FILES[0] - with open(sam_file, 'r', encoding='utf-8') as fh: + with open(sam_file, encoding='utf-8') as fh: sam_config = json.load(fh) with tempfile.TemporaryDirectory() as td: @@ -148,7 +149,7 @@ def _run_gen_with_and_without_losses( # undo UTC array rolling for ind, row in gen.meta.iterrows(): - time_shift = row['timezone'] + time_shift = row[MetaKeyName.TIMEZONE] gen_profiles_with_losses[:, ind] = np.roll( gen_profiles_with_losses[:, ind], time_shift ) @@ -167,18 +168,18 @@ def _run_gen_with_and_without_losses( gen_profiles = gen.out['gen_profile'] for ind, row in gen.meta.iterrows(): - time_shift = row['timezone'] + time_shift = row[MetaKeyName.TIMEZONE] gen_profiles[:, ind] = np.roll(gen_profiles[:, ind], time_shift) return gen_profiles, gen_profiles_with_losses def _make_site_data_df(site_data): - """Make site data DataFrame for a specific power curve loss input. """ + """Make site data DataFrame for a specific power curve loss input.""" if site_data is not None: site_specific_losses = [json.dumps(site_data)] * len(REV_POINTS) site_data_dict = { - 'gid': REV_POINTS, + MetaKeyName.GID: REV_POINTS, PowerCurveLossesMixin.POWER_CURVE_CONFIG_KEY: site_specific_losses } site_data = pd.DataFrame(site_data_dict) @@ -186,7 +187,7 @@ def _make_site_data_df(site_data): def test_power_curve_losses_witch_scheduled_outages(): - """Test full gen run with scheduled losses. """ + """Test full gen run with scheduled losses.""" gen_profiles, gen_profiles_with_losses = _run_gen_with_and_without_losses( generic_losses=0.2, target_losses=20, transformation='exponential_stretching', @@ -199,9 +200,9 @@ def test_power_curve_losses_witch_scheduled_outages(): @pytest.mark.parametrize('config', SAM_FILES) def test_power_curve_losses_mixin_class_add_power_curve_losses(config): - """Test mixin class behavior when adding losses. """ + """Test mixin class behavior when adding losses.""" - with open(config, 'r') as fh: + with open(config) as fh: sam_config = json.load(fh) og_power_curve = np.array(sam_config["wind_turbine_powercurve_powerout"]) @@ -236,7 +237,7 @@ def test_power_curve_losses_mixin_class_wind_resource_no_power(config, ws): """Test mixin class behavior when there's no power generation from wind resource always below the cutin speed or above the cutout speed. """ - with open(config, 'r') as fh: + with open(config) as fh: sam_config = json.load(fh) og_power_curve = np.array(sam_config["wind_turbine_powercurve_powerout"]) @@ -268,9 +269,9 @@ def get_item_patch(self, key): @pytest.mark.parametrize('config', SAM_FILES) def test_power_curve_losses_mixin_class_no_losses_input(config): - """Test mixin class behavior when no losses should be added. """ + """Test mixin class behavior when no losses should be added.""" - with open(config, 'r') as fh: + with open(config) as fh: sam_config = json.load(fh) og_power_curve = np.array(sam_config["wind_turbine_powercurve_powerout"]) @@ -288,7 +289,7 @@ def test_power_curve_losses_mixin_class_no_losses_input(config): @pytest.mark.parametrize('bad_wind_speed', ([], [-10, 10])) def test_power_curve_class_bad_wind_speed_input(bad_wind_speed): - """Test that error is raised for bad wind speed inputs. """ + """Test that error is raised for bad wind speed inputs.""" power_curve = [10, 100] with pytest.raises(reVLossesValueError) as excinfo: @@ -298,7 +299,7 @@ def test_power_curve_class_bad_wind_speed_input(bad_wind_speed): @pytest.mark.parametrize('bad_generation', ([], [0, 0, 0, 0], [0, 20, 0, 10])) def test_power_curve_class_bad_generation_input(bad_generation): - """Test that error is raised for bad generation inputs. """ + """Test that error is raised for bad generation inputs.""" wind_speed = [0, 10, 20, 30] with pytest.raises(reVLossesValueError) as excinfo: @@ -308,7 +309,7 @@ def test_power_curve_class_bad_generation_input(bad_generation): @pytest.mark.parametrize('bad_wind_res', ([], [-10, 10])) def test_power_curve_losses_class_bad_wind_res_input(bad_wind_res): - """Test that error is raised for bad wind resource inputs. """ + """Test that error is raised for bad wind resource inputs.""" wind_speed = [0, 10] generation = [10, 100] power_curve = PowerCurve(wind_speed, generation) @@ -319,7 +320,7 @@ def test_power_curve_losses_class_bad_wind_res_input(bad_wind_res): @pytest.mark.parametrize('bad_weights', ([], [1])) def test_power_curve_losses_class_bad_weights_input(bad_weights): - """Test that error is raised for bad weights input. """ + """Test that error is raised for bad weights input.""" wind_speed = [0, 10] generation = [10, 100] wind_res = [0, 0, 5] @@ -331,7 +332,7 @@ def test_power_curve_losses_class_bad_weights_input(bad_weights): @pytest.mark.parametrize('pc_transformation', TRANSFORMATIONS.values()) def test_transformation_classes_apply(pc_transformation, real_power_curve): - """Test that the power curve transformations are applied correctly. """ + """Test that the power curve transformations are applied correctly.""" real_power_curve.generation[-1] = real_power_curve.generation[-2] transformation = pc_transformation(real_power_curve) @@ -375,7 +376,7 @@ def test_horizontal_transformation_class_apply(real_power_curve): def test_power_curve_losses_class_annual_losses_with_transformed_power_curve(): - """Test that the average difference is calculated correctly. """ + """Test that the average difference is calculated correctly.""" wind_speed = [0, 10, 20, 30, 40] generation = [0, 10, 15, 20, 0] @@ -398,9 +399,9 @@ def test_power_curve_losses_class_annual_losses_with_transformed_power_curve(): @pytest.mark.parametrize('sam_file', SAM_FILES) @pytest.mark.parametrize('pc_transformation', TRANSFORMATIONS.values()) def test_transformation_classes_bounds(sam_file, pc_transformation): - """Test that shift_bounds are set correctly. """ + """Test that shift_bounds are set correctly.""" - with open(sam_file, 'r') as fh: + with open(sam_file) as fh: sam_config = json.load(fh) wind_speed = sam_config['wind_turbine_powercurve_windspeeds'] @@ -416,7 +417,7 @@ def test_transformation_classes_bounds(sam_file, pc_transformation): def test_transformation_invalid_result(real_power_curve): - """Test a transformation with invalid result. """ + """Test a transformation with invalid result.""" transformation = HorizontalTranslation(real_power_curve) with pytest.raises(reVLossesValueError) as excinfo: @@ -428,7 +429,7 @@ def test_transformation_invalid_result(real_power_curve): def test_power_curve_loss_input_class_valid_inputs(): - """Test PowerCurveLossesInput class with valid input. """ + """Test PowerCurveLossesInput class with valid input.""" specs = {'target_losses_percent': 50} pc_input = PowerCurveLossesInput(specs) @@ -442,7 +443,7 @@ def test_power_curve_loss_input_class_valid_inputs(): @pytest.mark.parametrize('bad_percent', [-10, 105]) def test_power_curve_loss_input_class_bad_percent_input(bad_percent): - """Test PowerCurveLossesInput class with bad percent input. """ + """Test PowerCurveLossesInput class with bad percent input.""" bad_specs = {'target_losses_percent': bad_percent} @@ -453,7 +454,7 @@ def test_power_curve_loss_input_class_bad_percent_input(bad_percent): def test_power_curve_loss_input_class_bad_transformation_input(): - """Test PowerCurveLossesInput class with bad transformation input. """ + """Test PowerCurveLossesInput class with bad transformation input.""" bad_specs = {'target_losses_percent': 50, 'transformation': 'DNE'} @@ -464,7 +465,7 @@ def test_power_curve_loss_input_class_bad_transformation_input(): def test_power_curve_loss_input_class_missing_required_keys(): - """Test PowerCurveLossesInput class with missing keys input. """ + """Test PowerCurveLossesInput class with missing keys input.""" with pytest.raises(reVLossesValueError) as excinfo: PowerCurveLossesInput({}) @@ -472,9 +473,9 @@ def test_power_curve_loss_input_class_missing_required_keys(): def test_power_curve_loss_invalid_pressure_values(): - """Test mixin class behavior when adding losses. """ + """Test mixin class behavior when adding losses.""" - with open(SAM_FILES[0], 'r') as fh: + with open(SAM_FILES[0]) as fh: sam_config = json.load(fh) # patch required for 'wind_resource_data' access below @@ -498,7 +499,7 @@ def get_item_patch(self, key): def test_power_curve_losses_class_power_gen_no_losses(simple_power_curve): - """Test that power_gen_no_losses is calculated correctly. """ + """Test that power_gen_no_losses is calculated correctly.""" pc_losses = PowerCurveLosses(simple_power_curve, BASIC_WIND_RES) @@ -509,7 +510,7 @@ def test_power_curve_losses_class_power_gen_no_losses(simple_power_curve): def test_power_curve_class_cutoff_wind_speed( simple_power_curve, real_power_curve ): - """Test that cutoff_wind_speed is calculated correctly. """ + """Test that cutoff_wind_speed is calculated correctly.""" assert simple_power_curve.cutoff_wind_speed == np.inf assert ( @@ -523,7 +524,7 @@ def test_power_curve_class_cutoff_wind_speed( def test_power_curve_class_comparisons(simple_power_curve): - """Test power curve class comparison and call operators. """ + """Test power curve class comparison and call operators.""" assert simple_power_curve == [0, 20, 15, 10] assert simple_power_curve != [0, 20, 15, 0] @@ -536,10 +537,11 @@ def test_power_curve_class_comparisons(simple_power_curve): def test_bad_transformation_implementation(real_power_curve): - """Test an invalid transformation implementation. """ + """Test an invalid transformation implementation.""" class NewTransformation(AbstractPowerCurveTransformation): """Test class""" + # pylint: disable=useless-super-delegation def apply(self, *args, **kwargs): """Test apply method.""" diff --git a/tests/test_losses_scheduled.py b/tests/test_losses_scheduled.py index 9171664d9..4f7699303 100644 --- a/tests/test_losses_scheduled.py +++ b/tests/test_losses_scheduled.py @@ -7,29 +7,31 @@ @author: ppinchuk """ -import os -import pytest -import tempfile -import json -import random import copy import glob +import json +import os +import random +import tempfile import traceback import numpy as np import pandas as pd +import pytest +from rex.utilities.utilities import safe_json_load from reV import TESTDATADIR -from reV.generation.generation import Gen -from reV.utilities.exceptions import reVLossesValueError, reVLossesWarning -from reV.losses.utils import hourly_indices_for_months -from reV.losses.scheduled import (Outage, OutageScheduler, - SingleOutageScheduler, ScheduledLossesMixin) from reV.cli import main +from reV.generation.generation import Gen from reV.handlers.outputs import Outputs - -from rex.utilities.utilities import safe_json_load - +from reV.losses.scheduled import ( + Outage, + OutageScheduler, + ScheduledLossesMixin, + SingleOutageScheduler, +) +from reV.losses.utils import hourly_indices_for_months +from reV.utilities.exceptions import reVLossesValueError, reVLossesWarning REV_POINTS = list(range(3)) RTOL = 0 @@ -130,7 +132,7 @@ def so_scheduler(basic_outage_dict): (PV_SAM_FILE, PV_RES_FILE, 'pvwattsv7') ]) def test_scheduled_losses(generic_losses, outages, haf, files): - """Test full gen run with scheduled losses. """ + """Test full gen run with scheduled losses.""" gen_profiles, gen_profiles_with_losses = _run_gen_with_and_without_losses( generic_losses, outages, None, haf, files @@ -235,7 +237,7 @@ def test_scheduled_losses(generic_losses, outages, haf, files): (PV_SAM_FILE, PV_RES_FILE, 'pvwattsv7') ]) def test_scheduled_losses_site_specific(generic_losses, haf, files): - """Test full gen run with scheduled losses. """ + """Test full gen run with scheduled losses.""" gen_profiles, gen_profiles_with_losses = _run_gen_with_and_without_losses( generic_losses, NOMINAL_OUTAGES[0], SINGLE_SITE_OUTAGE, haf, files @@ -316,9 +318,9 @@ def test_scheduled_losses_site_specific(generic_losses, haf, files): def _run_gen_with_and_without_losses( generic_losses, outages, site_outages, haf, files ): - """Run generation with and without losses for testing. """ + """Run generation with and without losses for testing.""" sam_file, res_file, tech = files - with open(sam_file, 'r') as fh: + with open(sam_file) as fh: sam_config = json.load(fh) with tempfile.TemporaryDirectory() as td: @@ -347,7 +349,7 @@ def _run_gen_with_and_without_losses( gen_profiles_with_losses = gen_profiles_with_losses[::time_steps_in_hour] # undo UTC array rolling for ind, row in gen.meta.iterrows(): - time_shift = row['timezone'] + time_shift = row[MetaKeyName.TIMEZONE] gen_profiles_with_losses[:, ind] = np.roll( gen_profiles_with_losses[:, ind], time_shift ) @@ -372,18 +374,18 @@ def _run_gen_with_and_without_losses( time_steps_in_hour = int(round(gen_profiles.shape[0] / 8760)) gen_profiles = gen_profiles[::time_steps_in_hour] for ind, row in gen.meta.iterrows(): - time_shift = row['timezone'] + time_shift = row[MetaKeyName.TIMEZONE] gen_profiles[:, ind] = np.roll(gen_profiles[:, ind], time_shift) return gen_profiles, gen_profiles_with_losses def _make_site_data_df(site_data): - """Make site data DataFrame for a specific outage input. """ + """Make site data DataFrame for a specific outage input.""" if site_data is not None: site_specific_outages = [json.dumps(site_data)] * len(REV_POINTS) site_data_dict = { - 'gid': REV_POINTS, + MetaKeyName.GID: REV_POINTS, ScheduledLossesMixin.OUTAGE_CONFIG_KEY: site_specific_outages } site_data = pd.DataFrame(site_data_dict) @@ -401,9 +403,9 @@ def _make_site_data_df(site_data): def test_scheduled_losses_repeatability( generic_losses, outages, site_outages, files ): - """Test that losses are reproducible between runs. """ + """Test that losses are reproducible between runs.""" sam_file, res_file, tech = files - with open(sam_file, 'r') as fh: + with open(sam_file) as fh: sam_config = json.load(fh) with tempfile.TemporaryDirectory() as td: @@ -447,10 +449,10 @@ def test_scheduled_losses_repeatability( (PV_SAM_FILE, PV_RES_FILE, 'pvwattsv7') ]) def test_scheduled_losses_repeatability_with_seed(files): - """Test that losses are reproducible between runs. """ + """Test that losses are reproducible between runs.""" sam_file, res_file, tech = files outages = copy.deepcopy(NOMINAL_OUTAGES[0]) - with open(sam_file, 'r') as fh: + with open(sam_file) as fh: sam_config = json.load(fh) with tempfile.TemporaryDirectory() as td: @@ -506,7 +508,7 @@ def test_scheduled_losses_repeatability_with_seed(files): @pytest.mark.parametrize('outages', NOMINAL_OUTAGES) def test_scheduled_losses_mixin_class_add_scheduled_losses(outages): - """Test mixin class behavior when adding losses. """ + """Test mixin class behavior when adding losses.""" mixin = ScheduledLossesMixin() mixin.sam_sys_inputs = {mixin.OUTAGE_CONFIG_KEY: outages} @@ -518,7 +520,7 @@ def test_scheduled_losses_mixin_class_add_scheduled_losses(outages): def test_scheduled_losses_mixin_class_no_losses_input(): - """Test mixin class behavior when adding losses. """ + """Test mixin class behavior when adding losses.""" mixin = ScheduledLossesMixin() mixin.sam_sys_inputs = {} @@ -533,7 +535,7 @@ def test_scheduled_losses_mixin_class_no_losses_input(): def test_single_outage_scheduler_normal_run( allow_outage_overlap, so_scheduler ): - """Test that single outage is scheduled correctly. """ + """Test that single outage is scheduled correctly.""" so_scheduler.outage._specs['allow_outage_overlap'] = allow_outage_overlap outage = so_scheduler.outage @@ -562,7 +564,7 @@ def test_single_outage_scheduler_normal_run( def test_single_outage_scheduler_update_when_can_schedule_from_months( so_scheduler ): - """Test that single outage is scheduled correctly. """ + """Test that single outage is scheduled correctly.""" so_scheduler.update_when_can_schedule_from_months() @@ -571,7 +573,7 @@ def test_single_outage_scheduler_update_when_can_schedule_from_months( def test_single_outage_scheduler_update_when_can_schedule(so_scheduler): - """Test that single outage is scheduled correctly. """ + """Test that single outage is scheduled correctly.""" so_scheduler.update_when_can_schedule_from_months() @@ -585,7 +587,7 @@ def test_single_outage_scheduler_update_when_can_schedule(so_scheduler): def test_single_outage_scheduler_find_random_outage_slice(so_scheduler): - """Test single outage class method. """ + """Test single outage class method.""" so_scheduler.update_when_can_schedule_from_months() random_slice = so_scheduler.find_random_outage_slice() @@ -600,7 +602,7 @@ def test_single_outage_scheduler_find_random_outage_slice(so_scheduler): def test_single_outage_scheduler_schedule_losses( allow_outage_overlap, so_scheduler ): - """Test single outage class method. """ + """Test single outage class method.""" so_scheduler.outage._specs['allow_outage_overlap'] = allow_outage_overlap so_scheduler.update_when_can_schedule_from_months() @@ -615,7 +617,7 @@ def test_single_outage_scheduler_schedule_losses( @pytest.mark.parametrize('outages_info', NOMINAL_OUTAGES) def test_outage_scheduler_normal_run(outages_info): - """Test hourly outage losses for a reasonable outage info input. """ + """Test hourly outage losses for a reasonable outage info input.""" outages = [Outage(spec) for spec in outages_info] losses = OutageScheduler(outages).calculate() @@ -641,7 +643,7 @@ def test_outage_scheduler_normal_run(outages_info): def test_outage_scheduler_no_outages(): - """Test hourly outage losses for no outage input. """ + """Test hourly outage losses for no outage input.""" losses = OutageScheduler([]).calculate() @@ -650,7 +652,7 @@ def test_outage_scheduler_no_outages(): def test_outage_scheduler_cannot_schedule_any_more(): - """Test scheduler when little or no outages are allowed. """ + """Test scheduler when little or no outages are allowed.""" outage_info = { 'count': 5, @@ -675,7 +677,7 @@ def test_outage_scheduler_cannot_schedule_any_more(): def test_outage_class_missing_keys(basic_outage_dict): - """Test Outage class behavior for inputs with missing keys. """ + """Test Outage class behavior for inputs with missing keys.""" for key in basic_outage_dict: bad_input = basic_outage_dict.copy() @@ -686,7 +688,7 @@ def test_outage_class_missing_keys(basic_outage_dict): def test_outage_class_count(basic_outage_dict): - """Test Outage class behavior for different count inputs. """ + """Test Outage class behavior for different count inputs.""" basic_outage_dict['count'] = 0 with pytest.raises(reVLossesValueError) as excinfo: @@ -700,7 +702,7 @@ def test_outage_class_count(basic_outage_dict): def test_outage_class_allowed_months(basic_outage_dict): - """Test Outage class behavior for different allowed_month inputs. """ + """Test Outage class behavior for different allowed_month inputs.""" basic_outage_dict['allowed_months'] = [] with pytest.raises(reVLossesValueError) as excinfo: @@ -731,7 +733,7 @@ def test_outage_class_allowed_months(basic_outage_dict): def test_outage_class_duration(basic_outage_dict): - """Test Outage class behavior for different duration inputs. """ + """Test Outage class behavior for different duration inputs.""" err_msg = "Duration of outage must be between 1 and the total available" @@ -755,7 +757,7 @@ def test_outage_class_duration(basic_outage_dict): def test_outage_class_percentage(basic_outage_dict): - """Test Outage class behavior for different percentage inputs. """ + """Test Outage class behavior for different percentage inputs.""" err_msg = "Percentage of farm down during outage must be in the range" @@ -791,9 +793,9 @@ def test_outage_class_allow_outage_overlap(basic_outage_dict): (PV_SAM_FILE, TESTDATADIR + '/nsrdb/ri_100_nsrdb_{}.h5', 'pvwattsv7') ]) def test_scheduled_outages_multi_year(runner, files, clear_loggers): - """Test that scheduled outages are different year to year. """ + """Test that scheduled outages are different year to year.""" sam_file, res_file, tech = files - with open(sam_file, 'r') as fh: + with open(sam_file) as fh: sam_config = json.load(fh) outages = NOMINAL_OUTAGES[0] diff --git a/tests/test_nrwal.py b/tests/test_nrwal.py index 691287f3e..8b92f8c41 100644 --- a/tests/test_nrwal.py +++ b/tests/test_nrwal.py @@ -6,25 +6,22 @@ @author: gbuster """ - -import os import json -import traceback -import numpy as np +import os import shutil -import pytest import tempfile +import traceback +import numpy as np import pandas as pd - +import pytest from rex.utilities.utilities import pd_date_range +from reV import TESTDATADIR from reV.cli import main from reV.handlers.outputs import Outputs from reV.nrwal.nrwal import RevNrwal -from reV.utilities import ModuleName -from reV import TESTDATADIR - +from reV.utilities import MetaKeyName, ModuleName SOURCE_DIR = os.path.join(TESTDATADIR, 'nrwal/') @@ -41,16 +38,17 @@ def test_nrwal(): offshore_config = os.path.join(td, 'offshore.json') onshore_config = os.path.join(td, 'onshore.json') sam_configs = {'onshore': onshore_config, - 'offshore': offshore_config} - nrwal_configs = {'offshore': os.path.join(td, 'nrwal_offshore.yaml')} + MetaKeyName.OFFSHORE: offshore_config} + nrwal_configs = {MetaKeyName.OFFSHORE: + os.path.join(td, 'nrwal_offshore.yaml')} with Outputs(gen_fpath, 'a') as f: f.time_index = pd_date_range('20100101', '20110101', closed='right', freq='1h') - f._add_dset('cf_profile', np.random.random(f.shape), + f._add_dset(MetaKeyName.CF_PROFILE, np.random.random(f.shape), np.uint32, attrs={'scale_factor': 1000}, chunks=(None, 10)) - f._add_dset('fixed_charge_rate', + f._add_dset(MetaKeyName.FIXED_CHARGE_RATE, 0.09 * np.ones(f.shape[1], dtype=np.float32), np.float32, attrs={'scale_factor': 1}, chunks=None) @@ -59,14 +57,15 @@ def test_nrwal(): original_dsets = [d for d in f.dsets if d not in ('meta', 'time_index')] meta_raw = f.meta - lcoe_raw = f['lcoe_fcr'] - cf_mean_raw = f['cf_mean'] - cf_profile_raw = f['cf_profile'] + lcoe_raw = f[MetaKeyName.LCOE_FCR] + cf_mean_raw = f[MetaKeyName.CF_MEAN] + cf_profile_raw = f[MetaKeyName.CF_PROFILE] mask = meta_raw.offshore == 1 - output_request = ['fixed_charge_rate', 'depth', 'total_losses', + output_request = [MetaKeyName.FIXED_CHARGE_RATE, 'depth', + 'total_losses', 'array', 'export', 'gcf_adjustment', - 'lcoe_fcr', 'cf_mean', 'cf_profile'] + MetaKeyName.LCOE_FCR, MetaKeyName.CF_MEAN, ] obj = RevNrwal(gen_fpath, site_data, sam_configs, nrwal_configs, output_request, site_meta_cols=['depth']) @@ -74,14 +73,14 @@ def test_nrwal(): with Outputs(gen_fpath, 'r') as f: meta_new = f.meta - lcoe_new = f['lcoe_fcr'] + lcoe_new = f[MetaKeyName.LCOE_FCR] losses = f['total_losses'] gcf_adjustment = f['gcf_adjustment'] assert np.allclose(cf_mean_raw, f['cf_mean_raw']) assert np.allclose(cf_profile_raw, f['cf_profile_raw']) - cf_mean_new = f['cf_mean'] - cf_profile_new = f['cf_profile'] - fcr = f['fixed_charge_rate'] + cf_mean_new = f[MetaKeyName.CF_MEAN] + cf_profile_new = f[MetaKeyName.CF_PROFILE] + fcr = f[MetaKeyName.FIXED_CHARGE_RATE] depth = f['depth'] for key in [d for d in original_dsets if d in f]: @@ -105,9 +104,9 @@ def test_nrwal(): # make sure the second offshore compute gives same results as first with Outputs(gen_fpath, 'r') as f: - assert np.allclose(lcoe_new, f['lcoe_fcr']) - assert np.allclose(cf_mean_new, f['cf_mean']) - assert np.allclose(cf_profile_new, f['cf_profile']) + assert np.allclose(lcoe_new, f[MetaKeyName.LCOE_FCR]) + assert np.allclose(cf_mean_new, f[MetaKeyName.CF_MEAN]) + assert np.allclose(cf_profile_new, f[MetaKeyName.CF_PROFILE]) assert np.allclose(cf_mean_raw, f['cf_mean_raw']) assert np.allclose(cf_profile_raw, f['cf_profile_raw']) @@ -147,7 +146,7 @@ def test_nrwal(): @pytest.mark.parametrize("out_fn", ["nrwal_meta.csv", None]) def test_nrwal_csv(out_fn): - """Test the reV nrwal class with csv output. """ + """Test the reV nrwal class with csv output.""" with tempfile.TemporaryDirectory() as td: for fn in os.listdir(SOURCE_DIR): shutil.copy(os.path.join(SOURCE_DIR, fn), os.path.join(td, fn)) @@ -157,8 +156,9 @@ def test_nrwal_csv(out_fn): offshore_config = os.path.join(td, 'offshore.json') onshore_config = os.path.join(td, 'onshore.json') sam_configs = {'onshore': onshore_config, - 'offshore': offshore_config} - nrwal_configs = {'offshore': os.path.join(td, 'nrwal_offshore.yaml')} + MetaKeyName.OFFSHORE: offshore_config} + nrwal_configs = {MetaKeyName.OFFSHORE: + os.path.join(td, 'nrwal_offshore.yaml')} with Outputs(gen_fpath, 'a') as f: f.time_index = pd_date_range('20100101', '20110101', @@ -169,15 +169,15 @@ def test_nrwal_csv(out_fn): f._add_dset('cf_mean_raw', np.random.random(f.shape[1]), np.uint32, attrs={'scale_factor': 1000}, chunks=None) - f._add_dset('fixed_charge_rate', + f._add_dset(MetaKeyName.FIXED_CHARGE_RATE, 0.09 * np.ones(f.shape[1], dtype=np.float32), np.float32, attrs={'scale_factor': 1}, chunks=None) compatible = ['depth', 'total_losses', 'array', 'export', - 'gcf_adjustment', 'fixed_charge_rate', 'lcoe_fcr', - 'cf_mean'] - incompatible = ['cf_profile'] + 'gcf_adjustment', MetaKeyName.FIXED_CHARGE_RATE, + MetaKeyName.LCOE_FCR, MetaKeyName.CF_MEAN] + incompatible = [] output_request = compatible + incompatible with pytest.warns(Warning) as record: @@ -217,16 +217,17 @@ def test_nrwal_constant_eq_output_request(): offshore_config = os.path.join(td, 'offshore.json') onshore_config = os.path.join(td, 'onshore.json') sam_configs = {'onshore': onshore_config, - 'offshore': offshore_config} - nrwal_configs = {'offshore': os.path.join(td, 'nrwal_offshore.yaml')} + MetaKeyName.OFFSHORE: offshore_config} + nrwal_configs = {MetaKeyName.OFFSHORE: + os.path.join(td, 'nrwal_offshore.yaml')} with Outputs(gen_fpath, 'a') as f: f.time_index = pd_date_range('20100101', '20110101', closed='right', freq='1h') - f._add_dset('cf_profile', np.random.random(f.shape), + f._add_dset(MetaKeyName.CF_PROFILE, np.random.random(f.shape), np.uint32, attrs={'scale_factor': 1000}, chunks=(None, 10)) - f._add_dset('fixed_charge_rate', + f._add_dset(MetaKeyName.FIXED_CHARGE_RATE, 0.09 * np.ones(f.shape[1], dtype=np.float32), np.float32, attrs={'scale_factor': 1}, chunks=None) @@ -235,7 +236,7 @@ def test_nrwal_constant_eq_output_request(): meta_raw = f.meta mask = meta_raw.offshore == 1 - output_request = ['cf_mean', 'cf_profile', + output_request = [MetaKeyName.CF_MEAN, MetaKeyName.CF_PROFILE, 'lease_price', 'lease_price_mil'] RevNrwal(gen_fpath, site_data, sam_configs, nrwal_configs, @@ -260,16 +261,17 @@ def test_nrwal_cli(runner, clear_loggers): offshore_config = os.path.join(td, 'offshore.json') onshore_config = os.path.join(td, 'onshore.json') sam_configs = {'onshore': onshore_config, - 'offshore': offshore_config} - nrwal_configs = {'offshore': os.path.join(td, 'nrwal_offshore.yaml')} + MetaKeyName.OFFSHORE: offshore_config} + nrwal_configs = {MetaKeyName.OFFSHORE: + os.path.join(td, 'nrwal_offshore.yaml')} with Outputs(gen_fpath, 'a') as f: f.time_index = pd_date_range('20100101', '20110101', closed='right', freq='1h') - f._add_dset('cf_profile', np.random.random(f.shape), + f._add_dset(MetaKeyName.CF_PROFILE, np.random.random(f.shape), np.uint32, attrs={'scale_factor': 1000}, chunks=(None, 10)) - f._add_dset('fixed_charge_rate', + f._add_dset(MetaKeyName.FIXED_CHARGE_RATE, 0.09 * np.ones(f.shape[1], dtype=np.float32), np.float32, attrs={'scale_factor': 1}, chunks=None) @@ -278,14 +280,15 @@ def test_nrwal_cli(runner, clear_loggers): original_dsets = [d for d in f.dsets if d not in ('meta', 'time_index')] meta_raw = f.meta - lcoe_raw = f['lcoe_fcr'] - cf_mean_raw = f['cf_mean'] - cf_profile_raw = f['cf_profile'] + lcoe_raw = f[MetaKeyName.LCOE_FCR] + cf_mean_raw = f[MetaKeyName.CF_MEAN] + cf_profile_raw = f[MetaKeyName.CF_PROFILE] mask = meta_raw.offshore == 1 - output_request = ['fixed_charge_rate', 'depth', 'total_losses', + output_request = [MetaKeyName.FIXED_CHARGE_RATE, 'depth', + 'total_losses', 'array', 'export', 'gcf_adjustment', - 'lcoe_fcr', 'cf_mean', 'cf_profile'] + MetaKeyName.LCOE_FCR, MetaKeyName.CF_MEAN, ] config = { "execution_control": { @@ -314,14 +317,14 @@ def test_nrwal_cli(runner, clear_loggers): with Outputs(gen_fpath, 'r') as f: meta_new = f.meta - lcoe_new = f['lcoe_fcr'] + lcoe_new = f[MetaKeyName.LCOE_FCR] losses = f['total_losses'] gcf_adjustment = f['gcf_adjustment'] assert np.allclose(cf_mean_raw, f['cf_mean_raw']) assert np.allclose(cf_profile_raw, f['cf_profile_raw']) - cf_mean_new = f['cf_mean'] - cf_profile_new = f['cf_profile'] - fcr = f['fixed_charge_rate'] + cf_mean_new = f[MetaKeyName.CF_MEAN] + cf_profile_new = f[MetaKeyName.CF_PROFILE] + fcr = f[MetaKeyName.FIXED_CHARGE_RATE] depth = f['depth'] for key in [d for d in original_dsets if d in f]: @@ -349,9 +352,9 @@ def test_nrwal_cli(runner, clear_loggers): # make sure the second offshore compute gives same results as first with Outputs(gen_fpath, 'r') as f: - assert np.allclose(lcoe_new, f['lcoe_fcr']) - assert np.allclose(cf_mean_new, f['cf_mean']) - assert np.allclose(cf_profile_new, f['cf_profile']) + assert np.allclose(lcoe_new, f[MetaKeyName.LCOE_FCR]) + assert np.allclose(cf_mean_new, f[MetaKeyName.CF_MEAN]) + assert np.allclose(cf_profile_new, f[MetaKeyName.CF_PROFILE]) assert np.allclose(cf_mean_raw, f['cf_mean_raw']) assert np.allclose(cf_profile_raw, f['cf_profile_raw']) @@ -403,8 +406,9 @@ def test_nrwal_cli_csv(runner, clear_loggers): offshore_config = os.path.join(td, 'offshore.json') onshore_config = os.path.join(td, 'onshore.json') sam_configs = {'onshore': onshore_config, - 'offshore': offshore_config} - nrwal_configs = {'offshore': os.path.join(td, 'nrwal_offshore.yaml')} + MetaKeyName.OFFSHORE: offshore_config} + nrwal_configs = {MetaKeyName.OFFSHORE: + os.path.join(td, 'nrwal_offshore.yaml')} with Outputs(gen_fpath, 'a') as f: f.time_index = pd_date_range('20100101', '20110101', @@ -415,14 +419,14 @@ def test_nrwal_cli_csv(runner, clear_loggers): f._add_dset('cf_mean_raw', np.random.random(f.shape[1]), np.uint32, attrs={'scale_factor': 1000}, chunks=None) - f._add_dset('fixed_charge_rate', + f._add_dset(MetaKeyName.FIXED_CHARGE_RATE, 0.09 * np.ones(f.shape[1], dtype=np.float32), np.float32, attrs={'scale_factor': 1}, chunks=None) - output_request = ['fixed_charge_rate', 'depth', 'total_losses', - 'array', 'export', 'gcf_adjustment', - 'lcoe_fcr', 'cf_mean', 'cf_profile'] + output_request = [MetaKeyName.FIXED_CHARGE_RATE, 'depth', + 'total_losses', 'array', 'export', 'gcf_adjustment', + MetaKeyName.LCOE_FCR, MetaKeyName.CF_MEAN, ] config = { "execution_control": { @@ -455,7 +459,7 @@ def test_nrwal_cli_csv(runner, clear_loggers): new_data = pd.read_csv(os.path.join(td, out_fn)) for col in output_request[:-1]: assert col in new_data - assert 'cf_profile' not in new_data + assert MetaKeyName.CF_PROFILE not in new_data clear_loggers() diff --git a/tests/test_qa_qc_summary.py b/tests/test_qa_qc_summary.py index 4fdac1183..3a2a5d8c8 100644 --- a/tests/test_qa_qc_summary.py +++ b/tests/test_qa_qc_summary.py @@ -3,19 +3,22 @@ QA/QC tests """ import os + import pandas as pd -from pandas.testing import assert_frame_equal import pytest +from pandas.testing import assert_frame_equal from reV import TESTDATADIR from reV.qa_qc.summary import SummarizeH5, SummarizeSupplyCurve +from reV.utilities import MetaKeyName H5_FILE = os.path.join(TESTDATADIR, 'gen_out', 'ri_wind_gen_profiles_2010.h5') SC_TABLE = os.path.join(TESTDATADIR, 'sc_out', 'sc_full_out_1.csv') SUMMARY_DIR = os.path.join(TESTDATADIR, 'qa_qc') -@pytest.mark.parametrize('dataset', ['cf_mean', 'cf_profile', None]) +@pytest.mark.parametrize('dataset', + [MetaKeyName.CF_MEAN, MetaKeyName.CF_PROFILE, None]) def test_summarize(dataset): """Run QA/QC Summarize and compare with baseline""" @@ -26,7 +29,7 @@ def test_summarize(dataset): 'ri_wind_gen_profiles_2010_summary.csv') baseline = pd.read_csv(baseline) test = summary.summarize_means() - elif dataset == 'cf_mean': + elif dataset == MetaKeyName.CF_MEAN: baseline = os.path.join(SUMMARY_DIR, 'cf_mean_summary.csv') baseline = pd.read_csv(baseline, index_col=0) test = summary.summarize_dset( diff --git a/tests/test_rep_profiles.py b/tests/test_rep_profiles.py index 7fb8087b4..c4a2029f0 100644 --- a/tests/test_rep_profiles.py +++ b/tests/test_rep_profiles.py @@ -1,20 +1,23 @@ # -*- coding: utf-8 -*- """reVX representative profile tests. """ -import os -import pytest -import pandas as pd -import numpy as np import json +import os import tempfile -from pandas.testing import assert_frame_equal - -from reV.rep_profiles.rep_profiles import (RegionRepProfile, RepProfiles, - RepresentativeMethods) -from reV import TESTDATADIR +import numpy as np +import pandas as pd +import pytest +from pandas.testing import assert_frame_equal from rex.resource import Resource +from reV import TESTDATADIR +from reV.rep_profiles.rep_profiles import ( + RegionRepProfile, + RepProfiles, + RepresentativeMethods, +) +from reV.utilities import MetaKeyName GEN_FPATH = os.path.join(TESTDATADIR, 'gen_out/gen_ri_pv_2012_x000.h5') @@ -22,8 +25,8 @@ def test_rep_region_interval(): """Test the rep profile with a weird interval of gids""" sites = np.arange(40) * 2 - rev_summary = pd.DataFrame({'gen_gids': sites, - 'res_gids': sites}) + rev_summary = pd.DataFrame({MetaKeyName.GEN_GIDS: sites, + MetaKeyName.RES_GIDS: sites}) r = RegionRepProfile(GEN_FPATH, rev_summary, weight=None) assert r.i_reps[0] == 14 @@ -31,8 +34,8 @@ def test_rep_region_interval(): def test_rep_methods(): """Test integrated rep methods against baseline rep profile result""" sites = np.arange(100) - rev_summary = pd.DataFrame({'gen_gids': sites, - 'res_gids': sites}) + rev_summary = pd.DataFrame({MetaKeyName.GEN_GIDS: sites, + MetaKeyName.RES_GIDS: sites}) r = RegionRepProfile(GEN_FPATH, rev_summary, rep_method='meanoid', err_method='rmse', weight=None) @@ -58,8 +61,8 @@ def test_rep_methods(): def test_meanoid(): """Test the simple meanoid method""" sites = np.arange(100) - rev_summary = pd.DataFrame({'gen_gids': sites, - 'res_gids': sites}) + rev_summary = pd.DataFrame({MetaKeyName.GEN_GIDS: sites, + MetaKeyName.RES_GIDS: sites}) r = RegionRepProfile(GEN_FPATH, rev_summary, weight=None) meanoid = RepresentativeMethods.meanoid(r.source_profiles) @@ -74,18 +77,18 @@ def test_weighted_meanoid(): """Test a meanoid weighted by gid_counts vs. a non-weighted meanoid.""" sites = np.arange(100) - rev_summary = pd.DataFrame({'gen_gids': sites, - 'res_gids': sites, - 'gid_counts': [1] * 50 + [0] * 50}) + rev_summary = pd.DataFrame({MetaKeyName.GEN_GIDS: sites, + MetaKeyName.RES_GIDS: sites, + MetaKeyName.GID_COUNTS: [1] * 50 + [0] * 50}) r = RegionRepProfile(GEN_FPATH, rev_summary) - weights = r._get_region_attr(r._rev_summary, 'gid_counts') + weights = r._get_region_attr(r._rev_summary, MetaKeyName.GID_COUNTS) w_meanoid = RepresentativeMethods.meanoid(r.source_profiles, weights=weights) sites = np.arange(50) - rev_summary = pd.DataFrame({'gen_gids': sites, - 'res_gids': sites}) + rev_summary = pd.DataFrame({MetaKeyName.GEN_GIDS: sites, + MetaKeyName.RES_GIDS: sites}) r = RegionRepProfile(GEN_FPATH, rev_summary, weight=None) meanoid = RepresentativeMethods.meanoid(r.source_profiles, weights=None) @@ -101,8 +104,8 @@ def test_integrated(): zeros = np.zeros((100,)) regions = (['r0'] * 7) + (['r1'] * 33) + (['r2'] * 60) timezone = np.random.choice([-4, -5, -6, -7], 100) - rev_summary = pd.DataFrame({'gen_gids': sites, - 'res_gids': sites, + rev_summary = pd.DataFrame({MetaKeyName.GEN_GIDS: sites, + MetaKeyName.RES_GIDS: sites, 'res_class': zeros, 'weight': ones, 'region': regions, @@ -125,12 +128,12 @@ def test_sc_points(): """Test rep profiles for each SC point.""" sites = np.arange(10) timezone = np.random.choice([-4, -5, -6, -7], 10) - rev_summary = pd.DataFrame({'sc_gid': sites, - 'gen_gids': sites, - 'res_gids': sites, + rev_summary = pd.DataFrame({MetaKeyName.SC_GID: sites, + MetaKeyName.GEN_GIDS: sites, + MetaKeyName.RES_GIDS: sites, 'timezone': timezone}) - rp = RepProfiles(GEN_FPATH, rev_summary, 'sc_gid', weight=None) + rp = RepProfiles(GEN_FPATH, rev_summary, MetaKeyName.SC_GID, weight=None) rp.run(max_workers=1) with Resource(GEN_FPATH) as res: @@ -149,28 +152,29 @@ def test_agg_profile(): res_gids = [json.dumps(x) for x in res_gids] gid_counts = [json.dumps(x) for x in gid_counts] timezone = np.random.choice([-4, -5, -6, -7], 4) - rev_summary = pd.DataFrame({'sc_gid': np.arange(4), - 'gen_gids': gen_gids, - 'res_gids': res_gids, - 'gid_counts': gid_counts, - 'timezone': timezone}) - - rp = RepProfiles(GEN_FPATH, rev_summary, 'sc_gid', cf_dset='cf_profile', - err_method=None) + rev_summary = pd.DataFrame({MetaKeyName.SC_GID: np.arange(4), + MetaKeyName.GEN_GIDS: gen_gids, + MetaKeyName.RES_GIDS: res_gids, + MetaKeyName.GID_COUNTS: gid_counts, + MetaKeyName.TIMEZONE: timezone}) + + rp = RepProfiles(GEN_FPATH, rev_summary, MetaKeyName.SC_GID, + cf_dset=MetaKeyName.CF_PROFILE, err_method=None) rp.run(scaled_precision=False, max_workers=1) for index in rev_summary.index: - gen_gids = json.loads(rev_summary.loc[index, 'gen_gids']) - res_gids = json.loads(rev_summary.loc[index, 'res_gids']) - weights = np.array(json.loads(rev_summary.loc[index, 'gid_counts'])) + gen_gids = json.loads(rev_summary.loc[index, MetaKeyName.GEN_GIDS]) + res_gids = json.loads(rev_summary.loc[index, MetaKeyName.RES_GIDS]) + weights = np.array( + json.loads(rev_summary.loc[index, MetaKeyName.GID_COUNTS])) with Resource(GEN_FPATH) as res: meta = res.meta raw_profiles = [] for gid in res_gids: - iloc = np.where(meta['gid'] == gid)[0][0] - prof = np.expand_dims(res['cf_profile', :, iloc], 1) + iloc = np.where(meta[MetaKeyName.GID] == gid)[0][0] + prof = np.expand_dims(res[MetaKeyName.CF_PROFILE, :, iloc], 1) raw_profiles.append(prof) last = raw_profiles[-1].flatten() @@ -186,7 +190,8 @@ def test_agg_profile(): assert np.allclose(rp.profiles[0][:, index], truth) - passthrough_cols = ['gen_gids', 'res_gids', 'gid_counts'] + passthrough_cols = [MetaKeyName.GEN_GIDS, MetaKeyName.RES_GIDS, + MetaKeyName.GID_COUNTS] for col in passthrough_cols: assert col in rp.meta @@ -204,8 +209,8 @@ def test_many_regions(use_weights): region1 = (['r0'] * 7) + (['r1'] * 33) + (['r2'] * 60) region2 = (['a0'] * 20) + (['b1'] * 10) + (['c2'] * 20) + (['d3'] * 50) timezone = np.random.choice([-4, -5, -6, -7], 100) - rev_summary = pd.DataFrame({'gen_gids': sites, - 'res_gids': sites, + rev_summary = pd.DataFrame({MetaKeyName.GEN_GIDS: sites, + MetaKeyName.RES_GIDS: sites, 'res_class': zeros, 'region1': region1, 'region2': region2, @@ -239,13 +244,13 @@ def test_many_regions_with_list_weights(): region1 = (['r0'] * 7) + (['r1'] * 33) + (['r2'] * 60) region2 = (['a0'] * 20) + (['b1'] * 10) + (['c2'] * 20) + (['d3'] * 50) timezone = np.random.choice([-4, -5, -6, -7], 100) - rev_summary = pd.DataFrame({'gen_gids': sites, - 'res_gids': sites, + rev_summary = pd.DataFrame({MetaKeyName.GEN_GIDS: sites, + MetaKeyName.RES_GIDS: sites, 'res_class': zeros, 'region1': region1, 'region2': region2, 'weights': weights, - 'timezone': timezone}) + MetaKeyName.TIMEZONE: timezone}) reg_cols = ['region1', 'region2'] rp = RepProfiles(GEN_FPATH, rev_summary, reg_cols, weight='weights') rp.run() @@ -267,11 +272,11 @@ def test_write_to_file(): zeros = np.zeros((100,)) regions = (['r0'] * 7) + (['r1'] * 33) + (['r2'] * 60) timezone = np.random.choice([-4, -5, -6, -7], 100) - rev_summary = pd.DataFrame({'gen_gids': sites, - 'res_gids': sites, + rev_summary = pd.DataFrame({MetaKeyName.GEN_GIDS: sites, + MetaKeyName.RES_GIDS: sites, 'res_class': zeros, 'region': regions, - 'timezone': timezone}) + MetaKeyName.TIMEZONE: timezone}) fout = os.path.join(td, 'temp_rep_profiles.h5') rp = RepProfiles(GEN_FPATH, rev_summary, 'region', n_profiles=3, weight=None) @@ -300,11 +305,11 @@ def test_file_options(): zeros = np.zeros((100,)) regions = (['r0'] * 7) + (['r1'] * 33) + (['r2'] * 60) timezone = np.random.choice([-4, -5, -6, -7], 100) - rev_summary = pd.DataFrame({'gen_gids': sites, - 'res_gids': sites, + rev_summary = pd.DataFrame({MetaKeyName.GEN_GIDS: sites, + MetaKeyName.RES_GIDS: sites, 'res_class': zeros, 'region': regions, - 'timezone': timezone}) + MetaKeyName.TIMEZONE: timezone}) fout = os.path.join(td, 'temp_rep_profiles.h5') rp = RepProfiles(GEN_FPATH, rev_summary, 'region', n_profiles=3, weight=None) diff --git a/tests/test_sam.py b/tests/test_sam.py index 8ff39d846..6525a4887 100644 --- a/tests/test_sam.py +++ b/tests/test_sam.py @@ -3,23 +3,26 @@ """reV SAM unit test module """ import os -from pkg_resources import get_distribution -from packaging import version -import pytest -import numpy as np import warnings -from reV.SAM.defaults import (DefaultPvWattsv5, DefaultPvWattsv8, - DefaultWindPower) -from reV.SAM.generation import PvWattsv5, PvWattsv7, PvWattsv8 +import numpy as np +import pytest +from packaging import version +from pkg_resources import get_distribution +from rex.renewable_resource import NSRDB +from rex.utilities.utilities import pd_date_range + from reV import TESTDATADIR from reV.config.project_points import ProjectPoints +from reV.SAM.defaults import ( + DefaultPvWattsv5, + DefaultPvWattsv8, + DefaultWindPower, +) +from reV.SAM.generation import PvWattsv5, PvWattsv7, PvWattsv8 from reV.SAM.version_checker import PySamVersionChecker -from reV.utilities.exceptions import PySAMVersionWarning -from reV.utilities.exceptions import InputError - -from rex.renewable_resource import NSRDB -from rex.utilities.utilities import pd_date_range +from reV.utilities import MetaKeyName +from reV.utilities.exceptions import InputError, PySAMVersionWarning @pytest.fixture @@ -94,18 +97,18 @@ def test_PV_lat_tilt(res, site_index): # get SAM inputs from project_points based on the current site site = res_df.name config, inputs = pp[site] - inputs['tilt'] = 'latitude' + inputs['tilt'] = MetaKeyName.LATITUDE # iterate through requested sites. with warnings.catch_warnings(): warnings.simplefilter("ignore") sim = PvWattsv5(resource=res_df, meta=meta, sam_sys_inputs=inputs, - output_request=('cf_mean',)) + output_request=(MetaKeyName.CF_MEAN,)) break else: pass - assert sim.sam_sys_inputs['tilt'] == meta['latitude'] + assert sim.sam_sys_inputs['tilt'] == meta[MetaKeyName.LATITUDE] @pytest.mark.parametrize('dt', ('1h', '30min', '5min')) diff --git a/tests/test_supply_curve_aggregation.py b/tests/test_supply_curve_aggregation.py index e669e6223..7771cbeb5 100644 --- a/tests/test_supply_curve_aggregation.py +++ b/tests/test_supply_curve_aggregation.py @@ -3,21 +3,22 @@ """ Aggregation tests """ -import numpy as np import os -from pandas.testing import assert_frame_equal + +import numpy as np import pytest +from pandas.testing import assert_frame_equal +from rex.resource import Resource -from reV.supply_curve.aggregation import Aggregation from reV import TESTDATADIR - -from rex.resource import Resource +from reV.supply_curve.aggregation import Aggregation +from reV.utilities import MetaKeyName EXCL = os.path.join(TESTDATADIR, 'ri_exclusions/ri_exclusions.h5') RES = os.path.join(TESTDATADIR, 'nsrdb/ri_100_nsrdb_2012.h5') GEN = os.path.join(TESTDATADIR, 'gen_out/ri_wind_gen_profiles_2010.h5') TM_DSET = 'techmap_wtk' -AGG_DSET = ('cf_mean', 'cf_profile') +AGG_DSET = ('cf_mean', ) EXCL_DICT = {'ri_srtm_slope': {'inclusion_range': (None, 5), 'exclude_nodata': True}, @@ -46,7 +47,7 @@ def check_agg(agg_out, baseline_h5): truth = f[dset] if dset == 'meta': truth = truth.set_index('sc_gid') - for c in ['source_gids', 'gid_counts']: + for c in [MetaKeyName.SOURCE_GIDS, MetaKeyName.GID_COUNTS]: test[c] = test[c].astype(str) truth = truth.fillna('none') @@ -74,9 +75,7 @@ def test_aggregation_serial(excl_dict, baseline_name): [(None, 'baseline_agg.h5'), (EXCL_DICT, 'baseline_agg_excl.h5')]) def test_aggregation_parallel(excl_dict, baseline_name): - """ - test aggregation run in parallel - """ + """Test aggregation run in parallel""" baseline_h5 = os.path.join(TESTDATADIR, "sc_out", baseline_name) agg_out = Aggregation.run(EXCL, GEN, TM_DSET, *AGG_DSET, excl_dict=excl_dict, max_workers=None, @@ -86,9 +85,7 @@ def test_aggregation_parallel(excl_dict, baseline_name): @pytest.mark.parametrize('pre_extract_inclusions', (True, False)) def test_pre_extract_inclusions(pre_extract_inclusions): - """ - test aggregation w/ and w/out pre-extracting inclusions - """ + """Test aggregation w/ and w/out pre-extracting inclusions""" baseline_h5 = os.path.join(TESTDATADIR, "sc_out", "baseline_agg_excl.h5") agg_out = Aggregation.run(EXCL, GEN, TM_DSET, *AGG_DSET, excl_dict=EXCL_DICT, @@ -98,16 +95,14 @@ def test_pre_extract_inclusions(pre_extract_inclusions): @pytest.mark.parametrize('excl_dict', [None, EXCL_DICT]) def test_gid_counts(excl_dict): - """ - test counting of exclusion gids during aggregation - """ + """Test counting of exclusion gids during aggregation""" agg_out = Aggregation.run(EXCL, GEN, TM_DSET, *AGG_DSET, excl_dict=excl_dict, max_workers=1) for i, row in agg_out['meta'].iterrows(): - n_gids = row['n_gids'] - gid_counts = np.sum(row['gid_counts']) - area = row['area_sq_km'] + n_gids = row[MetaKeyName.N_GIDS] + gid_counts = np.sum(row[MetaKeyName.GID_COUNTS]) + area = row[MetaKeyName.AREA_SQ_KM] msg = ('For sc_gid {}: the sum of gid_counts ({}), does not match ' 'n_gids ({})'.format(i, n_gids, gid_counts)) @@ -116,9 +111,8 @@ def test_gid_counts(excl_dict): def compute_mean_wind_dirs(res_path, dset, gids, fracs): - """ - Compute mean wind directions for given dset and gids - """ + """Compute mean wind directions for given dset and gids""" + with Resource(res_path) as f: wind_dirs = np.radians(f[dset, :, gids]) @@ -148,8 +142,8 @@ def test_mean_wind_dirs(excl_dict): for i, row in out_meta.iterrows(): test = mean_wind_dirs[:, i] - gids = row['source_gids'] - fracs = row['gid_counts'] / row['n_gids'] + gids = row[MetaKeyName.SOURCE_GIDS] + fracs = row[MetaKeyName.GID_COUNTS] / row[MetaKeyName.N_GIDS] truth = compute_mean_wind_dirs(RES, DSET, gids, fracs) diff --git a/tests/test_supply_curve_aggregation_friction.py b/tests/test_supply_curve_aggregation_friction.py index 49158a742..85f565bf8 100644 --- a/tests/test_supply_curve_aggregation_friction.py +++ b/tests/test_supply_curve_aggregation_friction.py @@ -5,16 +5,18 @@ @author: gbuster """ +import os +import warnings + import h5py import numpy as np import pytest -import os -import warnings -from reV.supply_curve.extent import SupplyCurveExtent +from reV import TESTDATADIR from reV.supply_curve.exclusions import ExclusionMaskFromDict, FrictionMask +from reV.supply_curve.extent import SupplyCurveExtent from reV.supply_curve.sc_aggregation import SupplyCurveAggregation -from reV import TESTDATADIR +from reV.utilities import MetaKeyName EXCL_FPATH = os.path.join(TESTDATADIR, 'ri_exclusions/ri_exclusions.h5') FRICTION_FPATH = os.path.join(TESTDATADIR, 'ri_exclusions/ri_friction.h5') @@ -63,7 +65,7 @@ def test_friction_mask(): assert diff < 0.0001, m -@pytest.mark.parametrize('gid', [100, 114, 130, 181]) +@pytest.mark.parametrize(MetaKeyName.GID, [100, 114, 130, 181]) def test_agg_friction(gid): """Test SC Aggregation with friction by checking friction factors and LCOE against a hand calc.""" @@ -92,18 +94,20 @@ def test_agg_friction(gid): m = ('SC point gid {} does not match mean friction hand calc' .format(gid)) - assert np.isclose(s['mean_friction'].values[0], mean_friction), m + assert np.isclose(s[MetaKeyName.MEAN_FRICTION].values[0], + mean_friction), m m = ('SC point gid {} does not match mean LCOE with friction hand calc' .format(gid)) - assert np.allclose(s['mean_lcoe_friction'], - s['mean_lcoe'] * mean_friction), m + assert np.allclose(s[MetaKeyName.MEAN_LCOE_FRICTION], + s[MetaKeyName.MEAN_LCOE] * mean_friction), m # pylint: disable=no-member def make_friction_file(): """Script to make a test friction file""" - import matplotlib.pyplot as plt import shutil + + import matplotlib.pyplot as plt shutil.copy(EXCL, FRICTION_FPATH) with h5py.File(FRICTION_FPATH, 'a') as f: f[FRICTION_DSET] = f['ri_srtm_slope'] @@ -121,7 +125,8 @@ def make_friction_file(): f[FRICTION_DSET][...] = data for d in f: - if d not in [FRICTION_DSET, 'latitude', 'longitude']: + if d not in [FRICTION_DSET, MetaKeyName.LATITUDE, + MetaKeyName.LONGITUDE]: del f[d] with h5py.File(FRICTION_FPATH, 'r') as f: diff --git a/tests/test_supply_curve_compute.py b/tests/test_supply_curve_compute.py index 6b80f7e54..ce7339d34 100644 --- a/tests/test_supply_curve_compute.py +++ b/tests/test_supply_curve_compute.py @@ -3,15 +3,17 @@ Supply Curve computation integrated tests """ import os -import pandas as pd -from pandas.testing import assert_frame_equal -import pytest +import tempfile import warnings + import numpy as np -import tempfile +import pandas as pd +import pytest +from pandas.testing import assert_frame_equal from reV import TESTDATADIR from reV.supply_curve.supply_curve import SupplyCurve +from reV.utilities import MetaKeyName from reV.utilities.exceptions import SupplyCurveInputError TRANS_COSTS_1 = {'line_tie_in_cost': 200, 'line_cost': 1000, @@ -117,13 +119,13 @@ def test_integrated_sc_full_friction(): transmission_costs=tcosts, avail_cap_frac=avail_cap_frac, columns=SC_FULL_COLUMNS, - sort_on='total_lcoe_friction') + sort_on=MetaKeyName.TOTAL_LCOE_FRICTION) sc_full = pd.read_csv(sc_full) - assert 'mean_lcoe_friction' in sc_full - assert 'total_lcoe_friction' in sc_full - test = sc_full['mean_lcoe_friction'] + sc_full['lcot'] - assert np.allclose(test, sc_full['total_lcoe_friction']) + assert MetaKeyName.MEAN_LCOE_FRICTION in sc_full + assert MetaKeyName.TOTAL_LCOE_FRICTION in sc_full + test = sc_full[MetaKeyName.MEAN_LCOE_FRICTION] + sc_full['lcot'] + assert np.allclose(test, sc_full[MetaKeyName.TOTAL_LCOE_FRICTION]) fpath_baseline = os.path.join(TESTDATADIR, 'sc_out/sc_full_out_friction.csv') @@ -139,12 +141,12 @@ def test_integrated_sc_simple_friction(): out_fpath = os.path.join(td, "sc") sc_simple = sc.run(out_fpath, fixed_charge_rate=0.1, simple=True, transmission_costs=tcosts, - sort_on='total_lcoe_friction') + sort_on=MetaKeyName.TOTAL_LCOE_FRICTION) sc_simple = pd.read_csv(sc_simple) - assert 'mean_lcoe_friction' in sc_simple - assert 'total_lcoe_friction' in sc_simple - test = sc_simple['mean_lcoe_friction'] + sc_simple['lcot'] - assert np.allclose(test, sc_simple['total_lcoe_friction']) + assert MetaKeyName.MEAN_LCOE_FRICTION in sc_simple + assert MetaKeyName.TOTAL_LCOE_FRICTION in sc_simple + test = sc_simple[MetaKeyName.MEAN_LCOE_FRICTION] + sc_simple['lcot'] + assert np.allclose(test, sc_simple[MetaKeyName.TOTAL_LCOE_FRICTION]) fpath_baseline = os.path.join(TESTDATADIR, 'sc_out/sc_simple_out_friction.csv') @@ -153,7 +155,7 @@ def test_integrated_sc_simple_friction(): def test_sc_warning1(): """Run the full SC test with missing connections and verify warning.""" - mask = TRANS_TABLE['sc_point_gid'].isin(list(range(10))) + mask = TRANS_TABLE[MetaKeyName.SC_POINT_GID].isin(list(range(10))) trans_table = TRANS_TABLE[~mask] tcosts = TRANS_COSTS_1.copy() avail_cap_frac = tcosts.pop('available_capacity', 1) @@ -218,7 +220,7 @@ def test_parallel(): assert_frame_equal(sc_full_parallel, sc_full_serial) -def verify_trans_cap(sc_table, trans_tables, cap_col='capacity'): +def verify_trans_cap(sc_table, trans_tables, cap_col=MetaKeyName.CAPACITY): """ Verify that sc_points are connected to features in the correct capacity bins @@ -239,7 +241,7 @@ def verify_trans_cap(sc_table, trans_tables, cap_col='capacity'): test = sc_table.merge(trans_features, on='trans_gid', how='left') mask = test[cap_col] > test['max_cap'] - cols = ['sc_gid', 'trans_gid', cap_col, 'max_cap'] + cols = [MetaKeyName.SC_GID, 'trans_gid', cap_col, 'max_cap'] msg = ("SC points connected to transmission features with " "max_cap < sc_cap:\n{}" .format(test.loc[mask, cols])) @@ -258,7 +260,7 @@ def test_least_cost_full(): out_fpath = os.path.join(td, "sc") sc_full = sc.run(out_fpath, fixed_charge_rate=0.1, simple=False, avail_cap_frac=0.1, - columns=list(SC_FULL_COLUMNS) + ["max_cap"]) + columns=[*list(SC_FULL_COLUMNS), "max_cap"] fpath_baseline = os.path.join(TESTDATADIR, 'sc_out/sc_full_lc.csv') baseline_verify(sc_full, fpath_baseline) @@ -348,32 +350,42 @@ def test_multi_parallel_trans(): sc = SupplyCurve(SC_POINTS, trans_tables) sc_2 = sc.simple_sort(fcr=0.1, columns=columns) - assert not set(SC_POINTS['sc_gid']) - set(sc_1['sc_gid']) - assert not set(SC_POINTS['sc_gid']) - set(sc_2['sc_gid']) - assert not set(SC_POINTS['sc_point_gid']) - set(sc_1['sc_point_gid']) - assert not set(SC_POINTS['sc_point_gid']) - set(sc_2['sc_point_gid']) - assert not set(sc_1['sc_point_gid']) - set(SC_POINTS['sc_point_gid']) - assert not set(sc_2['sc_point_gid']) - set(SC_POINTS['sc_point_gid']) + assert not (set(SC_POINTS[MetaKeyName.SC_GID]) + - set(sc_1[MetaKeyName.SC_GID])) + assert not (set(SC_POINTS[MetaKeyName.SC_GID]) + - set(sc_2[MetaKeyName.SC_GID])) + assert not (set(SC_POINTS[MetaKeyName.SC_POINT_GID]) + - set(sc_1[MetaKeyName.SC_POINT_GID])) + assert not (set(SC_POINTS[MetaKeyName.SC_POINT_GID]) + - set(sc_2[MetaKeyName.SC_POINT_GID])) + assert not (set(sc_1[MetaKeyName.SC_POINT_GID]) + - set(SC_POINTS[MetaKeyName.SC_POINT_GID])) + assert not (set(sc_2[MetaKeyName.SC_POINT_GID]) + - set(SC_POINTS[MetaKeyName.SC_POINT_GID])) assert (sc_2.n_parallel_trans > 1).any() mask_2 = sc_2['n_parallel_trans'] > 1 - for gid in sc_2.loc[mask_2, 'sc_gid']: - nx_1 = sc_1.loc[(sc_1['sc_gid'] == gid), 'n_parallel_trans'].values[0] - nx_2 = sc_2.loc[(sc_2['sc_gid'] == gid), 'n_parallel_trans'].values[0] + for gid in sc_2.loc[mask_2, MetaKeyName.SC_GID]: + nx_1 = sc_1.loc[(sc_1[MetaKeyName.SC_GID] == gid), + 'n_parallel_trans'].values[0] + nx_2 = sc_2.loc[(sc_2[MetaKeyName.SC_GID] == gid), + 'n_parallel_trans'].values[0] assert nx_2 >= nx_1 if nx_1 != nx_2: - lcot_1 = sc_1.loc[(sc_1['sc_gid'] == gid), 'lcot'].values[0] - lcot_2 = sc_2.loc[(sc_2['sc_gid'] == gid), 'lcot'].values[0] + lcot_1 = sc_1.loc[(sc_1[MetaKeyName.SC_GID] == gid), + 'lcot'].values[0] + lcot_2 = sc_2.loc[(sc_2[MetaKeyName.SC_GID] == gid), + 'lcot'].values[0] assert lcot_2 > lcot_1 # pylint: disable=no-member def test_least_cost_full_with_reinforcement(): """ - Test full supply curve sorting with reinforcement costs in the - least-cost path transmission tables + Test full supply curve sorting with reinforcement costs in the least-cost + path transmission tables """ with tempfile.TemporaryDirectory() as td: trans_tables = [] @@ -394,7 +406,8 @@ def test_least_cost_full_with_reinforcement(): columns=list(SC_FULL_COLUMNS) + ["max_cap"]) sc_full = pd.read_csv(sc_full) - fpath_baseline = os.path.join(TESTDATADIR, 'sc_out/sc_full_lc.csv') + fpath_baseline = os.path.join(TESTDATADIR, + 'sc_out/sc_full_lc.csv') baseline_verify(sc_full, fpath_baseline) verify_trans_cap(sc_full, trans_tables) @@ -441,10 +454,12 @@ def test_least_cost_simple_with_reinforcement(): out_fpath = os.path.join(td, "sc") sc = SupplyCurve(SC_POINTS, trans_tables) - sc_simple = sc.run(out_fpath, fixed_charge_rate=0.1, simple=True) + sc_simple = sc.run(out_fpath, fixed_charge_rate=0.1, + simple=True) sc_simple = pd.read_csv(sc_simple) - fpath_baseline = os.path.join(TESTDATADIR, 'sc_out/sc_simple_lc.csv') + fpath_baseline = os.path.join(TESTDATADIR, + 'sc_out/sc_simple_lc.csv') baseline_verify(sc_simple, fpath_baseline) verify_trans_cap(sc_simple, trans_tables) @@ -467,7 +482,8 @@ def test_least_cost_simple_with_reinforcement(): verify_trans_cap(sc_simple_r, trans_tables) assert np.allclose(sc_simple.trans_gid, sc_simple_r.trans_gid) - assert not np.allclose(sc_simple.total_lcoe, sc_simple_r.total_lcoe) + assert not np.allclose(sc_simple.total_lcoe, + sc_simple_r.total_lcoe) def test_least_cost_full_pass_through(): diff --git a/tests/test_supply_curve_points.py b/tests/test_supply_curve_points.py index 1d1f47ea1..a9c718bad 100644 --- a/tests/test_supply_curve_points.py +++ b/tests/test_supply_curve_points.py @@ -6,15 +6,19 @@ """ # pylint: disable=no-member import os + import numpy as np import pytest -from reV.supply_curve.sc_aggregation import SupplyCurveAggregation -from reV.supply_curve.extent import SupplyCurveExtent -from reV.supply_curve.points import (SupplyCurvePoint, - GenerationSupplyCurvePoint) -from reV.handlers.outputs import Outputs from reV import TESTDATADIR +from reV.handlers.outputs import Outputs +from reV.supply_curve.extent import SupplyCurveExtent +from reV.supply_curve.points import ( + GenerationSupplyCurvePoint, + SupplyCurvePoint, +) +from reV.supply_curve.sc_aggregation import SupplyCurveAggregation +from reV.utilities import MetaKeyName F_EXCL = os.path.join(TESTDATADIR, 'ri_exclusions/ri_exclusions.h5') F_GEN = os.path.join(TESTDATADIR, 'gen_out/gen_ri_pv_2012_x000.h5') @@ -58,7 +62,7 @@ def test_slicer(gids, resolution): assert col_slice0 == col_slice1, msg -@pytest.mark.parametrize(('gid', 'resolution', 'excl_dict', 'time_series'), +@pytest.mark.parametrize((MetaKeyName.GID, 'resolution', 'excl_dict', 'time_series'), [(37, 64, None, None), (37, 64, EXCL_DICT, None), (37, 64, None, 100), @@ -96,7 +100,7 @@ def test_weighted_means(gid, resolution, excl_dict, time_series): assert np.allclose(test, means, rtol=RTOL) -@pytest.mark.parametrize(('gid', 'resolution', 'excl_dict', 'time_series'), +@pytest.mark.parametrize((MetaKeyName.GID, 'resolution', 'excl_dict', 'time_series'), [(37, 64, None, None), (37, 64, EXCL_DICT, None), (37, 64, None, 100), @@ -145,11 +149,11 @@ def plot_all_sc_points(resolution=64): colors *= len(sc) for gid in range(len(sc)): excl_meta = sc.get_excl_points('meta', gid) - axs.scatter(excl_meta['longitude'], excl_meta['latitude'], + axs.scatter(excl_meta[MetaKeyName.LONGITUDE], excl_meta[MetaKeyName.LATITUDE], c=colors[gid], s=0.01) with Outputs(F_GEN) as f: - axs.scatter(f.meta['longitude'], f.meta['latitude'], c='k', s=25) + axs.scatter(f.meta[MetaKeyName.LONGITUDE], f.meta[MetaKeyName.LATITUDE], c='k', s=25) axs.axis('equal') plt.show() @@ -175,12 +179,12 @@ def plot_single_gen_sc_point(gid=2, resolution=64): for i, gen_gid in enumerate(all_gen_gids): if gen_gid != -1: mask = (sc._gen_gids == gen_gid) - axs.scatter(excl_meta.loc[mask, 'longitude'], - excl_meta.loc[mask, 'latitude'], + axs.scatter(excl_meta.loc[mask, MetaKeyName.LONGITUDE], + excl_meta.loc[mask, MetaKeyName.LATITUDE], marker='s', c=colors[i], s=1) - axs.scatter(sc.gen.meta.loc[gen_gid, 'longitude'], - sc.gen.meta.loc[gen_gid, 'latitude'], + axs.scatter(sc.gen.meta.loc[gen_gid, MetaKeyName.LONGITUDE], + sc.gen.meta.loc[gen_gid, MetaKeyName.LATITUDE], c='k', s=100) axs.scatter(sc.centroid[1], sc.centroid[0], marker='x', c='k', s=200) diff --git a/tests/test_supply_curve_sc_aggregation.py b/tests/test_supply_curve_sc_aggregation.py index 277843e2e..547e8ebd8 100644 --- a/tests/test_supply_curve_sc_aggregation.py +++ b/tests/test_supply_curve_sc_aggregation.py @@ -7,26 +7,25 @@ """ import json import os +import shutil +import tempfile +import traceback + +import h5py import numpy as np import pandas as pd -from pandas.testing import assert_frame_equal import pytest -import tempfile -import shutil -import h5py -import json -import shutil -import traceback +from pandas.testing import assert_frame_equal +from rex import Outputs, Resource +from reV import TESTDATADIR from reV.cli import main from reV.econ.utilities import lcoe_fcr -from reV.supply_curve.sc_aggregation import (SupplyCurveAggregation, - _warn_about_large_datasets) -from reV.utilities import ModuleName -from reV import TESTDATADIR -from rex import Resource, Outputs -from rex.utilities.loggers import LOGGERS - +from reV.supply_curve.sc_aggregation import ( + SupplyCurveAggregation, + _warn_about_large_datasets, +) +from reV.utilities import MetaKeyName, ModuleName EXCL = os.path.join(TESTDATADIR, 'ri_exclusions/ri_exclusions.h5') RES = os.path.join(TESTDATADIR, 'nsrdb/ri_100_nsrdb_2012.h5') @@ -63,12 +62,12 @@ def test_agg_extent(resolution=64): summary = sca.summarize(GEN) all_res_gids = [] - for gids in summary['res_gids']: + for gids in summary[MetaKeyName.RES_GIDS]: all_res_gids += gids - assert 'sc_col_ind' in summary - assert 'sc_row_ind' in summary - assert 'gen_gids' in summary + assert MetaKeyName.SC_COL_IND in summary + assert MetaKeyName.SC_ROW_IND in summary + assert MetaKeyName.GEN_GIDS in summary assert len(set(all_res_gids)) == 177 @@ -103,7 +102,8 @@ def test_agg_summary(): 'Created: {}'.format(AGG_BASELINE)) else: - for c in ['res_gids', 'gen_gids', 'gid_counts']: + for c in [MetaKeyName.RES_GIDS, MetaKeyName.GEN_GIDS, + MetaKeyName.GID_COUNTS]: summary[c] = summary[c].astype(str) s_baseline = pd.read_csv(AGG_BASELINE, index_col=0) @@ -159,7 +159,7 @@ def test_multi_file_excl(): shutil.copy(EXCL, excl_temp_2) with h5py.File(excl_temp_1, 'a') as f: - shape = f['latitude'].shape + shape = f[MetaKeyName.LATITUDE].shape attrs = dict(f['ri_srtm_slope'].attrs) data = np.ones(shape) test_dset = 'excl_test' @@ -179,7 +179,8 @@ def test_multi_file_excl(): summary = summary.fillna('None') s_baseline = s_baseline.fillna('None') - assert np.allclose(summary['area_sq_km'] * 2, s_baseline['area_sq_km']) + assert np.allclose(summary[MetaKeyName.AREA_SQ_KM] * 2, + s_baseline[MetaKeyName.AREA_SQ_KM]) @pytest.mark.parametrize('pre_extract', (True, False)) @@ -199,7 +200,8 @@ def test_pre_extract_inclusions(pre_extract): 'Created: {}'.format(AGG_BASELINE)) else: - for c in ['res_gids', 'gen_gids', 'gid_counts']: + for c in [MetaKeyName.RES_GIDS, MetaKeyName.GEN_GIDS, + MetaKeyName.GID_COUNTS]: summary[c] = summary[c].astype(str) s_baseline = pd.read_csv(AGG_BASELINE, index_col=0) @@ -244,13 +246,13 @@ def test_agg_extra_dsets(): for dset in h5_dsets: assert 'mean_{}'.format(dset) in summary.columns - check = summary['mean_lcoe_fcr-2012'] == summary['mean_lcoe'] + check = summary['mean_lcoe_fcr-2012'] == summary[MetaKeyName.MEAN_LCOE] assert not any(check) - check = summary['mean_lcoe_fcr-2013'] == summary['mean_lcoe'] + check = summary['mean_lcoe_fcr-2013'] == summary[MetaKeyName.MEAN_LCOE] assert not any(check) avg = (summary['mean_lcoe_fcr-2012'] + summary['mean_lcoe_fcr-2013']) / 2 - assert np.allclose(avg.values, summary['mean_lcoe'].values) + assert np.allclose(avg.values, summary[MetaKeyName.MEAN_LCOE].values) def test_agg_extra_2D_dsets(): @@ -288,7 +290,7 @@ def test_agg_scalar_excl(): data_layers=DATA_LAYERS, gids=gids_subset) summary_with_weights = sca.summarize(GEN, max_workers=1) - dsets = ['area_sq_km', 'capacity'] + dsets = [MetaKeyName.AREA_SQ_KM, MetaKeyName.CAPACITY] for dset in dsets: diff = (summary_base[dset].values / summary_with_weights[dset].values) msg = ('Fractional exclusions failed for {} which has values {} and {}' @@ -297,8 +299,8 @@ def test_agg_scalar_excl(): assert all(diff == 2), msg for i in summary_base.index: - counts_full = summary_base.loc[i, 'gid_counts'] - counts_half = summary_with_weights.loc[i, 'gid_counts'] + counts_full = summary_base.loc[i, MetaKeyName.GID_COUNTS] + counts_half = summary_with_weights.loc[i, MetaKeyName.GID_COUNTS] for j, counts in enumerate(counts_full): msg = ('GID counts for fractional exclusions failed for index {}!' @@ -328,7 +330,7 @@ def test_data_layer_methods(): for i in summary.index.values: # Check categorical data layers - counts = summary.loc[i, 'gid_counts'] + counts = summary.loc[i, MetaKeyName.GID_COUNTS] rr = summary.loc[i, 'reeds_region'] assert isinstance(rr, str) rr = json.loads(rr) @@ -348,7 +350,7 @@ def test_data_layer_methods(): raise RuntimeError(e) # Check min/mean/max of the same data layer - n = summary.loc[i, 'n_gids'] + n = summary.loc[i, MetaKeyName.N_GIDS] slope_mean = summary.loc[i, 'pct_slope_mean'] slope_max = summary.loc[i, 'pct_slope_max'] slope_min = summary.loc[i, 'pct_slope_min'] @@ -364,10 +366,10 @@ def test_recalc_lcoe(cap_cost_scale): """Test supply curve aggregation with the re-calculation of lcoe using the multi-year mean capacity factor""" - data = {'capital_cost': 34900000, - 'fixed_operating_cost': 280000, - 'fixed_charge_rate': 0.09606382995843887, - 'variable_operating_cost': 0, + data = {MetaKeyName.CAPITAL_COST: 34900000, + MetaKeyName.FIXED_OPERATING_COST: 280000, + MetaKeyName.FIXED_CHARGE_RATE: 0.09606382995843887, + MetaKeyName.VARIABLE_OPERATING_COST: 0, 'system_capacity': 20000} annual_cf = [0.24, 0.26, 0.37, 0.15] annual_lcoe = [] @@ -384,11 +386,11 @@ def test_recalc_lcoe(cap_cost_scale): arr = np.full(res['meta'].shape, v) res.create_dataset(k, res['meta'].shape, data=arr) for year, cf in zip(years, annual_cf): - lcoe = lcoe_fcr(data['fixed_charge_rate'], - data['capital_cost'], - data['fixed_operating_cost'], + lcoe = lcoe_fcr(data[MetaKeyName.FIXED_CHARGE_RATE], + data[MetaKeyName.CAPITAL_COST], + data[MetaKeyName.FIXED_OPERATING_COST], data['system_capacity'] * cf * 8760, - data['variable_operating_cost']) + data[MetaKeyName.VARIABLE_OPERATING_COST]) cf_arr = np.full(res['meta'].shape, cf) lcoe_arr = np.full(res['meta'].shape, lcoe) annual_lcoe.append(lcoe) @@ -405,8 +407,10 @@ def test_recalc_lcoe(cap_cost_scale): res.create_dataset('lcoe_fcr-means', res['meta'].shape, data=lcoe_arr) - h5_dsets = ['capital_cost', 'fixed_operating_cost', - 'fixed_charge_rate', 'variable_operating_cost', + h5_dsets = [MetaKeyName.CAPITAL_COST, + MetaKeyName.FIXED_OPERATING_COST, + MetaKeyName.FIXED_CHARGE_RATE, + MetaKeyName.VARIABLE_OPERATING_COST, 'system_capacity'] base = SupplyCurveAggregation(EXCL, TM_DSET, excl_dict=EXCL_DICT, @@ -427,17 +431,19 @@ def test_recalc_lcoe(cap_cost_scale): cap_cost_scale=cap_cost_scale) summary = sca.summarize(gen_temp, max_workers=1) - assert not np.allclose(summary_base['mean_lcoe'], summary['mean_lcoe']) + assert not np.allclose(summary_base[MetaKeyName.MEAN_LCOE], + summary[MetaKeyName.MEAN_LCOE]) if cap_cost_scale == '1': - cc_dset = 'sc_point_capital_cost' + cc_dset = MetaKeyName.SC_POINT_CAPITAL_COST else: - cc_dset = 'scaled_sc_point_capital_cost' - lcoe = lcoe_fcr(summary['mean_fixed_charge_rate'], summary[cc_dset], - summary['sc_point_fixed_operating_cost'], - summary['sc_point_annual_energy'], - summary['mean_variable_operating_cost']) - assert np.allclose(lcoe, summary['mean_lcoe']) + cc_dset = MetaKeyName.SCALED_SC_POINT_CAPITAL_COST + lcoe = lcoe_fcr(summary[MetaKeyName.MEAN_FIXED_CHARGE_RATE], + summary[cc_dset], + summary[MetaKeyName.SC_POINT_FIXED_OPERATING_COST], + summary[MetaKeyName.SC_POINT_ANNUAL_ENERGY], + summary[MetaKeyName.MEAN_VARIABLE_OPERATING_COST]) + assert np.allclose(lcoe, summary[MetaKeyName.MEAN_LCOE]) @pytest.mark.parametrize('tm_dset', ("techmap_ri", "techmap_ri_new")) diff --git a/tests/test_supply_curve_tech_mapping.py b/tests/test_supply_curve_tech_mapping.py index 034b86780..8c2d5d431 100644 --- a/tests/test_supply_curve_tech_mapping.py +++ b/tests/test_supply_curve_tech_mapping.py @@ -4,11 +4,12 @@ @author: gbuster """ +import os + import h5py import numpy as np import pandas as pd import pytest -import os from reV import TESTDATADIR from reV.handlers.exclusions import ExclusionLayers @@ -43,8 +44,8 @@ def plot_tech_mapping(dist_margin=1.05): import matplotlib.pyplot as plt with h5py.File(EXCL, 'r') as f: - lats = f['latitude'][...].flatten() - lons = f['longitude'][...].flatten() + lats = f[MetaKeyName.LATITUDE][...].flatten() + lons = f[MetaKeyName.LONGITUDE][...].flatten() ind_truth = f[TM_DSET][...].flatten() with Outputs(GEN) as fgen: @@ -53,8 +54,8 @@ def plot_tech_mapping(dist_margin=1.05): ind_test = TechMapping.run(EXCL, RES, dset=None, max_workers=2, dist_margin=dist_margin) - df = pd.DataFrame({'latitude': lats, - 'longitude': lons, + df = pd.DataFrame({MetaKeyName.LATITUDE: lats, + MetaKeyName.LONGITUDE: lons, TM_DSET: ind_truth, 'test': ind_test.flatten()}) @@ -65,31 +66,31 @@ def plot_tech_mapping(dist_margin=1.05): for i, ind in enumerate(df[TM_DSET].unique()): if ind != -1: mask = df[TM_DSET] == ind - axs.scatter(df.loc[mask, 'longitude'], - df.loc[mask, 'latitude'], + axs.scatter(df.loc[mask, MetaKeyName.LONGITUDE], + df.loc[mask, MetaKeyName.LATITUDE], c=colors[i], s=0.001) elif ind == -1: mask = df[TM_DSET] == ind - axs.scatter(df.loc[mask, 'longitude'], - df.loc[mask, 'latitude'], + axs.scatter(df.loc[mask, MetaKeyName.LONGITUDE], + df.loc[mask, MetaKeyName.LATITUDE], c='r', s=0.001) for ind in df[TM_DSET].unique(): if ind != -1: - axs.scatter(gen_meta.loc[ind, 'longitude'], - gen_meta.loc[ind, 'latitude'], + axs.scatter(gen_meta.loc[ind, MetaKeyName.LONGITUDE], + gen_meta.loc[ind, MetaKeyName.LATITUDE], c='w', s=1) for ind in df['test'].unique(): if ind != -1: - axs.scatter(gen_meta.loc[ind, 'longitude'], - gen_meta.loc[ind, 'latitude'], + axs.scatter(gen_meta.loc[ind, MetaKeyName.LONGITUDE], + gen_meta.loc[ind, MetaKeyName.LATITUDE], c='r', s=1) mask = df[TM_DSET].values != df['test'] - axs.scatter(df.loc[mask, 'longitude'], - df.loc[mask, 'latitude'], + axs.scatter(df.loc[mask, MetaKeyName.LONGITUDE], + df.loc[mask, MetaKeyName.LATITUDE], c='k', s=1) axs.axis('equal') diff --git a/tests/test_supply_curve_vpd.py b/tests/test_supply_curve_vpd.py index ae2b48389..43a018fb2 100644 --- a/tests/test_supply_curve_vpd.py +++ b/tests/test_supply_curve_vpd.py @@ -3,13 +3,15 @@ Test Variable Power Density @author: gbuster """ -import pandas as pd +import os + import numpy as np +import pandas as pd import pytest -import os -from reV.supply_curve.sc_aggregation import SupplyCurveAggregation from reV import TESTDATADIR +from reV.supply_curve.sc_aggregation import SupplyCurveAggregation +from reV.utilities import MetaKeyName from reV.utilities.exceptions import FileInputError EXCL = os.path.join(TESTDATADIR, 'ri_exclusions/ri_exclusions.h5') @@ -41,10 +43,10 @@ def test_vpd(): vpd = pd.read_csv(FVPD, index_col=0) for i in summary.index: - capacity = summary.loc[i, 'capacity'] - area = summary.loc[i, 'area_sq_km'] + capacity = summary.loc[i, MetaKeyName.CAPACITY] + area = summary.loc[i, MetaKeyName.AREA_SQ_KM] res_gids = np.array(summary.loc[i, 'res_gids']) - gid_counts = np.array(summary.loc[i, 'gid_counts']) + gid_counts = np.array(summary.loc[i, MetaKeyName.GID_COUNTS]) vpd_per_gid = vpd.loc[res_gids, 'power_density'].values truth = area * (vpd_per_gid * gid_counts).sum() / gid_counts.sum() @@ -77,8 +79,8 @@ def test_vpd_fractional_excl(): summary_2 = sca_2.summarize(GEN, max_workers=1) for i in summary_1.index: - cap_full = summary_1.loc[i, 'capacity'] - cap_half = summary_2.loc[i, 'capacity'] + cap_full = summary_1.loc[i, MetaKeyName.CAPACITY] + cap_half = summary_2.loc[i, MetaKeyName.CAPACITY] msg = ('Variable power density for fractional exclusions failed! ' 'Index {} has cap full {} and cap half {}' diff --git a/tests/test_supply_curve_wind_dirs.py b/tests/test_supply_curve_wind_dirs.py index 766bbfbc6..049791651 100644 --- a/tests/test_supply_curve_wind_dirs.py +++ b/tests/test_supply_curve_wind_dirs.py @@ -3,12 +3,14 @@ Supply Curve computation integrated tests """ import os + import pandas as pd -from pandas.testing import assert_frame_equal import pytest +from pandas.testing import assert_frame_equal from reV import TESTDATADIR from reV.supply_curve.supply_curve import CompetitiveWindFarms, SupplyCurve +from reV.utilities import MetaKeyName TRANS_COSTS = {'line_tie_in_cost': 200, 'line_cost': 1000, 'station_tie_in_cost': 50, 'center_tie_in_cost': 10, @@ -28,7 +30,8 @@ def test_competitive_wind_dirs(downwind): """Run CompetitiveWindFarms and verify results against baseline file.""" sc_points = CompetitiveWindFarms.run(WIND_DIRS, SC_POINTS, - n_dirs=2, sort_on='mean_lcoe', + n_dirs=2, + sort_on=MetaKeyName.MEAN_LCOE, downwind=downwind) if downwind: @@ -105,11 +108,11 @@ def test_upwind_exclusion(): 'sc_full_upwind.csv') sc_out = pd.read_csv(sc_out).sort_values('total_lcoe') - sc_point_gids = sc_out['sc_point_gid'].values.tolist() + sc_point_gids = sc_out[MetaKeyName.SC_POINT_GID].values.tolist() for _, row in sc_out.iterrows(): - sc_gid = row['sc_gid'] - sc_point_gids.remove(row['sc_point_gid']) - sc_point_gid = cwf['sc_point_gid', sc_gid] + sc_gid = row[MetaKeyName.SC_GID] + sc_point_gids.remove(row[MetaKeyName.SC_POINT_GID]) + sc_point_gid = cwf[MetaKeyName.SC_POINT_GID, sc_gid] for gid in cwf['upwind', sc_point_gid]: msg = 'Upwind gid {} was not excluded!'.format(gid) assert gid not in sc_point_gids, msg @@ -125,11 +128,11 @@ def test_upwind_downwind_exclusion(): 'sc_full_downwind.csv') sc_out = pd.read_csv(sc_out).sort_values('total_lcoe') - sc_point_gids = sc_out['sc_point_gid'].values.tolist() + sc_point_gids = sc_out[MetaKeyName.SC_POINT_GID].values.tolist() for _, row in sc_out.iterrows(): - sc_gid = row['sc_gid'] - sc_point_gids.remove(row['sc_point_gid']) - sc_point_gid = cwf['sc_point_gid', sc_gid] + sc_gid = row[MetaKeyName.SC_GID] + sc_point_gids.remove(row[MetaKeyName.SC_POINT_GID]) + sc_point_gid = cwf[MetaKeyName.SC_POINT_GID, sc_gid] for gid in cwf['upwind', sc_point_gid]: msg = 'Upwind gid {} was not excluded!'.format(gid) assert gid not in sc_point_gids, msg From 2b6b919680f28d95f1d80b84299485b4c0f3621e Mon Sep 17 00:00:00 2001 From: bnb32 Date: Sun, 12 May 2024 06:08:26 -0600 Subject: [PATCH 03/61] fix: bad replacements --- examples/bespoke_wind_plants/single_run.py | 7 +- reV/SAM/SAM.py | 44 ++- reV/generation/generation.py | 16 +- reV/handlers/multi_year.py | 4 +- reV/rep_profiles/rep_profiles.py | 40 +-- reV/supply_curve/extent.py | 116 ++++--- reV/supply_curve/points.py | 49 ++- tests/eagle.py | 15 +- tests/test_bespoke.py | 3 +- tests/test_config.py | 256 ++++++++------ tests/test_curtailment.py | 12 +- tests/test_econ_of_scale.py | 4 +- tests/test_econ_windbos.py | 317 ++++++++++------- tests/test_gen_5min.py | 2 +- tests/test_gen_csp.py | 2 +- tests/test_gen_forecast.py | 2 +- tests/test_gen_geothermal.py | 16 +- tests/test_gen_linear.py | 2 +- tests/test_gen_pv.py | 45 +-- tests/test_gen_swh.py | 4 +- tests/test_gen_time_scale.py | 8 +- tests/test_gen_trough.py | 2 +- tests/test_gen_wave.py | 6 +- tests/test_gen_wind.py | 386 +++++++++++++-------- tests/test_gen_wind_losses.py | 6 +- tests/test_handlers_multiyear.py | 30 +- tests/test_nrwal.py | 42 +-- tests/test_qa_qc_summary.py | 5 +- tests/test_rep_profiles.py | 4 +- tests/test_sam.py | 5 +- tests/test_supply_curve_compute.py | 2 +- tests/test_supply_curve_points.py | 189 ++++++---- tests/test_supply_curve_tech_mapping.py | 1 + tests/test_supply_curve_wind_dirs.py | 2 +- 34 files changed, 964 insertions(+), 680 deletions(-) diff --git a/examples/bespoke_wind_plants/single_run.py b/examples/bespoke_wind_plants/single_run.py index 4a205999f..ff53a2faf 100644 --- a/examples/bespoke_wind_plants/single_run.py +++ b/examples/bespoke_wind_plants/single_run.py @@ -18,13 +18,12 @@ plot_windrose, ) from reV.supply_curve.tech_mapping import TechMapping -from reV.utilities import MetaKeyName SAM = os.path.join(TESTDATADIR, 'SAM/i_windpower.json') EXCL = os.path.join(TESTDATADIR, 'ri_exclusions/ri_exclusions.h5') RES = os.path.join(TESTDATADIR, 'wtk/ri_100_wtk_{}.h5') TM_DSET = 'techmap_wtk_ri_100' -AGG_DSET = (MetaKeyName.CF_MEAN, MetaKeyName.CF_PROFILE) +AGG_DSET = ('cf_mean', 'cf_profile') # note that this differs from the EXCL_DICT = {'ri_srtm_slope': {'inclusion_range': (None, 5), @@ -61,8 +60,8 @@ 1E5 * 0.1 + (1 - 0.1))""" objective_function = "cost / aep" - output_request = ('system_capacity', MetaKeyName.CF_MEAN, - MetaKeyName.CF_PROFILE) + output_request = ('system_capacity', 'cf_mean', + 'cf_profile') gid = 33 with tempfile.TemporaryDirectory() as td: excl_fp = os.path.join(td, 'ri_exclusions.h5') diff --git a/reV/SAM/SAM.py b/reV/SAM/SAM.py index 927af2d05..2714d856e 100644 --- a/reV/SAM/SAM.py +++ b/reV/SAM/SAM.py @@ -27,7 +27,6 @@ ) from rex.utilities.utilities import check_res_file -from reV.utilities import MetaKeyName from reV.utilities.exceptions import ( ResourceError, SAMExecutionError, @@ -376,19 +375,18 @@ def __setitem__(self, key, value): .format(key, self.pysam)) logger.exception(msg) raise SAMInputError(msg) - else: - self.sam_sys_inputs[key] = value - group = self._get_group(key, outputs=False) - try: - setattr(getattr(self.pysam, group), key, value) - except Exception as e: - msg = ('Could not set input key "{}" to ' - 'group "{}" in "{}".\n' - 'Data is: {} ({})\n' - 'Received the following error: "{}"' - .format(key, group, self.pysam, value, type(value), e)) - logger.exception(msg) - raise SAMInputError(msg) from e + self.sam_sys_inputs[key] = value + group = self._get_group(key, outputs=False) + try: + setattr(getattr(self.pysam, group), key, value) + except Exception as e: + msg = ('Could not set input key "{}" to ' + 'group "{}" in "{}".\n' + 'Data is: {} ({})\n' + 'Received the following error: "{}"' + .format(key, group, self.pysam, value, type(value), e)) + logger.exception(msg) + raise SAMInputError(msg) from e @property def pysam(self): @@ -739,7 +737,7 @@ def get_time_interval(cls, time_index): if t == 1.0: time_interval += 1 break - elif t == 0.0: + if t == 0.0: time_interval += 1 return int(time_interval) @@ -797,22 +795,20 @@ def _is_arr_like(val): """Returns true if SAM data is array-like. False if scalar.""" if isinstance(val, (int, float, str)): return False + try: + len(val) + except TypeError: + return False else: - try: - len(val) - except TypeError: - return False - else: - return True + return True @classmethod def _is_hourly(cls, val): """Returns true if SAM data is hourly or sub-hourly. False otherise.""" if not cls._is_arr_like(val): return False - else: - L = len(val) - return L >= 8760 + L = len(val) + return L >= 8760 def outputs_to_utc_arr(self): """Convert array-like SAM outputs to UTC np.ndarrays""" diff --git a/reV/generation/generation.py b/reV/generation/generation.py index b26ac0c89..91dbe7462 100644 --- a/reV/generation/generation.py +++ b/reV/generation/generation.py @@ -82,7 +82,7 @@ class Gen(BaseGen): def __init__(self, technology, project_points, sam_files, resource_file, low_res_resource_file=None, - output_request=(MetaKeyName.CF_MEAN,), + output_request=('cf_mean',), site_data=None, curtailment=None, gid_map=None, drop_leap=False, sites_per_worker=None, memory_utilization_limit=0.4, scale_outputs=True, @@ -123,10 +123,10 @@ def __init__(self, technology, project_points, sam_files, resource_file, >>> gen.run() >>> >>> gen.out - {MetaKeyName.CF_MEAN: array([0.16966143], dtype=float32)} + {'cf_mean': array([0.16966143], dtype=float32)} >>> >>> sites = [3, 4, 7, 9] - >>> req = (MetaKeyName.CF_MEAN, , MetaKeyName.LCOE_FCR) + >>> req = ('cf_mean', , MetaKeyName.LCOE_FCR) >>> gen = Gen(sam_tech, sites, fp_sam, fp_res, output_request=req) >>> gen.run() >>> @@ -283,7 +283,7 @@ def __init__(self, technology, project_points, sam_files, resource_file, aggregation/supply curve step if the ``"dc_ac_ratio"`` dataset is detected in the generation file. - By default, ``(MetaKeyName.CF_MEAN,)``. + By default, ``('cf_mean',)``. site_data : str | pd.DataFrame, optional Site-specific input data for SAM calculation. If this input is a string, it should be a path that points to a CSV file. @@ -547,7 +547,7 @@ def handle_lifetime_index(self, ti): var for var, attrs in GEN_ATTRS.items() if attrs['type'] == 'array' ] - valid_vars = [MetaKeyName.GEN_PROFILE, MetaKeyName.CF_PROFILE, + valid_vars = [MetaKeyName.GEN_PROFILE, 'cf_profile', 'cf_profile_ac'] invalid_vars = set(array_vars) - set(valid_vars) invalid_requests = [var for var in self.output_request @@ -836,7 +836,7 @@ def _parse_bc(bias_correct): if isinstance(bias_correct, type(None)): return bias_correct - elif isinstance(bias_correct, str): + if isinstance(bias_correct, str): bias_correct = pd.read_csv(bias_correct) msg = ('Bias correction data must be a filepath to csv or a dataframe ' @@ -874,8 +874,8 @@ def _parse_output_request(self, req): output_request = self._output_request_type_check(req) # ensure that cf_mean is requested from output - if MetaKeyName.CF_MEAN not in output_request: - output_request.append(MetaKeyName.CF_MEAN) + if 'cf_mean' not in output_request: + output_request.append('cf_mean') for request in output_request: if request not in self.OUT_ATTRS: diff --git a/reV/handlers/multi_year.py b/reV/handlers/multi_year.py index dd2d3c9b2..e8d3e52d5 100644 --- a/reV/handlers/multi_year.py +++ b/reV/handlers/multi_year.py @@ -34,7 +34,7 @@ class MultiYearGroup: def __init__(self, name, out_dir, source_files=None, source_dir=None, source_prefix=None, source_pattern=None, - dsets=(MetaKeyName.CF_MEAN,), pass_through_dsets=None): + dsets=('cf_mean',), pass_through_dsets=None): """ Parameters ---------- @@ -61,7 +61,7 @@ def __init__(self, name, out_dir, source_files=None, `source_prefix` but is not used if `source_files` are specified explicitly. By default, ``None``. dsets : list | tuple, optional - List of datasets to collect. By default, ``(MetaKeyName.CF_MEAN,)``. + List of datasets to collect. By default, ``('cf_mean',)``. pass_through_dsets : list | tuple, optional Optional list of datasets that are identical in the multi-year files (e.g. input datasets that don't vary from diff --git a/reV/rep_profiles/rep_profiles.py b/reV/rep_profiles/rep_profiles.py index 5e0195e86..c13103dd5 100644 --- a/reV/rep_profiles/rep_profiles.py +++ b/reV/rep_profiles/rep_profiles.py @@ -302,7 +302,7 @@ class RegionRepProfile: RES_GID_COL = MetaKeyName.RES_GIDS GEN_GID_COL = MetaKeyName.GEN_GIDS - def __init__(self, gen_fpath, rev_summary, cf_dset=MetaKeyName.CF_PROFILE, + def __init__(self, gen_fpath, rev_summary, cf_dset='cf_profile', rep_method='meanoid', err_method='rmse', weight=MetaKeyName.GID_COUNTS, n_profiles=1): @@ -442,7 +442,7 @@ def _get_region_attr(rev_summary, attr_name): if any(data): if isinstance(data[0], str): # pylint: disable=simplifiable-condition - if ('[' and ']' in data[0]) or ('(' and ')' in data[0 + if ('[' and ']' in data[0]) or ('(' and ')' in data[0]): data = [json.loads(s) for s in data] if isinstance(data[0], (list, tuple)): @@ -474,7 +474,7 @@ def _run_rep_methods(self): rep_method=self._rep_method, err_method=self._err_method, n_profiles=self._n_profiles) - @property + @ property def rep_profiles(self): """Get the representative profiles of this region.""" if self._profiles is None: @@ -482,7 +482,7 @@ def rep_profiles(self): return self._profiles - @property + @ property def i_reps(self): """Get the representative profile index(es) of this region.""" if self._i_reps is None: @@ -490,7 +490,7 @@ def i_reps(self): return self._i_reps - @property + @ property def rep_gen_gids(self): """Get the representative profile gen gids of this region.""" gids = self._gen_gids @@ -501,7 +501,7 @@ def rep_gen_gids(self): return rep_gids - @property + @ property def rep_res_gids(self): """Get the representative profile resource gids of this region.""" gids = self._res_gids @@ -512,9 +512,9 @@ def rep_res_gids(self): return rep_gids - @classmethod + @ classmethod def get_region_rep_profile(cls, gen_fpath, rev_summary, - cf_dset=MetaKeyName.CF_PROFILE, + cf_dset='cf_profile', rep_method='meanoid', err_method='rmse', weight=MetaKeyName.GID_COUNTS, @@ -568,7 +568,7 @@ class RepProfilesBase(ABC): """Abstract utility framework for representative profile run classes.""" def __init__(self, gen_fpath, rev_summary, reg_cols=None, - cf_dset=MetaKeyName.CF_PROFILE, rep_method='meanoid', + cf_dset='cf_profile', rep_method='meanoid', err_method='rmse', weight=MetaKeyName.GID_COUNTS, n_profiles=1): """ @@ -634,7 +634,7 @@ def __init__(self, gen_fpath, rev_summary, reg_cols=None, self._rep_method = rep_method self._err_method = err_method - @staticmethod + @ staticmethod def _parse_rev_summary(rev_summary): """Extract, parse, and check the rev summary table. @@ -670,7 +670,7 @@ def _parse_rev_summary(rev_summary): return rev_summary - @staticmethod + @ staticmethod def _check_req_cols(df, cols): """Check a dataframe for required columns. @@ -693,7 +693,7 @@ def _check_req_cols(df, cols): logger.error(e) raise KeyError(e) - @staticmethod + @ staticmethod def _check_rev_gen(gen_fpath, cf_dset, rev_summary): """Check rev gen file for requisite datasets. @@ -736,7 +736,7 @@ def _init_profiles(self): dtype=np.float32) for k in range(self._n_profiles)} - @property + @ property def time_index(self): """Get the time index for the rep profiles. @@ -756,7 +756,7 @@ def time_index(self): return self._time_index - @property + @ property def meta(self): """Meta data for the representative profiles. @@ -768,7 +768,7 @@ def meta(self): """ return self._meta - @property + @ property def profiles(self): """Get the arrays of representative CF profiles corresponding to meta. @@ -866,15 +866,15 @@ def save_profiles(self, fout, save_rev_summary=True, scaled_precision=scaled_precision) self._write_h5_out(fout, save_rev_summary=save_rev_summary) - @abstractmethod + @ abstractmethod def _run_serial(self): """Abstract method for serial run method.""" - @abstractmethod + @ abstractmethod def _run_parallel(self): """Abstract method for parallel run method.""" - @abstractmethod + @ abstractmethod def run(self): """Abstract method for generic run method.""" @@ -883,7 +883,7 @@ class RepProfiles(RepProfilesBase): """RepProfiles""" def __init__(self, gen_fpath, rev_summary, reg_cols, - cf_dset=MetaKeyName.CF_PROFILE, + cf_dset='cf_profile', rep_method='meanoid', err_method='rmse', weight=MetaKeyName.GID_COUNTS, n_profiles=1, aggregate_profiles=False): @@ -1001,7 +1001,7 @@ def __init__(self, gen_fpath, rev_summary, reg_cols, 'key such as "sc_gid".') logger.error(e) raise ValueError(e) - elif isinstance(reg_cols, str): + if isinstance(reg_cols, str): reg_cols = [reg_cols] elif not isinstance(reg_cols, list): reg_cols = list(reg_cols) diff --git a/reV/supply_curve/extent.py b/reV/supply_curve/extent.py index b63e25cac..19ffac8b1 100644 --- a/reV/supply_curve/extent.py +++ b/reV/supply_curve/extent.py @@ -2,6 +2,7 @@ """ reV supply curve extent """ + import logging import numpy as np @@ -32,13 +33,17 @@ def __init__(self, f_excl, resolution=64): SC point. """ - logger.debug('Initializing SupplyCurveExtent with res {} from: {}' - .format(resolution, f_excl)) + logger.debug( + "Initializing SupplyCurveExtent with res {} from: {}".format( + resolution, f_excl + ) + ) if not isinstance(resolution, int): - raise SupplyCurveInputError('Supply Curve resolution needs to be ' - 'an integer but received: {}' - .format(type(resolution))) + raise SupplyCurveInputError( + "Supply Curve resolution needs to be " + "an integer but received: {}".format(type(resolution)) + ) if isinstance(f_excl, (str, list, tuple)): self._excl_fpath = f_excl @@ -47,11 +52,12 @@ def __init__(self, f_excl, resolution=64): self._excl_fpath = f_excl.h5_file self._excls = f_excl else: - raise SupplyCurveInputError('SupplyCurvePoints needs an ' - 'exclusions file path, or ' - 'ExclusionLayers handler but ' - 'received: {}' - .format(type(f_excl))) + raise SupplyCurveInputError( + "SupplyCurvePoints needs an " + "exclusions file path, or " + "ExclusionLayers handler but " + "received: {}".format(type(f_excl)) + ) self._excl_shape = self.exclusions.shape # limit the resolution to the exclusion shape. @@ -68,13 +74,15 @@ def __init__(self, f_excl, resolution=64): self._points = None self._sc_col_ind, self._sc_row_ind = np.meshgrid( - np.arange(self.n_cols), np.arange(self.n_rows)) + np.arange(self.n_cols), np.arange(self.n_rows) + ) self._sc_col_ind = self._sc_col_ind.flatten() self._sc_row_ind = self._sc_row_ind.flatten() - logger.debug('Initialized SupplyCurveExtent with shape {} from ' - 'exclusions with shape {}' - .format(self.shape, self.excl_shape)) + logger.debug( + "Initialized SupplyCurveExtent with shape {} from " + "exclusions with shape {}".format(self.shape, self.excl_shape) + ) def __len__(self): """Total number of supply curve points.""" @@ -91,8 +99,10 @@ def __exit__(self, type, value, traceback): def __getitem__(self, gid): """Get SC extent meta data corresponding to an SC point gid.""" if gid >= len(self): - raise KeyError('SC extent with {} points does not contain SC ' - 'point gid {}.'.format(len(self), gid)) + raise KeyError( + "SC extent with {} points does not contain SC " + "point gid {}.".format(len(self), gid) + ) return self.points.loc[gid] @@ -181,8 +191,9 @@ def rows_of_excl(self): point. """ if self._rows_of_excl is None: - self._rows_of_excl = self._chunk_excl(self.excl_rows, - self.resolution) + self._rows_of_excl = self._chunk_excl( + self.excl_rows, self.resolution + ) return self._rows_of_excl @@ -199,8 +210,9 @@ def cols_of_excl(self): point. """ if self._cols_of_excl is None: - self._cols_of_excl = self._chunk_excl(self.excl_cols, - self.resolution) + self._cols_of_excl = self._chunk_excl( + self.excl_cols, self.resolution + ) return self._cols_of_excl @@ -218,8 +230,9 @@ def excl_row_slices(self): point. """ if self._excl_row_slices is None: - self._excl_row_slices = self._excl_slices(self.excl_rows, - self.resolution) + self._excl_row_slices = self._excl_slices( + self.excl_rows, self.resolution + ) return self._excl_row_slices @@ -237,8 +250,9 @@ def excl_col_slices(self): point. """ if self._excl_col_slices is None: - self._excl_col_slices = self._excl_slices(self.excl_cols, - self.resolution) + self._excl_col_slices = self._excl_slices( + self.excl_cols, self.resolution + ) return self._excl_col_slices @@ -283,16 +297,19 @@ def latitude(self): lats = [] lons = [] - sc_cols, sc_rows = np.meshgrid(np.arange(self.n_cols), - np.arange(self.n_rows)) + sc_cols, sc_rows = np.meshgrid( + np.arange(self.n_cols), np.arange(self.n_rows) + ) for r, c in zip(sc_rows.flatten(), sc_cols.flatten()): r = self.excl_row_slices[r] c = self.excl_col_slices[c] lats.append(self.exclusions[MetaKeyName.LATITUDE, r, c].mean()) - lons.append(self.exclusions[MetaKeyName.LONGITUDE, r, c].mean()) + lons.append( + self.exclusions[MetaKeyName.LONGITUDE, r, c].mean() + ) - self._latitude = np.array(lats, dtype='float32') - self._longitude = np.array(lons, dtype='float32') + self._latitude = np.array(lats, dtype="float32") + self._longitude = np.array(lons, dtype="float32") return self._latitude @@ -309,16 +326,19 @@ def longitude(self): lats = [] lons = [] - sc_cols, sc_rows = np.meshgrid(np.arange(self.n_cols), - np.arange(self.n_rows)) + sc_cols, sc_rows = np.meshgrid( + np.arange(self.n_cols), np.arange(self.n_rows) + ) for r, c in zip(sc_rows.flatten(), sc_cols.flatten()): r = self.excl_row_slices[r] c = self.excl_col_slices[c] lats.append(self.exclusions[MetaKeyName.LATITUDE, r, c].mean()) - lons.append(self.exclusions[MetaKeyName.LONGITUDE, r, c].mean()) + lons.append( + self.exclusions[MetaKeyName.LONGITUDE, r, c].mean() + ) - self._latitude = np.array(lats, dtype='float32') - self._longitude = np.array(lons, dtype='float32') + self._latitude = np.array(lats, dtype="float32") + self._longitude = np.array(lons, dtype="float32") return self._longitude @@ -368,8 +388,12 @@ def points(self): """ if self._points is None: - self._points = pd.DataFrame({'row_ind': self.row_indices.copy(), - 'col_ind': self.col_indices.copy()}) + self._points = pd.DataFrame( + { + "row_ind": self.row_indices.copy(), + "col_ind": self.col_indices.copy(), + } + ) self._points.index.name = MetaKeyName.GID # sc_point_gid @@ -437,8 +461,8 @@ def get_sc_row_col_ind(self, gid): col_ind : int Column index that the gid is located at in the sc grid. """ - row_ind = self.points.loc[gid, 'row_ind'] - col_ind = self.points.loc[gid, 'col_ind'] + row_ind = self.points.loc[gid, "row_ind"] + col_ind = self.points.loc[gid, "col_ind"] return row_ind, col_ind def get_excl_slices(self, gid): @@ -459,9 +483,10 @@ def get_excl_slices(self, gid): """ if gid >= len(self): - raise SupplyCurveError('Requested gid "{}" is out of bounds for ' - 'supply curve points with length "{}".' - .format(gid, len(self))) + raise SupplyCurveError( + 'Requested gid "{}" is out of bounds for ' + 'supply curve points with length "{}".'.format(gid, len(self)) + ) row_slice = self.excl_row_slices[self.row_indices[gid]] col_slice = self.excl_col_slices[self.col_indices[gid]] @@ -560,9 +585,12 @@ def valid_sc_points(self, tm_dset): valid_gids = np.where(valid_bool == 1)[0].astype(np.uint32) - logger.info('Found {} valid SC points out of {} total possible ' - '(valid SC points that map to valid resource gids)' - .format(len(valid_gids), len(valid_bool))) + logger.info( + "Found {} valid SC points out of {} total possible " + "(valid SC points that map to valid resource gids)".format( + len(valid_gids), len(valid_bool) + ) + ) return valid_gids diff --git a/reV/supply_curve/points.py b/reV/supply_curve/points.py index 2210d379b..395ca6b7a 100644 --- a/reV/supply_curve/points.py +++ b/reV/supply_curve/points.py @@ -546,7 +546,7 @@ def exclusion_weighted_mean(self, arr, drop_nan=True): if np.isnan(x).all(): return np.nan - elif drop_nan and np.isnan(x).any(): + if drop_nan and np.isnan(x).any(): nan_mask = np.isnan(x) x = x[~nan_mask] incl = incl[~nan_mask] @@ -723,9 +723,8 @@ def _mode(data): """ if not data.size: return None - else: - # pd series is more flexible with non-numeric than stats mode - return pd.Series(data).mode().values[0] + # pd series is more flexible with non-numeric than stats mode + return pd.Series(data).mode().values[0] @staticmethod def _categorize(data, incl_mult): @@ -1512,7 +1511,7 @@ def res_data(self): if isinstance(self._res_class_dset, np.ndarray): return self._res_class_dset - elif self._res_data is None: + if self._res_data is None: if self._res_class_dset in self.gen.datasets: self._res_data = self.gen[self._res_class_dset] @@ -1532,7 +1531,7 @@ def gen_data(self): if isinstance(self._cf_dset, np.ndarray): return self._cf_dset - elif self._gen_data is None: + if self._gen_data is None: if self._cf_dset in self.gen.datasets: self._gen_data = self.gen[self._cf_dset] @@ -1552,7 +1551,7 @@ def lcoe_data(self): if isinstance(self._lcoe_dset, np.ndarray): return self._lcoe_dset - elif self._lcoe_data is None: + if self._lcoe_data is None: if self._lcoe_dset in self.gen.datasets: self._lcoe_data = self.gen[self._lcoe_dset] @@ -1597,8 +1596,8 @@ def mean_lcoe(self): 'fixed_operating_cost', 'variable_operating_cost', 'system_capacity') - if (self.mean_h5_dsets_data is not None and - all(k in self.mean_h5_dsets_data for k in required)): + if (self.mean_h5_dsets_data is not None + and all(k in self.mean_h5_dsets_data for k in required)): aep = (self.mean_h5_dsets_data['system_capacity'] * self.mean_cf * 8760) # Note the AEP computation uses the SAM config @@ -2021,22 +2020,22 @@ def point_summary(self, args=None): """ ARGS = { - MetaKeyName.LATITUDE: self.sc_point.latitude, - MetaKeyName.LONGITUDE: self.sc_point.longitude, - MetaKeyName.TIMEZONE: self.sc_point.timezone, - MetaKeyName.COUNTRY: self.sc_point.country, - MetaKeyName.STATE: self.sc_point.state, - MetaKeyName.COUNTY: self.sc_point.county, - MetaKeyName.ELEVATION: self.sc_point.elevation, - MetaKeyName.RES_GIDS: self.res_gid_set, - MetaKeyName.GEN_GIDS: self.gen_gid_set, - MetaKeyName.GID_COUNTS: self.gid_counts, - MetaKeyName.N_GIDS: self.sc_point.n_gids, - MetaKeyName.MEAN_CF: self.mean_cf, - MetaKeyName.MEAN_LCOE: self.mean_lcoe, - MetaKeyName.MEAN_RES: self.mean_res, - MetaKeyName.CAPACITY: self.capacity, - MetaKeyName.AREA_SQ_KM: self.sc_point.area} + MetaKeyName.LATITUDE: self.sc_point.latitude, + MetaKeyName.LONGITUDE: self.sc_point.longitude, + MetaKeyName.TIMEZONE: self.sc_point.timezone, + MetaKeyName.COUNTRY: self.sc_point.country, + MetaKeyName.STATE: self.sc_point.state, + MetaKeyName.COUNTY: self.sc_point.county, + MetaKeyName.ELEVATION: self.sc_point.elevation, + MetaKeyName.RES_GIDS: self.res_gid_set, + MetaKeyName.GEN_GIDS: self.gen_gid_set, + MetaKeyName.GID_COUNTS: self.gid_counts, + MetaKeyName.N_GIDS: self.sc_point.n_gids, + MetaKeyName.MEAN_CF: self.mean_cf, + MetaKeyName.MEAN_LCOE: self.mean_lcoe, + MetaKeyName.MEAN_RES: self.mean_res, + MetaKeyName.CAPACITY: self.capacity, + MetaKeyName.AREA_SQ_KM: self.sc_point.area} extra_atts = [MetaKeyName.CAPACITY_AC, MetaKeyName.OFFSHORE, diff --git a/tests/eagle.py b/tests/eagle.py index a0c56e56c..10de238ae 100644 --- a/tests/eagle.py +++ b/tests/eagle.py @@ -54,7 +54,7 @@ def years(self): def get_cf_mean(self, site, year): """Get a cf mean based on site and year""" iy = self.years.index(year) - out = self._h5['pv'][MetaKeyName.CF_MEAN][iy, site] + out = self._h5['pv']['cf_mean'][iy, site] return out @@ -70,10 +70,10 @@ def is_num(n): def to_list(gen_out): """Generation output handler that converts to the rev 1.0 format.""" if isinstance(gen_out, list) and len(gen_out) == 1: - out = [c[MetaKeyName.CF_MEAN] for c in gen_out[0].values()] + out = [c['cf_mean'] for c in gen_out[0].values()] if isinstance(gen_out, dict): - out = [c[MetaKeyName.CF_MEAN] for c in gen_out.values()] + out = [c['cf_mean'] for c in gen_out.values()] return out @@ -104,7 +104,7 @@ def test_eagle(year): sam_files=sam_files, res_file=res_file, sites_per_worker=None, max_workers=None, fout=rev2_out, dirout=rev2_out_dir, logdir=rev2_out_dir, - output_request=(, MetaKeyName.CF_MEAN), + output_request=('cf_profile', 'cf_mean'), verbose=verbose) # create and submit the SLURM job @@ -115,8 +115,7 @@ def test_eagle(year): status = slurm.check_status(name, var='name') if status == 'CG': break - else: - time.sleep(5) + time.sleep(5) # get reV 2.0 generation profiles from disk flist = os.listdir(rev2_out_dir) @@ -125,7 +124,7 @@ def test_eagle(year): if rev2_out.strip('.h5') in fname: full_f = os.path.join(rev2_out_dir, fname) with Outputs(full_f, 'r') as cf: - rev2_profiles = cf[] + rev2_profiles = cf['cf_profile'] break # get reV 1.0 generation profiles @@ -147,7 +146,7 @@ def get_r1_profiles(year=2012): rev1 = os.path.join(TESTDATADIR, 'ri_pv', 'profile_outputs', 'pv_{}_0.h5'.format(year)) with Outputs(rev1) as cf: - data = cf[][...] / 10000 + data = cf['cf_profile'][...] / 10000 return data diff --git a/tests/test_bespoke.py b/tests/test_bespoke.py index 9bc15ccbe..e5ec31215 100644 --- a/tests/test_bespoke.py +++ b/tests/test_bespoke.py @@ -74,8 +74,7 @@ def test_turbine_placement(gid=33): """Test turbine placement with zero available area.""" - output_request = ('system_capacity'ame.CF_MEAN, - 'cf_profile') + output_request = ('system_capacity', 'cf_mean', 'cf_profile') with tempfile.TemporaryDirectory() as td: res_fp = os.path.join(td, 'ri_100_wtk_{}.h5') excl_fp = os.path.join(td, 'ri_exclusions.h5') diff --git a/tests/test_config.py b/tests/test_config.py index 73c045067..09610db6a 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -7,6 +7,7 @@ @author: gbuster """ + import os import tempfile @@ -22,6 +23,7 @@ from reV.config.project_points import PointsControl, ProjectPoints from reV.generation.generation import Gen from reV.SAM.SAM import RevPySam +from reV.utilities import MetaKeyName from reV.utilities.exceptions import ConfigError @@ -29,7 +31,7 @@ def test_config_entries(): """ Test BaseConfig check_entry test """ - config_path = os.path.join(TESTDATADIR, 'config/collection.json') + config_path = os.path.join(TESTDATADIR, "config/collection.json") with pytest.raises(ConfigError): AnalysisConfig(config_path) @@ -38,25 +40,27 @@ def test_clearsky(): """ Test Clearsky """ - res_file = os.path.join(TESTDATADIR, 'nsrdb/', 'ri_100_nsrdb_2012.h5') - sam_files = os.path.join(TESTDATADIR, 'SAM/naris_pv_1axis_inv13_cs.json') - pp = ProjectPoints(slice(0, 10), sam_files, 'pvwattsv5', - res_file=res_file) + res_file = os.path.join(TESTDATADIR, "nsrdb/", "ri_100_nsrdb_2012.h5") + sam_files = os.path.join(TESTDATADIR, "SAM/naris_pv_1axis_inv13_cs.json") + pp = ProjectPoints(slice(0, 10), sam_files, "pvwattsv5", res_file=res_file) with pytest.raises(ResourceRuntimeError): # Get the SAM resource object RevPySam.get_sam_res(res_file, pp, pp.tech) -@pytest.mark.parametrize(('start', 'interval'), - [[0, 1], [13, 1], [10, 2], [13, 3]]) +@pytest.mark.parametrize( + ("start", "interval"), [[0, 1], [13, 1], [10, 2], [13, 3]] +) def test_proj_control_iter(start, interval): """Test the iteration of the points control.""" n = 3 - res_file = os.path.join(TESTDATADIR, 'wtk/ri_100_wtk_2012.h5') - sam_files = os.path.join(TESTDATADIR, - 'SAM/wind_gen_standard_losses_0.json') - pp = ProjectPoints(slice(start, 100, interval), sam_files, 'windpower', - res_file=res_file) + res_file = os.path.join(TESTDATADIR, "wtk/ri_100_wtk_2012.h5") + sam_files = os.path.join( + TESTDATADIR, "SAM/wind_gen_standard_losses_0.json" + ) + pp = ProjectPoints( + slice(start, 100, interval), sam_files, "windpower", res_file=res_file + ) pc = PointsControl(pp, sites_per_split=n) for i, pp_split in enumerate(pc): @@ -64,19 +68,22 @@ def test_proj_control_iter(start, interval): i1_nom = i * n + n split = pp_split.project_points.df target = pp.df.iloc[i0_nom:i1_nom, :] - msg = 'PointsControl iterator split did not function correctly!' + msg = "PointsControl iterator split did not function correctly!" assert all(split == target), msg -@pytest.mark.parametrize(('start', 'interval'), - [[0, 1], [13, 1], [10, 2], [13, 3]]) +@pytest.mark.parametrize( + ("start", "interval"), [[0, 1], [13, 1], [10, 2], [13, 3]] +) def test_proj_points_split(start, interval): """Test the split operation of project points.""" - res_file = os.path.join(TESTDATADIR, 'wtk/ri_100_wtk_2012.h5') - sam_files = os.path.join(TESTDATADIR, - 'SAM/wind_gen_standard_losses_0.json') - pp = ProjectPoints(slice(start, 100, interval), sam_files, 'windpower', - res_file=res_file) + res_file = os.path.join(TESTDATADIR, "wtk/ri_100_wtk_2012.h5") + sam_files = os.path.join( + TESTDATADIR, "SAM/wind_gen_standard_losses_0.json" + ) + pp = ProjectPoints( + slice(start, 100, interval), sam_files, "windpower", res_file=res_file + ) iter_interval = 5 for i0 in range(0, len(pp), iter_interval): @@ -86,18 +93,20 @@ def test_proj_points_split(start, interval): pp_0 = ProjectPoints.split(i0, i1, pp) - msg = 'ProjectPoints split did not function correctly!' + msg = "ProjectPoints split did not function correctly!" assert pp_0.sites == pp.sites[i0:i1], msg assert all(pp_0.df == pp.df.iloc[i0:i1]), msg def test_split_iter(): """Test Points_Control on two slices of ProjectPoints""" - res_file = os.path.join(TESTDATADIR, 'wtk/ri_100_wtk_2012.h5') - sam_files = os.path.join(TESTDATADIR, - 'SAM/wind_gen_standard_losses_0.json') - pp = ProjectPoints(slice(0, 500, 5), sam_files, 'windpower', - res_file=res_file) + res_file = os.path.join(TESTDATADIR, "wtk/ri_100_wtk_2012.h5") + sam_files = os.path.join( + TESTDATADIR, "SAM/wind_gen_standard_losses_0.json" + ) + pp = ProjectPoints( + slice(0, 500, 5), sam_files, "windpower", res_file=res_file + ) n = 3 for s, e in [(0, 50), (50, 100)]: @@ -106,25 +115,28 @@ def test_split_iter(): for i, pp_split in enumerate(pc): i0_nom = s + i * n i1_nom = s + i * n + n - if i1_nom >= e: - i1_nom = e + i1_nom = min(e, i1_nom) split = pp_split.project_points.df target = pp.df.iloc[i0_nom:i1_nom] - msg = 'PointsControl iterator split did not function correctly!' + msg = "PointsControl iterator split did not function correctly!" assert split.equals(target), msg def test_config_mapping(): """Test the mapping of multiple configs in the project points.""" - fpp = os.path.join(TESTDATADIR, 'project_points/pp_offshore.csv') - sam_files = {'onshore': os.path.join( - TESTDATADIR, 'SAM/wind_gen_standard_losses_0.json'), - MetaKeyName.OFFSHORE: os.path.join( - TESTDATADIR, 'SAM/wind_gen_standard_losses_1.json')} + fpp = os.path.join(TESTDATADIR, "project_points/pp_offshore.csv") + sam_files = { + "onshore": os.path.join( + TESTDATADIR, "SAM/wind_gen_standard_losses_0.json" + ), + MetaKeyName.OFFSHORE: os.path.join( + TESTDATADIR, "SAM/wind_gen_standard_losses_1.json" + ), + } df = pd.read_csv(fpp, index_col=0) - pp = ProjectPoints(fpp, sam_files, 'windpower') + pp = ProjectPoints(fpp, sam_files, "windpower") pc = PointsControl(pp, sites_per_split=100) for i, pc_split in enumerate(pc): for site in pc_split.sites: @@ -136,69 +148,78 @@ def test_sam_config_kw_replace(): """Test that the SAM config with old keys from pysam v1 gets updated on the fly and gets propogated to downstream splits.""" - fpp = os.path.join(TESTDATADIR, 'project_points/pp_offshore.csv') - sam_files = {'onshore': os.path.join( - TESTDATADIR, 'SAM/wind_gen_standard_losses_0.json'), - MetaKeyName.OFFSHORE: os.path.join( - TESTDATADIR, 'SAM/wind_gen_standard_losses_1.json')} - res_file = os.path.join(TESTDATADIR, 'wtk/ri_100_wtk_2012.h5') - pp = ProjectPoints(fpp, sam_files, 'windpower') - - gen = Gen('windpower', pp, sam_files, resource_file=res_file, - sites_per_worker=100) - config_on = gen.project_points.sam_inputs['onshore'] + fpp = os.path.join(TESTDATADIR, "project_points/pp_offshore.csv") + sam_files = { + "onshore": os.path.join( + TESTDATADIR, "SAM/wind_gen_standard_losses_0.json" + ), + MetaKeyName.OFFSHORE: os.path.join( + TESTDATADIR, "SAM/wind_gen_standard_losses_1.json" + ), + } + res_file = os.path.join(TESTDATADIR, "wtk/ri_100_wtk_2012.h5") + pp = ProjectPoints(fpp, sam_files, "windpower") + + gen = Gen( + "windpower", + pp, + sam_files, + resource_file=res_file, + sites_per_worker=100, + ) + config_on = gen.project_points.sam_inputs["onshore"] config_of = gen.project_points.sam_inputs[MetaKeyName.OFFSHORE] - assert 'turb_generic_loss' in config_on - assert 'turb_generic_loss' in config_of + assert "turb_generic_loss" in config_on + assert "turb_generic_loss" in config_of pp_split = ProjectPoints.split(0, 10000, gen.project_points) - config_on = pp_split.sam_inputs['onshore'] + config_on = pp_split.sam_inputs["onshore"] config_of = pp_split.sam_inputs[MetaKeyName.OFFSHORE] - assert 'turb_generic_loss' in config_on - assert 'turb_generic_loss' in config_of + assert "turb_generic_loss" in config_on + assert "turb_generic_loss" in config_of pc_split = PointsControl.split(0, 10000, gen.project_points) - config_on = pc_split.project_points.sam_inputs['onshore'] + config_on = pc_split.project_points.sam_inputs["onshore"] config_of = pc_split.project_points.sam_inputs[MetaKeyName.OFFSHORE] - assert 'turb_generic_loss' in config_on - assert 'turb_generic_loss' in config_of + assert "turb_generic_loss" in config_on + assert "turb_generic_loss" in config_of for ipc in pc_split: - if 'onshore' in ipc.project_points.sam_inputs: - config = ipc.project_points.sam_inputs['onshore'] - assert 'turb_generic_loss' in config + if "onshore" in ipc.project_points.sam_inputs: + config = ipc.project_points.sam_inputs["onshore"] + assert "turb_generic_loss" in config if MetaKeyName.OFFSHORE in ipc.project_points.sam_inputs: config = ipc.project_points.sam_inputs[MetaKeyName.OFFSHORE] - assert 'turb_generic_loss' in config + assert "turb_generic_loss" in config -@pytest.mark.parametrize('counties', [['Washington'], ['Providence', 'Kent']]) +@pytest.mark.parametrize("counties", [["Washington"], ["Providence", "Kent"]]) def test_regions(counties): """ Test ProjectPoint.regions class method """ - res_file = os.path.join(TESTDATADIR, 'nsrdb/', 'ri_100_nsrdb_2012.h5') - sam_files = os.path.join(TESTDATADIR, 'SAM/naris_pv_1axis_inv13_cs.json') + res_file = os.path.join(TESTDATADIR, "nsrdb/", "ri_100_nsrdb_2012.h5") + sam_files = os.path.join(TESTDATADIR, "SAM/naris_pv_1axis_inv13_cs.json") with Resource(res_file) as f: meta = f.meta - baseline = meta.loc[meta['county'].isin(counties)].index.values.tolist() + baseline = meta.loc[meta["county"].isin(counties)].index.values.tolist() - regions = {c: 'county' for c in counties} + regions = dict.fromkeys(counties, "county") pp = ProjectPoints.regions(regions, res_file, sam_files) assert sorted(baseline) == pp.sites -@pytest.mark.parametrize('sites', [1, 2, 5, 10]) +@pytest.mark.parametrize("sites", [1, 2, 5, 10]) def test_coords(sites): """ Test ProjectPoint.lat_lon_coords class method """ - res_file = os.path.join(TESTDATADIR, 'nsrdb/', 'ri_100_nsrdb_2012.h5') - sam_files = os.path.join(TESTDATADIR, 'SAM/naris_pv_1axis_inv13_cs.json') + res_file = os.path.join(TESTDATADIR, "nsrdb/", "ri_100_nsrdb_2012.h5") + sam_files = os.path.join(TESTDATADIR, "SAM/naris_pv_1axis_inv13_cs.json") with Resource(res_file) as f: meta = f.meta @@ -207,7 +228,9 @@ def test_coords(sites): if not isinstance(gids, list): gids = [gids] - lat_lons = meta.loc[gids, [MetaKeyName.LATITUDE, MetaKeyName.LONGITUDE]].values + lat_lons = meta.loc[ + gids, [MetaKeyName.LATITUDE, MetaKeyName.LONGITUDE] + ].values pp = ProjectPoints.lat_lon_coords(lat_lons, res_file, sam_files) assert sorted(gids) == pp.sites @@ -216,8 +239,8 @@ def test_coords(sites): def test_coords_from_file(): """Test ProjectPoint.lat_lon_coords read from file.""" - res_file = os.path.join(TESTDATADIR, 'nsrdb/', 'ri_100_nsrdb_2012.h5') - sam_files = os.path.join(TESTDATADIR, 'SAM/naris_pv_1axis_inv13_cs.json') + res_file = os.path.join(TESTDATADIR, "nsrdb/", "ri_100_nsrdb_2012.h5") + sam_files = os.path.join(TESTDATADIR, "SAM/naris_pv_1axis_inv13_cs.json") with Resource(res_file) as f: meta = f.meta @@ -239,18 +262,20 @@ def test_duplicate_coords(): """ Test ProjectPoint.lat_lon_coords duplicate coords error """ - res_file = os.path.join(TESTDATADIR, 'nsrdb/', 'ri_100_nsrdb_2012.h5') - sam_files = os.path.join(TESTDATADIR, 'SAM/naris_pv_1axis_inv13_cs.json') + res_file = os.path.join(TESTDATADIR, "nsrdb/", "ri_100_nsrdb_2012.h5") + sam_files = os.path.join(TESTDATADIR, "SAM/naris_pv_1axis_inv13_cs.json") with Resource(res_file) as f: meta = f.meta - duplicates = meta.loc[[2, 3, 3, 4], [MetaKeyName.LATITUDE, MetaKeyName.LONGITUDE]].values + duplicates = meta.loc[ + [2, 3, 3, 4], [MetaKeyName.LATITUDE, MetaKeyName.LONGITUDE] + ].values with pytest.raises(RuntimeError): ProjectPoints.lat_lon_coords(duplicates, res_file, sam_files) - regions = {'Kent': 'county', 'Rhode Island': 'state'} + regions = {"Kent": "county", "Rhode Island": "state"} with pytest.raises(RuntimeError): ProjectPoints.regions(regions, res_file, sam_files) @@ -259,15 +284,19 @@ def test_sam_configs(): """ Test supplying SAM config as a JSON or a dict """ - fpp = os.path.join(TESTDATADIR, 'project_points/pp_offshore.csv') - sam_files = {'onshore': os.path.join( - TESTDATADIR, 'SAM/wind_gen_standard_losses_0.json'), - MetaKeyName.OFFSHORE: os.path.join( - TESTDATADIR, 'SAM/wind_gen_standard_losses_1.json')} - pp_json = ProjectPoints(fpp, sam_files, 'windpower') + fpp = os.path.join(TESTDATADIR, "project_points/pp_offshore.csv") + sam_files = { + "onshore": os.path.join( + TESTDATADIR, "SAM/wind_gen_standard_losses_0.json" + ), + MetaKeyName.OFFSHORE: os.path.join( + TESTDATADIR, "SAM/wind_gen_standard_losses_1.json" + ), + } + pp_json = ProjectPoints(fpp, sam_files, "windpower") sam_configs = {k: safe_json_load(v) for k, v in sam_files.items()} - pp_dict = ProjectPoints(fpp, sam_configs, 'windpower') + pp_dict = ProjectPoints(fpp, sam_configs, "windpower") assert pp_json.sam_inputs == pp_dict.sam_inputs @@ -276,68 +305,71 @@ def test_bad_sam_configs(): """ Test supplying SAM config as a JSON or a dict """ - fpp = os.path.join(TESTDATADIR, 'project_points/pp_offshore.csv') - sam_files = {'onshore': os.path.join( - TESTDATADIR, 'SAM/wind_gen_standard_losses_0.json')} + fpp = os.path.join(TESTDATADIR, "project_points/pp_offshore.csv") + sam_files = { + "onshore": os.path.join( + TESTDATADIR, "SAM/wind_gen_standard_losses_0.json" + ) + } # Assert ProjecPoints fails with unequal config entries in points # vs sam_configs (as files) with pytest.raises(ConfigError): - ProjectPoints(fpp, sam_files, 'windpower') + ProjectPoints(fpp, sam_files, "windpower") sam_configs = {k: safe_json_load(v) for k, v in sam_files.items()} # Assert ProjecPoints fails with unequal config entries in points # vs sam_configs (as dicts) with pytest.raises(ConfigError): - ProjectPoints(fpp, sam_configs, 'windpower') + ProjectPoints(fpp, sam_configs, "windpower") sites = slice(0, 100) - sam_file = os.path.join(TESTDATADIR, 'SAM/wind_gen_standard_losses_0.csv') + sam_file = os.path.join(TESTDATADIR, "SAM/wind_gen_standard_losses_0.csv") # Assert SAMConfig fails when the SAM config is not a json with pytest.raises(IOError): - ProjectPoints(sites, sam_file, 'windpower') + ProjectPoints(sites, sam_file, "windpower") sites = slice(0, 100) - sam_file = os.path.join(TESTDATADIR, 'SAM/wind_gen_standard_losses_0.json') + sam_file = os.path.join(TESTDATADIR, "SAM/wind_gen_standard_losses_0.json") sam_config = safe_json_load(sam_file) # Assert SAMConfig fails when supplying a raw SAM config dictionary. # The SAM config dict should be mapped to a config ID with pytest.raises(RuntimeError): - ProjectPoints(sites, sam_config, 'windpower') + ProjectPoints(sites, sam_config, "windpower") - fpp = os.path.join(TESTDATADIR, 'project_points/pp_offshore.csv') - sam_files = [os.path.join(TESTDATADIR, - 'SAM/wind_gen_standard_losses_0.json'), - os.path.join(TESTDATADIR, - 'SAM/wind_gen_standard_losses_1.json')] + fpp = os.path.join(TESTDATADIR, "project_points/pp_offshore.csv") + sam_files = [ + os.path.join(TESTDATADIR, "SAM/wind_gen_standard_losses_0.json"), + os.path.join(TESTDATADIR, "SAM/wind_gen_standard_losses_1.json"), + ] # Assert ProjecPoints fails with a list of configs is provided instead # of a dictionary mapping the config files to config IDs with pytest.raises(ValueError): - ProjectPoints(fpp, sam_files, 'windpower') + ProjectPoints(fpp, sam_files, "windpower") def test_nested_sites(): """ Test check for nested points list """ - res_file = os.path.join(TESTDATADIR, 'wtk/ri_100_wtk_2012.h5') + res_file = os.path.join(TESTDATADIR, "wtk/ri_100_wtk_2012.h5") with pytest.raises(RuntimeError): points = [[1, 2, 3, 5]] - sam_file = os.path.join(TESTDATADIR, - 'SAM/wind_gen_standard_losses_0.json') - ProjectPoints(points, sam_file, 'windpower', res_file) + sam_file = os.path.join( + TESTDATADIR, "SAM/wind_gen_standard_losses_0.json" + ) + ProjectPoints(points, sam_file, "windpower", res_file) def test_project_points_h(): """ Test hub heights in project points """ - res_file = os.path.join(TESTDATADIR, 'wtk/ri_100_wtk_2012.h5') + res_file = os.path.join(TESTDATADIR, "wtk/ri_100_wtk_2012.h5") points = [1, 2, 3, 5] - sam_file = os.path.join(TESTDATADIR, - 'SAM/wind_gen_standard_losses_0.json') - assert ProjectPoints(points, sam_file, 'pvwattsv8', res_file).h is None + sam_file = os.path.join(TESTDATADIR, "SAM/wind_gen_standard_losses_0.json") + assert ProjectPoints(points, sam_file, "pvwattsv8", res_file).h is None - pp = ProjectPoints(points, sam_file, 'windpower', res_file) + pp = ProjectPoints(points, sam_file, "windpower", res_file) assert pp.h == [80] * 4 @@ -346,20 +378,20 @@ def test_project_points_d(): Test depth in project points """ points = [1, 2, 3, 5] - sam_file = os.path.join(TESTDATADIR, 'SAM/geothermal_default.json') - assert ProjectPoints(points, sam_file, 'windpower').d is None + sam_file = os.path.join(TESTDATADIR, "SAM/geothermal_default.json") + assert ProjectPoints(points, sam_file, "windpower").d is None - pp = ProjectPoints(points, sam_file, 'geothermal') + pp = ProjectPoints(points, sam_file, "geothermal") assert pp.d == [4500] * 4 depths_in_data = list(range(len(pp.df))) - pp = ProjectPoints(points, sam_file, 'geothermal') - pp.df['resource_depth'] = depths_in_data + pp = ProjectPoints(points, sam_file, "geothermal") + pp.df["resource_depth"] = depths_in_data assert pp.d == depths_in_data -def execute_pytest(capture='all', flags='-rapP'): +def execute_pytest(capture="all", flags="-rapP"): """Execute module as pytest with detailed summary report. Parameters @@ -372,8 +404,8 @@ def execute_pytest(capture='all', flags='-rapP'): """ fname = os.path.basename(__file__) - pytest.main(['-q', '--show-capture={}'.format(capture), fname, flags]) + pytest.main(["-q", "--show-capture={}".format(capture), fname, flags]) -if __name__ == '__main__': +if __name__ == "__main__": execute_pytest() diff --git a/tests/test_curtailment.py b/tests/test_curtailment.py index 893f4f051..3f6c5dbab 100644 --- a/tests/test_curtailment.py +++ b/tests/test_curtailment.py @@ -63,15 +63,15 @@ def test_cf_curtailment(year, site): # run reV 2.0 generation gen = Gen('windpower', points, sam_files, res_file, - output_request=(MetaKeyName.CF_PROFILE,), + output_request=('cf_profile',), curtailment=curtailment, sites_per_worker=50, scale_outputs=True) gen.run(max_workers=1) results, check_curtailment = test_res_curtailment(year, site=site) - results[MetaKeyName.CF_PROFILE] = gen.out[MetaKeyName.CF_PROFILE].flatten() + results['cf_profile'] = gen.out['cf_profile'].flatten() # was capacity factor NOT curtailed? - check_cf = (gen.out[MetaKeyName.CF_PROFILE].flatten() != 0) + check_cf = (gen.out['cf_profile'].flatten() != 0) # Were all thresholds met and windspeed NOT curtailed? check = check_curtailment & check_cf @@ -98,7 +98,7 @@ def test_curtailment_res_mean(year): curtailment = os.path.join(TESTDATADIR, 'config/', 'curtailment.json') points = slice(0, 100) - output_request = (MetaKeyName.CF_MEAN, 'ws_mean') + output_request = ('cf_mean', 'ws_mean') pc = Gen.get_pc(points, None, sam_files, 'windpower', sites_per_worker=50, res_file=res_file, curtailment=curtailment) @@ -146,11 +146,11 @@ def test_random(year, site): # run reV 2.0 generation and write to disk gen = Gen('windpower', points, sam_files, res_file, - output_request=(MetaKeyName.CF_PROFILE,), curtailment=c, + output_request=('cf_profile',), curtailment=c, sites_per_worker=50, scale_outputs=True) gen.run(max_workers=1) - results.append(gen.out[MetaKeyName.CF_MEAN]) + results.append(gen.out['cf_mean']) assert results[0] > results[1], 'Curtailment did not decrease cf_mean!' diff --git a/tests/test_econ_of_scale.py b/tests/test_econ_of_scale.py index d435c46e3..1aade9dc0 100644 --- a/tests/test_econ_of_scale.py +++ b/tests/test_econ_of_scale.py @@ -47,7 +47,7 @@ def test_pass_through_lcoe_args(): res_file = os.path.join(TESTDATADIR, 'wtk/ri_100_wtk_{}.h5'.format(year)) sam_files = os.path.join(TESTDATADIR, 'SAM/i_windpower_lcoe.json') - output_request = (MetaKeyName.CF_MEAN, + output_request = ('cf_mean', MetaKeyName.LCOE_FCR, 'system_capacity', 'capital_cost', @@ -63,7 +63,7 @@ def test_pass_through_lcoe_args(): checks = [x in gen.out for x in Gen.LCOE_ARGS] assert all(checks) assert MetaKeyName.LCOE_FCR in gen.out - assert MetaKeyName.CF_MEAN in gen.out + assert 'cf_mean' in gen.out def test_lcoe_calc_simple(): diff --git a/tests/test_econ_windbos.py b/tests/test_econ_windbos.py index 4824be563..e6a76b18b 100644 --- a/tests/test_econ_windbos.py +++ b/tests/test_econ_windbos.py @@ -6,6 +6,7 @@ @author: gbuster """ + import json import os import tempfile @@ -18,82 +19,128 @@ from reV.econ.econ import Econ from reV.generation.generation import Gen from reV.SAM.windbos import WindBos +from reV.utilities import MetaKeyName RTOL = 0.000001 ATOL = 0.001 -OUT_DIR = os.path.join(TESTDATADIR, 'ri_wind_reV2/') - -DEFAULTS = {'tech_model': 'windbos', - 'financial_model': 'none', - 'machine_rating': 1500.0, - 'rotor_diameter': 77.0, - 'hub_height': 80.0, - 'number_of_turbines': 32, - 'interconnect_voltage': 137.0, - 'distance_to_interconnect': 5.0, - 'site_terrain': 0, - 'turbine_layout': 1, - 'soil_condition': 0, - 'construction_time': 6, - 'om_building_size': 3000.0, - 'quantity_test_met_towers': 1, - 'quantity_permanent_met_towers': 1, - 'weather_delay_days': 6, - 'crane_breakdowns': 2, - 'access_road_entrances': 2, - 'turbine_capital_cost': 0.0, - 'turbine_cost_per_kw': 1094.0, # not in SDK tool, but important - 'tower_top_mass': 88.0, # tonnes, tuned to default foundation cost - 'delivery_assist_required': 0, - 'pad_mount_transformer_required': 1, - 'new_switchyard_required': 1, - 'rock_trenching_required': 10, - 'mv_thermal_backfill': 0.0, - 'mv_overhead_collector': 0.0, - 'performance_bond': 0, - 'contingency': 3.0, - 'warranty_management': 0.02, - 'sales_and_use_tax': 5.0, - 'sales_tax_basis': 0.0, - 'overhead': 5.0, - 'profit_margin': 5.0, - 'development_fee': 5.0, # Millions of dollars - 'turbine_transportation': 0.0, - } +OUT_DIR = os.path.join(TESTDATADIR, "ri_wind_reV2/") + +DEFAULTS = { + "tech_model": "windbos", + "financial_model": "none", + "machine_rating": 1500.0, + "rotor_diameter": 77.0, + "hub_height": 80.0, + "number_of_turbines": 32, + "interconnect_voltage": 137.0, + "distance_to_interconnect": 5.0, + "site_terrain": 0, + "turbine_layout": 1, + "soil_condition": 0, + "construction_time": 6, + "om_building_size": 3000.0, + "quantity_test_met_towers": 1, + "quantity_permanent_met_towers": 1, + "weather_delay_days": 6, + "crane_breakdowns": 2, + "access_road_entrances": 2, + "turbine_capital_cost": 0.0, + "turbine_cost_per_kw": 1094.0, # not in SDK tool, but important + "tower_top_mass": 88.0, # tonnes, tuned to default foundation cost + "delivery_assist_required": 0, + "pad_mount_transformer_required": 1, + "new_switchyard_required": 1, + "rock_trenching_required": 10, + "mv_thermal_backfill": 0.0, + "mv_overhead_collector": 0.0, + "performance_bond": 0, + "contingency": 3.0, + "warranty_management": 0.02, + "sales_and_use_tax": 5.0, + "sales_tax_basis": 0.0, + "overhead": 5.0, + "profit_margin": 5.0, + "development_fee": 5.0, # Millions of dollars + "turbine_transportation": 0.0, +} # baseline single owner + windbos results using 2012 RI WTK data. -BASELINE = {'project_return_aftertax_npv': np.array([7876459.5, 7875551.5, - 7874505., 7875270., - 7875349.5, 7872819.5, - 7871078.5, 7871352.5, - 7871153.5, 7869134.5]), - 'lcoe_real': np.array([71.007614, 69.21741, 67.24552, 68.67675, - 68.829605, 64.257965, 61.39551, 61.83265, - 61.51415, 58.43599]), - 'lcoe_nom': np.array([89.433525, 87.17878, 84.6952, 86.497826, - 86.69034, 80.932396, 77.327156, 77.87773, - 77.476585, 73.59966]), - 'flip_actual_irr': np.array([10.999977, 10.999978, 10.999978, - 10.999978, 10.999978, 10.999978, - 10.999979, 10.999979, 10.999979, - 10.99998]), - 'total_installed_cost': np.array(10 * [88892234.91311586]), - 'turbine_cost': np.array(10 * [52512000.0]), - 'sales_tax_cost': np.array(10 * [0.0]), - 'bos_cost': np.array(10 * [36380234.91311585]), - } +BASELINE = { + "project_return_aftertax_npv": np.array( + [ + 7876459.5, + 7875551.5, + 7874505.0, + 7875270.0, + 7875349.5, + 7872819.5, + 7871078.5, + 7871352.5, + 7871153.5, + 7869134.5, + ] + ), + "lcoe_real": np.array( + [ + 71.007614, + 69.21741, + 67.24552, + 68.67675, + 68.829605, + 64.257965, + 61.39551, + 61.83265, + 61.51415, + 58.43599, + ] + ), + "lcoe_nom": np.array( + [ + 89.433525, + 87.17878, + 84.6952, + 86.497826, + 86.69034, + 80.932396, + 77.327156, + 77.87773, + 77.476585, + 73.59966, + ] + ), + "flip_actual_irr": np.array( + [ + 10.999977, + 10.999978, + 10.999978, + 10.999978, + 10.999978, + 10.999978, + 10.999979, + 10.999979, + 10.999979, + 10.99998, + ] + ), + "total_installed_cost": np.array(10 * [88892234.91311586]), + "turbine_cost": np.array(10 * [52512000.0]), + "sales_tax_cost": np.array(10 * [0.0]), + "bos_cost": np.array(10 * [36380234.91311585]), +} # baseline single owner + windbos results when sweeping sales tax basis -BASELINE_SITE_BOS = {'total_installed_cost': np.array([88892230., 88936680., - 88981130., 89025576., - 89070020.]), - 'turbine_cost': np.array(5 * [52512000.0]), - 'sales_tax_cost': np.array([0., 44446.117, 88892.234, - 133338.36, 177784.47]), - 'bos_cost': np.array(5 * [36380234.91311585]), - } +BASELINE_SITE_BOS = { + "total_installed_cost": np.array( + [88892230.0, 88936680.0, 88981130.0, 89025576.0, 89070020.0] + ), + "turbine_cost": np.array(5 * [52512000.0]), + "sales_tax_cost": np.array( + [0.0, 44446.117, 88892.234, 133338.36, 177784.47] + ), + "bos_cost": np.array(5 * [36380234.91311585]), +} def test_sam_windbos(): @@ -102,34 +149,36 @@ def test_sam_windbos(): out = ssc_sim_from_dict(DEFAULTS) - tcost = ((DEFAULTS['turbine_cost_per_kw'] - + DEFAULTS['turbine_capital_cost']) - * DEFAULTS['machine_rating'] * DEFAULTS['number_of_turbines']) - total_installed_cost = tcost + out['project_total_budgeted_cost'] + tcost = ( + (DEFAULTS["turbine_cost_per_kw"] + DEFAULTS["turbine_capital_cost"]) + * DEFAULTS["machine_rating"] + * DEFAULTS["number_of_turbines"] + ) + total_installed_cost = tcost + out["project_total_budgeted_cost"] - assert np.allclose(total_installed_cost, 88892240.00, - atol=ATOL, rtol=RTOL) + assert np.allclose(total_installed_cost, 88892240.00, atol=ATOL, rtol=RTOL) def test_rev_windbos(): """Test baseline windbos calc with single owner defaults""" - fpath = TESTDATADIR + '/SAM/i_singleowner_windbos.json' + fpath = TESTDATADIR + "/SAM/i_singleowner_windbos.json" with open(fpath) as f: inputs = json.load(f) wb = WindBos(inputs) assert np.allclose(wb.turbine_cost, 52512000.00, atol=ATOL, rtol=RTOL) assert np.allclose(wb.bos_cost, 36380236.00, atol=ATOL, rtol=RTOL) - assert np.allclose(wb.total_installed_cost, 88892240.00, atol=ATOL, - rtol=RTOL) + assert np.allclose( + wb.total_installed_cost, 88892240.00, atol=ATOL, rtol=RTOL + ) def test_standalone_json(): """Test baseline windbos calc with standalone json file""" - fpath = TESTDATADIR + '/SAM/i_singleowner_windbos.json' + fpath = TESTDATADIR + "/SAM/i_singleowner_windbos.json" with open(fpath) as f: inputs = json.load(f) wb1 = WindBos(inputs) - fpath = TESTDATADIR + '/SAM/i_windbos.json' + fpath = TESTDATADIR + "/SAM/i_windbos.json" with open(fpath) as f: inputs = json.load(f) wb2 = WindBos(inputs) @@ -140,41 +189,44 @@ def test_standalone_json(): def test_rev_windbos_perf_bond(): """Test windbos calc with performance bonds""" - fpath = TESTDATADIR + '/SAM/i_singleowner_windbos.json' + fpath = TESTDATADIR + "/SAM/i_singleowner_windbos.json" with open(fpath) as f: inputs = json.load(f) - inputs['performance_bond'] = 10.0 + inputs["performance_bond"] = 10.0 wb = WindBos(inputs) assert np.allclose(wb.turbine_cost, 52512000.00, atol=ATOL, rtol=RTOL) assert np.allclose(wb.bos_cost, 36686280.00, atol=ATOL, rtol=RTOL) - assert np.allclose(wb.total_installed_cost, 89198280.00, atol=ATOL, - rtol=RTOL) + assert np.allclose( + wb.total_installed_cost, 89198280.00, atol=ATOL, rtol=RTOL + ) def test_rev_windbos_transport(): """Test windbos calc with turbine transport costs""" - fpath = TESTDATADIR + '/SAM/i_singleowner_windbos.json' + fpath = TESTDATADIR + "/SAM/i_singleowner_windbos.json" with open(fpath) as f: inputs = json.load(f) - inputs['turbine_transportation'] = 100.0 + inputs["turbine_transportation"] = 100.0 wb = WindBos(inputs) assert np.allclose(wb.turbine_cost, 52512000.00, atol=ATOL, rtol=RTOL) assert np.allclose(wb.bos_cost, 37720412.00, atol=ATOL, rtol=RTOL) - assert np.allclose(wb.total_installed_cost, 90232416.00, atol=ATOL, - rtol=RTOL) + assert np.allclose( + wb.total_installed_cost, 90232416.00, atol=ATOL, rtol=RTOL + ) def test_rev_windbos_sales(): """Test windbos calc with turbine transport costs""" - fpath = TESTDATADIR + '/SAM/i_singleowner_windbos.json' + fpath = TESTDATADIR + "/SAM/i_singleowner_windbos.json" with open(fpath) as f: inputs = json.load(f) - inputs['sales_tax_basis'] = 5.0 + inputs["sales_tax_basis"] = 5.0 wb = WindBos(inputs) assert np.allclose(wb.turbine_cost, 52512000.00, atol=ATOL, rtol=RTOL) assert np.allclose(wb.bos_cost, 36380236.00, atol=ATOL, rtol=RTOL) - assert np.allclose(wb.total_installed_cost, 89114464.00, atol=ATOL, - rtol=RTOL) + assert np.allclose( + wb.total_installed_cost, 89114464.00, atol=ATOL, rtol=RTOL + ) def test_run_gen_econ(points=slice(0, 10), year=2012, max_workers=1): @@ -182,27 +234,46 @@ def test_run_gen_econ(points=slice(0, 10), year=2012, max_workers=1): against baseline results.""" with tempfile.TemporaryDirectory() as td: # get full file paths. - sam_files = os.path.join(TESTDATADIR, 'SAM/i_singleowner_windbos.json') - res_file = os.path.join(TESTDATADIR, - 'wtk/ri_100_wtk_{}.h5'.format(year)) - fn_gen = 'windbos_generation_{}.h5'.format(year) + sam_files = os.path.join(TESTDATADIR, "SAM/i_singleowner_windbos.json") + res_file = os.path.join( + TESTDATADIR, "wtk/ri_100_wtk_{}.h5".format(year) + ) + fn_gen = "windbos_generation_{}.h5".format(year) cf_file = os.path.join(td, fn_gen) # run reV 2.0 generation - gen = Gen('windpower', points, sam_files, res_file, - output_request=(MetaKeyName.CF_MEAN, ), - sites_per_worker=3) + gen = Gen( + "windpower", + points, + sam_files, + res_file, + output_request=('cf_mean',), + sites_per_worker=3, + ) gen.run(max_workers=max_workers, out_fpath=cf_file) - econ_outs = ('lcoe_nom', 'lcoe_real', 'flip_actual_irr', - 'project_return_aftertax_npv', 'total_installed_cost', - 'turbine_cost', 'sales_tax_cost', 'bos_cost') - e = Econ(points, sam_files, cf_file, site_data=None, - output_request=econ_outs, sites_per_worker=3) + econ_outs = ( + "lcoe_nom", + "lcoe_real", + "flip_actual_irr", + "project_return_aftertax_npv", + "total_installed_cost", + "turbine_cost", + "sales_tax_cost", + "bos_cost", + ) + e = Econ( + points, + sam_files, + cf_file, + site_data=None, + output_request=econ_outs, + sites_per_worker=3, + ) e.run(max_workers=max_workers) for k in econ_outs: - msg = 'Failed for {}'.format(k) + msg = "Failed for {}".format(k) test = np.allclose(e.out[k], BASELINE[k], atol=ATOL, rtol=RTOL) assert test, msg @@ -214,26 +285,38 @@ def test_run_bos(points=slice(0, 5), max_workers=1): against baseline results.""" # get full file paths. - sam_files = TESTDATADIR + '/SAM/i_singleowner_windbos.json' - site_data = pd.DataFrame({MetaKeyName.GID: range(5), - 'sales_tax_basis': range(5)}) - - econ_outs = ('total_installed_cost', 'turbine_cost', 'sales_tax_cost', - 'bos_cost') - e = Econ(points, sam_files, None, site_data=site_data, - output_request=econ_outs, sites_per_worker=3) + sam_files = TESTDATADIR + "/SAM/i_singleowner_windbos.json" + site_data = pd.DataFrame( + {MetaKeyName.GID: range(5), "sales_tax_basis": range(5)} + ) + + econ_outs = ( + "total_installed_cost", + "turbine_cost", + "sales_tax_cost", + "bos_cost", + ) + e = Econ( + points, + sam_files, + None, + site_data=site_data, + output_request=econ_outs, + sites_per_worker=3, + ) e.run(max_workers=max_workers) for k in econ_outs: - check = np.allclose(e.out[k], BASELINE_SITE_BOS[k], - atol=ATOL, rtol=RTOL) - msg = 'Failed for {}'.format(k) + check = np.allclose( + e.out[k], BASELINE_SITE_BOS[k], atol=ATOL, rtol=RTOL + ) + msg = "Failed for {}".format(k) assert check, msg return e -def execute_pytest(capture='all', flags='-rapP'): +def execute_pytest(capture="all", flags="-rapP"): """Execute module as pytest with detailed summary report. Parameters @@ -246,8 +329,8 @@ def execute_pytest(capture='all', flags='-rapP'): """ fname = os.path.basename(__file__) - pytest.main(['-q', '--show-capture={}'.format(capture), fname, flags]) + pytest.main(["-q", "--show-capture={}".format(capture), fname, flags]) -if __name__ == '__main__': +if __name__ == "__main__": execute_pytest() diff --git a/tests/test_gen_5min.py b/tests/test_gen_5min.py index 59239aa41..8d2eda4fd 100644 --- a/tests/test_gen_5min.py +++ b/tests/test_gen_5min.py @@ -28,7 +28,7 @@ def test_gen_downscaling(): # run reV 2.0 generation gen = Gen('pvwattsv5', slice(0, None), sam_files, res_file, - output_request=(MetaKeyName.CF_MEAN, ), sites_per_worker=100) + output_request=('cf_mean', ), sites_per_worker=100) gen.run(max_workers=1) gen_outs = gen.out[].astype(np.int32) diff --git a/tests/test_gen_csp.py b/tests/test_gen_csp.py index c75bedc0b..82c8f6570 100644 --- a/tests/test_gen_csp.py +++ b/tests/test_gen_csp.py @@ -29,7 +29,7 @@ def test_gen_csp(): res_file = TESTDATADIR + '/nsrdb/ri_100_nsrdb_{}.h5'.format(2012) # run reV 2.0 generation - output_request = (MetaKeyName.CF_MEAN, MetaKeyName.CF_PROFILE, + output_request = ('cf_mean', 'cf_profile', MetaKeyName.GEN_PROFILE) gen = Gen('tcsmoltensalt', points, sam_files, res_file, output_request=output_request, sites_per_worker=1, diff --git a/tests/test_gen_forecast.py b/tests/test_gen_forecast.py index cd47101f3..41edec897 100644 --- a/tests/test_gen_forecast.py +++ b/tests/test_gen_forecast.py @@ -52,7 +52,7 @@ def test_forecast(): points = ProjectPoints(slice(0, 5), sam_files, 'pvwattsv7', res_file=res_file) - output_request = (MetaKeyName.CF_MEAN, 'ghi_mean') + output_request = ('cf_mean', 'ghi_mean') site_data = pd.DataFrame({MetaKeyName.GID: np.arange(5), MetaKeyName.TIMEZONE: -5, 'elevation': 0}) diff --git a/tests/test_gen_geothermal.py b/tests/test_gen_geothermal.py index c1427c2ec..e57ebf2ff 100644 --- a/tests/test_gen_geothermal.py +++ b/tests/test_gen_geothermal.py @@ -63,8 +63,8 @@ def test_gen_geothermal(depth, sample_resource_data): with open(geo_sam_file, "w") as fh: json.dump(geo_config, fh) - output_request = ('annual_energy', MetaKeyName.CF_MEAN, - MetaKeyName.CF_PROFILE, + output_request = ('annual_energy', 'cf_mean', + 'cf_profile', MetaKeyName.GEN_PROFILE, MetaKeyName.LCOE_FCR, 'nameplate') @@ -97,8 +97,8 @@ def test_gen_geothermal_temp_too_low(sample_resource_data): geo_sam_file, geo_res_file = sample_resource_data shutil.copy(DEFAULT_GEO_SAM_FILE, geo_sam_file) - output_request = ('annual_energy', MetaKeyName.CF_MEAN, - MetaKeyName.CF_PROFILE, + output_request = ('annual_energy', 'cf_mean', + 'cf_profile', MetaKeyName.GEN_PROFILE, MetaKeyName.LCOE_FCR, 'nameplate') gen = Gen('geothermal', points, geo_sam_file, geo_res_file, @@ -213,8 +213,8 @@ def test_gen_with_nameplate_input(sample_resource_data): with open(geo_sam_file, "w") as fh: json.dump(geo_config, fh) - output_request = ('annual_energy', MetaKeyName.CF_MEAN, - MetaKeyName.CF_PROFILE, + output_request = ('annual_energy', 'cf_mean', + 'cf_profile', MetaKeyName.GEN_PROFILE, MetaKeyName.LCOE_FCR, 'nameplate') gen = Gen('geothermal', points, geo_sam_file, geo_res_file, @@ -365,8 +365,8 @@ def test_gen_with_time_index_step_input(sample_resource_data): with open(geo_sam_file, "w") as fh: json.dump(geo_config, fh) - output_request = ('annual_energy', MetaKeyName.CF_MEAN, - MetaKeyName.CF_PROFILE, + output_request = ('annual_energy', 'cf_mean', + 'cf_profile', MetaKeyName.GEN_PROFILE, MetaKeyName.LCOE_FCR, 'nameplate') gen = Gen('geothermal', points, geo_sam_file, geo_res_file, diff --git a/tests/test_gen_linear.py b/tests/test_gen_linear.py index 21fde7a89..b42717e10 100644 --- a/tests/test_gen_linear.py +++ b/tests/test_gen_linear.py @@ -40,7 +40,7 @@ def test_gen_linear(): # sequence: Receiver mass flow rate [kg/s] output_request = ('q_dot_to_heat_sink', 'gen', 'm_dot_field', 'q_dot_sf_out', 'W_dot_heat_sink_pump', 'm_dot_loop', - 'q_dot_rec_inc', MetaKeyName.CF_MEAN, 'gen_profile', + 'q_dot_rec_inc', 'cf_mean', 'gen_profile', 'annual_field_energy', 'annual_thermal_consumption',) # run reV 2.0 generation diff --git a/tests/test_gen_pv.py b/tests/test_gen_pv.py index 1adac55b6..122097ecd 100644 --- a/tests/test_gen_pv.py +++ b/tests/test_gen_pv.py @@ -22,6 +22,7 @@ from reV.config.project_points import ProjectPoints from reV.generation.generation import Gen from reV.handlers.outputs import Outputs +from reV.utililities import MetaKeyName from reV.utilities.exceptions import ConfigError, ExecutionError RTOL = 0.0 @@ -169,11 +170,11 @@ def test_pv_gen_profiles(year): # run reV 2.0 generation and write to disk gen = Gen('pvwattsv5', points, sam_files, res_file, - output_request=(,), sites_per_worker=50) + output_request=('cf_profile',), sites_per_worker=50) gen.run(max_workers=2, out_fpath=rev2_out) with Outputs(rev2_out, 'r') as cf: - rev2_profiles = cf[] + rev2_profiles = cf['cf_profile'] # get reV 1.0 generation profiles rev1_profiles = get_r1_profiles(year=year) @@ -195,11 +196,11 @@ def test_smart(year): # run reV 2.0 generation and write to disk gen = Gen('pvwattsv5', points, sam_files, res_file, - output_request=(,), sites_per_worker=50) + output_request=('cf_profile',), sites_per_worker=50) gen.run(max_workers=2, out_fpath=rev2_out) with Outputs(rev2_out, 'r') as cf: - rev2_profiles = cf[] + rev2_profiles = cf['cf_profile'] # get reV 1.0 generation profiles rev1_profiles = get_r1_profiles(year=year) @@ -225,7 +226,7 @@ def test_multi_file_nsrdb_2018(model): assert len(means_outs) == 10 assert np.mean(means_outs) > 0.14 - profiles_out = gen.out[] + profiles_out = gen.out['cf_profile'] assert profiles_out.shape == (105120, 10) assert np.mean(profiles_out) > 0.14 @@ -235,7 +236,7 @@ def get_r1_profiles(year=2012): rev1 = os.path.join(TESTDATADIR, 'ri_pv', 'profile_outputs', 'pv_{}_0.h5'.format(year)) with Outputs(rev1) as cf: - data = cf[][...] / 10000 + data = cf['cf_profile'][...] / 10000 return data @@ -261,7 +262,7 @@ def test_southern_hemisphere(): rev2_points = slice(0, 1) res_file = TESTDATADIR + '/nsrdb/brazil_solar.h5' sam_files = TESTDATADIR + '/SAM/i_pvwatts_fixed_lat_tilt.json' - output_request = ('cf_mean', , 'dni_mean', 'dhi_mean', + output_request = ('cf_mean', 'cf_profile', 'dni_mean', 'dhi_mean', 'ghi_mean', 'ac', 'dc', 'azimuth') gen = Gen('pvwattsv7', rev2_points, sam_files, res_file, @@ -286,7 +287,7 @@ def test_pvwattsv7_baseline(): res_file = TESTDATADIR + '/nsrdb/ri_100_nsrdb_{}.h5'.format(year) sam_files = TESTDATADIR + '/SAM/i_pvwattsv7.json' - output_request = ('cf_mean', , 'dni_mean', 'dhi_mean', + output_request = ('cf_mean', 'cf_profile', 'dni_mean', 'dhi_mean', 'ghi_mean', 'ac', 'dc') # run reV 2.0 generation @@ -335,7 +336,7 @@ def test_pvwatts_v8_lifetime(): res_file = TESTDATADIR + '/nsrdb/ri_100_nsrdb_{}.h5'.format(year) sam_files = TESTDATADIR + '/SAM/i_pvwattsv8_degradation.json' - output_request = ('cf_mean', , 'dni_mean', 'dhi_mean', + output_request = ('cf_mean', 'cf_profile', 'dni_mean', 'dhi_mean', 'ghi_mean') # run reV 2.0 generation with valid output request @@ -362,7 +363,7 @@ def test_pvwatts_v8_lifetime_invalid_request(): res_file = TESTDATADIR + '/nsrdb/ri_100_nsrdb_{}.h5'.format(year) sam_files = TESTDATADIR + '/SAM/i_pvwattsv8_degradation.json' - output_request_invalid = ('cf_mean', , 'dni_mean', 'dhi_mean', + output_request_invalid = ('cf_mean', 'cf_profile', 'dni_mean', 'dhi_mean', 'ghi_mean', 'ac', 'dc') # run reV 2.0 generation with invalid output request @@ -388,7 +389,7 @@ def test_bifacial(): sam_files = TESTDATADIR + '/SAM/i_pvwattsv7_bifacial.json' # run reV 2.0 generation - output_request = ('cf_mean', , 'surface_albedo') + output_request = ('cf_mean', 'cf_profile', 'surface_albedo') gen_bi = Gen('pvwattsv7', rev2_points, sam_files, res_file, sites_per_worker=1, output_request=output_request) gen_bi.run(max_workers=1) @@ -499,7 +500,7 @@ def test_detailed_pv_baseline(): res_file = TESTDATADIR + '/nsrdb/ri_100_nsrdb_{}.h5'.format(year) sam_files = TESTDATADIR + '/SAM/i_pvsamv1.json' - output_request = ('cf_mean', , 'dni_mean', 'dhi_mean', + output_request = ('cf_mean', 'cf_profile', 'dni_mean', 'dhi_mean', 'ghi_mean', 'ac', 'dc') # run reV 2.0 generation @@ -526,7 +527,7 @@ def test_detailed_pv_bifacial(): res_file = TESTDATADIR + '/nsrdb/ri_100_nsrdb_{}.h5'.format(year) sam_files = TESTDATADIR + '/SAM/i_pvsamv1_bifacial.json' - output_request = ('cf_mean', , 'dni_mean', 'dhi_mean', + output_request = ('cf_mean', 'cf_profile', 'dni_mean', 'dhi_mean', 'ghi_mean', 'ac', 'dc', 'surface_albedo') # run reV 2.0 generation @@ -552,9 +553,9 @@ def test_pv_clearsky(): sam_files = TESTDATADIR + '/SAM/naris_pv_1axis_inv13.json' sam_files_cs = TESTDATADIR + '/SAM/naris_pv_1axis_inv13_cs.json' - output_request = ('cf_mean', , 'dni_mean', 'dhi_mean', + output_request = ('cf_mean', 'cf_profile', 'dni_mean', 'dhi_mean', 'ghi_mean', 'ac', 'dc') - output_request_cs = ('cf_mean', , 'clearsky_dni_mean', + output_request_cs = ('cf_mean', 'cf_profile', 'clearsky_dni_mean', 'clearsky_dhi_mean', 'clearsky_ghi_mean', 'ac', 'dc') with tempfile.TemporaryDirectory() as td: @@ -590,7 +591,7 @@ def test_irrad_bias_correct(): res_file = TESTDATADIR + '/nsrdb/ri_100_nsrdb_{}.h5'.format(year) sam_files = TESTDATADIR + '/SAM/i_pvwattsv7.json' - output_request = ('cf_mean', , 'dni_mean', 'dhi_mean', + output_request = ('cf_mean', 'cf_profile', 'dni_mean', 'dhi_mean', 'ghi_mean', 'ac', 'dc') gen_base = Gen('pvwattsv7', points, sam_files, res_file, @@ -607,13 +608,13 @@ def test_irrad_bias_correct(): assert (gen_base.out['cf_mean'][0] == gen.out['cf_mean'][0]).all() assert (gen_base.out['ghi_mean'][0] == gen.out['ghi_mean'][0]).all() - assert np.allclose(gen_base.out[][:, 0], - gen.out[][:, 0]) + assert np.allclose(gen_base.out['cf_profile'][:, 0], + gen.out['cf_profile'][:, 0]) assert (gen_base.out['cf_mean'][1:] < gen.out['cf_mean'][1:]).all() assert (gen_base.out['ghi_mean'][1:] < gen.out['ghi_mean'][1:]).all() - mask = (gen_base.out[][:, 1:] <= gen.out[][:, 1:]) + mask = (gen_base.out['cf_profile'][:, 1:] <= gen.out['cf_profile'][:, 1:]) assert (mask.sum() / mask.size) > 0.99 bc_df = pd.DataFrame({MetaKeyName.GID: np.arange(100), @@ -635,7 +636,7 @@ def test_ac_outputs(): res_file = TESTDATADIR + '/nsrdb/ri_100_nsrdb_{}.h5'.format(year) sam_files = TESTDATADIR + '/SAM/i_pvwattsv8.json' - output_request = ('cf_mean', 'cf_mean_ac', , 'cf_profile_ac', + output_request = ('cf_mean', 'cf_mean_ac', 'cf_profile', 'cf_profile_ac', 'system_capacity', 'system_capacity_ac', 'ac', 'dc', 'dc_ac_ratio') @@ -659,7 +660,7 @@ def test_ac_outputs(): assert np.allclose(gen.out['system_capacity'] / gen.out['dc_ac_ratio'], gen.out['system_capacity_ac']) - assert not np.isclose(gen.out[], 1).any() + assert not np.isclose(gen.out['cf_profile'], 1).any() assert np.isclose(gen.out['cf_profile_ac'], 1).any() @@ -681,7 +682,7 @@ def test_bad_loc_inputs(bad_input): f.meta = meta gen = Gen('pvwattsv8', slice(0, 3), sam_files, res_file_bad, - output_request=(,), sites_per_worker=50) + output_request=('cf_profile',), sites_per_worker=50) with pytest.raises(ValueError): gen.run(max_workers=1) diff --git a/tests/test_gen_swh.py b/tests/test_gen_swh.py index 4c78bb48d..0de443f8a 100644 --- a/tests/test_gen_swh.py +++ b/tests/test_gen_swh.py @@ -29,7 +29,7 @@ def test_gen_swh_non_leap_year(): output_request = ('T_amb', 'T_cold', 'T_deliv', 'T_hot', 'draw', 'beam', 'diffuse', 'I_incident', 'I_transmitted', - 'annual_Q_deliv', 'Q_deliv', MetaKeyName.CF_MEAN, 'solar_fraction', + 'annual_Q_deliv', 'Q_deliv', 'cf_mean', 'solar_fraction', 'gen_profile') # run reV 2.0 generation @@ -61,7 +61,7 @@ def test_gen_swh_leap_year(): output_request = ('T_amb', 'T_cold', 'T_deliv', 'T_hot', 'draw', 'beam', 'diffuse', 'I_incident', 'I_transmitted', - 'annual_Q_deliv', 'Q_deliv', MetaKeyName.CF_MEAN, 'solar_fraction') + 'annual_Q_deliv', 'Q_deliv', 'cf_mean', 'solar_fraction') # run reV 2.0 generation gen = Gen('solarwaterheat', points, sam_files, res_file, diff --git a/tests/test_gen_time_scale.py b/tests/test_gen_time_scale.py index d04ec6eae..13fb0c01f 100644 --- a/tests/test_gen_time_scale.py +++ b/tests/test_gen_time_scale.py @@ -32,17 +32,17 @@ def test_time_index_step(): # run reV 2.0 generation gen = Gen('pvwattsv5', slice(0, None), sam_input, res_file, - output_request=(MetaKeyName.CF_MEAN, ), + output_request=('cf_mean', ), sites_per_worker=100) gen.run(max_workers=1) - gen_outs = gen.out[].astype(np.int32) + gen_outs = gen.out['cf_profile'].astype(np.int32) if not os.path.exists(baseline): with h5py.File(baseline, 'w') as f: - f.create_dataset(, data=gen_outs, dtype=gen_outs.dtype) + f.create_dataset('cf_profile', data=gen_outs, dtype=gen_outs.dtype) else: with h5py.File(baseline, 'r') as f: - baseline = f[][...].astype(np.int32) + baseline = f['cf_profile'][...].astype(np.int32) assert np.allclose(gen_outs, baseline) diff --git a/tests/test_gen_trough.py b/tests/test_gen_trough.py index 244a49c9d..1040be085 100644 --- a/tests/test_gen_trough.py +++ b/tests/test_gen_trough.py @@ -39,7 +39,7 @@ def test_gen_tph(): 'm_dot_field_delivered', 'm_dot_field_recirc', 'q_dot_htf_sf_out', 'q_dot_to_heat_sink', 'q_dot_rec_inc', 'qinc_costh', 'dni_costh', 'beam', - MetaKeyName.CF_MEAN, 'annual_gross_energy', + 'cf_mean', 'annual_gross_energy', 'annual_thermal_consumption', 'annual_energy') # run reV 2.0 generation diff --git a/tests/test_gen_wave.py b/tests/test_gen_wave.py index e50a64a81..54d816f79 100644 --- a/tests/test_gen_wave.py +++ b/tests/test_gen_wave.py @@ -38,16 +38,16 @@ def test_mhkwave(): sam_files = TESTDATADIR + '/SAM/mhkwave_default.json' res_file = TESTDATADIR + '/wave/ri_wave_2010.h5' points = slice(0, 100) - output_request = (MetaKeyName.CF_MEAN, ) + output_request = ('cf_mean', ) test = Gen('mhkwave', points, sam_files, res_file, sites_per_worker=3, output_request=output_request) test.run(max_workers=1) with Resource(BASELINE) as f: - assert np.allclose(test.out[MetaKeyName.CF_MEAN], f[MetaKeyName.CF_MEAN], + assert np.allclose(test.out['cf_mean'], f['cf_mean'], atol=0.01, rtol=0.01) - assert np.allclose(test.out[], f[], + assert np.allclose(test.out['cf_profile'], f['cf_profile'], atol=0.01, rtol=0.01) diff --git a/tests/test_gen_wind.py b/tests/test_gen_wind.py index 15ec42d99..56c1687a4 100644 --- a/tests/test_gen_wind.py +++ b/tests/test_gen_wind.py @@ -31,7 +31,7 @@ class wind_results: """Class to retrieve results from the rev 1.0 pv files""" def __init__(self, f): - self._h5 = h5py.File(f, 'r') + self._h5 = h5py.File(f, "r") def __enter__(self): return self @@ -45,15 +45,15 @@ def __exit__(self, type, value, traceback): @property def years(self): """Get a list of year strings.""" - if not hasattr(self, '_years'): - year_index = self._h5['wind']['year_index'][...] + if not hasattr(self, "_years"): + year_index = self._h5["wind"]["year_index"][...] self._years = [y.decode() for y in year_index] return self._years def get_cf_mean(self, site, year): """Get a cf mean based on site and year""" iy = self.years.index(year) - out = self._h5['wind'][MetaKeyName.CF_MEAN][iy, site] + out = self._h5["wind"]["cf_mean"][iy, site] return out @@ -69,32 +69,37 @@ def is_num(n): def to_list(gen_out): """Generation output handler that converts to the rev 1.0 format.""" if isinstance(gen_out, list) and len(gen_out) == 1: - out = [c[MetaKeyName.CF_MEAN] for c in gen_out[0].values()] + out = [c["cf_mean"] for c in gen_out[0].values()] if isinstance(gen_out, dict): - out = [c[MetaKeyName.CF_MEAN] for c in gen_out.values()] + out = [c["cf_mean"] for c in gen_out.values()] return out -@pytest.mark.parametrize(('f_rev1_out', 'rev2_points', 'year', 'max_workers'), - [ - ('project_outputs.h5', slice(0, 10), '2012', 1), - ('project_outputs.h5', slice(0, 100, 10), '2013', 2)]) +@pytest.mark.parametrize( + ("f_rev1_out", "rev2_points", "year", "max_workers"), + [ + ("project_outputs.h5", slice(0, 10), "2012", 1), + ("project_outputs.h5", slice(0, 100, 10), "2013", 2), + ], +) def test_wind_gen_slice(f_rev1_out, rev2_points, year, max_workers): """Test reV 2.0 generation for PV and benchmark against reV 1.0 results.""" # get full file paths. - rev1_outs = os.path.join(TESTDATADIR, 'ri_wind', 'scalar_outputs', - f_rev1_out) - sam_files = TESTDATADIR + '/SAM/wind_gen_standard_losses_0.json' - res_file = TESTDATADIR + '/wtk/ri_100_wtk_{}.h5'.format(year) + rev1_outs = os.path.join( + TESTDATADIR, "ri_wind", "scalar_outputs", f_rev1_out + ) + sam_files = TESTDATADIR + "/SAM/wind_gen_standard_losses_0.json" + res_file = TESTDATADIR + "/wtk/ri_100_wtk_{}.h5".format(year) # run reV 2.0 generation - pp = ProjectPoints(rev2_points, sam_files, 'windpower', res_file=res_file) - gen = Gen('windpower', rev2_points, sam_files, res_file, - sites_per_worker=3) + pp = ProjectPoints(rev2_points, sam_files, "windpower", res_file=res_file) + gen = Gen( + "windpower", rev2_points, sam_files, res_file, sites_per_worker=3 + ) gen.run(max_workers=max_workers) - gen_outs = list(gen.out[MetaKeyName.CF_MEAN]) + gen_outs = list(gen.out["cf_mean"]) # initialize the rev1 output hander with wind_results(rev1_outs) as wind: @@ -102,10 +107,12 @@ def test_wind_gen_slice(f_rev1_out, rev2_points, year, max_workers): cf_mean_list = wind.get_cf_mean(pp.sites, year) # benchmark the results - msg = 'Wind cf_means results did not match reV 1.0 results!' + msg = "Wind cf_means results did not match reV 1.0 results!" assert np.allclose(gen_outs, cf_mean_list, rtol=RTOL, atol=ATOL), msg - assert np.allclose(pp.sites, gen.meta.index.values), 'bad gen meta!' - assert np.allclose(pp.sites, gen.meta[MetaKeyName.GID].values), 'bad gen meta!' + assert np.allclose(pp.sites, gen.meta.index.values), "bad gen meta!" + assert np.allclose( + pp.sites, gen.meta[MetaKeyName.GID].values + ), "bad gen meta!" labels = [MetaKeyName.LATITUDE, MetaKeyName.LONGITUDE] with Resource(res_file) as res: @@ -118,15 +125,18 @@ def test_wind_gen_slice(f_rev1_out, rev2_points, year, max_workers): assert site_meta[MetaKeyName.GID] == res_gid -@pytest.mark.parametrize('gid_map', - [{0: 0, 1: 1, 2: 1, 3: 3, 4: 4}, - {0: 4, 1: 3, 2: 2, 3: 1, 4: 0}, - {10: 14, 11: 13, 12: 12, 13: 11, 20: 0}, - {0: 59, 1: 1, 2: 1, 3: 0, 4: 4}, - {0: 59, 1: 1, 2: 0, 3: 0, 4: 4}, - {0: 1, 1: 1, 2: 0, 3: 0, 4: 0}, - {0: 0, 1: 0, 2: 0, 3: 0, 4: 0}, - ]) +@pytest.mark.parametrize( + "gid_map", + [ + {0: 0, 1: 1, 2: 1, 3: 3, 4: 4}, + {0: 4, 1: 3, 2: 2, 3: 1, 4: 0}, + {10: 14, 11: 13, 12: 12, 13: 11, 20: 0}, + {0: 59, 1: 1, 2: 1, 3: 0, 4: 4}, + {0: 59, 1: 1, 2: 0, 3: 0, 4: 4}, + {0: 1, 1: 1, 2: 0, 3: 0, 4: 0}, + {0: 0, 1: 0, 2: 0, 3: 0, 4: 0}, + ], +) def test_gid_map(gid_map): """Test gid mapping feature where the unique gen_gids are mapped to non-unique res_gids @@ -135,47 +145,77 @@ def test_gid_map(gid_map): points_test = sorted(list(set(gid_map.keys()))) year = 2012 max_workers = 1 - sam_files = TESTDATADIR + '/SAM/wind_gen_standard_losses_0.json' - res_file = TESTDATADIR + '/wtk/ri_100_wtk_{}.h5'.format(year) - - output_request = (MetaKeyName.CF_MEAN, , 'ws_mean', 'windspeed', - 'monthly_energy') - - baseline = Gen('windpower', points_base, sam_files, res_file, - sites_per_worker=3, output_request=output_request) + sam_files = TESTDATADIR + "/SAM/wind_gen_standard_losses_0.json" + res_file = TESTDATADIR + "/wtk/ri_100_wtk_{}.h5".format(year) + + output_request = ( + "cf_mean", + "cf_profile", + "ws_mean", + "windspeed", + "monthly_energy", + ) + + baseline = Gen( + "windpower", + points_base, + sam_files, + res_file, + sites_per_worker=3, + output_request=output_request, + ) baseline.run(max_workers=max_workers) - map_test = Gen('windpower', points_test, sam_files, res_file, - sites_per_worker=3, output_request=output_request, - gid_map=gid_map) + map_test = Gen( + "windpower", + points_test, + sam_files, + res_file, + sites_per_worker=3, + output_request=output_request, + gid_map=gid_map, + ) map_test.run(max_workers=max_workers) - write_gid_test = Gen('windpower', points_test, sam_files, res_file, - sites_per_worker=3, output_request=output_request, - gid_map=gid_map, write_mapped_gids=True) + write_gid_test = Gen( + "windpower", + points_test, + sam_files, + res_file, + sites_per_worker=3, + output_request=output_request, + gid_map=gid_map, + write_mapped_gids=True, + ) write_gid_test.run(max_workers=max_workers) for key in output_request: assert np.allclose(map_test.out[key], write_gid_test.out[key]) - for map_test_gid, write_test_gid in zip(map_test.meta[MetaKeyName.GID], - write_gid_test.meta[MetaKeyName.GID]): + for map_test_gid, write_test_gid in zip( + map_test.meta[MetaKeyName.GID], write_gid_test.meta[MetaKeyName.GID] + ): assert map_test_gid == gid_map[write_test_gid] - if len(baseline.out[MetaKeyName.CF_MEAN]) == len(map_test.out[MetaKeyName.CF_MEAN]): - assert not np.allclose(baseline.out[MetaKeyName.CF_MEAN], - map_test.out[MetaKeyName.CF_MEAN]) + if len(baseline.out["cf_mean"]) == len(map_test.out["cf_mean"]): + assert not np.allclose( + baseline.out["cf_mean"], map_test.out["cf_mean"] + ) for gen_gid_test, res_gid in gid_map.items(): gen_gid_test = points_test.index(gen_gid_test) gen_gid_base = points_base.index(res_gid) for key in output_request: if len(map_test.out[key].shape) == 2: - assert np.allclose(baseline.out[key][:, gen_gid_base], - map_test.out[key][:, gen_gid_test]) + assert np.allclose( + baseline.out[key][:, gen_gid_base], + map_test.out[key][:, gen_gid_test], + ) else: - assert np.allclose(baseline.out[key][gen_gid_base], - map_test.out[key][gen_gid_test]) + assert np.allclose( + baseline.out[key][gen_gid_base], + map_test.out[key][gen_gid_test], + ) labels = [MetaKeyName.LATITUDE, MetaKeyName.LONGITUDE] with Resource(res_file) as res: @@ -197,100 +237,126 @@ def test_gid_map(gid_map): def test_wind_gen_new_outputs(points=slice(0, 10), year=2012, max_workers=1): """Test reV 2.0 generation for wind with new outputs.""" # get full file paths. - sam_files = TESTDATADIR + '/SAM/wind_gen_standard_losses_0.json' - res_file = TESTDATADIR + '/wtk/ri_100_wtk_{}.h5'.format(year) + sam_files = TESTDATADIR + "/SAM/wind_gen_standard_losses_0.json" + res_file = TESTDATADIR + "/wtk/ri_100_wtk_{}.h5".format(year) - output_request = (MetaKeyName.CF_MEAN, , 'monthly_energy') + output_request = ("cf_mean", "cf_profile", "monthly_energy") # run reV 2.0 generation - gen = Gen('windpower', points, sam_files, res_file, sites_per_worker=3, - output_request=output_request) + gen = Gen( + "windpower", + points, + sam_files, + res_file, + sites_per_worker=3, + output_request=output_request, + ) gen.run(max_workers=max_workers) - assert gen.out[MetaKeyName.CF_MEAN].shape == (10, ) - assert gen.out[].shape == (8760, 10) - assert gen.out['monthly_energy'].shape == (12, 10) + assert gen.out["cf_mean"].shape == (10,) + assert gen.out["cf_profile"].shape == (8760, 10) + assert gen.out["monthly_energy"].shape == (12, 10) - assert gen._out[MetaKeyName.CF_MEAN].dtype == np.float32 - assert gen._out[].dtype == np.uint16 - assert gen._out['monthly_energy'].dtype == np.float32 + assert gen._out["cf_mean"].dtype == np.float32 + assert gen._out["cf_profile"].dtype == np.uint16 + assert gen._out["monthly_energy"].dtype == np.float32 -def test_windspeed_pass_through(rev2_points=slice(0, 10), year=2012, - max_workers=1): +def test_windspeed_pass_through( + rev2_points=slice(0, 10), year=2012, max_workers=1 +): """Test a windspeed output request so that resource array is passed through to output dict.""" - sam_files = TESTDATADIR + '/SAM/wind_gen_standard_losses_0.json' - res_file = TESTDATADIR + '/wtk/ri_100_wtk_{}.h5'.format(year) + sam_files = TESTDATADIR + "/SAM/wind_gen_standard_losses_0.json" + res_file = TESTDATADIR + "/wtk/ri_100_wtk_{}.h5".format(year) - output_requests = (MetaKeyName.CF_MEAN, 'windspeed') + output_requests = ("cf_mean", "windspeed") # run reV 2.0 generation - gen = Gen('windpower', rev2_points, sam_files, res_file, - sites_per_worker=3, output_request=output_requests) + gen = Gen( + "windpower", + rev2_points, + sam_files, + res_file, + sites_per_worker=3, + output_request=output_requests, + ) gen.run(max_workers=max_workers) - assert 'windspeed' in gen.out - assert gen.out['windspeed'].shape == (8760, 10) - assert gen._out['windspeed'].max() == 2597 - assert gen._out['windspeed'].min() == 1 + assert "windspeed" in gen.out + assert gen.out["windspeed"].shape == (8760, 10) + assert gen._out["windspeed"].max() == 2597 + assert gen._out["windspeed"].min() == 1 def test_multi_file_5min_wtk(): """Test running reV gen from a multi-h5 directory with prefix and suffix""" points = slice(0, 10) max_workers = 1 - sam_files = TESTDATADIR + '/SAM/wind_gen_standard_losses_0.json' - res_file = TESTDATADIR + '/wtk/wtk_{}_*m.h5'.format(2010) + sam_files = TESTDATADIR + "/SAM/wind_gen_standard_losses_0.json" + res_file = TESTDATADIR + "/wtk/wtk_{}_*m.h5".format(2010) # run reV 2.0 generation - gen = Gen('windpower', points, sam_files, res_file, sites_per_worker=3) + gen = Gen("windpower", points, sam_files, res_file, sites_per_worker=3) gen.run(max_workers=max_workers) - gen_outs = list(gen._out[MetaKeyName.CF_MEAN]) + gen_outs = list(gen._out["cf_mean"]) assert len(gen_outs) == 10 assert np.mean(gen_outs) > 0.55 def test_wind_gen_site_data(points=slice(0, 5), year=2012, max_workers=1): """Test site specific SAM input config via site_data arg""" - sam_files = TESTDATADIR + '/SAM/wind_gen_standard_losses_0.json' - res_file = TESTDATADIR + '/wtk/ri_100_wtk_{}.h5'.format(year) - - output_request = (MetaKeyName.CF_MEAN, 'turb_generic_loss') - - baseline = Gen('windpower', points, sam_files, res_file, - sites_per_worker=3, output_request=output_request) + sam_files = TESTDATADIR + "/SAM/wind_gen_standard_losses_0.json" + res_file = TESTDATADIR + "/wtk/ri_100_wtk_{}.h5".format(year) + + output_request = ("cf_mean", "turb_generic_loss") + + baseline = Gen( + "windpower", + points, + sam_files, + res_file, + sites_per_worker=3, + output_request=output_request, + ) baseline.run(max_workers=max_workers) - site_data = pd.DataFrame({MetaKeyName.GID: np.arange(2), - 'turb_generic_loss': np.zeros(2)}) - test = Gen('windpower', points, sam_files, res_file, sites_per_worker=3, - output_request=output_request, site_data=site_data) + site_data = pd.DataFrame( + {MetaKeyName.GID: np.arange(2), "turb_generic_loss": np.zeros(2)} + ) + test = Gen( + "windpower", + points, + sam_files, + res_file, + sites_per_worker=3, + output_request=output_request, + site_data=site_data, + ) test.run(max_workers=max_workers) - assert all(test.out[MetaKeyName.CF_MEAN][0:2] > baseline.out[MetaKeyName.CF_MEAN][0:2]) - assert np.allclose(test.out[MetaKeyName.CF_MEAN][2:], baseline.out[MetaKeyName.CF_MEAN][2:]) - assert np.allclose(test.out['turb_generic_loss'][0:2], np.zeros(2)) - assert np.allclose(test.out['turb_generic_loss'][2:], 16.7 * np.ones(3)) + assert all(test.out["cf_mean"][0:2] > baseline.out["cf_mean"][0:2]) + assert np.allclose(test.out["cf_mean"][2:], baseline.out["cf_mean"][2:]) + assert np.allclose(test.out["turb_generic_loss"][0:2], np.zeros(2)) + assert np.allclose(test.out["turb_generic_loss"][2:], 16.7 * np.ones(3)) def test_multi_resolution_wtk(): """Test windpower analysis with wind at 5min and t+p at hourly""" with tempfile.TemporaryDirectory() as td: - points = slice(0, 2) max_workers = 1 - sam_files = TESTDATADIR + '/SAM/wind_gen_standard_losses_0.json' - source_fp_100m = TESTDATADIR + '/wtk/wtk_2010_100m.h5' - source_fp_200m = TESTDATADIR + '/wtk/wtk_2010_200m.h5' - - fp_hr_100m = os.path.join(td, 'wtk_2010_100m_hr.h5') - fp_hr_200m = os.path.join(td, 'wtk_2010_200m_hr.h5') - fp_hr = os.path.join(td, 'wtk_2010_*hr.h5') - fp_lr = os.path.join(td, 'wtk_2010_lr.h5') + sam_files = TESTDATADIR + "/SAM/wind_gen_standard_losses_0.json" + source_fp_100m = TESTDATADIR + "/wtk/wtk_2010_100m.h5" + source_fp_200m = TESTDATADIR + "/wtk/wtk_2010_200m.h5" + + fp_hr_100m = os.path.join(td, "wtk_2010_100m_hr.h5") + fp_hr_200m = os.path.join(td, "wtk_2010_200m_hr.h5") + fp_hr = os.path.join(td, "wtk_2010_*hr.h5") + fp_lr = os.path.join(td, "wtk_2010_lr.h5") shutil.copy(source_fp_100m, fp_hr_100m) shutil.copy(source_fp_200m, fp_hr_200m) - lr_dsets = ['temperature_100m', 'pressure_100m'] + lr_dsets = ["temperature_100m", "pressure_100m"] with WindResource(fp_hr_100m) as hr_res: ti = hr_res.time_index meta = hr_res.meta @@ -306,62 +372,110 @@ def test_multi_resolution_wtk(): lr_data = [d[t_slice, s_slice] for d in lr_data] lr_shapes = {d: (len(lr_ti), len(lr_meta)) for d in lr_dsets} - Outputs.init_h5(fp_lr, lr_dsets, lr_shapes, lr_attrs, lr_chunks, - lr_dtypes, lr_meta, lr_ti) + Outputs.init_h5( + fp_lr, + lr_dsets, + lr_shapes, + lr_attrs, + lr_chunks, + lr_dtypes, + lr_meta, + lr_ti, + ) for name, arr in zip(lr_dsets, lr_data): - Outputs.add_dataset(fp_lr, name, arr, lr_dtypes[name], - attrs=lr_attrs[name], chunks=lr_chunks[name]) + Outputs.add_dataset( + fp_lr, + name, + arr, + lr_dtypes[name], + attrs=lr_attrs[name], + chunks=lr_chunks[name], + ) for fp in (fp_hr_100m, fp_hr_200m): - with h5py.File(fp, 'a') as f: + with h5py.File(fp, "a") as f: for dset in lr_dsets: if dset in f: del f[dset] # run reV 2.0 generation - gen = Gen('windpower', points, sam_files, fp_hr, - low_res_resource_file=fp_lr, - sites_per_worker=3) + gen = Gen( + "windpower", + points, + sam_files, + fp_hr, + low_res_resource_file=fp_lr, + sites_per_worker=3, + ) gen.run(max_workers=max_workers) - gen_outs = list(gen._out[MetaKeyName.CF_MEAN]) + gen_outs = list(gen._out["cf_mean"]) assert len(gen_outs) == 2 assert np.mean(gen_outs) > 0.55 def test_wind_bias_correct(): """Test rev generation with bias correction.""" - sam_files = TESTDATADIR + '/SAM/wind_gen_standard_losses_0.json' - res_file = TESTDATADIR + '/wtk/ri_100_wtk_2012.h5' + sam_files = TESTDATADIR + "/SAM/wind_gen_standard_losses_0.json" + res_file = TESTDATADIR + "/wtk/ri_100_wtk_2012.h5" # run reV 2.0 generation points = slice(0, 10) - gen_base = Gen('windpower', points, sam_files, res_file, - output_request=(MetaKeyName.CF_MEAN, , 'ws_mean'), - sites_per_worker=3) + gen_base = Gen( + "windpower", + points, + sam_files, + res_file, + output_request=("cf_mean", "cf_profile", "ws_mean"), + sites_per_worker=3, + ) gen_base.run(max_workers=1) - outs_base = np.array(list(gen_base.out[MetaKeyName.CF_MEAN])) - - bc_df = pd.DataFrame({MetaKeyName.GID: np.arange(100), 'method': 'lin_ws', - 'scalar': 1, 'adder': 2}) - gen = Gen('windpower', points, sam_files, res_file, - output_request=(MetaKeyName.CF_MEAN, , 'ws_mean'), - sites_per_worker=3, bias_correct=bc_df) + outs_base = np.array(list(gen_base.out["cf_mean"])) + + bc_df = pd.DataFrame( + { + MetaKeyName.GID: np.arange(100), + "method": "lin_ws", + "scalar": 1, + "adder": 2, + } + ) + gen = Gen( + "windpower", + points, + sam_files, + res_file, + output_request=("cf_mean", "cf_profile", "ws_mean"), + sites_per_worker=3, + bias_correct=bc_df, + ) gen.run(max_workers=1) - outs_bc = np.array(list(gen.out[MetaKeyName.CF_MEAN])) + outs_bc = np.array(list(gen.out["cf_mean"])) assert all(outs_bc > outs_base) - assert np.allclose(gen_base.out['ws_mean'] + 2, gen.out['ws_mean']) - - bc_df = pd.DataFrame({MetaKeyName.GID: np.arange(100), 'method': 'lin_ws', - 'scalar': 1, 'adder': -100}) - gen = Gen('windpower', points, sam_files, res_file, - output_request=(MetaKeyName.CF_MEAN, , 'ws_mean'), - sites_per_worker=3, bias_correct=bc_df) + assert np.allclose(gen_base.out["ws_mean"] + 2, gen.out["ws_mean"]) + + bc_df = pd.DataFrame( + { + MetaKeyName.GID: np.arange(100), + "method": "lin_ws", + "scalar": 1, + "adder": -100, + } + ) + gen = Gen( + "windpower", + points, + sam_files, + res_file, + output_request=("cf_mean", "cf_profile", "ws_mean"), + sites_per_worker=3, + bias_correct=bc_df, + ) gen.run(max_workers=1) for k, arr in gen.out.items(): assert (np.array(arr) == 0).all() -def execute_pytest(capture='all', flags='-rapP'): +def execute_pytest(capture="all", flags="-rapP"): """Execute module as pytest with detailed summary report. Parameters @@ -374,8 +488,8 @@ def execute_pytest(capture='all', flags='-rapP'): """ fname = os.path.basename(__file__) - pytest.main(['-q', '--show-capture={}'.format(capture), fname, flags]) + pytest.main(["-q", "--show-capture={}".format(capture), fname, flags]) -if __name__ == '__main__': +if __name__ == "__main__": execute_pytest() diff --git a/tests/test_gen_wind_losses.py b/tests/test_gen_wind_losses.py index fb83c320b..51d70ff91 100644 --- a/tests/test_gen_wind_losses.py +++ b/tests/test_gen_wind_losses.py @@ -58,7 +58,7 @@ def test_wind_generic_losses(loss): gen = Gen('windpower', pc, SAM_FILE, RES_FILE, sites_per_worker=3) gen.run(max_workers=1) - gen_outs = list(gen.out[MetaKeyName.CF_MEAN]) + gen_outs = list(gen.out['cf_mean']) assert np.allclose(gen_outs, LOSS_BASELINE[loss], rtol=RTOL, atol=ATOL) @@ -77,7 +77,7 @@ def test_wind_icing_losses(i): gen = Gen('windpower', pc, SAM_FILE, RES_FILE, sites_per_worker=3) gen.run(max_workers=1) - gen_outs = list(gen.out[MetaKeyName.CF_MEAN]) + gen_outs = list(gen.out['cf_mean']) assert np.allclose(gen_outs, ICING_BASELINE[i]['output'], rtol=RTOL, atol=ATOL) @@ -95,7 +95,7 @@ def test_wind_low_temp_cutoff(i): gen = Gen('windpower', pc, SAM_FILE, RES_FILE, sites_per_worker=3) gen.run(max_workers=1) - gen_outs = list(gen.out[MetaKeyName.CF_MEAN]) + gen_outs = list(gen.out['cf_mean']) assert np.allclose(gen_outs, LOW_TEMP_BASELINE[i]['output'], rtol=RTOL, atol=ATOL) diff --git a/tests/test_handlers_multiyear.py b/tests/test_handlers_multiyear.py index e3b1a2a23..40d3340c8 100644 --- a/tests/test_handlers_multiyear.py +++ b/tests/test_handlers_multiyear.py @@ -18,7 +18,7 @@ from reV.cli import main from reV.handlers.multi_year import MultiYear from reV.handlers.outputs import Outputs -from reV.utilities import MetaKeyName, ModuleName +from reV.utilities import ModuleName H5_DIR = os.path.join(TESTDATADIR, 'gen_out') YEARS = [2012, 2013] @@ -96,14 +96,14 @@ def compare_arrays(my_arr, test_arr, desc): @pytest.mark.parametrize(('source', 'dset', 'group'), [ - (H5_FILES, , None), - (H5_FILES, MetaKeyName.CF_MEAN, None), - (H5_FILES, , 'pytest'), - (H5_FILES, MetaKeyName.CF_MEAN, 'pytest'), - (H5_PATTERN, , None), - (H5_PATTERN, MetaKeyName.CF_MEAN, None), - (H5_PATTERN, , 'pytest'), - (H5_PATTERN, MetaKeyName.CF_MEAN, 'pytest'), + (H5_FILES, 'cf_profile', None), + (H5_FILES, 'cf_mean', None), + (H5_FILES, 'cf_profile', 'pytest'), + (H5_FILES, 'cf_mean', 'pytest'), + (H5_PATTERN, 'cf_profile', None), + (H5_PATTERN, 'cf_mean', None), + (H5_PATTERN, 'cf_profile', 'pytest'), + (H5_PATTERN, 'cf_mean', 'pytest'), ]) def test_my_collection(source, dset, group): """ @@ -207,8 +207,8 @@ def test_cli(runner, clear_loggers): @pytest.mark.parametrize(('dset', 'group'), [ - (MetaKeyName.CF_MEAN, None), - (MetaKeyName.CF_MEAN, 'pytest')]) + ('cf_mean', None), + ('cf_mean', 'pytest')]) def test_my_means(dset, group): """ Test computation of multi-year means @@ -237,8 +237,8 @@ def test_my_means(dset, group): @pytest.mark.parametrize(('dset', 'group'), [ - (MetaKeyName.CF_MEAN, None), - (MetaKeyName.CF_MEAN, 'pytest')]) + ('cf_mean', None), + ('cf_mean', 'pytest')]) def test_update(dset, group): """ Test computation of multi-year means @@ -278,8 +278,8 @@ def test_update(dset, group): @pytest.mark.parametrize(('dset', 'group'), [ - (MetaKeyName.CF_MEAN, None), - (MetaKeyName.CF_MEAN, 'pytest')]) + ('cf_mean', None), + ('cf_mean', 'pytest')]) def test_my_stdev(dset, group): """ Test computation of multi-year means diff --git a/tests/test_nrwal.py b/tests/test_nrwal.py index 8b92f8c41..903f5aadc 100644 --- a/tests/test_nrwal.py +++ b/tests/test_nrwal.py @@ -45,7 +45,7 @@ def test_nrwal(): with Outputs(gen_fpath, 'a') as f: f.time_index = pd_date_range('20100101', '20110101', closed='right', freq='1h') - f._add_dset(MetaKeyName.CF_PROFILE, np.random.random(f.shape), + f._add_dset('cf_profile', np.random.random(f.shape), np.uint32, attrs={'scale_factor': 1000}, chunks=(None, 10)) f._add_dset(MetaKeyName.FIXED_CHARGE_RATE, @@ -58,14 +58,14 @@ def test_nrwal(): if d not in ('meta', 'time_index')] meta_raw = f.meta lcoe_raw = f[MetaKeyName.LCOE_FCR] - cf_mean_raw = f[MetaKeyName.CF_MEAN] - cf_profile_raw = f[MetaKeyName.CF_PROFILE] + cf_mean_raw = f['cf_mean'] + cf_profile_raw = f['cf_profile'] mask = meta_raw.offshore == 1 output_request = [MetaKeyName.FIXED_CHARGE_RATE, 'depth', 'total_losses', 'array', 'export', 'gcf_adjustment', - MetaKeyName.LCOE_FCR, MetaKeyName.CF_MEAN, ] + MetaKeyName.LCOE_FCR, 'cf_mean', ] obj = RevNrwal(gen_fpath, site_data, sam_configs, nrwal_configs, output_request, site_meta_cols=['depth']) @@ -78,8 +78,8 @@ def test_nrwal(): gcf_adjustment = f['gcf_adjustment'] assert np.allclose(cf_mean_raw, f['cf_mean_raw']) assert np.allclose(cf_profile_raw, f['cf_profile_raw']) - cf_mean_new = f[MetaKeyName.CF_MEAN] - cf_profile_new = f[MetaKeyName.CF_PROFILE] + cf_mean_new = f['cf_mean'] + cf_profile_new = f['cf_profile'] fcr = f[MetaKeyName.FIXED_CHARGE_RATE] depth = f['depth'] @@ -105,8 +105,8 @@ def test_nrwal(): # make sure the second offshore compute gives same results as first with Outputs(gen_fpath, 'r') as f: assert np.allclose(lcoe_new, f[MetaKeyName.LCOE_FCR]) - assert np.allclose(cf_mean_new, f[MetaKeyName.CF_MEAN]) - assert np.allclose(cf_profile_new, f[MetaKeyName.CF_PROFILE]) + assert np.allclose(cf_mean_new, f['cf_mean']) + assert np.allclose(cf_profile_new, f['cf_profile']) assert np.allclose(cf_mean_raw, f['cf_mean_raw']) assert np.allclose(cf_profile_raw, f['cf_profile_raw']) @@ -176,7 +176,7 @@ def test_nrwal_csv(out_fn): compatible = ['depth', 'total_losses', 'array', 'export', 'gcf_adjustment', MetaKeyName.FIXED_CHARGE_RATE, - MetaKeyName.LCOE_FCR, MetaKeyName.CF_MEAN] + MetaKeyName.LCOE_FCR, 'cf_mean'] incompatible = [] output_request = compatible + incompatible @@ -224,7 +224,7 @@ def test_nrwal_constant_eq_output_request(): with Outputs(gen_fpath, 'a') as f: f.time_index = pd_date_range('20100101', '20110101', closed='right', freq='1h') - f._add_dset(MetaKeyName.CF_PROFILE, np.random.random(f.shape), + f._add_dset('cf_profile', np.random.random(f.shape), np.uint32, attrs={'scale_factor': 1000}, chunks=(None, 10)) f._add_dset(MetaKeyName.FIXED_CHARGE_RATE, @@ -236,7 +236,7 @@ def test_nrwal_constant_eq_output_request(): meta_raw = f.meta mask = meta_raw.offshore == 1 - output_request = [MetaKeyName.CF_MEAN, MetaKeyName.CF_PROFILE, + output_request = ['cf_mean', 'cf_profile', 'lease_price', 'lease_price_mil'] RevNrwal(gen_fpath, site_data, sam_configs, nrwal_configs, @@ -268,7 +268,7 @@ def test_nrwal_cli(runner, clear_loggers): with Outputs(gen_fpath, 'a') as f: f.time_index = pd_date_range('20100101', '20110101', closed='right', freq='1h') - f._add_dset(MetaKeyName.CF_PROFILE, np.random.random(f.shape), + f._add_dset('cf_profile', np.random.random(f.shape), np.uint32, attrs={'scale_factor': 1000}, chunks=(None, 10)) f._add_dset(MetaKeyName.FIXED_CHARGE_RATE, @@ -281,14 +281,14 @@ def test_nrwal_cli(runner, clear_loggers): if d not in ('meta', 'time_index')] meta_raw = f.meta lcoe_raw = f[MetaKeyName.LCOE_FCR] - cf_mean_raw = f[MetaKeyName.CF_MEAN] - cf_profile_raw = f[MetaKeyName.CF_PROFILE] + cf_mean_raw = f['cf_mean'] + cf_profile_raw = f['cf_profile'] mask = meta_raw.offshore == 1 output_request = [MetaKeyName.FIXED_CHARGE_RATE, 'depth', 'total_losses', 'array', 'export', 'gcf_adjustment', - MetaKeyName.LCOE_FCR, MetaKeyName.CF_MEAN, ] + MetaKeyName.LCOE_FCR, 'cf_mean', ] config = { "execution_control": { @@ -322,8 +322,8 @@ def test_nrwal_cli(runner, clear_loggers): gcf_adjustment = f['gcf_adjustment'] assert np.allclose(cf_mean_raw, f['cf_mean_raw']) assert np.allclose(cf_profile_raw, f['cf_profile_raw']) - cf_mean_new = f[MetaKeyName.CF_MEAN] - cf_profile_new = f[MetaKeyName.CF_PROFILE] + cf_mean_new = f['cf_mean'] + cf_profile_new = f['cf_profile'] fcr = f[MetaKeyName.FIXED_CHARGE_RATE] depth = f['depth'] @@ -353,8 +353,8 @@ def test_nrwal_cli(runner, clear_loggers): # make sure the second offshore compute gives same results as first with Outputs(gen_fpath, 'r') as f: assert np.allclose(lcoe_new, f[MetaKeyName.LCOE_FCR]) - assert np.allclose(cf_mean_new, f[MetaKeyName.CF_MEAN]) - assert np.allclose(cf_profile_new, f[MetaKeyName.CF_PROFILE]) + assert np.allclose(cf_mean_new, f['cf_mean']) + assert np.allclose(cf_profile_new, f['cf_profile']) assert np.allclose(cf_mean_raw, f['cf_mean_raw']) assert np.allclose(cf_profile_raw, f['cf_profile_raw']) @@ -426,7 +426,7 @@ def test_nrwal_cli_csv(runner, clear_loggers): output_request = [MetaKeyName.FIXED_CHARGE_RATE, 'depth', 'total_losses', 'array', 'export', 'gcf_adjustment', - MetaKeyName.LCOE_FCR, MetaKeyName.CF_MEAN, ] + MetaKeyName.LCOE_FCR, 'cf_mean', ] config = { "execution_control": { @@ -459,7 +459,7 @@ def test_nrwal_cli_csv(runner, clear_loggers): new_data = pd.read_csv(os.path.join(td, out_fn)) for col in output_request[:-1]: assert col in new_data - assert MetaKeyName.CF_PROFILE not in new_data + assert 'cf_profile' not in new_data clear_loggers() diff --git a/tests/test_qa_qc_summary.py b/tests/test_qa_qc_summary.py index 3a2a5d8c8..f663325e4 100644 --- a/tests/test_qa_qc_summary.py +++ b/tests/test_qa_qc_summary.py @@ -10,7 +10,6 @@ from reV import TESTDATADIR from reV.qa_qc.summary import SummarizeH5, SummarizeSupplyCurve -from reV.utilities import MetaKeyName H5_FILE = os.path.join(TESTDATADIR, 'gen_out', 'ri_wind_gen_profiles_2010.h5') SC_TABLE = os.path.join(TESTDATADIR, 'sc_out', 'sc_full_out_1.csv') @@ -18,7 +17,7 @@ @pytest.mark.parametrize('dataset', - [MetaKeyName.CF_MEAN, MetaKeyName.CF_PROFILE, None]) + ['cf_mean', 'cf_profile', None]) def test_summarize(dataset): """Run QA/QC Summarize and compare with baseline""" @@ -29,7 +28,7 @@ def test_summarize(dataset): 'ri_wind_gen_profiles_2010_summary.csv') baseline = pd.read_csv(baseline) test = summary.summarize_means() - elif dataset == MetaKeyName.CF_MEAN: + elif dataset == 'cf_mean': baseline = os.path.join(SUMMARY_DIR, 'cf_mean_summary.csv') baseline = pd.read_csv(baseline, index_col=0) test = summary.summarize_dset( diff --git a/tests/test_rep_profiles.py b/tests/test_rep_profiles.py index c4a2029f0..e26c8b860 100644 --- a/tests/test_rep_profiles.py +++ b/tests/test_rep_profiles.py @@ -159,7 +159,7 @@ def test_agg_profile(): MetaKeyName.TIMEZONE: timezone}) rp = RepProfiles(GEN_FPATH, rev_summary, MetaKeyName.SC_GID, - cf_dset=MetaKeyName.CF_PROFILE, err_method=None) + cf_dset='cf_profile', err_method=None) rp.run(scaled_precision=False, max_workers=1) for index in rev_summary.index: @@ -174,7 +174,7 @@ def test_agg_profile(): raw_profiles = [] for gid in res_gids: iloc = np.where(meta[MetaKeyName.GID] == gid)[0][0] - prof = np.expand_dims(res[MetaKeyName.CF_PROFILE, :, iloc], 1) + prof = np.expand_dims(res['cf_profile', :, iloc], 1) raw_profiles.append(prof) last = raw_profiles[-1].flatten() diff --git a/tests/test_sam.py b/tests/test_sam.py index 6525a4887..3cad13a41 100644 --- a/tests/test_sam.py +++ b/tests/test_sam.py @@ -103,10 +103,9 @@ def test_PV_lat_tilt(res, site_index): warnings.simplefilter("ignore") sim = PvWattsv5(resource=res_df, meta=meta, sam_sys_inputs=inputs, - output_request=(MetaKeyName.CF_MEAN,)) + output_request=('cf_mean',)) break - else: - pass + pass assert sim.sam_sys_inputs['tilt'] == meta[MetaKeyName.LATITUDE] diff --git a/tests/test_supply_curve_compute.py b/tests/test_supply_curve_compute.py index ce7339d34..0c43bdcc8 100644 --- a/tests/test_supply_curve_compute.py +++ b/tests/test_supply_curve_compute.py @@ -260,7 +260,7 @@ def test_least_cost_full(): out_fpath = os.path.join(td, "sc") sc_full = sc.run(out_fpath, fixed_charge_rate=0.1, simple=False, avail_cap_frac=0.1, - columns=[*list(SC_FULL_COLUMNS), "max_cap"] + columns=[*list(SC_FULL_COLUMNS), "max_cap"]) fpath_baseline = os.path.join(TESTDATADIR, 'sc_out/sc_full_lc.csv') baseline_verify(sc_full, fpath_baseline) diff --git a/tests/test_supply_curve_points.py b/tests/test_supply_curve_points.py index a9c718bad..6390e63cc 100644 --- a/tests/test_supply_curve_points.py +++ b/tests/test_supply_curve_points.py @@ -4,6 +4,7 @@ @author: gbuster """ + # pylint: disable=no-member import os @@ -20,22 +21,24 @@ from reV.supply_curve.sc_aggregation import SupplyCurveAggregation from reV.utilities import MetaKeyName -F_EXCL = os.path.join(TESTDATADIR, 'ri_exclusions/ri_exclusions.h5') -F_GEN = os.path.join(TESTDATADIR, 'gen_out/gen_ri_pv_2012_x000.h5') -TM_DSET = 'techmap_nsrdb' -EXCL_DICT = {'ri_srtm_slope': {'inclusion_range': (None, 5), - 'exclude_nodata': True}, - 'ri_padus': {'exclude_values': [1], - 'exclude_nodata': True}, - 'ri_reeds_regions': {'inclusion_range': (None, 400), - 'exclude_nodata': True}} - -F_TECHMAP = os.path.join(TESTDATADIR, 'sc_out/baseline_ri_tech_map.h5') -DSET_TM = 'res_ri_pv' +F_EXCL = os.path.join(TESTDATADIR, "ri_exclusions/ri_exclusions.h5") +F_GEN = os.path.join(TESTDATADIR, "gen_out/gen_ri_pv_2012_x000.h5") +TM_DSET = "techmap_nsrdb" +EXCL_DICT = { + "ri_srtm_slope": {"inclusion_range": (None, 5), "exclude_nodata": True}, + "ri_padus": {"exclude_values": [1], "exclude_nodata": True}, + "ri_reeds_regions": { + "inclusion_range": (None, 400), + "exclude_nodata": True, + }, +} + +F_TECHMAP = os.path.join(TESTDATADIR, "sc_out/baseline_ri_tech_map.h5") +DSET_TM = "res_ri_pv" RTOL = 0.001 -@pytest.mark.parametrize('resolution', [7, 32, 50, 64, 163]) +@pytest.mark.parametrize("resolution", [7, 32, 50, 64, 163]) def test_points_calc(resolution): """Test the calculation of the SC points setup from exclusions tiff.""" @@ -45,41 +48,46 @@ def test_points_calc(resolution): assert len(sc) == (sc.n_rows * sc.n_cols) -@pytest.mark.parametrize(('gids', 'resolution'), - [(range(361), 64), (range(12), 377)]) +@pytest.mark.parametrize( + ("gids", "resolution"), [(range(361), 64), (range(12), 377)] +) def test_slicer(gids, resolution): """Run tests on the different extent slicing algorithms.""" with SupplyCurveExtent(F_EXCL, resolution=resolution) as sc: - for gid in gids: row_slice0, col_slice0 = sc.get_excl_slices(gid) row_slice1, col_slice1 = SupplyCurvePoint.get_agg_slices( - gid, sc.exclusions.shape, resolution) - msg = ('Slicing failed for gid {} and res {}' - .format(gid, resolution)) + gid, sc.exclusions.shape, resolution + ) + msg = "Slicing failed for gid {} and res {}".format( + gid, resolution + ) assert row_slice0 == row_slice1, msg assert col_slice0 == col_slice1, msg -@pytest.mark.parametrize((MetaKeyName.GID, 'resolution', 'excl_dict', 'time_series'), - [(37, 64, None, None), - (37, 64, EXCL_DICT, None), - (37, 64, None, 100), - (37, 64, EXCL_DICT, 100), - (37, 37, None, None), - (37, 37, EXCL_DICT, None), - (37, 37, None, 100), - (37, 37, EXCL_DICT, 100)]) +@pytest.mark.parametrize( + (MetaKeyName.GID, "resolution", "excl_dict", "time_series"), + [ + (37, 64, None, None), + (37, 64, EXCL_DICT, None), + (37, 64, None, 100), + (37, 64, EXCL_DICT, 100), + (37, 37, None, None), + (37, 37, EXCL_DICT, None), + (37, 37, None, 100), + (37, 37, EXCL_DICT, 100), + ], +) def test_weighted_means(gid, resolution, excl_dict, time_series): - """ - Test Supply Curve Point exclusions weighted mean calculation - """ - with SupplyCurvePoint(gid, F_EXCL, TM_DSET, excl_dict=excl_dict, - resolution=resolution) as point: - shape = (point._gids.max() + 1, ) + """Test Supply Curve Point exclusions weighted mean calculation""" + with SupplyCurvePoint( + gid, F_EXCL, TM_DSET, excl_dict=excl_dict, resolution=resolution + ) as point: + shape = (point._gids.max() + 1,) if time_series: - shape = (time_series, ) + shape + shape = (time_series,) + shape arr = np.random.random(shape) means = point.exclusion_weighted_mean(arr.copy()) @@ -100,24 +108,29 @@ def test_weighted_means(gid, resolution, excl_dict, time_series): assert np.allclose(test, means, rtol=RTOL) -@pytest.mark.parametrize((MetaKeyName.GID, 'resolution', 'excl_dict', 'time_series'), - [(37, 64, None, None), - (37, 64, EXCL_DICT, None), - (37, 64, None, 100), - (37, 64, EXCL_DICT, 100), - (37, 37, None, None), - (37, 37, EXCL_DICT, None), - (37, 37, None, 100), - (37, 37, EXCL_DICT, 100)]) +@pytest.mark.parametrize( + (MetaKeyName.GID, "resolution", "excl_dict", "time_series"), + [ + (37, 64, None, None), + (37, 64, EXCL_DICT, None), + (37, 64, None, 100), + (37, 64, EXCL_DICT, 100), + (37, 37, None, None), + (37, 37, EXCL_DICT, None), + (37, 37, None, 100), + (37, 37, EXCL_DICT, 100), + ], +) def test_aggregate(gid, resolution, excl_dict, time_series): """ Test Supply Curve Point aggregate calculation """ - with SupplyCurvePoint(gid, F_EXCL, TM_DSET, excl_dict=excl_dict, - resolution=resolution) as point: - shape = (point._gids.max() + 1, ) + with SupplyCurvePoint( + gid, F_EXCL, TM_DSET, excl_dict=excl_dict, resolution=resolution + ) as point: + shape = (point._gids.max() + 1,) if time_series: - shape = (time_series, ) + shape + shape = (time_series,) + shape arr = np.random.random(shape) total = point.aggregate(arr.copy()) @@ -141,21 +154,31 @@ def plot_all_sc_points(resolution=64): """Test the calculation of the SC points setup from exclusions tiff.""" import matplotlib.pyplot as plt - prop_cycle = plt.rcParams['axes.prop_cycle'] - colors = prop_cycle.by_key()['color'] + + prop_cycle = plt.rcParams["axes.prop_cycle"] + colors = prop_cycle.by_key()["color"] _, axs = plt.subplots(1, 1) with SupplyCurveExtent(F_EXCL, resolution=resolution) as sc: colors *= len(sc) for gid in range(len(sc)): - excl_meta = sc.get_excl_points('meta', gid) - axs.scatter(excl_meta[MetaKeyName.LONGITUDE], excl_meta[MetaKeyName.LATITUDE], - c=colors[gid], s=0.01) + excl_meta = sc.get_excl_points("meta", gid) + axs.scatter( + excl_meta[MetaKeyName.LONGITUDE], + excl_meta[MetaKeyName.LATITUDE], + c=colors[gid], + s=0.01, + ) with Outputs(F_GEN) as f: - axs.scatter(f.meta[MetaKeyName.LONGITUDE], f.meta[MetaKeyName.LATITUDE], c='k', s=25) - - axs.axis('equal') + axs.scatter( + f.meta[MetaKeyName.LONGITUDE], + f.meta[MetaKeyName.LATITUDE], + c="k", + s=25, + ) + + axs.axis("equal") plt.show() @@ -163,37 +186,49 @@ def plot_single_gen_sc_point(gid=2, resolution=64): """Test the calculation of the SC points setup from exclusions tiff.""" import matplotlib.pyplot as plt - colors = ['b', 'g', 'c', 'y', 'm'] + colors = ["b", "g", "c", "y", "m"] colors *= 100 _, axs = plt.subplots(1, 1) gen_index = SupplyCurveAggregation._parse_gen_index(F_GEN) - with GenerationSupplyCurvePoint(gid, F_EXCL, F_GEN, F_TECHMAP, DSET_TM, - gen_index, - resolution=resolution) as sc: - + with GenerationSupplyCurvePoint( + gid, + F_EXCL, + F_GEN, + F_TECHMAP, + DSET_TM, + gen_index, + resolution=resolution, + ) as sc: all_gen_gids = list(set(sc._gen_gids)) - excl_meta = sc.exclusions['meta', sc.rows, sc.cols] + excl_meta = sc.exclusions["meta", sc.rows, sc.cols] for i, gen_gid in enumerate(all_gen_gids): if gen_gid != -1: - mask = (sc._gen_gids == gen_gid) - axs.scatter(excl_meta.loc[mask, MetaKeyName.LONGITUDE], - excl_meta.loc[mask, MetaKeyName.LATITUDE], - marker='s', c=colors[i], s=1) - - axs.scatter(sc.gen.meta.loc[gen_gid, MetaKeyName.LONGITUDE], - sc.gen.meta.loc[gen_gid, MetaKeyName.LATITUDE], - c='k', s=100) - - axs.scatter(sc.centroid[1], sc.centroid[0], marker='x', c='k', s=200) - - axs.axis('equal') + mask = sc._gen_gids == gen_gid + axs.scatter( + excl_meta.loc[mask, MetaKeyName.LONGITUDE], + excl_meta.loc[mask, MetaKeyName.LATITUDE], + marker="s", + c=colors[i], + s=1, + ) + + axs.scatter( + sc.gen.meta.loc[gen_gid, MetaKeyName.LONGITUDE], + sc.gen.meta.loc[gen_gid, MetaKeyName.LATITUDE], + c="k", + s=100, + ) + + axs.scatter(sc.centroid[1], sc.centroid[0], marker="x", c="k", s=200) + + axs.axis("equal") plt.show() -def execute_pytest(capture='all', flags='-rapP'): +def execute_pytest(capture="all", flags="-rapP"): """Execute module as pytest with detailed summary report. Parameters @@ -206,8 +241,8 @@ def execute_pytest(capture='all', flags='-rapP'): """ fname = os.path.basename(__file__) - pytest.main(['-q', '--show-capture={}'.format(capture), fname, flags]) + pytest.main(["-q", "--show-capture={}".format(capture), fname, flags]) -if __name__ == '__main__': +if __name__ == "__main__": execute_pytest() diff --git a/tests/test_supply_curve_tech_mapping.py b/tests/test_supply_curve_tech_mapping.py index 8c2d5d431..c795142a5 100644 --- a/tests/test_supply_curve_tech_mapping.py +++ b/tests/test_supply_curve_tech_mapping.py @@ -15,6 +15,7 @@ from reV.handlers.exclusions import ExclusionLayers from reV.handlers.outputs import Outputs from reV.supply_curve.tech_mapping import TechMapping +from reV.utilities import MetaKeyName EXCL = os.path.join(TESTDATADIR, 'ri_exclusions/ri_exclusions.h5') RES = os.path.join(TESTDATADIR, 'nsrdb/ri_100_nsrdb_2012.h5') diff --git a/tests/test_supply_curve_wind_dirs.py b/tests/test_supply_curve_wind_dirs.py index 049791651..33962344c 100644 --- a/tests/test_supply_curve_wind_dirs.py +++ b/tests/test_supply_curve_wind_dirs.py @@ -80,7 +80,7 @@ def test_sc_full_wind_dirs(downwind): def test_sc_simple_wind_dirs(downwind): """Run the simple SC test and verify results against baseline file.""" sc = SupplyCurve(SC_POINTS, TRANS_TABLE, sc_features=MULTIPLIERS) - sc_out = sc.simple_sort(fcr=0.1,transmission_costs=TRANS_COSTS, + sc_out = sc.simple_sort(fcr=0.1, transmission_costs=TRANS_COSTS, wind_dirs=WIND_DIRS, downwind=downwind) if downwind: From 477aa7328b950f5875861acf5a6c099844a9188d Mon Sep 17 00:00:00 2001 From: bnb32 Date: Sun, 12 May 2024 06:20:52 -0600 Subject: [PATCH 04/61] lint: line breaks after binary operators --- reV/SAM/SAM.py | 249 ++++++++++++++++++++--------------- reV/generation/generation.py | 2 +- reV/supply_curve/points.py | 18 +-- tests/test_gen_5min.py | 8 +- tests/test_gen_csp.py | 3 +- tests/test_gen_geothermal.py | 8 +- 6 files changed, 163 insertions(+), 125 deletions(-) diff --git a/reV/SAM/SAM.py b/reV/SAM/SAM.py index 2714d856e..46f60e0ee 100644 --- a/reV/SAM/SAM.py +++ b/reV/SAM/SAM.py @@ -3,6 +3,7 @@ Wraps the NREL-PySAM library with additional reV features. """ + import copy import json import logging @@ -42,18 +43,19 @@ class SamResourceRetriever: # Mapping for reV technology and SAM module to h5 resource handler type # SolarResource is swapped for NSRDB if the res_file contains "nsrdb" - RESOURCE_TYPES = {'geothermal': GeothermalResource, - 'pvwattsv5': SolarResource, - 'pvwattsv7': SolarResource, - 'pvwattsv8': SolarResource, - 'pvsamv1': SolarResource, - 'tcsmoltensalt': SolarResource, - 'solarwaterheat': SolarResource, - 'troughphysicalheat': SolarResource, - 'lineardirectsteam': SolarResource, - 'windpower': WindResource, - 'mhkwave': WaveResource - } + RESOURCE_TYPES = { + "geothermal": GeothermalResource, + "pvwattsv5": SolarResource, + "pvwattsv7": SolarResource, + "pvwattsv8": SolarResource, + "pvsamv1": SolarResource, + "tcsmoltensalt": SolarResource, + "solarwaterheat": SolarResource, + "troughphysicalheat": SolarResource, + "lineardirectsteam": SolarResource, + "windpower": WindResource, + "mhkwave": WaveResource, + } @staticmethod def _get_base_handler(res_file, module): @@ -80,15 +82,17 @@ def _get_base_handler(res_file, module): res_handler = SamResourceRetriever.RESOURCE_TYPES[module.lower()] except KeyError as e: - msg = ('Cannot interpret what kind of resource handler the SAM ' - 'module or reV technology "{}" requires. Expecting one of ' - 'the following SAM modules or reV technologies: {}' - .format(module, - list(SamResourceRetriever.RESOURCE_TYPES.keys()))) + msg = ( + "Cannot interpret what kind of resource handler the SAM " + 'module or reV technology "{}" requires. Expecting one of ' + "the following SAM modules or reV technologies: {}".format( + module, list(SamResourceRetriever.RESOURCE_TYPES.keys()) + ) + ) logger.exception(msg) raise SAMExecutionError(msg) from e - if res_handler == SolarResource and 'nsrdb' in res_file.lower(): + if res_handler == SolarResource and "nsrdb" in res_file.lower(): # Use NSRDB handler if definitely an NSRDB file res_handler = NSRDB @@ -124,8 +128,9 @@ def _parse_gid_map_sites(gen_gids, gid_map=None): return res_gids @classmethod - def _make_res_kwargs(cls, res_handler, project_points, output_request, - gid_map): + def _make_res_kwargs( + cls, res_handler, project_points, output_request, gid_map + ): """ Make Resource.preloadSam args and kwargs @@ -157,9 +162,9 @@ def _make_res_kwargs(cls, res_handler, project_points, output_request, kwargs = {} if res_handler in (SolarResource, NSRDB): # check for clearsky irradiation analysis for NSRDB - kwargs['clearsky'] = project_points.sam_config_obj.clearsky - kwargs['bifacial'] = project_points.sam_config_obj.bifacial - kwargs['tech'] = project_points.tech + kwargs["clearsky"] = project_points.sam_config_obj.clearsky + kwargs["bifacial"] = project_points.sam_config_obj.bifacial + kwargs["tech"] = project_points.tech downscale = project_points.sam_config_obj.downscale # check for downscaling request @@ -167,30 +172,34 @@ def _make_res_kwargs(cls, res_handler, project_points, output_request, # make sure that downscaling is only requested for NSRDB # resource if res_handler != NSRDB: - msg = ('Downscaling was requested for a non-NSRDB ' - 'resource file. reV does not have this capability ' - 'at the current time. Please contact a developer ' - 'for more information on this feature.') + msg = ( + "Downscaling was requested for a non-NSRDB " + "resource file. reV does not have this capability " + "at the current time. Please contact a developer " + "for more information on this feature." + ) logger.warning(msg) warn(msg, SAMInputWarning) else: # pass through the downscaling request - kwargs['downscale'] = downscale + kwargs["downscale"] = downscale elif res_handler == WindResource: - args += (project_points.h, ) - kwargs['icing'] = project_points.sam_config_obj.icing - if (project_points.curtailment is not None and - project_points.curtailment.precipitation): + args += (project_points.h,) + kwargs["icing"] = project_points.sam_config_obj.icing + if ( + project_points.curtailment is not None + and project_points.curtailment.precipitation + ): # make precip rate available for curtailment analysis - kwargs['precip_rate'] = True + kwargs["precip_rate"] = True elif res_handler == GeothermalResource: - args += (project_points.d, ) + args += (project_points.d,) # Check for resource means - if any(req.endswith('_mean') for req in output_request): - kwargs['means'] = True + if any(req.endswith("_mean") for req in output_request): + kwargs["means"] = True return kwargs, args @@ -229,9 +238,17 @@ def _multi_file_mods(res_handler, kwargs, res_file): return res_handler, kwargs, res_file @classmethod - def get(cls, res_file, project_points, module, - output_request=('cf_mean', ), gid_map=None, - lr_res_file=None, nn_map=None, bias_correct=None): + def get( + cls, + res_file, + project_points, + module, + output_request=("cf_mean",), + gid_map=None, + lr_res_file=None, + nn_map=None, + bias_correct=None, + ): """Get the SAM resource iterator object (single year, single file). Parameters @@ -291,27 +308,30 @@ def get(cls, res_file, project_points, module, """ res_handler = cls._get_base_handler(res_file, module) - kwargs, args = cls._make_res_kwargs(res_handler, project_points, - output_request, gid_map) + kwargs, args = cls._make_res_kwargs( + res_handler, project_points, output_request, gid_map + ) multi_h5_res, hsds = check_res_file(res_file) if multi_h5_res: - res_handler, kwargs, res_file = cls._multi_file_mods(res_handler, - kwargs, - res_file) + res_handler, kwargs, res_file = cls._multi_file_mods( + res_handler, kwargs, res_file + ) else: - kwargs['hsds'] = hsds + kwargs["hsds"] = hsds - kwargs['time_index_step'] = \ + kwargs["time_index_step"] = ( project_points.sam_config_obj.time_index_step + ) if lr_res_file is None: res = res_handler.preload_SAM(res_file, *args, **kwargs) else: - kwargs['handler_class'] = res_handler - kwargs['nn_map'] = nn_map - res = MultiResolutionResource.preload_SAM(res_file, lr_res_file, - *args, **kwargs) + kwargs["handler_class"] = res_handler + kwargs["nn_map"] = nn_map + res = MultiResolutionResource.preload_SAM( + res_file, lr_res_file, *args, **kwargs + ) if bias_correct is not None: res.bias_correct(bias_correct) @@ -326,15 +346,15 @@ class Sam: PYSAM = generic # callable attributes to be ignored in the get/set logic - IGNORE_ATTRS = ['assign', 'execute', 'export'] + IGNORE_ATTRS = ["assign", "execute", "export"] def __init__(self): self._pysam = self.PYSAM.new() self._attr_dict = None self._inputs = [] self.sam_sys_inputs = {} - if 'constant' in self.input_list: - self['constant'] = 0.0 + if "constant" in self.input_list: + self["constant"] = 0.0 def __getitem__(self, key): """Get the value of a PySAM attribute (either input or output). @@ -370,9 +390,10 @@ def __setitem__(self, key, value): """ if key not in self.input_list: - msg = ('Could not set input key "{}". Attribute not ' - 'found in PySAM object: "{}"' - .format(key, self.pysam)) + msg = ( + 'Could not set input key "{}". Attribute not ' + 'found in PySAM object: "{}"'.format(key, self.pysam) + ) logger.exception(msg) raise SAMInputError(msg) self.sam_sys_inputs[key] = value @@ -380,11 +401,14 @@ def __setitem__(self, key, value): try: setattr(getattr(self.pysam, group), key, value) except Exception as e: - msg = ('Could not set input key "{}" to ' - 'group "{}" in "{}".\n' - 'Data is: {} ({})\n' - 'Received the following error: "{}"' - .format(key, group, self.pysam, value, type(value), e)) + msg = ( + 'Could not set input key "{}" to ' + 'group "{}" in "{}".\n' + "Data is: {} ({})\n" + 'Received the following error: "{}"'.format( + key, group, self.pysam, value, type(value), e + ) + ) logger.exception(msg) raise SAMInputError(msg) from e @@ -401,7 +425,7 @@ def default(cls): ------- PySAM.GenericSystem """ - obj = cls.PYSAM.default('GenericSystemNone') + obj = cls.PYSAM.default("GenericSystemNone") obj.execute() return obj @@ -419,8 +443,9 @@ def attr_dict(self): """ if self._attr_dict is None: keys = self._get_pysam_attrs(self.pysam) - self._attr_dict = {k: self._get_pysam_attrs(getattr(self.pysam, k)) - for k in keys} + self._attr_dict = { + k: self._get_pysam_attrs(getattr(self.pysam, k)) for k in keys + } return self._attr_dict @@ -435,7 +460,7 @@ def input_list(self): """ if not any(self._inputs): for k, v in self.attr_dict.items(): - if k.lower() != 'outputs': + if k.lower() != "outputs": self._inputs += v return self._inputs @@ -460,8 +485,7 @@ def _get_group(self, key, outputs=True): temp = self.attr_dict if not outputs: - temp = {k: v for (k, v) in temp.items() - if k.lower() != 'outputs'} + temp = {k: v for (k, v) in temp.items() if k.lower() != "outputs"} for k, v in temp.items(): if key in v: @@ -484,8 +508,11 @@ def _get_pysam_attrs(self, obj): List of attrs belonging to obj with dunder attrs and IGNORE_ATTRS not included. """ - attrs = [a for a in dir(obj) if not a.startswith('__') - and a not in self.IGNORE_ATTRS] + attrs = [ + a + for a in dir(obj) + if not a.startswith("__") and a not in self.IGNORE_ATTRS + ] return attrs def execute(self): @@ -516,19 +543,21 @@ def _filter_inputs(key, value): Filtered Input value associated with key. """ - if '.' in key: - key = key.replace('.', '_') + if "." in key: + key = key.replace(".", "_") - if ':constant' in key and 'adjust:' in key: - key = key.replace('adjust:', '') + if ":constant" in key and "adjust:" in key: + key = key.replace("adjust:", "") - if isinstance(value, str) and '[' in value and ']' in value: + if isinstance(value, str) and "[" in value and "]" in value: try: value = json.loads(value) except json.JSONDecodeError: - msg = ('Found a weird SAM config input for "{}" that looks ' - 'like a stringified-list but could not run through ' - 'json.loads() so skipping: {}'.format(key, value)) + msg = ( + 'Found a weird SAM config input for "{}" that looks ' + "like a stringified-list but could not run through " + "json.loads() so skipping: {}".format(key, value) + ) logger.warning(msg) warn(msg) @@ -551,8 +580,7 @@ def assign_inputs(self, inputs, raise_warning=False): if k in self.input_list and v is not None: self[k] = v elif raise_warning: - wmsg = ('Not setting input "{}" to: {}.' - .format(k, v)) + wmsg = 'Not setting input "{}" to: {}.'.format(k, v) warn(wmsg, SAMInputWarning) logger.warning(wmsg) @@ -563,8 +591,9 @@ class RevPySam(Sam): DIR = os.path.dirname(os.path.realpath(__file__)) MODULE = None - def __init__(self, meta, sam_sys_inputs, output_request, - site_sys_inputs=None): + def __init__( + self, meta, sam_sys_inputs, output_request, site_sys_inputs=None + ): """Initialize a SAM object. Parameters @@ -633,11 +662,13 @@ def drop_leap(resource): Resource dataframe with all February 29th timesteps removed. """ - if hasattr(resource, 'index'): - if (hasattr(resource.index, 'month') - and hasattr(resource.index, 'day')): - leap_day = ((resource.index.month == 2) - & (resource.index.day == 29)) + if hasattr(resource, "index"): + if hasattr(resource.index, "month") and hasattr( + resource.index, "day" + ): + leap_day = (resource.index.month == 2) & ( + resource.index.day == 29 + ) resource = resource.drop(resource.index[leap_day]) return resource @@ -661,13 +692,15 @@ def ensure_res_len(arr, time_index): arr : ndarray Truncated array of data such that there are 365 days """ - msg = ('A valid time_index must be supplied to ensure the proper ' - 'resource length! Instead {} was supplied' - .format(type(time_index))) + msg = ( + "A valid time_index must be supplied to ensure the proper " + "resource length! Instead {} was supplied".format(type(time_index)) + ) assert isinstance(time_index, pd.DatetimeIndex) - msg = ('arr length {} does not match time_index length {}!' - .format(len(arr), len(time_index))) + msg = "arr length {} does not match time_index length {}!".format( + len(arr), len(time_index) + ) assert len(arr) == len(time_index) if time_index.is_leap_year.all(): @@ -679,16 +712,18 @@ def ensure_res_len(arr, time_index): s = np.where(mask)[0][-1] freq = pd.infer_freq(time_index[:s]) - msg = 'frequencies do not match before and after 2/29' - assert freq == pd.infer_freq(time_index[s + 1:]), msg + msg = "frequencies do not match before and after 2/29" + assert freq == pd.infer_freq(time_index[s + 1 :]), msg else: freq = pd.infer_freq(time_index) else: freq = pd.infer_freq(time_index) if freq is None: - msg = ('Resource time_index does not have a consistent time-step ' - '(frequency)!') + msg = ( + "Resource time_index does not have a consistent time-step " + "(frequency)!" + ) logger.error(msg) raise ResourceError(msg) @@ -706,7 +741,7 @@ def ensure_res_len(arr, time_index): @staticmethod def make_datetime(series): """Ensure that pd series is a datetime series with dt accessor""" - if not hasattr(series, 'dt'): + if not hasattr(series, "dt"): series = pd.to_datetime(pd.Series(series)) return series @@ -763,8 +798,9 @@ def _parse_meta(meta): """ if isinstance(meta, pd.DataFrame): - msg = ('Meta data must only be for a single site but received: {}' - .format(meta)) + msg = "Meta data must only be for a single site but received: {}".format( + meta + ) assert len(meta) == 1, msg meta = meta.iloc[0] @@ -823,8 +859,9 @@ def outputs_to_utc_arr(self): output = output.astype(np.int32) if self._is_hourly(output): - n_roll = int(-1 * self.meta['timezone'] - * self.time_interval) + n_roll = int( + -1 * self.meta["timezone"] * self.time_interval + ) output = np.roll(output, n_roll) self.outputs[key] = output @@ -851,8 +888,9 @@ def collect_outputs(self, output_lookup): bad_requests.append(req) if any(bad_requests): - msg = ('Could not retrieve outputs "{}" from PySAM object "{}".' - .format(bad_requests, self.pysam)) + msg = 'Could not retrieve outputs "{}" from PySAM object "{}".'.format( + bad_requests, self.pysam + ) logger.error(msg) raise SAMExecutionError(msg) @@ -869,9 +907,10 @@ def execute(self): try: self.pysam.execute() except Exception as e: - msg = ('PySAM raised an error while executing: "{}"' - .format(self.module)) + msg = 'PySAM raised an error while executing: "{}"'.format( + self.module + ) if self.site is not None: - msg += ' for site {}'.format(self.site) + msg += " for site {}".format(self.site) logger.exception(msg) raise SAMExecutionError(msg) from e diff --git a/reV/generation/generation.py b/reV/generation/generation.py index 91dbe7462..7c99361a8 100644 --- a/reV/generation/generation.py +++ b/reV/generation/generation.py @@ -547,7 +547,7 @@ def handle_lifetime_index(self, ti): var for var, attrs in GEN_ATTRS.items() if attrs['type'] == 'array' ] - valid_vars = [MetaKeyName.GEN_PROFILE, 'cf_profile', + valid_vars = ['gen_profile', 'cf_profile', 'cf_profile_ac'] invalid_vars = set(array_vars) - set(valid_vars) invalid_requests = [var for var in self.output_request diff --git a/reV/supply_curve/points.py b/reV/supply_curve/points.py index 395ca6b7a..8a67e6c27 100644 --- a/reV/supply_curve/points.py +++ b/reV/supply_curve/points.py @@ -2020,22 +2020,22 @@ def point_summary(self, args=None): """ ARGS = { - MetaKeyName.LATITUDE: self.sc_point.latitude, - MetaKeyName.LONGITUDE: self.sc_point.longitude, - MetaKeyName.TIMEZONE: self.sc_point.timezone, - MetaKeyName.COUNTRY: self.sc_point.country, - MetaKeyName.STATE: self.sc_point.state, - MetaKeyName.COUNTY: self.sc_point.county, - MetaKeyName.ELEVATION: self.sc_point.elevation, + MetaKeyName.LATITUDE: self.latitude, + MetaKeyName.LONGITUDE: self.longitude, + MetaKeyName.TIMEZONE: self.timezone, + MetaKeyName.COUNTRY: self.country, + MetaKeyName.STATE: self.state, + MetaKeyName.COUNTY: self.county, + MetaKeyName.ELEVATION: self.elevation, MetaKeyName.RES_GIDS: self.res_gid_set, MetaKeyName.GEN_GIDS: self.gen_gid_set, MetaKeyName.GID_COUNTS: self.gid_counts, - MetaKeyName.N_GIDS: self.sc_point.n_gids, + MetaKeyName.N_GIDS: self.n_gids, MetaKeyName.MEAN_CF: self.mean_cf, MetaKeyName.MEAN_LCOE: self.mean_lcoe, MetaKeyName.MEAN_RES: self.mean_res, MetaKeyName.CAPACITY: self.capacity, - MetaKeyName.AREA_SQ_KM: self.sc_point.area} + MetaKeyName.AREA_SQ_KM: self.area} extra_atts = [MetaKeyName.CAPACITY_AC, MetaKeyName.OFFSHORE, diff --git a/tests/test_gen_5min.py b/tests/test_gen_5min.py index 8d2eda4fd..3078c00f6 100644 --- a/tests/test_gen_5min.py +++ b/tests/test_gen_5min.py @@ -28,16 +28,16 @@ def test_gen_downscaling(): # run reV 2.0 generation gen = Gen('pvwattsv5', slice(0, None), sam_files, res_file, - output_request=('cf_mean', ), sites_per_worker=100) + output_request=('cf_mean', 'cf_profile'), sites_per_worker=100) gen.run(max_workers=1) - gen_outs = gen.out[].astype(np.int32) + gen_outs = gen.out['cf_profile'].astype(np.int32) if not os.path.exists(baseline): with h5py.File(baseline, 'w') as f: - f.create_dataset(, data=gen_outs, dtype=gen_outs.dtype) + f.create_dataset('cf_profile', data=gen_outs, dtype=gen_outs.dtype) else: with h5py.File(baseline, 'r') as f: - baseline = f[][...].astype(np.int32) + baseline = f['cf_profile'][...].astype(np.int32) x = mae_perc(gen_outs, baseline) msg = 'Mean absolute error is {}% from the baseline data'.format(x) diff --git a/tests/test_gen_csp.py b/tests/test_gen_csp.py index 82c8f6570..b7e96b125 100644 --- a/tests/test_gen_csp.py +++ b/tests/test_gen_csp.py @@ -15,7 +15,6 @@ from reV import TESTDATADIR from reV.generation.generation import Gen -from reV.utilities import MetaKeyName BASELINE = os.path.join(TESTDATADIR, 'gen_out', 'gen_ri_csp_2012.h5') RTOL = 0.1 @@ -30,7 +29,7 @@ def test_gen_csp(): # run reV 2.0 generation output_request = ('cf_mean', 'cf_profile', - MetaKeyName.GEN_PROFILE) + 'gen_profile') gen = Gen('tcsmoltensalt', points, sam_files, res_file, output_request=output_request, sites_per_worker=1, scale_outputs=True) diff --git a/tests/test_gen_geothermal.py b/tests/test_gen_geothermal.py index e57ebf2ff..cbca81b7d 100644 --- a/tests/test_gen_geothermal.py +++ b/tests/test_gen_geothermal.py @@ -65,7 +65,7 @@ def test_gen_geothermal(depth, sample_resource_data): output_request = ('annual_energy', 'cf_mean', 'cf_profile', - MetaKeyName.GEN_PROFILE, + 'gen_profile', MetaKeyName.LCOE_FCR, 'nameplate') gen = Gen('geothermal', points, geo_sam_file, geo_res_file, @@ -99,7 +99,7 @@ def test_gen_geothermal_temp_too_low(sample_resource_data): output_request = ('annual_energy', 'cf_mean', 'cf_profile', - MetaKeyName.GEN_PROFILE, + 'gen_profile', MetaKeyName.LCOE_FCR, 'nameplate') gen = Gen('geothermal', points, geo_sam_file, geo_res_file, output_request=output_request, sites_per_worker=1, @@ -215,7 +215,7 @@ def test_gen_with_nameplate_input(sample_resource_data): output_request = ('annual_energy', 'cf_mean', 'cf_profile', - MetaKeyName.GEN_PROFILE, + 'gen_profile', MetaKeyName.LCOE_FCR, 'nameplate') gen = Gen('geothermal', points, geo_sam_file, geo_res_file, output_request=output_request, sites_per_worker=1, @@ -367,7 +367,7 @@ def test_gen_with_time_index_step_input(sample_resource_data): output_request = ('annual_energy', 'cf_mean', 'cf_profile', - MetaKeyName.GEN_PROFILE, + 'gen_profile', MetaKeyName.LCOE_FCR, 'nameplate') gen = Gen('geothermal', points, geo_sam_file, geo_res_file, output_request=output_request, sites_per_worker=1, From 227e1af72a448cfe18c698f6d4e6ededcda79b40 Mon Sep 17 00:00:00 2001 From: bnb32 Date: Sun, 12 May 2024 06:28:25 -0600 Subject: [PATCH 05/61] lint: lines too long --- reV/SAM/SAM.py | 12 +- reV/supply_curve/points.py | 8 +- tests/test_gen_geothermal.py | 383 +++++++++++++++++++++++------------ 3 files changed, 263 insertions(+), 140 deletions(-) diff --git a/reV/SAM/SAM.py b/reV/SAM/SAM.py index 46f60e0ee..7496538e0 100644 --- a/reV/SAM/SAM.py +++ b/reV/SAM/SAM.py @@ -713,7 +713,7 @@ def ensure_res_len(arr, time_index): freq = pd.infer_freq(time_index[:s]) msg = "frequencies do not match before and after 2/29" - assert freq == pd.infer_freq(time_index[s + 1 :]), msg + assert freq == pd.infer_freq(time_index[s + 1:]), msg else: freq = pd.infer_freq(time_index) else: @@ -798,8 +798,9 @@ def _parse_meta(meta): """ if isinstance(meta, pd.DataFrame): - msg = "Meta data must only be for a single site but received: {}".format( - meta + msg = ( + "Meta data must only be for a single site but received: {}" + .format(meta) ) assert len(meta) == 1, msg meta = meta.iloc[0] @@ -888,9 +889,8 @@ def collect_outputs(self, output_lookup): bad_requests.append(req) if any(bad_requests): - msg = 'Could not retrieve outputs "{}" from PySAM object "{}".'.format( - bad_requests, self.pysam - ) + msg = ('Could not retrieve outputs "{}" from PySAM object "{}".' + .format(bad_requests, self.pysam)) logger.error(msg) raise SAMExecutionError(msg) diff --git a/reV/supply_curve/points.py b/reV/supply_curve/points.py index 8a67e6c27..622926ea9 100644 --- a/reV/supply_curve/points.py +++ b/reV/supply_curve/points.py @@ -1836,8 +1836,8 @@ def sc_point_capital_cost(self): return None cap_cost_per_mw = ( - self.mean_h5_dsets_data['capital_cost'] / - self.mean_h5_dsets_data['system_capacity']) + self.mean_h5_dsets_data['capital_cost'] + / self.mean_h5_dsets_data['system_capacity']) return cap_cost_per_mw * self.capacity @property @@ -1864,8 +1864,8 @@ def sc_point_fixed_operating_cost(self): return None fixed_cost_per_mw = ( - self.mean_h5_dsets_data['fixed_operating_cost'] / - self.mean_h5_dsets_data['system_capacity']) + self.mean_h5_dsets_data['fixed_operating_cost'] + / self.mean_h5_dsets_data['system_capacity']) return fixed_cost_per_mw * self.capacity @property diff --git a/tests/test_gen_geothermal.py b/tests/test_gen_geothermal.py index cbca81b7d..c495a4cd8 100644 --- a/tests/test_gen_geothermal.py +++ b/tests/test_gen_geothermal.py @@ -3,6 +3,7 @@ """ PyTest file for geothermal generation. """ + import json import os import shutil @@ -18,7 +19,7 @@ from reV.SAM.generation import Geothermal from reV.utilities import MetaKeyName -DEFAULT_GEO_SAM_FILE = TESTDATADIR + '/SAM/geothermal_default.json' +DEFAULT_GEO_SAM_FILE = TESTDATADIR + "/SAM/geothermal_default.json" RTOL = 0.1 ATOL = 0.01 @@ -26,31 +27,42 @@ @pytest.fixture def sample_resource_data(request): """Create geothermal resource data for testing""" - meta = pd.DataFrame({"latitude": [41.29], "longitude": [-71.86], - "timezone": [-5]}) + meta = pd.DataFrame( + {"latitude": [41.29], "longitude": [-71.86], "timezone": [-5]} + ) meta.index.name = "gid" with TemporaryDirectory() as td: geo_sam_file = os.path.join(td, "geothermal_sam.json") geo_res_file = os.path.join(td, "test_geo.h5") - with Outputs(geo_res_file, 'w') as f: + with Outputs(geo_res_file, "w") as f: f.meta = meta - f.time_index = pd.date_range(start='1/1/2018', end='1/1/2019', - freq='H')[:-1] - - Outputs.add_dataset(geo_res_file, 'temperature_2000m', - np.array([request.param["temp"]]), - np.float32, attrs={"units": "C"}) - Outputs.add_dataset(geo_res_file, 'potential_MW_2000m', - np.array([request.param["potential"]]), - np.float32, attrs={"units": "MW"}) + f.time_index = pd.date_range( + start="1/1/2018", end="1/1/2019", freq="H" + )[:-1] + + Outputs.add_dataset( + geo_res_file, + "temperature_2000m", + np.array([request.param["temp"]]), + np.float32, + attrs={"units": "C"}, + ) + Outputs.add_dataset( + geo_res_file, + "potential_MW_2000m", + np.array([request.param["potential"]]), + np.float32, + attrs={"units": "MW"}, + ) yield geo_sam_file, geo_res_file @pytest.mark.parametrize("depth", [2000, 4500]) -@pytest.mark.parametrize("sample_resource_data", - [{"temp": 150, "potential": 200}], indirect=True) +@pytest.mark.parametrize( + "sample_resource_data", [{"temp": 150, "potential": 200}], indirect=True +) def test_gen_geothermal(depth, sample_resource_data): """Test generation for geothermal module""" points = slice(0, 1) @@ -63,66 +75,100 @@ def test_gen_geothermal(depth, sample_resource_data): with open(geo_sam_file, "w") as fh: json.dump(geo_config, fh) - output_request = ('annual_energy', 'cf_mean', - 'cf_profile', - 'gen_profile', - MetaKeyName.LCOE_FCR, - 'nameplate') - gen = Gen('geothermal', points, geo_sam_file, geo_res_file, - output_request=output_request, sites_per_worker=1, - scale_outputs=True) + output_request = ( + "annual_energy", + "cf_mean", + "cf_profile", + "gen_profile", + MetaKeyName.LCOE_FCR, + "nameplate", + ) + gen = Gen( + "geothermal", + points, + geo_sam_file, + geo_res_file, + output_request=output_request, + sites_per_worker=1, + scale_outputs=True, + ) gen.run(max_workers=1) - truth_vals = {"annual_energy": 1.74e+09, "cf_mean": 0.993, - "cf_profile": 0.993, "gen_profile": 198653.64, - "lcoe_fcr": 12.52, "nameplate": 200_000, - "resource_temp": 150} + truth_vals = { + "annual_energy": 1.74e09, + "cf_mean": 0.993, + "cf_profile": 0.993, + "gen_profile": 198653.64, + "lcoe_fcr": 12.52, + "nameplate": 200_000, + "resource_temp": 150, + } for dset in output_request: truth = truth_vals[dset] test = gen.out[dset] if len(test.shape) == 2: test = np.mean(test, axis=0) - msg = ('{} outputs do not match baseline value! Values differ ' - 'at most by: {}' - .format(dset, np.max(np.abs(truth - test)))) + msg = ( + "{} outputs do not match baseline value! Values differ " + "at most by: {}".format(dset, np.max(np.abs(truth - test))) + ) assert np.allclose(truth, test, rtol=RTOL, atol=ATOL), msg -@pytest.mark.parametrize("sample_resource_data", - [{"temp": 60, "potential": 200}], indirect=True) +@pytest.mark.parametrize( + "sample_resource_data", [{"temp": 60, "potential": 200}], indirect=True +) def test_gen_geothermal_temp_too_low(sample_resource_data): """Test generation for geothermal module when temp too low""" points = slice(0, 1) geo_sam_file, geo_res_file = sample_resource_data shutil.copy(DEFAULT_GEO_SAM_FILE, geo_sam_file) - output_request = ('annual_energy', 'cf_mean', - 'cf_profile', - 'gen_profile', - MetaKeyName.LCOE_FCR, 'nameplate') - gen = Gen('geothermal', points, geo_sam_file, geo_res_file, - output_request=output_request, sites_per_worker=1, - scale_outputs=True) + output_request = ( + "annual_energy", + "cf_mean", + "cf_profile", + "gen_profile", + MetaKeyName.LCOE_FCR, + "nameplate", + ) + gen = Gen( + "geothermal", + points, + geo_sam_file, + geo_res_file, + output_request=output_request, + sites_per_worker=1, + scale_outputs=True, + ) gen.run(max_workers=1) - truth_vals = {"annual_energy": 0, "cf_mean": 0, "cf_profile": 0, - "gen_profile": 0, "lcoe_fcr": 0, "nameplate": 0, - "resource_temp": 60} + truth_vals = { + "annual_energy": 0, + "cf_mean": 0, + "cf_profile": 0, + "gen_profile": 0, + "lcoe_fcr": 0, + "nameplate": 0, + "resource_temp": 60, + } for dset in output_request: truth = truth_vals[dset] test = gen.out[dset] if len(test.shape) == 2: test = np.mean(test, axis=0) - msg = ('{} outputs do not match baseline value! Values differ ' - 'at most by: {}' - .format(dset, np.max(np.abs(truth - test)))) + msg = ( + "{} outputs do not match baseline value! Values differ " + "at most by: {}".format(dset, np.max(np.abs(truth - test))) + ) assert np.allclose(truth, test, rtol=RTOL, atol=ATOL), msg -@pytest.mark.parametrize("sample_resource_data", - [{"temp": 150, "potential": 100}], indirect=True) +@pytest.mark.parametrize( + "sample_resource_data", [{"temp": 150, "potential": 100}], indirect=True +) def test_per_kw_cost_inputs(sample_resource_data): """Test per_kw cost inputs for geothermal module""" points = slice(0, 1) @@ -139,28 +185,40 @@ def test_per_kw_cost_inputs(sample_resource_data): with open(geo_sam_file, "w") as fh: json.dump(geo_config, fh) - output_request = (MetaKeyName.CAPITAL_COST, - MetaKeyName.FIXED_OPERATING_COST, - MetaKeyName.LCOE_FCR) - gen = Gen('geothermal', points, geo_sam_file, geo_res_file, - output_request=output_request, sites_per_worker=1, - scale_outputs=True) + output_request = ( + MetaKeyName.CAPITAL_COST, + MetaKeyName.FIXED_OPERATING_COST, + MetaKeyName.LCOE_FCR, + ) + gen = Gen( + "geothermal", + points, + geo_sam_file, + geo_res_file, + output_request=output_request, + sites_per_worker=1, + scale_outputs=True, + ) gen.run(max_workers=1) - truth_vals = {"capital_cost": 383_086_656, - "fixed_operating_cost": 25539104, - "lcoe_fcr": 72.5092} + truth_vals = { + "capital_cost": 383_086_656, + "fixed_operating_cost": 25539104, + "lcoe_fcr": 72.5092, + } for dset in output_request: truth = truth_vals[dset] test = gen.out[dset] - msg = ('{} outputs do not match baseline value! Values differ ' - 'at most by: {}' - .format(dset, np.max(np.abs(truth - test)))) + msg = ( + "{} outputs do not match baseline value! Values differ " + "at most by: {}".format(dset, np.max(np.abs(truth - test))) + ) assert np.allclose(truth, test, rtol=1e-6, atol=ATOL), msg -@pytest.mark.parametrize("sample_resource_data", - [{"temp": 150, "potential": 100}], indirect=True) +@pytest.mark.parametrize( + "sample_resource_data", [{"temp": 150, "potential": 100}], indirect=True +) def test_drill_cost_inputs(sample_resource_data): """Test per_kw cost inputs for geothermal module""" points = slice(0, 1) @@ -178,28 +236,40 @@ def test_drill_cost_inputs(sample_resource_data): with open(geo_sam_file, "w") as fh: json.dump(geo_config, fh) - output_request = (MetaKeyName.CAPITAL_COST, - MetaKeyName.FIXED_OPERATING_COST, - MetaKeyName.LCOE_FCR) - gen = Gen('geothermal', points, geo_sam_file, geo_res_file, - output_request=output_request, sites_per_worker=1, - scale_outputs=True) + output_request = ( + MetaKeyName.CAPITAL_COST, + MetaKeyName.FIXED_OPERATING_COST, + MetaKeyName.LCOE_FCR, + ) + gen = Gen( + "geothermal", + points, + geo_sam_file, + geo_res_file, + output_request=output_request, + sites_per_worker=1, + scale_outputs=True, + ) gen.run(max_workers=1) - truth_vals = {"capital_cost": 466_134_733, - "fixed_operating_cost": 25539104, - "lcoe_fcr": 81.8643} + truth_vals = { + "capital_cost": 466_134_733, + "fixed_operating_cost": 25539104, + "lcoe_fcr": 81.8643, + } for dset in output_request: truth = truth_vals[dset] test = gen.out[dset] - msg = ('{} outputs do not match baseline value! Values differ ' - 'at most by: {}' - .format(dset, np.max(np.abs(truth - test)))) + msg = ( + "{} outputs do not match baseline value! Values differ " + "at most by: {}".format(dset, np.max(np.abs(truth - test))) + ) assert np.allclose(truth, test, rtol=1e-6, atol=ATOL), msg -@pytest.mark.parametrize("sample_resource_data", - [{"temp": 150, "potential": 20}], indirect=True) +@pytest.mark.parametrize( + "sample_resource_data", [{"temp": 150, "potential": 20}], indirect=True +) def test_gen_with_nameplate_input(sample_resource_data): """Test generation for geothermal module with user nameplate input""" points = slice(0, 1) @@ -213,33 +283,50 @@ def test_gen_with_nameplate_input(sample_resource_data): with open(geo_sam_file, "w") as fh: json.dump(geo_config, fh) - output_request = ('annual_energy', 'cf_mean', - 'cf_profile', - 'gen_profile', - MetaKeyName.LCOE_FCR, 'nameplate') - gen = Gen('geothermal', points, geo_sam_file, geo_res_file, - output_request=output_request, sites_per_worker=1, - scale_outputs=True) + output_request = ( + "annual_energy", + "cf_mean", + "cf_profile", + "gen_profile", + MetaKeyName.LCOE_FCR, + "nameplate", + ) + gen = Gen( + "geothermal", + points, + geo_sam_file, + geo_res_file, + output_request=output_request, + sites_per_worker=1, + scale_outputs=True, + ) gen.run(max_workers=1) - truth_vals = {"annual_energy": 3.47992e+08, "cf_mean": 0.993, - "cf_profile": 0.993, "gen_profile": 39725.117, - "lcoe_fcr": 62.613, "nameplate": 40_000, - "resource_temp": 150} + truth_vals = { + "annual_energy": 3.47992e08, + "cf_mean": 0.993, + "cf_profile": 0.993, + "gen_profile": 39725.117, + "lcoe_fcr": 62.613, + "nameplate": 40_000, + "resource_temp": 150, + } for dset in output_request: truth = truth_vals[dset] test = gen.out[dset] if len(test.shape) == 2: test = np.mean(test, axis=0) - msg = ('{} outputs do not match baseline value! Values differ ' - 'at most by: {}' - .format(dset, np.max(np.abs(truth - test)))) + msg = ( + "{} outputs do not match baseline value! Values differ " + "at most by: {}".format(dset, np.max(np.abs(truth - test))) + ) assert np.allclose(truth, test, rtol=RTOL, atol=ATOL), msg -@pytest.mark.parametrize("sample_resource_data", - [{"temp": 150, "potential": 20}], indirect=True) +@pytest.mark.parametrize( + "sample_resource_data", [{"temp": 150, "potential": 20}], indirect=True +) def test_gen_egs_too_high_egs_plant_design_temp(sample_resource_data): """Test generation for EGS too high plant design temp""" points = slice(0, 1) @@ -254,11 +341,17 @@ def test_gen_egs_too_high_egs_plant_design_temp(sample_resource_data): with open(geo_sam_file, "w") as fh: json.dump(geo_config, fh) - output_request = ('design_temp',) + output_request = ("design_temp",) with pytest.warns(UserWarning): - gen = Gen('geothermal', points, geo_sam_file, geo_res_file, - output_request=output_request, sites_per_worker=1, - scale_outputs=True) + gen = Gen( + "geothermal", + points, + geo_sam_file, + geo_res_file, + output_request=output_request, + sites_per_worker=1, + scale_outputs=True, + ) gen.run(max_workers=1) truth_vals = {"design_temp": 150} @@ -268,15 +361,18 @@ def test_gen_egs_too_high_egs_plant_design_temp(sample_resource_data): if len(test.shape) == 2: test = np.mean(test, axis=0) - msg = ('{} outputs do not match baseline value! Values differ ' - 'at most by: {}' - .format(dset, np.max(np.abs(truth - test)))) + msg = ( + "{} outputs do not match baseline value! Values differ " + "at most by: {}".format(dset, np.max(np.abs(truth - test))) + ) assert np.allclose(truth, test, rtol=RTOL, atol=ATOL), msg -@pytest.mark.parametrize("sample_resource_data", - [{"temp": 200 * Geothermal.MAX_RT_TO_EGS_RATIO + 10, - "potential": 20}], indirect=True) +@pytest.mark.parametrize( + "sample_resource_data", + [{"temp": 200 * Geothermal.MAX_RT_TO_EGS_RATIO + 10, "potential": 20}], + indirect=True, +) def test_gen_egs_too_low_egs_plant_design_temp(sample_resource_data): """Test generation for EGS too low plant design temp""" points = slice(0, 1) @@ -292,11 +388,17 @@ def test_gen_egs_too_low_egs_plant_design_temp(sample_resource_data): with open(geo_sam_file, "w") as fh: json.dump(geo_config, fh) - output_request = ('design_temp',) + output_request = ("design_temp",) with pytest.warns(UserWarning): - gen = Gen('geothermal', points, geo_sam_file, geo_res_file, - output_request=output_request, sites_per_worker=1, - scale_outputs=True) + gen = Gen( + "geothermal", + points, + geo_sam_file, + geo_res_file, + output_request=output_request, + sites_per_worker=1, + scale_outputs=True, + ) gen.run(max_workers=1) truth_vals = {"design_temp": high_temp} @@ -306,15 +408,18 @@ def test_gen_egs_too_low_egs_plant_design_temp(sample_resource_data): if len(test.shape) == 2: test = np.mean(test, axis=0) - msg = ('{} outputs do not match baseline value! Values differ ' - 'at most by: {}' - .format(dset, np.max(np.abs(truth - test)))) + msg = ( + "{} outputs do not match baseline value! Values differ " + "at most by: {}".format(dset, np.max(np.abs(truth - test))) + ) assert np.allclose(truth, test, rtol=RTOL, atol=ATOL), msg -@pytest.mark.parametrize("sample_resource_data", - [{"temp": 200 * Geothermal.MAX_RT_TO_EGS_RATIO - 1, - "potential": 20}], indirect=True) +@pytest.mark.parametrize( + "sample_resource_data", + [{"temp": 200 * Geothermal.MAX_RT_TO_EGS_RATIO - 1, "potential": 20}], + indirect=True, +) def test_gen_egs_plant_design_temp_adjusted_from_user(sample_resource_data): """Test generation for user-requested match of EGS plant design and RT""" points = slice(0, 1) @@ -331,11 +436,17 @@ def test_gen_egs_plant_design_temp_adjusted_from_user(sample_resource_data): with open(geo_sam_file, "w") as fh: json.dump(geo_config, fh) - output_request = ('design_temp',) + output_request = ("design_temp",) with pytest.warns(UserWarning): - gen = Gen('geothermal', points, geo_sam_file, geo_res_file, - output_request=output_request, sites_per_worker=1, - scale_outputs=True) + gen = Gen( + "geothermal", + points, + geo_sam_file, + geo_res_file, + output_request=output_request, + sites_per_worker=1, + scale_outputs=True, + ) gen.run(max_workers=1) truth_vals = {"design_temp": not_too_high_temp} @@ -345,14 +456,16 @@ def test_gen_egs_plant_design_temp_adjusted_from_user(sample_resource_data): if len(test.shape) == 2: test = np.mean(test, axis=0) - msg = ('{} outputs do not match baseline value! Values differ ' - 'at most by: {}' - .format(dset, np.max(np.abs(truth - test)))) + msg = ( + "{} outputs do not match baseline value! Values differ " + "at most by: {}".format(dset, np.max(np.abs(truth - test))) + ) assert np.allclose(truth, test, rtol=RTOL, atol=ATOL), msg -@pytest.mark.parametrize("sample_resource_data", - [{"temp": 150, "potential": 20}], indirect=True) +@pytest.mark.parametrize( + "sample_resource_data", [{"temp": 150, "potential": 20}], indirect=True +) def test_gen_with_time_index_step_input(sample_resource_data): """Test generation for geothermal module with time_index_step=2""" points = slice(0, 1) @@ -365,19 +478,29 @@ def test_gen_with_time_index_step_input(sample_resource_data): with open(geo_sam_file, "w") as fh: json.dump(geo_config, fh) - output_request = ('annual_energy', 'cf_mean', - 'cf_profile', - 'gen_profile', - MetaKeyName.LCOE_FCR, 'nameplate') - gen = Gen('geothermal', points, geo_sam_file, geo_res_file, - output_request=output_request, sites_per_worker=1, - scale_outputs=True) + output_request = ( + "annual_energy", + "cf_mean", + "cf_profile", + "gen_profile", + MetaKeyName.LCOE_FCR, + "nameplate", + ) + gen = Gen( + "geothermal", + points, + geo_sam_file, + geo_res_file, + output_request=output_request, + sites_per_worker=1, + scale_outputs=True, + ) gen.run(max_workers=1) assert gen.out["cf_profile"].shape[0] == 8760 // 2 -def execute_pytest(capture='all', flags='-rapP'): +def execute_pytest(capture="all", flags="-rapP"): """Execute module as pytest with detailed summary report. Parameters @@ -390,8 +513,8 @@ def execute_pytest(capture='all', flags='-rapP'): """ fname = os.path.basename(__file__) - pytest.main(['-q', '--show-capture={}'.format(capture), fname, flags]) + pytest.main(["-q", "--show-capture={}".format(capture), fname, flags]) -if __name__ == '__main__': +if __name__ == "__main__": execute_pytest() From d3253c36555d36b38748405df0389f18f8576bc3 Mon Sep 17 00:00:00 2001 From: bnb32 Date: Sun, 12 May 2024 10:46:21 -0600 Subject: [PATCH 06/61] probably dont need aliases for country, state, and county --- reV/SAM/generation.py | 8 +- reV/bespoke/bespoke.py | 1049 ++++++++++++++++++++++-------------- reV/hybrids/hybrids.py | 22 +- reV/supply_curve/points.py | 40 +- reV/utilities/__init__.py | 4 - 5 files changed, 677 insertions(+), 446 deletions(-) diff --git a/reV/SAM/generation.py b/reV/SAM/generation.py index b4ec2c413..e99e0db70 100644 --- a/reV/SAM/generation.py +++ b/reV/SAM/generation.py @@ -528,8 +528,8 @@ class AbstractSamGenerationFromWeatherFile(AbstractSamGeneration, ABC): """Base class for running sam generation with a weather file on disk.""" WF_META_DROP_COLS = {MetaKeyName.ELEVATION, MetaKeyName.TIMEZONE, - MetaKeyName.COUNTRY, MetaKeyName.STATE, - MetaKeyName.COUNTY, 'urban', 'population', + 'country', 'state', + 'county', 'urban', 'population', 'landcover', MetaKeyName.LATITUDE, MetaKeyName.LONGITUDE} @@ -605,9 +605,9 @@ def _create_pysam_wfile(self, resource, meta): m['Source'] = 'NSRDB' m['Location ID'] = meta.name m['City'] = '-' - m['State'] = m[MetaKeyName.STATE].apply( + m['State'] = m['state'].apply( lambda x: '-' if x == 'None' else x) - m['Country'] = m[MetaKeyName.COUNTRY].apply( + m['Country'] = m['country'].apply( lambda x: '-' if x == 'None' else x) m['Latitude'] = m[MetaKeyName.LATITUDE] m['Longitude'] = m[MetaKeyName.LONGITUDE] diff --git a/reV/bespoke/bespoke.py b/reV/bespoke/bespoke.py index de8d42c4c..8a48b40b3 100644 --- a/reV/bespoke/bespoke.py +++ b/reV/bespoke/bespoke.py @@ -2,6 +2,7 @@ """ reV bespoke wind plant analysis tools """ + # pylint: disable=anomalous-backslash-in-string import copy import json @@ -84,28 +85,38 @@ def _pre_load_data(self): hh = self.sc_gid_to_hh[sc_gid] self.hh_to_res_gids.setdefault(hh, set()).update(gids) - self.hh_to_res_gids = {hh: sorted(gids) - for hh, gids in self.hh_to_res_gids.items()} + self.hh_to_res_gids = { + hh: sorted(gids) for hh, gids in self.hh_to_res_gids.items() + } start_time = time.time() - if '*' in self.res_fpath: + if "*" in self.res_fpath: handler = MultiYearWindResource else: handler = WindResource with handler(self.res_fpath) as res: - self._wind_dirs = {hh: res[f"winddirection_{hh}m", :, gids] - for hh, gids in self.hh_to_res_gids.items()} - self._wind_speeds = {hh: res[f"windspeed_{hh}m", :, gids] - for hh, gids in self.hh_to_res_gids.items()} - self._temps = {hh: res[f"temperature_{hh}m", :, gids] - for hh, gids in self.hh_to_res_gids.items()} - self._pressures = {hh: res[f"pressure_{hh}m", :, gids] - for hh, gids in self.hh_to_res_gids.items()} + self._wind_dirs = { + hh: res[f"winddirection_{hh}m", :, gids] + for hh, gids in self.hh_to_res_gids.items() + } + self._wind_speeds = { + hh: res[f"windspeed_{hh}m", :, gids] + for hh, gids in self.hh_to_res_gids.items() + } + self._temps = { + hh: res[f"temperature_{hh}m", :, gids] + for hh, gids in self.hh_to_res_gids.items() + } + self._pressures = { + hh: res[f"pressure_{hh}m", :, gids] + for hh, gids in self.hh_to_res_gids.items() + } self._time_index = res.time_index - logger.debug(f"Data took {(time.time() - start_time) / 60:.2f} " - f"min to load") + logger.debug( + f"Data took {(time.time() - start_time) / 60:.2f} " f"min to load" + ) def get_preloaded_data_for_gid(self, sc_gid): """Get the pre-loaded data for a single SC GID. @@ -124,12 +135,14 @@ def get_preloaded_data_for_gid(self, sc_gid): hh = self.sc_gid_to_hh[sc_gid] sc_point_res_gids = sorted(self.sc_gid_to_res_gid[sc_gid]) data_inds = np.searchsorted(self.hh_to_res_gids[hh], sc_point_res_gids) - return BespokeSinglePlantData(sc_point_res_gids, - self._wind_dirs[hh][:, data_inds], - self._wind_speeds[hh][:, data_inds], - self._temps[hh][:, data_inds], - self._pressures[hh][:, data_inds], - self._time_index) + return BespokeSinglePlantData( + sc_point_res_gids, + self._wind_dirs[hh][:, data_inds], + self._wind_speeds[hh][:, data_inds], + self._temps[hh][:, data_inds], + self._pressures[hh][:, data_inds], + self._time_index, + ) class BespokeSinglePlantData: @@ -140,8 +153,9 @@ class BespokeSinglePlantData: reads to a single HDF5 file. """ - def __init__(self, data_inds, wind_dirs, wind_speeds, temps, pressures, - time_index): + def __init__( + self, data_inds, wind_dirs, wind_speeds, temps, pressures, time_index + ): """Initialize BespokeSinglePlantData Parameters @@ -189,21 +203,39 @@ class BespokeSinglePlant: the local wind resource and exclusions for a single reV supply curve point. """ - DEPENDENCIES = ('shapely',) + DEPENDENCIES = ("shapely",) OUT_ATTRS = copy.deepcopy(Gen.OUT_ATTRS) - def __init__(self, gid, excl, res, tm_dset, sam_sys_inputs, - objective_function, capital_cost_function, - fixed_operating_cost_function, - variable_operating_cost_function, - min_spacing='5x', wake_loss_multiplier=1, ga_kwargs=None, - output_request=('system_capacity', - 'cf_mean'), - ws_bins=(0.0, 20.0, 5.0), wd_bins=(0.0, 360.0, 45.0), - excl_dict=None, inclusion_mask=None, data_layers=None, - resolution=64, excl_area=None, exclusion_shape=None, - eos_mult_baseline_cap_mw=200, prior_meta=None, gid_map=None, - bias_correct=None, pre_loaded_data=None, close=True): + def __init__( + self, + gid, + excl, + res, + tm_dset, + sam_sys_inputs, + objective_function, + capital_cost_function, + fixed_operating_cost_function, + variable_operating_cost_function, + min_spacing="5x", + wake_loss_multiplier=1, + ga_kwargs=None, + output_request=("system_capacity", "cf_mean"), + ws_bins=(0.0, 20.0, 5.0), + wd_bins=(0.0, 360.0, 45.0), + excl_dict=None, + inclusion_mask=None, + data_layers=None, + resolution=64, + excl_area=None, + exclusion_shape=None, + eos_mult_baseline_cap_mw=200, + prior_meta=None, + gid_map=None, + bias_correct=None, + pre_loaded_data=None, + close=True, + ): """ Parameters ---------- @@ -352,38 +384,48 @@ def __init__(self, gid, excl, res, tm_dset, sam_sys_inputs, Flag to close object file handlers on exit. """ - logger.debug('Initializing BespokeSinglePlant for gid {}...' - .format(gid)) - logger.debug('Resource filepath: {}'.format(res)) - logger.debug('Exclusion filepath: {}'.format(excl)) - logger.debug('Exclusion dict: {}'.format(excl_dict)) - logger.debug('Bespoke objective function: {}' - .format(objective_function)) - logger.debug('Bespoke cost function: {}'.format(objective_function)) - logger.debug('Bespoke wake loss multiplier: {}' - .format(wake_loss_multiplier)) - logger.debug('Bespoke GA initialization kwargs: {}'.format(ga_kwargs)) - logger.debug('Bespoke EOS multiplier baseline capacity: {:,} MW' - .format(eos_mult_baseline_cap_mw)) - - if isinstance(min_spacing, str) and min_spacing.endswith('x'): + logger.debug( + "Initializing BespokeSinglePlant for gid {}...".format(gid) + ) + logger.debug("Resource filepath: {}".format(res)) + logger.debug("Exclusion filepath: {}".format(excl)) + logger.debug("Exclusion dict: {}".format(excl_dict)) + logger.debug( + "Bespoke objective function: {}".format(objective_function) + ) + logger.debug("Bespoke cost function: {}".format(objective_function)) + logger.debug( + "Bespoke wake loss multiplier: {}".format(wake_loss_multiplier) + ) + logger.debug("Bespoke GA initialization kwargs: {}".format(ga_kwargs)) + logger.debug( + "Bespoke EOS multiplier baseline capacity: {:,} MW".format( + eos_mult_baseline_cap_mw + ) + ) + + if isinstance(min_spacing, str) and min_spacing.endswith("x"): rotor_diameter = sam_sys_inputs["wind_turbine_rotor_diameter"] - min_spacing = float(min_spacing.strip('x')) * rotor_diameter + min_spacing = float(min_spacing.strip("x")) * rotor_diameter if not isinstance(min_spacing, (int, float)): try: min_spacing = float(min_spacing) except Exception as e: - msg = ('min_spacing must be numeric but received: {}, {}' - .format(min_spacing, type(min_spacing))) + msg = ( + "min_spacing must be numeric but received: {}, {}".format( + min_spacing, type(min_spacing) + ) + ) logger.error(msg) raise TypeError(msg) from e self.objective_function = objective_function self.capital_cost_function = capital_cost_function self.fixed_operating_cost_function = fixed_operating_cost_function - self.variable_operating_cost_function = \ + self.variable_operating_cost_function = ( variable_operating_cost_function + ) self.min_spacing = min_spacing self.wake_loss_multiplier = wake_loss_multiplier self.ga_kwargs = ga_kwargs or {} @@ -411,26 +453,33 @@ def __init__(self, gid, excl, res, tm_dset, sam_sys_inputs, Handler = self.get_wind_handler(res) res = res if not isinstance(res, str) else Handler(res) - self._sc_point = AggSCPoint(gid, excl, res, tm_dset, - excl_dict=excl_dict, - inclusion_mask=inclusion_mask, - resolution=resolution, - excl_area=excl_area, - exclusion_shape=exclusion_shape, - close=close) + self._sc_point = AggSCPoint( + gid, + excl, + res, + tm_dset, + excl_dict=excl_dict, + inclusion_mask=inclusion_mask, + resolution=resolution, + excl_area=excl_area, + exclusion_shape=exclusion_shape, + close=close, + ) self._parse_output_req() self._data_layers = data_layers self._parse_prior_run() def __str__(self): - s = ('BespokeSinglePlant for reV SC gid {} with resolution {}' - .format(self.sc_point.gid, self.sc_point.resolution)) + s = "BespokeSinglePlant for reV SC gid {} with resolution {}".format( + self.sc_point.gid, self.sc_point.resolution + ) return s def __repr__(self): - s = ('BespokeSinglePlant for reV SC gid {} with resolution {}' - .format(self.sc_point.gid, self.sc_point.resolution)) + s = "BespokeSinglePlant for reV SC gid {} with resolution {}".format( + self.sc_point.gid, self.sc_point.resolution + ) return s def __enter__(self): @@ -447,14 +496,14 @@ def _parse_output_req(self): (ws_mean, *_mean) if requested. """ - required = ('cf_mean', 'annual_energy') + required = ("cf_mean", "annual_energy") for req in required: if req not in self._out_req: self._out_req.append(req) - if 'ws_mean' in self._out_req: - self._out_req.remove('ws_mean') - self._outputs['ws_mean'] = self.res_df['windspeed'].mean() + if "ws_mean" in self._out_req: + self._out_req.remove("ws_mean") + self._outputs["ws_mean"] = self.res_df["windspeed"].mean() for req in copy.deepcopy(self._out_req): if req in self.res_df: @@ -463,18 +512,20 @@ def _parse_output_req(self): year = annual_ti.year[0] mask = self.res_df.index.isin(annual_ti) arr = self.res_df.loc[mask, req].values.flatten() - self._outputs[req + f'-{year}'] = arr + self._outputs[req + f"-{year}"] = arr - elif req.replace('_mean', '') in self.res_df: + elif req.replace("_mean", "") in self.res_df: self._out_req.remove(req) - dset = req.replace('_mean', '') + dset = req.replace("_mean", "") self._outputs[req] = self.res_df[dset].mean() - if ('lcoe_fcr' in self._out_req and - ('fixed_charge_rate' not in - self.original_sam_sys_inputs)): - msg = ('User requested "lcoe_fcr" but did not input ' - '"fixed_charge_rate" in the SAM system config.') + if "lcoe_fcr" in self._out_req and ( + "fixed_charge_rate" not in self.original_sam_sys_inputs + ): + msg = ( + 'User requested "lcoe_fcr" but did not input ' + '"fixed_charge_rate" in the SAM system config.' + ) logger.error(msg) raise KeyError(msg) @@ -484,14 +535,17 @@ def _parse_prior_run(self): # {meta_column: sam_sys_input_key} required = { - MetaKeyName.CAPACITY: 'system_capacity' - MetaKeyName.TURBINE_X_COORDS: 'wind_farrm_xCoordinates', - MetaKeyName.TURBINE_Y_COORDS: 'wind_farrm_yCoordinates'} + MetaKeyName.CAPACITY: "system_capacity", + MetaKeyName.TURBINE_X_COORDS: "wind_farm_xCoordinates", + MetaKeyName.TURBINE_Y_COORDS: "wind_farm_yCoordinates", + } if self._prior_meta: missing = [k for k in required if k not in self.meta] - msg = ('Prior bespoke run meta data is missing the following ' - 'required columns: {}'.format(missing)) + msg = ( + "Prior bespoke run meta data is missing the following " + "required columns: {}".format(missing) + ) assert not any(missing), msg for meta_col, sam_sys_key in required.items(): @@ -499,7 +553,7 @@ def _parse_prior_run(self): self._sam_sys_inputs[sam_sys_key] = prior_value # convert reV supply curve cap in MW to SAM capacity in kW - self._sam_sys_inputs['system_capacity' + self._sam_sys_inputs["system_capacity"] *= 1e3 @staticmethod def _parse_gid_map(gid_map): @@ -524,14 +578,18 @@ def _parse_gid_map(gid_map): """ if isinstance(gid_map, str): - if gid_map.endswith('.csv'): + if gid_map.endswith(".csv"): gid_map = pd.read_csv(gid_map).to_dict() - assert MetaKeyName.GID in gid_map, 'Need "gid" in gid_map column' - assert 'gid_map' in gid_map, 'Need "gid_map" in gid_map column' - gid_map = {gid_map[MetaKeyName.GID][i]: gid_map['gid_map'][i] - for i in gid_map[MetaKeyName.GID].keys()} + assert ( + MetaKeyName.GID in gid_map + ), 'Need "gid" in gid_map column' + assert "gid_map" in gid_map, 'Need "gid_map" in gid_map column' + gid_map = { + gid_map[MetaKeyName.GID][i]: gid_map["gid_map"][i] + for i in gid_map[MetaKeyName.GID].keys() + } - elif gid_map.endswith('.json'): + elif gid_map.endswith(".json"): with open(gid_map) as f: gid_map = json.load(f) @@ -565,19 +623,23 @@ def bias_correct_ws(self, ws, dset, h5_gids): Bias corrected windspeed data in same shape as input """ - if self._bias_correct is not None and dset.startswith('windspeed_'): - + if self._bias_correct is not None and dset.startswith("windspeed_"): out = parse_bc_table(self._bias_correct, h5_gids) bc_fun, bc_fun_kwargs, bool_bc = out if bool_bc.any(): - logger.debug('Bias correcting windspeed with function {} ' - 'for h5 gids: {}'.format(bc_fun, h5_gids)) + logger.debug( + "Bias correcting windspeed with function {} " + "for h5 gids: {}".format(bc_fun, h5_gids) + ) - bc_fun_kwargs['ws'] = ws[:, bool_bc] + bc_fun_kwargs["ws"] = ws[:, bool_bc] sig = signature(bc_fun) - bc_fun_kwargs = {k: v for k, v in bc_fun_kwargs.items() - if k in sig.parameters} + bc_fun_kwargs = { + k: v + for k, v in bc_fun_kwargs.items() + if k in sig.parameters + } ws[:, bool_bc] = bc_fun(**bc_fun_kwargs) @@ -633,7 +695,7 @@ def get_weighted_res_dir(self): of degrees from north. """ - dset = f'winddirection_{self.hub_height}m' + dset = f"winddirection_{self.hub_height}m" gids = self.sc_point.h5_gid_set h5_gids = copy.deepcopy(gids) if self._gid_map is not None: @@ -738,31 +800,36 @@ def meta(self): """ if self._meta is None: res_gids = json.dumps([int(g) for g in self.sc_point.h5_gid_set]) - gid_counts = json.dumps([float(np.round(n, 1)) - for n in self.sc_point.gid_counts]) + gid_counts = json.dumps( + [float(np.round(n, 1)) for n in self.sc_point.gid_counts] + ) - with SupplyCurveExtent(self.sc_point._excl_fpath, - resolution=self.sc_point.resolution) as sc: + with SupplyCurveExtent( + self.sc_point._excl_fpath, resolution=self.sc_point.resolution + ) as sc: row_ind, col_ind = sc.get_sc_row_col_ind(self.sc_point.gid) self._meta = pd.DataFrame( - {MetaKeyName.SC_POINT_GID: self.sc_point.gid, - MetaKeyName.SC_ROW_IND: row_ind, - MetaKeyName.SC_COL_IND: col_ind, - MetaKeyName.GID: self.sc_point.gid, - MetaKeyName.LATITUDE: self.sc_point.latitude, - MetaKeyName.LONGITUDE: self.sc_point.longitude, - MetaKeyName.TIMEZONE: self.sc_point.timezone, - MetaKeyName.COUNTRY: self.sc_point.country, - MetaKeyName.STATE: self.sc_point.state, - MetaKeyName.COUNTY: self.sc_point.county, - MetaKeyName.ELEVATION: self.sc_point.elevation, - MetaKeyName.OFFSHORE: self.sc_point.offshore, - MetaKeyName.RES_GIDS: res_gids, - MetaKeyName.GID_COUNTS: gid_counts, - MetaKeyName.N_GIDS: self.sc_point.n_gids, - MetaKeyName.AREA_SQ_KM: self.sc_point.area, - }, index=[self.sc_point.gid]) + { + MetaKeyName.SC_POINT_GID: self.sc_point.gid, + MetaKeyName.SC_ROW_IND: row_ind, + MetaKeyName.SC_COL_IND: col_ind, + MetaKeyName.GID: self.sc_point.gid, + MetaKeyName.LATITUDE: self.sc_point.latitude, + MetaKeyName.LONGITUDE: self.sc_point.longitude, + MetaKeyName.TIMEZONE: self.sc_point.timezone, + MetaKeyName.COUNTRY: self.sc_point.country, + MetaKeyName.STATE: self.sc_point.state, + MetaKeyName.COUNTY: self.sc_point.county, + MetaKeyName.ELEVATION: self.sc_point.elevation, + MetaKeyName.OFFSHORE: self.sc_point.offshore, + MetaKeyName.RES_GIDS: res_gids, + MetaKeyName.GID_COUNTS: gid_counts, + MetaKeyName.N_GIDS: self.sc_point.n_gids, + MetaKeyName.AREA_SQ_KM: self.sc_point.area, + }, + index=[self.sc_point.gid], + ) return self._meta @@ -774,7 +841,7 @@ def hub_height(self): ------- int """ - return int(self.sam_sys_inputs['wind_turbine_hub_ht']) + return int(self.sam_sys_inputs["wind_turbine_hub_ht"]) @property def res_df(self): @@ -794,21 +861,26 @@ def res_df(self): ti = self._pre_loaded_data.time_index wd = self.get_weighted_res_dir() - ws = self.get_weighted_res_ts(f'windspeed_{self.hub_height}m') - temp = self.get_weighted_res_ts(f'temperature_{self.hub_height}m') - pres = self.get_weighted_res_ts(f'pressure_{self.hub_height}m') + ws = self.get_weighted_res_ts(f"windspeed_{self.hub_height}m") + temp = self.get_weighted_res_ts(f"temperature_{self.hub_height}m") + pres = self.get_weighted_res_ts(f"pressure_{self.hub_height}m") # convert mbar to atm if np.nanmax(pres) > 1000: pres *= 9.86923e-6 - self._res_df = pd.DataFrame({'temperature': temp, - 'pressure': pres, - 'windspeed': ws, - 'winddirection': wd}, index=ti) - - if 'time_index_step' in self.original_sam_sys_inputs: - ti_step = self.original_sam_sys_inputs['time_index_step'] + self._res_df = pd.DataFrame( + { + "temperature": temp, + "pressure": pres, + "windspeed": ws, + "winddirection": wd, + }, + index=ti, + ) + + if "time_index_step" in self.original_sam_sys_inputs: + ti_step = self.original_sam_sys_inputs["time_index_step"] self._res_df = self._res_df.iloc[::ti_step] return self._res_df @@ -859,9 +931,11 @@ def wind_dist(self): ws_bins = JointPD._make_bins(*self._ws_bins) wd_bins = JointPD._make_bins(*self._wd_bins) - hist_out = np.histogram2d(self.res_df['windspeed'], - self.res_df['winddirection'], - bins=(ws_bins, wd_bins)) + hist_out = np.histogram2d( + self.res_df["windspeed"], + self.res_df["winddirection"], + bins=(ws_bins, wd_bins), + ) self._wind_dist, self._ws_edges, self._wd_edges = hist_out self._wind_dist /= self._wind_dist.sum() @@ -882,12 +956,13 @@ def initialize_wind_plant_ts(self): res_df = self.res_df[(self.res_df.index.year == year)] sam_inputs = copy.deepcopy(self.sam_sys_inputs) - if 'lcoe_fcr' in self._out_req: + if "lcoe_fcr" in self._out_req: lcoe_kwargs = self.get_lcoe_kwargs() sam_inputs.update(lcoe_kwargs) - i_wp = WindPower(res_df, self.meta, sam_inputs, - output_request=self._out_req) + i_wp = WindPower( + res_df, self.meta, sam_inputs, output_request=self._out_req + ) wind_plant_ts[year] = i_wp return wind_plant_ts @@ -904,9 +979,14 @@ def wind_plant_pd(self): if self._wind_plant_pd is None: wind_dist, ws_edges, wd_edges = self.wind_dist - self._wind_plant_pd = WindPowerPD(ws_edges, wd_edges, wind_dist, - self.meta, self.sam_sys_inputs, - output_request=self._out_req) + self._wind_plant_pd = WindPowerPD( + ws_edges, + wd_edges, + wind_dist, + self.meta, + self.sam_sys_inputs, + output_request=self._out_req, + ) return self._wind_plant_pd @property @@ -931,6 +1011,7 @@ def plant_optimizer(self): if self._plant_optm is None: # put import here to delay breaking due to special dependencies from reV.bespoke.place_turbines import PlaceTurbines + self._plant_optm = PlaceTurbines( self.wind_plant_pd, self.objective_function, @@ -940,7 +1021,8 @@ def plant_optimizer(self): self.include_mask, self.pixel_side_length, self.min_spacing, - self.wake_loss_multiplier) + self.wake_loss_multiplier, + ) return self._plant_optm @@ -948,21 +1030,23 @@ def recalc_lcoe(self): """Recalculate the multi-year mean LCOE based on the multi-year mean annual energy production (AEP)""" - if 'lcoe_fcr-means' in self.outputs: + if "lcoe_fcr-means" in self.outputs: lcoe_kwargs = self.get_lcoe_kwargs() - logger.debug('Recalulating multi-year mean LCOE using ' - 'multi-year mean AEP.') + logger.debug( + "Recalulating multi-year mean LCOE using " + "multi-year mean AEP." + ) - fcr = lcoe_kwargs['fixed_charge_rate'] - cap_cost = lcoe_kwargs['capital_cost'] - foc = lcoe_kwargs['fixed_operating_cost'] - voc = lcoe_kwargs['variable_operating_cost'] - aep = self.outputs['annual_energy-means'] + fcr = lcoe_kwargs["fixed_charge_rate"] + cap_cost = lcoe_kwargs["capital_cost"] + foc = lcoe_kwargs["fixed_operating_cost"] + voc = lcoe_kwargs["variable_operating_cost"] + aep = self.outputs["annual_energy-means"] my_mean_lcoe = lcoe_fcr(fcr, cap_cost, foc, aep, voc) - self._outputs['lcoe_fcr-means'] = my_mean_lcoe + self._outputs["lcoe_fcr-means"] = my_mean_lcoe self._meta[MetaKeyName.MEAN_LCOE] = my_mean_lcoe def get_lcoe_kwargs(self): @@ -981,8 +1065,13 @@ def get_lcoe_kwargs(self): original_sam_sys_inputs, meta """ - kwargs_list = ['fixed_charge_rate', 'system_capacity', 'capital_cost', - 'fixed_operating_cost', 'variable_operating_cost'] + kwargs_list = [ + "fixed_charge_rate", + "system_capacity", + "capital_cost", + "fixed_operating_cost", + "variable_operating_cost", + ] lcoe_kwargs = {} for kwarg in kwargs_list: @@ -1001,9 +1090,12 @@ def get_lcoe_kwargs(self): missing = [k for k in kwargs_list if k not in lcoe_kwargs] if any(missing): - msg = ('Could not find these LCOE kwargs in outputs, ' - 'plant_optimizer, original_sam_sys_inputs, or meta: {}' - .format(missing)) + msg = ( + "Could not find these LCOE kwargs in outputs, " + "plant_optimizer, original_sam_sys_inputs, or meta: {}".format( + missing + ) + ) logger.error(msg) raise KeyError(msg) @@ -1026,7 +1118,7 @@ def get_wind_handler(res): """ handler = res if isinstance(res, str): - if '*' in res: + if "*" in res: handler = MultiYearWindResource else: handler = WindResource @@ -1044,21 +1136,28 @@ def check_dependencies(cls): missing.append(name) if any(missing): - msg = ('The reV bespoke module depends on the following special ' - 'dependencies that were not found in the active ' - 'environment: {}'.format(missing)) + msg = ( + "The reV bespoke module depends on the following special " + "dependencies that were not found in the active " + "environment: {}".format(missing) + ) logger.error(msg) raise ModuleNotFoundError(msg) @staticmethod - def _check_sys_inputs(plant1, plant2, - ignore=('wind_resource_model_choice', - 'wind_resource_data', - 'wind_turbine_powercurve_powerout', - 'hourly', - 'capital_cost', - 'fixed_operating_cost', - 'variable_operating_cost')): + def _check_sys_inputs( + plant1, + plant2, + ignore=( + "wind_resource_model_choice", + "wind_resource_data", + "wind_turbine_powercurve_powerout", + "hourly", + "capital_cost", + "fixed_operating_cost", + "variable_operating_cost", + ), + ): """Check two reV-SAM models for matching system inputs. Parameters @@ -1068,12 +1167,13 @@ def _check_sys_inputs(plant1, plant2, """ bad = [] for k, v in plant1.sam_sys_inputs.items(): - if (k not in plant2.sam_sys_inputs or - str(v) != str(plant2.sam_sys_inputs[k])): + if k not in plant2.sam_sys_inputs or str(v) != str( + plant2.sam_sys_inputs[k] + ): bad.append(k) bad = [b for b in bad if b not in ignore] if any(bad): - msg = 'Inputs no longer match: {}'.format(bad) + msg = "Inputs no longer match: {}".format(bad) logger.error(msg) raise RuntimeError(msg) @@ -1089,41 +1189,51 @@ def run_wind_plant_ts(self): BespokeSinglePlant.outputs property. """ - logger.debug('Running {} years of SAM timeseries analysis for {}' - .format(len(self.years), self)) + logger.debug( + "Running {} years of SAM timeseries analysis for {}".format( + len(self.years), self + ) + ) self._wind_plant_ts = self.initialize_wind_plant_ts() for year, plant in self.wind_plant_ts.items(): self._check_sys_inputs(plant, self.wind_plant_pd) try: plant.run_gen_and_econ() except Exception as e: - msg = ('{} failed while trying to run SAM WindPower ' - 'timeseries analysis for {}'.format(self, year)) + msg = ( + "{} failed while trying to run SAM WindPower " + "timeseries analysis for {}".format(self, year) + ) logger.exception(msg) raise RuntimeError(msg) from e for k, v in plant.outputs.items(): - self._outputs[k + '-{}'.format(year)] = v + self._outputs[k + "-{}".format(year)] = v means = {} for k1, v1 in self._outputs.items(): - if isinstance(v1, Number) and parse_year(k1, option='boolean'): + if isinstance(v1, Number) and parse_year(k1, option="boolean"): year = parse_year(k1) - base_str = k1.replace(str(year), '') - all_values = [v2 for k2, v2 in self._outputs.items() - if base_str in k2] - means[base_str + 'means'] = np.mean(all_values) + base_str = k1.replace(str(year), "") + all_values = [ + v2 for k2, v2 in self._outputs.items() if base_str in k2 + ] + means[base_str + "means"] = np.mean(all_values) self._outputs.update(means) # copy dataset outputs to meta data for supply curve table summary - if 'cf_mean-means' in self.outputs: - self._meta.loc[:, MetaKeyName.MEAN_CF] = self.outputs['cf_mean-means'] - if 'lcoe_fcr-means' in self.outputs: - self._meta.loc[:, MetaKeyName.MEAN_LCOE] = self.outputs['lcoe_fcr-means'] + if "cf_mean-means" in self.outputs: + self._meta.loc[:, MetaKeyName.MEAN_CF] = self.outputs[ + "cf_mean-means" + ] + if "lcoe_fcr-means" in self.outputs: + self._meta.loc[:, MetaKeyName.MEAN_LCOE] = self.outputs[ + "lcoe_fcr-means" + ] self.recalc_lcoe() - logger.debug('Timeseries analysis complete!') + logger.debug("Timeseries analysis complete!") return self.outputs @@ -1139,13 +1249,14 @@ def run_plant_optimization(self): BespokeSinglePlant.outputs property. """ - logger.debug('Running plant layout optimization for {}'.format(self)) + logger.debug("Running plant layout optimization for {}".format(self)) try: self.plant_optimizer.place_turbines(**self.ga_kwargs) except Exception as e: - msg = ('{} failed while trying to run the ' - 'turbine placement optimizer' - .format(self)) + msg = ( + "{} failed while trying to run the " + "turbine placement optimizer".format(self) + ) logger.exception(msg) raise RuntimeError(msg) from e @@ -1169,56 +1280,70 @@ def run_plant_optimization(self): self._meta["possible_y_coords"] = pyc self._outputs["full_polygons"] = self.plant_optimizer.full_polygons - self._outputs["packing_polygons"] = \ + self._outputs["packing_polygons"] = ( self.plant_optimizer.packing_polygons + ) self._outputs["system_capacity"] = self.plant_optimizer.capacity self._meta["n_turbines"] = self.plant_optimizer.nturbs self._meta["bespoke_aep"] = self.plant_optimizer.aep self._meta["bespoke_objective"] = self.plant_optimizer.objective - self._meta["bespoke_capital_cost"] = \ - self.plant_optimizer.capital_cost - self._meta["bespoke_fixed_operating_cost"] = \ + self._meta["bespoke_capital_cost"] = self.plant_optimizer.capital_cost + self._meta["bespoke_fixed_operating_cost"] = ( self.plant_optimizer.fixed_operating_cost - self._meta["bespoke_variable_operating_cost"] = \ + ) + self._meta["bespoke_variable_operating_cost"] = ( self.plant_optimizer.variable_operating_cost + ) self._meta["included_area"] = self.plant_optimizer.area - self._meta["included_area_capacity_density"] = \ + self._meta["included_area_capacity_density"] = ( self.plant_optimizer.capacity_density - self._meta["convex_hull_area"] = \ - self.plant_optimizer.convex_hull_area - self._meta["convex_hull_capacity_density"] = \ + ) + self._meta["convex_hull_area"] = self.plant_optimizer.convex_hull_area + self._meta["convex_hull_capacity_density"] = ( self.plant_optimizer.convex_hull_capacity_density - self._meta["full_cell_capacity_density"] = \ + ) + self._meta["full_cell_capacity_density"] = ( self.plant_optimizer.full_cell_capacity_density + ) - logger.debug('Plant layout optimization complete!') + logger.debug("Plant layout optimization complete!") # copy dataset outputs to meta data for supply curve table summary # convert SAM system capacity in kW to reV supply curve cap in MW - self._meta[MetaKeyName.CAPACITY] = self.outputs['system_capacity'] / 1e3 + self._meta[MetaKeyName.CAPACITY] = ( + self.outputs["system_capacity"] / 1e3 + ) # add required ReEDS multipliers to meta baseline_cost = self.plant_optimizer.capital_cost_per_kw( - capacity_mw=self._baseline_cap_mw) - self._meta['eos_mult'] = (self.plant_optimizer.capital_cost - / self.plant_optimizer.capacity - / baseline_cost) - self._meta['reg_mult'] = (self.sam_sys_inputs - .get("capital_cost_multiplier", 1)) + capacity_mw=self._baseline_cap_mw + ) + self._meta["eos_mult"] = ( + self.plant_optimizer.capital_cost + / self.plant_optimizer.capacity + / baseline_cost + ) + self._meta["reg_mult"] = self.sam_sys_inputs.get( + "capital_cost_multiplier", 1 + ) return self.outputs def agg_data_layers(self): """Aggregate optional data layers if requested and save to self.meta""" if self._data_layers is not None: - logger.debug('Aggregating {} extra data layers.' - .format(len(self._data_layers))) + logger.debug( + "Aggregating {} extra data layers.".format( + len(self._data_layers) + ) + ) point_summary = self.meta.to_dict() - point_summary = self.sc_point.agg_data_layers(point_summary, - self._data_layers) + point_summary = self.sc_point.agg_data_layers( + point_summary, self._data_layers + ) self._meta = pd.DataFrame(point_summary) - logger.debug('Finished aggregating extra data layers.') + logger.debug("Finished aggregating extra data layers.") @property def outputs(self): @@ -1247,9 +1372,10 @@ def run(cls, *args, **kwargs): with cls(*args, **kwargs) as bsp: if bsp._prior_meta: - logger.debug('Skipping bespoke plant optimization for gid {}. ' - 'Received prior meta data for this point.' - .format(bsp.gid)) + logger.debug( + "Skipping bespoke plant optimization for gid {}. " + "Received prior meta data for this point.".format(bsp.gid) + ) else: _ = bsp.run_plant_optimization() @@ -1258,9 +1384,9 @@ def run(cls, *args, **kwargs): meta = bsp.meta out = bsp.outputs - out['meta'] = meta + out["meta"] = meta for year, ti in zip(bsp.years, bsp.annual_time_indexes): - out['time_index-{}'.format(year)] = ti + out["time_index-{}".format(year)] = ti return out @@ -1268,16 +1394,35 @@ def run(cls, *args, **kwargs): class BespokeWindPlants(BaseAggregation): """BespokeWindPlants""" - def __init__(self, excl_fpath, res_fpath, tm_dset, objective_function, - capital_cost_function, fixed_operating_cost_function, - variable_operating_cost_function, project_points, - sam_files, min_spacing='5x', wake_loss_multiplier=1, - ga_kwargs=None, output_request=('system_capacity', 'cf_mean'), - ws_bins=(0.0, 20.0, 5.0), wd_bins=(0.0, 360.0, 45.0), - excl_dict=None, area_filter_kernel='queen', min_area=None, - resolution=64, excl_area=None, data_layers=None, - pre_extract_inclusions=False, prior_run=None, gid_map=None, - bias_correct=None, pre_load_data=False): + def __init__( + self, + excl_fpath, + res_fpath, + tm_dset, + objective_function, + capital_cost_function, + fixed_operating_cost_function, + variable_operating_cost_function, + project_points, + sam_files, + min_spacing="5x", + wake_loss_multiplier=1, + ga_kwargs=None, + output_request=("system_capacity", "cf_mean"), + ws_bins=(0.0, 20.0, 5.0), + wd_bins=(0.0, 360.0, 45.0), + excl_dict=None, + area_filter_kernel="queen", + min_area=None, + resolution=64, + excl_area=None, + data_layers=None, + pre_extract_inclusions=False, + prior_run=None, + gid_map=None, + bias_correct=None, + pre_load_data=False, + ): r"""ReV bespoke analysis class. Much like generation, ``reV`` bespoke analysis runs SAM @@ -1686,39 +1831,58 @@ def __init__(self, excl_fpath, res_fpath, tm_dset, objective_function, """ log_versions(logger) - logger.info('Initializing BespokeWindPlants...') - logger.info('Resource filepath: {}'.format(res_fpath)) - logger.info('Exclusion filepath: {}'.format(excl_fpath)) - logger.debug('Exclusion dict: {}'.format(excl_dict)) - logger.info('Bespoke objective function: {}' - .format(objective_function)) - logger.info('Bespoke capital cost function: {}' - .format(capital_cost_function)) - logger.info('Bespoke fixed operating cost function: {}' - .format(fixed_operating_cost_function)) - logger.info('Bespoke variable operating cost function: {}' - .format(variable_operating_cost_function)) - logger.info('Bespoke wake loss multiplier: {}' - .format(wake_loss_multiplier)) - logger.info('Bespoke GA initialization kwargs: {}'.format(ga_kwargs)) - - logger.info('Bespoke pre-extracting exclusions: {}' - .format(pre_extract_inclusions)) - logger.info('Bespoke pre-extracting resource data: {}' - .format(pre_load_data)) - logger.info('Bespoke prior run: {}'.format(prior_run)) - logger.info('Bespoke GID map: {}'.format(gid_map)) - logger.info('Bespoke bias correction table: {}'.format(bias_correct)) + logger.info("Initializing BespokeWindPlants...") + logger.info("Resource filepath: {}".format(res_fpath)) + logger.info("Exclusion filepath: {}".format(excl_fpath)) + logger.debug("Exclusion dict: {}".format(excl_dict)) + logger.info( + "Bespoke objective function: {}".format(objective_function) + ) + logger.info( + "Bespoke capital cost function: {}".format(capital_cost_function) + ) + logger.info( + "Bespoke fixed operating cost function: {}".format( + fixed_operating_cost_function + ) + ) + logger.info( + "Bespoke variable operating cost function: {}".format( + variable_operating_cost_function + ) + ) + logger.info( + "Bespoke wake loss multiplier: {}".format(wake_loss_multiplier) + ) + logger.info("Bespoke GA initialization kwargs: {}".format(ga_kwargs)) + + logger.info( + "Bespoke pre-extracting exclusions: {}".format( + pre_extract_inclusions + ) + ) + logger.info( + "Bespoke pre-extracting resource data: {}".format(pre_load_data) + ) + logger.info("Bespoke prior run: {}".format(prior_run)) + logger.info("Bespoke GID map: {}".format(gid_map)) + logger.info("Bespoke bias correction table: {}".format(bias_correct)) BespokeSinglePlant.check_dependencies() self._project_points = self._parse_points(project_points, sam_files) - super().__init__(excl_fpath, tm_dset, excl_dict=excl_dict, - area_filter_kernel=area_filter_kernel, - min_area=min_area, resolution=resolution, - excl_area=excl_area, gids=self._project_points.gids, - pre_extract_inclusions=pre_extract_inclusions) + super().__init__( + excl_fpath, + tm_dset, + excl_dict=excl_dict, + area_filter_kernel=area_filter_kernel, + min_area=min_area, + resolution=resolution, + excl_area=excl_area, + gids=self._project_points.gids, + pre_extract_inclusions=pre_extract_inclusions, + ) self._res_fpath = res_fpath self._obj_fun = objective_function @@ -1743,8 +1907,11 @@ def __init__(self, excl_fpath, res_fpath, tm_dset, objective_function, self._slice_lookup = None - logger.info('Initialized BespokeWindPlants with project points: {}' - .format(self._project_points)) + logger.info( + "Initialized BespokeWindPlants with project points: {}".format( + self._project_points + ) + ) @staticmethod def _parse_points(points, sam_configs): @@ -1756,7 +1923,7 @@ def _parse_points(points, sam_configs): Slice or list specifying project points, string pointing to a project points csv, or a fully instantiated PointsControl object. Can also be a single site integer value. Points csv should have - MetaKeyName.GID and 'config' column, the config maps to the sam_configs dict + 'gid' and 'config' column, the config maps to the sam_configs dict keys. sam_configs : dict | str | SAMConfig SAM input configuration ID(s) and file path(s). Keys are the SAM @@ -1771,8 +1938,13 @@ def _parse_points(points, sam_configs): Project points object laying out the supply curve gids to analyze. """ - pc = Gen.get_pc(points, points_range=None, sam_configs=sam_configs, - tech='windpower', sites_per_worker=1) + pc = Gen.get_pc( + points, + points_range=None, + sam_configs=sam_configs, + tech="windpower", + sites_per_worker=1, + ) return pc.project_points @@ -1802,15 +1974,15 @@ def _parse_prior_run(prior_run): if prior_run is not None: assert os.path.isfile(prior_run) - assert prior_run.endswith('.h5') + assert prior_run.endswith(".h5") - with Outputs(prior_run, mode='r') as f: + with Outputs(prior_run, mode="r") as f: meta = f.meta # pylint: disable=no-member for col in meta.columns: val = meta[col].values[0] - if isinstance(val, str) and val[0] == '[' and val[-1] == ']': + if isinstance(val, str) and val[0] == "[" and val[-1] == "]": meta[col] = meta[col].apply(json.loads) return meta @@ -1847,14 +2019,19 @@ def _check_files(self): for path in paths: if not os.path.exists(path): raise FileNotFoundError( - 'Could not find required exclusions file: ' - '{}'.format(path)) + "Could not find required exclusions file: " "{}".format( + path + ) + ) with ExclusionLayers(paths) as excl: if self._tm_dset not in excl: - raise FileInputError('Could not find techmap dataset "{}" ' - 'in the exclusions file(s): {}' - .format(self._tm_dset, paths)) + raise FileInputError( + 'Could not find techmap dataset "{}" ' + "in the exclusions file(s): {}".format( + self._tm_dset, paths + ) + ) # just check that this file exists, cannot check res_fpath if *glob Handler = BespokeSinglePlant.get_wind_handler(self._res_fpath) @@ -1866,18 +2043,24 @@ def _pre_load_data(self, pre_load_data): if not pre_load_data: return - sc_gid_to_hh = {gid: self._hh_for_sc_gid(gid) - for gid in self._project_points.df["gid"]} + sc_gid_to_hh = { + gid: self._hh_for_sc_gid(gid) + for gid in self._project_points.df["gid"] + } with ExclusionLayers(self._excl_fpath) as excl: tm = excl[self._tm_dset] scp_kwargs = {"shape": self.shape, "resolution": self._resolution} - slices = {gid: SupplyCurvePoint.get_agg_slices(gid=gid, **scp_kwargs) - for gid in self._project_points.df["gid"]} + slices = { + gid: SupplyCurvePoint.get_agg_slices(gid=gid, **scp_kwargs) + for gid in self._project_points.df["gid"] + } - sc_gid_to_res_gid = {gid: sorted(set(tm[slx, sly].flatten())) - for gid, (slx, sly) in slices.items()} + sc_gid_to_res_gid = { + gid: sorted(set(tm[slx, sly].flatten())) + for gid, (slx, sly) in slices.items() + } for sc_gid, res_gids in sc_gid_to_res_gid.items(): if res_gids[0] < 0: @@ -1885,13 +2068,14 @@ def _pre_load_data(self, pre_load_data): if self._gid_map is not None: for sc_gid, res_gids in sc_gid_to_res_gid.items(): - sc_gid_to_res_gid[sc_gid] = sorted(self._gid_map[g] - for g in res_gids) + sc_gid_to_res_gid[sc_gid] = sorted( + self._gid_map[g] for g in res_gids + ) logger.info("Pre-loading resource data for Bespoke run... ") - self._pre_loaded_data = BespokeMultiPlantData(self._res_fpath, - sc_gid_to_hh, - sc_gid_to_res_gid) + self._pre_loaded_data = BespokeMultiPlantData( + self._res_fpath, sc_gid_to_hh, sc_gid_to_res_gid + ) def _hh_for_sc_gid(self, sc_gid): """Fetch the hh for a given sc_gid""" @@ -1927,9 +2111,12 @@ def _get_bc_for_gid(self, gid): if self._bias_correct is not None: h5_gids = [] try: - scp_kwargs = dict(gid=gid, excl=self._excl_fpath, - tm_dset=self._tm_dset, - resolution=self._resolution) + scp_kwargs = dict( + gid=gid, + excl=self._excl_fpath, + tm_dset=self._tm_dset, + resolution=self._resolution, + ) with SupplyCurvePoint(**scp_kwargs) as scp: h5_gids = scp.h5_gid_set except EmptySupplyCurvePointError: @@ -1973,7 +2160,7 @@ def meta(self): ------- pd.DataFrame """ - meta = [self.outputs[g]['meta'] for g in self.completed_gids] + meta = [self.outputs[g]["meta"] for g in self.completed_gids] if len(self.completed_gids) > 1: meta = pd.concat(meta, axis=0) else: @@ -1984,8 +2171,9 @@ def meta(self): def slice_lookup(self): """Dict | None: Lookup mapping sc_point_gid to exclusion slice.""" if self._slice_lookup is None and self._inclusion_mask is not None: - with SupplyCurveExtent(self._excl_fpath, - resolution=self._resolution) as sc: + with SupplyCurveExtent( + self._excl_fpath, resolution=self._resolution + ) as sc: assert self.shape == self._inclusion_mask.shape self._slice_lookup = sc.get_slice_lookup(self.gids) @@ -2014,8 +2202,13 @@ def sam_sys_inputs_with_site_data(self, gid): site_data = self._project_points.df.iloc[gid_idx] site_sys_inputs = self._project_points[gid][1] - site_sys_inputs.update({k: v for k, v in site_data.to_dict().items() - if not (isinstance(v, float) and np.isnan(v))}) + site_sys_inputs.update( + { + k: v + for k, v in site_data.to_dict().items() + if not (isinstance(v, float) and np.isnan(v)) + } + ) return site_sys_inputs def _init_fout(self, out_fpath, sample): @@ -2034,13 +2227,14 @@ def _init_fout(self, out_fpath, sample): if not os.path.exists(out_dir): create_dirs(out_dir) - with Outputs(out_fpath, mode='w') as f: - f._set_meta('meta', self.meta, attrs={}) - ti_dsets = [d for d in sample.keys() - if d.startswith('time_index-')] + with Outputs(out_fpath, mode="w") as f: + f._set_meta("meta", self.meta, attrs={}) + ti_dsets = [ + d for d in sample.keys() if d.startswith("time_index-") + ] for dset in ti_dsets: f._set_time_index(dset, sample[dset], attrs={}) - f._set_time_index('time_index', sample[dset], attrs={}) + f._set_time_index("time_index", sample[dset], attrs={}) def _collect_out_arr(self, dset, sample): """Collect single-plant data arrays into complete arrays with data from @@ -2071,8 +2265,9 @@ def _collect_out_arr(self, dset, sample): shape = (len(single_arr), len(self.completed_gids)) sample_num = single_arr[0] else: - msg = ('Not writing dataset "{}" of type "{}" to disk.' - .format(dset, type(single_arr))) + msg = 'Not writing dataset "{}" of type "{}" to disk.'.format( + dset, type(single_arr) + ) logger.info(msg) return None @@ -2083,8 +2278,9 @@ def _collect_out_arr(self, dset, sample): full_arr = np.zeros(shape, dtype=dtype) # collect data from all wind plants - logger.info('Collecting dataset "{}" with final shape {}' - .format(dset, shape)) + logger.info( + 'Collecting dataset "{}" with final shape {}'.format(dset, shape) + ) for i, gid in enumerate(self.completed_gids): if len(full_arr.shape) == 1: full_arr[i] = self.outputs[gid][dset] @@ -2108,16 +2304,18 @@ def save_outputs(self, out_fpath): Full filepath to desired .h5 output file, the .h5 extension has been added if it was not already present. """ - if not out_fpath.endswith('.h5'): - out_fpath += '.h5' + if not out_fpath.endswith(".h5"): + out_fpath += ".h5" if ModuleName.BESPOKE not in out_fpath: extension_with_module = "_{}.h5".format(ModuleName.BESPOKE) out_fpath = out_fpath.replace(".h5", extension_with_module) if not self.completed_gids: - msg = ("No output data found! It is likely that all requested " - "points are excluded.") + msg = ( + "No output data found! It is likely that all requested " + "points are excluded." + ) logger.warning(msg) warn(msg) return out_fpath @@ -2125,50 +2323,69 @@ def save_outputs(self, out_fpath): sample = self.outputs[self.completed_gids[0]] self._init_fout(out_fpath, sample) - dsets = [d for d in sample.keys() - if not d.startswith('time_index-') - and d != 'meta'] - with Outputs(out_fpath, mode='a') as f: + dsets = [ + d + for d in sample.keys() + if not d.startswith("time_index-") and d != "meta" + ] + with Outputs(out_fpath, mode="a") as f: for dset in dsets: full_arr = self._collect_out_arr(dset, sample) if full_arr is not None: dset_no_year = dset - if parse_year(dset, option='boolean'): + if parse_year(dset, option="boolean"): year = parse_year(dset) - dset_no_year = dset.replace('-{}'.format(year), '') + dset_no_year = dset.replace("-{}".format(year), "") attrs = BespokeSinglePlant.OUT_ATTRS.get(dset_no_year, {}) attrs = copy.deepcopy(attrs) - dtype = attrs.pop('dtype', np.float32) - chunks = attrs.pop('chunks', None) + dtype = attrs.pop("dtype", np.float32) + chunks = attrs.pop("chunks", None) try: - f.write_dataset(dset, full_arr, dtype, chunks=chunks, - attrs=attrs) + f.write_dataset( + dset, full_arr, dtype, chunks=chunks, attrs=attrs + ) except Exception as e: msg = 'Failed to write "{}" to disk.'.format(dset) logger.exception(msg) raise OSError(msg) from e - logger.info('Saved output data to: {}'.format(out_fpath)) + logger.info("Saved output data to: {}".format(out_fpath)) return out_fpath # pylint: disable=arguments-renamed @classmethod - def run_serial(cls, excl_fpath, res_fpath, tm_dset, - sam_sys_inputs, objective_function, - capital_cost_function, - fixed_operating_cost_function, - variable_operating_cost_function, - min_spacing='5x', wake_loss_multiplier=1, ga_kwargs=None, - output_request=('system_capacity', - 'cf_mean'), - ws_bins=(0.0, 20.0, 5.0), wd_bins=(0.0, 360.0, 45.0), - excl_dict=None, inclusion_mask=None, - area_filter_kernel='queen', min_area=None, - resolution=64, excl_area=0.0081, data_layers=None, - gids=None, exclusion_shape=None, slice_lookup=None, - prior_meta=None, gid_map=None, bias_correct=None, - pre_loaded_data=None): + def run_serial( + cls, + excl_fpath, + res_fpath, + tm_dset, + sam_sys_inputs, + objective_function, + capital_cost_function, + fixed_operating_cost_function, + variable_operating_cost_function, + min_spacing="5x", + wake_loss_multiplier=1, + ga_kwargs=None, + output_request=("system_capacity", "cf_mean"), + ws_bins=(0.0, 20.0, 5.0), + wd_bins=(0.0, 360.0, 45.0), + excl_dict=None, + inclusion_mask=None, + area_filter_kernel="queen", + min_area=None, + resolution=64, + excl_area=0.0081, + data_layers=None, + gids=None, + exclusion_shape=None, + slice_lookup=None, + prior_meta=None, + gid_map=None, + bias_correct=None, + pre_loaded_data=None, + ): """ Standalone serial method to run bespoke optimization. See BespokeWindPlants docstring for parameter description. @@ -2197,18 +2414,19 @@ def run_serial(cls, excl_fpath, res_fpath, tm_dset, Handler = BespokeSinglePlant.get_wind_handler(res_fpath) # pre-extract handlers so they are not repeatedly initialized - file_kwargs = {'excl_dict': excl_dict, - 'area_filter_kernel': area_filter_kernel, - 'min_area': min_area, - 'h5_handler': Handler, - } + file_kwargs = { + "excl_dict": excl_dict, + "area_filter_kernel": area_filter_kernel, + "min_area": min_area, + "h5_handler": Handler, + } with AggFileHandler(excl_fpath, res_fpath, **file_kwargs) as fh: n_finished = 0 for gid in gids: gid_inclusions = cls._get_gid_inclusion_mask( - inclusion_mask, gid, slice_lookup, - resolution=resolution) + inclusion_mask, gid, slice_lookup, resolution=resolution + ) try: bsp_plant_out = BespokeSinglePlant.run( gid, @@ -2236,20 +2454,26 @@ def run_serial(cls, excl_fpath, res_fpath, tm_dset, gid_map=gid_map, bias_correct=bias_correct, pre_loaded_data=pre_loaded_data, - close=False) + close=False, + ) except EmptySupplyCurvePointError: - logger.debug('SC gid {} is fully excluded or does not ' - 'have any valid source data!'.format(gid)) + logger.debug( + "SC gid {} is fully excluded or does not " + "have any valid source data!".format(gid) + ) except Exception as e: - msg = 'SC gid {} failed!'.format(gid) + msg = "SC gid {} failed!".format(gid) logger.exception(msg) raise RuntimeError(msg) from e else: n_finished += 1 - logger.debug('Serial bespoke: ' - '{} out of {} points complete' - .format(n_finished, len(gids))) + logger.debug( + "Serial bespoke: " + "{} out of {} points complete".format( + n_finished, len(gids) + ) + ) log_mem(logger) out[gid] = bsp_plant_out @@ -2271,17 +2495,18 @@ def run_parallel(self, max_workers=None): Bespoke outputs keyed by sc point gid """ - logger.info('Running bespoke optimization for points {} through {} ' - 'at a resolution of {} on {} cores.' - .format(self.gids[0], self.gids[-1], self._resolution, - max_workers)) + logger.info( + "Running bespoke optimization for points {} through {} " + "at a resolution of {} on {} cores.".format( + self.gids[0], self.gids[-1], self._resolution, max_workers + ) + ) futures = [] out = {} n_finished = 0 - loggers = [__name__, 'reV.supply_curve.point_summary', 'reV'] + loggers = [__name__, "reV.supply_curve.point_summary", "reV"] with SpawnProcessPool(max_workers=max_workers, loggers=loggers) as exe: - # iterate through split executions, submitting each to worker for gid in self.gids: # submit executions and append to futures list @@ -2290,36 +2515,39 @@ def run_parallel(self, max_workers=None): rs, cs = self.slice_lookup[gid] gid_incl_mask = self._inclusion_mask[rs, cs] - futures.append(exe.submit( - self.run_serial, - self._excl_fpath, - self._res_fpath, - self._tm_dset, - self.sam_sys_inputs_with_site_data(gid), - self._obj_fun, - self._cap_cost_fun, - self._foc_fun, - self._voc_fun, - self._min_spacing, - wake_loss_multiplier=self._wake_loss_multiplier, - ga_kwargs=self._ga_kwargs, - output_request=self._output_request, - ws_bins=self._ws_bins, - wd_bins=self._wd_bins, - excl_dict=self._excl_dict, - inclusion_mask=gid_incl_mask, - area_filter_kernel=self._area_filter_kernel, - min_area=self._min_area, - resolution=self._resolution, - excl_area=self._excl_area, - data_layers=self._data_layers, - gids=gid, - exclusion_shape=self.shape, - slice_lookup=copy.deepcopy(self.slice_lookup), - prior_meta=self._get_prior_meta(gid), - gid_map=self._gid_map, - bias_correct=self._get_bc_for_gid(gid), - pre_loaded_data=self._pre_loaded_data_for_sc_gid(gid))) + futures.append( + exe.submit( + self.run_serial, + self._excl_fpath, + self._res_fpath, + self._tm_dset, + self.sam_sys_inputs_with_site_data(gid), + self._obj_fun, + self._cap_cost_fun, + self._foc_fun, + self._voc_fun, + self._min_spacing, + wake_loss_multiplier=self._wake_loss_multiplier, + ga_kwargs=self._ga_kwargs, + output_request=self._output_request, + ws_bins=self._ws_bins, + wd_bins=self._wd_bins, + excl_dict=self._excl_dict, + inclusion_mask=gid_incl_mask, + area_filter_kernel=self._area_filter_kernel, + min_area=self._min_area, + resolution=self._resolution, + excl_area=self._excl_area, + data_layers=self._data_layers, + gids=gid, + exclusion_shape=self.shape, + slice_lookup=copy.deepcopy(self.slice_lookup), + prior_meta=self._get_prior_meta(gid), + gid_map=self._gid_map, + bias_correct=self._get_bc_for_gid(gid), + pre_loaded_data=self._pre_loaded_data_for_sc_gid(gid), + ) + ) # gather results for future in as_completed(futures): @@ -2327,12 +2555,17 @@ def run_parallel(self, max_workers=None): out.update(future.result()) if n_finished % 10 == 0: mem = psutil.virtual_memory() - logger.info('Parallel bespoke futures collected: ' - '{} out of {}. Memory usage is {:.3f} GB out ' - 'of {:.3f} GB ({:.2f}% utilized).' - .format(n_finished, len(futures), - mem.used / 1e9, mem.total / 1e9, - 100 * mem.used / mem.total)) + logger.info( + "Parallel bespoke futures collected: " + "{} out of {}. Memory usage is {:.3f} GB out " + "of {:.3f} GB ({:.2f}% utilized).".format( + n_finished, + len(futures), + mem.used / 1e9, + mem.total / 1e9, + 100 * mem.used / mem.total, + ) + ) return out @@ -2358,7 +2591,7 @@ def run(self, out_fpath=None, max_workers=None): """ # parallel job distribution test. - if self._obj_fun == 'test': + if self._obj_fun == "test": return True if max_workers == 1: @@ -2376,33 +2609,35 @@ def run(self, out_fpath=None, max_workers=None): wlm = self._wake_loss_multiplier i_bc = self._get_bc_for_gid(gid) - si = self.run_serial(self._excl_fpath, - self._res_fpath, - self._tm_dset, - sam_inputs, - self._obj_fun, - self._cap_cost_fun, - self._foc_fun, - self._voc_fun, - min_spacing=self._min_spacing, - wake_loss_multiplier=wlm, - ga_kwargs=self._ga_kwargs, - output_request=self._output_request, - ws_bins=self._ws_bins, - wd_bins=self._wd_bins, - excl_dict=self._excl_dict, - inclusion_mask=gid_incl_mask, - area_filter_kernel=afk, - min_area=self._min_area, - resolution=self._resolution, - excl_area=self._excl_area, - data_layers=self._data_layers, - slice_lookup=slice_lookup, - prior_meta=prior_meta, - gid_map=self._gid_map, - bias_correct=i_bc, - gids=gid, - pre_loaded_data=pre_loaded_data) + si = self.run_serial( + self._excl_fpath, + self._res_fpath, + self._tm_dset, + sam_inputs, + self._obj_fun, + self._cap_cost_fun, + self._foc_fun, + self._voc_fun, + min_spacing=self._min_spacing, + wake_loss_multiplier=wlm, + ga_kwargs=self._ga_kwargs, + output_request=self._output_request, + ws_bins=self._ws_bins, + wd_bins=self._wd_bins, + excl_dict=self._excl_dict, + inclusion_mask=gid_incl_mask, + area_filter_kernel=afk, + min_area=self._min_area, + resolution=self._resolution, + excl_area=self._excl_area, + data_layers=self._data_layers, + slice_lookup=slice_lookup, + prior_meta=prior_meta, + gid_map=self._gid_map, + bias_correct=i_bc, + gids=gid, + pre_loaded_data=pre_loaded_data, + ) self._outputs.update(si) else: self._outputs = self.run_parallel(max_workers=max_workers) diff --git a/reV/hybrids/hybrids.py b/reV/hybrids/hybrids.py index 4d5c11708..59ecdcc5e 100644 --- a/reV/hybrids/hybrids.py +++ b/reV/hybrids/hybrids.py @@ -16,6 +16,7 @@ from reV.handlers.outputs import Outputs from reV.hybrids.hybrid_methods import HYBRID_METHODS +from reV.utilities import MetaKeyName from reV.utilities.exceptions import ( FileInputError, InputError, @@ -30,8 +31,10 @@ SOLAR_PREFIX = 'solar_' WIND_PREFIX = 'wind_' NON_DUPLICATE_COLS = { - MetaKeyName.LATITUDE, MetaKeyName.LONGITUDE, MetaKeyName.COUNTRY, 'state', 'county', 'elevation', - MetaKeyName.TIMEZONE, MetaKeyName.SC_POINT_GID, MetaKeyName.SC_ROW_IND, MetaKeyName.SC_COL_IND + MetaKeyName.LATITUDE, MetaKeyName.LONGITUDE, + 'country', 'state', 'county', MetaKeyName.ELEVATION, + MetaKeyName.TIMEZONE, MetaKeyName.SC_POINT_GID, + MetaKeyName.SC_ROW_IND, MetaKeyName.SC_COL_IND } DROPPED_COLUMNS = [MetaKeyName.GID] DEFAULT_FILL_VALUES = {'solar_capacity': 0, 'wind_capacity': 0, @@ -235,7 +238,7 @@ def _validate_num_profiles(self): e = msg.format(PROFILE_DSET_REGEX, fp) logger.error(e) raise FileInputError(e) - elif len(profile_dset_names) > 1: + if len(profile_dset_names) > 1: msg = ("Found more than one profile in {!r}: {}. " "This module is not intended for hybridization of " "multiple representative profiles. Please re-run " @@ -243,8 +246,7 @@ def _validate_num_profiles(self): e = msg.format(fp, profile_dset_names) logger.error(e) raise FileInputError(e) - else: - self.profile_dset_names += profile_dset_names + self.profile_dset_names += profile_dset_names def _validate_merge_col_exists(self): """Validate the existence of the merge column. @@ -403,8 +405,7 @@ def hybrid_meta(self): """ if self._hybrid_meta is None or self.__hybrid_meta_cols is None: return self._hybrid_meta - else: - return self._hybrid_meta[self.__hybrid_meta_cols] + return self._hybrid_meta[self.__hybrid_meta_cols] def validate_input(self): """Validate the input parameters. @@ -647,9 +648,9 @@ def _merge_type(self): """Determine the type of merge to use for meta based on user input.""" if self._allow_solar_only and self._allow_wind_only: return 'outer' - elif self._allow_solar_only and not self._allow_wind_only: + if self._allow_solar_only and not self._allow_wind_only: return 'left' - elif not self._allow_solar_only and self._allow_wind_only: + if not self._allow_solar_only and self._allow_wind_only: return 'right' return 'inner' @@ -721,8 +722,7 @@ def _verify_col_match_post_merge(self, col_name): & (self._hybrid_meta[c2].notnull()) ] return np.allclose(compare_df[c1], compare_df[c2]) - else: - return True + return True def _fillna_meta_cols(self): """Fill N/A values as specified by user (and internals).""" diff --git a/reV/supply_curve/points.py b/reV/supply_curve/points.py index 622926ea9..de8676391 100644 --- a/reV/supply_curve/points.py +++ b/reV/supply_curve/points.py @@ -1047,18 +1047,18 @@ def h5(self): def country(self): """Get the SC point country based on the resource meta data.""" country = None - if MetaKeyName.COUNTRY in self.h5.meta and self.county is not None: + if 'country' in self.h5.meta and self.county is not None: # make sure country and county are coincident counties = self.h5.meta.loc[self.h5_gid_set, - MetaKeyName.COUNTY].values + 'county'].values iloc = np.where(counties == self.county)[0][0] country = self.h5.meta.loc[self.h5_gid_set, - MetaKeyName.COUNTRY].values + 'country'].values country = country[iloc] - elif MetaKeyName.COUNTRY in self.h5.meta: + elif 'country' in self.h5.meta: country = self.h5.meta.loc[self.h5_gid_set, - MetaKeyName.COUNTRY].mode() + 'country'].mode() country = country.values[0] return country @@ -1067,16 +1067,16 @@ def country(self): def state(self): """Get the SC point state based on the resource meta data.""" state = None - if MetaKeyName.STATE in self.h5.meta and self.county is not None: + if 'state' in self.h5.meta and self.county is not None: # make sure state and county are coincident counties = self.h5.meta.loc[self.h5_gid_set, - MetaKeyName.COUNTY].values + 'county'].values iloc = np.where(counties == self.county)[0][0] - state = self.h5.meta.loc[self.h5_gid_set, MetaKeyName.STATE].values + state = self.h5.meta.loc[self.h5_gid_set, 'state'].values state = state[iloc] - elif MetaKeyName.STATE in self.h5.meta: - state = self.h5.meta.loc[self.h5_gid_set, MetaKeyName.STATE].mode() + elif 'state' in self.h5.meta: + state = self.h5.meta.loc[self.h5_gid_set, 'state'].mode() state = state.values[0] return state @@ -1085,9 +1085,9 @@ def state(self): def county(self): """Get the SC point county based on the resource meta data.""" county = None - if MetaKeyName.COUNTY in self.h5.meta: + if 'county' in self.h5.meta: county = self.h5.meta.loc[self.h5_gid_set, - MetaKeyName.COUNTY].mode() + 'county'].mode() county = county.values[0] return county @@ -1109,7 +1109,7 @@ def timezone(self): if MetaKeyName.TIMEZONE in self.h5.meta and self.county is not None: # make sure timezone flag and county are coincident counties = self.h5.meta.loc[self.h5_gid_set, - MetaKeyName.COUNTY].values + 'county'].values iloc = np.where(counties == self.county)[0][0] timezone = self.h5.meta.loc[self.h5_gid_set, MetaKeyName.TIMEZONE].values @@ -1130,7 +1130,7 @@ def offshore(self): if MetaKeyName.OFFSHORE in self.h5.meta and self.county is not None: # make sure offshore flag and county are coincident counties = self.h5.meta.loc[self.h5_gid_set, - MetaKeyName.COUNTY].values + 'county'].values iloc = np.where(counties == self.county)[0][0] offshore = self.h5.meta.loc[self.h5_gid_set, MetaKeyName.OFFSHORE].values @@ -1176,9 +1176,9 @@ def summary(self): MetaKeyName.AREA_SQ_KM: self.area, MetaKeyName.LATITUDE: self.latitude, MetaKeyName.LONGITUDE: self.longitude, - MetaKeyName.COUNTRY: self.country, - MetaKeyName.STATE: self.state, - MetaKeyName.COUNTY: self.county, + 'country': self.country, + 'state': self.state, + 'county': self.county, MetaKeyName.ELEVATION: self.elevation, MetaKeyName.TIMEZONE: self.timezone, } @@ -2023,9 +2023,9 @@ def point_summary(self, args=None): MetaKeyName.LATITUDE: self.latitude, MetaKeyName.LONGITUDE: self.longitude, MetaKeyName.TIMEZONE: self.timezone, - MetaKeyName.COUNTRY: self.country, - MetaKeyName.STATE: self.state, - MetaKeyName.COUNTY: self.county, + 'country': self.country, + 'state': self.state, + 'county': self.county, MetaKeyName.ELEVATION: self.elevation, MetaKeyName.RES_GIDS: self.res_gid_set, MetaKeyName.GEN_GIDS: self.gen_gid_set, diff --git a/reV/utilities/__init__.py b/reV/utilities/__init__.py index 8d53e9fe9..48c51d8cb 100644 --- a/reV/utilities/__init__.py +++ b/reV/utilities/__init__.py @@ -28,10 +28,6 @@ class MetaKeyName(str, Enum): AREA_SQ_KM = 'area_sq_km' LATITUDE = 'latitude' LONGITUDE = 'longitude' - COUNTRY = 'country' - STATE = 'state' - COUNTY = 'county' - COUNTRY = 'country', ELEVATION = 'elevation' TIMEZONE = 'timezone' MEAN_CF = 'mean_cf' From 73515d2ab6c968b1a16c65d24ee9693d3ac7461c Mon Sep 17 00:00:00 2001 From: bnb32 Date: Sun, 12 May 2024 10:57:26 -0600 Subject: [PATCH 07/61] MetaKeyName enum: init commit --- reV/utilities/__init__.py | 53 +++++++++++++++++++++++++++++++++++++++ tests/test_gen_config.py | 18 ++++++------- 2 files changed, 62 insertions(+), 9 deletions(-) diff --git a/reV/utilities/__init__.py b/reV/utilities/__init__.py index ae58cdb7f..3ad340833 100644 --- a/reV/utilities/__init__.py +++ b/reV/utilities/__init__.py @@ -3,11 +3,64 @@ reV utilities. """ from enum import Enum + import PySAM from rex.utilities.loggers import log_versions as rex_log_versions + from reV.version import __version__ +class MetaKeyName(str, Enum): + """An enumerated map to summary/meta keys. + + Each output name should match the name of a key in + meth:`AggregationSupplyCurvePoint.summary` or + meth:`GenerationSupplyCurvePoint.point_summary` or + meth:`BespokeSinglePlant.meta` + """ + + SC_POINT_GID = 'sc_point_gid' + SOURCE_GIDS = 'source_gids' + SC_GID = 'sc_gid' + GID_COUNTS = 'gid_counts' + GID = 'gid' + N_GIDS = 'n_gids' + RES_GIDS = 'res_gids' + GEN_GIDS = 'gen_gids' + AREA_SQ_KM = 'area_sq_km' + LATITUDE = 'latitude' + LONGITUDE = 'longitude' + ELEVATION = 'elevation' + TIMEZONE = 'timezone' + MEAN_CF = 'mean_cf' + MEAN_LCOE = 'mean_lcoe' + MEAN_RES = 'mean_res' + CAPACITY = 'capacity' + OFFSHORE = 'offshore' + SC_ROW_IND = 'sc_row_ind' + SC_COL_IND = 'sc_col_ind' + CAPACITY_AC = 'capacity_ac' + SC_POINT_CAPITAL_COST = 'sc_point_capital_cost' + SC_POINT_FIXED_OPERATING_COST = 'sc_point_fixed_operating_cost' + SC_POINT_ANNUAL_ENERGY = 'sc_point_annual_energy' + SC_POINT_ANNUAL_ENERGY_AC = 'sc_point_annual_energy_ac' + MEAN_FRICTION = 'mean_friction' + MEAN_LCOE_FRICTION = 'mean_lcoe_friction' + TOTAL_LCOE_FRICTION = 'total_lcoe_friction' + RAW_LCOE = 'raw_lcoe' + CAPITAL_COST_SCALAR = 'capital_cost_scalar' + SCALED_CAPITAL_COST = 'scaled_capital_cost' + SCALED_SC_POINT_CAPITAL_COST = 'scaled_sc_point_capital_cost' + TURBINE_X_COORDS = 'turbine_x_coords' + TURBINE_Y_COORDS = 'turbine_y_coords' + + def __str__(self): + return self.value + + def __format__(self, format_spec): + return str.__format__(self.value, format_spec) + + class ModuleName(str, Enum): """A collection of the module names available in reV. diff --git a/tests/test_gen_config.py b/tests/test_gen_config.py index 277589a92..9b9f9812a 100644 --- a/tests/test_gen_config.py +++ b/tests/test_gen_config.py @@ -8,21 +8,21 @@ @author: gbuster """ import json -import numpy as np import os -import pandas as pd -import pytest import tempfile import traceback +import numpy as np +import pandas as pd +import pytest +from rex.utilities.utilities import safe_json_load + +from reV import TESTDATADIR from reV.cli import main from reV.config.project_points import ProjectPoints from reV.generation.generation import Gen -from reV import TESTDATADIR from reV.handlers.outputs import Outputs -from rex.utilities.utilities import safe_json_load - RTOL = 0.0 ATOL = 0.04 @@ -43,8 +43,8 @@ def get_r1_profiles(year=2012, tech='pv'): return data -@pytest.mark.parametrize('tech', ['pv', 'wind']) # noqa: C901 -def test_gen_from_config(runner, tech, clear_loggers): # noqa: C901 +@pytest.mark.parametrize('tech', ['pv', 'wind']) +def test_gen_from_config(runner, tech, clear_loggers): """Gen PV CF profiles with write to disk and compare against rev1.""" with tempfile.TemporaryDirectory() as td: @@ -219,7 +219,7 @@ def test_gen_mw_config_input(runner, clear_loggers, expected_log_message): assert result.exit_code == 0, msg log_file = os.path.join(run_dir, 'generation_generation.log') - with open(log_file, "r") as fh: + with open(log_file) as fh: assert expected_log_message in fh.read() clear_loggers() From 664eaf4ec719b17e7cca10bf428dc9b199492227 Mon Sep 17 00:00:00 2001 From: bnb32 Date: Sun, 12 May 2024 11:58:31 -0600 Subject: [PATCH 08/61] all MetaKeyName replacements and imports --- examples/aws_pcluster/make_project_points.py | 4 +- examples/marine_energy/plot_lcoe.py | 7 +- reV/SAM/SAM.py | 77 +++++---- reV/SAM/econ.py | 4 +- reV/SAM/generation.py | 147 ++++++++++-------- reV/bespoke/bespoke.py | 117 +++++++------- reV/config/project_points.py | 69 ++++---- reV/econ/econ.py | 36 ++--- reV/econ/economies_of_scale.py | 9 +- reV/generation/base.py | 61 ++++---- reV/generation/generation.py | 22 +-- reV/handlers/exclusions.py | 31 ++-- reV/handlers/outputs.py | 11 +- reV/hybrids/hybrids.py | 99 ++++++------ reV/nrwal/nrwal.py | 28 ++-- reV/qa_qc/cli_qa_qc.py | 24 +-- reV/qa_qc/qa_qc.py | 30 ++-- reV/qa_qc/summary.py | 85 +++++----- reV/rep_profiles/rep_profiles.py | 45 +++--- reV/supply_curve/aggregation.py | 39 ++--- reV/supply_curve/competitive_wind_farms.py | 38 +++-- reV/supply_curve/exclusions.py | 35 ++--- reV/supply_curve/extent.py | 15 +- reV/supply_curve/points.py | 145 +++++++++-------- reV/supply_curve/sc_aggregation.py | 65 ++++---- reV/supply_curve/supply_curve.py | 109 +++++++------ reV/supply_curve/tech_mapping.py | 21 +-- reV/utilities/pytest_utils.py | 9 +- tests/hsds.py | 7 +- tests/test_bespoke.py | 130 ++++++++-------- tests/test_config.py | 38 ++--- tests/test_curtailment.py | 18 ++- tests/test_econ_of_scale.py | 44 +++--- tests/test_econ_windbos.py | 2 +- tests/test_gen_forecast.py | 29 ++-- tests/test_gen_pv.py | 27 ++-- tests/test_gen_wind.py | 41 ++--- tests/test_handlers_outputs.py | 15 +- tests/test_handlers_transmission.py | 6 +- tests/test_hybrids.py | 113 +++++++------- tests/test_losses_power_curve.py | 95 +++++------ tests/test_losses_scheduled.py | 87 ++++++----- tests/test_nrwal.py | 36 ++--- tests/test_rep_profiles.py | 107 ++++++------- tests/test_sam.py | 33 ++-- tests/test_supply_curve_aggregation.py | 24 +-- .../test_supply_curve_aggregation_friction.py | 23 +-- tests/test_supply_curve_compute.py | 58 +++---- tests/test_supply_curve_points.py | 30 ++-- tests/test_supply_curve_sc_aggregation.py | 97 ++++++------ tests/test_supply_curve_tech_mapping.py | 32 ++-- tests/test_supply_curve_vpd.py | 20 +-- tests/test_supply_curve_wind_dirs.py | 24 +-- 53 files changed, 1306 insertions(+), 1212 deletions(-) diff --git a/examples/aws_pcluster/make_project_points.py b/examples/aws_pcluster/make_project_points.py index 8b9d08514..0c0e25c80 100644 --- a/examples/aws_pcluster/make_project_points.py +++ b/examples/aws_pcluster/make_project_points.py @@ -22,7 +22,7 @@ print(meta[mask]) pp = meta[mask] - pp['gid'] = pp.index.values + pp[MetaKeyName.GID] = pp.index.values pp['config'] = 'def' - pp = pp[['gid', 'config']] + pp = pp[[MetaKeyName.GID, 'config']] pp.to_csv('./points_front_range.csv', index=False) diff --git a/examples/marine_energy/plot_lcoe.py b/examples/marine_energy/plot_lcoe.py index 8842228d5..8d6e987a0 100644 --- a/examples/marine_energy/plot_lcoe.py +++ b/examples/marine_energy/plot_lcoe.py @@ -2,8 +2,11 @@ Simple plot script for wave LCOE """ import os -import pandas as pd + import matplotlib.pyplot as plt +import pandas as pd + +from reV.utilities import MetaKeyName fps = ['./atlantic_rm5/atlantic_rm5_agg.csv', './pacific_rm5/pacific_rm5_agg.csv', @@ -11,7 +14,7 @@ for fp in fps: df = pd.read_csv(fp) - a = plt.scatter(df.longitude, df.latitude, c=df['mean_lcoe'], + a = plt.scatter(df.longitude, df.latitude, c=df[MetaKeyName.MEAN_LCOE], s=0.5, vmin=0, vmax=1500) plt.colorbar(a, label='lcoe_fcr ($/MWh)') tag = os.path.basename(fp).replace('_agg.csv', '') diff --git a/reV/SAM/SAM.py b/reV/SAM/SAM.py index e8bb9438e..ddd08039a 100644 --- a/reV/SAM/SAM.py +++ b/reV/SAM/SAM.py @@ -6,22 +6,34 @@ import copy import json import logging -import numpy as np import os -import pandas as pd from warnings import warn -import PySAM.GenericSystem as generic -from reV.utilities.exceptions import (SAMInputWarning, SAMInputError, - SAMExecutionError, ResourceError) - -from rex.multi_file_resource import (MultiFileResource, MultiFileNSRDB, - MultiFileWTK) -from rex.renewable_resource import (WindResource, SolarResource, NSRDB, - WaveResource, GeothermalResource) +import numpy as np +import pandas as pd +import PySAM.GenericSystem as generic +from rex.multi_file_resource import ( + MultiFileNSRDB, + MultiFileResource, + MultiFileWTK, +) from rex.multi_res_resource import MultiResolutionResource +from rex.renewable_resource import ( + NSRDB, + GeothermalResource, + SolarResource, + WaveResource, + WindResource, +) from rex.utilities.utilities import check_res_file +from reV.utilities import MetaKeyName +from reV.utilities.exceptions import ( + ResourceError, + SAMExecutionError, + SAMInputError, + SAMInputWarning, +) logger = logging.getLogger(__name__) @@ -364,19 +376,18 @@ def __setitem__(self, key, value): .format(key, self.pysam)) logger.exception(msg) raise SAMInputError(msg) - else: - self.sam_sys_inputs[key] = value - group = self._get_group(key, outputs=False) - try: - setattr(getattr(self.pysam, group), key, value) - except Exception as e: - msg = ('Could not set input key "{}" to ' - 'group "{}" in "{}".\n' - 'Data is: {} ({})\n' - 'Received the following error: "{}"' - .format(key, group, self.pysam, value, type(value), e)) - logger.exception(msg) - raise SAMInputError(msg) from e + self.sam_sys_inputs[key] = value + group = self._get_group(key, outputs=False) + try: + setattr(getattr(self.pysam, group), key, value) + except Exception as e: + msg = ('Could not set input key "{}" to ' + 'group "{}" in "{}".\n' + 'Data is: {} ({})\n' + 'Received the following error: "{}"' + .format(key, group, self.pysam, value, type(value), e)) + logger.exception(msg) + raise SAMInputError(msg) from e @property def pysam(self): @@ -727,7 +738,7 @@ def get_time_interval(cls, time_index): if t == 1.0: time_interval += 1 break - elif t == 0.0: + if t == 0.0: time_interval += 1 return int(time_interval) @@ -785,22 +796,20 @@ def _is_arr_like(val): """Returns true if SAM data is array-like. False if scalar.""" if isinstance(val, (int, float, str)): return False + try: + len(val) + except TypeError: + return False else: - try: - len(val) - except TypeError: - return False - else: - return True + return True @classmethod def _is_hourly(cls, val): """Returns true if SAM data is hourly or sub-hourly. False otherise.""" if not cls._is_arr_like(val): return False - else: - L = len(val) - return L >= 8760 + L = len(val) + return L >= 8760 def outputs_to_utc_arr(self): """Convert array-like SAM outputs to UTC np.ndarrays""" @@ -815,7 +824,7 @@ def outputs_to_utc_arr(self): output = output.astype(np.int32) if self._is_hourly(output): - n_roll = int(-1 * self.meta['timezone'] + n_roll = int(-1 * self.meta[MetaKeyName.TIMEZONE] * self.time_interval) output = np.roll(output, n_roll) diff --git a/reV/SAM/econ.py b/reV/SAM/econ.py index 648606812..4469ff0c6 100644 --- a/reV/SAM/econ.py +++ b/reV/SAM/econ.py @@ -172,7 +172,7 @@ def _get_cf_profiles(sites, cf_file, year): with Outputs(cf_file) as cfh: # get the index location of the site in question - site_gids = list(cfh.get_meta_arr('gid')) + site_gids = list(cfh.get_meta_arr(MetaKeyName.GID)) isites = [site_gids.index(s) for s in sites] # look for the cf_profile dataset @@ -375,7 +375,7 @@ def _parse_lcoe_inputs(site_df, cf_file, year): # get the cf_file meta data gid's to use as indexing tools with Outputs(cf_file) as cfh: - site_gids = list(cfh.meta['gid']) + site_gids = list(cfh.meta[MetaKeyName.GID]) calc_aey = False if 'annual_energy' not in site_df: diff --git a/reV/SAM/generation.py b/reV/SAM/generation.py index 5ebfa8803..d39d7c97c 100644 --- a/reV/SAM/generation.py +++ b/reV/SAM/generation.py @@ -5,9 +5,8 @@ additional reV features. """ import copy -import os import logging - +import os from abc import ABC, abstractmethod from tempfile import TemporaryDirectory from warnings import warn @@ -26,22 +25,28 @@ import PySAM.TroughPhysicalProcessHeat as PySamTpph import PySAM.Windpower as PySamWindPower -from reV.losses import ScheduledLossesMixin, PowerCurveLossesMixin -from reV.SAM.defaults import (DefaultGeothermal, - DefaultPvWattsv5, - DefaultPvWattsv8, - DefaultPvSamv1, - DefaultWindPower, - DefaultTcsMoltenSalt, - DefaultSwh, - DefaultTroughPhysicalProcessHeat, - DefaultLinearFresnelDsgIph, - DefaultMhkWave) +from reV.losses import PowerCurveLossesMixin, ScheduledLossesMixin +from reV.SAM.defaults import ( + DefaultGeothermal, + DefaultLinearFresnelDsgIph, + DefaultMhkWave, + DefaultPvSamv1, + DefaultPvWattsv5, + DefaultPvWattsv8, + DefaultSwh, + DefaultTcsMoltenSalt, + DefaultTroughPhysicalProcessHeat, + DefaultWindPower, +) from reV.SAM.econ import LCOE, SingleOwner from reV.SAM.SAM import RevPySam +from reV.utilities import MetaKeyName from reV.utilities.curtailment import curtail -from reV.utilities.exceptions import (SAMInputWarning, SAMExecutionError, - InputError) +from reV.utilities.exceptions import ( + InputError, + SAMExecutionError, + SAMInputWarning, +) logger = logging.getLogger(__name__) @@ -240,19 +245,19 @@ def tz_elev_check(sam_sys_inputs, site_sys_inputs, meta): if meta is not None: if sam_sys_inputs is not None: - if 'elevation' in sam_sys_inputs: - meta['elevation'] = sam_sys_inputs['elevation'] - if 'timezone' in sam_sys_inputs: - meta['timezone'] = int(sam_sys_inputs['timezone']) + if MetaKeyName.ELEVATION in sam_sys_inputs: + meta[MetaKeyName.ELEVATION] = sam_sys_inputs[MetaKeyName.ELEVATION] + if MetaKeyName.TIMEZONE in sam_sys_inputs: + meta[MetaKeyName.TIMEZONE] = int(sam_sys_inputs[MetaKeyName.TIMEZONE]) # site-specific inputs take priority over generic system inputs if site_sys_inputs is not None: - if 'elevation' in site_sys_inputs: - meta['elevation'] = site_sys_inputs['elevation'] - if 'timezone' in site_sys_inputs: - meta['timezone'] = int(site_sys_inputs['timezone']) + if MetaKeyName.ELEVATION in site_sys_inputs: + meta[MetaKeyName.ELEVATION] = site_sys_inputs[MetaKeyName.ELEVATION] + if MetaKeyName.TIMEZONE in site_sys_inputs: + meta[MetaKeyName.TIMEZONE] = int(site_sys_inputs[MetaKeyName.TIMEZONE]) - if 'timezone' not in meta: + if MetaKeyName.TIMEZONE not in meta: msg = ('Need timezone input to run SAM gen. Not found in ' 'resource meta or technology json input config.') raise SAMExecutionError(msg) @@ -261,9 +266,9 @@ def tz_elev_check(sam_sys_inputs, site_sys_inputs, meta): @property def has_timezone(self): - """ Returns true if instance has a timezone set """ + """Returns true if instance has a timezone set""" if self._meta is not None: - if 'timezone' in self.meta: + if MetaKeyName.TIMEZONE in self.meta: return True return False @@ -514,10 +519,11 @@ def reV_run(cls, points_control, res_file, site_df, class AbstractSamGenerationFromWeatherFile(AbstractSamGeneration, ABC): - """Base class for running sam generation with a weather file on disk. """ - WF_META_DROP_COLS = {'elevation', 'timezone', 'country', 'state', 'county', - 'urban', 'population', 'landcover', 'latitude', - 'longitude'} + """Base class for running sam generation with a weather file on disk.""" + + WF_META_DROP_COLS = {MetaKeyName.ELEVATION, MetaKeyName.TIMEZONE, 'country', 'state', 'county', + 'urban', 'population', 'landcover', MetaKeyName.LATITUDE, + MetaKeyName.LONGITUDE} @property @abstractmethod @@ -587,16 +593,16 @@ def _create_pysam_wfile(self, resource, meta): # ------- Process metadata m = pd.DataFrame(meta).T - timezone = m['timezone'] + timezone = m[MetaKeyName.TIMEZONE] m['Source'] = 'NSRDB' m['Location ID'] = meta.name m['City'] = '-' m['State'] = m['state'].apply(lambda x: '-' if x == 'None' else x) m['Country'] = m['country'].apply(lambda x: '-' if x == 'None' else x) - m['Latitude'] = m['latitude'] - m['Longitude'] = m['longitude'] + m['Latitude'] = m[MetaKeyName.LATITUDE] + m[MetaKeyName.LONGITUDE] = m[MetaKeyName.LONGITUDE] m['Time Zone'] = timezone - m['Elevation'] = m['elevation'] + m[MetaKeyName.ELEVATION] = m[MetaKeyName.ELEVATION] m['Local Time Zone'] = timezone m['Dew Point Units'] = 'c' m['DHI Units'] = 'w/m2' @@ -734,7 +740,7 @@ def set_resource_data(self, resource, meta): # ensure that resource array length is multiple of 8760 arr = self.ensure_res_len(arr, time_index) - n_roll = int(self._meta['timezone'] * self.time_interval) + n_roll = int(self._meta[MetaKeyName.TIMEZONE] * self.time_interval) arr = np.roll(arr, n_roll) if var in irrad_vars: @@ -746,12 +752,12 @@ def set_resource_data(self, resource, meta): resource[var] = arr.tolist() - resource['lat'] = meta['latitude'] - resource['lon'] = meta['longitude'] - resource['tz'] = meta['timezone'] + resource['lat'] = meta[MetaKeyName.LATITUDE] + resource['lon'] = meta[MetaKeyName.LONGITUDE] + resource['tz'] = meta[MetaKeyName.TIMEZONE] - if 'elevation' in meta: - resource['elev'] = meta['elevation'] + if MetaKeyName.ELEVATION in meta: + resource['elev'] = meta[MetaKeyName.ELEVATION] else: resource['elev'] = 0.0 @@ -905,10 +911,10 @@ def set_resource_data(self, resource, meta): respectively. """ - bad_location_input = ((meta['latitude'] < -90) - | (meta['latitude'] > 90) - | (meta['longitude'] < -180) - | (meta['longitude'] > 180)) + bad_location_input = ((meta[MetaKeyName.LATITUDE] < -90) + | (meta[MetaKeyName.LATITUDE] > 90) + | (meta[MetaKeyName.LONGITUDE] < -180) + | (meta[MetaKeyName.LONGITUDE] > 180)) if bad_location_input.any(): raise ValueError("Detected latitude/longitude values outside of " "the range -90 to 90 and -180 to 180, " @@ -934,7 +940,7 @@ def set_latitude_tilt_az(sam_sys_inputs, meta): sam_sys_inputs : dict Site-agnostic SAM system model inputs arguments. If for a pv simulation the "tilt" parameter was originally not - present or set to 'lat' or 'latitude', the tilt will be set to + present or set to 'lat' or MetaKeyName.LATITUDE, the tilt will be set to the absolute value of the latitude found in meta and the azimuth will be 180 if lat>0, 0 if lat<0. """ @@ -945,15 +951,14 @@ def set_latitude_tilt_az(sam_sys_inputs, meta): warn('No tilt specified, setting at latitude.', SAMInputWarning) set_tilt = True - else: - if (sam_sys_inputs['tilt'] == 'lat' - or sam_sys_inputs['tilt'] == 'latitude'): - set_tilt = True + elif (sam_sys_inputs['tilt'] == 'lat' + or sam_sys_inputs['tilt'] == MetaKeyName.LATITUDE): + set_tilt = True if set_tilt: # set tilt to abs(latitude) - sam_sys_inputs['tilt'] = np.abs(meta['latitude']) - if meta['latitude'] > 0: + sam_sys_inputs['tilt'] = np.abs(meta[MetaKeyName.LATITUDE]) + if meta[MetaKeyName.LATITUDE] > 0: # above the equator, az = 180 sam_sys_inputs['azimuth'] = 180 else: @@ -1127,6 +1132,7 @@ def collect_outputs(self, output_lookup=None): class PvWattsv5(AbstractSamPv): """Photovoltaic (PV) generation with pvwattsv5. """ + MODULE = 'pvwattsv5' PYSAM = PySamPv5 @@ -1144,6 +1150,7 @@ def default(): class PvWattsv7(AbstractSamPv): """Photovoltaic (PV) generation with pvwattsv7. """ + MODULE = 'pvwattsv7' PYSAM = PySamPv7 @@ -1161,6 +1168,7 @@ def default(): class PvWattsv8(AbstractSamPv): """Photovoltaic (PV) generation with pvwattsv8. """ + MODULE = 'pvwattsv8' PYSAM = PySamPv8 @@ -1220,6 +1228,7 @@ def default(): class TcsMoltenSalt(AbstractSamSolar): """Concentrated Solar Power (CSP) generation with tower molten salt """ + MODULE = 'tcsmolten_salt' PYSAM = PySamCSP @@ -1252,6 +1261,7 @@ class SolarWaterHeat(AbstractSamGenerationFromWeatherFile): """ Solar Water Heater generation """ + MODULE = 'solarwaterheat' PYSAM = PySamSwh PYSAM_WEATHER_TAG = 'solar_resource_file' @@ -1271,6 +1281,7 @@ class LinearDirectSteam(AbstractSamGenerationFromWeatherFile): """ Process heat linear Fresnel direct steam generation """ + MODULE = 'lineardirectsteam' PYSAM = PySamLds PYSAM_WEATHER_TAG = 'file_name' @@ -1305,6 +1316,7 @@ class TroughPhysicalHeat(AbstractSamGenerationFromWeatherFile): """ Trough Physical Process Heat generation """ + MODULE = 'troughphysicalheat' PYSAM = PySamTpph PYSAM_WEATHER_TAG = 'file_name' @@ -1508,6 +1520,7 @@ class Geothermal(AbstractSamGenerationFromWeatherFile): ``time_index_step=2`` yields hourly output, and so forth). """ + # Per Matt Prilliman on 2/22/24, it's unclear where this ratio originates, # but SAM errors out if it's exceeded. MAX_RT_TO_EGS_RATIO = 1.134324 @@ -1580,7 +1593,7 @@ def set_resource_data(self, resource, meta): self._set_costs() def _set_resource_temperature(self, resource): - """Set resource temp from data if user did not specify it. """ + """Set resource temp from data if user did not specify it.""" if "resource_temp" in self.sam_sys_inputs: logger.debug("Found 'resource_temp' value in SAM config: {:.2f}" @@ -1640,7 +1653,7 @@ def _set_egs_plant_design_temperature(self): self.sam_sys_inputs["design_temp"] = resource_temp def _set_nameplate_to_match_resource_potential(self, resource): - """Set the nameplate capacity to match the resource potential. """ + """Set the nameplate capacity to match the resource potential.""" if "nameplate" in self.sam_sys_inputs: msg = ('Found "nameplate" input in config! Resource potential ' @@ -1679,7 +1692,7 @@ def _set_resource_potential_to_match_gross_output(self): self.sam_sys_inputs["resource_potential"] = -1 return - gross_gen = (getattr(self.pysam.Outputs, "gross_output") + gross_gen = (self.pysam.Outputs.gross_output * self._RESOURCE_POTENTIAL_MULT) if "resource_potential" in self.sam_sys_inputs: msg = ('Setting "resource_potential" is not allowed! Updating ' @@ -1696,9 +1709,9 @@ def _set_resource_potential_to_match_gross_output(self): ncw = self.sam_sys_inputs.pop("num_confirmation_wells", self._DEFAULT_NUM_CONFIRMATION_WELLS) self.sam_sys_inputs["prod_and_inj_wells_to_drill"] = ( - getattr(self.pysam.Outputs, "num_wells_getem_output") + self.pysam.Outputs.num_wells_getem_output - ncw - + getattr(self.pysam.Outputs, "num_wells_getem_inj")) + + self.pysam.Outputs.num_wells_getem_inj) self["ui_calculations_only"] = 0 def _set_costs(self): @@ -1902,6 +1915,7 @@ def __init__(self, *args, **kwargs): class WindPower(AbstractSamWind): """Class for Wind generation from SAM """ + MODULE = 'windpower' PYSAM = PySamWindPower @@ -1952,7 +1966,7 @@ def set_resource_data(self, resource, meta): if 'rh' in resource: # set relative humidity for icing. rh = self.ensure_res_len(resource['rh'].values, time_index) - n_roll = int(meta['timezone'] * self.time_interval) + n_roll = int(meta[MetaKeyName.TIMEZONE] * self.time_interval) rh = np.roll(rh, n_roll, axis=0) data_dict['rh'] = rh.tolist() @@ -1960,14 +1974,14 @@ def set_resource_data(self, resource, meta): # ensure that resource array length is multiple of 8760 # roll the truncated resource array to local timezone temp = self.ensure_res_len(resource[var_list].values, time_index) - n_roll = int(meta['timezone'] * self.time_interval) + n_roll = int(meta[MetaKeyName.TIMEZONE] * self.time_interval) temp = np.roll(temp, n_roll, axis=0) data_dict['data'] = temp.tolist() - data_dict['lat'] = float(meta['latitude']) - data_dict['lon'] = float(meta['longitude']) - data_dict['tz'] = int(meta['timezone']) - data_dict['elev'] = float(meta['elevation']) + data_dict['lat'] = float(meta[MetaKeyName.LATITUDE]) + data_dict['lon'] = float(meta[MetaKeyName.LONGITUDE]) + data_dict['tz'] = int(meta[MetaKeyName.TIMEZONE]) + data_dict['elev'] = float(meta[MetaKeyName.ELEVATION]) time_index = self.ensure_res_len(time_index, time_index) data_dict['minute'] = time_index.minute.tolist() @@ -2086,6 +2100,7 @@ def set_resource_data(self, ws_edges, wd_edges, wind_dist): class MhkWave(AbstractSamGeneration): """Class for Wave generation from SAM """ + MODULE = 'mhkwave' PYSAM = PySamMhkWave @@ -2131,12 +2146,12 @@ def set_resource_data(self, resource, meta): # roll the truncated resource array to local timezone for var in ['significant_wave_height', 'energy_period']: arr = self.ensure_res_len(resource[var].values, time_index) - n_roll = int(meta['timezone'] * self.time_interval) + n_roll = int(meta[MetaKeyName.TIMEZONE] * self.time_interval) data_dict[var] = np.roll(arr, n_roll, axis=0).tolist() - data_dict['lat'] = meta['latitude'] - data_dict['lon'] = meta['longitude'] - data_dict['tz'] = meta['timezone'] + data_dict['lat'] = meta[MetaKeyName.LATITUDE] + data_dict['lon'] = meta[MetaKeyName.LONGITUDE] + data_dict['tz'] = meta[MetaKeyName.TIMEZONE] time_index = self.ensure_res_len(time_index, time_index) data_dict['minute'] = time_index.minute diff --git a/reV/bespoke/bespoke.py b/reV/bespoke/bespoke.py index e88a9db58..e9a0d6d93 100644 --- a/reV/bespoke/bespoke.py +++ b/reV/bespoke/bespoke.py @@ -3,41 +3,40 @@ reV bespoke wind plant analysis tools """ # pylint: disable=anomalous-backslash-in-string -from inspect import signature -import time -import logging import copy -import pandas as pd -import numpy as np -import os import json -import psutil +import logging +import os +import time +from concurrent.futures import as_completed from importlib import import_module +from inspect import signature from numbers import Number -from concurrent.futures import as_completed from warnings import warn +import numpy as np +import pandas as pd +import psutil +from rex.joint_pd.joint_pd import JointPD +from rex.multi_year_resource import MultiYearWindResource +from rex.renewable_resource import WindResource +from rex.utilities.bc_parse_table import parse_bc_table +from rex.utilities.execution import SpawnProcessPool +from rex.utilities.loggers import create_dirs, log_mem +from rex.utilities.utilities import parse_year + from reV.config.output_request import SAMOutputRequest -from reV.generation.generation import Gen -from reV.SAM.generation import WindPower, WindPowerPD from reV.econ.utilities import lcoe_fcr -from reV.handlers.outputs import Outputs +from reV.generation.generation import Gen from reV.handlers.exclusions import ExclusionLayers +from reV.handlers.outputs import Outputs +from reV.SAM.generation import WindPower, WindPowerPD +from reV.supply_curve.aggregation import AggFileHandler, BaseAggregation from reV.supply_curve.extent import SupplyCurveExtent from reV.supply_curve.points import AggregationSupplyCurvePoint as AggSCPoint from reV.supply_curve.points import SupplyCurvePoint -from reV.supply_curve.aggregation import BaseAggregation, AggFileHandler -from reV.utilities.exceptions import (EmptySupplyCurvePointError, - FileInputError) -from reV.utilities import log_versions, ModuleName - -from rex.utilities.bc_parse_table import parse_bc_table -from rex.joint_pd.joint_pd import JointPD -from rex.renewable_resource import WindResource -from rex.multi_year_resource import MultiYearWindResource -from rex.utilities.loggers import log_mem, create_dirs -from rex.utilities.utilities import parse_year -from rex.utilities.execution import SpawnProcessPool +from reV.utilities import MetaKeyName, ModuleName, log_versions +from reV.utilities.exceptions import EmptySupplyCurvePointError, FileInputError logger = logging.getLogger(__name__) @@ -79,7 +78,7 @@ def __init__(self, res_fpath, sc_gid_to_hh, sc_gid_to_res_gid): self._pre_load_data() def _pre_load_data(self): - """Pre-load the resource data. """ + """Pre-load the resource data.""" for sc_gid, gids in self.sc_gid_to_res_gid.items(): hh = self.sc_gid_to_hh[sc_gid] @@ -482,9 +481,9 @@ def _parse_prior_run(self): sure the SAM system inputs are set accordingly.""" # {meta_column: sam_sys_input_key} - required = {'capacity': 'system_capacity', - 'turbine_x_coords': 'wind_farm_xCoordinates', - 'turbine_y_coords': 'wind_farm_yCoordinates'} + required = {MetaKeyName.CAPACITY: 'system_capacity', + MetaKeyName.TURBINE_X_COORDS: 'wind_farm_xCoordinates', + MetaKeyName.TURBINE_Y_COORDS: 'wind_farm_yCoordinates'} if self._prior_meta: missing = [k for k in required if k not in self.meta] @@ -524,13 +523,13 @@ def _parse_gid_map(gid_map): if isinstance(gid_map, str): if gid_map.endswith('.csv'): gid_map = pd.read_csv(gid_map).to_dict() - assert 'gid' in gid_map, 'Need "gid" in gid_map column' + assert MetaKeyName.GID in gid_map, 'Need "gid" in gid_map column' assert 'gid_map' in gid_map, 'Need "gid_map" in gid_map column' - gid_map = {gid_map['gid'][i]: gid_map['gid_map'][i] - for i in gid_map['gid'].keys()} + gid_map = {gid_map[MetaKeyName.GID][i]: gid_map['gid_map'][i] + for i in gid_map[MetaKeyName.GID].keys()} elif gid_map.endswith('.json'): - with open(gid_map, 'r') as f: + with open(gid_map) as f: gid_map = json.load(f) return gid_map @@ -744,22 +743,22 @@ def meta(self): row_ind, col_ind = sc.get_sc_row_col_ind(self.sc_point.gid) self._meta = pd.DataFrame( - {'sc_point_gid': self.sc_point.gid, - 'sc_row_ind': row_ind, - 'sc_col_ind': col_ind, - 'gid': self.sc_point.gid, - 'latitude': self.sc_point.latitude, - 'longitude': self.sc_point.longitude, - 'timezone': self.sc_point.timezone, + {MetaKeyName.SC_POINT_GID: self.sc_point.gid, + MetaKeyName.SC_ROW_IND: row_ind, + MetaKeyName.SC_COL_IND: col_ind, + MetaKeyName.GID: self.sc_point.gid, + MetaKeyName.LATITUDE: self.sc_point.latitude, + MetaKeyName.LONGITUDE: self.sc_point.longitude, + MetaKeyName.TIMEZONE: self.sc_point.timezone, 'country': self.sc_point.country, 'state': self.sc_point.state, 'county': self.sc_point.county, - 'elevation': self.sc_point.elevation, - 'offshore': self.sc_point.offshore, - 'res_gids': res_gids, - 'gid_counts': gid_counts, - 'n_gids': self.sc_point.n_gids, - 'area_sq_km': self.sc_point.area, + MetaKeyName.ELEVATION: self.sc_point.elevation, + MetaKeyName.OFFSHORE: self.sc_point.offshore, + MetaKeyName.RES_GIDS: res_gids, + MetaKeyName.GID_COUNTS: gid_counts, + MetaKeyName.N_GIDS: self.sc_point.n_gids, + MetaKeyName.AREA_SQ_KM: self.sc_point.area, }, index=[self.sc_point.gid]) return self._meta @@ -892,7 +891,7 @@ def initialize_wind_plant_ts(self): @property def wind_plant_pd(self): - """reV WindPowerPD compute object for plant layout optimization based + """ReV WindPowerPD compute object for plant layout optimization based on wind joint probability distribution Returns @@ -909,7 +908,7 @@ def wind_plant_pd(self): @property def wind_plant_ts(self): - """reV WindPower compute object(s) based on wind resource timeseries + """ReV WindPower compute object(s) based on wind resource timeseries data keyed by year Returns @@ -961,7 +960,7 @@ def recalc_lcoe(self): my_mean_lcoe = lcoe_fcr(fcr, cap_cost, foc, aep, voc) self._outputs['lcoe_fcr-means'] = my_mean_lcoe - self._meta['mean_lcoe'] = my_mean_lcoe + self._meta[MetaKeyName.MEAN_LCOE] = my_mean_lcoe def get_lcoe_kwargs(self): """Get a namespace of arguments for calculating LCOE based on the @@ -1066,9 +1065,7 @@ def _check_sys_inputs(plant1, plant2, """ bad = [] for k, v in plant1.sam_sys_inputs.items(): - if k not in plant2.sam_sys_inputs: - bad.append(k) - elif str(v) != str(plant2.sam_sys_inputs[k]): + if k not in plant2.sam_sys_inputs or str(v) != str(plant2.sam_sys_inputs[k]): bad.append(k) bad = [b for b in bad if b not in ignore] if any(bad): @@ -1117,9 +1114,9 @@ def run_wind_plant_ts(self): # copy dataset outputs to meta data for supply curve table summary if 'cf_mean-means' in self.outputs: - self._meta.loc[:, 'mean_cf'] = self.outputs['cf_mean-means'] + self._meta.loc[:, MetaKeyName.MEAN_CF] = self.outputs['cf_mean-means'] if 'lcoe_fcr-means' in self.outputs: - self._meta.loc[:, 'mean_lcoe'] = self.outputs['lcoe_fcr-means'] + self._meta.loc[:, MetaKeyName.MEAN_LCOE] = self.outputs['lcoe_fcr-means'] self.recalc_lcoe() logger.debug('Timeseries analysis complete!') @@ -1195,7 +1192,7 @@ def run_plant_optimization(self): # copy dataset outputs to meta data for supply curve table summary # convert SAM system capacity in kW to reV supply curve cap in MW - self._meta['capacity'] = self.outputs['system_capacity'] / 1e3 + self._meta[MetaKeyName.CAPACITY] = self.outputs['system_capacity'] / 1e3 # add required ReEDS multipliers to meta baseline_cost = self.plant_optimizer.capital_cost_per_kw( @@ -1277,7 +1274,7 @@ def __init__(self, excl_fpath, res_fpath, tm_dset, objective_function, resolution=64, excl_area=None, data_layers=None, pre_extract_inclusions=False, prior_run=None, gid_map=None, bias_correct=None, pre_load_data=False): - """reV bespoke analysis class. + r"""ReV bespoke analysis class. Much like generation, ``reV`` bespoke analysis runs SAM simulations by piping in renewable energy resource data (usually @@ -1755,7 +1752,7 @@ def _parse_points(points, sam_configs): Slice or list specifying project points, string pointing to a project points csv, or a fully instantiated PointsControl object. Can also be a single site integer value. Points csv should have - 'gid' and 'config' column, the config maps to the sam_configs dict + MetaKeyName.GID and 'config' column, the config maps to the sam_configs dict keys. sam_configs : dict | str | SAMConfig SAM input configuration ID(s) and file path(s). Keys are the SAM @@ -1830,7 +1827,7 @@ def _get_prior_meta(self, gid): meta = None if self._prior_meta is not None: - mask = self._prior_meta['gid'] == gid + mask = self._prior_meta[MetaKeyName.GID] == gid if any(mask): meta = self._prior_meta[mask] @@ -1861,7 +1858,7 @@ def _check_files(self): assert any(f.dsets) def _pre_load_data(self, pre_load_data): - """Pre-load resource data, if requested. """ + """Pre-load resource data, if requested.""" if not pre_load_data: return @@ -1898,7 +1895,7 @@ def _hh_for_sc_gid(self, sc_gid): return int(config["wind_turbine_hub_ht"]) def _pre_loaded_data_for_sc_gid(self, sc_gid): - """Pre-load data for a given SC GID, if requested. """ + """Pre-load data for a given SC GID, if requested.""" if self._pre_loaded_data is None: return None @@ -1981,7 +1978,7 @@ def meta(self): @property def slice_lookup(self): - """dict | None: Lookup mapping sc_point_gid to exclusion slice. """ + """Dict | None: Lookup mapping sc_point_gid to exclusion slice.""" if self._slice_lookup is None and self._inclusion_mask is not None: with SupplyCurveExtent(self._excl_fpath, resolution=self._resolution) as sc: @@ -2146,7 +2143,7 @@ def save_outputs(self, out_fpath): except Exception as e: msg = 'Failed to write "{}" to disk.'.format(dset) logger.exception(msg) - raise IOError(msg) from e + raise OSError(msg) from e logger.info('Saved output data to: {}'.format(out_fpath)) return out_fpath diff --git a/reV/config/project_points.py b/reV/config/project_points.py index c8bb7a8b1..63b97f022 100644 --- a/reV/config/project_points.py +++ b/reV/config/project_points.py @@ -4,26 +4,30 @@ """ import copy import logging -import numpy as np import os -import pandas as pd from warnings import warn +import numpy as np +import pandas as pd +from rex.multi_file_resource import MultiFileResource +from rex.resource import Resource +from rex.resource_extraction.resource_extraction import ( + MultiFileResourceX, + ResourceX, +) +from rex.utilities import check_res_file, parse_table + from reV.config.curtailment import Curtailment from reV.config.sam_config import SAMConfig +from reV.utilities import MetaKeyName from reV.utilities.exceptions import ConfigError, ConfigWarning -from rex.resource import Resource -from rex.multi_file_resource import MultiFileResource -from rex.resource_extraction.resource_extraction import (ResourceX, - MultiFileResourceX) -from rex.utilities import check_res_file, parse_table - logger = logging.getLogger(__name__) class PointsControl: """Class to manage and split ProjectPoints.""" + def __init__(self, project_points, sites_per_split=100): """ Parameters @@ -269,7 +273,7 @@ def __getitem__(self, site): names (keys) and values. """ - site_bool = (self.df['gid'] == site) + site_bool = (self.df[MetaKeyName.GID] == site) try: config_id = self.df.loc[site_bool, 'config'].values[0] except (KeyError, IndexError) as ex: @@ -299,7 +303,7 @@ def df(self): ------- _df : pd.DataFrame Table of sites and corresponding SAM configuration IDs. - Has columns 'gid' and 'config'. + Has columns MetaKeyName.GID and 'config'. """ return self._df @@ -384,7 +388,7 @@ def sites(self): List of integer sites (resource file gids) belonging to this instance of ProjectPoints. """ - return self.df['gid'].values.tolist() + return self.df[MetaKeyName.GID].values.tolist() @property def sites_as_slice(self): @@ -485,7 +489,7 @@ def _parse_csv(fname): Parameters ---------- fname : str - Project points .csv file (with path). Must have 'gid' and 'config' + Project points .csv file (with path). Must have MetaKeyName.GID and 'config' column names. Returns @@ -522,7 +526,7 @@ def _parse_sites(points, res_file=None): df : pd.DataFrame DataFrame mapping sites (gids) to SAM technology (config) """ - df = pd.DataFrame(columns=['gid', 'config']) + df = pd.DataFrame(columns=[MetaKeyName.GID, 'config']) if isinstance(points, int): points = [points] if isinstance(points, (list, tuple, np.ndarray)): @@ -532,7 +536,7 @@ def _parse_sites(points, res_file=None): logger.error(msg) raise RuntimeError(msg) - df['gid'] = points + df[MetaKeyName.GID] = points elif isinstance(points, slice): stop = points.stop if stop is None: @@ -547,7 +551,7 @@ def _parse_sites(points, res_file=None): else: stop = Resource(res_file).shape[1] - df['gid'] = list(range(*points.indices(stop))) + df[MetaKeyName.GID] = list(range(*points.indices(stop))) else: raise TypeError('Project Points sites needs to be set as a list, ' 'tuple, or slice, but was set as: {}' @@ -588,14 +592,14 @@ def _parse_points(cls, points, res_file=None): raise ValueError('Cannot parse Project points data from {}' .format(type(points))) - if 'gid' not in df.columns: + if MetaKeyName.GID not in df.columns: raise KeyError('Project points data must contain "gid" column.') # pylint: disable=no-member if 'config' not in df.columns: df = cls._parse_sites(points["gid"].values, res_file=res_file) - gids = df['gid'].values + gids = df[MetaKeyName.GID].values if not np.array_equal(np.sort(gids), gids): msg = ('WARNING: points are not in sequential order and will be ' 'sorted! The original order is being preserved under ' @@ -603,7 +607,7 @@ def _parse_points(cls, points, res_file=None): logger.warning(msg) warn(msg) df['points_order'] = df.index.values - df = df.sort_values('gid').reset_index(drop=True) + df = df.sort_values(MetaKeyName.GID).reset_index(drop=True) return df @@ -628,16 +632,15 @@ def _parse_sam_config(sam_config): if isinstance(sam_config, SAMConfig): return sam_config + if isinstance(sam_config, dict): + config_dict = sam_config + elif isinstance(sam_config, str): + config_dict = {sam_config: sam_config} else: - if isinstance(sam_config, dict): - config_dict = sam_config - elif isinstance(sam_config, str): - config_dict = {sam_config: sam_config} - else: - raise ValueError('Cannot parse SAM configs from {}' - .format(type(sam_config))) + raise ValueError('Cannot parse SAM configs from {}' + .format(type(sam_config))) - return SAMConfig(config_dict) + return SAMConfig(config_dict) @staticmethod def _parse_curtailment(curtailment_input): @@ -691,13 +694,13 @@ def index(self, gid): ind : int Row index of gid in the project points dataframe. """ - if gid not in self._df['gid'].values: + if gid not in self._df[MetaKeyName.GID].values: e = ('Requested resource gid {} is not present in the project ' 'points dataframe. Cannot return row index.'.format(gid)) logger.error(e) raise ConfigError(e) - ind = np.where(self._df['gid'] == gid)[0][0] + ind = np.where(self._df[MetaKeyName.GID] == gid)[0][0] return ind @@ -747,7 +750,7 @@ def _check_points_config_mapping(self): logger.error(msg) raise ConfigError(msg) - def join_df(self, df2, key='gid'): + def join_df(self, df2, key=MetaKeyName.GID): """Join new df2 to the _df attribute using the _df's gid as pkey. This can be used to add site-specific data to the project_points, @@ -767,7 +770,7 @@ def join_df(self, df2, key='gid'): """ # ensure df2 doesnt have any duplicate columns for suffix reasons. df2_cols = [c for c in df2.columns if c not in self._df or c == key] - self._df = pd.merge(self._df, df2[df2_cols], how='left', left_on='gid', + self._df = pd.merge(self._df, df2[df2_cols], how='left', left_on=MetaKeyName.GID, right_on=key, copy=False, validate='1:1') def get_sites_from_config(self, config): @@ -784,7 +787,7 @@ def get_sites_from_config(self, config): List of sites associated with the requested configuration ID. If the configuration ID is not recognized, an empty list is returned. """ - sites = self.df.loc[(self.df['config'] == config), 'gid'].values + sites = self.df.loc[(self.df['config'] == config), MetaKeyName.GID].values return list(sites) @@ -938,8 +941,8 @@ def lat_lon_coords(cls, lat_lons, res_file, sam_configs, tech=None, if 'points_order' in pp.df: lat_lons = lat_lons[pp.df['points_order'].values] - pp._df['latitude'] = lat_lons[:, 0] - pp._df['longitude'] = lat_lons[:, 1] + pp._df[MetaKeyName.LATITUDE] = lat_lons[:, 0] + pp._df[MetaKeyName.LONGITUDE] = lat_lons[:, 1] return pp diff --git a/reV/econ/econ.py b/reV/econ/econ.py index 662798dcb..6c87796fe 100644 --- a/reV/econ/econ.py +++ b/reV/econ/econ.py @@ -3,24 +3,24 @@ reV econ module (lcoe-fcr, single owner, etc...) """ import logging -import numpy as np import os -import pandas as pd import pprint from warnings import warn +import numpy as np +import pandas as pd +from rex.multi_file_resource import MultiFileResource +from rex.resource import Resource +from rex.utilities.utilities import check_res_file + from reV.config.project_points import PointsControl from reV.generation.base import BaseGen from reV.handlers.outputs import Outputs from reV.SAM.econ import LCOE as SAM_LCOE from reV.SAM.econ import SingleOwner from reV.SAM.windbos import WindBos -from reV.utilities.exceptions import (ExecutionError, OffshoreWindInputWarning) -from reV.utilities import ModuleName - -from rex.resource import Resource -from rex.multi_file_resource import MultiFileResource -from rex.utilities.utilities import check_res_file +from reV.utilities import MetaKeyName, ModuleName +from reV.utilities.exceptions import ExecutionError, OffshoreWindInputWarning logger = logging.getLogger(__name__) @@ -54,7 +54,7 @@ class Econ(BaseGen): def __init__(self, project_points, sam_files, cf_file, site_data=None, output_request=('lcoe_fcr',), sites_per_worker=100, memory_utilization_limit=0.4, append=False): - """reV econ analysis class. + """ReV econ analysis class. ``reV`` econ analysis runs SAM econ calculations, typically to compute LCOE (using :py:class:`PySAM.Lcoefcr.Lcoefcr`), though @@ -212,10 +212,10 @@ def meta(self): with Outputs(self.cf_file) as cfh: # only take meta that belongs to this project's site list self._meta = cfh.meta[ - cfh.meta['gid'].isin(self.points_control.sites)] + cfh.meta[MetaKeyName.GID].isin(self.points_control.sites)] - if 'offshore' in self._meta: - if self._meta['offshore'].sum() > 1: + if MetaKeyName.OFFSHORE in self._meta: + if self._meta[MetaKeyName.OFFSHORE].sum() > 1: w = ('Found offshore sites in econ meta data. ' 'This functionality has been deprecated. ' 'Please run the reV offshore module to ' @@ -224,7 +224,7 @@ def meta(self): logger.warning(w) elif self._meta is None and self.cf_file is None: - self._meta = pd.DataFrame({'gid': self.points_control.sites}) + self._meta = pd.DataFrame({MetaKeyName.GID: self.points_control.sites}) return self._meta @@ -267,8 +267,8 @@ def _econ_append_pc(pp, cf_file, sites_per_worker=None): res_kwargs = {'hsds': hsds} with res_cls(cf_file, **res_kwargs) as f: - gid0 = f.meta['gid'].values[0] - gid1 = f.meta['gid'].values[-1] + gid0 = f.meta[MetaKeyName.GID].values[0] + gid1 = f.meta[MetaKeyName.GID].values[-1] i0 = pp.index(gid0) i1 = pp.index(gid1) + 1 @@ -330,7 +330,7 @@ def _run_single_worker(pc, econ_fun, output_request, **kwargs): pc : reV.config.project_points.PointsControl Iterable points control object from reV config module. Must have project_points with df property with all relevant - site-specific inputs and a 'gid' column. By passing site-specific + site-specific inputs and a MetaKeyName.GID column. By passing site-specific inputs in this dataframe, which was split using points_control, only the data relevant to the current sites is passed. econ_fun : method @@ -354,7 +354,7 @@ def _run_single_worker(pc, econ_fun, output_request, **kwargs): # Extract the site df from the project points df. site_df = pc.project_points.df - site_df = site_df.set_index('gid', drop=True) + site_df = site_df.set_index(MetaKeyName.GID, drop=True) # SAM execute econ analysis based on output request try: @@ -498,7 +498,7 @@ def run(self, out_fpath=None, max_workers=1, timeout=1800, self._init_out_arrays() diff = list(set(self.points_control.sites) - - set(self.meta['gid'].values)) + - set(self.meta[MetaKeyName.GID].values)) if diff: raise Exception('The following analysis sites were requested ' 'through project points for econ but are not ' diff --git a/reV/econ/economies_of_scale.py b/reV/econ/economies_of_scale.py index 326c1d6d3..cfcde6223 100644 --- a/reV/econ/economies_of_scale.py +++ b/reV/econ/economies_of_scale.py @@ -3,14 +3,15 @@ reV module for calculating economies of scale where larger power plants will have reduced capital cost. """ -import logging import copy +import logging import re -import numpy as np # pylint: disable=unused-import + import pandas as pd +from rex.utilities.utilities import check_eval_str from reV.econ.utilities import lcoe_fcr -from rex.utilities.utilities import check_eval_str +from reV.utilities import MetaKeyName logger = logging.getLogger(__name__) @@ -284,7 +285,7 @@ def raw_lcoe(self): ------- lcoe : float | np.ndarray """ - key_list = ['raw_lcoe', 'mean_lcoe'] + key_list = [MetaKeyName.RAW_LCOE, MetaKeyName.MEAN_LCOE] return copy.deepcopy(self._get_prioritized_keys(self._data, key_list)) @property diff --git a/reV/generation/base.py b/reV/generation/base.py index 6f7955e58..d170940a2 100644 --- a/reV/generation/base.py +++ b/reV/generation/base.py @@ -2,44 +2,47 @@ """ reV base gen and econ module. """ -from abc import ABC, abstractmethod import copy -from concurrent.futures import TimeoutError +import json import logging -import pandas as pd -import numpy as np import os -import psutil -import json import sys +from abc import ABC, abstractmethod +from concurrent.futures import TimeoutError from warnings import warn +import numpy as np +import pandas as pd +import psutil +from rex.resource import Resource +from rex.utilities.execution import SpawnProcessPool + from reV.config.output_request import SAMOutputRequest -from reV.config.project_points import ProjectPoints, PointsControl +from reV.config.project_points import PointsControl, ProjectPoints from reV.handlers.outputs import Outputs from reV.SAM.version_checker import PySamVersionChecker -from reV.utilities.exceptions import (OutputWarning, ExecutionError, - ParallelExecutionWarning, - OffshoreWindInputWarning) -from reV.utilities import log_versions, ModuleName - -from rex.resource import Resource -from rex.utilities.execution import SpawnProcessPool +from reV.utilities import MetaKeyName, ModuleName, log_versions +from reV.utilities.exceptions import ( + ExecutionError, + OffshoreWindInputWarning, + OutputWarning, + ParallelExecutionWarning, +) logger = logging.getLogger(__name__) ATTR_DIR = os.path.dirname(os.path.realpath(__file__)) ATTR_DIR = os.path.join(ATTR_DIR, 'output_attributes') -with open(os.path.join(ATTR_DIR, 'other.json'), 'r') as f: +with open(os.path.join(ATTR_DIR, 'other.json')) as f: OTHER_ATTRS = json.load(f) -with open(os.path.join(ATTR_DIR, 'lcoe_fcr.json'), 'r') as f: +with open(os.path.join(ATTR_DIR, 'lcoe_fcr.json')) as f: LCOE_ATTRS = json.load(f) -with open(os.path.join(ATTR_DIR, 'single_owner.json'), 'r') as f: +with open(os.path.join(ATTR_DIR, 'single_owner.json')) as f: SO_ATTRS = json.load(f) -with open(os.path.join(ATTR_DIR, 'windbos.json'), 'r') as f: +with open(os.path.join(ATTR_DIR, 'windbos.json')) as f: BOS_ATTRS = json.load(f) -with open(os.path.join(ATTR_DIR, 'lcoe_fcr_inputs.json'), 'r') as f: +with open(os.path.join(ATTR_DIR, 'lcoe_fcr_inputs.json')) as f: LCOE_IN_ATTRS = json.load(f) @@ -284,7 +287,7 @@ def meta(self): Meta data df for sites in project points. Column names are meta data variables, rows are different sites. The row index does not indicate the site number if the project points are - non-sequential or do not start from 0, so a 'gid' column is added. + non-sequential or do not start from 0, so a MetaKeyName.GID column is added. """ return self._meta @@ -735,7 +738,7 @@ def _parse_site_data(self, inp): if inp is None or inp is False: # no input, just initialize dataframe with site gids as index site_data = pd.DataFrame(index=self.project_points.sites) - site_data.index.name = 'gid' + site_data.index.name = MetaKeyName.GID else: # explicit input, initialize df if isinstance(inp, str): @@ -748,18 +751,18 @@ def _parse_site_data(self, inp): raise Exception('Site data input must be .csv or ' 'dataframe, but received: {}'.format(inp)) - if 'gid' not in site_data and site_data.index.name != 'gid': + if MetaKeyName.GID not in site_data and site_data.index.name != MetaKeyName.GID: # require gid as column label or index raise KeyError('Site data input must have "gid" column ' 'to match reV site gid.') # pylint: disable=no-member - if site_data.index.name != 'gid': + if site_data.index.name != MetaKeyName.GID: # make gid the dataframe index if not already - site_data = site_data.set_index('gid', drop=True) + site_data = site_data.set_index(MetaKeyName.GID, drop=True) - if 'offshore' in site_data: - if site_data['offshore'].sum() > 1: + if MetaKeyName.OFFSHORE in site_data: + if site_data[MetaKeyName.OFFSHORE].sum() > 1: w = ('Found offshore sites in econ site data input. ' 'This functionality has been deprecated. ' 'Please run the reV offshore module to ' @@ -829,7 +832,7 @@ def _get_data_shape_from_out_attrs(self, dset, n_sites): return (n_sites,) def _get_data_shape_from_sam_config(self, dset, n_sites): - """Get data shape from SAM input config """ + """Get data shape from SAM input config""" data = list(self.project_points.sam_inputs.values())[0][dset] if isinstance(data, (list, tuple, np.ndarray)): return (*np.array(data).shape, n_sites) @@ -1249,8 +1252,8 @@ def _handle_failed_future(self, future, i, sites, timeout): logger.warning(w) warn(w, OutputWarning) - site_out = {k: 0 for k in self.output_request} - result = {site: site_out for site in sites} + site_out = dict.fromkeys(self.output_request, 0) + result = dict.fromkeys(sites, site_out) try: cancelled = future.cancel() diff --git a/reV/generation/generation.py b/reV/generation/generation.py index f505fda71..344f8f079 100644 --- a/reV/generation/generation.py +++ b/reV/generation/generation.py @@ -424,7 +424,7 @@ def meta(self): Meta data df for sites in project points. Column names are meta data variables, rows are different sites. The row index does not indicate the site number if the project points are - non-sequential or do not start from 0, so a 'gid' column is added. + non-sequential or do not start from 0, so a MetaKeyName.GID column is added. """ if self._meta is None: res_cls = Resource @@ -451,11 +451,11 @@ def meta(self): self._meta = res['meta', res_gids] - self._meta.loc[:, 'gid'] = res_gids + self._meta.loc[:, MetaKeyName.GID] = res_gids if self.write_mapped_gids: - self._meta.loc[:, 'gid'] = self.project_points.sites + self._meta.loc[:, MetaKeyName.GID] = self.project_points.sites self._meta.index = self.project_points.sites - self._meta.index.name = 'gid' + self._meta.index.name = MetaKeyName.GID self._meta.loc[:, 'reV_tech'] = self.project_points.tech return self._meta @@ -637,7 +637,7 @@ def _run_single_worker(cls, points_control, tech=None, res_file=None, # Extract the site df from the project points df. site_df = points_control.project_points.df - site_df = site_df.set_index('gid', drop=True) + site_df = site_df.set_index(MetaKeyName.GID, drop=True) # run generation method for specified technology try: @@ -703,10 +703,10 @@ def _parse_gid_map(self, gid_map): if isinstance(gid_map, str): if gid_map.endswith('.csv'): gid_map = pd.read_csv(gid_map).to_dict() - assert 'gid' in gid_map, 'Need "gid" in gid_map column' + assert MetaKeyName.GID in gid_map, 'Need "gid" in gid_map column' assert 'gid_map' in gid_map, 'Need "gid_map" in gid_map column' - gid_map = {gid_map['gid'][i]: gid_map['gid_map'][i] - for i in gid_map['gid'].keys()} + gid_map = {gid_map[MetaKeyName.GID][i]: gid_map['gid_map'][i] + for i in gid_map[MetaKeyName.GID].keys()} elif gid_map.endswith('.json'): with open(gid_map, 'r') as f: @@ -838,10 +838,10 @@ def _parse_bc(bias_correct): msg = ('Bias correction table must have "gid" column but only found: ' '{}'.format(list(bias_correct.columns))) - assert 'gid' in bias_correct or bias_correct.index.name == 'gid', msg + assert MetaKeyName.GID in bias_correct or bias_correct.index.name == MetaKeyName.GID, msg - if bias_correct.index.name != 'gid': - bias_correct = bias_correct.set_index('gid') + if bias_correct.index.name != MetaKeyName.GID: + bias_correct = bias_correct.set_index(MetaKeyName.GID) msg = ('Bias correction table must have "method" column but only ' 'found: {}'.format(list(bias_correct.columns))) diff --git a/reV/handlers/exclusions.py b/reV/handlers/exclusions.py index d7bde9b04..484ce1db6 100644 --- a/reV/handlers/exclusions.py +++ b/reV/handlers/exclusions.py @@ -2,16 +2,17 @@ """ Exclusion layers handler """ -import logging import json +import logging + import numpy as np +from rex.multi_file_resource import MultiFileResource +from rex.resource import Resource +from rex.utilities.parse_keys import parse_keys +from reV.utilities import MetaKeyName from reV.utilities.exceptions import HandlerKeyError, MultiFileExclusionError -from rex.utilities.parse_keys import parse_keys -from rex.resource import Resource -from rex.multi_file_resource import MultiFileResource - logger = logging.getLogger(__name__) @@ -81,8 +82,8 @@ def __contains__(self, layer): def _preflight_multi_file(self): """Run simple multi-file exclusion checks.""" - lat_shape = self.h5.shapes['latitude'] - lon_shape = self.h5.shapes['longitude'] + lat_shape = self.h5.shapes[MetaKeyName.LATITUDE] + lon_shape = self.h5.shapes[MetaKeyName.LONGITUDE] for layer in self.layers: lshape = self.h5.shapes[layer] lshape = lshape[1:] if len(lshape) > 2 else lshape @@ -231,7 +232,7 @@ def shape(self): """ shape = self.h5.attrs.get('shape', None) if shape is None: - shape = self.h5.shapes['latitude'] + shape = self.h5.shapes[MetaKeyName.LATITUDE] return tuple(shape) @@ -247,7 +248,7 @@ def chunks(self): """ chunks = self.h5.attrs.get('chunks', None) if chunks is None: - chunks = self.h5.chunks['latitude'] + chunks = self.h5.chunks[MetaKeyName.LATITUDE] return chunks @@ -260,7 +261,7 @@ def latitude(self): ------- ndarray """ - return self['latitude'] + return self[MetaKeyName.LATITUDE] @property def longitude(self): @@ -271,7 +272,7 @@ def longitude(self): ------- ndarray """ - return self['longitude'] + return self[MetaKeyName.LONGITUDE] def get_layer_profile(self, layer): """ @@ -384,13 +385,13 @@ def _get_latitude(self, *ds_slice): lat : ndarray Latitude coordinates """ - if 'latitude' not in self.h5: + if MetaKeyName.LATITUDE not in self.h5: msg = ('"latitude" is missing from {}' .format(self.h5_file)) logger.error(msg) raise HandlerKeyError(msg) - ds_slice = ('latitude', ) + ds_slice + ds_slice = (MetaKeyName.LATITUDE, ) + ds_slice lat = self.h5[ds_slice] @@ -410,13 +411,13 @@ def _get_longitude(self, *ds_slice): lon : ndarray Longitude coordinates """ - if 'longitude' not in self.h5: + if MetaKeyName.LONGITUDE not in self.h5: msg = ('"longitude" is missing from {}' .format(self.h5_file)) logger.error(msg) raise HandlerKeyError(msg) - ds_slice = ('longitude', ) + ds_slice + ds_slice = (MetaKeyName.LONGITUDE, ) + ds_slice lon = self.h5[ds_slice] diff --git a/reV/handlers/outputs.py b/reV/handlers/outputs.py index ffbf795be..34c91dd46 100644 --- a/reV/handlers/outputs.py +++ b/reV/handlers/outputs.py @@ -2,15 +2,16 @@ """ Classes to handle reV h5 output files. """ +import json import logging +import sys + import NRWAL import PySAM import rex -import sys -import json +from rex.outputs import Outputs as rexOutputs from reV.version import __version__ -from rex.outputs import Outputs as rexOutputs logger = logging.getLogger(__name__) @@ -28,8 +29,8 @@ class Outputs(rexOutputs): >>> import pandas as pd >>> import numpy as np >>> - >>> meta = pd.DataFrame({'latitude': np.ones(100), - >>> 'longitude': np.ones(100)}) + >>> meta = pd.DataFrame({MetaKeyName.LATITUDE: np.ones(100), + >>> MetaKeyName.LONGITUDE: np.ones(100)}) >>> >>> time_index = pd.date_range('20210101', '20220101', freq='1h', >>> closed='right') diff --git a/reV/hybrids/hybrids.py b/reV/hybrids/hybrids.py index 7a8f2d207..d5374d2d0 100644 --- a/reV/hybrids/hybrids.py +++ b/reV/hybrids/hybrids.py @@ -4,32 +4,37 @@ @author: ppinchuk """ import logging -import numpy as np import re -import pandas as pd +from collections import namedtuple from string import ascii_letters from warnings import warn -from collections import namedtuple - -from reV.handlers.outputs import Outputs -from reV.utilities.exceptions import (FileInputError, InputError, - InputWarning, OutputWarning) -from reV.hybrids.hybrid_methods import HYBRID_METHODS +import numpy as np +import pandas as pd from rex.resource import Resource from rex.utilities.utilities import to_records_array +from reV.handlers.outputs import Outputs +from reV.hybrids.hybrid_methods import HYBRID_METHODS +from reV.utilities import MetaKeyName +from reV.utilities.exceptions import ( + FileInputError, + InputError, + InputWarning, + OutputWarning, +) + logger = logging.getLogger(__name__) -MERGE_COLUMN = 'sc_point_gid' +MERGE_COLUMN = MetaKeyName.SC_POINT_GID PROFILE_DSET_REGEX = 'rep_profiles_[0-9]+$' SOLAR_PREFIX = 'solar_' WIND_PREFIX = 'wind_' NON_DUPLICATE_COLS = { - 'latitude', 'longitude', 'country', 'state', 'county', 'elevation', - 'timezone', 'sc_point_gid', 'sc_row_ind', 'sc_col_ind' + MetaKeyName.LATITUDE, MetaKeyName.LONGITUDE, 'country', 'state', 'county', MetaKeyName.ELEVATION, + MetaKeyName.TIMEZONE, MetaKeyName.SC_POINT_GID, MetaKeyName.SC_ROW_IND, MetaKeyName.SC_COL_IND } -DROPPED_COLUMNS = ['gid'] +DROPPED_COLUMNS = [MetaKeyName.GID] DEFAULT_FILL_VALUES = {'solar_capacity': 0, 'wind_capacity': 0, 'solar_mean_cf': 0, 'wind_mean_cf': 0} OUTPUT_PROFILE_NAMES = ['hybrid_profile', @@ -40,7 +45,8 @@ class ColNameFormatter: - """Column name formatting helper class. """ + """Column name formatting helper class.""" + ALLOWED = set(ascii_letters) @classmethod @@ -65,7 +71,7 @@ def fmt(cls, n): class HybridsData: - """Hybrids input data container. """ + """Hybrids input data container.""" def __init__(self, solar_fpath, wind_fpath): """ @@ -230,7 +236,7 @@ def _validate_num_profiles(self): e = msg.format(PROFILE_DSET_REGEX, fp) logger.error(e) raise FileInputError(e) - elif len(profile_dset_names) > 1: + if len(profile_dset_names) > 1: msg = ("Found more than one profile in {!r}: {}. " "This module is not intended for hybridization of " "multiple representative profiles. Please re-run " @@ -238,8 +244,7 @@ def _validate_num_profiles(self): e = msg.format(fp, profile_dset_names) logger.error(e) raise FileInputError(e) - else: - self.profile_dset_names += profile_dset_names + self.profile_dset_names += profile_dset_names def _validate_merge_col_exists(self): """Validate the existence of the merge column. @@ -398,8 +403,7 @@ def hybrid_meta(self): """ if self._hybrid_meta is None or self.__hybrid_meta_cols is None: return self._hybrid_meta - else: - return self._hybrid_meta[self.__hybrid_meta_cols] + return self._hybrid_meta[self.__hybrid_meta_cols] def validate_input(self): """Validate the input parameters. @@ -431,7 +435,7 @@ def _validate_limits_cols_prefixed(self): @staticmethod def __validate_col_prefix(col, prefixes, input_name): - """Validate the the col starts with the correct prefix. """ + """Validate the the col starts with the correct prefix.""" missing = [not col.startswith(p) for p in prefixes] if all(missing): @@ -585,7 +589,7 @@ def _validate_ratio_cols_exist(self): @property def _ratio_cols(self): - """Get the ratio columns from the ratio input. """ + """Get the ratio columns from the ratio input.""" if self._ratio is None: return [] return self._ratio.strip().split('/') @@ -603,7 +607,7 @@ def hybridize(self): self._sort_hybrid_meta_cols() def _format_meta_pre_merge(self): - """Prepare solar and wind meta for merging. """ + """Prepare solar and wind meta for merging.""" self.__col_name_map = { ColNameFormatter.fmt(c): c for c in self.data.solar_meta.columns.values @@ -616,7 +620,7 @@ def _format_meta_pre_merge(self): @staticmethod def _rename_cols(df, prefix): - """Replace column names with the ColNameFormatter.fmt is needed. """ + """Replace column names with the ColNameFormatter.fmt is needed.""" df.columns = [ ColNameFormatter.fmt(col_name) if col_name in NON_DUPLICATE_COLS @@ -625,13 +629,13 @@ def _rename_cols(df, prefix): ] def _save_rep_prof_index_internally(self): - """Save rep profiles index in hybrid meta for access later. """ + """Save rep profiles index in hybrid meta for access later.""" self.data.solar_meta[self.__solar_rpi_n] = self.data.solar_meta.index self.data.wind_meta[self.__wind_rpi_n] = self.data.wind_meta.index def _merge_solar_wind_meta(self): - """Merge the wind and solar meta DataFrames. """ + """Merge the wind and solar meta DataFrames.""" self._hybrid_meta = self.data.solar_meta.merge( self.data.wind_meta, on=ColNameFormatter.fmt(MERGE_COLUMN), @@ -639,26 +643,26 @@ def _merge_solar_wind_meta(self): ) def _merge_type(self): - """Determine the type of merge to use for meta based on user input. """ + """Determine the type of merge to use for meta based on user input.""" if self._allow_solar_only and self._allow_wind_only: return 'outer' - elif self._allow_solar_only and not self._allow_wind_only: + if self._allow_solar_only and not self._allow_wind_only: return 'left' - elif not self._allow_solar_only and self._allow_wind_only: + if not self._allow_solar_only and self._allow_wind_only: return 'right' return 'inner' def _format_meta_post_merge(self): - """Format hybrid meta after merging. """ + """Format hybrid meta after merging.""" duplicate_cols = [n for n in self._hybrid_meta.columns if "_x" in n] self._propagate_duplicate_cols(duplicate_cols) self._drop_cols(duplicate_cols) self._hybrid_meta.rename(self.__col_name_map, inplace=True, axis=1) - self._hybrid_meta.index.name = 'gid' + self._hybrid_meta.index.name = MetaKeyName.GID def _propagate_duplicate_cols(self, duplicate_cols): - """Fill missing column values from outer merge. """ + """Fill missing column values from outer merge.""" for duplicate in duplicate_cols: no_suffix = "_".join(duplicate.split("_")[:-1]) null_idx = self._hybrid_meta[no_suffix].isnull() @@ -666,14 +670,14 @@ def _propagate_duplicate_cols(self, duplicate_cols): self._hybrid_meta.loc[null_idx, no_suffix] = non_null_vals def _drop_cols(self, duplicate_cols): - """Drop any remaning duplicate and 'DROPPED_COLUMNS' columns. """ + """Drop any remaning duplicate and 'DROPPED_COLUMNS' columns.""" self._hybrid_meta.drop( duplicate_cols + DROPPED_COLUMNS, axis=1, inplace=True, errors='ignore' ) def _sort_hybrid_meta_cols(self): - """Sort the columns of the hybrid meta. """ + """Sort the columns of the hybrid meta.""" self.__hybrid_meta_cols = sorted( [c for c in self._hybrid_meta.columns if not c.startswith(self._INTERNAL_COL_PREFIX)], @@ -681,7 +685,7 @@ def _sort_hybrid_meta_cols(self): ) def _column_sorting_key(self, c): - """Helper function to sort hybrid meta columns. """ + """Helper function to sort hybrid meta columns.""" first_index = 0 if c.startswith('hybrid'): first_index = 1 @@ -695,8 +699,8 @@ def _column_sorting_key(self, c): def _verify_lat_long_match_post_merge(self): """Verify that all the lat/lon values match post merge.""" - lat = self._verify_col_match_post_merge(col_name='latitude') - lon = self._verify_col_match_post_merge(col_name='longitude') + lat = self._verify_col_match_post_merge(col_name=MetaKeyName.LATITUDE) + lon = self._verify_col_match_post_merge(col_name=MetaKeyName.LONGITUDE) if not lat or not lon: msg = ("Detected mismatched coordinate values (latitude or " "longitude) post merge. Please ensure that all matching " @@ -708,7 +712,7 @@ def _verify_lat_long_match_post_merge(self): raise FileInputError(e) def _verify_col_match_post_merge(self, col_name): - """Verify that all (non-null) values in a column match post merge. """ + """Verify that all (non-null) values in a column match post merge.""" c1, c2 = col_name, '{}_x'.format(col_name) if c1 in self._hybrid_meta.columns and c2 in self._hybrid_meta.columns: compare_df = self._hybrid_meta[ @@ -716,11 +720,10 @@ def _verify_col_match_post_merge(self, col_name): & (self._hybrid_meta[c2].notnull()) ] return np.allclose(compare_df[c1], compare_df[c2]) - else: - return True + return True def _fillna_meta_cols(self): - """Fill N/A values as specified by user (and internals). """ + """Fill N/A values as specified by user (and internals).""" for col_name, fill_value in self._fillna.items(): if col_name in self._hybrid_meta.columns: self._hybrid_meta[col_name].fillna(fill_value, inplace=True) @@ -732,7 +735,7 @@ def _fillna_meta_cols(self): @staticmethod def __warn_missing_col(col_name, action): - """Warn that a column the user request an action for is missing. """ + """Warn that a column the user request an action for is missing.""" msg = ("Skipping {} values for {!r}: Unable to find column " "in hybrid meta. Did you forget to prefix with " "{!r} or {!r}? ") @@ -741,7 +744,7 @@ def __warn_missing_col(col_name, action): warn(w, InputWarning) def _apply_limits(self): - """Clip column values as specified by user. """ + """Clip column values as specified by user.""" for col_name, max_value in self._limits.items(): if col_name in self._hybrid_meta.columns: self._hybrid_meta[col_name].clip(upper=max_value, inplace=True) @@ -749,7 +752,7 @@ def _apply_limits(self): self.__warn_missing_col(col_name, action='limit') def _limit_by_ratio(self): - """ Limit the given pair of ratio columns based on input ratio. """ + """Limit the given pair of ratio columns based on input ratio.""" if self._ratio_bounds is None: return @@ -784,7 +787,7 @@ def _limit_by_ratio(self): self._hybrid_meta[h_denom_name] = denominator_vals.values def _add_hybrid_cols(self): - """Add new hybrid columns using registered hybrid methods. """ + """Add new hybrid columns using registered hybrid methods.""" for new_col_name, method in HYBRID_METHODS.items(): out = method(self) if out is not None: @@ -936,7 +939,7 @@ def __init__(self, solar_fpath, wind_fpath, allow_solar_only=False, self._validate_input() def _validate_input(self): - """Validate the user input and input files. """ + """Validate the user input and input files.""" self.data.validate() self.meta_hybridizer.validate_input() @@ -1085,7 +1088,7 @@ def _init_profiles(self): for k in OUTPUT_PROFILE_NAMES} def _compute_hybridized_profile_components(self): - """Compute the resource components of the hybridized profiles. """ + """Compute the resource components of the hybridized profiles.""" for params in self.__rep_profile_hybridization_params: col, (hybrid_idxs, solar_idxs), fpath, p_name, dset_name = params @@ -1099,7 +1102,7 @@ def _compute_hybridized_profile_components(self): @property def __rep_profile_hybridization_params(self): - """Zip the rep profile hybridization parameters. """ + """Zip the rep profile hybridization parameters.""" cap_col_names = ['hybrid_solar_capacity', 'hybrid_wind_capacity'] idx_maps = [self.meta_hybridizer.solar_profile_indices_map, @@ -1110,7 +1113,7 @@ def __rep_profile_hybridization_params(self): return zipped def _compute_hybridized_profiles_from_components(self): - """Compute the hybridized profiles from the resource components. """ + """Compute the hybridized profiles from the resource components.""" hp_name, sp_name, wp_name = OUTPUT_PROFILE_NAMES self._profiles[hp_name] = (self._profiles[sp_name] diff --git a/reV/nrwal/nrwal.py b/reV/nrwal/nrwal.py index 182ce3d31..f4b6592ab 100644 --- a/reV/nrwal/nrwal.py +++ b/reV/nrwal/nrwal.py @@ -34,7 +34,7 @@ class RevNrwal: """Columns from the `site_data` table to join to the output meta data""" def __init__(self, gen_fpath, site_data, sam_files, nrwal_configs, - output_request, save_raw=True, meta_gid_col='gid', + output_request, save_raw=True, meta_gid_col=MetaKeyName.GID, site_meta_cols=None): """Framework to handle reV-NRWAL analysis. @@ -178,7 +178,7 @@ def __init__(self, gen_fpath, site_data, sam_files, nrwal_configs, self._meta_source = self._parse_gen_data() self._analysis_gids, self._site_data = self._parse_analysis_gids() - pc = Gen.get_pc(self._site_data[['gid', 'config']], points_range=None, + pc = Gen.get_pc(self._site_data[[MetaKeyName.GID, 'config']], points_range=None, sam_configs=sam_files, tech='windpower') self._project_points = pc.project_points @@ -191,7 +191,7 @@ def __init__(self, gen_fpath, site_data, sam_files, nrwal_configs, meta_gids.max(), len(self.meta_source), len(self.analysis_gids))) - def _parse_site_data(self, required_columns=('gid', 'config')): + def _parse_site_data(self, required_columns=(MetaKeyName.GID, 'config')): """Parse the site-specific spatial input data file Parameters @@ -227,7 +227,7 @@ def _parse_site_data(self, required_columns=('gid', 'config')): logger.error(msg) raise KeyError(msg) - self._site_data = self._site_data.sort_values('gid') + self._site_data = self._site_data.sort_values(MetaKeyName.GID) return self._site_data @@ -272,7 +272,7 @@ def _parse_analysis_gids(self): meta_gids = self.meta_source[self._meta_gid_col].values - missing = ~np.isin(meta_gids, self._site_data['gid']) + missing = ~np.isin(meta_gids, self._site_data[MetaKeyName.GID]) if any(missing): msg = ('{} sites from the generation meta data input were ' 'missing from the "site_data" input and will not be ' @@ -280,19 +280,19 @@ def _parse_analysis_gids(self): .format(missing.sum(), meta_gids[missing])) logger.info(msg) - missing = ~np.isin(self._site_data['gid'], meta_gids) + missing = ~np.isin(self._site_data[MetaKeyName.GID], meta_gids) if any(missing): - missing = self._site_data['gid'].values[missing] + missing = self._site_data[MetaKeyName.GID].values[missing] msg = ('{} sites from the "site_data" input were missing from the ' 'generation meta data and will not be run through NRWAL: {}' .format(len(missing), missing)) logger.info(msg) - analysis_gids = set(meta_gids) & set(self._site_data['gid']) + analysis_gids = set(meta_gids) & set(self._site_data[MetaKeyName.GID]) analysis_gids = np.array(sorted(list(analysis_gids))) # reduce the site data table to only those sites being analyzed - mask = np.isin(self._site_data['gid'], meta_gids) + mask = np.isin(self._site_data[MetaKeyName.GID], meta_gids) self._site_data = self._site_data[mask] return analysis_gids, self._site_data @@ -315,9 +315,9 @@ def _parse_sam_sys_inputs(self): system_inputs = pd.DataFrame(system_inputs).T system_inputs = system_inputs.sort_index() - system_inputs['gid'] = system_inputs.index.values - system_inputs.index.name = 'gid' - mask = system_inputs['gid'].isin(self.analysis_gids) + system_inputs[MetaKeyName.GID] = system_inputs.index.values + system_inputs.index.name = MetaKeyName.GID + mask = system_inputs[MetaKeyName.GID].isin(self.analysis_gids) system_inputs = system_inputs[mask] return system_inputs @@ -390,8 +390,8 @@ def _preflight_checks(self): logger.error(msg) raise OffshoreWindInputError(msg) - check_gid_order = (self._site_data['gid'].values - == self._sam_sys_inputs['gid'].values) + check_gid_order = (self._site_data[MetaKeyName.GID].values + == self._sam_sys_inputs[MetaKeyName.GID].values) msg = 'NRWAL site_data and system input dataframe had bad order' assert (check_gid_order).all(), msg diff --git a/reV/qa_qc/cli_qa_qc.py b/reV/qa_qc/cli_qa_qc.py index 8ec411dad..abd9187f3 100644 --- a/reV/qa_qc/cli_qa_qc.py +++ b/reV/qa_qc/cli_qa_qc.py @@ -2,20 +2,24 @@ """ QA/QC CLI utility functions. """ -import click import logging -import numpy as np import os -from rex.utilities.cli_dtypes import STR, STRLIST, INT +import click +import numpy as np +from gaps.cli import CLICommandFromFunction, as_click_command +from rex.utilities.cli_dtypes import INT, STR, STRLIST from rex.utilities.loggers import init_logger -from gaps.cli import as_click_command, CLICommandFromFunction -from reV.utilities import ModuleName -from reV.qa_qc.qa_qc import QaQc, QaQcModule -from reV.qa_qc.summary import (SummarizeH5, SummarizeSupplyCurve, - SupplyCurvePlot, ExclusionsMask) from reV import __version__ +from reV.qa_qc.qa_qc import QaQc, QaQcModule +from reV.qa_qc.summary import ( + ExclusionsMask, + SummarizeH5, + SummarizeSupplyCurve, + SupplyCurvePlot, +) +from reV.utilities import MetaKeyName, ModuleName logger = logging.getLogger(__name__) @@ -190,8 +194,8 @@ def supply_curve_table(ctx, sc_table, columns): show_default=True, help=(" plot_type of plot to create 'plot' or 'plotly', by " "default 'plot'")) -@click.option('--lcoe', '-lcoe', type=STR, default='mean_lcoe', - help="LCOE value to plot, by default 'mean_lcoe'") +@click.option('--lcoe', '-lcoe', type=STR, default=MetaKeyName.MEAN_LCOE, + help=f"LCOE value to plot, by default {MetaKeyName.MEAN_LCOE}") @click.pass_context def supply_curve_plot(ctx, sc_table, plot_type, lcoe): """ diff --git a/reV/qa_qc/qa_qc.py b/reV/qa_qc/qa_qc.py index 061b7cb04..fd69fc687 100644 --- a/reV/qa_qc/qa_qc.py +++ b/reV/qa_qc/qa_qc.py @@ -3,19 +3,24 @@ reV quality assurance and control classes """ import logging -import numpy as np import os -import pandas as pd from warnings import warn -from reV.qa_qc.summary import (SummarizeH5, SummarizeSupplyCurve, SummaryPlots, - SupplyCurvePlot, ExclusionsMask) +import numpy as np +import pandas as pd +from gaps.status import Status + +from reV.qa_qc.summary import ( + ExclusionsMask, + SummarizeH5, + SummarizeSupplyCurve, + SummaryPlots, + SupplyCurvePlot, +) from reV.supply_curve.exclusions import ExclusionMaskFromDict -from reV.utilities import log_versions, ModuleName +from reV.utilities import MetaKeyName, ModuleName, log_versions from reV.utilities.exceptions import PipelineError -from gaps.status import Status - logger = logging.getLogger(__name__) @@ -23,6 +28,7 @@ class QaQc: """ reV QA/QC """ + def __init__(self, out_dir): """ Parameters @@ -94,8 +100,8 @@ def create_scatter_plots(self, plot_type='plotly', cmap='viridis', if file.endswith('.csv'): summary_csv = os.path.join(self.out_dir, file) summary = pd.read_csv(summary_csv) - if ('gid' in summary and 'latitude' in summary - and 'longitude' in summary): + if (MetaKeyName.GID in summary and MetaKeyName.LATITUDE in summary + and MetaKeyName.LONGITUDE in summary): self._scatter_plot(summary_csv, self.out_dir, plot_type=plot_type, cmap=cmap, **kwargs) @@ -145,7 +151,7 @@ def h5(cls, h5_file, out_dir, dsets=None, group=None, process_size=None, .format(os.path.basename(h5_file), out_dir)) @classmethod - def supply_curve(cls, sc_table, out_dir, columns=None, lcoe='mean_lcoe', + def supply_curve(cls, sc_table, out_dir, columns=None, lcoe=MetaKeyName.MEAN_LCOE, plot_type='plotly', cmap='viridis', sc_plot_kwargs=None, scatter_plot_kwargs=None): """ @@ -161,7 +167,7 @@ def supply_curve(cls, sc_table, out_dir, columns=None, lcoe='mean_lcoe', Column(s) to summarize, if None summarize all numeric columns, by default None lcoe : str, optional - LCOE value to plot, by default 'mean_lcoe' + LCOE value to plot, by default MetaKeyName.MEAN_LCOE plot_type : str, optional plot_type of plot to create 'plot' or 'plotly', by default 'plotly' cmap : str, optional @@ -266,7 +272,7 @@ def __init__(self, module_name, config, out_root): self._default_plot_type = 'plotly' self._default_cmap = 'viridis' self._default_plot_step = 100 - self._default_lcoe = 'mean_lcoe' + self._default_lcoe = MetaKeyName.MEAN_LCOE self._default_area_filter_kernel = 'queen' @property diff --git a/reV/qa_qc/summary.py b/reV/qa_qc/summary.py index fb194b47f..98d27e357 100644 --- a/reV/qa_qc/summary.py +++ b/reV/qa_qc/summary.py @@ -3,15 +3,17 @@ Compute and plot summary data """ import logging -import numpy as np import os + +import numpy as np import pandas as pd -import plotting as mplt import plotly.express as px - +import plotting as mplt from rex import Resource from rex.utilities import SpawnProcessPool, parse_table +from reV.utilities import MetaKeyName + logger = logging.getLogger(__name__) @@ -19,6 +21,7 @@ class SummarizeH5: """ reV Summary data for QA/QC """ + def __init__(self, h5_file, group=None): """ Parameters @@ -160,26 +163,25 @@ def summarize_dset(self, ds_name, process_size=None, max_workers=None, summary = [future.result() for future in futures] summary = pd.concat(summary) + elif process_size is None: + summary = self._compute_sites_summary(self.h5_file, + ds_name, + sites=sites, + group=self._group) else: - if process_size is None: - summary = self._compute_sites_summary(self.h5_file, - ds_name, - sites=sites, - group=self._group) - else: - sites = np.array_split( - sites, int(np.ceil(len(sites) / process_size))) - - summary = [] - for site_slice in sites: - summary.append(self._compute_sites_summary( - self.h5_file, ds_name, - sites=site_slice, - group=self._group)) + sites = np.array_split( + sites, int(np.ceil(len(sites) / process_size))) + + summary = [] + for site_slice in sites: + summary.append(self._compute_sites_summary( + self.h5_file, ds_name, + sites=site_slice, + group=self._group)) - summary = pd.concat(summary) + summary = pd.concat(summary) - summary.index.name = 'gid' + summary.index.name = MetaKeyName.GID else: summary = self._compute_ds_summary(self.h5_file, ds_name, @@ -206,9 +208,9 @@ def summarize_means(self, out_path=None): """ with Resource(self.h5_file, group=self._group) as f: meta = f.meta - if 'gid' not in meta: - if meta.index.name != 'gid': - meta.index.name = 'gid' + if MetaKeyName.GID not in meta: + if meta.index.name != MetaKeyName.GID: + meta.index.name = MetaKeyName.GID meta = meta.reset_index() @@ -270,6 +272,7 @@ class SummarizeSupplyCurve: """ Summarize Supply Curve table """ + def __init__(self, sc_table): self._sc_table = self._parse_summary(sc_table) @@ -401,6 +404,7 @@ class PlotBase: """ QA/QC Plotting base class """ + def __init__(self, data): """ Parameters @@ -461,7 +465,7 @@ def _check_value(df, values, scatter=True): values = [values] if scatter: - values += ['latitude', 'longitude'] + values += [MetaKeyName.LATITUDE, MetaKeyName.LONGITUDE] for value in values: if value not in df: @@ -475,6 +479,7 @@ class SummaryPlots(PlotBase): """ Plot summary data for QA/QC """ + def __init__(self, summary): """ Parameters @@ -523,7 +528,7 @@ def scatter_plot(self, value, cmap='viridis', out_path=None, **kwargs): Additional kwargs for plotting.dataframes.df_scatter """ self._check_value(self.summary, value) - mplt.df_scatter(self.summary, x='longitude', y='latitude', c=value, + mplt.df_scatter(self.summary, x=MetaKeyName.LONGITUDE, y=MetaKeyName.LATITUDE, c=value, colormap=cmap, filename=out_path, **kwargs) def scatter_plotly(self, value, cmap='Viridis', out_path=None, **kwargs): @@ -544,7 +549,7 @@ def scatter_plotly(self, value, cmap='Viridis', out_path=None, **kwargs): Additional kwargs for plotly.express.scatter """ self._check_value(self.summary, value) - fig = px.scatter(self.summary, x='longitude', y='latitude', + fig = px.scatter(self.summary, x=MetaKeyName.LONGITUDE, y=MetaKeyName.LATITUDE, color=value, color_continuous_scale=cmap, **kwargs) fig.update_layout(font=dict(family="Arial", size=18, color="black")) @@ -553,24 +558,24 @@ def scatter_plotly(self, value, cmap='Viridis', out_path=None, **kwargs): fig.show() - def _extract_sc_data(self, lcoe='mean_lcoe'): + def _extract_sc_data(self, lcoe=MetaKeyName.MEAN_LCOE): """ Extract supply curve data Parameters ---------- lcoe : str, optional - LCOE value to use for supply curve, by default 'mean_lcoe' + LCOE value to use for supply curve, by default MetaKeyName.MEAN_LCOE Returns ------- sc_df : pandas.DataFrame Supply curve data """ - values = ['capacity', lcoe] + values = [MetaKeyName.CAPACITY, lcoe] self._check_value(self.summary, values, scatter=False) sc_df = self.summary[values].sort_values(lcoe) - sc_df['cumulative_capacity'] = sc_df['capacity'].cumsum() + sc_df['cumulative_capacity'] = sc_df[MetaKeyName.CAPACITY].cumsum() return sc_df @@ -735,35 +740,35 @@ def columns(self): """ return list(self.sc_table.columns) - def _extract_sc_data(self, lcoe='mean_lcoe'): + def _extract_sc_data(self, lcoe=MetaKeyName.MEAN_LCOE): """ Extract supply curve data Parameters ---------- lcoe : str, optional - LCOE value to use for supply curve, by default 'mean_lcoe' + LCOE value to use for supply curve, by default MetaKeyName.MEAN_LCOE Returns ------- sc_df : pandas.DataFrame Supply curve data """ - values = ['capacity', lcoe] + values = [MetaKeyName.CAPACITY, lcoe] self._check_value(self.sc_table, values, scatter=False) sc_df = self.sc_table[values].sort_values(lcoe) - sc_df['cumulative_capacity'] = sc_df['capacity'].cumsum() + sc_df['cumulative_capacity'] = sc_df[MetaKeyName.CAPACITY].cumsum() return sc_df - def supply_curve_plot(self, lcoe='mean_lcoe', out_path=None, **kwargs): + def supply_curve_plot(self, lcoe=MetaKeyName.MEAN_LCOE, out_path=None, **kwargs): """ Plot supply curve (cumulative capacity vs lcoe) using seaborn.scatter Parameters ---------- lcoe : str, optional - LCOE value to plot, by default 'mean_lcoe' + LCOE value to plot, by default MetaKeyName.MEAN_LCOE out_path : str, optional File path to save plot to, by default None kwargs : dict @@ -773,14 +778,14 @@ def supply_curve_plot(self, lcoe='mean_lcoe', out_path=None, **kwargs): mplt.df_scatter(sc_df, x='cumulative_capacity', y=lcoe, filename=out_path, **kwargs) - def supply_curve_plotly(self, lcoe='mean_lcoe', out_path=None, **kwargs): + def supply_curve_plotly(self, lcoe=MetaKeyName.MEAN_LCOE, out_path=None, **kwargs): """ Plot supply curve (cumulative capacity vs lcoe) using plotly Parameters ---------- lcoe : str, optional - LCOE value to plot, by default 'mean_lcoe' + LCOE value to plot, by default MetaKeyName.MEAN_LCOE out_path : str, optional File path to save plot to, can be a .html or static image, by default None @@ -797,7 +802,7 @@ def supply_curve_plotly(self, lcoe='mean_lcoe', out_path=None, **kwargs): fig.show() @classmethod - def plot(cls, sc_table, out_dir, plot_type='plotly', lcoe='mean_lcoe', + def plot(cls, sc_table, out_dir, plot_type='plotly', lcoe=MetaKeyName.MEAN_LCOE, **kwargs): """ Create supply curve plot from supply curve table using lcoe value @@ -812,7 +817,7 @@ def plot(cls, sc_table, out_dir, plot_type='plotly', lcoe='mean_lcoe', plot_type : str, optional plot_type of plot to create 'plot' or 'plotly', by default 'plotly' lcoe : str, optional - LCOE value to plot, by default 'mean_lcoe' + LCOE value to plot, by default MetaKeyName.MEAN_LCOE kwargs : dict Additional plotting kwargs """ diff --git a/reV/rep_profiles/rep_profiles.py b/reV/rep_profiles/rep_profiles.py index 305cc4548..38db15411 100644 --- a/reV/rep_profiles/rep_profiles.py +++ b/reV/rep_profiles/rep_profiles.py @@ -5,26 +5,25 @@ @author: gbuster """ -from abc import ABC, abstractmethod -from concurrent.futures import as_completed -from copy import deepcopy import json import logging -import numpy as np import os -import pandas as pd -from scipy import stats +from abc import ABC, abstractmethod +from concurrent.futures import as_completed +from copy import deepcopy from warnings import warn - -from reV.handlers.outputs import Outputs -from reV.utilities.exceptions import FileInputError, DataShapeError -from reV.utilities import log_versions - +import numpy as np +import pandas as pd from rex.resource import Resource from rex.utilities.execution import SpawnProcessPool from rex.utilities.loggers import log_mem from rex.utilities.utilities import parse_year, to_records_array +from scipy import stats + +from reV.handlers.outputs import Outputs +from reV.utilities import MetaKeyName, log_versions +from reV.utilities.exceptions import DataShapeError, FileInputError logger = logging.getLogger(__name__) @@ -299,11 +298,11 @@ def run(cls, profiles, weights=None, rep_method='meanoid', class RegionRepProfile: """Framework to handle rep profile for one resource region""" - RES_GID_COL = 'res_gids' - GEN_GID_COL = 'gen_gids' + RES_GID_COL = MetaKeyName.RES_GIDS + GEN_GID_COL = MetaKeyName.GEN_GIDS def __init__(self, gen_fpath, rev_summary, cf_dset='cf_profile', - rep_method='meanoid', err_method='rmse', weight='gid_counts', + rep_method='meanoid', err_method='rmse', weight=MetaKeyName.GID_COUNTS, n_profiles=1): """ Parameters @@ -372,8 +371,8 @@ def _init_profiles_weights(self): with Resource(self._gen_fpath) as res: meta = res.meta - assert 'gid' in meta - source_res_gids = meta['gid'].values + assert MetaKeyName.GID in meta + source_res_gids = meta[MetaKeyName.GID].values msg = ('Resource gids from "gid" column in meta data from "{}" ' 'must be sorted! reV generation should always be run with ' 'sequential project points.'.format(self._gen_fpath)) @@ -514,7 +513,7 @@ def rep_res_gids(self): @classmethod def get_region_rep_profile(cls, gen_fpath, rev_summary, cf_dset='cf_profile', rep_method='meanoid', - err_method='rmse', weight='gid_counts', + err_method='rmse', weight=MetaKeyName.GID_COUNTS, n_profiles=1): """Class method for parallelization of rep profile calc. @@ -566,7 +565,7 @@ class RepProfilesBase(ABC): def __init__(self, gen_fpath, rev_summary, reg_cols=None, cf_dset='cf_profile', rep_method='meanoid', err_method='rmse', - weight='gid_counts', n_profiles=1): + weight=MetaKeyName.GID_COUNTS, n_profiles=1): """ Parameters ---------- @@ -884,9 +883,9 @@ class RepProfiles(RepProfilesBase): """RepProfiles""" def __init__(self, gen_fpath, rev_summary, reg_cols, cf_dset='cf_profile', - rep_method='meanoid', err_method='rmse', weight='gid_counts', + rep_method='meanoid', err_method='rmse', weight=MetaKeyName.GID_COUNTS, n_profiles=1, aggregate_profiles=False): - """reV rep profiles class. + """ReV rep profiles class. ``reV`` rep profiles compute representative generation profiles for each supply curve point output by ``reV`` supply curve @@ -977,7 +976,7 @@ def __init__(self, gen_fpath, rev_summary, reg_cols, cf_dset='cf_profile', *equally* to the meanoid profile unless these weights are specified. - By default, ``'gid_counts'``. + By default, ``MetaKeyName.GID_COUNTS``. n_profiles : int, optional Number of representative profiles to save to the output file. By default, ``1``. @@ -1000,7 +999,7 @@ def __init__(self, gen_fpath, rev_summary, reg_cols, cf_dset='cf_profile', 'key such as "sc_gid".') logger.error(e) raise ValueError(e) - elif isinstance(reg_cols, str): + if isinstance(reg_cols, str): reg_cols = [reg_cols] elif not isinstance(reg_cols, list): reg_cols = list(reg_cols) @@ -1030,7 +1029,7 @@ def _set_meta(self): else: self._meta = self._rev_summary.groupby(self._reg_cols) self._meta = ( - self._meta['timezone'] + self._meta[MetaKeyName.TIMEZONE] .apply(lambda x: stats.mode(x, keepdims=True).mode[0]) ) self._meta = self._meta.reset_index() diff --git a/reV/supply_curve/aggregation.py b/reV/supply_curve/aggregation.py index 77cfeb4ca..5c14ac9e0 100644 --- a/reV/supply_curve/aggregation.py +++ b/reV/supply_curve/aggregation.py @@ -2,26 +2,29 @@ """ reV aggregation framework. """ +import logging +import os from abc import ABC, abstractmethod + import h5py -import logging import numpy as np -import os import pandas as pd +from rex.resource import Resource +from rex.utilities.execution import SpawnProcessPool +from rex.utilities.loggers import log_mem -from reV.handlers.outputs import Outputs from reV.handlers.exclusions import ExclusionLayers +from reV.handlers.outputs import Outputs from reV.supply_curve.exclusions import ExclusionMaskFromDict from reV.supply_curve.extent import SupplyCurveExtent -from reV.supply_curve.tech_mapping import TechMapping from reV.supply_curve.points import AggregationSupplyCurvePoint -from reV.utilities.exceptions import (EmptySupplyCurvePointError, - FileInputError, SupplyCurveInputError) +from reV.supply_curve.tech_mapping import TechMapping from reV.utilities import log_versions - -from rex.resource import Resource -from rex.utilities.execution import SpawnProcessPool -from rex.utilities.loggers import log_mem +from reV.utilities.exceptions import ( + EmptySupplyCurvePointError, + FileInputError, + SupplyCurveInputError, +) logger = logging.getLogger(__name__) @@ -419,14 +422,14 @@ def _parse_gen_index(gen_fpath): logger.error(msg) raise FileInputError(msg) - if 'gid' in gen_index: - gen_index = gen_index.rename(columns={'gid': 'res_gids'}) - gen_index['gen_gids'] = gen_index.index - gen_index = gen_index[['res_gids', 'gen_gids']] - gen_index = gen_index.set_index(keys='res_gids') + if MetaKeyName.GID in gen_index: + gen_index = gen_index.rename(columns={MetaKeyName.GID: MetaKeyName.RES_GIDS}) + gen_index[MetaKeyName.GEN_GIDS] = gen_index.index + gen_index = gen_index[[MetaKeyName.RES_GIDS, MetaKeyName.GEN_GIDS]] + gen_index = gen_index.set_index(keys=MetaKeyName.RES_GIDS) gen_index = \ gen_index.reindex(range(int(gen_index.index.max() + 1))) - gen_index = gen_index['gen_gids'].values + gen_index = gen_index[MetaKeyName.GEN_GIDS].values gen_index[np.isnan(gen_index)] = -1 gen_index = gen_index.astype(np.int32) else: @@ -795,9 +798,9 @@ def aggregate(self, h5_fpath, agg_method='mean', max_workers=None, for k, v in agg.items(): if k == 'meta': v = pd.concat(v, axis=1).T - v = v.sort_values('sc_point_gid') + v = v.sort_values(MetaKeyName.SC_POINT_GID) v = v.reset_index(drop=True) - v.index.name = 'sc_gid' + v.index.name = MetaKeyName.SC_GID agg[k] = v else: v = np.dstack(v)[0] diff --git a/reV/supply_curve/competitive_wind_farms.py b/reV/supply_curve/competitive_wind_farms.py index 455368715..d93fae6cd 100644 --- a/reV/supply_curve/competitive_wind_farms.py +++ b/reV/supply_curve/competitive_wind_farms.py @@ -3,10 +3,12 @@ Competitive Wind Farms exclusion handler """ import logging -import numpy as np +import numpy as np from rex.utilities.utilities import parse_table +from reV.utilities import MetaKeyName + logger = logging.getLogger(__name__) @@ -76,22 +78,22 @@ def __getitem__(self, keys): """ if not isinstance(keys, tuple): msg = ("{} must be a tuple of form (source, gid) where source is: " - "'sc_gid', 'sc_point_gid', or 'upwind', 'downwind'" + "MetaKeyName.SC_GID, MetaKeyName.SC_POINT_GID, or 'upwind', 'downwind'" .format(keys)) logger.error(msg) raise ValueError(msg) source, gid = keys - if source == 'sc_point_gid': + if source == MetaKeyName.SC_POINT_GID: out = self.map_sc_gid_to_sc_point_gid(gid) - elif source == 'sc_gid': + elif source == MetaKeyName.SC_GID: out = self.map_sc_point_gid_to_sc_gid(gid) elif source == 'upwind': out = self.map_upwind(gid) elif source == 'downwind': out = self.map_downwind(gid) else: - msg = ("{} must be: 'sc_gid', 'sc_point_gid', or 'upwind', " + msg = ("{} must be: MetaKeyName.SC_GID, MetaKeyName.SC_POINT_GID, or 'upwind', " "'downwind'".format(source)) logger.error(msg) raise ValueError(msg) @@ -182,7 +184,7 @@ def _parse_wind_dirs(cls, wind_dirs): """ wind_dirs = cls._parse_table(wind_dirs) - wind_dirs = wind_dirs.set_index('sc_point_gid') + wind_dirs = wind_dirs.set_index(MetaKeyName.SC_POINT_GID) columns = [c for c in wind_dirs if c.endswith(('_gid', '_pr'))] wind_dirs = wind_dirs[columns] @@ -212,21 +214,21 @@ def _parse_sc_points(cls, sc_points, offshore=False): Mask array to mask excluded sc_point_gids """ sc_points = cls._parse_table(sc_points) - if 'offshore' in sc_points and not offshore: + if MetaKeyName.OFFSHORE in sc_points and not offshore: logger.debug('Not including offshore supply curve points in ' 'CompetitiveWindFarm') - mask = sc_points['offshore'] == 0 + mask = sc_points[MetaKeyName.OFFSHORE] == 0 sc_points = sc_points.loc[mask] - mask = np.ones(int(1 + sc_points['sc_point_gid'].max()), dtype=bool) + mask = np.ones(int(1 + sc_points[MetaKeyName.SC_POINT_GID].max()), dtype=bool) - sc_points = sc_points[['sc_gid', 'sc_point_gid']] - sc_gids = sc_points.set_index('sc_gid') + sc_points = sc_points[[MetaKeyName.SC_GID, MetaKeyName.SC_POINT_GID]] + sc_gids = sc_points.set_index(MetaKeyName.SC_GID) sc_gids = {k: int(v[0]) for k, v in sc_gids.iterrows()} sc_point_gids = \ - sc_points.groupby('sc_point_gid')['sc_gid'].unique().to_frame() - sc_point_gids = {int(k): v['sc_gid'] + sc_points.groupby(MetaKeyName.SC_POINT_GID)[MetaKeyName.SC_GID].unique().to_frame() + sc_point_gids = {int(k): v[MetaKeyName.SC_GID] for k, v in sc_point_gids.iterrows()} return sc_gids, sc_point_gids, mask @@ -338,6 +340,7 @@ def map_upwind(self, sc_point_gid): ---------- sc_point_gid : int Supply point curve gid to get upwind neighbors + Returns ------- int | list @@ -353,6 +356,7 @@ def map_downwind(self, sc_point_gid): ---------- sc_point_gid : int Supply point curve gid to get downwind neighbors + Returns ------- int | list @@ -407,13 +411,13 @@ def remove_noncompetitive_farm(self, sc_points, sort_on='total_lcoe', wind farms """ sc_points = self._parse_table(sc_points) - if 'offshore' in sc_points and not self._offshore: - mask = sc_points['offshore'] == 0 + if MetaKeyName.OFFSHORE in sc_points and not self._offshore: + mask = sc_points[MetaKeyName.OFFSHORE] == 0 sc_points = sc_points.loc[mask] sc_points = sc_points.sort_values(sort_on) - sc_point_gids = sc_points['sc_point_gid'].values.astype(int) + sc_point_gids = sc_points[MetaKeyName.SC_POINT_GID].values.astype(int) for i in range(len(sc_points)): gid = sc_point_gids[i] @@ -428,7 +432,7 @@ def remove_noncompetitive_farm(self, sc_points, sort_on='total_lcoe', self.exclude_sc_point_gid(n) sc_gids = self.sc_gids - mask = sc_points['sc_gid'].isin(sc_gids) + mask = sc_points[MetaKeyName.SC_GID].isin(sc_gids) return sc_points.loc[mask].reset_index(drop=True) diff --git a/reV/supply_curve/exclusions.py b/reV/supply_curve/exclusions.py index 2342998ac..e65a44e3f 100644 --- a/reV/supply_curve/exclusions.py +++ b/reV/supply_curve/exclusions.py @@ -3,14 +3,15 @@ Generate reV inclusion mask from exclusion layers """ import logging -import numpy as np -from scipy import ndimage from warnings import warn +import numpy as np from rex.utilities.loggers import log_mem +from scipy import ndimage + from reV.handlers.exclusions import ExclusionLayers -from reV.utilities.exceptions import ExclusionLayerError -from reV.utilities.exceptions import SupplyCurveInputError +from reV.utilities import MetaKeyName +from reV.utilities.exceptions import ExclusionLayerError, SupplyCurveInputError logger = logging.getLogger(__name__) @@ -246,8 +247,7 @@ def min_value(self): if all(isinstance(x, (int, float)) for x in range_var): return min(range_var) - else: - return range_var[0] + return range_var[0] @property def max_value(self): @@ -265,8 +265,7 @@ def max_value(self): if all(isinstance(x, (int, float)) for x in range_var): return max(range_var) - else: - return range_var[1] + return range_var[1] @property def exclude_values(self): @@ -370,7 +369,7 @@ def _check_mask_type(self): contradictory Returns - ------ + ------- mask : str Mask type """ @@ -689,7 +688,7 @@ def excl_h5(self): Returns ------- _excl_h5 : ExclusionLayers - """ + """ return self._excl_h5 @property @@ -714,7 +713,7 @@ def layer_names(self): Returns ------- list - """ + """ return self._layers.keys() @property @@ -725,7 +724,7 @@ def layers(self): Returns ------- list - """ + """ return self._layers.values() @property @@ -749,7 +748,7 @@ def latitude(self): ------- ndarray """ - return self.excl_h5['latitude'] + return self.excl_h5[MetaKeyName.LATITUDE] @property def longitude(self): @@ -760,7 +759,7 @@ def longitude(self): ------- ndarray """ - return self.excl_h5['longitude'] + return self.excl_h5[MetaKeyName.LONGITUDE] def add_layer(self, layer, replace=False): """ @@ -939,7 +938,7 @@ def _generate_ones_mask(self, ds_slice): def _add_layer_to_mask(self, mask, layer, ds_slice, check_layers, combine_func): - """Add layer mask to full mask. """ + """Add layer mask to full mask.""" layer_mask = self._compute_layer_mask(layer, ds_slice, check_layers) if mask is None: return layer_mask @@ -947,7 +946,7 @@ def _add_layer_to_mask(self, mask, layer, ds_slice, check_layers, return combine_func(mask, layer_mask, dtype='float32') def _compute_layer_mask(self, layer, ds_slice, check_layers=False): - """Compute mask for single layer, including extent. """ + """Compute mask for single layer, including extent.""" layer_mask = self._masked_layer_data(layer, ds_slice) layer_mask = self._apply_layer_mask_extent(layer, layer_mask, ds_slice) @@ -964,7 +963,7 @@ def _compute_layer_mask(self, layer, ds_slice, check_layers=False): return layer_mask def _apply_layer_mask_extent(self, layer, layer_mask, ds_slice): - """Apply extent to layer mask, if any. """ + """Apply extent to layer mask, if any.""" if layer.extent is None: return layer_mask @@ -983,7 +982,7 @@ def _apply_layer_mask_extent(self, layer, layer_mask, ds_slice): return layer_mask def _masked_layer_data(self, layer, ds_slice): - """Extract masked data for layer. """ + """Extract masked data for layer.""" return layer[self.excl_h5[(layer.name, ) + ds_slice]] def _generate_mask(self, *ds_slice, check_layers=False): diff --git a/reV/supply_curve/extent.py b/reV/supply_curve/extent.py index 1e02b1511..b63e25cac 100644 --- a/reV/supply_curve/extent.py +++ b/reV/supply_curve/extent.py @@ -3,14 +3,15 @@ reV supply curve extent """ import logging + import numpy as np import pandas as pd +from rex.utilities.utilities import get_chunk_ranges from reV.handlers.exclusions import ExclusionLayers +from reV.utilities import MetaKeyName from reV.utilities.exceptions import SupplyCurveError, SupplyCurveInputError -from rex.utilities.utilities import get_chunk_ranges - logger = logging.getLogger(__name__) @@ -287,8 +288,8 @@ def latitude(self): for r, c in zip(sc_rows.flatten(), sc_cols.flatten()): r = self.excl_row_slices[r] c = self.excl_col_slices[c] - lats.append(self.exclusions['latitude', r, c].mean()) - lons.append(self.exclusions['longitude', r, c].mean()) + lats.append(self.exclusions[MetaKeyName.LATITUDE, r, c].mean()) + lons.append(self.exclusions[MetaKeyName.LONGITUDE, r, c].mean()) self._latitude = np.array(lats, dtype='float32') self._longitude = np.array(lons, dtype='float32') @@ -313,8 +314,8 @@ def longitude(self): for r, c in zip(sc_rows.flatten(), sc_cols.flatten()): r = self.excl_row_slices[r] c = self.excl_col_slices[c] - lats.append(self.exclusions['latitude', r, c].mean()) - lons.append(self.exclusions['longitude', r, c].mean()) + lats.append(self.exclusions[MetaKeyName.LATITUDE, r, c].mean()) + lons.append(self.exclusions[MetaKeyName.LONGITUDE, r, c].mean()) self._latitude = np.array(lats, dtype='float32') self._longitude = np.array(lons, dtype='float32') @@ -370,7 +371,7 @@ def points(self): self._points = pd.DataFrame({'row_ind': self.row_indices.copy(), 'col_ind': self.col_indices.copy()}) - self._points.index.name = 'gid' # sc_point_gid + self._points.index.name = MetaKeyName.GID # sc_point_gid return self._points diff --git a/reV/supply_curve/points.py b/reV/supply_curve/points.py index 7f229b1e3..b864c44cf 100644 --- a/reV/supply_curve/points.py +++ b/reV/supply_curve/points.py @@ -2,26 +2,29 @@ """ reV supply curve points frameworks. """ -from abc import ABC import logging +from abc import ABC +from warnings import warn + import numpy as np import pandas as pd -from warnings import warn +from rex.multi_time_resource import MultiTimeResource +from rex.resource import BaseResource, Resource +from rex.utilities.utilities import jsonify_dict from reV.econ.economies_of_scale import EconomiesOfScale from reV.econ.utilities import lcoe_fcr from reV.handlers.exclusions import ExclusionLayers from reV.supply_curve.exclusions import ExclusionMask, ExclusionMaskFromDict -from reV.utilities.exceptions import (SupplyCurveInputError, - EmptySupplyCurvePointError, - InputWarning, - FileInputError, - DataShapeError, - OutputWarning) - -from rex.resource import Resource, BaseResource -from rex.multi_time_resource import MultiTimeResource -from rex.utilities.utilities import jsonify_dict +from reV.utilities import MetaKeyName +from reV.utilities.exceptions import ( + DataShapeError, + EmptySupplyCurvePointError, + FileInputError, + InputWarning, + OutputWarning, + SupplyCurveInputError, +) logger = logging.getLogger(__name__) @@ -98,7 +101,7 @@ def _parse_slices(self, gid, resolution, exclusion_shape): @property def gid(self): - """supply curve point gid""" + """Supply curve point gid""" return self._gid @property @@ -359,8 +362,8 @@ def centroid(self): """ if self._centroid is None: - lats = self.exclusions.excl_h5['latitude', self.rows, self.cols] - lons = self.exclusions.excl_h5['longitude', self.rows, self.cols] + lats = self.exclusions.excl_h5[MetaKeyName.LATITUDE, self.rows, self.cols] + lons = self.exclusions.excl_h5[MetaKeyName.LONGITUDE, self.rows, self.cols] self._centroid = (lats.mean(), lons.mean()) return self._centroid @@ -541,7 +544,7 @@ def exclusion_weighted_mean(self, arr, drop_nan=True): if np.isnan(x).all(): return np.nan - elif drop_nan and np.isnan(x).any(): + if drop_nan and np.isnan(x).any(): nan_mask = np.isnan(x) x = x[~nan_mask] incl = incl[~nan_mask] @@ -718,9 +721,8 @@ def _mode(data): """ if not data.size: return None - else: - # pd series is more flexible with non-numeric than stats mode - return pd.Series(data).mode().values[0] + # pd series is more flexible with non-numeric than stats mode + return pd.Series(data).mode().values[0] @staticmethod def _categorize(data, incl_mult): @@ -1087,8 +1089,8 @@ def county(self): def elevation(self): """Get the SC point elevation based on the resource meta data.""" elevation = None - if 'elevation' in self.h5.meta: - elevation = self.h5.meta.loc[self.h5_gid_set, 'elevation'].mean() + if MetaKeyName.ELEVATION in self.h5.meta: + elevation = self.h5.meta.loc[self.h5_gid_set, MetaKeyName.ELEVATION].mean() return elevation @@ -1096,15 +1098,15 @@ def elevation(self): def timezone(self): """Get the SC point timezone based on the resource meta data.""" timezone = None - if 'timezone' in self.h5.meta and self.county is not None: + if MetaKeyName.TIMEZONE in self.h5.meta and self.county is not None: # make sure timezone flag and county are coincident counties = self.h5.meta.loc[self.h5_gid_set, 'county'].values iloc = np.where(counties == self.county)[0][0] - timezone = self.h5.meta.loc[self.h5_gid_set, 'timezone'].values + timezone = self.h5.meta.loc[self.h5_gid_set, MetaKeyName.TIMEZONE].values timezone = timezone[iloc] - elif 'timezone' in self.h5.meta: - timezone = self.h5.meta.loc[self.h5_gid_set, 'timezone'].mode() + elif MetaKeyName.TIMEZONE in self.h5.meta: + timezone = self.h5.meta.loc[self.h5_gid_set, MetaKeyName.TIMEZONE].mode() timezone = timezone.values[0] return timezone @@ -1114,15 +1116,15 @@ def offshore(self): """Get the SC point offshore flag based on the resource meta data (if offshore column is present).""" offshore = None - if 'offshore' in self.h5.meta and self.county is not None: + if MetaKeyName.OFFSHORE in self.h5.meta and self.county is not None: # make sure offshore flag and county are coincident counties = self.h5.meta.loc[self.h5_gid_set, 'county'].values iloc = np.where(counties == self.county)[0][0] - offshore = self.h5.meta.loc[self.h5_gid_set, 'offshore'].values + offshore = self.h5.meta.loc[self.h5_gid_set, MetaKeyName.OFFSHORE].values offshore = offshore[iloc] - elif 'offshore' in self.h5.meta: - offshore = self.h5.meta.loc[self.h5_gid_set, 'offshore'].mode() + elif MetaKeyName.OFFSHORE in self.h5.meta: + offshore = self.h5.meta.loc[self.h5_gid_set, MetaKeyName.OFFSHORE].mode() offshore = offshore.values[0] return offshore @@ -1153,18 +1155,18 @@ def summary(self): pandas.Series List of supply curve point's meta data """ - meta = {'sc_point_gid': self.sc_point_gid, - 'source_gids': self.h5_gid_set, - 'gid_counts': self.gid_counts, - 'n_gids': self.n_gids, - 'area_sq_km': self.area, - 'latitude': self.latitude, - 'longitude': self.longitude, + meta = {MetaKeyName.SC_POINT_GID: self.sc_point_gid, + MetaKeyName.SOURCE_GIDS: self.h5_gid_set, + MetaKeyName.GID_COUNTS: self.gid_counts, + MetaKeyName.N_GIDS: self.n_gids, + MetaKeyName.AREA_SQ_KM: self.area, + MetaKeyName.LATITUDE: self.latitude, + MetaKeyName.LONGITUDE: self.longitude, 'country': self.country, 'state': self.state, 'county': self.county, - 'elevation': self.elevation, - 'timezone': self.timezone, + MetaKeyName.ELEVATION: self.elevation, + MetaKeyName.TIMEZONE: self.timezone, } meta = pd.Series(meta) @@ -1495,10 +1497,9 @@ def res_data(self): if isinstance(self._res_class_dset, np.ndarray): return self._res_class_dset - else: - if self._res_data is None: - if self._res_class_dset in self.gen.datasets: - self._res_data = self.gen[self._res_class_dset] + if self._res_data is None: + if self._res_class_dset in self.gen.datasets: + self._res_data = self.gen[self._res_class_dset] return self._res_data @@ -1516,10 +1517,9 @@ def gen_data(self): if isinstance(self._cf_dset, np.ndarray): return self._cf_dset - else: - if self._gen_data is None: - if self._cf_dset in self.gen.datasets: - self._gen_data = self.gen[self._cf_dset] + if self._gen_data is None: + if self._cf_dset in self.gen.datasets: + self._gen_data = self.gen[self._cf_dset] return self._gen_data @@ -1537,10 +1537,9 @@ def lcoe_data(self): if isinstance(self._lcoe_dset, np.ndarray): return self._lcoe_dset - else: - if self._lcoe_data is None: - if self._lcoe_dset in self.gen.datasets: - self._lcoe_data = self.gen[self._lcoe_dset] + if self._lcoe_data is None: + if self._lcoe_dset in self.gen.datasets: + self._lcoe_data = self.gen[self._lcoe_dset] return self._lcoe_data @@ -2001,35 +2000,35 @@ def point_summary(self, args=None): Dictionary of summary outputs for this sc point. """ - ARGS = {'res_gids': self.res_gid_set, - 'gen_gids': self.gen_gid_set, - 'gid_counts': self.gid_counts, - 'n_gids': self.n_gids, - 'mean_cf': self.mean_cf, - 'mean_lcoe': self.mean_lcoe, - 'mean_res': self.mean_res, - 'capacity': self.capacity, - 'area_sq_km': self.area, - 'latitude': self.latitude, - 'longitude': self.longitude, + ARGS = {MetaKeyName.RES_GIDS: self.res_gid_set, + MetaKeyName.GEN_GIDS: self.gen_gid_set, + MetaKeyName.GID_COUNTS: self.gid_counts, + MetaKeyName.N_GIDS: self.n_gids, + MetaKeyName.MEAN_CF: self.mean_cf, + MetaKeyName.MEAN_LCOE: self.mean_lcoe, + MetaKeyName.MEAN_RES: self.mean_res, + MetaKeyName.CAPACITY: self.capacity, + MetaKeyName.AREA_SQ_KM: self.area, + MetaKeyName.LATITUDE: self.latitude, + MetaKeyName.LONGITUDE: self.longitude, 'country': self.country, 'state': self.state, 'county': self.county, - 'elevation': self.elevation, - 'timezone': self.timezone, + MetaKeyName.ELEVATION: self.elevation, + MetaKeyName.TIMEZONE: self.timezone, } - extra_atts = ['capacity_ac', 'offshore', 'sc_point_capital_cost', - 'sc_point_fixed_operating_cost', - 'sc_point_annual_energy', 'sc_point_annual_energy_ac'] + extra_atts = [MetaKeyName.CAPACITY_AC, MetaKeyName.OFFSHORE, MetaKeyName.SC_POINT_CAPITAL_COST, + MetaKeyName.SC_POINT_FIXED_OPERATING_COST, + MetaKeyName.SC_POINT_ANNUAL_ENERGY, MetaKeyName.SC_POINT_ANNUAL_ENERGY_AC] for attr in extra_atts: value = getattr(self, attr) if value is not None: ARGS[attr] = value if self._friction_layer is not None: - ARGS['mean_friction'] = self.mean_friction - ARGS['mean_lcoe_friction'] = self.mean_lcoe_friction + ARGS[MetaKeyName.MEAN_FRICTION] = self.mean_friction + ARGS[MetaKeyName.MEAN_LCOE_FRICTION] = self.mean_lcoe_friction if self._h5_dsets is not None: for dset, data in self.mean_h5_dsets_data.items(): @@ -2070,14 +2069,14 @@ def economies_of_scale(cap_cost_scale, summary): """ eos = EconomiesOfScale(cap_cost_scale, summary) - summary['raw_lcoe'] = eos.raw_lcoe - summary['mean_lcoe'] = eos.scaled_lcoe - summary['capital_cost_scalar'] = eos.capital_cost_scalar - summary['scaled_capital_cost'] = eos.scaled_capital_cost + summary[MetaKeyName.RAW_LCOE] = eos.raw_lcoe + summary[MetaKeyName.MEAN_LCOE] = eos.scaled_lcoe + summary[MetaKeyName.CAPITAL_COST_SCALAR] = eos.capital_cost_scalar + summary[MetaKeyName.SCALED_CAPITAL_COST] = eos.scaled_capital_cost if "sc_point_capital_cost" in summary: scaled_costs = (summary["sc_point_capital_cost"] * eos.capital_cost_scalar) - summary['scaled_sc_point_capital_cost'] = scaled_costs + summary[MetaKeyName.SCALED_SC_POINT_CAPITAL_COST] = scaled_costs return summary diff --git a/reV/supply_curve/sc_aggregation.py b/reV/supply_curve/sc_aggregation.py index 6621205a1..1500ffad9 100644 --- a/reV/supply_curve/sc_aggregation.py +++ b/reV/supply_curve/sc_aggregation.py @@ -6,29 +6,35 @@ @author: gbuster """ -from concurrent.futures import as_completed import logging -import numpy as np -import psutil import os -import pandas as pd +from concurrent.futures import as_completed from warnings import warn +import numpy as np +import pandas as pd +import psutil +from rex.multi_file_resource import MultiFileResource +from rex.resource import Resource +from rex.utilities.execution import SpawnProcessPool + from reV.generation.base import BaseGen from reV.handlers.exclusions import ExclusionLayers -from reV.supply_curve.aggregation import (AbstractAggFileHandler, - BaseAggregation, Aggregation) +from reV.supply_curve.aggregation import ( + AbstractAggFileHandler, + Aggregation, + BaseAggregation, +) from reV.supply_curve.exclusions import FrictionMask from reV.supply_curve.extent import SupplyCurveExtent from reV.supply_curve.points import GenerationSupplyCurvePoint -from reV.utilities.exceptions import (EmptySupplyCurvePointError, - OutputWarning, FileInputError, - InputWarning) -from reV.utilities import log_versions - -from rex.resource import Resource -from rex.multi_file_resource import MultiFileResource -from rex.utilities.execution import SpawnProcessPool +from reV.utilities import MetaKeyName, log_versions +from reV.utilities.exceptions import ( + EmptySupplyCurvePointError, + FileInputError, + InputWarning, + OutputWarning, +) logger = logging.getLogger(__name__) @@ -151,9 +157,9 @@ def _parse_power_density(self): if self._pdf.endswith('.csv'): self._power_density = pd.read_csv(self._pdf) - if ('gid' in self._power_density + if (MetaKeyName.GID in self._power_density and 'power_density' in self._power_density): - self._power_density = self._power_density.set_index('gid') + self._power_density = self._power_density.set_index(MetaKeyName.GID) else: msg = ('Variable power density file must include "gid" ' 'and "power_density" columns, but received: {}' @@ -231,7 +237,7 @@ def __init__(self, excl_fpath, tm_dset, econ_fpath=None, lcoe_dset='lcoe_fcr-means', h5_dsets=None, data_layers=None, power_density=None, friction_fpath=None, friction_dset=None, cap_cost_scale=None, recalc_lcoe=True): - """reV supply curve points aggregation framework. + r"""ReV supply curve points aggregation framework. ``reV`` supply curve aggregation combines a high-resolution (e.g. 90m) exclusion dataset with a (typically) lower resolution @@ -721,7 +727,7 @@ def _check_data_layers(self, methods=('mean', 'max', 'min', if 'method' not in v: raise KeyError('Data aggregation "method" data layer "{}" ' 'must be specified.'.format(k)) - elif v['method'].lower() not in methods: + if v['method'].lower() not in methods: raise ValueError('Cannot recognize data layer agg method: ' '"{}". Can only do: {}.' .format(v['method'], methods)) @@ -1067,9 +1073,9 @@ def run_serial(cls, excl_fpath, gen_fpath, tm_dset, gen_index, except EmptySupplyCurvePointError: logger.debug('SC point {} is empty'.format(gid)) else: - pointsum['sc_point_gid'] = gid - pointsum['sc_row_ind'] = points.loc[gid, 'row_ind'] - pointsum['sc_col_ind'] = points.loc[gid, 'col_ind'] + pointsum[MetaKeyName.SC_POINT_GID] = gid + pointsum[MetaKeyName.SC_ROW_IND] = points.loc[gid, 'row_ind'] + pointsum[MetaKeyName.SC_COL_IND] = points.loc[gid, 'col_ind'] pointsum['res_class'] = ri summary.append(pointsum) @@ -1201,17 +1207,16 @@ def _convert_bins(bins): if all(type_check): return bins - elif any(type_check): + if any(type_check): raise TypeError('Resource class bins has inconsistent ' 'entry type: {}'.format(bins)) - else: - bbins = [] - for i, b in enumerate(sorted(bins)): - if i < len(bins) - 1: - bbins.append([b, bins[i + 1]]) + bbins = [] + for i, b in enumerate(sorted(bins)): + if i < len(bins) - 1: + bbins.append([b, bins[i + 1]]) - return bbins + return bbins @staticmethod def _summary_to_df(summary): @@ -1228,10 +1233,10 @@ def _summary_to_df(summary): Summary of the SC points. """ summary = pd.DataFrame(summary) - sort_by = [x for x in ('sc_point_gid', 'res_class') if x in summary] + sort_by = [x for x in (MetaKeyName.SC_POINT_GID, 'res_class') if x in summary] summary = summary.sort_values(sort_by) summary = summary.reset_index(drop=True) - summary.index.name = 'sc_gid' + summary.index.name = MetaKeyName.SC_GID return summary diff --git a/reV/supply_curve/supply_curve.py b/reV/supply_curve/supply_curve.py index edf6a1c66..071d3461a 100644 --- a/reV/supply_curve/supply_curve.py +++ b/reV/supply_curve/supply_curve.py @@ -4,23 +4,22 @@ - Calculation of LCOT - Supply Curve creation """ -from copy import deepcopy import json import logging -import numpy as np import os -import pandas as pd +from copy import deepcopy from warnings import warn +import numpy as np +import pandas as pd +from rex import Resource +from rex.utilities import SpawnProcessPool, parse_table + from reV.handlers.transmission import TransmissionCosts as TC from reV.handlers.transmission import TransmissionFeatures as TF from reV.supply_curve.competitive_wind_farms import CompetitiveWindFarms -from reV.utilities.exceptions import SupplyCurveInputError, SupplyCurveError -from reV.utilities import log_versions - -from rex import Resource -from rex.utilities import parse_table, SpawnProcessPool - +from reV.utilities import MetaKeyName, log_versions +from reV.utilities.exceptions import SupplyCurveError, SupplyCurveInputError logger = logging.getLogger(__name__) @@ -29,8 +28,8 @@ class SupplyCurve: """SupplyCurve""" def __init__(self, sc_points, trans_table, sc_features=None, - sc_capacity_col='capacity'): - """reV LCOT calculation and SupplyCurve sorting class. + sc_capacity_col=MetaKeyName.CAPACITY): + """ReV LCOT calculation and SupplyCurve sorting class. ``reV`` supply curve computes the transmission costs associated with each supply curve point output by ``reV`` supply curve @@ -180,7 +179,7 @@ def _parse_sc_points(sc_points, sc_features=None): if isinstance(sc_points, str) and sc_points.endswith('.h5'): with Resource(sc_points) as res: sc_points = res.meta - sc_points.index.name = 'sc_gid' + sc_points.index.name = MetaKeyName.SC_GID sc_points = sc_points.reset_index() else: sc_points = parse_table(sc_points) @@ -275,7 +274,7 @@ def _parse_trans_table(trans_table): trans_table = trans_table.rename(columns={'dist_mi': 'dist_km'}) trans_table['dist_km'] *= 1.60934 - drop_cols = ['sc_gid', 'cap_left', 'sc_point_gid'] + drop_cols = [MetaKeyName.SC_GID, 'cap_left', MetaKeyName.SC_POINT_GID] drop_cols = [c for c in drop_cols if c in trans_table] if drop_cols: trans_table = trans_table.drop(columns=drop_cols) @@ -283,7 +282,7 @@ def _parse_trans_table(trans_table): return trans_table @staticmethod - def _map_trans_capacity(trans_sc_table, sc_capacity_col='capacity'): + def _map_trans_capacity(trans_sc_table, sc_capacity_col=MetaKeyName.CAPACITY): """ Map SC gids to transmission features based on capacity. For any SC gids with capacity > the maximum transmission feature capacity, map @@ -396,7 +395,7 @@ def _check_sub_trans_lines(cls, features): return line_gids[~test].tolist() @classmethod - def _check_substation_conns(cls, trans_table, sc_cols='sc_gid'): + def _check_substation_conns(cls, trans_table, sc_cols=MetaKeyName.SC_GID): """ Run checks on substation transmission features to make sure that every sc point connecting to a substation can also connect to its @@ -409,7 +408,7 @@ def _check_substation_conns(cls, trans_table, sc_cols='sc_gid'): (should already be merged with SC points). sc_cols : str | list, optional Column(s) in trans_table with unique supply curve id, - by default 'sc_gid' + by default MetaKeyName.SC_GID """ missing = {} for sc_point, sc_table in trans_table.groupby(sc_cols): @@ -437,8 +436,8 @@ def _check_sc_trans_table(cls, sc_points, trans_table): Table mapping supply curve points to transmission features (should already be merged with SC points). """ - sc_gids = set(sc_points['sc_gid'].unique()) - trans_sc_gids = set(trans_table['sc_gid'].unique()) + sc_gids = set(sc_points[MetaKeyName.SC_GID].unique()) + trans_sc_gids = set(trans_table[MetaKeyName.SC_GID].unique()) missing = sorted(list(sc_gids - trans_sc_gids)) if any(missing): msg = ("There are {} Supply Curve points with missing " @@ -465,9 +464,9 @@ def _check_sc_trans_table(cls, sc_points, trans_table): @classmethod def _merge_sc_trans_tables(cls, sc_points, trans_table, - sc_cols=('sc_gid', 'capacity', 'mean_cf', - 'mean_lcoe'), - sc_capacity_col='capacity'): + sc_cols=(MetaKeyName.SC_GID, MetaKeyName.CAPACITY, MetaKeyName.MEAN_CF, + MetaKeyName.MEAN_LCOE), + sc_capacity_col=MetaKeyName.CAPACITY): """ Merge the supply curve table with the transmission features table. @@ -482,7 +481,7 @@ def _merge_sc_trans_tables(cls, sc_points, trans_table, sc_cols : tuple | list, optional List of column from sc_points to transfer into the trans table, If the `sc_capacity_col` is not included, it will get added. - by default ('sc_gid', 'capacity', 'mean_cf', 'mean_lcoe') + by default (MetaKeyName.SC_GID, MetaKeyName.CAPACITY, MetaKeyName.MEAN_CF, MetaKeyName.MEAN_LCOE) sc_capacity_col : str, optional Name of capacity column in `trans_sc_table`. The values in this column determine the size of transmission lines built. @@ -524,8 +523,8 @@ def _merge_sc_trans_tables(cls, sc_points, trans_table, if isinstance(sc_cols, tuple): sc_cols = list(sc_cols) - if 'mean_lcoe_friction' in sc_points: - sc_cols.append('mean_lcoe_friction') + if MetaKeyName.MEAN_LCOE_FRICTION in sc_points: + sc_cols.append(MetaKeyName.MEAN_LCOE_FRICTION) if 'transmission_multiplier' in sc_points: sc_cols.append('transmission_multiplier') @@ -539,8 +538,8 @@ def _merge_sc_trans_tables(cls, sc_points, trans_table, @classmethod def _map_tables(cls, sc_points, trans_table, - sc_cols=('sc_gid', 'capacity', 'mean_cf', 'mean_lcoe'), - sc_capacity_col='capacity'): + sc_cols=(MetaKeyName.SC_GID, MetaKeyName.CAPACITY, MetaKeyName.MEAN_CF, MetaKeyName.MEAN_LCOE), + sc_capacity_col=MetaKeyName.CAPACITY): """ Map supply curve points to transmission features @@ -555,7 +554,7 @@ def _map_tables(cls, sc_points, trans_table, sc_cols : tuple | list, optional List of column from sc_points to transfer into the trans table, If the `sc_capacity_col` is not included, it will get added. - by default ('sc_gid', 'capacity', 'mean_cf', 'mean_lcoe') + by default (MetaKeyName.SC_GID, MetaKeyName.CAPACITY, MetaKeyName.MEAN_CF, MetaKeyName.MEAN_LCOE) sc_capacity_col : str, optional Name of capacity column in `trans_sc_table`. The values in this column determine the size of transmission lines built. @@ -583,7 +582,7 @@ def _map_tables(cls, sc_points, trans_table, trans_sc_table = \ trans_sc_table.sort_values( - ['sc_gid', 'trans_gid']).reset_index(drop=True) + [MetaKeyName.SC_GID, 'trans_gid']).reset_index(drop=True) cls._check_sc_trans_table(sc_points, trans_sc_table) @@ -625,7 +624,7 @@ def _create_handler(trans_table, trans_costs=None, avail_cap_frac=1): return trans_features @staticmethod - def _parse_sc_gids(trans_table, gid_key='sc_gid'): + def _parse_sc_gids(trans_table, gid_key=MetaKeyName.SC_GID): """Extract unique sc gids, make bool mask from tranmission table Parameters @@ -652,7 +651,7 @@ def _parse_sc_gids(trans_table, gid_key='sc_gid'): @staticmethod def _get_capacity(sc_gid, sc_table, connectable=True, - sc_capacity_col='capacity'): + sc_capacity_col=MetaKeyName.CAPACITY): """ Get capacity of supply curve point @@ -700,7 +699,7 @@ def _get_capacity(sc_gid, sc_table, connectable=True, def _compute_trans_cap_cost(cls, trans_table, trans_costs=None, avail_cap_frac=1, max_workers=None, connectable=True, line_limited=False, - sc_capacity_col='capacity'): + sc_capacity_col=MetaKeyName.CAPACITY): """ Compute levelized cost of transmission for all combinations of supply curve points and tranmission features in trans_table @@ -762,7 +761,7 @@ def _compute_trans_cap_cost(cls, trans_table, trans_costs=None, max_workers = os.cpu_count() logger.info('Computing LCOT costs for all possible connections...') - groups = trans_table.groupby('sc_gid') + groups = trans_table.groupby(MetaKeyName.SC_GID) if max_workers > 1: loggers = [__name__, 'reV.handlers.transmission', 'reV'] with SpawnProcessPool(max_workers=max_workers, @@ -844,29 +843,29 @@ def compute_total_lcoe(self, fcr, transmission_costs=None, self._trans_table['trans_cap_cost_per_mw'] = cost cost *= self._trans_table[self._sc_capacity_col] - cost /= self._trans_table['capacity'] # align with "mean_cf" + cost /= self._trans_table[MetaKeyName.CAPACITY] # align with "mean_cf" if 'reinforcement_cost_per_mw' in self._trans_table: logger.info("'reinforcement_cost_per_mw' column found in " "transmission table. Adding reinforcement costs " "to total LCOE.") - cf_mean_arr = self._trans_table['mean_cf'].values + cf_mean_arr = self._trans_table[MetaKeyName.MEAN_CF].values lcot = (cost * fcr) / (cf_mean_arr * 8760) - lcoe = lcot + self._trans_table['mean_lcoe'] + lcoe = lcot + self._trans_table[MetaKeyName.MEAN_LCOE] self._trans_table['lcot_no_reinforcement'] = lcot self._trans_table['lcoe_no_reinforcement'] = lcoe r_cost = (self._trans_table['reinforcement_cost_per_mw'] .values.copy()) r_cost *= self._trans_table[self._sc_capacity_col] - r_cost /= self._trans_table['capacity'] # align with "mean_cf" + r_cost /= self._trans_table[MetaKeyName.CAPACITY] # align with "mean_cf" cost += r_cost # $/MW - cf_mean_arr = self._trans_table['mean_cf'].values + cf_mean_arr = self._trans_table[MetaKeyName.MEAN_CF].values lcot = (cost * fcr) / (cf_mean_arr * 8760) self._trans_table['lcot'] = lcot self._trans_table['total_lcoe'] = (self._trans_table['lcot'] - + self._trans_table['mean_lcoe']) + + self._trans_table[MetaKeyName.MEAN_LCOE]) if consider_friction: self._calculate_total_lcoe_friction() @@ -875,10 +874,10 @@ def _calculate_total_lcoe_friction(self): """Look for site mean LCOE with friction in the trans table and if found make a total LCOE column with friction.""" - if 'mean_lcoe_friction' in self._trans_table: + if MetaKeyName.MEAN_LCOE_FRICTION in self._trans_table: lcoe_friction = (self._trans_table['lcot'] - + self._trans_table['mean_lcoe_friction']) - self._trans_table['total_lcoe_friction'] = lcoe_friction + + self._trans_table[MetaKeyName.MEAN_LCOE_FRICTION]) + self._trans_table[MetaKeyName.TOTAL_LCOE_FRICTION] = lcoe_friction logger.info('Found mean LCOE with friction. Adding key ' '"total_lcoe_friction" to trans table.') @@ -912,7 +911,7 @@ def _exclude_noncompetitive_wind_farms(self, comp_wind_dirs, sc_gid, for n in exclude_gids: check = comp_wind_dirs.exclude_sc_point_gid(n) if check: - sc_gids = comp_wind_dirs['sc_gid', n] + sc_gids = comp_wind_dirs[MetaKeyName.SC_GID, n] for sc_id in sc_gids: if self._mask[sc_id]: logger.debug('Excluding sc_gid {}' @@ -1012,7 +1011,7 @@ def _full_sort(self, trans_table, trans_costs=None, conn_lists = {k: deepcopy(init_list) for k in columns} - trans_sc_gids = trans_table['sc_gid'].values.astype(int) + trans_sc_gids = trans_table[MetaKeyName.SC_GID].values.astype(int) # syntax is final_key: source_key (source from trans_table) all_cols = {k: k for k in columns} @@ -1047,7 +1046,7 @@ def _full_sort(self, trans_table, trans_costs=None, conn_lists[col_name][sc_gid] = data_arr[i] if total_lcoe_fric is not None: - conn_lists['total_lcoe_friction'][sc_gid] = \ + conn_lists[MetaKeyName.TOTAL_LCOE_FRICTION][sc_gid] = \ total_lcoe_fric[i] current_prog = connected // (len(self) / 100) @@ -1063,12 +1062,12 @@ def _full_sort(self, trans_table, trans_costs=None, index = range(0, int(1 + np.max(self._sc_gids))) connections = pd.DataFrame(conn_lists, index=index) - connections.index.name = 'sc_gid' + connections.index.name = MetaKeyName.SC_GID connections = connections.dropna(subset=[sort_on]) connections = connections[columns].reset_index() - sc_gids = self._sc_points['sc_gid'].values - connected = connections['sc_gid'].values + sc_gids = self._sc_points[MetaKeyName.SC_GID].values + connected = connections[MetaKeyName.SC_GID].values logger.debug('Connected gids {} out of total supply curve gids {}' .format(len(connected), len(sc_gids))) unconnected = ~np.isin(sc_gids, connected) @@ -1081,7 +1080,7 @@ def _full_sort(self, trans_table, trans_costs=None, logger.warning(msg) warn(msg) - supply_curve = self._sc_points.merge(connections, on='sc_gid') + supply_curve = self._sc_points.merge(connections, on=MetaKeyName.SC_GID) return supply_curve.reset_index(drop=True) @@ -1096,15 +1095,15 @@ def _check_feature_capacity(self, avail_cap_frac=1): self._trans_table = self._trans_table.merge(fc, on='trans_gid') def _adjust_output_columns(self, columns, consider_friction): - """Add extra output columns, if needed. """ + """Add extra output columns, if needed.""" # These are essentially should-be-defaults that are not # backwards-compatible, so have to explicitly check for them extra_cols = ['ba_str', 'poi_lat', 'poi_lon', 'reinforcement_poi_lat', 'reinforcement_poi_lon', 'eos_mult', 'reg_mult', 'reinforcement_cost_per_mw', 'reinforcement_dist_km', - 'n_parallel_trans', 'total_lcoe_friction'] + 'n_parallel_trans', MetaKeyName.TOTAL_LCOE_FRICTION] if not consider_friction: - extra_cols -= {'total_lcoe_friction'} + extra_cols -= {MetaKeyName.TOTAL_LCOE_FRICTION} extra_cols = [col for col in extra_cols if col in self._trans_table and col not in columns] @@ -1201,8 +1200,8 @@ def full_sort(self, fcr, transmission_costs=None, trans_table = trans_table.loc[~pos].sort_values([sort_on, 'trans_gid']) total_lcoe_fric = None - if consider_friction and 'mean_lcoe_friction' in trans_table: - total_lcoe_fric = trans_table['total_lcoe_friction'].values + if consider_friction and MetaKeyName.MEAN_LCOE_FRICTION in trans_table: + total_lcoe_fric = trans_table[MetaKeyName.TOTAL_LCOE_FRICTION].values comp_wind_dirs = None if wind_dirs is not None: @@ -1307,13 +1306,13 @@ def simple_sort(self, fcr, transmission_costs=None, sort_on = self._determine_sort_on(sort_on) connections = trans_table.sort_values([sort_on, 'trans_gid']) - connections = connections.groupby('sc_gid').first() + connections = connections.groupby(MetaKeyName.SC_GID).first() rename = {'trans_gid': 'trans_gid', 'category': 'trans_type'} connections = connections.rename(columns=rename) connections = connections[columns].reset_index() - supply_curve = self._sc_points.merge(connections, on='sc_gid') + supply_curve = self._sc_points.merge(connections, on=MetaKeyName.SC_GID) if wind_dirs is not None: supply_curve = \ CompetitiveWindFarms.run(wind_dirs, diff --git a/reV/supply_curve/tech_mapping.py b/reV/supply_curve/tech_mapping.py index 5dabb88c1..46326a876 100644 --- a/reV/supply_curve/tech_mapping.py +++ b/reV/supply_curve/tech_mapping.py @@ -8,21 +8,22 @@ @author: gbuster """ -from concurrent.futures import as_completed -import h5py import logging -from math import ceil -import numpy as np import os -from scipy.spatial import cKDTree +from concurrent.futures import as_completed +from math import ceil from warnings import warn -from reV.supply_curve.extent import SupplyCurveExtent -from reV.utilities.exceptions import FileInputWarning, FileInputError - +import h5py +import numpy as np from rex.resource import Resource from rex.utilities.execution import SpawnProcessPool from rex.utilities.utilities import res_dist_threshold +from scipy.spatial import cKDTree + +from reV.supply_curve.extent import SupplyCurveExtent +from reV.utilities import MetaKeyName +from reV.utilities.exceptions import FileInputError, FileInputWarning logger = logging.getLogger(__name__) @@ -175,7 +176,7 @@ def _get_excl_slices(gid, sc_row_indices, sc_col_indices, excl_row_slices, @classmethod def _get_excl_coords(cls, excl_fpath, gids, sc_row_indices, sc_col_indices, excl_row_slices, excl_col_slices, - coord_labels=('latitude', 'longitude')): + coord_labels=(MetaKeyName.LATITUDE, MetaKeyName.LONGITUDE)): """ Extract the exclusion coordinates for teh desired gids for TechMapping. @@ -339,7 +340,7 @@ def save_tech_map(excl_fpath, dset, indices, distance_threshold=None, def _check_fout(self): """Check the TechMapping output file for cached data.""" with h5py.File(self._excl_fpath, 'r') as f: - if 'latitude' not in f or 'longitude' not in f: + if MetaKeyName.LATITUDE not in f or MetaKeyName.LONGITUDE not in f: emsg = ('Datasets "latitude" and/or "longitude" not in ' 'pre-existing Exclusions TechMapping file "{}". ' 'Cannot proceed.' diff --git a/reV/utilities/pytest_utils.py b/reV/utilities/pytest_utils.py index 3ba318a41..c95294944 100644 --- a/reV/utilities/pytest_utils.py +++ b/reV/utilities/pytest_utils.py @@ -1,8 +1,9 @@ # -*- coding: utf-8 -*- """Functions used for pytests""" -import numpy as np import os + +import numpy as np import pandas as pd from packaging import version from rex.outputs import Outputs as RexOutputs @@ -89,9 +90,9 @@ def make_fake_h5_chunks(td, features, shuffle=False): for i, s1 in enumerate(s_slices): for j, s2 in enumerate(s_slices): out_file = out_pattern.format(i=i, j=j) - meta = pd.DataFrame({'latitude': lat[s1, s2].flatten(), - 'longitude': lon[s1, s2].flatten(), - 'gid': gids[s1, s2].flatten()}) + meta = pd.DataFrame({MetaKeyName.LATITUDE: lat[s1, s2].flatten(), + MetaKeyName.LONGITUDE: lon[s1, s2].flatten(), + MetaKeyName.GID: gids[s1, s2].flatten()}) write_chunk(meta=meta, times=times, data=data[s1, s2], features=features, out_file=out_file) diff --git a/tests/hsds.py b/tests/hsds.py index 51474781d..d06780dcc 100644 --- a/tests/hsds.py +++ b/tests/hsds.py @@ -5,12 +5,14 @@ https://github.com/NREL/hsds-examples """ from datetime import datetime + import numpy as np import pandas as pd import pytest - from rex.renewable_resource import NSRDB +from reV.utilities import MetaKeyName + @pytest.fixture def NSRDB_hsds(): @@ -54,7 +56,7 @@ def check_meta(res_cls): assert isinstance(meta, pd.DataFrame) assert meta.shape == (len(sites), meta_shape[1]) # select columns - meta = res_cls['meta', :, ['latitude', 'longitude']] + meta = res_cls['meta', :, [MetaKeyName.LATITUDE, MetaKeyName.LONGITUDE]] assert isinstance(meta, pd.DataFrame) assert meta.shape == (meta_shape[0], 2) @@ -160,6 +162,7 @@ class TestNSRDB: """ NSRDB Resource handler tests """ + @staticmethod def test_res(NSRDB_hsds): """ diff --git a/tests/test_bespoke.py b/tests/test_bespoke.py index 6c17a6698..70052b7de 100644 --- a/tests/test_bespoke.py +++ b/tests/test_bespoke.py @@ -2,31 +2,31 @@ """reV bespoke wind plant optimization tests """ import copy -from glob import glob import json import os import shutil import tempfile +import traceback +from glob import glob + +import h5py import numpy as np import pandas as pd import pytest -import h5py -import traceback - from gaps.collection import Collector +from rex import Resource + from reV import TESTDATADIR -from reV.cli import main from reV.bespoke.bespoke import BespokeSinglePlant, BespokeWindPlants from reV.bespoke.place_turbines import PlaceTurbines +from reV.cli import main from reV.handlers.outputs import Outputs -from reV.supply_curve.tech_mapping import TechMapping -from reV.supply_curve.supply_curve import SupplyCurve -from reV.SAM.generation import WindPower from reV.losses.power_curve import PowerCurveLossesMixin from reV.losses.scheduled import ScheduledLossesMixin -from reV.utilities import ModuleName - -from rex import Resource +from reV.SAM.generation import WindPower +from reV.supply_curve.supply_curve import SupplyCurve +from reV.supply_curve.tech_mapping import TechMapping +from reV.utilities import MetaKeyName, ModuleName pytest.importorskip("shapely") @@ -54,7 +54,7 @@ 'ri_reeds_regions': {'inclusion_range': (None, 400), 'exclude_nodata': False}} -with open(SAM, 'r') as f: +with open(SAM) as f: SAM_SYS_INPUTS = json.load(f) SAM_SYS_INPUTS['wind_farm_wake_model'] = 2 @@ -74,7 +74,7 @@ def test_turbine_placement(gid=33): - """Test turbine placement with zero available area. """ + """Test turbine placement with zero available area.""" output_request = ('system_capacity', 'cf_mean', 'cf_profile') with tempfile.TemporaryDirectory() as td: res_fp = os.path.join(td, 'ri_100_wtk_{}.h5') @@ -151,7 +151,7 @@ def test_turbine_placement(gid=33): def test_zero_area(gid=33): - """Test turbine placement with zero available area. """ + """Test turbine placement with zero available area.""" output_request = ('system_capacity', 'cf_mean', 'cf_profile') objective_function = ( @@ -194,7 +194,7 @@ def test_zero_area(gid=33): def test_correct_turb_location(gid=33): - """Test turbine location is reported correctly. """ + """Test turbine location is reported correctly.""" output_request = ('system_capacity', 'cf_mean', 'cf_profile') objective_function = ( @@ -237,7 +237,7 @@ def test_correct_turb_location(gid=33): def test_packing_algorithm(gid=33): - """Test turbine placement with zero available area. """ + """Test turbine placement with zero available area.""" output_request = () cap_cost_fun = "" foc_fun = "" @@ -277,13 +277,13 @@ def test_packing_algorithm(gid=33): def test_bespoke_points(): """Test the bespoke points input options""" # pylint: disable=W0612 - points = pd.DataFrame({'gid': [33, 34, 35], 'config': ['default'] * 3}) + points = pd.DataFrame({MetaKeyName.GID: [33, 34, 35], 'config': ['default'] * 3}) pp = BespokeWindPlants._parse_points(points, {'default': SAM}) assert len(pp) == 3 for gid in pp.gids: assert pp[gid][0] == 'default' - points = pd.DataFrame({'gid': [33, 34, 35]}) + points = pd.DataFrame({MetaKeyName.GID: [33, 34, 35]}) pp = BespokeWindPlants._parse_points(points, {'default': SAM}) assert len(pp) == 3 assert 'config' in pp.df.columns @@ -326,8 +326,8 @@ def test_single(gid=33): assert (TURB_RATING * bsp.meta['n_turbines'].values[0] == out['system_capacity']) - x_coords = json.loads(bsp.meta['turbine_x_coords'].values[0]) - y_coords = json.loads(bsp.meta['turbine_y_coords'].values[0]) + x_coords = json.loads(bsp.meta[MetaKeyName.TURBINE_X_COORDS].values[0]) + y_coords = json.loads(bsp.meta[MetaKeyName.TURBINE_Y_COORDS].values[0]) assert bsp.meta['n_turbines'].values[0] == len(x_coords) assert bsp.meta['n_turbines'].values[0] == len(y_coords) @@ -417,9 +417,9 @@ def test_extra_outputs(gid=33): assert 'lcoe_fcr-2013' in out assert 'lcoe_fcr-means' in out - assert 'capacity' in bsp.meta - assert 'mean_cf' in bsp.meta - assert 'mean_lcoe' in bsp.meta + assert MetaKeyName.CAPACITY in bsp.meta + assert MetaKeyName.MEAN_CF in bsp.meta + assert MetaKeyName.MEAN_LCOE in bsp.meta assert 'pct_slope' in bsp.meta assert 'reeds_region' in bsp.meta @@ -451,9 +451,9 @@ def test_extra_outputs(gid=33): assert 'lcoe_fcr-2013' in out assert 'lcoe_fcr-means' in out - assert 'capacity' in bsp.meta - assert 'mean_cf' in bsp.meta - assert 'mean_lcoe' in bsp.meta + assert MetaKeyName.CAPACITY in bsp.meta + assert MetaKeyName.MEAN_CF in bsp.meta + assert MetaKeyName.MEAN_LCOE in bsp.meta assert 'pct_slope' in bsp.meta assert 'reeds_region' in bsp.meta @@ -492,9 +492,9 @@ def test_bespoke(): shutil.copy(RES.format(2013), res_fp.format(2013)) res_fp = res_fp.format('*') # both 33 and 35 are included, 37 is fully excluded - points = pd.DataFrame({'gid': [33, 35], 'config': ['default'] * 2, + points = pd.DataFrame({MetaKeyName.GID: [33, 35], 'config': ['default'] * 2, 'extra_unused_data': [0, 42]}) - fully_excluded_points = pd.DataFrame({'gid': [37], + fully_excluded_points = pd.DataFrame({MetaKeyName.GID: [37], 'config': ['default'], 'extra_unused_data': [0]}) @@ -525,12 +525,12 @@ def test_bespoke(): with Resource(out_fpath_truth) as f: meta = f.meta assert len(meta) <= len(points) - assert 'sc_point_gid' in meta - assert 'turbine_x_coords' in meta - assert 'turbine_y_coords' in meta + assert MetaKeyName.SC_POINT_GID in meta + assert MetaKeyName.TURBINE_X_COORDS in meta + assert MetaKeyName.TURBINE_Y_COORDS in meta assert 'possible_x_coords' in meta assert 'possible_y_coords' in meta - assert 'res_gids' in meta + assert MetaKeyName.RES_GIDS in meta dsets_1d = ('system_capacity', 'cf_mean-2012', 'annual_energy-2012', 'cf_mean-means', @@ -568,7 +568,7 @@ def test_bespoke(): def test_collect_bespoke(): - """Test the collection of multiple chunked bespoke files. """ + """Test the collection of multiple chunked bespoke files.""" with tempfile.TemporaryDirectory() as td: source_dir = os.path.join(TESTDATADIR, 'bespoke/') source_pattern = source_dir + '/test_bespoke*.h5' @@ -581,7 +581,7 @@ def test_collect_bespoke(): with Resource(h5_file) as fout: meta = fout.meta - assert all(meta['gid'].values == sorted(meta['gid'].values)) + assert all(meta[MetaKeyName.GID].values == sorted(meta[MetaKeyName.GID].values)) ti = fout.time_index assert len(ti) == 8760 assert 'time_index-2012' in fout @@ -590,10 +590,10 @@ def test_collect_bespoke(): for fp in source_fps: with Resource(fp) as source: - assert all(np.isin(source.meta['gid'].values, - meta['gid'].values)) - for isource, gid in enumerate(source.meta['gid'].values): - iout = np.where(meta['gid'].values == gid)[0] + assert all(np.isin(source.meta[MetaKeyName.GID].values, + meta[MetaKeyName.GID].values)) + for isource, gid in enumerate(source.meta[MetaKeyName.GID].values): + iout = np.where(meta[MetaKeyName.GID].values == gid)[0] truth = source['cf_profile-2012', :, isource].flatten() test = data[:, iout].flatten() assert np.allclose(truth, test) @@ -650,7 +650,7 @@ def test_bespoke_supply_curve(): del f['meta'] with Outputs(bespoke_sc_fp, mode='a') as f: bespoke_meta = normal_sc_points.copy() - bespoke_meta = bespoke_meta.drop('sc_gid', axis=1) + bespoke_meta = bespoke_meta.drop(MetaKeyName.SC_GID, axis=1) f.meta = bespoke_meta # this is basically copied from test_supply_curve_compute.py @@ -661,15 +661,15 @@ def test_bespoke_supply_curve(): sc = SupplyCurve(bespoke_sc_fp, trans_tables) sc_full = sc.full_sort(fcr=0.1, avail_cap_frac=0.1) - assert all(gid in sc_full['sc_gid'] - for gid in normal_sc_points['sc_gid']) + assert all(gid in sc_full[MetaKeyName.SC_GID] + for gid in normal_sc_points[MetaKeyName.SC_GID]) for _, inp_row in normal_sc_points.iterrows(): - sc_gid = inp_row['sc_gid'] - assert sc_gid in sc_full['sc_gid'] - test_ind = np.where(sc_full['sc_gid'] == sc_gid)[0] + sc_gid = inp_row[MetaKeyName.SC_GID] + assert sc_gid in sc_full[MetaKeyName.SC_GID] + test_ind = np.where(sc_full[MetaKeyName.SC_GID] == sc_gid)[0] assert len(test_ind) == 1 test_row = sc_full.iloc[test_ind] - assert test_row['total_lcoe'].values[0] > inp_row['mean_lcoe'] + assert test_row['total_lcoe'].values[0] > inp_row[MetaKeyName.MEAN_LCOE] fpath_baseline = os.path.join(TESTDATADIR, 'sc_out/sc_full_lc.csv') sc_baseline = pd.read_csv(fpath_baseline) @@ -678,7 +678,7 @@ def test_bespoke_supply_curve(): @pytest.mark.parametrize('wlm', [2, 100]) def test_wake_loss_multiplier(wlm): - """Test wake loss multiplier. """ + """Test wake loss multiplier.""" output_request = ('system_capacity', 'cf_mean', 'cf_profile') with tempfile.TemporaryDirectory() as td: res_fp = os.path.join(td, 'ri_100_wtk_{}.h5') @@ -743,7 +743,7 @@ def test_wake_loss_multiplier(wlm): def test_bespoke_wind_plant_with_power_curve_losses(): - """Test bespoke ``wind_plant`` with power curve losses. """ + """Test bespoke ``wind_plant`` with power curve losses.""" output_request = ('system_capacity', 'cf_mean', 'cf_profile') with tempfile.TemporaryDirectory() as td: res_fp = os.path.join(td, 'ri_100_wtk_{}.h5') @@ -807,7 +807,7 @@ def test_bespoke_wind_plant_with_power_curve_losses(): def test_bespoke_run_with_power_curve_losses(): - """Test bespoke run with power curve losses. """ + """Test bespoke run with power curve losses.""" output_request = ('system_capacity', 'cf_mean', 'cf_profile') with tempfile.TemporaryDirectory() as td: res_fp = os.path.join(td, 'ri_100_wtk_{}.h5') @@ -858,7 +858,7 @@ def test_bespoke_run_with_power_curve_losses(): def test_bespoke_run_with_scheduled_losses(): - """Test bespoke run with scheduled losses. """ + """Test bespoke run with scheduled losses.""" output_request = ('system_capacity', 'cf_mean', 'cf_profile') with tempfile.TemporaryDirectory() as td: res_fp = os.path.join(td, 'ri_100_wtk_{}.h5') @@ -918,7 +918,7 @@ def test_bespoke_run_with_scheduled_losses(): def test_bespoke_aep_is_zero_if_no_turbines_placed(): - """Test that bespoke aep output is zero if no turbines placed. """ + """Test that bespoke aep output is zero if no turbines placed.""" output_request = ('system_capacity', 'cf_mean', 'cf_profile') objective_function = 'aep' @@ -987,7 +987,7 @@ def test_bespoke_prior_run(): res_fp_2013 = res_fp.format('2013') # gids 33 and 35 are included, 37 is fully excluded - points = pd.DataFrame({'gid': [33], 'config': ['default'], + points = pd.DataFrame({MetaKeyName.GID: [33], 'config': ['default'], 'extra_unused_data': [42]}) TechMapping.run(excl_fp, RES.format(2012), dset=TM_DSET, max_workers=1) @@ -1022,8 +1022,8 @@ def test_bespoke_prior_run(): meta2 = f2.meta data2 = {k: f2[k] for k in f2.dsets} - cols = ['turbine_x_coords', 'turbine_y_coords', 'capacity', - 'n_gids', 'gid_counts', 'res_gids'] + cols = [MetaKeyName.TURBINE_X_COORDS, MetaKeyName.TURBINE_Y_COORDS, MetaKeyName.CAPACITY, + MetaKeyName.N_GIDS, MetaKeyName.GID_COUNTS, MetaKeyName.RES_GIDS] pd.testing.assert_frame_equal(meta1[cols], meta2[cols]) # multi-year means should not match the 2nd run with 2013 only. @@ -1055,10 +1055,10 @@ def test_gid_map(): res_fp_2013 = res_fp.format('2013') # gids 33 and 35 are included, 37 is fully excluded - points = pd.DataFrame({'gid': [33], 'config': ['default'], + points = pd.DataFrame({MetaKeyName.GID: [33], 'config': ['default'], 'extra_unused_data': [42]}) - gid_map = pd.DataFrame({'gid': [3, 4, 13, 12, 11, 10, 9]}) + gid_map = pd.DataFrame({MetaKeyName.GID: [3, 4, 13, 12, 11, 10, 9]}) new_gid = 50 gid_map['gid_map'] = new_gid fp_gid_map = os.path.join(td, 'gid_map.csv') @@ -1100,7 +1100,7 @@ def test_gid_map(): with Resource(res_fp_2013) as f3: ws = f3[f'windspeed_{hh}m', :, new_gid] - cols = ['n_gids', 'gid_counts', 'res_gids'] + cols = [MetaKeyName.N_GIDS, MetaKeyName.GID_COUNTS, MetaKeyName.RES_GIDS] pd.testing.assert_frame_equal(meta1[cols], meta2[cols]) assert not np.allclose(data1['cf_mean-2013'], data2['cf_mean-2013']) @@ -1137,12 +1137,12 @@ def test_bespoke_bias_correct(): res_fp_2013 = res_fp.format('2013') # gids 33 and 35 are included, 37 is fully excluded - points = pd.DataFrame({'gid': [33], 'config': ['default'], + points = pd.DataFrame({MetaKeyName.GID: [33], 'config': ['default'], 'extra_unused_data': [42]}) # intentionally leaving out WTK gid 13 which only has 5 included 90m # pixels in order to check that this is dynamically patched. - bias_correct = pd.DataFrame({'gid': [3, 4, 12, 11, 10, 9]}) + bias_correct = pd.DataFrame({MetaKeyName.GID: [3, 4, 12, 11, 10, 9]}) bias_correct['method'] = 'lin_ws' bias_correct['scalar'] = 0.5 fp_bc = os.path.join(td, 'bc.csv') @@ -1180,7 +1180,7 @@ def test_bespoke_bias_correct(): meta2 = f2.meta data2 = {k: f2[k] for k in f2.dsets} - cols = ['n_gids', 'gid_counts', 'res_gids'] + cols = [MetaKeyName.N_GIDS, MetaKeyName.GID_COUNTS, MetaKeyName.RES_GIDS] pd.testing.assert_frame_equal(meta1[cols], meta2[cols]) assert data1['cf_mean-2013'] * 0.5 > data2['cf_mean-2013'] @@ -1256,12 +1256,12 @@ def test_cli(runner, clear_loggers): with Resource(out_fpath) as f: meta = f.meta assert len(meta) == 2 - assert 'sc_point_gid' in meta - assert 'turbine_x_coords' in meta - assert 'turbine_y_coords' in meta + assert MetaKeyName.SC_POINT_GID in meta + assert MetaKeyName.TURBINE_X_COORDS in meta + assert MetaKeyName.TURBINE_Y_COORDS in meta assert 'possible_x_coords' in meta assert 'possible_y_coords' in meta - assert 'res_gids' in meta + assert MetaKeyName.RES_GIDS in meta dsets_1d = ('system_capacity', 'cf_mean-2012', 'annual_energy-2012', 'cf_mean-means', 'ws_mean') @@ -1298,7 +1298,7 @@ def test_bespoke_5min_sample(): shutil.copy(EXCL, excl_fp) res_fp = os.path.join(TESTDATADIR, 'wtk/wtk_2010_*m.h5') - points = pd.DataFrame({'gid': [33, 35], 'config': ['default'] * 2, + points = pd.DataFrame({MetaKeyName.GID: [33, 35], 'config': ['default'] * 2, 'extra_unused_data': [0, 42]}) sam_sys_inputs = copy.deepcopy(SAM_SYS_INPUTS) sam_sys_inputs['time_index_step'] = 12 @@ -1306,7 +1306,7 @@ def test_bespoke_5min_sample(): # hack techmap because 5min data only has 10 wind resource pixels with h5py.File(excl_fp, 'a') as excl_file: - arr = np.random.choice(10, size=excl_file['latitude'].shape) + arr = np.random.choice(10, size=excl_file[MetaKeyName.LATITUDE].shape) excl_file.create_dataset(name=tm_dset, data=arr) bsp = BespokeWindPlants(excl_fp, res_fp, tm_dset, OBJECTIVE_FUNCTION, diff --git a/tests/test_config.py b/tests/test_config.py index eff40d846..be0f8010e 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -7,21 +7,22 @@ @author: gbuster """ -import numpy as np import os -import pandas as pd -import pytest import tempfile +import numpy as np +import pandas as pd +import pytest from rex import Resource from rex.utilities.exceptions import ResourceRuntimeError from rex.utilities.utilities import safe_json_load +from reV import TESTDATADIR from reV.config.base_analysis_config import AnalysisConfig -from reV.config.project_points import ProjectPoints, PointsControl +from reV.config.project_points import PointsControl, ProjectPoints from reV.generation.generation import Gen from reV.SAM.SAM import RevPySam -from reV import TESTDATADIR +from reV.utilities import MetaKeyName from reV.utilities.exceptions import ConfigError @@ -106,8 +107,7 @@ def test_split_iter(): for i, pp_split in enumerate(pc): i0_nom = s + i * n i1_nom = s + i * n + n - if i1_nom >= e: - i1_nom = e + i1_nom = min(e, i1_nom) split = pp_split.project_points.df target = pp.df.iloc[i0_nom:i1_nom] @@ -121,7 +121,7 @@ def test_config_mapping(): fpp = os.path.join(TESTDATADIR, 'project_points/pp_offshore.csv') sam_files = {'onshore': os.path.join( TESTDATADIR, 'SAM/wind_gen_standard_losses_0.json'), - 'offshore': os.path.join( + MetaKeyName.OFFSHORE: os.path.join( TESTDATADIR, 'SAM/wind_gen_standard_losses_1.json')} df = pd.read_csv(fpp, index_col=0) pp = ProjectPoints(fpp, sam_files, 'windpower') @@ -139,7 +139,7 @@ def test_sam_config_kw_replace(): fpp = os.path.join(TESTDATADIR, 'project_points/pp_offshore.csv') sam_files = {'onshore': os.path.join( TESTDATADIR, 'SAM/wind_gen_standard_losses_0.json'), - 'offshore': os.path.join( + MetaKeyName.OFFSHORE: os.path.join( TESTDATADIR, 'SAM/wind_gen_standard_losses_1.json')} res_file = os.path.join(TESTDATADIR, 'wtk/ri_100_wtk_2012.h5') pp = ProjectPoints(fpp, sam_files, 'windpower') @@ -147,19 +147,19 @@ def test_sam_config_kw_replace(): gen = Gen('windpower', pp, sam_files, resource_file=res_file, sites_per_worker=100) config_on = gen.project_points.sam_inputs['onshore'] - config_of = gen.project_points.sam_inputs['offshore'] + config_of = gen.project_points.sam_inputs[MetaKeyName.OFFSHORE] assert 'turb_generic_loss' in config_on assert 'turb_generic_loss' in config_of pp_split = ProjectPoints.split(0, 10000, gen.project_points) config_on = pp_split.sam_inputs['onshore'] - config_of = pp_split.sam_inputs['offshore'] + config_of = pp_split.sam_inputs[MetaKeyName.OFFSHORE] assert 'turb_generic_loss' in config_on assert 'turb_generic_loss' in config_of pc_split = PointsControl.split(0, 10000, gen.project_points) config_on = pc_split.project_points.sam_inputs['onshore'] - config_of = pc_split.project_points.sam_inputs['offshore'] + config_of = pc_split.project_points.sam_inputs[MetaKeyName.OFFSHORE] assert 'turb_generic_loss' in config_on assert 'turb_generic_loss' in config_of @@ -168,8 +168,8 @@ def test_sam_config_kw_replace(): config = ipc.project_points.sam_inputs['onshore'] assert 'turb_generic_loss' in config - if 'offshore' in ipc.project_points.sam_inputs: - config = ipc.project_points.sam_inputs['offshore'] + if MetaKeyName.OFFSHORE in ipc.project_points.sam_inputs: + config = ipc.project_points.sam_inputs[MetaKeyName.OFFSHORE] assert 'turb_generic_loss' in config @@ -186,7 +186,7 @@ def test_regions(counties): baseline = meta.loc[meta['county'].isin(counties)].index.values.tolist() - regions = {c: 'county' for c in counties} + regions = dict.fromkeys(counties, 'county') pp = ProjectPoints.regions(regions, res_file, sam_files) assert sorted(baseline) == pp.sites @@ -207,7 +207,7 @@ def test_coords(sites): if not isinstance(gids, list): gids = [gids] - lat_lons = meta.loc[gids, ['latitude', 'longitude']].values + lat_lons = meta.loc[gids, [MetaKeyName.LATITUDE, MetaKeyName.LONGITUDE]].values pp = ProjectPoints.lat_lon_coords(lat_lons, res_file, sam_files) assert sorted(gids) == pp.sites @@ -226,7 +226,7 @@ def test_coords_from_file(): if not isinstance(gids, list): gids = [gids] - lat_lons = meta.loc[gids, ['latitude', 'longitude']] + lat_lons = meta.loc[gids, [MetaKeyName.LATITUDE, MetaKeyName.LONGITUDE]] with tempfile.TemporaryDirectory() as td: coords_fp = os.path.join(td, "lat_lon.csv") lat_lons.to_csv(coords_fp, index=False) @@ -245,7 +245,7 @@ def test_duplicate_coords(): with Resource(res_file) as f: meta = f.meta - duplicates = meta.loc[[2, 3, 3, 4], ['latitude', 'longitude']].values + duplicates = meta.loc[[2, 3, 3, 4], [MetaKeyName.LATITUDE, MetaKeyName.LONGITUDE]].values with pytest.raises(RuntimeError): ProjectPoints.lat_lon_coords(duplicates, res_file, sam_files) @@ -262,7 +262,7 @@ def test_sam_configs(): fpp = os.path.join(TESTDATADIR, 'project_points/pp_offshore.csv') sam_files = {'onshore': os.path.join( TESTDATADIR, 'SAM/wind_gen_standard_losses_0.json'), - 'offshore': os.path.join( + MetaKeyName.OFFSHORE: os.path.join( TESTDATADIR, 'SAM/wind_gen_standard_losses_1.json')} pp_json = ProjectPoints(fpp, sam_files, 'windpower') diff --git a/tests/test_curtailment.py b/tests/test_curtailment.py index aff02da18..39558f581 100644 --- a/tests/test_curtailment.py +++ b/tests/test_curtailment.py @@ -5,19 +5,21 @@ @author: gbuster """ -from copy import deepcopy import os +from copy import deepcopy + import numpy as np import pandas as pd import pytest -from reV.SAM.SAM import RevPySam -from reV.config.project_points import ProjectPoints +from rex.utilities import safe_json_load +from rex.utilities.solar_position import SolarPosition + from reV import TESTDATADIR -from reV.utilities.curtailment import curtail +from reV.config.project_points import ProjectPoints from reV.generation.generation import Gen - -from rex.utilities.solar_position import SolarPosition -from rex.utilities import safe_json_load +from reV.SAM.SAM import RevPySam +from reV.utilities import MetaKeyName +from reV.utilities.curtailment import curtail def get_curtailment(year, curt_fn='curtailment.json'): @@ -168,7 +170,7 @@ def test_res_curtailment(year, site): sza = SolarPosition( non_curtailed_res.time_index, - non_curtailed_res.meta[['latitude', 'longitude']].values).zenith + non_curtailed_res.meta[[MetaKeyName.LATITUDE, MetaKeyName.LONGITUDE]].values).zenith ti = non_curtailed_res.time_index diff --git a/tests/test_econ_of_scale.py b/tests/test_econ_of_scale.py index 272df05ed..00ebd4345 100644 --- a/tests/test_econ_of_scale.py +++ b/tests/test_econ_of_scale.py @@ -3,19 +3,21 @@ """ PyTest file for reV LCOE economies of scale """ -import h5py -import numpy as np -import pandas as pd -import pytest import os import shutil import tempfile +import h5py +import numpy as np +import pandas as pd +import pytest from rex import Resource -from reV.generation.generation import Gen + +from reV import TESTDATADIR from reV.econ.economies_of_scale import EconomiesOfScale +from reV.generation.generation import Gen from reV.supply_curve.sc_aggregation import SupplyCurveAggregation -from reV import TESTDATADIR +from reV.utilities import MetaKeyName EXCL = os.path.join(TESTDATADIR, 'ri_exclusions/ri_exclusions.h5') GEN = os.path.join(TESTDATADIR, 'gen_out/ri_my_pv_gen.h5') @@ -76,7 +78,7 @@ def test_lcoe_calc_simple(): true_lcoe = ((data['fcr'] * data['capital_cost'] + data['foc']) / (data['aep'] / 1000)) - data['mean_lcoe'] = true_lcoe + data[MetaKeyName.MEAN_LCOE] = true_lcoe eos = EconomiesOfScale(eqn, data) assert eos.raw_capital_cost == eos.scaled_capital_cost @@ -160,10 +162,10 @@ def test_econ_of_scale_baseline(): base_df = pd.read_csv(out_fp_base + ".csv") sc_df = pd.read_csv(out_fp_sc + ".csv") - assert np.allclose(base_df['mean_lcoe'], sc_df['mean_lcoe']) - assert (sc_df['capital_cost_scalar'] == 1).all() + assert np.allclose(base_df[MetaKeyName.MEAN_LCOE], sc_df[MetaKeyName.MEAN_LCOE]) + assert (sc_df[MetaKeyName.CAPITAL_COST_SCALAR] == 1).all() assert np.allclose(sc_df['mean_capital_cost'], - sc_df['scaled_capital_cost']) + sc_df[MetaKeyName.SCALED_CAPITAL_COST]) def test_sc_agg_econ_scale(): @@ -207,11 +209,11 @@ def test_sc_agg_econ_scale(): # check that econ of scale saved the raw lcoe and that it reduced all # of the mean lcoe values from baseline - assert np.allclose(sc_df['raw_lcoe'], base_df['mean_lcoe']) - assert all(sc_df['mean_lcoe'] < base_df['mean_lcoe']) + assert np.allclose(sc_df[MetaKeyName.RAW_LCOE], base_df[MetaKeyName.MEAN_LCOE]) + assert all(sc_df[MetaKeyName.MEAN_LCOE] < base_df[MetaKeyName.MEAN_LCOE]) aep = ((sc_df['mean_fixed_charge_rate'] * sc_df['mean_capital_cost'] - + sc_df['mean_fixed_operating_cost']) / sc_df['raw_lcoe']) + + sc_df['mean_fixed_operating_cost']) / sc_df[MetaKeyName.RAW_LCOE]) true_raw_lcoe = ((data['fixed_charge_rate'] * data['capital_cost'] + data['fixed_operating_cost']) @@ -226,19 +228,19 @@ def test_sc_agg_econ_scale(): + data['fixed_operating_cost']) / aep + data['variable_operating_cost']) - assert np.allclose(scalars, sc_df['capital_cost_scalar']) + assert np.allclose(scalars, sc_df[MetaKeyName.CAPITAL_COST_SCALAR]) assert np.allclose(scalars * sc_df['mean_capital_cost'], - sc_df['scaled_capital_cost']) + sc_df[MetaKeyName.SCALED_CAPITAL_COST]) - assert np.allclose(true_scaled_lcoe, sc_df['mean_lcoe']) - assert np.allclose(true_raw_lcoe, sc_df['raw_lcoe']) - sc_df = sc_df.sort_values('capacity') - assert all(sc_df['mean_lcoe'].diff()[1:] < 0) + assert np.allclose(true_scaled_lcoe, sc_df[MetaKeyName.MEAN_LCOE]) + assert np.allclose(true_raw_lcoe, sc_df[MetaKeyName.RAW_LCOE]) + sc_df = sc_df.sort_values(MetaKeyName.CAPACITY) + assert all(sc_df[MetaKeyName.MEAN_LCOE].diff()[1:] < 0) for i in sc_df.index.values: if sc_df.loc[i, 'scalars'] < 1: - assert sc_df.loc[i, 'mean_lcoe'] < sc_df.loc[i, 'raw_lcoe'] + assert sc_df.loc[i, MetaKeyName.MEAN_LCOE] < sc_df.loc[i, MetaKeyName.RAW_LCOE] else: - assert sc_df.loc[i, 'mean_lcoe'] >= sc_df.loc[i, 'raw_lcoe'] + assert sc_df.loc[i, MetaKeyName.MEAN_LCOE] >= sc_df.loc[i, MetaKeyName.RAW_LCOE] def execute_pytest(capture='all', flags='-rapP'): diff --git a/tests/test_econ_windbos.py b/tests/test_econ_windbos.py index a96cad8bb..b073f9cd4 100644 --- a/tests/test_econ_windbos.py +++ b/tests/test_econ_windbos.py @@ -214,7 +214,7 @@ def test_run_bos(points=slice(0, 5), max_workers=1): # get full file paths. sam_files = TESTDATADIR + '/SAM/i_singleowner_windbos.json' - site_data = pd.DataFrame({'gid': range(5), + site_data = pd.DataFrame({MetaKeyName.GID: range(5), 'sales_tax_basis': range(5)}) econ_outs = ('total_installed_cost', 'turbine_cost', 'sales_tax_cost', diff --git a/tests/test_gen_forecast.py b/tests/test_gen_forecast.py index a190aa02a..ff8d8e2a8 100644 --- a/tests/test_gen_forecast.py +++ b/tests/test_gen_forecast.py @@ -8,21 +8,22 @@ """ import os -import pytest -import pandas as pd -import numpy as np import shutil import tempfile -from reV.handlers.outputs import Outputs -from reV.generation.generation import Gen -from reV.config.project_points import ProjectPoints -from reV import TESTDATADIR -from reV.utilities.exceptions import SAMExecutionError - +import numpy as np +import pandas as pd +import pytest from rex import Resource from rex.utilities.utilities import mean_irrad +from reV import TESTDATADIR +from reV.config.project_points import ProjectPoints +from reV.generation.generation import Gen +from reV.handlers.outputs import Outputs +from reV.utilities import MetaKeyName +from reV.utilities.exceptions import SAMExecutionError + def test_forecast(): """Test several forecast features implemented in reV gen including @@ -37,14 +38,14 @@ def test_forecast(): with Outputs(res_file, mode='a') as f: meta = f.meta - meta = meta.drop(['timezone', 'elevation'], axis=1) + meta = meta.drop([MetaKeyName.TIMEZONE, MetaKeyName.ELEVATION], axis=1) del f._h5['meta'] f._meta = None f.meta = meta with Outputs(res_file, mode='r') as f: - assert 'timezone' not in f.meta - assert 'elevation' not in f.meta + assert MetaKeyName.TIMEZONE not in f.meta + assert MetaKeyName.ELEVATION not in f.meta with Resource(res_file) as res: ghi = res['ghi'] @@ -52,8 +53,8 @@ def test_forecast(): points = ProjectPoints(slice(0, 5), sam_files, 'pvwattsv7', res_file=res_file) output_request = ('cf_mean', 'ghi_mean') - site_data = pd.DataFrame({'gid': np.arange(5), 'timezone': -5, - 'elevation': 0}) + site_data = pd.DataFrame({MetaKeyName.GID: np.arange(5), MetaKeyName.TIMEZONE: -5, + MetaKeyName.ELEVATION: 0}) gid_map = {0: 20, 1: 20, 2: 50, 3: 51, 4: 51} # test that this raises an error with missing timezone diff --git a/tests/test_gen_pv.py b/tests/test_gen_pv.py index f282eb5cf..d55375845 100644 --- a/tests/test_gen_pv.py +++ b/tests/test_gen_pv.py @@ -9,19 +9,20 @@ """ import os -import h5py -import pytest -import pandas as pd -import numpy as np -import tempfile import shutil +import tempfile +import h5py +import numpy as np +import pandas as pd +import pytest from rex.utilities.exceptions import ResourceRuntimeError -from reV.utilities.exceptions import ConfigError, ExecutionError -from reV.generation.generation import Gen -from reV.config.project_points import ProjectPoints + from reV import TESTDATADIR +from reV.config.project_points import ProjectPoints +from reV.generation.generation import Gen from reV.handlers.outputs import Outputs +from reV.utilities.exceptions import ConfigError, ExecutionError RTOL = 0.0 ATOL = 0.04 @@ -410,7 +411,7 @@ def test_gen_input_mods(): gen.run(max_workers=1) for i in range(5): inputs = gen.project_points[i][1] - assert inputs['tilt'] == 'latitude' + assert inputs['tilt'] == MetaKeyName.LATITUDE def test_gen_input_pass_through(): @@ -447,7 +448,7 @@ def test_gen_pv_site_data(): sites_per_worker=1, output_request=output_request) baseline.run(max_workers=1) - site_data = pd.DataFrame({'gid': np.arange(2), + site_data = pd.DataFrame({MetaKeyName.GID: np.arange(2), 'losses': np.ones(2)}) test = Gen('pvwattsv7', rev2_points, sam_files, res_file, sites_per_worker=1, output_request=output_request, @@ -540,7 +541,7 @@ def test_detailed_pv_bifacial(): def test_pv_clearsky(): - """test basic clearsky functionality""" + """Test basic clearsky functionality""" year = 2012 rev2_points = slice(0, 3) res_file = TESTDATADIR + '/nsrdb/ri_100_nsrdb_{}.h5'.format(year) @@ -592,7 +593,7 @@ def test_irrad_bias_correct(): sites_per_worker=1, output_request=output_request) gen_base.run(max_workers=1) - bc_df = pd.DataFrame({'gid': np.arange(1, 10), 'method': 'lin_irrad', + bc_df = pd.DataFrame({MetaKeyName.GID: np.arange(1, 10), 'method': 'lin_irrad', 'scalar': 1, 'adder': 50}) gen = Gen('pvwattsv7', points, sam_files, res_file, sites_per_worker=1, output_request=output_request, @@ -609,7 +610,7 @@ def test_irrad_bias_correct(): mask = (gen_base.out['cf_profile'][:, 1:] <= gen.out['cf_profile'][:, 1:]) assert (mask.sum() / mask.size) > 0.99 - bc_df = pd.DataFrame({'gid': np.arange(100), 'method': 'lin_irrad', + bc_df = pd.DataFrame({MetaKeyName.GID: np.arange(100), 'method': 'lin_irrad', 'scalar': 1, 'adder': -1500}) gen = Gen('pvwattsv7', points, sam_files, res_file, sites_per_worker=1, output_request=output_request, bias_correct=bc_df) diff --git a/tests/test_gen_wind.py b/tests/test_gen_wind.py index 47e986ca6..72da38035 100644 --- a/tests/test_gen_wind.py +++ b/tests/test_gen_wind.py @@ -10,17 +10,18 @@ import os import shutil +import tempfile + import h5py -import pytest -import pandas as pd import numpy as np -import tempfile +import pandas as pd +import pytest +from rex import Outputs, Resource, WindResource -from reV.generation.generation import Gen -from reV.config.project_points import ProjectPoints from reV import TESTDATADIR - -from rex import Resource, WindResource, Outputs +from reV.config.project_points import ProjectPoints +from reV.generation.generation import Gen +from reV.utilities import MetaKeyName RTOL = 0 ATOL = 0.001 @@ -104,17 +105,17 @@ def test_wind_gen_slice(f_rev1_out, rev2_points, year, max_workers): msg = 'Wind cf_means results did not match reV 1.0 results!' assert np.allclose(gen_outs, cf_mean_list, rtol=RTOL, atol=ATOL), msg assert np.allclose(pp.sites, gen.meta.index.values), 'bad gen meta!' - assert np.allclose(pp.sites, gen.meta['gid'].values), 'bad gen meta!' + assert np.allclose(pp.sites, gen.meta[MetaKeyName.GID].values), 'bad gen meta!' - labels = ['latitude', 'longitude'] + labels = [MetaKeyName.LATITUDE, MetaKeyName.LONGITUDE] with Resource(res_file) as res: for i, (gen_gid, site_meta) in enumerate(gen.meta.iterrows()): - res_gid = site_meta['gid'] + res_gid = site_meta[MetaKeyName.GID] assert gen_gid == res_gid test_coords = site_meta[labels].values.astype(float) true_coords = res.meta.loc[res_gid, labels].values.astype(float) assert np.allclose(test_coords, true_coords) - assert site_meta['gid'] == res_gid + assert site_meta[MetaKeyName.GID] == res_gid @pytest.mark.parametrize('gid_map', @@ -157,8 +158,8 @@ def test_gid_map(gid_map): for key in output_request: assert np.allclose(map_test.out[key], write_gid_test.out[key]) - for map_test_gid, write_test_gid in zip(map_test.meta['gid'], - write_gid_test.meta['gid']): + for map_test_gid, write_test_gid in zip(map_test.meta[MetaKeyName.GID], + write_gid_test.meta[MetaKeyName.GID]): assert map_test_gid == gid_map[write_test_gid] if len(baseline.out['cf_mean']) == len(map_test.out['cf_mean']): @@ -176,21 +177,21 @@ def test_gid_map(gid_map): assert np.allclose(baseline.out[key][gen_gid_base], map_test.out[key][gen_gid_test]) - labels = ['latitude', 'longitude'] + labels = [MetaKeyName.LATITUDE, MetaKeyName.LONGITUDE] with Resource(res_file) as res: for i, (gen_gid, site_meta) in enumerate(baseline.meta.iterrows()): - res_gid = site_meta['gid'] + res_gid = site_meta[MetaKeyName.GID] test_coords = site_meta[labels].values.astype(float) true_coords = res.meta.loc[res_gid, labels].values.astype(float) assert np.allclose(test_coords, true_coords) - assert site_meta['gid'] == res_gid + assert site_meta[MetaKeyName.GID] == res_gid for i, (gen_gid, site_meta) in enumerate(map_test.meta.iterrows()): res_gid = gid_map[gen_gid] test_coords = site_meta[labels].values.astype(float) true_coords = res.meta.loc[res_gid, labels].values.astype(float) assert np.allclose(test_coords, true_coords) - assert site_meta['gid'] == res_gid + assert site_meta[MetaKeyName.GID] == res_gid def test_wind_gen_new_outputs(points=slice(0, 10), year=2012, max_workers=1): @@ -260,7 +261,7 @@ def test_wind_gen_site_data(points=slice(0, 5), year=2012, max_workers=1): sites_per_worker=3, output_request=output_request) baseline.run(max_workers=max_workers) - site_data = pd.DataFrame({'gid': np.arange(2), + site_data = pd.DataFrame({MetaKeyName.GID: np.arange(2), 'turb_generic_loss': np.zeros(2)}) test = Gen('windpower', points, sam_files, res_file, sites_per_worker=3, output_request=output_request, site_data=site_data) @@ -340,7 +341,7 @@ def test_wind_bias_correct(): gen_base.run(max_workers=1) outs_base = np.array(list(gen_base.out['cf_mean'])) - bc_df = pd.DataFrame({'gid': np.arange(100), 'method': 'lin_ws', + bc_df = pd.DataFrame({MetaKeyName.GID: np.arange(100), 'method': 'lin_ws', 'scalar': 1, 'adder': 2}) gen = Gen('windpower', points, sam_files, res_file, output_request=('cf_mean', 'cf_profile', 'ws_mean'), @@ -350,7 +351,7 @@ def test_wind_bias_correct(): assert all(outs_bc > outs_base) assert np.allclose(gen_base.out['ws_mean'] + 2, gen.out['ws_mean']) - bc_df = pd.DataFrame({'gid': np.arange(100), 'method': 'lin_ws', + bc_df = pd.DataFrame({MetaKeyName.GID: np.arange(100), 'method': 'lin_ws', 'scalar': 1, 'adder': -100}) gen = Gen('windpower', points, sam_files, res_file, output_request=('cf_mean', 'cf_profile', 'ws_mean'), diff --git a/tests/test_handlers_outputs.py b/tests/test_handlers_outputs.py index 135674ef2..f90790de4 100644 --- a/tests/test_handlers_outputs.py +++ b/tests/test_handlers_outputs.py @@ -2,23 +2,24 @@ """ PyTest file for reV LCOE economies of scale """ +import os +import tempfile + import h5py import numpy as np import pandas as pd import pytest -import os -import tempfile - -from reV.version import __version__ -from reV.handlers.outputs import Outputs from rex.utilities.utilities import pd_date_range +from reV.handlers.outputs import Outputs +from reV.utilities import MetaKeyName +from reV.version import __version__ arr1 = np.ones(100) arr2 = np.ones((8760, 100)) arr3 = np.ones((8760, 100), dtype=float) * 42.42 -meta = pd.DataFrame({'latitude': np.ones(100), - 'longitude': np.zeros(100)}) +meta = pd.DataFrame({MetaKeyName.LATITUDE: np.ones(100), + MetaKeyName.LONGITUDE: np.zeros(100)}) time_index = pd_date_range('20210101', '20220101', freq='1h', closed='right') diff --git a/tests/test_handlers_transmission.py b/tests/test_handlers_transmission.py index c421148e8..d65583cbd 100644 --- a/tests/test_handlers_transmission.py +++ b/tests/test_handlers_transmission.py @@ -3,11 +3,13 @@ Transmission Feature Tests """ import os + import pandas as pd import pytest from reV import TESTDATADIR from reV.handlers.transmission import TransmissionFeatures as TF +from reV.utilities import MetaKeyName TRANS_COSTS_1 = {'line_tie_in_cost': 200, 'line_cost': 1000, 'station_tie_in_cost': 50, 'center_tie_in_cost': 10, @@ -45,7 +47,7 @@ def trans_table(): return trans_table -@pytest.mark.parametrize(('i', 'trans_costs', 'distance', 'gid'), +@pytest.mark.parametrize(('i', 'trans_costs', 'distance', MetaKeyName.GID), ((1, TRANS_COSTS_1, 0, 43300), (2, TRANS_COSTS_2, 0, 43300), (1, TRANS_COSTS_1, 100, 43300), @@ -75,7 +77,7 @@ def test_cost_calculation(i, trans_costs, distance, gid, trans_table): assert true_cost == trans_cost -@pytest.mark.parametrize(('trans_costs', 'capacity', 'gid'), +@pytest.mark.parametrize(('trans_costs', MetaKeyName.CAPACITY, MetaKeyName.GID), ((TRANS_COSTS_1, 350, 43300), (TRANS_COSTS_2, 350, 43300), (TRANS_COSTS_1, 100, 43300), diff --git a/tests/test_hybrids.py b/tests/test_hybrids.py index f2ad5a049..344d10d5b 100644 --- a/tests/test_hybrids.py +++ b/tests/test_hybrids.py @@ -1,21 +1,20 @@ # -*- coding: utf-8 -*- """reV hybrids tests. """ +import json import os -import pytest -import numpy as np import tempfile -import json - -from reV.hybrids import Hybridization, HYBRID_METHODS -from reV.hybrids.hybrids import HybridsData, MERGE_COLUMN, OUTPUT_PROFILE_NAMES -from reV.utilities.exceptions import FileInputError, InputError, OutputWarning -from reV.utilities import ModuleName -from reV.cli import main -from reV import Outputs, TESTDATADIR +import numpy as np +import pytest from rex.resource import Resource +from reV import TESTDATADIR, Outputs +from reV.cli import main +from reV.hybrids import HYBRID_METHODS, Hybridization +from reV.hybrids.hybrids import MERGE_COLUMN, OUTPUT_PROFILE_NAMES, HybridsData +from reV.utilities import MetaKeyName, ModuleName +from reV.utilities.exceptions import FileInputError, InputError, OutputWarning SOLAR_FPATH = os.path.join( TESTDATADIR, 'rep_profiles_out', 'rep_profiles_solar.h5') @@ -26,22 +25,22 @@ SOLAR_FPATH_MULT = os.path.join( TESTDATADIR, 'rep_profiles_out', 'rep_profiles_solar_multiple.h5') with Resource(SOLAR_FPATH) as res: - SOLAR_SCPGIDS = set(res.meta['sc_point_gid']) + SOLAR_SCPGIDS = set(res.meta[MetaKeyName.SC_POINT_GID]) with Resource(WIND_FPATH) as res: - WIND_SCPGIDS = set(res.meta['sc_point_gid']) + WIND_SCPGIDS = set(res.meta[MetaKeyName.SC_POINT_GID]) def test_hybridization_profile_output_single_resource(): - """Test that the hybridization calculation is correct (1 resource). """ + """Test that the hybridization calculation is correct (1 resource).""" sc_point_gid = 40005 with Resource(SOLAR_FPATH) as res: solar_idx = np.where( - res.meta['sc_point_gid'] == sc_point_gid + res.meta[MetaKeyName.SC_POINT_GID] == sc_point_gid )[0][0] - solar_cap = res.meta.loc[solar_idx, 'capacity'] + solar_cap = res.meta.loc[solar_idx, MetaKeyName.CAPACITY] solar_test_profile = res['rep_profiles_0', :, solar_idx] weighted_solar = solar_cap * solar_test_profile @@ -50,7 +49,7 @@ def test_hybridization_profile_output_single_resource(): h.run() hp, hsp, hwp, = h.profiles.values() h_meta = h.hybrid_meta - h_idx = np.where(h_meta['sc_point_gid'] == sc_point_gid)[0][0] + h_idx = np.where(h_meta[MetaKeyName.SC_POINT_GID] == sc_point_gid)[0][0] assert np.allclose(hp[:, h_idx], weighted_solar) assert np.allclose(hsp[:, h_idx], weighted_solar) @@ -59,16 +58,16 @@ def test_hybridization_profile_output_single_resource(): def test_hybridization_profile_output_with_ratio_none(): - """Test that the hybridization calculation is correct (1 resource). """ + """Test that the hybridization calculation is correct (1 resource).""" sc_point_gid = 40005 with Resource(SOLAR_FPATH) as res: solar_idx = np.where( - res.meta['sc_point_gid'] == sc_point_gid + res.meta[MetaKeyName.SC_POINT_GID] == sc_point_gid )[0][0] - solar_cap = res.meta.loc[solar_idx, 'capacity'] + solar_cap = res.meta.loc[solar_idx, MetaKeyName.CAPACITY] solar_test_profile = res['rep_profiles_0', :, solar_idx] weighted_solar = solar_cap * solar_test_profile @@ -80,7 +79,7 @@ def test_hybridization_profile_output_with_ratio_none(): h.run() hp, hsp, hwp, = h.profiles.values() h_meta = h.hybrid_meta - h_idx = np.where(h_meta['sc_point_gid'] == sc_point_gid)[0][0] + h_idx = np.where(h_meta[MetaKeyName.SC_POINT_GID] == sc_point_gid)[0][0] assert np.allclose(hp[:, h_idx], weighted_solar) assert np.allclose(hsp[:, h_idx], weighted_solar) @@ -89,21 +88,21 @@ def test_hybridization_profile_output_with_ratio_none(): def test_hybridization_profile_output(): - """Test that the hybridization calculation is correct. """ + """Test that the hybridization calculation is correct.""" common_sc_point_gid = 38883 with Resource(SOLAR_FPATH) as res: solar_idx = np.where( - res.meta['sc_point_gid'] == common_sc_point_gid + res.meta[MetaKeyName.SC_POINT_GID] == common_sc_point_gid )[0][0] - solar_cap = res.meta.loc[solar_idx, 'capacity'] + solar_cap = res.meta.loc[solar_idx, MetaKeyName.CAPACITY] solar_test_profile = res['rep_profiles_0', :, solar_idx] with Resource(WIND_FPATH) as res: wind_idx = np.where( - res.meta['sc_point_gid'] == common_sc_point_gid + res.meta[MetaKeyName.SC_POINT_GID] == common_sc_point_gid )[0][0] - wind_cap = res.meta.loc[wind_idx, 'capacity'] + wind_cap = res.meta.loc[wind_idx, MetaKeyName.CAPACITY] wind_test_profile = res['rep_profiles_0', :, wind_idx] weighted_solar = solar_cap * solar_test_profile @@ -113,7 +112,7 @@ def test_hybridization_profile_output(): h.run() hp, hsp, hwp, = h.profiles.values() h_meta = h.hybrid_meta - h_idx = np.where(h_meta['sc_point_gid'] == common_sc_point_gid)[0][0] + h_idx = np.where(h_meta[MetaKeyName.SC_POINT_GID] == common_sc_point_gid)[0][0] assert np.allclose(hp[:, h_idx], weighted_solar + weighted_wind) assert np.allclose(hsp[:, h_idx], weighted_solar) @@ -123,7 +122,7 @@ def test_hybridization_profile_output(): @pytest.mark.parametrize("input_files", [(SOLAR_FPATH, WIND_FPATH), (SOLAR_FPATH_30_MIN, WIND_FPATH)]) def test_hybridization_output_shapes(input_files): - """Test that the output shapes are as expected. """ + """Test that the output shapes are as expected.""" sfp, wfp = input_files h = Hybridization(sfp, wfp) @@ -169,11 +168,11 @@ def test_meta_hybridization(input_combination, expected_shape, overlap): ) h.run() assert h.hybrid_meta.shape == expected_shape - assert set(h.hybrid_meta['sc_point_gid']) == overlap + assert set(h.hybrid_meta[MetaKeyName.SC_POINT_GID]) == overlap def test_limits_and_ratios_output_values(): - """Test that limits and ratios are properly applied in succession. """ + """Test that limits and ratios are properly applied in succession.""" limits = {'solar_capacity': 50, 'wind_capacity': 0.5} ratio_numerator = 'solar_capacity' @@ -212,7 +211,7 @@ def test_limits_and_ratios_output_values(): ((0.3, 3.6), (0.3 - 1e6, 3.6 + 1e6)) ]) def test_ratios_input(ratio_cols, ratio_bounds, bounds): - """Test that the hybrid meta limits the ratio columns correctly. """ + """Test that the hybrid meta limits the ratio columns correctly.""" ratio_numerator, ratio_denominator = ratio_cols ratio = '{}/{}'.format(ratio_numerator, ratio_denominator) h = Hybridization( @@ -231,7 +230,7 @@ def test_ratios_input(ratio_cols, ratio_bounds, bounds): assert np.all(h.hybrid_meta['hybrid_{}'.format(ratio_denominator)] <= h.hybrid_meta[ratio_denominator]) - if 'capacity' in ratio: + if MetaKeyName.CAPACITY in ratio: max_solar_capacities = h.hybrid_meta['hybrid_solar_capacity'] max_solar_capacities = max_solar_capacities.values.reshape(1, -1) assert np.all(h.profiles['hybrid_solar_profile'] @@ -243,7 +242,7 @@ def test_ratios_input(ratio_cols, ratio_bounds, bounds): def test_rep_profile_idx_map(): - """Test that rep profile index mappings are correct shape. """ + """Test that rep profile index mappings are correct shape.""" h = Hybridization(SOLAR_FPATH, WIND_FPATH, allow_wind_only=True) for h_idxs, r_idxs in (h.meta_hybridizer.solar_profile_indices_map, @@ -262,7 +261,7 @@ def test_rep_profile_idx_map(): def test_limits_values(): - """Test that column values are properly limited on user input. """ + """Test that column values are properly limited on user input.""" limits = {'solar_capacity': 100, 'wind_capacity': 0.5} @@ -274,7 +273,7 @@ def test_limits_values(): def test_invalid_limits_column_name(): - """Test invalid inputs for limits columns. """ + """Test invalid inputs for limits columns.""" test_limits = {'un_prefixed_col': 0, 'wind_capacity': 10} with pytest.raises(InputError) as excinfo: @@ -285,7 +284,7 @@ def test_invalid_limits_column_name(): def test_fillna_values(): - """Test that N/A values are filled properly based on user input. """ + """Test that N/A values are filled properly based on user input.""" fill_vals = {'solar_n_gids': 0, 'wind_capacity': -1} @@ -302,7 +301,7 @@ def test_fillna_values(): def test_invalid_fillna_column_name(): - """Test invalid inputs for fillna columns. """ + """Test invalid inputs for fillna columns.""" test_fillna = {'un_prefixed_col': 0, 'wind_capacity': 10} with pytest.raises(InputError) as excinfo: @@ -318,7 +317,7 @@ def test_invalid_fillna_column_name(): ((False, True), (True, False)), ((True, True), (True, True))]) def test_all_allow_solar_allow_wind_combinations(input_combination, na_vals): - """Test that "allow_x_only" options perform the intended merges. """ + """Test that "allow_x_only" options perform the intended merges.""" allow_solar_only, allow_wind_only = input_combination h = Hybridization( @@ -337,7 +336,7 @@ def test_all_allow_solar_allow_wind_combinations(input_combination, na_vals): def test_warning_for_improper_data_output_from_hybrid_method(): - """Test that hybrid function with incorrect output throws warning. """ + """Test that hybrid function with incorrect output throws warning.""" def some_new_hybrid_func(__): return [0] @@ -355,10 +354,10 @@ def some_new_hybrid_func(__): def test_hybrid_col_additional_method(): - """Test that function decorated with 'hybrid_col' adds to hybrid meta. """ + """Test that function decorated with 'hybrid_col' adds to hybrid meta.""" def some_new_hybrid_func(h): - return h.hybrid_meta['elevation'] * 1000 + return h.hybrid_meta[MetaKeyName.ELEVATION] * 1000 HYBRID_METHODS['scaled_elevation'] = some_new_hybrid_func h = Hybridization(SOLAR_FPATH, WIND_FPATH) @@ -366,14 +365,14 @@ def some_new_hybrid_func(h): assert 'scaled_elevation' in HYBRID_METHODS assert 'scaled_elevation' in h.hybrid_meta.columns - assert np.allclose(h.hybrid_meta['elevation'] * 1000, + assert np.allclose(h.hybrid_meta[MetaKeyName.ELEVATION] * 1000, h.hybrid_meta['scaled_elevation']) HYBRID_METHODS.pop('scaled_elevation') def test_duplicate_lat_long_values(): - """Test duplicate lat/long values corresponding to unique merge column. """ + """Test duplicate lat/long values corresponding to unique merge column.""" with tempfile.TemporaryDirectory() as td: fout_solar = os.path.join(td, 'rep_profiles_solar.h5') @@ -387,7 +386,7 @@ def test_duplicate_lat_long_values(): def test_invalid_ratio_bounds_length_input(): - """Test improper ratios input. """ + """Test improper ratios input.""" ratio = 'solar_capacity/wind_capacity' with pytest.raises(InputError) as excinfo: @@ -401,7 +400,7 @@ def test_invalid_ratio_bounds_length_input(): def test_ratio_column_missing(): - """Test missing ratio column. """ + """Test missing ratio column.""" ratio = 'solar_col_dne/wind_capacity' with pytest.raises(FileInputError) as excinfo: @@ -415,7 +414,7 @@ def test_ratio_column_missing(): @pytest.mark.parametrize("ratio", [None, ('solar_capacity', 'wind_capacity')]) def test_ratio_not_string(ratio): - """Test ratio input is not string. """ + """Test ratio input is not string.""" with pytest.raises(InputError) as excinfo: Hybridization( @@ -432,7 +431,7 @@ def test_ratio_not_string(ratio): 'solar_capacity/wind_capacity/solar_capacity'] ) def test_invalid_ratio_format(ratio): - """Test ratio input is not string. """ + """Test ratio input is not string.""" with pytest.raises(InputError) as excinfo: Hybridization( @@ -446,7 +445,7 @@ def test_invalid_ratio_format(ratio): def test_invalid_ratio_column_name(): - """Test invalid inputs for ratio columns. """ + """Test invalid inputs for ratio columns.""" ratio = 'un_prefixed_col/wind_capacity' with pytest.raises(InputError) as excinfo: @@ -459,7 +458,7 @@ def test_invalid_ratio_column_name(): def test_no_overlap_in_merge_column_values(): - """Test duplicate values in merge column. """ + """Test duplicate values in merge column.""" with tempfile.TemporaryDirectory() as td: fout_solar = os.path.join(td, 'rep_profiles_solar.h5') @@ -474,7 +473,7 @@ def test_no_overlap_in_merge_column_values(): def test_duplicate_merge_column_values(): - """Test duplicate values in merge column. """ + """Test duplicate values in merge column.""" with tempfile.TemporaryDirectory() as td: fout_solar = os.path.join(td, 'rep_profiles_solar.h5') @@ -487,7 +486,7 @@ def test_duplicate_merge_column_values(): def test_merge_columns_missing(): - """Test missing merge column. """ + """Test missing merge column.""" with tempfile.TemporaryDirectory() as td: fout_solar = os.path.join(td, 'rep_profiles_solar.h5') @@ -502,7 +501,7 @@ def test_merge_columns_missing(): def test_invalid_num_profiles(): - """Test input files with an invalid number of profiles (>1). """ + """Test input files with an invalid number of profiles (>1).""" with pytest.raises(FileInputError) as excinfo: Hybridization(SOLAR_FPATH_MULT, WIND_FPATH) @@ -514,7 +513,7 @@ def test_invalid_num_profiles(): def test_invalid_time_index_overlap(): - """Test input files with an invalid time index overlap. """ + """Test input files with an invalid time index overlap.""" with tempfile.TemporaryDirectory() as td: fout_solar = os.path.join(td, 'rep_profiles_solar.h5') @@ -531,7 +530,7 @@ def test_invalid_time_index_overlap(): def test_valid_time_index_overlap(): - """Test input files with a valid time index overlap. """ + """Test input files with a valid time index overlap.""" h = Hybridization(SOLAR_FPATH_30_MIN, WIND_FPATH) @@ -565,7 +564,7 @@ def test_write_to_file(): def test_hybrids_data_content(): - """Test HybridsData class content. """ + """Test HybridsData class content.""" fv = -999 h_data = HybridsData(SOLAR_FPATH, WIND_FPATH) @@ -669,7 +668,7 @@ def test_hybrids_cli_from_config(runner, input_files, ratio, ratio_bounds, os.path.join(TESTDATADIR, 'rep_profiles_out', 'rep_profiles_dne.h5'), ]) def test_hybrids_cli_bad_fpath_input(runner, bad_fpath, clear_loggers): - """Test cli when filepath input is ambiguous or invalid. """ + """Test cli when filepath input is ambiguous or invalid.""" with tempfile.TemporaryDirectory() as td: config = { @@ -740,8 +739,8 @@ def make_test_file(in_fp, out_fp, p_slice=slice(None), t_slice=slice(None), half_n_rows = n_rows // 2 meta.iloc[-half_n_rows:] = meta.iloc[:half_n_rows].values if duplicate_coord_values: - meta.loc[0, 'latitude'] = meta['latitude'].iloc[-1] - meta.loc[0, 'latitude'] = meta['latitude'].iloc[-1] + meta.loc[0, MetaKeyName.LATITUDE] = meta[MetaKeyName.LATITUDE].iloc[-1] + meta.loc[0, MetaKeyName.LATITUDE] = meta[MetaKeyName.LATITUDE].iloc[-1] shapes['meta'] = len(meta) for d in dset_names: shapes[d] = (len(res.time_index[t_slice]), len(meta)) diff --git a/tests/test_losses_power_curve.py b/tests/test_losses_power_curve.py index c62414710..007998b71 100644 --- a/tests/test_losses_power_curve.py +++ b/tests/test_losses_power_curve.py @@ -7,27 +7,29 @@ @author: ppinchuk """ +import copy +import json import os -import pytest import tempfile -import json -import copy import numpy as np import pandas as pd +import pytest from reV import TESTDATADIR from reV.generation.generation import Gen -from reV.utilities.exceptions import reVLossesValueError, reVLossesWarning -from reV.losses.power_curve import (PowerCurve, PowerCurveLosses, - PowerCurveLossesMixin, - PowerCurveLossesInput, - TRANSFORMATIONS, - HorizontalTranslation, - AbstractPowerCurveTransformation, - ) +from reV.losses.power_curve import ( + TRANSFORMATIONS, + AbstractPowerCurveTransformation, + HorizontalTranslation, + PowerCurve, + PowerCurveLosses, + PowerCurveLossesInput, + PowerCurveLossesMixin, +) from reV.losses.scheduled import ScheduledLossesMixin - +from reV.utilities import MetaKeyName +from reV.utilities.exceptions import reVLossesValueError, reVLossesWarning REV_POINTS = list(range(3)) RES_FILE = TESTDATADIR + '/wtk/ri_100_wtk_2012.h5' @@ -55,7 +57,7 @@ def simple_power_curve(): @pytest.fixture def real_power_curve(): """Return a basic power curve.""" - with open(SAM_FILES[0], 'r') as fh: + with open(SAM_FILES[0]) as fh: sam_config = json.load(fh) wind_speed = sam_config['wind_turbine_powercurve_windspeeds'] @@ -67,7 +69,7 @@ def real_power_curve(): @pytest.mark.parametrize('target_losses', [0, 10, 50]) @pytest.mark.parametrize('transformation', TRANSFORMATIONS) def test_power_curve_losses(generic_losses, target_losses, transformation): - """Test full gen run with scheduled losses. """ + """Test full gen run with scheduled losses.""" gen_profiles, gen_profiles_with_losses = _run_gen_with_and_without_losses( generic_losses, target_losses=target_losses, @@ -88,7 +90,7 @@ def test_power_curve_losses(generic_losses, target_losses, transformation): @pytest.mark.parametrize('generic_losses', [0, 0.2]) def test_power_curve_losses_site_specific(generic_losses): - """Test full gen run with scheduled losses. """ + """Test full gen run with scheduled losses.""" gen_profiles, gen_profiles_with_losses = _run_gen_with_and_without_losses( generic_losses, target_losses=10, @@ -110,11 +112,11 @@ def _run_gen_with_and_without_losses( generic_losses, target_losses, transformation, include_outages=False, site_losses=None ): - """Run generation with and without losses for testing. """ + """Run generation with and without losses for testing.""" sam_file = SAM_FILES[0] - with open(sam_file, 'r', encoding='utf-8') as fh: + with open(sam_file, encoding='utf-8') as fh: sam_config = json.load(fh) with tempfile.TemporaryDirectory() as td: @@ -148,7 +150,7 @@ def _run_gen_with_and_without_losses( # undo UTC array rolling for ind, row in gen.meta.iterrows(): - time_shift = row['timezone'] + time_shift = row[MetaKeyName.TIMEZONE] gen_profiles_with_losses[:, ind] = np.roll( gen_profiles_with_losses[:, ind], time_shift ) @@ -167,18 +169,18 @@ def _run_gen_with_and_without_losses( gen_profiles = gen.out['gen_profile'] for ind, row in gen.meta.iterrows(): - time_shift = row['timezone'] + time_shift = row[MetaKeyName.TIMEZONE] gen_profiles[:, ind] = np.roll(gen_profiles[:, ind], time_shift) return gen_profiles, gen_profiles_with_losses def _make_site_data_df(site_data): - """Make site data DataFrame for a specific power curve loss input. """ + """Make site data DataFrame for a specific power curve loss input.""" if site_data is not None: site_specific_losses = [json.dumps(site_data)] * len(REV_POINTS) site_data_dict = { - 'gid': REV_POINTS, + MetaKeyName.GID: REV_POINTS, PowerCurveLossesMixin.POWER_CURVE_CONFIG_KEY: site_specific_losses } site_data = pd.DataFrame(site_data_dict) @@ -186,7 +188,7 @@ def _make_site_data_df(site_data): def test_power_curve_losses_witch_scheduled_outages(): - """Test full gen run with scheduled losses. """ + """Test full gen run with scheduled losses.""" gen_profiles, gen_profiles_with_losses = _run_gen_with_and_without_losses( generic_losses=0.2, target_losses=20, transformation='exponential_stretching', @@ -199,9 +201,9 @@ def test_power_curve_losses_witch_scheduled_outages(): @pytest.mark.parametrize('config', SAM_FILES) def test_power_curve_losses_mixin_class_add_power_curve_losses(config): - """Test mixin class behavior when adding losses. """ + """Test mixin class behavior when adding losses.""" - with open(config, 'r') as fh: + with open(config) as fh: sam_config = json.load(fh) og_power_curve = np.array(sam_config["wind_turbine_powercurve_powerout"]) @@ -236,7 +238,7 @@ def test_power_curve_losses_mixin_class_wind_resource_no_power(config, ws): """Test mixin class behavior when there's no power generation from wind resource always below the cutin speed or above the cutout speed. """ - with open(config, 'r') as fh: + with open(config) as fh: sam_config = json.load(fh) og_power_curve = np.array(sam_config["wind_turbine_powercurve_powerout"]) @@ -268,9 +270,9 @@ def get_item_patch(self, key): @pytest.mark.parametrize('config', SAM_FILES) def test_power_curve_losses_mixin_class_no_losses_input(config): - """Test mixin class behavior when no losses should be added. """ + """Test mixin class behavior when no losses should be added.""" - with open(config, 'r') as fh: + with open(config) as fh: sam_config = json.load(fh) og_power_curve = np.array(sam_config["wind_turbine_powercurve_powerout"]) @@ -288,7 +290,7 @@ def test_power_curve_losses_mixin_class_no_losses_input(config): @pytest.mark.parametrize('bad_wind_speed', ([], [-10, 10])) def test_power_curve_class_bad_wind_speed_input(bad_wind_speed): - """Test that error is raised for bad wind speed inputs. """ + """Test that error is raised for bad wind speed inputs.""" power_curve = [10, 100] with pytest.raises(reVLossesValueError) as excinfo: @@ -298,7 +300,7 @@ def test_power_curve_class_bad_wind_speed_input(bad_wind_speed): @pytest.mark.parametrize('bad_generation', ([], [0, 0, 0, 0], [0, 20, 0, 10])) def test_power_curve_class_bad_generation_input(bad_generation): - """Test that error is raised for bad generation inputs. """ + """Test that error is raised for bad generation inputs.""" wind_speed = [0, 10, 20, 30] with pytest.raises(reVLossesValueError) as excinfo: @@ -308,7 +310,7 @@ def test_power_curve_class_bad_generation_input(bad_generation): @pytest.mark.parametrize('bad_wind_res', ([], [-10, 10])) def test_power_curve_losses_class_bad_wind_res_input(bad_wind_res): - """Test that error is raised for bad wind resource inputs. """ + """Test that error is raised for bad wind resource inputs.""" wind_speed = [0, 10] generation = [10, 100] power_curve = PowerCurve(wind_speed, generation) @@ -319,7 +321,7 @@ def test_power_curve_losses_class_bad_wind_res_input(bad_wind_res): @pytest.mark.parametrize('bad_weights', ([], [1])) def test_power_curve_losses_class_bad_weights_input(bad_weights): - """Test that error is raised for bad weights input. """ + """Test that error is raised for bad weights input.""" wind_speed = [0, 10] generation = [10, 100] wind_res = [0, 0, 5] @@ -331,7 +333,7 @@ def test_power_curve_losses_class_bad_weights_input(bad_weights): @pytest.mark.parametrize('pc_transformation', TRANSFORMATIONS.values()) def test_transformation_classes_apply(pc_transformation, real_power_curve): - """Test that the power curve transformations are applied correctly. """ + """Test that the power curve transformations are applied correctly.""" real_power_curve.generation[-1] = real_power_curve.generation[-2] transformation = pc_transformation(real_power_curve) @@ -375,7 +377,7 @@ def test_horizontal_transformation_class_apply(real_power_curve): def test_power_curve_losses_class_annual_losses_with_transformed_power_curve(): - """Test that the average difference is calculated correctly. """ + """Test that the average difference is calculated correctly.""" wind_speed = [0, 10, 20, 30, 40] generation = [0, 10, 15, 20, 0] @@ -398,9 +400,9 @@ def test_power_curve_losses_class_annual_losses_with_transformed_power_curve(): @pytest.mark.parametrize('sam_file', SAM_FILES) @pytest.mark.parametrize('pc_transformation', TRANSFORMATIONS.values()) def test_transformation_classes_bounds(sam_file, pc_transformation): - """Test that shift_bounds are set correctly. """ + """Test that shift_bounds are set correctly.""" - with open(sam_file, 'r') as fh: + with open(sam_file) as fh: sam_config = json.load(fh) wind_speed = sam_config['wind_turbine_powercurve_windspeeds'] @@ -416,7 +418,7 @@ def test_transformation_classes_bounds(sam_file, pc_transformation): def test_transformation_invalid_result(real_power_curve): - """Test a transformation with invalid result. """ + """Test a transformation with invalid result.""" transformation = HorizontalTranslation(real_power_curve) with pytest.raises(reVLossesValueError) as excinfo: @@ -428,7 +430,7 @@ def test_transformation_invalid_result(real_power_curve): def test_power_curve_loss_input_class_valid_inputs(): - """Test PowerCurveLossesInput class with valid input. """ + """Test PowerCurveLossesInput class with valid input.""" specs = {'target_losses_percent': 50} pc_input = PowerCurveLossesInput(specs) @@ -442,7 +444,7 @@ def test_power_curve_loss_input_class_valid_inputs(): @pytest.mark.parametrize('bad_percent', [-10, 105]) def test_power_curve_loss_input_class_bad_percent_input(bad_percent): - """Test PowerCurveLossesInput class with bad percent input. """ + """Test PowerCurveLossesInput class with bad percent input.""" bad_specs = {'target_losses_percent': bad_percent} @@ -453,7 +455,7 @@ def test_power_curve_loss_input_class_bad_percent_input(bad_percent): def test_power_curve_loss_input_class_bad_transformation_input(): - """Test PowerCurveLossesInput class with bad transformation input. """ + """Test PowerCurveLossesInput class with bad transformation input.""" bad_specs = {'target_losses_percent': 50, 'transformation': 'DNE'} @@ -464,7 +466,7 @@ def test_power_curve_loss_input_class_bad_transformation_input(): def test_power_curve_loss_input_class_missing_required_keys(): - """Test PowerCurveLossesInput class with missing keys input. """ + """Test PowerCurveLossesInput class with missing keys input.""" with pytest.raises(reVLossesValueError) as excinfo: PowerCurveLossesInput({}) @@ -472,9 +474,9 @@ def test_power_curve_loss_input_class_missing_required_keys(): def test_power_curve_loss_invalid_pressure_values(): - """Test mixin class behavior when adding losses. """ + """Test mixin class behavior when adding losses.""" - with open(SAM_FILES[0], 'r') as fh: + with open(SAM_FILES[0]) as fh: sam_config = json.load(fh) # patch required for 'wind_resource_data' access below @@ -498,7 +500,7 @@ def get_item_patch(self, key): def test_power_curve_losses_class_power_gen_no_losses(simple_power_curve): - """Test that power_gen_no_losses is calculated correctly. """ + """Test that power_gen_no_losses is calculated correctly.""" pc_losses = PowerCurveLosses(simple_power_curve, BASIC_WIND_RES) @@ -509,7 +511,7 @@ def test_power_curve_losses_class_power_gen_no_losses(simple_power_curve): def test_power_curve_class_cutoff_wind_speed( simple_power_curve, real_power_curve ): - """Test that cutoff_wind_speed is calculated correctly. """ + """Test that cutoff_wind_speed is calculated correctly.""" assert simple_power_curve.cutoff_wind_speed == np.inf assert ( @@ -523,7 +525,7 @@ def test_power_curve_class_cutoff_wind_speed( def test_power_curve_class_comparisons(simple_power_curve): - """Test power curve class comparison and call operators. """ + """Test power curve class comparison and call operators.""" assert simple_power_curve == [0, 20, 15, 10] assert simple_power_curve != [0, 20, 15, 0] @@ -536,10 +538,11 @@ def test_power_curve_class_comparisons(simple_power_curve): def test_bad_transformation_implementation(real_power_curve): - """Test an invalid transformation implementation. """ + """Test an invalid transformation implementation.""" class NewTransformation(AbstractPowerCurveTransformation): """Test class""" + # pylint: disable=useless-super-delegation def apply(self, *args, **kwargs): """Test apply method.""" diff --git a/tests/test_losses_scheduled.py b/tests/test_losses_scheduled.py index 9171664d9..75846bf0f 100644 --- a/tests/test_losses_scheduled.py +++ b/tests/test_losses_scheduled.py @@ -7,29 +7,32 @@ @author: ppinchuk """ -import os -import pytest -import tempfile -import json -import random import copy import glob +import json +import os +import random +import tempfile import traceback import numpy as np import pandas as pd +import pytest +from rex.utilities.utilities import safe_json_load from reV import TESTDATADIR -from reV.generation.generation import Gen -from reV.utilities.exceptions import reVLossesValueError, reVLossesWarning -from reV.losses.utils import hourly_indices_for_months -from reV.losses.scheduled import (Outage, OutageScheduler, - SingleOutageScheduler, ScheduledLossesMixin) from reV.cli import main +from reV.generation.generation import Gen from reV.handlers.outputs import Outputs - -from rex.utilities.utilities import safe_json_load - +from reV.losses.scheduled import ( + Outage, + OutageScheduler, + ScheduledLossesMixin, + SingleOutageScheduler, +) +from reV.losses.utils import hourly_indices_for_months +from reV.utilities import MetaKeyName +from reV.utilities.exceptions import reVLossesValueError, reVLossesWarning REV_POINTS = list(range(3)) RTOL = 0 @@ -130,7 +133,7 @@ def so_scheduler(basic_outage_dict): (PV_SAM_FILE, PV_RES_FILE, 'pvwattsv7') ]) def test_scheduled_losses(generic_losses, outages, haf, files): - """Test full gen run with scheduled losses. """ + """Test full gen run with scheduled losses.""" gen_profiles, gen_profiles_with_losses = _run_gen_with_and_without_losses( generic_losses, outages, None, haf, files @@ -235,7 +238,7 @@ def test_scheduled_losses(generic_losses, outages, haf, files): (PV_SAM_FILE, PV_RES_FILE, 'pvwattsv7') ]) def test_scheduled_losses_site_specific(generic_losses, haf, files): - """Test full gen run with scheduled losses. """ + """Test full gen run with scheduled losses.""" gen_profiles, gen_profiles_with_losses = _run_gen_with_and_without_losses( generic_losses, NOMINAL_OUTAGES[0], SINGLE_SITE_OUTAGE, haf, files @@ -316,9 +319,9 @@ def test_scheduled_losses_site_specific(generic_losses, haf, files): def _run_gen_with_and_without_losses( generic_losses, outages, site_outages, haf, files ): - """Run generation with and without losses for testing. """ + """Run generation with and without losses for testing.""" sam_file, res_file, tech = files - with open(sam_file, 'r') as fh: + with open(sam_file) as fh: sam_config = json.load(fh) with tempfile.TemporaryDirectory() as td: @@ -347,7 +350,7 @@ def _run_gen_with_and_without_losses( gen_profiles_with_losses = gen_profiles_with_losses[::time_steps_in_hour] # undo UTC array rolling for ind, row in gen.meta.iterrows(): - time_shift = row['timezone'] + time_shift = row[MetaKeyName.TIMEZONE] gen_profiles_with_losses[:, ind] = np.roll( gen_profiles_with_losses[:, ind], time_shift ) @@ -372,18 +375,18 @@ def _run_gen_with_and_without_losses( time_steps_in_hour = int(round(gen_profiles.shape[0] / 8760)) gen_profiles = gen_profiles[::time_steps_in_hour] for ind, row in gen.meta.iterrows(): - time_shift = row['timezone'] + time_shift = row[MetaKeyName.TIMEZONE] gen_profiles[:, ind] = np.roll(gen_profiles[:, ind], time_shift) return gen_profiles, gen_profiles_with_losses def _make_site_data_df(site_data): - """Make site data DataFrame for a specific outage input. """ + """Make site data DataFrame for a specific outage input.""" if site_data is not None: site_specific_outages = [json.dumps(site_data)] * len(REV_POINTS) site_data_dict = { - 'gid': REV_POINTS, + MetaKeyName.GID: REV_POINTS, ScheduledLossesMixin.OUTAGE_CONFIG_KEY: site_specific_outages } site_data = pd.DataFrame(site_data_dict) @@ -401,9 +404,9 @@ def _make_site_data_df(site_data): def test_scheduled_losses_repeatability( generic_losses, outages, site_outages, files ): - """Test that losses are reproducible between runs. """ + """Test that losses are reproducible between runs.""" sam_file, res_file, tech = files - with open(sam_file, 'r') as fh: + with open(sam_file) as fh: sam_config = json.load(fh) with tempfile.TemporaryDirectory() as td: @@ -447,10 +450,10 @@ def test_scheduled_losses_repeatability( (PV_SAM_FILE, PV_RES_FILE, 'pvwattsv7') ]) def test_scheduled_losses_repeatability_with_seed(files): - """Test that losses are reproducible between runs. """ + """Test that losses are reproducible between runs.""" sam_file, res_file, tech = files outages = copy.deepcopy(NOMINAL_OUTAGES[0]) - with open(sam_file, 'r') as fh: + with open(sam_file) as fh: sam_config = json.load(fh) with tempfile.TemporaryDirectory() as td: @@ -506,7 +509,7 @@ def test_scheduled_losses_repeatability_with_seed(files): @pytest.mark.parametrize('outages', NOMINAL_OUTAGES) def test_scheduled_losses_mixin_class_add_scheduled_losses(outages): - """Test mixin class behavior when adding losses. """ + """Test mixin class behavior when adding losses.""" mixin = ScheduledLossesMixin() mixin.sam_sys_inputs = {mixin.OUTAGE_CONFIG_KEY: outages} @@ -518,7 +521,7 @@ def test_scheduled_losses_mixin_class_add_scheduled_losses(outages): def test_scheduled_losses_mixin_class_no_losses_input(): - """Test mixin class behavior when adding losses. """ + """Test mixin class behavior when adding losses.""" mixin = ScheduledLossesMixin() mixin.sam_sys_inputs = {} @@ -533,7 +536,7 @@ def test_scheduled_losses_mixin_class_no_losses_input(): def test_single_outage_scheduler_normal_run( allow_outage_overlap, so_scheduler ): - """Test that single outage is scheduled correctly. """ + """Test that single outage is scheduled correctly.""" so_scheduler.outage._specs['allow_outage_overlap'] = allow_outage_overlap outage = so_scheduler.outage @@ -562,7 +565,7 @@ def test_single_outage_scheduler_normal_run( def test_single_outage_scheduler_update_when_can_schedule_from_months( so_scheduler ): - """Test that single outage is scheduled correctly. """ + """Test that single outage is scheduled correctly.""" so_scheduler.update_when_can_schedule_from_months() @@ -571,7 +574,7 @@ def test_single_outage_scheduler_update_when_can_schedule_from_months( def test_single_outage_scheduler_update_when_can_schedule(so_scheduler): - """Test that single outage is scheduled correctly. """ + """Test that single outage is scheduled correctly.""" so_scheduler.update_when_can_schedule_from_months() @@ -585,7 +588,7 @@ def test_single_outage_scheduler_update_when_can_schedule(so_scheduler): def test_single_outage_scheduler_find_random_outage_slice(so_scheduler): - """Test single outage class method. """ + """Test single outage class method.""" so_scheduler.update_when_can_schedule_from_months() random_slice = so_scheduler.find_random_outage_slice() @@ -600,7 +603,7 @@ def test_single_outage_scheduler_find_random_outage_slice(so_scheduler): def test_single_outage_scheduler_schedule_losses( allow_outage_overlap, so_scheduler ): - """Test single outage class method. """ + """Test single outage class method.""" so_scheduler.outage._specs['allow_outage_overlap'] = allow_outage_overlap so_scheduler.update_when_can_schedule_from_months() @@ -615,7 +618,7 @@ def test_single_outage_scheduler_schedule_losses( @pytest.mark.parametrize('outages_info', NOMINAL_OUTAGES) def test_outage_scheduler_normal_run(outages_info): - """Test hourly outage losses for a reasonable outage info input. """ + """Test hourly outage losses for a reasonable outage info input.""" outages = [Outage(spec) for spec in outages_info] losses = OutageScheduler(outages).calculate() @@ -641,7 +644,7 @@ def test_outage_scheduler_normal_run(outages_info): def test_outage_scheduler_no_outages(): - """Test hourly outage losses for no outage input. """ + """Test hourly outage losses for no outage input.""" losses = OutageScheduler([]).calculate() @@ -650,7 +653,7 @@ def test_outage_scheduler_no_outages(): def test_outage_scheduler_cannot_schedule_any_more(): - """Test scheduler when little or no outages are allowed. """ + """Test scheduler when little or no outages are allowed.""" outage_info = { 'count': 5, @@ -675,7 +678,7 @@ def test_outage_scheduler_cannot_schedule_any_more(): def test_outage_class_missing_keys(basic_outage_dict): - """Test Outage class behavior for inputs with missing keys. """ + """Test Outage class behavior for inputs with missing keys.""" for key in basic_outage_dict: bad_input = basic_outage_dict.copy() @@ -686,7 +689,7 @@ def test_outage_class_missing_keys(basic_outage_dict): def test_outage_class_count(basic_outage_dict): - """Test Outage class behavior for different count inputs. """ + """Test Outage class behavior for different count inputs.""" basic_outage_dict['count'] = 0 with pytest.raises(reVLossesValueError) as excinfo: @@ -700,7 +703,7 @@ def test_outage_class_count(basic_outage_dict): def test_outage_class_allowed_months(basic_outage_dict): - """Test Outage class behavior for different allowed_month inputs. """ + """Test Outage class behavior for different allowed_month inputs.""" basic_outage_dict['allowed_months'] = [] with pytest.raises(reVLossesValueError) as excinfo: @@ -731,7 +734,7 @@ def test_outage_class_allowed_months(basic_outage_dict): def test_outage_class_duration(basic_outage_dict): - """Test Outage class behavior for different duration inputs. """ + """Test Outage class behavior for different duration inputs.""" err_msg = "Duration of outage must be between 1 and the total available" @@ -755,7 +758,7 @@ def test_outage_class_duration(basic_outage_dict): def test_outage_class_percentage(basic_outage_dict): - """Test Outage class behavior for different percentage inputs. """ + """Test Outage class behavior for different percentage inputs.""" err_msg = "Percentage of farm down during outage must be in the range" @@ -791,9 +794,9 @@ def test_outage_class_allow_outage_overlap(basic_outage_dict): (PV_SAM_FILE, TESTDATADIR + '/nsrdb/ri_100_nsrdb_{}.h5', 'pvwattsv7') ]) def test_scheduled_outages_multi_year(runner, files, clear_loggers): - """Test that scheduled outages are different year to year. """ + """Test that scheduled outages are different year to year.""" sam_file, res_file, tech = files - with open(sam_file, 'r') as fh: + with open(sam_file) as fh: sam_config = json.load(fh) outages = NOMINAL_OUTAGES[0] diff --git a/tests/test_nrwal.py b/tests/test_nrwal.py index 691287f3e..c6c74677e 100644 --- a/tests/test_nrwal.py +++ b/tests/test_nrwal.py @@ -7,24 +7,22 @@ @author: gbuster """ -import os import json -import traceback -import numpy as np +import os import shutil -import pytest import tempfile +import traceback +import numpy as np import pandas as pd - +import pytest from rex.utilities.utilities import pd_date_range +from reV import TESTDATADIR from reV.cli import main from reV.handlers.outputs import Outputs from reV.nrwal.nrwal import RevNrwal -from reV.utilities import ModuleName -from reV import TESTDATADIR - +from reV.utilities import MetaKeyName, ModuleName SOURCE_DIR = os.path.join(TESTDATADIR, 'nrwal/') @@ -41,8 +39,8 @@ def test_nrwal(): offshore_config = os.path.join(td, 'offshore.json') onshore_config = os.path.join(td, 'onshore.json') sam_configs = {'onshore': onshore_config, - 'offshore': offshore_config} - nrwal_configs = {'offshore': os.path.join(td, 'nrwal_offshore.yaml')} + MetaKeyName.OFFSHORE: offshore_config} + nrwal_configs = {MetaKeyName.OFFSHORE: os.path.join(td, 'nrwal_offshore.yaml')} with Outputs(gen_fpath, 'a') as f: f.time_index = pd_date_range('20100101', '20110101', @@ -147,7 +145,7 @@ def test_nrwal(): @pytest.mark.parametrize("out_fn", ["nrwal_meta.csv", None]) def test_nrwal_csv(out_fn): - """Test the reV nrwal class with csv output. """ + """Test the reV nrwal class with csv output.""" with tempfile.TemporaryDirectory() as td: for fn in os.listdir(SOURCE_DIR): shutil.copy(os.path.join(SOURCE_DIR, fn), os.path.join(td, fn)) @@ -157,8 +155,8 @@ def test_nrwal_csv(out_fn): offshore_config = os.path.join(td, 'offshore.json') onshore_config = os.path.join(td, 'onshore.json') sam_configs = {'onshore': onshore_config, - 'offshore': offshore_config} - nrwal_configs = {'offshore': os.path.join(td, 'nrwal_offshore.yaml')} + MetaKeyName.OFFSHORE: offshore_config} + nrwal_configs = {MetaKeyName.OFFSHORE: os.path.join(td, 'nrwal_offshore.yaml')} with Outputs(gen_fpath, 'a') as f: f.time_index = pd_date_range('20100101', '20110101', @@ -217,8 +215,8 @@ def test_nrwal_constant_eq_output_request(): offshore_config = os.path.join(td, 'offshore.json') onshore_config = os.path.join(td, 'onshore.json') sam_configs = {'onshore': onshore_config, - 'offshore': offshore_config} - nrwal_configs = {'offshore': os.path.join(td, 'nrwal_offshore.yaml')} + MetaKeyName.OFFSHORE: offshore_config} + nrwal_configs = {MetaKeyName.OFFSHORE: os.path.join(td, 'nrwal_offshore.yaml')} with Outputs(gen_fpath, 'a') as f: f.time_index = pd_date_range('20100101', '20110101', @@ -260,8 +258,8 @@ def test_nrwal_cli(runner, clear_loggers): offshore_config = os.path.join(td, 'offshore.json') onshore_config = os.path.join(td, 'onshore.json') sam_configs = {'onshore': onshore_config, - 'offshore': offshore_config} - nrwal_configs = {'offshore': os.path.join(td, 'nrwal_offshore.yaml')} + MetaKeyName.OFFSHORE: offshore_config} + nrwal_configs = {MetaKeyName.OFFSHORE: os.path.join(td, 'nrwal_offshore.yaml')} with Outputs(gen_fpath, 'a') as f: f.time_index = pd_date_range('20100101', '20110101', @@ -403,8 +401,8 @@ def test_nrwal_cli_csv(runner, clear_loggers): offshore_config = os.path.join(td, 'offshore.json') onshore_config = os.path.join(td, 'onshore.json') sam_configs = {'onshore': onshore_config, - 'offshore': offshore_config} - nrwal_configs = {'offshore': os.path.join(td, 'nrwal_offshore.yaml')} + MetaKeyName.OFFSHORE: offshore_config} + nrwal_configs = {MetaKeyName.OFFSHORE: os.path.join(td, 'nrwal_offshore.yaml')} with Outputs(gen_fpath, 'a') as f: f.time_index = pd_date_range('20100101', '20110101', diff --git a/tests/test_rep_profiles.py b/tests/test_rep_profiles.py index 7fb8087b4..d80b3c3cd 100644 --- a/tests/test_rep_profiles.py +++ b/tests/test_rep_profiles.py @@ -1,20 +1,23 @@ # -*- coding: utf-8 -*- """reVX representative profile tests. """ -import os -import pytest -import pandas as pd -import numpy as np import json +import os import tempfile -from pandas.testing import assert_frame_equal - -from reV.rep_profiles.rep_profiles import (RegionRepProfile, RepProfiles, - RepresentativeMethods) -from reV import TESTDATADIR +import numpy as np +import pandas as pd +import pytest +from pandas.testing import assert_frame_equal from rex.resource import Resource +from reV import TESTDATADIR +from reV.rep_profiles.rep_profiles import ( + RegionRepProfile, + RepProfiles, + RepresentativeMethods, +) +from reV.utilities import MetaKeyName GEN_FPATH = os.path.join(TESTDATADIR, 'gen_out/gen_ri_pv_2012_x000.h5') @@ -22,8 +25,8 @@ def test_rep_region_interval(): """Test the rep profile with a weird interval of gids""" sites = np.arange(40) * 2 - rev_summary = pd.DataFrame({'gen_gids': sites, - 'res_gids': sites}) + rev_summary = pd.DataFrame({MetaKeyName.GEN_GIDS: sites, + MetaKeyName.RES_GIDS: sites}) r = RegionRepProfile(GEN_FPATH, rev_summary, weight=None) assert r.i_reps[0] == 14 @@ -31,8 +34,8 @@ def test_rep_region_interval(): def test_rep_methods(): """Test integrated rep methods against baseline rep profile result""" sites = np.arange(100) - rev_summary = pd.DataFrame({'gen_gids': sites, - 'res_gids': sites}) + rev_summary = pd.DataFrame({MetaKeyName.GEN_GIDS: sites, + MetaKeyName.RES_GIDS: sites}) r = RegionRepProfile(GEN_FPATH, rev_summary, rep_method='meanoid', err_method='rmse', weight=None) @@ -58,8 +61,8 @@ def test_rep_methods(): def test_meanoid(): """Test the simple meanoid method""" sites = np.arange(100) - rev_summary = pd.DataFrame({'gen_gids': sites, - 'res_gids': sites}) + rev_summary = pd.DataFrame({MetaKeyName.GEN_GIDS: sites, + MetaKeyName.RES_GIDS: sites}) r = RegionRepProfile(GEN_FPATH, rev_summary, weight=None) meanoid = RepresentativeMethods.meanoid(r.source_profiles) @@ -74,18 +77,18 @@ def test_weighted_meanoid(): """Test a meanoid weighted by gid_counts vs. a non-weighted meanoid.""" sites = np.arange(100) - rev_summary = pd.DataFrame({'gen_gids': sites, - 'res_gids': sites, - 'gid_counts': [1] * 50 + [0] * 50}) + rev_summary = pd.DataFrame({MetaKeyName.GEN_GIDS: sites, + MetaKeyName.RES_GIDS: sites, + MetaKeyName.GID_COUNTS: [1] * 50 + [0] * 50}) r = RegionRepProfile(GEN_FPATH, rev_summary) - weights = r._get_region_attr(r._rev_summary, 'gid_counts') + weights = r._get_region_attr(r._rev_summary, MetaKeyName.GID_COUNTS) w_meanoid = RepresentativeMethods.meanoid(r.source_profiles, weights=weights) sites = np.arange(50) - rev_summary = pd.DataFrame({'gen_gids': sites, - 'res_gids': sites}) + rev_summary = pd.DataFrame({MetaKeyName.GEN_GIDS: sites, + MetaKeyName.RES_GIDS: sites}) r = RegionRepProfile(GEN_FPATH, rev_summary, weight=None) meanoid = RepresentativeMethods.meanoid(r.source_profiles, weights=None) @@ -101,12 +104,12 @@ def test_integrated(): zeros = np.zeros((100,)) regions = (['r0'] * 7) + (['r1'] * 33) + (['r2'] * 60) timezone = np.random.choice([-4, -5, -6, -7], 100) - rev_summary = pd.DataFrame({'gen_gids': sites, - 'res_gids': sites, + rev_summary = pd.DataFrame({MetaKeyName.GEN_GIDS: sites, + MetaKeyName.RES_GIDS: sites, 'res_class': zeros, 'weight': ones, 'region': regions, - 'timezone': timezone}) + MetaKeyName.TIMEZONE: timezone}) rp = RepProfiles(GEN_FPATH, rev_summary, 'region', weight='weight') rp.run(max_workers=1) p1, m1 = rp.profiles, rp.meta @@ -125,12 +128,12 @@ def test_sc_points(): """Test rep profiles for each SC point.""" sites = np.arange(10) timezone = np.random.choice([-4, -5, -6, -7], 10) - rev_summary = pd.DataFrame({'sc_gid': sites, - 'gen_gids': sites, - 'res_gids': sites, - 'timezone': timezone}) + rev_summary = pd.DataFrame({MetaKeyName.SC_GID: sites, + MetaKeyName.GEN_GIDS: sites, + MetaKeyName.RES_GIDS: sites, + MetaKeyName.TIMEZONE: timezone}) - rp = RepProfiles(GEN_FPATH, rev_summary, 'sc_gid', weight=None) + rp = RepProfiles(GEN_FPATH, rev_summary, MetaKeyName.SC_GID, weight=None) rp.run(max_workers=1) with Resource(GEN_FPATH) as res: @@ -149,27 +152,27 @@ def test_agg_profile(): res_gids = [json.dumps(x) for x in res_gids] gid_counts = [json.dumps(x) for x in gid_counts] timezone = np.random.choice([-4, -5, -6, -7], 4) - rev_summary = pd.DataFrame({'sc_gid': np.arange(4), - 'gen_gids': gen_gids, - 'res_gids': res_gids, - 'gid_counts': gid_counts, - 'timezone': timezone}) + rev_summary = pd.DataFrame({MetaKeyName.SC_GID: np.arange(4), + MetaKeyName.GEN_GIDS: gen_gids, + MetaKeyName.RES_GIDS: res_gids, + MetaKeyName.GID_COUNTS: gid_counts, + MetaKeyName.TIMEZONE: timezone}) - rp = RepProfiles(GEN_FPATH, rev_summary, 'sc_gid', cf_dset='cf_profile', + rp = RepProfiles(GEN_FPATH, rev_summary, MetaKeyName.SC_GID, cf_dset='cf_profile', err_method=None) rp.run(scaled_precision=False, max_workers=1) for index in rev_summary.index: - gen_gids = json.loads(rev_summary.loc[index, 'gen_gids']) - res_gids = json.loads(rev_summary.loc[index, 'res_gids']) - weights = np.array(json.loads(rev_summary.loc[index, 'gid_counts'])) + gen_gids = json.loads(rev_summary.loc[index, MetaKeyName.GEN_GIDS]) + res_gids = json.loads(rev_summary.loc[index, MetaKeyName.RES_GIDS]) + weights = np.array(json.loads(rev_summary.loc[index, MetaKeyName.GID_COUNTS])) with Resource(GEN_FPATH) as res: meta = res.meta raw_profiles = [] for gid in res_gids: - iloc = np.where(meta['gid'] == gid)[0][0] + iloc = np.where(meta[MetaKeyName.GID] == gid)[0][0] prof = np.expand_dims(res['cf_profile', :, iloc], 1) raw_profiles.append(prof) @@ -186,7 +189,7 @@ def test_agg_profile(): assert np.allclose(rp.profiles[0][:, index], truth) - passthrough_cols = ['gen_gids', 'res_gids', 'gid_counts'] + passthrough_cols = [MetaKeyName.GEN_GIDS, MetaKeyName.RES_GIDS, MetaKeyName.GID_COUNTS] for col in passthrough_cols: assert col in rp.meta @@ -204,13 +207,13 @@ def test_many_regions(use_weights): region1 = (['r0'] * 7) + (['r1'] * 33) + (['r2'] * 60) region2 = (['a0'] * 20) + (['b1'] * 10) + (['c2'] * 20) + (['d3'] * 50) timezone = np.random.choice([-4, -5, -6, -7], 100) - rev_summary = pd.DataFrame({'gen_gids': sites, - 'res_gids': sites, + rev_summary = pd.DataFrame({MetaKeyName.GEN_GIDS: sites, + MetaKeyName.RES_GIDS: sites, 'res_class': zeros, 'region1': region1, 'region2': region2, 'weight': sites + 1, - 'timezone': timezone}) + MetaKeyName.TIMEZONE: timezone}) reg_cols = ['region1', 'region2'] if use_weights: rp = RepProfiles(GEN_FPATH, rev_summary, reg_cols, weight="weight") @@ -239,13 +242,13 @@ def test_many_regions_with_list_weights(): region1 = (['r0'] * 7) + (['r1'] * 33) + (['r2'] * 60) region2 = (['a0'] * 20) + (['b1'] * 10) + (['c2'] * 20) + (['d3'] * 50) timezone = np.random.choice([-4, -5, -6, -7], 100) - rev_summary = pd.DataFrame({'gen_gids': sites, - 'res_gids': sites, + rev_summary = pd.DataFrame({MetaKeyName.GEN_GIDS: sites, + MetaKeyName.RES_GIDS: sites, 'res_class': zeros, 'region1': region1, 'region2': region2, 'weights': weights, - 'timezone': timezone}) + MetaKeyName.TIMEZONE: timezone}) reg_cols = ['region1', 'region2'] rp = RepProfiles(GEN_FPATH, rev_summary, reg_cols, weight='weights') rp.run() @@ -267,11 +270,11 @@ def test_write_to_file(): zeros = np.zeros((100,)) regions = (['r0'] * 7) + (['r1'] * 33) + (['r2'] * 60) timezone = np.random.choice([-4, -5, -6, -7], 100) - rev_summary = pd.DataFrame({'gen_gids': sites, - 'res_gids': sites, + rev_summary = pd.DataFrame({MetaKeyName.GEN_GIDS: sites, + MetaKeyName.RES_GIDS: sites, 'res_class': zeros, 'region': regions, - 'timezone': timezone}) + MetaKeyName.TIMEZONE: timezone}) fout = os.path.join(td, 'temp_rep_profiles.h5') rp = RepProfiles(GEN_FPATH, rev_summary, 'region', n_profiles=3, weight=None) @@ -300,11 +303,11 @@ def test_file_options(): zeros = np.zeros((100,)) regions = (['r0'] * 7) + (['r1'] * 33) + (['r2'] * 60) timezone = np.random.choice([-4, -5, -6, -7], 100) - rev_summary = pd.DataFrame({'gen_gids': sites, - 'res_gids': sites, + rev_summary = pd.DataFrame({MetaKeyName.GEN_GIDS: sites, + MetaKeyName.RES_GIDS: sites, 'res_class': zeros, 'region': regions, - 'timezone': timezone}) + MetaKeyName.TIMEZONE: timezone}) fout = os.path.join(td, 'temp_rep_profiles.h5') rp = RepProfiles(GEN_FPATH, rev_summary, 'region', n_profiles=3, weight=None) diff --git a/tests/test_sam.py b/tests/test_sam.py index 8ff39d846..4413e30d9 100644 --- a/tests/test_sam.py +++ b/tests/test_sam.py @@ -3,23 +3,25 @@ """reV SAM unit test module """ import os -from pkg_resources import get_distribution -from packaging import version -import pytest -import numpy as np import warnings -from reV.SAM.defaults import (DefaultPvWattsv5, DefaultPvWattsv8, - DefaultWindPower) -from reV.SAM.generation import PvWattsv5, PvWattsv7, PvWattsv8 +import numpy as np +import pytest +from packaging import version +from pkg_resources import get_distribution +from rex.renewable_resource import NSRDB +from rex.utilities.utilities import pd_date_range + from reV import TESTDATADIR from reV.config.project_points import ProjectPoints +from reV.SAM.defaults import ( + DefaultPvWattsv5, + DefaultPvWattsv8, + DefaultWindPower, +) +from reV.SAM.generation import PvWattsv5, PvWattsv7, PvWattsv8 from reV.SAM.version_checker import PySamVersionChecker -from reV.utilities.exceptions import PySAMVersionWarning -from reV.utilities.exceptions import InputError - -from rex.renewable_resource import NSRDB -from rex.utilities.utilities import pd_date_range +from reV.utilities.exceptions import InputError, PySAMVersionWarning @pytest.fixture @@ -94,7 +96,7 @@ def test_PV_lat_tilt(res, site_index): # get SAM inputs from project_points based on the current site site = res_df.name config, inputs = pp[site] - inputs['tilt'] = 'latitude' + inputs['tilt'] = MetaKeyName.LATITUDE # iterate through requested sites. with warnings.catch_warnings(): warnings.simplefilter("ignore") @@ -102,10 +104,9 @@ def test_PV_lat_tilt(res, site_index): sam_sys_inputs=inputs, output_request=('cf_mean',)) break - else: - pass + pass - assert sim.sam_sys_inputs['tilt'] == meta['latitude'] + assert sim.sam_sys_inputs['tilt'] == meta[MetaKeyName.LATITUDE] @pytest.mark.parametrize('dt', ('1h', '30min', '5min')) diff --git a/tests/test_supply_curve_aggregation.py b/tests/test_supply_curve_aggregation.py index e669e6223..9ef1ba243 100644 --- a/tests/test_supply_curve_aggregation.py +++ b/tests/test_supply_curve_aggregation.py @@ -3,15 +3,15 @@ """ Aggregation tests """ -import numpy as np import os -from pandas.testing import assert_frame_equal + +import numpy as np import pytest +from pandas.testing import assert_frame_equal +from rex.resource import Resource -from reV.supply_curve.aggregation import Aggregation from reV import TESTDATADIR - -from rex.resource import Resource +from reV.supply_curve.aggregation import Aggregation EXCL = os.path.join(TESTDATADIR, 'ri_exclusions/ri_exclusions.h5') RES = os.path.join(TESTDATADIR, 'nsrdb/ri_100_nsrdb_2012.h5') @@ -45,8 +45,8 @@ def check_agg(agg_out, baseline_h5): for dset, test in agg_out.items(): truth = f[dset] if dset == 'meta': - truth = truth.set_index('sc_gid') - for c in ['source_gids', 'gid_counts']: + truth = truth.set_index(MetaKeyName.SC_GID) + for c in [MetaKeyName.SOURCE_GIDS, MetaKeyName.GID_COUNTS]: test[c] = test[c].astype(str) truth = truth.fillna('none') @@ -105,9 +105,9 @@ def test_gid_counts(excl_dict): excl_dict=excl_dict, max_workers=1) for i, row in agg_out['meta'].iterrows(): - n_gids = row['n_gids'] - gid_counts = np.sum(row['gid_counts']) - area = row['area_sq_km'] + n_gids = row[MetaKeyName.N_GIDS] + gid_counts = np.sum(row[MetaKeyName.GID_COUNTS]) + area = row[MetaKeyName.AREA_SQ_KM] msg = ('For sc_gid {}: the sum of gid_counts ({}), does not match ' 'n_gids ({})'.format(i, n_gids, gid_counts)) @@ -148,8 +148,8 @@ def test_mean_wind_dirs(excl_dict): for i, row in out_meta.iterrows(): test = mean_wind_dirs[:, i] - gids = row['source_gids'] - fracs = row['gid_counts'] / row['n_gids'] + gids = row[MetaKeyName.SOURCE_GIDS] + fracs = row[MetaKeyName.GID_COUNTS] / row[MetaKeyName.N_GIDS] truth = compute_mean_wind_dirs(RES, DSET, gids, fracs) diff --git a/tests/test_supply_curve_aggregation_friction.py b/tests/test_supply_curve_aggregation_friction.py index 49158a742..648c6dafd 100644 --- a/tests/test_supply_curve_aggregation_friction.py +++ b/tests/test_supply_curve_aggregation_friction.py @@ -5,16 +5,18 @@ @author: gbuster """ +import os +import warnings + import h5py import numpy as np import pytest -import os -import warnings -from reV.supply_curve.extent import SupplyCurveExtent +from reV import TESTDATADIR from reV.supply_curve.exclusions import ExclusionMaskFromDict, FrictionMask +from reV.supply_curve.extent import SupplyCurveExtent from reV.supply_curve.sc_aggregation import SupplyCurveAggregation -from reV import TESTDATADIR +from reV.utilities import MetaKeyName EXCL_FPATH = os.path.join(TESTDATADIR, 'ri_exclusions/ri_exclusions.h5') FRICTION_FPATH = os.path.join(TESTDATADIR, 'ri_exclusions/ri_friction.h5') @@ -63,7 +65,7 @@ def test_friction_mask(): assert diff < 0.0001, m -@pytest.mark.parametrize('gid', [100, 114, 130, 181]) +@pytest.mark.parametrize(MetaKeyName.GID, [100, 114, 130, 181]) def test_agg_friction(gid): """Test SC Aggregation with friction by checking friction factors and LCOE against a hand calc.""" @@ -92,18 +94,19 @@ def test_agg_friction(gid): m = ('SC point gid {} does not match mean friction hand calc' .format(gid)) - assert np.isclose(s['mean_friction'].values[0], mean_friction), m + assert np.isclose(s[MetaKeyName.MEAN_FRICTION].values[0], mean_friction), m m = ('SC point gid {} does not match mean LCOE with friction hand calc' .format(gid)) - assert np.allclose(s['mean_lcoe_friction'], - s['mean_lcoe'] * mean_friction), m + assert np.allclose(s[MetaKeyName.MEAN_LCOE_FRICTION], + s[MetaKeyName.MEAN_LCOE] * mean_friction), m # pylint: disable=no-member def make_friction_file(): """Script to make a test friction file""" - import matplotlib.pyplot as plt import shutil + + import matplotlib.pyplot as plt shutil.copy(EXCL, FRICTION_FPATH) with h5py.File(FRICTION_FPATH, 'a') as f: f[FRICTION_DSET] = f['ri_srtm_slope'] @@ -121,7 +124,7 @@ def make_friction_file(): f[FRICTION_DSET][...] = data for d in f: - if d not in [FRICTION_DSET, 'latitude', 'longitude']: + if d not in [FRICTION_DSET, MetaKeyName.LATITUDE, MetaKeyName.LONGITUDE]: del f[d] with h5py.File(FRICTION_FPATH, 'r') as f: diff --git a/tests/test_supply_curve_compute.py b/tests/test_supply_curve_compute.py index 6b80f7e54..b10e7ad77 100644 --- a/tests/test_supply_curve_compute.py +++ b/tests/test_supply_curve_compute.py @@ -3,15 +3,17 @@ Supply Curve computation integrated tests """ import os -import pandas as pd -from pandas.testing import assert_frame_equal -import pytest +import tempfile import warnings + import numpy as np -import tempfile +import pandas as pd +import pytest +from pandas.testing import assert_frame_equal from reV import TESTDATADIR from reV.supply_curve.supply_curve import SupplyCurve +from reV.utilities import MetaKeyName from reV.utilities.exceptions import SupplyCurveInputError TRANS_COSTS_1 = {'line_tie_in_cost': 200, 'line_cost': 1000, @@ -117,13 +119,13 @@ def test_integrated_sc_full_friction(): transmission_costs=tcosts, avail_cap_frac=avail_cap_frac, columns=SC_FULL_COLUMNS, - sort_on='total_lcoe_friction') + sort_on=MetaKeyName.TOTAL_LCOE_FRICTION) sc_full = pd.read_csv(sc_full) - assert 'mean_lcoe_friction' in sc_full - assert 'total_lcoe_friction' in sc_full - test = sc_full['mean_lcoe_friction'] + sc_full['lcot'] - assert np.allclose(test, sc_full['total_lcoe_friction']) + assert MetaKeyName.MEAN_LCOE_FRICTION in sc_full + assert MetaKeyName.TOTAL_LCOE_FRICTION in sc_full + test = sc_full[MetaKeyName.MEAN_LCOE_FRICTION] + sc_full['lcot'] + assert np.allclose(test, sc_full[MetaKeyName.TOTAL_LCOE_FRICTION]) fpath_baseline = os.path.join(TESTDATADIR, 'sc_out/sc_full_out_friction.csv') @@ -139,12 +141,12 @@ def test_integrated_sc_simple_friction(): out_fpath = os.path.join(td, "sc") sc_simple = sc.run(out_fpath, fixed_charge_rate=0.1, simple=True, transmission_costs=tcosts, - sort_on='total_lcoe_friction') + sort_on=MetaKeyName.TOTAL_LCOE_FRICTION) sc_simple = pd.read_csv(sc_simple) - assert 'mean_lcoe_friction' in sc_simple - assert 'total_lcoe_friction' in sc_simple - test = sc_simple['mean_lcoe_friction'] + sc_simple['lcot'] - assert np.allclose(test, sc_simple['total_lcoe_friction']) + assert MetaKeyName.MEAN_LCOE_FRICTION in sc_simple + assert MetaKeyName.TOTAL_LCOE_FRICTION in sc_simple + test = sc_simple[MetaKeyName.MEAN_LCOE_FRICTION] + sc_simple['lcot'] + assert np.allclose(test, sc_simple[MetaKeyName.TOTAL_LCOE_FRICTION]) fpath_baseline = os.path.join(TESTDATADIR, 'sc_out/sc_simple_out_friction.csv') @@ -153,7 +155,7 @@ def test_integrated_sc_simple_friction(): def test_sc_warning1(): """Run the full SC test with missing connections and verify warning.""" - mask = TRANS_TABLE['sc_point_gid'].isin(list(range(10))) + mask = TRANS_TABLE[MetaKeyName.SC_POINT_GID].isin(list(range(10))) trans_table = TRANS_TABLE[~mask] tcosts = TRANS_COSTS_1.copy() avail_cap_frac = tcosts.pop('available_capacity', 1) @@ -218,7 +220,7 @@ def test_parallel(): assert_frame_equal(sc_full_parallel, sc_full_serial) -def verify_trans_cap(sc_table, trans_tables, cap_col='capacity'): +def verify_trans_cap(sc_table, trans_tables, cap_col=MetaKeyName.CAPACITY): """ Verify that sc_points are connected to features in the correct capacity bins @@ -239,7 +241,7 @@ def verify_trans_cap(sc_table, trans_tables, cap_col='capacity'): test = sc_table.merge(trans_features, on='trans_gid', how='left') mask = test[cap_col] > test['max_cap'] - cols = ['sc_gid', 'trans_gid', cap_col, 'max_cap'] + cols = [MetaKeyName.SC_GID, 'trans_gid', cap_col, 'max_cap'] msg = ("SC points connected to transmission features with " "max_cap < sc_cap:\n{}" .format(test.loc[mask, cols])) @@ -348,24 +350,24 @@ def test_multi_parallel_trans(): sc = SupplyCurve(SC_POINTS, trans_tables) sc_2 = sc.simple_sort(fcr=0.1, columns=columns) - assert not set(SC_POINTS['sc_gid']) - set(sc_1['sc_gid']) - assert not set(SC_POINTS['sc_gid']) - set(sc_2['sc_gid']) - assert not set(SC_POINTS['sc_point_gid']) - set(sc_1['sc_point_gid']) - assert not set(SC_POINTS['sc_point_gid']) - set(sc_2['sc_point_gid']) - assert not set(sc_1['sc_point_gid']) - set(SC_POINTS['sc_point_gid']) - assert not set(sc_2['sc_point_gid']) - set(SC_POINTS['sc_point_gid']) + assert not set(SC_POINTS[MetaKeyName.SC_GID]) - set(sc_1[MetaKeyName.SC_GID]) + assert not set(SC_POINTS[MetaKeyName.SC_GID]) - set(sc_2[MetaKeyName.SC_GID]) + assert not set(SC_POINTS[MetaKeyName.SC_POINT_GID]) - set(sc_1[MetaKeyName.SC_POINT_GID]) + assert not set(SC_POINTS[MetaKeyName.SC_POINT_GID]) - set(sc_2[MetaKeyName.SC_POINT_GID]) + assert not set(sc_1[MetaKeyName.SC_POINT_GID]) - set(SC_POINTS[MetaKeyName.SC_POINT_GID]) + assert not set(sc_2[MetaKeyName.SC_POINT_GID]) - set(SC_POINTS[MetaKeyName.SC_POINT_GID]) assert (sc_2.n_parallel_trans > 1).any() mask_2 = sc_2['n_parallel_trans'] > 1 - for gid in sc_2.loc[mask_2, 'sc_gid']: - nx_1 = sc_1.loc[(sc_1['sc_gid'] == gid), 'n_parallel_trans'].values[0] - nx_2 = sc_2.loc[(sc_2['sc_gid'] == gid), 'n_parallel_trans'].values[0] + for gid in sc_2.loc[mask_2, MetaKeyName.SC_GID]: + nx_1 = sc_1.loc[(sc_1[MetaKeyName.SC_GID] == gid), 'n_parallel_trans'].values[0] + nx_2 = sc_2.loc[(sc_2[MetaKeyName.SC_GID] == gid), 'n_parallel_trans'].values[0] assert nx_2 >= nx_1 if nx_1 != nx_2: - lcot_1 = sc_1.loc[(sc_1['sc_gid'] == gid), 'lcot'].values[0] - lcot_2 = sc_2.loc[(sc_2['sc_gid'] == gid), 'lcot'].values[0] + lcot_1 = sc_1.loc[(sc_1[MetaKeyName.SC_GID] == gid), 'lcot'].values[0] + lcot_2 = sc_2.loc[(sc_2[MetaKeyName.SC_GID] == gid), 'lcot'].values[0] assert lcot_2 > lcot_1 diff --git a/tests/test_supply_curve_points.py b/tests/test_supply_curve_points.py index 1d1f47ea1..a9c718bad 100644 --- a/tests/test_supply_curve_points.py +++ b/tests/test_supply_curve_points.py @@ -6,15 +6,19 @@ """ # pylint: disable=no-member import os + import numpy as np import pytest -from reV.supply_curve.sc_aggregation import SupplyCurveAggregation -from reV.supply_curve.extent import SupplyCurveExtent -from reV.supply_curve.points import (SupplyCurvePoint, - GenerationSupplyCurvePoint) -from reV.handlers.outputs import Outputs from reV import TESTDATADIR +from reV.handlers.outputs import Outputs +from reV.supply_curve.extent import SupplyCurveExtent +from reV.supply_curve.points import ( + GenerationSupplyCurvePoint, + SupplyCurvePoint, +) +from reV.supply_curve.sc_aggregation import SupplyCurveAggregation +from reV.utilities import MetaKeyName F_EXCL = os.path.join(TESTDATADIR, 'ri_exclusions/ri_exclusions.h5') F_GEN = os.path.join(TESTDATADIR, 'gen_out/gen_ri_pv_2012_x000.h5') @@ -58,7 +62,7 @@ def test_slicer(gids, resolution): assert col_slice0 == col_slice1, msg -@pytest.mark.parametrize(('gid', 'resolution', 'excl_dict', 'time_series'), +@pytest.mark.parametrize((MetaKeyName.GID, 'resolution', 'excl_dict', 'time_series'), [(37, 64, None, None), (37, 64, EXCL_DICT, None), (37, 64, None, 100), @@ -96,7 +100,7 @@ def test_weighted_means(gid, resolution, excl_dict, time_series): assert np.allclose(test, means, rtol=RTOL) -@pytest.mark.parametrize(('gid', 'resolution', 'excl_dict', 'time_series'), +@pytest.mark.parametrize((MetaKeyName.GID, 'resolution', 'excl_dict', 'time_series'), [(37, 64, None, None), (37, 64, EXCL_DICT, None), (37, 64, None, 100), @@ -145,11 +149,11 @@ def plot_all_sc_points(resolution=64): colors *= len(sc) for gid in range(len(sc)): excl_meta = sc.get_excl_points('meta', gid) - axs.scatter(excl_meta['longitude'], excl_meta['latitude'], + axs.scatter(excl_meta[MetaKeyName.LONGITUDE], excl_meta[MetaKeyName.LATITUDE], c=colors[gid], s=0.01) with Outputs(F_GEN) as f: - axs.scatter(f.meta['longitude'], f.meta['latitude'], c='k', s=25) + axs.scatter(f.meta[MetaKeyName.LONGITUDE], f.meta[MetaKeyName.LATITUDE], c='k', s=25) axs.axis('equal') plt.show() @@ -175,12 +179,12 @@ def plot_single_gen_sc_point(gid=2, resolution=64): for i, gen_gid in enumerate(all_gen_gids): if gen_gid != -1: mask = (sc._gen_gids == gen_gid) - axs.scatter(excl_meta.loc[mask, 'longitude'], - excl_meta.loc[mask, 'latitude'], + axs.scatter(excl_meta.loc[mask, MetaKeyName.LONGITUDE], + excl_meta.loc[mask, MetaKeyName.LATITUDE], marker='s', c=colors[i], s=1) - axs.scatter(sc.gen.meta.loc[gen_gid, 'longitude'], - sc.gen.meta.loc[gen_gid, 'latitude'], + axs.scatter(sc.gen.meta.loc[gen_gid, MetaKeyName.LONGITUDE], + sc.gen.meta.loc[gen_gid, MetaKeyName.LATITUDE], c='k', s=100) axs.scatter(sc.centroid[1], sc.centroid[0], marker='x', c='k', s=200) diff --git a/tests/test_supply_curve_sc_aggregation.py b/tests/test_supply_curve_sc_aggregation.py index 277843e2e..e6189b7ca 100644 --- a/tests/test_supply_curve_sc_aggregation.py +++ b/tests/test_supply_curve_sc_aggregation.py @@ -7,26 +7,25 @@ """ import json import os +import shutil +import tempfile +import traceback + +import h5py import numpy as np import pandas as pd -from pandas.testing import assert_frame_equal import pytest -import tempfile -import shutil -import h5py -import json -import shutil -import traceback +from pandas.testing import assert_frame_equal +from rex import Outputs, Resource +from reV import TESTDATADIR from reV.cli import main from reV.econ.utilities import lcoe_fcr -from reV.supply_curve.sc_aggregation import (SupplyCurveAggregation, - _warn_about_large_datasets) -from reV.utilities import ModuleName -from reV import TESTDATADIR -from rex import Resource, Outputs -from rex.utilities.loggers import LOGGERS - +from reV.supply_curve.sc_aggregation import ( + SupplyCurveAggregation, + _warn_about_large_datasets, +) +from reV.utilities import MetaKeyName, ModuleName EXCL = os.path.join(TESTDATADIR, 'ri_exclusions/ri_exclusions.h5') RES = os.path.join(TESTDATADIR, 'nsrdb/ri_100_nsrdb_2012.h5') @@ -63,12 +62,12 @@ def test_agg_extent(resolution=64): summary = sca.summarize(GEN) all_res_gids = [] - for gids in summary['res_gids']: + for gids in summary[MetaKeyName.RES_GIDS]: all_res_gids += gids - assert 'sc_col_ind' in summary - assert 'sc_row_ind' in summary - assert 'gen_gids' in summary + assert MetaKeyName.SC_COL_IND in summary + assert MetaKeyName.SC_ROW_IND in summary + assert MetaKeyName.GEN_GIDS in summary assert len(set(all_res_gids)) == 177 @@ -102,17 +101,16 @@ def test_agg_summary(): raise Exception('Aggregation summary baseline file did not exist. ' 'Created: {}'.format(AGG_BASELINE)) - else: - for c in ['res_gids', 'gen_gids', 'gid_counts']: - summary[c] = summary[c].astype(str) + for c in [MetaKeyName.RES_GIDS, MetaKeyName.GEN_GIDS, MetaKeyName.GID_COUNTS]: + summary[c] = summary[c].astype(str) - s_baseline = pd.read_csv(AGG_BASELINE, index_col=0) + s_baseline = pd.read_csv(AGG_BASELINE, index_col=0) - summary = summary.fillna('None') - s_baseline = s_baseline.fillna('None') - summary = summary[list(s_baseline.columns)] + summary = summary.fillna('None') + s_baseline = s_baseline.fillna('None') + summary = summary[list(s_baseline.columns)] - assert_frame_equal(summary, s_baseline, check_dtype=False, rtol=0.0001) + assert_frame_equal(summary, s_baseline, check_dtype=False, rtol=0.0001) assert "capacity_ac" not in summary @@ -159,7 +157,7 @@ def test_multi_file_excl(): shutil.copy(EXCL, excl_temp_2) with h5py.File(excl_temp_1, 'a') as f: - shape = f['latitude'].shape + shape = f[MetaKeyName.LATITUDE].shape attrs = dict(f['ri_srtm_slope'].attrs) data = np.ones(shape) test_dset = 'excl_test' @@ -179,7 +177,7 @@ def test_multi_file_excl(): summary = summary.fillna('None') s_baseline = s_baseline.fillna('None') - assert np.allclose(summary['area_sq_km'] * 2, s_baseline['area_sq_km']) + assert np.allclose(summary[MetaKeyName.AREA_SQ_KM] * 2, s_baseline[MetaKeyName.AREA_SQ_KM]) @pytest.mark.parametrize('pre_extract', (True, False)) @@ -198,17 +196,16 @@ def test_pre_extract_inclusions(pre_extract): raise Exception('Aggregation summary baseline file did not exist. ' 'Created: {}'.format(AGG_BASELINE)) - else: - for c in ['res_gids', 'gen_gids', 'gid_counts']: - summary[c] = summary[c].astype(str) + for c in [MetaKeyName.RES_GIDS, MetaKeyName.GEN_GIDS, MetaKeyName.GID_COUNTS]: + summary[c] = summary[c].astype(str) - s_baseline = pd.read_csv(AGG_BASELINE, index_col=0) + s_baseline = pd.read_csv(AGG_BASELINE, index_col=0) - summary = summary.fillna('None') - s_baseline = s_baseline.fillna('None') - summary = summary[list(s_baseline.columns)] + summary = summary.fillna('None') + s_baseline = s_baseline.fillna('None') + summary = summary[list(s_baseline.columns)] - assert_frame_equal(summary, s_baseline, check_dtype=False, rtol=0.0001) + assert_frame_equal(summary, s_baseline, check_dtype=False, rtol=0.0001) def test_agg_gen_econ(): @@ -244,13 +241,13 @@ def test_agg_extra_dsets(): for dset in h5_dsets: assert 'mean_{}'.format(dset) in summary.columns - check = summary['mean_lcoe_fcr-2012'] == summary['mean_lcoe'] + check = summary['mean_lcoe_fcr-2012'] == summary[MetaKeyName.MEAN_LCOE] assert not any(check) - check = summary['mean_lcoe_fcr-2013'] == summary['mean_lcoe'] + check = summary['mean_lcoe_fcr-2013'] == summary[MetaKeyName.MEAN_LCOE] assert not any(check) avg = (summary['mean_lcoe_fcr-2012'] + summary['mean_lcoe_fcr-2013']) / 2 - assert np.allclose(avg.values, summary['mean_lcoe'].values) + assert np.allclose(avg.values, summary[MetaKeyName.MEAN_LCOE].values) def test_agg_extra_2D_dsets(): @@ -288,7 +285,7 @@ def test_agg_scalar_excl(): data_layers=DATA_LAYERS, gids=gids_subset) summary_with_weights = sca.summarize(GEN, max_workers=1) - dsets = ['area_sq_km', 'capacity'] + dsets = [MetaKeyName.AREA_SQ_KM, MetaKeyName.CAPACITY] for dset in dsets: diff = (summary_base[dset].values / summary_with_weights[dset].values) msg = ('Fractional exclusions failed for {} which has values {} and {}' @@ -297,8 +294,8 @@ def test_agg_scalar_excl(): assert all(diff == 2), msg for i in summary_base.index: - counts_full = summary_base.loc[i, 'gid_counts'] - counts_half = summary_with_weights.loc[i, 'gid_counts'] + counts_full = summary_base.loc[i, MetaKeyName.GID_COUNTS] + counts_half = summary_with_weights.loc[i, MetaKeyName.GID_COUNTS] for j, counts in enumerate(counts_full): msg = ('GID counts for fractional exclusions failed for index {}!' @@ -328,7 +325,7 @@ def test_data_layer_methods(): for i in summary.index.values: # Check categorical data layers - counts = summary.loc[i, 'gid_counts'] + counts = summary.loc[i, MetaKeyName.GID_COUNTS] rr = summary.loc[i, 'reeds_region'] assert isinstance(rr, str) rr = json.loads(rr) @@ -348,7 +345,7 @@ def test_data_layer_methods(): raise RuntimeError(e) # Check min/mean/max of the same data layer - n = summary.loc[i, 'n_gids'] + n = summary.loc[i, MetaKeyName.N_GIDS] slope_mean = summary.loc[i, 'pct_slope_mean'] slope_max = summary.loc[i, 'pct_slope_max'] slope_min = summary.loc[i, 'pct_slope_min'] @@ -427,17 +424,17 @@ def test_recalc_lcoe(cap_cost_scale): cap_cost_scale=cap_cost_scale) summary = sca.summarize(gen_temp, max_workers=1) - assert not np.allclose(summary_base['mean_lcoe'], summary['mean_lcoe']) + assert not np.allclose(summary_base[MetaKeyName.MEAN_LCOE], summary[MetaKeyName.MEAN_LCOE]) if cap_cost_scale == '1': - cc_dset = 'sc_point_capital_cost' + cc_dset = MetaKeyName.SC_POINT_CAPITAL_COST else: - cc_dset = 'scaled_sc_point_capital_cost' + cc_dset = MetaKeyName.SCALED_SC_POINT_CAPITAL_COST lcoe = lcoe_fcr(summary['mean_fixed_charge_rate'], summary[cc_dset], - summary['sc_point_fixed_operating_cost'], - summary['sc_point_annual_energy'], + summary[MetaKeyName.SC_POINT_FIXED_OPERATING_COST], + summary[MetaKeyName.SC_POINT_ANNUAL_ENERGY], summary['mean_variable_operating_cost']) - assert np.allclose(lcoe, summary['mean_lcoe']) + assert np.allclose(lcoe, summary[MetaKeyName.MEAN_LCOE]) @pytest.mark.parametrize('tm_dset', ("techmap_ri", "techmap_ri_new")) diff --git a/tests/test_supply_curve_tech_mapping.py b/tests/test_supply_curve_tech_mapping.py index 034b86780..c795142a5 100644 --- a/tests/test_supply_curve_tech_mapping.py +++ b/tests/test_supply_curve_tech_mapping.py @@ -4,16 +4,18 @@ @author: gbuster """ +import os + import h5py import numpy as np import pandas as pd import pytest -import os from reV import TESTDATADIR from reV.handlers.exclusions import ExclusionLayers from reV.handlers.outputs import Outputs from reV.supply_curve.tech_mapping import TechMapping +from reV.utilities import MetaKeyName EXCL = os.path.join(TESTDATADIR, 'ri_exclusions/ri_exclusions.h5') RES = os.path.join(TESTDATADIR, 'nsrdb/ri_100_nsrdb_2012.h5') @@ -43,8 +45,8 @@ def plot_tech_mapping(dist_margin=1.05): import matplotlib.pyplot as plt with h5py.File(EXCL, 'r') as f: - lats = f['latitude'][...].flatten() - lons = f['longitude'][...].flatten() + lats = f[MetaKeyName.LATITUDE][...].flatten() + lons = f[MetaKeyName.LONGITUDE][...].flatten() ind_truth = f[TM_DSET][...].flatten() with Outputs(GEN) as fgen: @@ -53,8 +55,8 @@ def plot_tech_mapping(dist_margin=1.05): ind_test = TechMapping.run(EXCL, RES, dset=None, max_workers=2, dist_margin=dist_margin) - df = pd.DataFrame({'latitude': lats, - 'longitude': lons, + df = pd.DataFrame({MetaKeyName.LATITUDE: lats, + MetaKeyName.LONGITUDE: lons, TM_DSET: ind_truth, 'test': ind_test.flatten()}) @@ -65,31 +67,31 @@ def plot_tech_mapping(dist_margin=1.05): for i, ind in enumerate(df[TM_DSET].unique()): if ind != -1: mask = df[TM_DSET] == ind - axs.scatter(df.loc[mask, 'longitude'], - df.loc[mask, 'latitude'], + axs.scatter(df.loc[mask, MetaKeyName.LONGITUDE], + df.loc[mask, MetaKeyName.LATITUDE], c=colors[i], s=0.001) elif ind == -1: mask = df[TM_DSET] == ind - axs.scatter(df.loc[mask, 'longitude'], - df.loc[mask, 'latitude'], + axs.scatter(df.loc[mask, MetaKeyName.LONGITUDE], + df.loc[mask, MetaKeyName.LATITUDE], c='r', s=0.001) for ind in df[TM_DSET].unique(): if ind != -1: - axs.scatter(gen_meta.loc[ind, 'longitude'], - gen_meta.loc[ind, 'latitude'], + axs.scatter(gen_meta.loc[ind, MetaKeyName.LONGITUDE], + gen_meta.loc[ind, MetaKeyName.LATITUDE], c='w', s=1) for ind in df['test'].unique(): if ind != -1: - axs.scatter(gen_meta.loc[ind, 'longitude'], - gen_meta.loc[ind, 'latitude'], + axs.scatter(gen_meta.loc[ind, MetaKeyName.LONGITUDE], + gen_meta.loc[ind, MetaKeyName.LATITUDE], c='r', s=1) mask = df[TM_DSET].values != df['test'] - axs.scatter(df.loc[mask, 'longitude'], - df.loc[mask, 'latitude'], + axs.scatter(df.loc[mask, MetaKeyName.LONGITUDE], + df.loc[mask, MetaKeyName.LATITUDE], c='k', s=1) axs.axis('equal') diff --git a/tests/test_supply_curve_vpd.py b/tests/test_supply_curve_vpd.py index ae2b48389..c5fbbb7cc 100644 --- a/tests/test_supply_curve_vpd.py +++ b/tests/test_supply_curve_vpd.py @@ -3,13 +3,15 @@ Test Variable Power Density @author: gbuster """ -import pandas as pd +import os + import numpy as np +import pandas as pd import pytest -import os -from reV.supply_curve.sc_aggregation import SupplyCurveAggregation from reV import TESTDATADIR +from reV.supply_curve.sc_aggregation import SupplyCurveAggregation +from reV.utilities import MetaKeyName from reV.utilities.exceptions import FileInputError EXCL = os.path.join(TESTDATADIR, 'ri_exclusions/ri_exclusions.h5') @@ -41,10 +43,10 @@ def test_vpd(): vpd = pd.read_csv(FVPD, index_col=0) for i in summary.index: - capacity = summary.loc[i, 'capacity'] - area = summary.loc[i, 'area_sq_km'] - res_gids = np.array(summary.loc[i, 'res_gids']) - gid_counts = np.array(summary.loc[i, 'gid_counts']) + capacity = summary.loc[i, MetaKeyName.CAPACITY] + area = summary.loc[i, MetaKeyName.AREA_SQ_KM] + res_gids = np.array(summary.loc[i, MetaKeyName.RES_GIDS]) + gid_counts = np.array(summary.loc[i, MetaKeyName.GID_COUNTS]) vpd_per_gid = vpd.loc[res_gids, 'power_density'].values truth = area * (vpd_per_gid * gid_counts).sum() / gid_counts.sum() @@ -77,8 +79,8 @@ def test_vpd_fractional_excl(): summary_2 = sca_2.summarize(GEN, max_workers=1) for i in summary_1.index: - cap_full = summary_1.loc[i, 'capacity'] - cap_half = summary_2.loc[i, 'capacity'] + cap_full = summary_1.loc[i, MetaKeyName.CAPACITY] + cap_half = summary_2.loc[i, MetaKeyName.CAPACITY] msg = ('Variable power density for fractional exclusions failed! ' 'Index {} has cap full {} and cap half {}' diff --git a/tests/test_supply_curve_wind_dirs.py b/tests/test_supply_curve_wind_dirs.py index 766bbfbc6..8d543ead2 100644 --- a/tests/test_supply_curve_wind_dirs.py +++ b/tests/test_supply_curve_wind_dirs.py @@ -3,12 +3,14 @@ Supply Curve computation integrated tests """ import os + import pandas as pd -from pandas.testing import assert_frame_equal import pytest +from pandas.testing import assert_frame_equal from reV import TESTDATADIR from reV.supply_curve.supply_curve import CompetitiveWindFarms, SupplyCurve +from reV.utilities import MetaKeyName TRANS_COSTS = {'line_tie_in_cost': 200, 'line_cost': 1000, 'station_tie_in_cost': 50, 'center_tie_in_cost': 10, @@ -28,7 +30,7 @@ def test_competitive_wind_dirs(downwind): """Run CompetitiveWindFarms and verify results against baseline file.""" sc_points = CompetitiveWindFarms.run(WIND_DIRS, SC_POINTS, - n_dirs=2, sort_on='mean_lcoe', + n_dirs=2, sort_on=MetaKeyName.MEAN_LCOE, downwind=downwind) if downwind: @@ -77,7 +79,7 @@ def test_sc_full_wind_dirs(downwind): def test_sc_simple_wind_dirs(downwind): """Run the simple SC test and verify results against baseline file.""" sc = SupplyCurve(SC_POINTS, TRANS_TABLE, sc_features=MULTIPLIERS) - sc_out = sc.simple_sort(fcr=0.1,transmission_costs=TRANS_COSTS, + sc_out = sc.simple_sort(fcr=0.1, transmission_costs=TRANS_COSTS, wind_dirs=WIND_DIRS, downwind=downwind) if downwind: @@ -105,11 +107,11 @@ def test_upwind_exclusion(): 'sc_full_upwind.csv') sc_out = pd.read_csv(sc_out).sort_values('total_lcoe') - sc_point_gids = sc_out['sc_point_gid'].values.tolist() + sc_point_gids = sc_out[MetaKeyName.SC_POINT_GID].values.tolist() for _, row in sc_out.iterrows(): - sc_gid = row['sc_gid'] - sc_point_gids.remove(row['sc_point_gid']) - sc_point_gid = cwf['sc_point_gid', sc_gid] + sc_gid = row[MetaKeyName.SC_GID] + sc_point_gids.remove(row[MetaKeyName.SC_POINT_GID]) + sc_point_gid = cwf[MetaKeyName.SC_POINT_GID, sc_gid] for gid in cwf['upwind', sc_point_gid]: msg = 'Upwind gid {} was not excluded!'.format(gid) assert gid not in sc_point_gids, msg @@ -125,11 +127,11 @@ def test_upwind_downwind_exclusion(): 'sc_full_downwind.csv') sc_out = pd.read_csv(sc_out).sort_values('total_lcoe') - sc_point_gids = sc_out['sc_point_gid'].values.tolist() + sc_point_gids = sc_out[MetaKeyName.SC_POINT_GID].values.tolist() for _, row in sc_out.iterrows(): - sc_gid = row['sc_gid'] - sc_point_gids.remove(row['sc_point_gid']) - sc_point_gid = cwf['sc_point_gid', sc_gid] + sc_gid = row[MetaKeyName.SC_GID] + sc_point_gids.remove(row[MetaKeyName.SC_POINT_GID]) + sc_point_gid = cwf[MetaKeyName.SC_POINT_GID, sc_gid] for gid in cwf['upwind', sc_point_gid]: msg = 'Upwind gid {} was not excluded!'.format(gid) assert gid not in sc_point_gids, msg From cc4b85e18486f998c43c62f5582a3d3548d8aa09 Mon Sep 17 00:00:00 2001 From: bnb32 Date: Mon, 13 May 2024 07:39:05 -0600 Subject: [PATCH 09/61] linting after replacements --- reV/SAM/econ.py | 13 +- reV/SAM/generation.py | 912 ++++++---- reV/econ/econ.py | 259 +-- reV/generation/base.py | 461 +++-- reV/hybrids/hybrids.py | 442 +++-- reV/qa_qc/qa_qc.py | 334 ++-- reV/qa_qc/summary.py | 347 ++-- reV/rep_profiles/rep_profiles.py | 551 +++--- reV/supply_curve/aggregation.py | 491 ++++-- reV/supply_curve/competitive_wind_farms.py | 124 +- reV/supply_curve/extent.py | 116 +- reV/supply_curve/points.py | 716 +++++--- reV/supply_curve/sc_aggregation.py | 588 ++++--- reV/supply_curve/supply_curve.py | 841 ++++++---- reV/supply_curve/tech_mapping.py | 233 ++- tests/eagle.py | 15 +- tests/test_bespoke.py | 1487 ++++++++++------- tests/test_config.py | 252 +-- tests/test_curtailment.py | 276 +-- tests/test_econ_lcoe.py | 13 +- tests/test_econ_of_scale.py | 305 ++-- tests/test_econ_single_owner.py | 6 +- tests/test_econ_windbos.py | 22 +- tests/test_gen_5min.py | 5 +- tests/test_gen_forecast.py | 89 +- tests/test_gen_time_scale.py | 7 +- tests/test_gen_wave.py | 8 +- tests/test_gen_wind.py | 386 +++-- tests/test_handlers_multiyear.py | 18 +- tests/test_handlers_transmission.py | 196 ++- tests/test_hybrids.py | 462 +++-- tests/test_nrwal.py | 583 ++++--- tests/test_rep_profiles.py | 316 ++-- .../test_supply_curve_aggregation_friction.py | 104 +- tests/test_supply_curve_compute.py | 523 +++--- tests/test_supply_curve_points.py | 185 +- tests/test_supply_curve_sc_aggregation.py | 534 +++--- 37 files changed, 7645 insertions(+), 4575 deletions(-) diff --git a/reV/SAM/econ.py b/reV/SAM/econ.py index 4469ff0c6..355bc6619 100644 --- a/reV/SAM/econ.py +++ b/reV/SAM/econ.py @@ -4,17 +4,19 @@ Wraps the NREL-PySAM lcoefcr and singleowner modules with additional reV features. """ -from copy import deepcopy import logging -import numpy as np +from copy import deepcopy from warnings import warn + +import numpy as np import PySAM.Lcoefcr as PySamLCOE import PySAM.Singleowner as PySamSingleOwner -from reV.SAM.defaults import DefaultSingleOwner, DefaultLCOE from reV.handlers.outputs import Outputs -from reV.SAM.windbos import WindBos +from reV.SAM.defaults import DefaultLCOE, DefaultSingleOwner from reV.SAM.SAM import RevPySam +from reV.SAM.windbos import WindBos +from reV.utilities import MetaKeyName from reV.utilities.exceptions import SAMExecutionError logger = logging.getLogger(__name__) @@ -22,6 +24,7 @@ class Economic(RevPySam): """Base class for SAM economic models.""" + MODULE = None def __init__(self, sam_sys_inputs, site_sys_inputs=None, @@ -335,6 +338,7 @@ def reV_run(cls, site, site_df, inputs, output_request): class LCOE(Economic): """SAM LCOE model. """ + MODULE = 'lcoefcr' PYSAM = PySamLCOE @@ -463,6 +467,7 @@ def reV_run(cls, points_control, site_df, cf_file, year, class SingleOwner(Economic): """SAM single owner economic model. """ + MODULE = 'singleowner' PYSAM = PySamSingleOwner diff --git a/reV/SAM/generation.py b/reV/SAM/generation.py index d39d7c97c..67698483d 100644 --- a/reV/SAM/generation.py +++ b/reV/SAM/generation.py @@ -4,6 +4,7 @@ Wraps the NREL-PySAM pvwattsv5, windpower, and tcsmolensalt modules with additional reV features. """ + import copy import logging import os @@ -54,8 +55,15 @@ class AbstractSamGeneration(RevPySam, ScheduledLossesMixin, ABC): """Base class for SAM generation simulations.""" - def __init__(self, resource, meta, sam_sys_inputs, site_sys_inputs=None, - output_request=None, drop_leap=False): + def __init__( + self, + resource, + meta, + sam_sys_inputs, + site_sys_inputs=None, + output_request=None, + drop_leap=False, + ): """Initialize a SAM generation object. Parameters @@ -93,11 +101,15 @@ def __init__(self, resource, meta, sam_sys_inputs, site_sys_inputs=None, # don't pass resource to base class, # set in concrete generation classes instead - super().__init__(meta, sam_sys_inputs, output_request, - site_sys_inputs=site_sys_inputs) + super().__init__( + meta, + sam_sys_inputs, + output_request, + site_sys_inputs=site_sys_inputs, + ) # Set the site number using resource - if hasattr(resource, 'name'): + if hasattr(resource, "name"): self._site = resource.name else: self._site = None @@ -169,19 +181,24 @@ def _get_res_mean(resource, res_gid, output_request): out_req_nomeans = copy.deepcopy(output_request) res_mean = None idx = resource.sites.index(res_gid) - irrad_means = ('dni_mean', 'dhi_mean', 'ghi_mean', - 'clearsky_dni_mean', 'clearsky_dhi_mean', - 'clearsky_ghi_mean') - - if 'ws_mean' in out_req_nomeans: - out_req_nomeans.remove('ws_mean') + irrad_means = ( + "dni_mean", + "dhi_mean", + "ghi_mean", + "clearsky_dni_mean", + "clearsky_dhi_mean", + "clearsky_ghi_mean", + ) + + if "ws_mean" in out_req_nomeans: + out_req_nomeans.remove("ws_mean") res_mean = {} - res_mean['ws_mean'] = resource['mean_windspeed', idx] + res_mean["ws_mean"] = resource["mean_windspeed", idx] else: for var in resource.var_list: - label_1 = '{}_mean'.format(var) - label_2 = 'mean_{}'.format(var) + label_1 = "{}_mean".format(var) + label_2 = "mean_{}".format(var) if label_1 in out_req_nomeans: out_req_nomeans.remove(label_1) if res_mean is None: @@ -210,8 +227,9 @@ def check_resource_data(self, resource): if pd.isna(resource).any().any(): bad_vars = pd.isna(resource).any(axis=0) bad_vars = resource.columns[bad_vars].values.tolist() - msg = ('Found NaN values for site {} in variables {}' - .format(self.site, bad_vars)) + msg = "Found NaN values for site {} in variables {}".format( + self.site, bad_vars + ) logger.error(msg) raise InputError(msg) @@ -246,20 +264,30 @@ def tz_elev_check(sam_sys_inputs, site_sys_inputs, meta): if meta is not None: if sam_sys_inputs is not None: if MetaKeyName.ELEVATION in sam_sys_inputs: - meta[MetaKeyName.ELEVATION] = sam_sys_inputs[MetaKeyName.ELEVATION] + meta[MetaKeyName.ELEVATION] = sam_sys_inputs[ + MetaKeyName.ELEVATION + ] if MetaKeyName.TIMEZONE in sam_sys_inputs: - meta[MetaKeyName.TIMEZONE] = int(sam_sys_inputs[MetaKeyName.TIMEZONE]) + meta[MetaKeyName.TIMEZONE] = int( + sam_sys_inputs[MetaKeyName.TIMEZONE] + ) # site-specific inputs take priority over generic system inputs if site_sys_inputs is not None: if MetaKeyName.ELEVATION in site_sys_inputs: - meta[MetaKeyName.ELEVATION] = site_sys_inputs[MetaKeyName.ELEVATION] + meta[MetaKeyName.ELEVATION] = site_sys_inputs[ + MetaKeyName.ELEVATION + ] if MetaKeyName.TIMEZONE in site_sys_inputs: - meta[MetaKeyName.TIMEZONE] = int(site_sys_inputs[MetaKeyName.TIMEZONE]) + meta[MetaKeyName.TIMEZONE] = int( + site_sys_inputs[MetaKeyName.TIMEZONE] + ) if MetaKeyName.TIMEZONE not in meta: - msg = ('Need timezone input to run SAM gen. Not found in ' - 'resource meta or technology json input config.') + msg = ( + "Need timezone input to run SAM gen. Not found in " + "resource meta or technology json input config." + ) raise SAMExecutionError(msg) return meta @@ -281,7 +309,7 @@ def cf_mean(self): output : float Mean capacity factor (fractional). """ - return self['capacity_factor'] / 100 + return self["capacity_factor"] / 100 def cf_profile(self): """Get hourly capacity factor (frac) profile in local timezone. @@ -293,7 +321,7 @@ def cf_profile(self): 1D numpy array of capacity factor profile. Datatype is float32 and array length is 8760*time_interval. """ - return self.gen_profile() / self.sam_sys_inputs['system_capacity'] + return self.gen_profile() / self.sam_sys_inputs["system_capacity"] def annual_energy(self): """Get annual energy generation value in kWh from SAM. @@ -303,7 +331,7 @@ def annual_energy(self): output : float Annual energy generation (kWh). """ - return self['annual_energy'] + return self["annual_energy"] def energy_yield(self): """Get annual energy yield value in kwh/kw from SAM. @@ -313,7 +341,7 @@ def energy_yield(self): output : float Annual energy yield (kwh/kw). """ - return self['kwh_per_kw'] + return self["kwh_per_kw"] def gen_profile(self): """Get power generation profile (local timezone) in kW. @@ -325,7 +353,7 @@ def gen_profile(self): 1D array of hourly power generation in kW. Datatype is float32 and array length is 8760*time_interval. """ - return np.array(self['gen'], dtype=np.float32) + return np.array(self["gen"], dtype=np.float32) def collect_outputs(self, output_lookup=None): """Collect SAM output_request, convert timeseries outputs to UTC, and @@ -340,12 +368,13 @@ def collect_outputs(self, output_lookup=None): """ if output_lookup is None: - output_lookup = {'cf_mean': self.cf_mean, - 'cf_profile': self.cf_profile, - 'annual_energy': self.annual_energy, - 'energy_yield': self.energy_yield, - 'gen_profile': self.gen_profile, - } + output_lookup = { + "cf_mean": self.cf_mean, + "cf_profile": self.cf_profile, + "annual_energy": self.annual_energy, + "energy_yield": self.energy_yield, + "gen_profile": self.gen_profile, + } super().collect_outputs(output_lookup=output_lookup) @@ -354,19 +383,31 @@ def run_gen_and_econ(self): lcoe_out_reqs = None so_out_reqs = None - lcoe_vars = ('lcoe_fcr', 'fixed_charge_rate', 'capital_cost', - 'fixed_operating_cost', 'variable_operating_cost') - so_vars = ('ppa_price', 'lcoe_real', 'lcoe_nom', - 'project_return_aftertax_npv', 'flip_actual_irr', - 'gross_revenue') - if 'lcoe_fcr' in self.output_request: + lcoe_vars = ( + "lcoe_fcr", + "fixed_charge_rate", + "capital_cost", + "fixed_operating_cost", + "variable_operating_cost", + ) + so_vars = ( + "ppa_price", + "lcoe_real", + "lcoe_nom", + "project_return_aftertax_npv", + "flip_actual_irr", + "gross_revenue", + ) + if "lcoe_fcr" in self.output_request: lcoe_out_reqs = [r for r in self.output_request if r in lcoe_vars] - self.output_request = [r for r in self.output_request - if r not in lcoe_out_reqs] + self.output_request = [ + r for r in self.output_request if r not in lcoe_out_reqs + ] elif any(x in self.output_request for x in so_vars): so_out_reqs = [r for r in self.output_request if r in so_vars] - self.output_request = [r for r in self.output_request - if r not in so_out_reqs] + self.output_request = [ + r for r in self.output_request if r not in so_out_reqs + ] # Execute the SAM generation compute module (pvwattsv7, windpower, etc) self.run() @@ -374,7 +415,7 @@ def run_gen_and_econ(self): # Execute a follow-on SAM econ compute module # (lcoe_fcr, singleowner, etc) if lcoe_out_reqs is not None: - self.sam_sys_inputs['annual_energy'] = self.annual_energy() + self.sam_sys_inputs["annual_energy"] = self.annual_energy() lcoe = LCOE(self.sam_sys_inputs, output_request=lcoe_out_reqs) lcoe.assign_inputs() lcoe.execute() @@ -382,7 +423,7 @@ def run_gen_and_econ(self): self.outputs.update(lcoe.outputs) elif so_out_reqs is not None: - self.sam_sys_inputs['gen'] = self.gen_profile() + self.sam_sys_inputs["gen"] = self.gen_profile() so = SingleOwner(self.sam_sys_inputs, output_request=so_out_reqs) so.assign_inputs() so.execute() @@ -398,10 +439,18 @@ def run(self): self.collect_outputs() @classmethod - def reV_run(cls, points_control, res_file, site_df, - lr_res_file=None, output_request=('cf_mean',), - drop_leap=False, gid_map=None, nn_map=None, - bias_correct=None): + def reV_run( + cls, + points_control, + res_file, + site_df, + lr_res_file=None, + output_request=("cf_mean",), + drop_leap=False, + gid_map=None, + nn_map=None, + bias_correct=None, + ): """Execute SAM generation based on a reV points control instance. Parameters @@ -465,24 +514,26 @@ def reV_run(cls, points_control, res_file, site_df, out = {} # Get the RevPySam resource object - resources = RevPySam.get_sam_res(res_file, - points_control.project_points, - points_control.project_points.tech, - output_request=output_request, - gid_map=gid_map, - lr_res_file=lr_res_file, - nn_map=nn_map, - bias_correct=bias_correct) + resources = RevPySam.get_sam_res( + res_file, + points_control.project_points, + points_control.project_points.tech, + output_request=output_request, + gid_map=gid_map, + lr_res_file=lr_res_file, + nn_map=nn_map, + bias_correct=bias_correct, + ) # run resource through curtailment filter if applicable curtailment = points_control.project_points.curtailment if curtailment is not None: - resources = curtail(resources, curtailment, - random_seed=curtailment.random_seed) + resources = curtail( + resources, curtailment, random_seed=curtailment.random_seed + ) # iterate through project_points gen_gid values for gen_gid in points_control.project_points.sites: - # Lookup the resource gid if there's a mapping and get the resource # data from the SAMResource object using the res_gid. res_gid = gen_gid if gid_map is None else gid_map[gen_gid] @@ -495,15 +546,21 @@ def reV_run(cls, points_control, res_file, site_df, _, inputs = points_control.project_points[gen_gid] # get resource data pass-throughs and resource means - res_outs, out_req_cleaned = cls._get_res(site_res_df, - output_request) - res_mean, out_req_cleaned = cls._get_res_mean(resources, res_gid, - out_req_cleaned) + res_outs, out_req_cleaned = cls._get_res( + site_res_df, output_request + ) + res_mean, out_req_cleaned = cls._get_res_mean( + resources, res_gid, out_req_cleaned + ) # iterate through requested sites. - sim = cls(resource=site_res_df, meta=site_meta, - sam_sys_inputs=inputs, output_request=out_req_cleaned, - site_sys_inputs=dict(site_df.loc[gen_gid, :])) + sim = cls( + resource=site_res_df, + meta=site_meta, + sam_sys_inputs=inputs, + output_request=out_req_cleaned, + site_sys_inputs=dict(site_df.loc[gen_gid, :]), + ) sim.run_gen_and_econ() # collect outputs to dictout @@ -521,9 +578,18 @@ def reV_run(cls, points_control, res_file, site_df, class AbstractSamGenerationFromWeatherFile(AbstractSamGeneration, ABC): """Base class for running sam generation with a weather file on disk.""" - WF_META_DROP_COLS = {MetaKeyName.ELEVATION, MetaKeyName.TIMEZONE, 'country', 'state', 'county', - 'urban', 'population', 'landcover', MetaKeyName.LATITUDE, - MetaKeyName.LONGITUDE} + WF_META_DROP_COLS = { + MetaKeyName.ELEVATION, + MetaKeyName.TIMEZONE, + "country", + "state", + "county", + "urban", + "population", + "landcover", + MetaKeyName.LATITUDE, + MetaKeyName.LONGITUDE, + } @property @abstractmethod @@ -588,59 +654,62 @@ def _create_pysam_wfile(self, resource, meta): """ # pylint: disable=attribute-defined-outside-init,consider-using-with self._temp_dir = TemporaryDirectory() - fname = os.path.join(self._temp_dir.name, 'weather.csv') - logger.debug('Creating PySAM weather data file: {}'.format(fname)) + fname = os.path.join(self._temp_dir.name, "weather.csv") + logger.debug("Creating PySAM weather data file: {}".format(fname)) # ------- Process metadata m = pd.DataFrame(meta).T timezone = m[MetaKeyName.TIMEZONE] - m['Source'] = 'NSRDB' - m['Location ID'] = meta.name - m['City'] = '-' - m['State'] = m['state'].apply(lambda x: '-' if x == 'None' else x) - m['Country'] = m['country'].apply(lambda x: '-' if x == 'None' else x) - m['Latitude'] = m[MetaKeyName.LATITUDE] + m["Source"] = "NSRDB" + m["Location ID"] = meta.name + m["City"] = "-" + m["State"] = m["state"].apply(lambda x: "-" if x == "None" else x) + m["Country"] = m["country"].apply(lambda x: "-" if x == "None" else x) + m["Latitude"] = m[MetaKeyName.LATITUDE] m[MetaKeyName.LONGITUDE] = m[MetaKeyName.LONGITUDE] - m['Time Zone'] = timezone + m["Time Zone"] = timezone m[MetaKeyName.ELEVATION] = m[MetaKeyName.ELEVATION] - m['Local Time Zone'] = timezone - m['Dew Point Units'] = 'c' - m['DHI Units'] = 'w/m2' - m['DNI Units'] = 'w/m2' - m['Temperature Units'] = 'c' - m['Pressure Units'] = 'mbar' - m['Wind Speed'] = 'm/s' + m["Local Time Zone"] = timezone + m["Dew Point Units"] = "c" + m["DHI Units"] = "w/m2" + m["DNI Units"] = "w/m2" + m["Temperature Units"] = "c" + m["Pressure Units"] = "mbar" + m["Wind Speed"] = "m/s" keep_cols = [c for c in m.columns if c not in self.WF_META_DROP_COLS] - m[keep_cols].to_csv(fname, index=False, mode='w') + m[keep_cols].to_csv(fname, index=False, mode="w") # --------- Process data - var_map = {'dni': 'DNI', - 'dhi': 'DHI', - 'wind_speed': 'Wind Speed', - 'air_temperature': 'Temperature', - 'dew_point': 'Dew Point', - 'surface_pressure': 'Pressure', - } - resource = resource.rename(mapper=var_map, axis='columns') + var_map = { + "dni": "DNI", + "dhi": "DHI", + "wind_speed": "Wind Speed", + "air_temperature": "Temperature", + "dew_point": "Dew Point", + "surface_pressure": "Pressure", + } + resource = resource.rename(mapper=var_map, axis="columns") time_index = resource.index # Adjust from UTC to local time - local = np.roll(resource.values, int(timezone * self.time_interval), - axis=0) - resource = pd.DataFrame(local, columns=resource.columns, - index=time_index) + local = np.roll( + resource.values, int(timezone * self.time_interval), axis=0 + ) + resource = pd.DataFrame( + local, columns=resource.columns, index=time_index + ) mask = (time_index.month == 2) & (time_index.day == 29) time_index = time_index[~mask] df = pd.DataFrame(index=time_index) - df['Year'] = time_index.year - df['Month'] = time_index.month - df['Day'] = time_index.day - df['Hour'] = time_index.hour - df['Minute'] = time_index.minute + df["Year"] = time_index.year + df["Month"] = time_index.month + df["Day"] = time_index.day + df["Hour"] = time_index.hour + df["Minute"] = time_index.minute df = df.join(resource.loc[~mask]) - df.to_csv(fname, index=False, mode='a') + df.to_csv(fname, index=False, mode="a") return fname @@ -709,83 +778,100 @@ def set_resource_data(self, resource, meta): self.time_interval = self.get_time_interval(resource.index.values) # map resource data names to SAM required data names - var_map = {'dni': 'dn', - 'dhi': 'df', - 'ghi': 'gh', - 'clearskydni': 'dn', - 'clearskydhi': 'df', - 'clearskyghi': 'gh', - 'windspeed': 'wspd', - 'airtemperature': 'tdry', - 'temperature': 'tdry', - 'temp': 'tdry', - 'dewpoint': 'tdew', - 'surfacepressure': 'pres', - 'pressure': 'pres', - 'surfacealbedo': 'albedo', - } - lower_case = {k: k.lower().replace(' ', '').replace('_', '') - for k in resource.columns} - irrad_vars = ['dn', 'df', 'gh'] - - resource = resource.rename(mapper=lower_case, axis='columns') - resource = resource.rename(mapper=var_map, axis='columns') + var_map = { + "dni": "dn", + "dhi": "df", + "ghi": "gh", + "clearskydni": "dn", + "clearskydhi": "df", + "clearskyghi": "gh", + "windspeed": "wspd", + "airtemperature": "tdry", + "temperature": "tdry", + "temp": "tdry", + "dewpoint": "tdew", + "surfacepressure": "pres", + "pressure": "pres", + "surfacealbedo": "albedo", + } + lower_case = { + k: k.lower().replace(" ", "").replace("_", "") + for k in resource.columns + } + irrad_vars = ["dn", "df", "gh"] + + resource = resource.rename(mapper=lower_case, axis="columns") + resource = resource.rename(mapper=var_map, axis="columns") time_index = resource.index - resource = {k: np.array(v) for (k, v) in - resource.to_dict(orient='list').items()} + resource = { + k: np.array(v) + for (k, v) in resource.to_dict(orient="list").items() + } # set resource variables for var, arr in resource.items(): - if var != 'time_index': - + if var != "time_index": # ensure that resource array length is multiple of 8760 arr = self.ensure_res_len(arr, time_index) - n_roll = int(self._meta[MetaKeyName.TIMEZONE] * self.time_interval) + n_roll = int( + self._meta[MetaKeyName.TIMEZONE] * self.time_interval + ) arr = np.roll(arr, n_roll) if var in irrad_vars: if np.min(arr) < 0: - warn('Solar irradiance variable "{}" has a minimum ' - 'value of {}. Truncating to zero.' - .format(var, np.min(arr)), SAMInputWarning) + warn( + 'Solar irradiance variable "{}" has a minimum ' + "value of {}. Truncating to zero.".format( + var, np.min(arr) + ), + SAMInputWarning, + ) arr = np.where(arr < 0, 0, arr) resource[var] = arr.tolist() - resource['lat'] = meta[MetaKeyName.LATITUDE] - resource['lon'] = meta[MetaKeyName.LONGITUDE] - resource['tz'] = meta[MetaKeyName.TIMEZONE] + resource["lat"] = meta[MetaKeyName.LATITUDE] + resource["lon"] = meta[MetaKeyName.LONGITUDE] + resource["tz"] = meta[MetaKeyName.TIMEZONE] if MetaKeyName.ELEVATION in meta: - resource['elev'] = meta[MetaKeyName.ELEVATION] + resource["elev"] = meta[MetaKeyName.ELEVATION] else: - resource['elev'] = 0.0 + resource["elev"] = 0.0 time_index = self.ensure_res_len(time_index, time_index) - resource['minute'] = time_index.minute - resource['hour'] = time_index.hour - resource['month'] = time_index.month - resource['year'] = time_index.year - resource['day'] = time_index.day + resource["minute"] = time_index.minute + resource["hour"] = time_index.hour + resource["month"] = time_index.month + resource["year"] = time_index.year + resource["day"] = time_index.day - if 'albedo' in resource: - self['albedo'] = self.agg_albedo( - time_index, resource.pop('albedo')) + if "albedo" in resource: + self["albedo"] = self.agg_albedo( + time_index, resource.pop("albedo") + ) - self['solar_resource_data'] = resource + self["solar_resource_data"] = resource class AbstractSamPv(AbstractSamSolar, ABC): - """Photovoltaic (PV) generation with either pvwatts of detailed pv. - """ + """Photovoltaic (PV) generation with either pvwatts of detailed pv.""" # set these class attrs in concrete subclasses MODULE = None PYSAM = None # pylint: disable=line-too-long - def __init__(self, resource, meta, sam_sys_inputs, site_sys_inputs=None, - output_request=None, drop_leap=False): + def __init__( + self, + resource, + meta, + sam_sys_inputs, + site_sys_inputs=None, + output_request=None, + drop_leap=False, + ): """Initialize a SAM solar object. See the PySAM :py:class:`~PySAM.Pvwattsv8.Pvwattsv8` (or older @@ -884,10 +970,14 @@ def __init__(self, resource, meta, sam_sys_inputs, site_sys_inputs=None, meta = self._parse_meta(meta) sam_sys_inputs = self.set_latitude_tilt_az(sam_sys_inputs, meta) - super().__init__(resource, meta, sam_sys_inputs, - site_sys_inputs=site_sys_inputs, - output_request=output_request, - drop_leap=drop_leap) + super().__init__( + resource, + meta, + sam_sys_inputs, + site_sys_inputs=site_sys_inputs, + output_request=output_request, + drop_leap=drop_leap, + ) def set_resource_data(self, resource, meta): """Set NSRDB resource data arrays. @@ -911,15 +1001,19 @@ def set_resource_data(self, resource, meta): respectively. """ - bad_location_input = ((meta[MetaKeyName.LATITUDE] < -90) - | (meta[MetaKeyName.LATITUDE] > 90) - | (meta[MetaKeyName.LONGITUDE] < -180) - | (meta[MetaKeyName.LONGITUDE] > 180)) + bad_location_input = ( + (meta[MetaKeyName.LATITUDE] < -90) + | (meta[MetaKeyName.LATITUDE] > 90) + | (meta[MetaKeyName.LONGITUDE] < -180) + | (meta[MetaKeyName.LONGITUDE] > 180) + ) if bad_location_input.any(): - raise ValueError("Detected latitude/longitude values outside of " - "the range -90 to 90 and -180 to 180, " - "respectively. Please ensure input resource data" - "locations conform to these ranges. ") + raise ValueError( + "Detected latitude/longitude values outside of " + "the range -90 to 90 and -180 to 180, " + "respectively. Please ensure input resource data" + "locations conform to these ranges. " + ) return super().set_resource_data(resource, meta) @staticmethod @@ -940,35 +1034,40 @@ def set_latitude_tilt_az(sam_sys_inputs, meta): sam_sys_inputs : dict Site-agnostic SAM system model inputs arguments. If for a pv simulation the "tilt" parameter was originally not - present or set to 'lat' or MetaKeyName.LATITUDE, the tilt will be set to - the absolute value of the latitude found in meta and the azimuth - will be 180 if lat>0, 0 if lat<0. + present or set to 'lat' or MetaKeyName.LATITUDE, the tilt will be + set to the absolute value of the latitude found in meta and the + azimuth will be 180 if lat>0, 0 if lat<0. """ set_tilt = False if sam_sys_inputs is not None and meta is not None: - if 'tilt' not in sam_sys_inputs: - warn('No tilt specified, setting at latitude.', - SAMInputWarning) + if "tilt" not in sam_sys_inputs: + warn( + "No tilt specified, setting at latitude.", SAMInputWarning + ) set_tilt = True - elif (sam_sys_inputs['tilt'] == 'lat' - or sam_sys_inputs['tilt'] == MetaKeyName.LATITUDE): + elif ( + sam_sys_inputs["tilt"] == "lat" + or sam_sys_inputs["tilt"] == MetaKeyName.LATITUDE + ): set_tilt = True if set_tilt: # set tilt to abs(latitude) - sam_sys_inputs['tilt'] = np.abs(meta[MetaKeyName.LATITUDE]) + sam_sys_inputs["tilt"] = np.abs(meta[MetaKeyName.LATITUDE]) if meta[MetaKeyName.LATITUDE] > 0: # above the equator, az = 180 - sam_sys_inputs['azimuth'] = 180 + sam_sys_inputs["azimuth"] = 180 else: # below the equator, az = 0 - sam_sys_inputs['azimuth'] = 0 - - logger.debug('Tilt specified at "latitude", setting tilt to: {}, ' - 'azimuth to: {}' - .format(sam_sys_inputs['tilt'], - sam_sys_inputs['azimuth'])) + sam_sys_inputs["azimuth"] = 0 + + logger.debug( + 'Tilt specified at "latitude", setting tilt to: {}, ' + "azimuth to: {}".format( + sam_sys_inputs["tilt"], sam_sys_inputs["azimuth"] + ) + ) return sam_sys_inputs def system_capacity_ac(self): @@ -981,8 +1080,10 @@ def system_capacity_ac(self): cf_profile : float AC nameplate = DC nameplate / ILR """ - return (self.sam_sys_inputs['system_capacity'] - / self.sam_sys_inputs['dc_ac_ratio']) + return ( + self.sam_sys_inputs["system_capacity"] + / self.sam_sys_inputs["dc_ac_ratio"] + ) def cf_mean(self): """Get mean capacity factor (fractional) from SAM. @@ -995,7 +1096,7 @@ def cf_mean(self): Mean capacity factor (fractional). PV CF is calculated as AC power / DC nameplate. """ - return self['capacity_factor'] / 100 + return self["capacity_factor"] / 100 def cf_mean_ac(self): """Get mean AC capacity factor (fractional) from SAM. @@ -1008,7 +1109,7 @@ def cf_mean_ac(self): Mean AC capacity factor (fractional). PV AC CF is calculated as AC power / AC nameplate. """ - return self['capacity_factor_ac'] / 100 + return self["capacity_factor_ac"] / 100 def cf_profile(self): """Get hourly capacity factor (frac) profile in local timezone. @@ -1023,7 +1124,7 @@ def cf_profile(self): Datatype is float32 and array length is 8760*time_interval. PV CF is calculated as AC power / DC nameplate. """ - return self.gen_profile() / self.sam_sys_inputs['system_capacity'] + return self.gen_profile() / self.sam_sys_inputs["system_capacity"] def cf_profile_ac(self): """Get hourly AC capacity factor (frac) profile in local timezone. @@ -1052,7 +1153,7 @@ def gen_profile(self): 1D array of AC inverter power generation in kW. Datatype is float32 and array length is 8760*time_interval. """ - return np.array(self['gen'], dtype=np.float32) + return np.array(self["gen"], dtype=np.float32) def ac(self): """Get AC inverter power generation profile (local timezone) in kW. @@ -1064,7 +1165,7 @@ def ac(self): 1D array of AC inverter power generation in kW. Datatype is float32 and array length is 8760*time_interval. """ - return np.array(self['ac'], dtype=np.float32) / 1000 + return np.array(self["ac"], dtype=np.float32) / 1000 def dc(self): """ @@ -1077,7 +1178,7 @@ def dc(self): 1D array of DC array power generation in kW. Datatype is float32 and array length is 8760*time_interval. """ - return np.array(self['dc'], dtype=np.float32) / 1000 + return np.array(self["dc"], dtype=np.float32) / 1000 def clipped_power(self): """ @@ -1113,27 +1214,27 @@ def collect_outputs(self, output_lookup=None): """ if output_lookup is None: - output_lookup = {'cf_mean': self.cf_mean, - 'cf_mean_ac': self.cf_mean_ac, - 'cf_profile': self.cf_profile, - 'cf_profile_ac': self.cf_profile_ac, - 'annual_energy': self.annual_energy, - 'energy_yield': self.energy_yield, - 'gen_profile': self.gen_profile, - 'ac': self.ac, - 'dc': self.dc, - 'clipped_power': self.clipped_power, - 'system_capacity_ac': self.system_capacity_ac, - } + output_lookup = { + "cf_mean": self.cf_mean, + "cf_mean_ac": self.cf_mean_ac, + "cf_profile": self.cf_profile, + "cf_profile_ac": self.cf_profile_ac, + "annual_energy": self.annual_energy, + "energy_yield": self.energy_yield, + "gen_profile": self.gen_profile, + "ac": self.ac, + "dc": self.dc, + "clipped_power": self.clipped_power, + "system_capacity_ac": self.system_capacity_ac, + } super().collect_outputs(output_lookup=output_lookup) class PvWattsv5(AbstractSamPv): - """Photovoltaic (PV) generation with pvwattsv5. - """ + """Photovoltaic (PV) generation with pvwattsv5.""" - MODULE = 'pvwattsv5' + MODULE = "pvwattsv5" PYSAM = PySamPv5 @staticmethod @@ -1148,10 +1249,9 @@ def default(): class PvWattsv7(AbstractSamPv): - """Photovoltaic (PV) generation with pvwattsv7. - """ + """Photovoltaic (PV) generation with pvwattsv7.""" - MODULE = 'pvwattsv7' + MODULE = "pvwattsv7" PYSAM = PySamPv7 @staticmethod @@ -1166,10 +1266,9 @@ def default(): class PvWattsv8(AbstractSamPv): - """Photovoltaic (PV) generation with pvwattsv8. - """ + """Photovoltaic (PV) generation with pvwattsv8.""" - MODULE = 'pvwattsv8' + MODULE = "pvwattsv8" PYSAM = PySamPv8 @staticmethod @@ -1186,7 +1285,7 @@ def default(): class PvSamv1(AbstractSamPv): """Detailed PV model""" - MODULE = 'Pvsamv1' + MODULE = "Pvsamv1" PYSAM = PySamDetailedPv def ac(self): @@ -1199,7 +1298,7 @@ def ac(self): 1D array of AC inverter power generation in kW. Datatype is float32 and array length is 8760*time_interval. """ - return np.array(self['gen'], dtype=np.float32) + return np.array(self["gen"], dtype=np.float32) def dc(self): """ @@ -1212,7 +1311,7 @@ def dc(self): 1D array of DC array power generation in kW. Datatype is float32 and array length is 8760*time_interval. """ - return np.array(self['dc_net'], dtype=np.float32) + return np.array(self["dc_net"], dtype=np.float32) @staticmethod def default(): @@ -1226,10 +1325,9 @@ def default(): class TcsMoltenSalt(AbstractSamSolar): - """Concentrated Solar Power (CSP) generation with tower molten salt - """ + """Concentrated Solar Power (CSP) generation with tower molten salt""" - MODULE = 'tcsmolten_salt' + MODULE = "tcsmolten_salt" PYSAM = PySamCSP def cf_profile(self): @@ -1243,7 +1341,7 @@ def cf_profile(self): 1D numpy array of capacity factor profile. Datatype is float32 and array length is 8760*time_interval. """ - x = np.abs(self.gen_profile() / self.sam_sys_inputs['system_capacity']) + x = np.abs(self.gen_profile() / self.sam_sys_inputs["system_capacity"]) return x @staticmethod @@ -1262,9 +1360,9 @@ class SolarWaterHeat(AbstractSamGenerationFromWeatherFile): Solar Water Heater generation """ - MODULE = 'solarwaterheat' + MODULE = "solarwaterheat" PYSAM = PySamSwh - PYSAM_WEATHER_TAG = 'solar_resource_file' + PYSAM_WEATHER_TAG = "solar_resource_file" @staticmethod def default(): @@ -1282,9 +1380,9 @@ class LinearDirectSteam(AbstractSamGenerationFromWeatherFile): Process heat linear Fresnel direct steam generation """ - MODULE = 'lineardirectsteam' + MODULE = "lineardirectsteam" PYSAM = PySamLds - PYSAM_WEATHER_TAG = 'file_name' + PYSAM_WEATHER_TAG = "file_name" def cf_mean(self): """Calculate mean capacity factor (fractional) from SAM. @@ -1294,10 +1392,11 @@ def cf_mean(self): output : float Mean capacity factor (fractional). """ - net_power = self['annual_field_energy'] \ - - self['annual_thermal_consumption'] # kW-hr + net_power = ( + self["annual_field_energy"] - self["annual_thermal_consumption"] + ) # kW-hr # q_pb_des is in MW, convert to kW-hr - name_plate = self['q_pb_des'] * 8760 * 1000 + name_plate = self["q_pb_des"] * 8760 * 1000 return net_power / name_plate @@ -1317,9 +1416,9 @@ class TroughPhysicalHeat(AbstractSamGenerationFromWeatherFile): Trough Physical Process Heat generation """ - MODULE = 'troughphysicalheat' + MODULE = "troughphysicalheat" PYSAM = PySamTpph - PYSAM_WEATHER_TAG = 'file_name' + PYSAM_WEATHER_TAG = "file_name" def cf_mean(self): """Calculate mean capacity factor (fractional) from SAM. @@ -1329,10 +1428,11 @@ def cf_mean(self): output : float Mean capacity factor (fractional). """ - net_power = self['annual_gross_energy'] \ - - self['annual_thermal_consumption'] # kW-hr + net_power = ( + self["annual_gross_energy"] - self["annual_thermal_consumption"] + ) # kW-hr # q_pb_des is in MW, convert to kW-hr - name_plate = self['q_pb_design'] * 8760 * 1000 + name_plate = self["q_pb_design"] * 8760 * 1000 return net_power / name_plate @@ -1525,7 +1625,7 @@ class Geothermal(AbstractSamGenerationFromWeatherFile): # but SAM errors out if it's exceeded. MAX_RT_TO_EGS_RATIO = 1.134324 """Max value of ``resource_temperature``/``EGS_plan_design_temperature``""" - MODULE = 'geothermal' + MODULE = "geothermal" PYSAM = PySamGeothermal PYSAM_WEATHER_TAG = "file_name" _RESOURCE_POTENTIAL_MULT = 1.001 @@ -1551,14 +1651,16 @@ def cf_profile(self): 1D numpy array of capacity factor profile. Datatype is float32 and array length is 8760*time_interval. """ - return self.gen_profile() / self.sam_sys_inputs['nameplate'] + return self.gen_profile() / self.sam_sys_inputs["nameplate"] def assign_inputs(self): """Assign the self.sam_sys_inputs attribute to the PySAM object.""" if self.sam_sys_inputs.get("ui_calculations_only"): - msg = ('reV requires model run - cannot set ' - '"ui_calculations_only" to `True` (1). Automatically ' - 'setting to `False` (0)!') + msg = ( + "reV requires model run - cannot set " + '"ui_calculations_only" to `True` (1). Automatically ' + "setting to `False` (0)!" + ) logger.warning(msg) warn(msg) self.sam_sys_inputs["ui_calculations_only"] = 0 @@ -1596,13 +1698,19 @@ def _set_resource_temperature(self, resource): """Set resource temp from data if user did not specify it.""" if "resource_temp" in self.sam_sys_inputs: - logger.debug("Found 'resource_temp' value in SAM config: {:.2f}" - .format(self.sam_sys_inputs["resource_temp"])) + logger.debug( + "Found 'resource_temp' value in SAM config: {:.2f}".format( + self.sam_sys_inputs["resource_temp"] + ) + ) return val = set(resource["temperature"].unique()) - logger.debug("Found {} value(s) for 'temperature' in resource data" - .format(len(val))) + logger.debug( + "Found {} value(s) for 'temperature' in resource data".format( + len(val) + ) + ) if len(val) > 1: msg = ("Found multiple values for 'temperature' for site {}: {}" .format(self.site, val)) @@ -1610,9 +1718,10 @@ def _set_resource_temperature(self, resource): raise InputError(msg) val = val.pop() - logger.debug("Input 'resource_temp' not found in SAM config - setting " - "to {:.2f} based on input resource data." - .format(val)) + logger.debug( + "Input 'resource_temp' not found in SAM config - setting " + "to {:.2f} based on input resource data.".format(val) + ) self.sam_sys_inputs["resource_temp"] = val def _set_egs_plant_design_temperature(self): @@ -1625,29 +1734,40 @@ def _set_egs_plant_design_temperature(self): resource_temp = self.sam_sys_inputs["resource_temp"] if set_egs_pdt_to_rt: - msg = ('Setting EGS plant design temperature ({}C) to match ' - 'resource temperature ({}C)' - .format(egs_plant_design_temp, resource_temp)) + msg = ( + "Setting EGS plant design temperature ({}C) to match " + "resource temperature ({}C)".format( + egs_plant_design_temp, resource_temp + ) + ) logger.info(msg) self.sam_sys_inputs["design_temp"] = resource_temp return if egs_plant_design_temp > resource_temp: - msg = ('EGS plant design temperature ({}C) exceeds resource ' - 'temperature ({}C). Lowering EGS plant design temperature ' - 'to match resource temperature' - .format(egs_plant_design_temp, resource_temp)) + msg = ( + "EGS plant design temperature ({}C) exceeds resource " + "temperature ({}C). Lowering EGS plant design temperature " + "to match resource temperature".format( + egs_plant_design_temp, resource_temp + ) + ) logger.warning(msg) warn(msg) self.sam_sys_inputs["design_temp"] = resource_temp return if resource_temp / egs_plant_design_temp > self.MAX_RT_TO_EGS_RATIO: - msg = ('EGS plant design temperature ({}C) is lower than resource ' - 'temperature ({}C) by more than a factor of {}. Increasing ' - 'EGS plant design temperature to match resource temperature' - .format(egs_plant_design_temp, resource_temp, - self.MAX_RT_TO_EGS_RATIO)) + msg = ( + "EGS plant design temperature ({}C) is lower than resource " + "temperature ({}C) by more than a factor of {}. Increasing " + "EGS plant design temperature to match resource temperature" + .format( + egs_plant_design_temp, + resource_temp, + self.MAX_RT_TO_EGS_RATIO, + ) + ) logger.warning(msg) warn(msg) self.sam_sys_inputs["design_temp"] = resource_temp @@ -1656,9 +1776,13 @@ def _set_nameplate_to_match_resource_potential(self, resource): """Set the nameplate capacity to match the resource potential.""" if "nameplate" in self.sam_sys_inputs: - msg = ('Found "nameplate" input in config! Resource potential ' - 'from input data will be ignored. Nameplate capacity is {}' - .format(self.sam_sys_inputs["nameplate"])) + msg = ( + 'Found "nameplate" input in config! Resource potential ' + "from input data will be ignored. Nameplate capacity is {}" + .format( + self.sam_sys_inputs["nameplate"] + ) + ) logger.info(msg) return @@ -1692,63 +1816,82 @@ def _set_resource_potential_to_match_gross_output(self): self.sam_sys_inputs["resource_potential"] = -1 return - gross_gen = (self.pysam.Outputs.gross_output - * self._RESOURCE_POTENTIAL_MULT) + gross_gen = ( + self.pysam.Outputs.gross_output * self._RESOURCE_POTENTIAL_MULT + ) if "resource_potential" in self.sam_sys_inputs: - msg = ('Setting "resource_potential" is not allowed! Updating ' - 'user input of {} to match the gross generation: {}' - .format(self.sam_sys_inputs["resource_potential"], - gross_gen)) + msg = ( + 'Setting "resource_potential" is not allowed! Updating ' + "user input of {} to match the gross generation: {}".format( + self.sam_sys_inputs["resource_potential"], gross_gen + ) + ) logger.warning(msg) warn(msg) - logger.debug("Setting the resource potential to {} MW" - .format(gross_gen)) + logger.debug( + "Setting the resource potential to {} MW".format(gross_gen) + ) self.sam_sys_inputs["resource_potential"] = gross_gen - ncw = self.sam_sys_inputs.pop("num_confirmation_wells", - self._DEFAULT_NUM_CONFIRMATION_WELLS) + ncw = self.sam_sys_inputs.pop( + "num_confirmation_wells", self._DEFAULT_NUM_CONFIRMATION_WELLS + ) self.sam_sys_inputs["prod_and_inj_wells_to_drill"] = ( self.pysam.Outputs.num_wells_getem_output - ncw - + self.pysam.Outputs.num_wells_getem_inj) + + self.pysam.Outputs.num_wells_getem_inj + ) self["ui_calculations_only"] = 0 def _set_costs(self): """Set the costs based on gross plant generation.""" - plant_size_kw = (self.sam_sys_inputs["resource_potential"] - / self._RESOURCE_POTENTIAL_MULT) * 1000 + plant_size_kw = ( + self.sam_sys_inputs["resource_potential"] + / self._RESOURCE_POTENTIAL_MULT + ) * 1000 cc_per_kw = self.sam_sys_inputs.pop("capital_cost_per_kw", None) if cc_per_kw is not None: capital_cost = cc_per_kw * plant_size_kw - logger.debug("Setting the capital_cost to ${:,.2f}" - .format(capital_cost)) + logger.debug( + "Setting the capital_cost to ${:,.2f}".format(capital_cost) + ) self.sam_sys_inputs["capital_cost"] = capital_cost dc_per_well = self.sam_sys_inputs.pop("drill_cost_per_well", None) - num_wells = self.sam_sys_inputs.pop("prod_and_inj_wells_to_drill", - None) + num_wells = self.sam_sys_inputs.pop( + "prod_and_inj_wells_to_drill", None + ) if dc_per_well is not None: if num_wells is None: - msg = ('Could not determine number of wells to be drilled. ' - 'No drilling costs added!') + msg = ( + "Could not determine number of wells to be drilled. " + "No drilling costs added!" + ) logger.warning(msg) warn(msg) else: capital_cost = self.sam_sys_inputs["capital_cost"] drill_cost = dc_per_well * num_wells - logger.debug("Setting the drilling cost to ${:,.2f} " - "({:.2f} wells at ${:,.2f} per well)" - .format(drill_cost, num_wells, dc_per_well)) + logger.debug( + "Setting the drilling cost to ${:,.2f} " + "({:.2f} wells at ${:,.2f} per well)".format( + drill_cost, num_wells, dc_per_well + ) + ) self.sam_sys_inputs["capital_cost"] = capital_cost + drill_cost - foc_per_kw = self.sam_sys_inputs.pop("fixed_operating_cost_per_kw", - None) + foc_per_kw = self.sam_sys_inputs.pop( + "fixed_operating_cost_per_kw", None + ) if foc_per_kw is not None: fixed_operating_cost = foc_per_kw * plant_size_kw - logger.debug("Setting the fixed_operating_cost to ${:,.2f}" - .format(capital_cost)) + logger.debug( + "Setting the fixed_operating_cost to ${:,.2f}".format( + capital_cost + ) + ) self.sam_sys_inputs["fixed_operating_cost"] = fixed_operating_cost def _create_pysam_wfile(self, resource, meta): @@ -1782,16 +1925,23 @@ def _create_pysam_wfile(self, resource, meta): """ # pylint: disable=attribute-defined-outside-init, consider-using-with self._temp_dir = TemporaryDirectory() - fname = os.path.join(self._temp_dir.name, 'weather.csv') - logger.debug('Creating PySAM weather data file: {}'.format(fname)) + fname = os.path.join(self._temp_dir.name, "weather.csv") + logger.debug("Creating PySAM weather data file: {}".format(fname)) # ------- Process metadata m = pd.DataFrame(meta).T - m = m.rename({"latitude": "Latitude", "longitude": "Longitude", - "timezone": "Time Zone"}, axis=1) - - m[["Latitude", "Longitude", "Time Zone"]].to_csv(fname, index=False, - mode='w') + m = m.rename( + { + "latitude": "Latitude", + "longitude": "Longitude", + "timezone": "Time Zone", + }, + axis=1, + ) + + m[["Latitude", "Longitude", "Time Zone"]].to_csv( + fname, index=False, mode="w" + ) # --------- Process data, blank for geothermal time_index = resource.index @@ -1799,12 +1949,12 @@ def _create_pysam_wfile(self, resource, meta): time_index = time_index[~mask] df = pd.DataFrame(index=time_index) - df['Year'] = time_index.year - df['Month'] = time_index.month - df['Day'] = time_index.day - df['Hour'] = time_index.hour - df['Minute'] = time_index.minute - df.to_csv(fname, index=False, mode='a') + df["Year"] = time_index.year + df["Month"] = time_index.month + df["Day"] = time_index.day + df["Hour"] = time_index.hour + df["Minute"] = time_index.minute + df.to_csv(fname, index=False, mode="a") return fname @@ -1813,8 +1963,11 @@ def run_gen_and_econ(self): try: super().run_gen_and_econ() except SAMExecutionError as e: - logger.error("Skipping site {}; received sam error: {}" - .format(self._site, str(e))) + logger.error( + "Skipping site {}; received sam error: {}".format( + self._site, str(e) + ) + ) self.outputs = {} @@ -1913,10 +2066,9 @@ def __init__(self, *args, **kwargs): class WindPower(AbstractSamWind): - """Class for Wind generation from SAM - """ + """Class for Wind generation from SAM""" - MODULE = 'windpower' + MODULE = "windpower" PYSAM = PySamWindPower def set_resource_data(self, resource, meta): @@ -1939,36 +2091,39 @@ def set_resource_data(self, resource, meta): meta = self._parse_meta(meta) # map resource data names to SAM required data names - var_map = {'speed': 'windspeed', - 'direction': 'winddirection', - 'airtemperature': 'temperature', - 'temp': 'temperature', - 'surfacepressure': 'pressure', - 'relativehumidity': 'rh', - 'humidity': 'rh', - } - lower_case = {k: k.lower().replace(' ', '').replace('_', '') - for k in resource.columns} - resource = resource.rename(mapper=lower_case, axis='columns') - resource = resource.rename(mapper=var_map, axis='columns') + var_map = { + "speed": "windspeed", + "direction": "winddirection", + "airtemperature": "temperature", + "temp": "temperature", + "surfacepressure": "pressure", + "relativehumidity": "rh", + "humidity": "rh", + } + lower_case = { + k: k.lower().replace(" ", "").replace("_", "") + for k in resource.columns + } + resource = resource.rename(mapper=lower_case, axis="columns") + resource = resource.rename(mapper=var_map, axis="columns") data_dict = {} - var_list = ['temperature', 'pressure', 'windspeed', 'winddirection'] - if 'winddirection' not in resource: - resource['winddirection'] = 0.0 + var_list = ["temperature", "pressure", "windspeed", "winddirection"] + if "winddirection" not in resource: + resource["winddirection"] = 0.0 time_index = resource.index self.time_interval = self.get_time_interval(resource.index.values) - data_dict['fields'] = [1, 2, 3, 4] - data_dict['heights'] = 4 * [self.sam_sys_inputs['wind_turbine_hub_ht']] + data_dict["fields"] = [1, 2, 3, 4] + data_dict["heights"] = 4 * [self.sam_sys_inputs["wind_turbine_hub_ht"]] - if 'rh' in resource: + if "rh" in resource: # set relative humidity for icing. - rh = self.ensure_res_len(resource['rh'].values, time_index) + rh = self.ensure_res_len(resource["rh"].values, time_index) n_roll = int(meta[MetaKeyName.TIMEZONE] * self.time_interval) rh = np.roll(rh, n_roll, axis=0) - data_dict['rh'] = rh.tolist() + data_dict["rh"] = rh.tolist() # must be set as matrix in [temperature, pres, speed, direction] order # ensure that resource array length is multiple of 8760 @@ -1976,23 +2131,23 @@ def set_resource_data(self, resource, meta): temp = self.ensure_res_len(resource[var_list].values, time_index) n_roll = int(meta[MetaKeyName.TIMEZONE] * self.time_interval) temp = np.roll(temp, n_roll, axis=0) - data_dict['data'] = temp.tolist() + data_dict["data"] = temp.tolist() - data_dict['lat'] = float(meta[MetaKeyName.LATITUDE]) - data_dict['lon'] = float(meta[MetaKeyName.LONGITUDE]) - data_dict['tz'] = int(meta[MetaKeyName.TIMEZONE]) - data_dict['elev'] = float(meta[MetaKeyName.ELEVATION]) + data_dict["lat"] = float(meta[MetaKeyName.LATITUDE]) + data_dict["lon"] = float(meta[MetaKeyName.LONGITUDE]) + data_dict["tz"] = int(meta[MetaKeyName.TIMEZONE]) + data_dict["elev"] = float(meta[MetaKeyName.ELEVATION]) time_index = self.ensure_res_len(time_index, time_index) - data_dict['minute'] = time_index.minute.tolist() - data_dict['hour'] = time_index.hour.tolist() - data_dict['year'] = time_index.year.tolist() - data_dict['month'] = time_index.month.tolist() - data_dict['day'] = time_index.day.tolist() + data_dict["minute"] = time_index.minute.tolist() + data_dict["hour"] = time_index.hour.tolist() + data_dict["year"] = time_index.year.tolist() + data_dict["month"] = time_index.month.tolist() + data_dict["day"] = time_index.day.tolist() # add resource data to self.data and clear - self['wind_resource_data'] = data_dict - self['wind_resource_model_choice'] = 0 + self["wind_resource_data"] = data_dict + self["wind_resource_model_choice"] = 0 @staticmethod def default(): @@ -2010,12 +2165,19 @@ class WindPowerPD(AbstractSamGeneration, PowerCurveLossesMixin): """WindPower analysis with wind speed/direction joint probabilty distrubtion input""" - MODULE = 'windpower' + MODULE = "windpower" PYSAM = PySamWindPower - def __init__(self, ws_edges, wd_edges, wind_dist, - meta, sam_sys_inputs, - site_sys_inputs=None, output_request=None): + def __init__( + self, + ws_edges, + wd_edges, + wind_dist, + meta, + sam_sys_inputs, + site_sys_inputs=None, + output_request=None, + ): """Initialize a SAM generation object for windpower with a speed/direction joint probability distribution. @@ -2049,13 +2211,17 @@ def __init__(self, ws_edges, wd_edges, wind_dist, # don't pass resource to base class, # set in concrete generation classes instead - super().__init__(None, meta, sam_sys_inputs, - site_sys_inputs=site_sys_inputs, - output_request=output_request, - drop_leap=False) + super().__init__( + None, + meta, + sam_sys_inputs, + site_sys_inputs=site_sys_inputs, + output_request=output_request, + drop_leap=False, + ) # Set the site number using meta data - if hasattr(meta, 'name'): + if hasattr(meta, "name"): self._site = meta.name else: self._site = None @@ -2088,20 +2254,21 @@ def set_resource_data(self, ws_edges, wd_edges, wind_dist): wd_points = wd_edges[:-1] + np.diff(wd_edges) / 2 wd_points, ws_points = np.meshgrid(wd_points, ws_points) - vstack = (ws_points.flatten(), - wd_points.flatten(), - wind_dist.flatten()) + vstack = ( + ws_points.flatten(), + wd_points.flatten(), + wind_dist.flatten(), + ) wrd = np.vstack(vstack).T.tolist() - self['wind_resource_model_choice'] = 2 - self['wind_resource_distribution'] = wrd + self["wind_resource_model_choice"] = 2 + self["wind_resource_distribution"] = wrd class MhkWave(AbstractSamGeneration): - """Class for Wave generation from SAM - """ + """Class for Wave generation from SAM""" - MODULE = 'mhkwave' + MODULE = "mhkwave" PYSAM = PySamMhkWave def set_resource_data(self, resource, meta): @@ -2122,19 +2289,22 @@ def set_resource_data(self, resource, meta): meta = self._parse_meta(meta) # map resource data names to SAM required data names - var_map = {'significantwaveheight': 'significant_wave_height', - 'waveheight': 'significant_wave_height', - 'height': 'significant_wave_height', - 'swh': 'significant_wave_height', - 'energyperiod': 'energy_period', - 'waveperiod': 'energy_period', - 'period': 'energy_period', - 'ep': 'energy_period', - } - lower_case = {k: k.lower().replace(' ', '').replace('_', '') - for k in resource.columns} - resource = resource.rename(mapper=lower_case, axis='columns') - resource = resource.rename(mapper=var_map, axis='columns') + var_map = { + "significantwaveheight": "significant_wave_height", + "waveheight": "significant_wave_height", + "height": "significant_wave_height", + "swh": "significant_wave_height", + "energyperiod": "energy_period", + "waveperiod": "energy_period", + "period": "energy_period", + "ep": "energy_period", + } + lower_case = { + k: k.lower().replace(" ", "").replace("_", "") + for k in resource.columns + } + resource = resource.rename(mapper=lower_case, axis="columns") + resource = resource.rename(mapper=var_map, axis="columns") data_dict = {} @@ -2144,24 +2314,24 @@ def set_resource_data(self, resource, meta): # must be set as matrix in [temperature, pres, speed, direction] order # ensure that resource array length is multiple of 8760 # roll the truncated resource array to local timezone - for var in ['significant_wave_height', 'energy_period']: + for var in ["significant_wave_height", "energy_period"]: arr = self.ensure_res_len(resource[var].values, time_index) n_roll = int(meta[MetaKeyName.TIMEZONE] * self.time_interval) data_dict[var] = np.roll(arr, n_roll, axis=0).tolist() - data_dict['lat'] = meta[MetaKeyName.LATITUDE] - data_dict['lon'] = meta[MetaKeyName.LONGITUDE] - data_dict['tz'] = meta[MetaKeyName.TIMEZONE] + data_dict["lat"] = meta[MetaKeyName.LATITUDE] + data_dict["lon"] = meta[MetaKeyName.LONGITUDE] + data_dict["tz"] = meta[MetaKeyName.TIMEZONE] time_index = self.ensure_res_len(time_index, time_index) - data_dict['minute'] = time_index.minute - data_dict['hour'] = time_index.hour - data_dict['year'] = time_index.year - data_dict['month'] = time_index.month - data_dict['day'] = time_index.day + data_dict["minute"] = time_index.minute + data_dict["hour"] = time_index.hour + data_dict["year"] = time_index.year + data_dict["month"] = time_index.month + data_dict["day"] = time_index.day # add resource data to self.data and clear - self['wave_resource_data'] = data_dict + self["wave_resource_data"] = data_dict @staticmethod def default(): diff --git a/reV/econ/econ.py b/reV/econ/econ.py index 6c87796fe..5b729d466 100644 --- a/reV/econ/econ.py +++ b/reV/econ/econ.py @@ -2,6 +2,7 @@ """ reV econ module (lcoe-fcr, single owner, etc...) """ + import logging import os import pprint @@ -29,31 +30,40 @@ class Econ(BaseGen): """Econ""" # Mapping of reV econ output strings to SAM econ modules - OPTIONS = {'lcoe_fcr': SAM_LCOE, - 'ppa_price': SingleOwner, - 'project_return_aftertax_npv': SingleOwner, - 'lcoe_real': SingleOwner, - 'lcoe_nom': SingleOwner, - 'flip_actual_irr': SingleOwner, - 'gross_revenue': SingleOwner, - 'total_installed_cost': WindBos, - 'turbine_cost': WindBos, - 'sales_tax_cost': WindBos, - 'bos_cost': WindBos, - 'fixed_charge_rate': SAM_LCOE, - 'capital_cost': SAM_LCOE, - 'fixed_operating_cost': SAM_LCOE, - 'variable_operating_cost': SAM_LCOE, - } + OPTIONS = { + "lcoe_fcr": SAM_LCOE, + "ppa_price": SingleOwner, + "project_return_aftertax_npv": SingleOwner, + "lcoe_real": SingleOwner, + "lcoe_nom": SingleOwner, + "flip_actual_irr": SingleOwner, + "gross_revenue": SingleOwner, + "total_installed_cost": WindBos, + "turbine_cost": WindBos, + "sales_tax_cost": WindBos, + "bos_cost": WindBos, + "fixed_charge_rate": SAM_LCOE, + "capital_cost": SAM_LCOE, + "fixed_operating_cost": SAM_LCOE, + "variable_operating_cost": SAM_LCOE, + } """Available ``reV`` econ `output_request` options""" # Mapping of reV econ outputs to scale factors and units. # Type is scalar or array and corresponds to the SAM single-site output OUT_ATTRS = BaseGen.ECON_ATTRS - def __init__(self, project_points, sam_files, cf_file, site_data=None, - output_request=('lcoe_fcr',), sites_per_worker=100, - memory_utilization_limit=0.4, append=False): + def __init__( + self, + project_points, + sam_files, + cf_file, + site_data=None, + output_request=("lcoe_fcr",), + sites_per_worker=100, + memory_utilization_limit=0.4, + append=False, + ): """ReV econ analysis class. ``reV`` econ analysis runs SAM econ calculations, typically to @@ -176,17 +186,26 @@ def __init__(self, project_points, sam_files, cf_file, site_data=None, """ # get a points control instance - pc = self.get_pc(points=project_points, points_range=None, - sam_configs=sam_files, cf_file=cf_file, - sites_per_worker=sites_per_worker, append=append) - - super().__init__(pc, output_request, site_data=site_data, - memory_utilization_limit=memory_utilization_limit) + pc = self.get_pc( + points=project_points, + points_range=None, + sam_configs=sam_files, + cf_file=cf_file, + sites_per_worker=sites_per_worker, + append=append, + ) + + super().__init__( + pc, + output_request, + site_data=site_data, + memory_utilization_limit=memory_utilization_limit, + ) self._cf_file = cf_file self._append = append - self._run_attrs['cf_file'] = cf_file - self._run_attrs['sam_module'] = self._sam_module.MODULE + self._run_attrs["cf_file"] = cf_file + self._run_attrs["sam_module"] = self._sam_module.MODULE @property def cf_file(self): @@ -212,19 +231,24 @@ def meta(self): with Outputs(self.cf_file) as cfh: # only take meta that belongs to this project's site list self._meta = cfh.meta[ - cfh.meta[MetaKeyName.GID].isin(self.points_control.sites)] + cfh.meta[MetaKeyName.GID].isin(self.points_control.sites) + ] if MetaKeyName.OFFSHORE in self._meta: if self._meta[MetaKeyName.OFFSHORE].sum() > 1: - w = ('Found offshore sites in econ meta data. ' - 'This functionality has been deprecated. ' - 'Please run the reV offshore module to ' - 'calculate offshore wind lcoe.') + w = ( + "Found offshore sites in econ meta data. " + "This functionality has been deprecated. " + "Please run the reV offshore module to " + "calculate offshore wind lcoe." + ) warn(w, OffshoreWindInputWarning) logger.warning(w) elif self._meta is None and self.cf_file is None: - self._meta = pd.DataFrame({MetaKeyName.GID: self.points_control.sites}) + self._meta = pd.DataFrame( + {MetaKeyName.GID: self.points_control.sites} + ) return self._meta @@ -233,7 +257,7 @@ def time_index(self): """Get the generation resource time index data.""" if self._time_index is None and self.cf_file is not None: with Outputs(self.cf_file) as cfh: - if 'time_index' in cfh.datasets: + if "time_index" in cfh.datasets: self._time_index = cfh.time_index return self._time_index @@ -264,7 +288,7 @@ def _econ_append_pc(pp, cf_file, sites_per_worker=None): res_kwargs = {} else: res_cls = Resource - res_kwargs = {'hsds': hsds} + res_kwargs = {"hsds": hsds} with res_cls(cf_file, **res_kwargs) as f: gid0 = f.meta[MetaKeyName.GID].values[0] @@ -277,8 +301,15 @@ def _econ_append_pc(pp, cf_file, sites_per_worker=None): return pc @classmethod - def get_pc(cls, points, points_range, sam_configs, cf_file, - sites_per_worker=None, append=False): + def get_pc( + cls, + points, + points_range, + sam_configs, + cf_file, + sites_per_worker=None, + append=False, + ): """ Get a PointsControl instance. @@ -311,13 +342,19 @@ def get_pc(cls, points, points_range, sam_configs, cf_file, pc : reV.config.project_points.PointsControl PointsControl object instance. """ - pc = super().get_pc(points, points_range, sam_configs, ModuleName.ECON, - sites_per_worker=sites_per_worker, - res_file=cf_file) + pc = super().get_pc( + points, + points_range, + sam_configs, + ModuleName.ECON, + sites_per_worker=sites_per_worker, + res_file=cf_file, + ) if append: - pc = cls._econ_append_pc(pc.project_points, cf_file, - sites_per_worker=sites_per_worker) + pc = cls._econ_append_pc( + pc.project_points, cf_file, sites_per_worker=sites_per_worker + ) return pc @@ -330,9 +367,10 @@ def _run_single_worker(pc, econ_fun, output_request, **kwargs): pc : reV.config.project_points.PointsControl Iterable points control object from reV config module. Must have project_points with df property with all relevant - site-specific inputs and a MetaKeyName.GID column. By passing site-specific - inputs in this dataframe, which was split using points_control, - only the data relevant to the current sites is passed. + site-specific inputs and a `MetaKeyName.GID` column. By passing + site-specific inputs in this dataframe, which was split using + points_control, only the data relevant to the current sites is + passed. econ_fun : method reV_run() method from one of the econ modules (SingleOwner, SAM_LCOE, WindBos). @@ -358,11 +396,12 @@ def _run_single_worker(pc, econ_fun, output_request, **kwargs): # SAM execute econ analysis based on output request try: - out = econ_fun(pc, site_df, output_request=output_request, - **kwargs) + out = econ_fun( + pc, site_df, output_request=output_request, **kwargs + ) except Exception as e: out = {} - logger.exception('Worker failed for PC: {}'.format(pc)) + logger.exception("Worker failed for PC: {}".format(pc)) raise e return out @@ -385,8 +424,10 @@ def _parse_output_request(self, req): for request in output_request: if request not in self.OUT_ATTRS: - msg = ('User output request "{}" not recognized. ' - 'Will attempt to extract from PySAM.'.format(request)) + msg = ( + 'User output request "{}" not recognized. ' + "Will attempt to extract from PySAM.".format(request) + ) logger.debug(msg) modules = [] @@ -395,10 +436,13 @@ def _parse_output_request(self, req): modules.append(self.OPTIONS[request]) if not any(modules): - msg = ('None of the user output requests were recognized. ' - 'Cannot run reV econ. ' - 'At least one of the following must be requested: {}' - .format(list(self.OPTIONS.keys()))) + msg = ( + "None of the user output requests were recognized. " + "Cannot run reV econ. " + "At least one of the following must be requested: {}".format( + list(self.OPTIONS.keys()) + ) + ) logger.exception(msg) raise ExecutionError(msg) @@ -413,9 +457,11 @@ def _parse_output_request(self, req): self._sam_module = SingleOwner self._fun = SingleOwner.reV_run else: - msg = ('Econ outputs requested from different SAM modules not ' - 'currently supported. Output request variables require ' - 'SAM methods: {}'.format(modules)) + msg = ( + "Econ outputs requested from different SAM modules not " + "currently supported. Output request variables require " + "SAM methods: {}".format(modules) + ) raise ValueError(msg) return list(set(output_request)) @@ -441,12 +487,14 @@ def _get_data_shape(self, dset, n_sites): """ if dset in self.site_data: - data_shape = (n_sites, ) + data_shape = (n_sites,) data = self.site_data[dset].values[0] if isinstance(data, (list, tuple, np.ndarray, str)): - msg = ('Cannot pass through non-scalar site_data ' - 'input key "{}" as an output_request!'.format(dset)) + msg = ( + "Cannot pass through non-scalar site_data " + 'input key "{}" as an output_request!'.format(dset) + ) logger.error(msg) raise ExecutionError(msg) @@ -455,8 +503,7 @@ def _get_data_shape(self, dset, n_sites): return data_shape - def run(self, out_fpath=None, max_workers=1, timeout=1800, - pool_size=None): + def run(self, out_fpath=None, max_workers=1, timeout=1800, pool_size=None): """Execute a parallel reV econ run with smart data flushing. Parameters @@ -494,52 +541,76 @@ def run(self, out_fpath=None, max_workers=1, timeout=1800, else: self._init_fpath(out_fpath, ModuleName.ECON) - self._init_h5(mode='a' if self._append else 'w') + self._init_h5(mode="a" if self._append else "w") self._init_out_arrays() - diff = list(set(self.points_control.sites) - - set(self.meta[MetaKeyName.GID].values)) + diff = list( + set(self.points_control.sites) + - set(self.meta[MetaKeyName.GID].values) + ) if diff: - raise Exception('The following analysis sites were requested ' - 'through project points for econ but are not ' - 'found in the CF file ("{}"): {}' - .format(self.cf_file, diff)) + raise Exception( + "The following analysis sites were requested " + "through project points for econ but are not " + 'found in the CF file ("{}"): {}'.format(self.cf_file, diff) + ) # make a kwarg dict - kwargs = {'output_request': self.output_request, - 'cf_file': self.cf_file, - 'year': self.year} - - logger.info('Running econ with smart data flushing ' - 'for: {}'.format(self.points_control)) - logger.debug('The following project points were specified: "{}"' - .format(self.project_points)) - logger.debug('The following SAM configs are available to this run:\n{}' - .format(pprint.pformat(self.sam_configs, indent=4))) - logger.debug('The SAM output variables have been requested:\n{}' - .format(self.output_request)) + kwargs = { + "output_request": self.output_request, + "cf_file": self.cf_file, + "year": self.year, + } + + logger.info( + "Running econ with smart data flushing " "for: {}".format( + self.points_control + ) + ) + logger.debug( + 'The following project points were specified: "{}"'.format( + self.project_points + ) + ) + logger.debug( + "The following SAM configs are available to this run:\n{}".format( + pprint.pformat(self.sam_configs, indent=4) + ) + ) + logger.debug( + "The SAM output variables have been requested:\n{}".format( + self.output_request + ) + ) try: - kwargs['econ_fun'] = self._fun + kwargs["econ_fun"] = self._fun if max_workers == 1: - logger.debug('Running serial econ for: {}' - .format(self.points_control)) + logger.debug( + "Running serial econ for: {}".format(self.points_control) + ) for i, pc_sub in enumerate(self.points_control): self.out = self._run_single_worker(pc_sub, **kwargs) - logger.info('Finished reV econ serial compute for: {} ' - '(iteration {} out of {})' - .format(pc_sub, i + 1, - len(self.points_control))) + logger.info( + "Finished reV econ serial compute for: {} " + "(iteration {} out of {})".format( + pc_sub, i + 1, len(self.points_control) + ) + ) self.flush() else: - logger.debug('Running parallel econ for: {}' - .format(self.points_control)) - self._parallel_run(max_workers=max_workers, - pool_size=pool_size, timeout=timeout, - **kwargs) + logger.debug( + "Running parallel econ for: {}".format(self.points_control) + ) + self._parallel_run( + max_workers=max_workers, + pool_size=pool_size, + timeout=timeout, + **kwargs, + ) except Exception as e: - logger.exception('SmartParallelJob.execute() failed for econ.') + logger.exception("SmartParallelJob.execute() failed for econ.") raise e return self._out_fpath diff --git a/reV/generation/base.py b/reV/generation/base.py index d170940a2..c25b85f23 100644 --- a/reV/generation/base.py +++ b/reV/generation/base.py @@ -2,6 +2,7 @@ """ reV base gen and econ module. """ + import copy import json import logging @@ -33,16 +34,16 @@ ATTR_DIR = os.path.dirname(os.path.realpath(__file__)) -ATTR_DIR = os.path.join(ATTR_DIR, 'output_attributes') -with open(os.path.join(ATTR_DIR, 'other.json')) as f: +ATTR_DIR = os.path.join(ATTR_DIR, "output_attributes") +with open(os.path.join(ATTR_DIR, "other.json")) as f: OTHER_ATTRS = json.load(f) -with open(os.path.join(ATTR_DIR, 'lcoe_fcr.json')) as f: +with open(os.path.join(ATTR_DIR, "lcoe_fcr.json")) as f: LCOE_ATTRS = json.load(f) -with open(os.path.join(ATTR_DIR, 'single_owner.json')) as f: +with open(os.path.join(ATTR_DIR, "single_owner.json")) as f: SO_ATTRS = json.load(f) -with open(os.path.join(ATTR_DIR, 'windbos.json')) as f: +with open(os.path.join(ATTR_DIR, "windbos.json")) as f: BOS_ATTRS = json.load(f) -with open(os.path.join(ATTR_DIR, 'lcoe_fcr_inputs.json')) as f: +with open(os.path.join(ATTR_DIR, "lcoe_fcr_inputs.json")) as f: LCOE_IN_ATTRS = json.load(f) @@ -68,12 +69,22 @@ class BaseGen(ABC): # SAM argument names used to calculate LCOE # Note that system_capacity is not included here because it is never used # downstream and could be confused with the supply_curve point capacity - LCOE_ARGS = ('fixed_charge_rate', 'capital_cost', 'fixed_operating_cost', - 'variable_operating_cost') - - def __init__(self, points_control, output_request, site_data=None, - drop_leap=False, memory_utilization_limit=0.4, - scale_outputs=True): + LCOE_ARGS = ( + "fixed_charge_rate", + "capital_cost", + "fixed_operating_cost", + "variable_operating_cost", + ) + + def __init__( + self, + points_control, + output_request, + site_data=None, + drop_leap=False, + memory_utilization_limit=0.4, + scale_outputs=True, + ): """ Parameters ---------- @@ -109,11 +120,13 @@ def __init__(self, points_control, output_request, site_data=None, self.mem_util_lim = memory_utilization_limit self.scale_outputs = scale_outputs - self._run_attrs = {'points_control': str(points_control), - 'output_request': output_request, - 'site_data': str(site_data), - 'drop_leap': str(drop_leap), - 'memory_utilization_limit': self.mem_util_lim} + self._run_attrs = { + "points_control": str(points_control), + "output_request": output_request, + "site_data": str(site_data), + "drop_leap": str(drop_leap), + "memory_utilization_limit": self.mem_util_lim, + } self._site_data = self._parse_site_data(site_data) self.add_site_data_to_pp(self._site_data) @@ -177,11 +190,16 @@ def site_limit(self): tot_mem = psutil.virtual_memory().total / 1e6 avail_mem = self.mem_util_lim * tot_mem self._site_limit = int(np.floor(avail_mem / self.site_mem)) - logger.info('Limited to storing {0} sites in memory ' - '({1:.1f} GB total hardware, {2:.1f} GB available ' - 'with {3:.1f}% utilization).' - .format(self._site_limit, tot_mem / 1e3, - avail_mem / 1e3, self.mem_util_lim * 100)) + logger.info( + "Limited to storing {0} sites in memory " + "({1:.1f} GB total hardware, {2:.1f} GB available " + "with {3:.1f}% utilization).".format( + self._site_limit, + tot_mem / 1e3, + avail_mem / 1e3, + self.mem_util_lim * 100, + ) + ) return self._site_limit @@ -202,17 +220,18 @@ def site_mem(self): n = 100 self._site_mem = 0 for request in self.output_request: - dtype = 'float32' + dtype = "float32" if request in self.OUT_ATTRS: - dtype = self.OUT_ATTRS[request].get('dtype', 'float32') + dtype = self.OUT_ATTRS[request].get("dtype", "float32") shape = self._get_data_shape(request, n) self._site_mem += sys.getsizeof(np.ones(shape, dtype=dtype)) self._site_mem = self._site_mem / 1e6 / n - logger.info('Output results from a single site are calculated to ' - 'use {0:.1f} KB of memory.' - .format(self._site_mem / 1000)) + logger.info( + "Output results from a single site are calculated to " + "use {0:.1f} KB of memory.".format(self._site_mem / 1000) + ) return self._site_mem @@ -262,7 +281,7 @@ def sam_metas(self): """ sam_metas = self.sam_configs.copy() for v in sam_metas.values(): - v.update({'module': self._sam_module.MODULE}) + v.update({"module": self._sam_module.MODULE}) return sam_metas @@ -287,7 +306,8 @@ def meta(self): Meta data df for sites in project points. Column names are meta data variables, rows are different sites. The row index does not indicate the site number if the project points are - non-sequential or do not start from 0, so a MetaKeyName.GID column is added. + non-sequential or do not start from 0, so a `MetaKeyName.GID` + column is added. """ return self._meta @@ -354,12 +374,12 @@ def out(self): out = {} for k, v in self._out.items(): if k in self.OUT_ATTRS: - scale_factor = self.OUT_ATTRS[k].get('scale_factor', 1) + scale_factor = self.OUT_ATTRS[k].get("scale_factor", 1) else: scale_factor = 1 if scale_factor != 1 and self.scale_outputs: - v = v.astype('float32') + v = v.astype("float32") v /= scale_factor out[k] = v @@ -384,16 +404,14 @@ def out(self, result): result = self.unpack_futures(result) if isinstance(result, dict): - # iterate through dict where sites are keys and values are # corresponding results for site_gid, site_output in result.items(): - # check that the sites are stored sequentially then add to # the finished site list if self._finished_sites: if int(site_gid) < np.max(self._finished_sites): - raise Exception('Site results are non sequential!') + raise Exception("Site results are non sequential!") # unpack site output object self.unpack_output(site_gid, site_output) @@ -405,9 +423,11 @@ def out(self, result): self._out.clear() self._finished_sites.clear() else: - raise TypeError('Did not recognize the type of output. ' - 'Tried to set output type "{}", but requires ' - 'list, dict or None.'.format(type(result))) + raise TypeError( + "Did not recognize the type of output. " + 'Tried to set output type "{}", but requires ' + "list, dict or None.".format(type(result)) + ) @staticmethod def _output_request_type_check(req): @@ -431,8 +451,10 @@ def _output_request_type_check(req): elif isinstance(req, str): output_request = [req] else: - raise TypeError('Output request must be str, list, or tuple but ' - 'received: {}'.format(type(req))) + raise TypeError( + "Output request must be str, list, or tuple but " + "received: {}".format(type(req)) + ) return output_request @@ -455,7 +477,7 @@ def handle_leap_ti(ti, drop_leap=False): """ # Drop leap day or last day - leap_day = ((ti.month == 2) & (ti.day == 29)) + leap_day = (ti.month == 2) & (ti.day == 29) leap_year = ti.year % 4 == 0 last_day = ((ti.month == 12) & (ti.day == 31)) * leap_year if drop_leap: @@ -466,14 +488,23 @@ def handle_leap_ti(ti, drop_leap=False): ti = ti.drop(ti[last_day]) if len(ti) % 365 != 0: - raise ValueError('Bad time index with length not a multiple of ' - '365: {}'.format(ti)) + raise ValueError( + "Bad time index with length not a multiple of " + "365: {}".format(ti) + ) return ti @staticmethod - def _pp_to_pc(points, points_range, sam_configs, tech, - sites_per_worker=None, res_file=None, curtailment=None): + def _pp_to_pc( + points, + points_range, + sam_configs, + tech, + sites_per_worker=None, + res_file=None, + curtailment=None, + ): """ Create ProjectControl from ProjectPoints @@ -522,16 +553,25 @@ def _pp_to_pc(points, points_range, sam_configs, tech, if hasattr(points, "df"): points = points.df - pp = ProjectPoints(points, sam_configs, tech=tech, res_file=res_file, - curtailment=curtailment) + pp = ProjectPoints( + points, + sam_configs, + tech=tech, + res_file=res_file, + curtailment=curtailment, + ) # make Points Control instance if points_range is not None: # PointsControl is for just a subset of the project points... # this is the case if generation is being initialized on one # of many HPC nodes in a large project - pc = PointsControl.split(points_range[0], points_range[1], pp, - sites_per_split=sites_per_worker) + pc = PointsControl.split( + points_range[0], + points_range[1], + pp, + sites_per_split=sites_per_worker, + ) else: # PointsControl is for all of the project points pc = PointsControl(pp, sites_per_split=sites_per_worker) @@ -539,8 +579,16 @@ def _pp_to_pc(points, points_range, sam_configs, tech, return pc @classmethod - def get_pc(cls, points, points_range, sam_configs, tech, - sites_per_worker=None, res_file=None, curtailment=None): + def get_pc( + cls, + points, + points_range, + sam_configs, + tech, + sites_per_worker=None, + res_file=None, + curtailment=None, + ): """Get a PointsControl instance. Parameters @@ -588,9 +636,12 @@ def get_pc(cls, points, points_range, sam_configs, tech, """ if tech not in cls.OPTIONS and tech.lower() != ModuleName.ECON: - msg = ('Did not recognize reV-SAM technology string "{}". ' - 'Technology string options are: {}' - .format(tech, list(cls.OPTIONS.keys()))) + msg = ( + 'Did not recognize reV-SAM technology string "{}". ' + "Technology string options are: {}".format( + tech, list(cls.OPTIONS.keys()) + ) + ) logger.error(msg) raise KeyError(msg) @@ -598,16 +649,25 @@ def get_pc(cls, points, points_range, sam_configs, tech, # get the optimal sites per split based on res file chunk size sites_per_worker = cls.get_sites_per_worker(res_file) - logger.debug('Sites per worker being set to {} for ' - 'PointsControl.'.format(sites_per_worker)) + logger.debug( + "Sites per worker being set to {} for " "PointsControl.".format( + sites_per_worker + ) + ) if isinstance(points, PointsControl): # received a pre-intialized instance of pointscontrol pc = points else: - pc = cls._pp_to_pc(points, points_range, sam_configs, tech, - sites_per_worker=sites_per_worker, - res_file=res_file, curtailment=curtailment) + pc = cls._pp_to_pc( + points, + points_range, + sam_configs, + tech, + sites_per_worker=sites_per_worker, + res_file=res_file, + curtailment=curtailment, + ) return pc @@ -640,31 +700,37 @@ def get_sites_per_worker(res_file, default=100): return default with Resource(res_file) as res: - if 'wtk' in res_file.lower(): + if "wtk" in res_file.lower(): for dset in res.datasets: - if 'speed' in dset: + if "speed" in dset: # take nominal WTK chunks from windspeed _, _, chunks = res.get_dset_properties(dset) break - elif 'nsrdb' in res_file.lower(): + elif "nsrdb" in res_file.lower(): # take nominal NSRDB chunks from dni - _, _, chunks = res.get_dset_properties('dni') + _, _, chunks = res.get_dset_properties("dni") else: - warn('Could not infer dataset chunk size as the resource type ' - 'could not be determined from the filename: {}' - .format(res_file)) + warn( + "Could not infer dataset chunk size as the resource type " + "could not be determined from the filename: {}".format( + res_file + ) + ) chunks = None if chunks is None: # if chunks not set, go to default sites_per_worker = default - logger.debug('Sites per worker being set to {} (default) based on ' - 'no set chunk size in {}.' - .format(sites_per_worker, res_file)) + logger.debug( + "Sites per worker being set to {} (default) based on " + "no set chunk size in {}.".format(sites_per_worker, res_file) + ) else: sites_per_worker = chunks[1] - logger.debug('Sites per worker being set to {} based on chunk ' - 'size of {}.'.format(sites_per_worker, res_file)) + logger.debug( + "Sites per worker being set to {} based on chunk " + "size of {}.".format(sites_per_worker, res_file) + ) return sites_per_worker @@ -691,8 +757,13 @@ def unpack_futures(futures): @staticmethod @abstractmethod - def _run_single_worker(points_control, tech=None, res_file=None, - output_request=None, scale_outputs=True): + def _run_single_worker( + points_control, + tech=None, + res_file=None, + output_request=None, + scale_outputs=True, + ): """Run a reV-SAM analysis based on the points_control iterator. Parameters @@ -742,19 +813,26 @@ def _parse_site_data(self, inp): else: # explicit input, initialize df if isinstance(inp, str): - if inp.endswith('.csv'): + if inp.endswith(".csv"): site_data = pd.read_csv(inp) elif isinstance(inp, pd.DataFrame): site_data = inp else: # site data was not able to be set. Raise error. - raise Exception('Site data input must be .csv or ' - 'dataframe, but received: {}'.format(inp)) - - if MetaKeyName.GID not in site_data and site_data.index.name != MetaKeyName.GID: + raise Exception( + "Site data input must be .csv or " + "dataframe, but received: {}".format(inp) + ) + + if ( + MetaKeyName.GID not in site_data + and site_data.index.name != MetaKeyName.GID + ): # require gid as column label or index - raise KeyError('Site data input must have "gid" column ' - 'to match reV site gid.') + raise KeyError( + 'Site data input must have "gid" column ' + "to match reV site gid." + ) # pylint: disable=no-member if site_data.index.name != MetaKeyName.GID: @@ -763,10 +841,12 @@ def _parse_site_data(self, inp): if MetaKeyName.OFFSHORE in site_data: if site_data[MetaKeyName.OFFSHORE].sum() > 1: - w = ('Found offshore sites in econ site data input. ' - 'This functionality has been deprecated. ' - 'Please run the reV offshore module to ' - 'calculate offshore wind lcoe.') + w = ( + "Found offshore sites in econ site data input. " + "This functionality has been deprecated. " + "Please run the reV offshore module to " + "calculate offshore wind lcoe." + ) warn(w, OffshoreWindInputWarning) logger.warning(w) @@ -827,7 +907,7 @@ def _get_data_shape(self, dset, n_sites): def _get_data_shape_from_out_attrs(self, dset, n_sites): """Get data shape from ``OUT_ATTRS`` variable""" - if self.OUT_ATTRS[dset]['type'] == 'array': + if self.OUT_ATTRS[dset]["type"] == "array": return (len(self.time_index), n_sites) return (n_sites,) @@ -838,12 +918,14 @@ def _get_data_shape_from_sam_config(self, dset, n_sites): return (*np.array(data).shape, n_sites) if isinstance(data, str): - msg = ('Cannot pass through non-scalar SAM input key "{}" ' - 'as an output_request!'.format(dset)) + msg = ( + 'Cannot pass through non-scalar SAM input key "{}" ' + "as an output_request!".format(dset) + ) logger.error(msg) raise ExecutionError(msg) - return (n_sites, ) + return (n_sites,) def _get_data_shape_from_pysam(self, dset, n_sites): """Get data shape from PySAM output object""" @@ -853,10 +935,13 @@ def _get_data_shape_from_pysam(self, dset, n_sites): try: out_data = getattr(self._sam_obj_default.Outputs, dset) except AttributeError as e: - msg = ('Could not get data shape for dset "{}" ' - 'from object "{}". ' - 'Received the following error: "{}"' - .format(dset, self._sam_obj_default, e)) + msg = ( + 'Could not get data shape for dset "{}" ' + 'from object "{}". ' + 'Received the following error: "{}"'.format( + dset, self._sam_obj_default, e + ) + ) logger.error(msg) raise ExecutionError(msg) from e @@ -876,8 +961,8 @@ def _init_fpath(self, out_fpath, module): project_dir, out_fn = os.path.split(out_fpath) # ensure output file is an h5 - if not out_fn.endswith('.h5'): - out_fn += '.h5' + if not out_fn.endswith(".h5"): + out_fn += ".h5" if module not in out_fn: extension_with_module = "_{}.h5".format(module) @@ -894,9 +979,9 @@ def _init_fpath(self, out_fpath, module): os.makedirs(project_dir, exist_ok=True) self._out_fpath = os.path.join(project_dir, out_fn) - self._run_attrs['out_fpath'] = out_fpath + self._run_attrs["out_fpath"] = out_fpath - def _init_h5(self, mode='w'): + def _init_h5(self, mode="w"): """Initialize the single h5 output file with all output requests. Parameters @@ -908,12 +993,18 @@ def _init_h5(self, mode='w'): if self._out_fpath is None: return - if 'w' in mode: - logger.info('Initializing full output file: "{}" with mode: {}' - .format(self._out_fpath, mode)) - elif 'a' in mode: - logger.info('Appending data to output file: "{}" with mode: {}' - .format(self._out_fpath, mode)) + if "w" in mode: + logger.info( + 'Initializing full output file: "{}" with mode: {}'.format( + self._out_fpath, mode + ) + ) + elif "a" in mode: + logger.info( + 'Appending data to output file: "{}" with mode: {}'.format( + self._out_fpath, mode + ) + ) attrs = {d: {} for d in self.output_request} chunks = {} @@ -924,17 +1015,16 @@ def _init_h5(self, mode='w'): write_ti = False for dset in self.output_request: - - tmp = 'other' + tmp = "other" if dset in self.OUT_ATTRS: tmp = dset - attrs[dset]['units'] = self.OUT_ATTRS[tmp].get('units', - 'unknown') - attrs[dset]['scale_factor'] = \ - self.OUT_ATTRS[tmp].get('scale_factor', 1) - chunks[dset] = self.OUT_ATTRS[tmp].get('chunks', None) - dtypes[dset] = self.OUT_ATTRS[tmp].get('dtype', 'float32') + attrs[dset]["units"] = self.OUT_ATTRS[tmp].get("units", "unknown") + attrs[dset]["scale_factor"] = self.OUT_ATTRS[tmp].get( + "scale_factor", 1 + ) + chunks[dset] = self.OUT_ATTRS[tmp].get("chunks", None) + dtypes[dset] = self.OUT_ATTRS[tmp].get("dtype", "float32") shapes[dset] = self._get_data_shape(dset, len(self.meta)) if len(shapes[dset]) > 1: write_ti = True @@ -945,10 +1035,19 @@ def _init_h5(self, mode='w'): else: ti = None - Outputs.init_h5(self._out_fpath, self.output_request, shapes, - attrs, chunks, dtypes, self.meta, time_index=ti, - configs=self.sam_metas, run_attrs=self.run_attrs, - mode=mode) + Outputs.init_h5( + self._out_fpath, + self.output_request, + shapes, + attrs, + chunks, + dtypes, + self.meta, + time_index=ti, + configs=self.sam_metas, + run_attrs=self.run_attrs, + mode=mode, + ) def _init_out_arrays(self, index_0=0): """Initialize output arrays based on the number of sites that can be @@ -966,21 +1065,27 @@ def _init_out_arrays(self, index_0=0): self._finished_sites = [] # Output chunk is the index range (inclusive) of this set of site outs - self._out_chunk = (index_0, np.min((index_0 + self.site_limit, - len(self.project_points) - 1))) + self._out_chunk = ( + index_0, + np.min((index_0 + self.site_limit, len(self.project_points) - 1)), + ) self._out_n_sites = int(self.out_chunk[1] - self.out_chunk[0]) + 1 - logger.info('Initializing in-memory outputs for {} sites with gids ' - '{} through {} inclusive (site list index {} through {})' - .format(self._out_n_sites, - self.project_points.sites[self.out_chunk[0]], - self.project_points.sites[self.out_chunk[1]], - self.out_chunk[0], self.out_chunk[1])) + logger.info( + "Initializing in-memory outputs for {} sites with gids " + "{} through {} inclusive (site list index {} through {})".format( + self._out_n_sites, + self.project_points.sites[self.out_chunk[0]], + self.project_points.sites[self.out_chunk[1]], + self.out_chunk[0], + self.out_chunk[1], + ) + ) for request in self.output_request: - dtype = 'float32' + dtype = "float32" if request in self.OUT_ATTRS and self.scale_outputs: - dtype = self.OUT_ATTRS[request].get('dtype', 'float32') + dtype = self.OUT_ATTRS[request].get("dtype", "float32") shape = self._get_data_shape(request, self._out_n_sites) @@ -1008,9 +1113,11 @@ def unpack_output(self, site_gid, site_output): # iterate through the site results for var, value in site_output.items(): if var not in self._out: - raise KeyError('Tried to collect output variable "{}", but it ' - 'was not yet initialized in the output ' - 'dictionary.') + raise KeyError( + 'Tried to collect output variable "{}", but it ' + "was not yet initialized in the output " + "dictionary." + ) # get the index in the output array for the current site i = self.site_index(site_gid, out_index=True) @@ -1059,12 +1166,14 @@ def site_index(self, site_gid, out_index=False): else: output_index = global_site_index - self.out_chunk[0] if output_index < 0: - raise ValueError('Attempting to set output data for site with ' - 'gid {} to global site index {}, which was ' - 'already set based on the current output ' - 'index chunk of {}' - .format(site_gid, global_site_index, - self.out_chunk)) + raise ValueError( + "Attempting to set output data for site with " + "gid {} to global site index {}, which was " + "already set based on the current output " + "index chunk of {}".format( + site_gid, global_site_index, self.out_chunk + ) + ) return output_index @@ -1079,15 +1188,17 @@ def flush(self): # handle output file request if file is specified and .out is not empty if isinstance(self._out_fpath, str) and self._out: - logger.info('Flushing outputs to disk, target file: "{}"' - .format(self._out_fpath)) + logger.info( + 'Flushing outputs to disk, target file: "{}"'.format( + self._out_fpath + ) + ) # get the slice of indices to write outputs to islice = slice(self.out_chunk[0], self.out_chunk[1] + 1) # open output file in append mode to add output results to - with Outputs(self._out_fpath, mode='a') as f: - + with Outputs(self._out_fpath, mode="a") as f: # iterate through all output requests writing each as a dataset for dset, arr in self._out.items(): if len(arr.shape) == 1: @@ -1097,7 +1208,7 @@ def flush(self): # write 2D array of profiles f[dset, :, islice] = arr - logger.debug('Flushed output successfully to disk.') + logger.debug("Flushed output successfully to disk.") def _pre_split_pc(self, pool_size=None): """Pre-split project control iterator into sub chunks to further @@ -1133,9 +1244,12 @@ def _pre_split_pc(self, pool_size=None): if i_chunk: pc_chunks.append(i_chunk) - logger.debug('Pre-splitting points control into {} chunks with the ' - 'following chunk sizes: {}' - .format(len(pc_chunks), [len(x) for x in pc_chunks])) + logger.debug( + "Pre-splitting points control into {} chunks with the " + "following chunk sizes: {}".format( + len(pc_chunks), [len(x) for x in pc_chunks] + ) + ) return N, pc_chunks # pylint: disable=unused-argument @@ -1158,8 +1272,9 @@ def _reduce_kwargs(self, pc, **kwargs): """ return kwargs - def _parallel_run(self, max_workers=None, pool_size=None, timeout=1800, - **kwargs): + def _parallel_run( + self, max_workers=None, pool_size=None, timeout=1800, **kwargs + ): """Execute parallel compute. Parameters @@ -1181,25 +1296,31 @@ def _parallel_run(self, max_workers=None, pool_size=None, timeout=1800, pool_size = os.cpu_count() * 2 if max_workers is None: max_workers = os.cpu_count() - logger.info('Running parallel execution with max_workers={}' - .format(max_workers)) + logger.info( + "Running parallel execution with max_workers={}".format( + max_workers + ) + ) i = 0 N, pc_chunks = self._pre_split_pc(pool_size=pool_size) for j, pc_chunk in enumerate(pc_chunks): - logger.debug('Starting process pool for points control ' - 'iteration {} out of {}' - .format(j + 1, len(pc_chunks))) + logger.debug( + "Starting process pool for points control " + "iteration {} out of {}".format(j + 1, len(pc_chunks)) + ) failed_futures = False chunks = {} futures = [] - loggers = [__name__, 'reV.gen', 'reV.econ', 'reV'] - with SpawnProcessPool(max_workers=max_workers, - loggers=loggers) as exe: + loggers = [__name__, "reV.gen", "reV.econ", "reV"] + with SpawnProcessPool( + max_workers=max_workers, loggers=loggers + ) as exe: for pc in pc_chunk: pc_kwargs = self._reduce_kwargs(pc, **kwargs) - future = exe.submit(self._run_single_worker, pc, - **pc_kwargs) + future = exe.submit( + self._run_single_worker, pc, **pc_kwargs + ) futures.append(future) chunks[future] = pc @@ -1210,24 +1331,32 @@ def _parallel_run(self, max_workers=None, pool_size=None, timeout=1800, except TimeoutError: failed_futures = True sites = chunks[future].project_points.sites - result = self._handle_failed_future(future, i, sites, - timeout) + result = self._handle_failed_future( + future, i, sites, timeout + ) self.out = result mem = psutil.virtual_memory() - m = ('Parallel run at iteration {0} out of {1}. ' - 'Memory utilization is {2:.3f} GB out of {3:.3f} GB ' - 'total ({4:.1f}% used, intended limit of {5:.1f}%)' - .format(i, N, mem.used / 1e9, mem.total / 1e9, - 100 * mem.used / mem.total, - 100 * self.mem_util_lim)) + m = ( + "Parallel run at iteration {0} out of {1}. " + "Memory utilization is {2:.3f} GB out of {3:.3f} GB " + "total ({4:.1f}% used, intended limit of {5:.1f}%)" + .format( + i, + N, + mem.used / 1e9, + mem.total / 1e9, + 100 * mem.used / mem.total, + 100 * self.mem_util_lim, + ) + ) logger.info(m) if failed_futures: - logger.info('Forcing pool shutdown after failed futures.') + logger.info("Forcing pool shutdown after failed futures.") exe.shutdown(wait=False) - logger.info('Forced pool shutdown complete.') + logger.info("Forced pool shutdown complete.") self.flush() @@ -1247,8 +1376,8 @@ def _handle_failed_future(self, future, i, sites, timeout): before returning zeros. """ - w = ('Iteration {} hit the timeout limit of {} seconds! Passing zeros.' - .format(i, timeout)) + w = ("Iteration {} hit the timeout limit of {} seconds! " + "Passing zeros.".format(i, timeout)) logger.warning(w) warn(w, OutputWarning) @@ -1258,12 +1387,12 @@ def _handle_failed_future(self, future, i, sites, timeout): try: cancelled = future.cancel() except Exception as e: - w = 'Could not cancel future! Received exception: {}'.format(e) + w = "Could not cancel future! Received exception: {}".format(e) logger.warning(w) warn(w, ParallelExecutionWarning) if not cancelled: - w = 'Could not cancel future!' + w = "Could not cancel future!" logger.warning(w) warn(w, ParallelExecutionWarning) diff --git a/reV/hybrids/hybrids.py b/reV/hybrids/hybrids.py index d5374d2d0..1638e62fe 100644 --- a/reV/hybrids/hybrids.py +++ b/reV/hybrids/hybrids.py @@ -3,6 +3,7 @@ @author: ppinchuk """ + import logging import re from collections import namedtuple @@ -27,21 +28,36 @@ logger = logging.getLogger(__name__) MERGE_COLUMN = MetaKeyName.SC_POINT_GID -PROFILE_DSET_REGEX = 'rep_profiles_[0-9]+$' -SOLAR_PREFIX = 'solar_' -WIND_PREFIX = 'wind_' +PROFILE_DSET_REGEX = "rep_profiles_[0-9]+$" +SOLAR_PREFIX = "solar_" +WIND_PREFIX = "wind_" NON_DUPLICATE_COLS = { - MetaKeyName.LATITUDE, MetaKeyName.LONGITUDE, 'country', 'state', 'county', MetaKeyName.ELEVATION, - MetaKeyName.TIMEZONE, MetaKeyName.SC_POINT_GID, MetaKeyName.SC_ROW_IND, MetaKeyName.SC_COL_IND + MetaKeyName.LATITUDE, + MetaKeyName.LONGITUDE, + "country", + "state", + "county", + MetaKeyName.ELEVATION, + MetaKeyName.TIMEZONE, + MetaKeyName.SC_POINT_GID, + MetaKeyName.SC_ROW_IND, + MetaKeyName.SC_COL_IND, } DROPPED_COLUMNS = [MetaKeyName.GID] -DEFAULT_FILL_VALUES = {'solar_capacity': 0, 'wind_capacity': 0, - 'solar_mean_cf': 0, 'wind_mean_cf': 0} -OUTPUT_PROFILE_NAMES = ['hybrid_profile', - 'hybrid_solar_profile', - 'hybrid_wind_profile'] -RatioColumns = namedtuple('RatioColumns', ['num', 'denom', 'fixed'], - defaults=(None, None, None)) +DEFAULT_FILL_VALUES = { + "solar_capacity": 0, + "wind_capacity": 0, + "solar_mean_cf": 0, + "wind_mean_cf": 0, +} +OUTPUT_PROFILE_NAMES = [ + "hybrid_profile", + "hybrid_solar_profile", + "hybrid_wind_profile", +] +RatioColumns = namedtuple( + "RatioColumns", ["num", "denom", "fixed"], defaults=(None, None, None) +) class ColNameFormatter: @@ -67,7 +83,7 @@ def fmt(cls, n): The column name with all characters except ascii stripped and all lowercase. """ - return ''.join(c for c in n if c in cls.ALLOWED).lower() + return "".join(c for c in n if c in cls.ALLOWED).lower() class HybridsData: @@ -164,7 +180,8 @@ def hybrid_time_index(self): """ if self._hybrid_time_index is None: self._hybrid_time_index = self.solar_time_index.join( - self.wind_time_index, how='inner') + self.wind_time_index, how="inner" + ) return self._hybrid_time_index def contains_col(self, col_name): @@ -208,9 +225,11 @@ def _validate_time_index(self): If len(time_index) < 8760 for the hybrid profile. """ if len(self.hybrid_time_index) < 8760: - msg = ("The length of the merged time index ({}) is less than " - "8760. Please ensure that the input profiles have a " - "time index that overlaps >= 8760 times.") + msg = ( + "The length of the merged time index ({}) is less than " + "8760. Please ensure that the input profiles have a " + "time index that overlaps >= 8760 times." + ) e = msg.format(len(self.hybrid_time_index)) logger.error(e) raise FileInputError(e) @@ -226,21 +245,24 @@ def _validate_num_profiles(self): for fp in [self.solar_fpath, self.wind_fpath]: with Resource(fp) as res: profile_dset_names = [ - n for n in res.dsets - if self.__profile_reg_check.match(n) + n for n in res.dsets if self.__profile_reg_check.match(n) ] if not profile_dset_names: - msg = ("Did not find any data sets matching the regex: " - "{!r} in {!r}. Please ensure that the profile data " - "exists and that the data set is named correctly.") + msg = ( + "Did not find any data sets matching the regex: " + "{!r} in {!r}. Please ensure that the profile data " + "exists and that the data set is named correctly." + ) e = msg.format(PROFILE_DSET_REGEX, fp) logger.error(e) raise FileInputError(e) if len(profile_dset_names) > 1: - msg = ("Found more than one profile in {!r}: {}. " - "This module is not intended for hybridization of " - "multiple representative profiles. Please re-run " - "on a single aggregated profile.") + msg = ( + "Found more than one profile in {!r}: {}. " + "This module is not intended for hybridization of " + "multiple representative profiles. Please re-run " + "on a single aggregated profile." + ) e = msg.format(fp, profile_dset_names) logger.error(e) raise FileInputError(e) @@ -255,13 +277,17 @@ def _validate_merge_col_exists(self): If merge column is missing from either the solar or the wind meta data. """ - msg = ("Cannot hybridize: merge column {!r} missing from the " - "{} meta data! ({!r})") + msg = ( + "Cannot hybridize: merge column {!r} missing from the " + "{} meta data! ({!r})" + ) mc = ColNameFormatter.fmt(MERGE_COLUMN) - for cols, fp, res in zip([self.__solar_cols, self.__wind_cols], - [self.solar_fpath, self.wind_fpath], - ['solar', 'wind']): + for cols, fp, res in zip( + [self.__solar_cols, self.__wind_cols], + [self.solar_fpath, self.wind_fpath], + ["solar", "wind"], + ): if mc not in cols: e = msg.format(MERGE_COLUMN, res, fp) logger.error(e) @@ -276,16 +302,20 @@ def _validate_unique_merge_col(self): If merge column contains duplicate values in either the solar or the wind meta data. """ - msg = ("Duplicate {}s were found. This is likely due to resource " - "class binning, which is not supported at this time. " - "Please re-run supply curve aggregation without " - "resource class binning and ensure there are no duplicate " - "values in {!r}. File: {!r}") + msg = ( + "Duplicate {}s were found. This is likely due to resource " + "class binning, which is not supported at this time. " + "Please re-run supply curve aggregation without " + "resource class binning and ensure there are no duplicate " + "values in {!r}. File: {!r}" + ) mc = ColNameFormatter.fmt(MERGE_COLUMN) - for ds, cols, fp in zip([self.solar_meta, self.wind_meta], - [self.__solar_cols, self.__wind_cols], - [self.solar_fpath, self.wind_fpath]): + for ds, cols, fp in zip( + [self.solar_meta, self.wind_meta], + [self.__solar_cols, self.__wind_cols], + [self.solar_fpath, self.wind_fpath], + ): merge_col = ds.columns[cols == mc].item() if not ds[merge_col].is_unique: e = msg.format(merge_col, merge_col, fp) @@ -308,11 +338,14 @@ def _validate_merge_col_overlaps(self): self.merge_col_overlap_values = solar_vals & wind_vals if not self.merge_col_overlap_values: - msg = ("No overlap detected in the values of {!r} across the " - "input files. Please ensure that at least one of the " - "{!r} values is the same for input files {!r} and {!r}") - e = msg.format(merge_col, merge_col, self.solar_fpath, - self.wind_fpath) + msg = ( + "No overlap detected in the values of {!r} across the " + "input files. Please ensure that at least one of the " + "{!r} values is the same for input files {!r} and {!r}" + ) + e = msg.format( + merge_col, merge_col, self.solar_fpath, self.wind_fpath + ) logger.error(e) raise FileInputError(e) @@ -320,12 +353,18 @@ def _validate_merge_col_overlaps(self): class MetaHybridizer: """Framework to handle hybridization of meta data.""" - _INTERNAL_COL_PREFIX = '_h_internal' - - def __init__(self, data, allow_solar_only=False, - allow_wind_only=False, fillna=None, - limits=None, ratio_bounds=None, - ratio='solar_capacity/wind_capacity'): + _INTERNAL_COL_PREFIX = "_h_internal" + + def __init__( + self, + data, + allow_solar_only=False, + allow_wind_only=False, + fillna=None, + limits=None, + ratio_bounds=None, + ratio="solar_capacity/wind_capacity", + ): """ Parameters ---------- @@ -388,8 +427,8 @@ def __init__(self, data, allow_solar_only=False, self._hybrid_meta = None self.__hybrid_meta_cols = None self.__col_name_map = None - self.__solar_rpi_n = '{}_solar_rpidx'.format(self._INTERNAL_COL_PREFIX) - self.__wind_rpi_n = '{}_wind_rpidx'.format(self._INTERNAL_COL_PREFIX) + self.__solar_rpi_n = "{}_solar_rpidx".format(self._INTERNAL_COL_PREFIX) + self.__wind_rpi_n = "{}_wind_rpidx".format(self._INTERNAL_COL_PREFIX) @property def hybrid_meta(self): @@ -430,7 +469,7 @@ def _validate_limits_cols_prefixed(self): """ for col in self._limits: self.__validate_col_prefix( - col, (SOLAR_PREFIX, WIND_PREFIX), input_name='limits' + col, (SOLAR_PREFIX, WIND_PREFIX), input_name="limits" ) @staticmethod @@ -439,9 +478,11 @@ def __validate_col_prefix(col, prefixes, input_name): missing = [not col.startswith(p) for p in prefixes] if all(missing): - msg = ("Input {0} column {1!r} does not start with a valid " - "prefix: {2!r}. Please ensure that the {0} column " - "names specify the correct resource prefix.") + msg = ( + "Input {0} column {1!r} does not start with a valid " + "prefix: {2!r}. Please ensure that the {0} column " + "names specify the correct resource prefix." + ) e = msg.format(input_name, col, prefixes) logger.error(e) raise InputError(e) @@ -461,7 +502,7 @@ def _validate_fillna_cols_prefixed(self): """ for col in self._fillna: self.__validate_col_prefix( - col, (SOLAR_PREFIX, WIND_PREFIX), input_name='fillna' + col, (SOLAR_PREFIX, WIND_PREFIX), input_name="fillna" ) def _validate_ratio_input(self): @@ -491,18 +532,22 @@ def _validate_ratio_bounds(self): try: if len(self._ratio_bounds) != 2: - msg = ("Length of input for ratio_bounds is {} - but is " - "required to be of length 2. Please make sure this " - "input is a len 2 container of floats. If you would " - "like to specify a single ratio value, use the same " - "float for both limits (i.e. ratio_bounds=(1, 1)).") + msg = ( + "Length of input for ratio_bounds is {} - but is " + "required to be of length 2. Please make sure this " + "input is a len 2 container of floats. If you would " + "like to specify a single ratio value, use the same " + "float for both limits (i.e. ratio_bounds=(1, 1))." + ) e = msg.format(len(self._ratio_bounds)) logger.error(e) raise InputError(e) except TypeError: - msg = ("Input for ratio_bounds not understood: {!r}. " - "Please make sure this value is a len 2 container " - "of floats.") + msg = ( + "Input for ratio_bounds not understood: {!r}. " + "Please make sure this value is a len 2 container " + "of floats." + ) e = msg.format(self._ratio_bounds) logger.error(e) raise InputError(e) from None @@ -516,10 +561,12 @@ def _validate_ratio_type(self): If `ratio` is not a string. """ if not isinstance(self._ratio, str): - msg = ("Ratio input type {} not understood. Please make sure " - "the ratio input is a string in the form " - "'numerator_column_name/denominator_column_name'. Ratio " - "input: {!r}") + msg = ( + "Ratio input type {} not understood. Please make sure " + "the ratio input is a string in the form " + "'numerator_column_name/denominator_column_name'. Ratio " + "input: {!r}" + ) e = msg.format(type(self._ratio), self._ratio) logger.error(e) raise InputError(e) @@ -533,18 +580,22 @@ def _validate_ratio_format(self): If the '/' character is missing or of there are too many '/' characters. """ - if '/' not in self._ratio: - msg = ("Ratio input {} does not contain the '/' character. " - "Please make sure the ratio input is a string in the form " - "'numerator_column_name/denominator_column_name'") + if "/" not in self._ratio: + msg = ( + "Ratio input {} does not contain the '/' character. " + "Please make sure the ratio input is a string in the form " + "'numerator_column_name/denominator_column_name'" + ) e = msg.format(self._ratio) logger.error(e) raise InputError(e) if len(self._ratio_cols) != 2: - msg = ("Ratio input {} contains too many '/' characters. Please " - "make sure the ratio input is a string in the form " - "'numerator_column_name/denominator_column_name'.") + msg = ( + "Ratio input {} contains too many '/' characters. Please " + "make sure the ratio input is a string in the form " + "'numerator_column_name/denominator_column_name'." + ) e = msg.format(self._ratio) logger.error(e) raise InputError(e) @@ -565,7 +616,7 @@ def _validate_ratio_cols_prefixed(self): for col in self._ratio_cols: self.__validate_col_prefix( - col, (SOLAR_PREFIX, WIND_PREFIX), input_name='ratios' + col, (SOLAR_PREFIX, WIND_PREFIX), input_name="ratios" ) def _validate_ratio_cols_exist(self): @@ -578,12 +629,15 @@ def _validate_ratio_cols_exist(self): """ for col in self._ratio_cols: - no_prefix_name = "_".join(col.split('_')[1:]) + no_prefix_name = "_".join(col.split("_")[1:]) if not self.data.contains_col(no_prefix_name): - msg = ("Input ratios column {!r} not found in either meta " - "data! Please check the input files {!r} and {!r}") - e = msg.format(no_prefix_name, self.data.solar_fpath, - self.data.wind_fpath) + msg = ( + "Input ratios column {!r} not found in either meta " + "data! Please check the input files {!r} and {!r}" + ) + e = msg.format( + no_prefix_name, self.data.solar_fpath, self.data.wind_fpath + ) logger.error(e) raise FileInputError(e) @@ -592,7 +646,7 @@ def _ratio_cols(self): """Get the ratio columns from the ratio input.""" if self._ratio is None: return [] - return self._ratio.strip().split('/') + return self._ratio.strip().split("/") def hybridize(self): """Combine the solar and wind metas and run hybridize methods.""" @@ -624,7 +678,7 @@ def _rename_cols(df, prefix): df.columns = [ ColNameFormatter.fmt(col_name) if col_name in NON_DUPLICATE_COLS - else '{}{}'.format(prefix, col_name) + else "{}{}".format(prefix, col_name) for col_name in df.columns.values ] @@ -639,18 +693,19 @@ def _merge_solar_wind_meta(self): self._hybrid_meta = self.data.solar_meta.merge( self.data.wind_meta, on=ColNameFormatter.fmt(MERGE_COLUMN), - suffixes=[None, '_x'], how=self._merge_type() + suffixes=[None, "_x"], + how=self._merge_type(), ) def _merge_type(self): """Determine the type of merge to use for meta based on user input.""" if self._allow_solar_only and self._allow_wind_only: - return 'outer' + return "outer" if self._allow_solar_only and not self._allow_wind_only: - return 'left' + return "left" if not self._allow_solar_only and self._allow_wind_only: - return 'right' - return 'inner' + return "right" + return "inner" def _format_meta_post_merge(self): """Format hybrid meta after merging.""" @@ -673,25 +728,30 @@ def _drop_cols(self, duplicate_cols): """Drop any remaning duplicate and 'DROPPED_COLUMNS' columns.""" self._hybrid_meta.drop( duplicate_cols + DROPPED_COLUMNS, - axis=1, inplace=True, errors='ignore' + axis=1, + inplace=True, + errors="ignore", ) def _sort_hybrid_meta_cols(self): """Sort the columns of the hybrid meta.""" self.__hybrid_meta_cols = sorted( - [c for c in self._hybrid_meta.columns - if not c.startswith(self._INTERNAL_COL_PREFIX)], - key=self._column_sorting_key + [ + c + for c in self._hybrid_meta.columns + if not c.startswith(self._INTERNAL_COL_PREFIX) + ], + key=self._column_sorting_key, ) def _column_sorting_key(self, c): """Helper function to sort hybrid meta columns.""" first_index = 0 - if c.startswith('hybrid'): + if c.startswith("hybrid"): first_index = 1 - elif c.startswith('solar'): + elif c.startswith("solar"): first_index = 2 - elif c.startswith('wind'): + elif c.startswith("wind"): first_index = 3 elif c == MERGE_COLUMN: first_index = -1 @@ -702,18 +762,21 @@ def _verify_lat_long_match_post_merge(self): lat = self._verify_col_match_post_merge(col_name=MetaKeyName.LATITUDE) lon = self._verify_col_match_post_merge(col_name=MetaKeyName.LONGITUDE) if not lat or not lon: - msg = ("Detected mismatched coordinate values (latitude or " - "longitude) post merge. Please ensure that all matching " - "values of {!r} correspond to the same values of latitude " - "and longitude across the input files {!r} and {!r}") - e = msg.format(MERGE_COLUMN, self.data.solar_fpath, - self.data.wind_fpath) + msg = ( + "Detected mismatched coordinate values (latitude or " + "longitude) post merge. Please ensure that all matching " + "values of {!r} correspond to the same values of latitude " + "and longitude across the input files {!r} and {!r}" + ) + e = msg.format( + MERGE_COLUMN, self.data.solar_fpath, self.data.wind_fpath + ) logger.error(e) raise FileInputError(e) def _verify_col_match_post_merge(self, col_name): """Verify that all (non-null) values in a column match post merge.""" - c1, c2 = col_name, '{}_x'.format(col_name) + c1, c2 = col_name, "{}_x".format(col_name) if c1 in self._hybrid_meta.columns and c2 in self._hybrid_meta.columns: compare_df = self._hybrid_meta[ (self._hybrid_meta[c1].notnull()) @@ -728,7 +791,7 @@ def _fillna_meta_cols(self): if col_name in self._hybrid_meta.columns: self._hybrid_meta[col_name].fillna(fill_value, inplace=True) else: - self.__warn_missing_col(col_name, action='fill') + self.__warn_missing_col(col_name, action="fill") self._hybrid_meta[self.__solar_rpi_n].fillna(-1, inplace=True) self._hybrid_meta[self.__wind_rpi_n].fillna(-1, inplace=True) @@ -736,9 +799,11 @@ def _fillna_meta_cols(self): @staticmethod def __warn_missing_col(col_name, action): """Warn that a column the user request an action for is missing.""" - msg = ("Skipping {} values for {!r}: Unable to find column " - "in hybrid meta. Did you forget to prefix with " - "{!r} or {!r}? ") + msg = ( + "Skipping {} values for {!r}: Unable to find column " + "in hybrid meta. Did you forget to prefix with " + "{!r} or {!r}? " + ) w = msg.format(action, col_name, SOLAR_PREFIX, WIND_PREFIX) logger.warning(w) warn(w, InputWarning) @@ -749,7 +814,7 @@ def _apply_limits(self): if col_name in self._hybrid_meta.columns: self._hybrid_meta[col_name].clip(upper=max_value, inplace=True) else: - self.__warn_missing_col(col_name, action='limit') + self.__warn_missing_col(col_name, action="limit") def _limit_by_ratio(self): """Limit the given pair of ratio columns based on input ratio.""" @@ -768,8 +833,7 @@ def _limit_by_ratio(self): denominator_vals = self._hybrid_meta[denominator_col].copy() ratios = ( - numerator_vals.loc[overlap_idx] - / denominator_vals.loc[overlap_idx] + numerator_vals.loc[overlap_idx] / denominator_vals.loc[overlap_idx] ) ratio_too_low = (ratios < min_ratio) & overlap_idx ratio_too_high = (ratios > max_ratio) & overlap_idx @@ -794,9 +858,11 @@ def _add_hybrid_cols(self): try: self._hybrid_meta[new_col_name] = out except ValueError as e: - msg = ("Unable to add {!r} column to hybrid meta. The " - "following exception was raised when adding " - "the data output by '{}': {!r}.") + msg = ( + "Unable to add {!r} column to hybrid meta. The " + "following exception was raised when adding " + "the data output by '{}': {!r}." + ) w = msg.format(new_col_name, method.__name__, e) logger.warning(w) warn(w, OutputWarning) @@ -846,9 +912,17 @@ def wind_profile_indices_map(self): class Hybridization: """Hybridization""" - def __init__(self, solar_fpath, wind_fpath, allow_solar_only=False, - allow_wind_only=False, fillna=None, limits=None, - ratio_bounds=None, ratio='solar_capacity/wind_capacity'): + def __init__( + self, + solar_fpath, + wind_fpath, + allow_solar_only=False, + allow_wind_only=False, + fillna=None, + limits=None, + ratio_bounds=None, + ratio="solar_capacity/wind_capacity", + ): """Framework to handle hybridization of SC and corresponding profiles. ``reV`` hybrids computes a "hybrid" wind and solar supply curve, @@ -912,28 +986,51 @@ def __init__(self, solar_fpath, wind_fpath, allow_solar_only=False, variables. By default ``'solar_capacity/wind_capacity'``. """ - logger.info('Running hybridization of rep profiles with solar_fpath: ' - '"{}"'.format(solar_fpath)) - logger.info('Running hybridization of rep profiles with solar_fpath: ' - '"{}"'.format(wind_fpath)) - logger.info('Running hybridization of rep profiles with ' - 'allow_solar_only: "{}"'.format(allow_solar_only)) - logger.info('Running hybridization of rep profiles with ' - 'allow_wind_only: "{}"'.format(allow_wind_only)) - logger.info('Running hybridization of rep profiles with fillna: "{}"' - .format(fillna)) - logger.info('Running hybridization of rep profiles with limits: "{}"' - .format(limits)) - logger.info('Running hybridization of rep profiles with ratio_bounds: ' - '"{}"'.format(ratio_bounds)) - logger.info('Running hybridization of rep profiles with ratio: "{}"' - .format(ratio)) + logger.info( + "Running hybridization of rep profiles with solar_fpath: " + '"{}"'.format(solar_fpath) + ) + logger.info( + "Running hybridization of rep profiles with solar_fpath: " + '"{}"'.format(wind_fpath) + ) + logger.info( + "Running hybridization of rep profiles with " + 'allow_solar_only: "{}"'.format(allow_solar_only) + ) + logger.info( + "Running hybridization of rep profiles with " + 'allow_wind_only: "{}"'.format(allow_wind_only) + ) + logger.info( + 'Running hybridization of rep profiles with fillna: "{}"'.format( + fillna + ) + ) + logger.info( + 'Running hybridization of rep profiles with limits: "{}"'.format( + limits + ) + ) + logger.info( + "Running hybridization of rep profiles with ratio_bounds: " + '"{}"'.format(ratio_bounds) + ) + logger.info( + 'Running hybridization of rep profiles with ratio: "{}"'.format( + ratio + ) + ) self.data = HybridsData(solar_fpath, wind_fpath) self.meta_hybridizer = MetaHybridizer( - data=self.data, allow_solar_only=allow_solar_only, - allow_wind_only=allow_wind_only, fillna=fillna, limits=limits, - ratio_bounds=ratio_bounds, ratio=ratio + data=self.data, + allow_solar_only=allow_solar_only, + allow_wind_only=allow_wind_only, + fillna=fillna, + limits=limits, + ratio_bounds=ratio_bounds, + ratio=ratio, ) self._profiles = None self._validate_input() @@ -1045,7 +1142,7 @@ def run(self, fout=None, save_hybrid_meta=True): if fout is not None: self.save_profiles(fout, save_hybrid_meta=save_hybrid_meta) - logger.info('Hybridization of representative profiles complete!') + logger.info("Hybridization of representative profiles complete!") return fout def run_meta(self): @@ -1070,22 +1167,25 @@ def run_profiles(self): hybridized profiles as attributes. """ - logger.info('Running hybrid profile calculations.') + logger.info("Running hybrid profile calculations.") self._init_profiles() self._compute_hybridized_profile_components() self._compute_hybridized_profiles_from_components() - logger.info('Profile hybridization complete.') + logger.info("Profile hybridization complete.") return self def _init_profiles(self): """Initialize the output rep profiles attribute.""" self._profiles = { - k: np.zeros((len(self.hybrid_time_index), len(self.hybrid_meta)), - dtype=np.float32) - for k in OUTPUT_PROFILE_NAMES} + k: np.zeros( + (len(self.hybrid_time_index), len(self.hybrid_meta)), + dtype=np.float32, + ) + for k in OUTPUT_PROFILE_NAMES + } def _compute_hybridized_profile_components(self): """Compute the resource components of the hybridized profiles.""" @@ -1095,29 +1195,39 @@ def _compute_hybridized_profile_components(self): capacity = self.hybrid_meta.loc[hybrid_idxs, col].values with Resource(fpath) as res: - data = res[dset_name, - res.time_index.isin(self.hybrid_time_index)] - self._profiles[p_name][:, hybrid_idxs] = (data[:, solar_idxs] - * capacity) + data = res[ + dset_name, res.time_index.isin(self.hybrid_time_index) + ] + self._profiles[p_name][:, hybrid_idxs] = ( + data[:, solar_idxs] * capacity + ) @property def __rep_profile_hybridization_params(self): """Zip the rep profile hybridization parameters.""" - cap_col_names = ['hybrid_solar_capacity', 'hybrid_wind_capacity'] - idx_maps = [self.meta_hybridizer.solar_profile_indices_map, - self.meta_hybridizer.wind_profile_indices_map] + cap_col_names = ["hybrid_solar_capacity", "hybrid_wind_capacity"] + idx_maps = [ + self.meta_hybridizer.solar_profile_indices_map, + self.meta_hybridizer.wind_profile_indices_map, + ] fpaths = [self.data.solar_fpath, self.data.wind_fpath] - zipped = zip(cap_col_names, idx_maps, fpaths, OUTPUT_PROFILE_NAMES[1:], - self.data.profile_dset_names) + zipped = zip( + cap_col_names, + idx_maps, + fpaths, + OUTPUT_PROFILE_NAMES[1:], + self.data.profile_dset_names, + ) return zipped def _compute_hybridized_profiles_from_components(self): """Compute the hybridized profiles from the resource components.""" hp_name, sp_name, wp_name = OUTPUT_PROFILE_NAMES - self._profiles[hp_name] = (self._profiles[sp_name] - + self._profiles[wp_name]) + self._profiles[hp_name] = ( + self._profiles[sp_name] + self._profiles[wp_name] + ) def _init_h5_out(self, fout, save_hybrid_meta=True): """Initialize an output h5 file for hybrid profiles. @@ -1149,14 +1259,26 @@ def _init_h5_out(self, fout, save_hybrid_meta=True): except ValueError: pass - Outputs.init_h5(fout, dsets, shapes, attrs, chunks, dtypes, - meta, time_index=self.hybrid_time_index) + Outputs.init_h5( + fout, + dsets, + shapes, + attrs, + chunks, + dtypes, + meta, + time_index=self.hybrid_time_index, + ) if save_hybrid_meta: - with Outputs(fout, mode='a') as out: + with Outputs(fout, mode="a") as out: hybrid_meta = to_records_array(self.hybrid_meta) - out._create_dset('meta', hybrid_meta.shape, - hybrid_meta.dtype, data=hybrid_meta) + out._create_dset( + "meta", + hybrid_meta.shape, + hybrid_meta.dtype, + data=hybrid_meta, + ) def _write_h5_out(self, fout, save_hybrid_meta=True): """Write hybrid profiles and meta to an output file. @@ -1169,10 +1291,10 @@ def _write_h5_out(self, fout, save_hybrid_meta=True): Flag to save hybrid SC table to hybrid rep profile output. """ - with Outputs(fout, mode='a') as out: - if 'meta' in out.datasets and save_hybrid_meta: + with Outputs(fout, mode="a") as out: + if "meta" in out.datasets and save_hybrid_meta: hybrid_meta = to_records_array(self.hybrid_meta) - out['meta'] = hybrid_meta + out["meta"] = hybrid_meta for dset, data in self.profiles.items(): out[dset] = data diff --git a/reV/qa_qc/qa_qc.py b/reV/qa_qc/qa_qc.py index fd69fc687..899ae00ed 100644 --- a/reV/qa_qc/qa_qc.py +++ b/reV/qa_qc/qa_qc.py @@ -2,6 +2,7 @@ """ reV quality assurance and control classes """ + import logging import os from warnings import warn @@ -37,7 +38,7 @@ def __init__(self, out_dir): Directory path to save summary data and plots too """ log_versions(logger) - logger.info('QA/QC results to be saved to: {}'.format(out_dir)) + logger.info("QA/QC results to be saved to: {}".format(out_dir)) if not os.path.exists(out_dir): os.makedirs(out_dir, exist_ok=True) @@ -55,8 +56,9 @@ def out_dir(self): return self._out_dir @staticmethod - def _scatter_plot(summary_csv, out_root, plot_type='plotly', - cmap='viridis', **kwargs): + def _scatter_plot( + summary_csv, out_root, plot_type="plotly", cmap="viridis", **kwargs + ): """ Create scatter plot for all summary stats in summary table and save to out_dir @@ -74,16 +76,19 @@ def _scatter_plot(summary_csv, out_root, plot_type='plotly', kwargs : dict Additional plotting kwargs """ - out_dir = os.path.join(out_root, - os.path.basename(summary_csv).rstrip('.csv')) + out_dir = os.path.join( + out_root, os.path.basename(summary_csv).rstrip(".csv") + ) if not os.path.exists(out_dir): os.makedirs(out_dir, exist_ok=True) - SummaryPlots.scatter_all(summary_csv, out_dir, plot_type=plot_type, - cmap=cmap, **kwargs) + SummaryPlots.scatter_all( + summary_csv, out_dir, plot_type=plot_type, cmap=cmap, **kwargs + ) - def create_scatter_plots(self, plot_type='plotly', cmap='viridis', - **kwargs): + def create_scatter_plots( + self, plot_type="plotly", cmap="viridis", **kwargs + ): """ Create scatter plot for all compatible summary .csv files @@ -97,18 +102,35 @@ def create_scatter_plots(self, plot_type='plotly', cmap='viridis', Additional plotting kwargs """ for file in os.listdir(self.out_dir): - if file.endswith('.csv'): + if file.endswith(".csv"): summary_csv = os.path.join(self.out_dir, file) summary = pd.read_csv(summary_csv) - if (MetaKeyName.GID in summary and MetaKeyName.LATITUDE in summary - and MetaKeyName.LONGITUDE in summary): - self._scatter_plot(summary_csv, self.out_dir, - plot_type=plot_type, cmap=cmap, - **kwargs) + if ( + MetaKeyName.GID in summary + and MetaKeyName.LATITUDE in summary + and MetaKeyName.LONGITUDE in summary + ): + self._scatter_plot( + summary_csv, + self.out_dir, + plot_type=plot_type, + cmap=cmap, + **kwargs, + ) @classmethod - def h5(cls, h5_file, out_dir, dsets=None, group=None, process_size=None, - max_workers=None, plot_type='plotly', cmap='viridis', **kwargs): + def h5( + cls, + h5_file, + out_dir, + dsets=None, + group=None, + process_size=None, + max_workers=None, + plot_type="plotly", + cmap="viridis", + **kwargs, + ): """ Run QA/QC by computing summary stats from dsets in h5_file and plotting scatters plots of compatible summary stats @@ -137,23 +159,43 @@ def h5(cls, h5_file, out_dir, dsets=None, group=None, process_size=None, """ try: qa_qc = cls(out_dir) - SummarizeH5.run(h5_file, out_dir, group=group, - dsets=dsets, process_size=process_size, - max_workers=max_workers) - qa_qc.create_scatter_plots(plot_type=plot_type, cmap=cmap, - **kwargs) + SummarizeH5.run( + h5_file, + out_dir, + group=group, + dsets=dsets, + process_size=process_size, + max_workers=max_workers, + ) + qa_qc.create_scatter_plots( + plot_type=plot_type, cmap=cmap, **kwargs + ) except Exception as e: - logger.exception('QAQC failed on file: {}. Received exception:\n{}' - .format(os.path.basename(h5_file), e)) + logger.exception( + "QAQC failed on file: {}. Received exception:\n{}".format( + os.path.basename(h5_file), e + ) + ) raise e else: - logger.info('Finished QAQC on file: {} output directory: {}' - .format(os.path.basename(h5_file), out_dir)) + logger.info( + "Finished QAQC on file: {} output directory: {}".format( + os.path.basename(h5_file), out_dir + ) + ) @classmethod - def supply_curve(cls, sc_table, out_dir, columns=None, lcoe=MetaKeyName.MEAN_LCOE, - plot_type='plotly', cmap='viridis', sc_plot_kwargs=None, - scatter_plot_kwargs=None): + def supply_curve( + cls, + sc_table, + out_dir, + columns=None, + lcoe=MetaKeyName.MEAN_LCOE, + plot_type="plotly", + cmap="viridis", + sc_plot_kwargs=None, + scatter_plot_kwargs=None, + ): """ Plot supply curve @@ -186,22 +228,48 @@ def supply_curve(cls, sc_table, out_dir, columns=None, lcoe=MetaKeyName.MEAN_LCO try: qa_qc = cls(out_dir) SummarizeSupplyCurve.run(sc_table, out_dir, columns=columns) - SupplyCurvePlot.plot(sc_table, out_dir, plot_type=plot_type, - lcoe=lcoe, **sc_plot_kwargs) - qa_qc._scatter_plot(sc_table, out_dir, plot_type=plot_type, - cmap=cmap, **scatter_plot_kwargs) + SupplyCurvePlot.plot( + sc_table, + out_dir, + plot_type=plot_type, + lcoe=lcoe, + **sc_plot_kwargs, + ) + qa_qc._scatter_plot( + sc_table, + out_dir, + plot_type=plot_type, + cmap=cmap, + **scatter_plot_kwargs, + ) except Exception as e: - logger.exception('QAQC failed on file: {}. Received exception:\n{}' - .format(os.path.basename(sc_table), e)) + logger.exception( + "QAQC failed on file: {}. Received exception:\n{}".format( + os.path.basename(sc_table), e + ) + ) raise e else: - logger.info('Finished QAQC on file: {} output directory: {}' - .format(os.path.basename(sc_table), out_dir)) + logger.info( + "Finished QAQC on file: {} output directory: {}".format( + os.path.basename(sc_table), out_dir + ) + ) @classmethod - def exclusions_mask(cls, excl_h5, out_dir, layers_dict=None, min_area=None, - kernel='queen', hsds=False, plot_type='plotly', - cmap='viridis', plot_step=100, **kwargs): + def exclusions_mask( + cls, + excl_h5, + out_dir, + layers_dict=None, + min_area=None, + kernel="queen", + hsds=False, + plot_type="plotly", + cmap="viridis", + plot_step=100, + **kwargs, + ): """ Create inclusion mask from given layers dictionary, dump to disk and plot @@ -230,26 +298,40 @@ def exclusions_mask(cls, excl_h5, out_dir, layers_dict=None, min_area=None, """ try: cls(out_dir) - excl_mask = ExclusionMaskFromDict.run(excl_h5, - layers_dict=layers_dict, - min_area=min_area, - kernel=kernel, - hsds=hsds) - excl_mask = np.round(excl_mask * 100).astype('uint8') - - out_file = os.path.basename(excl_h5).replace('.h5', '_mask.npy') + excl_mask = ExclusionMaskFromDict.run( + excl_h5, + layers_dict=layers_dict, + min_area=min_area, + kernel=kernel, + hsds=hsds, + ) + excl_mask = np.round(excl_mask * 100).astype("uint8") + + out_file = os.path.basename(excl_h5).replace(".h5", "_mask.npy") out_file = os.path.join(out_dir, out_file) np.save(out_file, excl_mask) - ExclusionsMask.plot(excl_mask, out_dir, plot_type=plot_type, - cmap=cmap, plot_step=plot_step, **kwargs) + ExclusionsMask.plot( + excl_mask, + out_dir, + plot_type=plot_type, + cmap=cmap, + plot_step=plot_step, + **kwargs, + ) except Exception as e: - logger.exception('QAQC failed on file: {}. Received exception:\n{}' - .format(os.path.basename(excl_h5), e)) + logger.exception( + "QAQC failed on file: {}. Received exception:\n{}".format( + os.path.basename(excl_h5), e + ) + ) raise e else: - logger.info('Finished QAQC on file: {} output directory: {}' - .format(os.path.basename(excl_h5), out_dir)) + logger.info( + "Finished QAQC on file: {} output directory: {}".format( + os.path.basename(excl_h5), out_dir + ) + ) class QaQcModule: @@ -263,17 +345,20 @@ def __init__(self, module_name, config, out_root): Dictionary with pre-extracted config input group. """ if not isinstance(config, dict): - raise TypeError('Config input must be a dict but received: {}' - .format(type(config))) + raise TypeError( + "Config input must be a dict but received: {}".format( + type(config) + ) + ) self._name = module_name self._config = config self._out_root = out_root - self._default_plot_type = 'plotly' - self._default_cmap = 'viridis' + self._default_plot_type = "plotly" + self._default_cmap = "viridis" self._default_plot_step = 100 self._default_lcoe = MetaKeyName.MEAN_LCOE - self._default_area_filter_kernel = 'queen' + self._default_area_filter_kernel = "queen" @property def fpath(self): @@ -285,20 +370,23 @@ def fpath(self): One or more filepaths output by current module being QA'd """ - fpath = self._config['fpath'] + fpath = self._config["fpath"] - if fpath == 'PIPELINE': + if fpath == "PIPELINE": target_modules = [self._name] for target_module in target_modules: fpath = Status.parse_step_status(self._out_root, target_module) if fpath: break else: - raise PipelineError('Could not parse fpath from previous ' - 'pipeline jobs.') + raise PipelineError( + "Could not parse fpath from previous " "pipeline jobs." + ) fpath = fpath[0] - logger.info('QA/QC using the following ' - 'pipeline input for fpath: {}'.format(fpath)) + logger.info( + "QA/QC using the following " + "pipeline input for fpath: {}".format(fpath) + ) return fpath @@ -307,141 +395,157 @@ def sub_dir(self): """ QA/QC sub directory for this module's outputs """ - return self._config.get('sub_dir', None) + return self._config.get("sub_dir", None) @property def plot_type(self): """Get the QA/QC plot type: either 'plot' or 'plotly'""" - return self._config.get('plot_type', self._default_plot_type) + return self._config.get("plot_type", self._default_plot_type) @property def dsets(self): """Get the reV_h5 dsets to QA/QC""" - return self._config.get('dsets', None) + return self._config.get("dsets", None) @property def group(self): """Get the reV_h5 group to QA/QC""" - return self._config.get('group', None) + return self._config.get("group", None) @property def process_size(self): """Get the reV_h5 process_size for QA/QC""" - return self._config.get('process_size', None) + return self._config.get("process_size", None) @property def cmap(self): """Get the QA/QC plot colormap""" - return self._config.get('cmap', self._default_cmap) + return self._config.get("cmap", self._default_cmap) @property def plot_step(self): """Get the QA/QC step between exclusion mask points to plot""" - return self._config.get('cmap', self._default_plot_step) + return self._config.get("cmap", self._default_plot_step) @property def columns(self): """Get the supply_curve columns to QA/QC""" - return self._config.get('columns', None) + return self._config.get("columns", None) @property def lcoe(self): """Get the supply_curve lcoe column to plot""" - return self._config.get('lcoe', self._default_lcoe) + return self._config.get("lcoe", self._default_lcoe) @property def excl_fpath(self): """Get the source exclusions filepath""" - excl_fpath = self._config.get('excl_fpath', 'PIPELINE') + excl_fpath = self._config.get("excl_fpath", "PIPELINE") - if excl_fpath == 'PIPELINE': + if excl_fpath == "PIPELINE": target_module = ModuleName.SUPPLY_CURVE_AGGREGATION - excl_fpath = Status.parse_step_status(self._out_root, - target_module, - key='excl_fpath') + excl_fpath = Status.parse_step_status( + self._out_root, target_module, key="excl_fpath" + ) if not excl_fpath: excl_fpath = None - msg = ('Could not parse excl_fpath from previous ' - 'pipeline jobs, defaulting to: {}'.format(excl_fpath)) + msg = ( + "Could not parse excl_fpath from previous " + "pipeline jobs, defaulting to: {}".format(excl_fpath) + ) logger.warning(msg) warn(msg) else: excl_fpath = excl_fpath[0] - logger.info('QA/QC using the following ' - 'pipeline input for excl_fpath: {}' - .format(excl_fpath)) + logger.info( + "QA/QC using the following " + "pipeline input for excl_fpath: {}".format(excl_fpath) + ) return excl_fpath @property def excl_dict(self): """Get the exclusions dictionary""" - excl_dict = self._config.get('excl_dict', 'PIPELINE') + excl_dict = self._config.get("excl_dict", "PIPELINE") - if excl_dict == 'PIPELINE': + if excl_dict == "PIPELINE": target_module = ModuleName.SUPPLY_CURVE_AGGREGATION - excl_dict = Status.parse_step_status(self._out_root, target_module, - key='excl_dict') + excl_dict = Status.parse_step_status( + self._out_root, target_module, key="excl_dict" + ) if not excl_dict: excl_dict = None - msg = ('Could not parse excl_dict from previous ' - 'pipeline jobs, defaulting to: {}'.format(excl_dict)) + msg = ( + "Could not parse excl_dict from previous " + "pipeline jobs, defaulting to: {}".format(excl_dict) + ) logger.warning(msg) warn(msg) else: excl_dict = excl_dict[0] - logger.info('QA/QC using the following ' - 'pipeline input for excl_dict: {}' - .format(excl_dict)) + logger.info( + "QA/QC using the following " + "pipeline input for excl_dict: {}".format(excl_dict) + ) return excl_dict @property def area_filter_kernel(self): """Get the minimum area filter kernel name ('queen' or 'rook').""" - area_filter_kernel = self._config.get('area_filter_kernel', 'PIPELINE') + area_filter_kernel = self._config.get("area_filter_kernel", "PIPELINE") - if area_filter_kernel == 'PIPELINE': + if area_filter_kernel == "PIPELINE": target_module = ModuleName.SUPPLY_CURVE_AGGREGATION - key = 'area_filter_kernel' - area_filter_kernel = Status.parse_step_status(self._out_root, - target_module, - key=key) + key = "area_filter_kernel" + area_filter_kernel = Status.parse_step_status( + self._out_root, target_module, key=key + ) if not area_filter_kernel: area_filter_kernel = self._default_area_filter_kernel - msg = ('Could not parse area_filter_kernel from previous ' - 'pipeline jobs, defaulting to: {}' - .format(area_filter_kernel)) + msg = ( + "Could not parse area_filter_kernel from previous " + "pipeline jobs, defaulting to: {}".format( + area_filter_kernel + ) + ) logger.warning(msg) warn(msg) else: area_filter_kernel = area_filter_kernel[0] - logger.info('QA/QC using the following ' - 'pipeline input for area_filter_kernel: {}' - .format(area_filter_kernel)) + logger.info( + "QA/QC using the following " + "pipeline input for area_filter_kernel: {}".format( + area_filter_kernel + ) + ) return area_filter_kernel @property def min_area(self): """Get the minimum area filter minimum area in km2.""" - min_area = self._config.get('min_area', 'PIPELINE') + min_area = self._config.get("min_area", "PIPELINE") - if min_area == 'PIPELINE': + if min_area == "PIPELINE": target_module = ModuleName.SUPPLY_CURVE_AGGREGATION - min_area = Status.parse_step_status(self._out_root, target_module, - key='min_area') + min_area = Status.parse_step_status( + self._out_root, target_module, key="min_area" + ) if not min_area: min_area = None - msg = ('Could not parse min_area from previous ' - 'pipeline jobs, defaulting to: {}' - .format(min_area)) + msg = ( + "Could not parse min_area from previous " + "pipeline jobs, defaulting to: {}".format(min_area) + ) logger.warning(msg) warn(msg) else: min_area = min_area[0] - logger.info('QA/QC using the following ' - 'pipeline input for min_area: {}' - .format(min_area)) + logger.info( + "QA/QC using the following " + "pipeline input for min_area: {}".format(min_area) + ) return min_area diff --git a/reV/qa_qc/summary.py b/reV/qa_qc/summary.py index 98d27e357..43bcb529f 100644 --- a/reV/qa_qc/summary.py +++ b/reV/qa_qc/summary.py @@ -2,6 +2,7 @@ """ Compute and plot summary data """ + import logging import os @@ -31,7 +32,7 @@ def __init__(self, h5_file, group=None): group : str, optional Group within h5_file to summarize datasets for, by default None """ - logger.info('QAQC Summarize initializing on: {}'.format(h5_file)) + logger.info("QAQC Summarize initializing on: {}".format(h5_file)) self._h5_file = h5_file self._group = group @@ -76,12 +77,12 @@ def _compute_sites_summary(h5_file, ds_name, sites=None, group=None): sites = slice(None) with Resource(h5_file, group=group) as f: - sites_meta = f['meta', sites] + sites_meta = f["meta", sites] sites_data = f[ds_name, :, sites] sites_summary = pd.DataFrame(sites_data, columns=sites_meta.index) - sites_summary = sites_summary.describe().T.drop(columns=['count']) - sites_summary['sum'] = sites_data.sum(axis=0) + sites_summary = sites_summary.describe().T.drop(columns=["count"]) + sites_summary["sum"] = sites_data.sum(axis=0) return sites_summary @@ -108,13 +109,14 @@ def _compute_ds_summary(h5_file, ds_name, group=None): ds_data = f[ds_name, :] ds_summary = pd.DataFrame(ds_data, columns=[ds_name]) - ds_summary = ds_summary.describe().drop(['count']) - ds_summary.at['sum', ds_name] = ds_data.sum() + ds_summary = ds_summary.describe().drop(["count"]) + ds_summary.at["sum", ds_name] = ds_data.sum() return ds_summary - def summarize_dset(self, ds_name, process_size=None, max_workers=None, - out_path=None): + def summarize_dset( + self, ds_name, process_size=None, max_workers=None, out_path=None + ): """ Compute dataset summary. If dataset is 2D compute temporal statistics for each site @@ -147,45 +149,56 @@ def summarize_dset(self, ds_name, process_size=None, max_workers=None, if process_size is None: process_size = ds_shape[-1] - sites = \ - np.array_split(sites, - int(np.ceil(len(sites) / process_size))) - loggers = [__name__, 'reV'] - with SpawnProcessPool(max_workers=max_workers, - loggers=loggers) as ex: + sites = np.array_split( + sites, int(np.ceil(len(sites) / process_size)) + ) + loggers = [__name__, "reV"] + with SpawnProcessPool( + max_workers=max_workers, loggers=loggers + ) as ex: futures = [] for site_slice in sites: - futures.append(ex.submit( - self._compute_sites_summary, - self.h5_file, ds_name, sites=site_slice, - group=self._group)) + futures.append( + ex.submit( + self._compute_sites_summary, + self.h5_file, + ds_name, + sites=site_slice, + group=self._group, + ) + ) summary = [future.result() for future in futures] summary = pd.concat(summary) elif process_size is None: - summary = self._compute_sites_summary(self.h5_file, - ds_name, - sites=sites, - group=self._group) + summary = self._compute_sites_summary( + self.h5_file, ds_name, sites=sites, group=self._group + ) else: sites = np.array_split( - sites, int(np.ceil(len(sites) / process_size))) + sites, int(np.ceil(len(sites) / process_size)) + ) summary = [] for site_slice in sites: - summary.append(self._compute_sites_summary( - self.h5_file, ds_name, - sites=site_slice, - group=self._group)) + summary.append( + self._compute_sites_summary( + self.h5_file, + ds_name, + sites=site_slice, + group=self._group, + ) + ) summary = pd.concat(summary) summary.index.name = MetaKeyName.GID else: - summary = self._compute_ds_summary(self.h5_file, ds_name, - group=self._group) + summary = self._compute_ds_summary( + self.h5_file, ds_name, group=self._group + ) if out_path is not None: summary.to_csv(out_path) @@ -225,8 +238,15 @@ def summarize_means(self, out_path=None): return meta @classmethod - def run(cls, h5_file, out_dir, group=None, dsets=None, - process_size=None, max_workers=None): + def run( + cls, + h5_file, + out_dir, + group=None, + dsets=None, + process_size=None, + max_workers=None, + ): """ Summarize all datasets in h5_file and dump to out_dir @@ -251,19 +271,25 @@ def run(cls, h5_file, out_dir, group=None, dsets=None, if dsets is None: with Resource(h5_file, group=group) as f: - dsets = [dset for dset in f.datasets - if dset not in ['meta', 'time_index']] + dsets = [ + dset + for dset in f.datasets + if dset not in ["meta", "time_index"] + ] elif isinstance(dsets, str): dsets = [dsets] summary = cls(h5_file) for ds_name in dsets: - out_path = os.path.join(out_dir, - "{}_summary.csv".format(ds_name)) - summary.summarize_dset(ds_name, process_size=process_size, - max_workers=max_workers, out_path=out_path) - - out_path = os.path.basename(h5_file).replace('.h5', '_summary.csv') + out_path = os.path.join(out_dir, "{}_summary.csv".format(ds_name)) + summary.summarize_dset( + ds_name, + process_size=process_size, + max_workers=max_workers, + out_path=out_path, + ) + + out_path = os.path.basename(h5_file).replace(".h5", "_summary.csv") out_path = os.path.join(out_dir, out_path) summary.summarize_means(out_path=out_path) @@ -346,27 +372,27 @@ def supply_curve_summary(self, columns=None, out_path=None): sc_summary = [] sc_stat = sc_table.mean(axis=0) - sc_stat.name = 'mean' + sc_stat.name = "mean" sc_summary.append(sc_stat) sc_stat = sc_table.std(axis=0) - sc_stat.name = 'stdev' + sc_stat.name = "stdev" sc_summary.append(sc_stat) sc_stat = sc_table.median(axis=0) - sc_stat.name = 'median' + sc_stat.name = "median" sc_summary.append(sc_stat) sc_stat = sc_table.min(axis=0) - sc_stat.name = 'min' + sc_stat.name = "min" sc_summary.append(sc_stat) sc_stat = sc_table.max(axis=0) - sc_stat.name = 'max' + sc_stat.name = "max" sc_summary.append(sc_stat) sc_stat = sc_table.sum(axis=0) - sc_stat.name = 'sum' + sc_stat.name = "sum" sc_summary.append(sc_stat) sc_summary = pd.concat(sc_summary, axis=1).T @@ -395,7 +421,7 @@ def run(cls, sc_table, out_dir, columns=None): os.makedirs(out_dir, exist_ok=True) summary = cls(sc_table) - out_path = os.path.basename(sc_table).replace('.csv', '_summary.csv') + out_path = os.path.basename(sc_table).replace(".csv", "_summary.csv") out_path = os.path.join(out_dir, out_path) summary.supply_curve_summary(columns=columns, out_path=out_path) @@ -442,7 +468,7 @@ def _save_plotly(fig, out_path): out_path : str File path to save plot to, can be a .html or static image """ - if out_path.endswith('.html'): + if out_path.endswith(".html"): fig.write_html(out_path) else: fig.write_image(out_path) @@ -469,8 +495,9 @@ def _check_value(df, values, scatter=True): for value in values: if value not in df: - msg = ("{} is not a valid column in summary table:\n{}" - .format(value, df)) + msg = "{} is not a valid column in summary table:\n{}".format( + value, df + ) logger.error(msg) raise ValueError(msg) @@ -511,7 +538,7 @@ def columns(self): """ return list(self.summary.columns) - def scatter_plot(self, value, cmap='viridis', out_path=None, **kwargs): + def scatter_plot(self, value, cmap="viridis", out_path=None, **kwargs): """ Plot scatter plot of value versus longitude and latitude using pandas.plot.scatter @@ -528,10 +555,17 @@ def scatter_plot(self, value, cmap='viridis', out_path=None, **kwargs): Additional kwargs for plotting.dataframes.df_scatter """ self._check_value(self.summary, value) - mplt.df_scatter(self.summary, x=MetaKeyName.LONGITUDE, y=MetaKeyName.LATITUDE, c=value, - colormap=cmap, filename=out_path, **kwargs) - - def scatter_plotly(self, value, cmap='Viridis', out_path=None, **kwargs): + mplt.df_scatter( + self.summary, + x=MetaKeyName.LONGITUDE, + y=MetaKeyName.LATITUDE, + c=value, + colormap=cmap, + filename=out_path, + **kwargs, + ) + + def scatter_plotly(self, value, cmap="Viridis", out_path=None, **kwargs): """ Plot scatter plot of value versus longitude and latitude using plotly @@ -549,8 +583,14 @@ def scatter_plotly(self, value, cmap='Viridis', out_path=None, **kwargs): Additional kwargs for plotly.express.scatter """ self._check_value(self.summary, value) - fig = px.scatter(self.summary, x=MetaKeyName.LONGITUDE, y=MetaKeyName.LATITUDE, - color=value, color_continuous_scale=cmap, **kwargs) + fig = px.scatter( + self.summary, + x=MetaKeyName.LONGITUDE, + y=MetaKeyName.LATITUDE, + color=value, + color_continuous_scale=cmap, + **kwargs, + ) fig.update_layout(font=dict(family="Arial", size=18, color="black")) if out_path is not None: @@ -565,7 +605,8 @@ def _extract_sc_data(self, lcoe=MetaKeyName.MEAN_LCOE): Parameters ---------- lcoe : str, optional - LCOE value to use for supply curve, by default MetaKeyName.MEAN_LCOE + LCOE value to use for supply curve, by default + MetaKeyName.MEAN_LCOE Returns ------- @@ -575,7 +616,7 @@ def _extract_sc_data(self, lcoe=MetaKeyName.MEAN_LCOE): values = [MetaKeyName.CAPACITY, lcoe] self._check_value(self.summary, values, scatter=False) sc_df = self.summary[values].sort_values(lcoe) - sc_df['cumulative_capacity'] = sc_df[MetaKeyName.CAPACITY].cumsum() + sc_df["cumulative_capacity"] = sc_df[MetaKeyName.CAPACITY].cumsum() return sc_df @@ -619,8 +660,15 @@ def dist_plotly(self, value, out_path=None, **kwargs): fig.show() @classmethod - def scatter(cls, summary_csv, out_dir, value, plot_type='plotly', - cmap='viridis', **kwargs): + def scatter( + cls, + summary_csv, + out_dir, + value, + plot_type="plotly", + cmap="viridis", + **kwargs, + ): """ Create scatter plot for given value in summary table and save to out_dir @@ -641,25 +689,31 @@ def scatter(cls, summary_csv, out_dir, value, plot_type='plotly', Additional plotting kwargs """ splt = cls(summary_csv) - if plot_type == 'plot': - out_path = os.path.basename(summary_csv).replace('.csv', '.png') + if plot_type == "plot": + out_path = os.path.basename(summary_csv).replace(".csv", ".png") out_path = os.path.join(out_dir, out_path) - splt.scatter_plot(value, cmap=cmap.lower(), out_path=out_path, - **kwargs) - elif plot_type == 'plotly': - out_path = os.path.basename(summary_csv).replace('.csv', '.html') + splt.scatter_plot( + value, cmap=cmap.lower(), out_path=out_path, **kwargs + ) + elif plot_type == "plotly": + out_path = os.path.basename(summary_csv).replace(".csv", ".html") out_path = os.path.join(out_dir, out_path) - splt.scatter_plotly(value, cmap=cmap.capitalize(), - out_path=out_path, **kwargs) + splt.scatter_plotly( + value, cmap=cmap.capitalize(), out_path=out_path, **kwargs + ) else: - msg = ("plot_type must be 'plot' or 'plotly' but {} was given" - .format(plot_type)) + msg = ( + "plot_type must be 'plot' or 'plotly' but {} was given".format( + plot_type + ) + ) logger.error(msg) raise ValueError(msg) @classmethod - def scatter_all(cls, summary_csv, out_dir, plot_type='plotly', - cmap='viridis', **kwargs): + def scatter_all( + cls, summary_csv, out_dir, plot_type="plotly", cmap="viridis", **kwargs + ): """ Create scatter plot for all summary stats in summary table and save to out_dir @@ -679,24 +733,29 @@ def scatter_all(cls, summary_csv, out_dir, plot_type='plotly', """ splt = cls(summary_csv) splt._data = splt.summary.select_dtypes(include=np.number) - datasets = [c for c in splt.summary.columns - if not c.startswith(('lat', 'lon'))] + datasets = [ + c for c in splt.summary.columns if not c.startswith(("lat", "lon")) + ] for value in datasets: - if plot_type == 'plot': - out_path = '_{}.png'.format(value) - out_path = \ - os.path.basename(summary_csv).replace('.csv', out_path) + if plot_type == "plot": + out_path = "_{}.png".format(value) + out_path = os.path.basename(summary_csv).replace( + ".csv", out_path + ) out_path = os.path.join(out_dir, out_path) - splt.scatter_plot(value, cmap=cmap.lower(), out_path=out_path, - **kwargs) - elif plot_type == 'plotly': - out_path = '_{}.html'.format(value) - out_path = \ - os.path.basename(summary_csv).replace('.csv', out_path) + splt.scatter_plot( + value, cmap=cmap.lower(), out_path=out_path, **kwargs + ) + elif plot_type == "plotly": + out_path = "_{}.html".format(value) + out_path = os.path.basename(summary_csv).replace( + ".csv", out_path + ) out_path = os.path.join(out_dir, out_path) - splt.scatter_plotly(value, cmap=cmap.capitalize(), - out_path=out_path, **kwargs) + splt.scatter_plotly( + value, cmap=cmap.capitalize(), out_path=out_path, **kwargs + ) else: msg = ("plot_type must be 'plot' or 'plotly' but {} was given" .format(plot_type)) @@ -747,7 +806,8 @@ def _extract_sc_data(self, lcoe=MetaKeyName.MEAN_LCOE): Parameters ---------- lcoe : str, optional - LCOE value to use for supply curve, by default MetaKeyName.MEAN_LCOE + LCOE value to use for supply curve, by default + `MetaKeyName.MEAN_LCOE` Returns ------- @@ -757,11 +817,13 @@ def _extract_sc_data(self, lcoe=MetaKeyName.MEAN_LCOE): values = [MetaKeyName.CAPACITY, lcoe] self._check_value(self.sc_table, values, scatter=False) sc_df = self.sc_table[values].sort_values(lcoe) - sc_df['cumulative_capacity'] = sc_df[MetaKeyName.CAPACITY].cumsum() + sc_df["cumulative_capacity"] = sc_df[MetaKeyName.CAPACITY].cumsum() return sc_df - def supply_curve_plot(self, lcoe=MetaKeyName.MEAN_LCOE, out_path=None, **kwargs): + def supply_curve_plot( + self, lcoe=MetaKeyName.MEAN_LCOE, out_path=None, **kwargs + ): """ Plot supply curve (cumulative capacity vs lcoe) using seaborn.scatter @@ -775,10 +837,13 @@ def supply_curve_plot(self, lcoe=MetaKeyName.MEAN_LCOE, out_path=None, **kwargs) Additional kwargs for plotting.dataframes.df_scatter """ sc_df = self._extract_sc_data(lcoe=lcoe) - mplt.df_scatter(sc_df, x='cumulative_capacity', y=lcoe, - filename=out_path, **kwargs) + mplt.df_scatter( + sc_df, x="cumulative_capacity", y=lcoe, filename=out_path, **kwargs + ) - def supply_curve_plotly(self, lcoe=MetaKeyName.MEAN_LCOE, out_path=None, **kwargs): + def supply_curve_plotly( + self, lcoe=MetaKeyName.MEAN_LCOE, out_path=None, **kwargs + ): """ Plot supply curve (cumulative capacity vs lcoe) using plotly @@ -793,7 +858,7 @@ def supply_curve_plotly(self, lcoe=MetaKeyName.MEAN_LCOE, out_path=None, **kwarg Additional kwargs for plotly.express.scatter """ sc_df = self._extract_sc_data(lcoe=lcoe) - fig = px.scatter(sc_df, x='cumulative_capacity', y=lcoe, **kwargs) + fig = px.scatter(sc_df, x="cumulative_capacity", y=lcoe, **kwargs) fig.update_layout(font=dict(family="Arial", size=18, color="black")) if out_path is not None: @@ -802,8 +867,14 @@ def supply_curve_plotly(self, lcoe=MetaKeyName.MEAN_LCOE, out_path=None, **kwarg fig.show() @classmethod - def plot(cls, sc_table, out_dir, plot_type='plotly', lcoe=MetaKeyName.MEAN_LCOE, - **kwargs): + def plot( + cls, + sc_table, + out_dir, + plot_type="plotly", + lcoe=MetaKeyName.MEAN_LCOE, + **kwargs, + ): """ Create supply curve plot from supply curve table using lcoe value and save to out_dir @@ -822,17 +893,20 @@ def plot(cls, sc_table, out_dir, plot_type='plotly', lcoe=MetaKeyName.MEAN_LCOE, Additional plotting kwargs """ splt = cls(sc_table) - if plot_type == 'plot': - out_path = os.path.basename(sc_table).replace('.csv', '.png') + if plot_type == "plot": + out_path = os.path.basename(sc_table).replace(".csv", ".png") out_path = os.path.join(out_dir, out_path) splt.supply_curve_plot(lcoe=lcoe, out_path=out_path, **kwargs) - elif plot_type == 'plotly': - out_path = os.path.basename(sc_table).replace('.csv', '.html') + elif plot_type == "plotly": + out_path = os.path.basename(sc_table).replace(".csv", ".html") out_path = os.path.join(out_dir, out_path) splt.supply_curve_plotly(lcoe=lcoe, out_path=out_path, **kwargs) else: - msg = ("plot_type must be 'plot' or 'plotly' but {} was given" - .format(plot_type)) + msg = ( + "plot_type must be 'plot' or 'plotly' but {} was given".format( + plot_type + ) + ) logger.error(msg) raise ValueError(msg) @@ -884,8 +958,9 @@ def _parse_mask(excl_mask): return excl_mask - def exclusions_plot(self, cmap='Viridis', plot_step=100, out_path=None, - **kwargs): + def exclusions_plot( + self, cmap="Viridis", plot_step=100, out_path=None, **kwargs + ): """ Plot exclusions mask as a seaborn heatmap @@ -901,11 +976,16 @@ def exclusions_plot(self, cmap='Viridis', plot_step=100, out_path=None, kwargs : dict Additional kwargs for plotting.colormaps.heatmap_plot """ - mplt.heatmap_plot(self.mask[::plot_step, ::plot_step], cmap=cmap, - filename=out_path, **kwargs) + mplt.heatmap_plot( + self.mask[::plot_step, ::plot_step], + cmap=cmap, + filename=out_path, + **kwargs, + ) - def exclusions_plotly(self, cmap='Viridis', plot_step=100, out_path=None, - **kwargs): + def exclusions_plotly( + self, cmap="Viridis", plot_step=100, out_path=None, **kwargs + ): """ Plot exclusions mask as a plotly heatmap @@ -921,8 +1001,11 @@ def exclusions_plotly(self, cmap='Viridis', plot_step=100, out_path=None, kwargs : dict Additional kwargs for plotly.express.imshow """ - fig = px.imshow(self.mask[::plot_step, ::plot_step], - color_continuous_scale=cmap, **kwargs) + fig = px.imshow( + self.mask[::plot_step, ::plot_step], + color_continuous_scale=cmap, + **kwargs, + ) fig.update_layout(font=dict(family="Arial", size=18, color="black")) if out_path is not None: @@ -931,8 +1014,15 @@ def exclusions_plotly(self, cmap='Viridis', plot_step=100, out_path=None, fig.show() @classmethod - def plot(cls, mask, out_dir, plot_type='plotly', cmap='Viridis', - plot_step=100, **kwargs): + def plot( + cls, + mask, + out_dir, + plot_type="plotly", + cmap="Viridis", + plot_step=100, + **kwargs, + ): """ Plot exclusions mask and save to out_dir @@ -952,22 +1042,29 @@ def plot(cls, mask, out_dir, plot_type='plotly', cmap='Viridis', Additional plotting kwargs """ excl_mask = cls(mask) - if plot_type == 'plot': - out_path = 'exclusions_mask.png' + if plot_type == "plot": + out_path = "exclusions_mask.png" out_path = os.path.join(out_dir, out_path) - excl_mask.exclusions_plot(cmap=cmap.lower(), - plot_step=plot_step, - out_path=out_path, - **kwargs) - elif plot_type == 'plotly': - out_path = 'exclusions_mask.html' + excl_mask.exclusions_plot( + cmap=cmap.lower(), + plot_step=plot_step, + out_path=out_path, + **kwargs, + ) + elif plot_type == "plotly": + out_path = "exclusions_mask.html" out_path = os.path.join(out_dir, out_path) - excl_mask.exclusions_plotly(cmap=cmap.capitalize(), - plot_step=plot_step, - out_path=out_path, - **kwargs) + excl_mask.exclusions_plotly( + cmap=cmap.capitalize(), + plot_step=plot_step, + out_path=out_path, + **kwargs, + ) else: - msg = ("plot_type must be 'plot' or 'plotly' but {} was given" - .format(plot_type)) + msg = ( + "plot_type must be 'plot' or 'plotly' but {} was given".format( + plot_type + ) + ) logger.error(msg) raise ValueError(msg) diff --git a/reV/rep_profiles/rep_profiles.py b/reV/rep_profiles/rep_profiles.py index 38db15411..ca0d646db 100644 --- a/reV/rep_profiles/rep_profiles.py +++ b/reV/rep_profiles/rep_profiles.py @@ -5,6 +5,7 @@ @author: gbuster """ + import json import logging import os @@ -31,8 +32,9 @@ class RepresentativeMethods: """Class for organizing the methods to determine representative-ness""" - def __init__(self, profiles, weights=None, rep_method='meanoid', - err_method='rmse'): + def __init__( + self, profiles, weights=None, rep_method="meanoid", err_method="rmse" + ): """ Parameters ---------- @@ -60,30 +62,35 @@ def _parse_weights(self): self._weights = np.array(self._weights) if self._weights is not None: - emsg = ('Weighting factors array of length {} does not match ' - 'profiles of shape {}' - .format(len(self._weights), self._profiles.shape[1])) + emsg = ( + "Weighting factors array of length {} does not match " + "profiles of shape {}".format( + len(self._weights), self._profiles.shape[1] + ) + ) assert len(self._weights) == self._profiles.shape[1], emsg @property def rep_methods(self): """Lookup table of representative methods""" - methods = {'mean': self.meanoid, - 'meanoid': self.meanoid, - 'median': self.medianoid, - 'medianoid': self.medianoid, - } + methods = { + "mean": self.meanoid, + "meanoid": self.meanoid, + "median": self.medianoid, + "medianoid": self.medianoid, + } return methods @property def err_methods(self): """Lookup table of error methods""" - methods = {'mbe': self.mbe, - 'mae': self.mae, - 'rmse': self.rmse, - None: None, - } + methods = { + "mbe": self.mbe, + "mae": self.mae, + "rmse": self.rmse, + None: None, + } return methods @@ -104,7 +111,7 @@ def nargmin(arr, n): i : int Location of the Nth min value in arr. """ - return arr.argsort()[:(n + 1)][-1] + return arr.argsort()[: (n + 1)][-1] @staticmethod def meanoid(profiles, weights=None): @@ -240,8 +247,14 @@ def rmse(cls, profiles, baseline, i_profile=0): return profiles[:, i_rep], i_rep @classmethod - def run(cls, profiles, weights=None, rep_method='meanoid', - err_method='rmse', n_profiles=1): + def run( + cls, + profiles, + weights=None, + rep_method="meanoid", + err_method="rmse", + n_profiles=1, + ): """Run representative profile methods. Parameters @@ -269,8 +282,12 @@ def run(cls, profiles, weights=None, rep_method='meanoid', representative profile(s). If err_method is None, this value is also set to None. """ - inst = cls(profiles, weights=weights, rep_method=rep_method, - err_method=err_method) + inst = cls( + profiles, + weights=weights, + rep_method=rep_method, + err_method=err_method, + ) if inst._weights is not None: baseline = inst._rep_method(inst._profiles, weights=inst._weights) @@ -301,9 +318,16 @@ class RegionRepProfile: RES_GID_COL = MetaKeyName.RES_GIDS GEN_GID_COL = MetaKeyName.GEN_GIDS - def __init__(self, gen_fpath, rev_summary, cf_dset='cf_profile', - rep_method='meanoid', err_method='rmse', weight=MetaKeyName.GID_COUNTS, - n_profiles=1): + def __init__( + self, + gen_fpath, + rev_summary, + cf_dset="cf_profile", + rep_method="meanoid", + err_method="rmse", + weight=MetaKeyName.GID_COUNTS, + n_profiles=1, + ): """ Parameters ---------- @@ -354,17 +378,22 @@ def _init_profiles_weights(self): self._weights = np.ones(len(res_gids)) if self._weight is not None: - self._weights = self._get_region_attr(self._rev_summary, - self._weight) + self._weights = self._get_region_attr( + self._rev_summary, self._weight + ) - df = pd.DataFrame({self.GEN_GID_COL: gen_gids, - self.RES_GID_COL: res_gids, - 'weights': self._weights}) + df = pd.DataFrame( + { + self.GEN_GID_COL: gen_gids, + self.RES_GID_COL: res_gids, + "weights": self._weights, + } + ) df = df.sort_values(self.RES_GID_COL) self._gen_gids = df[self.GEN_GID_COL].values self._res_gids = df[self.RES_GID_COL].values if self._weight is not None: - self._weights = df['weights'].values + self._weights = df["weights"].values else: self._weights = None @@ -373,19 +402,24 @@ def _init_profiles_weights(self): assert MetaKeyName.GID in meta source_res_gids = meta[MetaKeyName.GID].values - msg = ('Resource gids from "gid" column in meta data from "{}" ' - 'must be sorted! reV generation should always be run with ' - 'sequential project points.'.format(self._gen_fpath)) + msg = ( + 'Resource gids from "gid" column in meta data from "{}" ' + "must be sorted! reV generation should always be run with " + "sequential project points.".format(self._gen_fpath) + ) assert np.all(source_res_gids[:-1] <= source_res_gids[1:]), msg missing = set(self._res_gids) - set(source_res_gids) - msg = ('The following resource gids were found in the rev summary ' - 'supply curve file but not in the source generation meta ' - 'data: {}'.format(missing)) + msg = ( + "The following resource gids were found in the rev summary " + "supply curve file but not in the source generation meta " + "data: {}".format(missing) + ) assert not any(missing), msg - unique_res_gids, u_idxs = np.unique(self._res_gids, - return_inverse=True) + unique_res_gids, u_idxs = np.unique( + self._res_gids, return_inverse=True + ) iloc = np.where(np.isin(source_res_gids, unique_res_gids))[0] self._source_profiles = res[self._cf_dset, :, iloc[u_idxs]] @@ -440,7 +474,7 @@ def _get_region_attr(rev_summary, attr_name): if any(data): if isinstance(data[0], str): # pylint: disable=simplifiable-condition - if ('[' and ']' in data[0]) or ('(' and ')' in data[0]): + if ("[" and "]" in data[0]) or ("(" and ")" in data[0]): data = [json.loads(s) for s in data] if isinstance(data[0], (list, tuple)): @@ -454,23 +488,35 @@ def _run_rep_methods(self): if self.weights is not None: if len(self.weights) != self.source_profiles.shape[1]: - e = ('Weights column "{}" resulted in {} weight scalars ' - 'which doesnt match gid column which yields ' - 'profiles with shape {}.' - .format(self._weight, len(self.weights), - self.source_profiles.shape)) - logger.debug('Gids from column "res_gids" with len {}: {}' - .format(len(self._res_gids), self._res_gids)) - logger.debug('Weights from column "{}" with len {}: {}' - .format(self._weight, len(self.weights), - self.weights)) + e = ( + 'Weights column "{}" resulted in {} weight scalars ' + "which doesnt match gid column which yields " + "profiles with shape {}.".format( + self._weight, + len(self.weights), + self.source_profiles.shape, + ) + ) + logger.debug( + 'Gids from column "res_gids" with len {}: {}'.format( + len(self._res_gids), self._res_gids + ) + ) + logger.debug( + 'Weights from column "{}" with len {}: {}'.format( + self._weight, len(self.weights), self.weights + ) + ) logger.error(e) raise DataShapeError(e) self._profiles, self._i_reps = RepresentativeMethods.run( - self.source_profiles, weights=self.weights, - rep_method=self._rep_method, err_method=self._err_method, - n_profiles=self._n_profiles) + self.source_profiles, + weights=self.weights, + rep_method=self._rep_method, + err_method=self._err_method, + n_profiles=self._n_profiles, + ) @property def rep_profiles(self): @@ -511,10 +557,16 @@ def rep_res_gids(self): return rep_gids @classmethod - def get_region_rep_profile(cls, gen_fpath, rev_summary, - cf_dset='cf_profile', rep_method='meanoid', - err_method='rmse', weight=MetaKeyName.GID_COUNTS, - n_profiles=1): + def get_region_rep_profile( + cls, + gen_fpath, + rev_summary, + cf_dset="cf_profile", + rep_method="meanoid", + err_method="rmse", + weight=MetaKeyName.GID_COUNTS, + n_profiles=1, + ): """Class method for parallelization of rep profile calc. Parameters @@ -553,9 +605,15 @@ def get_region_rep_profile(cls, gen_fpath, rev_summary, res_gid_reps : list Resource gid(s) of the representative profile(s). """ - r = cls(gen_fpath, rev_summary, cf_dset=cf_dset, - rep_method=rep_method, err_method=err_method, weight=weight, - n_profiles=n_profiles) + r = cls( + gen_fpath, + rev_summary, + cf_dset=cf_dset, + rep_method=rep_method, + err_method=err_method, + weight=weight, + n_profiles=n_profiles, + ) return r.rep_profiles, r.i_reps, r.rep_gen_gids, r.rep_res_gids @@ -563,9 +621,17 @@ def get_region_rep_profile(cls, gen_fpath, rev_summary, class RepProfilesBase(ABC): """Abstract utility framework for representative profile run classes.""" - def __init__(self, gen_fpath, rev_summary, reg_cols=None, - cf_dset='cf_profile', rep_method='meanoid', err_method='rmse', - weight=MetaKeyName.GID_COUNTS, n_profiles=1): + def __init__( + self, + gen_fpath, + rev_summary, + reg_cols=None, + cf_dset="cf_profile", + rep_method="meanoid", + err_method="rmse", + weight=MetaKeyName.GID_COUNTS, + n_profiles=1, + ): """ Parameters ---------- @@ -596,18 +662,26 @@ def __init__(self, gen_fpath, rev_summary, reg_cols=None, Number of representative profiles to save to fout. """ - logger.info('Running rep profiles with gen_fpath: "{}"' - .format(gen_fpath)) - logger.info('Running rep profiles with rev_summary: "{}"' - .format(rev_summary)) - logger.info('Running rep profiles with region columns: "{}"' - .format(reg_cols)) - logger.info('Running rep profiles with representative method: "{}"' - .format(rep_method)) - logger.info('Running rep profiles with error method: "{}"' - .format(err_method)) - logger.info('Running rep profiles with weight factor: "{}"' - .format(weight)) + logger.info( + 'Running rep profiles with gen_fpath: "{}"'.format(gen_fpath) + ) + logger.info( + 'Running rep profiles with rev_summary: "{}"'.format(rev_summary) + ) + logger.info( + 'Running rep profiles with region columns: "{}"'.format(reg_cols) + ) + logger.info( + 'Running rep profiles with representative method: "{}"'.format( + rep_method + ) + ) + logger.info( + 'Running rep profiles with error method: "{}"'.format(err_method) + ) + logger.info( + 'Running rep profiles with weight factor: "{}"'.format(weight) + ) self._weight = weight self._n_profiles = n_profiles @@ -649,17 +723,18 @@ def _parse_rev_summary(rev_summary): """ if isinstance(rev_summary, str): - if os.path.exists(rev_summary) and rev_summary.endswith('.csv'): + if os.path.exists(rev_summary) and rev_summary.endswith(".csv"): rev_summary = pd.read_csv(rev_summary) - elif os.path.exists(rev_summary) and rev_summary.endswith('.json'): + elif os.path.exists(rev_summary) and rev_summary.endswith(".json"): rev_summary = pd.read_json(rev_summary) else: - e = 'Could not parse reV summary file: {}'.format(rev_summary) + e = "Could not parse reV summary file: {}".format(rev_summary) logger.error(e) raise FileInputError(e) elif not isinstance(rev_summary, pd.DataFrame): - e = ('Bad input dtype for rev_summary input: {}' - .format(type(rev_summary))) + e = "Bad input dtype for rev_summary input: {}".format( + type(rev_summary) + ) logger.error(e) raise TypeError(e) @@ -686,8 +761,9 @@ def _check_req_cols(df, cols): missing.append(c) if any(missing): - e = ('Column labels not found in rev_summary table: {}' - .format(missing)) + e = "Column labels not found in rev_summary table: {}".format( + missing + ) logger.error(e) raise KeyError(e) @@ -709,30 +785,40 @@ def _check_rev_gen(gen_fpath, cf_dset, rev_summary): with Resource(gen_fpath) as res: dsets = res.datasets if cf_dset not in dsets: - raise KeyError('reV gen file needs to have "{}" ' - 'dataset to calculate representative profiles!' - .format(cf_dset)) - - if 'time_index' not in str(dsets): - raise KeyError('reV gen file needs to have "time_index" ' - 'dataset to calculate representative profiles!') + raise KeyError( + 'reV gen file needs to have "{}" ' + "dataset to calculate representative profiles!".format( + cf_dset + ) + ) + + if "time_index" not in str(dsets): + raise KeyError( + 'reV gen file needs to have "time_index" ' + "dataset to calculate representative profiles!" + ) shape = res.get_dset_properties(cf_dset)[0] if len(rev_summary) > shape[1]: - msg = ('WARNING: reV SC summary table has {} sc points and CF ' - 'dataset "{}" has {} profiles. There should never be more ' - 'SC points than CF profiles.' - .format(len(rev_summary), cf_dset, shape[1])) + msg = ( + "WARNING: reV SC summary table has {} sc points and CF " + 'dataset "{}" has {} profiles. There should never be more ' + "SC points than CF profiles.".format( + len(rev_summary), cf_dset, shape[1] + ) + ) logger.warning(msg) warn(msg) def _init_profiles(self): """Initialize the output rep profiles attribute.""" - self._profiles = {k: np.zeros((len(self.time_index), - len(self.meta)), - dtype=np.float32) - for k in range(self._n_profiles)} + self._profiles = { + k: np.zeros( + (len(self.time_index), len(self.meta)), dtype=np.float32 + ) + for k in range(self._n_profiles) + } @property def time_index(self): @@ -745,10 +831,10 @@ def time_index(self): """ if self._time_index is None: with Resource(self._gen_fpath) as res: - ds = 'time_index' - if parse_year(self._cf_dset, option='bool'): - year = parse_year(self._cf_dset, option='raise') - ds += '-{}'.format(year) + ds = "time_index" + if parse_year(self._cf_dset, option="bool"): + year = parse_year(self._cf_dset, option="raise") + ds += "-{}".format(year) self._time_index = res._get_time_index(ds, slice(None)) @@ -778,8 +864,9 @@ def profiles(self): """ return self._profiles - def _init_h5_out(self, fout, save_rev_summary=True, - scaled_precision=False): + def _init_h5_out( + self, fout, save_rev_summary=True, scaled_precision=False + ): """Initialize an output h5 file for n_profiles Parameters @@ -798,13 +885,13 @@ def _init_h5_out(self, fout, save_rev_summary=True, dtypes = {} for i in range(self._n_profiles): - dset = 'rep_profiles_{}'.format(i) + dset = "rep_profiles_{}".format(i) dsets.append(dset) shapes[dset] = self.profiles[0].shape chunks[dset] = None if scaled_precision: - attrs[dset] = {'scale_factor': 1000} + attrs[dset] = {"scale_factor": 1000} dtypes[dset] = np.uint16 else: attrs[dset] = None @@ -817,14 +904,23 @@ def _init_h5_out(self, fout, save_rev_summary=True, except ValueError: pass - Outputs.init_h5(fout, dsets, shapes, attrs, chunks, dtypes, - meta, time_index=self.time_index) + Outputs.init_h5( + fout, + dsets, + shapes, + attrs, + chunks, + dtypes, + meta, + time_index=self.time_index, + ) if save_rev_summary: - with Outputs(fout, mode='a') as out: + with Outputs(fout, mode="a") as out: rev_sum = to_records_array(self._rev_summary) - out._create_dset('rev_summary', rev_sum.shape, - rev_sum.dtype, data=rev_sum) + out._create_dset( + "rev_summary", rev_sum.shape, rev_sum.dtype, data=rev_sum + ) def _write_h5_out(self, fout, save_rev_summary=True): """Write profiles and meta to an output file. @@ -838,18 +934,18 @@ def _write_h5_out(self, fout, save_rev_summary=True): scaled_precision : bool Flag to scale cf_profiles by 1000 and save as uint16. """ - with Outputs(fout, mode='a') as out: - - if 'rev_summary' in out.datasets and save_rev_summary: + with Outputs(fout, mode="a") as out: + if "rev_summary" in out.datasets and save_rev_summary: rev_sum = to_records_array(self._rev_summary) - out['rev_summary'] = rev_sum + out["rev_summary"] = rev_sum for i in range(self._n_profiles): - dset = 'rep_profiles_{}'.format(i) + dset = "rep_profiles_{}".format(i) out[dset] = self.profiles[i] - def save_profiles(self, fout, save_rev_summary=True, - scaled_precision=False): + def save_profiles( + self, fout, save_rev_summary=True, scaled_precision=False + ): """Initialize fout and save profiles. Parameters @@ -862,8 +958,11 @@ def save_profiles(self, fout, save_rev_summary=True, Flag to scale cf_profiles by 1000 and save as uint16. """ - self._init_h5_out(fout, save_rev_summary=save_rev_summary, - scaled_precision=scaled_precision) + self._init_h5_out( + fout, + save_rev_summary=save_rev_summary, + scaled_precision=scaled_precision, + ) self._write_h5_out(fout, save_rev_summary=save_rev_summary) @abstractmethod @@ -882,9 +981,18 @@ def run(self): class RepProfiles(RepProfilesBase): """RepProfiles""" - def __init__(self, gen_fpath, rev_summary, reg_cols, cf_dset='cf_profile', - rep_method='meanoid', err_method='rmse', weight=MetaKeyName.GID_COUNTS, - n_profiles=1, aggregate_profiles=False): + def __init__( + self, + gen_fpath, + rev_summary, + reg_cols, + cf_dset="cf_profile", + rep_method="meanoid", + err_method="rmse", + weight=MetaKeyName.GID_COUNTS, + n_profiles=1, + aggregate_profiles=False, + ): """ReV rep profiles class. ``reV`` rep profiles compute representative generation profiles @@ -990,13 +1098,17 @@ def __init__(self, gen_fpath, rev_summary, reg_cols, cf_dset='cf_profile', """ log_versions(logger) - logger.info('Finding representative profiles that are most similar ' - 'to the weighted meanoid for each supply curve region.') + logger.info( + "Finding representative profiles that are most similar " + "to the weighted meanoid for each supply curve region." + ) if reg_cols is None: - e = ('Need to define "reg_cols"! If you want a profile for each ' - 'supply curve point, try setting "reg_cols" to a primary ' - 'key such as "sc_gid".') + e = ( + 'Need to define "reg_cols"! If you want a profile for each ' + 'supply curve point, try setting "reg_cols" to a primary ' + 'key such as "sc_gid".' + ) logger.error(e) raise ValueError(e) if isinstance(reg_cols, str): @@ -1006,17 +1118,25 @@ def __init__(self, gen_fpath, rev_summary, reg_cols, cf_dset='cf_profile', self._aggregate_profiles = aggregate_profiles if self._aggregate_profiles: - logger.info("Aggregate profiles input set to `True`. Setting " - "'rep_method' to `'meanoid'`, 'err_method' to `None`, " - "and 'n_profiles' to `1`") - rep_method = 'meanoid' + logger.info( + "Aggregate profiles input set to `True`. Setting " + "'rep_method' to `'meanoid'`, 'err_method' to `None`, " + "and 'n_profiles' to `1`" + ) + rep_method = "meanoid" err_method = None n_profiles = 1 - super().__init__(gen_fpath, rev_summary, reg_cols=reg_cols, - cf_dset=cf_dset, - rep_method=rep_method, err_method=err_method, - weight=weight, n_profiles=n_profiles) + super().__init__( + gen_fpath, + rev_summary, + reg_cols=reg_cols, + cf_dset=cf_dset, + rep_method=rep_method, + err_method=err_method, + weight=weight, + n_profiles=n_profiles, + ) self._set_meta() self._init_profiles() @@ -1028,14 +1148,13 @@ def _set_meta(self): self._meta = self._rev_summary else: self._meta = self._rev_summary.groupby(self._reg_cols) - self._meta = ( - self._meta[MetaKeyName.TIMEZONE] - .apply(lambda x: stats.mode(x, keepdims=True).mode[0]) + self._meta = self._meta[MetaKeyName.TIMEZONE].apply( + lambda x: stats.mode(x, keepdims=True).mode[0] ) self._meta = self._meta.reset_index() - self._meta['rep_gen_gid'] = None - self._meta['rep_res_gid'] = None + self._meta["rep_gen_gid"] = None + self._meta["rep_res_gid"] = None def _get_mask(self, region_dict): """Get the mask for a given region and res class. @@ -1053,54 +1172,70 @@ def _get_mask(self, region_dict): """ mask = None for k, v in region_dict.items(): - temp = (self._rev_summary[k] == v) + temp = self._rev_summary[k] == v if mask is None: mask = temp else: - mask = (mask & temp) + mask = mask & temp return mask def _run_serial(self): """Compute all representative profiles in serial.""" - logger.info('Running {} rep profile calculations in serial.' - .format(len(self.meta))) + logger.info( + "Running {} rep profile calculations in serial.".format( + len(self.meta) + ) + ) meta_static = deepcopy(self.meta) for i, row in meta_static.iterrows(): - region_dict = {k: v for (k, v) in row.to_dict().items() - if k in self._reg_cols} + region_dict = { + k: v for (k, v) in row.to_dict().items() if k in self._reg_cols + } mask = self._get_mask(region_dict) if not any(mask): - logger.warning('Skipping profile {} out of {} ' - 'for region: {} with no valid mask.' - .format(i + 1, len(meta_static), region_dict)) + logger.warning( + "Skipping profile {} out of {} " + "for region: {} with no valid mask.".format( + i + 1, len(meta_static), region_dict + ) + ) else: - logger.debug('Working on profile {} out of {} for region: {}' - .format(i + 1, len(meta_static), region_dict)) + logger.debug( + "Working on profile {} out of {} for region: {}".format( + i + 1, len(meta_static), region_dict + ) + ) out = RegionRepProfile.get_region_rep_profile( - self._gen_fpath, self._rev_summary[mask], - cf_dset=self._cf_dset, rep_method=self._rep_method, - err_method=self._err_method, weight=self._weight, - n_profiles=self._n_profiles) + self._gen_fpath, + self._rev_summary[mask], + cf_dset=self._cf_dset, + rep_method=self._rep_method, + err_method=self._err_method, + weight=self._weight, + n_profiles=self._n_profiles, + ) profiles, _, ggids, rgids = out - logger.info('Profile {} out of {} complete ' - 'for region: {}' - .format(i + 1, len(meta_static), region_dict)) + logger.info( + "Profile {} out of {} complete " "for region: {}".format( + i + 1, len(meta_static), region_dict + ) + ) for n in range(profiles.shape[1]): self._profiles[n][:, i] = profiles[:, n] if ggids is None: - self._meta.at[i, 'rep_gen_gid'] = None - self._meta.at[i, 'rep_res_gid'] = None + self._meta.at[i, "rep_gen_gid"] = None + self._meta.at[i, "rep_res_gid"] = None elif len(ggids) == 1: - self._meta.at[i, 'rep_gen_gid'] = ggids[0] - self._meta.at[i, 'rep_res_gid'] = rgids[0] + self._meta.at[i, "rep_gen_gid"] = ggids[0] + self._meta.at[i, "rep_res_gid"] = rgids[0] else: - self._meta.at[i, 'rep_gen_gid'] = str(ggids) - self._meta.at[i, 'rep_res_gid'] = str(rgids) + self._meta.at[i, "rep_gen_gid"] = str(ggids) + self._meta.at[i, "rep_res_gid"] = str(rgids) def _run_parallel(self, max_workers=None, pool_size=72): """Compute all representative profiles in parallel. @@ -1115,39 +1250,49 @@ def _run_parallel(self, max_workers=None, pool_size=72): parallel futures. """ - logger.info('Kicking off {} rep profile futures.' - .format(len(self.meta))) + logger.info( + "Kicking off {} rep profile futures.".format(len(self.meta)) + ) - iter_chunks = np.array_split(self.meta.index.values, - np.ceil(len(self.meta) / pool_size)) + iter_chunks = np.array_split( + self.meta.index.values, np.ceil(len(self.meta) / pool_size) + ) n_complete = 0 for iter_chunk in iter_chunks: - logger.debug('Starting process pool...') + logger.debug("Starting process pool...") futures = {} - loggers = [__name__, 'reV'] - with SpawnProcessPool(max_workers=max_workers, - loggers=loggers) as exe: + loggers = [__name__, "reV"] + with SpawnProcessPool( + max_workers=max_workers, loggers=loggers + ) as exe: for i in iter_chunk: row = self.meta.loc[i, :] - region_dict = {k: v for (k, v) in row.to_dict().items() - if k in self._reg_cols} + region_dict = { + k: v + for (k, v) in row.to_dict().items() + if k in self._reg_cols + } mask = self._get_mask(region_dict) if not any(mask): - logger.info('Skipping profile {} out of {} ' - 'for region: {} with no valid mask.' - .format(i + 1, len(self.meta), - region_dict)) + logger.info( + "Skipping profile {} out of {} " + "for region: {} with no valid mask.".format( + i + 1, len(self.meta), region_dict + ) + ) else: future = exe.submit( RegionRepProfile.get_region_rep_profile, - self._gen_fpath, self._rev_summary[mask], + self._gen_fpath, + self._rev_summary[mask], cf_dset=self._cf_dset, rep_method=self._rep_method, err_method=self._err_method, weight=self._weight, - n_profiles=self._n_profiles) + n_profiles=self._n_profiles, + ) futures[future] = [i, region_dict] @@ -1155,27 +1300,34 @@ def _run_parallel(self, max_workers=None, pool_size=72): i, region_dict = futures[future] profiles, _, ggids, rgids = future.result() n_complete += 1 - logger.info('Future {} out of {} complete ' - 'for region: {}' - .format(n_complete, len(self.meta), - region_dict)) - log_mem(logger, log_level='DEBUG') + logger.info( + "Future {} out of {} complete " + "for region: {}".format( + n_complete, len(self.meta), region_dict + ) + ) + log_mem(logger, log_level="DEBUG") for n in range(profiles.shape[1]): self._profiles[n][:, i] = profiles[:, n] if ggids is None: - self._meta.at[i, 'rep_gen_gid'] = None - self._meta.at[i, 'rep_res_gid'] = None + self._meta.at[i, "rep_gen_gid"] = None + self._meta.at[i, "rep_res_gid"] = None elif len(ggids) == 1: - self._meta.at[i, 'rep_gen_gid'] = ggids[0] - self._meta.at[i, 'rep_res_gid'] = rgids[0] + self._meta.at[i, "rep_gen_gid"] = ggids[0] + self._meta.at[i, "rep_res_gid"] = rgids[0] else: - self._meta.at[i, 'rep_gen_gid'] = str(ggids) - self._meta.at[i, 'rep_res_gid'] = str(rgids) - - def run(self, fout=None, save_rev_summary=True, scaled_precision=False, - max_workers=None): + self._meta.at[i, "rep_gen_gid"] = str(ggids) + self._meta.at[i, "rep_res_gid"] = str(rgids) + + def run( + self, + fout=None, + save_rev_summary=True, + scaled_precision=False, + max_workers=None, + ): """ Run representative profiles in serial or parallel and save to disc @@ -1203,12 +1355,17 @@ def run(self, fout=None, save_rev_summary=True, scaled_precision=False, if fout is not None: if self._aggregate_profiles: - logger.info("Aggregate profiles input set to `True`. Setting " - "'save_rev_summary' input to `False`") + logger.info( + "Aggregate profiles input set to `True`. Setting " + "'save_rev_summary' input to `False`" + ) save_rev_summary = False - self.save_profiles(fout, save_rev_summary=save_rev_summary, - scaled_precision=scaled_precision) + self.save_profiles( + fout, + save_rev_summary=save_rev_summary, + scaled_precision=scaled_precision, + ) - logger.info('Representative profiles complete!') + logger.info("Representative profiles complete!") return fout diff --git a/reV/supply_curve/aggregation.py b/reV/supply_curve/aggregation.py index 5c14ac9e0..564ba8a58 100644 --- a/reV/supply_curve/aggregation.py +++ b/reV/supply_curve/aggregation.py @@ -2,6 +2,7 @@ """ reV aggregation framework. """ + import logging import os from abc import ABC, abstractmethod @@ -19,7 +20,7 @@ from reV.supply_curve.extent import SupplyCurveExtent from reV.supply_curve.points import AggregationSupplyCurvePoint from reV.supply_curve.tech_mapping import TechMapping -from reV.utilities import log_versions +from reV.utilities import MetaKeyName, log_versions from reV.utilities.exceptions import ( EmptySupplyCurvePointError, FileInputError, @@ -32,8 +33,13 @@ class AbstractAggFileHandler(ABC): """Simple framework to handle aggregation file context managers.""" - def __init__(self, excl_fpath, excl_dict=None, area_filter_kernel='queen', - min_area=None): + def __init__( + self, + excl_fpath, + excl_dict=None, + area_filter_kernel="queen", + min_area=None, + ): """ Parameters ---------- @@ -54,9 +60,12 @@ def __init__(self, excl_fpath, excl_dict=None, area_filter_kernel='queen', by default None """ self._excl_fpath = excl_fpath - self._excl = ExclusionMaskFromDict(excl_fpath, layers_dict=excl_dict, - min_area=min_area, - kernel=area_filter_kernel) + self._excl = ExclusionMaskFromDict( + excl_fpath, + layers_dict=excl_dict, + min_area=min_area, + kernel=area_filter_kernel, + ) def __enter__(self): return self @@ -98,9 +107,15 @@ class AggFileHandler(AbstractAggFileHandler): DEFAULT_H5_HANDLER = Resource - def __init__(self, excl_fpath, h5_fpath, excl_dict=None, - area_filter_kernel='queen', min_area=None, - h5_handler=None): + def __init__( + self, + excl_fpath, + h5_fpath, + excl_dict=None, + area_filter_kernel="queen", + min_area=None, + h5_handler=None, + ): """ Parameters ---------- @@ -124,9 +139,12 @@ def __init__(self, excl_fpath, h5_fpath, excl_dict=None, Optional special handler similar to the rex.Resource handler which is default. """ - super().__init__(excl_fpath, excl_dict=excl_dict, - area_filter_kernel=area_filter_kernel, - min_area=min_area) + super().__init__( + excl_fpath, + excl_dict=excl_dict, + area_filter_kernel=area_filter_kernel, + min_area=min_area, + ) if h5_handler is None: self._h5 = Resource(h5_fpath) @@ -155,10 +173,19 @@ class BaseAggregation(ABC): """Abstract supply curve points aggregation framework based on only an exclusion file and techmap.""" - def __init__(self, excl_fpath, tm_dset, excl_dict=None, - area_filter_kernel='queen', min_area=None, - resolution=64, excl_area=None, res_fpath=None, gids=None, - pre_extract_inclusions=False): + def __init__( + self, + excl_fpath, + tm_dset, + excl_dict=None, + area_filter_kernel="queen", + min_area=None, + resolution=64, + excl_area=None, + res_fpath=None, + gids=None, + pre_extract_inclusions=False, + ): """ Parameters ---------- @@ -211,12 +238,15 @@ def __init__(self, excl_fpath, tm_dset, excl_dict=None, self._validate_tech_mapping() if pre_extract_inclusions: - self._inclusion_mask = \ + self._inclusion_mask = ( ExclusionMaskFromDict.extract_inclusion_mask( - excl_fpath, tm_dset, + excl_fpath, + tm_dset, excl_dict=excl_dict, area_filter_kernel=area_filter_kernel, - min_area=min_area) + min_area=min_area, + ) + ) else: self._inclusion_mask = None @@ -231,20 +261,28 @@ def _validate_tech_mapping(self): if tm_in_excl: logger.info('Found techmap "{}".'.format(self._tm_dset)) elif not tm_in_excl and not excl_fp_is_str: - msg = ('Could not find techmap dataset "{}" and cannot run ' - 'techmap with arbitrary multiple exclusion filepaths ' - 'to write to: {}'.format(self._tm_dset, self._excl_fpath)) + msg = ( + 'Could not find techmap dataset "{}" and cannot run ' + "techmap with arbitrary multiple exclusion filepaths " + "to write to: {}".format(self._tm_dset, self._excl_fpath) + ) logger.error(msg) raise RuntimeError(msg) else: - logger.info('Could not find techmap "{}". Running techmap module.' - .format(self._tm_dset)) + logger.info( + 'Could not find techmap "{}". Running techmap module.'.format( + self._tm_dset + ) + ) try: - TechMapping.run(self._excl_fpath, self._res_fpath, - dset=self._tm_dset) + TechMapping.run( + self._excl_fpath, self._res_fpath, dset=self._tm_dset + ) except Exception as e: - msg = ('TechMapping process failed. Received the ' - 'following error:\n{}'.format(e)) + msg = ( + "TechMapping process failed. Received the " + "following error:\n{}".format(e) + ) logger.exception(msg) raise RuntimeError(msg) from e @@ -258,8 +296,9 @@ def gids(self): ndarray """ if self._gids is None: - with SupplyCurveExtent(self._excl_fpath, - resolution=self._resolution) as sc: + with SupplyCurveExtent( + self._excl_fpath, resolution=self._resolution + ) as sc: self._gids = sc.valid_sc_points(self._tm_dset) elif np.issubdtype(type(self._gids), np.number): self._gids = np.array([self._gids]) @@ -277,8 +316,9 @@ def shape(self): tuple """ if self._shape is None: - with SupplyCurveExtent(self._excl_fpath, - resolution=self._resolution) as sc: + with SupplyCurveExtent( + self._excl_fpath, resolution=self._resolution + ) as sc: self._shape = sc.exclusions.shape return self._shape @@ -304,14 +344,18 @@ def _get_excl_area(excl_fpath, excl_area=None): Area of an exclusion pixel in km2 """ if excl_area is None: - logger.debug('Setting the exclusion area from the area of a pixel ' - 'in {}'.format(excl_fpath)) + logger.debug( + "Setting the exclusion area from the area of a pixel " + "in {}".format(excl_fpath) + ) with ExclusionLayers(excl_fpath) as excl: excl_area = excl.pixel_area if excl_area is None: - e = ('No exclusion pixel area was input and could not parse ' - 'area from the exclusion file attributes!') + e = ( + "No exclusion pixel area was input and could not parse " + "area from the exclusion file attributes!" + ) logger.error(e) raise SupplyCurveInputError(e) @@ -340,14 +384,17 @@ def _check_inclusion_mask(inclusion_mask, gids, excl_shape): elif isinstance(inclusion_mask, np.ndarray): assert inclusion_mask.shape == excl_shape elif inclusion_mask is not None: - msg = ('Expected inclusion_mask to be dict or array but received ' - '{}'.format(type(inclusion_mask))) + msg = ( + "Expected inclusion_mask to be dict or array but received " + "{}".format(type(inclusion_mask)) + ) logger.error(msg) raise SupplyCurveInputError(msg) @staticmethod - def _get_gid_inclusion_mask(inclusion_mask, gid, slice_lookup, - resolution=64): + def _get_gid_inclusion_mask( + inclusion_mask, gid, slice_lookup, resolution=64 + ): """ Get inclusion mask for desired gid @@ -384,8 +431,10 @@ def _get_gid_inclusion_mask(inclusion_mask, gid, slice_lookup, row_slice, col_slice = slice_lookup[gid] gid_inclusions = inclusion_mask[row_slice, col_slice] elif inclusion_mask is not None: - msg = ('Expected inclusion_mask to be dict or array but received ' - '{}'.format(type(inclusion_mask))) + msg = ( + "Expected inclusion_mask to be dict or array but received " + "{}".format(type(inclusion_mask)) + ) logger.error(msg) raise SupplyCurveInputError(msg) @@ -410,25 +459,31 @@ def _parse_gen_index(gen_fpath): generation run. """ - if gen_fpath.endswith('.h5'): + if gen_fpath.endswith(".h5"): with Resource(gen_fpath) as f: gen_index = f.meta - elif gen_fpath.endswith('.csv'): + elif gen_fpath.endswith(".csv"): gen_index = pd.read_csv(gen_fpath) else: - msg = ('Could not recognize gen_fpath input, needs to be reV gen ' - 'output h5 or project points csv but received: {}' - .format(gen_fpath)) + msg = ( + "Could not recognize gen_fpath input, needs to be reV gen " + "output h5 or project points csv but received: {}".format( + gen_fpath + ) + ) logger.error(msg) raise FileInputError(msg) if MetaKeyName.GID in gen_index: - gen_index = gen_index.rename(columns={MetaKeyName.GID: MetaKeyName.RES_GIDS}) + gen_index = gen_index.rename( + columns={MetaKeyName.GID: MetaKeyName.RES_GIDS} + ) gen_index[MetaKeyName.GEN_GIDS] = gen_index.index gen_index = gen_index[[MetaKeyName.RES_GIDS, MetaKeyName.GEN_GIDS]] gen_index = gen_index.set_index(keys=MetaKeyName.RES_GIDS) - gen_index = \ - gen_index.reindex(range(int(gen_index.index.max() + 1))) + gen_index = gen_index.reindex( + range(int(gen_index.index.max() + 1)) + ) gen_index = gen_index[MetaKeyName.GEN_GIDS].values gen_index[np.isnan(gen_index)] = -1 gen_index = gen_index.astype(np.int32) @@ -442,10 +497,19 @@ class Aggregation(BaseAggregation): """Concrete but generalized aggregation framework to aggregate ANY reV h5 file to a supply curve grid (based on an aggregated exclusion grid).""" - def __init__(self, excl_fpath, tm_dset, *agg_dset, - excl_dict=None, area_filter_kernel='queen', min_area=None, - resolution=64, excl_area=None, gids=None, - pre_extract_inclusions=False): + def __init__( + self, + excl_fpath, + tm_dset, + *agg_dset, + excl_dict=None, + area_filter_kernel="queen", + min_area=None, + resolution=64, + excl_area=None, + gids=None, + pre_extract_inclusions=False, + ): """ Parameters ---------- @@ -489,18 +553,24 @@ def __init__(self, excl_fpath, tm_dset, *agg_dset, the inclusion mask on the fly with parallel workers. """ log_versions(logger) - logger.info('Initializing Aggregation...') - logger.debug('Exclusion filepath: {}'.format(excl_fpath)) - logger.debug('Exclusion dict: {}'.format(excl_dict)) - - super().__init__(excl_fpath, tm_dset, excl_dict=excl_dict, - area_filter_kernel=area_filter_kernel, - min_area=min_area, resolution=resolution, - excl_area=excl_area, gids=gids, - pre_extract_inclusions=pre_extract_inclusions) + logger.info("Initializing Aggregation...") + logger.debug("Exclusion filepath: {}".format(excl_fpath)) + logger.debug("Exclusion dict: {}".format(excl_dict)) + + super().__init__( + excl_fpath, + tm_dset, + excl_dict=excl_dict, + area_filter_kernel=area_filter_kernel, + min_area=min_area, + resolution=resolution, + excl_area=excl_area, + gids=gids, + pre_extract_inclusions=pre_extract_inclusions, + ) if isinstance(agg_dset, str): - agg_dset = (agg_dset, ) + agg_dset = (agg_dset,) self._agg_dsets = agg_dset @@ -508,33 +578,51 @@ def _check_files(self, h5_fpath): """Do a preflight check on input files""" if not os.path.exists(self._excl_fpath): - raise FileNotFoundError('Could not find required exclusions file: ' - '{}'.format(self._excl_fpath)) + raise FileNotFoundError( + "Could not find required exclusions file: " "{}".format( + self._excl_fpath + ) + ) if not os.path.exists(h5_fpath): - raise FileNotFoundError('Could not find required h5 file: ' - '{}'.format(h5_fpath)) + raise FileNotFoundError( + "Could not find required h5 file: " "{}".format(h5_fpath) + ) - with h5py.File(self._excl_fpath, 'r') as f: + with h5py.File(self._excl_fpath, "r") as f: if self._tm_dset not in f: - raise FileInputError('Could not find techmap dataset "{}" ' - 'in exclusions file: {}' - .format(self._tm_dset, - self._excl_fpath)) + raise FileInputError( + 'Could not find techmap dataset "{}" ' + "in exclusions file: {}".format( + self._tm_dset, self._excl_fpath + ) + ) with Resource(h5_fpath) as f: for dset in self._agg_dsets: if dset not in f: - raise FileInputError('Could not find provided dataset "{}"' - ' in h5 file: {}' - .format(dset, h5_fpath)) + raise FileInputError( + 'Could not find provided dataset "{}"' + " in h5 file: {}".format(dset, h5_fpath) + ) @classmethod - def run_serial(cls, excl_fpath, h5_fpath, tm_dset, *agg_dset, - agg_method='mean', excl_dict=None, inclusion_mask=None, - area_filter_kernel='queen', min_area=None, - resolution=64, excl_area=0.0081, gids=None, - gen_index=None): + def run_serial( + cls, + excl_fpath, + h5_fpath, + tm_dset, + *agg_dset, + agg_method="mean", + excl_dict=None, + inclusion_mask=None, + area_filter_kernel="queen", + min_area=None, + resolution=64, + excl_area=0.0081, + gids=None, + gen_index=None, + ): """ Standalone method to aggregate - can be parallelized. @@ -605,17 +693,19 @@ def run_serial(cls, excl_fpath, h5_fpath, tm_dset, *agg_dset, cls._check_inclusion_mask(inclusion_mask, gids, exclusion_shape) # pre-extract handlers so they are not repeatedly initialized - file_kwargs = {'excl_dict': excl_dict, - 'area_filter_kernel': area_filter_kernel, - 'min_area': min_area} - dsets = agg_dset + ('meta', ) + file_kwargs = { + "excl_dict": excl_dict, + "area_filter_kernel": area_filter_kernel, + "min_area": min_area, + } + dsets = agg_dset + ("meta",) agg_out = {ds: [] for ds in dsets} with AggFileHandler(excl_fpath, h5_fpath, **file_kwargs) as fh: n_finished = 0 for gid in gids: gid_inclusions = cls._get_gid_inclusion_mask( - inclusion_mask, gid, slice_lookup, - resolution=resolution) + inclusion_mask, gid, slice_lookup, resolution=resolution + ) try: gid_out = AggregationSupplyCurvePoint.run( gid, @@ -630,28 +720,40 @@ def run_serial(cls, excl_fpath, h5_fpath, tm_dset, *agg_dset, excl_area=excl_area, exclusion_shape=exclusion_shape, close=False, - gen_index=gen_index) + gen_index=gen_index, + ) except EmptySupplyCurvePointError: - logger.debug('SC gid {} is fully excluded or does not ' - 'have any valid source data!'.format(gid)) + logger.debug( + "SC gid {} is fully excluded or does not " + "have any valid source data!".format(gid) + ) except Exception as e: - msg = 'SC gid {} failed!'.format(gid) + msg = "SC gid {} failed!".format(gid) logger.exception(msg) raise RuntimeError(msg) from e else: n_finished += 1 - logger.debug('Serial aggregation: ' - '{} out of {} points complete' - .format(n_finished, len(gids))) + logger.debug( + "Serial aggregation: " + "{} out of {} points complete".format( + n_finished, len(gids) + ) + ) log_mem(logger) for k, v in gid_out.items(): agg_out[k].append(v) return agg_out - def run_parallel(self, h5_fpath, agg_method='mean', excl_area=None, - max_workers=None, sites_per_worker=100): + def run_parallel( + self, + h5_fpath, + agg_method="mean", + excl_area=None, + max_workers=None, + sites_per_worker=100, + ): """ Aggregate in parallel @@ -684,22 +786,29 @@ def run_parallel(self, h5_fpath, agg_method='mean', excl_area=None, chunks = np.array_split(self.gids, chunks) if self._inclusion_mask is not None: - with SupplyCurveExtent(self._excl_fpath, - resolution=self._resolution) as sc: + with SupplyCurveExtent( + self._excl_fpath, resolution=self._resolution + ) as sc: assert sc.exclusions.shape == self._inclusion_mask.shape slice_lookup = sc.get_slice_lookup(self.gids) - logger.info('Running supply curve point aggregation for ' - 'points {} through {} at a resolution of {} ' - 'on {} cores in {} chunks.' - .format(self.gids[0], self.gids[-1], self._resolution, - max_workers, len(chunks))) + logger.info( + "Running supply curve point aggregation for " + "points {} through {} at a resolution of {} " + "on {} cores in {} chunks.".format( + self.gids[0], + self.gids[-1], + self._resolution, + max_workers, + len(chunks), + ) + ) n_finished = 0 futures = [] - dsets = self._agg_dsets + ('meta', ) + dsets = self._agg_dsets + ("meta",) agg_out = {ds: [] for ds in dsets} - loggers = [__name__, 'reV.supply_curve.points', 'reV'] + loggers = [__name__, "reV.supply_curve.points", "reV"] with SpawnProcessPool(max_workers=max_workers, loggers=loggers) as exe: # iterate through split executions, submitting each to worker for gid_set in chunks: @@ -712,36 +821,45 @@ def run_parallel(self, h5_fpath, agg_method='mean', excl_area=None, chunk_incl_masks[gid] = self._inclusion_mask[rs, cs] # submit executions and append to futures list - futures.append(exe.submit( - self.run_serial, - self._excl_fpath, - h5_fpath, - self._tm_dset, - *self._agg_dsets, - agg_method=agg_method, - excl_dict=self._excl_dict, - inclusion_mask=chunk_incl_masks, - area_filter_kernel=self._area_filter_kernel, - min_area=self._min_area, - resolution=self._resolution, - excl_area=excl_area, - gids=gid_set, - gen_index=gen_index)) + futures.append( + exe.submit( + self.run_serial, + self._excl_fpath, + h5_fpath, + self._tm_dset, + *self._agg_dsets, + agg_method=agg_method, + excl_dict=self._excl_dict, + inclusion_mask=chunk_incl_masks, + area_filter_kernel=self._area_filter_kernel, + min_area=self._min_area, + resolution=self._resolution, + excl_area=excl_area, + gids=gid_set, + gen_index=gen_index, + ) + ) # gather results for future in futures: n_finished += 1 - logger.info('Parallel aggregation futures collected: ' - '{} out of {}' - .format(n_finished, len(chunks))) + logger.info( + "Parallel aggregation futures collected: " + "{} out of {}".format(n_finished, len(chunks)) + ) for k, v in future.result().items(): if v: agg_out[k].extend(v) return agg_out - def aggregate(self, h5_fpath, agg_method='mean', max_workers=None, - sites_per_worker=100): + def aggregate( + self, + h5_fpath, + agg_method="mean", + max_workers=None, + sites_per_worker=100, + ): """ Aggregate with given agg_method @@ -769,34 +887,40 @@ def aggregate(self, h5_fpath, agg_method='mean', max_workers=None, if max_workers == 1: self._check_files(h5_fpath) gen_index = self._parse_gen_index(h5_fpath) - agg = self.run_serial(self._excl_fpath, - h5_fpath, - self._tm_dset, - *self._agg_dsets, - agg_method=agg_method, - excl_dict=self._excl_dict, - gids=self.gids, - inclusion_mask=self._inclusion_mask, - area_filter_kernel=self._area_filter_kernel, - min_area=self._min_area, - resolution=self._resolution, - excl_area=self._excl_area, - gen_index=gen_index) + agg = self.run_serial( + self._excl_fpath, + h5_fpath, + self._tm_dset, + *self._agg_dsets, + agg_method=agg_method, + excl_dict=self._excl_dict, + gids=self.gids, + inclusion_mask=self._inclusion_mask, + area_filter_kernel=self._area_filter_kernel, + min_area=self._min_area, + resolution=self._resolution, + excl_area=self._excl_area, + gen_index=gen_index, + ) else: - agg = self.run_parallel(h5_fpath=h5_fpath, - agg_method=agg_method, - excl_area=self._excl_area, - max_workers=max_workers, - sites_per_worker=sites_per_worker) - - if not agg['meta']: - e = ('Supply curve aggregation found no non-excluded SC points. ' - 'Please check your exclusions or subset SC GID selection.') + agg = self.run_parallel( + h5_fpath=h5_fpath, + agg_method=agg_method, + excl_area=self._excl_area, + max_workers=max_workers, + sites_per_worker=sites_per_worker, + ) + + if not agg["meta"]: + e = ( + "Supply curve aggregation found no non-excluded SC points. " + "Please check your exclusions or subset SC GID selection." + ) logger.error(e) raise EmptySupplyCurvePointError(e) for k, v in agg.items(): - if k == 'meta': + if k == "meta": v = pd.concat(v, axis=1).T v = v.sort_values(MetaKeyName.SC_POINT_GID) v = v.reset_index(drop=True) @@ -824,7 +948,7 @@ def save_agg_to_h5(h5_fpath, out_fpath, aggregation): Aggregated values for each aggregation dataset """ agg_out = aggregation.copy() - meta = agg_out.pop('meta').reset_index() + meta = agg_out.pop("meta").reset_index() for c in meta.columns: try: meta[c] = pd.to_numeric(meta[c]) @@ -843,7 +967,7 @@ def save_agg_to_h5(h5_fpath, out_fpath, aggregation): shape = data.shape shapes[dset] = shape if len(data.shape) == 2: - if ('time_index' in f) and (shape[0] == f.shape[0]): + if ("time_index" in f) and (shape[0] == f.shape[0]): if time_index is None: time_index = f.time_index @@ -852,19 +976,40 @@ def save_agg_to_h5(h5_fpath, out_fpath, aggregation): chunks[dset] = chunk dtypes[dset] = dtype - Outputs.init_h5(out_fpath, dsets, shapes, attrs, chunks, dtypes, - meta, time_index=time_index) - - with Outputs(out_fpath, mode='a') as out: + Outputs.init_h5( + out_fpath, + dsets, + shapes, + attrs, + chunks, + dtypes, + meta, + time_index=time_index, + ) + + with Outputs(out_fpath, mode="a") as out: for dset, data in agg_out.items(): out[dset] = data @classmethod - def run(cls, excl_fpath, h5_fpath, tm_dset, *agg_dset, - excl_dict=None, area_filter_kernel='queen', min_area=None, - resolution=64, excl_area=None, gids=None, - pre_extract_inclusions=False, agg_method='mean', max_workers=None, - sites_per_worker=100, out_fpath=None): + def run( + cls, + excl_fpath, + h5_fpath, + tm_dset, + *agg_dset, + excl_dict=None, + area_filter_kernel="queen", + min_area=None, + resolution=64, + excl_area=None, + gids=None, + pre_extract_inclusions=False, + agg_method="mean", + max_workers=None, + sites_per_worker=100, + out_fpath=None, + ): """Get the supply curve points aggregation summary. Parameters @@ -926,15 +1071,25 @@ def run(cls, excl_fpath, h5_fpath, tm_dset, *agg_dset, Aggregated values for each aggregation dataset """ - agg = cls(excl_fpath, tm_dset, *agg_dset, - excl_dict=excl_dict, area_filter_kernel=area_filter_kernel, - min_area=min_area, resolution=resolution, - excl_area=excl_area, gids=gids, - pre_extract_inclusions=pre_extract_inclusions) - - aggregation = agg.aggregate(h5_fpath=h5_fpath, agg_method=agg_method, - max_workers=max_workers, - sites_per_worker=sites_per_worker) + agg = cls( + excl_fpath, + tm_dset, + *agg_dset, + excl_dict=excl_dict, + area_filter_kernel=area_filter_kernel, + min_area=min_area, + resolution=resolution, + excl_area=excl_area, + gids=gids, + pre_extract_inclusions=pre_extract_inclusions, + ) + + aggregation = agg.aggregate( + h5_fpath=h5_fpath, + agg_method=agg_method, + max_workers=max_workers, + sites_per_worker=sites_per_worker, + ) if out_fpath is not None: agg.save_agg_to_h5(h5_fpath, out_fpath, aggregation) diff --git a/reV/supply_curve/competitive_wind_farms.py b/reV/supply_curve/competitive_wind_farms.py index d93fae6cd..898ab94c6 100644 --- a/reV/supply_curve/competitive_wind_farms.py +++ b/reV/supply_curve/competitive_wind_farms.py @@ -2,6 +2,7 @@ """ Competitive Wind Farms exclusion handler """ + import logging import numpy as np @@ -35,30 +36,36 @@ def __init__(self, wind_dirs, sc_points, n_dirs=2, offshore=False): """ self._wind_dirs = self._parse_wind_dirs(wind_dirs) - self._sc_gids, self._sc_point_gids, self._mask = \ - self._parse_sc_points(sc_points, offshore=offshore) + self._sc_gids, self._sc_point_gids, self._mask = self._parse_sc_points( + sc_points, offshore=offshore + ) self._offshore = offshore valid = np.isin(self.sc_point_gids, self._wind_dirs.index) if not np.all(valid): - msg = ("'sc_points contains sc_point_gid values that do not " - "correspond to valid 'wind_dirs' sc_point_gids:\n{}" - .format(self.sc_point_gids[~valid])) + msg = ( + "'sc_points contains sc_point_gid values that do not " + "correspond to valid 'wind_dirs' sc_point_gids:\n{}".format( + self.sc_point_gids[~valid] + ) + ) logger.error(msg) raise RuntimeError(msg) mask = self._wind_dirs.index.isin(self._sc_point_gids.keys()) self._wind_dirs = self._wind_dirs.loc[mask] - self._upwind, self._downwind = self._get_neighbors(self._wind_dirs, - n_dirs=n_dirs) + self._upwind, self._downwind = self._get_neighbors( + self._wind_dirs, n_dirs=n_dirs + ) def __repr__(self): gids = len(self._upwind) # pylint: disable=unsubscriptable-object neighbors = len(self._upwind.values[0]) - msg = ("{} with {} sc_point_gids and {} prominent directions" - .format(self.__class__.__name__, gids, neighbors)) + msg = "{} with {} sc_point_gids and {} prominent directions".format( + self.__class__.__name__, gids, neighbors + ) return msg @@ -77,9 +84,13 @@ def __getitem__(self, keys): Mapped gid(s) for given mapping """ if not isinstance(keys, tuple): - msg = ("{} must be a tuple of form (source, gid) where source is: " - "MetaKeyName.SC_GID, MetaKeyName.SC_POINT_GID, or 'upwind', 'downwind'" - .format(keys)) + msg = ( + "{} must be a tuple of form (source, gid) where source is: " + "MetaKeyName.SC_GID, MetaKeyName.SC_POINT_GID, or 'upwind', " + "'downwind'".format( + keys + ) + ) logger.error(msg) raise ValueError(msg) @@ -88,13 +99,15 @@ def __getitem__(self, keys): out = self.map_sc_gid_to_sc_point_gid(gid) elif source == MetaKeyName.SC_GID: out = self.map_sc_point_gid_to_sc_gid(gid) - elif source == 'upwind': + elif source == "upwind": out = self.map_upwind(gid) - elif source == 'downwind': + elif source == "downwind": out = self.map_downwind(gid) else: - msg = ("{} must be: MetaKeyName.SC_GID, MetaKeyName.SC_POINT_GID, or 'upwind', " - "'downwind'".format(source)) + msg = ( + "{} must be: MetaKeyName.SC_GID, MetaKeyName.SC_POINT_GID, " + "or 'upwind', 'downwind'".format(source) + ) logger.error(msg) raise ValueError(msg) @@ -135,9 +148,9 @@ def sc_gids(self): ------- ndarray """ - sc_gids = \ - np.concatenate([self._sc_point_gids[gid] - for gid in self.sc_point_gids]) + sc_gids = np.concatenate( + [self._sc_point_gids[gid] for gid in self.sc_point_gids] + ) return sc_gids @@ -185,7 +198,7 @@ def _parse_wind_dirs(cls, wind_dirs): wind_dirs = cls._parse_table(wind_dirs) wind_dirs = wind_dirs.set_index(MetaKeyName.SC_POINT_GID) - columns = [c for c in wind_dirs if c.endswith(('_gid', '_pr'))] + columns = [c for c in wind_dirs if c.endswith(("_gid", "_pr"))] wind_dirs = wind_dirs[columns] return wind_dirs @@ -215,21 +228,29 @@ def _parse_sc_points(cls, sc_points, offshore=False): """ sc_points = cls._parse_table(sc_points) if MetaKeyName.OFFSHORE in sc_points and not offshore: - logger.debug('Not including offshore supply curve points in ' - 'CompetitiveWindFarm') + logger.debug( + "Not including offshore supply curve points in " + "CompetitiveWindFarm" + ) mask = sc_points[MetaKeyName.OFFSHORE] == 0 sc_points = sc_points.loc[mask] - mask = np.ones(int(1 + sc_points[MetaKeyName.SC_POINT_GID].max()), dtype=bool) + mask = np.ones( + int(1 + sc_points[MetaKeyName.SC_POINT_GID].max()), dtype=bool + ) sc_points = sc_points[[MetaKeyName.SC_GID, MetaKeyName.SC_POINT_GID]] sc_gids = sc_points.set_index(MetaKeyName.SC_GID) sc_gids = {k: int(v[0]) for k, v in sc_gids.iterrows()} - sc_point_gids = \ - sc_points.groupby(MetaKeyName.SC_POINT_GID)[MetaKeyName.SC_GID].unique().to_frame() - sc_point_gids = {int(k): v[MetaKeyName.SC_GID] - for k, v in sc_point_gids.iterrows()} + sc_point_gids = ( + sc_points.groupby(MetaKeyName.SC_POINT_GID)[MetaKeyName.SC_GID] + .unique() + .to_frame() + ) + sc_point_gids = { + int(k): v[MetaKeyName.SC_GID] for k, v in sc_point_gids.iterrows() + } return sc_gids, sc_point_gids, mask @@ -253,19 +274,30 @@ def _get_neighbors(wind_dirs, n_dirs=2): downwind : pandas.DataFrame Downwind neighbor gids for n prominent wind directions """ - cols = [c for c in wind_dirs - if (c.endswith('_gid') and not c.startswith('sc'))] - directions = [c.split('_')[0] for c in cols] + cols = [ + c + for c in wind_dirs + if (c.endswith("_gid") and not c.startswith("sc")) + ] + directions = [c.split("_")[0] for c in cols] upwind_gids = wind_dirs[cols].values - cols = ['{}_pr'.format(d) for d in directions] + cols = ["{}_pr".format(d) for d in directions] neighbor_pr = wind_dirs[cols].values neighbors = np.argsort(neighbor_pr)[:, :n_dirs] upwind_gids = np.take_along_axis(upwind_gids, neighbors, axis=1) - downwind_map = {'N': 'S', 'NE': 'SW', 'E': 'W', 'SE': 'NW', 'S': 'N', - 'SW': 'NE', 'W': 'E', 'NW': 'SE'} + downwind_map = { + "N": "S", + "NE": "SW", + "E": "W", + "SE": "NW", + "S": "N", + "SW": "NE", + "W": "E", + "NW": "SE", + } cols = ["{}_gid".format(downwind_map[d]) for d in directions] downwind_gids = wind_dirs[cols].values downwind_gids = np.take_along_axis(downwind_gids, neighbors, axis=1) @@ -387,8 +419,9 @@ def exclude_sc_point_gid(self, sc_point_gid): return out - def remove_noncompetitive_farm(self, sc_points, sort_on='total_lcoe', - downwind=False): + def remove_noncompetitive_farm( + self, sc_points, sort_on="total_lcoe", downwind=False + ): """ Remove neighboring sc points for given number of prominent wind directions @@ -422,12 +455,12 @@ def remove_noncompetitive_farm(self, sc_points, sort_on='total_lcoe', for i in range(len(sc_points)): gid = sc_point_gids[i] if self.mask[gid]: - upwind_gids = self['upwind', gid] + upwind_gids = self["upwind", gid] for n in upwind_gids: self.exclude_sc_point_gid(n) if downwind: - downwind_gids = self['downwind', gid] + downwind_gids = self["downwind", gid] for n in downwind_gids: self.exclude_sc_point_gid(n) @@ -437,8 +470,16 @@ def remove_noncompetitive_farm(self, sc_points, sort_on='total_lcoe', return sc_points.loc[mask].reset_index(drop=True) @classmethod - def run(cls, wind_dirs, sc_points, n_dirs=2, offshore=False, - sort_on='total_lcoe', downwind=False, out_fpath=None): + def run( + cls, + wind_dirs, + sc_points, + n_dirs=2, + offshore=False, + sort_on="total_lcoe", + downwind=False, + out_fpath=None, + ): """ Exclude given number of neighboring Supply Point gids based on most prominent wind directions @@ -473,8 +514,9 @@ def run(cls, wind_dirs, sc_points, n_dirs=2, offshore=False, wind farms """ cwf = cls(wind_dirs, sc_points, n_dirs=n_dirs, offshore=offshore) - sc_points = cwf.remove_noncompetitive_farm(sc_points, sort_on=sort_on, - downwind=downwind) + sc_points = cwf.remove_noncompetitive_farm( + sc_points, sort_on=sort_on, downwind=downwind + ) if out_fpath is not None: sc_points.to_csv(out_fpath, index=False) diff --git a/reV/supply_curve/extent.py b/reV/supply_curve/extent.py index b63e25cac..19ffac8b1 100644 --- a/reV/supply_curve/extent.py +++ b/reV/supply_curve/extent.py @@ -2,6 +2,7 @@ """ reV supply curve extent """ + import logging import numpy as np @@ -32,13 +33,17 @@ def __init__(self, f_excl, resolution=64): SC point. """ - logger.debug('Initializing SupplyCurveExtent with res {} from: {}' - .format(resolution, f_excl)) + logger.debug( + "Initializing SupplyCurveExtent with res {} from: {}".format( + resolution, f_excl + ) + ) if not isinstance(resolution, int): - raise SupplyCurveInputError('Supply Curve resolution needs to be ' - 'an integer but received: {}' - .format(type(resolution))) + raise SupplyCurveInputError( + "Supply Curve resolution needs to be " + "an integer but received: {}".format(type(resolution)) + ) if isinstance(f_excl, (str, list, tuple)): self._excl_fpath = f_excl @@ -47,11 +52,12 @@ def __init__(self, f_excl, resolution=64): self._excl_fpath = f_excl.h5_file self._excls = f_excl else: - raise SupplyCurveInputError('SupplyCurvePoints needs an ' - 'exclusions file path, or ' - 'ExclusionLayers handler but ' - 'received: {}' - .format(type(f_excl))) + raise SupplyCurveInputError( + "SupplyCurvePoints needs an " + "exclusions file path, or " + "ExclusionLayers handler but " + "received: {}".format(type(f_excl)) + ) self._excl_shape = self.exclusions.shape # limit the resolution to the exclusion shape. @@ -68,13 +74,15 @@ def __init__(self, f_excl, resolution=64): self._points = None self._sc_col_ind, self._sc_row_ind = np.meshgrid( - np.arange(self.n_cols), np.arange(self.n_rows)) + np.arange(self.n_cols), np.arange(self.n_rows) + ) self._sc_col_ind = self._sc_col_ind.flatten() self._sc_row_ind = self._sc_row_ind.flatten() - logger.debug('Initialized SupplyCurveExtent with shape {} from ' - 'exclusions with shape {}' - .format(self.shape, self.excl_shape)) + logger.debug( + "Initialized SupplyCurveExtent with shape {} from " + "exclusions with shape {}".format(self.shape, self.excl_shape) + ) def __len__(self): """Total number of supply curve points.""" @@ -91,8 +99,10 @@ def __exit__(self, type, value, traceback): def __getitem__(self, gid): """Get SC extent meta data corresponding to an SC point gid.""" if gid >= len(self): - raise KeyError('SC extent with {} points does not contain SC ' - 'point gid {}.'.format(len(self), gid)) + raise KeyError( + "SC extent with {} points does not contain SC " + "point gid {}.".format(len(self), gid) + ) return self.points.loc[gid] @@ -181,8 +191,9 @@ def rows_of_excl(self): point. """ if self._rows_of_excl is None: - self._rows_of_excl = self._chunk_excl(self.excl_rows, - self.resolution) + self._rows_of_excl = self._chunk_excl( + self.excl_rows, self.resolution + ) return self._rows_of_excl @@ -199,8 +210,9 @@ def cols_of_excl(self): point. """ if self._cols_of_excl is None: - self._cols_of_excl = self._chunk_excl(self.excl_cols, - self.resolution) + self._cols_of_excl = self._chunk_excl( + self.excl_cols, self.resolution + ) return self._cols_of_excl @@ -218,8 +230,9 @@ def excl_row_slices(self): point. """ if self._excl_row_slices is None: - self._excl_row_slices = self._excl_slices(self.excl_rows, - self.resolution) + self._excl_row_slices = self._excl_slices( + self.excl_rows, self.resolution + ) return self._excl_row_slices @@ -237,8 +250,9 @@ def excl_col_slices(self): point. """ if self._excl_col_slices is None: - self._excl_col_slices = self._excl_slices(self.excl_cols, - self.resolution) + self._excl_col_slices = self._excl_slices( + self.excl_cols, self.resolution + ) return self._excl_col_slices @@ -283,16 +297,19 @@ def latitude(self): lats = [] lons = [] - sc_cols, sc_rows = np.meshgrid(np.arange(self.n_cols), - np.arange(self.n_rows)) + sc_cols, sc_rows = np.meshgrid( + np.arange(self.n_cols), np.arange(self.n_rows) + ) for r, c in zip(sc_rows.flatten(), sc_cols.flatten()): r = self.excl_row_slices[r] c = self.excl_col_slices[c] lats.append(self.exclusions[MetaKeyName.LATITUDE, r, c].mean()) - lons.append(self.exclusions[MetaKeyName.LONGITUDE, r, c].mean()) + lons.append( + self.exclusions[MetaKeyName.LONGITUDE, r, c].mean() + ) - self._latitude = np.array(lats, dtype='float32') - self._longitude = np.array(lons, dtype='float32') + self._latitude = np.array(lats, dtype="float32") + self._longitude = np.array(lons, dtype="float32") return self._latitude @@ -309,16 +326,19 @@ def longitude(self): lats = [] lons = [] - sc_cols, sc_rows = np.meshgrid(np.arange(self.n_cols), - np.arange(self.n_rows)) + sc_cols, sc_rows = np.meshgrid( + np.arange(self.n_cols), np.arange(self.n_rows) + ) for r, c in zip(sc_rows.flatten(), sc_cols.flatten()): r = self.excl_row_slices[r] c = self.excl_col_slices[c] lats.append(self.exclusions[MetaKeyName.LATITUDE, r, c].mean()) - lons.append(self.exclusions[MetaKeyName.LONGITUDE, r, c].mean()) + lons.append( + self.exclusions[MetaKeyName.LONGITUDE, r, c].mean() + ) - self._latitude = np.array(lats, dtype='float32') - self._longitude = np.array(lons, dtype='float32') + self._latitude = np.array(lats, dtype="float32") + self._longitude = np.array(lons, dtype="float32") return self._longitude @@ -368,8 +388,12 @@ def points(self): """ if self._points is None: - self._points = pd.DataFrame({'row_ind': self.row_indices.copy(), - 'col_ind': self.col_indices.copy()}) + self._points = pd.DataFrame( + { + "row_ind": self.row_indices.copy(), + "col_ind": self.col_indices.copy(), + } + ) self._points.index.name = MetaKeyName.GID # sc_point_gid @@ -437,8 +461,8 @@ def get_sc_row_col_ind(self, gid): col_ind : int Column index that the gid is located at in the sc grid. """ - row_ind = self.points.loc[gid, 'row_ind'] - col_ind = self.points.loc[gid, 'col_ind'] + row_ind = self.points.loc[gid, "row_ind"] + col_ind = self.points.loc[gid, "col_ind"] return row_ind, col_ind def get_excl_slices(self, gid): @@ -459,9 +483,10 @@ def get_excl_slices(self, gid): """ if gid >= len(self): - raise SupplyCurveError('Requested gid "{}" is out of bounds for ' - 'supply curve points with length "{}".' - .format(gid, len(self))) + raise SupplyCurveError( + 'Requested gid "{}" is out of bounds for ' + 'supply curve points with length "{}".'.format(gid, len(self)) + ) row_slice = self.excl_row_slices[self.row_indices[gid]] col_slice = self.excl_col_slices[self.col_indices[gid]] @@ -560,9 +585,12 @@ def valid_sc_points(self, tm_dset): valid_gids = np.where(valid_bool == 1)[0].astype(np.uint32) - logger.info('Found {} valid SC points out of {} total possible ' - '(valid SC points that map to valid resource gids)' - .format(len(valid_gids), len(valid_bool))) + logger.info( + "Found {} valid SC points out of {} total possible " + "(valid SC points that map to valid resource gids)".format( + len(valid_gids), len(valid_bool) + ) + ) return valid_gids diff --git a/reV/supply_curve/points.py b/reV/supply_curve/points.py index b864c44cf..0d3d4e495 100644 --- a/reV/supply_curve/points.py +++ b/reV/supply_curve/points.py @@ -2,6 +2,7 @@ """ reV supply curve points frameworks. """ + import logging from abc import ABC from warnings import warn @@ -51,7 +52,8 @@ def __init__(self, gid, exclusion_shape, resolution=64): self._gid = gid self._resolution = resolution self._rows, self._cols = self._parse_slices( - gid, resolution, exclusion_shape) + gid, resolution, exclusion_shape + ) @staticmethod def _ordered_unique(seq): @@ -175,8 +177,10 @@ def get_agg_slices(gid, shape, resolution): row = loc[0][0] col = loc[1][0] except IndexError as exc: - msg = ('Gid {} out of bounds for extent shape {} and ' - 'resolution {}.'.format(gid, shape, resolution)) + msg = ( + "Gid {} out of bounds for extent shape {} and " + "resolution {}.".format(gid, shape, resolution) + ) raise IndexError(msg) from exc if row + 1 != nrows: @@ -195,9 +199,18 @@ def get_agg_slices(gid, shape, resolution): class SupplyCurvePoint(AbstractSupplyCurvePoint): """Generic single SC point based on exclusions, resolution, and techmap""" - def __init__(self, gid, excl, tm_dset, excl_dict=None, inclusion_mask=None, - resolution=64, excl_area=None, exclusion_shape=None, - close=True): + def __init__( + self, + gid, + excl, + tm_dset, + excl_dict=None, + inclusion_mask=None, + resolution=64, + excl_area=None, + exclusion_shape=None, + close=True, + ): """ Parameters ---------- @@ -248,8 +261,10 @@ def __init__(self, gid, excl, tm_dset, excl_dict=None, inclusion_mask=None, self._incl_mask = inclusion_mask self._incl_mask_flat = None if inclusion_mask is not None: - msg = ('Bad inclusion mask input shape of {} with stated ' - 'resolution of {}'.format(inclusion_mask.shape, resolution)) + msg = ( + "Bad inclusion mask input shape of {} with stated " + "resolution of {}".format(inclusion_mask.shape, resolution) + ) assert len(inclusion_mask.shape) == 2, msg assert inclusion_mask.shape[0] <= resolution, msg assert inclusion_mask.shape[1] <= resolution, msg @@ -285,11 +300,12 @@ def _parse_excl_file(excl): excl_fpath = excl.excl_h5.h5_file exclusions = excl else: - raise SupplyCurveInputError('SupplyCurvePoints needs an ' - 'exclusions file path, or ' - 'ExclusionMask handler but ' - 'received: {}' - .format(type(excl))) + raise SupplyCurveInputError( + "SupplyCurvePoints needs an " + "exclusions file path, or " + "ExclusionMask handler but " + "received: {}".format(type(excl)) + ) return excl_fpath, exclusions @@ -315,9 +331,12 @@ def _parse_techmap(self, tm_dset): res_gids = res_gids.astype(np.int32).flatten() if (res_gids != -1).sum() == 0: - emsg = ('Supply curve point gid {} has no viable exclusion points ' - 'based on exclusions file: "{}"' - .format(self._gid, self._excl_fpath)) + emsg = ( + "Supply curve point gid {} has no viable exclusion points " + 'based on exclusions file: "{}"'.format( + self._gid, self._excl_fpath + ) + ) raise EmptySupplyCurvePointError(emsg) return res_gids @@ -346,8 +365,9 @@ def exclusions(self): ExclusionMask h5 handler object. """ if self._excls is None: - self._excls = ExclusionMaskFromDict(self._excl_fpath, - layers_dict=self._excl_dict) + self._excls = ExclusionMaskFromDict( + self._excl_fpath, layers_dict=self._excl_dict + ) return self._excls @@ -362,8 +382,12 @@ def centroid(self): """ if self._centroid is None: - lats = self.exclusions.excl_h5[MetaKeyName.LATITUDE, self.rows, self.cols] - lons = self.exclusions.excl_h5[MetaKeyName.LONGITUDE, self.rows, self.cols] + lats = self.exclusions.excl_h5[ + MetaKeyName.LATITUDE, self.rows, self.cols + ] + lons = self.exclusions.excl_h5[ + MetaKeyName.LONGITUDE, self.rows, self.cols + ] self._centroid = (lats.mean(), lons.mean()) return self._centroid @@ -441,8 +465,12 @@ def include_mask(self): self._incl_mask[out_of_extent] = 0.0 if self._incl_mask.max() > 1: - w = ('Exclusions data max value is > 1: {}' - .format(self._incl_mask.max()), InputWarning) + w = ( + "Exclusions data max value is > 1: {}".format( + self._incl_mask.max() + ), + InputWarning, + ) logger.warning(w) warn(w) @@ -508,8 +536,9 @@ def _check_excl(self): """ if all(self.include_mask_flat[self.bool_mask] == 0): - msg = ('Supply curve point gid {} is completely excluded!' - .format(self._gid)) + msg = "Supply curve point gid {} is completely excluded!".format( + self._gid + ) raise EmptySupplyCurvePointError(msg) def exclusion_weighted_mean(self, arr, drop_nan=True): @@ -533,13 +562,13 @@ def exclusion_weighted_mean(self, arr, drop_nan=True): """ if len(arr.shape) == 2: - x = arr[:, self._gids[self.bool_mask]].astype('float32') + x = arr[:, self._gids[self.bool_mask]].astype("float32") incl = self.include_mask_flat[self.bool_mask] x *= incl mean = x.sum(axis=1) / incl.sum() else: - x = arr[self._gids[self.bool_mask]].astype('float32') + x = arr[self._gids[self.bool_mask]].astype("float32") incl = self.include_mask_flat[self.bool_mask] if np.isnan(x).all(): @@ -603,10 +632,10 @@ def aggregate(self, arr): Sum of arr masked by the binary exclusions """ if len(arr.shape) == 2: - x = arr[:, self._gids[self.bool_mask]].astype('float32') + x = arr[:, self._gids[self.bool_mask]].astype("float32") ax = 1 else: - x = arr[self._gids[self.bool_mask]].astype('float32') + x = arr[self._gids[self.bool_mask]].astype("float32") ax = 0 x *= self.include_mask_flat[self.bool_mask] @@ -615,8 +644,17 @@ def aggregate(self, arr): return agg @classmethod - def sc_mean(cls, gid, excl, tm_dset, data, excl_dict=None, resolution=64, - exclusion_shape=None, close=True): + def sc_mean( + cls, + gid, + excl, + tm_dset, + data, + excl_dict=None, + resolution=64, + exclusion_shape=None, + close=True, + ): """ Compute exclusions weight mean for the sc point from data @@ -652,16 +690,29 @@ def sc_mean(cls, gid, excl, tm_dset, data, excl_dict=None, resolution=64, ndarray Exclusions weighted means of data for supply curve point """ - kwargs = {"excl_dict": excl_dict, "resolution": resolution, - "exclusion_shape": exclusion_shape, "close": close} + kwargs = { + "excl_dict": excl_dict, + "resolution": resolution, + "exclusion_shape": exclusion_shape, + "close": close, + } with cls(gid, excl, tm_dset, **kwargs) as point: means = point.exclusion_weighted_mean(data) return means @classmethod - def sc_sum(cls, gid, excl, tm_dset, data, excl_dict=None, resolution=64, - exclusion_shape=None, close=True): + def sc_sum( + cls, + gid, + excl, + tm_dset, + data, + excl_dict=None, + resolution=64, + exclusion_shape=None, + close=True, + ): """ Compute the aggregate (sum) of data for the sc point @@ -697,8 +748,12 @@ def sc_sum(cls, gid, excl, tm_dset, data, excl_dict=None, resolution=64, ndarray Sum / aggregation of data for supply curve point """ - kwargs = {"excl_dict": excl_dict, "resolution": resolution, - "exclusion_shape": exclusion_shape, "close": close} + kwargs = { + "excl_dict": excl_dict, + "resolution": resolution, + "exclusion_shape": exclusion_shape, + "close": close, + } with cls(gid, excl, tm_dset, **kwargs) as point: agg = point.aggregate(data) @@ -745,8 +800,10 @@ def _categorize(data, incl_mult): total inclusions """ - data = {category: float(incl_mult[(data == category)].sum()) - for category in np.unique(data)} + data = { + category: float(incl_mult[(data == category)].sum()) + for category in np.unique(data) + } data = jsonify_dict(data) return data @@ -772,18 +829,22 @@ def _agg_data_layer_method(cls, data, incl_mult, method): data : float | int | str | None Result of applying method to data. """ - method_func = {'mode': cls._mode, - 'mean': np.mean, - 'max': np.max, - 'min': np.min, - 'sum': np.sum, - 'category': cls._categorize} + method_func = { + "mode": cls._mode, + "mean": np.mean, + "max": np.max, + "min": np.min, + "sum": np.sum, + "category": cls._categorize, + } if data is not None: method = method.lower() if method not in method_func: - e = ('Cannot recognize data layer agg method: ' - '"{}". Can only {}'.format(method, list(method_func))) + e = ( + "Cannot recognize data layer agg method: " + '"{}". Can only {}'.format(method, list(method_func)) + ) logger.error(e) raise ValueError(e) @@ -791,14 +852,16 @@ def _agg_data_layer_method(cls, data, incl_mult, method): data = data.flatten() if data.shape != incl_mult.shape: - e = ('Cannot aggregate data with shape that doesnt ' - 'match excl mult!') + e = ( + "Cannot aggregate data with shape that doesnt " + "match excl mult!" + ) logger.error(e) raise DataShapeError(e) - if method == 'category': - data = method_func['category'](data, incl_mult) - elif method in ['mean', 'sum']: + if method == "category": + data = method_func["category"](data, incl_mult) + elif method in ["mean", "sum"]: data = data * incl_mult data = method_func[method](data) else: @@ -830,32 +893,36 @@ def agg_data_layers(self, summary, data_layers): if data_layers is not None: for name, attrs in data_layers.items(): - excl_fp = attrs.get('fpath', self._excl_fpath) + excl_fp = attrs.get("fpath", self._excl_fpath) if excl_fp != self._excl_fpath: - fh = ExclusionLayers(attrs['fpath']) + fh = ExclusionLayers(attrs["fpath"]) else: fh = self.exclusions.excl_h5 - raw = fh[attrs['dset'], self.rows, self.cols] - nodata = fh.get_nodata_value(attrs['dset']) + raw = fh[attrs["dset"], self.rows, self.cols] + nodata = fh.get_nodata_value(attrs["dset"]) data = raw.flatten()[self.bool_mask] incl_mult = self.include_mask_flat[self.bool_mask].copy() if nodata is not None: - valid_data_mask = (data != nodata) + valid_data_mask = data != nodata data = data[valid_data_mask] incl_mult = incl_mult[valid_data_mask] if not data.size: - m = ('Data layer "{}" has no valid data for ' - 'SC point gid {} because of exclusions ' - 'and/or nodata values in the data layer.' - .format(name, self._gid)) + m = ( + 'Data layer "{}" has no valid data for ' + "SC point gid {} because of exclusions " + "and/or nodata values in the data layer.".format( + name, self._gid + ) + ) logger.debug(m) - data = self._agg_data_layer_method(data, incl_mult, - attrs['method']) + data = self._agg_data_layer_method( + data, incl_mult, attrs["method"] + ) summary[name] = data if excl_fp != self._excl_fpath: @@ -867,10 +934,21 @@ def agg_data_layers(self, summary, data_layers): class AggregationSupplyCurvePoint(SupplyCurvePoint): """Generic single SC point to aggregate data from an h5 file.""" - def __init__(self, gid, excl, agg_h5, tm_dset, - excl_dict=None, inclusion_mask=None, - resolution=64, excl_area=None, exclusion_shape=None, - close=True, gen_index=None, apply_exclusions=True): + def __init__( + self, + gid, + excl, + agg_h5, + tm_dset, + excl_dict=None, + inclusion_mask=None, + resolution=64, + excl_area=None, + exclusion_shape=None, + close=True, + gen_index=None, + apply_exclusions=True, + ): """ Parameters ---------- @@ -913,13 +991,17 @@ def __init__(self, gid, excl, agg_h5, tm_dset, Flag to apply exclusions to the resource / generation gid's on initialization. """ - super().__init__(gid, excl, tm_dset, - excl_dict=excl_dict, - inclusion_mask=inclusion_mask, - resolution=resolution, - excl_area=excl_area, - exclusion_shape=exclusion_shape, - close=close) + super().__init__( + gid, + excl, + tm_dset, + excl_dict=excl_dict, + inclusion_mask=inclusion_mask, + resolution=resolution, + excl_area=excl_area, + exclusion_shape=exclusion_shape, + close=close, + ) self._h5_fpath, self._h5 = self._parse_h5_file(agg_h5) @@ -929,9 +1011,12 @@ def __init__(self, gid, excl, agg_h5, tm_dset, self._h5_gids = self._gids if (self._h5_gids != -1).sum() == 0: - emsg = ('Supply curve point gid {} has no viable exclusion ' - 'points based on exclusions file: "{}"' - .format(self._gid, self._excl_fpath)) + emsg = ( + "Supply curve point gid {} has no viable exclusion " + 'points based on exclusions file: "{}"'.format( + self._gid, self._excl_fpath + ) + ) raise EmptySupplyCurvePointError(emsg) if apply_exclusions: @@ -964,11 +1049,12 @@ def _parse_h5_file(h5): elif issubclass(h5.__class__, MultiTimeResource): h5_fpath = h5.h5_files else: - raise SupplyCurveInputError('SupplyCurvePoints needs a ' - '.h5 file path, or ' - 'Resource handler but ' - 'received: {}' - .format(type(h5))) + raise SupplyCurveInputError( + "SupplyCurvePoints needs a " + ".h5 file path, or " + "Resource handler but " + "received: {}".format(type(h5)) + ) return h5_fpath, h5 @@ -984,8 +1070,9 @@ def _apply_exclusions(self): self._h5_gids[exclude] = -1 if (self._gids != -1).sum() == 0: - msg = ('Supply curve point gid {} is completely excluded!' - .format(self._gid)) + msg = "Supply curve point gid {} is completely excluded!".format( + self._gid + ) raise EmptySupplyCurvePointError(msg) def close(self): @@ -1034,7 +1121,7 @@ def h5(self): _h5 : Resource Resource h5 handler object. """ - if self._h5 is None and '*' in self._h5_fpath: + if self._h5 is None and "*" in self._h5_fpath: self._h5 = MultiTimeResource(self._h5_fpath) elif self._h5 is None: self._h5 = Resource(self._h5_fpath) @@ -1045,15 +1132,15 @@ def h5(self): def country(self): """Get the SC point country based on the resource meta data.""" country = None - if 'country' in self.h5.meta and self.county is not None: + if "country" in self.h5.meta and self.county is not None: # make sure country and county are coincident - counties = self.h5.meta.loc[self.h5_gid_set, 'county'].values + counties = self.h5.meta.loc[self.h5_gid_set, "county"].values iloc = np.where(counties == self.county)[0][0] - country = self.h5.meta.loc[self.h5_gid_set, 'country'].values + country = self.h5.meta.loc[self.h5_gid_set, "country"].values country = country[iloc] - elif 'country' in self.h5.meta: - country = self.h5.meta.loc[self.h5_gid_set, 'country'].mode() + elif "country" in self.h5.meta: + country = self.h5.meta.loc[self.h5_gid_set, "country"].mode() country = country.values[0] return country @@ -1062,15 +1149,15 @@ def country(self): def state(self): """Get the SC point state based on the resource meta data.""" state = None - if 'state' in self.h5.meta and self.county is not None: + if "state" in self.h5.meta and self.county is not None: # make sure state and county are coincident - counties = self.h5.meta.loc[self.h5_gid_set, 'county'].values + counties = self.h5.meta.loc[self.h5_gid_set, "county"].values iloc = np.where(counties == self.county)[0][0] - state = self.h5.meta.loc[self.h5_gid_set, 'state'].values + state = self.h5.meta.loc[self.h5_gid_set, "state"].values state = state[iloc] - elif 'state' in self.h5.meta: - state = self.h5.meta.loc[self.h5_gid_set, 'state'].mode() + elif "state" in self.h5.meta: + state = self.h5.meta.loc[self.h5_gid_set, "state"].mode() state = state.values[0] return state @@ -1079,8 +1166,8 @@ def state(self): def county(self): """Get the SC point county based on the resource meta data.""" county = None - if 'county' in self.h5.meta: - county = self.h5.meta.loc[self.h5_gid_set, 'county'].mode() + if "county" in self.h5.meta: + county = self.h5.meta.loc[self.h5_gid_set, "county"].mode() county = county.values[0] return county @@ -1090,7 +1177,9 @@ def elevation(self): """Get the SC point elevation based on the resource meta data.""" elevation = None if MetaKeyName.ELEVATION in self.h5.meta: - elevation = self.h5.meta.loc[self.h5_gid_set, MetaKeyName.ELEVATION].mean() + elevation = self.h5.meta.loc[ + self.h5_gid_set, MetaKeyName.ELEVATION + ].mean() return elevation @@ -1100,13 +1189,17 @@ def timezone(self): timezone = None if MetaKeyName.TIMEZONE in self.h5.meta and self.county is not None: # make sure timezone flag and county are coincident - counties = self.h5.meta.loc[self.h5_gid_set, 'county'].values + counties = self.h5.meta.loc[self.h5_gid_set, "county"].values iloc = np.where(counties == self.county)[0][0] - timezone = self.h5.meta.loc[self.h5_gid_set, MetaKeyName.TIMEZONE].values + timezone = self.h5.meta.loc[ + self.h5_gid_set, MetaKeyName.TIMEZONE + ].values timezone = timezone[iloc] elif MetaKeyName.TIMEZONE in self.h5.meta: - timezone = self.h5.meta.loc[self.h5_gid_set, MetaKeyName.TIMEZONE].mode() + timezone = self.h5.meta.loc[ + self.h5_gid_set, MetaKeyName.TIMEZONE + ].mode() timezone = timezone.values[0] return timezone @@ -1118,13 +1211,17 @@ def offshore(self): offshore = None if MetaKeyName.OFFSHORE in self.h5.meta and self.county is not None: # make sure offshore flag and county are coincident - counties = self.h5.meta.loc[self.h5_gid_set, 'county'].values + counties = self.h5.meta.loc[self.h5_gid_set, "county"].values iloc = np.where(counties == self.county)[0][0] - offshore = self.h5.meta.loc[self.h5_gid_set, MetaKeyName.OFFSHORE].values + offshore = self.h5.meta.loc[ + self.h5_gid_set, MetaKeyName.OFFSHORE + ].values offshore = offshore[iloc] elif MetaKeyName.OFFSHORE in self.h5.meta: - offshore = self.h5.meta.loc[self.h5_gid_set, MetaKeyName.OFFSHORE].mode() + offshore = self.h5.meta.loc[ + self.h5_gid_set, MetaKeyName.OFFSHORE + ].mode() offshore = offshore.values[0] return offshore @@ -1140,8 +1237,10 @@ def gid_counts(self): ------- gid_counts : list """ - gid_counts = [self.include_mask_flat[(self._h5_gids == gid)].sum() - for gid in self.h5_gid_set] + gid_counts = [ + self.include_mask_flat[(self._h5_gids == gid)].sum() + for gid in self.h5_gid_set + ] return gid_counts @@ -1155,28 +1254,41 @@ def summary(self): pandas.Series List of supply curve point's meta data """ - meta = {MetaKeyName.SC_POINT_GID: self.sc_point_gid, - MetaKeyName.SOURCE_GIDS: self.h5_gid_set, - MetaKeyName.GID_COUNTS: self.gid_counts, - MetaKeyName.N_GIDS: self.n_gids, - MetaKeyName.AREA_SQ_KM: self.area, - MetaKeyName.LATITUDE: self.latitude, - MetaKeyName.LONGITUDE: self.longitude, - 'country': self.country, - 'state': self.state, - 'county': self.county, - MetaKeyName.ELEVATION: self.elevation, - MetaKeyName.TIMEZONE: self.timezone, - } + meta = { + MetaKeyName.SC_POINT_GID: self.sc_point_gid, + MetaKeyName.SOURCE_GIDS: self.h5_gid_set, + MetaKeyName.GID_COUNTS: self.gid_counts, + MetaKeyName.N_GIDS: self.n_gids, + MetaKeyName.AREA_SQ_KM: self.area, + MetaKeyName.LATITUDE: self.latitude, + MetaKeyName.LONGITUDE: self.longitude, + "country": self.country, + "state": self.state, + "county": self.county, + MetaKeyName.ELEVATION: self.elevation, + MetaKeyName.TIMEZONE: self.timezone, + } meta = pd.Series(meta) return meta @classmethod - def run(cls, gid, excl, agg_h5, tm_dset, *agg_dset, agg_method='mean', - excl_dict=None, inclusion_mask=None, - resolution=64, excl_area=None, - exclusion_shape=None, close=True, gen_index=None): + def run( + cls, + gid, + excl, + agg_h5, + tm_dset, + *agg_dset, + agg_method="mean", + excl_dict=None, + inclusion_mask=None, + resolution=64, + excl_area=None, + exclusion_shape=None, + close=True, + gen_index=None, + ): """ Compute exclusions weight mean for the sc point from data @@ -1230,30 +1342,34 @@ def run(cls, gid, excl, agg_h5, tm_dset, *agg_dset, agg_method='mean', Given datasets and meta data aggregated to supply curve points """ if isinstance(agg_dset, str): - agg_dset = (agg_dset, ) - - kwargs = {"excl_dict": excl_dict, - "inclusion_mask": inclusion_mask, - "resolution": resolution, - "excl_area": excl_area, - "exclusion_shape": exclusion_shape, - "close": close, - "gen_index": gen_index} + agg_dset = (agg_dset,) + + kwargs = { + "excl_dict": excl_dict, + "inclusion_mask": inclusion_mask, + "resolution": resolution, + "excl_area": excl_area, + "exclusion_shape": exclusion_shape, + "close": close, + "gen_index": gen_index, + } with cls(gid, excl, agg_h5, tm_dset, **kwargs) as point: - if agg_method.lower().startswith('mean'): + if agg_method.lower().startswith("mean"): agg_method = point.exclusion_weighted_mean - elif agg_method.lower().startswith(('sum', 'agg')): + elif agg_method.lower().startswith(("sum", "agg")): agg_method = point.aggregate - elif 'wind_dir' in agg_method.lower(): + elif "wind_dir" in agg_method.lower(): agg_method = point.mean_wind_dirs else: - msg = ('Aggregation method must be either mean, ' - 'sum/aggregate, or wind_dir') + msg = ( + "Aggregation method must be either mean, " + "sum/aggregate, or wind_dir" + ) logger.error(msg) raise ValueError(msg) - out = {'meta': point.summary} + out = {"meta": point.summary} for dset in agg_dset: ds = point.h5.open_dataset(dset) @@ -1267,15 +1383,31 @@ class GenerationSupplyCurvePoint(AggregationSupplyCurvePoint): respective generation and resource data.""" # technology-dependent power density estimates in MW/km2 - POWER_DENSITY = {'pv': 36, 'wind': 3} - - def __init__(self, gid, excl, gen, tm_dset, gen_index, - excl_dict=None, inclusion_mask=None, - res_class_dset=None, res_class_bin=None, excl_area=None, - power_density=None, cf_dset='cf_mean-means', - lcoe_dset='lcoe_fcr-means', h5_dsets=None, resolution=64, - exclusion_shape=None, close=False, friction_layer=None, - recalc_lcoe=True, apply_exclusions=True): + POWER_DENSITY = {"pv": 36, "wind": 3} + + def __init__( + self, + gid, + excl, + gen, + tm_dset, + gen_index, + excl_dict=None, + inclusion_mask=None, + res_class_dset=None, + res_class_bin=None, + excl_area=None, + power_density=None, + cf_dset="cf_mean-means", + lcoe_dset="lcoe_fcr-means", + h5_dsets=None, + resolution=64, + exclusion_shape=None, + close=False, + friction_layer=None, + recalc_lcoe=True, + apply_exclusions=True, + ): """ Parameters ---------- @@ -1362,26 +1494,36 @@ def __init__(self, gid, excl, gen, tm_dset, gen_index, self._friction_layer = friction_layer self._recalc_lcoe = recalc_lcoe - super().__init__(gid, excl, gen, tm_dset, - excl_dict=excl_dict, - inclusion_mask=inclusion_mask, - resolution=resolution, - excl_area=excl_area, - exclusion_shape=exclusion_shape, - close=close, apply_exclusions=False) + super().__init__( + gid, + excl, + gen, + tm_dset, + excl_dict=excl_dict, + inclusion_mask=inclusion_mask, + resolution=resolution, + excl_area=excl_area, + exclusion_shape=exclusion_shape, + close=close, + apply_exclusions=False, + ) self._res_gid_set = None self._gen_gid_set = None self._gen_fpath, self._gen = self._h5_fpath, self._h5 - self._gen_gids, self._res_gids = self._map_gen_gids(self._gids, - gen_index) + self._gen_gids, self._res_gids = self._map_gen_gids( + self._gids, gen_index + ) self._gids = self._gen_gids if (self._gen_gids != -1).sum() == 0: - emsg = ('Supply curve point gid {} has no viable exclusion ' - 'points based on exclusions file: "{}"' - .format(self._gid, self._excl_fpath)) + emsg = ( + "Supply curve point gid {} has no viable exclusion " + 'points based on exclusions file: "{}"'.format( + self._gid, self._excl_fpath + ) + ) raise EmptySupplyCurvePointError(emsg) if apply_exclusions: @@ -1403,7 +1545,7 @@ def exclusion_weighted_mean(self, flat_arr): Mean of flat_arr masked by the binary exclusions then weighted by the non-zero exclusions. """ - x = flat_arr[self._gen_gids[self.bool_mask]].astype('float32') + x = flat_arr[self._gen_gids[self.bool_mask]].astype("float32") incl = self.include_mask_flat[self.bool_mask] x *= incl mean = x.sum() / incl.sum() @@ -1478,8 +1620,10 @@ def gid_counts(self): gid_counts : list List of exclusion pixels in each resource/generation gid. """ - gid_counts = [self.include_mask_flat[(self._res_gids == gid)].sum() - for gid in self.res_gid_set] + gid_counts = [ + self.include_mask_flat[(self._res_gids == gid)].sum() + for gid in self.res_gid_set + ] return gid_counts @@ -1577,22 +1721,30 @@ def mean_lcoe(self): # year CF, but the output should be identical to the original LCOE and # so is not consequential). if self._recalc_lcoe: - required = ('fixed_charge_rate', 'capital_cost', - 'fixed_operating_cost', 'variable_operating_cost', - 'system_capacity') + required = ( + "fixed_charge_rate", + "capital_cost", + "fixed_operating_cost", + "variable_operating_cost", + "system_capacity", + ) if self.mean_h5_dsets_data is not None: if all(k in self.mean_h5_dsets_data for k in required): - aep = (self.mean_h5_dsets_data['system_capacity'] - * self.mean_cf * 8760) + aep = ( + self.mean_h5_dsets_data["system_capacity"] + * self.mean_cf + * 8760 + ) # Note the AEP computation uses the SAM config # `system_capacity`, so no need to scale `capital_cost` # or `fixed_operating_cost` by anything mean_lcoe = lcoe_fcr( - self.mean_h5_dsets_data['fixed_charge_rate'], - self.mean_h5_dsets_data['capital_cost'], - self.mean_h5_dsets_data['fixed_operating_cost'], + self.mean_h5_dsets_data["fixed_charge_rate"], + self.mean_h5_dsets_data["capital_cost"], + self.mean_h5_dsets_data["fixed_operating_cost"], aep, - self.mean_h5_dsets_data['variable_operating_cost']) + self.mean_h5_dsets_data["variable_operating_cost"], + ) # alternative if lcoe was not able to be re-calculated from # multi year mean CF @@ -1678,26 +1830,31 @@ def power_density(self): """ if self._power_density is None: - tech = self.gen.meta['reV_tech'][0] + tech = self.gen.meta["reV_tech"][0] if tech in self.POWER_DENSITY: self._power_density = self.POWER_DENSITY[tech] else: - warn('Could not recognize reV technology in generation meta ' - 'data: "{}". Cannot lookup an appropriate power density ' - 'to calculate SC point capacity.'.format(tech)) + warn( + "Could not recognize reV technology in generation meta " + 'data: "{}". Cannot lookup an appropriate power density ' + "to calculate SC point capacity.".format(tech) + ) elif isinstance(self._power_density, pd.DataFrame): self._pd_obj = self._power_density missing = set(self.res_gid_set) - set(self._pd_obj.index.values) if any(missing): - msg = ('Variable power density input is missing the ' - 'following resource GIDs: {}'.format(missing)) + msg = ( + "Variable power density input is missing the " + "following resource GIDs: {}".format(missing) + ) logger.error(msg) raise FileInputError(msg) - pds = self._pd_obj.loc[self._res_gids[self.bool_mask], - 'power_density'].values + pds = self._pd_obj.loc[ + self._res_gids[self.bool_mask], "power_density" + ].values pds = pds.astype(np.float32) pds *= self.include_mask_flat[self.bool_mask] denom = self.include_mask_flat[self.bool_mask].sum() @@ -1723,18 +1880,20 @@ def power_density_ac(self): return None ilr = self.gen["dc_ac_ratio", self._gen_gids[self.bool_mask]] - ilr = ilr.astype('float32') + ilr = ilr.astype("float32") weights = self.include_mask_flat[self.bool_mask] if self._power_density_ac is None: - tech = self.gen.meta['reV_tech'][0] + tech = self.gen.meta["reV_tech"][0] if tech in self.POWER_DENSITY: power_density_ac = self.POWER_DENSITY[tech] / ilr power_density_ac *= weights power_density_ac = power_density_ac.sum() / weights.sum() else: - warn('Could not recognize reV technology in generation meta ' - 'data: "{}". Cannot lookup an appropriate power density ' - 'to calculate SC point capacity.'.format(tech)) + warn( + "Could not recognize reV technology in generation meta " + 'data: "{}". Cannot lookup an appropriate power density ' + "to calculate SC point capacity.".format(tech) + ) power_density_ac = None elif isinstance(self._power_density_ac, pd.DataFrame): @@ -1742,13 +1901,16 @@ def power_density_ac(self): missing = set(self.res_gid_set) - set(self._pd_obj.index.values) if any(missing): - msg = ('Variable power density input is missing the ' - 'following resource GIDs: {}'.format(missing)) + msg = ( + "Variable power density input is missing the " + "following resource GIDs: {}".format(missing) + ) logger.error(msg) raise FileInputError(msg) - pds = self._pd_obj.loc[self._res_gids[self.bool_mask], - 'power_density'].values + pds = self._pd_obj.loc[ + self._res_gids[self.bool_mask], "power_density" + ].values power_density_ac = pds.astype(np.float32) / ilr power_density_ac *= weights power_density_ac = power_density_ac.sum() / weights.sum() @@ -1815,12 +1977,14 @@ def sc_point_capital_cost(self): if self.mean_h5_dsets_data is None: return None - required = ('capital_cost', 'system_capacity') + required = ("capital_cost", "system_capacity") if not all(k in self.mean_h5_dsets_data for k in required): return None - cap_cost_per_mw = (self.mean_h5_dsets_data['capital_cost'] - / self.mean_h5_dsets_data['system_capacity']) + cap_cost_per_mw = ( + self.mean_h5_dsets_data["capital_cost"] + / self.mean_h5_dsets_data["system_capacity"] + ) return cap_cost_per_mw * self.capacity @property @@ -1841,12 +2005,14 @@ def sc_point_fixed_operating_cost(self): if self.mean_h5_dsets_data is None: return None - required = ('fixed_operating_cost', 'system_capacity') + required = ("fixed_operating_cost", "system_capacity") if not all(k in self.mean_h5_dsets_data for k in required): return None - fixed_cost_per_mw = (self.mean_h5_dsets_data['fixed_operating_cost'] - / self.mean_h5_dsets_data['system_capacity']) + fixed_cost_per_mw = ( + self.mean_h5_dsets_data["fixed_operating_cost"] + / self.mean_h5_dsets_data["system_capacity"] + ) return fixed_cost_per_mw * self.capacity @property @@ -1908,10 +2074,13 @@ def h5_dsets_data(self): _h5_dsets_data = self._h5_dsets elif self._h5_dsets is not None: - e = ('Cannot recognize h5_dsets input type, should be None, ' - 'a list of dataset names, or a dictionary or ' - 'pre-extracted data. Received: {} {}' - .format(type(self._h5_dsets), self._h5_dsets)) + e = ( + "Cannot recognize h5_dsets input type, should be None, " + "a list of dataset names, or a dictionary or " + "pre-extracted data. Received: {} {}".format( + type(self._h5_dsets), self._h5_dsets + ) + ) logger.error(e) raise TypeError(e) @@ -1954,8 +2123,10 @@ def _apply_exclusions(self): self._incl_mask = self._incl_mask.flatten() if (self._gen_gids != -1).sum() == 0: - msg = ('Supply curve point gid {} is completely excluded for res ' - 'bin: {}'.format(self._gid, self._res_class_bin)) + msg = ( + "Supply curve point gid {} is completely excluded for res " + "bin: {}".format(self._gid, self._res_class_bin) + ) raise EmptySupplyCurvePointError(msg) def _resource_exclusion(self, boolean_exclude): @@ -1973,14 +2144,16 @@ def _resource_exclusion(self, boolean_exclude): outside of current resource class bin. """ - if (self._res_class_dset is not None - and self._res_class_bin is not None): - + if ( + self._res_class_dset is not None + and self._res_class_bin is not None + ): rex = self.res_data[self._gen_gids] - rex = ((rex < np.min(self._res_class_bin)) - | (rex >= np.max(self._res_class_bin))) + rex = (rex < np.min(self._res_class_bin)) | ( + rex >= np.max(self._res_class_bin) + ) - boolean_exclude = (boolean_exclude | rex) + boolean_exclude = boolean_exclude | rex return boolean_exclude @@ -2000,27 +2173,33 @@ def point_summary(self, args=None): Dictionary of summary outputs for this sc point. """ - ARGS = {MetaKeyName.RES_GIDS: self.res_gid_set, - MetaKeyName.GEN_GIDS: self.gen_gid_set, - MetaKeyName.GID_COUNTS: self.gid_counts, - MetaKeyName.N_GIDS: self.n_gids, - MetaKeyName.MEAN_CF: self.mean_cf, - MetaKeyName.MEAN_LCOE: self.mean_lcoe, - MetaKeyName.MEAN_RES: self.mean_res, - MetaKeyName.CAPACITY: self.capacity, - MetaKeyName.AREA_SQ_KM: self.area, - MetaKeyName.LATITUDE: self.latitude, - MetaKeyName.LONGITUDE: self.longitude, - 'country': self.country, - 'state': self.state, - 'county': self.county, - MetaKeyName.ELEVATION: self.elevation, - MetaKeyName.TIMEZONE: self.timezone, - } - - extra_atts = [MetaKeyName.CAPACITY_AC, MetaKeyName.OFFSHORE, MetaKeyName.SC_POINT_CAPITAL_COST, - MetaKeyName.SC_POINT_FIXED_OPERATING_COST, - MetaKeyName.SC_POINT_ANNUAL_ENERGY, MetaKeyName.SC_POINT_ANNUAL_ENERGY_AC] + ARGS = { + MetaKeyName.RES_GIDS: self.res_gid_set, + MetaKeyName.GEN_GIDS: self.gen_gid_set, + MetaKeyName.GID_COUNTS: self.gid_counts, + MetaKeyName.N_GIDS: self.n_gids, + MetaKeyName.MEAN_CF: self.mean_cf, + MetaKeyName.MEAN_LCOE: self.mean_lcoe, + MetaKeyName.MEAN_RES: self.mean_res, + MetaKeyName.CAPACITY: self.capacity, + MetaKeyName.AREA_SQ_KM: self.area, + MetaKeyName.LATITUDE: self.latitude, + MetaKeyName.LONGITUDE: self.longitude, + "country": self.country, + "state": self.state, + "county": self.county, + MetaKeyName.ELEVATION: self.elevation, + MetaKeyName.TIMEZONE: self.timezone, + } + + extra_atts = [ + MetaKeyName.CAPACITY_AC, + MetaKeyName.OFFSHORE, + MetaKeyName.SC_POINT_CAPITAL_COST, + MetaKeyName.SC_POINT_FIXED_OPERATING_COST, + MetaKeyName.SC_POINT_ANNUAL_ENERGY, + MetaKeyName.SC_POINT_ANNUAL_ENERGY_AC, + ] for attr in extra_atts: value = getattr(self, attr) if value is not None: @@ -2032,7 +2211,7 @@ def point_summary(self, args=None): if self._h5_dsets is not None: for dset, data in self.mean_h5_dsets_data.items(): - ARGS['mean_{}'.format(dset)] = data + ARGS["mean_{}".format(dset)] = data if args is None: args = list(ARGS.keys()) @@ -2042,8 +2221,11 @@ def point_summary(self, args=None): if arg in ARGS: summary[arg] = ARGS[arg] else: - warn('Cannot find "{}" as an available SC self summary ' - 'output', OutputWarning) + warn( + 'Cannot find "{}" as an available SC self summary ' + "output", + OutputWarning, + ) return summary @@ -2074,21 +2256,39 @@ def economies_of_scale(cap_cost_scale, summary): summary[MetaKeyName.CAPITAL_COST_SCALAR] = eos.capital_cost_scalar summary[MetaKeyName.SCALED_CAPITAL_COST] = eos.scaled_capital_cost if "sc_point_capital_cost" in summary: - scaled_costs = (summary["sc_point_capital_cost"] - * eos.capital_cost_scalar) + scaled_costs = ( + summary["sc_point_capital_cost"] * eos.capital_cost_scalar + ) summary[MetaKeyName.SCALED_SC_POINT_CAPITAL_COST] = scaled_costs return summary @classmethod - def summarize(cls, gid, excl_fpath, gen_fpath, tm_dset, gen_index, - excl_dict=None, inclusion_mask=None, - res_class_dset=None, res_class_bin=None, - excl_area=None, power_density=None, - cf_dset='cf_mean-means', lcoe_dset='lcoe_fcr-means', - h5_dsets=None, resolution=64, exclusion_shape=None, - close=False, friction_layer=None, args=None, - data_layers=None, cap_cost_scale=None, recalc_lcoe=True): + def summarize( + cls, + gid, + excl_fpath, + gen_fpath, + tm_dset, + gen_index, + excl_dict=None, + inclusion_mask=None, + res_class_dset=None, + res_class_bin=None, + excl_area=None, + power_density=None, + cf_dset="cf_mean-means", + lcoe_dset="lcoe_fcr-means", + h5_dsets=None, + resolution=64, + exclusion_shape=None, + close=False, + friction_layer=None, + args=None, + data_layers=None, + cap_cost_scale=None, + recalc_lcoe=True, + ): """Get a summary dictionary of a single supply curve point. Parameters @@ -2175,24 +2375,26 @@ def summarize(cls, gid, excl_fpath, gen_fpath, tm_dset, gen_index, summary : dict Dictionary of summary outputs for this sc point. """ - kwargs = {"excl_dict": excl_dict, - "inclusion_mask": inclusion_mask, - "res_class_dset": res_class_dset, - "res_class_bin": res_class_bin, - "excl_area": excl_area, - "power_density": power_density, - "cf_dset": cf_dset, - "lcoe_dset": lcoe_dset, - "h5_dsets": h5_dsets, - "resolution": resolution, - "exclusion_shape": exclusion_shape, - "close": close, - 'friction_layer': friction_layer, - 'recalc_lcoe': recalc_lcoe, - } - - with cls(gid, excl_fpath, gen_fpath, tm_dset, gen_index, - **kwargs) as point: + kwargs = { + "excl_dict": excl_dict, + "inclusion_mask": inclusion_mask, + "res_class_dset": res_class_dset, + "res_class_bin": res_class_bin, + "excl_area": excl_area, + "power_density": power_density, + "cf_dset": cf_dset, + "lcoe_dset": lcoe_dset, + "h5_dsets": h5_dsets, + "resolution": resolution, + "exclusion_shape": exclusion_shape, + "close": close, + "friction_layer": friction_layer, + "recalc_lcoe": recalc_lcoe, + } + + with cls( + gid, excl_fpath, gen_fpath, tm_dset, gen_index, **kwargs + ) as point: summary = point.point_summary(args=args) if data_layers is not None: diff --git a/reV/supply_curve/sc_aggregation.py b/reV/supply_curve/sc_aggregation.py index 1500ffad9..494761c0c 100644 --- a/reV/supply_curve/sc_aggregation.py +++ b/reV/supply_curve/sc_aggregation.py @@ -6,6 +6,7 @@ @author: gbuster """ + import logging import os from concurrent.futures import as_completed @@ -49,10 +50,19 @@ class SupplyCurveAggFileHandler(AbstractAggFileHandler): - variable power density .csv (optional) """ - def __init__(self, excl_fpath, gen_fpath, econ_fpath=None, - data_layers=None, power_density=None, excl_dict=None, - friction_fpath=None, friction_dset=None, - area_filter_kernel='queen', min_area=None): + def __init__( + self, + excl_fpath, + gen_fpath, + econ_fpath=None, + data_layers=None, + power_density=None, + excl_dict=None, + friction_fpath=None, + friction_dset=None, + area_filter_kernel="queen", + min_area=None, + ): """ Parameters ---------- @@ -95,9 +105,12 @@ def __init__(self, excl_fpath, gen_fpath, econ_fpath=None, min_area : float | None Minimum required contiguous area filter in sq-km """ - super().__init__(excl_fpath, excl_dict=excl_dict, - area_filter_kernel=area_filter_kernel, - min_area=min_area) + super().__init__( + excl_fpath, + excl_dict=excl_dict, + area_filter_kernel=area_filter_kernel, + min_area=min_area, + ) self._gen = self._open_gen_econ_resource(gen_fpath, econ_fpath) # pre-initialize the resource meta data @@ -112,7 +125,7 @@ def __init__(self, excl_fpath, gen_fpath, econ_fpath=None, self._friction_layer = FrictionMask(friction_fpath, friction_dset) if not np.all(self._friction_layer.shape == self._excl.shape): - e = ('Friction layer shape {} must match exclusions shape {}!' + e = ("Friction layer shape {} must match exclusions shape {}!" .format(self._friction_layer.shape, self._excl.shape)) logger.error(e) raise FileInputError(e) @@ -138,14 +151,15 @@ def _open_gen_econ_resource(gen_fpath, econ_fpath): """ handler = None - is_gen_h5 = isinstance(gen_fpath, str) and gen_fpath.endswith('.h5') - is_econ_h5 = isinstance(econ_fpath, str) and econ_fpath.endswith('.h5') + is_gen_h5 = isinstance(gen_fpath, str) and gen_fpath.endswith(".h5") + is_econ_h5 = isinstance(econ_fpath, str) and econ_fpath.endswith(".h5") if is_gen_h5 and not is_econ_h5: handler = Resource(gen_fpath) elif is_gen_h5 and is_econ_h5: - handler = MultiFileResource([gen_fpath, econ_fpath], - check_files=True) + handler = MultiFileResource( + [gen_fpath, econ_fpath], check_files=True + ) return handler @@ -155,20 +169,29 @@ def _parse_power_density(self): if isinstance(self._power_density, str): self._pdf = self._power_density - if self._pdf.endswith('.csv'): + if self._pdf.endswith(".csv"): self._power_density = pd.read_csv(self._pdf) - if (MetaKeyName.GID in self._power_density - and 'power_density' in self._power_density): - self._power_density = self._power_density.set_index(MetaKeyName.GID) + if ( + MetaKeyName.GID in self._power_density + and "power_density" in self._power_density + ): + self._power_density = self._power_density.set_index( + MetaKeyName.GID + ) else: - msg = ('Variable power density file must include "gid" ' - 'and "power_density" columns, but received: {}' - .format(self._power_density.columns.values)) + msg = ( + 'Variable power density file must include "gid" ' + 'and "power_density" columns, but received: {}'.format( + self._power_density.columns.values + ) + ) logger.error(msg) raise FileInputError(msg) else: - msg = ('Variable power density file must be csv but received: ' - '{}'.format(self._pdf)) + msg = ( + "Variable power density file must be csv but received: " + "{}".format(self._pdf) + ) logger.error(msg) raise FileInputError(msg) @@ -229,14 +252,31 @@ def friction_layer(self): class SupplyCurveAggregation(BaseAggregation): """SupplyCurveAggregation""" - def __init__(self, excl_fpath, tm_dset, econ_fpath=None, - excl_dict=None, area_filter_kernel='queen', min_area=None, - resolution=64, excl_area=None, res_fpath=None, gids=None, - pre_extract_inclusions=False, res_class_dset=None, - res_class_bins=None, cf_dset='cf_mean-means', - lcoe_dset='lcoe_fcr-means', h5_dsets=None, data_layers=None, - power_density=None, friction_fpath=None, friction_dset=None, - cap_cost_scale=None, recalc_lcoe=True): + def __init__( + self, + excl_fpath, + tm_dset, + econ_fpath=None, + excl_dict=None, + area_filter_kernel="queen", + min_area=None, + resolution=64, + excl_area=None, + res_fpath=None, + gids=None, + pre_extract_inclusions=False, + res_class_dset=None, + res_class_bins=None, + cf_dset="cf_mean-means", + lcoe_dset="lcoe_fcr-means", + h5_dsets=None, + data_layers=None, + power_density=None, + friction_fpath=None, + friction_dset=None, + cap_cost_scale=None, + recalc_lcoe=True, + ): r"""ReV supply curve points aggregation framework. ``reV`` supply curve aggregation combines a high-resolution @@ -663,15 +703,22 @@ def __init__(self, excl_fpath, tm_dset, econ_fpath=None, associated with all pixels with that unique value. """ log_versions(logger) - logger.info('Initializing SupplyCurveAggregation...') - logger.debug('Exclusion filepath: {}'.format(excl_fpath)) - logger.debug('Exclusion dict: {}'.format(excl_dict)) - - super().__init__(excl_fpath, tm_dset, excl_dict=excl_dict, - area_filter_kernel=area_filter_kernel, - min_area=min_area, resolution=resolution, - excl_area=excl_area, res_fpath=res_fpath, gids=gids, - pre_extract_inclusions=pre_extract_inclusions) + logger.info("Initializing SupplyCurveAggregation...") + logger.debug("Exclusion filepath: {}".format(excl_fpath)) + logger.debug("Exclusion dict: {}".format(excl_dict)) + + super().__init__( + excl_fpath, + tm_dset, + excl_dict=excl_dict, + area_filter_kernel=area_filter_kernel, + min_area=min_area, + resolution=resolution, + excl_area=excl_area, + res_fpath=res_fpath, + gids=gids, + pre_extract_inclusions=pre_extract_inclusions, + ) self._econ_fpath = econ_fpath self._res_class_dset = res_class_dset @@ -686,7 +733,7 @@ def __init__(self, excl_fpath, tm_dset, econ_fpath=None, self._data_layers = data_layers self._recalc_lcoe = recalc_lcoe - logger.debug('Resource class bins: {}'.format(self._res_class_bins)) + logger.debug("Resource class bins: {}".format(self._res_class_bins)) if self._cap_cost_scale is not None: if self._h5_dsets is None: @@ -696,16 +743,20 @@ def __init__(self, excl_fpath, tm_dset, econ_fpath=None, self._h5_dsets = list(set(self._h5_dsets)) if self._power_density is None: - msg = ('Supply curve aggregation power density not specified. ' - 'Will try to infer based on lookup table: {}' - .format(GenerationSupplyCurvePoint.POWER_DENSITY)) + msg = ( + "Supply curve aggregation power density not specified. " + "Will try to infer based on lookup table: {}".format( + GenerationSupplyCurvePoint.POWER_DENSITY + ) + ) logger.warning(msg) warn(msg, InputWarning) self._check_data_layers() - def _check_data_layers(self, methods=('mean', 'max', 'min', - 'mode', 'sum', 'category')): + def _check_data_layers( + self, methods=("mean", "max", "min", "mode", "sum", "category") + ): """Run pre-flight checks on requested aggregation data layers. Parameters @@ -715,40 +766,49 @@ def _check_data_layers(self, methods=('mean', 'max', 'min', """ if self._data_layers is not None: - logger.debug('Checking data layers...') + logger.debug("Checking data layers...") with ExclusionLayers(self._excl_fpath) as f: shape_base = f.shape for k, v in self._data_layers.items(): - if 'dset' not in v: - raise KeyError('Data aggregation "dset" data layer "{}" ' - 'must be specified.'.format(k)) - if 'method' not in v: - raise KeyError('Data aggregation "method" data layer "{}" ' - 'must be specified.'.format(k)) - if v['method'].lower() not in methods: - raise ValueError('Cannot recognize data layer agg method: ' - '"{}". Can only do: {}.' - .format(v['method'], methods)) - if 'fpath' in v: - with ExclusionLayers(v['fpath']) as f: + if "dset" not in v: + raise KeyError( + 'Data aggregation "dset" data layer "{}" ' + "must be specified.".format(k) + ) + if "method" not in v: + raise KeyError( + 'Data aggregation "method" data layer "{}" ' + "must be specified.".format(k) + ) + if v["method"].lower() not in methods: + raise ValueError( + "Cannot recognize data layer agg method: " + '"{}". Can only do: {}.'.format(v["method"], methods) + ) + if "fpath" in v: + with ExclusionLayers(v["fpath"]) as f: try: mismatched_shapes = any(f.shape != shape_base) except TypeError: mismatched_shapes = f.shape != shape_base if mismatched_shapes: - msg = ('Data shape of data layer "{}" is {}, ' - 'which does not match the baseline ' - 'exclusions shape {}.' - .format(k, f.shape, shape_base)) + msg = ( + 'Data shape of data layer "{}" is {}, ' + "which does not match the baseline " + "exclusions shape {}.".format( + k, f.shape, shape_base + ) + ) raise FileInputError(msg) - logger.debug('Finished checking data layers.') + logger.debug("Finished checking data layers.") @staticmethod - def _get_res_gen_lcoe_data(gen, res_class_dset, res_class_bins, - cf_dset, lcoe_dset): + def _get_res_gen_lcoe_data( + gen, res_class_dset, res_class_bins, cf_dset, lcoe_dset + ): """Extract the basic resource / generation / lcoe data to be used in the aggregation process. @@ -782,7 +842,7 @@ def _get_res_gen_lcoe_data(gen, res_class_dset, res_class_bins, dset_list = (res_class_dset, cf_dset, lcoe_dset) gen_dsets = [] if gen is None else gen.datasets - labels = ('res_class_dset', 'cf_dset', 'lcoe_dset') + labels = ("res_class_dset", "cf_dset", "lcoe_dset") temp = [None, None, None] if isinstance(gen, Resource): @@ -790,8 +850,9 @@ def _get_res_gen_lcoe_data(gen, res_class_dset, res_class_bins, elif isinstance(gen, MultiFileResource): source_fps = gen._h5_files else: - msg = ('Did not recognize gen object input of type "{}": {}' - .format(type(gen), gen)) + msg = 'Did not recognize gen object input of type "{}": {}'.format( + type(gen), gen + ) logger.error(msg) raise TypeError(msg) @@ -800,9 +861,12 @@ def _get_res_gen_lcoe_data(gen, res_class_dset, res_class_bins, _warn_about_large_datasets(gen, dset) temp[i] = gen[dset] elif dset not in gen_dsets and dset is not None: - w = ('Could not find "{}" input as "{}" in source files: {}. ' - 'Available datasets: {}' - .format(labels[i], dset, source_fps, gen_dsets)) + w = ( + 'Could not find "{}" input as "{}" in source files: {}. ' + "Available datasets: {}".format( + labels[i], dset, source_fps, gen_dsets + ) + ) logger.warning(w) warn(w, OutputWarning) @@ -838,11 +902,16 @@ def _get_extra_dsets(gen, h5_dsets): # look for the datasets required by the LCOE re-calculation and make # lists of the missing datasets gen_dsets = [] if gen is None else gen.datasets - lcoe_recalc_req = ('fixed_charge_rate', 'capital_cost', - 'fixed_operating_cost', 'variable_operating_cost', - 'system_capacity') - missing_lcoe_source = [k for k in lcoe_recalc_req - if k not in gen_dsets] + lcoe_recalc_req = ( + "fixed_charge_rate", + "capital_cost", + "fixed_operating_cost", + "variable_operating_cost", + "system_capacity", + ) + missing_lcoe_source = [ + k for k in lcoe_recalc_req if k not in gen_dsets + ] missing_lcoe_request = [] if isinstance(gen, Resource): @@ -850,61 +919,91 @@ def _get_extra_dsets(gen, h5_dsets): elif isinstance(gen, MultiFileResource): source_fps = gen._h5_files else: - msg = ('Did not recognize gen object input of type "{}": {}' - .format(type(gen), gen)) + msg = 'Did not recognize gen object input of type "{}": {}'.format( + type(gen), gen + ) logger.error(msg) raise TypeError(msg) h5_dsets_data = None if h5_dsets is not None: - missing_lcoe_request = [k for k in lcoe_recalc_req - if k not in h5_dsets] + missing_lcoe_request = [ + k for k in lcoe_recalc_req if k not in h5_dsets + ] if not isinstance(h5_dsets, (list, tuple)): - e = ('Additional h5_dsets argument must be a list or tuple ' - 'but received: {} {}'.format(type(h5_dsets), h5_dsets)) + e = ( + "Additional h5_dsets argument must be a list or tuple " + "but received: {} {}".format(type(h5_dsets), h5_dsets) + ) logger.error(e) raise TypeError(e) missing_h5_dsets = [k for k in h5_dsets if k not in gen_dsets] if any(missing_h5_dsets): - msg = ('Could not find requested h5_dsets "{}" in ' - 'source files: {}. Available datasets: {}' - .format(missing_h5_dsets, source_fps, gen_dsets)) + msg = ( + 'Could not find requested h5_dsets "{}" in ' + "source files: {}. Available datasets: {}".format( + missing_h5_dsets, source_fps, gen_dsets + ) + ) logger.error(msg) raise FileInputError(msg) h5_dsets_data = {dset: gen[dset] for dset in h5_dsets} if any(missing_lcoe_source): - msg = ('Could not find the datasets in the gen source file that ' - 'are required to re-calculate the multi-year LCOE. If you ' - 'are running a multi-year job, it is strongly suggested ' - 'you pass through these datasets to re-calculate the LCOE ' - 'from the multi-year mean CF: {}' - .format(missing_lcoe_source)) + msg = ( + "Could not find the datasets in the gen source file that " + "are required to re-calculate the multi-year LCOE. If you " + "are running a multi-year job, it is strongly suggested " + "you pass through these datasets to re-calculate the LCOE " + "from the multi-year mean CF: {}".format(missing_lcoe_source) + ) logger.warning(msg) warn(msg, InputWarning) if any(missing_lcoe_request): - msg = ('It is strongly advised that you include the following ' - 'datasets in the h5_dsets request in order to re-calculate ' - 'the LCOE from the multi-year mean CF and AEP: {}' - .format(missing_lcoe_request)) + msg = ( + "It is strongly advised that you include the following " + "datasets in the h5_dsets request in order to re-calculate " + "the LCOE from the multi-year mean CF and AEP: {}".format( + missing_lcoe_request + ) + ) logger.warning(msg) warn(msg, InputWarning) return h5_dsets_data @classmethod - def run_serial(cls, excl_fpath, gen_fpath, tm_dset, gen_index, - econ_fpath=None, excl_dict=None, inclusion_mask=None, - area_filter_kernel='queen', min_area=None, - resolution=64, gids=None, args=None, res_class_dset=None, - res_class_bins=None, cf_dset='cf_mean-means', - lcoe_dset='lcoe_fcr-means', h5_dsets=None, data_layers=None, - power_density=None, friction_fpath=None, friction_dset=None, - excl_area=None, cap_cost_scale=None, recalc_lcoe=True): + def run_serial( + cls, + excl_fpath, + gen_fpath, + tm_dset, + gen_index, + econ_fpath=None, + excl_dict=None, + inclusion_mask=None, + area_filter_kernel="queen", + min_area=None, + resolution=64, + gids=None, + args=None, + res_class_dset=None, + res_class_bins=None, + cf_dset="cf_mean-means", + lcoe_dset="lcoe_fcr-means", + h5_dsets=None, + data_layers=None, + power_density=None, + friction_fpath=None, + friction_dset=None, + excl_area=None, + cap_cost_scale=None, + recalc_lcoe=True, + ): """Standalone method to create agg summary - can be parallelized. Parameters @@ -1015,34 +1114,38 @@ def run_serial(cls, excl_fpath, gen_fpath, tm_dset, gen_index, slice_lookup = sc.get_slice_lookup(gids) - logger.debug('Starting SupplyCurveAggregation serial with ' - 'supply curve {} gids'.format(len(gids))) + logger.debug( + "Starting SupplyCurveAggregation serial with " + "supply curve {} gids".format(len(gids)) + ) cls._check_inclusion_mask(inclusion_mask, gids, exclusion_shape) # pre-extract handlers so they are not repeatedly initialized - file_kwargs = {'econ_fpath': econ_fpath, - 'data_layers': data_layers, - 'power_density': power_density, - 'excl_dict': excl_dict, - 'area_filter_kernel': area_filter_kernel, - 'min_area': min_area, - 'friction_fpath': friction_fpath, - 'friction_dset': friction_dset} - with SupplyCurveAggFileHandler(excl_fpath, gen_fpath, - **file_kwargs) as fh: - - temp = cls._get_res_gen_lcoe_data(fh.gen, res_class_dset, - res_class_bins, cf_dset, - lcoe_dset) + file_kwargs = { + "econ_fpath": econ_fpath, + "data_layers": data_layers, + "power_density": power_density, + "excl_dict": excl_dict, + "area_filter_kernel": area_filter_kernel, + "min_area": min_area, + "friction_fpath": friction_fpath, + "friction_dset": friction_dset, + } + with SupplyCurveAggFileHandler( + excl_fpath, gen_fpath, **file_kwargs + ) as fh: + temp = cls._get_res_gen_lcoe_data( + fh.gen, res_class_dset, res_class_bins, cf_dset, lcoe_dset + ) res_data, res_class_bins, cf_data, lcoe_data = temp h5_dsets_data = cls._get_extra_dsets(fh.gen, h5_dsets) n_finished = 0 for gid in gids: gid_inclusions = cls._get_gid_inclusion_mask( - inclusion_mask, gid, slice_lookup, - resolution=resolution) + inclusion_mask, gid, slice_lookup, resolution=resolution + ) for ri, res_bin in enumerate(res_class_bins): try: @@ -1068,27 +1171,36 @@ def run_serial(cls, excl_fpath, gen_fpath, tm_dset, gen_index, close=False, friction_layer=fh.friction_layer, cap_cost_scale=cap_cost_scale, - recalc_lcoe=recalc_lcoe) + recalc_lcoe=recalc_lcoe, + ) except EmptySupplyCurvePointError: - logger.debug('SC point {} is empty'.format(gid)) + logger.debug("SC point {} is empty".format(gid)) else: pointsum[MetaKeyName.SC_POINT_GID] = gid - pointsum[MetaKeyName.SC_ROW_IND] = points.loc[gid, 'row_ind'] - pointsum[MetaKeyName.SC_COL_IND] = points.loc[gid, 'col_ind'] - pointsum['res_class'] = ri + pointsum[MetaKeyName.SC_ROW_IND] = points.loc[ + gid, "row_ind" + ] + pointsum[MetaKeyName.SC_COL_IND] = points.loc[ + gid, "col_ind" + ] + pointsum["res_class"] = ri summary.append(pointsum) - logger.debug('Serial aggregation completed gid {}: ' - '{} out of {} points complete' - .format(gid, n_finished, len(gids))) + logger.debug( + "Serial aggregation completed gid {}: " + "{} out of {} points complete".format( + gid, n_finished, len(gids) + ) + ) n_finished += 1 return summary - def run_parallel(self, gen_fpath, args=None, max_workers=None, - sites_per_worker=100): + def run_parallel( + self, gen_fpath, args=None, max_workers=None, sites_per_worker=100 + ): """Get the supply curve points aggregation summary using futures. Parameters @@ -1114,25 +1226,31 @@ def run_parallel(self, gen_fpath, args=None, max_workers=None, chunks = int(np.ceil(len(self.gids) / sites_per_worker)) chunks = np.array_split(self.gids, chunks) - logger.info('Running supply curve point aggregation for ' - 'points {} through {} at a resolution of {} ' - 'on {} cores in {} chunks.' - .format(self.gids[0], self.gids[-1], self._resolution, - max_workers, len(chunks))) + logger.info( + "Running supply curve point aggregation for " + "points {} through {} at a resolution of {} " + "on {} cores in {} chunks.".format( + self.gids[0], + self.gids[-1], + self._resolution, + max_workers, + len(chunks), + ) + ) slice_lookup = None if self._inclusion_mask is not None: - with SupplyCurveExtent(self._excl_fpath, - resolution=self._resolution) as sc: + with SupplyCurveExtent( + self._excl_fpath, resolution=self._resolution + ) as sc: assert sc.exclusions.shape == self._inclusion_mask.shape slice_lookup = sc.get_slice_lookup(self.gids) futures = [] summary = [] n_finished = 0 - loggers = [__name__, 'reV.supply_curve.point_summary', 'reV'] + loggers = [__name__, "reV.supply_curve.point_summary", "reV"] with SpawnProcessPool(max_workers=max_workers, loggers=loggers) as exe: - # iterate through split executions, submitting each to worker for gid_set in chunks: # submit executions and append to futures list @@ -1143,30 +1261,35 @@ def run_parallel(self, gen_fpath, args=None, max_workers=None, rs, cs = slice_lookup[gid] chunk_incl_masks[gid] = self._inclusion_mask[rs, cs] - futures.append(exe.submit( - self.run_serial, - self._excl_fpath, gen_fpath, - self._tm_dset, gen_index, - econ_fpath=self._econ_fpath, - excl_dict=self._excl_dict, - inclusion_mask=chunk_incl_masks, - res_class_dset=self._res_class_dset, - res_class_bins=self._res_class_bins, - cf_dset=self._cf_dset, - lcoe_dset=self._lcoe_dset, - h5_dsets=self._h5_dsets, - data_layers=self._data_layers, - resolution=self._resolution, - power_density=self._power_density, - friction_fpath=self._friction_fpath, - friction_dset=self._friction_dset, - area_filter_kernel=self._area_filter_kernel, - min_area=self._min_area, - gids=gid_set, - args=args, - excl_area=self._excl_area, - cap_cost_scale=self._cap_cost_scale, - recalc_lcoe=self._recalc_lcoe)) + futures.append( + exe.submit( + self.run_serial, + self._excl_fpath, + gen_fpath, + self._tm_dset, + gen_index, + econ_fpath=self._econ_fpath, + excl_dict=self._excl_dict, + inclusion_mask=chunk_incl_masks, + res_class_dset=self._res_class_dset, + res_class_bins=self._res_class_bins, + cf_dset=self._cf_dset, + lcoe_dset=self._lcoe_dset, + h5_dsets=self._h5_dsets, + data_layers=self._data_layers, + resolution=self._resolution, + power_density=self._power_density, + friction_fpath=self._friction_fpath, + friction_dset=self._friction_dset, + area_filter_kernel=self._area_filter_kernel, + min_area=self._min_area, + gids=gid_set, + args=args, + excl_area=self._excl_area, + cap_cost_scale=self._cap_cost_scale, + recalc_lcoe=self._recalc_lcoe, + ) + ) # gather results for future in as_completed(futures): @@ -1174,12 +1297,17 @@ def run_parallel(self, gen_fpath, args=None, max_workers=None, summary += future.result() if n_finished % 10 == 0: mem = psutil.virtual_memory() - logger.info('Parallel aggregation futures collected: ' - '{} out of {}. Memory usage is {:.3f} GB out ' - 'of {:.3f} GB ({:.2f}% utilized).' - .format(n_finished, len(chunks), - mem.used / 1e9, mem.total / 1e9, - 100 * mem.used / mem.total)) + logger.info( + "Parallel aggregation futures collected: " + "{} out of {}. Memory usage is {:.3f} GB out " + "of {:.3f} GB ({:.2f}% utilized).".format( + n_finished, + len(chunks), + mem.used / 1e9, + mem.total / 1e9, + 100 * mem.used / mem.total, + ) + ) return summary @@ -1208,8 +1336,10 @@ def _convert_bins(bins): return bins if any(type_check): - raise TypeError('Resource class bins has inconsistent ' - 'entry type: {}'.format(bins)) + raise TypeError( + "Resource class bins has inconsistent " + "entry type: {}".format(bins) + ) bbins = [] for i, b in enumerate(sorted(bins)): @@ -1233,15 +1363,18 @@ def _summary_to_df(summary): Summary of the SC points. """ summary = pd.DataFrame(summary) - sort_by = [x for x in (MetaKeyName.SC_POINT_GID, 'res_class') if x in summary] + sort_by = [ + x for x in (MetaKeyName.SC_POINT_GID, "res_class") if x in summary + ] summary = summary.sort_values(sort_by) summary = summary.reset_index(drop=True) summary.index.name = MetaKeyName.SC_GID return summary - def summarize(self, gen_fpath, args=None, max_workers=None, - sites_per_worker=100): + def summarize( + self, gen_fpath, args=None, max_workers=None, sites_per_worker=100 + ): """ Get the supply curve points aggregation summary @@ -1269,35 +1402,45 @@ def summarize(self, gen_fpath, args=None, max_workers=None, if max_workers == 1: gen_index = self._parse_gen_index(gen_fpath) afk = self._area_filter_kernel - summary = self.run_serial(self._excl_fpath, gen_fpath, - self._tm_dset, gen_index, - econ_fpath=self._econ_fpath, - excl_dict=self._excl_dict, - inclusion_mask=self._inclusion_mask, - res_class_dset=self._res_class_dset, - res_class_bins=self._res_class_bins, - cf_dset=self._cf_dset, - lcoe_dset=self._lcoe_dset, - h5_dsets=self._h5_dsets, - data_layers=self._data_layers, - resolution=self._resolution, - power_density=self._power_density, - friction_fpath=self._friction_fpath, - friction_dset=self._friction_dset, - area_filter_kernel=afk, - min_area=self._min_area, - gids=self.gids, args=args, - excl_area=self._excl_area, - cap_cost_scale=self._cap_cost_scale, - recalc_lcoe=self._recalc_lcoe) + summary = self.run_serial( + self._excl_fpath, + gen_fpath, + self._tm_dset, + gen_index, + econ_fpath=self._econ_fpath, + excl_dict=self._excl_dict, + inclusion_mask=self._inclusion_mask, + res_class_dset=self._res_class_dset, + res_class_bins=self._res_class_bins, + cf_dset=self._cf_dset, + lcoe_dset=self._lcoe_dset, + h5_dsets=self._h5_dsets, + data_layers=self._data_layers, + resolution=self._resolution, + power_density=self._power_density, + friction_fpath=self._friction_fpath, + friction_dset=self._friction_dset, + area_filter_kernel=afk, + min_area=self._min_area, + gids=self.gids, + args=args, + excl_area=self._excl_area, + cap_cost_scale=self._cap_cost_scale, + recalc_lcoe=self._recalc_lcoe, + ) else: - summary = self.run_parallel(gen_fpath=gen_fpath, args=args, - max_workers=max_workers, - sites_per_worker=sites_per_worker) + summary = self.run_parallel( + gen_fpath=gen_fpath, + args=args, + max_workers=max_workers, + sites_per_worker=sites_per_worker, + ) if not any(summary): - e = ('Supply curve aggregation found no non-excluded SC points. ' - 'Please check your exclusions or subset SC GID selection.') + e = ( + "Supply curve aggregation found no non-excluded SC points. " + "Please check your exclusions or subset SC GID selection." + ) logger.error(e) raise EmptySupplyCurvePointError(e) @@ -1305,8 +1448,14 @@ def summarize(self, gen_fpath, args=None, max_workers=None, return summary - def run(self, out_fpath, gen_fpath=None, args=None, max_workers=None, - sites_per_worker=100): + def run( + self, + out_fpath, + gen_fpath=None, + args=None, + max_workers=None, + sites_per_worker=100, + ): """Run a supply curve aggregation. Parameters @@ -1345,7 +1494,9 @@ def run(self, out_fpath, gen_fpath=None, args=None, max_workers=None, if gen_fpath is None: out = Aggregation.run( - self._excl_fpath, self._res_fpath, self._tm_dset, + self._excl_fpath, + self._res_fpath, + self._tm_dset, excl_dict=self._excl_dict, resolution=self._resolution, excl_area=self._excl_area, @@ -1353,12 +1504,16 @@ def run(self, out_fpath, gen_fpath=None, args=None, max_workers=None, min_area=self._min_area, pre_extract_inclusions=self._pre_extract_inclusions, max_workers=max_workers, - sites_per_worker=sites_per_worker) - summary = out['meta'] + sites_per_worker=sites_per_worker, + ) + summary = out["meta"] else: - summary = self.summarize(gen_fpath=gen_fpath, args=args, - max_workers=max_workers, - sites_per_worker=sites_per_worker) + summary = self.summarize( + gen_fpath=gen_fpath, + args=args, + max_workers=max_workers, + sites_per_worker=sites_per_worker, + ) out_fpath = _format_sc_agg_out_fpath(out_fpath) summary.to_csv(out_fpath) @@ -1369,11 +1524,12 @@ def run(self, out_fpath, gen_fpath=None, args=None, max_workers=None, def _format_sc_agg_out_fpath(out_fpath): """Add CSV file ending and replace underscore, if necessary.""" if not out_fpath.endswith(".csv"): - out_fpath = '{}.csv'.format(out_fpath) + out_fpath = "{}.csv".format(out_fpath) project_dir, out_fn = os.path.split(out_fpath) - out_fn = out_fn.replace("supply_curve_aggregation", - "supply-curve-aggregation") + out_fn = out_fn.replace( + "supply_curve_aggregation", "supply-curve-aggregation" + ) return os.path.join(project_dir, out_fn) @@ -1381,9 +1537,11 @@ def _warn_about_large_datasets(gen, dset): """Warn user about multi-dimensional datasets in passthrough datasets""" dset_shape = gen.shapes.get(dset, (1,)) if len(dset_shape) > 1: - msg = ("Generation dataset {!r} is not 1-dimensional (shape: {})." - "You may run into memory errors during aggregation - use " - "rep-profiles for aggregating higher-order datasets instead!" - .format(dset, dset_shape)) + msg = ( + "Generation dataset {!r} is not 1-dimensional (shape: {})." + "You may run into memory errors during aggregation - use " + "rep-profiles for aggregating higher-order datasets instead!" + .format(dset, dset_shape) + ) logger.warning(msg) warn(msg, UserWarning) diff --git a/reV/supply_curve/supply_curve.py b/reV/supply_curve/supply_curve.py index 071d3461a..2e5c30de8 100644 --- a/reV/supply_curve/supply_curve.py +++ b/reV/supply_curve/supply_curve.py @@ -4,6 +4,7 @@ - Calculation of LCOT - Supply Curve creation """ + import json import logging import os @@ -27,8 +28,13 @@ class SupplyCurve: """SupplyCurve""" - def __init__(self, sc_points, trans_table, sc_features=None, - sc_capacity_col=MetaKeyName.CAPACITY): + def __init__( + self, + sc_points, + trans_table, + sc_features=None, + sc_capacity_col=MetaKeyName.CAPACITY, + ): """ReV LCOT calculation and SupplyCurve sorting class. ``reV`` supply curve computes the transmission costs associated @@ -126,15 +132,17 @@ def __init__(self, sc_points, trans_table, sc_features=None, (mean_lcoe_friction + lcot) ($/MWh). """ log_versions(logger) - logger.info('Supply curve points input: {}'.format(sc_points)) - logger.info('Transmission table input: {}'.format(trans_table)) - logger.info('Supply curve capacity column: {}'.format(sc_capacity_col)) + logger.info("Supply curve points input: {}".format(sc_points)) + logger.info("Transmission table input: {}".format(trans_table)) + logger.info("Supply curve capacity column: {}".format(sc_capacity_col)) self._sc_capacity_col = sc_capacity_col - self._sc_points = self._parse_sc_points(sc_points, - sc_features=sc_features) - self._trans_table = self._map_tables(self._sc_points, trans_table, - sc_capacity_col=sc_capacity_col) + self._sc_points = self._parse_sc_points( + sc_points, sc_features=sc_features + ) + self._trans_table = self._map_tables( + self._sc_points, trans_table, sc_capacity_col=sc_capacity_col + ) self._sc_gids, self._mask = self._parse_sc_gids(self._trans_table) def __repr__(self): @@ -176,7 +184,7 @@ def _parse_sc_points(sc_points, sc_features=None): DataFrame of supply curve point summary with additional features added if supplied """ - if isinstance(sc_points, str) and sc_points.endswith('.h5'): + if isinstance(sc_points, str) and sc_points.endswith(".h5"): with Resource(sc_points) as res: sc_points = res.meta sc_points.index.name = MetaKeyName.SC_GID @@ -184,23 +192,31 @@ def _parse_sc_points(sc_points, sc_features=None): else: sc_points = parse_table(sc_points) - logger.debug('Supply curve points table imported with columns: {}' - .format(sc_points.columns.values.tolist())) + logger.debug( + "Supply curve points table imported with columns: {}".format( + sc_points.columns.values.tolist() + ) + ) if sc_features is not None: sc_features = parse_table(sc_features) - merge_cols = [c for c in sc_features - if c in sc_points] - sc_points = sc_points.merge(sc_features, on=merge_cols, how='left') - logger.debug('Adding Supply Curve Features table with columns: {}' - .format(sc_features.columns.values.tolist())) - - if 'transmission_multiplier' in sc_points: - col = 'transmission_multiplier' + merge_cols = [c for c in sc_features if c in sc_points] + sc_points = sc_points.merge(sc_features, on=merge_cols, how="left") + logger.debug( + "Adding Supply Curve Features table with columns: {}".format( + sc_features.columns.values.tolist() + ) + ) + + if "transmission_multiplier" in sc_points: + col = "transmission_multiplier" sc_points.loc[:, col] = sc_points.loc[:, col].fillna(1) - logger.debug('Final supply curve points table has columns: {}' - .format(sc_points.columns.values.tolist())) + logger.debug( + "Final supply curve points table has columns: {}".format( + sc_points.columns.values.tolist() + ) + ) return sc_points @@ -221,18 +237,20 @@ def _get_merge_cols(sc_columns, trans_columns): Columns to merge on which maps the sc columns (keys) to the corresponding trans table columns (values) """ - sc_columns = [c for c in sc_columns if c.startswith('sc_')] - trans_columns = [c for c in trans_columns if c.startswith('sc_')] + sc_columns = [c for c in sc_columns if c.startswith("sc_")] + trans_columns = [c for c in trans_columns if c.startswith("sc_")] merge_cols = {} - for c_val in ['row', 'col']: + for c_val in ["row", "col"]: trans_col = [c for c in trans_columns if c_val in c] sc_col = [c for c in sc_columns if c_val in c] if trans_col and sc_col: merge_cols[sc_col[0]] = trans_col[0] if len(merge_cols) != 2: - msg = ('Did not find a unique set of sc row and column ids to ' - 'merge on: {}'.format(merge_cols)) + msg = ( + "Did not find a unique set of sc row and column ids to " + "merge on: {}".format(merge_cols) + ) logger.error(msg) raise RuntimeError(msg) @@ -265,16 +283,18 @@ def _parse_trans_table(trans_table): # legacy name: trans_gids # also xformer_cost_p_mw -> xformer_cost_per_mw (not sure why there # would be a *_p_mw but here we are...) - rename_map = {'trans_line_gid': 'trans_gid', - 'trans_gids': 'trans_line_gids', - 'xformer_cost_p_mw': 'xformer_cost_per_mw'} + rename_map = { + "trans_line_gid": "trans_gid", + "trans_gids": "trans_line_gids", + "xformer_cost_p_mw": "xformer_cost_per_mw", + } trans_table = trans_table.rename(columns=rename_map) - if 'dist_mi' in trans_table and 'dist_km' not in trans_table: - trans_table = trans_table.rename(columns={'dist_mi': 'dist_km'}) - trans_table['dist_km'] *= 1.60934 + if "dist_mi" in trans_table and "dist_km" not in trans_table: + trans_table = trans_table.rename(columns={"dist_mi": "dist_km"}) + trans_table["dist_km"] *= 1.60934 - drop_cols = [MetaKeyName.SC_GID, 'cap_left', MetaKeyName.SC_POINT_GID] + drop_cols = [MetaKeyName.SC_GID, "cap_left", MetaKeyName.SC_POINT_GID] drop_cols = [c for c in drop_cols if c in trans_table] if drop_cols: trans_table = trans_table.drop(columns=drop_cols) @@ -282,7 +302,9 @@ def _parse_trans_table(trans_table): return trans_table @staticmethod - def _map_trans_capacity(trans_sc_table, sc_capacity_col=MetaKeyName.CAPACITY): + def _map_trans_capacity( + trans_sc_table, sc_capacity_col=MetaKeyName.CAPACITY + ): """ Map SC gids to transmission features based on capacity. For any SC gids with capacity > the maximum transmission feature capacity, map @@ -309,33 +331,42 @@ def _map_trans_capacity(trans_sc_table, sc_capacity_col=MetaKeyName.CAPACITY): based on maximum capacity """ - nx = trans_sc_table[sc_capacity_col] / trans_sc_table['max_cap'] + nx = trans_sc_table[sc_capacity_col] / trans_sc_table["max_cap"] nx = np.ceil(nx).astype(int) - trans_sc_table['n_parallel_trans'] = nx + trans_sc_table["n_parallel_trans"] = nx if (nx > 1).any(): mask = nx > 1 - tie_line_cost = (trans_sc_table.loc[mask, 'tie_line_cost'] - * nx[mask]) - - xformer_cost = (trans_sc_table.loc[mask, 'xformer_cost_per_mw'] - * trans_sc_table.loc[mask, 'max_cap'] * nx[mask]) - - conn_cost = (xformer_cost - + trans_sc_table.loc[mask, 'sub_upgrade_cost'] - + trans_sc_table.loc[mask, 'new_sub_cost']) + tie_line_cost = ( + trans_sc_table.loc[mask, "tie_line_cost"] * nx[mask] + ) + + xformer_cost = ( + trans_sc_table.loc[mask, "xformer_cost_per_mw"] + * trans_sc_table.loc[mask, "max_cap"] + * nx[mask] + ) + + conn_cost = ( + xformer_cost + + trans_sc_table.loc[mask, "sub_upgrade_cost"] + + trans_sc_table.loc[mask, "new_sub_cost"] + ) trans_cap_cost = tie_line_cost + conn_cost - trans_sc_table.loc[mask, 'tie_line_cost'] = tie_line_cost - trans_sc_table.loc[mask, 'xformer_cost'] = xformer_cost - trans_sc_table.loc[mask, 'connection_cost'] = conn_cost - trans_sc_table.loc[mask, 'trans_cap_cost'] = trans_cap_cost - - msg = ("{} SC points have a capacity that exceeds the maximum " - "transmission feature capacity and will be connected with " - "multiple parallel transmission features." - .format((nx > 1).sum())) + trans_sc_table.loc[mask, "tie_line_cost"] = tie_line_cost + trans_sc_table.loc[mask, "xformer_cost"] = xformer_cost + trans_sc_table.loc[mask, "connection_cost"] = conn_cost + trans_sc_table.loc[mask, "trans_cap_cost"] = trans_cap_cost + + msg = ( + "{} SC points have a capacity that exceeds the maximum " + "transmission feature capacity and will be connected with " + "multiple parallel transmission features.".format( + (nx > 1).sum() + ) + ) logger.info(msg) return trans_sc_table @@ -378,19 +409,24 @@ def _check_sub_trans_lines(cls, features): List of missing transmission line 'trans_gid's for all substations in features table """ - features = features.rename(columns={'trans_line_gid': 'trans_gid', - 'trans_gids': 'trans_line_gids'}) - mask = features['category'].str.lower() == 'substation' + features = features.rename( + columns={ + "trans_line_gid": "trans_gid", + "trans_gids": "trans_line_gids", + } + ) + mask = features["category"].str.lower() == "substation" if not any(mask): return [] - line_gids = (features.loc[mask, 'trans_line_gids'] - .apply(cls._parse_trans_line_gids)) + line_gids = features.loc[mask, "trans_line_gids"].apply( + cls._parse_trans_line_gids + ) line_gids = np.unique(np.concatenate(line_gids.values)) - test = np.isin(line_gids, features['trans_gid'].values) + test = np.isin(line_gids, features["trans_gid"].values) return line_gids[~test].tolist() @@ -417,10 +453,13 @@ def _check_substation_conns(cls, trans_table, sc_cols=MetaKeyName.SC_GID): missing[sc_point] = tl_gids if any(missing): - msg = ('The following sc_gid (keys) were connected to substations ' - 'but were not connected to the respective transmission line' - ' gids (values) which is required for full SC sort: {}' - .format(missing)) + msg = ( + "The following sc_gid (keys) were connected to substations " + "but were not connected to the respective transmission line" + " gids (values) which is required for full SC sort: {}".format( + missing + ) + ) logger.error(msg) raise SupplyCurveInputError(msg) @@ -440,33 +479,51 @@ def _check_sc_trans_table(cls, sc_points, trans_table): trans_sc_gids = set(trans_table[MetaKeyName.SC_GID].unique()) missing = sorted(list(sc_gids - trans_sc_gids)) if any(missing): - msg = ("There are {} Supply Curve points with missing " - "transmission mappings. Supply curve points with no " - "transmission features will not be connected! " - "Missing sc_gid's: {}" - .format(len(missing), missing)) + msg = ( + "There are {} Supply Curve points with missing " + "transmission mappings. Supply curve points with no " + "transmission features will not be connected! " + "Missing sc_gid's: {}".format(len(missing), missing) + ) logger.warning(msg) warn(msg) if not any(trans_sc_gids) or not any(sc_gids): - msg = ('Merging of sc points table and transmission features ' - 'table failed with {} original sc gids and {} transmission ' - 'sc gids after table merge.' - .format(len(sc_gids), len(trans_sc_gids))) + msg = ( + "Merging of sc points table and transmission features " + "table failed with {} original sc gids and {} transmission " + "sc gids after table merge.".format( + len(sc_gids), len(trans_sc_gids) + ) + ) logger.error(msg) raise SupplyCurveError(msg) - logger.debug('There are {} original SC gids and {} sc gids in the ' - 'merged transmission table.' - .format(len(sc_gids), len(trans_sc_gids))) - logger.debug('Transmission Table created with columns: {}' - .format(trans_table.columns.values.tolist())) + logger.debug( + "There are {} original SC gids and {} sc gids in the " + "merged transmission table.".format( + len(sc_gids), len(trans_sc_gids) + ) + ) + logger.debug( + "Transmission Table created with columns: {}".format( + trans_table.columns.values.tolist() + ) + ) @classmethod - def _merge_sc_trans_tables(cls, sc_points, trans_table, - sc_cols=(MetaKeyName.SC_GID, MetaKeyName.CAPACITY, MetaKeyName.MEAN_CF, - MetaKeyName.MEAN_LCOE), - sc_capacity_col=MetaKeyName.CAPACITY): + def _merge_sc_trans_tables( + cls, + sc_points, + trans_table, + sc_cols=( + MetaKeyName.SC_GID, + MetaKeyName.CAPACITY, + MetaKeyName.MEAN_CF, + MetaKeyName.MEAN_LCOE, + ), + sc_capacity_col=MetaKeyName.CAPACITY, + ): """ Merge the supply curve table with the transmission features table. @@ -481,7 +538,8 @@ def _merge_sc_trans_tables(cls, sc_points, trans_table, sc_cols : tuple | list, optional List of column from sc_points to transfer into the trans table, If the `sc_capacity_col` is not included, it will get added. - by default (MetaKeyName.SC_GID, MetaKeyName.CAPACITY, MetaKeyName.MEAN_CF, MetaKeyName.MEAN_LCOE) + by default (MetaKeyName.SC_GID, MetaKeyName.CAPACITY, + MetaKeyName.MEAN_CF, MetaKeyName.MEAN_LCOE) sc_capacity_col : str, optional Name of capacity column in `trans_sc_table`. The values in this column determine the size of transmission lines built. @@ -504,19 +562,28 @@ def _merge_sc_trans_tables(cls, sc_points, trans_table, if isinstance(trans_table, (list, tuple)): trans_sc_table = [] for table in trans_table: - trans_sc_table.append(cls._merge_sc_trans_tables( - sc_points, table, sc_cols=sc_cols, - sc_capacity_col=sc_capacity_col)) + trans_sc_table.append( + cls._merge_sc_trans_tables( + sc_points, + table, + sc_cols=sc_cols, + sc_capacity_col=sc_capacity_col, + ) + ) trans_sc_table = pd.concat(trans_sc_table) else: trans_table = cls._parse_trans_table(trans_table) - merge_cols = cls._get_merge_cols(sc_points.columns, - trans_table.columns) - logger.info('Merging SC table and Trans Table with ' - '{} mapping: {}' - .format('sc_table_col: trans_table_col', merge_cols)) + merge_cols = cls._get_merge_cols( + sc_points.columns, trans_table.columns + ) + logger.info( + "Merging SC table and Trans Table with " + "{} mapping: {}".format( + "sc_table_col: trans_table_col", merge_cols + ) + ) sc_points = sc_points.rename(columns=merge_cols) merge_cols = list(merge_cols.values()) @@ -526,20 +593,30 @@ def _merge_sc_trans_tables(cls, sc_points, trans_table, if MetaKeyName.MEAN_LCOE_FRICTION in sc_points: sc_cols.append(MetaKeyName.MEAN_LCOE_FRICTION) - if 'transmission_multiplier' in sc_points: - sc_cols.append('transmission_multiplier') + if "transmission_multiplier" in sc_points: + sc_cols.append("transmission_multiplier") sc_cols += merge_cols sc_points = sc_points[sc_cols].copy() - trans_sc_table = trans_table.merge(sc_points, on=merge_cols, - how='inner') + trans_sc_table = trans_table.merge( + sc_points, on=merge_cols, how="inner" + ) return trans_sc_table @classmethod - def _map_tables(cls, sc_points, trans_table, - sc_cols=(MetaKeyName.SC_GID, MetaKeyName.CAPACITY, MetaKeyName.MEAN_CF, MetaKeyName.MEAN_LCOE), - sc_capacity_col=MetaKeyName.CAPACITY): + def _map_tables( + cls, + sc_points, + trans_table, + sc_cols=( + MetaKeyName.SC_GID, + MetaKeyName.CAPACITY, + MetaKeyName.MEAN_CF, + MetaKeyName.MEAN_LCOE, + ), + sc_capacity_col=MetaKeyName.CAPACITY, + ): """ Map supply curve points to transmission features @@ -554,7 +631,8 @@ def _map_tables(cls, sc_points, trans_table, sc_cols : tuple | list, optional List of column from sc_points to transfer into the trans table, If the `sc_capacity_col` is not included, it will get added. - by default (MetaKeyName.SC_GID, MetaKeyName.CAPACITY, MetaKeyName.MEAN_CF, MetaKeyName.MEAN_LCOE) + by default (MetaKeyName.SC_GID, MetaKeyName.CAPACITY, + MetaKeyName.MEAN_CF, MetaKeyName.MEAN_LCOE) sc_capacity_col : str, optional Name of capacity column in `trans_sc_table`. The values in this column determine the size of transmission lines built. @@ -572,17 +650,18 @@ def _map_tables(cls, sc_points, trans_table, This is performed by an inner merging with trans_table """ scc = sc_capacity_col - trans_sc_table = cls._merge_sc_trans_tables(sc_points, trans_table, - sc_cols=sc_cols, - sc_capacity_col=scc) + trans_sc_table = cls._merge_sc_trans_tables( + sc_points, trans_table, sc_cols=sc_cols, sc_capacity_col=scc + ) - if 'max_cap' in trans_sc_table: - trans_sc_table = cls._map_trans_capacity(trans_sc_table, - sc_capacity_col=scc) + if "max_cap" in trans_sc_table: + trans_sc_table = cls._map_trans_capacity( + trans_sc_table, sc_capacity_col=scc + ) - trans_sc_table = \ - trans_sc_table.sort_values( - [MetaKeyName.SC_GID, 'trans_gid']).reset_index(drop=True) + trans_sc_table = trans_sc_table.sort_values( + [MetaKeyName.SC_GID, "trans_gid"] + ).reset_index(drop=True) cls._check_sc_trans_table(sc_points, trans_sc_table) @@ -618,8 +697,9 @@ def _create_handler(trans_table, trans_costs=None, avail_cap_frac=1): else: kwargs = {} - trans_features = TF(trans_table, avail_cap_frac=avail_cap_frac, - **kwargs) + trans_features = TF( + trans_table, avail_cap_frac=avail_cap_frac, **kwargs + ) return trans_features @@ -650,8 +730,12 @@ def _parse_sc_gids(trans_table, gid_key=MetaKeyName.SC_GID): return sc_gids, mask @staticmethod - def _get_capacity(sc_gid, sc_table, connectable=True, - sc_capacity_col=MetaKeyName.CAPACITY): + def _get_capacity( + sc_gid, + sc_table, + connectable=True, + sc_capacity_col=MetaKeyName.CAPACITY, + ): """ Get capacity of supply curve point @@ -685,9 +769,10 @@ def _get_capacity(sc_gid, sc_table, connectable=True, if len(capacity) == 1: capacity = capacity[0] else: - msg = ('Each supply curve point should only have ' - 'a single capacity, but {} has {}' - .format(sc_gid, capacity)) + msg = ( + "Each supply curve point should only have " + "a single capacity, but {} has {}".format(sc_gid, capacity) + ) logger.error(msg) raise RuntimeError(msg) else: @@ -696,10 +781,16 @@ def _get_capacity(sc_gid, sc_table, connectable=True, return capacity @classmethod - def _compute_trans_cap_cost(cls, trans_table, trans_costs=None, - avail_cap_frac=1, max_workers=None, - connectable=True, line_limited=False, - sc_capacity_col=MetaKeyName.CAPACITY): + def _compute_trans_cap_cost( + cls, + trans_table, + trans_costs=None, + avail_cap_frac=1, + max_workers=None, + connectable=True, + line_limited=False, + sc_capacity_col=MetaKeyName.CAPACITY, + ): """ Compute levelized cost of transmission for all combinations of supply curve points and tranmission features in trans_table @@ -748,9 +839,11 @@ def _compute_trans_cap_cost(cls, trans_table, trans_costs=None, """ scc = sc_capacity_col if scc not in trans_table: - raise SupplyCurveInputError('Supply curve table must have ' - 'supply curve point capacity column' - '({}) to compute lcot'.format(scc)) + raise SupplyCurveInputError( + "Supply curve table must have " + "supply curve point capacity column" + "({}) to compute lcot".format(scc) + ) if trans_costs is not None: trans_costs = TF._parse_dictionary(trans_costs) @@ -760,45 +853,67 @@ def _compute_trans_cap_cost(cls, trans_table, trans_costs=None, if max_workers is None: max_workers = os.cpu_count() - logger.info('Computing LCOT costs for all possible connections...') + logger.info("Computing LCOT costs for all possible connections...") groups = trans_table.groupby(MetaKeyName.SC_GID) if max_workers > 1: - loggers = [__name__, 'reV.handlers.transmission', 'reV'] - with SpawnProcessPool(max_workers=max_workers, - loggers=loggers) as exe: + loggers = [__name__, "reV.handlers.transmission", "reV"] + with SpawnProcessPool( + max_workers=max_workers, loggers=loggers + ) as exe: futures = [] for sc_gid, sc_table in groups: - capacity = cls._get_capacity(sc_gid, sc_table, - connectable=connectable, - sc_capacity_col=scc) - futures.append(exe.submit(TC.feature_costs, sc_table, - capacity=capacity, - avail_cap_frac=avail_cap_frac, - line_limited=line_limited, - **trans_costs)) + capacity = cls._get_capacity( + sc_gid, + sc_table, + connectable=connectable, + sc_capacity_col=scc, + ) + futures.append( + exe.submit( + TC.feature_costs, + sc_table, + capacity=capacity, + avail_cap_frac=avail_cap_frac, + line_limited=line_limited, + **trans_costs, + ) + ) cost = [future.result() for future in futures] else: cost = [] for sc_gid, sc_table in groups: - capacity = cls._get_capacity(sc_gid, sc_table, - connectable=connectable, - sc_capacity_col=scc) - cost.append(TC.feature_costs(sc_table, - capacity=capacity, - avail_cap_frac=avail_cap_frac, - line_limited=line_limited, - **trans_costs)) - - cost = np.hstack(cost).astype('float32') - logger.info('LCOT cost calculation is complete.') + capacity = cls._get_capacity( + sc_gid, + sc_table, + connectable=connectable, + sc_capacity_col=scc, + ) + cost.append( + TC.feature_costs( + sc_table, + capacity=capacity, + avail_cap_frac=avail_cap_frac, + line_limited=line_limited, + **trans_costs, + ) + ) + + cost = np.hstack(cost).astype("float32") + logger.info("LCOT cost calculation is complete.") return cost - def compute_total_lcoe(self, fcr, transmission_costs=None, - avail_cap_frac=1, line_limited=False, - connectable=True, max_workers=None, - consider_friction=True): + def compute_total_lcoe( + self, + fcr, + transmission_costs=None, + avail_cap_frac=1, + line_limited=False, + connectable=True, + max_workers=None, + consider_friction=True, + ): """ Compute LCOT and total LCOE for all sc point to transmission feature connections @@ -827,45 +942,54 @@ def compute_total_lcoe(self, fcr, transmission_costs=None, Flag to consider friction layer on LCOE when "mean_lcoe_friction" is in the sc points input, by default True """ - if 'trans_cap_cost' not in self._trans_table: + if "trans_cap_cost" not in self._trans_table: scc = self._sc_capacity_col - cost = self._compute_trans_cap_cost(self._trans_table, - trans_costs=transmission_costs, - avail_cap_frac=avail_cap_frac, - line_limited=line_limited, - connectable=connectable, - max_workers=max_workers, - sc_capacity_col=scc) - self._trans_table['trans_cap_cost_per_mw'] = cost # $/MW + cost = self._compute_trans_cap_cost( + self._trans_table, + trans_costs=transmission_costs, + avail_cap_frac=avail_cap_frac, + line_limited=line_limited, + connectable=connectable, + max_workers=max_workers, + sc_capacity_col=scc, + ) + self._trans_table["trans_cap_cost_per_mw"] = cost # $/MW else: - cost = self._trans_table['trans_cap_cost'].values.copy() # $ + cost = self._trans_table["trans_cap_cost"].values.copy() # $ cost /= self._trans_table[self._sc_capacity_col] # $/MW - self._trans_table['trans_cap_cost_per_mw'] = cost + self._trans_table["trans_cap_cost_per_mw"] = cost cost *= self._trans_table[self._sc_capacity_col] cost /= self._trans_table[MetaKeyName.CAPACITY] # align with "mean_cf" - if 'reinforcement_cost_per_mw' in self._trans_table: - logger.info("'reinforcement_cost_per_mw' column found in " - "transmission table. Adding reinforcement costs " - "to total LCOE.") + if "reinforcement_cost_per_mw" in self._trans_table: + logger.info( + "'reinforcement_cost_per_mw' column found in " + "transmission table. Adding reinforcement costs " + "to total LCOE." + ) cf_mean_arr = self._trans_table[MetaKeyName.MEAN_CF].values lcot = (cost * fcr) / (cf_mean_arr * 8760) lcoe = lcot + self._trans_table[MetaKeyName.MEAN_LCOE] - self._trans_table['lcot_no_reinforcement'] = lcot - self._trans_table['lcoe_no_reinforcement'] = lcoe - r_cost = (self._trans_table['reinforcement_cost_per_mw'] - .values.copy()) + self._trans_table["lcot_no_reinforcement"] = lcot + self._trans_table["lcoe_no_reinforcement"] = lcoe + r_cost = self._trans_table[ + "reinforcement_cost_per_mw" + ].values.copy() r_cost *= self._trans_table[self._sc_capacity_col] - r_cost /= self._trans_table[MetaKeyName.CAPACITY] # align with "mean_cf" + r_cost /= self._trans_table[ + MetaKeyName.CAPACITY + ] # align with "mean_cf" cost += r_cost # $/MW cf_mean_arr = self._trans_table[MetaKeyName.MEAN_CF].values lcot = (cost * fcr) / (cf_mean_arr * 8760) - self._trans_table['lcot'] = lcot - self._trans_table['total_lcoe'] = (self._trans_table['lcot'] - + self._trans_table[MetaKeyName.MEAN_LCOE]) + self._trans_table["lcot"] = lcot + self._trans_table["total_lcoe"] = ( + self._trans_table["lcot"] + + self._trans_table[MetaKeyName.MEAN_LCOE] + ) if consider_friction: self._calculate_total_lcoe_friction() @@ -875,14 +999,19 @@ def _calculate_total_lcoe_friction(self): found make a total LCOE column with friction.""" if MetaKeyName.MEAN_LCOE_FRICTION in self._trans_table: - lcoe_friction = (self._trans_table['lcot'] - + self._trans_table[MetaKeyName.MEAN_LCOE_FRICTION]) + lcoe_friction = ( + self._trans_table["lcot"] + + self._trans_table[MetaKeyName.MEAN_LCOE_FRICTION] + ) self._trans_table[MetaKeyName.TOTAL_LCOE_FRICTION] = lcoe_friction - logger.info('Found mean LCOE with friction. Adding key ' - '"total_lcoe_friction" to trans table.') - - def _exclude_noncompetitive_wind_farms(self, comp_wind_dirs, sc_gid, - downwind=False): + logger.info( + "Found mean LCOE with friction. Adding key " + '"total_lcoe_friction" to trans table.' + ) + + def _exclude_noncompetitive_wind_farms( + self, comp_wind_dirs, sc_gid, downwind=False + ): """ Exclude non-competitive wind farms for given sc_gid @@ -904,18 +1033,20 @@ def _exclude_noncompetitive_wind_farms(self, comp_wind_dirs, sc_gid, gid = comp_wind_dirs.check_sc_gid(sc_gid) if gid is not None: if comp_wind_dirs.mask[gid]: - exclude_gids = comp_wind_dirs['upwind', gid] + exclude_gids = comp_wind_dirs["upwind", gid] if downwind: - exclude_gids = np.append(exclude_gids, - comp_wind_dirs['downwind', gid]) + exclude_gids = np.append( + exclude_gids, comp_wind_dirs["downwind", gid] + ) for n in exclude_gids: check = comp_wind_dirs.exclude_sc_point_gid(n) if check: sc_gids = comp_wind_dirs[MetaKeyName.SC_GID, n] for sc_id in sc_gids: if self._mask[sc_id]: - logger.debug('Excluding sc_gid {}' - .format(sc_id)) + logger.debug( + "Excluding sc_gid {}".format(sc_id) + ) self._mask[sc_id] = False return comp_wind_dirs @@ -944,8 +1075,11 @@ def add_sum_cols(table, sum_cols): missing = [s for s in sum_labels if s not in table] if any(missing): - logger.info('Could not make sum column "{}", missing: {}' - .format(new_label, missing)) + logger.info( + 'Could not make sum column "{}", missing: {}'.format( + new_label, missing + ) + ) else: sum_arr = np.zeros(len(table)) for s in sum_labels: @@ -957,13 +1091,25 @@ def add_sum_cols(table, sum_cols): return table - def _full_sort(self, trans_table, trans_costs=None, - avail_cap_frac=1, comp_wind_dirs=None, - total_lcoe_fric=None, sort_on='total_lcoe', - columns=('trans_gid', 'trans_capacity', 'trans_type', - 'trans_cap_cost_per_mw', 'dist_km', 'lcot', - 'total_lcoe'), - downwind=False): + def _full_sort( + self, + trans_table, + trans_costs=None, + avail_cap_frac=1, + comp_wind_dirs=None, + total_lcoe_fric=None, + sort_on="total_lcoe", + columns=( + "trans_gid", + "trans_capacity", + "trans_type", + "trans_cap_cost_per_mw", + "dist_km", + "lcot", + "total_lcoe", + ), + downwind=False, + ): """ Internal method to handle full supply curve sorting @@ -1001,9 +1147,11 @@ def _full_sort(self, trans_table, trans_costs=None, Updated sc_points table with transmission connections, LCOT and LCOE+LCOT based on full supply curve connections """ - trans_features = self._create_handler(self._trans_table, - trans_costs=trans_costs, - avail_cap_frac=avail_cap_frac) + trans_features = self._create_handler( + self._trans_table, + trans_costs=trans_costs, + avail_cap_frac=avail_cap_frac, + ) init_list = [np.nan] * int(1 + np.max(self._sc_gids)) columns = list(columns) if sort_on not in columns: @@ -1015,18 +1163,21 @@ def _full_sort(self, trans_table, trans_costs=None, # syntax is final_key: source_key (source from trans_table) all_cols = {k: k for k in columns} - essentials = {'trans_gid': 'trans_gid', - 'trans_capacity': 'avail_cap', - 'trans_type': 'category', - 'dist_km': 'dist_km', - 'trans_cap_cost_per_mw': 'trans_cap_cost_per_mw', - 'lcot': 'lcot', - 'total_lcoe': 'total_lcoe', - } + essentials = { + "trans_gid": "trans_gid", + "trans_capacity": "avail_cap", + "trans_type": "category", + "dist_km": "dist_km", + "trans_cap_cost_per_mw": "trans_cap_cost_per_mw", + "lcot": "lcot", + "total_lcoe": "total_lcoe", + } all_cols.update(essentials) - arrays = {final_key: trans_table[source_key].values - for final_key, source_key in all_cols.items()} + arrays = { + final_key: trans_table[source_key].values + for final_key, source_key in all_cols.items() + } sc_capacities = trans_table[self._sc_capacity_col].values @@ -1035,30 +1186,37 @@ def _full_sort(self, trans_table, trans_costs=None, for i in range(len(trans_table)): sc_gid = trans_sc_gids[i] if self._mask[sc_gid]: - connect = trans_features.connect(arrays['trans_gid'][i], - sc_capacities[i]) + connect = trans_features.connect( + arrays["trans_gid"][i], sc_capacities[i] + ) if connect: connected += 1 - logger.debug('Connecting sc gid {}'.format(sc_gid)) + logger.debug("Connecting sc gid {}".format(sc_gid)) self._mask[sc_gid] = False for col_name, data_arr in arrays.items(): conn_lists[col_name][sc_gid] = data_arr[i] if total_lcoe_fric is not None: - conn_lists[MetaKeyName.TOTAL_LCOE_FRICTION][sc_gid] = \ + conn_lists[MetaKeyName.TOTAL_LCOE_FRICTION][sc_gid] = ( total_lcoe_fric[i] + ) current_prog = connected // (len(self) / 100) if current_prog > progress: progress = current_prog - logger.info('{} % of supply curve points connected' - .format(progress)) + logger.info( + "{} % of supply curve points connected".format( + progress + ) + ) if comp_wind_dirs is not None: - comp_wind_dirs = \ + comp_wind_dirs = ( self._exclude_noncompetitive_wind_farms( - comp_wind_dirs, sc_gid, downwind=downwind) + comp_wind_dirs, sc_gid, downwind=downwind + ) + ) index = range(0, int(1 + np.max(self._sc_gids))) connections = pd.DataFrame(conn_lists, index=index) @@ -1068,19 +1226,27 @@ def _full_sort(self, trans_table, trans_costs=None, sc_gids = self._sc_points[MetaKeyName.SC_GID].values connected = connections[MetaKeyName.SC_GID].values - logger.debug('Connected gids {} out of total supply curve gids {}' - .format(len(connected), len(sc_gids))) + logger.debug( + "Connected gids {} out of total supply curve gids {}".format( + len(connected), len(sc_gids) + ) + ) unconnected = ~np.isin(sc_gids, connected) unconnected = sc_gids[unconnected].tolist() if unconnected: - msg = ("{} supply curve points were not connected to tranmission! " - "Unconnected sc_gid's: {}" - .format(len(unconnected), unconnected)) + msg = ( + "{} supply curve points were not connected to tranmission! " + "Unconnected sc_gid's: {}".format( + len(unconnected), unconnected + ) + ) logger.warning(msg) warn(msg) - supply_curve = self._sc_points.merge(connections, on=MetaKeyName.SC_GID) + supply_curve = self._sc_points.merge( + connections, on=MetaKeyName.SC_GID + ) return supply_curve.reset_index(drop=True) @@ -1089,42 +1255,69 @@ def _check_feature_capacity(self, avail_cap_frac=1): Add the transmission connection feature capacity to the trans table if needed """ - if 'avail_cap' not in self._trans_table: - kwargs = {'avail_cap_frac': avail_cap_frac} + if "avail_cap" not in self._trans_table: + kwargs = {"avail_cap_frac": avail_cap_frac} fc = TF.feature_capacity(self._trans_table, **kwargs) - self._trans_table = self._trans_table.merge(fc, on='trans_gid') + self._trans_table = self._trans_table.merge(fc, on="trans_gid") def _adjust_output_columns(self, columns, consider_friction): """Add extra output columns, if needed.""" # These are essentially should-be-defaults that are not # backwards-compatible, so have to explicitly check for them - extra_cols = ['ba_str', 'poi_lat', 'poi_lon', 'reinforcement_poi_lat', - 'reinforcement_poi_lon', 'eos_mult', 'reg_mult', - 'reinforcement_cost_per_mw', 'reinforcement_dist_km', - 'n_parallel_trans', MetaKeyName.TOTAL_LCOE_FRICTION] + extra_cols = [ + "ba_str", + "poi_lat", + "poi_lon", + "reinforcement_poi_lat", + "reinforcement_poi_lon", + "eos_mult", + "reg_mult", + "reinforcement_cost_per_mw", + "reinforcement_dist_km", + "n_parallel_trans", + MetaKeyName.TOTAL_LCOE_FRICTION, + ] if not consider_friction: extra_cols -= {MetaKeyName.TOTAL_LCOE_FRICTION} - extra_cols = [col for col in extra_cols - if col in self._trans_table and col not in columns] + extra_cols = [ + col + for col in extra_cols + if col in self._trans_table and col not in columns + ] return columns + extra_cols def _determine_sort_on(self, sort_on): """Determine the `sort_on` column from user input and trans table""" - if 'reinforcement_cost_per_mw' in self._trans_table: + if "reinforcement_cost_per_mw" in self._trans_table: sort_on = sort_on or "lcoe_no_reinforcement" - return sort_on or 'total_lcoe' - - def full_sort(self, fcr, transmission_costs=None, - avail_cap_frac=1, line_limited=False, - connectable=True, max_workers=None, - consider_friction=True, sort_on=None, - columns=('trans_gid', 'trans_capacity', 'trans_type', - 'trans_cap_cost_per_mw', 'dist_km', 'lcot', - 'total_lcoe'), - wind_dirs=None, n_dirs=2, downwind=False, - offshore_compete=False): + return sort_on or "total_lcoe" + + def full_sort( + self, + fcr, + transmission_costs=None, + avail_cap_frac=1, + line_limited=False, + connectable=True, + max_workers=None, + consider_friction=True, + sort_on=None, + columns=( + "trans_gid", + "trans_capacity", + "trans_type", + "trans_cap_cost_per_mw", + "dist_km", + "lcot", + "total_lcoe", + ), + wind_dirs=None, + n_dirs=2, + downwind=False, + offshore_compete=False, + ): """ run full supply curve sorting @@ -1179,14 +1372,17 @@ def full_sort(self, fcr, transmission_costs=None, Updated sc_points table with transmission connections, LCOT and LCOE+LCOT based on full supply curve connections """ - logger.info('Starting full competitive supply curve sort.') + logger.info("Starting full competitive supply curve sort.") self._check_substation_conns(self._trans_table) - self.compute_total_lcoe(fcr, transmission_costs=transmission_costs, - avail_cap_frac=avail_cap_frac, - line_limited=line_limited, - connectable=connectable, - max_workers=max_workers, - consider_friction=consider_friction) + self.compute_total_lcoe( + fcr, + transmission_costs=transmission_costs, + avail_cap_frac=avail_cap_frac, + line_limited=line_limited, + connectable=connectable, + max_workers=max_workers, + consider_friction=consider_friction, + ) self._check_feature_capacity(avail_cap_frac=avail_cap_frac) if isinstance(columns, tuple): @@ -1196,12 +1392,14 @@ def full_sort(self, fcr, transmission_costs=None, sort_on = self._determine_sort_on(sort_on) trans_table = self._trans_table.copy() - pos = trans_table['lcot'].isnull() - trans_table = trans_table.loc[~pos].sort_values([sort_on, 'trans_gid']) + pos = trans_table["lcot"].isnull() + trans_table = trans_table.loc[~pos].sort_values([sort_on, "trans_gid"]) total_lcoe_fric = None if consider_friction and MetaKeyName.MEAN_LCOE_FRICTION in trans_table: - total_lcoe_fric = trans_table[MetaKeyName.TOTAL_LCOE_FRICTION].values + total_lcoe_fric = trans_table[ + MetaKeyName.TOTAL_LCOE_FRICTION + ].values comp_wind_dirs = None if wind_dirs is not None: @@ -1215,28 +1413,47 @@ def full_sort(self, fcr, transmission_costs=None, msg += " windfarms" logger.info(msg) - comp_wind_dirs = CompetitiveWindFarms(wind_dirs, - self._sc_points, - n_dirs=n_dirs, - offshore=offshore_compete) - - supply_curve = self._full_sort(trans_table, - trans_costs=transmission_costs, - avail_cap_frac=avail_cap_frac, - comp_wind_dirs=comp_wind_dirs, - total_lcoe_fric=total_lcoe_fric, - sort_on=sort_on, columns=columns, - downwind=downwind) + comp_wind_dirs = CompetitiveWindFarms( + wind_dirs, + self._sc_points, + n_dirs=n_dirs, + offshore=offshore_compete, + ) + + supply_curve = self._full_sort( + trans_table, + trans_costs=transmission_costs, + avail_cap_frac=avail_cap_frac, + comp_wind_dirs=comp_wind_dirs, + total_lcoe_fric=total_lcoe_fric, + sort_on=sort_on, + columns=columns, + downwind=downwind, + ) return supply_curve - def simple_sort(self, fcr, transmission_costs=None, - avail_cap_frac=1, max_workers=None, - consider_friction=True, sort_on=None, - columns=('trans_gid', 'trans_type', 'lcot', 'total_lcoe', - 'dist_km', 'trans_cap_cost_per_mw'), - wind_dirs=None, n_dirs=2, downwind=False, - offshore_compete=False): + def simple_sort( + self, + fcr, + transmission_costs=None, + avail_cap_frac=1, + max_workers=None, + consider_friction=True, + sort_on=None, + columns=( + "trans_gid", + "trans_type", + "lcot", + "total_lcoe", + "dist_km", + "trans_cap_cost_per_mw", + ), + wind_dirs=None, + n_dirs=2, + downwind=False, + offshore_compete=False, + ): """ Run simple supply curve sorting that does not take into account available capacity @@ -1291,12 +1508,15 @@ def simple_sort(self, fcr, transmission_costs=None, Updated sc_points table with transmission connections, LCOT and LCOE+LCOT based on simple supply curve connections """ - logger.info('Starting simple supply curve sort (no capacity limits).') - self.compute_total_lcoe(fcr, transmission_costs=transmission_costs, - avail_cap_frac=avail_cap_frac, - connectable=False, - max_workers=max_workers, - consider_friction=consider_friction) + logger.info("Starting simple supply curve sort (no capacity limits).") + self.compute_total_lcoe( + fcr, + transmission_costs=transmission_costs, + avail_cap_frac=avail_cap_frac, + connectable=False, + max_workers=max_workers, + consider_friction=consider_friction, + ) trans_table = self._trans_table.copy() if isinstance(columns, tuple): @@ -1305,33 +1525,50 @@ def simple_sort(self, fcr, transmission_costs=None, columns = self._adjust_output_columns(columns, consider_friction) sort_on = self._determine_sort_on(sort_on) - connections = trans_table.sort_values([sort_on, 'trans_gid']) + connections = trans_table.sort_values([sort_on, "trans_gid"]) connections = connections.groupby(MetaKeyName.SC_GID).first() - rename = {'trans_gid': 'trans_gid', - 'category': 'trans_type'} + rename = {"trans_gid": "trans_gid", "category": "trans_type"} connections = connections.rename(columns=rename) connections = connections[columns].reset_index() - supply_curve = self._sc_points.merge(connections, on=MetaKeyName.SC_GID) + supply_curve = self._sc_points.merge( + connections, on=MetaKeyName.SC_GID + ) if wind_dirs is not None: - supply_curve = \ - CompetitiveWindFarms.run(wind_dirs, - supply_curve, - n_dirs=n_dirs, - offshore=offshore_compete, - sort_on=sort_on, - downwind=downwind) + supply_curve = CompetitiveWindFarms.run( + wind_dirs, + supply_curve, + n_dirs=n_dirs, + offshore=offshore_compete, + sort_on=sort_on, + downwind=downwind, + ) supply_curve = supply_curve.reset_index(drop=True) return supply_curve - def run(self, out_fpath, fixed_charge_rate, simple=True, avail_cap_frac=1, - line_limited=False, transmission_costs=None, - consider_friction=True, sort_on=None, - columns=('trans_gid', 'trans_type', 'trans_cap_cost_per_mw', - 'dist_km', 'lcot', 'total_lcoe'), - max_workers=None, competition=None): + def run( + self, + out_fpath, + fixed_charge_rate, + simple=True, + avail_cap_frac=1, + line_limited=False, + transmission_costs=None, + consider_friction=True, + sort_on=None, + columns=( + "trans_gid", + "trans_type", + "trans_cap_cost_per_mw", + "dist_km", + "lcot", + "total_lcoe", + ), + max_workers=None, + competition=None, + ): """Run Supply Curve Transmission calculations. Run full supply curve taking into account available capacity of @@ -1432,12 +1669,14 @@ def run(self, out_fpath, fixed_charge_rate, simple=True, avail_cap_frac=1, str Path to output supply curve. """ - kwargs = {"fcr": fixed_charge_rate, - "transmission_costs": transmission_costs, - "consider_friction": consider_friction, - "sort_on": sort_on, - "columns": columns, - "max_workers": max_workers} + kwargs = { + "fcr": fixed_charge_rate, + "transmission_costs": transmission_costs, + "consider_friction": consider_friction, + "sort_on": sort_on, + "columns": columns, + "max_workers": max_workers, + } kwargs.update(competition or {}) if simple: @@ -1456,7 +1695,7 @@ def run(self, out_fpath, fixed_charge_rate, simple=True, avail_cap_frac=1, def _format_sc_out_fpath(out_fpath): """Add CSV file ending and replace underscore, if necessary.""" if not out_fpath.endswith(".csv"): - out_fpath = '{}.csv'.format(out_fpath) + out_fpath = "{}.csv".format(out_fpath) project_dir, out_fn = os.path.split(out_fpath) out_fn = out_fn.replace("supply_curve", "supply-curve") diff --git a/reV/supply_curve/tech_mapping.py b/reV/supply_curve/tech_mapping.py index 46326a876..e0d87c978 100644 --- a/reV/supply_curve/tech_mapping.py +++ b/reV/supply_curve/tech_mapping.py @@ -8,6 +8,7 @@ @author: gbuster """ + import logging import os from concurrent.futures import as_completed @@ -31,8 +32,9 @@ class TechMapping: """Framework to create map between tech layer (exclusions), res, and gen""" - def __init__(self, excl_fpath, res_fpath, sc_resolution=2560, - dist_margin=1.05): + def __init__( + self, excl_fpath, res_fpath, sc_resolution=2560, dist_margin=1.05 + ): """ Parameters ---------- @@ -52,11 +54,13 @@ def __init__(self, excl_fpath, res_fpath, sc_resolution=2560, self._excl_fpath = excl_fpath self._check_fout() - self._tree, self._dist_thresh = \ - self._build_tree(res_fpath, dist_margin=dist_margin) + self._tree, self._dist_thresh = self._build_tree( + res_fpath, dist_margin=dist_margin + ) - with SupplyCurveExtent(self._excl_fpath, - resolution=sc_resolution) as sc: + with SupplyCurveExtent( + self._excl_fpath, resolution=sc_resolution + ) as sc: self._sc_resolution = sc.resolution self._gids = np.array(list(range(len(sc))), dtype=np.uint32) self._excl_shape = sc.exclusions.shape @@ -65,9 +69,12 @@ def __init__(self, excl_fpath, res_fpath, sc_resolution=2560, self._sc_col_indices = sc.col_indices self._excl_row_slices = sc.excl_row_slices self._excl_col_slices = sc.excl_col_slices - logger.info('Initialized TechMapping object with {} calc chunks ' - 'for {} tech exclusion points' - .format(len(self._gids), self._n_excl)) + logger.info( + "Initialized TechMapping object with {} calc chunks " + "for {} tech exclusion points".format( + len(self._gids), self._n_excl + ) + ) @property def distance_threshold(self): @@ -111,8 +118,9 @@ def _build_tree(res_fpath, dist_margin=1.05): # pylint: disable=not-callable tree = cKDTree(lat_lons) - dist_thresh = res_dist_threshold(lat_lons, tree=tree, - margin=dist_margin) + dist_thresh = res_dist_threshold( + lat_lons, tree=tree, margin=dist_margin + ) return tree, dist_thresh @@ -136,8 +144,9 @@ def _make_excl_iarr(shape): return iarr.reshape(shape) @staticmethod - def _get_excl_slices(gid, sc_row_indices, sc_col_indices, excl_row_slices, - excl_col_slices): + def _get_excl_slices( + gid, sc_row_indices, sc_col_indices, excl_row_slices, excl_col_slices + ): """ Get the row and column slices of the exclusions grid corresponding to the supply curve point gid. @@ -174,9 +183,16 @@ def _get_excl_slices(gid, sc_row_indices, sc_col_indices, excl_row_slices, return row_slice, col_slice @classmethod - def _get_excl_coords(cls, excl_fpath, gids, sc_row_indices, sc_col_indices, - excl_row_slices, excl_col_slices, - coord_labels=(MetaKeyName.LATITUDE, MetaKeyName.LONGITUDE)): + def _get_excl_coords( + cls, + excl_fpath, + gids, + sc_row_indices, + sc_col_indices, + excl_row_slices, + excl_col_slices, + coord_labels=(MetaKeyName.LATITUDE, MetaKeyName.LONGITUDE), + ): """ Extract the exclusion coordinates for teh desired gids for TechMapping. @@ -211,21 +227,25 @@ def _get_excl_coords(cls, excl_fpath, gids, sc_row_indices, sc_col_indices, tech exclusion points. List entries correspond to input gids. """ coords_out = [] - with h5py.File(excl_fpath, 'r') as f: + with h5py.File(excl_fpath, "r") as f: for gid in gids: - row_slice, col_slice = cls._get_excl_slices(gid, - sc_row_indices, - sc_col_indices, - excl_row_slices, - excl_col_slices) + row_slice, col_slice = cls._get_excl_slices( + gid, + sc_row_indices, + sc_col_indices, + excl_row_slices, + excl_col_slices, + ) try: lats = f[coord_labels[0]][row_slice, col_slice] lons = f[coord_labels[1]][row_slice, col_slice] emeta = np.vstack((lats.flatten(), lons.flatten())).T except Exception as e: - m = ('Could not unpack coordinates for gid {} with ' - 'row/col slice {}/{}. Received the following ' - 'error:\n{}'.format(gid, row_slice, col_slice, e)) + m = ( + "Could not unpack coordinates for gid {} with " + "row/col slice {}/{}. Received the following " + "error:\n{}".format(gid, row_slice, col_slice, e) + ) logger.error(m) raise e @@ -234,9 +254,17 @@ def _get_excl_coords(cls, excl_fpath, gids, sc_row_indices, sc_col_indices, return coords_out @classmethod - def map_resource_gids(cls, gids, excl_fpath, sc_row_indices, - sc_col_indices, excl_row_slices, excl_col_slices, - tree, dist_thresh): + def map_resource_gids( + cls, + gids, + excl_fpath, + sc_row_indices, + sc_col_indices, + excl_row_slices, + excl_col_slices, + tree, + dist_thresh, + ): """Map exclusion gids to the resource meta. Parameters @@ -273,15 +301,26 @@ def map_resource_gids(cls, gids, excl_fpath, sc_row_indices, List of arrays of index values from the NN. List entries correspond to input gids. """ - logger.debug('Getting tech map coordinates for chunks {} through {}' - .format(gids[0], gids[-1])) + logger.debug( + "Getting tech map coordinates for chunks {} through {}".format( + gids[0], gids[-1] + ) + ) ind_out = [] - coords_out = cls._get_excl_coords(excl_fpath, gids, sc_row_indices, - sc_col_indices, excl_row_slices, - excl_col_slices) - - logger.debug('Running tech mapping for chunks {} through {}' - .format(gids[0], gids[-1])) + coords_out = cls._get_excl_coords( + excl_fpath, + gids, + sc_row_indices, + sc_col_indices, + excl_row_slices, + excl_col_slices, + ) + + logger.debug( + "Running tech mapping for chunks {} through {}".format( + gids[0], gids[-1] + ) + ) for i, _ in enumerate(gids): dist, ind = tree.query(coords_out[i]) ind[(dist >= dist_thresh)] = -1 @@ -290,8 +329,14 @@ def map_resource_gids(cls, gids, excl_fpath, sc_row_indices, return ind_out @staticmethod - def save_tech_map(excl_fpath, dset, indices, distance_threshold=None, - res_fpath=None, chunks=(128, 128)): + def save_tech_map( + excl_fpath, + dset, + indices, + distance_threshold=None, + res_fpath=None, + chunks=(128, 128), + ): """Save tech mapping indices and coordinates to an h5 output file. Parameters @@ -316,35 +361,47 @@ def save_tech_map(excl_fpath, dset, indices, distance_threshold=None, shape = indices.shape chunks = (np.min((shape[0], chunks[0])), np.min((shape[1], chunks[1]))) - with h5py.File(excl_fpath, 'a') as f: + with h5py.File(excl_fpath, "a") as f: if dset in list(f): - wmsg = ('TechMap results dataset "{}" is being replaced ' - 'in pre-existing Exclusions TechMapping file "{}"' - .format(dset, excl_fpath)) + wmsg = ( + 'TechMap results dataset "{}" is being replaced ' + 'in pre-existing Exclusions TechMapping file "{}"'.format( + dset, excl_fpath + ) + ) logger.warning(wmsg) warn(wmsg, FileInputWarning) f[dset][...] = indices else: - f.create_dataset(dset, shape=shape, dtype=indices.dtype, - data=indices, chunks=chunks) + f.create_dataset( + dset, + shape=shape, + dtype=indices.dtype, + data=indices, + chunks=chunks, + ) if distance_threshold: - f[dset].attrs['distance_threshold'] = distance_threshold + f[dset].attrs["distance_threshold"] = distance_threshold if res_fpath: - f[dset].attrs['src_res_fpath'] = res_fpath + f[dset].attrs["src_res_fpath"] = res_fpath - logger.info('Successfully saved tech map "{}" to {}' - .format(dset, excl_fpath)) + logger.info( + 'Successfully saved tech map "{}" to {}'.format(dset, excl_fpath) + ) def _check_fout(self): """Check the TechMapping output file for cached data.""" - with h5py.File(self._excl_fpath, 'r') as f: + with h5py.File(self._excl_fpath, "r") as f: if MetaKeyName.LATITUDE not in f or MetaKeyName.LONGITUDE not in f: - emsg = ('Datasets "latitude" and/or "longitude" not in ' - 'pre-existing Exclusions TechMapping file "{}". ' - 'Cannot proceed.' - .format(os.path.basename(self._excl_fpath))) + emsg = ( + 'Datasets "latitude" and/or "longitude" not in ' + 'pre-existing Exclusions TechMapping file "{}". ' + "Cannot proceed.".format( + os.path.basename(self._excl_fpath) + ) + ) logger.exception(emsg) raise FileInputError(emsg) @@ -371,33 +428,36 @@ def map_resource(self, max_workers=None, points_per_worker=10): gid_chunks = np.array_split(self._gids, gid_chunks) # init full output arrays - indices = -1 * np.ones((self._n_excl, ), dtype=np.int32) + indices = -1 * np.ones((self._n_excl,), dtype=np.int32) iarr = self._make_excl_iarr(self._excl_shape) futures = {} - loggers = [__name__, 'reV'] - with SpawnProcessPool(max_workers=max_workers, - loggers=loggers) as exe: - + loggers = [__name__, "reV"] + with SpawnProcessPool(max_workers=max_workers, loggers=loggers) as exe: # iterate through split executions, submitting each to worker for i, gid_set in enumerate(gid_chunks): # submit executions and append to futures list - futures[exe.submit(self.map_resource_gids, - gid_set, - self._excl_fpath, - self._sc_row_indices, - self._sc_col_indices, - self._excl_row_slices, - self._excl_col_slices, - self._tree, - self.distance_threshold)] = i + futures[ + exe.submit( + self.map_resource_gids, + gid_set, + self._excl_fpath, + self._sc_row_indices, + self._sc_col_indices, + self._excl_row_slices, + self._excl_col_slices, + self._tree, + self.distance_threshold, + ) + ] = i n_finished = 0 for future in as_completed(futures): n_finished += 1 - logger.info('Parallel TechMapping futures collected: ' - '{} out of {}' - .format(n_finished, len(futures))) + logger.info( + "Parallel TechMapping futures collected: " + "{} out of {}".format(n_finished, len(futures)) + ) i = futures[future] result = future.result() @@ -408,7 +468,8 @@ def map_resource(self, max_workers=None, points_per_worker=10): self._sc_row_indices, self._sc_col_indices, self._excl_row_slices, - self._excl_col_slices) + self._excl_col_slices, + ) ind_slice = iarr[row_slice, col_slice].flatten() indices[ind_slice] = result[j] @@ -417,8 +478,16 @@ def map_resource(self, max_workers=None, points_per_worker=10): return indices @classmethod - def run(cls, excl_fpath, res_fpath, dset=None, sc_resolution=2560, - dist_margin=1.05, max_workers=None, points_per_worker=10): + def run( + cls, + excl_fpath, + res_fpath, + dset=None, + sc_resolution=2560, + dist_margin=1.05, + max_workers=None, + points_per_worker=10, + ): """Run parallel mapping and save to h5 file. Parameters @@ -451,15 +520,19 @@ def run(cls, excl_fpath, res_fpath, dset=None, sc_resolution=2560, Index values of the NN resource point. -1 if no res point found. 2D integer array with shape equal to the exclusions extent shape. """ - kwargs = {"dist_margin": dist_margin, - "sc_resolution": sc_resolution} + kwargs = {"dist_margin": dist_margin, "sc_resolution": sc_resolution} mapper = cls(excl_fpath, res_fpath, **kwargs) - indices = mapper.map_resource(max_workers=max_workers, - points_per_worker=points_per_worker) + indices = mapper.map_resource( + max_workers=max_workers, points_per_worker=points_per_worker + ) if dset: - mapper.save_tech_map(excl_fpath, dset, indices, - distance_threshold=mapper.distance_threshold, - res_fpath=res_fpath) + mapper.save_tech_map( + excl_fpath, + dset, + indices, + distance_threshold=mapper.distance_threshold, + res_fpath=res_fpath, + ) return indices diff --git a/tests/eagle.py b/tests/eagle.py index 96583ac2e..10de238ae 100644 --- a/tests/eagle.py +++ b/tests/eagle.py @@ -12,17 +12,17 @@ """ import os -import h5py -import pytest -import numpy as np import time -from reV import TESTDATADIR -from reV.handlers.outputs import Outputs +import h5py +import numpy as np +import pytest from rex.utilities.hpc import SLURM from rex.utilities.loggers import init_logger -from reV.generation.cli_gen import get_node_cmd +from reV import TESTDATADIR +from reV.generation.cli_gen import get_node_cmd +from reV.handlers.outputs import Outputs RTOL = 0.0 ATOL = 0.04 @@ -115,8 +115,7 @@ def test_eagle(year): status = slurm.check_status(name, var='name') if status == 'CG': break - else: - time.sleep(5) + time.sleep(5) # get reV 2.0 generation profiles from disk flist = os.listdir(rev2_out_dir) diff --git a/tests/test_bespoke.py b/tests/test_bespoke.py index 70052b7de..a5bab11f3 100644 --- a/tests/test_bespoke.py +++ b/tests/test_bespoke.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -"""reV bespoke wind plant optimization tests -""" +"""reV bespoke wind plant optimization tests""" + import copy import json import os @@ -30,74 +30,86 @@ pytest.importorskip("shapely") -SAM = os.path.join(TESTDATADIR, 'SAM/i_windpower.json') -EXCL = os.path.join(TESTDATADIR, 'ri_exclusions/ri_exclusions.h5') -RES = os.path.join(TESTDATADIR, 'wtk/ri_100_wtk_{}.h5') -TM_DSET = 'techmap_wtk_ri_100' -AGG_DSET = ('cf_mean', 'cf_profile') - -DATA_LAYERS = {'pct_slope': {'dset': 'ri_srtm_slope', - 'method': 'mean', - 'fpath': EXCL}, - 'reeds_region': {'dset': 'ri_reeds_regions', - 'method': 'mode', - 'fpath': EXCL}, - 'padus': {'dset': 'ri_padus', - 'method': 'mode', - 'fpath': EXCL}} +SAM = os.path.join(TESTDATADIR, "SAM/i_windpower.json") +EXCL = os.path.join(TESTDATADIR, "ri_exclusions/ri_exclusions.h5") +RES = os.path.join(TESTDATADIR, "wtk/ri_100_wtk_{}.h5") +TM_DSET = "techmap_wtk_ri_100" +AGG_DSET = ("cf_mean", "cf_profile") + +DATA_LAYERS = { + "pct_slope": {"dset": "ri_srtm_slope", "method": "mean", "fpath": EXCL}, + "reeds_region": { + "dset": "ri_reeds_regions", + "method": "mode", + "fpath": EXCL, + }, + "padus": {"dset": "ri_padus", "method": "mode", "fpath": EXCL}, +} # note that this differs from the -EXCL_DICT = {'ri_srtm_slope': {'inclusion_range': (None, 5), - 'exclude_nodata': False}, - 'ri_padus': {'exclude_values': [1], - 'exclude_nodata': False}, - 'ri_reeds_regions': {'inclusion_range': (None, 400), - 'exclude_nodata': False}} +EXCL_DICT = { + "ri_srtm_slope": {"inclusion_range": (None, 5), "exclude_nodata": False}, + "ri_padus": {"exclude_values": [1], "exclude_nodata": False}, + "ri_reeds_regions": { + "inclusion_range": (None, 400), + "exclude_nodata": False, + }, +} with open(SAM) as f: SAM_SYS_INPUTS = json.load(f) -SAM_SYS_INPUTS['wind_farm_wake_model'] = 2 -SAM_SYS_INPUTS['wind_farm_losses_percent'] = 0 -del SAM_SYS_INPUTS['wind_resource_filename'] -TURB_RATING = np.max(SAM_SYS_INPUTS['wind_turbine_powercurve_powerout']) -SAM_CONFIGS = {'default': SAM_SYS_INPUTS} +SAM_SYS_INPUTS["wind_farm_wake_model"] = 2 +SAM_SYS_INPUTS["wind_farm_losses_percent"] = 0 +del SAM_SYS_INPUTS["wind_resource_filename"] +TURB_RATING = np.max(SAM_SYS_INPUTS["wind_turbine_powercurve_powerout"]) +SAM_CONFIGS = {"default": SAM_SYS_INPUTS} -CAP_COST_FUN = ('140 * system_capacity ' - '* np.exp(-system_capacity / 1E5 * 0.1 + (1 - 0.1))') -FOC_FUN = ('60 * system_capacity ' - '* np.exp(-system_capacity / 1E5 * 0.1 + (1 - 0.1))') -VOC_FUN = '3' -OBJECTIVE_FUNCTION = ('(0.0975 * capital_cost + fixed_operating_cost) ' - '/ aep + variable_operating_cost') +CAP_COST_FUN = ( + "140 * system_capacity " + "* np.exp(-system_capacity / 1E5 * 0.1 + (1 - 0.1))" +) +FOC_FUN = ( + "60 * system_capacity " + "* np.exp(-system_capacity / 1E5 * 0.1 + (1 - 0.1))" +) +VOC_FUN = "3" +OBJECTIVE_FUNCTION = ( + "(0.0975 * capital_cost + fixed_operating_cost) " + "/ aep + variable_operating_cost" +) def test_turbine_placement(gid=33): """Test turbine placement with zero available area.""" - output_request = ('system_capacity', 'cf_mean', 'cf_profile') + output_request = ("system_capacity", "cf_mean", "cf_profile") with tempfile.TemporaryDirectory() as td: - res_fp = os.path.join(td, 'ri_100_wtk_{}.h5') - excl_fp = os.path.join(td, 'ri_exclusions.h5') + res_fp = os.path.join(td, "ri_100_wtk_{}.h5") + excl_fp = os.path.join(td, "ri_exclusions.h5") shutil.copy(EXCL, excl_fp) shutil.copy(RES.format(2012), res_fp.format(2012)) shutil.copy(RES.format(2013), res_fp.format(2013)) - res_fp = res_fp.format('*') + res_fp = res_fp.format("*") sam_sys_inputs = copy.deepcopy(SAM_SYS_INPUTS) - sam_sys_inputs['fixed_operating_cost_multiplier'] = 2 - sam_sys_inputs['variable_operating_cost_multiplier'] = 5 + sam_sys_inputs["fixed_operating_cost_multiplier"] = 2 + sam_sys_inputs["variable_operating_cost_multiplier"] = 5 TechMapping.run(excl_fp, RES.format(2012), dset=TM_DSET, max_workers=1) - bsp = BespokeSinglePlant(gid, excl_fp, res_fp, TM_DSET, - sam_sys_inputs, - OBJECTIVE_FUNCTION, - CAP_COST_FUN, - FOC_FUN, - VOC_FUN, - excl_dict=EXCL_DICT, - output_request=output_request, - ) + bsp = BespokeSinglePlant( + gid, + excl_fp, + res_fp, + TM_DSET, + sam_sys_inputs, + OBJECTIVE_FUNCTION, + CAP_COST_FUN, + FOC_FUN, + VOC_FUN, + excl_dict=EXCL_DICT, + output_request=output_request, + ) place_optimizer = bsp.plant_optimizer assert place_optimizer.turbine_x is None @@ -113,23 +125,31 @@ def test_turbine_placement(gid=33): place_optimizer.place_turbines(max_time=5) assert place_optimizer.nturbs == len(place_optimizer.turbine_x) - assert place_optimizer.capacity == place_optimizer.nturbs *\ - place_optimizer.turbine_capacity + assert ( + place_optimizer.capacity + == place_optimizer.nturbs * place_optimizer.turbine_capacity + ) assert place_optimizer.area == place_optimizer.full_polygons.area / 1e6 - assert place_optimizer.capacity_density == place_optimizer.capacity\ - / place_optimizer.area / 1E3 + assert ( + place_optimizer.capacity_density + == place_optimizer.capacity / place_optimizer.area / 1e3 + ) - place_optimizer.wind_plant["wind_farm_xCoordinates"] = \ + place_optimizer.wind_plant["wind_farm_xCoordinates"] = ( place_optimizer.turbine_x - place_optimizer.wind_plant["wind_farm_yCoordinates"] = \ + ) + place_optimizer.wind_plant["wind_farm_yCoordinates"] = ( place_optimizer.turbine_y - place_optimizer.wind_plant["system_capacity"] =\ + ) + place_optimizer.wind_plant["system_capacity"] = ( place_optimizer.capacity + ) place_optimizer.wind_plant.assign_inputs() place_optimizer.wind_plant.execute() - assert place_optimizer.aep == \ - place_optimizer.wind_plant.annual_energy() + assert ( + place_optimizer.aep == place_optimizer.wind_plant.annual_energy() + ) # pylint: disable=W0641 system_capacity = place_optimizer.capacity @@ -140,40 +160,49 @@ def test_turbine_placement(gid=33): fixed_operating_cost = eval(FOC_FUN, globals(), locals()) * 2 variable_operating_cost = eval(VOC_FUN, globals(), locals()) * 5 # pylint: disable=W0123 - assert place_optimizer.objective ==\ - eval(OBJECTIVE_FUNCTION, globals(), locals()) + assert place_optimizer.objective == eval( + OBJECTIVE_FUNCTION, globals(), locals() + ) assert place_optimizer.capital_cost == capital_cost assert place_optimizer.fixed_operating_cost == fixed_operating_cost - assert (place_optimizer.variable_operating_cost - == variable_operating_cost) + assert ( + place_optimizer.variable_operating_cost == variable_operating_cost + ) bsp.close() def test_zero_area(gid=33): """Test turbine placement with zero available area.""" - output_request = ('system_capacity', 'cf_mean', 'cf_profile') + output_request = ("system_capacity", "cf_mean", "cf_profile") objective_function = ( - '(0.0975 * capital_cost + fixed_operating_cost) ' - '/ (aep + 1E-6) + variable_operating_cost') + "(0.0975 * capital_cost + fixed_operating_cost) " + "/ (aep + 1E-6) + variable_operating_cost" + ) with tempfile.TemporaryDirectory() as td: - res_fp = os.path.join(td, 'ri_100_wtk_{}.h5') - excl_fp = os.path.join(td, 'ri_exclusions.h5') + res_fp = os.path.join(td, "ri_100_wtk_{}.h5") + excl_fp = os.path.join(td, "ri_exclusions.h5") shutil.copy(EXCL, excl_fp) shutil.copy(RES.format(2012), res_fp.format(2012)) shutil.copy(RES.format(2013), res_fp.format(2013)) - res_fp = res_fp.format('*') + res_fp = res_fp.format("*") TechMapping.run(excl_fp, RES.format(2012), dset=TM_DSET, max_workers=1) - bsp = BespokeSinglePlant(gid, excl_fp, res_fp, TM_DSET, - SAM_SYS_INPUTS, - objective_function, CAP_COST_FUN, - FOC_FUN, VOC_FUN, - excl_dict=EXCL_DICT, - output_request=output_request, - ) + bsp = BespokeSinglePlant( + gid, + excl_fp, + res_fp, + TM_DSET, + SAM_SYS_INPUTS, + objective_function, + CAP_COST_FUN, + FOC_FUN, + VOC_FUN, + excl_dict=EXCL_DICT, + output_request=output_request, + ) optimizer = bsp.plant_optimizer optimizer.include_mask = np.zeros_like(optimizer.include_mask) @@ -195,37 +224,48 @@ def test_zero_area(gid=33): def test_correct_turb_location(gid=33): """Test turbine location is reported correctly.""" - output_request = ('system_capacity', 'cf_mean', 'cf_profile') + output_request = ("system_capacity", "cf_mean", "cf_profile") objective_function = ( - '(0.0975 * capital_cost + fixed_operating_cost) ' - '/ (aep + 1E-6) + variable_operating_cost') + "(0.0975 * capital_cost + fixed_operating_cost) " + "/ (aep + 1E-6) + variable_operating_cost" + ) with tempfile.TemporaryDirectory() as td: - res_fp = os.path.join(td, 'ri_100_wtk_{}.h5') - excl_fp = os.path.join(td, 'ri_exclusions.h5') + res_fp = os.path.join(td, "ri_100_wtk_{}.h5") + excl_fp = os.path.join(td, "ri_exclusions.h5") shutil.copy(EXCL, excl_fp) shutil.copy(RES.format(2012), res_fp.format(2012)) shutil.copy(RES.format(2013), res_fp.format(2013)) - res_fp = res_fp.format('*') + res_fp = res_fp.format("*") TechMapping.run(excl_fp, RES.format(2012), dset=TM_DSET, max_workers=1) - bsp = BespokeSinglePlant(gid, excl_fp, res_fp, TM_DSET, - SAM_SYS_INPUTS, - objective_function, CAP_COST_FUN, - FOC_FUN, VOC_FUN, - excl_dict=EXCL_DICT, - output_request=output_request, - ) + bsp = BespokeSinglePlant( + gid, + excl_fp, + res_fp, + TM_DSET, + SAM_SYS_INPUTS, + objective_function, + CAP_COST_FUN, + FOC_FUN, + VOC_FUN, + excl_dict=EXCL_DICT, + output_request=output_request, + ) include_mask = np.zeros_like(bsp.include_mask) include_mask[1, -2] = 1 - pt = PlaceTurbines(bsp.wind_plant_pd, bsp.objective_function, - bsp.capital_cost_function, - bsp.fixed_operating_cost_function, - bsp.variable_operating_cost_function, - include_mask, pixel_side_length=90, - min_spacing=45) + pt = PlaceTurbines( + bsp.wind_plant_pd, + bsp.objective_function, + bsp.capital_cost_function, + bsp.fixed_operating_cost_function, + bsp.variable_operating_cost_function, + include_mask, + pixel_side_length=90, + min_spacing=45, + ) pt.define_exclusions() pt.initialize_packing() @@ -244,22 +284,28 @@ def test_packing_algorithm(gid=33): voc_fun = "" objective_function = "" with tempfile.TemporaryDirectory() as td: - res_fp = os.path.join(td, 'ri_100_wtk_{}.h5') - excl_fp = os.path.join(td, 'ri_exclusions.h5') + res_fp = os.path.join(td, "ri_100_wtk_{}.h5") + excl_fp = os.path.join(td, "ri_exclusions.h5") shutil.copy(EXCL, excl_fp) shutil.copy(RES.format(2012), res_fp.format(2012)) shutil.copy(RES.format(2013), res_fp.format(2013)) - res_fp = res_fp.format('*') + res_fp = res_fp.format("*") TechMapping.run(excl_fp, RES.format(2012), dset=TM_DSET, max_workers=1) - bsp = BespokeSinglePlant(gid, excl_fp, res_fp, TM_DSET, - SAM_SYS_INPUTS, - objective_function, cap_cost_fun, - foc_fun, voc_fun, - ga_kwargs={'max_time': 5}, - excl_dict=EXCL_DICT, - output_request=output_request, - ) + bsp = BespokeSinglePlant( + gid, + excl_fp, + res_fp, + TM_DSET, + SAM_SYS_INPUTS, + objective_function, + cap_cost_fun, + foc_fun, + voc_fun, + ga_kwargs={"max_time": 5}, + excl_dict=EXCL_DICT, + output_request=output_request, + ) optimizer = bsp.plant_optimizer optimizer.define_exclusions() @@ -268,8 +314,8 @@ def test_packing_algorithm(gid=33): assert len(optimizer.x_locations) < 165 assert len(optimizer.x_locations) > 145 assert np.sum(optimizer.include_mask) == ( - optimizer.safe_polygons.area - / (optimizer.pixel_side_length**2)) + optimizer.safe_polygons.area / (optimizer.pixel_side_length**2) + ) bsp.close() @@ -277,89 +323,100 @@ def test_packing_algorithm(gid=33): def test_bespoke_points(): """Test the bespoke points input options""" # pylint: disable=W0612 - points = pd.DataFrame({MetaKeyName.GID: [33, 34, 35], 'config': ['default'] * 3}) - pp = BespokeWindPlants._parse_points(points, {'default': SAM}) + points = pd.DataFrame( + {MetaKeyName.GID: [33, 34, 35], "config": ["default"] * 3} + ) + pp = BespokeWindPlants._parse_points(points, {"default": SAM}) assert len(pp) == 3 for gid in pp.gids: - assert pp[gid][0] == 'default' + assert pp[gid][0] == "default" points = pd.DataFrame({MetaKeyName.GID: [33, 34, 35]}) - pp = BespokeWindPlants._parse_points(points, {'default': SAM}) + pp = BespokeWindPlants._parse_points(points, {"default": SAM}) assert len(pp) == 3 - assert 'config' in pp.df.columns + assert "config" in pp.df.columns for gid in pp.gids: - assert pp[gid][0] == 'default' + assert pp[gid][0] == "default" def test_single(gid=33): """Test a single wind plant bespoke optimization run""" - output_request = ('system_capacity', 'cf_mean', 'cf_profile') + output_request = ("system_capacity", "cf_mean", "cf_profile") with tempfile.TemporaryDirectory() as td: - res_fp = os.path.join(td, 'ri_100_wtk_{}.h5') - excl_fp = os.path.join(td, 'ri_exclusions.h5') + res_fp = os.path.join(td, "ri_100_wtk_{}.h5") + excl_fp = os.path.join(td, "ri_exclusions.h5") shutil.copy(EXCL, excl_fp) shutil.copy(RES.format(2012), res_fp.format(2012)) shutil.copy(RES.format(2013), res_fp.format(2013)) - res_fp = res_fp.format('*') + res_fp = res_fp.format("*") TechMapping.run(excl_fp, RES.format(2012), dset=TM_DSET, max_workers=1) - bsp = BespokeSinglePlant(gid, excl_fp, res_fp, TM_DSET, - SAM_SYS_INPUTS, - OBJECTIVE_FUNCTION, CAP_COST_FUN, - FOC_FUN, VOC_FUN, - ga_kwargs={'max_time': 5}, - excl_dict=EXCL_DICT, - output_request=output_request, - ) + bsp = BespokeSinglePlant( + gid, + excl_fp, + res_fp, + TM_DSET, + SAM_SYS_INPUTS, + OBJECTIVE_FUNCTION, + CAP_COST_FUN, + FOC_FUN, + VOC_FUN, + ga_kwargs={"max_time": 5}, + excl_dict=EXCL_DICT, + output_request=output_request, + ) out = bsp.run_plant_optimization() out = bsp.run_wind_plant_ts() - assert 'cf_profile-2012' in out - assert 'cf_profile-2013' in out - assert 'cf_mean-2012' in out - assert 'cf_mean-2013' in out - assert 'cf_mean-means' in out - assert 'annual_energy-2012' in out - assert 'annual_energy-2013' in out - assert 'annual_energy-means' in out - - assert (TURB_RATING * bsp.meta['n_turbines'].values[0] - == out['system_capacity']) + assert "cf_profile-2012" in out + assert "cf_profile-2013" in out + assert "cf_mean-2012" in out + assert "cf_mean-2013" in out + assert "cf_mean-means" in out + assert "annual_energy-2012" in out + assert "annual_energy-2013" in out + assert "annual_energy-means" in out + + assert ( + TURB_RATING * bsp.meta["n_turbines"].values[0] + == out["system_capacity"] + ) x_coords = json.loads(bsp.meta[MetaKeyName.TURBINE_X_COORDS].values[0]) y_coords = json.loads(bsp.meta[MetaKeyName.TURBINE_Y_COORDS].values[0]) - assert bsp.meta['n_turbines'].values[0] == len(x_coords) - assert bsp.meta['n_turbines'].values[0] == len(y_coords) + assert bsp.meta["n_turbines"].values[0] == len(x_coords) + assert bsp.meta["n_turbines"].values[0] == len(y_coords) for y in (2012, 2013): - cf = out[f'cf_profile-{y}'] + cf = out[f"cf_profile-{y}"] assert cf.min() == 0 assert cf.max() == 1 - assert np.allclose(cf.mean(), out[f'cf_mean-{y}']) + assert np.allclose(cf.mean(), out[f"cf_mean-{y}"]) # simple windpower obj for comparison wp_sam_config = bsp.sam_sys_inputs - wp_sam_config['wind_farm_wake_model'] = 0 - wp_sam_config['wake_int_loss'] = 0 - wp_sam_config['wind_farm_xCoordinates'] = [0] - wp_sam_config['wind_farm_yCoordinates'] = [0] - wp_sam_config['system_capacity'] = TURB_RATING + wp_sam_config["wind_farm_wake_model"] = 0 + wp_sam_config["wake_int_loss"] = 0 + wp_sam_config["wind_farm_xCoordinates"] = [0] + wp_sam_config["wind_farm_yCoordinates"] = [0] + wp_sam_config["system_capacity"] = TURB_RATING res_df = bsp.res_df[(bsp.res_df.index.year == 2012)].copy() - wp = WindPower(res_df, bsp.meta, wp_sam_config, - output_request=bsp._out_req) + wp = WindPower( + res_df, bsp.meta, wp_sam_config, output_request=bsp._out_req + ) wp.run() # make sure the wind resource was loaded correctly - res_ideal = np.array(wp['wind_resource_data']['data']) + res_ideal = np.array(wp["wind_resource_data"]["data"]) bsp_2012 = bsp.wind_plant_ts[2012] - res_bsp = np.array(bsp_2012['wind_resource_data']['data']) + res_bsp = np.array(bsp_2012["wind_resource_data"]["data"]) ws_ideal = res_ideal[:, 2] ws_bsp = res_bsp[:, 2] assert np.allclose(ws_ideal, ws_bsp) # make sure that the zero-losses analysis has greater CF - cf_bespoke = out['cf_profile-2012'] - cf_ideal = wp.outputs['cf_profile'] + cf_bespoke = out["cf_profile-2012"] + cf_ideal = wp.outputs["cf_profile"] diff = cf_ideal - cf_bespoke assert all(diff > -0.00001) assert diff.mean() > 0.02 @@ -369,156 +426,212 @@ def test_single(gid=33): def test_extra_outputs(gid=33): """Test running bespoke single farm optimization with lcoe requests""" - output_request = ('system_capacity', 'cf_mean', 'cf_profile', 'lcoe_fcr') + output_request = ("system_capacity", "cf_mean", "cf_profile", "lcoe_fcr") objective_function = ( - '(fixed_charge_rate * capital_cost + fixed_operating_cost) ' - '/ aep + variable_operating_cost') + "(fixed_charge_rate * capital_cost + fixed_operating_cost) " + "/ aep + variable_operating_cost" + ) with tempfile.TemporaryDirectory() as td: - res_fp = os.path.join(td, 'ri_100_wtk_{}.h5') - excl_fp = os.path.join(td, 'ri_exclusions.h5') + res_fp = os.path.join(td, "ri_100_wtk_{}.h5") + excl_fp = os.path.join(td, "ri_exclusions.h5") shutil.copy(EXCL, excl_fp) shutil.copy(RES.format(2012), res_fp.format(2012)) shutil.copy(RES.format(2013), res_fp.format(2013)) - res_fp = res_fp.format('*') + res_fp = res_fp.format("*") TechMapping.run(excl_fp, RES.format(2012), dset=TM_DSET, max_workers=1) with pytest.raises(KeyError): - bsp = BespokeSinglePlant(gid, excl_fp, res_fp, TM_DSET, - SAM_SYS_INPUTS, - objective_function, CAP_COST_FUN, - FOC_FUN, VOC_FUN, - ga_kwargs={'max_time': 5}, - excl_dict=EXCL_DICT, - output_request=output_request, - ) + bsp = BespokeSinglePlant( + gid, + excl_fp, + res_fp, + TM_DSET, + SAM_SYS_INPUTS, + objective_function, + CAP_COST_FUN, + FOC_FUN, + VOC_FUN, + ga_kwargs={"max_time": 5}, + excl_dict=EXCL_DICT, + output_request=output_request, + ) sam_sys_inputs = copy.deepcopy(SAM_SYS_INPUTS) - sam_sys_inputs['fixed_charge_rate'] = 0.0975 + sam_sys_inputs["fixed_charge_rate"] = 0.0975 test_eos_cap = 200_000 - bsp = BespokeSinglePlant(gid, excl_fp, res_fp, TM_DSET, - sam_sys_inputs, - objective_function, CAP_COST_FUN, - FOC_FUN, VOC_FUN, - ga_kwargs={'max_time': 5}, - excl_dict=EXCL_DICT, - output_request=output_request, - data_layers=copy.deepcopy(DATA_LAYERS), - eos_mult_baseline_cap_mw=test_eos_cap * 1e-3 - ) + bsp = BespokeSinglePlant( + gid, + excl_fp, + res_fp, + TM_DSET, + sam_sys_inputs, + objective_function, + CAP_COST_FUN, + FOC_FUN, + VOC_FUN, + ga_kwargs={"max_time": 5}, + excl_dict=EXCL_DICT, + output_request=output_request, + data_layers=copy.deepcopy(DATA_LAYERS), + eos_mult_baseline_cap_mw=test_eos_cap * 1e-3, + ) out = bsp.run_plant_optimization() out = bsp.run_wind_plant_ts() bsp.agg_data_layers() - assert 'lcoe_fcr-2012' in out - assert 'lcoe_fcr-2013' in out - assert 'lcoe_fcr-means' in out + assert "lcoe_fcr-2012" in out + assert "lcoe_fcr-2013" in out + assert "lcoe_fcr-means" in out assert MetaKeyName.CAPACITY in bsp.meta assert MetaKeyName.MEAN_CF in bsp.meta assert MetaKeyName.MEAN_LCOE in bsp.meta - assert 'pct_slope' in bsp.meta - assert 'reeds_region' in bsp.meta - assert 'padus' in bsp.meta + assert "pct_slope" in bsp.meta + assert "reeds_region" in bsp.meta + assert "padus" in bsp.meta out = None data_layers = copy.deepcopy(DATA_LAYERS) for layer in data_layers: - data_layers[layer].pop('fpath', None) + data_layers[layer].pop("fpath", None) for layer in data_layers: - assert 'fpath' not in data_layers[layer] - - bsp = BespokeSinglePlant(gid, excl_fp, res_fp, TM_DSET, - sam_sys_inputs, - objective_function, CAP_COST_FUN, - FOC_FUN, VOC_FUN, - ga_kwargs={'max_time': 5}, - excl_dict=EXCL_DICT, - output_request=output_request, - data_layers=data_layers, - ) + assert "fpath" not in data_layers[layer] + + bsp = BespokeSinglePlant( + gid, + excl_fp, + res_fp, + TM_DSET, + sam_sys_inputs, + objective_function, + CAP_COST_FUN, + FOC_FUN, + VOC_FUN, + ga_kwargs={"max_time": 5}, + excl_dict=EXCL_DICT, + output_request=output_request, + data_layers=data_layers, + ) out = bsp.run_plant_optimization() out = bsp.run_wind_plant_ts() bsp.agg_data_layers() - assert 'lcoe_fcr-2012' in out - assert 'lcoe_fcr-2013' in out - assert 'lcoe_fcr-means' in out + assert "lcoe_fcr-2012" in out + assert "lcoe_fcr-2013" in out + assert "lcoe_fcr-means" in out assert MetaKeyName.CAPACITY in bsp.meta assert MetaKeyName.MEAN_CF in bsp.meta assert MetaKeyName.MEAN_LCOE in bsp.meta - assert 'pct_slope' in bsp.meta - assert 'reeds_region' in bsp.meta - assert 'padus' in bsp.meta + assert "pct_slope" in bsp.meta + assert "reeds_region" in bsp.meta + assert "padus" in bsp.meta - assert 'eos_mult' in bsp.meta - assert 'reg_mult' in bsp.meta - assert np.allclose(bsp.meta['reg_mult'], 1) + assert "eos_mult" in bsp.meta + assert "reg_mult" in bsp.meta + assert np.allclose(bsp.meta["reg_mult"], 1) n_turbs = round(test_eos_cap / TURB_RATING) test_eos_cap_kw = n_turbs * TURB_RATING - baseline_cost = (140 * test_eos_cap_kw - * np.exp(-test_eos_cap_kw / 1E5 * 0.1 + (1 - 0.1))) - eos_mult = (bsp.plant_optimizer.capital_cost - / bsp.plant_optimizer.capacity - / (baseline_cost / test_eos_cap_kw)) - assert np.allclose(bsp.meta['eos_mult'], eos_mult) + baseline_cost = ( + 140 + * test_eos_cap_kw + * np.exp(-test_eos_cap_kw / 1e5 * 0.1 + (1 - 0.1)) + ) + eos_mult = ( + bsp.plant_optimizer.capital_cost + / bsp.plant_optimizer.capacity + / (baseline_cost / test_eos_cap_kw) + ) + assert np.allclose(bsp.meta["eos_mult"], eos_mult) bsp.close() def test_bespoke(): """Test bespoke optimization with multiple plants, parallel processing, and - file output. """ - output_request = ('system_capacity', 'cf_mean', 'cf_profile', - 'extra_unused_data', 'winddirection', 'windspeed', - 'ws_mean') + file output.""" + output_request = ( + "system_capacity", + "cf_mean", + "cf_profile", + "extra_unused_data", + "winddirection", + "windspeed", + "ws_mean", + ) with tempfile.TemporaryDirectory() as td: - out_fpath_request = os.path.join(td, 'wind') - out_fpath_truth = os.path.join(td, 'wind_bespoke.h5') - res_fp = os.path.join(td, 'ri_100_wtk_{}.h5') - excl_fp = os.path.join(td, 'ri_exclusions.h5') + out_fpath_request = os.path.join(td, "wind") + out_fpath_truth = os.path.join(td, "wind_bespoke.h5") + res_fp = os.path.join(td, "ri_100_wtk_{}.h5") + excl_fp = os.path.join(td, "ri_exclusions.h5") shutil.copy(EXCL, excl_fp) shutil.copy(RES.format(2012), res_fp.format(2012)) shutil.copy(RES.format(2013), res_fp.format(2013)) - res_fp = res_fp.format('*') + res_fp = res_fp.format("*") # both 33 and 35 are included, 37 is fully excluded - points = pd.DataFrame({MetaKeyName.GID: [33, 35], 'config': ['default'] * 2, - 'extra_unused_data': [0, 42]}) - fully_excluded_points = pd.DataFrame({MetaKeyName.GID: [37], - 'config': ['default'], - 'extra_unused_data': [0]}) + points = pd.DataFrame( + { + MetaKeyName.GID: [33, 35], + "config": ["default"] * 2, + "extra_unused_data": [0, 42], + } + ) + fully_excluded_points = pd.DataFrame( + { + MetaKeyName.GID: [37], + "config": ["default"], + "extra_unused_data": [0], + } + ) TechMapping.run(excl_fp, RES.format(2012), dset=TM_DSET, max_workers=1) # test no outputs with pytest.warns(UserWarning) as record: assert not os.path.exists(out_fpath_truth) - bsp = BespokeWindPlants(excl_fp, res_fp, TM_DSET, - OBJECTIVE_FUNCTION, CAP_COST_FUN, - FOC_FUN, VOC_FUN, fully_excluded_points, - SAM_CONFIGS, ga_kwargs={'max_time': 5}, - excl_dict=EXCL_DICT, - output_request=output_request) + bsp = BespokeWindPlants( + excl_fp, + res_fp, + TM_DSET, + OBJECTIVE_FUNCTION, + CAP_COST_FUN, + FOC_FUN, + VOC_FUN, + fully_excluded_points, + SAM_CONFIGS, + ga_kwargs={"max_time": 5}, + excl_dict=EXCL_DICT, + output_request=output_request, + ) test_fpath = bsp.run(max_workers=2, out_fpath=out_fpath_request) assert out_fpath_truth == test_fpath - assert 'points are excluded' in str(record[0].message) + assert "points are excluded" in str(record[0].message) assert not os.path.exists(out_fpath_truth) - bsp = BespokeWindPlants(excl_fp, res_fp, TM_DSET, OBJECTIVE_FUNCTION, - CAP_COST_FUN, FOC_FUN, VOC_FUN, points, - SAM_CONFIGS, ga_kwargs={'max_time': 5}, - excl_dict=EXCL_DICT, - output_request=output_request) + bsp = BespokeWindPlants( + excl_fp, + res_fp, + TM_DSET, + OBJECTIVE_FUNCTION, + CAP_COST_FUN, + FOC_FUN, + VOC_FUN, + points, + SAM_CONFIGS, + ga_kwargs={"max_time": 5}, + excl_dict=EXCL_DICT, + output_request=output_request, + ) test_fpath = bsp.run(max_workers=2, out_fpath=out_fpath_request) assert out_fpath_truth == test_fpath assert os.path.exists(out_fpath_truth) @@ -528,13 +641,18 @@ def test_bespoke(): assert MetaKeyName.SC_POINT_GID in meta assert MetaKeyName.TURBINE_X_COORDS in meta assert MetaKeyName.TURBINE_Y_COORDS in meta - assert 'possible_x_coords' in meta - assert 'possible_y_coords' in meta + assert "possible_x_coords" in meta + assert "possible_y_coords" in meta assert MetaKeyName.RES_GIDS in meta - dsets_1d = ('system_capacity', 'cf_mean-2012', - 'annual_energy-2012', 'cf_mean-means', - 'extra_unused_data-2012', 'ws_mean') + dsets_1d = ( + "system_capacity", + "cf_mean-2012", + "annual_energy-2012", + "cf_mean-means", + "extra_unused_data-2012", + "ws_mean", + ) for dset in dsets_1d: assert dset in list(f) assert isinstance(f[dset], np.ndarray) @@ -542,8 +660,12 @@ def test_bespoke(): assert len(f[dset]) == len(meta) assert f[dset].any() # not all zeros - dsets_2d = ('cf_profile-2012', 'cf_profile-2013', - 'windspeed-2012', 'windspeed-2013') + dsets_2d = ( + "cf_profile-2012", + "cf_profile-2013", + "windspeed-2012", + "windspeed-2013", + ) for dset in dsets_2d: assert dset in list(f) assert isinstance(f[dset], np.ndarray) @@ -552,83 +674,112 @@ def test_bespoke(): assert f[dset].shape[1] == len(meta) assert f[dset].any() # not all zeros - out_fpath_pre = os.path.join(td, 'bespoke_out_pre.h5') - bsp = BespokeWindPlants(excl_fp, res_fp, TM_DSET, OBJECTIVE_FUNCTION, - CAP_COST_FUN, FOC_FUN, VOC_FUN, points, - SAM_CONFIGS, ga_kwargs={'max_time': 1}, - excl_dict=EXCL_DICT, - output_request=output_request, - pre_load_data=True) + out_fpath_pre = os.path.join(td, "bespoke_out_pre.h5") + bsp = BespokeWindPlants( + excl_fp, + res_fp, + TM_DSET, + OBJECTIVE_FUNCTION, + CAP_COST_FUN, + FOC_FUN, + VOC_FUN, + points, + SAM_CONFIGS, + ga_kwargs={"max_time": 1}, + excl_dict=EXCL_DICT, + output_request=output_request, + pre_load_data=True, + ) bsp.run(max_workers=1, out_fpath=out_fpath_pre) with Resource(out_fpath_truth) as f1, Resource(out_fpath_pre) as f2: - assert np.allclose(f1["winddirection-2012"], - f2["winddirection-2012"]) + assert np.allclose( + f1["winddirection-2012"], f2["winddirection-2012"] + ) assert np.allclose(f1["ws_mean"], f2["ws_mean"]) def test_collect_bespoke(): """Test the collection of multiple chunked bespoke files.""" with tempfile.TemporaryDirectory() as td: - source_dir = os.path.join(TESTDATADIR, 'bespoke/') - source_pattern = source_dir + '/test_bespoke*.h5' + source_dir = os.path.join(TESTDATADIR, "bespoke/") + source_pattern = source_dir + "/test_bespoke*.h5" source_fps = sorted(glob(source_pattern)) assert len(source_fps) > 1 - h5_file = os.path.join(td, 'collection.h5') + h5_file = os.path.join(td, "collection.h5") collector = Collector(h5_file, source_pattern, None) - collector.collect('cf_profile-2012') + collector.collect("cf_profile-2012") with Resource(h5_file) as fout: meta = fout.meta - assert all(meta[MetaKeyName.GID].values == sorted(meta[MetaKeyName.GID].values)) + assert all( + meta[MetaKeyName.GID].values + == sorted(meta[MetaKeyName.GID].values) + ) ti = fout.time_index assert len(ti) == 8760 - assert 'time_index-2012' in fout - assert 'time_index-2013' in fout - data = fout['cf_profile-2012'] + assert "time_index-2012" in fout + assert "time_index-2013" in fout + data = fout["cf_profile-2012"] for fp in source_fps: with Resource(fp) as source: - assert all(np.isin(source.meta[MetaKeyName.GID].values, - meta[MetaKeyName.GID].values)) - for isource, gid in enumerate(source.meta[MetaKeyName.GID].values): + assert all( + np.isin( + source.meta[MetaKeyName.GID].values, + meta[MetaKeyName.GID].values, + ) + ) + for isource, gid in enumerate( + source.meta[MetaKeyName.GID].values + ): iout = np.where(meta[MetaKeyName.GID].values == gid)[0] - truth = source['cf_profile-2012', :, isource].flatten() + truth = source["cf_profile-2012", :, isource].flatten() test = data[:, iout].flatten() assert np.allclose(truth, test) def test_consistent_eval_namespace(gid=33): """Test that all the same variables are available for every eval.""" - output_request = ('system_capacity', 'cf_mean', 'cf_profile') + output_request = ("system_capacity", "cf_mean", "cf_profile") cap_cost_fun = "2000" foc_fun = "0" voc_fun = "0" - objective_function = ("n_turbines + id(self.wind_plant) " - "+ system_capacity + capital_cost + aep") + objective_function = ( + "n_turbines + id(self.wind_plant) " + "+ system_capacity + capital_cost + aep" + ) with tempfile.TemporaryDirectory() as td: - res_fp = os.path.join(td, 'ri_100_wtk_{}.h5') - excl_fp = os.path.join(td, 'ri_exclusions.h5') + res_fp = os.path.join(td, "ri_100_wtk_{}.h5") + excl_fp = os.path.join(td, "ri_exclusions.h5") shutil.copy(EXCL, excl_fp) shutil.copy(RES.format(2012), res_fp.format(2012)) shutil.copy(RES.format(2013), res_fp.format(2013)) - res_fp = res_fp.format('*') + res_fp = res_fp.format("*") TechMapping.run(excl_fp, RES.format(2012), dset=TM_DSET, max_workers=1) - bsp = BespokeSinglePlant(gid, excl_fp, res_fp, TM_DSET, - SAM_SYS_INPUTS, - objective_function, cap_cost_fun, - foc_fun, voc_fun, - ga_kwargs={'max_time': 5}, - excl_dict=EXCL_DICT, - output_request=output_request, - ) + bsp = BespokeSinglePlant( + gid, + excl_fp, + res_fp, + TM_DSET, + SAM_SYS_INPUTS, + objective_function, + cap_cost_fun, + foc_fun, + voc_fun, + ga_kwargs={"max_time": 5}, + excl_dict=EXCL_DICT, + output_request=output_request, + ) _ = bsp.run_plant_optimization() assert bsp.meta["bespoke_aep"].values[0] == bsp.plant_optimizer.aep - assert (bsp.meta["bespoke_objective"].values[0] - == bsp.plant_optimizer.objective) + assert ( + bsp.meta["bespoke_objective"].values[0] + == bsp.plant_optimizer.objective + ) bsp.close() @@ -637,67 +788,78 @@ def test_bespoke_supply_curve(): """Test supply curve compute from a bespoke output that acts as the traditional reV-sc-aggregation output table.""" - bespoke_sample_fout = os.path.join(TESTDATADIR, - 'bespoke/test_bespoke_node00.h5') + bespoke_sample_fout = os.path.join( + TESTDATADIR, "bespoke/test_bespoke_node00.h5" + ) - normal_path = os.path.join(TESTDATADIR, 'sc_out/baseline_agg_summary.csv') + normal_path = os.path.join(TESTDATADIR, "sc_out/baseline_agg_summary.csv") normal_sc_points = pd.read_csv(normal_path) with tempfile.TemporaryDirectory() as td: - bespoke_sc_fp = os.path.join(td, 'bespoke_out.h5') + bespoke_sc_fp = os.path.join(td, "bespoke_out.h5") shutil.copy(bespoke_sample_fout, bespoke_sc_fp) - with h5py.File(bespoke_sc_fp, 'a') as f: - del f['meta'] - with Outputs(bespoke_sc_fp, mode='a') as f: + with h5py.File(bespoke_sc_fp, "a") as f: + del f["meta"] + with Outputs(bespoke_sc_fp, mode="a") as f: bespoke_meta = normal_sc_points.copy() bespoke_meta = bespoke_meta.drop(MetaKeyName.SC_GID, axis=1) f.meta = bespoke_meta # this is basically copied from test_supply_curve_compute.py - trans_tables = [os.path.join(TESTDATADIR, 'trans_tables', - f'costs_RI_{cap}MW.csv') - for cap in [100, 200, 400, 1000]] + trans_tables = [ + os.path.join(TESTDATADIR, "trans_tables", f"costs_RI_{cap}MW.csv") + for cap in [100, 200, 400, 1000] + ] sc = SupplyCurve(bespoke_sc_fp, trans_tables) sc_full = sc.full_sort(fcr=0.1, avail_cap_frac=0.1) - assert all(gid in sc_full[MetaKeyName.SC_GID] - for gid in normal_sc_points[MetaKeyName.SC_GID]) + assert all( + gid in sc_full[MetaKeyName.SC_GID] + for gid in normal_sc_points[MetaKeyName.SC_GID] + ) for _, inp_row in normal_sc_points.iterrows(): sc_gid = inp_row[MetaKeyName.SC_GID] assert sc_gid in sc_full[MetaKeyName.SC_GID] test_ind = np.where(sc_full[MetaKeyName.SC_GID] == sc_gid)[0] assert len(test_ind) == 1 test_row = sc_full.iloc[test_ind] - assert test_row['total_lcoe'].values[0] > inp_row[MetaKeyName.MEAN_LCOE] + assert ( + test_row["total_lcoe"].values[0] + > inp_row[MetaKeyName.MEAN_LCOE] + ) - fpath_baseline = os.path.join(TESTDATADIR, 'sc_out/sc_full_lc.csv') + fpath_baseline = os.path.join(TESTDATADIR, "sc_out/sc_full_lc.csv") sc_baseline = pd.read_csv(fpath_baseline) - assert np.allclose(sc_baseline['total_lcoe'], sc_full['total_lcoe']) + assert np.allclose(sc_baseline["total_lcoe"], sc_full["total_lcoe"]) -@pytest.mark.parametrize('wlm', [2, 100]) +@pytest.mark.parametrize("wlm", [2, 100]) def test_wake_loss_multiplier(wlm): """Test wake loss multiplier.""" - output_request = ('system_capacity', 'cf_mean', 'cf_profile') + output_request = ("system_capacity", "cf_mean", "cf_profile") with tempfile.TemporaryDirectory() as td: - res_fp = os.path.join(td, 'ri_100_wtk_{}.h5') - excl_fp = os.path.join(td, 'ri_exclusions.h5') + res_fp = os.path.join(td, "ri_100_wtk_{}.h5") + excl_fp = os.path.join(td, "ri_exclusions.h5") shutil.copy(EXCL, excl_fp) shutil.copy(RES.format(2012), res_fp.format(2012)) shutil.copy(RES.format(2013), res_fp.format(2013)) - res_fp = res_fp.format('*') + res_fp = res_fp.format("*") TechMapping.run(excl_fp, RES.format(2012), dset=TM_DSET, max_workers=1) - bsp = BespokeSinglePlant(33, excl_fp, res_fp, TM_DSET, - SAM_SYS_INPUTS, - OBJECTIVE_FUNCTION, - CAP_COST_FUN, - FOC_FUN, - VOC_FUN, - excl_dict=EXCL_DICT, - output_request=output_request, - ) + bsp = BespokeSinglePlant( + 33, + excl_fp, + res_fp, + TM_DSET, + SAM_SYS_INPUTS, + OBJECTIVE_FUNCTION, + CAP_COST_FUN, + FOC_FUN, + VOC_FUN, + excl_dict=EXCL_DICT, + output_request=output_request, + ) optimizer = bsp.plant_optimizer optimizer.define_exclusions() @@ -706,8 +868,9 @@ def test_wake_loss_multiplier(wlm): optimizer.wind_plant["wind_farm_xCoordinates"] = optimizer.x_locations optimizer.wind_plant["wind_farm_yCoordinates"] = optimizer.y_locations - system_capacity = (len(optimizer.x_locations) - * optimizer.turbine_capacity) + system_capacity = ( + len(optimizer.x_locations) * optimizer.turbine_capacity + ) optimizer.wind_plant["system_capacity"] = system_capacity optimizer.wind_plant.assign_inputs() @@ -715,22 +878,28 @@ def test_wake_loss_multiplier(wlm): aep = optimizer._aep_after_scaled_wake_losses() bsp.close() - bsp = BespokeSinglePlant(33, excl_fp, res_fp, TM_DSET, - SAM_SYS_INPUTS, - OBJECTIVE_FUNCTION, - CAP_COST_FUN, - FOC_FUN, - VOC_FUN, - excl_dict=EXCL_DICT, - output_request=output_request, - wake_loss_multiplier=wlm) + bsp = BespokeSinglePlant( + 33, + excl_fp, + res_fp, + TM_DSET, + SAM_SYS_INPUTS, + OBJECTIVE_FUNCTION, + CAP_COST_FUN, + FOC_FUN, + VOC_FUN, + excl_dict=EXCL_DICT, + output_request=output_request, + wake_loss_multiplier=wlm, + ) optimizer2 = bsp.plant_optimizer optimizer2.wind_plant["wind_farm_xCoordinates"] = optimizer.x_locations optimizer2.wind_plant["wind_farm_yCoordinates"] = optimizer.y_locations - system_capacity = (len(optimizer.x_locations) - * optimizer.turbine_capacity) + system_capacity = ( + len(optimizer.x_locations) * optimizer.turbine_capacity + ) optimizer2.wind_plant["system_capacity"] = system_capacity optimizer2.wind_plant.assign_inputs() @@ -744,25 +913,29 @@ def test_wake_loss_multiplier(wlm): def test_bespoke_wind_plant_with_power_curve_losses(): """Test bespoke ``wind_plant`` with power curve losses.""" - output_request = ('system_capacity', 'cf_mean', 'cf_profile') + output_request = ("system_capacity", "cf_mean", "cf_profile") with tempfile.TemporaryDirectory() as td: - res_fp = os.path.join(td, 'ri_100_wtk_{}.h5') - excl_fp = os.path.join(td, 'ri_exclusions.h5') + res_fp = os.path.join(td, "ri_100_wtk_{}.h5") + excl_fp = os.path.join(td, "ri_exclusions.h5") shutil.copy(EXCL, excl_fp) shutil.copy(RES.format(2012), res_fp.format(2012)) shutil.copy(RES.format(2013), res_fp.format(2013)) - res_fp = res_fp.format('*') + res_fp = res_fp.format("*") TechMapping.run(excl_fp, RES.format(2012), dset=TM_DSET, max_workers=1) - bsp = BespokeSinglePlant(33, excl_fp, res_fp, TM_DSET, - SAM_SYS_INPUTS, - OBJECTIVE_FUNCTION, - CAP_COST_FUN, - FOC_FUN, - VOC_FUN, - excl_dict=EXCL_DICT, - output_request=output_request, - ) + bsp = BespokeSinglePlant( + 33, + excl_fp, + res_fp, + TM_DSET, + SAM_SYS_INPUTS, + OBJECTIVE_FUNCTION, + CAP_COST_FUN, + FOC_FUN, + VOC_FUN, + excl_dict=EXCL_DICT, + output_request=output_request, + ) optimizer = bsp.plant_optimizer optimizer.wind_plant["wind_farm_xCoordinates"] = [1000, -1000] @@ -777,17 +950,22 @@ def test_bespoke_wind_plant_with_power_curve_losses(): sam_inputs = copy.deepcopy(SAM_SYS_INPUTS) sam_inputs[PowerCurveLossesMixin.POWER_CURVE_CONFIG_KEY] = { - 'target_losses_percent': 10, - 'transformation': 'exponential_stretching' + "target_losses_percent": 10, + "transformation": "exponential_stretching", } - bsp = BespokeSinglePlant(33, excl_fp, res_fp, TM_DSET, - sam_inputs, - OBJECTIVE_FUNCTION, - CAP_COST_FUN, - FOC_FUN, - VOC_FUN, - excl_dict=EXCL_DICT, - output_request=output_request) + bsp = BespokeSinglePlant( + 33, + excl_fp, + res_fp, + TM_DSET, + sam_inputs, + OBJECTIVE_FUNCTION, + CAP_COST_FUN, + FOC_FUN, + VOC_FUN, + excl_dict=EXCL_DICT, + output_request=output_request, + ) optimizer2 = bsp.plant_optimizer optimizer2.wind_plant["wind_farm_xCoordinates"] = [1000, -1000] @@ -808,23 +986,30 @@ def test_bespoke_wind_plant_with_power_curve_losses(): def test_bespoke_run_with_power_curve_losses(): """Test bespoke run with power curve losses.""" - output_request = ('system_capacity', 'cf_mean', 'cf_profile') + output_request = ("system_capacity", "cf_mean", "cf_profile") with tempfile.TemporaryDirectory() as td: - res_fp = os.path.join(td, 'ri_100_wtk_{}.h5') - excl_fp = os.path.join(td, 'ri_exclusions.h5') + res_fp = os.path.join(td, "ri_100_wtk_{}.h5") + excl_fp = os.path.join(td, "ri_exclusions.h5") shutil.copy(EXCL, excl_fp) shutil.copy(RES.format(2012), res_fp.format(2012)) shutil.copy(RES.format(2013), res_fp.format(2013)) - res_fp = res_fp.format('*') + res_fp = res_fp.format("*") TechMapping.run(excl_fp, RES.format(2012), dset=TM_DSET, max_workers=1) - bsp = BespokeSinglePlant(33, excl_fp, res_fp, TM_DSET, - SAM_SYS_INPUTS, - OBJECTIVE_FUNCTION, CAP_COST_FUN, - FOC_FUN, VOC_FUN, - ga_kwargs={'max_time': 5}, - excl_dict=EXCL_DICT, - output_request=output_request) + bsp = BespokeSinglePlant( + 33, + excl_fp, + res_fp, + TM_DSET, + SAM_SYS_INPUTS, + OBJECTIVE_FUNCTION, + CAP_COST_FUN, + FOC_FUN, + VOC_FUN, + ga_kwargs={"max_time": 5}, + excl_dict=EXCL_DICT, + output_request=output_request, + ) out = bsp.run_plant_optimization() out = bsp.run_wind_plant_ts() @@ -832,26 +1017,33 @@ def test_bespoke_run_with_power_curve_losses(): sam_inputs = copy.deepcopy(SAM_SYS_INPUTS) sam_inputs[PowerCurveLossesMixin.POWER_CURVE_CONFIG_KEY] = { - 'target_losses_percent': 10, - 'transformation': 'exponential_stretching' + "target_losses_percent": 10, + "transformation": "exponential_stretching", } - bsp = BespokeSinglePlant(33, excl_fp, res_fp, TM_DSET, - sam_inputs, - OBJECTIVE_FUNCTION, - CAP_COST_FUN, - FOC_FUN, - VOC_FUN, - ga_kwargs={'max_time': 5}, - excl_dict=EXCL_DICT, - output_request=output_request) + bsp = BespokeSinglePlant( + 33, + excl_fp, + res_fp, + TM_DSET, + sam_inputs, + OBJECTIVE_FUNCTION, + CAP_COST_FUN, + FOC_FUN, + VOC_FUN, + ga_kwargs={"max_time": 5}, + excl_dict=EXCL_DICT, + output_request=output_request, + ) out_losses = bsp.run_plant_optimization() out_losses = bsp.run_wind_plant_ts() bsp.close() - ae_dsets = ['annual_energy-2012', - 'annual_energy-2013', - 'annual_energy-means'] + ae_dsets = [ + "annual_energy-2012", + "annual_energy-2013", + "annual_energy-means", + ] for dset in ae_dsets: assert not np.isclose(out[dset], out_losses[dset]) assert out[dset] > out_losses[dset] @@ -859,88 +1051,117 @@ def test_bespoke_run_with_power_curve_losses(): def test_bespoke_run_with_scheduled_losses(): """Test bespoke run with scheduled losses.""" - output_request = ('system_capacity', 'cf_mean', 'cf_profile') + output_request = ("system_capacity", "cf_mean", "cf_profile") with tempfile.TemporaryDirectory() as td: - res_fp = os.path.join(td, 'ri_100_wtk_{}.h5') - excl_fp = os.path.join(td, 'ri_exclusions.h5') + res_fp = os.path.join(td, "ri_100_wtk_{}.h5") + excl_fp = os.path.join(td, "ri_exclusions.h5") shutil.copy(EXCL, excl_fp) shutil.copy(RES.format(2012), res_fp.format(2012)) shutil.copy(RES.format(2013), res_fp.format(2013)) - res_fp = res_fp.format('*') + res_fp = res_fp.format("*") TechMapping.run(excl_fp, RES.format(2012), dset=TM_DSET, max_workers=1) - bsp = BespokeSinglePlant(33, excl_fp, res_fp, TM_DSET, - SAM_SYS_INPUTS, - OBJECTIVE_FUNCTION, CAP_COST_FUN, - FOC_FUN, VOC_FUN, - ga_kwargs={'max_time': 5}, - excl_dict=EXCL_DICT, - output_request=output_request) + bsp = BespokeSinglePlant( + 33, + excl_fp, + res_fp, + TM_DSET, + SAM_SYS_INPUTS, + OBJECTIVE_FUNCTION, + CAP_COST_FUN, + FOC_FUN, + VOC_FUN, + ga_kwargs={"max_time": 5}, + excl_dict=EXCL_DICT, + output_request=output_request, + ) out = bsp.run_plant_optimization() out = bsp.run_wind_plant_ts() bsp.close() sam_inputs = copy.deepcopy(SAM_SYS_INPUTS) - sam_inputs[ScheduledLossesMixin.OUTAGE_CONFIG_KEY] = [{ - 'name': 'Environmental', - 'count': 115, - 'duration': 2, - 'percentage_of_capacity_lost': 100, - 'allowed_months': ['April', 'May', 'June', 'July', 'August', - 'September', 'October']}] - sam_inputs['hourly'] = [0] * 8760 # only needed for testing - output_request = ('system_capacity', 'cf_mean', 'cf_profile', 'hourly') - - bsp = BespokeSinglePlant(33, excl_fp, res_fp, TM_DSET, - sam_inputs, - OBJECTIVE_FUNCTION, - CAP_COST_FUN, - FOC_FUN, - VOC_FUN, - ga_kwargs={'max_time': 5}, - excl_dict=EXCL_DICT, - output_request=output_request) + sam_inputs[ScheduledLossesMixin.OUTAGE_CONFIG_KEY] = [ + { + "name": "Environmental", + "count": 115, + "duration": 2, + "percentage_of_capacity_lost": 100, + "allowed_months": [ + "April", + "May", + "June", + "July", + "August", + "September", + "October", + ], + } + ] + sam_inputs["hourly"] = [0] * 8760 # only needed for testing + output_request = ("system_capacity", "cf_mean", "cf_profile", "hourly") + + bsp = BespokeSinglePlant( + 33, + excl_fp, + res_fp, + TM_DSET, + sam_inputs, + OBJECTIVE_FUNCTION, + CAP_COST_FUN, + FOC_FUN, + VOC_FUN, + ga_kwargs={"max_time": 5}, + excl_dict=EXCL_DICT, + output_request=output_request, + ) out_losses = bsp.run_plant_optimization() out_losses = bsp.run_wind_plant_ts() bsp.close() - ae_dsets = ['annual_energy-2012', - 'annual_energy-2013', - 'annual_energy-means'] + ae_dsets = [ + "annual_energy-2012", + "annual_energy-2013", + "annual_energy-means", + ] for dset in ae_dsets: assert not np.isclose(out[dset], out_losses[dset]) assert out[dset] > out_losses[dset] - assert not np.allclose(out_losses['hourly-2012'], - out_losses['hourly-2013']) + assert not np.allclose( + out_losses["hourly-2012"], out_losses["hourly-2013"] + ) def test_bespoke_aep_is_zero_if_no_turbines_placed(): """Test that bespoke aep output is zero if no turbines placed.""" - output_request = ('system_capacity', 'cf_mean', 'cf_profile') + output_request = ("system_capacity", "cf_mean", "cf_profile") - objective_function = 'aep' + objective_function = "aep" with tempfile.TemporaryDirectory() as td: - res_fp = os.path.join(td, 'ri_100_wtk_{}.h5') - excl_fp = os.path.join(td, 'ri_exclusions.h5') + res_fp = os.path.join(td, "ri_100_wtk_{}.h5") + excl_fp = os.path.join(td, "ri_exclusions.h5") shutil.copy(EXCL, excl_fp) shutil.copy(RES.format(2012), res_fp.format(2012)) shutil.copy(RES.format(2013), res_fp.format(2013)) - res_fp = res_fp.format('*') + res_fp = res_fp.format("*") TechMapping.run(excl_fp, RES.format(2012), dset=TM_DSET, max_workers=1) - bsp = BespokeSinglePlant(33, excl_fp, res_fp, TM_DSET, - SAM_SYS_INPUTS, - objective_function, - CAP_COST_FUN, - FOC_FUN, - VOC_FUN, - excl_dict=EXCL_DICT, - output_request=output_request, - ) + bsp = BespokeSinglePlant( + 33, + excl_fp, + res_fp, + TM_DSET, + SAM_SYS_INPUTS, + objective_function, + CAP_COST_FUN, + FOC_FUN, + VOC_FUN, + excl_dict=EXCL_DICT, + output_request=output_request, + ) optimizer = bsp.plant_optimizer optimizer.define_exclusions() @@ -963,54 +1184,82 @@ def test_bespoke_prior_run(): single vertical level (e.g., with Sup3rCC data) """ sam_sys_inputs = copy.deepcopy(SAM_SYS_INPUTS) - sam_sys_inputs['fixed_charge_rate'] = 0.096 - sam_configs = {'default': sam_sys_inputs} - output_request = ('system_capacity', 'cf_mean', 'cf_profile', - 'extra_unused_data', 'lcoe_fcr') + sam_sys_inputs["fixed_charge_rate"] = 0.096 + sam_configs = {"default": sam_sys_inputs} + output_request = ( + "system_capacity", + "cf_mean", + "cf_profile", + "extra_unused_data", + "lcoe_fcr", + ) with tempfile.TemporaryDirectory() as td: - out_fpath1 = os.path.join(td, 'bespoke_out2.h5') - out_fpath2 = os.path.join(td, 'bespoke_out1.h5') - res_fp = os.path.join(td, 'ri_100_wtk_{}.h5') - excl_fp = os.path.join(td, 'ri_exclusions.h5') + out_fpath1 = os.path.join(td, "bespoke_out2.h5") + out_fpath2 = os.path.join(td, "bespoke_out1.h5") + res_fp = os.path.join(td, "ri_100_wtk_{}.h5") + excl_fp = os.path.join(td, "ri_exclusions.h5") shutil.copy(EXCL, excl_fp) shutil.copy(RES.format(2012), res_fp.format(2012)) shutil.copy(RES.format(2013), res_fp.format(2013)) # test t/p extrapolation from single level (e.g. with Sup3rCC data) - del_dsets = ('pressure_100m', 'pressure_200m', 'temperature_80m') + del_dsets = ("pressure_100m", "pressure_200m", "temperature_80m") for y in (2012, 2013): - with h5py.File(res_fp.format(y), 'a') as h5: + with h5py.File(res_fp.format(y), "a") as h5: for dset in del_dsets: del h5[dset] - res_fp_all = res_fp.format('*') - res_fp_2013 = res_fp.format('2013') + res_fp_all = res_fp.format("*") + res_fp_2013 = res_fp.format("2013") # gids 33 and 35 are included, 37 is fully excluded - points = pd.DataFrame({MetaKeyName.GID: [33], 'config': ['default'], - 'extra_unused_data': [42]}) + points = pd.DataFrame( + { + MetaKeyName.GID: [33], + "config": ["default"], + "extra_unused_data": [42], + } + ) TechMapping.run(excl_fp, RES.format(2012), dset=TM_DSET, max_workers=1) assert not os.path.exists(out_fpath1) assert not os.path.exists(out_fpath2) - bsp = BespokeWindPlants(excl_fp, res_fp_all, TM_DSET, - OBJECTIVE_FUNCTION, CAP_COST_FUN, - FOC_FUN, VOC_FUN, points, sam_configs, - ga_kwargs={'max_time': 1}, excl_dict=EXCL_DICT, - output_request=output_request) + bsp = BespokeWindPlants( + excl_fp, + res_fp_all, + TM_DSET, + OBJECTIVE_FUNCTION, + CAP_COST_FUN, + FOC_FUN, + VOC_FUN, + points, + sam_configs, + ga_kwargs={"max_time": 1}, + excl_dict=EXCL_DICT, + output_request=output_request, + ) bsp.run(max_workers=1, out_fpath=out_fpath1) assert os.path.exists(out_fpath1) assert not os.path.exists(out_fpath2) - bsp = BespokeWindPlants(excl_fp, res_fp_2013, TM_DSET, - OBJECTIVE_FUNCTION, CAP_COST_FUN, - FOC_FUN, VOC_FUN, points, sam_configs, - ga_kwargs={'max_time': 1}, excl_dict=EXCL_DICT, - output_request=output_request, - prior_run=out_fpath1) + bsp = BespokeWindPlants( + excl_fp, + res_fp_2013, + TM_DSET, + OBJECTIVE_FUNCTION, + CAP_COST_FUN, + FOC_FUN, + VOC_FUN, + points, + sam_configs, + ga_kwargs={"max_time": 1}, + excl_dict=EXCL_DICT, + output_request=output_request, + prior_run=out_fpath1, + ) bsp.run(max_workers=1, out_fpath=out_fpath2) assert os.path.exists(out_fpath2) @@ -1022,19 +1271,27 @@ def test_bespoke_prior_run(): meta2 = f2.meta data2 = {k: f2[k] for k in f2.dsets} - cols = [MetaKeyName.TURBINE_X_COORDS, MetaKeyName.TURBINE_Y_COORDS, MetaKeyName.CAPACITY, - MetaKeyName.N_GIDS, MetaKeyName.GID_COUNTS, MetaKeyName.RES_GIDS] + cols = [ + MetaKeyName.TURBINE_X_COORDS, + MetaKeyName.TURBINE_Y_COORDS, + MetaKeyName.CAPACITY, + MetaKeyName.N_GIDS, + MetaKeyName.GID_COUNTS, + MetaKeyName.RES_GIDS, + ] pd.testing.assert_frame_equal(meta1[cols], meta2[cols]) # multi-year means should not match the 2nd run with 2013 only. # 2013 values should match exactly - assert not np.allclose(data1['cf_mean-means'], data2['cf_mean-means']) - assert np.allclose(data1['cf_mean-2013'], data2['cf_mean-2013']) + assert not np.allclose(data1["cf_mean-means"], data2["cf_mean-means"]) + assert np.allclose(data1["cf_mean-2013"], data2["cf_mean-2013"]) - assert not np.allclose(data1['annual_energy-means'], - data2['annual_energy-means']) - assert np.allclose(data1['annual_energy-2013'], - data2['annual_energy-2013']) + assert not np.allclose( + data1["annual_energy-means"], data2["annual_energy-means"] + ) + assert np.allclose( + data1["annual_energy-2013"], data2["annual_energy-2013"] + ) def test_gid_map(): @@ -1042,26 +1299,37 @@ def test_gid_map(): new resource data files for example so you can run forecasted resource with the same spatial configuration.""" - output_request = ('system_capacity', 'cf_mean', 'cf_profile', - 'extra_unused_data', 'winddirection', 'ws_mean') + output_request = ( + "system_capacity", + "cf_mean", + "cf_profile", + "extra_unused_data", + "winddirection", + "ws_mean", + ) with tempfile.TemporaryDirectory() as td: - out_fpath1 = os.path.join(td, 'bespoke_out2.h5') - out_fpath2 = os.path.join(td, 'bespoke_out1.h5') - res_fp = os.path.join(td, 'ri_100_wtk_{}.h5') - excl_fp = os.path.join(td, 'ri_exclusions.h5') + out_fpath1 = os.path.join(td, "bespoke_out2.h5") + out_fpath2 = os.path.join(td, "bespoke_out1.h5") + res_fp = os.path.join(td, "ri_100_wtk_{}.h5") + excl_fp = os.path.join(td, "ri_exclusions.h5") shutil.copy(EXCL, excl_fp) shutil.copy(RES.format(2013), res_fp.format(2013)) - res_fp_2013 = res_fp.format('2013') + res_fp_2013 = res_fp.format("2013") # gids 33 and 35 are included, 37 is fully excluded - points = pd.DataFrame({MetaKeyName.GID: [33], 'config': ['default'], - 'extra_unused_data': [42]}) + points = pd.DataFrame( + { + MetaKeyName.GID: [33], + "config": ["default"], + "extra_unused_data": [42], + } + ) gid_map = pd.DataFrame({MetaKeyName.GID: [3, 4, 13, 12, 11, 10, 9]}) new_gid = 50 - gid_map['gid_map'] = new_gid - fp_gid_map = os.path.join(td, 'gid_map.csv') + gid_map["gid_map"] = new_gid + fp_gid_map = os.path.join(td, "gid_map.csv") gid_map.to_csv(fp_gid_map) TechMapping.run(excl_fp, RES.format(2013), dset=TM_DSET, max_workers=1) @@ -1069,22 +1337,40 @@ def test_gid_map(): assert not os.path.exists(out_fpath1) assert not os.path.exists(out_fpath2) - bsp = BespokeWindPlants(excl_fp, res_fp_2013, TM_DSET, - OBJECTIVE_FUNCTION, CAP_COST_FUN, - FOC_FUN, VOC_FUN, points, SAM_CONFIGS, - ga_kwargs={'max_time': 1}, excl_dict=EXCL_DICT, - output_request=output_request) + bsp = BespokeWindPlants( + excl_fp, + res_fp_2013, + TM_DSET, + OBJECTIVE_FUNCTION, + CAP_COST_FUN, + FOC_FUN, + VOC_FUN, + points, + SAM_CONFIGS, + ga_kwargs={"max_time": 1}, + excl_dict=EXCL_DICT, + output_request=output_request, + ) bsp.run(max_workers=1, out_fpath=out_fpath1) assert os.path.exists(out_fpath1) assert not os.path.exists(out_fpath2) - bsp = BespokeWindPlants(excl_fp, res_fp_2013, TM_DSET, - OBJECTIVE_FUNCTION, CAP_COST_FUN, - FOC_FUN, VOC_FUN, points, SAM_CONFIGS, - ga_kwargs={'max_time': 1}, excl_dict=EXCL_DICT, - output_request=output_request, - gid_map=fp_gid_map) + bsp = BespokeWindPlants( + excl_fp, + res_fp_2013, + TM_DSET, + OBJECTIVE_FUNCTION, + CAP_COST_FUN, + FOC_FUN, + VOC_FUN, + points, + SAM_CONFIGS, + ga_kwargs={"max_time": 1}, + excl_dict=EXCL_DICT, + output_request=output_request, + gid_map=fp_gid_map, + ) bsp.run(max_workers=1, out_fpath=out_fpath2) assert os.path.exists(out_fpath2) @@ -1096,56 +1382,81 @@ def test_gid_map(): meta2 = f2.meta data2 = {k: f2[k] for k in f2.dsets} - hh = SAM_CONFIGS['default']['wind_turbine_hub_ht'] + hh = SAM_CONFIGS["default"]["wind_turbine_hub_ht"] with Resource(res_fp_2013) as f3: - ws = f3[f'windspeed_{hh}m', :, new_gid] + ws = f3[f"windspeed_{hh}m", :, new_gid] - cols = [MetaKeyName.N_GIDS, MetaKeyName.GID_COUNTS, MetaKeyName.RES_GIDS] + cols = [ + MetaKeyName.N_GIDS, + MetaKeyName.GID_COUNTS, + MetaKeyName.RES_GIDS, + ] pd.testing.assert_frame_equal(meta1[cols], meta2[cols]) - assert not np.allclose(data1['cf_mean-2013'], data2['cf_mean-2013']) - assert not np.allclose(data1['ws_mean'], data2['ws_mean'], atol=0.2) - assert np.allclose(ws.mean(), data2['ws_mean'], atol=0.01) - - out_fpath_pre = os.path.join(td, 'bespoke_out_pre.h5') - bsp = BespokeWindPlants(excl_fp, res_fp_2013, TM_DSET, - OBJECTIVE_FUNCTION, CAP_COST_FUN, - FOC_FUN, VOC_FUN, points, SAM_CONFIGS, - ga_kwargs={'max_time': 1}, excl_dict=EXCL_DICT, - output_request=output_request, - gid_map=fp_gid_map, pre_load_data=True) + assert not np.allclose(data1["cf_mean-2013"], data2["cf_mean-2013"]) + assert not np.allclose(data1["ws_mean"], data2["ws_mean"], atol=0.2) + assert np.allclose(ws.mean(), data2["ws_mean"], atol=0.01) + + out_fpath_pre = os.path.join(td, "bespoke_out_pre.h5") + bsp = BespokeWindPlants( + excl_fp, + res_fp_2013, + TM_DSET, + OBJECTIVE_FUNCTION, + CAP_COST_FUN, + FOC_FUN, + VOC_FUN, + points, + SAM_CONFIGS, + ga_kwargs={"max_time": 1}, + excl_dict=EXCL_DICT, + output_request=output_request, + gid_map=fp_gid_map, + pre_load_data=True, + ) bsp.run(max_workers=1, out_fpath=out_fpath_pre) with Resource(out_fpath2) as f1, Resource(out_fpath_pre) as f2: - assert np.allclose(f1["winddirection-2013"], - f2["winddirection-2013"]) + assert np.allclose( + f1["winddirection-2013"], f2["winddirection-2013"] + ) assert np.allclose(f1["ws_mean"], f2["ws_mean"]) def test_bespoke_bias_correct(): """Test bespoke run with bias correction on windspeed data.""" - output_request = ('system_capacity', 'cf_mean', 'cf_profile', - 'extra_unused_data', 'ws_mean') + output_request = ( + "system_capacity", + "cf_mean", + "cf_profile", + "extra_unused_data", + "ws_mean", + ) with tempfile.TemporaryDirectory() as td: - out_fpath1 = os.path.join(td, 'bespoke_out2.h5') - out_fpath2 = os.path.join(td, 'bespoke_out1.h5') - res_fp = os.path.join(td, 'ri_100_wtk_{}.h5') - excl_fp = os.path.join(td, 'ri_exclusions.h5') + out_fpath1 = os.path.join(td, "bespoke_out2.h5") + out_fpath2 = os.path.join(td, "bespoke_out1.h5") + res_fp = os.path.join(td, "ri_100_wtk_{}.h5") + excl_fp = os.path.join(td, "ri_exclusions.h5") shutil.copy(EXCL, excl_fp) shutil.copy(RES.format(2013), res_fp.format(2013)) - res_fp_2013 = res_fp.format('2013') + res_fp_2013 = res_fp.format("2013") # gids 33 and 35 are included, 37 is fully excluded - points = pd.DataFrame({MetaKeyName.GID: [33], 'config': ['default'], - 'extra_unused_data': [42]}) + points = pd.DataFrame( + { + MetaKeyName.GID: [33], + "config": ["default"], + "extra_unused_data": [42], + } + ) # intentionally leaving out WTK gid 13 which only has 5 included 90m # pixels in order to check that this is dynamically patched. bias_correct = pd.DataFrame({MetaKeyName.GID: [3, 4, 12, 11, 10, 9]}) - bias_correct['method'] = 'lin_ws' - bias_correct['scalar'] = 0.5 - fp_bc = os.path.join(td, 'bc.csv') + bias_correct["method"] = "lin_ws" + bias_correct["scalar"] = 0.5 + fp_bc = os.path.join(td, "bc.csv") bias_correct.to_csv(fp_bc) TechMapping.run(excl_fp, RES.format(2013), dset=TM_DSET, max_workers=1) @@ -1153,22 +1464,40 @@ def test_bespoke_bias_correct(): assert not os.path.exists(out_fpath1) assert not os.path.exists(out_fpath2) - bsp = BespokeWindPlants(excl_fp, res_fp_2013, TM_DSET, - OBJECTIVE_FUNCTION, CAP_COST_FUN, - FOC_FUN, VOC_FUN, points, SAM_CONFIGS, - ga_kwargs={'max_time': 1}, excl_dict=EXCL_DICT, - output_request=output_request) + bsp = BespokeWindPlants( + excl_fp, + res_fp_2013, + TM_DSET, + OBJECTIVE_FUNCTION, + CAP_COST_FUN, + FOC_FUN, + VOC_FUN, + points, + SAM_CONFIGS, + ga_kwargs={"max_time": 1}, + excl_dict=EXCL_DICT, + output_request=output_request, + ) bsp.run(max_workers=1, out_fpath=out_fpath1) assert os.path.exists(out_fpath1) assert not os.path.exists(out_fpath2) - bsp = BespokeWindPlants(excl_fp, res_fp_2013, TM_DSET, - OBJECTIVE_FUNCTION, CAP_COST_FUN, - FOC_FUN, VOC_FUN, points, SAM_CONFIGS, - ga_kwargs={'max_time': 1}, excl_dict=EXCL_DICT, - output_request=output_request, - bias_correct=fp_bc) + bsp = BespokeWindPlants( + excl_fp, + res_fp_2013, + TM_DSET, + OBJECTIVE_FUNCTION, + CAP_COST_FUN, + FOC_FUN, + VOC_FUN, + points, + SAM_CONFIGS, + ga_kwargs={"max_time": 1}, + excl_dict=EXCL_DICT, + output_request=output_request, + bias_correct=fp_bc, + ) bsp.run(max_workers=1, out_fpath=out_fpath2) assert os.path.exists(out_fpath2) @@ -1180,29 +1509,39 @@ def test_bespoke_bias_correct(): meta2 = f2.meta data2 = {k: f2[k] for k in f2.dsets} - cols = [MetaKeyName.N_GIDS, MetaKeyName.GID_COUNTS, MetaKeyName.RES_GIDS] + cols = [ + MetaKeyName.N_GIDS, + MetaKeyName.GID_COUNTS, + MetaKeyName.RES_GIDS, + ] pd.testing.assert_frame_equal(meta1[cols], meta2[cols]) - assert data1['cf_mean-2013'] * 0.5 > data2['cf_mean-2013'] - assert np.allclose(data1['ws_mean'] * 0.5, data2['ws_mean'], atol=0.01) + assert data1["cf_mean-2013"] * 0.5 > data2["cf_mean-2013"] + assert np.allclose(data1["ws_mean"] * 0.5, data2["ws_mean"], atol=0.01) def test_cli(runner, clear_loggers): """Test bespoke CLI""" - output_request = ('system_capacity', 'cf_mean', 'cf_profile', - 'winddirection', 'windspeed', 'ws_mean') + output_request = ( + "system_capacity", + "cf_mean", + "cf_profile", + "winddirection", + "windspeed", + "ws_mean", + ) with tempfile.TemporaryDirectory() as td: dirname = os.path.basename(td) fn_out = "{}_{}.h5".format(dirname, ModuleName.BESPOKE) out_fpath = os.path.join(td, fn_out) - res_fp = os.path.join(td, 'ri_100_wtk_{}.h5') - excl_fp = os.path.join(td, 'ri_exclusions.h5') + res_fp = os.path.join(td, "ri_100_wtk_{}.h5") + excl_fp = os.path.join(td, "ri_exclusions.h5") shutil.copy(EXCL, excl_fp) shutil.copy(RES.format(2012), res_fp.format(2012)) shutil.copy(RES.format(2013), res_fp.format(2013)) - res_fp = res_fp.format('*') + res_fp = res_fp.format("*") TechMapping.run(excl_fp, RES.format(2012), dset=TM_DSET, max_workers=1) @@ -1222,14 +1561,14 @@ def test_cli(runner, clear_loggers): "variable_operating_cost_function": VOC_FUN, "project_points": [33, 35], "sam_files": SAM_CONFIGS, - "min_spacing": '5x', + "min_spacing": "5x", "wake_loss_multiplier": 1, - "ga_kwargs": {'max_time': 5}, + "ga_kwargs": {"max_time": 5}, "output_request": output_request, "ws_bins": (0, 20, 5), "wd_bins": (0, 360, 45), "excl_dict": EXCL_DICT, - "area_filter_kernel": 'queen', + "area_filter_kernel": "queen", "min_area": None, "resolution": 64, "excl_area": None, @@ -1240,15 +1579,16 @@ def test_cli(runner, clear_loggers): "bias_correct": None, "pre_load_data": False, } - config_path = os.path.join(td, 'config.json') - with open(config_path, 'w') as f: + config_path = os.path.join(td, "config.json") + with open(config_path, "w") as f: json.dump(config, f) assert not os.path.exists(out_fpath) - result = runner.invoke(main, ['bespoke', '-c', config_path]) + result = runner.invoke(main, ["bespoke", "-c", config_path]) if result.exit_code != 0: - msg = ('Failed with error {}' - .format(traceback.print_exception(*result.exc_info))) + msg = "Failed with error {}".format( + traceback.print_exception(*result.exc_info) + ) raise RuntimeError(msg) assert os.path.exists(out_fpath) @@ -1259,12 +1599,17 @@ def test_cli(runner, clear_loggers): assert MetaKeyName.SC_POINT_GID in meta assert MetaKeyName.TURBINE_X_COORDS in meta assert MetaKeyName.TURBINE_Y_COORDS in meta - assert 'possible_x_coords' in meta - assert 'possible_y_coords' in meta + assert "possible_x_coords" in meta + assert "possible_y_coords" in meta assert MetaKeyName.RES_GIDS in meta - dsets_1d = ('system_capacity', 'cf_mean-2012', - 'annual_energy-2012', 'cf_mean-means', 'ws_mean') + dsets_1d = ( + "system_capacity", + "cf_mean-2012", + "annual_energy-2012", + "cf_mean-means", + "ws_mean", + ) for dset in dsets_1d: assert dset in list(f) assert isinstance(f[dset], np.ndarray) @@ -1272,8 +1617,12 @@ def test_cli(runner, clear_loggers): assert len(f[dset]) == len(meta) assert f[dset].any() # not all zeros - dsets_2d = ('cf_profile-2012', 'cf_profile-2013', - 'windspeed-2012', 'windspeed-2013') + dsets_2d = ( + "cf_profile-2012", + "cf_profile-2013", + "windspeed-2012", + "windspeed-2013", + ) for dset in dsets_2d: assert dset in list(f) assert isinstance(f[dset], np.ndarray) @@ -1287,39 +1636,61 @@ def test_cli(runner, clear_loggers): def test_bespoke_5min_sample(): """Sample a 5min resource dataset for 60min outputs in bespoke""" - output_request = ('system_capacity', 'cf_mean', 'cf_profile', - 'extra_unused_data', 'winddirection', 'windspeed', - 'ws_mean') - tm_dset = 'test_wtk_5min' + output_request = ( + "system_capacity", + "cf_mean", + "cf_profile", + "extra_unused_data", + "winddirection", + "windspeed", + "ws_mean", + ) + tm_dset = "test_wtk_5min" with tempfile.TemporaryDirectory() as td: - out_fpath = os.path.join(td, 'wind_bespoke.h5') - excl_fp = os.path.join(td, 'ri_exclusions.h5') + out_fpath = os.path.join(td, "wind_bespoke.h5") + excl_fp = os.path.join(td, "ri_exclusions.h5") shutil.copy(EXCL, excl_fp) - res_fp = os.path.join(TESTDATADIR, 'wtk/wtk_2010_*m.h5') - - points = pd.DataFrame({MetaKeyName.GID: [33, 35], 'config': ['default'] * 2, - 'extra_unused_data': [0, 42]}) + res_fp = os.path.join(TESTDATADIR, "wtk/wtk_2010_*m.h5") + + points = pd.DataFrame( + { + MetaKeyName.GID: [33, 35], + "config": ["default"] * 2, + "extra_unused_data": [0, 42], + } + ) sam_sys_inputs = copy.deepcopy(SAM_SYS_INPUTS) - sam_sys_inputs['time_index_step'] = 12 - sam_configs = {'default': sam_sys_inputs} + sam_sys_inputs["time_index_step"] = 12 + sam_configs = {"default": sam_sys_inputs} # hack techmap because 5min data only has 10 wind resource pixels - with h5py.File(excl_fp, 'a') as excl_file: - arr = np.random.choice(10, size=excl_file[MetaKeyName.LATITUDE].shape) + with h5py.File(excl_fp, "a") as excl_file: + arr = np.random.choice( + 10, size=excl_file[MetaKeyName.LATITUDE].shape + ) excl_file.create_dataset(name=tm_dset, data=arr) - bsp = BespokeWindPlants(excl_fp, res_fp, tm_dset, OBJECTIVE_FUNCTION, - CAP_COST_FUN, FOC_FUN, VOC_FUN, points, - sam_configs, ga_kwargs={'max_time': 5}, - excl_dict=EXCL_DICT, - output_request=output_request) + bsp = BespokeWindPlants( + excl_fp, + res_fp, + tm_dset, + OBJECTIVE_FUNCTION, + CAP_COST_FUN, + FOC_FUN, + VOC_FUN, + points, + sam_configs, + ga_kwargs={"max_time": 5}, + excl_dict=EXCL_DICT, + output_request=output_request, + ) _ = bsp.run(max_workers=1, out_fpath=out_fpath) with Resource(out_fpath) as f: assert len(f.meta) == 2 assert len(f) == 8760 - assert len(f['cf_profile-2010']) == 8760 - assert len(f['time_index-2010']) == 8760 - assert len(f['windspeed-2010']) == 8760 - assert len(f['winddirection-2010']) == 8760 + assert len(f["cf_profile-2010"]) == 8760 + assert len(f["time_index-2010"]) == 8760 + assert len(f["windspeed-2010"]) == 8760 + assert len(f["winddirection-2010"]) == 8760 diff --git a/tests/test_config.py b/tests/test_config.py index be0f8010e..09610db6a 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -7,6 +7,7 @@ @author: gbuster """ + import os import tempfile @@ -30,7 +31,7 @@ def test_config_entries(): """ Test BaseConfig check_entry test """ - config_path = os.path.join(TESTDATADIR, 'config/collection.json') + config_path = os.path.join(TESTDATADIR, "config/collection.json") with pytest.raises(ConfigError): AnalysisConfig(config_path) @@ -39,25 +40,27 @@ def test_clearsky(): """ Test Clearsky """ - res_file = os.path.join(TESTDATADIR, 'nsrdb/', 'ri_100_nsrdb_2012.h5') - sam_files = os.path.join(TESTDATADIR, 'SAM/naris_pv_1axis_inv13_cs.json') - pp = ProjectPoints(slice(0, 10), sam_files, 'pvwattsv5', - res_file=res_file) + res_file = os.path.join(TESTDATADIR, "nsrdb/", "ri_100_nsrdb_2012.h5") + sam_files = os.path.join(TESTDATADIR, "SAM/naris_pv_1axis_inv13_cs.json") + pp = ProjectPoints(slice(0, 10), sam_files, "pvwattsv5", res_file=res_file) with pytest.raises(ResourceRuntimeError): # Get the SAM resource object RevPySam.get_sam_res(res_file, pp, pp.tech) -@pytest.mark.parametrize(('start', 'interval'), - [[0, 1], [13, 1], [10, 2], [13, 3]]) +@pytest.mark.parametrize( + ("start", "interval"), [[0, 1], [13, 1], [10, 2], [13, 3]] +) def test_proj_control_iter(start, interval): """Test the iteration of the points control.""" n = 3 - res_file = os.path.join(TESTDATADIR, 'wtk/ri_100_wtk_2012.h5') - sam_files = os.path.join(TESTDATADIR, - 'SAM/wind_gen_standard_losses_0.json') - pp = ProjectPoints(slice(start, 100, interval), sam_files, 'windpower', - res_file=res_file) + res_file = os.path.join(TESTDATADIR, "wtk/ri_100_wtk_2012.h5") + sam_files = os.path.join( + TESTDATADIR, "SAM/wind_gen_standard_losses_0.json" + ) + pp = ProjectPoints( + slice(start, 100, interval), sam_files, "windpower", res_file=res_file + ) pc = PointsControl(pp, sites_per_split=n) for i, pp_split in enumerate(pc): @@ -65,19 +68,22 @@ def test_proj_control_iter(start, interval): i1_nom = i * n + n split = pp_split.project_points.df target = pp.df.iloc[i0_nom:i1_nom, :] - msg = 'PointsControl iterator split did not function correctly!' + msg = "PointsControl iterator split did not function correctly!" assert all(split == target), msg -@pytest.mark.parametrize(('start', 'interval'), - [[0, 1], [13, 1], [10, 2], [13, 3]]) +@pytest.mark.parametrize( + ("start", "interval"), [[0, 1], [13, 1], [10, 2], [13, 3]] +) def test_proj_points_split(start, interval): """Test the split operation of project points.""" - res_file = os.path.join(TESTDATADIR, 'wtk/ri_100_wtk_2012.h5') - sam_files = os.path.join(TESTDATADIR, - 'SAM/wind_gen_standard_losses_0.json') - pp = ProjectPoints(slice(start, 100, interval), sam_files, 'windpower', - res_file=res_file) + res_file = os.path.join(TESTDATADIR, "wtk/ri_100_wtk_2012.h5") + sam_files = os.path.join( + TESTDATADIR, "SAM/wind_gen_standard_losses_0.json" + ) + pp = ProjectPoints( + slice(start, 100, interval), sam_files, "windpower", res_file=res_file + ) iter_interval = 5 for i0 in range(0, len(pp), iter_interval): @@ -87,18 +93,20 @@ def test_proj_points_split(start, interval): pp_0 = ProjectPoints.split(i0, i1, pp) - msg = 'ProjectPoints split did not function correctly!' + msg = "ProjectPoints split did not function correctly!" assert pp_0.sites == pp.sites[i0:i1], msg assert all(pp_0.df == pp.df.iloc[i0:i1]), msg def test_split_iter(): """Test Points_Control on two slices of ProjectPoints""" - res_file = os.path.join(TESTDATADIR, 'wtk/ri_100_wtk_2012.h5') - sam_files = os.path.join(TESTDATADIR, - 'SAM/wind_gen_standard_losses_0.json') - pp = ProjectPoints(slice(0, 500, 5), sam_files, 'windpower', - res_file=res_file) + res_file = os.path.join(TESTDATADIR, "wtk/ri_100_wtk_2012.h5") + sam_files = os.path.join( + TESTDATADIR, "SAM/wind_gen_standard_losses_0.json" + ) + pp = ProjectPoints( + slice(0, 500, 5), sam_files, "windpower", res_file=res_file + ) n = 3 for s, e in [(0, 50), (50, 100)]: @@ -112,19 +120,23 @@ def test_split_iter(): split = pp_split.project_points.df target = pp.df.iloc[i0_nom:i1_nom] - msg = 'PointsControl iterator split did not function correctly!' + msg = "PointsControl iterator split did not function correctly!" assert split.equals(target), msg def test_config_mapping(): """Test the mapping of multiple configs in the project points.""" - fpp = os.path.join(TESTDATADIR, 'project_points/pp_offshore.csv') - sam_files = {'onshore': os.path.join( - TESTDATADIR, 'SAM/wind_gen_standard_losses_0.json'), - MetaKeyName.OFFSHORE: os.path.join( - TESTDATADIR, 'SAM/wind_gen_standard_losses_1.json')} + fpp = os.path.join(TESTDATADIR, "project_points/pp_offshore.csv") + sam_files = { + "onshore": os.path.join( + TESTDATADIR, "SAM/wind_gen_standard_losses_0.json" + ), + MetaKeyName.OFFSHORE: os.path.join( + TESTDATADIR, "SAM/wind_gen_standard_losses_1.json" + ), + } df = pd.read_csv(fpp, index_col=0) - pp = ProjectPoints(fpp, sam_files, 'windpower') + pp = ProjectPoints(fpp, sam_files, "windpower") pc = PointsControl(pp, sites_per_split=100) for i, pc_split in enumerate(pc): for site in pc_split.sites: @@ -136,69 +148,78 @@ def test_sam_config_kw_replace(): """Test that the SAM config with old keys from pysam v1 gets updated on the fly and gets propogated to downstream splits.""" - fpp = os.path.join(TESTDATADIR, 'project_points/pp_offshore.csv') - sam_files = {'onshore': os.path.join( - TESTDATADIR, 'SAM/wind_gen_standard_losses_0.json'), - MetaKeyName.OFFSHORE: os.path.join( - TESTDATADIR, 'SAM/wind_gen_standard_losses_1.json')} - res_file = os.path.join(TESTDATADIR, 'wtk/ri_100_wtk_2012.h5') - pp = ProjectPoints(fpp, sam_files, 'windpower') - - gen = Gen('windpower', pp, sam_files, resource_file=res_file, - sites_per_worker=100) - config_on = gen.project_points.sam_inputs['onshore'] + fpp = os.path.join(TESTDATADIR, "project_points/pp_offshore.csv") + sam_files = { + "onshore": os.path.join( + TESTDATADIR, "SAM/wind_gen_standard_losses_0.json" + ), + MetaKeyName.OFFSHORE: os.path.join( + TESTDATADIR, "SAM/wind_gen_standard_losses_1.json" + ), + } + res_file = os.path.join(TESTDATADIR, "wtk/ri_100_wtk_2012.h5") + pp = ProjectPoints(fpp, sam_files, "windpower") + + gen = Gen( + "windpower", + pp, + sam_files, + resource_file=res_file, + sites_per_worker=100, + ) + config_on = gen.project_points.sam_inputs["onshore"] config_of = gen.project_points.sam_inputs[MetaKeyName.OFFSHORE] - assert 'turb_generic_loss' in config_on - assert 'turb_generic_loss' in config_of + assert "turb_generic_loss" in config_on + assert "turb_generic_loss" in config_of pp_split = ProjectPoints.split(0, 10000, gen.project_points) - config_on = pp_split.sam_inputs['onshore'] + config_on = pp_split.sam_inputs["onshore"] config_of = pp_split.sam_inputs[MetaKeyName.OFFSHORE] - assert 'turb_generic_loss' in config_on - assert 'turb_generic_loss' in config_of + assert "turb_generic_loss" in config_on + assert "turb_generic_loss" in config_of pc_split = PointsControl.split(0, 10000, gen.project_points) - config_on = pc_split.project_points.sam_inputs['onshore'] + config_on = pc_split.project_points.sam_inputs["onshore"] config_of = pc_split.project_points.sam_inputs[MetaKeyName.OFFSHORE] - assert 'turb_generic_loss' in config_on - assert 'turb_generic_loss' in config_of + assert "turb_generic_loss" in config_on + assert "turb_generic_loss" in config_of for ipc in pc_split: - if 'onshore' in ipc.project_points.sam_inputs: - config = ipc.project_points.sam_inputs['onshore'] - assert 'turb_generic_loss' in config + if "onshore" in ipc.project_points.sam_inputs: + config = ipc.project_points.sam_inputs["onshore"] + assert "turb_generic_loss" in config if MetaKeyName.OFFSHORE in ipc.project_points.sam_inputs: config = ipc.project_points.sam_inputs[MetaKeyName.OFFSHORE] - assert 'turb_generic_loss' in config + assert "turb_generic_loss" in config -@pytest.mark.parametrize('counties', [['Washington'], ['Providence', 'Kent']]) +@pytest.mark.parametrize("counties", [["Washington"], ["Providence", "Kent"]]) def test_regions(counties): """ Test ProjectPoint.regions class method """ - res_file = os.path.join(TESTDATADIR, 'nsrdb/', 'ri_100_nsrdb_2012.h5') - sam_files = os.path.join(TESTDATADIR, 'SAM/naris_pv_1axis_inv13_cs.json') + res_file = os.path.join(TESTDATADIR, "nsrdb/", "ri_100_nsrdb_2012.h5") + sam_files = os.path.join(TESTDATADIR, "SAM/naris_pv_1axis_inv13_cs.json") with Resource(res_file) as f: meta = f.meta - baseline = meta.loc[meta['county'].isin(counties)].index.values.tolist() + baseline = meta.loc[meta["county"].isin(counties)].index.values.tolist() - regions = dict.fromkeys(counties, 'county') + regions = dict.fromkeys(counties, "county") pp = ProjectPoints.regions(regions, res_file, sam_files) assert sorted(baseline) == pp.sites -@pytest.mark.parametrize('sites', [1, 2, 5, 10]) +@pytest.mark.parametrize("sites", [1, 2, 5, 10]) def test_coords(sites): """ Test ProjectPoint.lat_lon_coords class method """ - res_file = os.path.join(TESTDATADIR, 'nsrdb/', 'ri_100_nsrdb_2012.h5') - sam_files = os.path.join(TESTDATADIR, 'SAM/naris_pv_1axis_inv13_cs.json') + res_file = os.path.join(TESTDATADIR, "nsrdb/", "ri_100_nsrdb_2012.h5") + sam_files = os.path.join(TESTDATADIR, "SAM/naris_pv_1axis_inv13_cs.json") with Resource(res_file) as f: meta = f.meta @@ -207,7 +228,9 @@ def test_coords(sites): if not isinstance(gids, list): gids = [gids] - lat_lons = meta.loc[gids, [MetaKeyName.LATITUDE, MetaKeyName.LONGITUDE]].values + lat_lons = meta.loc[ + gids, [MetaKeyName.LATITUDE, MetaKeyName.LONGITUDE] + ].values pp = ProjectPoints.lat_lon_coords(lat_lons, res_file, sam_files) assert sorted(gids) == pp.sites @@ -216,8 +239,8 @@ def test_coords(sites): def test_coords_from_file(): """Test ProjectPoint.lat_lon_coords read from file.""" - res_file = os.path.join(TESTDATADIR, 'nsrdb/', 'ri_100_nsrdb_2012.h5') - sam_files = os.path.join(TESTDATADIR, 'SAM/naris_pv_1axis_inv13_cs.json') + res_file = os.path.join(TESTDATADIR, "nsrdb/", "ri_100_nsrdb_2012.h5") + sam_files = os.path.join(TESTDATADIR, "SAM/naris_pv_1axis_inv13_cs.json") with Resource(res_file) as f: meta = f.meta @@ -239,18 +262,20 @@ def test_duplicate_coords(): """ Test ProjectPoint.lat_lon_coords duplicate coords error """ - res_file = os.path.join(TESTDATADIR, 'nsrdb/', 'ri_100_nsrdb_2012.h5') - sam_files = os.path.join(TESTDATADIR, 'SAM/naris_pv_1axis_inv13_cs.json') + res_file = os.path.join(TESTDATADIR, "nsrdb/", "ri_100_nsrdb_2012.h5") + sam_files = os.path.join(TESTDATADIR, "SAM/naris_pv_1axis_inv13_cs.json") with Resource(res_file) as f: meta = f.meta - duplicates = meta.loc[[2, 3, 3, 4], [MetaKeyName.LATITUDE, MetaKeyName.LONGITUDE]].values + duplicates = meta.loc[ + [2, 3, 3, 4], [MetaKeyName.LATITUDE, MetaKeyName.LONGITUDE] + ].values with pytest.raises(RuntimeError): ProjectPoints.lat_lon_coords(duplicates, res_file, sam_files) - regions = {'Kent': 'county', 'Rhode Island': 'state'} + regions = {"Kent": "county", "Rhode Island": "state"} with pytest.raises(RuntimeError): ProjectPoints.regions(regions, res_file, sam_files) @@ -259,15 +284,19 @@ def test_sam_configs(): """ Test supplying SAM config as a JSON or a dict """ - fpp = os.path.join(TESTDATADIR, 'project_points/pp_offshore.csv') - sam_files = {'onshore': os.path.join( - TESTDATADIR, 'SAM/wind_gen_standard_losses_0.json'), - MetaKeyName.OFFSHORE: os.path.join( - TESTDATADIR, 'SAM/wind_gen_standard_losses_1.json')} - pp_json = ProjectPoints(fpp, sam_files, 'windpower') + fpp = os.path.join(TESTDATADIR, "project_points/pp_offshore.csv") + sam_files = { + "onshore": os.path.join( + TESTDATADIR, "SAM/wind_gen_standard_losses_0.json" + ), + MetaKeyName.OFFSHORE: os.path.join( + TESTDATADIR, "SAM/wind_gen_standard_losses_1.json" + ), + } + pp_json = ProjectPoints(fpp, sam_files, "windpower") sam_configs = {k: safe_json_load(v) for k, v in sam_files.items()} - pp_dict = ProjectPoints(fpp, sam_configs, 'windpower') + pp_dict = ProjectPoints(fpp, sam_configs, "windpower") assert pp_json.sam_inputs == pp_dict.sam_inputs @@ -276,68 +305,71 @@ def test_bad_sam_configs(): """ Test supplying SAM config as a JSON or a dict """ - fpp = os.path.join(TESTDATADIR, 'project_points/pp_offshore.csv') - sam_files = {'onshore': os.path.join( - TESTDATADIR, 'SAM/wind_gen_standard_losses_0.json')} + fpp = os.path.join(TESTDATADIR, "project_points/pp_offshore.csv") + sam_files = { + "onshore": os.path.join( + TESTDATADIR, "SAM/wind_gen_standard_losses_0.json" + ) + } # Assert ProjecPoints fails with unequal config entries in points # vs sam_configs (as files) with pytest.raises(ConfigError): - ProjectPoints(fpp, sam_files, 'windpower') + ProjectPoints(fpp, sam_files, "windpower") sam_configs = {k: safe_json_load(v) for k, v in sam_files.items()} # Assert ProjecPoints fails with unequal config entries in points # vs sam_configs (as dicts) with pytest.raises(ConfigError): - ProjectPoints(fpp, sam_configs, 'windpower') + ProjectPoints(fpp, sam_configs, "windpower") sites = slice(0, 100) - sam_file = os.path.join(TESTDATADIR, 'SAM/wind_gen_standard_losses_0.csv') + sam_file = os.path.join(TESTDATADIR, "SAM/wind_gen_standard_losses_0.csv") # Assert SAMConfig fails when the SAM config is not a json with pytest.raises(IOError): - ProjectPoints(sites, sam_file, 'windpower') + ProjectPoints(sites, sam_file, "windpower") sites = slice(0, 100) - sam_file = os.path.join(TESTDATADIR, 'SAM/wind_gen_standard_losses_0.json') + sam_file = os.path.join(TESTDATADIR, "SAM/wind_gen_standard_losses_0.json") sam_config = safe_json_load(sam_file) # Assert SAMConfig fails when supplying a raw SAM config dictionary. # The SAM config dict should be mapped to a config ID with pytest.raises(RuntimeError): - ProjectPoints(sites, sam_config, 'windpower') + ProjectPoints(sites, sam_config, "windpower") - fpp = os.path.join(TESTDATADIR, 'project_points/pp_offshore.csv') - sam_files = [os.path.join(TESTDATADIR, - 'SAM/wind_gen_standard_losses_0.json'), - os.path.join(TESTDATADIR, - 'SAM/wind_gen_standard_losses_1.json')] + fpp = os.path.join(TESTDATADIR, "project_points/pp_offshore.csv") + sam_files = [ + os.path.join(TESTDATADIR, "SAM/wind_gen_standard_losses_0.json"), + os.path.join(TESTDATADIR, "SAM/wind_gen_standard_losses_1.json"), + ] # Assert ProjecPoints fails with a list of configs is provided instead # of a dictionary mapping the config files to config IDs with pytest.raises(ValueError): - ProjectPoints(fpp, sam_files, 'windpower') + ProjectPoints(fpp, sam_files, "windpower") def test_nested_sites(): """ Test check for nested points list """ - res_file = os.path.join(TESTDATADIR, 'wtk/ri_100_wtk_2012.h5') + res_file = os.path.join(TESTDATADIR, "wtk/ri_100_wtk_2012.h5") with pytest.raises(RuntimeError): points = [[1, 2, 3, 5]] - sam_file = os.path.join(TESTDATADIR, - 'SAM/wind_gen_standard_losses_0.json') - ProjectPoints(points, sam_file, 'windpower', res_file) + sam_file = os.path.join( + TESTDATADIR, "SAM/wind_gen_standard_losses_0.json" + ) + ProjectPoints(points, sam_file, "windpower", res_file) def test_project_points_h(): """ Test hub heights in project points """ - res_file = os.path.join(TESTDATADIR, 'wtk/ri_100_wtk_2012.h5') + res_file = os.path.join(TESTDATADIR, "wtk/ri_100_wtk_2012.h5") points = [1, 2, 3, 5] - sam_file = os.path.join(TESTDATADIR, - 'SAM/wind_gen_standard_losses_0.json') - assert ProjectPoints(points, sam_file, 'pvwattsv8', res_file).h is None + sam_file = os.path.join(TESTDATADIR, "SAM/wind_gen_standard_losses_0.json") + assert ProjectPoints(points, sam_file, "pvwattsv8", res_file).h is None - pp = ProjectPoints(points, sam_file, 'windpower', res_file) + pp = ProjectPoints(points, sam_file, "windpower", res_file) assert pp.h == [80] * 4 @@ -346,20 +378,20 @@ def test_project_points_d(): Test depth in project points """ points = [1, 2, 3, 5] - sam_file = os.path.join(TESTDATADIR, 'SAM/geothermal_default.json') - assert ProjectPoints(points, sam_file, 'windpower').d is None + sam_file = os.path.join(TESTDATADIR, "SAM/geothermal_default.json") + assert ProjectPoints(points, sam_file, "windpower").d is None - pp = ProjectPoints(points, sam_file, 'geothermal') + pp = ProjectPoints(points, sam_file, "geothermal") assert pp.d == [4500] * 4 depths_in_data = list(range(len(pp.df))) - pp = ProjectPoints(points, sam_file, 'geothermal') - pp.df['resource_depth'] = depths_in_data + pp = ProjectPoints(points, sam_file, "geothermal") + pp.df["resource_depth"] = depths_in_data assert pp.d == depths_in_data -def execute_pytest(capture='all', flags='-rapP'): +def execute_pytest(capture="all", flags="-rapP"): """Execute module as pytest with detailed summary report. Parameters @@ -372,8 +404,8 @@ def execute_pytest(capture='all', flags='-rapP'): """ fname = os.path.basename(__file__) - pytest.main(['-q', '--show-capture={}'.format(capture), fname, flags]) + pytest.main(["-q", "--show-capture={}".format(capture), fname, flags]) -if __name__ == '__main__': +if __name__ == "__main__": execute_pytest() diff --git a/tests/test_curtailment.py b/tests/test_curtailment.py index 39558f581..2b8e34c0e 100644 --- a/tests/test_curtailment.py +++ b/tests/test_curtailment.py @@ -5,6 +5,7 @@ @author: gbuster """ + import os from copy import deepcopy @@ -22,18 +23,20 @@ from reV.utilities.curtailment import curtail -def get_curtailment(year, curt_fn='curtailment.json'): - """Get the curtailed and non-curtailed resource objects, and project points - """ - res_file = os.path.join(TESTDATADIR, 'wtk/', - 'ri_100_wtk_{}.h5'.format(year)) +def get_curtailment(year, curt_fn="curtailment.json"): + """Get curtailed and non-curtailed resource objects, and project points""" + res_file = os.path.join( + TESTDATADIR, "wtk/", "ri_100_wtk_{}.h5".format(year) + ) sam_files = os.path.join( - TESTDATADIR, 'SAM/wind_gen_standard_losses_0.json') - curtailment = os.path.join(TESTDATADIR, 'config/', curt_fn) - pp = ProjectPoints(slice(0, 100), sam_files, 'windpower', - curtailment=curtailment) - - resource = RevPySam.get_sam_res(res_file, pp, 'windpower') + TESTDATADIR, "SAM/wind_gen_standard_losses_0.json" + ) + curtailment = os.path.join(TESTDATADIR, "config/", curt_fn) + pp = ProjectPoints( + slice(0, 100), sam_files, "windpower", curtailment=curtailment + ) + + resource = RevPySam.get_sam_res(res_file, pp, "windpower") non_curtailed_res = deepcopy(resource) out = curtail(resource, pp.curtailment, random_seed=0) @@ -41,11 +44,9 @@ def get_curtailment(year, curt_fn='curtailment.json'): return out, non_curtailed_res, pp -@pytest.mark.parametrize(('year', 'site'), - [('2012', 0), - ('2012', 10), - ('2013', 0), - ('2013', 10)]) +@pytest.mark.parametrize( + ("year", "site"), [("2012", 0), ("2012", 10), ("2013", 0), ("2013", 10)] +) def test_cf_curtailment(year, site): """Run Wind generation and ensure that the cf_profile is zero when curtailment is expected. @@ -53,36 +54,45 @@ def test_cf_curtailment(year, site): Note that the probability of curtailment must be 1 for this to succeed. """ - res_file = os.path.join(TESTDATADIR, - 'wtk/ri_100_wtk_{}.h5'.format(year)) - sam_files = os.path.join(TESTDATADIR, - 'SAM/wind_gen_standard_losses_0.json') + res_file = os.path.join(TESTDATADIR, "wtk/ri_100_wtk_{}.h5".format(year)) + sam_files = os.path.join( + TESTDATADIR, "SAM/wind_gen_standard_losses_0.json" + ) - curtailment = os.path.join(TESTDATADIR, 'config/', 'curtailment.json') + curtailment = os.path.join(TESTDATADIR, "config/", "curtailment.json") points = slice(site, site + 1) # run reV 2.0 generation - gen = Gen('windpower', points, sam_files, res_file, - output_request=('cf_profile',), curtailment=curtailment, - sites_per_worker=50, scale_outputs=True) + gen = Gen( + "windpower", + points, + sam_files, + res_file, + output_request=("cf_profile",), + curtailment=curtailment, + sites_per_worker=50, + scale_outputs=True, + ) gen.run(max_workers=1) results, check_curtailment = test_res_curtailment(year, site=site) - results['cf_profile'] = gen.out['cf_profile'].flatten() + results["cf_profile"] = gen.out["cf_profile"].flatten() # was capacity factor NOT curtailed? - check_cf = (gen.out['cf_profile'].flatten() != 0) + check_cf = gen.out["cf_profile"].flatten() != 0 # Were all thresholds met and windspeed NOT curtailed? check = check_curtailment & check_cf - msg = ('All curtailment thresholds were met and cf_profile ' - 'was not curtailed!') + msg = ( + "All curtailment thresholds were met and cf_profile " + "was not curtailed!" + ) assert np.sum(check) == 0, msg return results -@pytest.mark.parametrize('year', ['2012', '2013']) +@pytest.mark.parametrize("year", ["2012", "2013"]) def test_curtailment_res_mean(year): """Run Wind generation and ensure that the cf_profile is zero when curtailment is expected. @@ -90,136 +100,171 @@ def test_curtailment_res_mean(year): Note that the probability of curtailment must be 1 for this to succeed. """ - res_file = os.path.join(TESTDATADIR, - 'wtk/ri_100_wtk_{}.h5'.format(year)) - sam_files = os.path.join(TESTDATADIR, - 'SAM/wind_gen_standard_losses_0.json') + res_file = os.path.join(TESTDATADIR, "wtk/ri_100_wtk_{}.h5".format(year)) + sam_files = os.path.join( + TESTDATADIR, "SAM/wind_gen_standard_losses_0.json" + ) - curtailment = os.path.join(TESTDATADIR, 'config/', 'curtailment.json') + curtailment = os.path.join(TESTDATADIR, "config/", "curtailment.json") points = slice(0, 100) - output_request = ('cf_mean', 'ws_mean') - pc = Gen.get_pc(points, None, sam_files, 'windpower', - sites_per_worker=50, res_file=res_file, - curtailment=curtailment) - - resources = RevPySam.get_sam_res(res_file, - pc.project_points, - pc.project_points.tech, - output_request) - truth = resources['mean_windspeed'] + output_request = ("cf_mean", "ws_mean") + pc = Gen.get_pc( + points, + None, + sam_files, + "windpower", + sites_per_worker=50, + res_file=res_file, + curtailment=curtailment, + ) + + resources = RevPySam.get_sam_res( + res_file, pc.project_points, pc.project_points.tech, output_request + ) + truth = resources["mean_windspeed"] # run reV 2.0 generation - gen = Gen('windpower', points, sam_files, res_file, - output_request=output_request, curtailment=curtailment, - sites_per_worker=50, scale_outputs=True) + gen = Gen( + "windpower", + points, + sam_files, + res_file, + output_request=output_request, + curtailment=curtailment, + sites_per_worker=50, + scale_outputs=True, + ) gen.run(max_workers=1) - test = gen.out['ws_mean'] + test = gen.out["ws_mean"] assert np.allclose(truth, test, rtol=0.001) -@pytest.mark.parametrize(('year', 'site'), - [('2012', 10), - ('2013', 10)]) +@pytest.mark.parametrize(("year", "site"), [("2012", 10), ("2013", 10)]) def test_random(year, site): """Run wind generation and ensure that no curtailment, 100% probability curtailment, and 50% probability curtailment result in expected decreases in the annual cf_mean. """ - res_file = os.path.join(TESTDATADIR, - 'wtk/ri_100_wtk_{}.h5'.format(year)) - sam_files = os.path.join(TESTDATADIR, - 'SAM/wind_gen_standard_losses_0.json') + res_file = os.path.join(TESTDATADIR, "wtk/ri_100_wtk_{}.h5".format(year)) + sam_files = os.path.join( + TESTDATADIR, "SAM/wind_gen_standard_losses_0.json" + ) results = [] no_curtail = None - curtailment = {"dawn_dusk": "nautical", "months": [4, 5, 6, 7], - "precipitation": None, "probability": 1, - "temperature": None, "wind_speed": 10.0} - prob_curtail = {"dawn_dusk": "nautical", "months": [4, 5, 6, 7], - "precipitation": None, "probability": 0.5, - "temperature": None, "wind_speed": 10.0} + curtailment = { + "dawn_dusk": "nautical", + "months": [4, 5, 6, 7], + "precipitation": None, + "probability": 1, + "temperature": None, + "wind_speed": 10.0, + } + prob_curtail = { + "dawn_dusk": "nautical", + "months": [4, 5, 6, 7], + "precipitation": None, + "probability": 0.5, + "temperature": None, + "wind_speed": 10.0, + } for c in [no_curtail, curtailment, prob_curtail]: - points = slice(site, site + 1) # run reV 2.0 generation and write to disk - gen = Gen('windpower', points, sam_files, res_file, - output_request=('cf_profile',), curtailment=c, - sites_per_worker=50, scale_outputs=True) + gen = Gen( + "windpower", + points, + sam_files, + res_file, + output_request=("cf_profile",), + curtailment=c, + sites_per_worker=50, + scale_outputs=True, + ) gen.run(max_workers=1) - results.append(gen.out['cf_mean']) + results.append(gen.out["cf_mean"]) - assert results[0] > results[1], 'Curtailment did not decrease cf_mean!' + assert results[0] > results[1], "Curtailment did not decrease cf_mean!" expected = (results[0] + results[1]) / 2 diff = expected - results[2] - msg = ('Curtailment with 50% probability did not result in 50% less ' - 'curtailment! No curtailment, curtailment, and 50% curtailment ' - 'have the following cf_means: {}'.format(results)) + msg = ( + "Curtailment with 50% probability did not result in 50% less " + "curtailment! No curtailment, curtailment, and 50% curtailment " + "have the following cf_means: {}".format(results) + ) assert diff <= 2, msg -@pytest.mark.parametrize(('year', 'site'), - [('2012', 50), - ('2013', 50)]) +@pytest.mark.parametrize(("year", "site"), [("2012", 50), ("2013", 50)]) def test_res_curtailment(year, site): """Test wind resource curtailment.""" out, non_curtailed_res, pp = get_curtailment(year) sza = SolarPosition( non_curtailed_res.time_index, - non_curtailed_res.meta[[MetaKeyName.LATITUDE, MetaKeyName.LONGITUDE]].values).zenith + non_curtailed_res.meta[ + [MetaKeyName.LATITUDE, MetaKeyName.LONGITUDE] + ].values, + ).zenith ti = non_curtailed_res.time_index # was it in a curtailment month? check1 = np.isin(non_curtailed_res.time_index.month, pp.curtailment.months) - check1 = np.tile(np.expand_dims(check1, axis=1), - non_curtailed_res.shape[1]) + check1 = np.tile( + np.expand_dims(check1, axis=1), non_curtailed_res.shape[1] + ) # was the non-curtailed wind speed threshold met? - check2 = (non_curtailed_res._res_arrays['windspeed'] - < pp.curtailment.wind_speed) + check2 = ( + non_curtailed_res._res_arrays["windspeed"] < pp.curtailment.wind_speed + ) # was it nighttime? - check3 = (sza > pp.curtailment.dawn_dusk) + check3 = sza > pp.curtailment.dawn_dusk # was the temperature threshold met? - check4 = (out._res_arrays['temperature'] > pp.curtailment.temperature) + check4 = out._res_arrays["temperature"] > pp.curtailment.temperature # thresholds for curtailment check_curtailment = check1 & check2 & check3 & check4 # was windspeed NOT curtailed? - check5 = (out._res_arrays['windspeed'] != 0) + check5 = out._res_arrays["windspeed"] != 0 # Were all thresholds met and windspeed NOT curtailed? check = check_curtailment & check5 - msg = ('All curtailment thresholds were met and windspeed ' - 'was not curtailed!') + msg = ( + "All curtailment thresholds were met and windspeed " + "was not curtailed!" + ) assert np.sum(check) == 0, msg # optional output df to help check results i = site - df = pd.DataFrame({'i': range(len(sza)), - 'curtailed_wind': out._res_arrays['windspeed'][:, i], - 'original_wind': - non_curtailed_res._res_arrays['windspeed'][:, i], - 'temperature': out._res_arrays['temperature'][:, i], - 'sza': sza[:, i], - 'wind_curtail': check2[:, i], - 'month_curtail': check1[:, i], - 'sza_curtail': check3[:, i], - 'temp_curtail': check4[:, i], - }, - index=ti) - - if str(year) == '2012': - drop_day = ((ti.month == 12) & (ti.day == 31)) + df = pd.DataFrame( + { + "i": range(len(sza)), + "curtailed_wind": out._res_arrays["windspeed"][:, i], + "original_wind": non_curtailed_res._res_arrays["windspeed"][:, i], + "temperature": out._res_arrays["temperature"][:, i], + "sza": sza[:, i], + "wind_curtail": check2[:, i], + "month_curtail": check1[:, i], + "sza_curtail": check3[:, i], + "temp_curtail": check4[:, i], + }, + index=ti, + ) + + if str(year) == "2012": + drop_day = (ti.month == 12) & (ti.day == 31) df = df.drop(df.index[drop_day]) check_curtailment = check_curtailment[~drop_day, :] @@ -229,28 +274,25 @@ def test_res_curtailment(year, site): def test_date_range(): """Test curtailment based on a date range vs. months list""" year = 2012 - cres_m = get_curtailment(year, curt_fn='curtailment.json')[0] - cres_dr = get_curtailment(year, curt_fn='curtailment_date_range.json')[0] + cres_m = get_curtailment(year, curt_fn="curtailment.json")[0] + cres_dr = get_curtailment(year, curt_fn="curtailment_date_range.json")[0] for df_res, site in cres_m: gid = int(site.name) - assert np.allclose(df_res['windspeed'], cres_dr[gid]['windspeed']) + assert np.allclose(df_res["windspeed"], cres_dr[gid]["windspeed"]) def test_eqn_curtailment(plot=False): """Test equation-based curtailment strategies.""" year = 2012 - curt_fn = 'curtailment_eqn.json' + curt_fn = "curtailment_eqn.json" curtailed, non_curtailed_res, pp = get_curtailment(year, curt_fn=curt_fn) - c_config = safe_json_load(os.path.join(TESTDATADIR, 'config/', curt_fn)) - c_eqn = c_config['equation'] + c_config = safe_json_load(os.path.join(TESTDATADIR, "config/", curt_fn)) + c_eqn = c_config["equation"] c_res = curtailed[0] nc_res = non_curtailed_res[0] c_mask = (c_res.windspeed == 0) & (nc_res.windspeed > 0) - temperature = nc_res['temperature'].values - wind_speed = nc_res['windspeed'].values - eval_mask = eval(c_eqn) # All curtailed windspeeds should satisfy the eqn eval but maybe not the @@ -259,17 +301,19 @@ def test_eqn_curtailment(plot=False): if plot: import matplotlib.pyplot as plt - fig, ax = plt.subplots() - ax.scatter(nc_res.loc[c_mask, 'windspeed'], - nc_res.loc[c_mask, 'temperature']) - ax.grid('on') + + _, ax = plt.subplots() + ax.scatter( + nc_res.loc[c_mask, "windspeed"], nc_res.loc[c_mask, "temperature"] + ) + ax.grid("on") ax.set_xlim([0, 7]) ax.set_ylim([0, 30]) - ax.set_legend(['Curtailed']) - plt.savefig('equation_based_curtailment.png') + ax.set_legend(["Curtailed"]) + plt.savefig("equation_based_curtailment.png") -def execute_pytest(capture='all', flags='-rapP'): +def execute_pytest(capture="all", flags="-rapP"): """Execute module as pytest with detailed summary report. Parameters @@ -282,8 +326,8 @@ def execute_pytest(capture='all', flags='-rapP'): """ fname = os.path.basename(__file__) - pytest.main(['-q', '--show-capture={}'.format(capture), fname, flags]) + pytest.main(["-q", "--show-capture={}".format(capture), fname, flags]) -if __name__ == '__main__': +if __name__ == "__main__": execute_pytest() diff --git a/tests/test_econ_lcoe.py b/tests/test_econ_lcoe.py index 8975ce453..84b378849 100644 --- a/tests/test_econ_lcoe.py +++ b/tests/test_econ_lcoe.py @@ -7,20 +7,21 @@ @author: gbuster """ -import h5py import json -import numpy as np import os -import pandas as pd import shutil -from pandas.testing import assert_frame_equal -import pytest import tempfile import traceback +import h5py +import numpy as np +import pandas as pd +import pytest +from pandas.testing import assert_frame_equal + +from reV import TESTDATADIR from reV.cli import main from reV.econ.econ import Econ -from reV import TESTDATADIR from reV.handlers.outputs import Outputs from reV.utilities import ModuleName diff --git a/tests/test_econ_of_scale.py b/tests/test_econ_of_scale.py index 00ebd4345..19e968b5a 100644 --- a/tests/test_econ_of_scale.py +++ b/tests/test_econ_of_scale.py @@ -3,6 +3,7 @@ """ PyTest file for reV LCOE economies of scale """ + import os import shutil import tempfile @@ -19,22 +20,21 @@ from reV.supply_curve.sc_aggregation import SupplyCurveAggregation from reV.utilities import MetaKeyName -EXCL = os.path.join(TESTDATADIR, 'ri_exclusions/ri_exclusions.h5') -GEN = os.path.join(TESTDATADIR, 'gen_out/ri_my_pv_gen.h5') -TM_DSET = 'techmap_nsrdb' -RES_CLASS_DSET = 'ghi_mean-means' +EXCL = os.path.join(TESTDATADIR, "ri_exclusions/ri_exclusions.h5") +GEN = os.path.join(TESTDATADIR, "gen_out/ri_my_pv_gen.h5") +TM_DSET = "techmap_nsrdb" +RES_CLASS_DSET = "ghi_mean-means" RES_CLASS_BINS = [0, 4, 100] -DATA_LAYERS = {'pct_slope': {'dset': 'ri_srtm_slope', - 'method': 'mean'}, - 'reeds_region': {'dset': 'ri_reeds_regions', - 'method': 'mode'}, - 'padus': {'dset': 'ri_padus', - 'method': 'mode'}} - -EXCL_DICT = {'ri_srtm_slope': {'inclusion_range': (None, 5), - 'exclude_nodata': True}, - 'ri_padus': {'exclude_values': [1], - 'exclude_nodata': True}} +DATA_LAYERS = { + "pct_slope": {"dset": "ri_srtm_slope", "method": "mean"}, + "reeds_region": {"dset": "ri_reeds_regions", "method": "mode"}, + "padus": {"dset": "ri_padus", "method": "mode"}, +} + +EXCL_DICT = { + "ri_srtm_slope": {"inclusion_range": (None, 5), "exclude_nodata": True}, + "ri_padus": {"exclude_values": [1], "exclude_nodata": True}, +} RTOL = 0.001 @@ -44,71 +44,86 @@ def test_pass_through_lcoe_args(): input to the reV output.""" year = 2012 rev2_points = slice(0, 3) - res_file = os.path.join(TESTDATADIR, 'wtk/ri_100_wtk_{}.h5'.format(year)) - sam_files = os.path.join(TESTDATADIR, 'SAM/i_windpower_lcoe.json') - - output_request = ('cf_mean', - 'lcoe_fcr', - 'system_capacity', - 'capital_cost', - 'fixed_charge_rate', - 'variable_operating_cost', - 'fixed_operating_cost') + res_file = os.path.join(TESTDATADIR, "wtk/ri_100_wtk_{}.h5".format(year)) + sam_files = os.path.join(TESTDATADIR, "SAM/i_windpower_lcoe.json") + + output_request = ( + "cf_mean", + "lcoe_fcr", + "system_capacity", + "capital_cost", + "fixed_charge_rate", + "variable_operating_cost", + "fixed_operating_cost", + ) # run reV 2.0 generation - gen = Gen('windpower', rev2_points, sam_files, res_file, - sites_per_worker=1, output_request=output_request) - gen.run(max_workers=1,) + gen = Gen( + "windpower", + rev2_points, + sam_files, + res_file, + sites_per_worker=1, + output_request=output_request, + ) + gen.run( + max_workers=1, + ) checks = [x in gen.out for x in Gen.LCOE_ARGS] assert all(checks) - assert 'lcoe_fcr' in gen.out - assert 'cf_mean' in gen.out + assert "lcoe_fcr" in gen.out + assert "cf_mean" in gen.out def test_lcoe_calc_simple(): """Test the EconomiesOfScale LCOE calculator without cap cost scalar""" eqn = None # from pvwattsv7 defaults - data = {'aep': 35188456.00, - 'capital_cost': 53455000.00, - 'foc': 360000.00, - 'voc': 0, - 'fcr': 0.096} - - true_lcoe = ((data['fcr'] * data['capital_cost'] + data['foc']) - / (data['aep'] / 1000)) + data = { + "aep": 35188456.00, + "capital_cost": 53455000.00, + "foc": 360000.00, + "voc": 0, + "fcr": 0.096, + } + + true_lcoe = (data["fcr"] * data["capital_cost"] + data["foc"]) / ( + data["aep"] / 1000 + ) data[MetaKeyName.MEAN_LCOE] = true_lcoe eos = EconomiesOfScale(eqn, data) assert eos.raw_capital_cost == eos.scaled_capital_cost - assert eos.raw_capital_cost == data['capital_cost'] + assert eos.raw_capital_cost == data["capital_cost"] assert np.allclose(eos.raw_lcoe, true_lcoe, rtol=0.001) assert np.allclose(eos.scaled_lcoe, true_lcoe, rtol=0.001) eqn = 1 eos = EconomiesOfScale(eqn, data) assert eos.raw_capital_cost == eos.scaled_capital_cost - assert eos.raw_capital_cost == data['capital_cost'] + assert eos.raw_capital_cost == data["capital_cost"] assert np.allclose(eos.raw_lcoe, true_lcoe, rtol=0.001) assert np.allclose(eos.scaled_lcoe, true_lcoe, rtol=0.001) eqn = 2 - true_scaled = ((data['fcr'] * eqn * data['capital_cost'] + data['foc']) - / (data['aep'] / 1000)) + true_scaled = (data["fcr"] * eqn * data["capital_cost"] + data["foc"]) / ( + data["aep"] / 1000 + ) eos = EconomiesOfScale(eqn, data) assert eqn * eos.raw_capital_cost == eos.scaled_capital_cost - assert eos.raw_capital_cost == data['capital_cost'] + assert eos.raw_capital_cost == data["capital_cost"] assert np.allclose(eos.raw_lcoe, true_lcoe, rtol=0.001) assert np.allclose(eos.scaled_lcoe, true_scaled, rtol=0.001) - data['system_capacity'] = 2 - eqn = '1 / system_capacity' - true_scaled = ((data['fcr'] * 0.5 * data['capital_cost'] + data['foc']) - / (data['aep'] / 1000)) + data["system_capacity"] = 2 + eqn = "1 / system_capacity" + true_scaled = (data["fcr"] * 0.5 * data["capital_cost"] + data["foc"]) / ( + data["aep"] / 1000 + ) eos = EconomiesOfScale(eqn, data) assert 0.5 * eos.raw_capital_cost == eos.scaled_capital_cost - assert eos.raw_capital_cost == data['capital_cost'] + assert eos.raw_capital_cost == data["capital_cost"] assert np.allclose(eos.raw_lcoe, true_lcoe, rtol=0.001) assert np.allclose(eos.scaled_lcoe, true_scaled, rtol=0.001) @@ -117,91 +132,118 @@ def test_econ_of_scale_baseline(): """Test an economies of scale calculation with scalar = 1 to ensure we can reproduce the lcoe values """ - data = {'capital_cost': 39767200, - 'fixed_operating_cost': 260000, - 'fixed_charge_rate': 0.096, - 'system_capacity': 20000, - 'variable_operating_cost': 0} + data = { + "capital_cost": 39767200, + "fixed_operating_cost": 260000, + "fixed_charge_rate": 0.096, + "system_capacity": 20000, + "variable_operating_cost": 0, + } with tempfile.TemporaryDirectory() as td: - gen_temp = os.path.join(td, 'ri_my_pv_gen.h5') + gen_temp = os.path.join(td, "ri_my_pv_gen.h5") shutil.copy(GEN, gen_temp) # overwrite the LCOE values since i dont know what econ inputs # the original test file was run with with Resource(GEN) as res: - cf = res['cf_mean-means'] - - lcoe = (1000 * (data['fixed_charge_rate'] * data['capital_cost'] - + data['fixed_operating_cost']) - / (cf * data['system_capacity'] * 8760)) - - with h5py.File(gen_temp, 'a') as res: - res['lcoe_fcr-means'][...] = lcoe + cf = res["cf_mean-means"] + + lcoe = ( + 1000 + * ( + data["fixed_charge_rate"] * data["capital_cost"] + + data["fixed_operating_cost"] + ) + / (cf * data["system_capacity"] * 8760) + ) + + with h5py.File(gen_temp, "a") as res: + res["lcoe_fcr-means"][...] = lcoe for k, v in data.items(): - arr = np.full(res['meta'].shape, v) - res.create_dataset(k, res['meta'].shape, data=arr) - res[k].attrs['scale_factor'] = 1.0 + arr = np.full(res["meta"].shape, v) + res.create_dataset(k, res["meta"].shape, data=arr) + res[k].attrs["scale_factor"] = 1.0 out_fp_base = os.path.join(td, "base") - base = SupplyCurveAggregation(EXCL, TM_DSET, excl_dict=EXCL_DICT, - res_class_dset=RES_CLASS_DSET, - res_class_bins=RES_CLASS_BINS, - data_layers=DATA_LAYERS, - gids=list(np.arange(10))) + base = SupplyCurveAggregation( + EXCL, + TM_DSET, + excl_dict=EXCL_DICT, + res_class_dset=RES_CLASS_DSET, + res_class_bins=RES_CLASS_BINS, + data_layers=DATA_LAYERS, + gids=list(np.arange(10)), + ) base.run(out_fp_base, gen_fpath=gen_temp, max_workers=1) out_fp_sc = os.path.join(td, "sc") - sc = SupplyCurveAggregation(EXCL, TM_DSET, excl_dict=EXCL_DICT, - res_class_dset=RES_CLASS_DSET, - res_class_bins=RES_CLASS_BINS, - data_layers=DATA_LAYERS, - gids=list(np.arange(10)), - cap_cost_scale='1') + sc = SupplyCurveAggregation( + EXCL, + TM_DSET, + excl_dict=EXCL_DICT, + res_class_dset=RES_CLASS_DSET, + res_class_bins=RES_CLASS_BINS, + data_layers=DATA_LAYERS, + gids=list(np.arange(10)), + cap_cost_scale="1", + ) sc.run(out_fp_sc, gen_fpath=gen_temp, max_workers=1) base_df = pd.read_csv(out_fp_base + ".csv") sc_df = pd.read_csv(out_fp_sc + ".csv") - assert np.allclose(base_df[MetaKeyName.MEAN_LCOE], sc_df[MetaKeyName.MEAN_LCOE]) + assert np.allclose( + base_df[MetaKeyName.MEAN_LCOE], sc_df[MetaKeyName.MEAN_LCOE] + ) assert (sc_df[MetaKeyName.CAPITAL_COST_SCALAR] == 1).all() - assert np.allclose(sc_df['mean_capital_cost'], - sc_df[MetaKeyName.SCALED_CAPITAL_COST]) + assert np.allclose( + sc_df["mean_capital_cost"], sc_df[MetaKeyName.SCALED_CAPITAL_COST] + ) def test_sc_agg_econ_scale(): - """Test supply curve aggregation with LCOE scaling based on plant capacity. - """ - data = {'capital_cost': 53455000, - 'fixed_operating_cost': 360000, - 'fixed_charge_rate': 0.096, - 'variable_operating_cost': 0} + """Test supply curve agg with LCOE scaling based on plant capacity.""" + data = { + "capital_cost": 53455000, + "fixed_operating_cost": 360000, + "fixed_charge_rate": 0.096, + "variable_operating_cost": 0, + } with tempfile.TemporaryDirectory() as td: - gen_temp = os.path.join(td, 'ri_my_pv_gen.h5') + gen_temp = os.path.join(td, "ri_my_pv_gen.h5") shutil.copy(GEN, gen_temp) - with h5py.File(gen_temp, 'a') as res: + with h5py.File(gen_temp, "a") as res: for k, v in data.items(): - arr = np.full(res['meta'].shape, v) - res.create_dataset(k, res['meta'].shape, data=arr) - res[k].attrs['scale_factor'] = 1.0 + arr = np.full(res["meta"].shape, v) + res.create_dataset(k, res["meta"].shape, data=arr) + res[k].attrs["scale_factor"] = 1.0 - eqn = '2 * np.multiply(1000, capacity) ** -0.3' + eqn = "2 * np.multiply(1000, capacity) ** -0.3" out_fp_base = os.path.join(td, "base") - base = SupplyCurveAggregation(EXCL, TM_DSET, excl_dict=EXCL_DICT, - res_class_dset=RES_CLASS_DSET, - res_class_bins=RES_CLASS_BINS, - data_layers=DATA_LAYERS, - gids=list(np.arange(10))) + base = SupplyCurveAggregation( + EXCL, + TM_DSET, + excl_dict=EXCL_DICT, + res_class_dset=RES_CLASS_DSET, + res_class_bins=RES_CLASS_BINS, + data_layers=DATA_LAYERS, + gids=list(np.arange(10)), + ) base.run(out_fp_base, gen_fpath=gen_temp, max_workers=1) out_fp_sc = os.path.join(td, "sc") - sc = SupplyCurveAggregation(EXCL, TM_DSET, excl_dict=EXCL_DICT, - res_class_dset=RES_CLASS_DSET, - res_class_bins=RES_CLASS_BINS, - data_layers=DATA_LAYERS, - gids=list(np.arange(10)), - cap_cost_scale=eqn) + sc = SupplyCurveAggregation( + EXCL, + TM_DSET, + excl_dict=EXCL_DICT, + res_class_dset=RES_CLASS_DSET, + res_class_bins=RES_CLASS_BINS, + data_layers=DATA_LAYERS, + gids=list(np.arange(10)), + cap_cost_scale=eqn, + ) sc.run(out_fp_sc, gen_fpath=gen_temp, max_workers=1) base_df = pd.read_csv(out_fp_base + ".csv") @@ -209,41 +251,56 @@ def test_sc_agg_econ_scale(): # check that econ of scale saved the raw lcoe and that it reduced all # of the mean lcoe values from baseline - assert np.allclose(sc_df[MetaKeyName.RAW_LCOE], base_df[MetaKeyName.MEAN_LCOE]) - assert all(sc_df[MetaKeyName.MEAN_LCOE] < base_df[MetaKeyName.MEAN_LCOE]) - - aep = ((sc_df['mean_fixed_charge_rate'] * sc_df['mean_capital_cost'] - + sc_df['mean_fixed_operating_cost']) / sc_df[MetaKeyName.RAW_LCOE]) - - true_raw_lcoe = ((data['fixed_charge_rate'] * data['capital_cost'] - + data['fixed_operating_cost']) - / aep + data['variable_operating_cost']) + assert np.allclose( + sc_df[MetaKeyName.RAW_LCOE], base_df[MetaKeyName.MEAN_LCOE] + ) + assert all( + sc_df[MetaKeyName.MEAN_LCOE] < base_df[MetaKeyName.MEAN_LCOE] + ) + + aep = ( + sc_df["mean_fixed_charge_rate"] * sc_df["mean_capital_cost"] + + sc_df["mean_fixed_operating_cost"] + ) / sc_df[MetaKeyName.RAW_LCOE] + + true_raw_lcoe = ( + data["fixed_charge_rate"] * data["capital_cost"] + + data["fixed_operating_cost"] + ) / aep + data["variable_operating_cost"] eval_inputs = {k: sc_df[k].values.flatten() for k in sc_df.columns} # pylint: disable=eval-used scalars = eval(str(eqn), globals(), eval_inputs) - sc_df['scalars'] = scalars - true_scaled_lcoe = ((data['fixed_charge_rate'] - * scalars * data['capital_cost'] - + data['fixed_operating_cost']) - / aep + data['variable_operating_cost']) + sc_df["scalars"] = scalars + true_scaled_lcoe = ( + data["fixed_charge_rate"] * scalars * data["capital_cost"] + + data["fixed_operating_cost"] + ) / aep + data["variable_operating_cost"] assert np.allclose(scalars, sc_df[MetaKeyName.CAPITAL_COST_SCALAR]) - assert np.allclose(scalars * sc_df['mean_capital_cost'], - sc_df[MetaKeyName.SCALED_CAPITAL_COST]) + assert np.allclose( + scalars * sc_df["mean_capital_cost"], + sc_df[MetaKeyName.SCALED_CAPITAL_COST], + ) assert np.allclose(true_scaled_lcoe, sc_df[MetaKeyName.MEAN_LCOE]) assert np.allclose(true_raw_lcoe, sc_df[MetaKeyName.RAW_LCOE]) sc_df = sc_df.sort_values(MetaKeyName.CAPACITY) assert all(sc_df[MetaKeyName.MEAN_LCOE].diff()[1:] < 0) for i in sc_df.index.values: - if sc_df.loc[i, 'scalars'] < 1: - assert sc_df.loc[i, MetaKeyName.MEAN_LCOE] < sc_df.loc[i, MetaKeyName.RAW_LCOE] + if sc_df.loc[i, "scalars"] < 1: + assert ( + sc_df.loc[i, MetaKeyName.MEAN_LCOE] + < sc_df.loc[i, MetaKeyName.RAW_LCOE] + ) else: - assert sc_df.loc[i, MetaKeyName.MEAN_LCOE] >= sc_df.loc[i, MetaKeyName.RAW_LCOE] + assert ( + sc_df.loc[i, MetaKeyName.MEAN_LCOE] + >= sc_df.loc[i, MetaKeyName.RAW_LCOE] + ) -def execute_pytest(capture='all', flags='-rapP'): +def execute_pytest(capture="all", flags="-rapP"): """Execute module as pytest with detailed summary report. Parameters @@ -256,8 +313,8 @@ def execute_pytest(capture='all', flags='-rapP'): """ fname = os.path.basename(__file__) - pytest.main(['-q', '--show-capture={}'.format(capture), fname, flags]) + pytest.main(["-q", "--show-capture={}".format(capture), fname, flags]) -if __name__ == '__main__': +if __name__ == "__main__": execute_pytest() diff --git a/tests/test_econ_single_owner.py b/tests/test_econ_single_owner.py index 0de36e853..758488deb 100644 --- a/tests/test_econ_single_owner.py +++ b/tests/test_econ_single_owner.py @@ -9,12 +9,12 @@ """ import os -import pytest + import numpy as np +import pytest -from reV.econ.econ import Econ from reV import TESTDATADIR - +from reV.econ.econ import Econ RTOL = 0.01 ATOL = 0.001 diff --git a/tests/test_econ_windbos.py b/tests/test_econ_windbos.py index b073f9cd4..f47f78b0c 100644 --- a/tests/test_econ_windbos.py +++ b/tests/test_econ_windbos.py @@ -8,15 +8,17 @@ """ import json import os -import pytest +import tempfile + import numpy as np import pandas as pd -import tempfile +import pytest -from reV.generation.generation import Gen +from reV import TESTDATADIR from reV.econ.econ import Econ +from reV.generation.generation import Gen from reV.SAM.windbos import WindBos -from reV import TESTDATADIR +from reV.utilities import MetaKeyName RTOL = 0.000001 ATOL = 0.001 @@ -113,7 +115,7 @@ def test_sam_windbos(): def test_rev_windbos(): """Test baseline windbos calc with single owner defaults""" fpath = TESTDATADIR + '/SAM/i_singleowner_windbos.json' - with open(fpath, 'r') as f: + with open(fpath) as f: inputs = json.load(f) wb = WindBos(inputs) assert np.allclose(wb.turbine_cost, 52512000.00, atol=ATOL, rtol=RTOL) @@ -125,11 +127,11 @@ def test_rev_windbos(): def test_standalone_json(): """Test baseline windbos calc with standalone json file""" fpath = TESTDATADIR + '/SAM/i_singleowner_windbos.json' - with open(fpath, 'r') as f: + with open(fpath) as f: inputs = json.load(f) wb1 = WindBos(inputs) fpath = TESTDATADIR + '/SAM/i_windbos.json' - with open(fpath, 'r') as f: + with open(fpath) as f: inputs = json.load(f) wb2 = WindBos(inputs) @@ -140,7 +142,7 @@ def test_standalone_json(): def test_rev_windbos_perf_bond(): """Test windbos calc with performance bonds""" fpath = TESTDATADIR + '/SAM/i_singleowner_windbos.json' - with open(fpath, 'r') as f: + with open(fpath) as f: inputs = json.load(f) inputs['performance_bond'] = 10.0 wb = WindBos(inputs) @@ -153,7 +155,7 @@ def test_rev_windbos_perf_bond(): def test_rev_windbos_transport(): """Test windbos calc with turbine transport costs""" fpath = TESTDATADIR + '/SAM/i_singleowner_windbos.json' - with open(fpath, 'r') as f: + with open(fpath) as f: inputs = json.load(f) inputs['turbine_transportation'] = 100.0 wb = WindBos(inputs) @@ -166,7 +168,7 @@ def test_rev_windbos_transport(): def test_rev_windbos_sales(): """Test windbos calc with turbine transport costs""" fpath = TESTDATADIR + '/SAM/i_singleowner_windbos.json' - with open(fpath, 'r') as f: + with open(fpath) as f: inputs = json.load(f) inputs['sales_tax_basis'] = 5.0 wb = WindBos(inputs) diff --git a/tests/test_gen_5min.py b/tests/test_gen_5min.py index 91168880a..3078c00f6 100644 --- a/tests/test_gen_5min.py +++ b/tests/test_gen_5min.py @@ -5,12 +5,13 @@ """ import os + import h5py -import pytest import numpy as np +import pytest -from reV.generation.generation import Gen from reV import TESTDATADIR +from reV.generation.generation import Gen pytest.importorskip("nsrdb") from nsrdb.utilities.statistics import mae_perc diff --git a/tests/test_gen_forecast.py b/tests/test_gen_forecast.py index ff8d8e2a8..38f75c10b 100644 --- a/tests/test_gen_forecast.py +++ b/tests/test_gen_forecast.py @@ -30,54 +30,89 @@ def test_forecast(): site_data for timezone and elevation input and gid_map for forecast meta mapping""" - res_files_source = TESTDATADIR + '/nsrdb/ri_100_nsrdb_2012.h5' - sam_files = TESTDATADIR + '/SAM/i_pvwattsv7.json' + res_files_source = TESTDATADIR + "/nsrdb/ri_100_nsrdb_2012.h5" + sam_files = TESTDATADIR + "/SAM/i_pvwattsv7.json" with tempfile.TemporaryDirectory() as td: - res_file = os.path.join(td, 'temp_2012.h5') + res_file = os.path.join(td, "temp_2012.h5") shutil.copy(res_files_source, res_file) - with Outputs(res_file, mode='a') as f: + with Outputs(res_file, mode="a") as f: meta = f.meta - meta = meta.drop([MetaKeyName.TIMEZONE, MetaKeyName.ELEVATION], axis=1) - del f._h5['meta'] + meta = meta.drop( + [MetaKeyName.TIMEZONE, MetaKeyName.ELEVATION], axis=1 + ) + del f._h5["meta"] f._meta = None f.meta = meta - with Outputs(res_file, mode='r') as f: + with Outputs(res_file, mode="r") as f: assert MetaKeyName.TIMEZONE not in f.meta assert MetaKeyName.ELEVATION not in f.meta with Resource(res_file) as res: - ghi = res['ghi'] - - points = ProjectPoints(slice(0, 5), sam_files, 'pvwattsv7', - res_file=res_file) - output_request = ('cf_mean', 'ghi_mean') - site_data = pd.DataFrame({MetaKeyName.GID: np.arange(5), MetaKeyName.TIMEZONE: -5, - MetaKeyName.ELEVATION: 0}) + ghi = res["ghi"] + + points = ProjectPoints( + slice(0, 5), sam_files, "pvwattsv7", res_file=res_file + ) + output_request = ("cf_mean", "ghi_mean") + site_data = pd.DataFrame( + { + MetaKeyName.GID: np.arange(5), + MetaKeyName.TIMEZONE: -5, + MetaKeyName.ELEVATION: 0, + } + ) gid_map = {0: 20, 1: 20, 2: 50, 3: 51, 4: 51} # test that this raises an error with missing timezone with pytest.raises(SAMExecutionError): - gen = Gen('pvwattsv7', points, sam_files, res_file, - sites_per_worker=3, output_request=output_request) + gen = Gen( + "pvwattsv7", + points, + sam_files, + res_file, + sites_per_worker=3, + output_request=output_request, + ) gen.run(max_workers=1) - gen1 = Gen('pvwattsv7', points, sam_files, res_file, - sites_per_worker=3, site_data=site_data, - output_request=output_request) + gen1 = Gen( + "pvwattsv7", + points, + sam_files, + res_file, + sites_per_worker=3, + site_data=site_data, + output_request=output_request, + ) gen1.run(max_workers=1) for i in range(5): - assert np.allclose(gen1.out['ghi_mean'][i], mean_irrad(ghi[:, i]), - atol=0.0, rtol=0.001) - - gen2 = Gen('pvwattsv7', points, sam_files, res_file, - sites_per_worker=3, site_data=site_data, - gid_map=gid_map, output_request=output_request) + assert np.allclose( + gen1.out["ghi_mean"][i], + mean_irrad(ghi[:, i]), + atol=0.0, + rtol=0.001, + ) + + gen2 = Gen( + "pvwattsv7", + points, + sam_files, + res_file, + sites_per_worker=3, + site_data=site_data, + gid_map=gid_map, + output_request=output_request, + ) gen2.run(max_workers=1) for i in range(5): j = gid_map[i] - assert np.allclose(gen2.out['ghi_mean'][i], mean_irrad(ghi[:, j]), - atol=0.0, rtol=0.001) + assert np.allclose( + gen2.out["ghi_mean"][i], + mean_irrad(ghi[:, j]), + atol=0.0, + rtol=0.001, + ) diff --git a/tests/test_gen_time_scale.py b/tests/test_gen_time_scale.py index 47e93bf3d..c9be4e984 100644 --- a/tests/test_gen_time_scale.py +++ b/tests/test_gen_time_scale.py @@ -4,14 +4,15 @@ Test resource up and down scaling """ -import os import json +import os + import h5py -import pytest import numpy as np +import pytest -from reV.generation.generation import Gen from reV import TESTDATADIR +from reV.generation.generation import Gen def test_time_index_step(): diff --git a/tests/test_gen_wave.py b/tests/test_gen_wave.py index a3b2f246d..8647f8476 100644 --- a/tests/test_gen_wave.py +++ b/tests/test_gen_wave.py @@ -9,14 +9,14 @@ """ import os -import pytest + import numpy as np +import pytest +from rex import Resource, safe_json_load +from reV import TESTDATADIR from reV.generation.generation import Gen from reV.SAM.defaults import DefaultMhkWave -from reV import TESTDATADIR - -from rex import Resource, safe_json_load BASELINE = os.path.join(TESTDATADIR, 'gen_out', 'ri_wave_2010.h5') diff --git a/tests/test_gen_wind.py b/tests/test_gen_wind.py index 72da38035..56c1687a4 100644 --- a/tests/test_gen_wind.py +++ b/tests/test_gen_wind.py @@ -31,7 +31,7 @@ class wind_results: """Class to retrieve results from the rev 1.0 pv files""" def __init__(self, f): - self._h5 = h5py.File(f, 'r') + self._h5 = h5py.File(f, "r") def __enter__(self): return self @@ -45,15 +45,15 @@ def __exit__(self, type, value, traceback): @property def years(self): """Get a list of year strings.""" - if not hasattr(self, '_years'): - year_index = self._h5['wind']['year_index'][...] + if not hasattr(self, "_years"): + year_index = self._h5["wind"]["year_index"][...] self._years = [y.decode() for y in year_index] return self._years def get_cf_mean(self, site, year): """Get a cf mean based on site and year""" iy = self.years.index(year) - out = self._h5['wind']['cf_mean'][iy, site] + out = self._h5["wind"]["cf_mean"][iy, site] return out @@ -69,32 +69,37 @@ def is_num(n): def to_list(gen_out): """Generation output handler that converts to the rev 1.0 format.""" if isinstance(gen_out, list) and len(gen_out) == 1: - out = [c['cf_mean'] for c in gen_out[0].values()] + out = [c["cf_mean"] for c in gen_out[0].values()] if isinstance(gen_out, dict): - out = [c['cf_mean'] for c in gen_out.values()] + out = [c["cf_mean"] for c in gen_out.values()] return out -@pytest.mark.parametrize(('f_rev1_out', 'rev2_points', 'year', 'max_workers'), - [ - ('project_outputs.h5', slice(0, 10), '2012', 1), - ('project_outputs.h5', slice(0, 100, 10), '2013', 2)]) +@pytest.mark.parametrize( + ("f_rev1_out", "rev2_points", "year", "max_workers"), + [ + ("project_outputs.h5", slice(0, 10), "2012", 1), + ("project_outputs.h5", slice(0, 100, 10), "2013", 2), + ], +) def test_wind_gen_slice(f_rev1_out, rev2_points, year, max_workers): """Test reV 2.0 generation for PV and benchmark against reV 1.0 results.""" # get full file paths. - rev1_outs = os.path.join(TESTDATADIR, 'ri_wind', 'scalar_outputs', - f_rev1_out) - sam_files = TESTDATADIR + '/SAM/wind_gen_standard_losses_0.json' - res_file = TESTDATADIR + '/wtk/ri_100_wtk_{}.h5'.format(year) + rev1_outs = os.path.join( + TESTDATADIR, "ri_wind", "scalar_outputs", f_rev1_out + ) + sam_files = TESTDATADIR + "/SAM/wind_gen_standard_losses_0.json" + res_file = TESTDATADIR + "/wtk/ri_100_wtk_{}.h5".format(year) # run reV 2.0 generation - pp = ProjectPoints(rev2_points, sam_files, 'windpower', res_file=res_file) - gen = Gen('windpower', rev2_points, sam_files, res_file, - sites_per_worker=3) + pp = ProjectPoints(rev2_points, sam_files, "windpower", res_file=res_file) + gen = Gen( + "windpower", rev2_points, sam_files, res_file, sites_per_worker=3 + ) gen.run(max_workers=max_workers) - gen_outs = list(gen.out['cf_mean']) + gen_outs = list(gen.out["cf_mean"]) # initialize the rev1 output hander with wind_results(rev1_outs) as wind: @@ -102,10 +107,12 @@ def test_wind_gen_slice(f_rev1_out, rev2_points, year, max_workers): cf_mean_list = wind.get_cf_mean(pp.sites, year) # benchmark the results - msg = 'Wind cf_means results did not match reV 1.0 results!' + msg = "Wind cf_means results did not match reV 1.0 results!" assert np.allclose(gen_outs, cf_mean_list, rtol=RTOL, atol=ATOL), msg - assert np.allclose(pp.sites, gen.meta.index.values), 'bad gen meta!' - assert np.allclose(pp.sites, gen.meta[MetaKeyName.GID].values), 'bad gen meta!' + assert np.allclose(pp.sites, gen.meta.index.values), "bad gen meta!" + assert np.allclose( + pp.sites, gen.meta[MetaKeyName.GID].values + ), "bad gen meta!" labels = [MetaKeyName.LATITUDE, MetaKeyName.LONGITUDE] with Resource(res_file) as res: @@ -118,15 +125,18 @@ def test_wind_gen_slice(f_rev1_out, rev2_points, year, max_workers): assert site_meta[MetaKeyName.GID] == res_gid -@pytest.mark.parametrize('gid_map', - [{0: 0, 1: 1, 2: 1, 3: 3, 4: 4}, - {0: 4, 1: 3, 2: 2, 3: 1, 4: 0}, - {10: 14, 11: 13, 12: 12, 13: 11, 20: 0}, - {0: 59, 1: 1, 2: 1, 3: 0, 4: 4}, - {0: 59, 1: 1, 2: 0, 3: 0, 4: 4}, - {0: 1, 1: 1, 2: 0, 3: 0, 4: 0}, - {0: 0, 1: 0, 2: 0, 3: 0, 4: 0}, - ]) +@pytest.mark.parametrize( + "gid_map", + [ + {0: 0, 1: 1, 2: 1, 3: 3, 4: 4}, + {0: 4, 1: 3, 2: 2, 3: 1, 4: 0}, + {10: 14, 11: 13, 12: 12, 13: 11, 20: 0}, + {0: 59, 1: 1, 2: 1, 3: 0, 4: 4}, + {0: 59, 1: 1, 2: 0, 3: 0, 4: 4}, + {0: 1, 1: 1, 2: 0, 3: 0, 4: 0}, + {0: 0, 1: 0, 2: 0, 3: 0, 4: 0}, + ], +) def test_gid_map(gid_map): """Test gid mapping feature where the unique gen_gids are mapped to non-unique res_gids @@ -135,47 +145,77 @@ def test_gid_map(gid_map): points_test = sorted(list(set(gid_map.keys()))) year = 2012 max_workers = 1 - sam_files = TESTDATADIR + '/SAM/wind_gen_standard_losses_0.json' - res_file = TESTDATADIR + '/wtk/ri_100_wtk_{}.h5'.format(year) - - output_request = ('cf_mean', 'cf_profile', 'ws_mean', 'windspeed', - 'monthly_energy') - - baseline = Gen('windpower', points_base, sam_files, res_file, - sites_per_worker=3, output_request=output_request) + sam_files = TESTDATADIR + "/SAM/wind_gen_standard_losses_0.json" + res_file = TESTDATADIR + "/wtk/ri_100_wtk_{}.h5".format(year) + + output_request = ( + "cf_mean", + "cf_profile", + "ws_mean", + "windspeed", + "monthly_energy", + ) + + baseline = Gen( + "windpower", + points_base, + sam_files, + res_file, + sites_per_worker=3, + output_request=output_request, + ) baseline.run(max_workers=max_workers) - map_test = Gen('windpower', points_test, sam_files, res_file, - sites_per_worker=3, output_request=output_request, - gid_map=gid_map) + map_test = Gen( + "windpower", + points_test, + sam_files, + res_file, + sites_per_worker=3, + output_request=output_request, + gid_map=gid_map, + ) map_test.run(max_workers=max_workers) - write_gid_test = Gen('windpower', points_test, sam_files, res_file, - sites_per_worker=3, output_request=output_request, - gid_map=gid_map, write_mapped_gids=True) + write_gid_test = Gen( + "windpower", + points_test, + sam_files, + res_file, + sites_per_worker=3, + output_request=output_request, + gid_map=gid_map, + write_mapped_gids=True, + ) write_gid_test.run(max_workers=max_workers) for key in output_request: assert np.allclose(map_test.out[key], write_gid_test.out[key]) - for map_test_gid, write_test_gid in zip(map_test.meta[MetaKeyName.GID], - write_gid_test.meta[MetaKeyName.GID]): + for map_test_gid, write_test_gid in zip( + map_test.meta[MetaKeyName.GID], write_gid_test.meta[MetaKeyName.GID] + ): assert map_test_gid == gid_map[write_test_gid] - if len(baseline.out['cf_mean']) == len(map_test.out['cf_mean']): - assert not np.allclose(baseline.out['cf_mean'], - map_test.out['cf_mean']) + if len(baseline.out["cf_mean"]) == len(map_test.out["cf_mean"]): + assert not np.allclose( + baseline.out["cf_mean"], map_test.out["cf_mean"] + ) for gen_gid_test, res_gid in gid_map.items(): gen_gid_test = points_test.index(gen_gid_test) gen_gid_base = points_base.index(res_gid) for key in output_request: if len(map_test.out[key].shape) == 2: - assert np.allclose(baseline.out[key][:, gen_gid_base], - map_test.out[key][:, gen_gid_test]) + assert np.allclose( + baseline.out[key][:, gen_gid_base], + map_test.out[key][:, gen_gid_test], + ) else: - assert np.allclose(baseline.out[key][gen_gid_base], - map_test.out[key][gen_gid_test]) + assert np.allclose( + baseline.out[key][gen_gid_base], + map_test.out[key][gen_gid_test], + ) labels = [MetaKeyName.LATITUDE, MetaKeyName.LONGITUDE] with Resource(res_file) as res: @@ -197,100 +237,126 @@ def test_gid_map(gid_map): def test_wind_gen_new_outputs(points=slice(0, 10), year=2012, max_workers=1): """Test reV 2.0 generation for wind with new outputs.""" # get full file paths. - sam_files = TESTDATADIR + '/SAM/wind_gen_standard_losses_0.json' - res_file = TESTDATADIR + '/wtk/ri_100_wtk_{}.h5'.format(year) + sam_files = TESTDATADIR + "/SAM/wind_gen_standard_losses_0.json" + res_file = TESTDATADIR + "/wtk/ri_100_wtk_{}.h5".format(year) - output_request = ('cf_mean', 'cf_profile', 'monthly_energy') + output_request = ("cf_mean", "cf_profile", "monthly_energy") # run reV 2.0 generation - gen = Gen('windpower', points, sam_files, res_file, sites_per_worker=3, - output_request=output_request) + gen = Gen( + "windpower", + points, + sam_files, + res_file, + sites_per_worker=3, + output_request=output_request, + ) gen.run(max_workers=max_workers) - assert gen.out['cf_mean'].shape == (10, ) - assert gen.out['cf_profile'].shape == (8760, 10) - assert gen.out['monthly_energy'].shape == (12, 10) + assert gen.out["cf_mean"].shape == (10,) + assert gen.out["cf_profile"].shape == (8760, 10) + assert gen.out["monthly_energy"].shape == (12, 10) - assert gen._out['cf_mean'].dtype == np.float32 - assert gen._out['cf_profile'].dtype == np.uint16 - assert gen._out['monthly_energy'].dtype == np.float32 + assert gen._out["cf_mean"].dtype == np.float32 + assert gen._out["cf_profile"].dtype == np.uint16 + assert gen._out["monthly_energy"].dtype == np.float32 -def test_windspeed_pass_through(rev2_points=slice(0, 10), year=2012, - max_workers=1): +def test_windspeed_pass_through( + rev2_points=slice(0, 10), year=2012, max_workers=1 +): """Test a windspeed output request so that resource array is passed through to output dict.""" - sam_files = TESTDATADIR + '/SAM/wind_gen_standard_losses_0.json' - res_file = TESTDATADIR + '/wtk/ri_100_wtk_{}.h5'.format(year) + sam_files = TESTDATADIR + "/SAM/wind_gen_standard_losses_0.json" + res_file = TESTDATADIR + "/wtk/ri_100_wtk_{}.h5".format(year) - output_requests = ('cf_mean', 'windspeed') + output_requests = ("cf_mean", "windspeed") # run reV 2.0 generation - gen = Gen('windpower', rev2_points, sam_files, res_file, - sites_per_worker=3, output_request=output_requests) + gen = Gen( + "windpower", + rev2_points, + sam_files, + res_file, + sites_per_worker=3, + output_request=output_requests, + ) gen.run(max_workers=max_workers) - assert 'windspeed' in gen.out - assert gen.out['windspeed'].shape == (8760, 10) - assert gen._out['windspeed'].max() == 2597 - assert gen._out['windspeed'].min() == 1 + assert "windspeed" in gen.out + assert gen.out["windspeed"].shape == (8760, 10) + assert gen._out["windspeed"].max() == 2597 + assert gen._out["windspeed"].min() == 1 def test_multi_file_5min_wtk(): """Test running reV gen from a multi-h5 directory with prefix and suffix""" points = slice(0, 10) max_workers = 1 - sam_files = TESTDATADIR + '/SAM/wind_gen_standard_losses_0.json' - res_file = TESTDATADIR + '/wtk/wtk_{}_*m.h5'.format(2010) + sam_files = TESTDATADIR + "/SAM/wind_gen_standard_losses_0.json" + res_file = TESTDATADIR + "/wtk/wtk_{}_*m.h5".format(2010) # run reV 2.0 generation - gen = Gen('windpower', points, sam_files, res_file, sites_per_worker=3) + gen = Gen("windpower", points, sam_files, res_file, sites_per_worker=3) gen.run(max_workers=max_workers) - gen_outs = list(gen._out['cf_mean']) + gen_outs = list(gen._out["cf_mean"]) assert len(gen_outs) == 10 assert np.mean(gen_outs) > 0.55 def test_wind_gen_site_data(points=slice(0, 5), year=2012, max_workers=1): """Test site specific SAM input config via site_data arg""" - sam_files = TESTDATADIR + '/SAM/wind_gen_standard_losses_0.json' - res_file = TESTDATADIR + '/wtk/ri_100_wtk_{}.h5'.format(year) - - output_request = ('cf_mean', 'turb_generic_loss') - - baseline = Gen('windpower', points, sam_files, res_file, - sites_per_worker=3, output_request=output_request) + sam_files = TESTDATADIR + "/SAM/wind_gen_standard_losses_0.json" + res_file = TESTDATADIR + "/wtk/ri_100_wtk_{}.h5".format(year) + + output_request = ("cf_mean", "turb_generic_loss") + + baseline = Gen( + "windpower", + points, + sam_files, + res_file, + sites_per_worker=3, + output_request=output_request, + ) baseline.run(max_workers=max_workers) - site_data = pd.DataFrame({MetaKeyName.GID: np.arange(2), - 'turb_generic_loss': np.zeros(2)}) - test = Gen('windpower', points, sam_files, res_file, sites_per_worker=3, - output_request=output_request, site_data=site_data) + site_data = pd.DataFrame( + {MetaKeyName.GID: np.arange(2), "turb_generic_loss": np.zeros(2)} + ) + test = Gen( + "windpower", + points, + sam_files, + res_file, + sites_per_worker=3, + output_request=output_request, + site_data=site_data, + ) test.run(max_workers=max_workers) - assert all(test.out['cf_mean'][0:2] > baseline.out['cf_mean'][0:2]) - assert np.allclose(test.out['cf_mean'][2:], baseline.out['cf_mean'][2:]) - assert np.allclose(test.out['turb_generic_loss'][0:2], np.zeros(2)) - assert np.allclose(test.out['turb_generic_loss'][2:], 16.7 * np.ones(3)) + assert all(test.out["cf_mean"][0:2] > baseline.out["cf_mean"][0:2]) + assert np.allclose(test.out["cf_mean"][2:], baseline.out["cf_mean"][2:]) + assert np.allclose(test.out["turb_generic_loss"][0:2], np.zeros(2)) + assert np.allclose(test.out["turb_generic_loss"][2:], 16.7 * np.ones(3)) def test_multi_resolution_wtk(): """Test windpower analysis with wind at 5min and t+p at hourly""" with tempfile.TemporaryDirectory() as td: - points = slice(0, 2) max_workers = 1 - sam_files = TESTDATADIR + '/SAM/wind_gen_standard_losses_0.json' - source_fp_100m = TESTDATADIR + '/wtk/wtk_2010_100m.h5' - source_fp_200m = TESTDATADIR + '/wtk/wtk_2010_200m.h5' - - fp_hr_100m = os.path.join(td, 'wtk_2010_100m_hr.h5') - fp_hr_200m = os.path.join(td, 'wtk_2010_200m_hr.h5') - fp_hr = os.path.join(td, 'wtk_2010_*hr.h5') - fp_lr = os.path.join(td, 'wtk_2010_lr.h5') + sam_files = TESTDATADIR + "/SAM/wind_gen_standard_losses_0.json" + source_fp_100m = TESTDATADIR + "/wtk/wtk_2010_100m.h5" + source_fp_200m = TESTDATADIR + "/wtk/wtk_2010_200m.h5" + + fp_hr_100m = os.path.join(td, "wtk_2010_100m_hr.h5") + fp_hr_200m = os.path.join(td, "wtk_2010_200m_hr.h5") + fp_hr = os.path.join(td, "wtk_2010_*hr.h5") + fp_lr = os.path.join(td, "wtk_2010_lr.h5") shutil.copy(source_fp_100m, fp_hr_100m) shutil.copy(source_fp_200m, fp_hr_200m) - lr_dsets = ['temperature_100m', 'pressure_100m'] + lr_dsets = ["temperature_100m", "pressure_100m"] with WindResource(fp_hr_100m) as hr_res: ti = hr_res.time_index meta = hr_res.meta @@ -306,62 +372,110 @@ def test_multi_resolution_wtk(): lr_data = [d[t_slice, s_slice] for d in lr_data] lr_shapes = {d: (len(lr_ti), len(lr_meta)) for d in lr_dsets} - Outputs.init_h5(fp_lr, lr_dsets, lr_shapes, lr_attrs, lr_chunks, - lr_dtypes, lr_meta, lr_ti) + Outputs.init_h5( + fp_lr, + lr_dsets, + lr_shapes, + lr_attrs, + lr_chunks, + lr_dtypes, + lr_meta, + lr_ti, + ) for name, arr in zip(lr_dsets, lr_data): - Outputs.add_dataset(fp_lr, name, arr, lr_dtypes[name], - attrs=lr_attrs[name], chunks=lr_chunks[name]) + Outputs.add_dataset( + fp_lr, + name, + arr, + lr_dtypes[name], + attrs=lr_attrs[name], + chunks=lr_chunks[name], + ) for fp in (fp_hr_100m, fp_hr_200m): - with h5py.File(fp, 'a') as f: + with h5py.File(fp, "a") as f: for dset in lr_dsets: if dset in f: del f[dset] # run reV 2.0 generation - gen = Gen('windpower', points, sam_files, fp_hr, - low_res_resource_file=fp_lr, - sites_per_worker=3) + gen = Gen( + "windpower", + points, + sam_files, + fp_hr, + low_res_resource_file=fp_lr, + sites_per_worker=3, + ) gen.run(max_workers=max_workers) - gen_outs = list(gen._out['cf_mean']) + gen_outs = list(gen._out["cf_mean"]) assert len(gen_outs) == 2 assert np.mean(gen_outs) > 0.55 def test_wind_bias_correct(): """Test rev generation with bias correction.""" - sam_files = TESTDATADIR + '/SAM/wind_gen_standard_losses_0.json' - res_file = TESTDATADIR + '/wtk/ri_100_wtk_2012.h5' + sam_files = TESTDATADIR + "/SAM/wind_gen_standard_losses_0.json" + res_file = TESTDATADIR + "/wtk/ri_100_wtk_2012.h5" # run reV 2.0 generation points = slice(0, 10) - gen_base = Gen('windpower', points, sam_files, res_file, - output_request=('cf_mean', 'cf_profile', 'ws_mean'), - sites_per_worker=3) + gen_base = Gen( + "windpower", + points, + sam_files, + res_file, + output_request=("cf_mean", "cf_profile", "ws_mean"), + sites_per_worker=3, + ) gen_base.run(max_workers=1) - outs_base = np.array(list(gen_base.out['cf_mean'])) - - bc_df = pd.DataFrame({MetaKeyName.GID: np.arange(100), 'method': 'lin_ws', - 'scalar': 1, 'adder': 2}) - gen = Gen('windpower', points, sam_files, res_file, - output_request=('cf_mean', 'cf_profile', 'ws_mean'), - sites_per_worker=3, bias_correct=bc_df) + outs_base = np.array(list(gen_base.out["cf_mean"])) + + bc_df = pd.DataFrame( + { + MetaKeyName.GID: np.arange(100), + "method": "lin_ws", + "scalar": 1, + "adder": 2, + } + ) + gen = Gen( + "windpower", + points, + sam_files, + res_file, + output_request=("cf_mean", "cf_profile", "ws_mean"), + sites_per_worker=3, + bias_correct=bc_df, + ) gen.run(max_workers=1) - outs_bc = np.array(list(gen.out['cf_mean'])) + outs_bc = np.array(list(gen.out["cf_mean"])) assert all(outs_bc > outs_base) - assert np.allclose(gen_base.out['ws_mean'] + 2, gen.out['ws_mean']) - - bc_df = pd.DataFrame({MetaKeyName.GID: np.arange(100), 'method': 'lin_ws', - 'scalar': 1, 'adder': -100}) - gen = Gen('windpower', points, sam_files, res_file, - output_request=('cf_mean', 'cf_profile', 'ws_mean'), - sites_per_worker=3, bias_correct=bc_df) + assert np.allclose(gen_base.out["ws_mean"] + 2, gen.out["ws_mean"]) + + bc_df = pd.DataFrame( + { + MetaKeyName.GID: np.arange(100), + "method": "lin_ws", + "scalar": 1, + "adder": -100, + } + ) + gen = Gen( + "windpower", + points, + sam_files, + res_file, + output_request=("cf_mean", "cf_profile", "ws_mean"), + sites_per_worker=3, + bias_correct=bc_df, + ) gen.run(max_workers=1) for k, arr in gen.out.items(): assert (np.array(arr) == 0).all() -def execute_pytest(capture='all', flags='-rapP'): +def execute_pytest(capture="all", flags="-rapP"): """Execute module as pytest with detailed summary report. Parameters @@ -374,8 +488,8 @@ def execute_pytest(capture='all', flags='-rapP'): """ fname = os.path.basename(__file__) - pytest.main(['-q', '--show-capture={}'.format(capture), fname, flags]) + pytest.main(["-q", "--show-capture={}".format(capture), fname, flags]) -if __name__ == '__main__': +if __name__ == "__main__": execute_pytest() diff --git a/tests/test_handlers_multiyear.py b/tests/test_handlers_multiyear.py index 019adf8e6..40d3340c8 100644 --- a/tests/test_handlers_multiyear.py +++ b/tests/test_handlers_multiyear.py @@ -2,24 +2,24 @@ """ pytests for MultiYear collection and computation """ -import h5py -import numpy as np +import json import os import shutil -import pytest import tempfile -import json import traceback +import h5py +import numpy as np +import pytest +from rex import Resource +from rex.utilities.loggers import init_logger + +from reV import TESTDATADIR from reV.cli import main -from reV.handlers.outputs import Outputs from reV.handlers.multi_year import MultiYear -from reV import TESTDATADIR +from reV.handlers.outputs import Outputs from reV.utilities import ModuleName -from rex import Resource -from rex.utilities.loggers import init_logger - H5_DIR = os.path.join(TESTDATADIR, 'gen_out') YEARS = [2012, 2013] H5_FILES = [os.path.join(H5_DIR, 'gen_ri_pv_{}_x000.h5'.format(year)) diff --git a/tests/test_handlers_transmission.py b/tests/test_handlers_transmission.py index d65583cbd..8f89a47de 100644 --- a/tests/test_handlers_transmission.py +++ b/tests/test_handlers_transmission.py @@ -2,6 +2,7 @@ """ Transmission Feature Tests """ + import os import pandas as pd @@ -11,96 +12,142 @@ from reV.handlers.transmission import TransmissionFeatures as TF from reV.utilities import MetaKeyName -TRANS_COSTS_1 = {'line_tie_in_cost': 200, 'line_cost': 1000, - 'station_tie_in_cost': 50, 'center_tie_in_cost': 10, - 'sink_tie_in_cost': 100, 'available_capacity': 0.1} - - -TRANS_COSTS_2 = {'line_tie_in_cost': 3000, 'line_cost': 2000, - 'station_tie_in_cost': 500, 'center_tie_in_cost': 100, - 'sink_tie_in_cost': 1e6, 'available_capacity': 0.9} - -COSTS = {'1-0-43300': 200, '2-0-43300': 3000, - '1-100-43300': 100200, '2-100-43300': 203000, - '1-0-68867': 50, '2-0-68867': 500, - '1-100-68867': 100050, '2-100-68867': 200500, - '1-0-80844': 10, '2-0-80844': 100, - '1-100-80844': 100010, '2-100-80844': 200100, - '1-0-80843': 100, '2-0-80843': 1000000.0, - '1-100-80843': 100100, '2-100-80843': 1200000.0} - -LINE_CAPS = {1: {43430: 0.0, 43439: 0.0, 43440: 81.41666666666666, - 43416: 81.41666666666666, 43420: 0.0, - 43428: 81.41666666666666, 43432: 0.0, - 43448: 81.41666666666666, 43451: 81.41666666666666, - 43636: 81.41666666666666}, - 2: {43430: 95.5, 43439: 95.5, 43440: 1099.0, 43416: 1099.0, - 43420: 316.0, 43428: 1099.0, 43432: 95.5, 43448: 1099.0, - 43451: 1099.0, 43636: 1099.0}} +TRANS_COSTS_1 = { + "line_tie_in_cost": 200, + "line_cost": 1000, + "station_tie_in_cost": 50, + "center_tie_in_cost": 10, + "sink_tie_in_cost": 100, + "available_capacity": 0.1, +} + + +TRANS_COSTS_2 = { + "line_tie_in_cost": 3000, + "line_cost": 2000, + "station_tie_in_cost": 500, + "center_tie_in_cost": 100, + "sink_tie_in_cost": 1e6, + "available_capacity": 0.9, +} + +COSTS = { + "1-0-43300": 200, + "2-0-43300": 3000, + "1-100-43300": 100200, + "2-100-43300": 203000, + "1-0-68867": 50, + "2-0-68867": 500, + "1-100-68867": 100050, + "2-100-68867": 200500, + "1-0-80844": 10, + "2-0-80844": 100, + "1-100-80844": 100010, + "2-100-80844": 200100, + "1-0-80843": 100, + "2-0-80843": 1000000.0, + "1-100-80843": 100100, + "2-100-80843": 1200000.0, +} + +LINE_CAPS = { + 1: { + 43430: 0.0, + 43439: 0.0, + 43440: 81.41666666666666, + 43416: 81.41666666666666, + 43420: 0.0, + 43428: 81.41666666666666, + 43432: 0.0, + 43448: 81.41666666666666, + 43451: 81.41666666666666, + 43636: 81.41666666666666, + }, + 2: { + 43430: 95.5, + 43439: 95.5, + 43440: 1099.0, + 43416: 1099.0, + 43420: 316.0, + 43428: 1099.0, + 43432: 95.5, + 43448: 1099.0, + 43451: 1099.0, + 43636: 1099.0, + }, +} @pytest.fixture def trans_table(): """Get the transmission mapping table""" - path = os.path.join(TESTDATADIR, 'trans_tables/ri_transmission_table.csv') + path = os.path.join(TESTDATADIR, "trans_tables/ri_transmission_table.csv") trans_table = pd.read_csv(path) return trans_table -@pytest.mark.parametrize(('i', 'trans_costs', 'distance', MetaKeyName.GID), - ((1, TRANS_COSTS_1, 0, 43300), - (2, TRANS_COSTS_2, 0, 43300), - (1, TRANS_COSTS_1, 100, 43300), - (2, TRANS_COSTS_2, 100, 43300), - (1, TRANS_COSTS_1, 0, 68867), - (2, TRANS_COSTS_2, 0, 68867), - (1, TRANS_COSTS_1, 100, 68867), - (2, TRANS_COSTS_2, 100, 68867), - (1, TRANS_COSTS_1, 0, 80844), - (2, TRANS_COSTS_2, 0, 80844), - (1, TRANS_COSTS_1, 100, 80844), - (2, TRANS_COSTS_2, 100, 80844), - (1, TRANS_COSTS_1, 0, 80843), - (2, TRANS_COSTS_2, 0, 80843), - (1, TRANS_COSTS_1, 100, 80843), - (2, TRANS_COSTS_2, 100, 80843))) +@pytest.mark.parametrize( + ("i", "trans_costs", "distance", MetaKeyName.GID), + ( + (1, TRANS_COSTS_1, 0, 43300), + (2, TRANS_COSTS_2, 0, 43300), + (1, TRANS_COSTS_1, 100, 43300), + (2, TRANS_COSTS_2, 100, 43300), + (1, TRANS_COSTS_1, 0, 68867), + (2, TRANS_COSTS_2, 0, 68867), + (1, TRANS_COSTS_1, 100, 68867), + (2, TRANS_COSTS_2, 100, 68867), + (1, TRANS_COSTS_1, 0, 80844), + (2, TRANS_COSTS_2, 0, 80844), + (1, TRANS_COSTS_1, 100, 80844), + (2, TRANS_COSTS_2, 100, 80844), + (1, TRANS_COSTS_1, 0, 80843), + (2, TRANS_COSTS_2, 0, 80843), + (1, TRANS_COSTS_1, 100, 80843), + (2, TRANS_COSTS_2, 100, 80843), + ), +) def test_cost_calculation(i, trans_costs, distance, gid, trans_table): """ Test tranmission capital cost calculation """ tcosts = trans_costs.copy() - avail_cap_frac = tcosts.pop('available_capacity') + avail_cap_frac = tcosts.pop("available_capacity") tf = TF(trans_table, avail_cap_frac=avail_cap_frac, **tcosts) - true_cost = COSTS['{}-{}-{}'.format(i, distance, gid)] + true_cost = COSTS["{}-{}-{}".format(i, distance, gid)] trans_cost = tf.cost(gid, distance) assert true_cost == trans_cost -@pytest.mark.parametrize(('trans_costs', MetaKeyName.CAPACITY, MetaKeyName.GID), - ((TRANS_COSTS_1, 350, 43300), - (TRANS_COSTS_2, 350, 43300), - (TRANS_COSTS_1, 100, 43300), - (TRANS_COSTS_2, 100, 43300), - (TRANS_COSTS_1, 350, 80844), - (TRANS_COSTS_2, 350, 80844), - (TRANS_COSTS_1, 100, 80844), - (TRANS_COSTS_2, 100, 80844))) +@pytest.mark.parametrize( + ("trans_costs", MetaKeyName.CAPACITY, MetaKeyName.GID), + ( + (TRANS_COSTS_1, 350, 43300), + (TRANS_COSTS_2, 350, 43300), + (TRANS_COSTS_1, 100, 43300), + (TRANS_COSTS_2, 100, 43300), + (TRANS_COSTS_1, 350, 80844), + (TRANS_COSTS_2, 350, 80844), + (TRANS_COSTS_1, 100, 80844), + (TRANS_COSTS_2, 100, 80844), + ), +) def test_connect(trans_costs, capacity, gid, trans_table): """ Test connection to transmission lines and load centers """ tcosts = trans_costs.copy() - avail_cap_frac = tcosts.pop('available_capacity') + avail_cap_frac = tcosts.pop("available_capacity") tf = TF(trans_table, avail_cap_frac=avail_cap_frac, **tcosts) - avail_cap = tf[gid].get('avail_cap', None) + avail_cap = tf[gid].get("avail_cap", None) if avail_cap is not None: if avail_cap > capacity: assert tf.connect(gid, capacity, apply=False) -@pytest.mark.parametrize('line_limited', (False, True)) +@pytest.mark.parametrize("line_limited", (False, True)) def test_substation_connect(line_limited, trans_table): """ Test connection to substation @@ -109,18 +156,23 @@ def test_substation_connect(line_limited, trans_table): gid = 68867 trans_costs = TRANS_COSTS_1.copy() - avail_cap_frac = trans_costs.pop('available_capacity') - - tf = TF(trans_table, avail_cap_frac=avail_cap_frac, - line_limited=line_limited, **trans_costs) + avail_cap_frac = trans_costs.pop("available_capacity") + + tf = TF( + trans_table, + avail_cap_frac=avail_cap_frac, + line_limited=line_limited, + **trans_costs, + ) if line_limited: assert not tf.connect(gid, capacity, apply=False) else: assert tf.connect(gid, capacity, apply=False) -@pytest.mark.parametrize(('i', 'trans_costs'), ((1, TRANS_COSTS_1), - (2, TRANS_COSTS_2))) +@pytest.mark.parametrize( + ("i", "trans_costs"), ((1, TRANS_COSTS_1), (2, TRANS_COSTS_2)) +) def test_substation_load_spreading(i, trans_costs, trans_table): """ Test load spreading on connection to substation @@ -129,22 +181,22 @@ def test_substation_load_spreading(i, trans_costs, trans_table): gid = 68867 tcosts = trans_costs.copy() - avail_cap_frac = tcosts.pop('available_capacity') + avail_cap_frac = tcosts.pop("available_capacity") tf = TF(trans_table, avail_cap_frac=avail_cap_frac, **tcosts) connect = tf.connect(gid, capacity, apply=True) assert connect - line_gids = tf[gid]['lines'] + line_gids = tf[gid]["lines"] missing = [gid for gid in line_gids if gid not in LINE_CAPS[i]] - assert not any(missing), 'New gids not in baseline: {}'.format(missing) + assert not any(missing), "New gids not in baseline: {}".format(missing) for line_id in line_gids: - msg = 'Bad line cap: {}'.format(line_id) - assert LINE_CAPS[i][line_id] == tf[line_id]['avail_cap'], msg + msg = "Bad line cap: {}".format(line_id) + assert LINE_CAPS[i][line_id] == tf[line_id]["avail_cap"], msg -def execute_pytest(capture='all', flags='-rapP'): +def execute_pytest(capture="all", flags="-rapP"): """Execute module as pytest with detailed summary report. Parameters @@ -157,8 +209,8 @@ def execute_pytest(capture='all', flags='-rapP'): """ fname = os.path.basename(__file__) - pytest.main(['-q', '--show-capture={}'.format(capture), fname, flags]) + pytest.main(["-q", "--show-capture={}".format(capture), fname, flags]) -if __name__ == '__main__': +if __name__ == "__main__": execute_pytest() diff --git a/tests/test_hybrids.py b/tests/test_hybrids.py index 344d10d5b..83e2b6069 100644 --- a/tests/test_hybrids.py +++ b/tests/test_hybrids.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -"""reV hybrids tests. -""" +"""reV hybrids tests.""" + import json import os import tempfile @@ -17,13 +17,17 @@ from reV.utilities.exceptions import FileInputError, InputError, OutputWarning SOLAR_FPATH = os.path.join( - TESTDATADIR, 'rep_profiles_out', 'rep_profiles_solar.h5') + TESTDATADIR, "rep_profiles_out", "rep_profiles_solar.h5" +) WIND_FPATH = os.path.join( - TESTDATADIR, 'rep_profiles_out', 'rep_profiles_wind.h5') + TESTDATADIR, "rep_profiles_out", "rep_profiles_wind.h5" +) SOLAR_FPATH_30_MIN = os.path.join( - TESTDATADIR, 'rep_profiles_out', 'rep_profiles_solar_30_min.h5') + TESTDATADIR, "rep_profiles_out", "rep_profiles_solar_30_min.h5" +) SOLAR_FPATH_MULT = os.path.join( - TESTDATADIR, 'rep_profiles_out', 'rep_profiles_solar_multiple.h5') + TESTDATADIR, "rep_profiles_out", "rep_profiles_solar_multiple.h5" +) with Resource(SOLAR_FPATH) as res: SOLAR_SCPGIDS = set(res.meta[MetaKeyName.SC_POINT_GID]) with Resource(WIND_FPATH) as res: @@ -41,13 +45,17 @@ def test_hybridization_profile_output_single_resource(): )[0][0] solar_cap = res.meta.loc[solar_idx, MetaKeyName.CAPACITY] - solar_test_profile = res['rep_profiles_0', :, solar_idx] + solar_test_profile = res["rep_profiles_0", :, solar_idx] weighted_solar = solar_cap * solar_test_profile h = Hybridization(SOLAR_FPATH, WIND_FPATH, allow_solar_only=True) h.run() - hp, hsp, hwp, = h.profiles.values() + ( + hp, + hsp, + hwp, + ) = h.profiles.values() h_meta = h.hybrid_meta h_idx = np.where(h_meta[MetaKeyName.SC_POINT_GID] == sc_point_gid)[0][0] @@ -68,16 +76,23 @@ def test_hybridization_profile_output_with_ratio_none(): )[0][0] solar_cap = res.meta.loc[solar_idx, MetaKeyName.CAPACITY] - solar_test_profile = res['rep_profiles_0', :, solar_idx] + solar_test_profile = res["rep_profiles_0", :, solar_idx] weighted_solar = solar_cap * solar_test_profile h = Hybridization( - SOLAR_FPATH, WIND_FPATH, allow_solar_only=True, - ratio=None, ratio_bounds=None + SOLAR_FPATH, + WIND_FPATH, + allow_solar_only=True, + ratio=None, + ratio_bounds=None, ) h.run() - hp, hsp, hwp, = h.profiles.values() + ( + hp, + hsp, + hwp, + ) = h.profiles.values() h_meta = h.hybrid_meta h_idx = np.where(h_meta[MetaKeyName.SC_POINT_GID] == sc_point_gid)[0][0] @@ -96,31 +111,39 @@ def test_hybridization_profile_output(): res.meta[MetaKeyName.SC_POINT_GID] == common_sc_point_gid )[0][0] solar_cap = res.meta.loc[solar_idx, MetaKeyName.CAPACITY] - solar_test_profile = res['rep_profiles_0', :, solar_idx] + solar_test_profile = res["rep_profiles_0", :, solar_idx] with Resource(WIND_FPATH) as res: wind_idx = np.where( res.meta[MetaKeyName.SC_POINT_GID] == common_sc_point_gid )[0][0] wind_cap = res.meta.loc[wind_idx, MetaKeyName.CAPACITY] - wind_test_profile = res['rep_profiles_0', :, wind_idx] + wind_test_profile = res["rep_profiles_0", :, wind_idx] weighted_solar = solar_cap * solar_test_profile weighted_wind = wind_cap * wind_test_profile h = Hybridization(SOLAR_FPATH, WIND_FPATH) h.run() - hp, hsp, hwp, = h.profiles.values() + ( + hp, + hsp, + hwp, + ) = h.profiles.values() h_meta = h.hybrid_meta - h_idx = np.where(h_meta[MetaKeyName.SC_POINT_GID] == common_sc_point_gid)[0][0] + h_idx = np.where(h_meta[MetaKeyName.SC_POINT_GID] == common_sc_point_gid)[ + 0 + ][0] assert np.allclose(hp[:, h_idx], weighted_solar + weighted_wind) assert np.allclose(hsp[:, h_idx], weighted_solar) assert np.allclose(hwp[:, h_idx], weighted_wind) -@pytest.mark.parametrize("input_files", [(SOLAR_FPATH, WIND_FPATH), - (SOLAR_FPATH_30_MIN, WIND_FPATH)]) +@pytest.mark.parametrize( + "input_files", + [(SOLAR_FPATH, WIND_FPATH), (SOLAR_FPATH_30_MIN, WIND_FPATH)], +) def test_hybridization_output_shapes(input_files): """Test that the output shapes are as expected.""" @@ -139,8 +162,7 @@ def test_hybridization_output_shapes(input_files): for arr, expected_shape in zip(out, expected_shapes): assert arr.shape == expected_shape - h = Hybridization(sfp, wfp, - allow_solar_only=True, allow_wind_only=True) + h = Hybridization(sfp, wfp, allow_solar_only=True, allow_wind_only=True) h.run() out = [*h.profiles.values(), h.hybrid_meta, h.hybrid_time_index] expected_shapes = [(8760, 147)] * 3 + [(147, 73), (8760,)] @@ -154,15 +176,16 @@ def test_hybridization_output_shapes(input_files): ((False, False), (53, 73), SOLAR_SCPGIDS & WIND_SCPGIDS), ((True, False), (100, 73), SOLAR_SCPGIDS), ((False, True), (100, 73), WIND_SCPGIDS), - ((True, True), (147, 73), SOLAR_SCPGIDS | WIND_SCPGIDS) - ] + ((True, True), (147, 73), SOLAR_SCPGIDS | WIND_SCPGIDS), + ], ) def test_meta_hybridization(input_combination, expected_shape, overlap): """Test that the meta is hybridized properly.""" allow_solar_only, allow_wind_only = input_combination h = Hybridization( - SOLAR_FPATH, WIND_FPATH, + SOLAR_FPATH, + WIND_FPATH, allow_solar_only=allow_solar_only, allow_wind_only=allow_wind_only, ) @@ -174,87 +197,110 @@ def test_meta_hybridization(input_combination, expected_shape, overlap): def test_limits_and_ratios_output_values(): """Test that limits and ratios are properly applied in succession.""" - limits = {'solar_capacity': 50, 'wind_capacity': 0.5} - ratio_numerator = 'solar_capacity' - ratio_denominator = 'wind_capacity' - ratio = '{}/{}'.format(ratio_numerator, ratio_denominator) + limits = {"solar_capacity": 50, "wind_capacity": 0.5} + ratio_numerator = "solar_capacity" + ratio_denominator = "wind_capacity" + ratio = "{}/{}".format(ratio_numerator, ratio_denominator) ratio_bounds = (0.3, 3.6) bounds = (0.3 - 1e6, 3.6 + 1e6) h = Hybridization( - SOLAR_FPATH, WIND_FPATH, + SOLAR_FPATH, + WIND_FPATH, limits=limits, ratio=ratio, - ratio_bounds=ratio_bounds + ratio_bounds=ratio_bounds, ) h.run() - ratios = (h.hybrid_meta['hybrid_{}'.format(ratio_numerator)] - / h.hybrid_meta['hybrid_{}'.format(ratio_denominator)]) + ratios = ( + h.hybrid_meta["hybrid_{}".format(ratio_numerator)] + / h.hybrid_meta["hybrid_{}".format(ratio_denominator)] + ) assert np.all(ratios.between(*bounds)) - assert np.all(h.hybrid_meta['hybrid_{}'.format(ratio_numerator)] - <= h.hybrid_meta[ratio_numerator]) - assert np.all(h.hybrid_meta['hybrid_{}'.format(ratio_denominator)] - <= h.hybrid_meta[ratio_denominator]) - assert np.all(h.hybrid_meta['solar_capacity'] <= limits['solar_capacity']) - assert np.all(h.hybrid_meta['wind_capacity'] <= limits['wind_capacity']) - - -@pytest.mark.parametrize("ratio_cols", [ - ('solar_capacity', 'wind_capacity'), - ('solar_area_sq_km', 'wind_area_sq_km') -]) -@pytest.mark.parametrize("ratio_bounds, bounds", [ - ((0.5, 0.5), (0.5 - 1e6, 0.5 + 1e6)), - ((1, 1), (1 - 1e6, 1 + 1e6)), - ((0.5, 1.5), (0.5 - 1e6, 1.5 + 1e6)), - ((0.3, 3.6), (0.3 - 1e6, 3.6 + 1e6)) -]) + assert np.all( + h.hybrid_meta["hybrid_{}".format(ratio_numerator)] + <= h.hybrid_meta[ratio_numerator] + ) + assert np.all( + h.hybrid_meta["hybrid_{}".format(ratio_denominator)] + <= h.hybrid_meta[ratio_denominator] + ) + assert np.all(h.hybrid_meta["solar_capacity"] <= limits["solar_capacity"]) + assert np.all(h.hybrid_meta["wind_capacity"] <= limits["wind_capacity"]) + + +@pytest.mark.parametrize( + "ratio_cols", + [ + ("solar_capacity", "wind_capacity"), + ("solar_area_sq_km", "wind_area_sq_km"), + ], +) +@pytest.mark.parametrize( + "ratio_bounds, bounds", + [ + ((0.5, 0.5), (0.5 - 1e6, 0.5 + 1e6)), + ((1, 1), (1 - 1e6, 1 + 1e6)), + ((0.5, 1.5), (0.5 - 1e6, 1.5 + 1e6)), + ((0.3, 3.6), (0.3 - 1e6, 3.6 + 1e6)), + ], +) def test_ratios_input(ratio_cols, ratio_bounds, bounds): """Test that the hybrid meta limits the ratio columns correctly.""" ratio_numerator, ratio_denominator = ratio_cols - ratio = '{}/{}'.format(ratio_numerator, ratio_denominator) + ratio = "{}/{}".format(ratio_numerator, ratio_denominator) h = Hybridization( - SOLAR_FPATH, WIND_FPATH, - ratio=ratio, - ratio_bounds=ratio_bounds + SOLAR_FPATH, WIND_FPATH, ratio=ratio, ratio_bounds=ratio_bounds ) h.run() - ratios = (h.hybrid_meta['hybrid_{}'.format(ratio_numerator)] - / h.hybrid_meta['hybrid_{}'.format(ratio_denominator)]) + ratios = ( + h.hybrid_meta["hybrid_{}".format(ratio_numerator)] + / h.hybrid_meta["hybrid_{}".format(ratio_denominator)] + ) assert np.all(ratios.between(*bounds)) - assert np.all(h.hybrid_meta['hybrid_{}'.format(ratio_numerator)] - <= h.hybrid_meta[ratio_numerator]) - assert np.all(h.hybrid_meta['hybrid_{}'.format(ratio_denominator)] - <= h.hybrid_meta[ratio_denominator]) + assert np.all( + h.hybrid_meta["hybrid_{}".format(ratio_numerator)] + <= h.hybrid_meta[ratio_numerator] + ) + assert np.all( + h.hybrid_meta["hybrid_{}".format(ratio_denominator)] + <= h.hybrid_meta[ratio_denominator] + ) if MetaKeyName.CAPACITY in ratio: - max_solar_capacities = h.hybrid_meta['hybrid_solar_capacity'] + max_solar_capacities = h.hybrid_meta["hybrid_solar_capacity"] max_solar_capacities = max_solar_capacities.values.reshape(1, -1) - assert np.all(h.profiles['hybrid_solar_profile'] - <= max_solar_capacities) - max_wind_capacities = h.hybrid_meta['hybrid_wind_capacity'] + assert np.all( + h.profiles["hybrid_solar_profile"] <= max_solar_capacities + ) + max_wind_capacities = h.hybrid_meta["hybrid_wind_capacity"] max_wind_capacities = max_wind_capacities.values.reshape(1, -1) - assert np.all(h.profiles['hybrid_wind_profile'] - <= max_wind_capacities) + assert np.all(h.profiles["hybrid_wind_profile"] <= max_wind_capacities) def test_rep_profile_idx_map(): """Test that rep profile index mappings are correct shape.""" h = Hybridization(SOLAR_FPATH, WIND_FPATH, allow_wind_only=True) - for h_idxs, r_idxs in (h.meta_hybridizer.solar_profile_indices_map, - h.meta_hybridizer.wind_profile_indices_map): + for h_idxs, r_idxs in ( + h.meta_hybridizer.solar_profile_indices_map, + h.meta_hybridizer.wind_profile_indices_map, + ): assert h_idxs.size == 0 assert r_idxs.size == 0 h.meta_hybridizer.hybridize() - for idxs, shape in zip((h.meta_hybridizer.solar_profile_indices_map, - h.meta_hybridizer.wind_profile_indices_map), - (53, 100)): + for idxs, shape in zip( + ( + h.meta_hybridizer.solar_profile_indices_map, + h.meta_hybridizer.wind_profile_indices_map, + ), + (53, 100), + ): h_idxs, r_idxs = idxs assert h_idxs.size == shape assert r_idxs.size == shape @@ -263,19 +309,19 @@ def test_rep_profile_idx_map(): def test_limits_values(): """Test that column values are properly limited on user input.""" - limits = {'solar_capacity': 100, 'wind_capacity': 0.5} + limits = {"solar_capacity": 100, "wind_capacity": 0.5} h = Hybridization(SOLAR_FPATH, WIND_FPATH, limits=limits) h.run() - assert np.all(h.hybrid_meta['solar_capacity'] <= limits['solar_capacity']) - assert np.all(h.hybrid_meta['wind_capacity'] <= limits['wind_capacity']) + assert np.all(h.hybrid_meta["solar_capacity"] <= limits["solar_capacity"]) + assert np.all(h.hybrid_meta["wind_capacity"] <= limits["wind_capacity"]) def test_invalid_limits_column_name(): """Test invalid inputs for limits columns.""" - test_limits = {'un_prefixed_col': 0, 'wind_capacity': 10} + test_limits = {"un_prefixed_col": 0, "wind_capacity": 10} with pytest.raises(InputError) as excinfo: Hybridization(SOLAR_FPATH, WIND_FPATH, limits=test_limits) @@ -286,24 +332,27 @@ def test_invalid_limits_column_name(): def test_fillna_values(): """Test that N/A values are filled properly based on user input.""" - fill_vals = {'solar_n_gids': 0, 'wind_capacity': -1} + fill_vals = {"solar_n_gids": 0, "wind_capacity": -1} h = Hybridization( - SOLAR_FPATH, WIND_FPATH, allow_solar_only=True, - allow_wind_only=True, fillna=fill_vals + SOLAR_FPATH, + WIND_FPATH, + allow_solar_only=True, + allow_wind_only=True, + fillna=fill_vals, ) h.run() - assert not np.any(h.hybrid_meta['solar_n_gids'].isna()) - assert not np.any(h.hybrid_meta['wind_capacity'].isna()) - assert np.any(h.hybrid_meta['solar_n_gids'].values == 0) - assert np.any(h.hybrid_meta['wind_capacity'].values == -1) + assert not np.any(h.hybrid_meta["solar_n_gids"].isna()) + assert not np.any(h.hybrid_meta["wind_capacity"].isna()) + assert np.any(h.hybrid_meta["solar_n_gids"].values == 0) + assert np.any(h.hybrid_meta["wind_capacity"].values == -1) def test_invalid_fillna_column_name(): """Test invalid inputs for fillna columns.""" - test_fillna = {'un_prefixed_col': 0, 'wind_capacity': 10} + test_fillna = {"un_prefixed_col": 0, "wind_capacity": 10} with pytest.raises(InputError) as excinfo: Hybridization(SOLAR_FPATH, WIND_FPATH, fillna=test_fillna) @@ -311,24 +360,30 @@ def test_invalid_fillna_column_name(): assert "does not start with a valid prefix" in str(excinfo.value) -@pytest.mark.parametrize("input_combination, na_vals", - [((False, False), (False, False)), - ((True, False), (False, True)), - ((False, True), (True, False)), - ((True, True), (True, True))]) +@pytest.mark.parametrize( + "input_combination, na_vals", + [ + ((False, False), (False, False)), + ((True, False), (False, True)), + ((False, True), (True, False)), + ((True, True), (True, True)), + ], +) def test_all_allow_solar_allow_wind_combinations(input_combination, na_vals): """Test that "allow_x_only" options perform the intended merges.""" allow_solar_only, allow_wind_only = input_combination h = Hybridization( - SOLAR_FPATH, WIND_FPATH, + SOLAR_FPATH, + WIND_FPATH, allow_solar_only=allow_solar_only, allow_wind_only=allow_wind_only, ) h.run() - for col_name, should_have_na_vals in zip(['solar_sc_gid', 'wind_sc_gid'], - na_vals): + for col_name, should_have_na_vals in zip( + ["solar_sc_gid", "wind_sc_gid"], na_vals + ): if should_have_na_vals: assert np.any(h.hybrid_meta[col_name].isna()) else: @@ -340,7 +395,8 @@ def test_warning_for_improper_data_output_from_hybrid_method(): def some_new_hybrid_func(__): return [0] - HYBRID_METHODS['scaled_elevation'] = some_new_hybrid_func + + HYBRID_METHODS["scaled_elevation"] = some_new_hybrid_func with pytest.warns(OutputWarning) as records: h = Hybridization(SOLAR_FPATH, WIND_FPATH) @@ -350,7 +406,7 @@ def some_new_hybrid_func(__): assert any("Unable to add" in msg for msg in messages) assert any("column to hybrid meta" in msg for msg in messages) - HYBRID_METHODS.pop('scaled_elevation') + HYBRID_METHODS.pop("scaled_elevation") def test_hybrid_col_additional_method(): @@ -358,24 +414,27 @@ def test_hybrid_col_additional_method(): def some_new_hybrid_func(h): return h.hybrid_meta[MetaKeyName.ELEVATION] * 1000 - HYBRID_METHODS['scaled_elevation'] = some_new_hybrid_func + + HYBRID_METHODS["scaled_elevation"] = some_new_hybrid_func h = Hybridization(SOLAR_FPATH, WIND_FPATH) h.run() - assert 'scaled_elevation' in HYBRID_METHODS - assert 'scaled_elevation' in h.hybrid_meta.columns - assert np.allclose(h.hybrid_meta[MetaKeyName.ELEVATION] * 1000, - h.hybrid_meta['scaled_elevation']) + assert "scaled_elevation" in HYBRID_METHODS + assert "scaled_elevation" in h.hybrid_meta.columns + assert np.allclose( + h.hybrid_meta[MetaKeyName.ELEVATION] * 1000, + h.hybrid_meta["scaled_elevation"], + ) - HYBRID_METHODS.pop('scaled_elevation') + HYBRID_METHODS.pop("scaled_elevation") def test_duplicate_lat_long_values(): """Test duplicate lat/long values corresponding to unique merge column.""" with tempfile.TemporaryDirectory() as td: - fout_solar = os.path.join(td, 'rep_profiles_solar.h5') + fout_solar = os.path.join(td, "rep_profiles_solar.h5") make_test_file(SOLAR_FPATH, fout_solar, duplicate_coord_values=True) with pytest.raises(FileInputError) as excinfo: @@ -388,21 +447,23 @@ def test_duplicate_lat_long_values(): def test_invalid_ratio_bounds_length_input(): """Test improper ratios input.""" - ratio = 'solar_capacity/wind_capacity' + ratio = "solar_capacity/wind_capacity" with pytest.raises(InputError) as excinfo: Hybridization( SOLAR_FPATH, WIND_FPATH, ratio=ratio, ratio_bounds=(1, 2, 3) ) - msg = ("Length of input for ratio_bounds is 3 " - "- but is required to be of length 2.") + msg = ( + "Length of input for ratio_bounds is 3 " + "- but is required to be of length 2." + ) assert msg in str(excinfo.value) def test_ratio_column_missing(): """Test missing ratio column.""" - ratio = 'solar_col_dne/wind_capacity' + ratio = "solar_col_dne/wind_capacity" with pytest.raises(FileInputError) as excinfo: Hybridization( SOLAR_FPATH, WIND_FPATH, ratio=ratio, ratio_bounds=(1, 1) @@ -412,7 +473,7 @@ def test_ratio_column_missing(): assert "not found" in str(excinfo.value) -@pytest.mark.parametrize("ratio", [None, ('solar_capacity', 'wind_capacity')]) +@pytest.mark.parametrize("ratio", [None, ("solar_capacity", "wind_capacity")]) def test_ratio_not_string(ratio): """Test ratio input is not string.""" @@ -426,9 +487,7 @@ def test_ratio_not_string(ratio): @pytest.mark.parametrize( - "ratio", - ['solar_capacity', - 'solar_capacity/wind_capacity/solar_capacity'] + "ratio", ["solar_capacity", "solar_capacity/wind_capacity/solar_capacity"] ) def test_invalid_ratio_format(ratio): """Test ratio input is not string.""" @@ -438,8 +497,10 @@ def test_invalid_ratio_format(ratio): SOLAR_FPATH, WIND_FPATH, ratio=ratio, ratio_bounds=(1, 1) ) - long_msg = ("Please make sure the ratio input is a string in the form " - "'numerator_column_name/denominator_column_name'") + long_msg = ( + "Please make sure the ratio input is a string in the form " + "'numerator_column_name/denominator_column_name'" + ) assert "Ratio input " in str(excinfo.value) assert long_msg in str(excinfo.value) @@ -447,7 +508,7 @@ def test_invalid_ratio_format(ratio): def test_invalid_ratio_column_name(): """Test invalid inputs for ratio columns.""" - ratio = 'un_prefixed_col/wind_capacity' + ratio = "un_prefixed_col/wind_capacity" with pytest.raises(InputError) as excinfo: Hybridization( SOLAR_FPATH, WIND_FPATH, ratio=ratio, ratio_bounds=(1, 1) @@ -461,8 +522,8 @@ def test_no_overlap_in_merge_column_values(): """Test duplicate values in merge column.""" with tempfile.TemporaryDirectory() as td: - fout_solar = os.path.join(td, 'rep_profiles_solar.h5') - fout_wind = os.path.join(td, 'rep_profiles_wind.h5') + fout_solar = os.path.join(td, "rep_profiles_solar.h5") + fout_wind = os.path.join(td, "rep_profiles_wind.h5") make_test_file(SOLAR_FPATH, fout_solar, p_slice=slice(0, 3)) make_test_file(WIND_FPATH, fout_wind, p_slice=slice(90, 100)) @@ -476,7 +537,7 @@ def test_duplicate_merge_column_values(): """Test duplicate values in merge column.""" with tempfile.TemporaryDirectory() as td: - fout_solar = os.path.join(td, 'rep_profiles_solar.h5') + fout_solar = os.path.join(td, "rep_profiles_solar.h5") make_test_file(SOLAR_FPATH, fout_solar, duplicate_rows=True) with pytest.raises(FileInputError) as excinfo: @@ -489,7 +550,7 @@ def test_merge_columns_missing(): """Test missing merge column.""" with tempfile.TemporaryDirectory() as td: - fout_solar = os.path.join(td, 'rep_profiles_solar.h5') + fout_solar = os.path.join(td, "rep_profiles_solar.h5") make_test_file(SOLAR_FPATH, fout_solar, drop_cols=[MERGE_COLUMN]) with pytest.raises(FileInputError) as excinfo: @@ -506,9 +567,11 @@ def test_invalid_num_profiles(): with pytest.raises(FileInputError) as excinfo: Hybridization(SOLAR_FPATH_MULT, WIND_FPATH) - msg = ("This module is not intended for hybridization of " - "multiple representative profiles. Please re-run " - "on a single aggregated profile.") + msg = ( + "This module is not intended for hybridization of " + "multiple representative profiles. Please re-run " + "on a single aggregated profile." + ) assert msg in str(excinfo.value) @@ -516,16 +579,18 @@ def test_invalid_time_index_overlap(): """Test input files with an invalid time index overlap.""" with tempfile.TemporaryDirectory() as td: - fout_solar = os.path.join(td, 'rep_profiles_solar.h5') - fout_wind = os.path.join(td, 'rep_profiles_wind.h5') + fout_solar = os.path.join(td, "rep_profiles_solar.h5") + fout_wind = os.path.join(td, "rep_profiles_wind.h5") make_test_file(SOLAR_FPATH, fout_solar, t_slice=slice(0, 1500)) make_test_file(WIND_FPATH, fout_wind, t_slice=slice(1000, 3000)) with pytest.raises(FileInputError) as excinfo: Hybridization(fout_solar, fout_wind) - msg = ("Please ensure that the input profiles have a " - "time index that overlaps >= 8760 times.") + msg = ( + "Please ensure that the input profiles have a " + "time index that overlaps >= 8760 times." + ) assert msg in str(excinfo.value) @@ -545,7 +610,7 @@ def test_valid_time_index_overlap(): def test_write_to_file(): """Test hybrid rep profiles with file write.""" with tempfile.TemporaryDirectory() as td: - fout = os.path.join(td, 'temp_hybrid_profiles.h5') + fout = os.path.join(td, "temp_hybrid_profiles.h5") h = Hybridization(SOLAR_FPATH, WIND_FPATH) h.run(fout=fout) @@ -556,11 +621,11 @@ def test_write_to_file(): disk_profiles = res[name] assert np.issubdtype(dtype, np.float32) - assert attrs['units'] == 'MW' + assert attrs["units"] == "MW" assert np.allclose(p, disk_profiles) disk_dsets = res.datasets - assert 'rep_profiles_0' not in disk_dsets + assert "rep_profiles_0" not in disk_dsets def test_hybrids_data_content(): @@ -574,7 +639,7 @@ def test_hybrids_data_content(): assert np.all(h_data.wind_meta.fillna(fv) == wr.meta.fillna(fv)) assert np.all(h_data.solar_time_index == sr.time_index) assert np.all(h_data.wind_time_index == wr.time_index) - hyb_idx = sr.time_index.join(wr.time_index, how='inner') + hyb_idx = sr.time_index.join(wr.time_index, how="inner") assert np.all(h_data.hybrid_time_index == hyb_idx) @@ -582,30 +647,31 @@ def test_hybrids_data_contains_col(): """Test the 'contains_col' method of HybridsData for accuracy.""" h_data = HybridsData(SOLAR_FPATH, WIND_FPATH) - assert h_data.contains_col('trans_capacity') - assert h_data.contains_col('dist_mi') - assert h_data.contains_col('dist_km') - assert not h_data.contains_col('dne_col_for_test') - - -@pytest.mark.parametrize("input_files", [ - (SOLAR_FPATH, WIND_FPATH), - (SOLAR_FPATH_30_MIN, WIND_FPATH) -]) -@pytest.mark.parametrize("ratio", [ - 'solar_capacity/wind_capacity', - 'solar_area_sq_km/wind_area_sq_km' -]) + assert h_data.contains_col("trans_capacity") + assert h_data.contains_col("dist_mi") + assert h_data.contains_col("dist_km") + assert not h_data.contains_col("dne_col_for_test") + + +@pytest.mark.parametrize( + "input_files", + [(SOLAR_FPATH, WIND_FPATH), (SOLAR_FPATH_30_MIN, WIND_FPATH)], +) +@pytest.mark.parametrize( + "ratio", + ["solar_capacity/wind_capacity", "solar_area_sq_km/wind_area_sq_km"], +) @pytest.mark.parametrize("ratio_bounds", [None, (0.5, 1.5), (0.3, 3.6)]) @pytest.mark.parametrize("input_combination", [(False, False), (True, True)]) -def test_hybrids_cli_from_config(runner, input_files, ratio, ratio_bounds, - input_combination, clear_loggers): +def test_hybrids_cli_from_config( + runner, input_files, ratio, ratio_bounds, input_combination, clear_loggers +): """Test hybrids cli from config""" fv = -999 sfp, wfp = input_files allow_solar_only, allow_wind_only = input_combination - fill_vals = {'solar_n_gids': 0, 'wind_capacity': -1} - limits = {'solar_capacity': 100} + fill_vals = {"solar_n_gids": 0, "wind_capacity": -1} + limits = {"solar_capacity": 100} with tempfile.TemporaryDirectory() as td: config = { @@ -615,58 +681,66 @@ def test_hybrids_cli_from_config(runner, input_files, ratio, ratio_bounds, "execution_control": { "nodes": 1, "option": "local", - "sites_per_worker": 10 + "sites_per_worker": 10, }, "log_level": "INFO", "allow_solar_only": allow_solar_only, "allow_wind_only": allow_wind_only, "fillna": fill_vals, - 'limits': limits, - 'ratio': ratio, - 'ratio_bounds': ratio_bounds + "limits": limits, + "ratio": ratio, + "ratio_bounds": ratio_bounds, } - config_path = os.path.join(td, 'config.json') - with open(config_path, 'w') as f: + config_path = os.path.join(td, "config.json") + with open(config_path, "w") as f: json.dump(config, f) - result = runner.invoke(main, [str(ModuleName.HYBRIDS), - '-c', config_path]) + result = runner.invoke( + main, [str(ModuleName.HYBRIDS), "-c", config_path] + ) if result.exit_code != 0: import traceback - msg = ('Failed with error {}' - .format(traceback.print_exception(*result.exc_info))) + + msg = "Failed with error {}".format( + traceback.print_exception(*result.exc_info) + ) clear_loggers() raise RuntimeError(msg) h = Hybridization( - sfp, wfp, + sfp, + wfp, allow_solar_only=allow_solar_only, allow_wind_only=allow_wind_only, - fillna=fill_vals, limits=limits, + fillna=fill_vals, + limits=limits, ratio=ratio, - ratio_bounds=ratio_bounds + ratio_bounds=ratio_bounds, ) h.run() dirname = os.path.basename(td) fn_out = "{}_{}.h5".format(dirname, ModuleName.HYBRIDS) out_fpath = os.path.join(td, fn_out) - with Outputs(out_fpath, 'r') as f: + with Outputs(out_fpath, "r") as f: for dset_name in OUTPUT_PROFILE_NAMES: assert dset_name in f.dsets - meta_from_file = f.meta.fillna(fv).replace('nan', fv) + meta_from_file = f.meta.fillna(fv).replace("nan", fv) assert np.all(meta_from_file == h.hybrid_meta.fillna(fv)) assert np.all(f.time_index.values == h.hybrid_time_index.values) clear_loggers() -@pytest.mark.parametrize("bad_fpath", [ - os.path.join(TESTDATADIR, 'rep_profiles_out', 'rep_profiles_sol*.h5'), - os.path.join(TESTDATADIR, 'rep_profiles_out', 'rep_profiles_dne.h5'), -]) +@pytest.mark.parametrize( + "bad_fpath", + [ + os.path.join(TESTDATADIR, "rep_profiles_out", "rep_profiles_sol*.h5"), + os.path.join(TESTDATADIR, "rep_profiles_out", "rep_profiles_dne.h5"), + ], +) def test_hybrids_cli_bad_fpath_input(runner, bad_fpath, clear_loggers): """Test cli when filepath input is ambiguous or invalid.""" @@ -678,17 +752,18 @@ def test_hybrids_cli_bad_fpath_input(runner, bad_fpath, clear_loggers): "execution_control": { "nodes": 1, "option": "local", - "sites_per_worker": 10 + "sites_per_worker": 10, }, "log_level": "INFO", } - config_path = os.path.join(td, 'config.json') - with open(config_path, 'w') as f: + config_path = os.path.join(td, "config.json") + with open(config_path, "w") as f: json.dump(config, f) - result = runner.invoke(main, [str(ModuleName.HYBRIDS), - '-c', config_path]) + result = runner.invoke( + main, [str(ModuleName.HYBRIDS), "-c", config_path] + ) assert result.exit_code != 0 assert "No files found" in str(result.exc_info) @@ -697,9 +772,15 @@ def test_hybrids_cli_bad_fpath_input(runner, bad_fpath, clear_loggers): # pylint: disable=no-member -def make_test_file(in_fp, out_fp, p_slice=slice(None), t_slice=slice(None), - drop_cols=None, duplicate_rows=False, - duplicate_coord_values=False): +def make_test_file( + in_fp, + out_fp, + p_slice=slice(None), + t_slice=slice(None), + drop_cols=None, + duplicate_rows=False, + duplicate_coord_values=False, +): """Generate a test file from existing input file. The new test file can have a subset of the data of the original file. @@ -729,7 +810,7 @@ def make_test_file(in_fp, out_fp, p_slice=slice(None), t_slice=slice(None), meta DataFrame, by default False. """ with Resource(in_fp) as res: - dset_names = [d for d in res.dsets if d not in ('meta', 'time_index')] + dset_names = [d for d in res.dsets if d not in ("meta", "time_index")] shapes = res.shapes meta = res.meta.iloc[p_slice] if drop_cols is not None: @@ -739,27 +820,40 @@ def make_test_file(in_fp, out_fp, p_slice=slice(None), t_slice=slice(None), half_n_rows = n_rows // 2 meta.iloc[-half_n_rows:] = meta.iloc[:half_n_rows].values if duplicate_coord_values: - meta.loc[0, MetaKeyName.LATITUDE] = meta[MetaKeyName.LATITUDE].iloc[-1] - meta.loc[0, MetaKeyName.LATITUDE] = meta[MetaKeyName.LATITUDE].iloc[-1] - shapes['meta'] = len(meta) + meta.loc[0, MetaKeyName.LATITUDE] = meta[ + MetaKeyName.LATITUDE + ].iloc[-1] + meta.loc[0, MetaKeyName.LATITUDE] = meta[ + MetaKeyName.LATITUDE + ].iloc[-1] + shapes["meta"] = len(meta) for d in dset_names: shapes[d] = (len(res.time_index[t_slice]), len(meta)) - Outputs.init_h5(out_fp, dset_names, shapes, res.attrs, res.chunks, - res.dtypes, meta, - time_index=res.time_index[t_slice]) + Outputs.init_h5( + out_fp, + dset_names, + shapes, + res.attrs, + res.chunks, + res.dtypes, + meta, + time_index=res.time_index[t_slice], + ) - with Outputs(out_fp, mode='a') as out: + with Outputs(out_fp, mode="a") as out: for d in dset_names: out[d] = res[d, t_slice, p_slice] - d = 'rep_profiles_0' - assert out._h5[d].shape == (len(res.time_index[t_slice]), - len(meta)) + d = "rep_profiles_0" + assert out._h5[d].shape == ( + len(res.time_index[t_slice]), + len(meta), + ) assert np.all(out[d].sum(axis=0) > 0) -def execute_pytest(capture='all', flags='-rapP'): +def execute_pytest(capture="all", flags="-rapP"): """Execute module as pytest with detailed summary report. Parameters @@ -770,8 +864,8 @@ def execute_pytest(capture='all', flags='-rapP'): flags : str Which tests to show logs and results for. """ - pytest.main(['-q', '--show-capture={}'.format(capture), __file__, flags]) + pytest.main(["-q", "--show-capture={}".format(capture), __file__, flags]) -if __name__ == '__main__': +if __name__ == "__main__": execute_pytest() diff --git a/tests/test_nrwal.py b/tests/test_nrwal.py index c6c74677e..8f2a6b201 100644 --- a/tests/test_nrwal.py +++ b/tests/test_nrwal.py @@ -24,7 +24,7 @@ from reV.nrwal.nrwal import RevNrwal from reV.utilities import MetaKeyName, ModuleName -SOURCE_DIR = os.path.join(TESTDATADIR, 'nrwal/') +SOURCE_DIR = os.path.join(TESTDATADIR, "nrwal/") def test_nrwal(): @@ -34,53 +34,80 @@ def test_nrwal(): for fn in os.listdir(SOURCE_DIR): shutil.copy(os.path.join(SOURCE_DIR, fn), os.path.join(td, fn)) - gen_fpath = os.path.join(td, 'gen_2010_node00.h5') - site_data = os.path.join(td, 'example_offshore_data.csv') - offshore_config = os.path.join(td, 'offshore.json') - onshore_config = os.path.join(td, 'onshore.json') - sam_configs = {'onshore': onshore_config, - MetaKeyName.OFFSHORE: offshore_config} - nrwal_configs = {MetaKeyName.OFFSHORE: os.path.join(td, 'nrwal_offshore.yaml')} - - with Outputs(gen_fpath, 'a') as f: - f.time_index = pd_date_range('20100101', '20110101', - closed='right', freq='1h') - f._add_dset('cf_profile', np.random.random(f.shape), - np.uint32, attrs={'scale_factor': 1000}, - chunks=(None, 10)) - f._add_dset('fixed_charge_rate', - 0.09 * np.ones(f.shape[1], dtype=np.float32), - np.float32, attrs={'scale_factor': 1}, - chunks=None) - - with Outputs(gen_fpath, 'r') as f: - original_dsets = [d for d in f.dsets - if d not in ('meta', 'time_index')] + gen_fpath = os.path.join(td, "gen_2010_node00.h5") + site_data = os.path.join(td, "example_offshore_data.csv") + offshore_config = os.path.join(td, "offshore.json") + onshore_config = os.path.join(td, "onshore.json") + sam_configs = { + "onshore": onshore_config, + MetaKeyName.OFFSHORE: offshore_config, + } + nrwal_configs = { + MetaKeyName.OFFSHORE: os.path.join(td, "nrwal_offshore.yaml") + } + + with Outputs(gen_fpath, "a") as f: + f.time_index = pd_date_range( + "20100101", "20110101", closed="right", freq="1h" + ) + f._add_dset( + "cf_profile", + np.random.random(f.shape), + np.uint32, + attrs={"scale_factor": 1000}, + chunks=(None, 10), + ) + f._add_dset( + "fixed_charge_rate", + 0.09 * np.ones(f.shape[1], dtype=np.float32), + np.float32, + attrs={"scale_factor": 1}, + chunks=None, + ) + + with Outputs(gen_fpath, "r") as f: + original_dsets = [ + d for d in f.dsets if d not in ("meta", "time_index") + ] meta_raw = f.meta - lcoe_raw = f['lcoe_fcr'] - cf_mean_raw = f['cf_mean'] - cf_profile_raw = f['cf_profile'] + lcoe_raw = f["lcoe_fcr"] + cf_mean_raw = f["cf_mean"] + cf_profile_raw = f["cf_profile"] mask = meta_raw.offshore == 1 - output_request = ['fixed_charge_rate', 'depth', 'total_losses', - 'array', 'export', 'gcf_adjustment', - 'lcoe_fcr', 'cf_mean', 'cf_profile'] - - obj = RevNrwal(gen_fpath, site_data, sam_configs, nrwal_configs, - output_request, site_meta_cols=['depth']) + output_request = [ + "fixed_charge_rate", + "depth", + "total_losses", + "array", + "export", + "gcf_adjustment", + "lcoe_fcr", + "cf_mean", + "cf_profile", + ] + + obj = RevNrwal( + gen_fpath, + site_data, + sam_configs, + nrwal_configs, + output_request, + site_meta_cols=["depth"], + ) obj.run() - with Outputs(gen_fpath, 'r') as f: + with Outputs(gen_fpath, "r") as f: meta_new = f.meta - lcoe_new = f['lcoe_fcr'] - losses = f['total_losses'] - gcf_adjustment = f['gcf_adjustment'] - assert np.allclose(cf_mean_raw, f['cf_mean_raw']) - assert np.allclose(cf_profile_raw, f['cf_profile_raw']) - cf_mean_new = f['cf_mean'] - cf_profile_new = f['cf_profile'] - fcr = f['fixed_charge_rate'] - depth = f['depth'] + lcoe_new = f["lcoe_fcr"] + losses = f["total_losses"] + gcf_adjustment = f["gcf_adjustment"] + assert np.allclose(cf_mean_raw, f["cf_mean_raw"]) + assert np.allclose(cf_profile_raw, f["cf_profile_raw"]) + cf_mean_new = f["cf_mean"] + cf_profile_new = f["cf_profile"] + fcr = f["fixed_charge_rate"] + depth = f["depth"] for key in [d for d in original_dsets if d in f]: assert key in f @@ -89,31 +116,36 @@ def test_nrwal(): # check nrwal keys requested as h5 dsets for key in obj._output_request: assert key in f - if 'profile' not in key: + if "profile" not in key: assert np.isnan(f[key][mask]).sum() == 0 else: assert np.isnan(f[key][:, mask]).sum() == 0 - if key in ('total_losses', 'array', 'export'): + if key in ("total_losses", "array", "export"): assert np.isnan(f[key][~mask]).all() # run offshore twice and make sure losses don't get doubled - _ = RevNrwal(gen_fpath, site_data, sam_configs, nrwal_configs, - output_request, site_meta_cols=['depth']).run() + _ = RevNrwal( + gen_fpath, + site_data, + sam_configs, + nrwal_configs, + output_request, + site_meta_cols=["depth"], + ).run() # make sure the second offshore compute gives same results as first - with Outputs(gen_fpath, 'r') as f: - assert np.allclose(lcoe_new, f['lcoe_fcr']) - assert np.allclose(cf_mean_new, f['cf_mean']) - assert np.allclose(cf_profile_new, f['cf_profile']) - assert np.allclose(cf_mean_raw, f['cf_mean_raw']) - assert np.allclose(cf_profile_raw, f['cf_profile_raw']) + with Outputs(gen_fpath, "r") as f: + assert np.allclose(lcoe_new, f["lcoe_fcr"]) + assert np.allclose(cf_mean_new, f["cf_mean"]) + assert np.allclose(cf_profile_new, f["cf_profile"]) + assert np.allclose(cf_mean_raw, f["cf_mean_raw"]) + assert np.allclose(cf_profile_raw, f["cf_profile_raw"]) # check offshore depth data - assert 'depth' in meta_new - assert all(meta_new.loc[(meta_new.offshore == 1), 'depth'] >= 0) - assert all(np.isnan( - meta_new.loc[(meta_new.offshore == 0), 'depth'])) + assert "depth" in meta_new + assert all(meta_new.loc[(meta_new.offshore == 1), "depth"] >= 0) + assert all(np.isnan(meta_new.loc[(meta_new.offshore == 0), "depth"])) assert all(depth[(meta_new.offshore == 1)] >= 0) assert all(np.isnan(depth[(meta_new.offshore == 0)])) @@ -132,10 +164,12 @@ def test_nrwal(): assert (lcoe_new[~mask] == lcoe_raw[~mask]).all() assert (cf_mean_new[mask] < cf_mean_raw[mask]).all() - cf_net = (gcf_adjustment[mask] * (1 - losses[mask]) - * cf_profile_raw[:, mask]) - assert np.allclose(cf_profile_new[:, mask], cf_net, rtol=0.005, - atol=0.001) + cf_net = ( + gcf_adjustment[mask] * (1 - losses[mask]) * cf_profile_raw[:, mask] + ) + assert np.allclose( + cf_profile_new[:, mask], cf_net, rtol=0.005, atol=0.001 + ) assert np.allclose(cf_profile_new[:, ~mask], cf_profile_raw[:, ~mask]) cf_net = gcf_adjustment[mask] * (1 - losses[mask]) * cf_mean_raw[mask] @@ -150,43 +184,72 @@ def test_nrwal_csv(out_fn): for fn in os.listdir(SOURCE_DIR): shutil.copy(os.path.join(SOURCE_DIR, fn), os.path.join(td, fn)) - gen_fpath = os.path.join(td, 'gen_2010_node00.h5') - site_data = os.path.join(td, 'example_offshore_data.csv') - offshore_config = os.path.join(td, 'offshore.json') - onshore_config = os.path.join(td, 'onshore.json') - sam_configs = {'onshore': onshore_config, - MetaKeyName.OFFSHORE: offshore_config} - nrwal_configs = {MetaKeyName.OFFSHORE: os.path.join(td, 'nrwal_offshore.yaml')} - - with Outputs(gen_fpath, 'a') as f: - f.time_index = pd_date_range('20100101', '20110101', - closed='right', freq='1h') - f._add_dset('cf_profile_raw', np.random.random(f.shape), - np.uint32, attrs={'scale_factor': 1000}, - chunks=(None, 10)) - f._add_dset('cf_mean_raw', np.random.random(f.shape[1]), - np.uint32, attrs={'scale_factor': 1000}, - chunks=None) - f._add_dset('fixed_charge_rate', - 0.09 * np.ones(f.shape[1], dtype=np.float32), - np.float32, attrs={'scale_factor': 1}, - chunks=None) - - compatible = ['depth', 'total_losses', 'array', 'export', - 'gcf_adjustment', 'fixed_charge_rate', 'lcoe_fcr', - 'cf_mean'] - incompatible = ['cf_profile'] + gen_fpath = os.path.join(td, "gen_2010_node00.h5") + site_data = os.path.join(td, "example_offshore_data.csv") + offshore_config = os.path.join(td, "offshore.json") + onshore_config = os.path.join(td, "onshore.json") + sam_configs = { + "onshore": onshore_config, + MetaKeyName.OFFSHORE: offshore_config, + } + nrwal_configs = { + MetaKeyName.OFFSHORE: os.path.join(td, "nrwal_offshore.yaml") + } + + with Outputs(gen_fpath, "a") as f: + f.time_index = pd_date_range( + "20100101", "20110101", closed="right", freq="1h" + ) + f._add_dset( + "cf_profile_raw", + np.random.random(f.shape), + np.uint32, + attrs={"scale_factor": 1000}, + chunks=(None, 10), + ) + f._add_dset( + "cf_mean_raw", + np.random.random(f.shape[1]), + np.uint32, + attrs={"scale_factor": 1000}, + chunks=None, + ) + f._add_dset( + "fixed_charge_rate", + 0.09 * np.ones(f.shape[1], dtype=np.float32), + np.float32, + attrs={"scale_factor": 1}, + chunks=None, + ) + + compatible = [ + "depth", + "total_losses", + "array", + "export", + "gcf_adjustment", + "fixed_charge_rate", + "lcoe_fcr", + "cf_mean", + ] + incompatible = ["cf_profile"] output_request = compatible + incompatible with pytest.warns(Warning) as record: - rev_nrwal = RevNrwal(gen_fpath, site_data, sam_configs, - nrwal_configs, output_request, - site_meta_cols=['depth']) + rev_nrwal = RevNrwal( + gen_fpath, + site_data, + sam_configs, + nrwal_configs, + output_request, + site_meta_cols=["depth"], + ) out_fpath = os.path.join(td, out_fn) if out_fn else None out_fpath = rev_nrwal.run(csv_output=True, out_fpath=out_fpath) - expected_message_out = ["`save_raw` option not allowed with " - "`csv_output`"] + expected_message_out = [ + "`save_raw` option not allowed with " "`csv_output`" + ] for r, m in zip(record, expected_message_out): warn_msg = r.message.args[0] assert m in warn_msg @@ -210,38 +273,60 @@ def test_nrwal_constant_eq_output_request(): for fn in os.listdir(SOURCE_DIR): shutil.copy(os.path.join(SOURCE_DIR, fn), os.path.join(td, fn)) - gen_fpath = os.path.join(td, 'gen_2010_node00.h5') - site_data = os.path.join(td, 'example_offshore_data.csv') - offshore_config = os.path.join(td, 'offshore.json') - onshore_config = os.path.join(td, 'onshore.json') - sam_configs = {'onshore': onshore_config, - MetaKeyName.OFFSHORE: offshore_config} - nrwal_configs = {MetaKeyName.OFFSHORE: os.path.join(td, 'nrwal_offshore.yaml')} - - with Outputs(gen_fpath, 'a') as f: - f.time_index = pd_date_range('20100101', '20110101', - closed='right', freq='1h') - f._add_dset('cf_profile', np.random.random(f.shape), - np.uint32, attrs={'scale_factor': 1000}, - chunks=(None, 10)) - f._add_dset('fixed_charge_rate', - 0.09 * np.ones(f.shape[1], dtype=np.float32), - np.float32, attrs={'scale_factor': 1}, - chunks=None) - - with Outputs(gen_fpath, 'r') as f: + gen_fpath = os.path.join(td, "gen_2010_node00.h5") + site_data = os.path.join(td, "example_offshore_data.csv") + offshore_config = os.path.join(td, "offshore.json") + onshore_config = os.path.join(td, "onshore.json") + sam_configs = { + "onshore": onshore_config, + MetaKeyName.OFFSHORE: offshore_config, + } + nrwal_configs = { + MetaKeyName.OFFSHORE: os.path.join(td, "nrwal_offshore.yaml") + } + + with Outputs(gen_fpath, "a") as f: + f.time_index = pd_date_range( + "20100101", "20110101", closed="right", freq="1h" + ) + f._add_dset( + "cf_profile", + np.random.random(f.shape), + np.uint32, + attrs={"scale_factor": 1000}, + chunks=(None, 10), + ) + f._add_dset( + "fixed_charge_rate", + 0.09 * np.ones(f.shape[1], dtype=np.float32), + np.float32, + attrs={"scale_factor": 1}, + chunks=None, + ) + + with Outputs(gen_fpath, "r") as f: meta_raw = f.meta mask = meta_raw.offshore == 1 - output_request = ['cf_mean', 'cf_profile', - 'lease_price', 'lease_price_mil'] - - RevNrwal(gen_fpath, site_data, sam_configs, nrwal_configs, - output_request, site_meta_cols=['depth']).run() - - with Outputs(gen_fpath, 'r') as f: - lease_price = f['lease_price'] - scaled_lease_price = f['lease_price_mil'] + output_request = [ + "cf_mean", + "cf_profile", + "lease_price", + "lease_price_mil", + ] + + RevNrwal( + gen_fpath, + site_data, + sam_configs, + nrwal_configs, + output_request, + site_meta_cols=["depth"], + ).run() + + with Outputs(gen_fpath, "r") as f: + lease_price = f["lease_price"] + scaled_lease_price = f["lease_price_mil"] assert np.allclose(lease_price[mask] / 1e6, scaled_lease_price[mask]) @@ -253,37 +338,58 @@ def test_nrwal_cli(runner, clear_loggers): for fn in os.listdir(SOURCE_DIR): shutil.copy(os.path.join(SOURCE_DIR, fn), os.path.join(td, fn)) - gen_fpath = os.path.join(td, 'gen_2010_node00.h5') - site_data = os.path.join(td, 'example_offshore_data.csv') - offshore_config = os.path.join(td, 'offshore.json') - onshore_config = os.path.join(td, 'onshore.json') - sam_configs = {'onshore': onshore_config, - MetaKeyName.OFFSHORE: offshore_config} - nrwal_configs = {MetaKeyName.OFFSHORE: os.path.join(td, 'nrwal_offshore.yaml')} - - with Outputs(gen_fpath, 'a') as f: - f.time_index = pd_date_range('20100101', '20110101', - closed='right', freq='1h') - f._add_dset('cf_profile', np.random.random(f.shape), - np.uint32, attrs={'scale_factor': 1000}, - chunks=(None, 10)) - f._add_dset('fixed_charge_rate', - 0.09 * np.ones(f.shape[1], dtype=np.float32), - np.float32, attrs={'scale_factor': 1}, - chunks=None) - - with Outputs(gen_fpath, 'r') as f: - original_dsets = [d for d in f.dsets - if d not in ('meta', 'time_index')] + gen_fpath = os.path.join(td, "gen_2010_node00.h5") + site_data = os.path.join(td, "example_offshore_data.csv") + offshore_config = os.path.join(td, "offshore.json") + onshore_config = os.path.join(td, "onshore.json") + sam_configs = { + "onshore": onshore_config, + MetaKeyName.OFFSHORE: offshore_config, + } + nrwal_configs = { + MetaKeyName.OFFSHORE: os.path.join(td, "nrwal_offshore.yaml") + } + + with Outputs(gen_fpath, "a") as f: + f.time_index = pd_date_range( + "20100101", "20110101", closed="right", freq="1h" + ) + f._add_dset( + "cf_profile", + np.random.random(f.shape), + np.uint32, + attrs={"scale_factor": 1000}, + chunks=(None, 10), + ) + f._add_dset( + "fixed_charge_rate", + 0.09 * np.ones(f.shape[1], dtype=np.float32), + np.float32, + attrs={"scale_factor": 1}, + chunks=None, + ) + + with Outputs(gen_fpath, "r") as f: + original_dsets = [ + d for d in f.dsets if d not in ("meta", "time_index") + ] meta_raw = f.meta - lcoe_raw = f['lcoe_fcr'] - cf_mean_raw = f['cf_mean'] - cf_profile_raw = f['cf_profile'] + lcoe_raw = f["lcoe_fcr"] + cf_mean_raw = f["cf_mean"] + cf_profile_raw = f["cf_profile"] mask = meta_raw.offshore == 1 - output_request = ['fixed_charge_rate', 'depth', 'total_losses', - 'array', 'export', 'gcf_adjustment', - 'lcoe_fcr', 'cf_mean', 'cf_profile'] + output_request = [ + "fixed_charge_rate", + "depth", + "total_losses", + "array", + "export", + "gcf_adjustment", + "lcoe_fcr", + "cf_mean", + "cf_profile", + ] config = { "execution_control": { @@ -297,30 +403,32 @@ def test_nrwal_cli(runner, clear_loggers): "sam_files": sam_configs, "nrwal_configs": nrwal_configs, "output_request": output_request, - 'site_meta_cols': ['depth'], + "site_meta_cols": ["depth"], } - config_path = os.path.join(td, 'config.json') - with open(config_path, 'w') as f: + config_path = os.path.join(td, "config.json") + with open(config_path, "w") as f: json.dump(config, f) - result = runner.invoke(main, [str(ModuleName.NRWAL), - '-c', config_path]) - msg = ('Failed with error {}' - .format(traceback.print_exception(*result.exc_info))) + result = runner.invoke( + main, [str(ModuleName.NRWAL), "-c", config_path] + ) + msg = "Failed with error {}".format( + traceback.print_exception(*result.exc_info) + ) assert result.exit_code == 0, msg - with Outputs(gen_fpath, 'r') as f: + with Outputs(gen_fpath, "r") as f: meta_new = f.meta - lcoe_new = f['lcoe_fcr'] - losses = f['total_losses'] - gcf_adjustment = f['gcf_adjustment'] - assert np.allclose(cf_mean_raw, f['cf_mean_raw']) - assert np.allclose(cf_profile_raw, f['cf_profile_raw']) - cf_mean_new = f['cf_mean'] - cf_profile_new = f['cf_profile'] - fcr = f['fixed_charge_rate'] - depth = f['depth'] + lcoe_new = f["lcoe_fcr"] + losses = f["total_losses"] + gcf_adjustment = f["gcf_adjustment"] + assert np.allclose(cf_mean_raw, f["cf_mean_raw"]) + assert np.allclose(cf_profile_raw, f["cf_profile_raw"]) + cf_mean_new = f["cf_mean"] + cf_profile_new = f["cf_profile"] + fcr = f["fixed_charge_rate"] + depth = f["depth"] for key in [d for d in original_dsets if d in f]: assert key in f @@ -329,35 +437,36 @@ def test_nrwal_cli(runner, clear_loggers): # check nrwal keys requested as h5 dsets for key in output_request: assert key in f - if 'profile' not in key: + if "profile" not in key: assert np.isnan(f[key][mask]).sum() == 0 else: assert np.isnan(f[key][:, mask]).sum() == 0 - if key in ('total_losses', 'array', 'export'): + if key in ("total_losses", "array", "export"): assert np.isnan(f[key][~mask]).all() clear_loggers() - result = runner.invoke(main, [str(ModuleName.NRWAL), - '-c', config_path]) - msg = ('Failed with error {}' - .format(traceback.print_exception(*result.exc_info))) + result = runner.invoke( + main, [str(ModuleName.NRWAL), "-c", config_path] + ) + msg = "Failed with error {}".format( + traceback.print_exception(*result.exc_info) + ) assert result.exit_code == 0, msg # make sure the second offshore compute gives same results as first - with Outputs(gen_fpath, 'r') as f: - assert np.allclose(lcoe_new, f['lcoe_fcr']) - assert np.allclose(cf_mean_new, f['cf_mean']) - assert np.allclose(cf_profile_new, f['cf_profile']) - assert np.allclose(cf_mean_raw, f['cf_mean_raw']) - assert np.allclose(cf_profile_raw, f['cf_profile_raw']) + with Outputs(gen_fpath, "r") as f: + assert np.allclose(lcoe_new, f["lcoe_fcr"]) + assert np.allclose(cf_mean_new, f["cf_mean"]) + assert np.allclose(cf_profile_new, f["cf_profile"]) + assert np.allclose(cf_mean_raw, f["cf_mean_raw"]) + assert np.allclose(cf_profile_raw, f["cf_profile_raw"]) # check offshore depth data - assert 'depth' in meta_new - assert all(meta_new.loc[(meta_new.offshore == 1), 'depth'] >= 0) - assert all(np.isnan( - meta_new.loc[(meta_new.offshore == 0), 'depth'])) + assert "depth" in meta_new + assert all(meta_new.loc[(meta_new.offshore == 1), "depth"] >= 0) + assert all(np.isnan(meta_new.loc[(meta_new.offshore == 0), "depth"])) assert all(depth[(meta_new.offshore == 1)] >= 0) assert all(np.isnan(depth[(meta_new.offshore == 0)])) @@ -368,17 +477,19 @@ def test_nrwal_cli(runner, clear_loggers): # make sure all of the requested offshore meta columns got # sent to the new meta data - assert 'depth' in meta_new + assert "depth" in meta_new # sanity check lcoe and cf values assert (lcoe_new[mask] != lcoe_raw[mask]).all() assert (lcoe_new[~mask] == lcoe_raw[~mask]).all() assert (cf_mean_new[mask] < cf_mean_raw[mask]).all() - cf_net = (gcf_adjustment[mask] * (1 - losses[mask]) - * cf_profile_raw[:, mask]) - assert np.allclose(cf_profile_new[:, mask], cf_net, rtol=0.005, - atol=0.001) + cf_net = ( + gcf_adjustment[mask] * (1 - losses[mask]) * cf_profile_raw[:, mask] + ) + assert np.allclose( + cf_profile_new[:, mask], cf_net, rtol=0.005, atol=0.001 + ) assert np.allclose(cf_profile_new[:, ~mask], cf_profile_raw[:, ~mask]) cf_net = gcf_adjustment[mask] * (1 - losses[mask]) * cf_mean_raw[mask] @@ -396,31 +507,55 @@ def test_nrwal_cli_csv(runner, clear_loggers): for fn in os.listdir(SOURCE_DIR): shutil.copy(os.path.join(SOURCE_DIR, fn), os.path.join(td, fn)) - gen_fpath = os.path.join(td, 'gen_2010_node00.h5') - site_data = os.path.join(td, 'example_offshore_data.csv') - offshore_config = os.path.join(td, 'offshore.json') - onshore_config = os.path.join(td, 'onshore.json') - sam_configs = {'onshore': onshore_config, - MetaKeyName.OFFSHORE: offshore_config} - nrwal_configs = {MetaKeyName.OFFSHORE: os.path.join(td, 'nrwal_offshore.yaml')} - - with Outputs(gen_fpath, 'a') as f: - f.time_index = pd_date_range('20100101', '20110101', - closed='right', freq='1h') - f._add_dset('cf_profile_raw', np.random.random(f.shape), - np.uint32, attrs={'scale_factor': 1000}, - chunks=(None, 10)) - f._add_dset('cf_mean_raw', np.random.random(f.shape[1]), - np.uint32, attrs={'scale_factor': 1000}, - chunks=None) - f._add_dset('fixed_charge_rate', - 0.09 * np.ones(f.shape[1], dtype=np.float32), - np.float32, attrs={'scale_factor': 1}, - chunks=None) - - output_request = ['fixed_charge_rate', 'depth', 'total_losses', - 'array', 'export', 'gcf_adjustment', - 'lcoe_fcr', 'cf_mean', 'cf_profile'] + gen_fpath = os.path.join(td, "gen_2010_node00.h5") + site_data = os.path.join(td, "example_offshore_data.csv") + offshore_config = os.path.join(td, "offshore.json") + onshore_config = os.path.join(td, "onshore.json") + sam_configs = { + "onshore": onshore_config, + MetaKeyName.OFFSHORE: offshore_config, + } + nrwal_configs = { + MetaKeyName.OFFSHORE: os.path.join(td, "nrwal_offshore.yaml") + } + + with Outputs(gen_fpath, "a") as f: + f.time_index = pd_date_range( + "20100101", "20110101", closed="right", freq="1h" + ) + f._add_dset( + "cf_profile_raw", + np.random.random(f.shape), + np.uint32, + attrs={"scale_factor": 1000}, + chunks=(None, 10), + ) + f._add_dset( + "cf_mean_raw", + np.random.random(f.shape[1]), + np.uint32, + attrs={"scale_factor": 1000}, + chunks=None, + ) + f._add_dset( + "fixed_charge_rate", + 0.09 * np.ones(f.shape[1], dtype=np.float32), + np.float32, + attrs={"scale_factor": 1}, + chunks=None, + ) + + output_request = [ + "fixed_charge_rate", + "depth", + "total_losses", + "array", + "export", + "gcf_adjustment", + "lcoe_fcr", + "cf_mean", + "cf_profile", + ] config = { "execution_control": { @@ -434,18 +569,20 @@ def test_nrwal_cli_csv(runner, clear_loggers): "sam_files": sam_configs, "nrwal_configs": nrwal_configs, "output_request": output_request, - 'site_meta_cols': ['depth'], - "csv_output": True + "site_meta_cols": ["depth"], + "csv_output": True, } - config_path = os.path.join(td, 'config.json') - with open(config_path, 'w') as f: + config_path = os.path.join(td, "config.json") + with open(config_path, "w") as f: json.dump(config, f) - result = runner.invoke(main, [str(ModuleName.NRWAL), - '-c', config_path]) - msg = ('Failed with error {}' - .format(traceback.print_exception(*result.exc_info))) + result = runner.invoke( + main, [str(ModuleName.NRWAL), "-c", config_path] + ) + msg = "Failed with error {}".format( + traceback.print_exception(*result.exc_info) + ) assert result.exit_code == 0, msg out_fn = "{}_{}.csv".format(os.path.basename(td), ModuleName.NRWAL) @@ -453,12 +590,12 @@ def test_nrwal_cli_csv(runner, clear_loggers): new_data = pd.read_csv(os.path.join(td, out_fn)) for col in output_request[:-1]: assert col in new_data - assert 'cf_profile' not in new_data + assert "cf_profile" not in new_data clear_loggers() -def execute_pytest(capture='all', flags='-rapP'): +def execute_pytest(capture="all", flags="-rapP"): """Execute module as pytest with detailed summary report. Parameters @@ -471,8 +608,8 @@ def execute_pytest(capture='all', flags='-rapP'): """ fname = os.path.basename(__file__) - pytest.main(['-q', '--show-capture={}'.format(capture), fname, flags]) + pytest.main(["-q", "--show-capture={}".format(capture), fname, flags]) -if __name__ == '__main__': +if __name__ == "__main__": execute_pytest() diff --git a/tests/test_rep_profiles.py b/tests/test_rep_profiles.py index d80b3c3cd..fe3cb818c 100644 --- a/tests/test_rep_profiles.py +++ b/tests/test_rep_profiles.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -"""reVX representative profile tests. -""" +"""reVX representative profile tests.""" + import json import os import tempfile @@ -19,14 +19,15 @@ ) from reV.utilities import MetaKeyName -GEN_FPATH = os.path.join(TESTDATADIR, 'gen_out/gen_ri_pv_2012_x000.h5') +GEN_FPATH = os.path.join(TESTDATADIR, "gen_out/gen_ri_pv_2012_x000.h5") def test_rep_region_interval(): """Test the rep profile with a weird interval of gids""" sites = np.arange(40) * 2 - rev_summary = pd.DataFrame({MetaKeyName.GEN_GIDS: sites, - MetaKeyName.RES_GIDS: sites}) + rev_summary = pd.DataFrame( + {MetaKeyName.GEN_GIDS: sites, MetaKeyName.RES_GIDS: sites} + ) r = RegionRepProfile(GEN_FPATH, rev_summary, weight=None) assert r.i_reps[0] == 14 @@ -34,41 +35,68 @@ def test_rep_region_interval(): def test_rep_methods(): """Test integrated rep methods against baseline rep profile result""" sites = np.arange(100) - rev_summary = pd.DataFrame({MetaKeyName.GEN_GIDS: sites, - MetaKeyName.RES_GIDS: sites}) + rev_summary = pd.DataFrame( + {MetaKeyName.GEN_GIDS: sites, MetaKeyName.RES_GIDS: sites} + ) - r = RegionRepProfile(GEN_FPATH, rev_summary, rep_method='meanoid', - err_method='rmse', weight=None) + r = RegionRepProfile( + GEN_FPATH, + rev_summary, + rep_method="meanoid", + err_method="rmse", + weight=None, + ) assert r.i_reps[0] == 15 - r = RegionRepProfile(GEN_FPATH, rev_summary, rep_method='meanoid', - err_method='mbe', weight=None) + r = RegionRepProfile( + GEN_FPATH, + rev_summary, + rep_method="meanoid", + err_method="mbe", + weight=None, + ) assert r.i_reps[0] == 13 - r = RegionRepProfile(GEN_FPATH, rev_summary, rep_method='meanoid', - err_method='mae', weight=None) + r = RegionRepProfile( + GEN_FPATH, + rev_summary, + rep_method="meanoid", + err_method="mae", + weight=None, + ) assert r.i_reps[0] == 15 - r = RegionRepProfile(GEN_FPATH, rev_summary, rep_method='median', - err_method='rmse', weight=None) + r = RegionRepProfile( + GEN_FPATH, + rev_summary, + rep_method="median", + err_method="rmse", + weight=None, + ) assert r.i_reps[0] == 15 - r = RegionRepProfile(GEN_FPATH, rev_summary, rep_method='median', - err_method='mbe', weight=None) + r = RegionRepProfile( + GEN_FPATH, + rev_summary, + rep_method="median", + err_method="mbe", + weight=None, + ) assert r.i_reps[0] == 13 def test_meanoid(): """Test the simple meanoid method""" sites = np.arange(100) - rev_summary = pd.DataFrame({MetaKeyName.GEN_GIDS: sites, - MetaKeyName.RES_GIDS: sites}) + rev_summary = pd.DataFrame( + {MetaKeyName.GEN_GIDS: sites, MetaKeyName.RES_GIDS: sites} + ) r = RegionRepProfile(GEN_FPATH, rev_summary, weight=None) meanoid = RepresentativeMethods.meanoid(r.source_profiles) with Resource(GEN_FPATH) as res: - truth_profiles = res['cf_profile', :, sites] + truth_profiles = res["cf_profile", :, sites] truth = truth_profiles.mean(axis=1).reshape(meanoid.shape) assert np.allclose(meanoid, truth) @@ -77,18 +105,24 @@ def test_weighted_meanoid(): """Test a meanoid weighted by gid_counts vs. a non-weighted meanoid.""" sites = np.arange(100) - rev_summary = pd.DataFrame({MetaKeyName.GEN_GIDS: sites, - MetaKeyName.RES_GIDS: sites, - MetaKeyName.GID_COUNTS: [1] * 50 + [0] * 50}) + rev_summary = pd.DataFrame( + { + MetaKeyName.GEN_GIDS: sites, + MetaKeyName.RES_GIDS: sites, + MetaKeyName.GID_COUNTS: [1] * 50 + [0] * 50, + } + ) r = RegionRepProfile(GEN_FPATH, rev_summary) weights = r._get_region_attr(r._rev_summary, MetaKeyName.GID_COUNTS) - w_meanoid = RepresentativeMethods.meanoid(r.source_profiles, - weights=weights) + w_meanoid = RepresentativeMethods.meanoid( + r.source_profiles, weights=weights + ) sites = np.arange(50) - rev_summary = pd.DataFrame({MetaKeyName.GEN_GIDS: sites, - MetaKeyName.RES_GIDS: sites}) + rev_summary = pd.DataFrame( + {MetaKeyName.GEN_GIDS: sites, MetaKeyName.RES_GIDS: sites} + ) r = RegionRepProfile(GEN_FPATH, rev_summary, weight=None) meanoid = RepresentativeMethods.meanoid(r.source_profiles, weights=None) @@ -102,42 +136,52 @@ def test_integrated(): sites = np.arange(100) ones = np.ones((100,)) zeros = np.zeros((100,)) - regions = (['r0'] * 7) + (['r1'] * 33) + (['r2'] * 60) + regions = (["r0"] * 7) + (["r1"] * 33) + (["r2"] * 60) timezone = np.random.choice([-4, -5, -6, -7], 100) - rev_summary = pd.DataFrame({MetaKeyName.GEN_GIDS: sites, - MetaKeyName.RES_GIDS: sites, - 'res_class': zeros, - 'weight': ones, - 'region': regions, - MetaKeyName.TIMEZONE: timezone}) - rp = RepProfiles(GEN_FPATH, rev_summary, 'region', weight='weight') + rev_summary = pd.DataFrame( + { + MetaKeyName.GEN_GIDS: sites, + MetaKeyName.RES_GIDS: sites, + "res_class": zeros, + "weight": ones, + "region": regions, + MetaKeyName.TIMEZONE: timezone, + } + ) + rp = RepProfiles(GEN_FPATH, rev_summary, "region", weight="weight") rp.run(max_workers=1) p1, m1 = rp.profiles, rp.meta rp.run(max_workers=None) p2, m2 = rp.profiles, rp.meta - assert np.allclose(m1['rep_res_gid'].values.astype(int), - m2['rep_res_gid'].values.astype(int)) + assert np.allclose( + m1["rep_res_gid"].values.astype(int), + m2["rep_res_gid"].values.astype(int), + ) assert np.allclose(p1[0], p2[0]) - assert m1.loc[0, 'rep_res_gid'] == 4 - assert m1.loc[1, 'rep_res_gid'] == 15 - assert m1.loc[2, 'rep_res_gid'] == 60 + assert m1.loc[0, "rep_res_gid"] == 4 + assert m1.loc[1, "rep_res_gid"] == 15 + assert m1.loc[2, "rep_res_gid"] == 60 def test_sc_points(): """Test rep profiles for each SC point.""" sites = np.arange(10) timezone = np.random.choice([-4, -5, -6, -7], 10) - rev_summary = pd.DataFrame({MetaKeyName.SC_GID: sites, - MetaKeyName.GEN_GIDS: sites, - MetaKeyName.RES_GIDS: sites, - MetaKeyName.TIMEZONE: timezone}) + rev_summary = pd.DataFrame( + { + MetaKeyName.SC_GID: sites, + MetaKeyName.GEN_GIDS: sites, + MetaKeyName.RES_GIDS: sites, + MetaKeyName.TIMEZONE: timezone, + } + ) rp = RepProfiles(GEN_FPATH, rev_summary, MetaKeyName.SC_GID, weight=None) rp.run(max_workers=1) with Resource(GEN_FPATH) as res: - truth = res['cf_profile', :, slice(0, 10)] + truth = res["cf_profile", :, slice(0, 10)] assert np.allclose(rp.profiles[0], truth) @@ -152,20 +196,31 @@ def test_agg_profile(): res_gids = [json.dumps(x) for x in res_gids] gid_counts = [json.dumps(x) for x in gid_counts] timezone = np.random.choice([-4, -5, -6, -7], 4) - rev_summary = pd.DataFrame({MetaKeyName.SC_GID: np.arange(4), - MetaKeyName.GEN_GIDS: gen_gids, - MetaKeyName.RES_GIDS: res_gids, - MetaKeyName.GID_COUNTS: gid_counts, - MetaKeyName.TIMEZONE: timezone}) - - rp = RepProfiles(GEN_FPATH, rev_summary, MetaKeyName.SC_GID, cf_dset='cf_profile', - err_method=None) + rev_summary = pd.DataFrame( + { + MetaKeyName.SC_GID: np.arange(4), + MetaKeyName.GEN_GIDS: gen_gids, + MetaKeyName.RES_GIDS: res_gids, + MetaKeyName.GID_COUNTS: gid_counts, + MetaKeyName.TIMEZONE: timezone, + } + ) + + rp = RepProfiles( + GEN_FPATH, + rev_summary, + MetaKeyName.SC_GID, + cf_dset="cf_profile", + err_method=None, + ) rp.run(scaled_precision=False, max_workers=1) for index in rev_summary.index: gen_gids = json.loads(rev_summary.loc[index, MetaKeyName.GEN_GIDS]) res_gids = json.loads(rev_summary.loc[index, MetaKeyName.RES_GIDS]) - weights = np.array(json.loads(rev_summary.loc[index, MetaKeyName.GID_COUNTS])) + weights = np.array( + json.loads(rev_summary.loc[index, MetaKeyName.GID_COUNTS]) + ) with Resource(GEN_FPATH) as res: meta = res.meta @@ -173,7 +228,7 @@ def test_agg_profile(): raw_profiles = [] for gid in res_gids: iloc = np.where(meta[MetaKeyName.GID] == gid)[0][0] - prof = np.expand_dims(res['cf_profile', :, iloc], 1) + prof = np.expand_dims(res["cf_profile", :, iloc], 1) raw_profiles.append(prof) last = raw_profiles[-1].flatten() @@ -189,13 +244,16 @@ def test_agg_profile(): assert np.allclose(rp.profiles[0][:, index], truth) - passthrough_cols = [MetaKeyName.GEN_GIDS, MetaKeyName.RES_GIDS, MetaKeyName.GID_COUNTS] + passthrough_cols = [ + MetaKeyName.GEN_GIDS, + MetaKeyName.RES_GIDS, + MetaKeyName.GID_COUNTS, + ] for col in passthrough_cols: assert col in rp.meta assert_frame_equal( - rev_summary[passthrough_cols], - rp.meta[passthrough_cols] + rev_summary[passthrough_cols], rp.meta[passthrough_cols] ) @@ -204,17 +262,21 @@ def test_many_regions(use_weights): """Test multiple complicated regions.""" sites = np.arange(100) zeros = np.zeros((100,)) - region1 = (['r0'] * 7) + (['r1'] * 33) + (['r2'] * 60) - region2 = (['a0'] * 20) + (['b1'] * 10) + (['c2'] * 20) + (['d3'] * 50) + region1 = (["r0"] * 7) + (["r1"] * 33) + (["r2"] * 60) + region2 = (["a0"] * 20) + (["b1"] * 10) + (["c2"] * 20) + (["d3"] * 50) timezone = np.random.choice([-4, -5, -6, -7], 100) - rev_summary = pd.DataFrame({MetaKeyName.GEN_GIDS: sites, - MetaKeyName.RES_GIDS: sites, - 'res_class': zeros, - 'region1': region1, - 'region2': region2, - 'weight': sites + 1, - MetaKeyName.TIMEZONE: timezone}) - reg_cols = ['region1', 'region2'] + rev_summary = pd.DataFrame( + { + MetaKeyName.GEN_GIDS: sites, + MetaKeyName.RES_GIDS: sites, + "res_class": zeros, + "region1": region1, + "region2": region2, + "weight": sites + 1, + MetaKeyName.TIMEZONE: timezone, + } + ) + reg_cols = ["region1", "region2"] if use_weights: rp = RepProfiles(GEN_FPATH, rev_summary, reg_cols, weight="weight") else: @@ -225,42 +287,49 @@ def test_many_regions(use_weights): assert len(rp.meta) == 6 for r1 in set(region1): - assert r1 in rp.meta['region1'].values + assert r1 in rp.meta["region1"].values for r2 in set(region2): - assert r2 in rp.meta['region2'].values + assert r2 in rp.meta["region2"].values def test_many_regions_with_list_weights(): """Test multiple complicated regions with multiple weights per row.""" - sites = [list(np.random.choice(np.arange(100), np.random.randint(10) + 10)) - for __ in np.arange(100)] - weights = [str(list(np.random.choice(np.arange(100), len(row)))) - for row in sites] + sites = [ + list(np.random.choice(np.arange(100), np.random.randint(10) + 10)) + for __ in np.arange(100) + ] + weights = [ + str(list(np.random.choice(np.arange(100), len(row)))) for row in sites + ] sites = [str(row) for row in sites] zeros = np.zeros((100,)) - region1 = (['r0'] * 7) + (['r1'] * 33) + (['r2'] * 60) - region2 = (['a0'] * 20) + (['b1'] * 10) + (['c2'] * 20) + (['d3'] * 50) + region1 = (["r0"] * 7) + (["r1"] * 33) + (["r2"] * 60) + region2 = (["a0"] * 20) + (["b1"] * 10) + (["c2"] * 20) + (["d3"] * 50) timezone = np.random.choice([-4, -5, -6, -7], 100) - rev_summary = pd.DataFrame({MetaKeyName.GEN_GIDS: sites, - MetaKeyName.RES_GIDS: sites, - 'res_class': zeros, - 'region1': region1, - 'region2': region2, - 'weights': weights, - MetaKeyName.TIMEZONE: timezone}) - reg_cols = ['region1', 'region2'] - rp = RepProfiles(GEN_FPATH, rev_summary, reg_cols, weight='weights') + rev_summary = pd.DataFrame( + { + MetaKeyName.GEN_GIDS: sites, + MetaKeyName.RES_GIDS: sites, + "res_class": zeros, + "region1": region1, + "region2": region2, + "weights": weights, + MetaKeyName.TIMEZONE: timezone, + } + ) + reg_cols = ["region1", "region2"] + rp = RepProfiles(GEN_FPATH, rev_summary, reg_cols, weight="weights") rp.run() assert rp.profiles[0].shape == (17520, 6) assert len(rp.meta) == 6 for r1 in set(region1): - assert r1 in rp.meta['region1'].values + assert r1 in rp.meta["region1"].values for r2 in set(region2): - assert r2 in rp.meta['region2'].values + assert r2 in rp.meta["region2"].values def test_write_to_file(): @@ -268,31 +337,35 @@ def test_write_to_file(): with tempfile.TemporaryDirectory() as td: sites = np.arange(100) zeros = np.zeros((100,)) - regions = (['r0'] * 7) + (['r1'] * 33) + (['r2'] * 60) + regions = (["r0"] * 7) + (["r1"] * 33) + (["r2"] * 60) timezone = np.random.choice([-4, -5, -6, -7], 100) - rev_summary = pd.DataFrame({MetaKeyName.GEN_GIDS: sites, - MetaKeyName.RES_GIDS: sites, - 'res_class': zeros, - 'region': regions, - MetaKeyName.TIMEZONE: timezone}) - fout = os.path.join(td, 'temp_rep_profiles.h5') - rp = RepProfiles(GEN_FPATH, rev_summary, 'region', n_profiles=3, - weight=None) + rev_summary = pd.DataFrame( + { + MetaKeyName.GEN_GIDS: sites, + MetaKeyName.RES_GIDS: sites, + "res_class": zeros, + "region": regions, + MetaKeyName.TIMEZONE: timezone, + } + ) + fout = os.path.join(td, "temp_rep_profiles.h5") + rp = RepProfiles( + GEN_FPATH, rev_summary, "region", n_profiles=3, weight=None + ) rp.run(fout=fout) with Resource(fout) as res: - disk_profiles = res['rep_profiles_0'] + disk_profiles = res["rep_profiles_0"] disk_meta = res.meta - assert 'rep_profiles_2' in res.datasets - test = np.array_equal(res['rep_profiles_0'], - res['rep_profiles_1']) + assert "rep_profiles_2" in res.datasets + test = np.array_equal(res["rep_profiles_0"], res["rep_profiles_1"]) assert not test assert np.allclose(rp.profiles[0], disk_profiles) assert len(disk_meta) == 3 for i in rp.meta.index: - v1 = json.loads(rp.meta.loc[i, 'rep_gen_gid']) - v2 = json.loads(disk_meta.loc[i, 'rep_gen_gid']) + v1 = json.loads(rp.meta.loc[i, "rep_gen_gid"]) + v2 = json.loads(disk_meta.loc[i, "rep_gen_gid"]) assert v1 == v2 @@ -301,30 +374,35 @@ def test_file_options(): with tempfile.TemporaryDirectory() as td: sites = np.arange(100) zeros = np.zeros((100,)) - regions = (['r0'] * 7) + (['r1'] * 33) + (['r2'] * 60) + regions = (["r0"] * 7) + (["r1"] * 33) + (["r2"] * 60) timezone = np.random.choice([-4, -5, -6, -7], 100) - rev_summary = pd.DataFrame({MetaKeyName.GEN_GIDS: sites, - MetaKeyName.RES_GIDS: sites, - 'res_class': zeros, - 'region': regions, - MetaKeyName.TIMEZONE: timezone}) - fout = os.path.join(td, 'temp_rep_profiles.h5') - rp = RepProfiles(GEN_FPATH, rev_summary, 'region', n_profiles=3, - weight=None) + rev_summary = pd.DataFrame( + { + MetaKeyName.GEN_GIDS: sites, + MetaKeyName.RES_GIDS: sites, + "res_class": zeros, + "region": regions, + MetaKeyName.TIMEZONE: timezone, + } + ) + fout = os.path.join(td, "temp_rep_profiles.h5") + rp = RepProfiles( + GEN_FPATH, rev_summary, "region", n_profiles=3, weight=None + ) rp.run(fout=fout, save_rev_summary=False, scaled_precision=True) with Resource(fout) as res: - dtype = res.get_dset_properties('rep_profiles_0')[1] - attrs = res.get_attrs('rep_profiles_0') - disk_profiles = res['rep_profiles_0'] + dtype = res.get_dset_properties("rep_profiles_0")[1] + attrs = res.get_attrs("rep_profiles_0") + disk_profiles = res["rep_profiles_0"] disk_dsets = res.datasets assert np.issubdtype(dtype, np.integer) - assert attrs['scale_factor'] == 1000 + assert attrs["scale_factor"] == 1000 assert np.allclose(rp.profiles[0], disk_profiles) - assert 'rev_summary' not in disk_dsets + assert "rev_summary" not in disk_dsets -def execute_pytest(capture='all', flags='-rapP'): +def execute_pytest(capture="all", flags="-rapP"): """Execute module as pytest with detailed summary report. Parameters @@ -337,8 +415,8 @@ def execute_pytest(capture='all', flags='-rapP'): """ fname = os.path.basename(__file__) - pytest.main(['-q', '--show-capture={}'.format(capture), fname, flags]) + pytest.main(["-q", "--show-capture={}".format(capture), fname, flags]) -if __name__ == '__main__': +if __name__ == "__main__": execute_pytest() diff --git a/tests/test_supply_curve_aggregation_friction.py b/tests/test_supply_curve_aggregation_friction.py index 648c6dafd..7d6bf7422 100644 --- a/tests/test_supply_curve_aggregation_friction.py +++ b/tests/test_supply_curve_aggregation_friction.py @@ -5,6 +5,7 @@ @author: gbuster """ + import os import warnings @@ -18,25 +19,24 @@ from reV.supply_curve.sc_aggregation import SupplyCurveAggregation from reV.utilities import MetaKeyName -EXCL_FPATH = os.path.join(TESTDATADIR, 'ri_exclusions/ri_exclusions.h5') -FRICTION_FPATH = os.path.join(TESTDATADIR, 'ri_exclusions/ri_friction.h5') -FRICTION_DSET = 'ri_friction_surface' -GEN = os.path.join(TESTDATADIR, 'gen_out/ri_my_pv_gen.h5') -AGG_BASELINE = os.path.join(TESTDATADIR, 'sc_out/baseline_agg_summary.csv') -TM_DSET = 'techmap_nsrdb' -RES_CLASS_DSET = 'ghi_mean-means' +EXCL_FPATH = os.path.join(TESTDATADIR, "ri_exclusions/ri_exclusions.h5") +FRICTION_FPATH = os.path.join(TESTDATADIR, "ri_exclusions/ri_friction.h5") +FRICTION_DSET = "ri_friction_surface" +GEN = os.path.join(TESTDATADIR, "gen_out/ri_my_pv_gen.h5") +AGG_BASELINE = os.path.join(TESTDATADIR, "sc_out/baseline_agg_summary.csv") +TM_DSET = "techmap_nsrdb" +RES_CLASS_DSET = "ghi_mean-means" RES_CLASS_BINS = [0, 4, 100] -DATA_LAYERS = {'pct_slope': {'dset': 'ri_srtm_slope', - 'method': 'mean'}, - 'reeds_region': {'dset': 'ri_reeds_regions', - 'method': 'mode'}, - 'padus': {'dset': 'ri_padus', - 'method': 'mode'}} - -EXCL_DICT = {'ri_srtm_slope': {'inclusion_range': (None, 5), - 'exclude_nodata': True}, - 'ri_padus': {'exclude_values': [1], - 'exclude_nodata': True}} +DATA_LAYERS = { + "pct_slope": {"dset": "ri_srtm_slope", "method": "mean"}, + "reeds_region": {"dset": "ri_reeds_regions", "method": "mode"}, + "padus": {"dset": "ri_padus", "method": "mode"}, +} + +EXCL_DICT = { + "ri_srtm_slope": {"inclusion_range": (None, 5), "exclude_nodata": True}, + "ri_padus": {"exclude_values": [1], "exclude_nodata": True}, +} RESOLUTION = 64 EXTENT = SupplyCurveExtent(EXCL_FPATH, resolution=RESOLUTION) @@ -48,20 +48,20 @@ def test_friction_mask(): """Test the friction mask on known quantities.""" x = FRICTION[slice(700, 800), slice(300, 400)].mean() - assert x == 20.0, 'Friction for region should be 20.0, but is {}'.format(x) + assert x == 20.0, "Friction for region should be 20.0, but is {}".format(x) x = FRICTION[slice(300, 400), slice(100, 300)].mean() - assert x == 1.0, 'Friction for nodata should be 1.0, but is {}'.format(x) + assert x == 1.0, "Friction for nodata should be 1.0, but is {}".format(x) x = FRICTION[slice(300, 400), slice(800, 900)].mean() - assert x == 1.0, 'Friction for nodata should be 1.0, but is {}'.format(x) + assert x == 1.0, "Friction for nodata should be 1.0, but is {}".format(x) x = FRICTION[slice(0, 10), slice(0, 10)].mean() - assert x == 10.0, 'Friction for region should be 10.0, but is {}'.format(x) + assert x == 10.0, "Friction for region should be 10.0, but is {}".format(x) x = FRICTION[slice(354, 360), slice(456, 460)].mean() diff = (x - 1.22769) / x - m = 'Friction for region should be 1.228, but is {}'.format(x) + m = "Friction for region should be 1.228, but is {}".format(x) assert diff < 0.0001, m @@ -70,17 +70,21 @@ def test_agg_friction(gid): """Test SC Aggregation with friction by checking friction factors and LCOE against a hand calc.""" - warnings.filterwarnings('ignore') + warnings.filterwarnings("ignore") for gid in [100, 114, 130, 181]: - sca = SupplyCurveAggregation(EXCL_FPATH, TM_DSET, excl_dict=EXCL_DICT, - res_class_dset=RES_CLASS_DSET, - res_class_bins=RES_CLASS_BINS, - data_layers=DATA_LAYERS, - resolution=RESOLUTION, - gids=[gid], - friction_fpath=FRICTION_FPATH, - friction_dset=FRICTION_DSET) + sca = SupplyCurveAggregation( + EXCL_FPATH, + TM_DSET, + excl_dict=EXCL_DICT, + res_class_dset=RES_CLASS_DSET, + res_class_bins=RES_CLASS_BINS, + data_layers=DATA_LAYERS, + resolution=RESOLUTION, + gids=[gid], + friction_fpath=FRICTION_FPATH, + friction_dset=FRICTION_DSET, + ) s = sca.summarize(GEN, max_workers=1) row_slice, col_slice = EXTENT.get_excl_slices(gid) @@ -92,13 +96,18 @@ def test_agg_friction(gid): x = x[(x != 0)] mean_friction = x.mean() - m = ('SC point gid {} does not match mean friction hand calc' + m = "SC point gid {} does not match mean friction hand calc".format( + gid + ) + assert np.isclose( + s[MetaKeyName.MEAN_FRICTION].values[0], mean_friction + ), m + m = ("SC point gid {} does not match mean LCOE with friction hand calc" .format(gid)) - assert np.isclose(s[MetaKeyName.MEAN_FRICTION].values[0], mean_friction), m - m = ('SC point gid {} does not match mean LCOE with friction hand calc' - .format(gid)) - assert np.allclose(s[MetaKeyName.MEAN_LCOE_FRICTION], - s[MetaKeyName.MEAN_LCOE] * mean_friction), m + assert np.allclose( + s[MetaKeyName.MEAN_LCOE_FRICTION], + s[MetaKeyName.MEAN_LCOE] * mean_friction, + ), m # pylint: disable=no-member @@ -107,9 +116,10 @@ def make_friction_file(): import shutil import matplotlib.pyplot as plt + shutil.copy(EXCL, FRICTION_FPATH) - with h5py.File(FRICTION_FPATH, 'a') as f: - f[FRICTION_DSET] = f['ri_srtm_slope'] + with h5py.File(FRICTION_FPATH, "a") as f: + f[FRICTION_DSET] = f["ri_srtm_slope"] shape = f[FRICTION_DSET].shape data = np.random.lognormal(mean=0.2, sigma=0.2, size=shape) data = data.astype(np.float32) @@ -124,16 +134,20 @@ def make_friction_file(): f[FRICTION_DSET][...] = data for d in f: - if d not in [FRICTION_DSET, MetaKeyName.LATITUDE, MetaKeyName.LONGITUDE]: + if d not in [ + FRICTION_DSET, + MetaKeyName.LATITUDE, + MetaKeyName.LONGITUDE, + ]: del f[d] - with h5py.File(FRICTION_FPATH, 'r') as f: + with h5py.File(FRICTION_FPATH, "r") as f: out = f[FRICTION_DSET][...] assert np.allclose(data, out) -def execute_pytest(capture='all', flags='-rapP'): +def execute_pytest(capture="all", flags="-rapP"): """Execute module as pytest with detailed summary report. Parameters @@ -146,8 +160,8 @@ def execute_pytest(capture='all', flags='-rapP'): """ fname = os.path.basename(__file__) - pytest.main(['-q', '--show-capture={}'.format(capture), fname, flags]) + pytest.main(["-q", "--show-capture={}".format(capture), fname, flags]) -if __name__ == '__main__': +if __name__ == "__main__": execute_pytest() diff --git a/tests/test_supply_curve_compute.py b/tests/test_supply_curve_compute.py index b10e7ad77..664eced60 100644 --- a/tests/test_supply_curve_compute.py +++ b/tests/test_supply_curve_compute.py @@ -2,6 +2,7 @@ """ Supply Curve computation integrated tests """ + import os import tempfile import warnings @@ -16,30 +17,46 @@ from reV.utilities import MetaKeyName from reV.utilities.exceptions import SupplyCurveInputError -TRANS_COSTS_1 = {'line_tie_in_cost': 200, 'line_cost': 1000, - 'station_tie_in_cost': 50, 'center_tie_in_cost': 10, - 'sink_tie_in_cost': 100, 'available_capacity': 0.3} - - -TRANS_COSTS_2 = {'line_tie_in_cost': 3000, 'line_cost': 2000, - 'station_tie_in_cost': 500, 'center_tie_in_cost': 100, - 'sink_tie_in_cost': 1e6, 'available_capacity': 0.9} - -path = os.path.join(TESTDATADIR, 'sc_out/baseline_agg_summary.csv') +TRANS_COSTS_1 = { + "line_tie_in_cost": 200, + "line_cost": 1000, + "station_tie_in_cost": 50, + "center_tie_in_cost": 10, + "sink_tie_in_cost": 100, + "available_capacity": 0.3, +} + + +TRANS_COSTS_2 = { + "line_tie_in_cost": 3000, + "line_cost": 2000, + "station_tie_in_cost": 500, + "center_tie_in_cost": 100, + "sink_tie_in_cost": 1e6, + "available_capacity": 0.9, +} + +path = os.path.join(TESTDATADIR, "sc_out/baseline_agg_summary.csv") SC_POINTS = pd.read_csv(path) -path = os.path.join(TESTDATADIR, 'sc_out/baseline_agg_summary_friction.csv') +path = os.path.join(TESTDATADIR, "sc_out/baseline_agg_summary_friction.csv") SC_POINTS_FRICTION = pd.read_csv(path) -path = os.path.join(TESTDATADIR, 'trans_tables/ri_transmission_table.csv') +path = os.path.join(TESTDATADIR, "trans_tables/ri_transmission_table.csv") TRANS_TABLE = pd.read_csv(path) -path = os.path.join(TESTDATADIR, 'trans_tables/transmission_multipliers.csv') +path = os.path.join(TESTDATADIR, "trans_tables/transmission_multipliers.csv") MULTIPLIERS = pd.read_csv(path) -SC_FULL_COLUMNS = ('trans_gid', 'trans_type', 'trans_capacity', - 'trans_cap_cost_per_mw', 'dist_km', 'lcot', - 'total_lcoe') +SC_FULL_COLUMNS = ( + "trans_gid", + "trans_type", + "trans_capacity", + "trans_cap_cost_per_mw", + "dist_km", + "lcot", + "total_lcoe", +) def baseline_verify(sc_full, fpath_baseline): @@ -52,104 +69,133 @@ def baseline_verify(sc_full, fpath_baseline): baseline = pd.read_csv(fpath_baseline) # double check useful for when tables are changing # but lcoe should be the same - check = np.allclose(baseline['total_lcoe'], sc_full['total_lcoe']) + check = np.allclose(baseline["total_lcoe"], sc_full["total_lcoe"]) if not check: - diff = np.abs(baseline['total_lcoe'].values - - sc_full['total_lcoe']) - rel_diff = 100 * diff / baseline['total_lcoe'].values - msg = ('Total LCOE values differed from baseline. ' - 'Maximum difference is {:.1f} ({:.1f}%), ' - 'mean difference is {:.1f} ({:.1f}%). ' - 'In total, {:.1f}% of all SC point connections changed' - .format(diff.max(), rel_diff.max(), - diff.mean(), rel_diff.mean(), - 100 * (diff > 0).sum() / len(diff))) + diff = np.abs( + baseline["total_lcoe"].values - sc_full["total_lcoe"] + ) + rel_diff = 100 * diff / baseline["total_lcoe"].values + msg = ( + "Total LCOE values differed from baseline. " + "Maximum difference is {:.1f} ({:.1f}%), " + "mean difference is {:.1f} ({:.1f}%). " + "In total, {:.1f}% of all SC point connections changed".format( + diff.max(), + rel_diff.max(), + diff.mean(), + rel_diff.mean(), + 100 * (diff > 0).sum() / len(diff), + ) + ) raise RuntimeError(msg) - assert_frame_equal(baseline, sc_full[baseline.columns], - check_dtype=False) + assert_frame_equal( + baseline, sc_full[baseline.columns], check_dtype=False + ) else: sc_full.to_csv(fpath_baseline, index=False) -@pytest.mark.parametrize(('i', 'trans_costs'), ((1, TRANS_COSTS_1), - (2, TRANS_COSTS_2))) +@pytest.mark.parametrize( + ("i", "trans_costs"), ((1, TRANS_COSTS_1), (2, TRANS_COSTS_2)) +) def test_integrated_sc_full(i, trans_costs): """Run the full SC test and verify results against baseline file.""" tcosts = trans_costs.copy() - avail_cap_frac = tcosts.pop('available_capacity', 1) + avail_cap_frac = tcosts.pop("available_capacity", 1) sc = SupplyCurve(SC_POINTS, TRANS_TABLE, sc_features=MULTIPLIERS) with tempfile.TemporaryDirectory() as td: out_fpath = os.path.join(td, "sc") - sc_full = sc.run(out_fpath, fixed_charge_rate=0.1, simple=False, - transmission_costs=tcosts, - avail_cap_frac=avail_cap_frac, - columns=SC_FULL_COLUMNS) - fpath_baseline = os.path.join(TESTDATADIR, - 'sc_out/sc_full_out_{}.csv'.format(i)) + sc_full = sc.run( + out_fpath, + fixed_charge_rate=0.1, + simple=False, + transmission_costs=tcosts, + avail_cap_frac=avail_cap_frac, + columns=SC_FULL_COLUMNS, + ) + fpath_baseline = os.path.join( + TESTDATADIR, "sc_out/sc_full_out_{}.csv".format(i) + ) baseline_verify(sc_full, fpath_baseline) -@pytest.mark.parametrize(('i', 'trans_costs'), ((1, TRANS_COSTS_1), - (2, TRANS_COSTS_2))) +@pytest.mark.parametrize( + ("i", "trans_costs"), ((1, TRANS_COSTS_1), (2, TRANS_COSTS_2)) +) def test_integrated_sc_simple(i, trans_costs): """Run the simple SC test and verify results against baseline file.""" tcosts = trans_costs.copy() - tcosts.pop('available_capacity', 1) + tcosts.pop("available_capacity", 1) sc = SupplyCurve(SC_POINTS, TRANS_TABLE, sc_features=MULTIPLIERS) with tempfile.TemporaryDirectory() as td: out_fpath = os.path.join(td, "sc") - sc_simple = sc.run(out_fpath, fixed_charge_rate=0.1, simple=True, - transmission_costs=tcosts) - - fpath_baseline = os.path.join(TESTDATADIR, - 'sc_out/sc_simple_out_{}.csv'.format(i)) + sc_simple = sc.run( + out_fpath, + fixed_charge_rate=0.1, + simple=True, + transmission_costs=tcosts, + ) + + fpath_baseline = os.path.join( + TESTDATADIR, "sc_out/sc_simple_out_{}.csv".format(i) + ) baseline_verify(sc_simple, fpath_baseline) def test_integrated_sc_full_friction(): """Run the full SC algorithm with friction""" tcosts = TRANS_COSTS_1.copy() - avail_cap_frac = tcosts.pop('available_capacity', 1) + avail_cap_frac = tcosts.pop("available_capacity", 1) sc = SupplyCurve(SC_POINTS_FRICTION, TRANS_TABLE, sc_features=MULTIPLIERS) with tempfile.TemporaryDirectory() as td: out_fpath = os.path.join(td, "sc") - sc_full = sc.run(out_fpath, fixed_charge_rate=0.1, simple=False, - transmission_costs=tcosts, - avail_cap_frac=avail_cap_frac, - columns=SC_FULL_COLUMNS, - sort_on=MetaKeyName.TOTAL_LCOE_FRICTION) + sc_full = sc.run( + out_fpath, + fixed_charge_rate=0.1, + simple=False, + transmission_costs=tcosts, + avail_cap_frac=avail_cap_frac, + columns=SC_FULL_COLUMNS, + sort_on=MetaKeyName.TOTAL_LCOE_FRICTION, + ) sc_full = pd.read_csv(sc_full) assert MetaKeyName.MEAN_LCOE_FRICTION in sc_full assert MetaKeyName.TOTAL_LCOE_FRICTION in sc_full - test = sc_full[MetaKeyName.MEAN_LCOE_FRICTION] + sc_full['lcot'] + test = sc_full[MetaKeyName.MEAN_LCOE_FRICTION] + sc_full["lcot"] assert np.allclose(test, sc_full[MetaKeyName.TOTAL_LCOE_FRICTION]) - fpath_baseline = os.path.join(TESTDATADIR, - 'sc_out/sc_full_out_friction.csv') + fpath_baseline = os.path.join( + TESTDATADIR, "sc_out/sc_full_out_friction.csv" + ) baseline_verify(sc_full, fpath_baseline) def test_integrated_sc_simple_friction(): """Run the simple SC algorithm with friction""" tcosts = TRANS_COSTS_1.copy() - tcosts.pop('available_capacity', 1) + tcosts.pop("available_capacity", 1) sc = SupplyCurve(SC_POINTS_FRICTION, TRANS_TABLE, sc_features=MULTIPLIERS) with tempfile.TemporaryDirectory() as td: out_fpath = os.path.join(td, "sc") - sc_simple = sc.run(out_fpath, fixed_charge_rate=0.1, simple=True, - transmission_costs=tcosts, - sort_on=MetaKeyName.TOTAL_LCOE_FRICTION) + sc_simple = sc.run( + out_fpath, + fixed_charge_rate=0.1, + simple=True, + transmission_costs=tcosts, + sort_on=MetaKeyName.TOTAL_LCOE_FRICTION, + ) sc_simple = pd.read_csv(sc_simple) assert MetaKeyName.MEAN_LCOE_FRICTION in sc_simple assert MetaKeyName.TOTAL_LCOE_FRICTION in sc_simple - test = sc_simple[MetaKeyName.MEAN_LCOE_FRICTION] + sc_simple['lcot'] + test = sc_simple[MetaKeyName.MEAN_LCOE_FRICTION] + sc_simple["lcot"] assert np.allclose(test, sc_simple[MetaKeyName.TOTAL_LCOE_FRICTION]) - fpath_baseline = os.path.join(TESTDATADIR, - 'sc_out/sc_simple_out_friction.csv') + fpath_baseline = os.path.join( + TESTDATADIR, "sc_out/sc_simple_out_friction.csv" + ) baseline_verify(sc_simple, fpath_baseline) @@ -158,41 +204,54 @@ def test_sc_warning1(): mask = TRANS_TABLE[MetaKeyName.SC_POINT_GID].isin(list(range(10))) trans_table = TRANS_TABLE[~mask] tcosts = TRANS_COSTS_1.copy() - avail_cap_frac = tcosts.pop('available_capacity', 1) + avail_cap_frac = tcosts.pop("available_capacity", 1) with warnings.catch_warnings(record=True) as w: - warnings.simplefilter('always') + warnings.simplefilter("always") sc = SupplyCurve(SC_POINTS, trans_table, sc_features=MULTIPLIERS) with tempfile.TemporaryDirectory() as td: out_fpath = os.path.join(td, "sc") - sc.run(out_fpath, fixed_charge_rate=0.1, simple=False, - transmission_costs=tcosts, avail_cap_frac=avail_cap_frac, - columns=SC_FULL_COLUMNS) - - s1 = str(list(range(10))).replace(']', '').replace('[', '') + sc.run( + out_fpath, + fixed_charge_rate=0.1, + simple=False, + transmission_costs=tcosts, + avail_cap_frac=avail_cap_frac, + columns=SC_FULL_COLUMNS, + ) + + s1 = str(list(range(10))).replace("]", "").replace("[", "") s2 = str(w[0].message) - msg = ('Warning failed! Should have had missing sc_gids 0 through 9: ' - '{}'.format(s2)) + msg = ( + "Warning failed! Should have had missing sc_gids 0 through 9: " + "{}".format(s2) + ) assert s1 in s2, msg def test_sc_warning2(): """Run the full SC test without PCA load centers and verify warning.""" - mask = TRANS_TABLE['category'] == 'PCALoadCen' + mask = TRANS_TABLE["category"] == "PCALoadCen" trans_table = TRANS_TABLE[~mask] tcosts = TRANS_COSTS_1.copy() - avail_cap_frac = tcosts.pop('available_capacity', 1) + avail_cap_frac = tcosts.pop("available_capacity", 1) with warnings.catch_warnings(record=True) as w: - warnings.simplefilter('always') + warnings.simplefilter("always") sc = SupplyCurve(SC_POINTS, trans_table, sc_features=MULTIPLIERS) with tempfile.TemporaryDirectory() as td: out_fpath = os.path.join(td, "sc") - sc.run(out_fpath, fixed_charge_rate=0.1, simple=False, - transmission_costs=tcosts, avail_cap_frac=avail_cap_frac, - columns=SC_FULL_COLUMNS) - s1 = 'Unconnected sc_gid' + sc.run( + out_fpath, + fixed_charge_rate=0.1, + simple=False, + transmission_costs=tcosts, + avail_cap_frac=avail_cap_frac, + columns=SC_FULL_COLUMNS, + ) + s1 = "Unconnected sc_gid" s2 = str(w[0].message) - msg = ('Warning failed! Should have Unconnected sc_gid: ' - '{}'.format(s2)) + msg = "Warning failed! Should have Unconnected sc_gid: " "{}".format( + s2 + ) assert s1 in s2, msg @@ -200,20 +259,28 @@ def test_parallel(): """Test a parallel compute against a serial compute""" tcosts = TRANS_COSTS_1.copy() - avail_cap_frac = tcosts.pop('available_capacity', 1) + avail_cap_frac = tcosts.pop("available_capacity", 1) sc = SupplyCurve(SC_POINTS, TRANS_TABLE, sc_features=MULTIPLIERS) with tempfile.TemporaryDirectory() as td: out_fpath = os.path.join(td, "sc") - sc_full_parallel = sc.run(out_fpath, fixed_charge_rate=0.1, - simple=False, transmission_costs=tcosts, - avail_cap_frac=avail_cap_frac, - columns=SC_FULL_COLUMNS, - max_workers=4) - sc_full_serial = sc.run(out_fpath, fixed_charge_rate=0.1, simple=False, - transmission_costs=tcosts, - avail_cap_frac=avail_cap_frac, - columns=SC_FULL_COLUMNS, - max_workers=1) + sc_full_parallel = sc.run( + out_fpath, + fixed_charge_rate=0.1, + simple=False, + transmission_costs=tcosts, + avail_cap_frac=avail_cap_frac, + columns=SC_FULL_COLUMNS, + max_workers=4, + ) + sc_full_serial = sc.run( + out_fpath, + fixed_charge_rate=0.1, + simple=False, + transmission_costs=tcosts, + avail_cap_frac=avail_cap_frac, + columns=SC_FULL_COLUMNS, + max_workers=1, + ) sc_full_parallel = pd.read_csv(sc_full_parallel) sc_full_serial = pd.read_csv(sc_full_serial) @@ -229,22 +296,23 @@ def verify_trans_cap(sc_table, trans_tables, cap_col=MetaKeyName.CAPACITY): trans_features = [] for path in trans_tables: df = pd.read_csv(path) - trans_features.append(df[['trans_gid', 'max_cap']]) + trans_features.append(df[["trans_gid", "max_cap"]]) trans_features = pd.concat(trans_features) if isinstance(sc_table, str) and os.path.exists(sc_table): sc_table = pd.read_csv(sc_table) - if 'max_cap' in sc_table and 'max_cap' in trans_features: - sc_table = sc_table.drop('max_cap', axis=1) + if "max_cap" in sc_table and "max_cap" in trans_features: + sc_table = sc_table.drop("max_cap", axis=1) - test = sc_table.merge(trans_features, on='trans_gid', how='left') - mask = test[cap_col] > test['max_cap'] - cols = [MetaKeyName.SC_GID, 'trans_gid', cap_col, 'max_cap'] - msg = ("SC points connected to transmission features with " - "max_cap < sc_cap:\n{}" - .format(test.loc[mask, cols])) + test = sc_table.merge(trans_features, on="trans_gid", how="left") + mask = test[cap_col] > test["max_cap"] + cols = [MetaKeyName.SC_GID, "trans_gid", cap_col, "max_cap"] + msg = ( + "SC points connected to transmission features with " + "max_cap < sc_cap:\n{}".format(test.loc[mask, cols]) + ) assert any(mask), msg @@ -252,17 +320,22 @@ def test_least_cost_full(): """ Test full supply curve sorting with least-cost path transmission tables """ - trans_tables = [os.path.join(TESTDATADIR, 'trans_tables', - f'costs_RI_{cap}MW.csv') - for cap in [100, 200, 400, 1000]] + trans_tables = [ + os.path.join(TESTDATADIR, "trans_tables", f"costs_RI_{cap}MW.csv") + for cap in [100, 200, 400, 1000] + ] sc = SupplyCurve(SC_POINTS, trans_tables, sc_features=None) with tempfile.TemporaryDirectory() as td: out_fpath = os.path.join(td, "sc") - sc_full = sc.run(out_fpath, fixed_charge_rate=0.1, simple=False, - avail_cap_frac=0.1, - columns=list(SC_FULL_COLUMNS) + ["max_cap"]) - - fpath_baseline = os.path.join(TESTDATADIR, 'sc_out/sc_full_lc.csv') + sc_full = sc.run( + out_fpath, + fixed_charge_rate=0.1, + simple=False, + avail_cap_frac=0.1, + columns=list(SC_FULL_COLUMNS) + ["max_cap"], + ) + + fpath_baseline = os.path.join(TESTDATADIR, "sc_out/sc_full_lc.csv") baseline_verify(sc_full, fpath_baseline) verify_trans_cap(sc_full, trans_tables) @@ -271,15 +344,16 @@ def test_least_cost_simple(): """ Test simple supply curve sorting with least-cost path transmission tables """ - trans_tables = [os.path.join(TESTDATADIR, 'trans_tables', - f'costs_RI_{cap}MW.csv') - for cap in [100, 200, 400, 1000]] + trans_tables = [ + os.path.join(TESTDATADIR, "trans_tables", f"costs_RI_{cap}MW.csv") + for cap in [100, 200, 400, 1000] + ] sc = SupplyCurve(SC_POINTS, trans_tables) with tempfile.TemporaryDirectory() as td: out_fpath = os.path.join(td, "sc") sc_simple = sc.run(out_fpath, fixed_charge_rate=0.1, simple=True) - fpath_baseline = os.path.join(TESTDATADIR, 'sc_out/sc_simple_lc.csv') + fpath_baseline = os.path.join(TESTDATADIR, "sc_out/sc_simple_lc.csv") baseline_verify(sc_simple, fpath_baseline) verify_trans_cap(sc_simple, trans_tables) @@ -289,16 +363,17 @@ def test_simple_trans_table(): Run the simple SC test using a simple transmission table and verify results against baseline file. """ - trans_table = os.path.join(TESTDATADIR, - 'trans_tables', - 'ri_simple_transmission_table.csv') + trans_table = os.path.join( + TESTDATADIR, "trans_tables", "ri_simple_transmission_table.csv" + ) sc = SupplyCurve(SC_POINTS, trans_table) with tempfile.TemporaryDirectory() as td: out_fpath = os.path.join(td, "sc") sc_simple = sc.run(out_fpath, fixed_charge_rate=0.1, simple=True) - fpath_baseline = os.path.join(TESTDATADIR, - 'sc_out/ri_sc_simple_lc.csv') + fpath_baseline = os.path.join( + TESTDATADIR, "sc_out/ri_sc_simple_lc.csv" + ) baseline_verify(sc_simple, fpath_baseline) @@ -307,8 +382,8 @@ def test_substation_conns(): Ensure missing trans lines are caught by SupplyCurveInputError """ tcosts = TRANS_COSTS_1.copy() - avail_cap_frac = tcosts.pop('available_capacity', 1) - drop_lines = np.where(TRANS_TABLE['category'] == 'TransLine')[0] + avail_cap_frac = tcosts.pop("available_capacity", 1) + drop_lines = np.where(TRANS_TABLE["category"] == "TransLine")[0] drop_lines = np.random.choice(drop_lines, 10, replace=False) trans_table = TRANS_TABLE.drop(labels=drop_lines) @@ -316,9 +391,14 @@ def test_substation_conns(): sc = SupplyCurve(SC_POINTS, trans_table, sc_features=MULTIPLIERS) with tempfile.TemporaryDirectory() as td: out_fpath = os.path.join(td, "sc") - sc.run(out_fpath, fixed_charge_rate=0.1, simple=False, - columns=SC_FULL_COLUMNS, avail_cap_frac=avail_cap_frac, - max_workers=4) + sc.run( + out_fpath, + fixed_charge_rate=0.1, + simple=False, + columns=SC_FULL_COLUMNS, + avail_cap_frac=avail_cap_frac, + max_workers=4, + ) def test_multi_parallel_trans(): @@ -334,40 +414,68 @@ def test_multi_parallel_trans(): https://github.com/NREL/reV/issues/336 """ - columns = ('trans_gid', 'trans_type', 'n_parallel_trans', - 'lcot', 'total_lcoe', 'trans_cap_cost_per_mw', - 'max_cap') - - trans_tables = [os.path.join(TESTDATADIR, 'trans_tables', - f'costs_RI_{cap}MW.csv') - for cap in [100, 200, 400, 1000]] + columns = ( + "trans_gid", + "trans_type", + "n_parallel_trans", + "lcot", + "total_lcoe", + "trans_cap_cost_per_mw", + "max_cap", + ) + + trans_tables = [ + os.path.join(TESTDATADIR, "trans_tables", f"costs_RI_{cap}MW.csv") + for cap in [100, 200, 400, 1000] + ] sc = SupplyCurve(SC_POINTS, trans_tables) sc_1 = sc.simple_sort(fcr=0.1, columns=columns) - trans_tables = [os.path.join(TESTDATADIR, 'trans_tables', - f'costs_RI_{cap}MW.csv') - for cap in [100]] + trans_tables = [ + os.path.join(TESTDATADIR, "trans_tables", f"costs_RI_{cap}MW.csv") + for cap in [100] + ] sc = SupplyCurve(SC_POINTS, trans_tables) sc_2 = sc.simple_sort(fcr=0.1, columns=columns) - assert not set(SC_POINTS[MetaKeyName.SC_GID]) - set(sc_1[MetaKeyName.SC_GID]) - assert not set(SC_POINTS[MetaKeyName.SC_GID]) - set(sc_2[MetaKeyName.SC_GID]) - assert not set(SC_POINTS[MetaKeyName.SC_POINT_GID]) - set(sc_1[MetaKeyName.SC_POINT_GID]) - assert not set(SC_POINTS[MetaKeyName.SC_POINT_GID]) - set(sc_2[MetaKeyName.SC_POINT_GID]) - assert not set(sc_1[MetaKeyName.SC_POINT_GID]) - set(SC_POINTS[MetaKeyName.SC_POINT_GID]) - assert not set(sc_2[MetaKeyName.SC_POINT_GID]) - set(SC_POINTS[MetaKeyName.SC_POINT_GID]) + assert not set(SC_POINTS[MetaKeyName.SC_GID]) - set( + sc_1[MetaKeyName.SC_GID] + ) + assert not set(SC_POINTS[MetaKeyName.SC_GID]) - set( + sc_2[MetaKeyName.SC_GID] + ) + assert not set(SC_POINTS[MetaKeyName.SC_POINT_GID]) - set( + sc_1[MetaKeyName.SC_POINT_GID] + ) + assert not set(SC_POINTS[MetaKeyName.SC_POINT_GID]) - set( + sc_2[MetaKeyName.SC_POINT_GID] + ) + assert not set(sc_1[MetaKeyName.SC_POINT_GID]) - set( + SC_POINTS[MetaKeyName.SC_POINT_GID] + ) + assert not set(sc_2[MetaKeyName.SC_POINT_GID]) - set( + SC_POINTS[MetaKeyName.SC_POINT_GID] + ) assert (sc_2.n_parallel_trans > 1).any() - mask_2 = sc_2['n_parallel_trans'] > 1 + mask_2 = sc_2["n_parallel_trans"] > 1 for gid in sc_2.loc[mask_2, MetaKeyName.SC_GID]: - nx_1 = sc_1.loc[(sc_1[MetaKeyName.SC_GID] == gid), 'n_parallel_trans'].values[0] - nx_2 = sc_2.loc[(sc_2[MetaKeyName.SC_GID] == gid), 'n_parallel_trans'].values[0] + nx_1 = sc_1.loc[ + (sc_1[MetaKeyName.SC_GID] == gid), "n_parallel_trans" + ].values[0] + nx_2 = sc_2.loc[ + (sc_2[MetaKeyName.SC_GID] == gid), "n_parallel_trans" + ].values[0] assert nx_2 >= nx_1 if nx_1 != nx_2: - lcot_1 = sc_1.loc[(sc_1[MetaKeyName.SC_GID] == gid), 'lcot'].values[0] - lcot_2 = sc_2.loc[(sc_2[MetaKeyName.SC_GID] == gid), 'lcot'].values[0] + lcot_1 = sc_1.loc[ + (sc_1[MetaKeyName.SC_GID] == gid), "lcot" + ].values[0] + lcot_2 = sc_2.loc[ + (sc_2[MetaKeyName.SC_GID] == gid), "lcot" + ].values[0] assert lcot_2 > lcot_1 @@ -380,10 +488,11 @@ def test_least_cost_full_with_reinforcement(): with tempfile.TemporaryDirectory() as td: trans_tables = [] for cap in [100, 200, 400, 1000]: - in_table = os.path.join(TESTDATADIR, 'trans_tables', - f'costs_RI_{cap}MW.csv') + in_table = os.path.join( + TESTDATADIR, "trans_tables", f"costs_RI_{cap}MW.csv" + ) in_table = pd.read_csv(in_table) - out_fp = os.path.join(td, f'costs_RI_{cap}MW.csv') + out_fp = os.path.join(td, f"costs_RI_{cap}MW.csv") in_table["reinforcement_cost_per_mw"] = 0 in_table["reinforcement_dist_km"] = 0 in_table.to_csv(out_fp, index=False) @@ -391,21 +500,26 @@ def test_least_cost_full_with_reinforcement(): out_fpath = os.path.join(td, "sc") sc = SupplyCurve(SC_POINTS, trans_tables) - sc_full = sc.run(out_fpath, fixed_charge_rate=0.1, simple=False, - avail_cap_frac=0.1, - columns=list(SC_FULL_COLUMNS) + ["max_cap"]) + sc_full = sc.run( + out_fpath, + fixed_charge_rate=0.1, + simple=False, + avail_cap_frac=0.1, + columns=list(SC_FULL_COLUMNS) + ["max_cap"], + ) sc_full = pd.read_csv(sc_full) - fpath_baseline = os.path.join(TESTDATADIR, 'sc_out/sc_full_lc.csv') + fpath_baseline = os.path.join(TESTDATADIR, "sc_out/sc_full_lc.csv") baseline_verify(sc_full, fpath_baseline) verify_trans_cap(sc_full, trans_tables) trans_tables = [] for cap in [100, 200, 400, 1000]: - in_table = os.path.join(TESTDATADIR, 'trans_tables', - f'costs_RI_{cap}MW.csv') + in_table = os.path.join( + TESTDATADIR, "trans_tables", f"costs_RI_{cap}MW.csv" + ) in_table = pd.read_csv(in_table) - out_fp = os.path.join(td, f'costs_RI_{cap}MW.csv') + out_fp = os.path.join(td, f"costs_RI_{cap}MW.csv") in_table["reinforcement_cost_per_mw"] = 1e6 in_table["reinforcement_dist_km"] = 10 in_table.to_csv(out_fp, index=False) @@ -413,9 +527,13 @@ def test_least_cost_full_with_reinforcement(): out_fpath = os.path.join(td, "sc_r") sc = SupplyCurve(SC_POINTS, trans_tables) - sc_full_r = sc.run(out_fpath, fixed_charge_rate=0.1, simple=False, - avail_cap_frac=0.1, - columns=list(SC_FULL_COLUMNS) + ["max_cap"]) + sc_full_r = sc.run( + out_fpath, + fixed_charge_rate=0.1, + simple=False, + avail_cap_frac=0.1, + columns=list(SC_FULL_COLUMNS) + ["max_cap"], + ) sc_full_r = pd.read_csv(sc_full_r) verify_trans_cap(sc_full, trans_tables) @@ -432,10 +550,11 @@ def test_least_cost_simple_with_reinforcement(): with tempfile.TemporaryDirectory() as td: trans_tables = [] for cap in [100, 200, 400, 1000]: - in_table = os.path.join(TESTDATADIR, 'trans_tables', - f'costs_RI_{cap}MW.csv') + in_table = os.path.join( + TESTDATADIR, "trans_tables", f"costs_RI_{cap}MW.csv" + ) in_table = pd.read_csv(in_table) - out_fp = os.path.join(td, f'costs_RI_{cap}MW.csv') + out_fp = os.path.join(td, f"costs_RI_{cap}MW.csv") in_table["reinforcement_cost_per_mw"] = 0 in_table["reinforcement_dist_km"] = 0 in_table.to_csv(out_fp, index=False) @@ -446,16 +565,17 @@ def test_least_cost_simple_with_reinforcement(): sc_simple = sc.run(out_fpath, fixed_charge_rate=0.1, simple=True) sc_simple = pd.read_csv(sc_simple) - fpath_baseline = os.path.join(TESTDATADIR, 'sc_out/sc_simple_lc.csv') + fpath_baseline = os.path.join(TESTDATADIR, "sc_out/sc_simple_lc.csv") baseline_verify(sc_simple, fpath_baseline) verify_trans_cap(sc_simple, trans_tables) trans_tables = [] for cap in [100, 200, 400, 1000]: - in_table = os.path.join(TESTDATADIR, 'trans_tables', - f'costs_RI_{cap}MW.csv') + in_table = os.path.join( + TESTDATADIR, "trans_tables", f"costs_RI_{cap}MW.csv" + ) in_table = pd.read_csv(in_table) - out_fp = os.path.join(td, f'costs_RI_{cap}MW.csv') + out_fp = os.path.join(td, f"costs_RI_{cap}MW.csv") in_table["reinforcement_cost_per_mw"] = 1e6 in_table["reinforcement_dist_km"] = 10 in_table.to_csv(out_fp, index=False) @@ -476,16 +596,24 @@ def test_least_cost_full_pass_through(): """ Test the full supply curve sorting passes through variables correctly """ - check_cols = {'poi_lat', 'poi_lon', 'reinforcement_poi_lat', - 'reinforcement_poi_lon', 'eos_mult', 'reg_mult', - 'reinforcement_cost_per_mw', 'reinforcement_dist_km'} + check_cols = { + "poi_lat", + "poi_lon", + "reinforcement_poi_lat", + "reinforcement_poi_lon", + "eos_mult", + "reg_mult", + "reinforcement_cost_per_mw", + "reinforcement_dist_km", + } with tempfile.TemporaryDirectory() as td: trans_tables = [] for cap in [100, 200, 400, 1000]: - in_table = os.path.join(TESTDATADIR, 'trans_tables', - f'costs_RI_{cap}MW.csv') + in_table = os.path.join( + TESTDATADIR, "trans_tables", f"costs_RI_{cap}MW.csv" + ) in_table = pd.read_csv(in_table) - out_fp = os.path.join(td, f'costs_RI_{cap}MW.csv') + out_fp = os.path.join(td, f"costs_RI_{cap}MW.csv") in_table["reinforcement_cost_per_mw"] = 0 for col in check_cols: in_table[col] = 0 @@ -494,9 +622,13 @@ def test_least_cost_full_pass_through(): out_fpath = os.path.join(td, "sc") sc = SupplyCurve(SC_POINTS, trans_tables) - sc_full = sc.run(out_fpath, fixed_charge_rate=0.1, simple=False, - avail_cap_frac=0.1, - columns=list(SC_FULL_COLUMNS) + ["max_cap"]) + sc_full = sc.run( + out_fpath, + fixed_charge_rate=0.1, + simple=False, + avail_cap_frac=0.1, + columns=list(SC_FULL_COLUMNS) + ["max_cap"], + ) sc_full = pd.read_csv(sc_full) for col in check_cols: @@ -508,16 +640,24 @@ def test_least_cost_simple_pass_through(): """ Test the simple supply curve sorting passes through variables correctly """ - check_cols = {'poi_lat', 'poi_lon', 'reinforcement_poi_lat', - 'reinforcement_poi_lon', 'eos_mult', 'reg_mult', - 'reinforcement_cost_per_mw', 'reinforcement_dist_km'} + check_cols = { + "poi_lat", + "poi_lon", + "reinforcement_poi_lat", + "reinforcement_poi_lon", + "eos_mult", + "reg_mult", + "reinforcement_cost_per_mw", + "reinforcement_dist_km", + } with tempfile.TemporaryDirectory() as td: trans_tables = [] for cap in [100, 200, 400, 1000]: - in_table = os.path.join(TESTDATADIR, 'trans_tables', - f'costs_RI_{cap}MW.csv') + in_table = os.path.join( + TESTDATADIR, "trans_tables", f"costs_RI_{cap}MW.csv" + ) in_table = pd.read_csv(in_table) - out_fp = os.path.join(td, f'costs_RI_{cap}MW.csv') + out_fp = os.path.join(td, f"costs_RI_{cap}MW.csv") in_table["reinforcement_cost_per_mw"] = 0 for col in check_cols: in_table[col] = 0 @@ -540,13 +680,13 @@ def test_least_cost_simple_with_ac_capacity_column(): least-cost path transmission tables and AC capacity column as capacity """ with tempfile.TemporaryDirectory() as td: - trans_tables = [] for cap in [100, 200, 400, 1000]: - in_table = os.path.join(TESTDATADIR, 'trans_tables', - f'costs_RI_{cap}MW.csv') + in_table = os.path.join( + TESTDATADIR, "trans_tables", f"costs_RI_{cap}MW.csv" + ) in_table = pd.read_csv(in_table) - out_fp = os.path.join(td, f'costs_RI_{cap}MW.csv') + out_fp = os.path.join(td, f"costs_RI_{cap}MW.csv") in_table["reinforcement_cost_per_mw"] = 1e6 in_table["reinforcement_dist_km"] = 10 in_table.to_csv(out_fp, index=False) @@ -558,10 +698,11 @@ def test_least_cost_simple_with_ac_capacity_column(): trans_tables = [] for cap in [100, 200, 400, 1000]: - in_table = os.path.join(TESTDATADIR, 'trans_tables', - f'costs_RI_{cap}MW.csv') + in_table = os.path.join( + TESTDATADIR, "trans_tables", f"costs_RI_{cap}MW.csv" + ) in_table = pd.read_csv(in_table) - out_fp = os.path.join(td, f'costs_RI_{cap}MW.csv') + out_fp = os.path.join(td, f"costs_RI_{cap}MW.csv") in_table["reinforcement_cost_per_mw"] = 1e6 in_table["reinforcement_dist_km"] = 10 in_table.to_csv(out_fp, index=False) @@ -574,10 +715,14 @@ def test_least_cost_simple_with_ac_capacity_column(): sc_simple_ac_cap = sc.simple_sort(fcr=0.1) verify_trans_cap(sc_simple_ac_cap, trans_tables, cap_col="capacity_ac") - assert np.allclose(sc_simple["trans_cap_cost_per_mw"] * 1.02, - sc_simple_ac_cap["trans_cap_cost_per_mw"]) - assert np.allclose(sc_simple["reinforcement_cost_per_mw"], - sc_simple_ac_cap["reinforcement_cost_per_mw"]) + assert np.allclose( + sc_simple["trans_cap_cost_per_mw"] * 1.02, + sc_simple_ac_cap["trans_cap_cost_per_mw"], + ) + assert np.allclose( + sc_simple["reinforcement_cost_per_mw"], + sc_simple_ac_cap["reinforcement_cost_per_mw"], + ) # Final reinforcement costs are slightly cheaper for AC capacity assert np.all(sc_simple["lcot"] > sc_simple_ac_cap["lcot"]) diff --git a/tests/test_supply_curve_points.py b/tests/test_supply_curve_points.py index a9c718bad..1d3b93116 100644 --- a/tests/test_supply_curve_points.py +++ b/tests/test_supply_curve_points.py @@ -4,6 +4,7 @@ @author: gbuster """ + # pylint: disable=no-member import os @@ -20,22 +21,24 @@ from reV.supply_curve.sc_aggregation import SupplyCurveAggregation from reV.utilities import MetaKeyName -F_EXCL = os.path.join(TESTDATADIR, 'ri_exclusions/ri_exclusions.h5') -F_GEN = os.path.join(TESTDATADIR, 'gen_out/gen_ri_pv_2012_x000.h5') -TM_DSET = 'techmap_nsrdb' -EXCL_DICT = {'ri_srtm_slope': {'inclusion_range': (None, 5), - 'exclude_nodata': True}, - 'ri_padus': {'exclude_values': [1], - 'exclude_nodata': True}, - 'ri_reeds_regions': {'inclusion_range': (None, 400), - 'exclude_nodata': True}} - -F_TECHMAP = os.path.join(TESTDATADIR, 'sc_out/baseline_ri_tech_map.h5') -DSET_TM = 'res_ri_pv' +F_EXCL = os.path.join(TESTDATADIR, "ri_exclusions/ri_exclusions.h5") +F_GEN = os.path.join(TESTDATADIR, "gen_out/gen_ri_pv_2012_x000.h5") +TM_DSET = "techmap_nsrdb" +EXCL_DICT = { + "ri_srtm_slope": {"inclusion_range": (None, 5), "exclude_nodata": True}, + "ri_padus": {"exclude_values": [1], "exclude_nodata": True}, + "ri_reeds_regions": { + "inclusion_range": (None, 400), + "exclude_nodata": True, + }, +} + +F_TECHMAP = os.path.join(TESTDATADIR, "sc_out/baseline_ri_tech_map.h5") +DSET_TM = "res_ri_pv" RTOL = 0.001 -@pytest.mark.parametrize('resolution', [7, 32, 50, 64, 163]) +@pytest.mark.parametrize("resolution", [7, 32, 50, 64, 163]) def test_points_calc(resolution): """Test the calculation of the SC points setup from exclusions tiff.""" @@ -45,41 +48,48 @@ def test_points_calc(resolution): assert len(sc) == (sc.n_rows * sc.n_cols) -@pytest.mark.parametrize(('gids', 'resolution'), - [(range(361), 64), (range(12), 377)]) +@pytest.mark.parametrize( + ("gids", "resolution"), [(range(361), 64), (range(12), 377)] +) def test_slicer(gids, resolution): """Run tests on the different extent slicing algorithms.""" with SupplyCurveExtent(F_EXCL, resolution=resolution) as sc: - for gid in gids: row_slice0, col_slice0 = sc.get_excl_slices(gid) row_slice1, col_slice1 = SupplyCurvePoint.get_agg_slices( - gid, sc.exclusions.shape, resolution) - msg = ('Slicing failed for gid {} and res {}' - .format(gid, resolution)) + gid, sc.exclusions.shape, resolution + ) + msg = "Slicing failed for gid {} and res {}".format( + gid, resolution + ) assert row_slice0 == row_slice1, msg assert col_slice0 == col_slice1, msg -@pytest.mark.parametrize((MetaKeyName.GID, 'resolution', 'excl_dict', 'time_series'), - [(37, 64, None, None), - (37, 64, EXCL_DICT, None), - (37, 64, None, 100), - (37, 64, EXCL_DICT, 100), - (37, 37, None, None), - (37, 37, EXCL_DICT, None), - (37, 37, None, 100), - (37, 37, EXCL_DICT, 100)]) +@pytest.mark.parametrize( + (MetaKeyName.GID, "resolution", "excl_dict", "time_series"), + [ + (37, 64, None, None), + (37, 64, EXCL_DICT, None), + (37, 64, None, 100), + (37, 64, EXCL_DICT, 100), + (37, 37, None, None), + (37, 37, EXCL_DICT, None), + (37, 37, None, 100), + (37, 37, EXCL_DICT, 100), + ], +) def test_weighted_means(gid, resolution, excl_dict, time_series): """ Test Supply Curve Point exclusions weighted mean calculation """ - with SupplyCurvePoint(gid, F_EXCL, TM_DSET, excl_dict=excl_dict, - resolution=resolution) as point: - shape = (point._gids.max() + 1, ) + with SupplyCurvePoint( + gid, F_EXCL, TM_DSET, excl_dict=excl_dict, resolution=resolution + ) as point: + shape = (point._gids.max() + 1,) if time_series: - shape = (time_series, ) + shape + shape = (time_series,) + shape arr = np.random.random(shape) means = point.exclusion_weighted_mean(arr.copy()) @@ -100,24 +110,29 @@ def test_weighted_means(gid, resolution, excl_dict, time_series): assert np.allclose(test, means, rtol=RTOL) -@pytest.mark.parametrize((MetaKeyName.GID, 'resolution', 'excl_dict', 'time_series'), - [(37, 64, None, None), - (37, 64, EXCL_DICT, None), - (37, 64, None, 100), - (37, 64, EXCL_DICT, 100), - (37, 37, None, None), - (37, 37, EXCL_DICT, None), - (37, 37, None, 100), - (37, 37, EXCL_DICT, 100)]) +@pytest.mark.parametrize( + (MetaKeyName.GID, "resolution", "excl_dict", "time_series"), + [ + (37, 64, None, None), + (37, 64, EXCL_DICT, None), + (37, 64, None, 100), + (37, 64, EXCL_DICT, 100), + (37, 37, None, None), + (37, 37, EXCL_DICT, None), + (37, 37, None, 100), + (37, 37, EXCL_DICT, 100), + ], +) def test_aggregate(gid, resolution, excl_dict, time_series): """ Test Supply Curve Point aggregate calculation """ - with SupplyCurvePoint(gid, F_EXCL, TM_DSET, excl_dict=excl_dict, - resolution=resolution) as point: - shape = (point._gids.max() + 1, ) + with SupplyCurvePoint( + gid, F_EXCL, TM_DSET, excl_dict=excl_dict, resolution=resolution + ) as point: + shape = (point._gids.max() + 1,) if time_series: - shape = (time_series, ) + shape + shape = (time_series,) + shape arr = np.random.random(shape) total = point.aggregate(arr.copy()) @@ -141,21 +156,31 @@ def plot_all_sc_points(resolution=64): """Test the calculation of the SC points setup from exclusions tiff.""" import matplotlib.pyplot as plt - prop_cycle = plt.rcParams['axes.prop_cycle'] - colors = prop_cycle.by_key()['color'] + + prop_cycle = plt.rcParams["axes.prop_cycle"] + colors = prop_cycle.by_key()["color"] _, axs = plt.subplots(1, 1) with SupplyCurveExtent(F_EXCL, resolution=resolution) as sc: colors *= len(sc) for gid in range(len(sc)): - excl_meta = sc.get_excl_points('meta', gid) - axs.scatter(excl_meta[MetaKeyName.LONGITUDE], excl_meta[MetaKeyName.LATITUDE], - c=colors[gid], s=0.01) + excl_meta = sc.get_excl_points("meta", gid) + axs.scatter( + excl_meta[MetaKeyName.LONGITUDE], + excl_meta[MetaKeyName.LATITUDE], + c=colors[gid], + s=0.01, + ) with Outputs(F_GEN) as f: - axs.scatter(f.meta[MetaKeyName.LONGITUDE], f.meta[MetaKeyName.LATITUDE], c='k', s=25) - - axs.axis('equal') + axs.scatter( + f.meta[MetaKeyName.LONGITUDE], + f.meta[MetaKeyName.LATITUDE], + c="k", + s=25, + ) + + axs.axis("equal") plt.show() @@ -163,37 +188,49 @@ def plot_single_gen_sc_point(gid=2, resolution=64): """Test the calculation of the SC points setup from exclusions tiff.""" import matplotlib.pyplot as plt - colors = ['b', 'g', 'c', 'y', 'm'] + colors = ["b", "g", "c", "y", "m"] colors *= 100 _, axs = plt.subplots(1, 1) gen_index = SupplyCurveAggregation._parse_gen_index(F_GEN) - with GenerationSupplyCurvePoint(gid, F_EXCL, F_GEN, F_TECHMAP, DSET_TM, - gen_index, - resolution=resolution) as sc: - + with GenerationSupplyCurvePoint( + gid, + F_EXCL, + F_GEN, + F_TECHMAP, + DSET_TM, + gen_index, + resolution=resolution, + ) as sc: all_gen_gids = list(set(sc._gen_gids)) - excl_meta = sc.exclusions['meta', sc.rows, sc.cols] + excl_meta = sc.exclusions["meta", sc.rows, sc.cols] for i, gen_gid in enumerate(all_gen_gids): if gen_gid != -1: - mask = (sc._gen_gids == gen_gid) - axs.scatter(excl_meta.loc[mask, MetaKeyName.LONGITUDE], - excl_meta.loc[mask, MetaKeyName.LATITUDE], - marker='s', c=colors[i], s=1) - - axs.scatter(sc.gen.meta.loc[gen_gid, MetaKeyName.LONGITUDE], - sc.gen.meta.loc[gen_gid, MetaKeyName.LATITUDE], - c='k', s=100) - - axs.scatter(sc.centroid[1], sc.centroid[0], marker='x', c='k', s=200) - - axs.axis('equal') + mask = sc._gen_gids == gen_gid + axs.scatter( + excl_meta.loc[mask, MetaKeyName.LONGITUDE], + excl_meta.loc[mask, MetaKeyName.LATITUDE], + marker="s", + c=colors[i], + s=1, + ) + + axs.scatter( + sc.gen.meta.loc[gen_gid, MetaKeyName.LONGITUDE], + sc.gen.meta.loc[gen_gid, MetaKeyName.LATITUDE], + c="k", + s=100, + ) + + axs.scatter(sc.centroid[1], sc.centroid[0], marker="x", c="k", s=200) + + axs.axis("equal") plt.show() -def execute_pytest(capture='all', flags='-rapP'): +def execute_pytest(capture="all", flags="-rapP"): """Execute module as pytest with detailed summary report. Parameters @@ -206,8 +243,8 @@ def execute_pytest(capture='all', flags='-rapP'): """ fname = os.path.basename(__file__) - pytest.main(['-q', '--show-capture={}'.format(capture), fname, flags]) + pytest.main(["-q", "--show-capture={}".format(capture), fname, flags]) -if __name__ == '__main__': +if __name__ == "__main__": execute_pytest() diff --git a/tests/test_supply_curve_sc_aggregation.py b/tests/test_supply_curve_sc_aggregation.py index e6189b7ca..9eed6323e 100644 --- a/tests/test_supply_curve_sc_aggregation.py +++ b/tests/test_supply_curve_sc_aggregation.py @@ -5,6 +5,7 @@ @author: gbuster """ + import json import os import shutil @@ -27,26 +28,25 @@ ) from reV.utilities import MetaKeyName, ModuleName -EXCL = os.path.join(TESTDATADIR, 'ri_exclusions/ri_exclusions.h5') -RES = os.path.join(TESTDATADIR, 'nsrdb/ri_100_nsrdb_2012.h5') -GEN = os.path.join(TESTDATADIR, 'gen_out/ri_my_pv_gen.h5') -ONLY_GEN = os.path.join(TESTDATADIR, 'gen_out/ri_my_pv_only_gen.h5') -ONLY_ECON = os.path.join(TESTDATADIR, 'gen_out/ri_my_pv_only_econ.h5') -AGG_BASELINE = os.path.join(TESTDATADIR, 'sc_out/baseline_agg_summary.csv') -TM_DSET = 'techmap_nsrdb' -RES_CLASS_DSET = 'ghi_mean-means' +EXCL = os.path.join(TESTDATADIR, "ri_exclusions/ri_exclusions.h5") +RES = os.path.join(TESTDATADIR, "nsrdb/ri_100_nsrdb_2012.h5") +GEN = os.path.join(TESTDATADIR, "gen_out/ri_my_pv_gen.h5") +ONLY_GEN = os.path.join(TESTDATADIR, "gen_out/ri_my_pv_only_gen.h5") +ONLY_ECON = os.path.join(TESTDATADIR, "gen_out/ri_my_pv_only_econ.h5") +AGG_BASELINE = os.path.join(TESTDATADIR, "sc_out/baseline_agg_summary.csv") +TM_DSET = "techmap_nsrdb" +RES_CLASS_DSET = "ghi_mean-means" RES_CLASS_BINS = [0, 4, 100] -DATA_LAYERS = {'pct_slope': {'dset': 'ri_srtm_slope', - 'method': 'mean'}, - 'reeds_region': {'dset': 'ri_reeds_regions', - 'method': 'mode'}, - 'padus': {'dset': 'ri_padus', - 'method': 'mode'}} - -EXCL_DICT = {'ri_srtm_slope': {'inclusion_range': (None, 5), - 'exclude_nodata': True}, - 'ri_padus': {'exclude_values': [1], - 'exclude_nodata': True}} +DATA_LAYERS = { + "pct_slope": {"dset": "ri_srtm_slope", "method": "mean"}, + "reeds_region": {"dset": "ri_reeds_regions", "method": "mode"}, + "padus": {"dset": "ri_padus", "method": "mode"}, +} + +EXCL_DICT = { + "ri_srtm_slope": {"inclusion_range": (None, 5), "exclude_nodata": True}, + "ri_padus": {"exclude_values": [1], "exclude_nodata": True}, +} RTOL = 0.001 @@ -55,10 +55,15 @@ def test_agg_extent(resolution=64): """Get the SC points aggregation summary and test that there are expected columns and that all resource gids were found""" - sca = SupplyCurveAggregation(EXCL, TM_DSET, excl_dict=EXCL_DICT, - res_class_dset=None, res_class_bins=None, - data_layers=DATA_LAYERS, - resolution=resolution) + sca = SupplyCurveAggregation( + EXCL, + TM_DSET, + excl_dict=EXCL_DICT, + res_class_dset=None, + res_class_bins=None, + data_layers=DATA_LAYERS, + resolution=resolution, + ) summary = sca.summarize(GEN) all_res_gids = [] @@ -76,13 +81,20 @@ def test_parallel_agg(resolution=64): aggregation.""" gids = list(range(50, 70)) - sca = SupplyCurveAggregation(EXCL, TM_DSET, excl_dict=EXCL_DICT, - res_class_dset=None, res_class_bins=None, - data_layers=DATA_LAYERS, gids=gids, - resolution=resolution) + sca = SupplyCurveAggregation( + EXCL, + TM_DSET, + excl_dict=EXCL_DICT, + res_class_dset=None, + res_class_bins=None, + data_layers=DATA_LAYERS, + gids=gids, + resolution=resolution, + ) summary_serial = sca.summarize(GEN, max_workers=1) - summary_parallel = sca.summarize(GEN, max_workers=None, - sites_per_worker=10) + summary_parallel = sca.summarize( + GEN, max_workers=None, sites_per_worker=10 + ) assert all(summary_serial == summary_parallel) @@ -90,24 +102,34 @@ def test_parallel_agg(resolution=64): def test_agg_summary(): """Test the aggregation summary method against a baseline file.""" - sca = SupplyCurveAggregation(EXCL, TM_DSET, excl_dict=EXCL_DICT, - res_class_dset=RES_CLASS_DSET, - res_class_bins=RES_CLASS_BINS, - data_layers=DATA_LAYERS) + sca = SupplyCurveAggregation( + EXCL, + TM_DSET, + excl_dict=EXCL_DICT, + res_class_dset=RES_CLASS_DSET, + res_class_bins=RES_CLASS_BINS, + data_layers=DATA_LAYERS, + ) summary = sca.summarize(GEN, max_workers=1) if not os.path.exists(AGG_BASELINE): summary.to_csv(AGG_BASELINE) - raise Exception('Aggregation summary baseline file did not exist. ' - 'Created: {}'.format(AGG_BASELINE)) - - for c in [MetaKeyName.RES_GIDS, MetaKeyName.GEN_GIDS, MetaKeyName.GID_COUNTS]: + raise Exception( + "Aggregation summary baseline file did not exist. " + "Created: {}".format(AGG_BASELINE) + ) + + for c in [ + MetaKeyName.RES_GIDS, + MetaKeyName.GEN_GIDS, + MetaKeyName.GID_COUNTS, + ]: summary[c] = summary[c].astype(str) s_baseline = pd.read_csv(AGG_BASELINE, index_col=0) - summary = summary.fillna('None') - s_baseline = s_baseline.fillna('None') + summary = summary.fillna("None") + s_baseline = s_baseline.fillna("None") summary = summary[list(s_baseline.columns)] assert_frame_equal(summary, s_baseline, check_dtype=False, rtol=0.0001) @@ -120,19 +142,24 @@ def test_agg_summary_solar_ac(pd): """Test the aggregation summary method for solar ac outputs.""" with tempfile.TemporaryDirectory() as td: - gen = os.path.join(td, 'gen.h5') + gen = os.path.join(td, "gen.h5") shutil.copy(GEN, gen) - Outputs.add_dataset(gen, 'dc_ac_ratio', np.array([1.3] * 188), - np.float32) + Outputs.add_dataset( + gen, "dc_ac_ratio", np.array([1.3] * 188), np.float32 + ) with Outputs(gen, "r") as out: assert "dc_ac_ratio" in out.datasets - sca = SupplyCurveAggregation(EXCL, TM_DSET, excl_dict=EXCL_DICT, - res_class_dset=RES_CLASS_DSET, - res_class_bins=RES_CLASS_BINS, - data_layers=DATA_LAYERS, - power_density=pd) + sca = SupplyCurveAggregation( + EXCL, + TM_DSET, + excl_dict=EXCL_DICT, + res_class_dset=RES_CLASS_DSET, + res_class_bins=RES_CLASS_BINS, + data_layers=DATA_LAYERS, + power_density=pd, + ) summary = sca.summarize(gen, max_workers=1) assert "capacity_ac" in summary @@ -142,67 +169,84 @@ def test_agg_summary_solar_ac(pd): def test_multi_file_excl(): """Test sc aggregation with multple exclusion file inputs.""" - excl_dict = {'ri_srtm_slope': {'inclusion_range': (None, 5), - 'exclude_nodata': True}, - 'ri_padus': {'exclude_values': [1], - 'exclude_nodata': True}, - 'excl_test': {'include_values': [1], - 'weight': 0.5}, - } + excl_dict = { + "ri_srtm_slope": { + "inclusion_range": (None, 5), + "exclude_nodata": True, + }, + "ri_padus": {"exclude_values": [1], "exclude_nodata": True}, + "excl_test": {"include_values": [1], "weight": 0.5}, + } with tempfile.TemporaryDirectory() as td: - excl_temp_1 = os.path.join(td, 'excl1.h5') - excl_temp_2 = os.path.join(td, 'excl2.h5') + excl_temp_1 = os.path.join(td, "excl1.h5") + excl_temp_2 = os.path.join(td, "excl2.h5") shutil.copy(EXCL, excl_temp_1) shutil.copy(EXCL, excl_temp_2) - with h5py.File(excl_temp_1, 'a') as f: + with h5py.File(excl_temp_1, "a") as f: shape = f[MetaKeyName.LATITUDE].shape - attrs = dict(f['ri_srtm_slope'].attrs) + attrs = dict(f["ri_srtm_slope"].attrs) data = np.ones(shape) - test_dset = 'excl_test' + test_dset = "excl_test" f.create_dataset(test_dset, shape, data=data) for k, v in attrs.items(): f[test_dset].attrs[k] = v - del f['ri_srtm_slope'] - - sca = SupplyCurveAggregation((excl_temp_1, excl_temp_2), TM_DSET, - excl_dict=excl_dict, - res_class_dset=RES_CLASS_DSET, - res_class_bins=RES_CLASS_BINS) + del f["ri_srtm_slope"] + + sca = SupplyCurveAggregation( + (excl_temp_1, excl_temp_2), + TM_DSET, + excl_dict=excl_dict, + res_class_dset=RES_CLASS_DSET, + res_class_bins=RES_CLASS_BINS, + ) summary = sca.summarize(GEN) s_baseline = pd.read_csv(AGG_BASELINE, index_col=0) - summary = summary.fillna('None') - s_baseline = s_baseline.fillna('None') + summary = summary.fillna("None") + s_baseline = s_baseline.fillna("None") - assert np.allclose(summary[MetaKeyName.AREA_SQ_KM] * 2, s_baseline[MetaKeyName.AREA_SQ_KM]) + assert np.allclose( + summary[MetaKeyName.AREA_SQ_KM] * 2, + s_baseline[MetaKeyName.AREA_SQ_KM], + ) -@pytest.mark.parametrize('pre_extract', (True, False)) +@pytest.mark.parametrize("pre_extract", (True, False)) def test_pre_extract_inclusions(pre_extract): """Test the aggregation summary w/ and w/out pre-extracting inclusions""" - sca = SupplyCurveAggregation(EXCL, TM_DSET, excl_dict=EXCL_DICT, - res_class_dset=RES_CLASS_DSET, - res_class_bins=RES_CLASS_BINS, - data_layers=DATA_LAYERS, - pre_extract_inclusions=pre_extract) + sca = SupplyCurveAggregation( + EXCL, + TM_DSET, + excl_dict=EXCL_DICT, + res_class_dset=RES_CLASS_DSET, + res_class_bins=RES_CLASS_BINS, + data_layers=DATA_LAYERS, + pre_extract_inclusions=pre_extract, + ) summary = sca.summarize(GEN, max_workers=1) if not os.path.exists(AGG_BASELINE): summary.to_csv(AGG_BASELINE) - raise Exception('Aggregation summary baseline file did not exist. ' - 'Created: {}'.format(AGG_BASELINE)) - - for c in [MetaKeyName.RES_GIDS, MetaKeyName.GEN_GIDS, MetaKeyName.GID_COUNTS]: + raise Exception( + "Aggregation summary baseline file did not exist. " + "Created: {}".format(AGG_BASELINE) + ) + + for c in [ + MetaKeyName.RES_GIDS, + MetaKeyName.GEN_GIDS, + MetaKeyName.GID_COUNTS, + ]: summary[c] = summary[c].astype(str) s_baseline = pd.read_csv(AGG_BASELINE, index_col=0) - summary = summary.fillna('None') - s_baseline = s_baseline.fillna('None') + summary = summary.fillna("None") + s_baseline = s_baseline.fillna("None") summary = summary[list(s_baseline.columns)] assert_frame_equal(summary, s_baseline, check_dtype=False, rtol=0.0001) @@ -212,17 +256,25 @@ def test_agg_gen_econ(): """Test the aggregation summary method with separate gen and econ input files.""" - sca = SupplyCurveAggregation(EXCL, TM_DSET, excl_dict=EXCL_DICT, - res_class_dset=RES_CLASS_DSET, - res_class_bins=RES_CLASS_BINS, - data_layers=DATA_LAYERS) + sca = SupplyCurveAggregation( + EXCL, + TM_DSET, + excl_dict=EXCL_DICT, + res_class_dset=RES_CLASS_DSET, + res_class_bins=RES_CLASS_BINS, + data_layers=DATA_LAYERS, + ) summary_base = sca.summarize(GEN, max_workers=1) - sca = SupplyCurveAggregation(EXCL, TM_DSET, econ_fpath=ONLY_ECON, - excl_dict=EXCL_DICT, - res_class_dset=RES_CLASS_DSET, - res_class_bins=RES_CLASS_BINS, - data_layers=DATA_LAYERS) + sca = SupplyCurveAggregation( + EXCL, + TM_DSET, + econ_fpath=ONLY_ECON, + excl_dict=EXCL_DICT, + res_class_dset=RES_CLASS_DSET, + res_class_bins=RES_CLASS_BINS, + data_layers=DATA_LAYERS, + ) summary_econ = sca.summarize(ONLY_GEN, max_workers=1) assert_frame_equal(summary_base, summary_econ) @@ -230,67 +282,87 @@ def test_agg_gen_econ(): def test_agg_extra_dsets(): """Test aggregation with extra datasets to aggregate.""" - h5_dsets = ['lcoe_fcr-2012', 'lcoe_fcr-2013', 'lcoe_fcr-stdev'] - sca = SupplyCurveAggregation(EXCL, TM_DSET, h5_dsets=h5_dsets, - econ_fpath=ONLY_ECON, excl_dict=EXCL_DICT, - res_class_dset=RES_CLASS_DSET, - res_class_bins=RES_CLASS_BINS, - data_layers=DATA_LAYERS) + h5_dsets = ["lcoe_fcr-2012", "lcoe_fcr-2013", "lcoe_fcr-stdev"] + sca = SupplyCurveAggregation( + EXCL, + TM_DSET, + h5_dsets=h5_dsets, + econ_fpath=ONLY_ECON, + excl_dict=EXCL_DICT, + res_class_dset=RES_CLASS_DSET, + res_class_bins=RES_CLASS_BINS, + data_layers=DATA_LAYERS, + ) summary = sca.summarize(ONLY_GEN, max_workers=1) for dset in h5_dsets: - assert 'mean_{}'.format(dset) in summary.columns + assert "mean_{}".format(dset) in summary.columns - check = summary['mean_lcoe_fcr-2012'] == summary[MetaKeyName.MEAN_LCOE] + check = summary["mean_lcoe_fcr-2012"] == summary[MetaKeyName.MEAN_LCOE] assert not any(check) - check = summary['mean_lcoe_fcr-2013'] == summary[MetaKeyName.MEAN_LCOE] + check = summary["mean_lcoe_fcr-2013"] == summary[MetaKeyName.MEAN_LCOE] assert not any(check) - avg = (summary['mean_lcoe_fcr-2012'] + summary['mean_lcoe_fcr-2013']) / 2 + avg = (summary["mean_lcoe_fcr-2012"] + summary["mean_lcoe_fcr-2013"]) / 2 assert np.allclose(avg.values, summary[MetaKeyName.MEAN_LCOE].values) def test_agg_extra_2D_dsets(): """Test that warning is thrown for 2D datasets.""" dset = "cf_profile" - fp = os.path.join(TESTDATADIR, 'gen_out/pv_gen_2018_node00.h5') + fp = os.path.join(TESTDATADIR, "gen_out/pv_gen_2018_node00.h5") with pytest.warns(UserWarning) as records: with Resource(fp) as res: _warn_about_large_datasets(res, dset) messages = [r.message.args[0] for r in records] - assert any("Generation dataset {!r} is not 1-dimensional (shape: {})" - .format(dset, (17520, 50)) - in msg for msg in messages) - assert any("You may run into memory errors during aggregation" - in msg for msg in messages) + assert any( + "Generation dataset {!r} is not 1-dimensional (shape: {})".format( + dset, (17520, 50) + ) + in msg + for msg in messages + ) + assert any( + "You may run into memory errors during aggregation" in msg + for msg in messages + ) def test_agg_scalar_excl(): """Test the aggregation summary with exclusions of 0.5""" gids_subset = list(range(0, 20)) - excl_dict_1 = {'ri_padus': {'exclude_values': [1]}} - sca = SupplyCurveAggregation(EXCL, TM_DSET, excl_dict=excl_dict_1, - res_class_dset=RES_CLASS_DSET, - res_class_bins=RES_CLASS_BINS, - data_layers=DATA_LAYERS, gids=gids_subset) + excl_dict_1 = {"ri_padus": {"exclude_values": [1]}} + sca = SupplyCurveAggregation( + EXCL, + TM_DSET, + excl_dict=excl_dict_1, + res_class_dset=RES_CLASS_DSET, + res_class_bins=RES_CLASS_BINS, + data_layers=DATA_LAYERS, + gids=gids_subset, + ) summary_base = sca.summarize(GEN, max_workers=1) - excl_dict_2 = {'ri_padus': {'exclude_values': [1], - 'weight': 0.5}} - sca = SupplyCurveAggregation(EXCL, TM_DSET, excl_dict=excl_dict_2, - res_class_dset=RES_CLASS_DSET, - res_class_bins=RES_CLASS_BINS, - data_layers=DATA_LAYERS, gids=gids_subset) + excl_dict_2 = {"ri_padus": {"exclude_values": [1], "weight": 0.5}} + sca = SupplyCurveAggregation( + EXCL, + TM_DSET, + excl_dict=excl_dict_2, + res_class_dset=RES_CLASS_DSET, + res_class_bins=RES_CLASS_BINS, + data_layers=DATA_LAYERS, + gids=gids_subset, + ) summary_with_weights = sca.summarize(GEN, max_workers=1) dsets = [MetaKeyName.AREA_SQ_KM, MetaKeyName.CAPACITY] for dset in dsets: - diff = (summary_base[dset].values / summary_with_weights[dset].values) - msg = ('Fractional exclusions failed for {} which has values {} and {}' + diff = summary_base[dset].values / summary_with_weights[dset].values + msg = ("Fractional exclusions failed for {} which has values {} and {}" .format(dset, summary_base[dset].values, - summary_with_weights[dset].values)) + summary_with_weights[dset].values)) assert all(diff == 2), msg for i in summary_base.index: @@ -298,40 +370,40 @@ def test_agg_scalar_excl(): counts_half = summary_with_weights.loc[i, MetaKeyName.GID_COUNTS] for j, counts in enumerate(counts_full): - msg = ('GID counts for fractional exclusions failed for index {}!' + msg = ("GID counts for fractional exclusions failed for index {}!" .format(i)) assert counts == 2 * counts_half[j], msg def test_data_layer_methods(): """Test aggregation of data layers with different methods""" - data_layers = {'pct_slope_mean': {'dset': 'ri_srtm_slope', - 'method': 'mean'}, - 'pct_slope_max': {'dset': 'ri_srtm_slope', - 'method': 'max'}, - 'pct_slope_min': {'dset': 'ri_srtm_slope', - 'method': 'min'}, - 'reeds_region': {'dset': 'ri_reeds_regions', - 'method': 'category'}, - 'padus': {'dset': 'ri_padus', - 'method': 'category'}} - - sca = SupplyCurveAggregation(EXCL, TM_DSET, excl_dict=EXCL_DICT, - res_class_dset=RES_CLASS_DSET, - res_class_bins=RES_CLASS_BINS, - data_layers=data_layers) + data_layers = { + "pct_slope_mean": {"dset": "ri_srtm_slope", "method": "mean"}, + "pct_slope_max": {"dset": "ri_srtm_slope", "method": "max"}, + "pct_slope_min": {"dset": "ri_srtm_slope", "method": "min"}, + "reeds_region": {"dset": "ri_reeds_regions", "method": "category"}, + "padus": {"dset": "ri_padus", "method": "category"}, + } + + sca = SupplyCurveAggregation( + EXCL, + TM_DSET, + excl_dict=EXCL_DICT, + res_class_dset=RES_CLASS_DSET, + res_class_bins=RES_CLASS_BINS, + data_layers=data_layers, + ) summary = sca.summarize(GEN, max_workers=1) for i in summary.index.values: - # Check categorical data layers counts = summary.loc[i, MetaKeyName.GID_COUNTS] - rr = summary.loc[i, 'reeds_region'] + rr = summary.loc[i, "reeds_region"] assert isinstance(rr, str) rr = json.loads(rr) assert isinstance(rr, dict) rr_sum = sum(list(rr.values())) - padus = summary.loc[i, 'padus'] + padus = summary.loc[i, "padus"] assert isinstance(padus, str) padus = json.loads(padus) assert isinstance(padus, dict) @@ -340,108 +412,137 @@ def test_data_layer_methods(): assert padus_sum == sum(counts) assert padus_sum >= rr_sum except AssertionError: - e = ('Categorical data layer aggregation failed:\n{}' - .format(summary.loc[i])) + e = "Categorical data layer aggregation failed:\n{}".format( + summary.loc[i] + ) raise RuntimeError(e) # Check min/mean/max of the same data layer n = summary.loc[i, MetaKeyName.N_GIDS] - slope_mean = summary.loc[i, 'pct_slope_mean'] - slope_max = summary.loc[i, 'pct_slope_max'] - slope_min = summary.loc[i, 'pct_slope_min'] + slope_mean = summary.loc[i, "pct_slope_mean"] + slope_max = summary.loc[i, "pct_slope_max"] + slope_min = summary.loc[i, "pct_slope_min"] if n > 3: # sc points with <= 3 90m pixels can have min == mean == max assert slope_min < slope_mean < slope_max else: assert slope_min <= slope_mean <= slope_max -@pytest.mark.parametrize("cap_cost_scale", - ['1', '2 * np.multiply(1000, capacity) ** -0.3']) +@pytest.mark.parametrize( + "cap_cost_scale", ["1", "2 * np.multiply(1000, capacity) ** -0.3"] +) def test_recalc_lcoe(cap_cost_scale): """Test supply curve aggregation with the re-calculation of lcoe using the multi-year mean capacity factor""" - data = {'capital_cost': 34900000, - 'fixed_operating_cost': 280000, - 'fixed_charge_rate': 0.09606382995843887, - 'variable_operating_cost': 0, - 'system_capacity': 20000} + data = { + "capital_cost": 34900000, + "fixed_operating_cost": 280000, + "fixed_charge_rate": 0.09606382995843887, + "variable_operating_cost": 0, + "system_capacity": 20000, + } annual_cf = [0.24, 0.26, 0.37, 0.15] annual_lcoe = [] years = list(range(2012, 2016)) with tempfile.TemporaryDirectory() as td: - gen_temp = os.path.join(td, 'ri_my_pv_gen.h5') + gen_temp = os.path.join(td, "ri_my_pv_gen.h5") shutil.copy(GEN, gen_temp) - with h5py.File(gen_temp, 'a') as res: - for k in [d for d in list(res) if d != 'meta']: + with h5py.File(gen_temp, "a") as res: + for k in [d for d in list(res) if d != "meta"]: del res[k] for k, v in data.items(): - arr = np.full(res['meta'].shape, v) - res.create_dataset(k, res['meta'].shape, data=arr) + arr = np.full(res["meta"].shape, v) + res.create_dataset(k, res["meta"].shape, data=arr) for year, cf in zip(years, annual_cf): - lcoe = lcoe_fcr(data['fixed_charge_rate'], - data['capital_cost'], - data['fixed_operating_cost'], - data['system_capacity'] * cf * 8760, - data['variable_operating_cost']) - cf_arr = np.full(res['meta'].shape, cf) - lcoe_arr = np.full(res['meta'].shape, lcoe) + lcoe = lcoe_fcr( + data["fixed_charge_rate"], + data["capital_cost"], + data["fixed_operating_cost"], + data["system_capacity"] * cf * 8760, + data["variable_operating_cost"], + ) + cf_arr = np.full(res["meta"].shape, cf) + lcoe_arr = np.full(res["meta"].shape, lcoe) annual_lcoe.append(lcoe) - res.create_dataset('cf_mean-{}'.format(year), - res['meta'].shape, data=cf_arr) - res.create_dataset('lcoe_fcr-{}'.format(year), - res['meta'].shape, data=lcoe_arr) - - cf_arr = np.full(res['meta'].shape, np.mean(annual_cf)) - lcoe_arr = np.full(res['meta'].shape, np.mean(annual_lcoe)) - res.create_dataset('cf_mean-means', - res['meta'].shape, data=cf_arr) - res.create_dataset('lcoe_fcr-means', - res['meta'].shape, data=lcoe_arr) - - h5_dsets = ['capital_cost', 'fixed_operating_cost', - 'fixed_charge_rate', 'variable_operating_cost', - 'system_capacity'] - - base = SupplyCurveAggregation(EXCL, TM_DSET, excl_dict=EXCL_DICT, - res_class_dset=None, res_class_bins=None, - data_layers=DATA_LAYERS, - h5_dsets=h5_dsets, - gids=list(np.arange(10)), - recalc_lcoe=False, - cap_cost_scale=cap_cost_scale) + res.create_dataset( + "cf_mean-{}".format(year), res["meta"].shape, data=cf_arr + ) + res.create_dataset( + "lcoe_fcr-{}".format(year), + res["meta"].shape, + data=lcoe_arr, + ) + + cf_arr = np.full(res["meta"].shape, np.mean(annual_cf)) + lcoe_arr = np.full(res["meta"].shape, np.mean(annual_lcoe)) + res.create_dataset("cf_mean-means", res["meta"].shape, data=cf_arr) + res.create_dataset( + "lcoe_fcr-means", res["meta"].shape, data=lcoe_arr + ) + + h5_dsets = [ + "capital_cost", + "fixed_operating_cost", + "fixed_charge_rate", + "variable_operating_cost", + "system_capacity", + ] + + base = SupplyCurveAggregation( + EXCL, + TM_DSET, + excl_dict=EXCL_DICT, + res_class_dset=None, + res_class_bins=None, + data_layers=DATA_LAYERS, + h5_dsets=h5_dsets, + gids=list(np.arange(10)), + recalc_lcoe=False, + cap_cost_scale=cap_cost_scale, + ) summary_base = base.summarize(gen_temp, max_workers=1) - sca = SupplyCurveAggregation(EXCL, TM_DSET, excl_dict=EXCL_DICT, - res_class_dset=None, res_class_bins=None, - data_layers=DATA_LAYERS, - h5_dsets=h5_dsets, - gids=list(np.arange(10)), - recalc_lcoe=True, - cap_cost_scale=cap_cost_scale) + sca = SupplyCurveAggregation( + EXCL, + TM_DSET, + excl_dict=EXCL_DICT, + res_class_dset=None, + res_class_bins=None, + data_layers=DATA_LAYERS, + h5_dsets=h5_dsets, + gids=list(np.arange(10)), + recalc_lcoe=True, + cap_cost_scale=cap_cost_scale, + ) summary = sca.summarize(gen_temp, max_workers=1) - assert not np.allclose(summary_base[MetaKeyName.MEAN_LCOE], summary[MetaKeyName.MEAN_LCOE]) + assert not np.allclose( + summary_base[MetaKeyName.MEAN_LCOE], summary[MetaKeyName.MEAN_LCOE] + ) - if cap_cost_scale == '1': + if cap_cost_scale == "1": cc_dset = MetaKeyName.SC_POINT_CAPITAL_COST else: cc_dset = MetaKeyName.SCALED_SC_POINT_CAPITAL_COST - lcoe = lcoe_fcr(summary['mean_fixed_charge_rate'], summary[cc_dset], - summary[MetaKeyName.SC_POINT_FIXED_OPERATING_COST], - summary[MetaKeyName.SC_POINT_ANNUAL_ENERGY], - summary['mean_variable_operating_cost']) + lcoe = lcoe_fcr( + summary["mean_fixed_charge_rate"], + summary[cc_dset], + summary[MetaKeyName.SC_POINT_FIXED_OPERATING_COST], + summary[MetaKeyName.SC_POINT_ANNUAL_ENERGY], + summary["mean_variable_operating_cost"], + ) assert np.allclose(lcoe, summary[MetaKeyName.MEAN_LCOE]) -@pytest.mark.parametrize('tm_dset', ("techmap_ri", "techmap_ri_new")) -@pytest.mark.parametrize('pre_extract', (True, False)) +@pytest.mark.parametrize("tm_dset", ("techmap_ri", "techmap_ri_new")) +@pytest.mark.parametrize("pre_extract", (True, False)) def test_cli_basic_agg(runner, clear_loggers, tm_dset, pre_extract): with tempfile.TemporaryDirectory() as td: - excl_fp = os.path.join(td, 'excl.h5') + excl_fp = os.path.join(td, "excl.h5") shutil.copy(EXCL, excl_fp) config = { "log_directory": td, @@ -455,31 +556,34 @@ def test_cli_basic_agg(runner, clear_loggers, tm_dset, pre_extract): "econ_fpath": None, "tm_dset": tm_dset, "res_fpath": RES, - 'excl_dict': EXCL_DICT, - 'resolution': 32, - 'pre_extract_inclusions': pre_extract + "excl_dict": EXCL_DICT, + "resolution": 32, + "pre_extract_inclusions": pre_extract, } - config_path = os.path.join(td, 'config.json') - with open(config_path, 'w') as f: + config_path = os.path.join(td, "config.json") + with open(config_path, "w") as f: json.dump(config, f) - result = runner.invoke(main, [ModuleName.SUPPLY_CURVE_AGGREGATION, - '-c', config_path]) + result = runner.invoke( + main, [ModuleName.SUPPLY_CURVE_AGGREGATION, "-c", config_path] + ) clear_loggers() if result.exit_code != 0: - msg = ('Failed with error {}' - .format(traceback.print_exception(*result.exc_info))) + msg = "Failed with error {}".format( + traceback.print_exception(*result.exc_info) + ) raise RuntimeError(msg) fn_list = os.listdir(td) dirname = os.path.basename(td) - out_csv_fn = ('{}_{}.csv' - .format(dirname, ModuleName.SUPPLY_CURVE_AGGREGATION)) + out_csv_fn = "{}_{}.csv".format( + dirname, ModuleName.SUPPLY_CURVE_AGGREGATION + ) assert out_csv_fn in fn_list -def execute_pytest(capture='all', flags='-rapP'): +def execute_pytest(capture="all", flags="-rapP"): """Execute module as pytest with detailed summary report. Parameters @@ -492,8 +596,8 @@ def execute_pytest(capture='all', flags='-rapP'): """ fname = os.path.basename(__file__) - pytest.main(['-q', '--show-capture={}'.format(capture), fname, flags]) + pytest.main(["-q", "--show-capture={}".format(capture), fname, flags]) -if __name__ == '__main__': +if __name__ == "__main__": execute_pytest() From 1a1243389e110105bf601c95a15089ea7dbb43e8 Mon Sep 17 00:00:00 2001 From: bnb32 Date: Mon, 13 May 2024 08:07:44 -0600 Subject: [PATCH 10/61] fix: missing imports and long lines. --- examples/aws_pcluster/make_project_points.py | 1 + reV/bespoke/bespoke.py | 1049 +++++++++++------- reV/config/project_points.py | 294 +++-- reV/generation/generation.py | 462 +++++--- reV/nrwal/nrwal.py | 470 +++++--- reV/qa_qc/qa_qc.py | 2 +- reV/utilities/pytest_utils.py | 2 + tests/test_gen_pv.py | 829 +++++++++----- tests/test_nrwal.py | 2 +- tests/test_sam.py | 1 + tests/test_supply_curve_aggregation.py | 1 + tests/test_supply_curve_sc_aggregation.py | 2 +- 12 files changed, 1963 insertions(+), 1152 deletions(-) diff --git a/examples/aws_pcluster/make_project_points.py b/examples/aws_pcluster/make_project_points.py index 0c0e25c80..1f234c7ee 100644 --- a/examples/aws_pcluster/make_project_points.py +++ b/examples/aws_pcluster/make_project_points.py @@ -3,6 +3,7 @@ """ from rex import Resource +from reV.utilities import MetaKeyName if __name__ == '__main__': fp = '/nrel/nsrdb/v3/nsrdb_2019.h5' diff --git a/reV/bespoke/bespoke.py b/reV/bespoke/bespoke.py index e9a0d6d93..029d95673 100644 --- a/reV/bespoke/bespoke.py +++ b/reV/bespoke/bespoke.py @@ -2,6 +2,7 @@ """ reV bespoke wind plant analysis tools """ + # pylint: disable=anomalous-backslash-in-string import copy import json @@ -84,28 +85,38 @@ def _pre_load_data(self): hh = self.sc_gid_to_hh[sc_gid] self.hh_to_res_gids.setdefault(hh, set()).update(gids) - self.hh_to_res_gids = {hh: sorted(gids) - for hh, gids in self.hh_to_res_gids.items()} + self.hh_to_res_gids = { + hh: sorted(gids) for hh, gids in self.hh_to_res_gids.items() + } start_time = time.time() - if '*' in self.res_fpath: + if "*" in self.res_fpath: handler = MultiYearWindResource else: handler = WindResource with handler(self.res_fpath) as res: - self._wind_dirs = {hh: res[f"winddirection_{hh}m", :, gids] - for hh, gids in self.hh_to_res_gids.items()} - self._wind_speeds = {hh: res[f"windspeed_{hh}m", :, gids] - for hh, gids in self.hh_to_res_gids.items()} - self._temps = {hh: res[f"temperature_{hh}m", :, gids] - for hh, gids in self.hh_to_res_gids.items()} - self._pressures = {hh: res[f"pressure_{hh}m", :, gids] - for hh, gids in self.hh_to_res_gids.items()} + self._wind_dirs = { + hh: res[f"winddirection_{hh}m", :, gids] + for hh, gids in self.hh_to_res_gids.items() + } + self._wind_speeds = { + hh: res[f"windspeed_{hh}m", :, gids] + for hh, gids in self.hh_to_res_gids.items() + } + self._temps = { + hh: res[f"temperature_{hh}m", :, gids] + for hh, gids in self.hh_to_res_gids.items() + } + self._pressures = { + hh: res[f"pressure_{hh}m", :, gids] + for hh, gids in self.hh_to_res_gids.items() + } self._time_index = res.time_index - logger.debug(f"Data took {(time.time() - start_time) / 60:.2f} " - f"min to load") + logger.debug( + f"Data took {(time.time() - start_time) / 60:.2f} " f"min to load" + ) def get_preloaded_data_for_gid(self, sc_gid): """Get the pre-loaded data for a single SC GID. @@ -124,12 +135,14 @@ def get_preloaded_data_for_gid(self, sc_gid): hh = self.sc_gid_to_hh[sc_gid] sc_point_res_gids = sorted(self.sc_gid_to_res_gid[sc_gid]) data_inds = np.searchsorted(self.hh_to_res_gids[hh], sc_point_res_gids) - return BespokeSinglePlantData(sc_point_res_gids, - self._wind_dirs[hh][:, data_inds], - self._wind_speeds[hh][:, data_inds], - self._temps[hh][:, data_inds], - self._pressures[hh][:, data_inds], - self._time_index) + return BespokeSinglePlantData( + sc_point_res_gids, + self._wind_dirs[hh][:, data_inds], + self._wind_speeds[hh][:, data_inds], + self._temps[hh][:, data_inds], + self._pressures[hh][:, data_inds], + self._time_index, + ) class BespokeSinglePlantData: @@ -140,8 +153,9 @@ class BespokeSinglePlantData: reads to a single HDF5 file. """ - def __init__(self, data_inds, wind_dirs, wind_speeds, temps, pressures, - time_index): + def __init__( + self, data_inds, wind_dirs, wind_speeds, temps, pressures, time_index + ): """Initialize BespokeSinglePlantData Parameters @@ -189,20 +203,39 @@ class BespokeSinglePlant: the local wind resource and exclusions for a single reV supply curve point. """ - DEPENDENCIES = ('shapely',) + DEPENDENCIES = ("shapely",) OUT_ATTRS = copy.deepcopy(Gen.OUT_ATTRS) - def __init__(self, gid, excl, res, tm_dset, sam_sys_inputs, - objective_function, capital_cost_function, - fixed_operating_cost_function, - variable_operating_cost_function, - min_spacing='5x', wake_loss_multiplier=1, ga_kwargs=None, - output_request=('system_capacity', 'cf_mean'), - ws_bins=(0.0, 20.0, 5.0), wd_bins=(0.0, 360.0, 45.0), - excl_dict=None, inclusion_mask=None, data_layers=None, - resolution=64, excl_area=None, exclusion_shape=None, - eos_mult_baseline_cap_mw=200, prior_meta=None, gid_map=None, - bias_correct=None, pre_loaded_data=None, close=True): + def __init__( + self, + gid, + excl, + res, + tm_dset, + sam_sys_inputs, + objective_function, + capital_cost_function, + fixed_operating_cost_function, + variable_operating_cost_function, + min_spacing="5x", + wake_loss_multiplier=1, + ga_kwargs=None, + output_request=("system_capacity", "cf_mean"), + ws_bins=(0.0, 20.0, 5.0), + wd_bins=(0.0, 360.0, 45.0), + excl_dict=None, + inclusion_mask=None, + data_layers=None, + resolution=64, + excl_area=None, + exclusion_shape=None, + eos_mult_baseline_cap_mw=200, + prior_meta=None, + gid_map=None, + bias_correct=None, + pre_loaded_data=None, + close=True, + ): """ Parameters ---------- @@ -351,38 +384,48 @@ def __init__(self, gid, excl, res, tm_dset, sam_sys_inputs, Flag to close object file handlers on exit. """ - logger.debug('Initializing BespokeSinglePlant for gid {}...' - .format(gid)) - logger.debug('Resource filepath: {}'.format(res)) - logger.debug('Exclusion filepath: {}'.format(excl)) - logger.debug('Exclusion dict: {}'.format(excl_dict)) - logger.debug('Bespoke objective function: {}' - .format(objective_function)) - logger.debug('Bespoke cost function: {}'.format(objective_function)) - logger.debug('Bespoke wake loss multiplier: {}' - .format(wake_loss_multiplier)) - logger.debug('Bespoke GA initialization kwargs: {}'.format(ga_kwargs)) - logger.debug('Bespoke EOS multiplier baseline capacity: {:,} MW' - .format(eos_mult_baseline_cap_mw)) - - if isinstance(min_spacing, str) and min_spacing.endswith('x'): + logger.debug( + "Initializing BespokeSinglePlant for gid {}...".format(gid) + ) + logger.debug("Resource filepath: {}".format(res)) + logger.debug("Exclusion filepath: {}".format(excl)) + logger.debug("Exclusion dict: {}".format(excl_dict)) + logger.debug( + "Bespoke objective function: {}".format(objective_function) + ) + logger.debug("Bespoke cost function: {}".format(objective_function)) + logger.debug( + "Bespoke wake loss multiplier: {}".format(wake_loss_multiplier) + ) + logger.debug("Bespoke GA initialization kwargs: {}".format(ga_kwargs)) + logger.debug( + "Bespoke EOS multiplier baseline capacity: {:,} MW".format( + eos_mult_baseline_cap_mw + ) + ) + + if isinstance(min_spacing, str) and min_spacing.endswith("x"): rotor_diameter = sam_sys_inputs["wind_turbine_rotor_diameter"] - min_spacing = float(min_spacing.strip('x')) * rotor_diameter + min_spacing = float(min_spacing.strip("x")) * rotor_diameter if not isinstance(min_spacing, (int, float)): try: min_spacing = float(min_spacing) except Exception as e: - msg = ('min_spacing must be numeric but received: {}, {}' - .format(min_spacing, type(min_spacing))) + msg = ( + "min_spacing must be numeric but received: {}, {}".format( + min_spacing, type(min_spacing) + ) + ) logger.error(msg) raise TypeError(msg) from e self.objective_function = objective_function self.capital_cost_function = capital_cost_function self.fixed_operating_cost_function = fixed_operating_cost_function - self.variable_operating_cost_function = \ + self.variable_operating_cost_function = ( variable_operating_cost_function + ) self.min_spacing = min_spacing self.wake_loss_multiplier = wake_loss_multiplier self.ga_kwargs = ga_kwargs or {} @@ -410,26 +453,33 @@ def __init__(self, gid, excl, res, tm_dset, sam_sys_inputs, Handler = self.get_wind_handler(res) res = res if not isinstance(res, str) else Handler(res) - self._sc_point = AggSCPoint(gid, excl, res, tm_dset, - excl_dict=excl_dict, - inclusion_mask=inclusion_mask, - resolution=resolution, - excl_area=excl_area, - exclusion_shape=exclusion_shape, - close=close) + self._sc_point = AggSCPoint( + gid, + excl, + res, + tm_dset, + excl_dict=excl_dict, + inclusion_mask=inclusion_mask, + resolution=resolution, + excl_area=excl_area, + exclusion_shape=exclusion_shape, + close=close, + ) self._parse_output_req() self._data_layers = data_layers self._parse_prior_run() def __str__(self): - s = ('BespokeSinglePlant for reV SC gid {} with resolution {}' - .format(self.sc_point.gid, self.sc_point.resolution)) + s = "BespokeSinglePlant for reV SC gid {} with resolution {}".format( + self.sc_point.gid, self.sc_point.resolution + ) return s def __repr__(self): - s = ('BespokeSinglePlant for reV SC gid {} with resolution {}' - .format(self.sc_point.gid, self.sc_point.resolution)) + s = "BespokeSinglePlant for reV SC gid {} with resolution {}".format( + self.sc_point.gid, self.sc_point.resolution + ) return s def __enter__(self): @@ -446,14 +496,14 @@ def _parse_output_req(self): (ws_mean, *_mean) if requested. """ - required = ('cf_mean', 'annual_energy') + required = ("cf_mean", "annual_energy") for req in required: if req not in self._out_req: self._out_req.append(req) - if 'ws_mean' in self._out_req: - self._out_req.remove('ws_mean') - self._outputs['ws_mean'] = self.res_df['windspeed'].mean() + if "ws_mean" in self._out_req: + self._out_req.remove("ws_mean") + self._outputs["ws_mean"] = self.res_df["windspeed"].mean() for req in copy.deepcopy(self._out_req): if req in self.res_df: @@ -462,17 +512,21 @@ def _parse_output_req(self): year = annual_ti.year[0] mask = self.res_df.index.isin(annual_ti) arr = self.res_df.loc[mask, req].values.flatten() - self._outputs[req + f'-{year}'] = arr + self._outputs[req + f"-{year}"] = arr - elif req.replace('_mean', '') in self.res_df: + elif req.replace("_mean", "") in self.res_df: self._out_req.remove(req) - dset = req.replace('_mean', '') + dset = req.replace("_mean", "") self._outputs[req] = self.res_df[dset].mean() - if ('lcoe_fcr' in self._out_req - and 'fixed_charge_rate' not in self.original_sam_sys_inputs): - msg = ('User requested "lcoe_fcr" but did not input ' - '"fixed_charge_rate" in the SAM system config.') + if ( + "lcoe_fcr" in self._out_req + and "fixed_charge_rate" not in self.original_sam_sys_inputs + ): + msg = ( + 'User requested "lcoe_fcr" but did not input ' + '"fixed_charge_rate" in the SAM system config.' + ) logger.error(msg) raise KeyError(msg) @@ -481,14 +535,18 @@ def _parse_prior_run(self): sure the SAM system inputs are set accordingly.""" # {meta_column: sam_sys_input_key} - required = {MetaKeyName.CAPACITY: 'system_capacity', - MetaKeyName.TURBINE_X_COORDS: 'wind_farm_xCoordinates', - MetaKeyName.TURBINE_Y_COORDS: 'wind_farm_yCoordinates'} + required = { + MetaKeyName.CAPACITY: "system_capacity", + MetaKeyName.TURBINE_X_COORDS: "wind_farm_xCoordinates", + MetaKeyName.TURBINE_Y_COORDS: "wind_farm_yCoordinates", + } if self._prior_meta: missing = [k for k in required if k not in self.meta] - msg = ('Prior bespoke run meta data is missing the following ' - 'required columns: {}'.format(missing)) + msg = ( + "Prior bespoke run meta data is missing the following " + "required columns: {}".format(missing) + ) assert not any(missing), msg for meta_col, sam_sys_key in required.items(): @@ -496,7 +554,7 @@ def _parse_prior_run(self): self._sam_sys_inputs[sam_sys_key] = prior_value # convert reV supply curve cap in MW to SAM capacity in kW - self._sam_sys_inputs['system_capacity'] *= 1e3 + self._sam_sys_inputs["system_capacity"] *= 1e3 @staticmethod def _parse_gid_map(gid_map): @@ -521,14 +579,18 @@ def _parse_gid_map(gid_map): """ if isinstance(gid_map, str): - if gid_map.endswith('.csv'): + if gid_map.endswith(".csv"): gid_map = pd.read_csv(gid_map).to_dict() - assert MetaKeyName.GID in gid_map, 'Need "gid" in gid_map column' - assert 'gid_map' in gid_map, 'Need "gid_map" in gid_map column' - gid_map = {gid_map[MetaKeyName.GID][i]: gid_map['gid_map'][i] - for i in gid_map[MetaKeyName.GID].keys()} + assert ( + MetaKeyName.GID in gid_map + ), 'Need "gid" in gid_map column' + assert "gid_map" in gid_map, 'Need "gid_map" in gid_map column' + gid_map = { + gid_map[MetaKeyName.GID][i]: gid_map["gid_map"][i] + for i in gid_map[MetaKeyName.GID].keys() + } - elif gid_map.endswith('.json'): + elif gid_map.endswith(".json"): with open(gid_map) as f: gid_map = json.load(f) @@ -562,19 +624,23 @@ def bias_correct_ws(self, ws, dset, h5_gids): Bias corrected windspeed data in same shape as input """ - if self._bias_correct is not None and dset.startswith('windspeed_'): - + if self._bias_correct is not None and dset.startswith("windspeed_"): out = parse_bc_table(self._bias_correct, h5_gids) bc_fun, bc_fun_kwargs, bool_bc = out if bool_bc.any(): - logger.debug('Bias correcting windspeed with function {} ' - 'for h5 gids: {}'.format(bc_fun, h5_gids)) + logger.debug( + "Bias correcting windspeed with function {} " + "for h5 gids: {}".format(bc_fun, h5_gids) + ) - bc_fun_kwargs['ws'] = ws[:, bool_bc] + bc_fun_kwargs["ws"] = ws[:, bool_bc] sig = signature(bc_fun) - bc_fun_kwargs = {k: v for k, v in bc_fun_kwargs.items() - if k in sig.parameters} + bc_fun_kwargs = { + k: v + for k, v in bc_fun_kwargs.items() + if k in sig.parameters + } ws[:, bool_bc] = bc_fun(**bc_fun_kwargs) @@ -630,7 +696,7 @@ def get_weighted_res_dir(self): of degrees from north. """ - dset = f'winddirection_{self.hub_height}m' + dset = f"winddirection_{self.hub_height}m" gids = self.sc_point.h5_gid_set h5_gids = copy.deepcopy(gids) if self._gid_map is not None: @@ -735,31 +801,36 @@ def meta(self): """ if self._meta is None: res_gids = json.dumps([int(g) for g in self.sc_point.h5_gid_set]) - gid_counts = json.dumps([float(np.round(n, 1)) - for n in self.sc_point.gid_counts]) + gid_counts = json.dumps( + [float(np.round(n, 1)) for n in self.sc_point.gid_counts] + ) - with SupplyCurveExtent(self.sc_point._excl_fpath, - resolution=self.sc_point.resolution) as sc: + with SupplyCurveExtent( + self.sc_point._excl_fpath, resolution=self.sc_point.resolution + ) as sc: row_ind, col_ind = sc.get_sc_row_col_ind(self.sc_point.gid) self._meta = pd.DataFrame( - {MetaKeyName.SC_POINT_GID: self.sc_point.gid, - MetaKeyName.SC_ROW_IND: row_ind, - MetaKeyName.SC_COL_IND: col_ind, - MetaKeyName.GID: self.sc_point.gid, - MetaKeyName.LATITUDE: self.sc_point.latitude, - MetaKeyName.LONGITUDE: self.sc_point.longitude, - MetaKeyName.TIMEZONE: self.sc_point.timezone, - 'country': self.sc_point.country, - 'state': self.sc_point.state, - 'county': self.sc_point.county, - MetaKeyName.ELEVATION: self.sc_point.elevation, - MetaKeyName.OFFSHORE: self.sc_point.offshore, - MetaKeyName.RES_GIDS: res_gids, - MetaKeyName.GID_COUNTS: gid_counts, - MetaKeyName.N_GIDS: self.sc_point.n_gids, - MetaKeyName.AREA_SQ_KM: self.sc_point.area, - }, index=[self.sc_point.gid]) + { + MetaKeyName.SC_POINT_GID: self.sc_point.gid, + MetaKeyName.SC_ROW_IND: row_ind, + MetaKeyName.SC_COL_IND: col_ind, + MetaKeyName.GID: self.sc_point.gid, + MetaKeyName.LATITUDE: self.sc_point.latitude, + MetaKeyName.LONGITUDE: self.sc_point.longitude, + MetaKeyName.TIMEZONE: self.sc_point.timezone, + "country": self.sc_point.country, + "state": self.sc_point.state, + "county": self.sc_point.county, + MetaKeyName.ELEVATION: self.sc_point.elevation, + MetaKeyName.OFFSHORE: self.sc_point.offshore, + MetaKeyName.RES_GIDS: res_gids, + MetaKeyName.GID_COUNTS: gid_counts, + MetaKeyName.N_GIDS: self.sc_point.n_gids, + MetaKeyName.AREA_SQ_KM: self.sc_point.area, + }, + index=[self.sc_point.gid], + ) return self._meta @@ -771,7 +842,7 @@ def hub_height(self): ------- int """ - return int(self.sam_sys_inputs['wind_turbine_hub_ht']) + return int(self.sam_sys_inputs["wind_turbine_hub_ht"]) @property def res_df(self): @@ -791,21 +862,26 @@ def res_df(self): ti = self._pre_loaded_data.time_index wd = self.get_weighted_res_dir() - ws = self.get_weighted_res_ts(f'windspeed_{self.hub_height}m') - temp = self.get_weighted_res_ts(f'temperature_{self.hub_height}m') - pres = self.get_weighted_res_ts(f'pressure_{self.hub_height}m') + ws = self.get_weighted_res_ts(f"windspeed_{self.hub_height}m") + temp = self.get_weighted_res_ts(f"temperature_{self.hub_height}m") + pres = self.get_weighted_res_ts(f"pressure_{self.hub_height}m") # convert mbar to atm if np.nanmax(pres) > 1000: pres *= 9.86923e-6 - self._res_df = pd.DataFrame({'temperature': temp, - 'pressure': pres, - 'windspeed': ws, - 'winddirection': wd}, index=ti) - - if 'time_index_step' in self.original_sam_sys_inputs: - ti_step = self.original_sam_sys_inputs['time_index_step'] + self._res_df = pd.DataFrame( + { + "temperature": temp, + "pressure": pres, + "windspeed": ws, + "winddirection": wd, + }, + index=ti, + ) + + if "time_index_step" in self.original_sam_sys_inputs: + ti_step = self.original_sam_sys_inputs["time_index_step"] self._res_df = self._res_df.iloc[::ti_step] return self._res_df @@ -856,9 +932,11 @@ def wind_dist(self): ws_bins = JointPD._make_bins(*self._ws_bins) wd_bins = JointPD._make_bins(*self._wd_bins) - hist_out = np.histogram2d(self.res_df['windspeed'], - self.res_df['winddirection'], - bins=(ws_bins, wd_bins)) + hist_out = np.histogram2d( + self.res_df["windspeed"], + self.res_df["winddirection"], + bins=(ws_bins, wd_bins), + ) self._wind_dist, self._ws_edges, self._wd_edges = hist_out self._wind_dist /= self._wind_dist.sum() @@ -879,12 +957,13 @@ def initialize_wind_plant_ts(self): res_df = self.res_df[(self.res_df.index.year == year)] sam_inputs = copy.deepcopy(self.sam_sys_inputs) - if 'lcoe_fcr' in self._out_req: + if "lcoe_fcr" in self._out_req: lcoe_kwargs = self.get_lcoe_kwargs() sam_inputs.update(lcoe_kwargs) - i_wp = WindPower(res_df, self.meta, sam_inputs, - output_request=self._out_req) + i_wp = WindPower( + res_df, self.meta, sam_inputs, output_request=self._out_req + ) wind_plant_ts[year] = i_wp return wind_plant_ts @@ -901,9 +980,14 @@ def wind_plant_pd(self): if self._wind_plant_pd is None: wind_dist, ws_edges, wd_edges = self.wind_dist - self._wind_plant_pd = WindPowerPD(ws_edges, wd_edges, wind_dist, - self.meta, self.sam_sys_inputs, - output_request=self._out_req) + self._wind_plant_pd = WindPowerPD( + ws_edges, + wd_edges, + wind_dist, + self.meta, + self.sam_sys_inputs, + output_request=self._out_req, + ) return self._wind_plant_pd @property @@ -928,6 +1012,7 @@ def plant_optimizer(self): if self._plant_optm is None: # put import here to delay breaking due to special dependencies from reV.bespoke.place_turbines import PlaceTurbines + self._plant_optm = PlaceTurbines( self.wind_plant_pd, self.objective_function, @@ -937,7 +1022,8 @@ def plant_optimizer(self): self.include_mask, self.pixel_side_length, self.min_spacing, - self.wake_loss_multiplier) + self.wake_loss_multiplier, + ) return self._plant_optm @@ -945,21 +1031,23 @@ def recalc_lcoe(self): """Recalculate the multi-year mean LCOE based on the multi-year mean annual energy production (AEP)""" - if 'lcoe_fcr-means' in self.outputs: + if "lcoe_fcr-means" in self.outputs: lcoe_kwargs = self.get_lcoe_kwargs() - logger.debug('Recalulating multi-year mean LCOE using ' - 'multi-year mean AEP.') + logger.debug( + "Recalulating multi-year mean LCOE using " + "multi-year mean AEP." + ) - fcr = lcoe_kwargs['fixed_charge_rate'] - cap_cost = lcoe_kwargs['capital_cost'] - foc = lcoe_kwargs['fixed_operating_cost'] - voc = lcoe_kwargs['variable_operating_cost'] - aep = self.outputs['annual_energy-means'] + fcr = lcoe_kwargs["fixed_charge_rate"] + cap_cost = lcoe_kwargs["capital_cost"] + foc = lcoe_kwargs["fixed_operating_cost"] + voc = lcoe_kwargs["variable_operating_cost"] + aep = self.outputs["annual_energy-means"] my_mean_lcoe = lcoe_fcr(fcr, cap_cost, foc, aep, voc) - self._outputs['lcoe_fcr-means'] = my_mean_lcoe + self._outputs["lcoe_fcr-means"] = my_mean_lcoe self._meta[MetaKeyName.MEAN_LCOE] = my_mean_lcoe def get_lcoe_kwargs(self): @@ -978,8 +1066,13 @@ def get_lcoe_kwargs(self): original_sam_sys_inputs, meta """ - kwargs_list = ['fixed_charge_rate', 'system_capacity', 'capital_cost', - 'fixed_operating_cost', 'variable_operating_cost'] + kwargs_list = [ + "fixed_charge_rate", + "system_capacity", + "capital_cost", + "fixed_operating_cost", + "variable_operating_cost", + ] lcoe_kwargs = {} for kwarg in kwargs_list: @@ -998,9 +1091,12 @@ def get_lcoe_kwargs(self): missing = [k for k in kwargs_list if k not in lcoe_kwargs] if any(missing): - msg = ('Could not find these LCOE kwargs in outputs, ' - 'plant_optimizer, original_sam_sys_inputs, or meta: {}' - .format(missing)) + msg = ( + "Could not find these LCOE kwargs in outputs, " + "plant_optimizer, original_sam_sys_inputs, or meta: {}".format( + missing + ) + ) logger.error(msg) raise KeyError(msg) @@ -1023,7 +1119,7 @@ def get_wind_handler(res): """ handler = res if isinstance(res, str): - if '*' in res: + if "*" in res: handler = MultiYearWindResource else: handler = WindResource @@ -1041,21 +1137,28 @@ def check_dependencies(cls): missing.append(name) if any(missing): - msg = ('The reV bespoke module depends on the following special ' - 'dependencies that were not found in the active ' - 'environment: {}'.format(missing)) + msg = ( + "The reV bespoke module depends on the following special " + "dependencies that were not found in the active " + "environment: {}".format(missing) + ) logger.error(msg) raise ModuleNotFoundError(msg) @staticmethod - def _check_sys_inputs(plant1, plant2, - ignore=('wind_resource_model_choice', - 'wind_resource_data', - 'wind_turbine_powercurve_powerout', - 'hourly', - 'capital_cost', - 'fixed_operating_cost', - 'variable_operating_cost')): + def _check_sys_inputs( + plant1, + plant2, + ignore=( + "wind_resource_model_choice", + "wind_resource_data", + "wind_turbine_powercurve_powerout", + "hourly", + "capital_cost", + "fixed_operating_cost", + "variable_operating_cost", + ), + ): """Check two reV-SAM models for matching system inputs. Parameters @@ -1065,11 +1168,13 @@ def _check_sys_inputs(plant1, plant2, """ bad = [] for k, v in plant1.sam_sys_inputs.items(): - if k not in plant2.sam_sys_inputs or str(v) != str(plant2.sam_sys_inputs[k]): + if k not in plant2.sam_sys_inputs or str(v) != str( + plant2.sam_sys_inputs[k] + ): bad.append(k) bad = [b for b in bad if b not in ignore] if any(bad): - msg = 'Inputs no longer match: {}'.format(bad) + msg = "Inputs no longer match: {}".format(bad) logger.error(msg) raise RuntimeError(msg) @@ -1085,41 +1190,51 @@ def run_wind_plant_ts(self): BespokeSinglePlant.outputs property. """ - logger.debug('Running {} years of SAM timeseries analysis for {}' - .format(len(self.years), self)) + logger.debug( + "Running {} years of SAM timeseries analysis for {}".format( + len(self.years), self + ) + ) self._wind_plant_ts = self.initialize_wind_plant_ts() for year, plant in self.wind_plant_ts.items(): self._check_sys_inputs(plant, self.wind_plant_pd) try: plant.run_gen_and_econ() except Exception as e: - msg = ('{} failed while trying to run SAM WindPower ' - 'timeseries analysis for {}'.format(self, year)) + msg = ( + "{} failed while trying to run SAM WindPower " + "timeseries analysis for {}".format(self, year) + ) logger.exception(msg) raise RuntimeError(msg) from e for k, v in plant.outputs.items(): - self._outputs[k + '-{}'.format(year)] = v + self._outputs[k + "-{}".format(year)] = v means = {} for k1, v1 in self._outputs.items(): - if isinstance(v1, Number) and parse_year(k1, option='boolean'): + if isinstance(v1, Number) and parse_year(k1, option="boolean"): year = parse_year(k1) - base_str = k1.replace(str(year), '') - all_values = [v2 for k2, v2 in self._outputs.items() - if base_str in k2] - means[base_str + 'means'] = np.mean(all_values) + base_str = k1.replace(str(year), "") + all_values = [ + v2 for k2, v2 in self._outputs.items() if base_str in k2 + ] + means[base_str + "means"] = np.mean(all_values) self._outputs.update(means) # copy dataset outputs to meta data for supply curve table summary - if 'cf_mean-means' in self.outputs: - self._meta.loc[:, MetaKeyName.MEAN_CF] = self.outputs['cf_mean-means'] - if 'lcoe_fcr-means' in self.outputs: - self._meta.loc[:, MetaKeyName.MEAN_LCOE] = self.outputs['lcoe_fcr-means'] + if "cf_mean-means" in self.outputs: + self._meta.loc[:, MetaKeyName.MEAN_CF] = self.outputs[ + "cf_mean-means" + ] + if "lcoe_fcr-means" in self.outputs: + self._meta.loc[:, MetaKeyName.MEAN_LCOE] = self.outputs[ + "lcoe_fcr-means" + ] self.recalc_lcoe() - logger.debug('Timeseries analysis complete!') + logger.debug("Timeseries analysis complete!") return self.outputs @@ -1135,13 +1250,14 @@ def run_plant_optimization(self): BespokeSinglePlant.outputs property. """ - logger.debug('Running plant layout optimization for {}'.format(self)) + logger.debug("Running plant layout optimization for {}".format(self)) try: self.plant_optimizer.place_turbines(**self.ga_kwargs) except Exception as e: - msg = ('{} failed while trying to run the ' - 'turbine placement optimizer' - .format(self)) + msg = ( + "{} failed while trying to run the " + "turbine placement optimizer".format(self) + ) logger.exception(msg) raise RuntimeError(msg) from e @@ -1165,56 +1281,70 @@ def run_plant_optimization(self): self._meta["possible_y_coords"] = pyc self._outputs["full_polygons"] = self.plant_optimizer.full_polygons - self._outputs["packing_polygons"] = \ + self._outputs["packing_polygons"] = ( self.plant_optimizer.packing_polygons + ) self._outputs["system_capacity"] = self.plant_optimizer.capacity self._meta["n_turbines"] = self.plant_optimizer.nturbs self._meta["bespoke_aep"] = self.plant_optimizer.aep self._meta["bespoke_objective"] = self.plant_optimizer.objective - self._meta["bespoke_capital_cost"] = \ - self.plant_optimizer.capital_cost - self._meta["bespoke_fixed_operating_cost"] = \ + self._meta["bespoke_capital_cost"] = self.plant_optimizer.capital_cost + self._meta["bespoke_fixed_operating_cost"] = ( self.plant_optimizer.fixed_operating_cost - self._meta["bespoke_variable_operating_cost"] = \ + ) + self._meta["bespoke_variable_operating_cost"] = ( self.plant_optimizer.variable_operating_cost + ) self._meta["included_area"] = self.plant_optimizer.area - self._meta["included_area_capacity_density"] = \ + self._meta["included_area_capacity_density"] = ( self.plant_optimizer.capacity_density - self._meta["convex_hull_area"] = \ - self.plant_optimizer.convex_hull_area - self._meta["convex_hull_capacity_density"] = \ + ) + self._meta["convex_hull_area"] = self.plant_optimizer.convex_hull_area + self._meta["convex_hull_capacity_density"] = ( self.plant_optimizer.convex_hull_capacity_density - self._meta["full_cell_capacity_density"] = \ + ) + self._meta["full_cell_capacity_density"] = ( self.plant_optimizer.full_cell_capacity_density + ) - logger.debug('Plant layout optimization complete!') + logger.debug("Plant layout optimization complete!") # copy dataset outputs to meta data for supply curve table summary # convert SAM system capacity in kW to reV supply curve cap in MW - self._meta[MetaKeyName.CAPACITY] = self.outputs['system_capacity'] / 1e3 + self._meta[MetaKeyName.CAPACITY] = ( + self.outputs["system_capacity"] / 1e3 + ) # add required ReEDS multipliers to meta baseline_cost = self.plant_optimizer.capital_cost_per_kw( - capacity_mw=self._baseline_cap_mw) - self._meta['eos_mult'] = (self.plant_optimizer.capital_cost - / self.plant_optimizer.capacity - / baseline_cost) - self._meta['reg_mult'] = (self.sam_sys_inputs - .get("capital_cost_multiplier", 1)) + capacity_mw=self._baseline_cap_mw + ) + self._meta["eos_mult"] = ( + self.plant_optimizer.capital_cost + / self.plant_optimizer.capacity + / baseline_cost + ) + self._meta["reg_mult"] = self.sam_sys_inputs.get( + "capital_cost_multiplier", 1 + ) return self.outputs def agg_data_layers(self): """Aggregate optional data layers if requested and save to self.meta""" if self._data_layers is not None: - logger.debug('Aggregating {} extra data layers.' - .format(len(self._data_layers))) + logger.debug( + "Aggregating {} extra data layers.".format( + len(self._data_layers) + ) + ) point_summary = self.meta.to_dict() - point_summary = self.sc_point.agg_data_layers(point_summary, - self._data_layers) + point_summary = self.sc_point.agg_data_layers( + point_summary, self._data_layers + ) self._meta = pd.DataFrame(point_summary) - logger.debug('Finished aggregating extra data layers.') + logger.debug("Finished aggregating extra data layers.") @property def outputs(self): @@ -1243,9 +1373,10 @@ def run(cls, *args, **kwargs): with cls(*args, **kwargs) as bsp: if bsp._prior_meta: - logger.debug('Skipping bespoke plant optimization for gid {}. ' - 'Received prior meta data for this point.' - .format(bsp.gid)) + logger.debug( + "Skipping bespoke plant optimization for gid {}. " + "Received prior meta data for this point.".format(bsp.gid) + ) else: _ = bsp.run_plant_optimization() @@ -1254,9 +1385,9 @@ def run(cls, *args, **kwargs): meta = bsp.meta out = bsp.outputs - out['meta'] = meta + out["meta"] = meta for year, ti in zip(bsp.years, bsp.annual_time_indexes): - out['time_index-{}'.format(year)] = ti + out["time_index-{}".format(year)] = ti return out @@ -1264,16 +1395,35 @@ def run(cls, *args, **kwargs): class BespokeWindPlants(BaseAggregation): """BespokeWindPlants""" - def __init__(self, excl_fpath, res_fpath, tm_dset, objective_function, - capital_cost_function, fixed_operating_cost_function, - variable_operating_cost_function, project_points, - sam_files, min_spacing='5x', wake_loss_multiplier=1, - ga_kwargs=None, output_request=('system_capacity', 'cf_mean'), - ws_bins=(0.0, 20.0, 5.0), wd_bins=(0.0, 360.0, 45.0), - excl_dict=None, area_filter_kernel='queen', min_area=None, - resolution=64, excl_area=None, data_layers=None, - pre_extract_inclusions=False, prior_run=None, gid_map=None, - bias_correct=None, pre_load_data=False): + def __init__( + self, + excl_fpath, + res_fpath, + tm_dset, + objective_function, + capital_cost_function, + fixed_operating_cost_function, + variable_operating_cost_function, + project_points, + sam_files, + min_spacing="5x", + wake_loss_multiplier=1, + ga_kwargs=None, + output_request=("system_capacity", "cf_mean"), + ws_bins=(0.0, 20.0, 5.0), + wd_bins=(0.0, 360.0, 45.0), + excl_dict=None, + area_filter_kernel="queen", + min_area=None, + resolution=64, + excl_area=None, + data_layers=None, + pre_extract_inclusions=False, + prior_run=None, + gid_map=None, + bias_correct=None, + pre_load_data=False, + ): r"""ReV bespoke analysis class. Much like generation, ``reV`` bespoke analysis runs SAM @@ -1682,39 +1832,58 @@ def __init__(self, excl_fpath, res_fpath, tm_dset, objective_function, """ log_versions(logger) - logger.info('Initializing BespokeWindPlants...') - logger.info('Resource filepath: {}'.format(res_fpath)) - logger.info('Exclusion filepath: {}'.format(excl_fpath)) - logger.debug('Exclusion dict: {}'.format(excl_dict)) - logger.info('Bespoke objective function: {}' - .format(objective_function)) - logger.info('Bespoke capital cost function: {}' - .format(capital_cost_function)) - logger.info('Bespoke fixed operating cost function: {}' - .format(fixed_operating_cost_function)) - logger.info('Bespoke variable operating cost function: {}' - .format(variable_operating_cost_function)) - logger.info('Bespoke wake loss multiplier: {}' - .format(wake_loss_multiplier)) - logger.info('Bespoke GA initialization kwargs: {}'.format(ga_kwargs)) - - logger.info('Bespoke pre-extracting exclusions: {}' - .format(pre_extract_inclusions)) - logger.info('Bespoke pre-extracting resource data: {}' - .format(pre_load_data)) - logger.info('Bespoke prior run: {}'.format(prior_run)) - logger.info('Bespoke GID map: {}'.format(gid_map)) - logger.info('Bespoke bias correction table: {}'.format(bias_correct)) + logger.info("Initializing BespokeWindPlants...") + logger.info("Resource filepath: {}".format(res_fpath)) + logger.info("Exclusion filepath: {}".format(excl_fpath)) + logger.debug("Exclusion dict: {}".format(excl_dict)) + logger.info( + "Bespoke objective function: {}".format(objective_function) + ) + logger.info( + "Bespoke capital cost function: {}".format(capital_cost_function) + ) + logger.info( + "Bespoke fixed operating cost function: {}".format( + fixed_operating_cost_function + ) + ) + logger.info( + "Bespoke variable operating cost function: {}".format( + variable_operating_cost_function + ) + ) + logger.info( + "Bespoke wake loss multiplier: {}".format(wake_loss_multiplier) + ) + logger.info("Bespoke GA initialization kwargs: {}".format(ga_kwargs)) + + logger.info( + "Bespoke pre-extracting exclusions: {}".format( + pre_extract_inclusions + ) + ) + logger.info( + "Bespoke pre-extracting resource data: {}".format(pre_load_data) + ) + logger.info("Bespoke prior run: {}".format(prior_run)) + logger.info("Bespoke GID map: {}".format(gid_map)) + logger.info("Bespoke bias correction table: {}".format(bias_correct)) BespokeSinglePlant.check_dependencies() self._project_points = self._parse_points(project_points, sam_files) - super().__init__(excl_fpath, tm_dset, excl_dict=excl_dict, - area_filter_kernel=area_filter_kernel, - min_area=min_area, resolution=resolution, - excl_area=excl_area, gids=self._project_points.gids, - pre_extract_inclusions=pre_extract_inclusions) + super().__init__( + excl_fpath, + tm_dset, + excl_dict=excl_dict, + area_filter_kernel=area_filter_kernel, + min_area=min_area, + resolution=resolution, + excl_area=excl_area, + gids=self._project_points.gids, + pre_extract_inclusions=pre_extract_inclusions, + ) self._res_fpath = res_fpath self._obj_fun = objective_function @@ -1739,8 +1908,11 @@ def __init__(self, excl_fpath, res_fpath, tm_dset, objective_function, self._slice_lookup = None - logger.info('Initialized BespokeWindPlants with project points: {}' - .format(self._project_points)) + logger.info( + "Initialized BespokeWindPlants with project points: {}".format( + self._project_points + ) + ) @staticmethod def _parse_points(points, sam_configs): @@ -1752,8 +1924,8 @@ def _parse_points(points, sam_configs): Slice or list specifying project points, string pointing to a project points csv, or a fully instantiated PointsControl object. Can also be a single site integer value. Points csv should have - MetaKeyName.GID and 'config' column, the config maps to the sam_configs dict - keys. + `MetaKeyName.GID` and 'config' column, the config maps to the + sam_configs dict keys. sam_configs : dict | str | SAMConfig SAM input configuration ID(s) and file path(s). Keys are the SAM config ID(s) which map to the config column in the project points @@ -1767,8 +1939,13 @@ def _parse_points(points, sam_configs): Project points object laying out the supply curve gids to analyze. """ - pc = Gen.get_pc(points, points_range=None, sam_configs=sam_configs, - tech='windpower', sites_per_worker=1) + pc = Gen.get_pc( + points, + points_range=None, + sam_configs=sam_configs, + tech="windpower", + sites_per_worker=1, + ) return pc.project_points @@ -1798,15 +1975,15 @@ def _parse_prior_run(prior_run): if prior_run is not None: assert os.path.isfile(prior_run) - assert prior_run.endswith('.h5') + assert prior_run.endswith(".h5") - with Outputs(prior_run, mode='r') as f: + with Outputs(prior_run, mode="r") as f: meta = f.meta # pylint: disable=no-member for col in meta.columns: val = meta[col].values[0] - if isinstance(val, str) and val[0] == '[' and val[-1] == ']': + if isinstance(val, str) and val[0] == "[" and val[-1] == "]": meta[col] = meta[col].apply(json.loads) return meta @@ -1843,14 +2020,19 @@ def _check_files(self): for path in paths: if not os.path.exists(path): raise FileNotFoundError( - 'Could not find required exclusions file: ' - '{}'.format(path)) + "Could not find required exclusions file: " "{}".format( + path + ) + ) with ExclusionLayers(paths) as excl: if self._tm_dset not in excl: - raise FileInputError('Could not find techmap dataset "{}" ' - 'in the exclusions file(s): {}' - .format(self._tm_dset, paths)) + raise FileInputError( + 'Could not find techmap dataset "{}" ' + "in the exclusions file(s): {}".format( + self._tm_dset, paths + ) + ) # just check that this file exists, cannot check res_fpath if *glob Handler = BespokeSinglePlant.get_wind_handler(self._res_fpath) @@ -1862,18 +2044,24 @@ def _pre_load_data(self, pre_load_data): if not pre_load_data: return - sc_gid_to_hh = {gid: self._hh_for_sc_gid(gid) - for gid in self._project_points.df["gid"]} + sc_gid_to_hh = { + gid: self._hh_for_sc_gid(gid) + for gid in self._project_points.df["gid"] + } with ExclusionLayers(self._excl_fpath) as excl: tm = excl[self._tm_dset] scp_kwargs = {"shape": self.shape, "resolution": self._resolution} - slices = {gid: SupplyCurvePoint.get_agg_slices(gid=gid, **scp_kwargs) - for gid in self._project_points.df["gid"]} + slices = { + gid: SupplyCurvePoint.get_agg_slices(gid=gid, **scp_kwargs) + for gid in self._project_points.df["gid"] + } - sc_gid_to_res_gid = {gid: sorted(set(tm[slx, sly].flatten())) - for gid, (slx, sly) in slices.items()} + sc_gid_to_res_gid = { + gid: sorted(set(tm[slx, sly].flatten())) + for gid, (slx, sly) in slices.items() + } for sc_gid, res_gids in sc_gid_to_res_gid.items(): if res_gids[0] < 0: @@ -1881,13 +2069,14 @@ def _pre_load_data(self, pre_load_data): if self._gid_map is not None: for sc_gid, res_gids in sc_gid_to_res_gid.items(): - sc_gid_to_res_gid[sc_gid] = sorted(self._gid_map[g] - for g in res_gids) + sc_gid_to_res_gid[sc_gid] = sorted( + self._gid_map[g] for g in res_gids + ) logger.info("Pre-loading resource data for Bespoke run... ") - self._pre_loaded_data = BespokeMultiPlantData(self._res_fpath, - sc_gid_to_hh, - sc_gid_to_res_gid) + self._pre_loaded_data = BespokeMultiPlantData( + self._res_fpath, sc_gid_to_hh, sc_gid_to_res_gid + ) def _hh_for_sc_gid(self, sc_gid): """Fetch the hh for a given sc_gid""" @@ -1923,9 +2112,12 @@ def _get_bc_for_gid(self, gid): if self._bias_correct is not None: h5_gids = [] try: - scp_kwargs = dict(gid=gid, excl=self._excl_fpath, - tm_dset=self._tm_dset, - resolution=self._resolution) + scp_kwargs = dict( + gid=gid, + excl=self._excl_fpath, + tm_dset=self._tm_dset, + resolution=self._resolution, + ) with SupplyCurvePoint(**scp_kwargs) as scp: h5_gids = scp.h5_gid_set except EmptySupplyCurvePointError: @@ -1969,7 +2161,7 @@ def meta(self): ------- pd.DataFrame """ - meta = [self.outputs[g]['meta'] for g in self.completed_gids] + meta = [self.outputs[g]["meta"] for g in self.completed_gids] if len(self.completed_gids) > 1: meta = pd.concat(meta, axis=0) else: @@ -1980,8 +2172,9 @@ def meta(self): def slice_lookup(self): """Dict | None: Lookup mapping sc_point_gid to exclusion slice.""" if self._slice_lookup is None and self._inclusion_mask is not None: - with SupplyCurveExtent(self._excl_fpath, - resolution=self._resolution) as sc: + with SupplyCurveExtent( + self._excl_fpath, resolution=self._resolution + ) as sc: assert self.shape == self._inclusion_mask.shape self._slice_lookup = sc.get_slice_lookup(self.gids) @@ -2010,8 +2203,13 @@ def sam_sys_inputs_with_site_data(self, gid): site_data = self._project_points.df.iloc[gid_idx] site_sys_inputs = self._project_points[gid][1] - site_sys_inputs.update({k: v for k, v in site_data.to_dict().items() - if not (isinstance(v, float) and np.isnan(v))}) + site_sys_inputs.update( + { + k: v + for k, v in site_data.to_dict().items() + if not (isinstance(v, float) and np.isnan(v)) + } + ) return site_sys_inputs def _init_fout(self, out_fpath, sample): @@ -2030,13 +2228,14 @@ def _init_fout(self, out_fpath, sample): if not os.path.exists(out_dir): create_dirs(out_dir) - with Outputs(out_fpath, mode='w') as f: - f._set_meta('meta', self.meta, attrs={}) - ti_dsets = [d for d in sample.keys() - if d.startswith('time_index-')] + with Outputs(out_fpath, mode="w") as f: + f._set_meta("meta", self.meta, attrs={}) + ti_dsets = [ + d for d in sample.keys() if d.startswith("time_index-") + ] for dset in ti_dsets: f._set_time_index(dset, sample[dset], attrs={}) - f._set_time_index('time_index', sample[dset], attrs={}) + f._set_time_index("time_index", sample[dset], attrs={}) def _collect_out_arr(self, dset, sample): """Collect single-plant data arrays into complete arrays with data from @@ -2067,8 +2266,9 @@ def _collect_out_arr(self, dset, sample): shape = (len(single_arr), len(self.completed_gids)) sample_num = single_arr[0] else: - msg = ('Not writing dataset "{}" of type "{}" to disk.' - .format(dset, type(single_arr))) + msg = 'Not writing dataset "{}" of type "{}" to disk.'.format( + dset, type(single_arr) + ) logger.info(msg) return None @@ -2079,8 +2279,9 @@ def _collect_out_arr(self, dset, sample): full_arr = np.zeros(shape, dtype=dtype) # collect data from all wind plants - logger.info('Collecting dataset "{}" with final shape {}' - .format(dset, shape)) + logger.info( + 'Collecting dataset "{}" with final shape {}'.format(dset, shape) + ) for i, gid in enumerate(self.completed_gids): if len(full_arr.shape) == 1: full_arr[i] = self.outputs[gid][dset] @@ -2104,16 +2305,18 @@ def save_outputs(self, out_fpath): Full filepath to desired .h5 output file, the .h5 extension has been added if it was not already present. """ - if not out_fpath.endswith('.h5'): - out_fpath += '.h5' + if not out_fpath.endswith(".h5"): + out_fpath += ".h5" if ModuleName.BESPOKE not in out_fpath: extension_with_module = "_{}.h5".format(ModuleName.BESPOKE) out_fpath = out_fpath.replace(".h5", extension_with_module) if not self.completed_gids: - msg = ("No output data found! It is likely that all requested " - "points are excluded.") + msg = ( + "No output data found! It is likely that all requested " + "points are excluded." + ) logger.warning(msg) warn(msg) return out_fpath @@ -2121,49 +2324,69 @@ def save_outputs(self, out_fpath): sample = self.outputs[self.completed_gids[0]] self._init_fout(out_fpath, sample) - dsets = [d for d in sample.keys() - if not d.startswith('time_index-') - and d != 'meta'] - with Outputs(out_fpath, mode='a') as f: + dsets = [ + d + for d in sample.keys() + if not d.startswith("time_index-") and d != "meta" + ] + with Outputs(out_fpath, mode="a") as f: for dset in dsets: full_arr = self._collect_out_arr(dset, sample) if full_arr is not None: dset_no_year = dset - if parse_year(dset, option='boolean'): + if parse_year(dset, option="boolean"): year = parse_year(dset) - dset_no_year = dset.replace('-{}'.format(year), '') + dset_no_year = dset.replace("-{}".format(year), "") attrs = BespokeSinglePlant.OUT_ATTRS.get(dset_no_year, {}) attrs = copy.deepcopy(attrs) - dtype = attrs.pop('dtype', np.float32) - chunks = attrs.pop('chunks', None) + dtype = attrs.pop("dtype", np.float32) + chunks = attrs.pop("chunks", None) try: - f.write_dataset(dset, full_arr, dtype, chunks=chunks, - attrs=attrs) + f.write_dataset( + dset, full_arr, dtype, chunks=chunks, attrs=attrs + ) except Exception as e: msg = 'Failed to write "{}" to disk.'.format(dset) logger.exception(msg) raise OSError(msg) from e - logger.info('Saved output data to: {}'.format(out_fpath)) + logger.info("Saved output data to: {}".format(out_fpath)) return out_fpath # pylint: disable=arguments-renamed @classmethod - def run_serial(cls, excl_fpath, res_fpath, tm_dset, - sam_sys_inputs, objective_function, - capital_cost_function, - fixed_operating_cost_function, - variable_operating_cost_function, - min_spacing='5x', wake_loss_multiplier=1, ga_kwargs=None, - output_request=('system_capacity', 'cf_mean'), - ws_bins=(0.0, 20.0, 5.0), wd_bins=(0.0, 360.0, 45.0), - excl_dict=None, inclusion_mask=None, - area_filter_kernel='queen', min_area=None, - resolution=64, excl_area=0.0081, data_layers=None, - gids=None, exclusion_shape=None, slice_lookup=None, - prior_meta=None, gid_map=None, bias_correct=None, - pre_loaded_data=None): + def run_serial( + cls, + excl_fpath, + res_fpath, + tm_dset, + sam_sys_inputs, + objective_function, + capital_cost_function, + fixed_operating_cost_function, + variable_operating_cost_function, + min_spacing="5x", + wake_loss_multiplier=1, + ga_kwargs=None, + output_request=("system_capacity", "cf_mean"), + ws_bins=(0.0, 20.0, 5.0), + wd_bins=(0.0, 360.0, 45.0), + excl_dict=None, + inclusion_mask=None, + area_filter_kernel="queen", + min_area=None, + resolution=64, + excl_area=0.0081, + data_layers=None, + gids=None, + exclusion_shape=None, + slice_lookup=None, + prior_meta=None, + gid_map=None, + bias_correct=None, + pre_loaded_data=None, + ): """ Standalone serial method to run bespoke optimization. See BespokeWindPlants docstring for parameter description. @@ -2192,18 +2415,19 @@ def run_serial(cls, excl_fpath, res_fpath, tm_dset, Handler = BespokeSinglePlant.get_wind_handler(res_fpath) # pre-extract handlers so they are not repeatedly initialized - file_kwargs = {'excl_dict': excl_dict, - 'area_filter_kernel': area_filter_kernel, - 'min_area': min_area, - 'h5_handler': Handler, - } + file_kwargs = { + "excl_dict": excl_dict, + "area_filter_kernel": area_filter_kernel, + "min_area": min_area, + "h5_handler": Handler, + } with AggFileHandler(excl_fpath, res_fpath, **file_kwargs) as fh: n_finished = 0 for gid in gids: gid_inclusions = cls._get_gid_inclusion_mask( - inclusion_mask, gid, slice_lookup, - resolution=resolution) + inclusion_mask, gid, slice_lookup, resolution=resolution + ) try: bsp_plant_out = BespokeSinglePlant.run( gid, @@ -2231,20 +2455,26 @@ def run_serial(cls, excl_fpath, res_fpath, tm_dset, gid_map=gid_map, bias_correct=bias_correct, pre_loaded_data=pre_loaded_data, - close=False) + close=False, + ) except EmptySupplyCurvePointError: - logger.debug('SC gid {} is fully excluded or does not ' - 'have any valid source data!'.format(gid)) + logger.debug( + "SC gid {} is fully excluded or does not " + "have any valid source data!".format(gid) + ) except Exception as e: - msg = 'SC gid {} failed!'.format(gid) + msg = "SC gid {} failed!".format(gid) logger.exception(msg) raise RuntimeError(msg) from e else: n_finished += 1 - logger.debug('Serial bespoke: ' - '{} out of {} points complete' - .format(n_finished, len(gids))) + logger.debug( + "Serial bespoke: " + "{} out of {} points complete".format( + n_finished, len(gids) + ) + ) log_mem(logger) out[gid] = bsp_plant_out @@ -2266,17 +2496,18 @@ def run_parallel(self, max_workers=None): Bespoke outputs keyed by sc point gid """ - logger.info('Running bespoke optimization for points {} through {} ' - 'at a resolution of {} on {} cores.' - .format(self.gids[0], self.gids[-1], self._resolution, - max_workers)) + logger.info( + "Running bespoke optimization for points {} through {} " + "at a resolution of {} on {} cores.".format( + self.gids[0], self.gids[-1], self._resolution, max_workers + ) + ) futures = [] out = {} n_finished = 0 - loggers = [__name__, 'reV.supply_curve.point_summary', 'reV'] + loggers = [__name__, "reV.supply_curve.point_summary", "reV"] with SpawnProcessPool(max_workers=max_workers, loggers=loggers) as exe: - # iterate through split executions, submitting each to worker for gid in self.gids: # submit executions and append to futures list @@ -2285,36 +2516,39 @@ def run_parallel(self, max_workers=None): rs, cs = self.slice_lookup[gid] gid_incl_mask = self._inclusion_mask[rs, cs] - futures.append(exe.submit( - self.run_serial, - self._excl_fpath, - self._res_fpath, - self._tm_dset, - self.sam_sys_inputs_with_site_data(gid), - self._obj_fun, - self._cap_cost_fun, - self._foc_fun, - self._voc_fun, - self._min_spacing, - wake_loss_multiplier=self._wake_loss_multiplier, - ga_kwargs=self._ga_kwargs, - output_request=self._output_request, - ws_bins=self._ws_bins, - wd_bins=self._wd_bins, - excl_dict=self._excl_dict, - inclusion_mask=gid_incl_mask, - area_filter_kernel=self._area_filter_kernel, - min_area=self._min_area, - resolution=self._resolution, - excl_area=self._excl_area, - data_layers=self._data_layers, - gids=gid, - exclusion_shape=self.shape, - slice_lookup=copy.deepcopy(self.slice_lookup), - prior_meta=self._get_prior_meta(gid), - gid_map=self._gid_map, - bias_correct=self._get_bc_for_gid(gid), - pre_loaded_data=self._pre_loaded_data_for_sc_gid(gid))) + futures.append( + exe.submit( + self.run_serial, + self._excl_fpath, + self._res_fpath, + self._tm_dset, + self.sam_sys_inputs_with_site_data(gid), + self._obj_fun, + self._cap_cost_fun, + self._foc_fun, + self._voc_fun, + self._min_spacing, + wake_loss_multiplier=self._wake_loss_multiplier, + ga_kwargs=self._ga_kwargs, + output_request=self._output_request, + ws_bins=self._ws_bins, + wd_bins=self._wd_bins, + excl_dict=self._excl_dict, + inclusion_mask=gid_incl_mask, + area_filter_kernel=self._area_filter_kernel, + min_area=self._min_area, + resolution=self._resolution, + excl_area=self._excl_area, + data_layers=self._data_layers, + gids=gid, + exclusion_shape=self.shape, + slice_lookup=copy.deepcopy(self.slice_lookup), + prior_meta=self._get_prior_meta(gid), + gid_map=self._gid_map, + bias_correct=self._get_bc_for_gid(gid), + pre_loaded_data=self._pre_loaded_data_for_sc_gid(gid), + ) + ) # gather results for future in as_completed(futures): @@ -2322,12 +2556,17 @@ def run_parallel(self, max_workers=None): out.update(future.result()) if n_finished % 10 == 0: mem = psutil.virtual_memory() - logger.info('Parallel bespoke futures collected: ' - '{} out of {}. Memory usage is {:.3f} GB out ' - 'of {:.3f} GB ({:.2f}% utilized).' - .format(n_finished, len(futures), - mem.used / 1e9, mem.total / 1e9, - 100 * mem.used / mem.total)) + logger.info( + "Parallel bespoke futures collected: " + "{} out of {}. Memory usage is {:.3f} GB out " + "of {:.3f} GB ({:.2f}% utilized).".format( + n_finished, + len(futures), + mem.used / 1e9, + mem.total / 1e9, + 100 * mem.used / mem.total, + ) + ) return out @@ -2353,7 +2592,7 @@ def run(self, out_fpath=None, max_workers=None): """ # parallel job distribution test. - if self._obj_fun == 'test': + if self._obj_fun == "test": return True if max_workers == 1: @@ -2371,33 +2610,35 @@ def run(self, out_fpath=None, max_workers=None): wlm = self._wake_loss_multiplier i_bc = self._get_bc_for_gid(gid) - si = self.run_serial(self._excl_fpath, - self._res_fpath, - self._tm_dset, - sam_inputs, - self._obj_fun, - self._cap_cost_fun, - self._foc_fun, - self._voc_fun, - min_spacing=self._min_spacing, - wake_loss_multiplier=wlm, - ga_kwargs=self._ga_kwargs, - output_request=self._output_request, - ws_bins=self._ws_bins, - wd_bins=self._wd_bins, - excl_dict=self._excl_dict, - inclusion_mask=gid_incl_mask, - area_filter_kernel=afk, - min_area=self._min_area, - resolution=self._resolution, - excl_area=self._excl_area, - data_layers=self._data_layers, - slice_lookup=slice_lookup, - prior_meta=prior_meta, - gid_map=self._gid_map, - bias_correct=i_bc, - gids=gid, - pre_loaded_data=pre_loaded_data) + si = self.run_serial( + self._excl_fpath, + self._res_fpath, + self._tm_dset, + sam_inputs, + self._obj_fun, + self._cap_cost_fun, + self._foc_fun, + self._voc_fun, + min_spacing=self._min_spacing, + wake_loss_multiplier=wlm, + ga_kwargs=self._ga_kwargs, + output_request=self._output_request, + ws_bins=self._ws_bins, + wd_bins=self._wd_bins, + excl_dict=self._excl_dict, + inclusion_mask=gid_incl_mask, + area_filter_kernel=afk, + min_area=self._min_area, + resolution=self._resolution, + excl_area=self._excl_area, + data_layers=self._data_layers, + slice_lookup=slice_lookup, + prior_meta=prior_meta, + gid_map=self._gid_map, + bias_correct=i_bc, + gids=gid, + pre_loaded_data=pre_loaded_data, + ) self._outputs.update(si) else: self._outputs = self.run_parallel(max_workers=max_workers) diff --git a/reV/config/project_points.py b/reV/config/project_points.py index 63b97f022..8385a2949 100644 --- a/reV/config/project_points.py +++ b/reV/config/project_points.py @@ -2,6 +2,7 @@ """ reV Project Points Configuration """ + import copy import logging import os @@ -50,9 +51,12 @@ def __iter__(self): last_site = 0 ilim = len(self.project_points) - logger.debug('PointsControl iterator initializing with sites ' - '{} through {}'.format(self.project_points.sites[0], - self.project_points.sites[-1])) + logger.debug( + "PointsControl iterator initializing with sites " + "{} through {}".format( + self.project_points.sites[0], self.project_points.sites[-1] + ) + ) # pre-initialize all iter objects while True: @@ -63,14 +67,19 @@ def __iter__(self): last_site = i1 - new = self.split(i0, i1, self.project_points, - sites_per_split=self.sites_per_split) + new = self.split( + i0, + i1, + self.project_points, + sites_per_split=self.sites_per_split, + ) new._split_range = [i0, i1] self._iter_list.append(new) - logger.debug('PointsControl stopped iteration at attempted ' - 'index of {}. Length of iterator is: {}' - .format(i1, len(self))) + logger.debug( + "PointsControl stopped iteration at attempted " + "index of {}. Length of iterator is: {}".format(i1, len(self)) + ) return self def __next__(self): @@ -89,17 +98,22 @@ def __next__(self): # No more points controllers left in initialized list raise StopIteration - logger.debug('PointsControl passing site project points ' - 'with indices {} to {} on iteration #{} ' - .format(next_pc.split_range[0], - next_pc.split_range[1], self._i)) + logger.debug( + "PointsControl passing site project points " + "with indices {} to {} on iteration #{} ".format( + next_pc.split_range[0], next_pc.split_range[1], self._i + ) + ) self._i += 1 return next_pc def __repr__(self): - msg = ("{} with {} sites from gid {} through {}" - .format(self.__class__.__name__, len(self.project_points), - self.sites[0], self.sites[-1])) + msg = "{} with {} sites from gid {} through {}".format( + self.__class__.__name__, + len(self.project_points), + self.sites[0], + self.sites[-1], + ) return msg def __len__(self): @@ -214,8 +228,9 @@ class ProjectPoints: >>> h_list = pp.h """ - def __init__(self, points, sam_configs, tech=None, res_file=None, - curtailment=None): + def __init__( + self, points, sam_configs, tech=None, res_file=None, curtailment=None + ): """ Parameters ---------- @@ -273,22 +288,25 @@ def __getitem__(self, site): names (keys) and values. """ - site_bool = (self.df[MetaKeyName.GID] == site) + site_bool = self.df[MetaKeyName.GID] == site try: - config_id = self.df.loc[site_bool, 'config'].values[0] + config_id = self.df.loc[site_bool, "config"].values[0] except (KeyError, IndexError) as ex: - msg = ('Site {} not found in this instance of ' - 'ProjectPoints. Available sites include: {}' - .format(site, self.sites)) + msg = ( + "Site {} not found in this instance of " + "ProjectPoints. Available sites include: {}".format( + site, self.sites + ) + ) logger.exception(msg) raise KeyError(msg) from ex return config_id, copy.deepcopy(self.sam_inputs[config_id]) def __repr__(self): - msg = ("{} with {} sites from gid {} through {}" - .format(self.__class__.__name__, len(self), - self.sites[0], self.sites[-1])) + msg = "{} with {} sites from gid {} through {}".format( + self.__class__.__name__, len(self), self.sites[0], self.sites[-1] + ) return msg def __len__(self): @@ -429,7 +447,7 @@ def tech(self): solarwaterheat, troughphysicalheat, lineardirectsteam) The string should be lower-cased with spaces and _ removed. """ - return 'windpower' if 'wind' in self._tech.lower() else self._tech + return "windpower" if "wind" in self._tech.lower() else self._tech @property def h(self): @@ -441,9 +459,9 @@ def h(self): Hub heights corresponding to each site, taken from the sam config for each site. This is None if the technology is not wind. """ - h_var = 'wind_turbine_hub_ht' + h_var = "wind_turbine_hub_ht" if self._h is None: - if 'wind' in self.tech: + if "wind" in self.tech: # wind technology, get a list of h values self._h = [self[site][1][h_var] for site in self.sites] @@ -460,9 +478,9 @@ def d(self): the sam config for each site. This is None if the technology is not geothermal. """ - d_var = 'resource_depth' + d_var = "resource_depth" if self._d is None: - if 'geothermal' in self.tech: + if "geothermal" in self.tech: if d_var in self.df: self._d = list(self.df[d_var]) else: @@ -489,8 +507,8 @@ def _parse_csv(fname): Parameters ---------- fname : str - Project points .csv file (with path). Must have MetaKeyName.GID and 'config' - column names. + Project points .csv file (with path). Must have `MetaKeyName.GID` + and 'config' column names. Returns ------- @@ -498,12 +516,13 @@ def _parse_csv(fname): DataFrame mapping sites (gids) to SAM technology (config) """ fname = fname.strip() - if fname.endswith('.csv'): + if fname.endswith(".csv"): df = pd.read_csv(fname) else: - raise ValueError('Config project points file must be ' - '.csv, but received: {}' - .format(fname)) + raise ValueError( + "Config project points file must be " + ".csv, but received: {}".format(fname) + ) return df @@ -526,7 +545,7 @@ def _parse_sites(points, res_file=None): df : pd.DataFrame DataFrame mapping sites (gids) to SAM technology (config) """ - df = pd.DataFrame(columns=[MetaKeyName.GID, 'config']) + df = pd.DataFrame(columns=[MetaKeyName.GID, "config"]) if isinstance(points, int): points = [points] if isinstance(points, (list, tuple, np.ndarray)): @@ -541,9 +560,11 @@ def _parse_sites(points, res_file=None): stop = points.stop if stop is None: if res_file is None: - raise ValueError('Must supply a resource file if ' - 'points is a slice of type ' - ' slice(*, None, *)') + raise ValueError( + "Must supply a resource file if " + "points is a slice of type " + " slice(*, None, *)" + ) multi_h5_res, _ = check_res_file(res_file) if multi_h5_res: @@ -553,11 +574,12 @@ def _parse_sites(points, res_file=None): df[MetaKeyName.GID] = list(range(*points.indices(stop))) else: - raise TypeError('Project Points sites needs to be set as a list, ' - 'tuple, or slice, but was set as: {}' - .format(type(points))) + raise TypeError( + "Project Points sites needs to be set as a list, " + "tuple, or slice, but was set as: {}".format(type(points)) + ) - df['config'] = None + df["config"] = None return df @@ -589,24 +611,27 @@ def _parse_points(cls, points, res_file=None): elif isinstance(points, pd.DataFrame): df = points else: - raise ValueError('Cannot parse Project points data from {}' - .format(type(points))) + raise ValueError( + "Cannot parse Project points data from {}".format(type(points)) + ) if MetaKeyName.GID not in df.columns: raise KeyError('Project points data must contain "gid" column.') # pylint: disable=no-member - if 'config' not in df.columns: + if "config" not in df.columns: df = cls._parse_sites(points["gid"].values, res_file=res_file) gids = df[MetaKeyName.GID].values if not np.array_equal(np.sort(gids), gids): - msg = ('WARNING: points are not in sequential order and will be ' - 'sorted! The original order is being preserved under ' - 'column "points_order"') + msg = ( + "WARNING: points are not in sequential order and will be " + "sorted! The original order is being preserved under " + 'column "points_order"' + ) logger.warning(msg) warn(msg) - df['points_order'] = df.index.values + df["points_order"] = df.index.values df = df.sort_values(MetaKeyName.GID).reset_index(drop=True) return df @@ -637,8 +662,9 @@ def _parse_sam_config(sam_config): elif isinstance(sam_config, str): config_dict = {sam_config: sam_config} else: - raise ValueError('Cannot parse SAM configs from {}' - .format(type(sam_config))) + raise ValueError( + "Cannot parse SAM configs from {}".format(type(sam_config)) + ) return SAMConfig(config_dict) @@ -673,10 +699,12 @@ def _parse_curtailment(curtailment_input): else: curtailment = None - warn('Curtailment inputs not recognized. Received curtailment ' - 'input of type: "{}". Expected None, dict, str, or ' - 'Curtailment object. Defaulting to no curtailment.', - ConfigWarning) + warn( + "Curtailment inputs not recognized. Received curtailment " + 'input of type: "{}". Expected None, dict, str, or ' + "Curtailment object. Defaulting to no curtailment.", + ConfigWarning, + ) return curtailment @@ -695,8 +723,10 @@ def index(self, gid): Row index of gid in the project points dataframe. """ if gid not in self._df[MetaKeyName.GID].values: - e = ('Requested resource gid {} is not present in the project ' - 'points dataframe. Cannot return row index.'.format(gid)) + e = ( + "Requested resource gid {} is not present in the project " + "points dataframe. Cannot return row index.".format(gid) + ) logger.error(e) raise ConfigError(e) @@ -710,21 +740,24 @@ def _check_points_config_mapping(self): (sam_config_obj) are compatible. Update as necessary or break """ # Extract unique config refences from project_points DataFrame - df_configs = self.df['config'].unique() + df_configs = self.df["config"].unique() sam_configs = self.sam_inputs # Checks to make sure that the same number of SAM config files # as references in project_points DataFrame if len(df_configs) > len(sam_configs): - msg = ('Points references {} configs while only ' - '{} SAM configs were provided!' - .format(len(df_configs), len(sam_configs))) + msg = ( + "Points references {} configs while only " + "{} SAM configs were provided!".format( + len(df_configs), len(sam_configs) + ) + ) logger.error(msg) raise ConfigError(msg) if len(df_configs) == 1 and df_configs[0] is None: - self._df['config'] = list(sam_configs)[0] - df_configs = self.df['config'].unique() + self._df["config"] = list(sam_configs)[0] + df_configs = self.df["config"].unique() # Check to see if config references in project_points DataFrame # are valid file paths, if compare with SAM configs @@ -736,17 +769,21 @@ def _check_points_config_mapping(self): elif config in sam_configs: configs[config] = sam_configs[config] else: - msg = ('{} does not map to a valid configuration file' - .format(config)) + msg = "{} does not map to a valid configuration file".format( + config + ) logger.error(msg) raise ConfigError(msg) # If configs has any keys that are not in sam_configs then # something really weird happened so raise an error. if any(set(configs) - set(sam_configs)): - msg = ('A wild config has appeared! Requested config keys for ' - 'ProjectPoints are {} and previous config keys are {}' - .format(list(configs), list(sam_configs))) + msg = ( + "A wild config has appeared! Requested config keys for " + "ProjectPoints are {} and previous config keys are {}".format( + list(configs), list(sam_configs) + ) + ) logger.error(msg) raise ConfigError(msg) @@ -770,8 +807,15 @@ def join_df(self, df2, key=MetaKeyName.GID): """ # ensure df2 doesnt have any duplicate columns for suffix reasons. df2_cols = [c for c in df2.columns if c not in self._df or c == key] - self._df = pd.merge(self._df, df2[df2_cols], how='left', left_on=MetaKeyName.GID, - right_on=key, copy=False, validate='1:1') + self._df = pd.merge( + self._df, + df2[df2_cols], + how="left", + left_on=MetaKeyName.GID, + right_on=key, + copy=False, + validate="1:1", + ) def get_sites_from_config(self, config): """Get a site list that corresponds to a config key. @@ -787,7 +831,9 @@ def get_sites_from_config(self, config): List of sites associated with the requested configuration ID. If the configuration ID is not recognized, an empty list is returned. """ - sites = self.df.loc[(self.df['config'] == config), MetaKeyName.GID].values + sites = self.df.loc[ + (self.df["config"] == config), MetaKeyName.GID + ].values return list(sites) @@ -820,32 +866,39 @@ def split(cls, i0, i1, project_points): # Extract DF subset with only index values between i0 and i1 n = len(project_points) if i0 > n or i1 > n: - raise ValueError('{} and {} must be within the range of ' - 'project_points (0 - {})'.format(i0, i1, n - 1)) + raise ValueError( + "{} and {} must be within the range of " + "project_points (0 - {})".format(i0, i1, n - 1) + ) points_df = project_points.df.iloc[i0:i1] # make a new instance of ProjectPoints with subset DF - sub = cls(points_df, - project_points.sam_config_obj, - project_points.tech, - curtailment=project_points.curtailment) + sub = cls( + points_df, + project_points.sam_config_obj, + project_points.tech, + curtailment=project_points.curtailment, + ) return sub @staticmethod def _parse_lat_lons(lat_lons): - msg = ('Expecting a pair or multiple pairs of latitude and ' - 'longitude coordinates!') + msg = ( + "Expecting a pair or multiple pairs of latitude and " + "longitude coordinates!" + ) if isinstance(lat_lons, str): lat_lons = parse_table(lat_lons) - cols = [c for c in lat_lons - if c.lower().startswith(('lat', 'lon'))] + cols = [ + c for c in lat_lons if c.lower().startswith(("lat", "lon")) + ] lat_lons = lat_lons[sorted(cols)].values elif isinstance(lat_lons, (list, tuple)): lat_lons = np.array(lat_lons) elif isinstance(lat_lons, (int, float)): - msg += ' Recieved a single coordinate value!' + msg += " Recieved a single coordinate value!" logger.error(msg) raise ValueError(msg) @@ -853,15 +906,16 @@ def _parse_lat_lons(lat_lons): lat_lons = np.expand_dims(lat_lons, axis=0) if lat_lons.shape[1] != 2: - msg += ' Received {} coordinate values!'.format(lat_lons.shape[1]) + msg += " Received {} coordinate values!".format(lat_lons.shape[1]) logger.error(msg) raise ValueError(msg) return lat_lons @classmethod - def lat_lon_coords(cls, lat_lons, res_file, sam_configs, tech=None, - curtailment=None): + def lat_lon_coords( + cls, lat_lons, res_file, sam_configs, tech=None, curtailment=None + ): """ Generate ProjectPoints for gids nearest to given latitude longitudes @@ -906,11 +960,13 @@ def lat_lon_coords(cls, lat_lons, res_file, sam_configs, tech=None, res_kwargs = {} else: res_cls = ResourceX - res_kwargs = {'hsds': hsds} + res_kwargs = {"hsds": hsds} - logger.info('Converting latitude longitude coordinates into nearest ' - 'ProjectPoints') - logger.debug('- (lat, lon) pairs:\n{}'.format(lat_lons)) + logger.info( + "Converting latitude longitude coordinates into nearest " + "ProjectPoints" + ) + logger.debug("- (lat, lon) pairs:\n{}".format(lat_lons)) with res_cls(res_file, **res_kwargs) as f: gids = f.lat_lon_gid(lat_lons) # pylint: disable=no-member @@ -918,28 +974,36 @@ def lat_lon_coords(cls, lat_lons, res_file, sam_configs, tech=None, gids = [gids] else: if len(gids) != len(np.unique(gids)): - uniques, pos, counts = np.unique(gids, return_counts=True, - return_inverse=True) + uniques, pos, counts = np.unique( + gids, return_counts=True, return_inverse=True + ) duplicates = {} for idx in np.where(counts > 1)[0]: duplicate_lat_lons = lat_lons[np.where(pos == idx)[0]] duplicates[uniques[idx]] = duplicate_lat_lons - msg = ('reV Cannot currently handle duplicate Resource gids! ' - 'The given latitude and longitudes map to the same ' - 'gids:\n{}'.format(duplicates)) + msg = ( + "reV Cannot currently handle duplicate Resource gids! " + "The given latitude and longitudes map to the same " + "gids:\n{}".format(duplicates) + ) logger.error(msg) raise RuntimeError(msg) gids = gids.tolist() - logger.debug('- Resource gids:\n{}'.format(gids)) + logger.debug("- Resource gids:\n{}".format(gids)) - pp = cls(gids, sam_configs, tech=tech, res_file=res_file, - curtailment=curtailment) + pp = cls( + gids, + sam_configs, + tech=tech, + res_file=res_file, + curtailment=curtailment, + ) - if 'points_order' in pp.df: - lat_lons = lat_lons[pp.df['points_order'].values] + if "points_order" in pp.df: + lat_lons = lat_lons[pp.df["points_order"].values] pp._df[MetaKeyName.LATITUDE] = lat_lons[:, 0] pp._df[MetaKeyName.LONGITUDE] = lat_lons[:, 1] @@ -947,8 +1011,9 @@ def lat_lon_coords(cls, lat_lons, res_file, sam_configs, tech=None, return pp @classmethod - def regions(cls, regions, res_file, sam_configs, tech=None, - curtailment=None): + def regions( + cls, regions, res_file, sam_configs, tech=None, curtailment=None + ): """ Generate ProjectPoints for gids nearest to given latitude longitudes @@ -992,28 +1057,35 @@ def regions(cls, regions, res_file, sam_configs, tech=None, else: res_cls = ResourceX - logger.info('Extracting ProjectPoints for desired regions') + logger.info("Extracting ProjectPoints for desired regions") points = [] with res_cls(res_file, hsds=hsds) as f: meta = f.meta for region, region_col in regions.items(): - logger.debug('- {}: {}'.format(region_col, region)) + logger.debug("- {}: {}".format(region_col, region)) # pylint: disable=no-member gids = f.region_gids(region, region_col=region_col) - logger.debug('- Resource gids:\n{}'.format(gids)) + logger.debug("- Resource gids:\n{}".format(gids)) if points: duplicates = np.intersect1d(gids, points).tolist() if duplicates: - msg = ('reV Cannot currently handle duplicate ' - 'Resource gids! The given regions containg the ' - 'same gids:\n{}'.format(duplicates)) + msg = ( + "reV Cannot currently handle duplicate " + "Resource gids! The given regions containg the " + "same gids:\n{}".format(duplicates) + ) logger.error(msg) raise RuntimeError(msg) points.extend(gids.tolist()) - pp = cls(points, sam_configs, tech=tech, res_file=res_file, - curtailment=curtailment) + pp = cls( + points, + sam_configs, + tech=tech, + res_file=res_file, + curtailment=curtailment, + ) meta = meta.loc[pp.sites] cols = list(set(regions.values())) diff --git a/reV/generation/generation.py b/reV/generation/generation.py index 344f8f079..4c0af7907 100644 --- a/reV/generation/generation.py +++ b/reV/generation/generation.py @@ -2,6 +2,7 @@ """ reV generation module. """ + import copy import json import logging @@ -10,42 +11,46 @@ import numpy as np import pandas as pd - -from reV.generation.base import BaseGen -from reV.SAM.generation import (Geothermal, - MhkWave, - LinearDirectSteam, - PvSamv1, - PvWattsv5, - PvWattsv7, - PvWattsv8, - SolarWaterHeat, - TcsMoltenSalt, - TroughPhysicalHeat, - WindPower) -from reV.utilities import ModuleName -from reV.utilities.exceptions import (ConfigError, - InputError, - ProjectPointsValueError) from rex.multi_file_resource import MultiFileResource from rex.multi_res_resource import MultiResolutionResource from rex.resource import Resource from rex.utilities.utilities import check_res_file +from reV.generation.base import BaseGen +from reV.SAM.generation import ( + Geothermal, + LinearDirectSteam, + MhkWave, + PvSamv1, + PvWattsv5, + PvWattsv7, + PvWattsv8, + SolarWaterHeat, + TcsMoltenSalt, + TroughPhysicalHeat, + WindPower, +) +from reV.utilities import MetaKeyName, ModuleName +from reV.utilities.exceptions import ( + ConfigError, + InputError, + ProjectPointsValueError, +) + logger = logging.getLogger(__name__) ATTR_DIR = os.path.dirname(os.path.realpath(__file__)) -ATTR_DIR = os.path.join(ATTR_DIR, 'output_attributes') -with open(os.path.join(ATTR_DIR, 'other.json'), 'r') as f: +ATTR_DIR = os.path.join(ATTR_DIR, "output_attributes") +with open(os.path.join(ATTR_DIR, "other.json")) as f: OTHER_ATTRS = json.load(f) -with open(os.path.join(ATTR_DIR, 'generation.json'), 'r') as f: +with open(os.path.join(ATTR_DIR, "generation.json")) as f: GEN_ATTRS = json.load(f) -with open(os.path.join(ATTR_DIR, 'linear_fresnel.json'), 'r') as f: +with open(os.path.join(ATTR_DIR, "linear_fresnel.json")) as f: LIN_ATTRS = json.load(f) -with open(os.path.join(ATTR_DIR, 'solar_water_heat.json'), 'r') as f: +with open(os.path.join(ATTR_DIR, "solar_water_heat.json")) as f: SWH_ATTRS = json.load(f) -with open(os.path.join(ATTR_DIR, 'trough_heat.json'), 'r') as f: +with open(os.path.join(ATTR_DIR, "trough_heat.json")) as f: TPPH_ATTRS = json.load(f) @@ -53,17 +58,19 @@ class Gen(BaseGen): """Gen""" # Mapping of reV technology strings to SAM generation objects - OPTIONS = {'geothermal': Geothermal, - 'lineardirectsteam': LinearDirectSteam, - 'mhkwave': MhkWave, - 'pvsamv1': PvSamv1, - 'pvwattsv5': PvWattsv5, - 'pvwattsv7': PvWattsv7, - 'pvwattsv8': PvWattsv8, - 'solarwaterheat': SolarWaterHeat, - 'tcsmoltensalt': TcsMoltenSalt, - 'troughphysicalheat': TroughPhysicalHeat, - 'windpower': WindPower} + OPTIONS = { + "geothermal": Geothermal, + "lineardirectsteam": LinearDirectSteam, + "mhkwave": MhkWave, + "pvsamv1": PvSamv1, + "pvwattsv5": PvWattsv5, + "pvwattsv7": PvWattsv7, + "pvwattsv8": PvWattsv8, + "solarwaterheat": SolarWaterHeat, + "tcsmoltensalt": TcsMoltenSalt, + "troughphysicalheat": TroughPhysicalHeat, + "windpower": WindPower, + } """reV technology options.""" @@ -76,13 +83,25 @@ class Gen(BaseGen): OUT_ATTRS.update(TPPH_ATTRS) OUT_ATTRS.update(BaseGen.ECON_ATTRS) - def __init__(self, technology, project_points, sam_files, resource_file, - low_res_resource_file=None, output_request=('cf_mean',), - site_data=None, curtailment=None, gid_map=None, - drop_leap=False, sites_per_worker=None, - memory_utilization_limit=0.4, scale_outputs=True, - write_mapped_gids=False, bias_correct=None): - """reV generation analysis class. + def __init__( + self, + technology, + project_points, + sam_files, + resource_file, + low_res_resource_file=None, + output_request=("cf_mean",), + site_data=None, + curtailment=None, + gid_map=None, + drop_leap=False, + sites_per_worker=None, + memory_utilization_limit=0.4, + scale_outputs=True, + write_mapped_gids=False, + bias_correct=None, + ): + """ReV generation analysis class. ``reV`` generation analysis runs SAM simulations by piping in renewable energy resource data (usually from the NSRDB or WTK), @@ -361,22 +380,33 @@ def __init__(self, technology, project_points, sam_files, resource_file, ``bias_correct`` table on a site-by-site basis. If ``None``, no corrections are applied. By default, ``None``. """ - pc = self.get_pc(points=project_points, points_range=None, - sam_configs=sam_files, tech=technology, - sites_per_worker=sites_per_worker, - res_file=resource_file, - curtailment=curtailment) - - super().__init__(pc, output_request, site_data=site_data, - drop_leap=drop_leap, - memory_utilization_limit=memory_utilization_limit, - scale_outputs=scale_outputs) + pc = self.get_pc( + points=project_points, + points_range=None, + sam_configs=sam_files, + tech=technology, + sites_per_worker=sites_per_worker, + res_file=resource_file, + curtailment=curtailment, + ) + + super().__init__( + pc, + output_request, + site_data=site_data, + drop_leap=drop_leap, + memory_utilization_limit=memory_utilization_limit, + scale_outputs=scale_outputs, + ) if self.tech not in self.OPTIONS: - msg = ('Requested technology "{}" is not available. ' - 'reV generation can analyze the following ' - 'SAM technologies: {}' - .format(self.tech, list(self.OPTIONS.keys()))) + msg = ( + 'Requested technology "{}" is not available. ' + "reV generation can analyze the following " + "SAM technologies: {}".format( + self.tech, list(self.OPTIONS.keys()) + ) + ) logger.error(msg) raise KeyError(msg) @@ -384,8 +414,8 @@ def __init__(self, technology, project_points, sam_files, resource_file, self._res_file = resource_file self._lr_res_file = low_res_resource_file self._sam_module = self.OPTIONS[self.tech] - self._run_attrs['sam_module'] = self._sam_module.MODULE - self._run_attrs['res_file'] = resource_file + self._run_attrs["sam_module"] = self._sam_module.MODULE + self._run_attrs["res_file"] = resource_file self._multi_h5_res, self._hsds = check_res_file(resource_file) self._gid_map = self._parse_gid_map(gid_map) @@ -424,11 +454,12 @@ def meta(self): Meta data df for sites in project points. Column names are meta data variables, rows are different sites. The row index does not indicate the site number if the project points are - non-sequential or do not start from 0, so a MetaKeyName.GID column is added. + non-sequential or do not start from 0, so a `MetaKeyName.GID` + column is added. """ if self._meta is None: res_cls = Resource - kwargs = {'hsds': self._hsds} + kwargs = {"hsds": self._hsds} if self._multi_h5_res: res_cls = MultiFileResource kwargs = {} @@ -438,25 +469,27 @@ def meta(self): res_gids = [self._gid_map[i] for i in res_gids] with res_cls(self.res_file, **kwargs) as res: - meta_len = res.shapes['meta'][0] + meta_len = res.shapes["meta"][0] if np.max(res_gids) > meta_len: - msg = ('ProjectPoints has a max site gid of {} which is ' - 'out of bounds for the meta data of len {} from ' - 'resource file: {}' - .format(np.max(res_gids), - meta_len, self.res_file)) + msg = ( + "ProjectPoints has a max site gid of {} which is " + "out of bounds for the meta data of len {} from " + "resource file: {}".format( + np.max(res_gids), meta_len, self.res_file + ) + ) logger.error(msg) raise ProjectPointsValueError(msg) - self._meta = res['meta', res_gids] + self._meta = res["meta", res_gids] self._meta.loc[:, MetaKeyName.GID] = res_gids if self.write_mapped_gids: self._meta.loc[:, MetaKeyName.GID] = self.project_points.sites self._meta.index = self.project_points.sites self._meta.index.name = MetaKeyName.GID - self._meta.loc[:, 'reV_tech'] = self.project_points.tech + self._meta.loc[:, "reV_tech"] = self.project_points.tech return self._meta @@ -472,7 +505,7 @@ def time_index(self): if self._time_index is None: if not self._multi_h5_res: res_cls = Resource - kwargs = {'hsds': self._hsds} + kwargs = {"hsds": self._hsds} else: res_cls = MultiFileResource kwargs = {} @@ -484,19 +517,22 @@ def time_index(self): step = self.project_points.sam_config_obj.time_index_step if downscale is not None: from rex.utilities.downscale import make_time_index + year = time_index.year[0] - ds_freq = downscale['frequency'] + ds_freq = downscale["frequency"] time_index = make_time_index(year, ds_freq) - logger.info('reV solar generation running with temporal ' - 'downscaling frequency "{}" with final ' - 'time_index length {}' - .format(ds_freq, len(time_index))) + logger.info( + "reV solar generation running with temporal " + 'downscaling frequency "{}" with final ' + "time_index length {}".format(ds_freq, len(time_index)) + ) elif step is not None: time_index = time_index[::step] time_index = self.handle_lifetime_index(time_index) - time_index = self.handle_leap_ti(time_index, - drop_leap=self._drop_leap) + time_index = self.handle_leap_ti( + time_index, drop_leap=self._drop_leap + ) self._time_index = time_index @@ -530,30 +566,32 @@ def handle_lifetime_index(self, ti): # Only one time index may be passed, check that lifetime periods match n_unique_periods = len(np.unique(lifetime_periods)) if n_unique_periods != 1: - msg = ('reV cannot handle multiple analysis_periods when ' - 'modeling with `system_use_lifetime_output` set ' - 'to 1. Found {} different analysis_periods in the SAM ' - 'configs'.format(n_unique_periods)) + msg = ( + "reV cannot handle multiple analysis_periods when " + "modeling with `system_use_lifetime_output` set " + "to 1. Found {} different analysis_periods in the SAM " + "configs".format(n_unique_periods) + ) logger.error(msg) raise ConfigError(msg) # Collect requested variables to check for lifetime compatibility array_vars = [ - var for var, attrs in GEN_ATTRS.items() - if attrs['type'] == 'array' + var for var, attrs in GEN_ATTRS.items() if attrs["type"] == "array" ] - valid_vars = ['gen_profile', 'cf_profile', 'cf_profile_ac'] + valid_vars = ["gen_profile", "cf_profile", "cf_profile_ac"] invalid_vars = set(array_vars) - set(valid_vars) - invalid_requests = [var for var in self.output_request - if var in invalid_vars] + invalid_requests = [ + var for var in self.output_request if var in invalid_vars + ] if invalid_requests: # SAM does not output full lifetime for all array variables msg = ( - 'reV can only handle the following output arrays ' - 'when modeling with `system_use_lifetime_output` set ' - 'to 1: {}. Try running without {}.'.format( - ', '.join(valid_vars), ', '.join(invalid_requests) + "reV can only handle the following output arrays " + "when modeling with `system_use_lifetime_output` set " + "to 1: {}. Try running without {}.".format( + ", ".join(valid_vars), ", ".join(invalid_requests) ) ) logger.error(msg) @@ -561,8 +599,10 @@ def handle_lifetime_index(self, ti): sam_meta = self.sam_metas[next(iter(self.sam_metas))] analysis_period = sam_meta["analysis_period"] - logger.info('reV generation running with a full system ' - 'life of {} years.'.format(analysis_period)) + logger.info( + "reV generation running with a full system " + "life of {} years.".format(analysis_period) + ) old_end = ti[-1] new_end = old_end + pd.DateOffset(years=analysis_period - 1) @@ -573,10 +613,18 @@ def handle_lifetime_index(self, ti): return ti @classmethod - def _run_single_worker(cls, points_control, tech=None, res_file=None, - lr_res_file=None, output_request=None, - scale_outputs=True, gid_map=None, nn_map=None, - bias_correct=None): + def _run_single_worker( + cls, + points_control, + tech=None, + res_file=None, + lr_res_file=None, + output_request=None, + scale_outputs=True, + gid_map=None, + nn_map=None, + bias_correct=None, + ): """Run a SAM generation analysis based on the points_control iterator. Parameters @@ -642,16 +690,19 @@ def _run_single_worker(cls, points_control, tech=None, res_file=None, # run generation method for specified technology try: out = cls.OPTIONS[tech].reV_run( - points_control, res_file, site_df, + points_control, + res_file, + site_df, lr_res_file=lr_res_file, output_request=output_request, - gid_map=gid_map, nn_map=nn_map, - bias_correct=bias_correct + gid_map=gid_map, + nn_map=nn_map, + bias_correct=bias_correct, ) except Exception as e: out = {} - logger.exception('Worker failed for PC: {}'.format(points_control)) + logger.exception("Worker failed for PC: {}".format(points_control)) raise e if scale_outputs: @@ -661,8 +712,8 @@ def _run_single_worker(cls, points_control, tech=None, res_file=None, # iterate through variable names in each site's output dict if k in cls.OUT_ATTRS: # get dtype and scale for output variable name - dtype = cls.OUT_ATTRS[k].get('dtype', 'float32') - scale_factor = cls.OUT_ATTRS[k].get('scale_factor', 1) + dtype = cls.OUT_ATTRS[k].get("dtype", "float32") + scale_factor = cls.OUT_ATTRS[k].get("scale_factor", 1) # apply scale factor and dtype out[site][k] *= scale_factor @@ -676,8 +727,9 @@ def _run_single_worker(cls, points_control, tech=None, res_file=None, out[site][k] = out[site][k].astype(dtype) else: # use numpy array conversion for scalar values - out[site][k] = np.array([out[site][k]], - dtype=dtype)[0] + out[site][k] = np.array( + [out[site][k]], dtype=dtype + )[0] return out @@ -701,44 +753,56 @@ def _parse_gid_map(self, gid_map): """ if isinstance(gid_map, str): - if gid_map.endswith('.csv'): + if gid_map.endswith(".csv"): gid_map = pd.read_csv(gid_map).to_dict() - assert MetaKeyName.GID in gid_map, 'Need "gid" in gid_map column' - assert 'gid_map' in gid_map, 'Need "gid_map" in gid_map column' - gid_map = {gid_map[MetaKeyName.GID][i]: gid_map['gid_map'][i] - for i in gid_map[MetaKeyName.GID].keys()} + m = 'Need "gid" in gid_map column' + assert MetaKeyName.GID in gid_map, m + assert "gid_map" in gid_map, 'Need "gid_map" in gid_map column' + gid_map = { + gid_map[MetaKeyName.GID][i]: gid_map["gid_map"][i] + for i in gid_map[MetaKeyName.GID].keys() + } - elif gid_map.endswith('.json'): - with open(gid_map, 'r') as f: + elif gid_map.endswith(".json"): + with open(gid_map) as f: gid_map = json.load(f) if isinstance(gid_map, dict): if not self._multi_h5_res: res_cls = Resource - kwargs = {'hsds': self._hsds} + kwargs = {"hsds": self._hsds} else: res_cls = MultiFileResource kwargs = {} with res_cls(self.res_file, **kwargs) as res: for gen_gid, res_gid in gid_map.items(): - msg1 = ('gid_map values must all be int but received ' - '{}: {}'.format(gen_gid, res_gid)) - msg2 = ('Could not find the gen_gid to res_gid mapping ' - '{}: {} in the resource meta data.' - .format(gen_gid, res_gid)) + msg1 = ( + "gid_map values must all be int but received " + "{}: {}".format(gen_gid, res_gid) + ) + msg2 = ( + "Could not find the gen_gid to res_gid mapping " + "{}: {} in the resource meta data.".format( + gen_gid, res_gid + ) + ) assert isinstance(gen_gid, int), msg1 assert isinstance(res_gid, int), msg1 assert res_gid in res.meta.index.values, msg2 for gen_gid in self.project_points.sites: - msg3 = ('Could not find the project points gid {} in the ' - 'gen_gid input of the gid_map.'.format(gen_gid)) + msg3 = ( + "Could not find the project points gid {} in the " + "gen_gid input of the gid_map.".format(gen_gid) + ) assert gen_gid in gid_map, msg3 elif gid_map is not None: - msg = ('Could not parse gid_map, must be None, dict, or path to ' - 'csv or json, but received: {}'.format(gid_map)) + msg = ( + "Could not parse gid_map, must be None, dict, or path to " + "csv or json, but received: {}".format(gid_map) + ) logger.error(msg) raise InputError(msg) @@ -758,24 +822,31 @@ def _parse_nn_map(self): """ nn_map = None if self.lr_res_file is not None: - handler_class = Resource - if '*' in self.res_file or '*' in self.lr_res_file: + if "*" in self.res_file or "*" in self.lr_res_file: handler_class = MultiFileResource with handler_class(self.res_file) as hr_res: with handler_class(self.lr_res_file) as lr_res: - logger.info('Making nearest neighbor map for multi ' - 'resolution resource data...') - nn_d, nn_map = MultiResolutionResource.make_nn_map(hr_res, - lr_res) - logger.info('Done making nearest neighbor map for multi ' - 'resolution resource data!') - - logger.info('Made nearest neighbor mapping between nominal-' - 'resolution and low-resolution resource files. ' - 'Min / mean / max dist: {:.3f} / {:.3f} / {:.3f}' - .format(nn_d.min(), nn_d.mean(), nn_d.max())) + logger.info( + "Making nearest neighbor map for multi " + "resolution resource data..." + ) + nn_d, nn_map = MultiResolutionResource.make_nn_map( + hr_res, lr_res + ) + logger.info( + "Done making nearest neighbor map for multi " + "resolution resource data!" + ) + + logger.info( + "Made nearest neighbor mapping between nominal-" + "resolution and low-resolution resource files. " + "Min / mean / max dist: {:.3f} / {:.3f} / {:.3f}".format( + nn_d.min(), nn_d.mean(), nn_d.max() + ) + ) return nn_map @@ -829,23 +900,32 @@ def _parse_bc(bias_correct): if isinstance(bias_correct, type(None)): return bias_correct - elif isinstance(bias_correct, str): + if isinstance(bias_correct, str): bias_correct = pd.read_csv(bias_correct) - msg = ('Bias correction data must be a filepath to csv or a dataframe ' - 'but received: {}'.format(type(bias_correct))) + msg = ( + "Bias correction data must be a filepath to csv or a dataframe " + "but received: {}".format(type(bias_correct)) + ) assert isinstance(bias_correct, pd.DataFrame), msg - msg = ('Bias correction table must have "gid" column but only found: ' - '{}'.format(list(bias_correct.columns))) - assert MetaKeyName.GID in bias_correct or bias_correct.index.name == MetaKeyName.GID, msg + msg = ( + 'Bias correction table must have "gid" column but only found: ' + "{}".format(list(bias_correct.columns)) + ) + assert ( + MetaKeyName.GID in bias_correct + or bias_correct.index.name == MetaKeyName.GID + ), msg if bias_correct.index.name != MetaKeyName.GID: bias_correct = bias_correct.set_index(MetaKeyName.GID) - msg = ('Bias correction table must have "method" column but only ' - 'found: {}'.format(list(bias_correct.columns))) - assert 'method' in bias_correct, msg + msg = ( + 'Bias correction table must have "method" column but only ' + "found: {}".format(list(bias_correct.columns)) + ) + assert "method" in bias_correct, msg return bias_correct @@ -866,13 +946,15 @@ def _parse_output_request(self, req): output_request = self._output_request_type_check(req) # ensure that cf_mean is requested from output - if 'cf_mean' not in output_request: - output_request.append('cf_mean') + if "cf_mean" not in output_request: + output_request.append("cf_mean") for request in output_request: if request not in self.OUT_ATTRS: - msg = ('User output request "{}" not recognized. ' - 'Will attempt to extract from PySAM.'.format(request)) + msg = ( + 'User output request "{}" not recognized. ' + "Will attempt to extract from PySAM.".format(request) + ) logger.debug(msg) return list(set(output_request)) @@ -896,20 +978,19 @@ def _reduce_kwargs(self, pc, **kwargs): """ gids = pc.project_points.gids - gid_map = kwargs.get('gid_map', None) - bias_correct = kwargs.get('bias_correct', None) + gid_map = kwargs.get("gid_map", None) + bias_correct = kwargs.get("bias_correct", None) if bias_correct is not None: if gid_map is not None: gids = [gid_map[gid] for gid in gids] mask = bias_correct.index.isin(gids) - kwargs['bias_correct'] = bias_correct[mask] + kwargs["bias_correct"] = bias_correct[mask] return kwargs - def run(self, out_fpath=None, max_workers=1, timeout=1800, - pool_size=None): + def run(self, out_fpath=None, max_workers=1, timeout=1800, pool_size=None): """Execute a parallel reV generation run with smart data flushing. Parameters @@ -947,45 +1028,68 @@ def run(self, out_fpath=None, max_workers=1, timeout=1800, if pool_size is None: pool_size = os.cpu_count() * 2 - kwargs = {'tech': self.tech, - 'res_file': self.res_file, - 'lr_res_file': self.lr_res_file, - 'output_request': self.output_request, - 'scale_outputs': self.scale_outputs, - 'gid_map': self._gid_map, - 'nn_map': self._nn_map, - 'bias_correct': self._bc} - - logger.info('Running reV generation for: {}' - .format(self.points_control)) - logger.debug('The following project points were specified: "{}"' - .format(self.project_points)) - logger.debug('The following SAM configs are available to this run:\n{}' - .format(pprint.pformat(self.sam_configs, indent=4))) - logger.debug('The SAM output variables have been requested:\n{}' - .format(self.output_request)) + kwargs = { + "tech": self.tech, + "res_file": self.res_file, + "lr_res_file": self.lr_res_file, + "output_request": self.output_request, + "scale_outputs": self.scale_outputs, + "gid_map": self._gid_map, + "nn_map": self._nn_map, + "bias_correct": self._bc, + } + + logger.info( + "Running reV generation for: {}".format(self.points_control) + ) + logger.debug( + 'The following project points were specified: "{}"'.format( + self.project_points + ) + ) + logger.debug( + "The following SAM configs are available to this run:\n{}".format( + pprint.pformat(self.sam_configs, indent=4) + ) + ) + logger.debug( + "The SAM output variables have been requested:\n{}".format( + self.output_request + ) + ) # use serial or parallel execution control based on max_workers try: if max_workers == 1: - logger.debug('Running serial generation for: {}' - .format(self.points_control)) + logger.debug( + "Running serial generation for: {}".format( + self.points_control + ) + ) for i, pc_sub in enumerate(self.points_control): self.out = self._run_single_worker(pc_sub, **kwargs) - logger.info('Finished reV gen serial compute for: {} ' - '(iteration {} out of {})' - .format(pc_sub, i + 1, - len(self.points_control))) + logger.info( + "Finished reV gen serial compute for: {} " + "(iteration {} out of {})".format( + pc_sub, i + 1, len(self.points_control) + ) + ) self.flush() else: - logger.debug('Running parallel generation for: {}' - .format(self.points_control)) - self._parallel_run(max_workers=max_workers, - pool_size=pool_size, timeout=timeout, - **kwargs) + logger.debug( + "Running parallel generation for: {}".format( + self.points_control + ) + ) + self._parallel_run( + max_workers=max_workers, + pool_size=pool_size, + timeout=timeout, + **kwargs, + ) except Exception as e: - logger.exception('reV generation failed!') + logger.exception("reV generation failed!") raise e return self._out_fpath diff --git a/reV/nrwal/nrwal.py b/reV/nrwal/nrwal.py index f4b6592ab..92e97c16f 100644 --- a/reV/nrwal/nrwal.py +++ b/reV/nrwal/nrwal.py @@ -11,18 +11,21 @@ generation output file. This is usually the wind or solar resource resolution but could be the supply curve resolution after representative profiles is run. """ -import numpy as np -import pandas as pd + import logging from warnings import warn +import numpy as np +import pandas as pd + from reV.generation.generation import Gen from reV.handlers.outputs import Outputs -from reV.utilities.exceptions import (DataShapeError, - OffshoreWindInputWarning, - OffshoreWindInputError) -from reV.utilities import log_versions - +from reV.utilities import MetaKeyName, log_versions +from reV.utilities.exceptions import ( + DataShapeError, + OffshoreWindInputError, + OffshoreWindInputWarning, +) logger = logging.getLogger(__name__) @@ -30,12 +33,20 @@ class RevNrwal: """RevNrwal""" - DEFAULT_META_COLS = ('config', ) + DEFAULT_META_COLS = ("config",) """Columns from the `site_data` table to join to the output meta data""" - def __init__(self, gen_fpath, site_data, sam_files, nrwal_configs, - output_request, save_raw=True, meta_gid_col=MetaKeyName.GID, - site_meta_cols=None): + def __init__( + self, + gen_fpath, + site_data, + sam_files, + nrwal_configs, + output_request, + save_raw=True, + meta_gid_col=MetaKeyName.GID, + site_meta_cols=None, + ): """Framework to handle reV-NRWAL analysis. ``reV`` NRWAL analysis runs ``reV`` data through the NRWAL @@ -163,8 +174,9 @@ def __init__(self, gen_fpath, site_data, sam_files, nrwal_configs, self._save_raw = save_raw self._nrwal_inputs = self._out = None - self._nrwal_configs = {k: NrwalConfig(v) for k, v in - nrwal_configs.items()} + self._nrwal_configs = { + k: NrwalConfig(v) for k, v in nrwal_configs.items() + } self._site_meta_cols = site_meta_cols if self._site_meta_cols is None: @@ -178,20 +190,29 @@ def __init__(self, gen_fpath, site_data, sam_files, nrwal_configs, self._meta_source = self._parse_gen_data() self._analysis_gids, self._site_data = self._parse_analysis_gids() - pc = Gen.get_pc(self._site_data[[MetaKeyName.GID, 'config']], points_range=None, - sam_configs=sam_files, tech='windpower') + pc = Gen.get_pc( + self._site_data[[MetaKeyName.GID, "config"]], + points_range=None, + sam_configs=sam_files, + tech="windpower", + ) self._project_points = pc.project_points self._sam_sys_inputs = self._parse_sam_sys_inputs() meta_gids = self.meta_source[self._meta_gid_col].values - logger.info('Finished initializing NRWAL analysis module for "{}" ' - '{} through {} with {} total generation points and ' - '{} NRWAL analysis points.' - .format(self._meta_gid_col, meta_gids.min(), - meta_gids.max(), len(self.meta_source), - len(self.analysis_gids))) - - def _parse_site_data(self, required_columns=(MetaKeyName.GID, 'config')): + logger.info( + 'Finished initializing NRWAL analysis module for "{}" ' + "{} through {} with {} total generation points and " + "{} NRWAL analysis points.".format( + self._meta_gid_col, + meta_gids.min(), + meta_gids.max(), + len(self.meta_source), + len(self.analysis_gids), + ) + ) + + def _parse_site_data(self, required_columns=(MetaKeyName.GID, "config")): """Parse the site-specific spatial input data file Parameters @@ -210,20 +231,23 @@ def _parse_site_data(self, required_columns=(MetaKeyName.GID, 'config')): if isinstance(self._site_data, str): self._site_data = pd.read_csv(self._site_data) - if 'dist_l_to_ts' in self._site_data: - if self._site_data['dist_l_to_ts'].sum() > 0: - w = ('Possible incorrect Offshore data input! "dist_l_to_ts" ' - '(distance land to transmission) input is non-zero. ' - 'Most reV runs set this to zero and input the cost ' - 'of transmission from landfall tie-in to ' - 'transmission feature in the supply curve module.') + if "dist_l_to_ts" in self._site_data: + if self._site_data["dist_l_to_ts"].sum() > 0: + w = ( + 'Possible incorrect Offshore data input! "dist_l_to_ts" ' + "(distance land to transmission) input is non-zero. " + "Most reV runs set this to zero and input the cost " + "of transmission from landfall tie-in to " + "transmission feature in the supply curve module." + ) logger.warning(w) warn(w, OffshoreWindInputWarning) for c in required_columns: if c not in self._site_data: - msg = ('Did not find required "{}" column in site_data!' - .format(c)) + msg = 'Did not find required "{}" column in site_data!'.format( + c + ) logger.error(msg) raise KeyError(msg) @@ -240,16 +264,19 @@ def _parse_gen_data(self): Full meta data from gen_fpath. """ - with Outputs(self._gen_fpath, mode='r') as out: + with Outputs(self._gen_fpath, mode="r") as out: meta = out.meta - msg = ('Could not find "{}" column in source generation h5 file ' - 'meta data! Available cols: {}' - .format(self._meta_gid_col, meta.columns.values.tolist())) + msg = ( + 'Could not find "{}" column in source generation h5 file ' + "meta data! Available cols: {}".format( + self._meta_gid_col, meta.columns.values.tolist() + ) + ) assert self._meta_gid_col in meta, msg # currently an assumption of sorted gids in the reV gen output - msg = ('Source capacity factor meta data is not ordered!') + msg = "Source capacity factor meta data is not ordered!" meta_gids = list(meta[self._meta_gid_col]) assert meta_gids == sorted(meta_gids), msg @@ -274,18 +301,25 @@ def _parse_analysis_gids(self): missing = ~np.isin(meta_gids, self._site_data[MetaKeyName.GID]) if any(missing): - msg = ('{} sites from the generation meta data input were ' - 'missing from the "site_data" input and will not be ' - 'run through NRWAL: {}' - .format(missing.sum(), meta_gids[missing])) + msg = ( + "{} sites from the generation meta data input were " + 'missing from the "site_data" input and will not be ' + "run through NRWAL: {}".format( + missing.sum(), meta_gids[missing] + ) + ) logger.info(msg) missing = ~np.isin(self._site_data[MetaKeyName.GID], meta_gids) if any(missing): missing = self._site_data[MetaKeyName.GID].values[missing] - msg = ('{} sites from the "site_data" input were missing from the ' - 'generation meta data and will not be run through NRWAL: {}' - .format(len(missing), missing)) + msg = ( + '{} sites from the "site_data" input were missing from the ' + "generation meta data and will not be run through NRWAL: {}" + .format( + len(missing), missing + ) + ) logger.info(msg) analysis_gids = set(meta_gids) & set(self._site_data[MetaKeyName.GID]) @@ -337,25 +371,30 @@ def _init_outputs(self): out = {} for key in self._output_request: - out[key] = np.full(len(self.analysis_gids), np.nan, - dtype=np.float32) + out[key] = np.full( + len(self.analysis_gids), np.nan, dtype=np.float32 + ) if key in self.gen_dsets and not self._save_raw: - msg = ('Output request "{0}" was also found in ' - 'the source gen file but save_raw=False! If ' - 'you are manipulating this ' - 'dset, make sure you set save_raw=False ' - 'and reference "{0}_raw" as the ' - 'input in the NRWAL equations and then define "{0}" ' - 'as the final manipulated dataset.'.format(key)) + msg = ( + 'Output request "{0}" was also found in ' + "the source gen file but save_raw=False! If " + "you are manipulating this " + "dset, make sure you set save_raw=False " + 'and reference "{0}_raw" as the ' + 'input in the NRWAL equations and then define "{0}" ' + "as the final manipulated dataset.".format(key) + ) logger.warning(msg) warn(msg) elif key in self.gen_dsets: - msg = ('Output request "{0}" was also found in ' - 'the source gen file. If you are manipulating this ' - 'dset, make sure you reference "{0}_raw" as the ' - 'input in the NRWAL equations and then define "{0}" ' - 'as the final manipulated dataset.'.format(key)) + msg = ( + 'Output request "{0}" was also found in ' + "the source gen file. If you are manipulating this " + 'dset, make sure you reference "{0}_raw" as the ' + 'input in the NRWAL equations and then define "{0}" ' + "as the final manipulated dataset.".format(key) + ) logger.info(msg) if key in self._nrwal_inputs: @@ -365,40 +404,50 @@ def _init_outputs(self): def _preflight_checks(self): """Run some preflight checks on the offshore inputs""" - sam_files = {k: v for k, v in - self._project_points.sam_inputs.items() - if k in self._nrwal_configs} + sam_files = { + k: v + for k, v in self._project_points.sam_inputs.items() + if k in self._nrwal_configs + } for cid, sys_in in sam_files.items(): - loss1 = sys_in.get('wind_farm_losses_percent', 0) - loss2 = sys_in.get('turb_generic_loss', 0) + loss1 = sys_in.get("wind_farm_losses_percent", 0) + loss2 = sys_in.get("turb_generic_loss", 0) if loss1 != 0 or loss2 != 0: - msg = ('Wind farm loss for config "{}" is not 0. When using ' - 'NRWAL for offshore analysis, consider using gross ' - 'capacity factors from reV generation and applying ' - 'spatially dependent losses from the NRWAL equations' - .format(cid)) + msg = ( + 'Wind farm loss for config "{}" is not 0. When using ' + "NRWAL for offshore analysis, consider using gross " + "capacity factors from reV generation and applying " + "spatially dependent losses from the NRWAL equations" + .format(cid) + ) logger.info(msg) available_ids = list(self._nrwal_configs.keys()) - requested_ids = list(self._site_data['config'].values) + requested_ids = list(self._site_data["config"].values) missing = set(requested_ids) - set(available_ids) if any(missing): - msg = ('The following config ids were requested in the offshore ' - 'data input but were not available in the NRWAL config ' - 'input dict: {}'.format(missing)) + msg = ( + "The following config ids were requested in the offshore " + "data input but were not available in the NRWAL config " + "input dict: {}".format(missing) + ) logger.error(msg) raise OffshoreWindInputError(msg) - check_gid_order = (self._site_data[MetaKeyName.GID].values - == self._sam_sys_inputs[MetaKeyName.GID].values) - msg = 'NRWAL site_data and system input dataframe had bad order' + check_gid_order = ( + self._site_data[MetaKeyName.GID].values + == self._sam_sys_inputs[MetaKeyName.GID].values + ) + msg = "NRWAL site_data and system input dataframe had bad order" assert (check_gid_order).all(), msg missing = [c for c in self._site_meta_cols if c not in self._site_data] if any(missing): - msg = ('Could not find requested NRWAL site data pass through ' - 'columns in offshore input data: {}'.format(missing)) + msg = ( + "Could not find requested NRWAL site data pass through " + "columns in offshore input data: {}".format(missing) + ) logger.error(msg) raise OffshoreWindInputError(msg) @@ -413,7 +462,7 @@ def _get_input_data(self): or 2D arrays of inputs for all the analysis_gids """ - logger.info('Setting up input data for NRWAL...') + logger.info("Setting up input data for NRWAL...") # preconditions for this to work properly assert len(self._site_data) == len(self.analysis_gids) @@ -424,50 +473,75 @@ def _get_input_data(self): all_required += list(nrwal_config.required_inputs) all_required = list(set(all_required)) - missing_vars = [var for var in nrwal_config.required_inputs - if var not in self._site_data - and var not in self.meta_source - and var not in self._sam_sys_inputs - and var not in self.gen_dsets] + missing_vars = [ + var + for var in nrwal_config.required_inputs + if var not in self._site_data + and var not in self.meta_source + and var not in self._sam_sys_inputs + and var not in self.gen_dsets + ] if any(missing_vars): - msg = ('Could not find required input variables {} ' - 'for NRWAL config "{}" in either the offshore ' - 'data or the SAM system data!' - .format(missing_vars, config_id)) + msg = ( + "Could not find required input variables {} " + 'for NRWAL config "{}" in either the offshore ' + "data or the SAM system data!".format( + missing_vars, config_id + ) + ) logger.error(msg) raise OffshoreWindInputError(msg) - meta_data_vars = [var for var in all_required - if var in self.meta_source] - logger.info('Pulling the following inputs from the gen meta data: {}' - .format(meta_data_vars)) - nrwal_inputs = {var: self.meta_source[var].values[self.analysis_mask] - for var in meta_data_vars} - - site_data_vars = [var for var in all_required - if var in self._site_data - and var not in nrwal_inputs] - site_data_vars.append('config') - logger.info('Pulling the following inputs from the site_data input: {}' - .format(site_data_vars)) + meta_data_vars = [ + var for var in all_required if var in self.meta_source + ] + logger.info( + "Pulling the following inputs from the gen meta data: {}".format( + meta_data_vars + ) + ) + nrwal_inputs = { + var: self.meta_source[var].values[self.analysis_mask] + for var in meta_data_vars + } + + site_data_vars = [ + var + for var in all_required + if var in self._site_data and var not in nrwal_inputs + ] + site_data_vars.append("config") + logger.info( + "Pulling the following inputs from the site_data input: {}".format( + site_data_vars + ) + ) for var in site_data_vars: nrwal_inputs[var] = self._site_data[var].values - sam_sys_vars = [var for var in all_required - if var in self._sam_sys_inputs - and var not in nrwal_inputs] - logger.info('Pulling the following inputs from the SAM system ' - 'configs: {}'.format(sam_sys_vars)) + sam_sys_vars = [ + var + for var in all_required + if var in self._sam_sys_inputs and var not in nrwal_inputs + ] + logger.info( + "Pulling the following inputs from the SAM system " + "configs: {}".format(sam_sys_vars) + ) for var in sam_sys_vars: nrwal_inputs[var] = self._sam_sys_inputs[var].values - gen_vars = [var for var in all_required - if var in self.gen_dsets - and var not in nrwal_inputs] - logger.info('Pulling the following inputs from the generation ' - 'h5 file: {}'.format(gen_vars)) - with Outputs(self._gen_fpath, mode='r') as f: + gen_vars = [ + var + for var in all_required + if var in self.gen_dsets and var not in nrwal_inputs + ] + logger.info( + "Pulling the following inputs from the generation " + "h5 file: {}".format(gen_vars) + ) + with Outputs(self._gen_fpath, mode="r") as f: source_gids = self.meta_source[self._meta_gid_col] gen_gids = np.where(source_gids.isin(self.analysis_gids))[0] for var in gen_vars: @@ -477,12 +551,14 @@ def _get_input_data(self): elif len(shape) == 2: nrwal_inputs[var] = f[var, :, gen_gids] else: - msg = ('Data shape for "{}" must be 1 or 2D but ' - 'received: {}'.format(var, shape)) + msg = ( + 'Data shape for "{}" must be 1 or 2D but ' + "received: {}".format(var, shape) + ) logger.error(msg) raise DataShapeError(msg) - logger.info('Finished setting up input data for NRWAL!') + logger.info("Finished setting up input data for NRWAL!") return nrwal_inputs @@ -490,7 +566,7 @@ def _get_input_data(self): def time_index(self): """Get the source time index.""" if self._time_index is None: - with Outputs(self._gen_fpath, mode='r') as out: + with Outputs(self._gen_fpath, mode="r") as out: self._time_index = out.time_index return self._time_index @@ -498,7 +574,7 @@ def time_index(self): @property def gen_dsets(self): """Get the available datasets from the gen source file""" - with Outputs(self._gen_fpath, mode='r') as out: + with Outputs(self._gen_fpath, mode="r") as out: dsets = out.dsets return dsets @@ -528,8 +604,9 @@ def analysis_mask(self): ------- np.ndarray """ - mask = np.isin(self.meta_source[self._meta_gid_col], - self.analysis_gids) + mask = np.isin( + self.meta_source[self._meta_gid_col], self.analysis_gids + ) return mask @property @@ -576,9 +653,12 @@ def _save_nrwal_out(self, name, nrwal_out, output_mask): elif len(value.shape) == 2: if len(self._out[name].shape) == 1: if not all(np.isnan(self._out[name])): - msg = ('Output dataset "{}" was initialized as 1D but was ' - 'later found to be 2D but was not all NaN!' - .format(name)) + msg = ( + 'Output dataset "{}" was initialized as 1D but was ' + "later found to be 2D but was not all NaN!".format( + name + ) + ) logger.error(msg) raise DataShapeError(msg) @@ -590,8 +670,10 @@ def _save_nrwal_out(self, name, nrwal_out, output_mask): self._out[name][:, output_mask] = value[:, output_mask] else: - msg = ('Could not make sense of NRWAL output "{}" ' - 'with shape {}'.format(name, value.shape)) + msg = ( + 'Could not make sense of NRWAL output "{}" ' + "with shape {}".format(name, value.shape) + ) logger.error(msg) raise DataShapeError(msg) @@ -614,11 +696,14 @@ def _save_nrwal_misc(self, name, nrwal_config, output_mask): """ from NRWAL import Equation + value = nrwal_config[name] if isinstance(value, Equation): - msg = ('Cannot retrieve Equation "{}" from NRWAL. ' - 'Must be a number!'.format(name)) + msg = ( + 'Cannot retrieve Equation "{}" from NRWAL. ' + "Must be a number!".format(name) + ) assert not any(value.variables), msg value = value.eval() @@ -631,8 +716,10 @@ def _value_to_array(self, value, name): value *= np.ones(len(self.analysis_gids), dtype=np.float32) if not isinstance(value, np.ndarray): - msg = ('NRWAL key "{}" returned bad type of "{}", needs to be ' - 'numeric or an output array.'.format(name, type(value))) + msg = ( + 'NRWAL key "{}" returned bad type of "{}", needs to be ' + "numeric or an output array.".format(name, type(value)) + ) logger.error(msg) raise TypeError(msg) return value @@ -646,11 +733,17 @@ def run_nrwal(self): self._out = self._init_outputs() for i, (cid, nrwal_config) in enumerate(self._nrwal_configs.items()): - output_mask = self._site_data['config'].values == cid - logger.info('Running NRWAL config {} of {}: "{}" and applying ' - 'to {} out of {} total sites' - .format(i + 1, len(self._nrwal_configs), cid, - output_mask.sum(), len(output_mask))) + output_mask = self._site_data["config"].values == cid + logger.info( + 'Running NRWAL config {} of {}: "{}" and applying ' + "to {} out of {} total sites".format( + i + 1, + len(self._nrwal_configs), + cid, + output_mask.sum(), + len(output_mask), + ) + ) nrwal_out = nrwal_config.eval(inputs=self._nrwal_inputs) @@ -663,8 +756,10 @@ def run_nrwal(self): self._save_nrwal_misc(name, nrwal_config, output_mask) elif name not in self._nrwal_inputs: - msg = ('Could not find "{}" in the output dict of NRWAL ' - 'config {}'.format(name, cid)) + msg = ( + 'Could not find "{}" in the output dict of NRWAL ' + "config {}".format(name, cid) + ) logger.warning(msg) warn(msg) @@ -672,36 +767,50 @@ def check_outputs(self): """Check the nrwal outputs for nan values and raise errors if found.""" for name, arr in self._out.items(): if np.isnan(arr).all(): - msg = ('Output array "{}" is all NaN! Probably was not ' - 'found in the available NRWAL keys.'.format(name)) + msg = ( + 'Output array "{}" is all NaN! Probably was not ' + "found in the available NRWAL keys.".format(name) + ) logger.warning(msg) warn(msg) elif np.isnan(arr).any(): mask = np.isnan(arr) nan_meta = self.meta_source[self.analysis_mask][mask] nan_gids = nan_meta[self._meta_gid_col].values - msg = ('NaN values ({} out of {}) persist in NRWAL ' - 'output "{}"!' - .format(np.isnan(arr).sum(), len(arr), name)) + msg = ( + "NaN values ({} out of {}) persist in NRWAL " + 'output "{}"!'.format(np.isnan(arr).sum(), len(arr), name) + ) logger.warning(msg) - logger.warning('This is the NRWAL meta that is causing NaN ' - 'outputs: {}'.format(nan_meta)) - logger.warning('These are the resource gids causing NaN ' - 'outputs: {}'.format(nan_gids)) + logger.warning( + "This is the NRWAL meta that is causing NaN " + "outputs: {}".format(nan_meta) + ) + logger.warning( + "These are the resource gids causing NaN " + "outputs: {}".format(nan_gids) + ) warn(msg) def save_raw_dsets(self): """If requested by save_raw=True, archive raw datasets that exist in the gen_fpath file and are also requested in the output_request""" if self._save_raw: - with Outputs(self._gen_fpath, 'a') as f: + with Outputs(self._gen_fpath, "a") as f: for dset in self._output_request: - dset_raw = '{}_raw'.format(dset) + dset_raw = "{}_raw".format(dset) if dset in f and dset_raw not in f: - logger.info('Saving raw data from "{}" to "{}"' - .format(dset, dset_raw)) - f._add_dset(dset_raw, f[dset], f.dtypes[dset], - attrs=f.attrs[dset]) + logger.info( + 'Saving raw data from "{}" to "{}"'.format( + dset, dset_raw + ) + ) + f._add_dset( + dset_raw, + f[dset], + f.dtypes[dset], + attrs=f.attrs[dset], + ) def write_to_gen_fpath(self): """Save NRWAL outputs to input generation fpath file. @@ -712,30 +821,34 @@ def write_to_gen_fpath(self): Path to output file. """ - logger.info('Writing NRWAL outputs to: {}'.format(self._gen_fpath)) + logger.info("Writing NRWAL outputs to: {}".format(self._gen_fpath)) write_all = self.analysis_mask.all() - with Outputs(self._gen_fpath, 'a') as f: - meta_attrs = f.attrs['meta'] - del f._h5['meta'] - f._set_meta('meta', self.meta_out, attrs=meta_attrs) + with Outputs(self._gen_fpath, "a") as f: + meta_attrs = f.attrs["meta"] + del f._h5["meta"] + f._set_meta("meta", self.meta_out, attrs=meta_attrs) for dset, arr in self._out.items(): if len(arr.shape) == 1: - data = np.full(len(self.meta_source), np.nan, - dtype=np.float32) + data = np.full( + len(self.meta_source), np.nan, dtype=np.float32 + ) else: - full_shape = (len(self.time_index), - len(self.meta_source)) + full_shape = (len(self.time_index), len(self.meta_source)) data = np.full(full_shape, np.nan, dtype=np.float32) - dset_attrs = {'scale_factor': 1} + dset_attrs = {"scale_factor": 1} dset_dtype = np.float32 if dset in f.dsets: - logger.info('Found "{}" in file, loading data and ' - 'overwriting data for {} out of {} sites.' - .format(dset, self.analysis_mask.sum(), - len(self.analysis_mask))) + logger.info( + 'Found "{}" in file, loading data and ' + "overwriting data for {} out of {} sites.".format( + dset, + self.analysis_mask.sum(), + len(self.analysis_mask), + ) + ) dset_attrs = f.attrs[dset] dset_dtype = f.dtypes[dset] if not write_all: @@ -746,12 +859,14 @@ def write_to_gen_fpath(self): else: data[:, self.analysis_mask] = arr - logger.info('Writing final "{}" to: {}' - .format(dset, self._gen_fpath)) + logger.info( + 'Writing final "{}" to: {}'.format(dset, self._gen_fpath) + ) f._add_dset(dset, data, dset_dtype, attrs=dset_attrs) - logger.info('Finished writing NRWAL outputs to: {}' - .format(self._gen_fpath)) + logger.info( + "Finished writing NRWAL outputs to: {}".format(self._gen_fpath) + ) return self._gen_fpath def write_meta_to_csv(self, out_fpath=None): @@ -776,21 +891,24 @@ def write_meta_to_csv(self, out_fpath=None): elif not out_fpath.endswith(".csv"): out_fpath = "{}.csv".format(out_fpath) - logger.info('Writing NRWAL outputs to: {}'.format(out_fpath)) + logger.info("Writing NRWAL outputs to: {}".format(out_fpath)) meta_out = self.meta_out[self.analysis_mask].copy() for dset, arr in self._out.items(): if len(arr.shape) != 1 or arr.shape[0] != meta_out.shape[0]: - msg = ('Skipping output {!r}: shape {} cannot be combined ' - 'with meta of shape {}!' - .format(dset, arr.shape, meta_out.shape)) + msg = ( + "Skipping output {!r}: shape {} cannot be combined " + "with meta of shape {}!".format( + dset, arr.shape, meta_out.shape + ) + ) logger.warning(msg) warn(msg) continue meta_out[dset] = arr meta_out.to_csv(out_fpath, index=False) - logger.info('Finished writing NRWAL outputs to: {}'.format(out_fpath)) + logger.info("Finished writing NRWAL outputs to: {}".format(out_fpath)) return out_fpath def run(self, csv_output=False, out_fpath=None): @@ -831,8 +949,10 @@ def run(self, csv_output=False, out_fpath=None): Path to output file. """ if csv_output and self._save_raw: - msg = ("`save_raw` option not allowed with `csv_output`. Setting" - "`save_raw=False`") + msg = ( + "`save_raw` option not allowed with `csv_output`. Setting" + "`save_raw=False`" + ) logger.warning(msg) warn(msg) self._save_raw = False @@ -845,6 +965,6 @@ def run(self, csv_output=False, out_fpath=None): else: out_fp = self.write_to_gen_fpath() - logger.info('NRWAL module complete!') + logger.info("NRWAL module complete!") return out_fp diff --git a/reV/qa_qc/qa_qc.py b/reV/qa_qc/qa_qc.py index 899ae00ed..c313198bb 100644 --- a/reV/qa_qc/qa_qc.py +++ b/reV/qa_qc/qa_qc.py @@ -380,7 +380,7 @@ def fpath(self): break else: raise PipelineError( - "Could not parse fpath from previous " "pipeline jobs." + "Could not parse fpath from previous pipeline jobs." ) fpath = fpath[0] logger.info( diff --git a/reV/utilities/pytest_utils.py b/reV/utilities/pytest_utils.py index c95294944..b67da4948 100644 --- a/reV/utilities/pytest_utils.py +++ b/reV/utilities/pytest_utils.py @@ -8,6 +8,8 @@ from packaging import version from rex.outputs import Outputs as RexOutputs +from reV.utilities import MetaKeyName + def pd_date_range(*args, **kwargs): """A simple wrapper on the pd.date_range() method that handles the closed diff --git a/tests/test_gen_pv.py b/tests/test_gen_pv.py index d55375845..9f0c52a5b 100644 --- a/tests/test_gen_pv.py +++ b/tests/test_gen_pv.py @@ -22,6 +22,7 @@ from reV.config.project_points import ProjectPoints from reV.generation.generation import Gen from reV.handlers.outputs import Outputs +from reV.utilities import MetaKeyName from reV.utilities.exceptions import ConfigError, ExecutionError RTOL = 0.0 @@ -32,7 +33,7 @@ class pv_results: """Class to retrieve results from the rev 1.0 pv files""" def __init__(self, f): - self._h5 = h5py.File(f, 'r') + self._h5 = h5py.File(f, "r") def __enter__(self): return self @@ -46,15 +47,15 @@ def __exit__(self, type, value, traceback): @property def years(self): """Get a list of year strings.""" - if not hasattr(self, '_years'): - year_index = self._h5['pv']['year_index'][...] + if not hasattr(self, "_years"): + year_index = self._h5["pv"]["year_index"][...] self._years = [y.decode() for y in year_index] return self._years def get_cf_mean(self, site, year): """Get a cf mean based on site and year""" iy = self.years.index(year) - out = self._h5['pv']['cf_mean'][iy, site] + out = self._h5["pv"]["cf_mean"][iy, site] return out @@ -70,34 +71,39 @@ def is_num(n): def _to_list(gen_out): """Generation output handler that converts to the rev 1.0 format.""" if isinstance(gen_out, list) and len(gen_out) == 1: - out = [c['cf_mean'] for c in gen_out[0].values()] + out = [c["cf_mean"] for c in gen_out[0].values()] if isinstance(gen_out, dict): - out = [c['cf_mean'] for c in gen_out.values()] + out = [c["cf_mean"] for c in gen_out.values()] return out -@pytest.mark.parametrize(('f_rev1_out', 'rev2_points', 'year', 'max_workers'), - [ - ('project_outputs.h5', slice(0, 10), '2012', 1), - ('project_outputs.h5', slice(0, None, 10), '2013', 1), - ('project_outputs.h5', slice(3, 25, 2), '2012', 2), - ('project_outputs.h5', slice(40, None, 10), '2013', 2)]) +@pytest.mark.parametrize( + ("f_rev1_out", "rev2_points", "year", "max_workers"), + [ + ("project_outputs.h5", slice(0, 10), "2012", 1), + ("project_outputs.h5", slice(0, None, 10), "2013", 1), + ("project_outputs.h5", slice(3, 25, 2), "2012", 2), + ("project_outputs.h5", slice(40, None, 10), "2013", 2), + ], +) def test_pv_gen_slice(f_rev1_out, rev2_points, year, max_workers): """Test reV 2.0 generation for PV and benchmark against reV 1.0 results.""" # get full file paths. - rev1_outs = os.path.join(TESTDATADIR, 'ri_pv', 'scalar_outputs', - f_rev1_out) - sam_files = TESTDATADIR + '/SAM/naris_pv_1axis_inv13.json' - res_file = TESTDATADIR + '/nsrdb/ri_100_nsrdb_{}.h5'.format(year) + rev1_outs = os.path.join( + TESTDATADIR, "ri_pv", "scalar_outputs", f_rev1_out + ) + sam_files = TESTDATADIR + "/SAM/naris_pv_1axis_inv13.json" + res_file = TESTDATADIR + "/nsrdb/ri_100_nsrdb_{}.h5".format(year) # run reV 2.0 generation - pp = ProjectPoints(rev2_points, sam_files, 'pvwattsv5', res_file=res_file) - gen = Gen('pvwattsv5', rev2_points, sam_files, res_file, - sites_per_worker=3) + pp = ProjectPoints(rev2_points, sam_files, "pvwattsv5", res_file=res_file) + gen = Gen( + "pvwattsv5", rev2_points, sam_files, res_file, sites_per_worker=3 + ) gen.run(max_workers=max_workers) - gen_outs = list(gen.out['cf_mean']) + gen_outs = list(gen.out["cf_mean"]) # initialize the rev1 output hander with pv_results(rev1_outs) as pv: @@ -108,72 +114,87 @@ def test_pv_gen_slice(f_rev1_out, rev2_points, year, max_workers): assert np.allclose(gen_outs, cf_mean_list, rtol=RTOL, atol=ATOL) -def test_pv_gen_csv1(f_rev1_out='project_outputs.h5', - rev2_points=TESTDATADIR + '/project_points/ri.csv', - res_file=TESTDATADIR + '/nsrdb/ri_100_nsrdb_2012.h5'): +def test_pv_gen_csv1( + f_rev1_out="project_outputs.h5", + rev2_points=TESTDATADIR + "/project_points/ri.csv", + res_file=TESTDATADIR + "/nsrdb/ri_100_nsrdb_2012.h5", +): """Test project points csv input with dictionary-based sam files.""" - rev1_outs = os.path.join(TESTDATADIR, 'ri_pv', 'scalar_outputs', - f_rev1_out) - sam_files = {'sam_param_0': TESTDATADIR + '/SAM/naris_pv_1axis_inv13.json', - 'sam_param_1': TESTDATADIR + '/SAM/naris_pv_1axis_inv13.json'} - pp = ProjectPoints(rev2_points, sam_files, 'pvwattsv5') + rev1_outs = os.path.join( + TESTDATADIR, "ri_pv", "scalar_outputs", f_rev1_out + ) + sam_files = { + "sam_param_0": TESTDATADIR + "/SAM/naris_pv_1axis_inv13.json", + "sam_param_1": TESTDATADIR + "/SAM/naris_pv_1axis_inv13.json", + } + pp = ProjectPoints(rev2_points, sam_files, "pvwattsv5") # run reV 2.0 generation - gen = Gen('pvwattsv5', rev2_points, sam_files, res_file) + gen = Gen("pvwattsv5", rev2_points, sam_files, res_file) gen.run() - gen_outs = list(gen.out['cf_mean']) + gen_outs = list(gen.out["cf_mean"]) # initialize the rev1 output hander with pv_results(rev1_outs) as pv: # get reV 1.0 results - cf_mean_list = pv.get_cf_mean(pp.sites, '2012') + cf_mean_list = pv.get_cf_mean(pp.sites, "2012") # benchmark the results assert np.allclose(gen_outs, cf_mean_list, rtol=RTOL, atol=ATOL) -def test_pv_gen_csv2(f_rev1_out='project_outputs.h5', - rev2_points=TESTDATADIR + '/project_points/ri.csv', - res_file=TESTDATADIR + '/nsrdb/ri_100_nsrdb_2012.h5'): +def test_pv_gen_csv2( + f_rev1_out="project_outputs.h5", + rev2_points=TESTDATADIR + "/project_points/ri.csv", + res_file=TESTDATADIR + "/nsrdb/ri_100_nsrdb_2012.h5", +): """Test project points csv input with list-based sam files.""" - rev1_outs = os.path.join(TESTDATADIR, 'ri_pv', 'scalar_outputs', - f_rev1_out) - sam_files = [TESTDATADIR + '/SAM/naris_pv_1axis_inv13.json', - TESTDATADIR + '/SAM/naris_pv_1axis_inv13.json'] - sam_files = {'sam_param_{}'.format(i): k for i, k in - enumerate(sam_files)} - pp = ProjectPoints(rev2_points, sam_files, 'pvwattsv5') - gen = Gen('pvwattsv5', rev2_points, sam_files, res_file) + rev1_outs = os.path.join( + TESTDATADIR, "ri_pv", "scalar_outputs", f_rev1_out + ) + sam_files = [ + TESTDATADIR + "/SAM/naris_pv_1axis_inv13.json", + TESTDATADIR + "/SAM/naris_pv_1axis_inv13.json", + ] + sam_files = {"sam_param_{}".format(i): k for i, k in enumerate(sam_files)} + pp = ProjectPoints(rev2_points, sam_files, "pvwattsv5") + gen = Gen("pvwattsv5", rev2_points, sam_files, res_file) gen.run() - gen_outs = list(gen.out['cf_mean']) + gen_outs = list(gen.out["cf_mean"]) # initialize the rev1 output hander with pv_results(rev1_outs) as pv: # get reV 1.0 results - cf_mean_list = pv.get_cf_mean(pp.sites, '2012') + cf_mean_list = pv.get_cf_mean(pp.sites, "2012") # benchmark the results assert np.allclose(gen_outs, cf_mean_list, rtol=RTOL, atol=ATOL) -@pytest.mark.parametrize('year', [('2012'), ('2013')]) +@pytest.mark.parametrize("year", [("2012"), ("2013")]) def test_pv_gen_profiles(year): """Gen PV CF profiles with write to disk and compare against rev1.""" with tempfile.TemporaryDirectory() as td: - res_file = TESTDATADIR + '/nsrdb/ri_100_nsrdb_{}.h5'.format(year) - sam_files = TESTDATADIR + '/SAM/naris_pv_1axis_inv13.json' - fn = 'gen_ri_pv_generation_{}.h5'.format(year) + res_file = TESTDATADIR + "/nsrdb/ri_100_nsrdb_{}.h5".format(year) + sam_files = TESTDATADIR + "/SAM/naris_pv_1axis_inv13.json" + fn = "gen_ri_pv_generation_{}.h5".format(year) rev2_out = os.path.join(td, fn) points = slice(0, 100) # run reV 2.0 generation and write to disk - gen = Gen('pvwattsv5', points, sam_files, res_file, - output_request=('cf_profile',), sites_per_worker=50) + gen = Gen( + "pvwattsv5", + points, + sam_files, + res_file, + output_request=("cf_profile",), + sites_per_worker=50, + ) gen.run(max_workers=2, out_fpath=rev2_out) - with Outputs(rev2_out, 'r') as cf: - rev2_profiles = cf['cf_profile'] + with Outputs(rev2_out, "r") as cf: + rev2_profiles = cf["cf_profile"] # get reV 1.0 generation profiles rev1_profiles = get_r1_profiles(year=year) @@ -182,24 +203,30 @@ def test_pv_gen_profiles(year): assert np.allclose(rev1_profiles, rev2_profiles, rtol=RTOL, atol=ATOL) -@pytest.mark.parametrize('year', [('2012'), ('2013')]) +@pytest.mark.parametrize("year", [("2012"), ("2013")]) def test_smart(year): """Gen PV CF profiles with write to disk and compare against rev1.""" with tempfile.TemporaryDirectory() as td: - res_file = TESTDATADIR + '/nsrdb/ri_100_nsrdb_{}.h5'.format(year) - sam_files = TESTDATADIR + '/SAM/naris_pv_1axis_inv13.json' - fn = 'gen_ri_pv_generation_smart_{}.h5'.format(year) + res_file = TESTDATADIR + "/nsrdb/ri_100_nsrdb_{}.h5".format(year) + sam_files = TESTDATADIR + "/SAM/naris_pv_1axis_inv13.json" + fn = "gen_ri_pv_generation_smart_{}.h5".format(year) rev2_out = os.path.join(td, fn) points = slice(0, 10) # run reV 2.0 generation and write to disk - gen = Gen('pvwattsv5', points, sam_files, res_file, - output_request=('cf_profile',), sites_per_worker=50) + gen = Gen( + "pvwattsv5", + points, + sam_files, + res_file, + output_request=("cf_profile",), + sites_per_worker=50, + ) gen.run(max_workers=2, out_fpath=rev2_out) - with Outputs(rev2_out, 'r') as cf: - rev2_profiles = cf['cf_profile'] + with Outputs(rev2_out, "r") as cf: + rev2_profiles = cf["cf_profile"] # get reV 1.0 generation profiles rev1_profiles = get_r1_profiles(year=year) @@ -208,34 +235,40 @@ def test_smart(year): assert np.allclose(rev1_profiles, rev2_profiles, rtol=RTOL, atol=ATOL) -@pytest.mark.parametrize('model', ['pvwattsv5', 'pvwattsv7']) +@pytest.mark.parametrize("model", ["pvwattsv5", "pvwattsv7"]) def test_multi_file_nsrdb_2018(model): """Test running reV gen from a multi-h5 directory with prefix and suffix""" points = slice(0, 10) max_workers = 1 - sam_files = TESTDATADIR + '/SAM/naris_pv_1axis_inv13.json' - res_file = TESTDATADIR + '/nsrdb/nsrdb_*{}.h5'.format(2018) + sam_files = TESTDATADIR + "/SAM/naris_pv_1axis_inv13.json" + res_file = TESTDATADIR + "/nsrdb/nsrdb_*{}.h5".format(2018) # run reV 2.0 generation - gen = Gen(model, points, sam_files, res_file, - output_request=('cf_mean', 'cf_profile'), - sites_per_worker=3) + gen = Gen( + model, + points, + sam_files, + res_file, + output_request=("cf_mean", "cf_profile"), + sites_per_worker=3, + ) gen.run(max_workers=max_workers) - means_outs = list(gen.out['cf_mean']) + means_outs = list(gen.out["cf_mean"]) assert len(means_outs) == 10 assert np.mean(means_outs) > 0.14 - profiles_out = gen.out['cf_profile'] + profiles_out = gen.out["cf_profile"] assert profiles_out.shape == (105120, 10) assert np.mean(profiles_out) > 0.14 def get_r1_profiles(year=2012): """Get the first 100 reV 1.0 ri pv generation profiles.""" - rev1 = os.path.join(TESTDATADIR, 'ri_pv', 'profile_outputs', - 'pv_{}_0.h5'.format(year)) + rev1 = os.path.join( + TESTDATADIR, "ri_pv", "profile_outputs", "pv_{}_0.h5".format(year) + ) with Outputs(rev1) as cf: - data = cf['cf_profile'][...] / 10000 + data = cf["cf_profile"][...] / 10000 return data @@ -245,35 +278,55 @@ def test_pv_name_error(): year = 2012 rev2_points = slice(0, 3) - sam_files = TESTDATADIR + '/SAM/naris_pv_1axis_inv13.json' - res_file = TESTDATADIR + '/nsrdb/ri_100_nsrdb_{}.h5'.format(year) + sam_files = TESTDATADIR + "/SAM/naris_pv_1axis_inv13.json" + res_file = TESTDATADIR + "/nsrdb/ri_100_nsrdb_{}.h5".format(year) # run reV 2.0 generation with pytest.raises(KeyError) as record: - gen = Gen('pv', rev2_points, sam_files, res_file, sites_per_worker=1) + gen = Gen("pv", rev2_points, sam_files, res_file, sites_per_worker=1) gen.run(max_workers=1) - assert 'Did not recognize' in record[0].message + assert "Did not recognize" in record[0].message def test_southern_hemisphere(): """Test reV pvwatts in the southern hemisphere with correct azimuth""" rev2_points = slice(0, 1) - res_file = TESTDATADIR + '/nsrdb/brazil_solar.h5' - sam_files = TESTDATADIR + '/SAM/i_pvwatts_fixed_lat_tilt.json' - output_request = ('cf_mean', 'cf_profile', 'dni_mean', 'dhi_mean', - 'ghi_mean', 'ac', 'dc', 'azimuth') - - gen = Gen('pvwattsv7', rev2_points, sam_files, res_file, - sites_per_worker=1, output_request=output_request) + res_file = TESTDATADIR + "/nsrdb/brazil_solar.h5" + sam_files = TESTDATADIR + "/SAM/i_pvwatts_fixed_lat_tilt.json" + output_request = ( + "cf_mean", + "cf_profile", + "dni_mean", + "dhi_mean", + "ghi_mean", + "ac", + "dc", + "azimuth", + ) + + gen = Gen( + "pvwattsv7", + rev2_points, + sam_files, + res_file, + sites_per_worker=1, + output_request=output_request, + ) gen.run(max_workers=1) - assert gen.out['azimuth'] == 0 - - res_file = TESTDATADIR + '/nsrdb/ri_100_nsrdb_2012.h5' - gen = Gen('pvwattsv7', rev2_points, sam_files, res_file, - sites_per_worker=1, output_request=output_request) + assert gen.out["azimuth"] == 0 + + res_file = TESTDATADIR + "/nsrdb/ri_100_nsrdb_2012.h5" + gen = Gen( + "pvwattsv7", + rev2_points, + sam_files, + res_file, + sites_per_worker=1, + output_request=output_request, + ) gen.run(max_workers=1) - assert gen.out['azimuth'] == 180 + assert gen.out["azimuth"] == 180 def test_pvwattsv7_baseline(): @@ -283,21 +336,36 @@ def test_pvwattsv7_baseline(): year = 2012 rev2_points = slice(0, 3) - res_file = TESTDATADIR + '/nsrdb/ri_100_nsrdb_{}.h5'.format(year) - sam_files = TESTDATADIR + '/SAM/i_pvwattsv7.json' - - output_request = ('cf_mean', 'cf_profile', 'dni_mean', 'dhi_mean', - 'ghi_mean', 'ac', 'dc') + res_file = TESTDATADIR + "/nsrdb/ri_100_nsrdb_{}.h5".format(year) + sam_files = TESTDATADIR + "/SAM/i_pvwattsv7.json" + + output_request = ( + "cf_mean", + "cf_profile", + "dni_mean", + "dhi_mean", + "ghi_mean", + "ac", + "dc", + ) # run reV 2.0 generation - gen = Gen('pvwattsv7', rev2_points, sam_files, res_file, - sites_per_worker=1, output_request=output_request) + gen = Gen( + "pvwattsv7", + rev2_points, + sam_files, + res_file, + sites_per_worker=1, + output_request=output_request, + ) gen.run(max_workers=1) - msg = ('PVWattsv7 cf_mean results {} did not match baseline: {}' - .format(gen.out['cf_mean'], baseline_cf_mean)) - assert np.allclose(gen.out['cf_mean'], baseline_cf_mean, - rtol=0.005, atol=0.0), msg + msg = "PVWattsv7 cf_mean results {} did not match baseline: {}".format( + gen.out["cf_mean"], baseline_cf_mean + ) + assert np.allclose( + gen.out["cf_mean"], baseline_cf_mean, rtol=0.005, atol=0.0 + ), msg for req in output_request: assert req in gen.out @@ -309,20 +377,22 @@ def test_pvwatts_v5_v7(): year = 2012 rev2_points = slice(0, 3) - res_file = TESTDATADIR + '/nsrdb/ri_100_nsrdb_{}.h5'.format(year) - sam_files = TESTDATADIR + '/SAM/naris_pv_1axis_inv13.json' + res_file = TESTDATADIR + "/nsrdb/ri_100_nsrdb_{}.h5".format(year) + sam_files = TESTDATADIR + "/SAM/naris_pv_1axis_inv13.json" # run reV 2.0 generation - gen7 = Gen('pvwattsv7', rev2_points, sam_files, res_file, - sites_per_worker=1) + gen7 = Gen( + "pvwattsv7", rev2_points, sam_files, res_file, sites_per_worker=1 + ) gen7.run(max_workers=1) - gen5 = Gen('pvwattsv5', rev2_points, sam_files, res_file, - sites_per_worker=1) + gen5 = Gen( + "pvwattsv5", rev2_points, sam_files, res_file, sites_per_worker=1 + ) gen5.run(max_workers=1) - msg = 'PVwatts v5 and v7 did not match within test tolerance' - assert np.allclose(gen7.out['cf_mean'], gen5.out['cf_mean'], atol=3), msg + msg = "PVwatts v5 and v7 did not match within test tolerance" + assert np.allclose(gen7.out["cf_mean"], gen5.out["cf_mean"], atol=3), msg def test_pvwatts_v8_lifetime(): @@ -331,21 +401,35 @@ def test_pvwatts_v8_lifetime(): year = 2012 rev2_points = slice(0, 3) - res_file = TESTDATADIR + '/nsrdb/ri_100_nsrdb_{}.h5'.format(year) - sam_files = TESTDATADIR + '/SAM/i_pvwattsv8_degradation.json' + res_file = TESTDATADIR + "/nsrdb/ri_100_nsrdb_{}.h5".format(year) + sam_files = TESTDATADIR + "/SAM/i_pvwattsv8_degradation.json" - output_request = ('cf_mean', 'cf_profile', 'dni_mean', 'dhi_mean', - 'ghi_mean') + output_request = ( + "cf_mean", + "cf_profile", + "dni_mean", + "dhi_mean", + "ghi_mean", + ) # run reV 2.0 generation with valid output request - gen = Gen('pvwattsv8', rev2_points, sam_files, res_file, - sites_per_worker=1, output_request=output_request) + gen = Gen( + "pvwattsv8", + rev2_points, + sam_files, + res_file, + sites_per_worker=1, + output_request=output_request, + ) gen.run(max_workers=1) - msg = ('PVWATTSV8 cf_mean with system lifetime results {} did not match ' - 'baseline: {}'.format(gen.out['cf_mean'], baseline_cf_mean)) - assert np.allclose(gen.out['cf_mean'], baseline_cf_mean, - rtol=0.005, atol=0.0), msg + msg = ( + "PVWATTSV8 cf_mean with system lifetime results {} did not match " + "baseline: {}".format(gen.out["cf_mean"], baseline_cf_mean) + ) + assert np.allclose( + gen.out["cf_mean"], baseline_cf_mean, rtol=0.005, atol=0.0 + ), msg for req in output_request: assert req in gen.out @@ -357,19 +441,34 @@ def test_pvwatts_v8_lifetime_invalid_request(): year = 2012 rev2_points = slice(0, 3) - res_file = TESTDATADIR + '/nsrdb/ri_100_nsrdb_{}.h5'.format(year) - sam_files = TESTDATADIR + '/SAM/i_pvwattsv8_degradation.json' - - output_request_invalid = ('cf_mean', 'cf_profile', 'dni_mean', 'dhi_mean', - 'ghi_mean', 'ac', 'dc') + res_file = TESTDATADIR + "/nsrdb/ri_100_nsrdb_{}.h5".format(year) + sam_files = TESTDATADIR + "/SAM/i_pvwattsv8_degradation.json" + + output_request_invalid = ( + "cf_mean", + "cf_profile", + "dni_mean", + "dhi_mean", + "ghi_mean", + "ac", + "dc", + ) # run reV 2.0 generation with invalid output request with pytest.raises(ConfigError) as record: - gen = Gen('pvwattsv8', rev2_points, sam_files, res_file, - sites_per_worker=1, output_request=output_request_invalid) + gen = Gen( + "pvwattsv8", + rev2_points, + sam_files, + res_file, + sites_per_worker=1, + output_request=output_request_invalid, + ) gen.run(max_workers=1) - msg_pattern = ('reV can only handle the following output arrays when ' - 'modeling with `system_use_lifetime_output`') + msg_pattern = ( + "reV can only handle the following output arrays when " + "modeling with `system_use_lifetime_output`" + ) assert msg_pattern in record[0].message @@ -377,107 +476,146 @@ def test_bifacial(): """Test pvwattsv7 with bifacial panel with albedo.""" year = 2012 rev2_points = slice(0, 1) - res_file = TESTDATADIR + '/nsrdb/ri_100_nsrdb_{}.h5'.format(year) - sam_files = TESTDATADIR + '/SAM/i_pvwattsv7.json' + res_file = TESTDATADIR + "/nsrdb/ri_100_nsrdb_{}.h5".format(year) + sam_files = TESTDATADIR + "/SAM/i_pvwattsv7.json" # run reV 2.0 generation - gen = Gen('pvwattsv7', rev2_points, sam_files, res_file, - sites_per_worker=1) + gen = Gen( + "pvwattsv7", rev2_points, sam_files, res_file, sites_per_worker=1 + ) gen.run(max_workers=1) - sam_files = TESTDATADIR + '/SAM/i_pvwattsv7_bifacial.json' + sam_files = TESTDATADIR + "/SAM/i_pvwattsv7_bifacial.json" # run reV 2.0 generation - output_request = ('cf_mean', 'cf_profile', 'surface_albedo') - gen_bi = Gen('pvwattsv7', rev2_points, sam_files, res_file, - sites_per_worker=1, output_request=output_request) + output_request = ("cf_mean", "cf_profile", "surface_albedo") + gen_bi = Gen( + "pvwattsv7", + rev2_points, + sam_files, + res_file, + sites_per_worker=1, + output_request=output_request, + ) gen_bi.run(max_workers=1) - assert 'surface_albedo' in gen_bi.out - assert all(gen_bi.out['cf_mean'] > gen.out['cf_mean']) - assert np.isclose(gen.out['cf_mean'][0], 0.151, atol=0.005) - assert np.isclose(gen_bi.out['cf_mean'][0], 0.162, atol=0.005) + assert "surface_albedo" in gen_bi.out + assert all(gen_bi.out["cf_mean"] > gen.out["cf_mean"]) + assert np.isclose(gen.out["cf_mean"][0], 0.151, atol=0.005) + assert np.isclose(gen_bi.out["cf_mean"][0], 0.162, atol=0.005) def test_gen_input_mods(): - """Test that the gen workers do not modify the top level input SAM config - """ + """Test that the gen workers do not modify the top level input SAM + config""" year = 2012 rev2_points = slice(0, 5) - res_file = TESTDATADIR + '/nsrdb/ri_100_nsrdb_{}.h5'.format(year) - sam_files = TESTDATADIR + '/SAM/i_pvwatts_fixed_lat_tilt.json' + res_file = TESTDATADIR + "/nsrdb/ri_100_nsrdb_{}.h5".format(year) + sam_files = TESTDATADIR + "/SAM/i_pvwatts_fixed_lat_tilt.json" # run reV 2.0 generation - gen = Gen('pvwattsv7', rev2_points, sam_files, res_file, - sites_per_worker=1) + gen = Gen( + "pvwattsv7", rev2_points, sam_files, res_file, sites_per_worker=1 + ) gen.run(max_workers=1) for i in range(5): inputs = gen.project_points[i][1] - assert inputs['tilt'] == MetaKeyName.LATITUDE + assert inputs["tilt"] == MetaKeyName.LATITUDE def test_gen_input_pass_through(): - """Test the ability for reV gen to pass through inputs from the sam config. - """ - output_request = ('cf_mean', 'gcr', 'azimuth') + """Test the ability for reV gen to pass through inputs from the sam + config.""" + output_request = ("cf_mean", "gcr", "azimuth") year = 2012 rev2_points = slice(0, 2) - res_file = TESTDATADIR + '/nsrdb/ri_100_nsrdb_{}.h5'.format(year) - sam_files = TESTDATADIR + '/SAM/i_pvwatts_fixed_lat_tilt.json' + res_file = TESTDATADIR + "/nsrdb/ri_100_nsrdb_{}.h5".format(year) + sam_files = TESTDATADIR + "/SAM/i_pvwatts_fixed_lat_tilt.json" # run reV 2.0 generation - gen = Gen('pvwattsv7', rev2_points, sam_files, res_file, - sites_per_worker=1, output_request=output_request) + gen = Gen( + "pvwattsv7", + rev2_points, + sam_files, + res_file, + sites_per_worker=1, + output_request=output_request, + ) gen.run(max_workers=1) - assert 'gcr' in gen.out - assert 'azimuth' in gen.out + assert "gcr" in gen.out + assert "azimuth" in gen.out - output_request = ('cf_mean', 'gcr', 'azimuth', 'tilt') + output_request = ("cf_mean", "gcr", "azimuth", "tilt") with pytest.raises(ExecutionError): - gen = Gen('pvwattsv7', rev2_points, sam_files, res_file, - sites_per_worker=1, output_request=output_request) + gen = Gen( + "pvwattsv7", + rev2_points, + sam_files, + res_file, + sites_per_worker=1, + output_request=output_request, + ) gen.run(max_workers=1) def test_gen_pv_site_data(): """Test site specific SAM input config via site_data arg""" - output_request = ('cf_mean', 'gcr', 'azimuth', 'losses') + output_request = ("cf_mean", "gcr", "azimuth", "losses") year = 2012 rev2_points = slice(0, 5) - res_file = TESTDATADIR + '/nsrdb/ri_100_nsrdb_{}.h5'.format(year) - sam_files = TESTDATADIR + '/SAM/i_pvwatts_fixed_lat_tilt.json' + res_file = TESTDATADIR + "/nsrdb/ri_100_nsrdb_{}.h5".format(year) + sam_files = TESTDATADIR + "/SAM/i_pvwatts_fixed_lat_tilt.json" # run reV 2.0 generation - baseline = Gen('pvwattsv7', rev2_points, sam_files, res_file, - sites_per_worker=1, output_request=output_request) + baseline = Gen( + "pvwattsv7", + rev2_points, + sam_files, + res_file, + sites_per_worker=1, + output_request=output_request, + ) baseline.run(max_workers=1) - site_data = pd.DataFrame({MetaKeyName.GID: np.arange(2), - 'losses': np.ones(2)}) - test = Gen('pvwattsv7', rev2_points, sam_files, res_file, - sites_per_worker=1, output_request=output_request, - site_data=site_data) + site_data = pd.DataFrame( + {MetaKeyName.GID: np.arange(2), "losses": np.ones(2)} + ) + test = Gen( + "pvwattsv7", + rev2_points, + sam_files, + res_file, + sites_per_worker=1, + output_request=output_request, + site_data=site_data, + ) test.run(max_workers=1) - assert all(test.out['cf_mean'][0:2] > baseline.out['cf_mean'][0:2]) - assert np.allclose(test.out['cf_mean'][2:], baseline.out['cf_mean'][2:]) - assert np.allclose(test.out['losses'][0:2], np.ones(2)) - assert np.allclose(test.out['losses'][2:], 14.07566 * np.ones(3)) + assert all(test.out["cf_mean"][0:2] > baseline.out["cf_mean"][0:2]) + assert np.allclose(test.out["cf_mean"][2:], baseline.out["cf_mean"][2:]) + assert np.allclose(test.out["losses"][0:2], np.ones(2)) + assert np.allclose(test.out["losses"][2:], 14.07566 * np.ones(3)) def test_clipping(): """Test reV pvwattsv7 generation against baseline data""" year = 2012 rev2_points = slice(0, 3) - res_file = TESTDATADIR + '/nsrdb/ri_100_nsrdb_{}.h5'.format(year) - sam_files = TESTDATADIR + '/SAM/i_pvwattsv7.json' + res_file = TESTDATADIR + "/nsrdb/ri_100_nsrdb_{}.h5".format(year) + sam_files = TESTDATADIR + "/SAM/i_pvwattsv7.json" - output_request = ('ac', 'dc', 'clipped_power') + output_request = ("ac", "dc", "clipped_power") # run reV 2.0 generation - gen = Gen('pvwattsv7', rev2_points, sam_files, res_file, - sites_per_worker=1, output_request=output_request) + gen = Gen( + "pvwattsv7", + rev2_points, + sam_files, + res_file, + sites_per_worker=1, + output_request=output_request, + ) gen.run(max_workers=1) - ac = gen.out['ac'] - dc = gen.out['dc'] - clipped = gen.out['clipped_power'] + ac = gen.out["ac"] + dc = gen.out["dc"] + clipped = gen.out["clipped_power"] mask = ac < ac.max() dc_ac = dc[~mask] - ac[~mask] @@ -492,21 +630,36 @@ def test_detailed_pv_baseline(): year = 2012 rev2_points = slice(0, 3) - res_file = TESTDATADIR + '/nsrdb/ri_100_nsrdb_{}.h5'.format(year) - sam_files = TESTDATADIR + '/SAM/i_pvsamv1.json' - - output_request = ('cf_mean', 'cf_profile', 'dni_mean', 'dhi_mean', - 'ghi_mean', 'ac', 'dc') + res_file = TESTDATADIR + "/nsrdb/ri_100_nsrdb_{}.h5".format(year) + sam_files = TESTDATADIR + "/SAM/i_pvsamv1.json" + + output_request = ( + "cf_mean", + "cf_profile", + "dni_mean", + "dhi_mean", + "ghi_mean", + "ac", + "dc", + ) # run reV 2.0 generation - gen = Gen('pvsamv1', rev2_points, sam_files, res_file, - sites_per_worker=1, output_request=output_request) + gen = Gen( + "pvsamv1", + rev2_points, + sam_files, + res_file, + sites_per_worker=1, + output_request=output_request, + ) gen.run(max_workers=1) - msg = ('PVSAMv1 cf_mean results {} did not match baseline: {}' - .format(gen.out['cf_mean'], baseline_cf_mean)) - assert np.allclose(gen.out['cf_mean'], baseline_cf_mean, - rtol=0.005, atol=0.0), msg + msg = "PVSAMv1 cf_mean results {} did not match baseline: {}".format( + gen.out["cf_mean"], baseline_cf_mean + ) + assert np.allclose( + gen.out["cf_mean"], baseline_cf_mean, rtol=0.005, atol=0.0 + ), msg for req in output_request: assert req in gen.out @@ -519,21 +672,37 @@ def test_detailed_pv_bifacial(): year = 2012 rev2_points = slice(0, 3) - res_file = TESTDATADIR + '/nsrdb/ri_100_nsrdb_{}.h5'.format(year) - sam_files = TESTDATADIR + '/SAM/i_pvsamv1_bifacial.json' - - output_request = ('cf_mean', 'cf_profile', 'dni_mean', 'dhi_mean', - 'ghi_mean', 'ac', 'dc', 'surface_albedo') + res_file = TESTDATADIR + "/nsrdb/ri_100_nsrdb_{}.h5".format(year) + sam_files = TESTDATADIR + "/SAM/i_pvsamv1_bifacial.json" + + output_request = ( + "cf_mean", + "cf_profile", + "dni_mean", + "dhi_mean", + "ghi_mean", + "ac", + "dc", + "surface_albedo", + ) # run reV 2.0 generation - gen = Gen('pvsamv1', rev2_points, sam_files, res_file, - sites_per_worker=1, output_request=output_request) + gen = Gen( + "pvsamv1", + rev2_points, + sam_files, + res_file, + sites_per_worker=1, + output_request=output_request, + ) gen.run(max_workers=1) - msg = ('PVSAMv1 cf_mean results {} did not match baseline: {}' - .format(gen.out['cf_mean'], baseline_cf_mean)) - assert np.allclose(gen.out['cf_mean'], baseline_cf_mean, - rtol=0.005, atol=0.0), msg + msg = "PVSAMv1 cf_mean results {} did not match baseline: {}".format( + gen.out["cf_mean"], baseline_cf_mean + ) + assert np.allclose( + gen.out["cf_mean"], baseline_cf_mean, rtol=0.005, atol=0.0 + ), msg for req in output_request: assert req in gen.out @@ -544,38 +713,70 @@ def test_pv_clearsky(): """Test basic clearsky functionality""" year = 2012 rev2_points = slice(0, 3) - res_file = TESTDATADIR + '/nsrdb/ri_100_nsrdb_{}.h5'.format(year) - sam_files = TESTDATADIR + '/SAM/naris_pv_1axis_inv13.json' - sam_files_cs = TESTDATADIR + '/SAM/naris_pv_1axis_inv13_cs.json' - - output_request = ('cf_mean', 'cf_profile', 'dni_mean', 'dhi_mean', - 'ghi_mean', 'ac', 'dc') - output_request_cs = ('cf_mean', 'cf_profile', 'clearsky_dni_mean', - 'clearsky_dhi_mean', 'clearsky_ghi_mean', 'ac', 'dc') + res_file = TESTDATADIR + "/nsrdb/ri_100_nsrdb_{}.h5".format(year) + sam_files = TESTDATADIR + "/SAM/naris_pv_1axis_inv13.json" + sam_files_cs = TESTDATADIR + "/SAM/naris_pv_1axis_inv13_cs.json" + + output_request = ( + "cf_mean", + "cf_profile", + "dni_mean", + "dhi_mean", + "ghi_mean", + "ac", + "dc", + ) + output_request_cs = ( + "cf_mean", + "cf_profile", + "clearsky_dni_mean", + "clearsky_dhi_mean", + "clearsky_ghi_mean", + "ac", + "dc", + ) with tempfile.TemporaryDirectory() as td: - res_cs = os.path.join(td, 'res_cs_{}.h5'.format(year)) + res_cs = os.path.join(td, "res_cs_{}.h5".format(year)) shutil.copy(res_file, res_cs) - with Outputs(res_cs, mode='a') as f: - f.write_dataset('clearsky_ghi', f['ghi'], np.float32) - f.write_dataset('clearsky_dni', f['dni'], np.float32) - f.write_dataset('clearsky_dhi', f['dhi'], np.float32) + with Outputs(res_cs, mode="a") as f: + f.write_dataset("clearsky_ghi", f["ghi"], np.float32) + f.write_dataset("clearsky_dni", f["dni"], np.float32) + f.write_dataset("clearsky_dhi", f["dhi"], np.float32) with pytest.raises(ResourceRuntimeError): - gen = Gen('pvwattsv7', rev2_points, sam_files_cs, res_file, - sites_per_worker=1, output_request=output_request_cs) + gen = Gen( + "pvwattsv7", + rev2_points, + sam_files_cs, + res_file, + sites_per_worker=1, + output_request=output_request_cs, + ) gen.run(max_workers=1) - gen_cs = Gen('pvwattsv7', rev2_points, sam_files_cs, res_cs, - sites_per_worker=1, output_request=output_request_cs) + gen_cs = Gen( + "pvwattsv7", + rev2_points, + sam_files_cs, + res_cs, + sites_per_worker=1, + output_request=output_request_cs, + ) gen_cs.run(max_workers=1) - gen = Gen('pvwattsv7', rev2_points, sam_files, res_file, - sites_per_worker=1, output_request=output_request) + gen = Gen( + "pvwattsv7", + rev2_points, + sam_files, + res_file, + sites_per_worker=1, + output_request=output_request, + ) gen.run(max_workers=1) for k, v in gen.out.items(): - if k in ('ghi_mean', 'dni_mean', 'dhi_mean'): - k = 'clearsky_' + k + if k in ("ghi_mean", "dni_mean", "dhi_mean"): + k = "clearsky_" + k assert np.allclose(v, gen_cs.out[k]) @@ -583,37 +784,76 @@ def test_irrad_bias_correct(): """Test reV generation with bias correction""" year = 2012 points = slice(0, 3) - res_file = TESTDATADIR + '/nsrdb/ri_100_nsrdb_{}.h5'.format(year) - sam_files = TESTDATADIR + '/SAM/i_pvwattsv7.json' - - output_request = ('cf_mean', 'cf_profile', 'dni_mean', 'dhi_mean', - 'ghi_mean', 'ac', 'dc') - - gen_base = Gen('pvwattsv7', points, sam_files, res_file, - sites_per_worker=1, output_request=output_request) + res_file = TESTDATADIR + "/nsrdb/ri_100_nsrdb_{}.h5".format(year) + sam_files = TESTDATADIR + "/SAM/i_pvwattsv7.json" + + output_request = ( + "cf_mean", + "cf_profile", + "dni_mean", + "dhi_mean", + "ghi_mean", + "ac", + "dc", + ) + + gen_base = Gen( + "pvwattsv7", + points, + sam_files, + res_file, + sites_per_worker=1, + output_request=output_request, + ) gen_base.run(max_workers=1) - bc_df = pd.DataFrame({MetaKeyName.GID: np.arange(1, 10), 'method': 'lin_irrad', - 'scalar': 1, 'adder': 50}) - gen = Gen('pvwattsv7', points, sam_files, res_file, - sites_per_worker=1, output_request=output_request, - bias_correct=bc_df) + bc_df = pd.DataFrame( + { + MetaKeyName.GID: np.arange(1, 10), + "method": "lin_irrad", + "scalar": 1, + "adder": 50, + } + ) + gen = Gen( + "pvwattsv7", + points, + sam_files, + res_file, + sites_per_worker=1, + output_request=output_request, + bias_correct=bc_df, + ) gen.run(max_workers=1) - assert (gen_base.out['cf_mean'][0] == gen.out['cf_mean'][0]).all() - assert (gen_base.out['ghi_mean'][0] == gen.out['ghi_mean'][0]).all() - assert np.allclose(gen_base.out['cf_profile'][:, 0], - gen.out['cf_profile'][:, 0]) + assert (gen_base.out["cf_mean"][0] == gen.out["cf_mean"][0]).all() + assert (gen_base.out["ghi_mean"][0] == gen.out["ghi_mean"][0]).all() + assert np.allclose( + gen_base.out["cf_profile"][:, 0], gen.out["cf_profile"][:, 0] + ) - assert (gen_base.out['cf_mean'][1:] < gen.out['cf_mean'][1:]).all() - assert (gen_base.out['ghi_mean'][1:] < gen.out['ghi_mean'][1:]).all() - mask = (gen_base.out['cf_profile'][:, 1:] <= gen.out['cf_profile'][:, 1:]) + assert (gen_base.out["cf_mean"][1:] < gen.out["cf_mean"][1:]).all() + assert (gen_base.out["ghi_mean"][1:] < gen.out["ghi_mean"][1:]).all() + mask = gen_base.out["cf_profile"][:, 1:] <= gen.out["cf_profile"][:, 1:] assert (mask.sum() / mask.size) > 0.99 - bc_df = pd.DataFrame({MetaKeyName.GID: np.arange(100), 'method': 'lin_irrad', - 'scalar': 1, 'adder': -1500}) - gen = Gen('pvwattsv7', points, sam_files, res_file, sites_per_worker=1, - output_request=output_request, bias_correct=bc_df) + bc_df = pd.DataFrame( + { + MetaKeyName.GID: np.arange(100), + "method": "lin_irrad", + "scalar": 1, + "adder": -1500, + } + ) + gen = Gen( + "pvwattsv7", + points, + sam_files, + res_file, + sites_per_worker=1, + output_request=output_request, + bias_correct=bc_df, + ) gen.run(max_workers=2) for arr in gen.out.values(): assert (arr == 0).all() @@ -625,61 +865,90 @@ def test_ac_outputs(): year = 2012 rev2_points = slice(0, 3) - res_file = TESTDATADIR + '/nsrdb/ri_100_nsrdb_{}.h5'.format(year) - sam_files = TESTDATADIR + '/SAM/i_pvwattsv8.json' - - output_request = ('cf_mean', 'cf_mean_ac', 'cf_profile', 'cf_profile_ac', - 'system_capacity', 'system_capacity_ac', 'ac', 'dc', - 'dc_ac_ratio') + res_file = TESTDATADIR + "/nsrdb/ri_100_nsrdb_{}.h5".format(year) + sam_files = TESTDATADIR + "/SAM/i_pvwattsv8.json" + + output_request = ( + "cf_mean", + "cf_mean_ac", + "cf_profile", + "cf_profile_ac", + "system_capacity", + "system_capacity_ac", + "ac", + "dc", + "dc_ac_ratio", + ) # run reV 2.0 generation - gen = Gen('pvwattsv8', rev2_points, sam_files, res_file, - sites_per_worker=1, output_request=output_request) + gen = Gen( + "pvwattsv8", + rev2_points, + sam_files, + res_file, + sites_per_worker=1, + output_request=output_request, + ) gen.run(max_workers=1) - msg = ('PVWattsv8 cf_mean results {} did not match baseline: {}' - .format(gen.out['cf_mean'], baseline_cf_mean)) - assert np.allclose(gen.out['cf_mean'], baseline_cf_mean, - rtol=0.005, atol=0.0), msg + msg = "PVWattsv8 cf_mean results {} did not match baseline: {}".format( + gen.out["cf_mean"], baseline_cf_mean + ) + assert np.allclose( + gen.out["cf_mean"], baseline_cf_mean, rtol=0.005, atol=0.0 + ), msg - for req in ['cf_mean', 'cf_profile']: - ac_req = '{}_ac'.format(req) + for req in ["cf_mean", "cf_profile"]: + ac_req = "{}_ac".format(req) assert req in gen.out assert ac_req in gen.out assert (gen.out[req] <= gen.out[ac_req]).all() assert (gen.out["dc"] >= gen.out["ac"]).all() - assert np.allclose(gen.out['system_capacity'] / gen.out['dc_ac_ratio'], - gen.out['system_capacity_ac']) - - assert not np.isclose(gen.out['cf_profile'], 1).any() - assert np.isclose(gen.out['cf_profile_ac'], 1).any() - - -@pytest.mark.parametrize("bad_input", [("latitude", -91), - ("latitude", 91), - ("longitude", -181), - ("longitude", 181)]) + assert np.allclose( + gen.out["system_capacity"] / gen.out["dc_ac_ratio"], + gen.out["system_capacity_ac"], + ) + + assert not np.isclose(gen.out["cf_profile"], 1).any() + assert np.isclose(gen.out["cf_profile_ac"], 1).any() + + +@pytest.mark.parametrize( + "bad_input", + [ + ("latitude", -91), + ("latitude", 91), + ("longitude", -181), + ("longitude", 181), + ], +) def test_bad_loc_inputs(bad_input): """Test that error is thrown for bad lat/lon inputs.""" - res_file = TESTDATADIR + '/nsrdb/ri_100_nsrdb_{}.h5'.format(2012) - sam_files = TESTDATADIR + '/SAM/naris_pv_1axis_inv13.json' + res_file = TESTDATADIR + "/nsrdb/ri_100_nsrdb_{}.h5".format(2012) + sam_files = TESTDATADIR + "/SAM/naris_pv_1axis_inv13.json" col, val = bad_input with tempfile.TemporaryDirectory() as td: - res_file_bad = os.path.join(td, 'res_bad_loc.h5') + res_file_bad = os.path.join(td, "res_bad_loc.h5") shutil.copy(res_file, res_file_bad) - with Outputs(res_file_bad, mode='a') as f: + with Outputs(res_file_bad, mode="a") as f: meta = f.meta.copy() meta.loc[0, col] = val f.meta = meta - gen = Gen('pvwattsv8', slice(0, 3), sam_files, res_file_bad, - output_request=('cf_profile',), sites_per_worker=50) + gen = Gen( + "pvwattsv8", + slice(0, 3), + sam_files, + res_file_bad, + output_request=("cf_profile",), + sites_per_worker=50, + ) with pytest.raises(ValueError): gen.run(max_workers=1) -def execute_pytest(capture='all', flags='-rapP'): +def execute_pytest(capture="all", flags="-rapP"): """Execute module as pytest with detailed summary report. Parameters @@ -692,8 +961,8 @@ def execute_pytest(capture='all', flags='-rapP'): """ fname = os.path.basename(__file__) - pytest.main(['-q', '--show-capture={}'.format(capture), fname, flags]) + pytest.main(["-q", "--show-capture={}".format(capture), fname, flags]) -if __name__ == '__main__': +if __name__ == "__main__": execute_pytest() diff --git a/tests/test_nrwal.py b/tests/test_nrwal.py index 8f2a6b201..fd4cb19ac 100644 --- a/tests/test_nrwal.py +++ b/tests/test_nrwal.py @@ -248,7 +248,7 @@ def test_nrwal_csv(out_fn): out_fpath = rev_nrwal.run(csv_output=True, out_fpath=out_fpath) expected_message_out = [ - "`save_raw` option not allowed with " "`csv_output`" + "`save_raw` option not allowed with `csv_output`" ] for r, m in zip(record, expected_message_out): warn_msg = r.message.args[0] diff --git a/tests/test_sam.py b/tests/test_sam.py index 4413e30d9..3cad13a41 100644 --- a/tests/test_sam.py +++ b/tests/test_sam.py @@ -21,6 +21,7 @@ ) from reV.SAM.generation import PvWattsv5, PvWattsv7, PvWattsv8 from reV.SAM.version_checker import PySamVersionChecker +from reV.utilities import MetaKeyName from reV.utilities.exceptions import InputError, PySAMVersionWarning diff --git a/tests/test_supply_curve_aggregation.py b/tests/test_supply_curve_aggregation.py index 9ef1ba243..3276a335b 100644 --- a/tests/test_supply_curve_aggregation.py +++ b/tests/test_supply_curve_aggregation.py @@ -12,6 +12,7 @@ from reV import TESTDATADIR from reV.supply_curve.aggregation import Aggregation +from reV.utilities import MetaKeyName EXCL = os.path.join(TESTDATADIR, 'ri_exclusions/ri_exclusions.h5') RES = os.path.join(TESTDATADIR, 'nsrdb/ri_100_nsrdb_2012.h5') diff --git a/tests/test_supply_curve_sc_aggregation.py b/tests/test_supply_curve_sc_aggregation.py index 9eed6323e..3539b99b3 100644 --- a/tests/test_supply_curve_sc_aggregation.py +++ b/tests/test_supply_curve_sc_aggregation.py @@ -362,7 +362,7 @@ def test_agg_scalar_excl(): diff = summary_base[dset].values / summary_with_weights[dset].values msg = ("Fractional exclusions failed for {} which has values {} and {}" .format(dset, summary_base[dset].values, - summary_with_weights[dset].values)) + summary_with_weights[dset].values)) assert all(diff == 2), msg for i in summary_base.index: From 45e8717851c018336dcc8a52b5d5c0bb7a4558b2 Mon Sep 17 00:00:00 2001 From: bnb32 Date: Mon, 13 May 2024 12:37:08 -0600 Subject: [PATCH 11/61] seemlingly unused numpy import actually needed for eval call on string with `np.` --- reV/econ/economies_of_scale.py | 1 + 1 file changed, 1 insertion(+) diff --git a/reV/econ/economies_of_scale.py b/reV/econ/economies_of_scale.py index cfcde6223..3a7241828 100644 --- a/reV/econ/economies_of_scale.py +++ b/reV/econ/economies_of_scale.py @@ -7,6 +7,7 @@ import logging import re +import numpy as np # noqa: F401 import pandas as pd from rex.utilities.utilities import check_eval_str From b64d8316dfb1969a6a15de1c89aedb4d4db4db8a Mon Sep 17 00:00:00 2001 From: bnb32 Date: Mon, 13 May 2024 12:41:19 -0600 Subject: [PATCH 12/61] need pylint disable=unused-import statement also. --- reV/econ/economies_of_scale.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reV/econ/economies_of_scale.py b/reV/econ/economies_of_scale.py index 3a7241828..a51a2c5e6 100644 --- a/reV/econ/economies_of_scale.py +++ b/reV/econ/economies_of_scale.py @@ -7,7 +7,7 @@ import logging import re -import numpy as np # noqa: F401 +import numpy as np # pylint: disable=unused-import # noqa: F401 import pandas as pd from rex.utilities.utilities import check_eval_str From 71e4858af5543652c0c307aabe316b429145d764 Mon Sep 17 00:00:00 2001 From: bnb32 Date: Mon, 13 May 2024 17:15:02 -0600 Subject: [PATCH 13/61] fix: case insensitive replacement broke some gen tests. --- reV/SAM/generation.py | 4 +- tests/test_curtailment.py | 278 ++++++++++++++++---------------------- tests/test_gen_linear.py | 7 +- tests/test_gen_swh.py | 8 +- 4 files changed, 128 insertions(+), 169 deletions(-) diff --git a/reV/SAM/generation.py b/reV/SAM/generation.py index 67698483d..08ec4aa3d 100644 --- a/reV/SAM/generation.py +++ b/reV/SAM/generation.py @@ -666,9 +666,9 @@ def _create_pysam_wfile(self, resource, meta): m["State"] = m["state"].apply(lambda x: "-" if x == "None" else x) m["Country"] = m["country"].apply(lambda x: "-" if x == "None" else x) m["Latitude"] = m[MetaKeyName.LATITUDE] - m[MetaKeyName.LONGITUDE] = m[MetaKeyName.LONGITUDE] + m["Longitude"] = m[MetaKeyName.LONGITUDE] m["Time Zone"] = timezone - m[MetaKeyName.ELEVATION] = m[MetaKeyName.ELEVATION] + m["Elevation"] = m[MetaKeyName.ELEVATION] m["Local Time Zone"] = timezone m["Dew Point Units"] = "c" m["DHI Units"] = "w/m2" diff --git a/tests/test_curtailment.py b/tests/test_curtailment.py index 2b8e34c0e..000b5a72d 100644 --- a/tests/test_curtailment.py +++ b/tests/test_curtailment.py @@ -5,7 +5,6 @@ @author: gbuster """ - import os from copy import deepcopy @@ -23,20 +22,18 @@ from reV.utilities.curtailment import curtail -def get_curtailment(year, curt_fn="curtailment.json"): - """Get curtailed and non-curtailed resource objects, and project points""" - res_file = os.path.join( - TESTDATADIR, "wtk/", "ri_100_wtk_{}.h5".format(year) - ) +def get_curtailment(year, curt_fn='curtailment.json'): + """Get the curtailed and non-curtailed resource objects, and project points + """ + res_file = os.path.join(TESTDATADIR, 'wtk/', + 'ri_100_wtk_{}.h5'.format(year)) sam_files = os.path.join( - TESTDATADIR, "SAM/wind_gen_standard_losses_0.json" - ) - curtailment = os.path.join(TESTDATADIR, "config/", curt_fn) - pp = ProjectPoints( - slice(0, 100), sam_files, "windpower", curtailment=curtailment - ) - - resource = RevPySam.get_sam_res(res_file, pp, "windpower") + TESTDATADIR, 'SAM/wind_gen_standard_losses_0.json') + curtailment = os.path.join(TESTDATADIR, 'config/', curt_fn) + pp = ProjectPoints(slice(0, 100), sam_files, 'windpower', + curtailment=curtailment) + + resource = RevPySam.get_sam_res(res_file, pp, 'windpower') non_curtailed_res = deepcopy(resource) out = curtail(resource, pp.curtailment, random_seed=0) @@ -44,9 +41,11 @@ def get_curtailment(year, curt_fn="curtailment.json"): return out, non_curtailed_res, pp -@pytest.mark.parametrize( - ("year", "site"), [("2012", 0), ("2012", 10), ("2013", 0), ("2013", 10)] -) +@pytest.mark.parametrize(('year', 'site'), + [('2012', 0), + ('2012', 10), + ('2013', 0), + ('2013', 10)]) def test_cf_curtailment(year, site): """Run Wind generation and ensure that the cf_profile is zero when curtailment is expected. @@ -54,45 +53,36 @@ def test_cf_curtailment(year, site): Note that the probability of curtailment must be 1 for this to succeed. """ - res_file = os.path.join(TESTDATADIR, "wtk/ri_100_wtk_{}.h5".format(year)) - sam_files = os.path.join( - TESTDATADIR, "SAM/wind_gen_standard_losses_0.json" - ) + res_file = os.path.join(TESTDATADIR, + 'wtk/ri_100_wtk_{}.h5'.format(year)) + sam_files = os.path.join(TESTDATADIR, + 'SAM/wind_gen_standard_losses_0.json') - curtailment = os.path.join(TESTDATADIR, "config/", "curtailment.json") + curtailment = os.path.join(TESTDATADIR, 'config/', 'curtailment.json') points = slice(site, site + 1) # run reV 2.0 generation - gen = Gen( - "windpower", - points, - sam_files, - res_file, - output_request=("cf_profile",), - curtailment=curtailment, - sites_per_worker=50, - scale_outputs=True, - ) + gen = Gen('windpower', points, sam_files, res_file, + output_request=('cf_profile',), curtailment=curtailment, + sites_per_worker=50, scale_outputs=True) gen.run(max_workers=1) results, check_curtailment = test_res_curtailment(year, site=site) - results["cf_profile"] = gen.out["cf_profile"].flatten() + results['cf_profile'] = gen.out['cf_profile'].flatten() # was capacity factor NOT curtailed? - check_cf = gen.out["cf_profile"].flatten() != 0 + check_cf = (gen.out['cf_profile'].flatten() != 0) # Were all thresholds met and windspeed NOT curtailed? check = check_curtailment & check_cf - msg = ( - "All curtailment thresholds were met and cf_profile " - "was not curtailed!" - ) + msg = ('All curtailment thresholds were met and cf_profile ' + 'was not curtailed!') assert np.sum(check) == 0, msg return results -@pytest.mark.parametrize("year", ["2012", "2013"]) +@pytest.mark.parametrize('year', ['2012', '2013']) def test_curtailment_res_mean(year): """Run Wind generation and ensure that the cf_profile is zero when curtailment is expected. @@ -100,171 +90,136 @@ def test_curtailment_res_mean(year): Note that the probability of curtailment must be 1 for this to succeed. """ - res_file = os.path.join(TESTDATADIR, "wtk/ri_100_wtk_{}.h5".format(year)) - sam_files = os.path.join( - TESTDATADIR, "SAM/wind_gen_standard_losses_0.json" - ) + res_file = os.path.join(TESTDATADIR, + 'wtk/ri_100_wtk_{}.h5'.format(year)) + sam_files = os.path.join(TESTDATADIR, + 'SAM/wind_gen_standard_losses_0.json') - curtailment = os.path.join(TESTDATADIR, "config/", "curtailment.json") + curtailment = os.path.join(TESTDATADIR, 'config/', 'curtailment.json') points = slice(0, 100) - output_request = ("cf_mean", "ws_mean") - pc = Gen.get_pc( - points, - None, - sam_files, - "windpower", - sites_per_worker=50, - res_file=res_file, - curtailment=curtailment, - ) - - resources = RevPySam.get_sam_res( - res_file, pc.project_points, pc.project_points.tech, output_request - ) - truth = resources["mean_windspeed"] + output_request = ('cf_mean', 'ws_mean') + pc = Gen.get_pc(points, None, sam_files, 'windpower', + sites_per_worker=50, res_file=res_file, + curtailment=curtailment) + + resources = RevPySam.get_sam_res(res_file, + pc.project_points, + pc.project_points.tech, + output_request) + truth = resources['mean_windspeed'] # run reV 2.0 generation - gen = Gen( - "windpower", - points, - sam_files, - res_file, - output_request=output_request, - curtailment=curtailment, - sites_per_worker=50, - scale_outputs=True, - ) + gen = Gen('windpower', points, sam_files, res_file, + output_request=output_request, curtailment=curtailment, + sites_per_worker=50, scale_outputs=True) gen.run(max_workers=1) - test = gen.out["ws_mean"] + test = gen.out['ws_mean'] assert np.allclose(truth, test, rtol=0.001) -@pytest.mark.parametrize(("year", "site"), [("2012", 10), ("2013", 10)]) +@pytest.mark.parametrize(('year', 'site'), + [('2012', 10), + ('2013', 10)]) def test_random(year, site): """Run wind generation and ensure that no curtailment, 100% probability curtailment, and 50% probability curtailment result in expected decreases in the annual cf_mean. """ - res_file = os.path.join(TESTDATADIR, "wtk/ri_100_wtk_{}.h5".format(year)) - sam_files = os.path.join( - TESTDATADIR, "SAM/wind_gen_standard_losses_0.json" - ) + res_file = os.path.join(TESTDATADIR, + 'wtk/ri_100_wtk_{}.h5'.format(year)) + sam_files = os.path.join(TESTDATADIR, + 'SAM/wind_gen_standard_losses_0.json') results = [] no_curtail = None - curtailment = { - "dawn_dusk": "nautical", - "months": [4, 5, 6, 7], - "precipitation": None, - "probability": 1, - "temperature": None, - "wind_speed": 10.0, - } - prob_curtail = { - "dawn_dusk": "nautical", - "months": [4, 5, 6, 7], - "precipitation": None, - "probability": 0.5, - "temperature": None, - "wind_speed": 10.0, - } + curtailment = {"dawn_dusk": "nautical", "months": [4, 5, 6, 7], + "precipitation": None, "probability": 1, + "temperature": None, "wind_speed": 10.0} + prob_curtail = {"dawn_dusk": "nautical", "months": [4, 5, 6, 7], + "precipitation": None, "probability": 0.5, + "temperature": None, "wind_speed": 10.0} for c in [no_curtail, curtailment, prob_curtail]: + points = slice(site, site + 1) # run reV 2.0 generation and write to disk - gen = Gen( - "windpower", - points, - sam_files, - res_file, - output_request=("cf_profile",), - curtailment=c, - sites_per_worker=50, - scale_outputs=True, - ) + gen = Gen('windpower', points, sam_files, res_file, + output_request=('cf_profile',), curtailment=c, + sites_per_worker=50, scale_outputs=True) gen.run(max_workers=1) - results.append(gen.out["cf_mean"]) + results.append(gen.out['cf_mean']) - assert results[0] > results[1], "Curtailment did not decrease cf_mean!" + assert results[0] > results[1], 'Curtailment did not decrease cf_mean!' expected = (results[0] + results[1]) / 2 diff = expected - results[2] - msg = ( - "Curtailment with 50% probability did not result in 50% less " - "curtailment! No curtailment, curtailment, and 50% curtailment " - "have the following cf_means: {}".format(results) - ) + msg = ('Curtailment with 50% probability did not result in 50% less ' + 'curtailment! No curtailment, curtailment, and 50% curtailment ' + 'have the following cf_means: {}'.format(results)) assert diff <= 2, msg -@pytest.mark.parametrize(("year", "site"), [("2012", 50), ("2013", 50)]) +@pytest.mark.parametrize(('year', 'site'), + [('2012', 50), + ('2013', 50)]) def test_res_curtailment(year, site): """Test wind resource curtailment.""" out, non_curtailed_res, pp = get_curtailment(year) sza = SolarPosition( non_curtailed_res.time_index, - non_curtailed_res.meta[ - [MetaKeyName.LATITUDE, MetaKeyName.LONGITUDE] - ].values, - ).zenith + non_curtailed_res.meta[[MetaKeyName.LATITUDE, MetaKeyName.LONGITUDE]].values).zenith ti = non_curtailed_res.time_index # was it in a curtailment month? check1 = np.isin(non_curtailed_res.time_index.month, pp.curtailment.months) - check1 = np.tile( - np.expand_dims(check1, axis=1), non_curtailed_res.shape[1] - ) + check1 = np.tile(np.expand_dims(check1, axis=1), + non_curtailed_res.shape[1]) # was the non-curtailed wind speed threshold met? - check2 = ( - non_curtailed_res._res_arrays["windspeed"] < pp.curtailment.wind_speed - ) + check2 = (non_curtailed_res._res_arrays['windspeed'] + < pp.curtailment.wind_speed) # was it nighttime? - check3 = sza > pp.curtailment.dawn_dusk + check3 = (sza > pp.curtailment.dawn_dusk) # was the temperature threshold met? - check4 = out._res_arrays["temperature"] > pp.curtailment.temperature + check4 = (out._res_arrays['temperature'] > pp.curtailment.temperature) # thresholds for curtailment check_curtailment = check1 & check2 & check3 & check4 # was windspeed NOT curtailed? - check5 = out._res_arrays["windspeed"] != 0 + check5 = (out._res_arrays['windspeed'] != 0) # Were all thresholds met and windspeed NOT curtailed? check = check_curtailment & check5 - msg = ( - "All curtailment thresholds were met and windspeed " - "was not curtailed!" - ) + msg = ('All curtailment thresholds were met and windspeed ' + 'was not curtailed!') assert np.sum(check) == 0, msg # optional output df to help check results i = site - df = pd.DataFrame( - { - "i": range(len(sza)), - "curtailed_wind": out._res_arrays["windspeed"][:, i], - "original_wind": non_curtailed_res._res_arrays["windspeed"][:, i], - "temperature": out._res_arrays["temperature"][:, i], - "sza": sza[:, i], - "wind_curtail": check2[:, i], - "month_curtail": check1[:, i], - "sza_curtail": check3[:, i], - "temp_curtail": check4[:, i], - }, - index=ti, - ) - - if str(year) == "2012": - drop_day = (ti.month == 12) & (ti.day == 31) + df = pd.DataFrame({'i': range(len(sza)), + 'curtailed_wind': out._res_arrays['windspeed'][:, i], + 'original_wind': + non_curtailed_res._res_arrays['windspeed'][:, i], + 'temperature': out._res_arrays['temperature'][:, i], + 'sza': sza[:, i], + 'wind_curtail': check2[:, i], + 'month_curtail': check1[:, i], + 'sza_curtail': check3[:, i], + 'temp_curtail': check4[:, i], + }, + index=ti) + + if str(year) == '2012': + drop_day = ((ti.month == 12) & (ti.day == 31)) df = df.drop(df.index[drop_day]) check_curtailment = check_curtailment[~drop_day, :] @@ -274,25 +229,30 @@ def test_res_curtailment(year, site): def test_date_range(): """Test curtailment based on a date range vs. months list""" year = 2012 - cres_m = get_curtailment(year, curt_fn="curtailment.json")[0] - cres_dr = get_curtailment(year, curt_fn="curtailment_date_range.json")[0] + cres_m = get_curtailment(year, curt_fn='curtailment.json')[0] + cres_dr = get_curtailment(year, curt_fn='curtailment_date_range.json')[0] for df_res, site in cres_m: gid = int(site.name) - assert np.allclose(df_res["windspeed"], cres_dr[gid]["windspeed"]) + assert np.allclose(df_res['windspeed'], cres_dr[gid]['windspeed']) def test_eqn_curtailment(plot=False): """Test equation-based curtailment strategies.""" year = 2012 - curt_fn = "curtailment_eqn.json" - curtailed, non_curtailed_res, pp = get_curtailment(year, curt_fn=curt_fn) - c_config = safe_json_load(os.path.join(TESTDATADIR, "config/", curt_fn)) - c_eqn = c_config["equation"] + curt_fn = 'curtailment_eqn.json' + curtailed, non_curtailed_res, pp = get_curtailment(year, curt_fn=curt_fn) # noqa: F841 + c_config = safe_json_load(os.path.join(TESTDATADIR, 'config/', curt_fn)) + c_eqn = c_config['equation'] c_res = curtailed[0] nc_res = non_curtailed_res[0] c_mask = (c_res.windspeed == 0) & (nc_res.windspeed > 0) + # these are used in the call to eval even though your linter might think + # they are unused + temperature = nc_res['temperature'].values # noqa: F841 + wind_speed = nc_res['windspeed'].values # noqa: F841 + eval_mask = eval(c_eqn) # All curtailed windspeeds should satisfy the eqn eval but maybe not the @@ -301,19 +261,17 @@ def test_eqn_curtailment(plot=False): if plot: import matplotlib.pyplot as plt - _, ax = plt.subplots() - ax.scatter( - nc_res.loc[c_mask, "windspeed"], nc_res.loc[c_mask, "temperature"] - ) - ax.grid("on") + ax.scatter(nc_res.loc[c_mask, 'windspeed'], + nc_res.loc[c_mask, 'temperature']) + ax.grid('on') ax.set_xlim([0, 7]) ax.set_ylim([0, 30]) - ax.set_legend(["Curtailed"]) - plt.savefig("equation_based_curtailment.png") + ax.set_legend(['Curtailed']) + plt.savefig('equation_based_curtailment.png') -def execute_pytest(capture="all", flags="-rapP"): +def execute_pytest(capture='all', flags='-rapP'): """Execute module as pytest with detailed summary report. Parameters @@ -326,8 +284,8 @@ def execute_pytest(capture="all", flags="-rapP"): """ fname = os.path.basename(__file__) - pytest.main(["-q", "--show-capture={}".format(capture), fname, flags]) + pytest.main(['-q', '--show-capture={}'.format(capture), fname, flags]) -if __name__ == "__main__": +if __name__ == '__main__': execute_pytest() diff --git a/tests/test_gen_linear.py b/tests/test_gen_linear.py index 63f3dbb6b..b42717e10 100644 --- a/tests/test_gen_linear.py +++ b/tests/test_gen_linear.py @@ -6,12 +6,13 @@ @author: Mike Bannister """ import os -import pytest + import numpy as np +import pytest +from rex import Resource -from reV.generation.generation import Gen from reV import TESTDATADIR -from rex import Resource +from reV.generation.generation import Gen BASELINE = os.path.join(TESTDATADIR, 'gen_out', 'gen_ri_linear_2012.h5') RTOL = 0.01 diff --git a/tests/test_gen_swh.py b/tests/test_gen_swh.py index efeddcc55..0de443f8a 100644 --- a/tests/test_gen_swh.py +++ b/tests/test_gen_swh.py @@ -7,14 +7,14 @@ Created on 2/6/2020 @author: Mike Bannister """ -import numpy as np import os + +import numpy as np import pytest -import json +from rex import Resource -from reV.generation.generation import Gen from reV import TESTDATADIR -from rex import Resource +from reV.generation.generation import Gen RTOL = 0.01 ATOL = 0 From 04aeca7c966e0183d65f1a1df75d7aa56b694f6c Mon Sep 17 00:00:00 2001 From: bnb32 Date: Mon, 13 May 2024 17:21:01 -0600 Subject: [PATCH 14/61] nobody likes long lines --- tests/test_curtailment.py | 278 ++++++++++++++++++++++---------------- 1 file changed, 163 insertions(+), 115 deletions(-) diff --git a/tests/test_curtailment.py b/tests/test_curtailment.py index 000b5a72d..73bdd4fa2 100644 --- a/tests/test_curtailment.py +++ b/tests/test_curtailment.py @@ -5,6 +5,7 @@ @author: gbuster """ + import os from copy import deepcopy @@ -22,18 +23,21 @@ from reV.utilities.curtailment import curtail -def get_curtailment(year, curt_fn='curtailment.json'): - """Get the curtailed and non-curtailed resource objects, and project points - """ - res_file = os.path.join(TESTDATADIR, 'wtk/', - 'ri_100_wtk_{}.h5'.format(year)) +def get_curtailment(year, curt_fn="curtailment.json"): + """Get the curtailed and non-curtailed resource objects, and project + points""" + res_file = os.path.join( + TESTDATADIR, "wtk/", "ri_100_wtk_{}.h5".format(year) + ) sam_files = os.path.join( - TESTDATADIR, 'SAM/wind_gen_standard_losses_0.json') - curtailment = os.path.join(TESTDATADIR, 'config/', curt_fn) - pp = ProjectPoints(slice(0, 100), sam_files, 'windpower', - curtailment=curtailment) - - resource = RevPySam.get_sam_res(res_file, pp, 'windpower') + TESTDATADIR, "SAM/wind_gen_standard_losses_0.json" + ) + curtailment = os.path.join(TESTDATADIR, "config/", curt_fn) + pp = ProjectPoints( + slice(0, 100), sam_files, "windpower", curtailment=curtailment + ) + + resource = RevPySam.get_sam_res(res_file, pp, "windpower") non_curtailed_res = deepcopy(resource) out = curtail(resource, pp.curtailment, random_seed=0) @@ -41,11 +45,9 @@ def get_curtailment(year, curt_fn='curtailment.json'): return out, non_curtailed_res, pp -@pytest.mark.parametrize(('year', 'site'), - [('2012', 0), - ('2012', 10), - ('2013', 0), - ('2013', 10)]) +@pytest.mark.parametrize( + ("year", "site"), [("2012", 0), ("2012", 10), ("2013", 0), ("2013", 10)] +) def test_cf_curtailment(year, site): """Run Wind generation and ensure that the cf_profile is zero when curtailment is expected. @@ -53,36 +55,45 @@ def test_cf_curtailment(year, site): Note that the probability of curtailment must be 1 for this to succeed. """ - res_file = os.path.join(TESTDATADIR, - 'wtk/ri_100_wtk_{}.h5'.format(year)) - sam_files = os.path.join(TESTDATADIR, - 'SAM/wind_gen_standard_losses_0.json') + res_file = os.path.join(TESTDATADIR, "wtk/ri_100_wtk_{}.h5".format(year)) + sam_files = os.path.join( + TESTDATADIR, "SAM/wind_gen_standard_losses_0.json" + ) - curtailment = os.path.join(TESTDATADIR, 'config/', 'curtailment.json') + curtailment = os.path.join(TESTDATADIR, "config/", "curtailment.json") points = slice(site, site + 1) # run reV 2.0 generation - gen = Gen('windpower', points, sam_files, res_file, - output_request=('cf_profile',), curtailment=curtailment, - sites_per_worker=50, scale_outputs=True) + gen = Gen( + "windpower", + points, + sam_files, + res_file, + output_request=("cf_profile",), + curtailment=curtailment, + sites_per_worker=50, + scale_outputs=True, + ) gen.run(max_workers=1) results, check_curtailment = test_res_curtailment(year, site=site) - results['cf_profile'] = gen.out['cf_profile'].flatten() + results["cf_profile"] = gen.out["cf_profile"].flatten() # was capacity factor NOT curtailed? - check_cf = (gen.out['cf_profile'].flatten() != 0) + check_cf = gen.out["cf_profile"].flatten() != 0 # Were all thresholds met and windspeed NOT curtailed? check = check_curtailment & check_cf - msg = ('All curtailment thresholds were met and cf_profile ' - 'was not curtailed!') + msg = ( + "All curtailment thresholds were met and cf_profile " + "was not curtailed!" + ) assert np.sum(check) == 0, msg return results -@pytest.mark.parametrize('year', ['2012', '2013']) +@pytest.mark.parametrize("year", ["2012", "2013"]) def test_curtailment_res_mean(year): """Run Wind generation and ensure that the cf_profile is zero when curtailment is expected. @@ -90,136 +101,171 @@ def test_curtailment_res_mean(year): Note that the probability of curtailment must be 1 for this to succeed. """ - res_file = os.path.join(TESTDATADIR, - 'wtk/ri_100_wtk_{}.h5'.format(year)) - sam_files = os.path.join(TESTDATADIR, - 'SAM/wind_gen_standard_losses_0.json') + res_file = os.path.join(TESTDATADIR, "wtk/ri_100_wtk_{}.h5".format(year)) + sam_files = os.path.join( + TESTDATADIR, "SAM/wind_gen_standard_losses_0.json" + ) - curtailment = os.path.join(TESTDATADIR, 'config/', 'curtailment.json') + curtailment = os.path.join(TESTDATADIR, "config/", "curtailment.json") points = slice(0, 100) - output_request = ('cf_mean', 'ws_mean') - pc = Gen.get_pc(points, None, sam_files, 'windpower', - sites_per_worker=50, res_file=res_file, - curtailment=curtailment) - - resources = RevPySam.get_sam_res(res_file, - pc.project_points, - pc.project_points.tech, - output_request) - truth = resources['mean_windspeed'] + output_request = ("cf_mean", "ws_mean") + pc = Gen.get_pc( + points, + None, + sam_files, + "windpower", + sites_per_worker=50, + res_file=res_file, + curtailment=curtailment, + ) + + resources = RevPySam.get_sam_res( + res_file, pc.project_points, pc.project_points.tech, output_request + ) + truth = resources["mean_windspeed"] # run reV 2.0 generation - gen = Gen('windpower', points, sam_files, res_file, - output_request=output_request, curtailment=curtailment, - sites_per_worker=50, scale_outputs=True) + gen = Gen( + "windpower", + points, + sam_files, + res_file, + output_request=output_request, + curtailment=curtailment, + sites_per_worker=50, + scale_outputs=True, + ) gen.run(max_workers=1) - test = gen.out['ws_mean'] + test = gen.out["ws_mean"] assert np.allclose(truth, test, rtol=0.001) -@pytest.mark.parametrize(('year', 'site'), - [('2012', 10), - ('2013', 10)]) +@pytest.mark.parametrize(("year", "site"), [("2012", 10), ("2013", 10)]) def test_random(year, site): """Run wind generation and ensure that no curtailment, 100% probability curtailment, and 50% probability curtailment result in expected decreases in the annual cf_mean. """ - res_file = os.path.join(TESTDATADIR, - 'wtk/ri_100_wtk_{}.h5'.format(year)) - sam_files = os.path.join(TESTDATADIR, - 'SAM/wind_gen_standard_losses_0.json') + res_file = os.path.join(TESTDATADIR, "wtk/ri_100_wtk_{}.h5".format(year)) + sam_files = os.path.join( + TESTDATADIR, "SAM/wind_gen_standard_losses_0.json" + ) results = [] no_curtail = None - curtailment = {"dawn_dusk": "nautical", "months": [4, 5, 6, 7], - "precipitation": None, "probability": 1, - "temperature": None, "wind_speed": 10.0} - prob_curtail = {"dawn_dusk": "nautical", "months": [4, 5, 6, 7], - "precipitation": None, "probability": 0.5, - "temperature": None, "wind_speed": 10.0} + curtailment = { + "dawn_dusk": "nautical", + "months": [4, 5, 6, 7], + "precipitation": None, + "probability": 1, + "temperature": None, + "wind_speed": 10.0, + } + prob_curtail = { + "dawn_dusk": "nautical", + "months": [4, 5, 6, 7], + "precipitation": None, + "probability": 0.5, + "temperature": None, + "wind_speed": 10.0, + } for c in [no_curtail, curtailment, prob_curtail]: - points = slice(site, site + 1) # run reV 2.0 generation and write to disk - gen = Gen('windpower', points, sam_files, res_file, - output_request=('cf_profile',), curtailment=c, - sites_per_worker=50, scale_outputs=True) + gen = Gen( + "windpower", + points, + sam_files, + res_file, + output_request=("cf_profile",), + curtailment=c, + sites_per_worker=50, + scale_outputs=True, + ) gen.run(max_workers=1) - results.append(gen.out['cf_mean']) + results.append(gen.out["cf_mean"]) - assert results[0] > results[1], 'Curtailment did not decrease cf_mean!' + assert results[0] > results[1], "Curtailment did not decrease cf_mean!" expected = (results[0] + results[1]) / 2 diff = expected - results[2] - msg = ('Curtailment with 50% probability did not result in 50% less ' - 'curtailment! No curtailment, curtailment, and 50% curtailment ' - 'have the following cf_means: {}'.format(results)) + msg = ( + "Curtailment with 50% probability did not result in 50% less " + "curtailment! No curtailment, curtailment, and 50% curtailment " + "have the following cf_means: {}".format(results) + ) assert diff <= 2, msg -@pytest.mark.parametrize(('year', 'site'), - [('2012', 50), - ('2013', 50)]) +@pytest.mark.parametrize(("year", "site"), [("2012", 50), ("2013", 50)]) def test_res_curtailment(year, site): """Test wind resource curtailment.""" out, non_curtailed_res, pp = get_curtailment(year) sza = SolarPosition( non_curtailed_res.time_index, - non_curtailed_res.meta[[MetaKeyName.LATITUDE, MetaKeyName.LONGITUDE]].values).zenith + non_curtailed_res.meta[ + [MetaKeyName.LATITUDE, MetaKeyName.LONGITUDE] + ].values, + ).zenith ti = non_curtailed_res.time_index # was it in a curtailment month? check1 = np.isin(non_curtailed_res.time_index.month, pp.curtailment.months) - check1 = np.tile(np.expand_dims(check1, axis=1), - non_curtailed_res.shape[1]) + check1 = np.tile( + np.expand_dims(check1, axis=1), non_curtailed_res.shape[1] + ) # was the non-curtailed wind speed threshold met? - check2 = (non_curtailed_res._res_arrays['windspeed'] - < pp.curtailment.wind_speed) + check2 = ( + non_curtailed_res._res_arrays["windspeed"] < pp.curtailment.wind_speed + ) # was it nighttime? - check3 = (sza > pp.curtailment.dawn_dusk) + check3 = sza > pp.curtailment.dawn_dusk # was the temperature threshold met? - check4 = (out._res_arrays['temperature'] > pp.curtailment.temperature) + check4 = out._res_arrays["temperature"] > pp.curtailment.temperature # thresholds for curtailment check_curtailment = check1 & check2 & check3 & check4 # was windspeed NOT curtailed? - check5 = (out._res_arrays['windspeed'] != 0) + check5 = out._res_arrays["windspeed"] != 0 # Were all thresholds met and windspeed NOT curtailed? check = check_curtailment & check5 - msg = ('All curtailment thresholds were met and windspeed ' - 'was not curtailed!') + msg = ( + "All curtailment thresholds were met and windspeed " + "was not curtailed!" + ) assert np.sum(check) == 0, msg # optional output df to help check results i = site - df = pd.DataFrame({'i': range(len(sza)), - 'curtailed_wind': out._res_arrays['windspeed'][:, i], - 'original_wind': - non_curtailed_res._res_arrays['windspeed'][:, i], - 'temperature': out._res_arrays['temperature'][:, i], - 'sza': sza[:, i], - 'wind_curtail': check2[:, i], - 'month_curtail': check1[:, i], - 'sza_curtail': check3[:, i], - 'temp_curtail': check4[:, i], - }, - index=ti) - - if str(year) == '2012': - drop_day = ((ti.month == 12) & (ti.day == 31)) + df = pd.DataFrame( + { + "i": range(len(sza)), + "curtailed_wind": out._res_arrays["windspeed"][:, i], + "original_wind": non_curtailed_res._res_arrays["windspeed"][:, i], + "temperature": out._res_arrays["temperature"][:, i], + "sza": sza[:, i], + "wind_curtail": check2[:, i], + "month_curtail": check1[:, i], + "sza_curtail": check3[:, i], + "temp_curtail": check4[:, i], + }, + index=ti, + ) + + if str(year) == "2012": + drop_day = (ti.month == 12) & (ti.day == 31) df = df.drop(df.index[drop_day]) check_curtailment = check_curtailment[~drop_day, :] @@ -229,20 +275,20 @@ def test_res_curtailment(year, site): def test_date_range(): """Test curtailment based on a date range vs. months list""" year = 2012 - cres_m = get_curtailment(year, curt_fn='curtailment.json')[0] - cres_dr = get_curtailment(year, curt_fn='curtailment_date_range.json')[0] + cres_m = get_curtailment(year, curt_fn="curtailment.json")[0] + cres_dr = get_curtailment(year, curt_fn="curtailment_date_range.json")[0] for df_res, site in cres_m: gid = int(site.name) - assert np.allclose(df_res['windspeed'], cres_dr[gid]['windspeed']) + assert np.allclose(df_res["windspeed"], cres_dr[gid]["windspeed"]) def test_eqn_curtailment(plot=False): """Test equation-based curtailment strategies.""" year = 2012 - curt_fn = 'curtailment_eqn.json' - curtailed, non_curtailed_res, pp = get_curtailment(year, curt_fn=curt_fn) # noqa: F841 - c_config = safe_json_load(os.path.join(TESTDATADIR, 'config/', curt_fn)) - c_eqn = c_config['equation'] + curt_fn = "curtailment_eqn.json" + curtailed, non_curtailed_res, pp = get_curtailment(year, curt_fn=curt_fn) + c_config = safe_json_load(os.path.join(TESTDATADIR, "config/", curt_fn)) + c_eqn = c_config["equation"] c_res = curtailed[0] nc_res = non_curtailed_res[0] @@ -250,8 +296,8 @@ def test_eqn_curtailment(plot=False): # these are used in the call to eval even though your linter might think # they are unused - temperature = nc_res['temperature'].values # noqa: F841 - wind_speed = nc_res['windspeed'].values # noqa: F841 + temperature = nc_res["temperature"].values # noqa: F841 + wind_speed = nc_res["windspeed"].values # noqa: F841 eval_mask = eval(c_eqn) @@ -261,17 +307,19 @@ def test_eqn_curtailment(plot=False): if plot: import matplotlib.pyplot as plt + _, ax = plt.subplots() - ax.scatter(nc_res.loc[c_mask, 'windspeed'], - nc_res.loc[c_mask, 'temperature']) - ax.grid('on') + ax.scatter( + nc_res.loc[c_mask, "windspeed"], nc_res.loc[c_mask, "temperature"] + ) + ax.grid("on") ax.set_xlim([0, 7]) ax.set_ylim([0, 30]) - ax.set_legend(['Curtailed']) - plt.savefig('equation_based_curtailment.png') + ax.set_legend(["Curtailed"]) + plt.savefig("equation_based_curtailment.png") -def execute_pytest(capture='all', flags='-rapP'): +def execute_pytest(capture="all", flags="-rapP"): """Execute module as pytest with detailed summary report. Parameters @@ -284,8 +332,8 @@ def execute_pytest(capture='all', flags='-rapP'): """ fname = os.path.basename(__file__) - pytest.main(['-q', '--show-capture={}'.format(capture), fname, flags]) + pytest.main(["-q", "--show-capture={}".format(capture), fname, flags]) -if __name__ == '__main__': +if __name__ == "__main__": execute_pytest() From 5c66da71badd87a88bfaf9606722078c22b1efa0 Mon Sep 17 00:00:00 2001 From: ppinchuk Date: Mon, 20 May 2024 10:31:47 -0600 Subject: [PATCH 15/61] Add eos mult and reg mult to meta enum --- reV/bespoke/bespoke.py | 4 ++-- reV/supply_curve/supply_curve.py | 3 ++- reV/utilities/__init__.py | 2 ++ tests/test_bespoke.py | 8 ++++---- tests/test_supply_curve_compute.py | 6 ++++-- 5 files changed, 14 insertions(+), 9 deletions(-) diff --git a/reV/bespoke/bespoke.py b/reV/bespoke/bespoke.py index 8a48b40b3..97206fe9d 100644 --- a/reV/bespoke/bespoke.py +++ b/reV/bespoke/bespoke.py @@ -1319,12 +1319,12 @@ def run_plant_optimization(self): baseline_cost = self.plant_optimizer.capital_cost_per_kw( capacity_mw=self._baseline_cap_mw ) - self._meta["eos_mult"] = ( + self._meta[MetaKeyName.EOS_MULT] = ( self.plant_optimizer.capital_cost / self.plant_optimizer.capacity / baseline_cost ) - self._meta["reg_mult"] = self.sam_sys_inputs.get( + self._meta[MetaKeyName.REG_MULT] = self.sam_sys_inputs.get( "capital_cost_multiplier", 1 ) diff --git a/reV/supply_curve/supply_curve.py b/reV/supply_curve/supply_curve.py index 201c0a02b..24c159192 100644 --- a/reV/supply_curve/supply_curve.py +++ b/reV/supply_curve/supply_curve.py @@ -1107,7 +1107,8 @@ def _adjust_output_columns(self, columns, consider_friction): # These are essentially should-be-defaults that are not # backwards-compatible, so have to explicitly check for them extra_cols = ['ba_str', 'poi_lat', 'poi_lon', 'reinforcement_poi_lat', - 'reinforcement_poi_lon', 'eos_mult', 'reg_mult', + 'reinforcement_poi_lon', MetaKeyName.EOS_MULT, + MetaKeyName.REG_MULT, 'reinforcement_cost_per_mw', 'reinforcement_dist_km', 'n_parallel_trans', MetaKeyName.TOTAL_LCOE_FRICTION] if not consider_friction: diff --git a/reV/utilities/__init__.py b/reV/utilities/__init__.py index 48c51d8cb..ce3d19838 100644 --- a/reV/utilities/__init__.py +++ b/reV/utilities/__init__.py @@ -51,6 +51,8 @@ class MetaKeyName(str, Enum): SCALED_SC_POINT_CAPITAL_COST = 'scaled_sc_point_capital_cost' TURBINE_X_COORDS = 'turbine_x_coords' TURBINE_Y_COORDS = 'turbine_y_coords' + EOS_MULT = "eos_mult" + REG_MULT = "reg_mult" def __str__(self): return self.value diff --git a/tests/test_bespoke.py b/tests/test_bespoke.py index e5ec31215..b384b58e4 100644 --- a/tests/test_bespoke.py +++ b/tests/test_bespoke.py @@ -463,9 +463,9 @@ def test_extra_outputs(gid=33): assert 'reeds_region' in bsp.meta assert 'padus' in bsp.meta - assert 'eos_mult' in bsp.meta - assert 'reg_mult' in bsp.meta - assert np.allclose(bsp.meta['reg_mult'], 1) + assert MetaKeyName.EOS_MULT in bsp.meta + assert MetaKeyName.REG_MULT in bsp.meta + assert np.allclose(bsp.meta[MetaKeyName.REG_MULT], 1) n_turbs = round(test_eos_cap / TURB_RATING) test_eos_cap_kw = n_turbs * TURB_RATING @@ -474,7 +474,7 @@ def test_extra_outputs(gid=33): eos_mult = (bsp.plant_optimizer.capital_cost / bsp.plant_optimizer.capacity / (baseline_cost / test_eos_cap_kw)) - assert np.allclose(bsp.meta['eos_mult'], eos_mult) + assert np.allclose(bsp.meta[MetaKeyName.EOS_MULT], eos_mult) bsp.close() diff --git a/tests/test_supply_curve_compute.py b/tests/test_supply_curve_compute.py index 0c43bdcc8..48222a2c6 100644 --- a/tests/test_supply_curve_compute.py +++ b/tests/test_supply_curve_compute.py @@ -491,7 +491,8 @@ def test_least_cost_full_pass_through(): Test the full supply curve sorting passes through variables correctly """ check_cols = {'poi_lat', 'poi_lon', 'reinforcement_poi_lat', - 'reinforcement_poi_lon', 'eos_mult', 'reg_mult', + 'reinforcement_poi_lon', MetaKeyName.EOS_MULT, + MetaKeyName.REG_MULT, 'reinforcement_cost_per_mw', 'reinforcement_dist_km'} with tempfile.TemporaryDirectory() as td: trans_tables = [] @@ -523,7 +524,8 @@ def test_least_cost_simple_pass_through(): Test the simple supply curve sorting passes through variables correctly """ check_cols = {'poi_lat', 'poi_lon', 'reinforcement_poi_lat', - 'reinforcement_poi_lon', 'eos_mult', 'reg_mult', + 'reinforcement_poi_lon', MetaKeyName.EOS_MULT, + MetaKeyName.REG_MULT, 'reinforcement_cost_per_mw', 'reinforcement_dist_km'} with tempfile.TemporaryDirectory() as td: trans_tables = [] From 414f68ed074f5637f212821006f197ec024fcf55 Mon Sep 17 00:00:00 2001 From: ppinchuk Date: Mon, 20 May 2024 10:50:48 -0600 Subject: [PATCH 16/61] Walk back changes pt 1 --- examples/aws_pcluster/make_project_points.py | 4 +-- examples/marine_energy/plot_lcoe.py | 2 +- reV/bespoke/bespoke.py | 6 ++-- reV/bespoke/place_turbines.py | 2 +- reV/config/project_points.py | 36 ++++++++++---------- reV/econ/econ.py | 18 +++++----- reV/econ/economies_of_scale.py | 4 +-- reV/generation/base.py | 15 ++++---- reV/generation/generation.py | 31 ++++++++--------- reV/handlers/exclusions.py | 21 ++++++------ 10 files changed, 68 insertions(+), 71 deletions(-) diff --git a/examples/aws_pcluster/make_project_points.py b/examples/aws_pcluster/make_project_points.py index 43a69173a..04c1c34ce 100644 --- a/examples/aws_pcluster/make_project_points.py +++ b/examples/aws_pcluster/make_project_points.py @@ -21,7 +21,7 @@ print(meta[mask]) pp = meta[mask] - pp[MetaKeyName.GID] = pp.index.values + pp['gid'] = pp.index.values pp['config'] = 'def' - pp = pp[[MetaKeyName.GID, 'config']] + pp = pp[['gid', 'config']] pp.to_csv('./points_front_range.csv', index=False) diff --git a/examples/marine_energy/plot_lcoe.py b/examples/marine_energy/plot_lcoe.py index c77b038cf..e4fb3b74a 100644 --- a/examples/marine_energy/plot_lcoe.py +++ b/examples/marine_energy/plot_lcoe.py @@ -12,7 +12,7 @@ for fp in fps: df = pd.read_csv(fp) - a = plt.scatter(df.longitude, df.latitude, c=df[MetaKeyName.MEAN_LCOE], + a = plt.scatter(df.longitude, df.latitude, c=df["mean_lcoe"], s=0.5, vmin=0, vmax=1500) plt.colorbar(a, label='lcoe_fcr ($/MWh)') tag = os.path.basename(fp).replace('_agg.csv', '') diff --git a/reV/bespoke/bespoke.py b/reV/bespoke/bespoke.py index 97206fe9d..a2602abdb 100644 --- a/reV/bespoke/bespoke.py +++ b/reV/bespoke/bespoke.py @@ -581,12 +581,12 @@ def _parse_gid_map(gid_map): if gid_map.endswith(".csv"): gid_map = pd.read_csv(gid_map).to_dict() assert ( - MetaKeyName.GID in gid_map + "gid" in gid_map ), 'Need "gid" in gid_map column' assert "gid_map" in gid_map, 'Need "gid_map" in gid_map column' gid_map = { - gid_map[MetaKeyName.GID][i]: gid_map["gid_map"][i] - for i in gid_map[MetaKeyName.GID].keys() + gid_map["gid"][i]: gid_map["gid_map"][i] + for i in gid_map["gid"].keys() } elif gid_map.endswith(".json"): diff --git a/reV/bespoke/place_turbines.py b/reV/bespoke/place_turbines.py index 432372aa1..3be797231 100644 --- a/reV/bespoke/place_turbines.py +++ b/reV/bespoke/place_turbines.py @@ -363,7 +363,7 @@ def capital_cost_per_kw(self, capacity_mw): def fixed_charge_rate(self): """Fixed charge rate if input to the SAM WindPowerPD object, None if not found in inputs.""" - return self.wind_plant.sam_sys_inputs.get(MetaKeyName.FIXED_CHARGE_RATE, None) + return self.wind_plant.sam_sys_inputs.get("fixed_charge_rate", None) @property @none_until_optimized diff --git a/reV/config/project_points.py b/reV/config/project_points.py index 587c7f718..fec9d8001 100644 --- a/reV/config/project_points.py +++ b/reV/config/project_points.py @@ -272,7 +272,7 @@ def __getitem__(self, site): names (keys) and values. """ - site_bool = (self.df[MetaKeyName.GID] == site) + site_bool = (self.df["gid"] == site) try: config_id = self.df.loc[site_bool, 'config'].values[0] except (KeyError, IndexError) as ex: @@ -302,7 +302,7 @@ def df(self): ------- _df : pd.DataFrame Table of sites and corresponding SAM configuration IDs. - Has columns MetaKeyName.GID and 'config'. + Has columns "gid" and 'config'. """ return self._df @@ -387,7 +387,7 @@ def sites(self): List of integer sites (resource file gids) belonging to this instance of ProjectPoints. """ - return self.df[MetaKeyName.GID].values.tolist() + return self.df["gid"].values.tolist() @property def sites_as_slice(self): @@ -488,8 +488,8 @@ def _parse_csv(fname): Parameters ---------- fname : str - Project points .csv file (with path). Must have MetaKeyName.GID and 'config' - column names. + Project points .csv file (with path). Must have 'gid' and + 'config' column names. Returns ------- @@ -525,7 +525,7 @@ def _parse_sites(points, res_file=None): df : pd.DataFrame DataFrame mapping sites (gids) to SAM technology (config) """ - df = pd.DataFrame(columns=[MetaKeyName.GID, 'config']) + df = pd.DataFrame(columns=["gid", "config"]) if isinstance(points, int): points = [points] if isinstance(points, (list, tuple, np.ndarray)): @@ -535,7 +535,7 @@ def _parse_sites(points, res_file=None): logger.error(msg) raise RuntimeError(msg) - df[MetaKeyName.GID] = points + df["gid"] = points elif isinstance(points, slice): stop = points.stop if stop is None: @@ -550,7 +550,7 @@ def _parse_sites(points, res_file=None): else: stop = Resource(res_file).shape[1] - df[MetaKeyName.GID] = list(range(*points.indices(stop))) + df["gid"] = list(range(*points.indices(stop))) else: raise TypeError('Project Points sites needs to be set as a list, ' 'tuple, or slice, but was set as: {}' @@ -591,14 +591,14 @@ def _parse_points(cls, points, res_file=None): raise ValueError('Cannot parse Project points data from {}' .format(type(points))) - if MetaKeyName.GID not in df.columns: + if "gid" not in df.columns: raise KeyError('Project points data must contain "gid" column.') # pylint: disable=no-member if 'config' not in df.columns: df = cls._parse_sites(points["gid"].values, res_file=res_file) - gids = df[MetaKeyName.GID].values + gids = df["gid"].values if not np.array_equal(np.sort(gids), gids): msg = ('WARNING: points are not in sequential order and will be ' 'sorted! The original order is being preserved under ' @@ -606,7 +606,7 @@ def _parse_points(cls, points, res_file=None): logger.warning(msg) warn(msg) df['points_order'] = df.index.values - df = df.sort_values(MetaKeyName.GID).reset_index(drop=True) + df = df.sort_values("gid").reset_index(drop=True) return df @@ -694,13 +694,13 @@ def index(self, gid): ind : int Row index of gid in the project points dataframe. """ - if gid not in self._df[MetaKeyName.GID].values: + if gid not in self._df["gid"].values: e = ('Requested resource gid {} is not present in the project ' 'points dataframe. Cannot return row index.'.format(gid)) logger.error(e) raise ConfigError(e) - ind = np.where(self._df[MetaKeyName.GID] == gid)[0][0] + ind = np.where(self._df["gid"] == gid)[0][0] return ind @@ -750,7 +750,7 @@ def _check_points_config_mapping(self): logger.error(msg) raise ConfigError(msg) - def join_df(self, df2, key=MetaKeyName.GID): + def join_df(self, df2, key="gid"): """Join new df2 to the _df attribute using the _df's gid as pkey. This can be used to add site-specific data to the project_points, @@ -770,7 +770,7 @@ def join_df(self, df2, key=MetaKeyName.GID): """ # ensure df2 doesnt have any duplicate columns for suffix reasons. df2_cols = [c for c in df2.columns if c not in self._df or c == key] - self._df = pd.merge(self._df, df2[df2_cols], how='left', left_on=MetaKeyName.GID, + self._df = pd.merge(self._df, df2[df2_cols], how='left', left_on="gid", right_on=key, copy=False, validate='1:1') def get_sites_from_config(self, config): @@ -787,7 +787,7 @@ def get_sites_from_config(self, config): List of sites associated with the requested configuration ID. If the configuration ID is not recognized, an empty list is returned. """ - sites = self.df.loc[(self.df['config'] == config), MetaKeyName.GID].values + sites = self.df.loc[(self.df['config'] == config), "gid"].values return list(sites) @@ -941,8 +941,8 @@ def lat_lon_coords(cls, lat_lons, res_file, sam_configs, tech=None, if 'points_order' in pp.df: lat_lons = lat_lons[pp.df['points_order'].values] - pp._df[MetaKeyName.LATITUDE] = lat_lons[:, 0] - pp._df[MetaKeyName.LONGITUDE] = lat_lons[:, 1] + pp._df["latitude"] = lat_lons[:, 0] + pp._df["longitude"] = lat_lons[:, 1] return pp diff --git a/reV/econ/econ.py b/reV/econ/econ.py index 8b0cd7403..8f5920983 100644 --- a/reV/econ/econ.py +++ b/reV/econ/econ.py @@ -17,7 +17,7 @@ from reV.SAM.econ import LCOE as SAM_LCOE from reV.SAM.econ import SingleOwner from reV.SAM.windbos import WindBos -from reV.utilities import MetaKeyName, ModuleName +from reV.utilities import ModuleName from reV.utilities.exceptions import ExecutionError, OffshoreWindInputWarning logger = logging.getLogger(__name__) @@ -210,10 +210,9 @@ def meta(self): with Outputs(self.cf_file) as cfh: # only take meta that belongs to this project's site list self._meta = cfh.meta[ - cfh.meta[MetaKeyName.GID].isin(self.points_control.sites)] + cfh.meta["gid"].isin(self.points_control.sites)] - if (MetaKeyName.OFFSHORE in self._meta and - self._meta[MetaKeyName.OFFSHORE].sum() > 1): + if ("offshore" in self._meta and self._meta["offshore"].sum() > 1): w = ('Found offshore sites in econ meta data. ' 'This functionality has been deprecated. ' 'Please run the reV offshore module to ' @@ -222,8 +221,7 @@ def meta(self): logger.warning(w) elif self._meta is None and self.cf_file is None: - self._meta = pd.DataFrame({MetaKeyName.GID: - self.points_control.sites}) + self._meta = pd.DataFrame({"gid": self.points_control.sites}) return self._meta @@ -266,8 +264,8 @@ def _econ_append_pc(pp, cf_file, sites_per_worker=None): res_kwargs = {'hsds': hsds} with res_cls(cf_file, **res_kwargs) as f: - gid0 = f.meta[MetaKeyName.GID].values[0] - gid1 = f.meta[MetaKeyName.GID].values[-1] + gid0 = f.meta["gid"].values[0] + gid1 = f.meta["gid"].values[-1] i0 = pp.index(gid0) i1 = pp.index(gid1) + 1 @@ -353,7 +351,7 @@ def _run_single_worker(pc, econ_fun, output_request, **kwargs): # Extract the site df from the project points df. site_df = pc.project_points.df - site_df = site_df.set_index(MetaKeyName.GID, drop=True) + site_df = site_df.set_index("gid", drop=True) # SAM execute econ analysis based on output request try: @@ -495,7 +493,7 @@ def run(self, out_fpath=None, max_workers=1, timeout=1800, self._init_out_arrays() diff = list(set(self.points_control.sites) - - set(self.meta[MetaKeyName.GID].values)) + - set(self.meta["gid"].values)) if diff: raise Exception('The following analysis sites were requested ' 'through project points for econ but are not ' diff --git a/reV/econ/economies_of_scale.py b/reV/econ/economies_of_scale.py index 539193aa1..92f230a7a 100644 --- a/reV/econ/economies_of_scale.py +++ b/reV/econ/economies_of_scale.py @@ -11,7 +11,7 @@ from rex.utilities.utilities import check_eval_str from reV.econ.utilities import lcoe_fcr -from reV.utilities import MetaKeyName + logger = logging.getLogger(__name__) @@ -284,7 +284,7 @@ def raw_lcoe(self): ------- lcoe : float | np.ndarray """ - key_list = [MetaKeyName.RAW_LCOE, MetaKeyName.MEAN_LCOE] + key_list = ["raw_lcoe", "mean_lcoe"] return copy.deepcopy(self._get_prioritized_keys(self._data, key_list)) @property diff --git a/reV/generation/base.py b/reV/generation/base.py index 6512f4a22..24606e13f 100644 --- a/reV/generation/base.py +++ b/reV/generation/base.py @@ -21,7 +21,7 @@ from reV.config.project_points import PointsControl, ProjectPoints from reV.handlers.outputs import Outputs from reV.SAM.version_checker import PySamVersionChecker -from reV.utilities import MetaKeyName, ModuleName, log_versions +from reV.utilities import ModuleName, log_versions from reV.utilities.exceptions import ( ExecutionError, OffshoreWindInputWarning, @@ -739,7 +739,7 @@ def _parse_site_data(self, inp): if inp is None or inp is False: # no input, just initialize dataframe with site gids as index site_data = pd.DataFrame(index=self.project_points.sites) - site_data.index.name = MetaKeyName.GID + site_data.index.name = "gid" else: # explicit input, initialize df if isinstance(inp, str): @@ -752,19 +752,18 @@ def _parse_site_data(self, inp): raise Exception('Site data input must be .csv or ' 'dataframe, but received: {}'.format(inp)) - if (MetaKeyName.GID not in site_data and - site_data.index.name != MetaKeyName.GID): + if ("gid" not in site_data and site_data.index.name != "gid"): # require gid as column label or index raise KeyError('Site data input must have "gid" column ' 'to match reV site gid.') # pylint: disable=no-member - if site_data.index.name != MetaKeyName.GID: + if site_data.index.name != "gid": # make gid the dataframe index if not already - site_data = site_data.set_index(MetaKeyName.GID, drop=True) + site_data = site_data.set_index("gid", drop=True) - if MetaKeyName.OFFSHORE in site_data: - if site_data[MetaKeyName.OFFSHORE].sum() > 1: + if "offshore" in site_data: + if site_data["offshore"].sum() > 1: w = ('Found offshore sites in econ site data input. ' 'This functionality has been deprecated. ' 'Please run the reV offshore module to ' diff --git a/reV/generation/generation.py b/reV/generation/generation.py index 7c99361a8..2eca15234 100644 --- a/reV/generation/generation.py +++ b/reV/generation/generation.py @@ -29,7 +29,7 @@ TroughPhysicalHeat, WindPower, ) -from reV.utilities import MetaKeyName, ModuleName +from reV.utilities import ModuleName from reV.utilities.exceptions import ( ConfigError, InputError, @@ -126,7 +126,7 @@ def __init__(self, technology, project_points, sam_files, resource_file, {'cf_mean': array([0.16966143], dtype=float32)} >>> >>> sites = [3, 4, 7, 9] - >>> req = ('cf_mean', , MetaKeyName.LCOE_FCR) + >>> req = ('cf_mean', 'lcoe_fcr') >>> gen = Gen(sam_tech, sites, fp_sam, fp_res, output_request=req) >>> gen.run() >>> @@ -456,11 +456,11 @@ def meta(self): self._meta = res['meta', res_gids] - self._meta.loc[:, MetaKeyName.GID] = res_gids + self._meta.loc[:, "gid"] = res_gids if self.write_mapped_gids: - self._meta.loc[:, MetaKeyName.GID] = self.project_points.sites + self._meta.loc[:, "gid"] = self.project_points.sites self._meta.index = self.project_points.sites - self._meta.index.name = MetaKeyName.GID + self._meta.index.name = "gid" self._meta.loc[:, 'reV_tech'] = self.project_points.tech return self._meta @@ -643,7 +643,7 @@ def _run_single_worker(cls, points_control, tech=None, res_file=None, # Extract the site df from the project points df. site_df = points_control.project_points.df - site_df = site_df.set_index(MetaKeyName.GID, drop=True) + site_df = site_df.set_index("gid", drop=True) # run generation method for specified technology try: @@ -709,11 +709,11 @@ def _parse_gid_map(self, gid_map): if isinstance(gid_map, str): if gid_map.endswith('.csv'): gid_map = pd.read_csv(gid_map).to_dict() - msg = f'Need "{MetaKeyName.GID}" in gid_map column' - assert MetaKeyName.GID in gid_map, msg + msg = f'Need "gid" in gid_map column' + assert "gid" in gid_map, msg assert 'gid_map' in gid_map, 'Need "gid_map" in gid_map column' - gid_map = {gid_map[MetaKeyName.GID][i]: gid_map['gid_map'][i] - for i in gid_map[MetaKeyName.GID].keys()} + gid_map = {gid_map["gid"][i]: gid_map['gid_map'][i] + for i in gid_map["gid"].keys()} elif gid_map.endswith('.json'): with open(gid_map) as f: @@ -770,8 +770,8 @@ def _parse_nn_map(self): if '*' in self.res_file or '*' in self.lr_res_file: handler_class = MultiFileResource - with (handler_class(self.res_file) as hr_res, - handler_class(self.lr_res_file) as lr_res): + with handler_class(self.res_file) as hr_res, \ + handler_class(self.lr_res_file) as lr_res: logger.info('Making nearest neighbor map for multi ' 'resolution resource data...') nn_d, nn_map = MultiResolutionResource.make_nn_map(hr_res, @@ -845,11 +845,10 @@ def _parse_bc(bias_correct): msg = ('Bias correction table must have "gid" column but only found: ' '{}'.format(list(bias_correct.columns))) - assert (MetaKeyName.GID in bias_correct or bias_correct.index.name == - MetaKeyName.GID), msg + assert ("gid" in bias_correct or bias_correct.index.name == "gid"), msg - if bias_correct.index.name != MetaKeyName.GID: - bias_correct = bias_correct.set_index(MetaKeyName.GID) + if bias_correct.index.name != "gid": + bias_correct = bias_correct.set_index("gid") msg = ('Bias correction table must have "method" column but only ' 'found: {}'.format(list(bias_correct.columns))) diff --git a/reV/handlers/exclusions.py b/reV/handlers/exclusions.py index 13d192ee2..6f4a620a3 100644 --- a/reV/handlers/exclusions.py +++ b/reV/handlers/exclusions.py @@ -13,6 +13,7 @@ from reV.utilities.exceptions import HandlerKeyError, MultiFileExclusionError logger = logging.getLogger(__name__) +LATITUDE, LONGITUDE = "latitude", "longitude" class ExclusionLayers: @@ -81,8 +82,8 @@ def __contains__(self, layer): def _preflight_multi_file(self): """Run simple multi-file exclusion checks.""" - lat_shape = self.h5.shapes[MetaKeyName.LATITUDE] - lon_shape = self.h5.shapes[MetaKeyName.LONGITUDE] + lat_shape = self.h5.shapes[LATITUDE] + lon_shape = self.h5.shapes[LONGITUDE] for layer in self.layers: lshape = self.h5.shapes[layer] lshape = lshape[1:] if len(lshape) > 2 else lshape @@ -231,7 +232,7 @@ def shape(self): """ shape = self.h5.attrs.get('shape', None) if shape is None: - shape = self.h5.shapes[MetaKeyName.LATITUDE] + shape = self.h5.shapes[LATITUDE] return tuple(shape) @@ -247,7 +248,7 @@ def chunks(self): """ chunks = self.h5.attrs.get('chunks', None) if chunks is None: - chunks = self.h5.chunks[MetaKeyName.LATITUDE] + chunks = self.h5.chunks[LATITUDE] return chunks @@ -260,7 +261,7 @@ def latitude(self): ------- ndarray """ - return self[MetaKeyName.LATITUDE] + return self[LATITUDE] @property def longitude(self): @@ -271,7 +272,7 @@ def longitude(self): ------- ndarray """ - return self[MetaKeyName.LONGITUDE] + return self[LONGITUDE] def get_layer_profile(self, layer): """ @@ -384,13 +385,13 @@ def _get_latitude(self, *ds_slice): lat : ndarray Latitude coordinates """ - if MetaKeyName.LATITUDE not in self.h5: + if LATITUDE not in self.h5: msg = ('"latitude" is missing from {}' .format(self.h5_file)) logger.error(msg) raise HandlerKeyError(msg) - ds_slice = (MetaKeyName.LATITUDE, ) + ds_slice + ds_slice = (LATITUDE, ) + ds_slice lat = self.h5[ds_slice] @@ -410,13 +411,13 @@ def _get_longitude(self, *ds_slice): lon : ndarray Longitude coordinates """ - if MetaKeyName.LONGITUDE not in self.h5: + if LONGITUDE not in self.h5: msg = ('"longitude" is missing from {}' .format(self.h5_file)) logger.error(msg) raise HandlerKeyError(msg) - ds_slice = (MetaKeyName.LONGITUDE, ) + ds_slice + ds_slice = (LONGITUDE, ) + ds_slice lon = self.h5[ds_slice] From 538f2601b8bcd0af66a0003558a69d92b7ac1ff4 Mon Sep 17 00:00:00 2001 From: ppinchuk Date: Mon, 20 May 2024 10:54:27 -0600 Subject: [PATCH 17/61] Add county, state, country, enum values --- reV/hybrids/hybrids.py | 4 ++-- reV/supply_curve/points.py | 6 +++--- reV/utilities/__init__.py | 4 +++- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/reV/hybrids/hybrids.py b/reV/hybrids/hybrids.py index 59ecdcc5e..5bc7262ae 100644 --- a/reV/hybrids/hybrids.py +++ b/reV/hybrids/hybrids.py @@ -32,8 +32,8 @@ WIND_PREFIX = 'wind_' NON_DUPLICATE_COLS = { MetaKeyName.LATITUDE, MetaKeyName.LONGITUDE, - 'country', 'state', 'county', MetaKeyName.ELEVATION, - MetaKeyName.TIMEZONE, MetaKeyName.SC_POINT_GID, + MetaKeyName.COUNTRY, MetaKeyName.STATE, MetaKeyName.COUNTY, + MetaKeyName.ELEVATION, MetaKeyName.TIMEZONE, MetaKeyName.SC_POINT_GID, MetaKeyName.SC_ROW_IND, MetaKeyName.SC_COL_IND } DROPPED_COLUMNS = [MetaKeyName.GID] diff --git a/reV/supply_curve/points.py b/reV/supply_curve/points.py index de8676391..d70e42ba9 100644 --- a/reV/supply_curve/points.py +++ b/reV/supply_curve/points.py @@ -1176,9 +1176,9 @@ def summary(self): MetaKeyName.AREA_SQ_KM: self.area, MetaKeyName.LATITUDE: self.latitude, MetaKeyName.LONGITUDE: self.longitude, - 'country': self.country, - 'state': self.state, - 'county': self.county, + MetaKeyName.COUNTRY: self.country, + MetaKeyName.STATE: self.state, + MetaKeyName.COUNTY: self.county, MetaKeyName.ELEVATION: self.elevation, MetaKeyName.TIMEZONE: self.timezone, } diff --git a/reV/utilities/__init__.py b/reV/utilities/__init__.py index ce3d19838..8a7afb366 100644 --- a/reV/utilities/__init__.py +++ b/reV/utilities/__init__.py @@ -16,7 +16,6 @@ class MetaKeyName(str, Enum): meth:`GenerationSupplyCurvePoint.point_summary` or meth:`BespokeSinglePlant.meta` """ - SC_POINT_GID = 'sc_point_gid' SOURCE_GIDS = 'source_gids' SC_GID = 'sc_gid' @@ -30,6 +29,9 @@ class MetaKeyName(str, Enum): LONGITUDE = 'longitude' ELEVATION = 'elevation' TIMEZONE = 'timezone' + COUNTY = "county" + STATE = "state" + COUNTRY = "country" MEAN_CF = 'mean_cf' MEAN_LCOE = 'mean_lcoe' MEAN_RES = 'mean_res' From 33cfd26d88fc0b7fbb4758dead7488634d54ec44 Mon Sep 17 00:00:00 2001 From: ppinchuk Date: Mon, 20 May 2024 11:07:14 -0600 Subject: [PATCH 18/61] Walk back changes pt 2 --- reV/SAM/econ.py | 5 ++--- reV/nrwal/nrwal.py | 28 ++++++++++++++-------------- reV/qa_qc/cli_qa_qc.py | 4 ++-- reV/qa_qc/qa_qc.py | 14 ++++++++------ reV/qa_qc/summary.py | 30 ++++++++++++++++++------------ reV/rep_profiles/rep_profiles.py | 2 +- 6 files changed, 45 insertions(+), 38 deletions(-) diff --git a/reV/SAM/econ.py b/reV/SAM/econ.py index 5a33c3d76..d69e33f89 100644 --- a/reV/SAM/econ.py +++ b/reV/SAM/econ.py @@ -16,7 +16,6 @@ from reV.SAM.defaults import DefaultLCOE, DefaultSingleOwner from reV.SAM.SAM import RevPySam from reV.SAM.windbos import WindBos -from reV.utilities import MetaKeyName from reV.utilities.exceptions import SAMExecutionError logger = logging.getLogger(__name__) @@ -175,7 +174,7 @@ def _get_cf_profiles(sites, cf_file, year): with Outputs(cf_file) as cfh: # get the index location of the site in question - site_gids = list(cfh.get_meta_arr(MetaKeyName.GID)) + site_gids = list(cfh.get_meta_arr("gid")) isites = [site_gids.index(s) for s in sites] # look for the cf_profile dataset @@ -379,7 +378,7 @@ def _parse_lcoe_inputs(site_df, cf_file, year): # get the cf_file meta data gid's to use as indexing tools with Outputs(cf_file) as cfh: - site_gids = list(cfh.meta[MetaKeyName.GID]) + site_gids = list(cfh.meta["gid"]) calc_aey = False if 'annual_energy' not in site_df: diff --git a/reV/nrwal/nrwal.py b/reV/nrwal/nrwal.py index b0c23853f..782a0c62e 100644 --- a/reV/nrwal/nrwal.py +++ b/reV/nrwal/nrwal.py @@ -19,7 +19,7 @@ from reV.generation.generation import Gen from reV.handlers.outputs import Outputs -from reV.utilities import log_versions +from reV.utilities import log_versions, MetaKeyName from reV.utilities.exceptions import ( DataShapeError, OffshoreWindInputError, @@ -180,7 +180,7 @@ def __init__(self, gen_fpath, site_data, sam_files, nrwal_configs, self._meta_source = self._parse_gen_data() self._analysis_gids, self._site_data = self._parse_analysis_gids() - pc = Gen.get_pc(self._site_data[[MetaKeyName.GID, 'config']], points_range=None, + pc = Gen.get_pc(self._site_data[["gid", 'config']], points_range=None, sam_configs=sam_files, tech='windpower') self._project_points = pc.project_points @@ -193,7 +193,7 @@ def __init__(self, gen_fpath, site_data, sam_files, nrwal_configs, meta_gids.max(), len(self.meta_source), len(self.analysis_gids))) - def _parse_site_data(self, required_columns=(MetaKeyName.GID, 'config')): + def _parse_site_data(self, required_columns=("gid", 'config')): """Parse the site-specific spatial input data file Parameters @@ -229,7 +229,7 @@ def _parse_site_data(self, required_columns=(MetaKeyName.GID, 'config')): logger.error(msg) raise KeyError(msg) - self._site_data = self._site_data.sort_values(MetaKeyName.GID) + self._site_data = self._site_data.sort_values("gid") return self._site_data @@ -274,7 +274,7 @@ def _parse_analysis_gids(self): meta_gids = self.meta_source[self._meta_gid_col].values - missing = ~np.isin(meta_gids, self._site_data[MetaKeyName.GID]) + missing = ~np.isin(meta_gids, self._site_data["gid"]) if any(missing): msg = ('{} sites from the generation meta data input were ' 'missing from the "site_data" input and will not be ' @@ -282,19 +282,19 @@ def _parse_analysis_gids(self): .format(missing.sum(), meta_gids[missing])) logger.info(msg) - missing = ~np.isin(self._site_data[MetaKeyName.GID], meta_gids) + missing = ~np.isin(self._site_data["gid"], meta_gids) if any(missing): - missing = self._site_data[MetaKeyName.GID].values[missing] + missing = self._site_data["gid"].values[missing] msg = ('{} sites from the "site_data" input were missing from the ' 'generation meta data and will not be run through NRWAL: {}' .format(len(missing), missing)) logger.info(msg) - analysis_gids = set(meta_gids) & set(self._site_data[MetaKeyName.GID]) + analysis_gids = set(meta_gids) & set(self._site_data["gid"]) analysis_gids = np.array(sorted(list(analysis_gids))) # reduce the site data table to only those sites being analyzed - mask = np.isin(self._site_data[MetaKeyName.GID], meta_gids) + mask = np.isin(self._site_data["gid"], meta_gids) self._site_data = self._site_data[mask] return analysis_gids, self._site_data @@ -317,9 +317,9 @@ def _parse_sam_sys_inputs(self): system_inputs = pd.DataFrame(system_inputs).T system_inputs = system_inputs.sort_index() - system_inputs[MetaKeyName.GID] = system_inputs.index.values - system_inputs.index.name = MetaKeyName.GID - mask = system_inputs[MetaKeyName.GID].isin(self.analysis_gids) + system_inputs["gid"] = system_inputs.index.values + system_inputs.index.name = "gid" + mask = system_inputs["gid"].isin(self.analysis_gids) system_inputs = system_inputs[mask] return system_inputs @@ -392,8 +392,8 @@ def _preflight_checks(self): logger.error(msg) raise OffshoreWindInputError(msg) - check_gid_order = (self._site_data[MetaKeyName.GID].values - == self._sam_sys_inputs[MetaKeyName.GID].values) + check_gid_order = (self._site_data["gid"].values + == self._sam_sys_inputs["gid"].values) msg = 'NRWAL site_data and system input dataframe had bad order' assert (check_gid_order).all(), msg diff --git a/reV/qa_qc/cli_qa_qc.py b/reV/qa_qc/cli_qa_qc.py index 2359b4738..ce566400b 100644 --- a/reV/qa_qc/cli_qa_qc.py +++ b/reV/qa_qc/cli_qa_qc.py @@ -19,7 +19,7 @@ SummarizeSupplyCurve, SupplyCurvePlot, ) -from reV.utilities import ModuleName +from reV.utilities import ModuleName, MetaKeyName logger = logging.getLogger(__name__) @@ -195,7 +195,7 @@ def supply_curve_table(ctx, sc_table, columns): help=(" plot_type of plot to create 'plot' or 'plotly', by " "default 'plot'")) @click.option('--lcoe', '-lcoe', type=STR, default=MetaKeyName.MEAN_LCOE, - help="LCOE value to plot, by default MetaKeyName.MEAN_LCOE") + help="LCOE value to plot, by default %(default)s") @click.pass_context def supply_curve_plot(ctx, sc_table, plot_type, lcoe): """ diff --git a/reV/qa_qc/qa_qc.py b/reV/qa_qc/qa_qc.py index 793964bd7..a32022609 100644 --- a/reV/qa_qc/qa_qc.py +++ b/reV/qa_qc/qa_qc.py @@ -18,7 +18,7 @@ SupplyCurvePlot, ) from reV.supply_curve.exclusions import ExclusionMaskFromDict -from reV.utilities import ModuleName, log_versions +from reV.utilities import ModuleName, MetaKeyName, log_versions from reV.utilities.exceptions import PipelineError logger = logging.getLogger(__name__) @@ -100,8 +100,9 @@ def create_scatter_plots(self, plot_type='plotly', cmap='viridis', if file.endswith('.csv'): summary_csv = os.path.join(self.out_dir, file) summary = pd.read_csv(summary_csv) - if (MetaKeyName.GID in summary and MetaKeyName.LATITUDE in summary - and MetaKeyName.LONGITUDE in summary): + if (MetaKeyName.GID in summary + and MetaKeyName.LATITUDE in summary + and MetaKeyName.LONGITUDE in summary): self._scatter_plot(summary_csv, self.out_dir, plot_type=plot_type, cmap=cmap, **kwargs) @@ -151,8 +152,9 @@ def h5(cls, h5_file, out_dir, dsets=None, group=None, process_size=None, .format(os.path.basename(h5_file), out_dir)) @classmethod - def supply_curve(cls, sc_table, out_dir, columns=None, lcoe=MetaKeyName.MEAN_LCOE, - plot_type='plotly', cmap='viridis', sc_plot_kwargs=None, + def supply_curve(cls, sc_table, out_dir, columns=None, + lcoe=MetaKeyName.MEAN_LCOE, plot_type='plotly', + cmap='viridis', sc_plot_kwargs=None, scatter_plot_kwargs=None): """ Plot supply curve @@ -167,7 +169,7 @@ def supply_curve(cls, sc_table, out_dir, columns=None, lcoe=MetaKeyName.MEAN_LCO Column(s) to summarize, if None summarize all numeric columns, by default None lcoe : str, optional - LCOE value to plot, by default MetaKeyName.MEAN_LCOE + LCOE value to plot, by default :obj:`MetaKeyName.MEAN_LCOE` plot_type : str, optional plot_type of plot to create 'plot' or 'plotly', by default 'plotly' cmap : str, optional diff --git a/reV/qa_qc/summary.py b/reV/qa_qc/summary.py index 98d27e357..94b197ce8 100644 --- a/reV/qa_qc/summary.py +++ b/reV/qa_qc/summary.py @@ -528,8 +528,9 @@ def scatter_plot(self, value, cmap='viridis', out_path=None, **kwargs): Additional kwargs for plotting.dataframes.df_scatter """ self._check_value(self.summary, value) - mplt.df_scatter(self.summary, x=MetaKeyName.LONGITUDE, y=MetaKeyName.LATITUDE, c=value, - colormap=cmap, filename=out_path, **kwargs) + mplt.df_scatter(self.summary, x=MetaKeyName.LONGITUDE, + y=MetaKeyName.LATITUDE, c=value, colormap=cmap, + filename=out_path, **kwargs) def scatter_plotly(self, value, cmap='Viridis', out_path=None, **kwargs): """ @@ -549,8 +550,9 @@ def scatter_plotly(self, value, cmap='Viridis', out_path=None, **kwargs): Additional kwargs for plotly.express.scatter """ self._check_value(self.summary, value) - fig = px.scatter(self.summary, x=MetaKeyName.LONGITUDE, y=MetaKeyName.LATITUDE, - color=value, color_continuous_scale=cmap, **kwargs) + fig = px.scatter(self.summary, x=MetaKeyName.LONGITUDE, + y=MetaKeyName.LATITUDE, color=value, + color_continuous_scale=cmap, **kwargs) fig.update_layout(font=dict(family="Arial", size=18, color="black")) if out_path is not None: @@ -565,7 +567,8 @@ def _extract_sc_data(self, lcoe=MetaKeyName.MEAN_LCOE): Parameters ---------- lcoe : str, optional - LCOE value to use for supply curve, by default MetaKeyName.MEAN_LCOE + LCOE value to use for supply curve, + by default :obj:`MetaKeyName.MEAN_LCOE` Returns ------- @@ -747,7 +750,8 @@ def _extract_sc_data(self, lcoe=MetaKeyName.MEAN_LCOE): Parameters ---------- lcoe : str, optional - LCOE value to use for supply curve, by default MetaKeyName.MEAN_LCOE + LCOE value to use for supply curve, + by default :obj:`MetaKeyName.MEAN_LCOE` Returns ------- @@ -761,14 +765,15 @@ def _extract_sc_data(self, lcoe=MetaKeyName.MEAN_LCOE): return sc_df - def supply_curve_plot(self, lcoe=MetaKeyName.MEAN_LCOE, out_path=None, **kwargs): + def supply_curve_plot(self, lcoe=MetaKeyName.MEAN_LCOE, out_path=None, + **kwargs): """ Plot supply curve (cumulative capacity vs lcoe) using seaborn.scatter Parameters ---------- lcoe : str, optional - LCOE value to plot, by default MetaKeyName.MEAN_LCOE + LCOE value to plot, by default :obj:`MetaKeyName.MEAN_LCOE` out_path : str, optional File path to save plot to, by default None kwargs : dict @@ -778,7 +783,8 @@ def supply_curve_plot(self, lcoe=MetaKeyName.MEAN_LCOE, out_path=None, **kwargs) mplt.df_scatter(sc_df, x='cumulative_capacity', y=lcoe, filename=out_path, **kwargs) - def supply_curve_plotly(self, lcoe=MetaKeyName.MEAN_LCOE, out_path=None, **kwargs): + def supply_curve_plotly(self, lcoe=MetaKeyName.MEAN_LCOE, out_path=None, + **kwargs): """ Plot supply curve (cumulative capacity vs lcoe) using plotly @@ -802,8 +808,8 @@ def supply_curve_plotly(self, lcoe=MetaKeyName.MEAN_LCOE, out_path=None, **kwarg fig.show() @classmethod - def plot(cls, sc_table, out_dir, plot_type='plotly', lcoe=MetaKeyName.MEAN_LCOE, - **kwargs): + def plot(cls, sc_table, out_dir, plot_type='plotly', + lcoe=MetaKeyName.MEAN_LCOE, **kwargs): """ Create supply curve plot from supply curve table using lcoe value and save to out_dir @@ -817,7 +823,7 @@ def plot(cls, sc_table, out_dir, plot_type='plotly', lcoe=MetaKeyName.MEAN_LCOE, plot_type : str, optional plot_type of plot to create 'plot' or 'plotly', by default 'plotly' lcoe : str, optional - LCOE value to plot, by default MetaKeyName.MEAN_LCOE + LCOE value to plot, by default :obj:`MetaKeyName.MEAN_LCOE` kwargs : dict Additional plotting kwargs """ diff --git a/reV/rep_profiles/rep_profiles.py b/reV/rep_profiles/rep_profiles.py index c13103dd5..667ea22a0 100644 --- a/reV/rep_profiles/rep_profiles.py +++ b/reV/rep_profiles/rep_profiles.py @@ -978,7 +978,7 @@ def __init__(self, gen_fpath, rev_summary, reg_cols, *equally* to the meanoid profile unless these weights are specified. - By default, ``MetaKeyName.GID_COUNTS``. + By default, :obj:`MetaKeyName.GID_COUNTS`. n_profiles : int, optional Number of representative profiles to save to the output file. By default, ``1``. From 7b47085ee3c397f8dac1b8035b1c0a8580777dce Mon Sep 17 00:00:00 2001 From: ppinchuk Date: Mon, 20 May 2024 11:26:51 -0600 Subject: [PATCH 19/61] Add `ResourceMetaField` enum --- reV/SAM/econ.py | 5 +- reV/SAM/generation.py | 99 +++++++++++++++++++----------------- reV/bespoke/bespoke.py | 16 +++--- reV/econ/econ.py | 16 +++--- reV/generation/base.py | 17 ++++--- reV/generation/generation.py | 31 ++++++----- reV/utilities/__init__.py | 17 +++++++ tests/test_gen_geothermal.py | 4 +- 8 files changed, 117 insertions(+), 88 deletions(-) diff --git a/reV/SAM/econ.py b/reV/SAM/econ.py index d69e33f89..04a7a4fc2 100644 --- a/reV/SAM/econ.py +++ b/reV/SAM/econ.py @@ -17,6 +17,7 @@ from reV.SAM.SAM import RevPySam from reV.SAM.windbos import WindBos from reV.utilities.exceptions import SAMExecutionError +from reV.utilities import ResourceMetaField logger = logging.getLogger(__name__) @@ -174,7 +175,7 @@ def _get_cf_profiles(sites, cf_file, year): with Outputs(cf_file) as cfh: # get the index location of the site in question - site_gids = list(cfh.get_meta_arr("gid")) + site_gids = list(cfh.get_meta_arr(ResourceMetaField.GID)) isites = [site_gids.index(s) for s in sites] # look for the cf_profile dataset @@ -378,7 +379,7 @@ def _parse_lcoe_inputs(site_df, cf_file, year): # get the cf_file meta data gid's to use as indexing tools with Outputs(cf_file) as cfh: - site_gids = list(cfh.meta["gid"]) + site_gids = list(cfh.meta[ResourceMetaField.GID]) calc_aey = False if 'annual_energy' not in site_df: diff --git a/reV/SAM/generation.py b/reV/SAM/generation.py index e99e0db70..1268623f8 100644 --- a/reV/SAM/generation.py +++ b/reV/SAM/generation.py @@ -40,7 +40,7 @@ ) from reV.SAM.econ import LCOE, SingleOwner from reV.SAM.SAM import RevPySam -from reV.utilities import MetaKeyName +from reV.utilities import ResourceMetaField from reV.utilities.curtailment import curtail from reV.utilities.exceptions import ( InputError, @@ -245,23 +245,23 @@ def tz_elev_check(sam_sys_inputs, site_sys_inputs, meta): if meta is not None: if sam_sys_inputs is not None: - if MetaKeyName.ELEVATION in sam_sys_inputs: - meta[MetaKeyName.ELEVATION] = \ - sam_sys_inputs[MetaKeyName.ELEVATION] - if MetaKeyName.TIMEZONE in sam_sys_inputs: - meta[MetaKeyName.TIMEZONE] = \ - int(sam_sys_inputs[MetaKeyName.TIMEZONE]) + if ResourceMetaField.ELEVATION in sam_sys_inputs: + meta[ResourceMetaField.ELEVATION] = \ + sam_sys_inputs[ResourceMetaField.ELEVATION] + if ResourceMetaField.TIMEZONE in sam_sys_inputs: + meta[ResourceMetaField.TIMEZONE] = \ + int(sam_sys_inputs[ResourceMetaField.TIMEZONE]) # site-specific inputs take priority over generic system inputs if site_sys_inputs is not None: - if MetaKeyName.ELEVATION in site_sys_inputs: - meta[MetaKeyName.ELEVATION] = \ - site_sys_inputs[MetaKeyName.ELEVATION] - if MetaKeyName.TIMEZONE in site_sys_inputs: - meta[MetaKeyName.TIMEZONE] = \ - int(site_sys_inputs[MetaKeyName.TIMEZONE]) - - if MetaKeyName.TIMEZONE not in meta: + if ResourceMetaField.ELEVATION in site_sys_inputs: + meta[ResourceMetaField.ELEVATION] = \ + site_sys_inputs[ResourceMetaField.ELEVATION] + if ResourceMetaField.TIMEZONE in site_sys_inputs: + meta[ResourceMetaField.TIMEZONE] = \ + int(site_sys_inputs[ResourceMetaField.TIMEZONE]) + + if ResourceMetaField.TIMEZONE not in meta: msg = ('Need timezone input to run SAM gen. Not found in ' 'resource meta or technology json input config.') raise SAMExecutionError(msg) @@ -271,7 +271,7 @@ def tz_elev_check(sam_sys_inputs, site_sys_inputs, meta): @property def has_timezone(self): """Returns true if instance has a timezone set""" - if self._meta is not None and MetaKeyName.TIMEZONE in self.meta: + if self._meta is not None and ResourceMetaField.TIMEZONE in self.meta: return True return False @@ -527,11 +527,14 @@ def reV_run(cls, points_control, res_file, site_df, class AbstractSamGenerationFromWeatherFile(AbstractSamGeneration, ABC): """Base class for running sam generation with a weather file on disk.""" - WF_META_DROP_COLS = {MetaKeyName.ELEVATION, MetaKeyName.TIMEZONE, - 'country', 'state', - 'county', 'urban', 'population', - 'landcover', MetaKeyName.LATITUDE, - MetaKeyName.LONGITUDE} + WF_META_DROP_COLS = {ResourceMetaField.LATITUDE, + ResourceMetaField.LONGITUDE, + ResourceMetaField.ELEVATION, + ResourceMetaField.TIMEZONE, + ResourceMetaField.COUNTRY, + ResourceMetaField.STATE, + ResourceMetaField.COUNTY, + 'urban', 'population', 'landcover'} @property @abstractmethod @@ -601,7 +604,7 @@ def _create_pysam_wfile(self, resource, meta): # ------- Process metadata m = pd.DataFrame(meta).T - timezone = m[MetaKeyName.TIMEZONE] + timezone = m[ResourceMetaField.TIMEZONE] m['Source'] = 'NSRDB' m['Location ID'] = meta.name m['City'] = '-' @@ -609,10 +612,10 @@ def _create_pysam_wfile(self, resource, meta): lambda x: '-' if x == 'None' else x) m['Country'] = m['country'].apply( lambda x: '-' if x == 'None' else x) - m['Latitude'] = m[MetaKeyName.LATITUDE] - m['Longitude'] = m[MetaKeyName.LONGITUDE] + m['Latitude'] = m[ResourceMetaField.LATITUDE] + m['Longitude'] = m[ResourceMetaField.LONGITUDE] m['Time Zone'] = timezone - m['Elevation'] = m[MetaKeyName.ELEVATION] + m['Elevation'] = m[ResourceMetaField.ELEVATION] m['Local Time Zone'] = timezone m['Dew Point Units'] = 'c' m['DHI Units'] = 'w/m2' @@ -750,7 +753,7 @@ def set_resource_data(self, resource, meta): # ensure that resource array length is multiple of 8760 arr = self.ensure_res_len(arr, time_index) - n_roll = int(self._meta[MetaKeyName.TIMEZONE] * + n_roll = int(self._meta[ResourceMetaField.TIMEZONE] * self.time_interval) arr = np.roll(arr, n_roll) @@ -762,12 +765,12 @@ def set_resource_data(self, resource, meta): resource[var] = arr.tolist() - resource['lat'] = meta[MetaKeyName.LATITUDE] - resource['lon'] = meta[MetaKeyName.LONGITUDE] - resource['tz'] = meta[MetaKeyName.TIMEZONE] + resource['lat'] = meta[ResourceMetaField.LATITUDE] + resource['lon'] = meta[ResourceMetaField.LONGITUDE] + resource['tz'] = meta[ResourceMetaField.TIMEZONE] - if MetaKeyName.ELEVATION in meta: - resource['elev'] = meta[MetaKeyName.ELEVATION] + if ResourceMetaField.ELEVATION in meta: + resource['elev'] = meta[ResourceMetaField.ELEVATION] else: resource['elev'] = 0.0 @@ -921,10 +924,10 @@ def set_resource_data(self, resource, meta): respectively. """ - bad_location_input = ((meta[MetaKeyName.LATITUDE] < -90) - | (meta[MetaKeyName.LATITUDE] > 90) - | (meta[MetaKeyName.LONGITUDE] < -180) - | (meta[MetaKeyName.LONGITUDE] > 180)) + bad_location_input = ((meta[ResourceMetaField.LATITUDE] < -90) + | (meta[ResourceMetaField.LATITUDE] > 90) + | (meta[ResourceMetaField.LONGITUDE] < -180) + | (meta[ResourceMetaField.LONGITUDE] > 180)) if bad_location_input.any(): raise ValueError("Detected latitude/longitude values outside of " "the range -90 to 90 and -180 to 180, " @@ -962,13 +965,13 @@ def set_latitude_tilt_az(sam_sys_inputs, meta): SAMInputWarning) set_tilt = True elif (sam_sys_inputs['tilt'] == 'lat' - or sam_sys_inputs['tilt'] == MetaKeyName.LATITUDE): + or sam_sys_inputs['tilt'] == ResourceMetaField.LATITUDE): set_tilt = True if set_tilt: # set tilt to abs(latitude) - sam_sys_inputs['tilt'] = np.abs(meta[MetaKeyName.LATITUDE]) - if meta[MetaKeyName.LATITUDE] > 0: + sam_sys_inputs['tilt'] = np.abs(meta[ResourceMetaField.LATITUDE]) + if meta[ResourceMetaField.LATITUDE] > 0: # above the equator, az = 180 sam_sys_inputs['azimuth'] = 180 else: @@ -1977,7 +1980,7 @@ def set_resource_data(self, resource, meta): if 'rh' in resource: # set relative humidity for icing. rh = self.ensure_res_len(resource['rh'].values, time_index) - n_roll = int(meta[MetaKeyName.TIMEZONE] * self.time_interval) + n_roll = int(meta[ResourceMetaField.TIMEZONE] * self.time_interval) rh = np.roll(rh, n_roll, axis=0) data_dict['rh'] = rh.tolist() @@ -1985,14 +1988,14 @@ def set_resource_data(self, resource, meta): # ensure that resource array length is multiple of 8760 # roll the truncated resource array to local timezone temp = self.ensure_res_len(resource[var_list].values, time_index) - n_roll = int(meta[MetaKeyName.TIMEZONE] * self.time_interval) + n_roll = int(meta[ResourceMetaField.TIMEZONE] * self.time_interval) temp = np.roll(temp, n_roll, axis=0) data_dict['data'] = temp.tolist() - data_dict['lat'] = float(meta[MetaKeyName.LATITUDE]) - data_dict['lon'] = float(meta[MetaKeyName.LONGITUDE]) - data_dict['tz'] = int(meta[MetaKeyName.TIMEZONE]) - data_dict['elev'] = float(meta[MetaKeyName.ELEVATION]) + data_dict['lat'] = float(meta[ResourceMetaField.LATITUDE]) + data_dict['lon'] = float(meta[ResourceMetaField.LONGITUDE]) + data_dict['tz'] = int(meta[ResourceMetaField.TIMEZONE]) + data_dict['elev'] = float(meta[ResourceMetaField.ELEVATION]) time_index = self.ensure_res_len(time_index, time_index) data_dict['minute'] = time_index.minute.tolist() @@ -2157,12 +2160,12 @@ def set_resource_data(self, resource, meta): # roll the truncated resource array to local timezone for var in ['significant_wave_height', 'energy_period']: arr = self.ensure_res_len(resource[var].values, time_index) - n_roll = int(meta[MetaKeyName.TIMEZONE] * self.time_interval) + n_roll = int(meta[ResourceMetaField.TIMEZONE] * self.time_interval) data_dict[var] = np.roll(arr, n_roll, axis=0).tolist() - data_dict['lat'] = meta[MetaKeyName.LATITUDE] - data_dict['lon'] = meta[MetaKeyName.LONGITUDE] - data_dict['tz'] = meta[MetaKeyName.TIMEZONE] + data_dict['lat'] = meta[ResourceMetaField.LATITUDE] + data_dict['lon'] = meta[ResourceMetaField.LONGITUDE] + data_dict['tz'] = meta[ResourceMetaField.TIMEZONE] time_index = self.ensure_res_len(time_index, time_index) data_dict['minute'] = time_index.minute diff --git a/reV/bespoke/bespoke.py b/reV/bespoke/bespoke.py index a2602abdb..db95d4e17 100644 --- a/reV/bespoke/bespoke.py +++ b/reV/bespoke/bespoke.py @@ -36,7 +36,8 @@ from reV.supply_curve.extent import SupplyCurveExtent from reV.supply_curve.points import AggregationSupplyCurvePoint as AggSCPoint from reV.supply_curve.points import SupplyCurvePoint -from reV.utilities import MetaKeyName, ModuleName, log_versions +from reV.utilities import (MetaKeyName, ModuleName, ResourceMetaField, + log_versions) from reV.utilities.exceptions import EmptySupplyCurvePointError, FileInputError logger = logging.getLogger(__name__) @@ -580,13 +581,12 @@ def _parse_gid_map(gid_map): if isinstance(gid_map, str): if gid_map.endswith(".csv"): gid_map = pd.read_csv(gid_map).to_dict() - assert ( - "gid" in gid_map - ), 'Need "gid" in gid_map column' + err_msg = 'Need "gid" in gid_map column' + assert ResourceMetaField.GID in gid_map, err_msg assert "gid_map" in gid_map, 'Need "gid_map" in gid_map column' gid_map = { - gid_map["gid"][i]: gid_map["gid_map"][i] - for i in gid_map["gid"].keys() + gid_map[ResourceMetaField.GID][i]: gid_map["gid_map"][i] + for i in gid_map[ResourceMetaField.GID].keys() } elif gid_map.endswith(".json"): @@ -2045,7 +2045,7 @@ def _pre_load_data(self, pre_load_data): sc_gid_to_hh = { gid: self._hh_for_sc_gid(gid) - for gid in self._project_points.df["gid"] + for gid in self._project_points.df[ResourceMetaField.GID] } with ExclusionLayers(self._excl_fpath) as excl: @@ -2054,7 +2054,7 @@ def _pre_load_data(self, pre_load_data): scp_kwargs = {"shape": self.shape, "resolution": self._resolution} slices = { gid: SupplyCurvePoint.get_agg_slices(gid=gid, **scp_kwargs) - for gid in self._project_points.df["gid"] + for gid in self._project_points.df[ResourceMetaField.GID] } sc_gid_to_res_gid = { diff --git a/reV/econ/econ.py b/reV/econ/econ.py index 8f5920983..9f759f9ec 100644 --- a/reV/econ/econ.py +++ b/reV/econ/econ.py @@ -17,7 +17,7 @@ from reV.SAM.econ import LCOE as SAM_LCOE from reV.SAM.econ import SingleOwner from reV.SAM.windbos import WindBos -from reV.utilities import ModuleName +from reV.utilities import ModuleName, ResourceMetaField from reV.utilities.exceptions import ExecutionError, OffshoreWindInputWarning logger = logging.getLogger(__name__) @@ -210,7 +210,8 @@ def meta(self): with Outputs(self.cf_file) as cfh: # only take meta that belongs to this project's site list self._meta = cfh.meta[ - cfh.meta["gid"].isin(self.points_control.sites)] + cfh.meta[ResourceMetaField.GID].isin( + self.points_control.sites)] if ("offshore" in self._meta and self._meta["offshore"].sum() > 1): w = ('Found offshore sites in econ meta data. ' @@ -221,7 +222,8 @@ def meta(self): logger.warning(w) elif self._meta is None and self.cf_file is None: - self._meta = pd.DataFrame({"gid": self.points_control.sites}) + self._meta = pd.DataFrame( + {ResourceMetaField.GID: self.points_control.sites}) return self._meta @@ -264,8 +266,8 @@ def _econ_append_pc(pp, cf_file, sites_per_worker=None): res_kwargs = {'hsds': hsds} with res_cls(cf_file, **res_kwargs) as f: - gid0 = f.meta["gid"].values[0] - gid1 = f.meta["gid"].values[-1] + gid0 = f.meta[ResourceMetaField.GID].values[0] + gid1 = f.meta[ResourceMetaField.GID].values[-1] i0 = pp.index(gid0) i1 = pp.index(gid1) + 1 @@ -351,7 +353,7 @@ def _run_single_worker(pc, econ_fun, output_request, **kwargs): # Extract the site df from the project points df. site_df = pc.project_points.df - site_df = site_df.set_index("gid", drop=True) + site_df = site_df.set_index(ResourceMetaField.GID, drop=True) # SAM execute econ analysis based on output request try: @@ -493,7 +495,7 @@ def run(self, out_fpath=None, max_workers=1, timeout=1800, self._init_out_arrays() diff = list(set(self.points_control.sites) - - set(self.meta["gid"].values)) + - set(self.meta[ResourceMetaField.GID].values)) if diff: raise Exception('The following analysis sites were requested ' 'through project points for econ but are not ' diff --git a/reV/generation/base.py b/reV/generation/base.py index 24606e13f..0bbd743d1 100644 --- a/reV/generation/base.py +++ b/reV/generation/base.py @@ -21,7 +21,7 @@ from reV.config.project_points import PointsControl, ProjectPoints from reV.handlers.outputs import Outputs from reV.SAM.version_checker import PySamVersionChecker -from reV.utilities import ModuleName, log_versions +from reV.utilities import ModuleName, ResourceMetaField, log_versions from reV.utilities.exceptions import ( ExecutionError, OffshoreWindInputWarning, @@ -739,7 +739,7 @@ def _parse_site_data(self, inp): if inp is None or inp is False: # no input, just initialize dataframe with site gids as index site_data = pd.DataFrame(index=self.project_points.sites) - site_data.index.name = "gid" + site_data.index.name = ResourceMetaField.GID else: # explicit input, initialize df if isinstance(inp, str): @@ -752,15 +752,18 @@ def _parse_site_data(self, inp): raise Exception('Site data input must be .csv or ' 'dataframe, but received: {}'.format(inp)) - if ("gid" not in site_data and site_data.index.name != "gid"): + if (ResourceMetaField.GID not in site_data + and site_data.index.name != ResourceMetaField.GID): # require gid as column label or index - raise KeyError('Site data input must have "gid" column ' - 'to match reV site gid.') + raise KeyError('Site data input must have ' + f'{ResourceMetaField.GID} column to match ' + 'reV site gid.') # pylint: disable=no-member - if site_data.index.name != "gid": + if site_data.index.name != ResourceMetaField.GID: # make gid the dataframe index if not already - site_data = site_data.set_index("gid", drop=True) + site_data = site_data.set_index(ResourceMetaField.GID, + drop=True) if "offshore" in site_data: if site_data["offshore"].sum() > 1: diff --git a/reV/generation/generation.py b/reV/generation/generation.py index 2eca15234..63e2339b3 100644 --- a/reV/generation/generation.py +++ b/reV/generation/generation.py @@ -29,7 +29,7 @@ TroughPhysicalHeat, WindPower, ) -from reV.utilities import ModuleName +from reV.utilities import ModuleName, ResourceMetaField from reV.utilities.exceptions import ( ConfigError, InputError, @@ -456,11 +456,12 @@ def meta(self): self._meta = res['meta', res_gids] - self._meta.loc[:, "gid"] = res_gids + self._meta.loc[:, ResourceMetaField.GID] = res_gids if self.write_mapped_gids: - self._meta.loc[:, "gid"] = self.project_points.sites + sites = self.project_points.sites + self._meta.loc[:, ResourceMetaField.GID] = sites self._meta.index = self.project_points.sites - self._meta.index.name = "gid" + self._meta.index.name = ResourceMetaField.GID self._meta.loc[:, 'reV_tech'] = self.project_points.tech return self._meta @@ -643,7 +644,7 @@ def _run_single_worker(cls, points_control, tech=None, res_file=None, # Extract the site df from the project points df. site_df = points_control.project_points.df - site_df = site_df.set_index("gid", drop=True) + site_df = site_df.set_index(ResourceMetaField.GID, drop=True) # run generation method for specified technology try: @@ -709,11 +710,12 @@ def _parse_gid_map(self, gid_map): if isinstance(gid_map, str): if gid_map.endswith('.csv'): gid_map = pd.read_csv(gid_map).to_dict() - msg = f'Need "gid" in gid_map column' - assert "gid" in gid_map, msg + msg = f'Need {ResourceMetaField.GID} in gid_map column' + assert ResourceMetaField.GID in gid_map, msg assert 'gid_map' in gid_map, 'Need "gid_map" in gid_map column' - gid_map = {gid_map["gid"][i]: gid_map['gid_map'][i] - for i in gid_map["gid"].keys()} + gid_map = { + gid_map[ResourceMetaField.GID][i]: gid_map['gid_map'][i] + for i in gid_map[ResourceMetaField.GID].keys()} elif gid_map.endswith('.json'): with open(gid_map) as f: @@ -843,12 +845,13 @@ def _parse_bc(bias_correct): 'but received: {}'.format(type(bias_correct))) assert isinstance(bias_correct, pd.DataFrame), msg - msg = ('Bias correction table must have "gid" column but only found: ' - '{}'.format(list(bias_correct.columns))) - assert ("gid" in bias_correct or bias_correct.index.name == "gid"), msg + msg = ('Bias correction table must have {!r} column but only found: ' + '{}'.format(ResourceMetaField.GID, list(bias_correct.columns))) + assert (ResourceMetaField.GID in bias_correct + or bias_correct.index.name == ResourceMetaField.GID), msg - if bias_correct.index.name != "gid": - bias_correct = bias_correct.set_index("gid") + if bias_correct.index.name != ResourceMetaField.GID: + bias_correct = bias_correct.set_index(ResourceMetaField.GID) msg = ('Bias correction table must have "method" column but only ' 'found: {}'.format(list(bias_correct.columns))) diff --git a/reV/utilities/__init__.py b/reV/utilities/__init__.py index 8a7afb366..1446eb018 100644 --- a/reV/utilities/__init__.py +++ b/reV/utilities/__init__.py @@ -8,6 +8,23 @@ from reV.version import __version__ +class ResourceMetaField(str, Enum): + """An enumerated map to resource meta column names. + + Each output name should match the name of a key the resource file + meta table. + """ + + GID = "gid" + LATITUDE = "latitude" + LONGITUDE = "longitude" + ELEVATION = "elevation" + TIMEZONE = "timezone" + COUNTY = "county" + STATE = "state" + COUNTRY = "country" + + class MetaKeyName(str, Enum): """An enumerated map to summary/meta keys. diff --git a/tests/test_gen_geothermal.py b/tests/test_gen_geothermal.py index c495a4cd8..26995e383 100644 --- a/tests/test_gen_geothermal.py +++ b/tests/test_gen_geothermal.py @@ -17,7 +17,7 @@ from reV import TESTDATADIR from reV.generation.generation import Gen from reV.SAM.generation import Geothermal -from reV.utilities import MetaKeyName +from reV.utilities import MetaKeyName, ResourceMetaField DEFAULT_GEO_SAM_FILE = TESTDATADIR + "/SAM/geothermal_default.json" RTOL = 0.1 @@ -30,7 +30,7 @@ def sample_resource_data(request): meta = pd.DataFrame( {"latitude": [41.29], "longitude": [-71.86], "timezone": [-5]} ) - meta.index.name = "gid" + meta.index.name = ResourceMetaField.GID with TemporaryDirectory() as td: geo_sam_file = os.path.join(td, "geothermal_sam.json") geo_res_file = os.path.join(td, "test_geo.h5") From 0fe7a6871e663711a2a0e3c47f5ac94d76a52c13 Mon Sep 17 00:00:00 2001 From: ppinchuk Date: Mon, 20 May 2024 11:28:42 -0600 Subject: [PATCH 20/61] Fix missing 'cf_profile' keys --- tests/test_gen_config.py | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/tests/test_gen_config.py b/tests/test_gen_config.py index f5363a4f7..37df6b7e6 100644 --- a/tests/test_gen_config.py +++ b/tests/test_gen_config.py @@ -38,7 +38,7 @@ def get_r1_profiles(year=2012, tech='pv'): 'wind_{}_0.h5'.format(year)) with Outputs(rev1) as cf: - data = cf[][...] / 10000 + data = cf['cf_profile'][...] / 10000 return data @@ -98,10 +98,8 @@ def test_gen_from_config(runner, tech, clear_loggers): # noqa: C901 with Outputs(path, 'r') as cf: msg = 'cf_profile not written to disk' - assert in cf.datasets, msg - print(cf.scale_factors[]) - print(cf.dtypes[]) - rev2_profiles = cf[] + assert 'cf_profile' in cf.datasets, msg + rev2_profiles = cf['cf_profile'] msg = 'monthly_energy not written to disk' assert 'monthly_energy' in cf.datasets, msg @@ -147,17 +145,17 @@ def test_sam_config(tech): "config": ['default'] * 100}) gen_json = Gen('pvwattsv5', points, sam_file, res_file, - output_request=(,), sites_per_worker=50) + output_request=('cf_profile',), sites_per_worker=50) gen_json.run(max_workers=2) gen_dict = Gen('pvwattsv5', points_config, sam_config, res_file, - output_request=(,), sites_per_worker=50) + output_request=('cf_profile',), sites_per_worker=50) gen_dict.run(max_workers=2) msg = ("reV {} generation run from JSON and SAM config dictionary do " "not match".format(tech)) - assert np.allclose(gen_json.out[], - gen_dict.out[]), msg + assert np.allclose(gen_json.out['cf_profile'], + gen_dict.out['cf_profile']), msg elif tech == 'wind': sam_file = TESTDATADIR + '/SAM/wind_gen_standard_losses_0.json' res_file = TESTDATADIR + '/wtk/ri_100_wtk_2012.h5' @@ -168,17 +166,17 @@ def test_sam_config(tech): "config": ['default'] * 10}) gen_json = Gen('windpower', points, sam_file, res_file, - output_request=(,), sites_per_worker=3) + output_request=('cf_profile',), sites_per_worker=3) gen_json.run(max_workers=2) gen_dict = Gen('windpower', points_config, sam_config, res_file, - output_request=(,), sites_per_worker=3) + output_request=('cf_profile',), sites_per_worker=3) gen_dict.run(max_workers=2) msg = ("reV {} generation run from JSON and SAM config dictionary do " "not match".format(tech)) - assert np.allclose(gen_json.out[], - gen_dict.out[]), msg + assert np.allclose(gen_json.out['cf_profile'], + gen_dict.out['cf_profile']), msg @pytest.mark.parametrize('expected_log_message', From dbfe47e1c43424f67ff47c20d0ce3feba4a72e2b Mon Sep 17 00:00:00 2001 From: ppinchuk Date: Mon, 20 May 2024 11:29:57 -0600 Subject: [PATCH 21/61] Format string --- reV/bespoke/bespoke.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reV/bespoke/bespoke.py b/reV/bespoke/bespoke.py index db95d4e17..e473f4ade 100644 --- a/reV/bespoke/bespoke.py +++ b/reV/bespoke/bespoke.py @@ -581,7 +581,7 @@ def _parse_gid_map(gid_map): if isinstance(gid_map, str): if gid_map.endswith(".csv"): gid_map = pd.read_csv(gid_map).to_dict() - err_msg = 'Need "gid" in gid_map column' + err_msg = f'Need {ResourceMetaField.GID} in gid_map column' assert ResourceMetaField.GID in gid_map, err_msg assert "gid_map" in gid_map, 'Need "gid_map" in gid_map column' gid_map = { From 906c6b7c03b5b0110416b3eacf8c0987cd60ca4a Mon Sep 17 00:00:00 2001 From: ppinchuk Date: Mon, 20 May 2024 11:39:08 -0600 Subject: [PATCH 22/61] Add `SiteDataField` enum --- reV/config/project_points.py | 49 ++++++++++++++++++++---------------- reV/nrwal/nrwal.py | 41 ++++++++++++++++-------------- reV/utilities/__init__.py | 5 ++++ tests/test_bespoke.py | 32 ++++++++++++----------- tests/test_gen_config.py | 9 ++++--- 5 files changed, 77 insertions(+), 59 deletions(-) diff --git a/reV/config/project_points.py b/reV/config/project_points.py index fec9d8001..ce1ae5982 100644 --- a/reV/config/project_points.py +++ b/reV/config/project_points.py @@ -20,6 +20,7 @@ from reV.config.curtailment import Curtailment from reV.config.sam_config import SAMConfig from reV.utilities.exceptions import ConfigError, ConfigWarning +from reV.utilities import SiteDataField logger = logging.getLogger(__name__) @@ -272,9 +273,9 @@ def __getitem__(self, site): names (keys) and values. """ - site_bool = (self.df["gid"] == site) + site_bool = (self.df[SiteDataField.GID] == site) try: - config_id = self.df.loc[site_bool, 'config'].values[0] + config_id = self.df.loc[site_bool, SiteDataField.CONFIG].values[0] except (KeyError, IndexError) as ex: msg = ('Site {} not found in this instance of ' 'ProjectPoints. Available sites include: {}' @@ -387,7 +388,7 @@ def sites(self): List of integer sites (resource file gids) belonging to this instance of ProjectPoints. """ - return self.df["gid"].values.tolist() + return self.df[SiteDataField.GID].values.tolist() @property def sites_as_slice(self): @@ -525,7 +526,7 @@ def _parse_sites(points, res_file=None): df : pd.DataFrame DataFrame mapping sites (gids) to SAM technology (config) """ - df = pd.DataFrame(columns=["gid", "config"]) + df = pd.DataFrame(columns=[SiteDataField.GID, SiteDataField.CONFIG]) if isinstance(points, int): points = [points] if isinstance(points, (list, tuple, np.ndarray)): @@ -535,7 +536,7 @@ def _parse_sites(points, res_file=None): logger.error(msg) raise RuntimeError(msg) - df["gid"] = points + df[SiteDataField.GID] = points elif isinstance(points, slice): stop = points.stop if stop is None: @@ -550,13 +551,13 @@ def _parse_sites(points, res_file=None): else: stop = Resource(res_file).shape[1] - df["gid"] = list(range(*points.indices(stop))) + df[SiteDataField.GID] = list(range(*points.indices(stop))) else: raise TypeError('Project Points sites needs to be set as a list, ' 'tuple, or slice, but was set as: {}' .format(type(points))) - df['config'] = None + df[SiteDataField.CONFIG] = None return df @@ -591,14 +592,16 @@ def _parse_points(cls, points, res_file=None): raise ValueError('Cannot parse Project points data from {}' .format(type(points))) - if "gid" not in df.columns: - raise KeyError('Project points data must contain "gid" column.') + if SiteDataField.GID not in df.columns: + raise KeyError('Project points data must contain ' + f'{SiteDataField.GID} column.') # pylint: disable=no-member - if 'config' not in df.columns: - df = cls._parse_sites(points["gid"].values, res_file=res_file) + if SiteDataField.CONFIG not in df.columns: + df = cls._parse_sites(points[SiteDataField.GID].values, + res_file=res_file) - gids = df["gid"].values + gids = df[SiteDataField.GID].values if not np.array_equal(np.sort(gids), gids): msg = ('WARNING: points are not in sequential order and will be ' 'sorted! The original order is being preserved under ' @@ -606,7 +609,7 @@ def _parse_points(cls, points, res_file=None): logger.warning(msg) warn(msg) df['points_order'] = df.index.values - df = df.sort_values("gid").reset_index(drop=True) + df = df.sort_values(SiteDataField.GID).reset_index(drop=True) return df @@ -694,13 +697,13 @@ def index(self, gid): ind : int Row index of gid in the project points dataframe. """ - if gid not in self._df["gid"].values: + if gid not in self._df[SiteDataField.GID].values: e = ('Requested resource gid {} is not present in the project ' 'points dataframe. Cannot return row index.'.format(gid)) logger.error(e) raise ConfigError(e) - ind = np.where(self._df["gid"] == gid)[0][0] + ind = np.where(self._df[SiteDataField.GID] == gid)[0][0] return ind @@ -710,7 +713,7 @@ def _check_points_config_mapping(self): (sam_config_obj) are compatible. Update as necessary or break """ # Extract unique config refences from project_points DataFrame - df_configs = self.df['config'].unique() + df_configs = self.df[SiteDataField.CONFIG].unique() sam_configs = self.sam_inputs # Checks to make sure that the same number of SAM config files @@ -723,8 +726,8 @@ def _check_points_config_mapping(self): raise ConfigError(msg) if len(df_configs) == 1 and df_configs[0] is None: - self._df['config'] = list(sam_configs)[0] - df_configs = self.df['config'].unique() + self._df[SiteDataField.CONFIG] = list(sam_configs)[0] + df_configs = self.df[SiteDataField.CONFIG].unique() # Check to see if config references in project_points DataFrame # are valid file paths, if compare with SAM configs @@ -750,7 +753,7 @@ def _check_points_config_mapping(self): logger.error(msg) raise ConfigError(msg) - def join_df(self, df2, key="gid"): + def join_df(self, df2, key=SiteDataField.GID): """Join new df2 to the _df attribute using the _df's gid as pkey. This can be used to add site-specific data to the project_points, @@ -770,8 +773,9 @@ def join_df(self, df2, key="gid"): """ # ensure df2 doesnt have any duplicate columns for suffix reasons. df2_cols = [c for c in df2.columns if c not in self._df or c == key] - self._df = pd.merge(self._df, df2[df2_cols], how='left', left_on="gid", - right_on=key, copy=False, validate='1:1') + self._df = pd.merge(self._df, df2[df2_cols], how='left', + left_on=SiteDataField.GID, right_on=key, + copy=False, validate='1:1') def get_sites_from_config(self, config): """Get a site list that corresponds to a config key. @@ -787,7 +791,8 @@ def get_sites_from_config(self, config): List of sites associated with the requested configuration ID. If the configuration ID is not recognized, an empty list is returned. """ - sites = self.df.loc[(self.df['config'] == config), "gid"].values + sites = self.df.loc[(self.df[SiteDataField.CONFIG] == config), + SiteDataField.GID].values return list(sites) diff --git a/reV/nrwal/nrwal.py b/reV/nrwal/nrwal.py index 782a0c62e..580c6726c 100644 --- a/reV/nrwal/nrwal.py +++ b/reV/nrwal/nrwal.py @@ -19,7 +19,7 @@ from reV.generation.generation import Gen from reV.handlers.outputs import Outputs -from reV.utilities import log_versions, MetaKeyName +from reV.utilities import log_versions, MetaKeyName, SiteDataField from reV.utilities.exceptions import ( DataShapeError, OffshoreWindInputError, @@ -32,7 +32,7 @@ class RevNrwal: """RevNrwal""" - DEFAULT_META_COLS = ('config', ) + DEFAULT_META_COLS = (SiteDataField.CONFIG, ) """Columns from the `site_data` table to join to the output meta data""" def __init__(self, gen_fpath, site_data, sam_files, nrwal_configs, @@ -180,8 +180,9 @@ def __init__(self, gen_fpath, site_data, sam_files, nrwal_configs, self._meta_source = self._parse_gen_data() self._analysis_gids, self._site_data = self._parse_analysis_gids() - pc = Gen.get_pc(self._site_data[["gid", 'config']], points_range=None, - sam_configs=sam_files, tech='windpower') + pc = Gen.get_pc( + self._site_data[[SiteDataField.GID, SiteDataField.CONFIG]], + points_range=None, sam_configs=sam_files, tech='windpower') self._project_points = pc.project_points self._sam_sys_inputs = self._parse_sam_sys_inputs() @@ -193,7 +194,8 @@ def __init__(self, gen_fpath, site_data, sam_files, nrwal_configs, meta_gids.max(), len(self.meta_source), len(self.analysis_gids))) - def _parse_site_data(self, required_columns=("gid", 'config')): + def _parse_site_data(self, required_columns=(SiteDataField.GID, + SiteDataField.CONFIG)): """Parse the site-specific spatial input data file Parameters @@ -229,7 +231,7 @@ def _parse_site_data(self, required_columns=("gid", 'config')): logger.error(msg) raise KeyError(msg) - self._site_data = self._site_data.sort_values("gid") + self._site_data = self._site_data.sort_values(SiteDataField.GID) return self._site_data @@ -274,7 +276,7 @@ def _parse_analysis_gids(self): meta_gids = self.meta_source[self._meta_gid_col].values - missing = ~np.isin(meta_gids, self._site_data["gid"]) + missing = ~np.isin(meta_gids, self._site_data[SiteDataField.GID]) if any(missing): msg = ('{} sites from the generation meta data input were ' 'missing from the "site_data" input and will not be ' @@ -282,19 +284,20 @@ def _parse_analysis_gids(self): .format(missing.sum(), meta_gids[missing])) logger.info(msg) - missing = ~np.isin(self._site_data["gid"], meta_gids) + missing = ~np.isin(self._site_data[SiteDataField.GID], meta_gids) if any(missing): - missing = self._site_data["gid"].values[missing] + missing = self._site_data[SiteDataField.GID].values[missing] msg = ('{} sites from the "site_data" input were missing from the ' 'generation meta data and will not be run through NRWAL: {}' .format(len(missing), missing)) logger.info(msg) - analysis_gids = set(meta_gids) & set(self._site_data["gid"]) + analysis_gids = (set(meta_gids) + & set(self._site_data[SiteDataField.GID])) analysis_gids = np.array(sorted(list(analysis_gids))) # reduce the site data table to only those sites being analyzed - mask = np.isin(self._site_data["gid"], meta_gids) + mask = np.isin(self._site_data[SiteDataField.GID], meta_gids) self._site_data = self._site_data[mask] return analysis_gids, self._site_data @@ -317,9 +320,9 @@ def _parse_sam_sys_inputs(self): system_inputs = pd.DataFrame(system_inputs).T system_inputs = system_inputs.sort_index() - system_inputs["gid"] = system_inputs.index.values - system_inputs.index.name = "gid" - mask = system_inputs["gid"].isin(self.analysis_gids) + system_inputs[SiteDataField.GID] = system_inputs.index.values + system_inputs.index.name = SiteDataField.GID + mask = system_inputs[SiteDataField.GID].isin(self.analysis_gids) system_inputs = system_inputs[mask] return system_inputs @@ -383,7 +386,7 @@ def _preflight_checks(self): logger.info(msg) available_ids = list(self._nrwal_configs.keys()) - requested_ids = list(self._site_data['config'].values) + requested_ids = list(self._site_data[SiteDataField.CONFIG].values) missing = set(requested_ids) - set(available_ids) if any(missing): msg = ('The following config ids were requested in the offshore ' @@ -392,8 +395,8 @@ def _preflight_checks(self): logger.error(msg) raise OffshoreWindInputError(msg) - check_gid_order = (self._site_data["gid"].values - == self._sam_sys_inputs["gid"].values) + check_gid_order = (self._site_data[SiteDataField.GID].values + == self._sam_sys_inputs[SiteDataField.GID].values) msg = 'NRWAL site_data and system input dataframe had bad order' assert (check_gid_order).all(), msg @@ -450,7 +453,7 @@ def _get_input_data(self): site_data_vars = [var for var in all_required if var in self._site_data and var not in nrwal_inputs] - site_data_vars.append('config') + site_data_vars.append(SiteDataField.CONFIG) logger.info('Pulling the following inputs from the site_data input: {}' .format(site_data_vars)) for var in site_data_vars: @@ -648,7 +651,7 @@ def run_nrwal(self): self._out = self._init_outputs() for i, (cid, nrwal_config) in enumerate(self._nrwal_configs.items()): - output_mask = self._site_data['config'].values == cid + output_mask = self._site_data[SiteDataField.CONFIG].values == cid logger.info('Running NRWAL config {} of {}: "{}" and applying ' 'to {} out of {} total sites' .format(i + 1, len(self._nrwal_configs), cid, diff --git a/reV/utilities/__init__.py b/reV/utilities/__init__.py index 1446eb018..f27627896 100644 --- a/reV/utilities/__init__.py +++ b/reV/utilities/__init__.py @@ -8,6 +8,11 @@ from reV.version import __version__ +class SiteDataField(str, Enum): + GID = "gid" + CONFIG = "config" + + class ResourceMetaField(str, Enum): """An enumerated map to resource meta column names. diff --git a/tests/test_bespoke.py b/tests/test_bespoke.py index b384b58e4..428760b98 100644 --- a/tests/test_bespoke.py +++ b/tests/test_bespoke.py @@ -25,7 +25,7 @@ from reV.SAM.generation import WindPower from reV.supply_curve.supply_curve import SupplyCurve from reV.supply_curve.tech_mapping import TechMapping -from reV.utilities import MetaKeyName, ModuleName +from reV.utilities import MetaKeyName, ModuleName, SiteDataField pytest.importorskip("shapely") @@ -278,8 +278,8 @@ def test_packing_algorithm(gid=33): def test_bespoke_points(): """Test the bespoke points input options""" # pylint: disable=W0612 - points = pd.DataFrame({MetaKeyName.GID: [33, 34, 35], - 'config': ['default'] * 3}) + points = pd.DataFrame({SiteDataField.GID: [33, 34, 35], + SiteDataField.CONFIG: ['default'] * 3}) pp = BespokeWindPlants._parse_points(points, {'default': SAM}) assert len(pp) == 3 for gid in pp.gids: @@ -288,7 +288,7 @@ def test_bespoke_points(): points = pd.DataFrame({MetaKeyName.GID: [33, 34, 35]}) pp = BespokeWindPlants._parse_points(points, {'default': SAM}) assert len(pp) == 3 - assert 'config' in pp.df.columns + assert SiteDataField.CONFIG in pp.df.columns for gid in pp.gids: assert pp[gid][0] == 'default' @@ -497,12 +497,13 @@ def test_bespoke(): shutil.copy(RES.format(2013), res_fp.format(2013)) res_fp = res_fp.format('*') # both 33 and 35 are included, 37 is fully excluded - points = pd.DataFrame({MetaKeyName.GID: [33, 35], - 'config': ['default'] * 2, + points = pd.DataFrame({SiteDataField.GID: [33, 35], + SiteDataField.CONFIG: ['default'] * 2, 'extra_unused_data': [0, 42]}) - fully_excluded_points = pd.DataFrame({MetaKeyName.GID: [37], - 'config': ['default'], - 'extra_unused_data': [0]}) + fully_excluded_points = pd.DataFrame( + {SiteDataField.GID: [37], + SiteDataField.CONFIG: ['default'], + 'extra_unused_data': [0]}) TechMapping.run(excl_fp, RES.format(2012), dset=TM_DSET, max_workers=1) @@ -1004,7 +1005,8 @@ def test_bespoke_prior_run(): res_fp_2013 = res_fp.format('2013') # gids 33 and 35 are included, 37 is fully excluded - points = pd.DataFrame({MetaKeyName.GID: [33], 'config': ['default'], + points = pd.DataFrame({SiteDataField.GID: [33], + SiteDataField.CONFIG: ['default'], 'extra_unused_data': [42]}) TechMapping.run(excl_fp, RES.format(2012), dset=TM_DSET, max_workers=1) @@ -1074,7 +1076,8 @@ def test_gid_map(): res_fp_2013 = res_fp.format('2013') # gids 33 and 35 are included, 37 is fully excluded - points = pd.DataFrame({MetaKeyName.GID: [33], 'config': ['default'], + points = pd.DataFrame({SiteDataField.GID: [33], + SiteDataField.CONFIG: ['default'], 'extra_unused_data': [42]}) gid_map = pd.DataFrame({MetaKeyName.GID: [3, 4, 13, 12, 11, 10, 9]}) @@ -1157,7 +1160,8 @@ def test_bespoke_bias_correct(): res_fp_2013 = res_fp.format('2013') # gids 33 and 35 are included, 37 is fully excluded - points = pd.DataFrame({MetaKeyName.GID: [33], 'config': ['default'], + points = pd.DataFrame({SiteDataField.GID: [33], + SiteDataField.CONFIG: ['default'], 'extra_unused_data': [42]}) # intentionally leaving out WTK gid 13 which only has 5 included 90m @@ -1320,8 +1324,8 @@ def test_bespoke_5min_sample(): shutil.copy(EXCL, excl_fp) res_fp = os.path.join(TESTDATADIR, 'wtk/wtk_2010_*m.h5') - points = pd.DataFrame({MetaKeyName.GID: [33, 35], - 'config': ['default'] * 2, + points = pd.DataFrame({SiteDataField.GID: [33, 35], + SiteDataField.CONFIG: ['default'] * 2, 'extra_unused_data': [0, 42]}) sam_sys_inputs = copy.deepcopy(SAM_SYS_INPUTS) sam_sys_inputs['time_index_step'] = 12 diff --git a/tests/test_gen_config.py b/tests/test_gen_config.py index 37df6b7e6..9a4ebe06a 100644 --- a/tests/test_gen_config.py +++ b/tests/test_gen_config.py @@ -20,6 +20,7 @@ from reV.generation.generation import Gen from reV import TESTDATADIR from reV.handlers.outputs import Outputs +from reV.utilities import SiteDataField from rex.utilities.utilities import safe_json_load @@ -141,8 +142,8 @@ def test_sam_config(tech): sam_config = {'default': safe_json_load(sam_file)} points = slice(0, 100) - points_config = pd.DataFrame({"gid": range(0, 100), - "config": ['default'] * 100}) + points_config = pd.DataFrame({SiteDataField.GID: range(0, 100), + SiteDataField.CONFIG: ['default'] * 100}) gen_json = Gen('pvwattsv5', points, sam_file, res_file, output_request=('cf_profile',), sites_per_worker=50) @@ -162,8 +163,8 @@ def test_sam_config(tech): sam_config = {'default': safe_json_load(sam_file)} points = slice(0, 10) - points_config = pd.DataFrame({"gid": range(0, 10), - "config": ['default'] * 10}) + points_config = pd.DataFrame({SiteDataField.GID: range(0, 10), + SiteDataField.CONFIG: ['default'] * 10}) gen_json = Gen('windpower', points, sam_file, res_file, output_request=('cf_profile',), sites_per_worker=3) From d5a6c70e5dd435062b665e11cc81a1de30cf2463 Mon Sep 17 00:00:00 2001 From: ppinchuk Date: Mon, 20 May 2024 12:17:56 -0600 Subject: [PATCH 23/61] Walk back changes final --- reV/supply_curve/competitive_wind_farms.py | 13 ++-- reV/supply_curve/exclusions.py | 7 +-- reV/supply_curve/extent.py | 14 ++--- reV/supply_curve/points.py | 69 ++++++++++---------- reV/supply_curve/tech_mapping.py | 8 +-- reV/utilities/__init__.py | 1 + reV/utilities/pytest_utils.py | 8 ++- tests/hsds.py | 6 +- tests/test_config.py | 25 ++++---- tests/test_curtailment.py | 6 +- tests/test_econ_of_scale.py | 6 +- tests/test_econ_windbos.py | 4 +- tests/test_gen_forecast.py | 15 ++--- tests/test_gen_geothermal.py | 22 +++---- tests/test_gen_pv.py | 15 +++-- tests/test_gen_wind.py | 27 ++++---- tests/test_handlers_outputs.py | 3 +- tests/test_handlers_transmission.py | 5 +- tests/test_hybrids.py | 12 ++-- tests/test_losses_power_curve.py | 8 ++- tests/test_losses_scheduled.py | 7 ++- tests/test_nrwal.py | 73 ++++++++++------------ tests/test_sam.py | 6 +- tests/test_supply_curve_tech_mapping.py | 31 +++++---- 24 files changed, 194 insertions(+), 197 deletions(-) diff --git a/reV/supply_curve/competitive_wind_farms.py b/reV/supply_curve/competitive_wind_farms.py index 674009d3d..4c66522ef 100644 --- a/reV/supply_curve/competitive_wind_farms.py +++ b/reV/supply_curve/competitive_wind_farms.py @@ -78,8 +78,8 @@ def __getitem__(self, keys): """ if not isinstance(keys, tuple): msg = ("{} must be a tuple of form (source, gid) where source is: " - "MetaKeyName.SC_GID, '{}', or 'upwind', 'downwind'" - .format(keys, MetaKeyName.SC_POINT_GID)) + "{}, '{}', or 'upwind', 'downwind'" + .format(keys, MetaKeyName.SC_GID, MetaKeyName.SC_POINT_GID)) logger.error(msg) raise ValueError(msg) @@ -93,8 +93,9 @@ def __getitem__(self, keys): elif source == 'downwind': out = self.map_downwind(gid) else: - msg = ("{} must be: MetaKeyName.SC_GID, {}, or 'upwind', " - "'downwind'".format(source, MetaKeyName.SC_POINT_GID)) + msg = ("{} must be: {}, {}, or 'upwind', " + "'downwind'".format(source, MetaKeyName.SC_GID, + MetaKeyName.SC_POINT_GID)) logger.error(msg) raise ValueError(msg) @@ -227,8 +228,8 @@ def _parse_sc_points(cls, sc_points, offshore=False): sc_gids = sc_points.set_index(MetaKeyName.SC_GID) sc_gids = {k: int(v[0]) for k, v in sc_gids.iterrows()} - sc_point_gids = \ - sc_points.groupby(MetaKeyName.SC_POINT_GID)[MetaKeyName.SC_GID].unique().to_frame() + groups = sc_points.groupby(MetaKeyName.SC_POINT_GID) + sc_point_gids = groups[MetaKeyName.SC_GID].unique().to_frame() sc_point_gids = {int(k): v[MetaKeyName.SC_GID] for k, v in sc_point_gids.iterrows()} diff --git a/reV/supply_curve/exclusions.py b/reV/supply_curve/exclusions.py index a1278de52..1176b172e 100644 --- a/reV/supply_curve/exclusions.py +++ b/reV/supply_curve/exclusions.py @@ -9,8 +9,7 @@ from rex.utilities.loggers import log_mem from scipy import ndimage -from reV.handlers.exclusions import ExclusionLayers -from reV.utilities import MetaKeyName +from reV.handlers.exclusions import ExclusionLayers, LATITUDE, LONGITUDE from reV.utilities.exceptions import ExclusionLayerError, SupplyCurveInputError logger = logging.getLogger(__name__) @@ -701,7 +700,7 @@ def latitude(self): ------- ndarray """ - return self.excl_h5[MetaKeyName.LATITUDE] + return self.excl_h5[LATITUDE] @property def longitude(self): @@ -712,7 +711,7 @@ def longitude(self): ------- ndarray """ - return self.excl_h5[MetaKeyName.LONGITUDE] + return self.excl_h5[LONGITUDE] def add_layer(self, layer, replace=False): """ diff --git a/reV/supply_curve/extent.py b/reV/supply_curve/extent.py index 19ffac8b1..229797f69 100644 --- a/reV/supply_curve/extent.py +++ b/reV/supply_curve/extent.py @@ -9,7 +9,7 @@ import pandas as pd from rex.utilities.utilities import get_chunk_ranges -from reV.handlers.exclusions import ExclusionLayers +from reV.handlers.exclusions import ExclusionLayers, LATITUDE, LONGITUDE from reV.utilities import MetaKeyName from reV.utilities.exceptions import SupplyCurveError, SupplyCurveInputError @@ -303,10 +303,8 @@ def latitude(self): for r, c in zip(sc_rows.flatten(), sc_cols.flatten()): r = self.excl_row_slices[r] c = self.excl_col_slices[c] - lats.append(self.exclusions[MetaKeyName.LATITUDE, r, c].mean()) - lons.append( - self.exclusions[MetaKeyName.LONGITUDE, r, c].mean() - ) + lats.append(self.exclusions[LATITUDE, r, c].mean()) + lons.append(self.exclusions[LONGITUDE, r, c].mean()) self._latitude = np.array(lats, dtype="float32") self._longitude = np.array(lons, dtype="float32") @@ -332,10 +330,8 @@ def longitude(self): for r, c in zip(sc_rows.flatten(), sc_cols.flatten()): r = self.excl_row_slices[r] c = self.excl_col_slices[c] - lats.append(self.exclusions[MetaKeyName.LATITUDE, r, c].mean()) - lons.append( - self.exclusions[MetaKeyName.LONGITUDE, r, c].mean() - ) + lats.append(self.exclusions[LATITUDE, r, c].mean()) + lons.append(self.exclusions[LONGITUDE, r, c].mean()) self._latitude = np.array(lats, dtype="float32") self._longitude = np.array(lons, dtype="float32") diff --git a/reV/supply_curve/points.py b/reV/supply_curve/points.py index d70e42ba9..24a5cb0cb 100644 --- a/reV/supply_curve/points.py +++ b/reV/supply_curve/points.py @@ -14,9 +14,9 @@ from reV.econ.economies_of_scale import EconomiesOfScale from reV.econ.utilities import lcoe_fcr -from reV.handlers.exclusions import ExclusionLayers +from reV.handlers.exclusions import ExclusionLayers, LATITUDE, LONGITUDE from reV.supply_curve.exclusions import ExclusionMask, ExclusionMaskFromDict -from reV.utilities import MetaKeyName +from reV.utilities import MetaKeyName, ResourceMetaField from reV.utilities.exceptions import ( DataShapeError, EmptySupplyCurvePointError, @@ -362,10 +362,8 @@ def centroid(self): """ if self._centroid is None: - lats = self.exclusions.excl_h5[MetaKeyName.LATITUDE, self.rows, - self.cols] - lons = self.exclusions.excl_h5[MetaKeyName.LONGITUDE, self.rows, - self.cols] + lats = self.exclusions.excl_h5[LATITUDE, self.rows, self.cols] + lons = self.exclusions.excl_h5[LONGITUDE, self.rows, self.cols] self._centroid = (lats.mean(), lons.mean()) return self._centroid @@ -1047,18 +1045,19 @@ def h5(self): def country(self): """Get the SC point country based on the resource meta data.""" country = None - if 'country' in self.h5.meta and self.county is not None: + if (ResourceMetaField.COUNTRY in self.h5.meta + and self.county is not None): # make sure country and county are coincident counties = self.h5.meta.loc[self.h5_gid_set, - 'county'].values + ResourceMetaField.COUNTY].values iloc = np.where(counties == self.county)[0][0] country = self.h5.meta.loc[self.h5_gid_set, - 'country'].values + ResourceMetaField.COUNTRY].values country = country[iloc] - elif 'country' in self.h5.meta: + elif ResourceMetaField.COUNTRY in self.h5.meta: country = self.h5.meta.loc[self.h5_gid_set, - 'country'].mode() + ResourceMetaField.COUNTRY].mode() country = country.values[0] return country @@ -1067,16 +1066,18 @@ def country(self): def state(self): """Get the SC point state based on the resource meta data.""" state = None - if 'state' in self.h5.meta and self.county is not None: + if ResourceMetaField.STATE in self.h5.meta and self.county is not None: # make sure state and county are coincident counties = self.h5.meta.loc[self.h5_gid_set, - 'county'].values + ResourceMetaField.COUNTY].values iloc = np.where(counties == self.county)[0][0] - state = self.h5.meta.loc[self.h5_gid_set, 'state'].values + state = self.h5.meta.loc[self.h5_gid_set, + ResourceMetaField.STATE].values state = state[iloc] - elif 'state' in self.h5.meta: - state = self.h5.meta.loc[self.h5_gid_set, 'state'].mode() + elif ResourceMetaField.STATE in self.h5.meta: + state = self.h5.meta.loc[self.h5_gid_set, + ResourceMetaField.STATE].mode() state = state.values[0] return state @@ -1085,9 +1086,9 @@ def state(self): def county(self): """Get the SC point county based on the resource meta data.""" county = None - if 'county' in self.h5.meta: + if ResourceMetaField.COUNTY in self.h5.meta: county = self.h5.meta.loc[self.h5_gid_set, - 'county'].mode() + ResourceMetaField.COUNTY].mode() county = county.values[0] return county @@ -1096,9 +1097,9 @@ def county(self): def elevation(self): """Get the SC point elevation based on the resource meta data.""" elevation = None - if MetaKeyName.ELEVATION in self.h5.meta: + if ResourceMetaField.ELEVATION in self.h5.meta: elevation = self.h5.meta.loc[self.h5_gid_set, - MetaKeyName.ELEVATION].mean() + ResourceMetaField.ELEVATION].mean() return elevation @@ -1106,18 +1107,19 @@ def elevation(self): def timezone(self): """Get the SC point timezone based on the resource meta data.""" timezone = None - if MetaKeyName.TIMEZONE in self.h5.meta and self.county is not None: + if (ResourceMetaField.TIMEZONE in self.h5.meta + and self.county is not None): # make sure timezone flag and county are coincident counties = self.h5.meta.loc[self.h5_gid_set, - 'county'].values + ResourceMetaField.COUNTY].values iloc = np.where(counties == self.county)[0][0] timezone = self.h5.meta.loc[self.h5_gid_set, - MetaKeyName.TIMEZONE].values + ResourceMetaField.TIMEZONE].values timezone = timezone[iloc] - elif MetaKeyName.TIMEZONE in self.h5.meta: + elif ResourceMetaField.TIMEZONE in self.h5.meta: timezone = self.h5.meta.loc[self.h5_gid_set, - MetaKeyName.TIMEZONE].mode() + ResourceMetaField.TIMEZONE].mode() timezone = timezone.values[0] return timezone @@ -1127,18 +1129,19 @@ def offshore(self): """Get the SC point offshore flag based on the resource meta data (if offshore column is present).""" offshore = None - if MetaKeyName.OFFSHORE in self.h5.meta and self.county is not None: + if (ResourceMetaField.OFFSHORE in self.h5.meta + and self.county is not None): # make sure offshore flag and county are coincident counties = self.h5.meta.loc[self.h5_gid_set, - 'county'].values + ResourceMetaField.COUNTY].values iloc = np.where(counties == self.county)[0][0] offshore = self.h5.meta.loc[self.h5_gid_set, - MetaKeyName.OFFSHORE].values + ResourceMetaField.OFFSHORE].values offshore = offshore[iloc] - elif MetaKeyName.OFFSHORE in self.h5.meta: + elif ResourceMetaField.OFFSHORE in self.h5.meta: offshore = self.h5.meta.loc[self.h5_gid_set, - MetaKeyName.OFFSHORE].mode() + ResourceMetaField.OFFSHORE].mode() offshore = offshore.values[0] return offshore @@ -2023,9 +2026,9 @@ def point_summary(self, args=None): MetaKeyName.LATITUDE: self.latitude, MetaKeyName.LONGITUDE: self.longitude, MetaKeyName.TIMEZONE: self.timezone, - 'country': self.country, - 'state': self.state, - 'county': self.county, + MetaKeyName.COUNTRY: self.country, + MetaKeyName.STATE: self.state, + MetaKeyName.COUNTY: self.county, MetaKeyName.ELEVATION: self.elevation, MetaKeyName.RES_GIDS: self.res_gid_set, MetaKeyName.GEN_GIDS: self.gen_gid_set, diff --git a/reV/supply_curve/tech_mapping.py b/reV/supply_curve/tech_mapping.py index b5b9e67c3..c529c1136 100644 --- a/reV/supply_curve/tech_mapping.py +++ b/reV/supply_curve/tech_mapping.py @@ -21,8 +21,7 @@ from rex.utilities.utilities import res_dist_threshold from scipy.spatial import cKDTree -from reV.supply_curve.extent import SupplyCurveExtent -from reV.utilities import MetaKeyName +from reV.supply_curve.extent import SupplyCurveExtent, LATITUDE, LONGITUDE from reV.utilities.exceptions import FileInputError, FileInputWarning logger = logging.getLogger(__name__) @@ -176,8 +175,7 @@ def _get_excl_slices(gid, sc_row_indices, sc_col_indices, excl_row_slices, @classmethod def _get_excl_coords(cls, excl_fpath, gids, sc_row_indices, sc_col_indices, excl_row_slices, excl_col_slices, - coord_labels=(MetaKeyName.LATITUDE, - MetaKeyName.LONGITUDE)): + coord_labels=(LATITUDE, LONGITUDE)): """ Extract the exclusion coordinates for teh desired gids for TechMapping. @@ -341,7 +339,7 @@ def save_tech_map(excl_fpath, dset, indices, distance_threshold=None, def _check_fout(self): """Check the TechMapping output file for cached data.""" with h5py.File(self._excl_fpath, 'r') as f: - if MetaKeyName.LATITUDE not in f or MetaKeyName.LONGITUDE not in f: + if LATITUDE not in f or LONGITUDE not in f: emsg = ('Datasets "latitude" and/or "longitude" not in ' 'pre-existing Exclusions TechMapping file "{}". ' 'Cannot proceed.' diff --git a/reV/utilities/__init__.py b/reV/utilities/__init__.py index f27627896..f15a98f2a 100644 --- a/reV/utilities/__init__.py +++ b/reV/utilities/__init__.py @@ -28,6 +28,7 @@ class ResourceMetaField(str, Enum): COUNTY = "county" STATE = "state" COUNTRY = "country" + OFFSHORE = 'offshore' class MetaKeyName(str, Enum): diff --git a/reV/utilities/pytest_utils.py b/reV/utilities/pytest_utils.py index c95294944..8e2e5878b 100644 --- a/reV/utilities/pytest_utils.py +++ b/reV/utilities/pytest_utils.py @@ -7,6 +7,7 @@ import pandas as pd from packaging import version from rex.outputs import Outputs as RexOutputs +from reV.utilities import ResourceMetaField def pd_date_range(*args, **kwargs): @@ -90,9 +91,10 @@ def make_fake_h5_chunks(td, features, shuffle=False): for i, s1 in enumerate(s_slices): for j, s2 in enumerate(s_slices): out_file = out_pattern.format(i=i, j=j) - meta = pd.DataFrame({MetaKeyName.LATITUDE: lat[s1, s2].flatten(), - MetaKeyName.LONGITUDE: lon[s1, s2].flatten(), - MetaKeyName.GID: gids[s1, s2].flatten()}) + meta = pd.DataFrame( + {ResourceMetaField.LATITUDE: lat[s1, s2].flatten(), + ResourceMetaField.LONGITUDE: lon[s1, s2].flatten(), + ResourceMetaField.GID: gids[s1, s2].flatten()}) write_chunk(meta=meta, times=times, data=data[s1, s2], features=features, out_file=out_file) diff --git a/tests/hsds.py b/tests/hsds.py index 64f2ada55..4267e0702 100644 --- a/tests/hsds.py +++ b/tests/hsds.py @@ -11,6 +11,8 @@ import pytest from rex.renewable_resource import NSRDB +from reV.utilities import ResourceMetaField + @pytest.fixture def NSRDB_hsds(): @@ -54,7 +56,9 @@ def check_meta(res_cls): assert isinstance(meta, pd.DataFrame) assert meta.shape == (len(sites), meta_shape[1]) # select columns - meta = res_cls['meta', :, [MetaKeyName.LATITUDE, MetaKeyName.LONGITUDE]] + meta = res_cls[ + 'meta', :, [ResourceMetaField.LATITUDE, ResourceMetaField.LONGITUDE] + ] assert isinstance(meta, pd.DataFrame) assert meta.shape == (meta_shape[0], 2) diff --git a/tests/test_config.py b/tests/test_config.py index 09610db6a..eceebf739 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -23,7 +23,7 @@ from reV.config.project_points import PointsControl, ProjectPoints from reV.generation.generation import Gen from reV.SAM.SAM import RevPySam -from reV.utilities import MetaKeyName +from reV.utilities import ResourceMetaField from reV.utilities.exceptions import ConfigError @@ -131,7 +131,7 @@ def test_config_mapping(): "onshore": os.path.join( TESTDATADIR, "SAM/wind_gen_standard_losses_0.json" ), - MetaKeyName.OFFSHORE: os.path.join( + "offshore": os.path.join( TESTDATADIR, "SAM/wind_gen_standard_losses_1.json" ), } @@ -153,7 +153,7 @@ def test_sam_config_kw_replace(): "onshore": os.path.join( TESTDATADIR, "SAM/wind_gen_standard_losses_0.json" ), - MetaKeyName.OFFSHORE: os.path.join( + "offshore": os.path.join( TESTDATADIR, "SAM/wind_gen_standard_losses_1.json" ), } @@ -168,19 +168,19 @@ def test_sam_config_kw_replace(): sites_per_worker=100, ) config_on = gen.project_points.sam_inputs["onshore"] - config_of = gen.project_points.sam_inputs[MetaKeyName.OFFSHORE] + config_of = gen.project_points.sam_inputs["offshore"] assert "turb_generic_loss" in config_on assert "turb_generic_loss" in config_of pp_split = ProjectPoints.split(0, 10000, gen.project_points) config_on = pp_split.sam_inputs["onshore"] - config_of = pp_split.sam_inputs[MetaKeyName.OFFSHORE] + config_of = pp_split.sam_inputs["offshore"] assert "turb_generic_loss" in config_on assert "turb_generic_loss" in config_of pc_split = PointsControl.split(0, 10000, gen.project_points) config_on = pc_split.project_points.sam_inputs["onshore"] - config_of = pc_split.project_points.sam_inputs[MetaKeyName.OFFSHORE] + config_of = pc_split.project_points.sam_inputs["offshore"] assert "turb_generic_loss" in config_on assert "turb_generic_loss" in config_of @@ -189,8 +189,8 @@ def test_sam_config_kw_replace(): config = ipc.project_points.sam_inputs["onshore"] assert "turb_generic_loss" in config - if MetaKeyName.OFFSHORE in ipc.project_points.sam_inputs: - config = ipc.project_points.sam_inputs[MetaKeyName.OFFSHORE] + if "offshore" in ipc.project_points.sam_inputs: + config = ipc.project_points.sam_inputs["offshore"] assert "turb_generic_loss" in config @@ -229,7 +229,7 @@ def test_coords(sites): gids = [gids] lat_lons = meta.loc[ - gids, [MetaKeyName.LATITUDE, MetaKeyName.LONGITUDE] + gids, [ResourceMetaField.LATITUDE, ResourceMetaField.LONGITUDE] ].values pp = ProjectPoints.lat_lon_coords(lat_lons, res_file, sam_files) @@ -249,7 +249,8 @@ def test_coords_from_file(): if not isinstance(gids, list): gids = [gids] - lat_lons = meta.loc[gids, [MetaKeyName.LATITUDE, MetaKeyName.LONGITUDE]] + lat_lons = meta.loc[gids, [ResourceMetaField.LATITUDE, + ResourceMetaField.LONGITUDE]] with tempfile.TemporaryDirectory() as td: coords_fp = os.path.join(td, "lat_lon.csv") lat_lons.to_csv(coords_fp, index=False) @@ -269,7 +270,7 @@ def test_duplicate_coords(): meta = f.meta duplicates = meta.loc[ - [2, 3, 3, 4], [MetaKeyName.LATITUDE, MetaKeyName.LONGITUDE] + [2, 3, 3, 4], [ResourceMetaField.LATITUDE, ResourceMetaField.LONGITUDE] ].values with pytest.raises(RuntimeError): @@ -289,7 +290,7 @@ def test_sam_configs(): "onshore": os.path.join( TESTDATADIR, "SAM/wind_gen_standard_losses_0.json" ), - MetaKeyName.OFFSHORE: os.path.join( + "offshore": os.path.join( TESTDATADIR, "SAM/wind_gen_standard_losses_1.json" ), } diff --git a/tests/test_curtailment.py b/tests/test_curtailment.py index 3f6c5dbab..aba06a385 100644 --- a/tests/test_curtailment.py +++ b/tests/test_curtailment.py @@ -18,7 +18,7 @@ from reV.config.project_points import ProjectPoints from reV.generation.generation import Gen from reV.SAM.SAM import RevPySam -from reV.utilities import MetaKeyName +from reV.utilities import ResourceMetaField from reV.utilities.curtailment import curtail @@ -171,8 +171,8 @@ def test_res_curtailment(year, site): sza = SolarPosition( non_curtailed_res.time_index, - non_curtailed_res.meta[[MetaKeyName.LATITUDE, - MetaKeyName.LONGITUDE]].values).zenith + non_curtailed_res.meta[[ResourceMetaField.LATITUDE, + ResourceMetaField.LONGITUDE]].values).zenith ti = non_curtailed_res.time_index diff --git a/tests/test_econ_of_scale.py b/tests/test_econ_of_scale.py index 1aade9dc0..74248e683 100644 --- a/tests/test_econ_of_scale.py +++ b/tests/test_econ_of_scale.py @@ -48,7 +48,7 @@ def test_pass_through_lcoe_args(): sam_files = os.path.join(TESTDATADIR, 'SAM/i_windpower_lcoe.json') output_request = ('cf_mean', - MetaKeyName.LCOE_FCR, + 'lcoe_fcr', 'system_capacity', 'capital_cost', 'fixed_charge_rate', @@ -62,7 +62,7 @@ def test_pass_through_lcoe_args(): checks = [x in gen.out for x in Gen.LCOE_ARGS] assert all(checks) - assert MetaKeyName.LCOE_FCR in gen.out + assert 'lcoe_fcr' in gen.out assert 'cf_mean' in gen.out @@ -78,7 +78,7 @@ def test_lcoe_calc_simple(): true_lcoe = ((data['fcr'] * data['capital_cost'] + data['foc']) / (data['aep'] / 1000)) - data[MetaKeyName.MEAN_LCOE] = true_lcoe + data['mean_lcoe'] = true_lcoe eos = EconomiesOfScale(eqn, data) assert eos.raw_capital_cost == eos.scaled_capital_cost diff --git a/tests/test_econ_windbos.py b/tests/test_econ_windbos.py index e6a76b18b..93aea522e 100644 --- a/tests/test_econ_windbos.py +++ b/tests/test_econ_windbos.py @@ -19,7 +19,7 @@ from reV.econ.econ import Econ from reV.generation.generation import Gen from reV.SAM.windbos import WindBos -from reV.utilities import MetaKeyName +from reV.utilities import SiteDataField RTOL = 0.000001 ATOL = 0.001 @@ -287,7 +287,7 @@ def test_run_bos(points=slice(0, 5), max_workers=1): # get full file paths. sam_files = TESTDATADIR + "/SAM/i_singleowner_windbos.json" site_data = pd.DataFrame( - {MetaKeyName.GID: range(5), "sales_tax_basis": range(5)} + {SiteDataField.GID: range(5), "sales_tax_basis": range(5)} ) econ_outs = ( diff --git a/tests/test_gen_forecast.py b/tests/test_gen_forecast.py index 41edec897..f0ca4637b 100644 --- a/tests/test_gen_forecast.py +++ b/tests/test_gen_forecast.py @@ -21,7 +21,7 @@ from reV.config.project_points import ProjectPoints from reV.generation.generation import Gen from reV.handlers.outputs import Outputs -from reV.utilities import MetaKeyName +from reV.utilities import ResourceMetaField from reV.utilities.exceptions import SAMExecutionError @@ -38,14 +38,15 @@ def test_forecast(): with Outputs(res_file, mode='a') as f: meta = f.meta - meta = meta.drop([MetaKeyName.TIMEZONE, 'elevation'], axis=1) + meta = meta.drop([ResourceMetaField.TIMEZONE, + ResourceMetaField.ELEVATION], axis=1) del f._h5['meta'] f._meta = None f.meta = meta with Outputs(res_file, mode='r') as f: - assert MetaKeyName.TIMEZONE not in f.meta - assert 'elevation' not in f.meta + assert ResourceMetaField.TIMEZONE not in f.meta + assert ResourceMetaField.ELEVATION not in f.meta with Resource(res_file) as res: ghi = res['ghi'] @@ -53,9 +54,9 @@ def test_forecast(): points = ProjectPoints(slice(0, 5), sam_files, 'pvwattsv7', res_file=res_file) output_request = ('cf_mean', 'ghi_mean') - site_data = pd.DataFrame({MetaKeyName.GID: np.arange(5), - MetaKeyName.TIMEZONE: -5, - 'elevation': 0}) + site_data = pd.DataFrame({ResourceMetaField.GID: np.arange(5), + ResourceMetaField.TIMEZONE: -5, + ResourceMetaField.ELEVATION: 0}) gid_map = {0: 20, 1: 20, 2: 50, 3: 51, 4: 51} # test that this raises an error with missing timezone diff --git a/tests/test_gen_geothermal.py b/tests/test_gen_geothermal.py index 26995e383..4f27550c7 100644 --- a/tests/test_gen_geothermal.py +++ b/tests/test_gen_geothermal.py @@ -17,7 +17,7 @@ from reV import TESTDATADIR from reV.generation.generation import Gen from reV.SAM.generation import Geothermal -from reV.utilities import MetaKeyName, ResourceMetaField +from reV.utilities import ResourceMetaField DEFAULT_GEO_SAM_FILE = TESTDATADIR + "/SAM/geothermal_default.json" RTOL = 0.1 @@ -80,7 +80,7 @@ def test_gen_geothermal(depth, sample_resource_data): "cf_mean", "cf_profile", "gen_profile", - MetaKeyName.LCOE_FCR, + "lcoe_fcr", "nameplate", ) gen = Gen( @@ -130,7 +130,7 @@ def test_gen_geothermal_temp_too_low(sample_resource_data): "cf_mean", "cf_profile", "gen_profile", - MetaKeyName.LCOE_FCR, + "lcoe_fcr", "nameplate", ) gen = Gen( @@ -185,11 +185,7 @@ def test_per_kw_cost_inputs(sample_resource_data): with open(geo_sam_file, "w") as fh: json.dump(geo_config, fh) - output_request = ( - MetaKeyName.CAPITAL_COST, - MetaKeyName.FIXED_OPERATING_COST, - MetaKeyName.LCOE_FCR, - ) + output_request = ("capital_cost", "fixed_operating_cost", "lcoe_fcr") gen = Gen( "geothermal", points, @@ -236,11 +232,7 @@ def test_drill_cost_inputs(sample_resource_data): with open(geo_sam_file, "w") as fh: json.dump(geo_config, fh) - output_request = ( - MetaKeyName.CAPITAL_COST, - MetaKeyName.FIXED_OPERATING_COST, - MetaKeyName.LCOE_FCR, - ) + output_request = ("capital_cost", "fixed_operating_cost", "lcoe_fcr") gen = Gen( "geothermal", points, @@ -288,7 +280,7 @@ def test_gen_with_nameplate_input(sample_resource_data): "cf_mean", "cf_profile", "gen_profile", - MetaKeyName.LCOE_FCR, + "lcoe_fcr", "nameplate", ) gen = Gen( @@ -483,7 +475,7 @@ def test_gen_with_time_index_step_input(sample_resource_data): "cf_mean", "cf_profile", "gen_profile", - MetaKeyName.LCOE_FCR, + "lcoe_fcr", "nameplate", ) gen = Gen( diff --git a/tests/test_gen_pv.py b/tests/test_gen_pv.py index 122097ecd..279a8f709 100644 --- a/tests/test_gen_pv.py +++ b/tests/test_gen_pv.py @@ -22,7 +22,7 @@ from reV.config.project_points import ProjectPoints from reV.generation.generation import Gen from reV.handlers.outputs import Outputs -from reV.utililities import MetaKeyName +from reV.utilities import SiteDataField from reV.utilities.exceptions import ConfigError, ExecutionError RTOL = 0.0 @@ -414,7 +414,7 @@ def test_gen_input_mods(): gen.run(max_workers=1) for i in range(5): inputs = gen.project_points[i][1] - assert inputs['tilt'] == MetaKeyName.LATITUDE + assert inputs['tilt'] == "latitude" def test_gen_input_pass_through(): @@ -451,7 +451,7 @@ def test_gen_pv_site_data(): sites_per_worker=1, output_request=output_request) baseline.run(max_workers=1) - site_data = pd.DataFrame({MetaKeyName.GID: np.arange(2), + site_data = pd.DataFrame({SiteDataField.GID: np.arange(2), 'losses': np.ones(2)}) test = Gen('pvwattsv7', rev2_points, sam_files, res_file, sites_per_worker=1, output_request=output_request, @@ -598,8 +598,8 @@ def test_irrad_bias_correct(): sites_per_worker=1, output_request=output_request) gen_base.run(max_workers=1) - bc_df = pd.DataFrame({MetaKeyName.GID: np.arange(1, 10), 'method': 'lin_irrad', - 'scalar': 1, 'adder': 50}) + bc_df = pd.DataFrame({SiteDataField.GID: np.arange(1, 10), + 'method': 'lin_irrad', 'scalar': 1, 'adder': 50}) gen = Gen('pvwattsv7', points, sam_files, res_file, sites_per_worker=1, output_request=output_request, bias_correct=bc_df) @@ -617,9 +617,8 @@ def test_irrad_bias_correct(): mask = (gen_base.out['cf_profile'][:, 1:] <= gen.out['cf_profile'][:, 1:]) assert (mask.sum() / mask.size) > 0.99 - bc_df = pd.DataFrame({MetaKeyName.GID: np.arange(100), - 'method': 'lin_irrad', - 'scalar': 1, 'adder': -1500}) + bc_df = pd.DataFrame({SiteDataField.GID: np.arange(100), + 'method': 'lin_irrad', 'scalar': 1, 'adder': -1500}) gen = Gen('pvwattsv7', points, sam_files, res_file, sites_per_worker=1, output_request=output_request, bias_correct=bc_df) gen.run(max_workers=2) diff --git a/tests/test_gen_wind.py b/tests/test_gen_wind.py index 56c1687a4..392a5ef25 100644 --- a/tests/test_gen_wind.py +++ b/tests/test_gen_wind.py @@ -21,7 +21,7 @@ from reV import TESTDATADIR from reV.config.project_points import ProjectPoints from reV.generation.generation import Gen -from reV.utilities import MetaKeyName +from reV.utilities import ResourceMetaField RTOL = 0 ATOL = 0.001 @@ -111,18 +111,18 @@ def test_wind_gen_slice(f_rev1_out, rev2_points, year, max_workers): assert np.allclose(gen_outs, cf_mean_list, rtol=RTOL, atol=ATOL), msg assert np.allclose(pp.sites, gen.meta.index.values), "bad gen meta!" assert np.allclose( - pp.sites, gen.meta[MetaKeyName.GID].values + pp.sites, gen.meta[ResourceMetaField.GID].values ), "bad gen meta!" - labels = [MetaKeyName.LATITUDE, MetaKeyName.LONGITUDE] + labels = [ResourceMetaField.LATITUDE, ResourceMetaField.LONGITUDE] with Resource(res_file) as res: for i, (gen_gid, site_meta) in enumerate(gen.meta.iterrows()): - res_gid = site_meta[MetaKeyName.GID] + res_gid = site_meta[ResourceMetaField.GID] assert gen_gid == res_gid test_coords = site_meta[labels].values.astype(float) true_coords = res.meta.loc[res_gid, labels].values.astype(float) assert np.allclose(test_coords, true_coords) - assert site_meta[MetaKeyName.GID] == res_gid + assert site_meta[ResourceMetaField.GID] == res_gid @pytest.mark.parametrize( @@ -193,7 +193,8 @@ def test_gid_map(gid_map): assert np.allclose(map_test.out[key], write_gid_test.out[key]) for map_test_gid, write_test_gid in zip( - map_test.meta[MetaKeyName.GID], write_gid_test.meta[MetaKeyName.GID] + map_test.meta[ResourceMetaField.GID], + write_gid_test.meta[ResourceMetaField.GID] ): assert map_test_gid == gid_map[write_test_gid] @@ -217,21 +218,21 @@ def test_gid_map(gid_map): map_test.out[key][gen_gid_test], ) - labels = [MetaKeyName.LATITUDE, MetaKeyName.LONGITUDE] + labels = [ResourceMetaField.LATITUDE, ResourceMetaField.LONGITUDE] with Resource(res_file) as res: for i, (gen_gid, site_meta) in enumerate(baseline.meta.iterrows()): - res_gid = site_meta[MetaKeyName.GID] + res_gid = site_meta[ResourceMetaField.GID] test_coords = site_meta[labels].values.astype(float) true_coords = res.meta.loc[res_gid, labels].values.astype(float) assert np.allclose(test_coords, true_coords) - assert site_meta[MetaKeyName.GID] == res_gid + assert site_meta[ResourceMetaField.GID] == res_gid for i, (gen_gid, site_meta) in enumerate(map_test.meta.iterrows()): res_gid = gid_map[gen_gid] test_coords = site_meta[labels].values.astype(float) true_coords = res.meta.loc[res_gid, labels].values.astype(float) assert np.allclose(test_coords, true_coords) - assert site_meta[MetaKeyName.GID] == res_gid + assert site_meta[ResourceMetaField.GID] == res_gid def test_wind_gen_new_outputs(points=slice(0, 10), year=2012, max_workers=1): @@ -321,7 +322,7 @@ def test_wind_gen_site_data(points=slice(0, 5), year=2012, max_workers=1): baseline.run(max_workers=max_workers) site_data = pd.DataFrame( - {MetaKeyName.GID: np.arange(2), "turb_generic_loss": np.zeros(2)} + {ResourceMetaField.GID: np.arange(2), "turb_generic_loss": np.zeros(2)} ) test = Gen( "windpower", @@ -433,7 +434,7 @@ def test_wind_bias_correct(): bc_df = pd.DataFrame( { - MetaKeyName.GID: np.arange(100), + ResourceMetaField.GID: np.arange(100), "method": "lin_ws", "scalar": 1, "adder": 2, @@ -455,7 +456,7 @@ def test_wind_bias_correct(): bc_df = pd.DataFrame( { - MetaKeyName.GID: np.arange(100), + ResourceMetaField.GID: np.arange(100), "method": "lin_ws", "scalar": 1, "adder": -100, diff --git a/tests/test_handlers_outputs.py b/tests/test_handlers_outputs.py index 7f53efd14..8b4eaf1f2 100644 --- a/tests/test_handlers_outputs.py +++ b/tests/test_handlers_outputs.py @@ -17,8 +17,7 @@ arr1 = np.ones(100) arr2 = np.ones((8760, 100)) arr3 = np.ones((8760, 100), dtype=float) * 42.42 -meta = pd.DataFrame({MetaKeyName.LATITUDE: np.ones(100), - MetaKeyName.LONGITUDE: np.zeros(100)}) +meta = pd.DataFrame({"latitude": np.ones(100), "longitude": np.zeros(100)}) time_index = pd_date_range('20210101', '20220101', freq='1h', closed='right') diff --git a/tests/test_handlers_transmission.py b/tests/test_handlers_transmission.py index 1ce11717d..8d30703ac 100644 --- a/tests/test_handlers_transmission.py +++ b/tests/test_handlers_transmission.py @@ -9,6 +9,8 @@ from reV import TESTDATADIR from reV.handlers.transmission import TransmissionFeatures as TF +from reV.utilities import MetaKeyName + TRANS_COSTS_1 = {'line_tie_in_cost': 200, 'line_cost': 1000, 'station_tie_in_cost': 50, 'center_tie_in_cost': 10, @@ -76,7 +78,8 @@ def test_cost_calculation(i, trans_costs, distance, gid, trans_table): assert true_cost == trans_cost -@pytest.mark.parametrize(('trans_costs', MetaKeyName.CAPACITY, MetaKeyName.GID), +@pytest.mark.parametrize(('trans_costs', MetaKeyName.CAPACITY, + MetaKeyName.GID), ((TRANS_COSTS_1, 350, 43300), (TRANS_COSTS_2, 350, 43300), (TRANS_COSTS_1, 100, 43300), diff --git a/tests/test_hybrids.py b/tests/test_hybrids.py index a31e959e0..55e0747cd 100644 --- a/tests/test_hybrids.py +++ b/tests/test_hybrids.py @@ -13,7 +13,7 @@ from reV.cli import main from reV.hybrids import HYBRID_METHODS, Hybridization from reV.hybrids.hybrids import MERGE_COLUMN, OUTPUT_PROFILE_NAMES, HybridsData -from reV.utilities import ModuleName +from reV.utilities import ModuleName, MetaKeyName from reV.utilities.exceptions import FileInputError, InputError, OutputWarning SOLAR_FPATH = os.path.join( @@ -112,7 +112,9 @@ def test_hybridization_profile_output(): h.run() hp, hsp, hwp, = h.profiles.values() h_meta = h.hybrid_meta - h_idx = np.where(h_meta[MetaKeyName.SC_POINT_GID] == common_sc_point_gid)[0][0] + h_idx = np.where( + h_meta[MetaKeyName.SC_POINT_GID] == common_sc_point_gid + )[0][0] assert np.allclose(hp[:, h_idx], weighted_solar + weighted_wind) assert np.allclose(hsp[:, h_idx], weighted_solar) @@ -739,8 +741,10 @@ def make_test_file(in_fp, out_fp, p_slice=slice(None), t_slice=slice(None), half_n_rows = n_rows // 2 meta.iloc[-half_n_rows:] = meta.iloc[:half_n_rows].values if duplicate_coord_values: - meta.loc[0, MetaKeyName.LATITUDE] = meta[MetaKeyName.LATITUDE].iloc[-1] - meta.loc[0, MetaKeyName.LATITUDE] = meta[MetaKeyName.LATITUDE].iloc[-1] + lat = meta[MetaKeyName.LATITUDE].iloc[-1] + meta.loc[0, MetaKeyName.LATITUDE] = lat + lon = meta[MetaKeyName.LATITUDE].iloc[-1] + meta.loc[0, MetaKeyName.LATITUDE] = lon shapes['meta'] = len(meta) for d in dset_names: shapes[d] = (len(res.time_index[t_slice]), len(meta)) diff --git a/tests/test_losses_power_curve.py b/tests/test_losses_power_curve.py index 1685e217f..faee603aa 100644 --- a/tests/test_losses_power_curve.py +++ b/tests/test_losses_power_curve.py @@ -29,6 +29,8 @@ ) from reV.losses.scheduled import ScheduledLossesMixin from reV.utilities.exceptions import reVLossesValueError, reVLossesWarning +from reV.utilities import ResourceMetaField + REV_POINTS = list(range(3)) RES_FILE = TESTDATADIR + '/wtk/ri_100_wtk_2012.h5' @@ -149,7 +151,7 @@ def _run_gen_with_and_without_losses( # undo UTC array rolling for ind, row in gen.meta.iterrows(): - time_shift = row[MetaKeyName.TIMEZONE] + time_shift = row[ResourceMetaField.TIMEZONE] gen_profiles_with_losses[:, ind] = np.roll( gen_profiles_with_losses[:, ind], time_shift ) @@ -168,7 +170,7 @@ def _run_gen_with_and_without_losses( gen_profiles = gen.out['gen_profile'] for ind, row in gen.meta.iterrows(): - time_shift = row[MetaKeyName.TIMEZONE] + time_shift = row[ResourceMetaField.TIMEZONE] gen_profiles[:, ind] = np.roll(gen_profiles[:, ind], time_shift) return gen_profiles, gen_profiles_with_losses @@ -179,7 +181,7 @@ def _make_site_data_df(site_data): if site_data is not None: site_specific_losses = [json.dumps(site_data)] * len(REV_POINTS) site_data_dict = { - MetaKeyName.GID: REV_POINTS, + ResourceMetaField.GID: REV_POINTS, PowerCurveLossesMixin.POWER_CURVE_CONFIG_KEY: site_specific_losses } site_data = pd.DataFrame(site_data_dict) diff --git a/tests/test_losses_scheduled.py b/tests/test_losses_scheduled.py index 4f7699303..00aa0ad18 100644 --- a/tests/test_losses_scheduled.py +++ b/tests/test_losses_scheduled.py @@ -32,6 +32,7 @@ ) from reV.losses.utils import hourly_indices_for_months from reV.utilities.exceptions import reVLossesValueError, reVLossesWarning +from reV.utilities import ResourceMetaField REV_POINTS = list(range(3)) RTOL = 0 @@ -349,7 +350,7 @@ def _run_gen_with_and_without_losses( gen_profiles_with_losses = gen_profiles_with_losses[::time_steps_in_hour] # undo UTC array rolling for ind, row in gen.meta.iterrows(): - time_shift = row[MetaKeyName.TIMEZONE] + time_shift = row[ResourceMetaField.TIMEZONE] gen_profiles_with_losses[:, ind] = np.roll( gen_profiles_with_losses[:, ind], time_shift ) @@ -374,7 +375,7 @@ def _run_gen_with_and_without_losses( time_steps_in_hour = int(round(gen_profiles.shape[0] / 8760)) gen_profiles = gen_profiles[::time_steps_in_hour] for ind, row in gen.meta.iterrows(): - time_shift = row[MetaKeyName.TIMEZONE] + time_shift = row[ResourceMetaField.TIMEZONE] gen_profiles[:, ind] = np.roll(gen_profiles[:, ind], time_shift) return gen_profiles, gen_profiles_with_losses @@ -385,7 +386,7 @@ def _make_site_data_df(site_data): if site_data is not None: site_specific_outages = [json.dumps(site_data)] * len(REV_POINTS) site_data_dict = { - MetaKeyName.GID: REV_POINTS, + ResourceMetaField.GID: REV_POINTS, ScheduledLossesMixin.OUTAGE_CONFIG_KEY: site_specific_outages } site_data = pd.DataFrame(site_data_dict) diff --git a/tests/test_nrwal.py b/tests/test_nrwal.py index 903f5aadc..cfcf672d6 100644 --- a/tests/test_nrwal.py +++ b/tests/test_nrwal.py @@ -21,7 +21,7 @@ from reV.cli import main from reV.handlers.outputs import Outputs from reV.nrwal.nrwal import RevNrwal -from reV.utilities import MetaKeyName, ModuleName +from reV.utilities import ModuleName SOURCE_DIR = os.path.join(TESTDATADIR, 'nrwal/') @@ -37,10 +37,8 @@ def test_nrwal(): site_data = os.path.join(td, 'example_offshore_data.csv') offshore_config = os.path.join(td, 'offshore.json') onshore_config = os.path.join(td, 'onshore.json') - sam_configs = {'onshore': onshore_config, - MetaKeyName.OFFSHORE: offshore_config} - nrwal_configs = {MetaKeyName.OFFSHORE: - os.path.join(td, 'nrwal_offshore.yaml')} + sam_configs = {'onshore': onshore_config, "offshore": offshore_config} + nrwal_configs = {"offshore": os.path.join(td, 'nrwal_offshore.yaml')} with Outputs(gen_fpath, 'a') as f: f.time_index = pd_date_range('20100101', '20110101', @@ -48,7 +46,7 @@ def test_nrwal(): f._add_dset('cf_profile', np.random.random(f.shape), np.uint32, attrs={'scale_factor': 1000}, chunks=(None, 10)) - f._add_dset(MetaKeyName.FIXED_CHARGE_RATE, + f._add_dset('fixed_charge_rate', 0.09 * np.ones(f.shape[1], dtype=np.float32), np.float32, attrs={'scale_factor': 1}, chunks=None) @@ -57,15 +55,15 @@ def test_nrwal(): original_dsets = [d for d in f.dsets if d not in ('meta', 'time_index')] meta_raw = f.meta - lcoe_raw = f[MetaKeyName.LCOE_FCR] + lcoe_raw = f['lcoe_fcr'] cf_mean_raw = f['cf_mean'] cf_profile_raw = f['cf_profile'] mask = meta_raw.offshore == 1 - output_request = [MetaKeyName.FIXED_CHARGE_RATE, 'depth', + output_request = ['fixed_charge_rate', 'depth', 'total_losses', 'array', 'export', 'gcf_adjustment', - MetaKeyName.LCOE_FCR, 'cf_mean', ] + 'lcoe_fcr', 'cf_mean', ] obj = RevNrwal(gen_fpath, site_data, sam_configs, nrwal_configs, output_request, site_meta_cols=['depth']) @@ -73,14 +71,14 @@ def test_nrwal(): with Outputs(gen_fpath, 'r') as f: meta_new = f.meta - lcoe_new = f[MetaKeyName.LCOE_FCR] + lcoe_new = f['lcoe_fcr'] losses = f['total_losses'] gcf_adjustment = f['gcf_adjustment'] assert np.allclose(cf_mean_raw, f['cf_mean_raw']) assert np.allclose(cf_profile_raw, f['cf_profile_raw']) cf_mean_new = f['cf_mean'] cf_profile_new = f['cf_profile'] - fcr = f[MetaKeyName.FIXED_CHARGE_RATE] + fcr = f['fixed_charge_rate'] depth = f['depth'] for key in [d for d in original_dsets if d in f]: @@ -104,7 +102,7 @@ def test_nrwal(): # make sure the second offshore compute gives same results as first with Outputs(gen_fpath, 'r') as f: - assert np.allclose(lcoe_new, f[MetaKeyName.LCOE_FCR]) + assert np.allclose(lcoe_new, f['lcoe_fcr']) assert np.allclose(cf_mean_new, f['cf_mean']) assert np.allclose(cf_profile_new, f['cf_profile']) assert np.allclose(cf_mean_raw, f['cf_mean_raw']) @@ -155,10 +153,8 @@ def test_nrwal_csv(out_fn): site_data = os.path.join(td, 'example_offshore_data.csv') offshore_config = os.path.join(td, 'offshore.json') onshore_config = os.path.join(td, 'onshore.json') - sam_configs = {'onshore': onshore_config, - MetaKeyName.OFFSHORE: offshore_config} - nrwal_configs = {MetaKeyName.OFFSHORE: - os.path.join(td, 'nrwal_offshore.yaml')} + sam_configs = {'onshore': onshore_config, "offshore": offshore_config} + nrwal_configs = {"offshore": os.path.join(td, 'nrwal_offshore.yaml')} with Outputs(gen_fpath, 'a') as f: f.time_index = pd_date_range('20100101', '20110101', @@ -169,14 +165,14 @@ def test_nrwal_csv(out_fn): f._add_dset('cf_mean_raw', np.random.random(f.shape[1]), np.uint32, attrs={'scale_factor': 1000}, chunks=None) - f._add_dset(MetaKeyName.FIXED_CHARGE_RATE, + f._add_dset('fixed_charge_rate', 0.09 * np.ones(f.shape[1], dtype=np.float32), np.float32, attrs={'scale_factor': 1}, chunks=None) compatible = ['depth', 'total_losses', 'array', 'export', - 'gcf_adjustment', MetaKeyName.FIXED_CHARGE_RATE, - MetaKeyName.LCOE_FCR, 'cf_mean'] + 'gcf_adjustment', 'fixed_charge_rate', + 'lcoe_fcr', 'cf_mean'] incompatible = [] output_request = compatible + incompatible @@ -216,10 +212,8 @@ def test_nrwal_constant_eq_output_request(): site_data = os.path.join(td, 'example_offshore_data.csv') offshore_config = os.path.join(td, 'offshore.json') onshore_config = os.path.join(td, 'onshore.json') - sam_configs = {'onshore': onshore_config, - MetaKeyName.OFFSHORE: offshore_config} - nrwal_configs = {MetaKeyName.OFFSHORE: - os.path.join(td, 'nrwal_offshore.yaml')} + sam_configs = {'onshore': onshore_config, "offshore": offshore_config} + nrwal_configs = {"offshore": os.path.join(td, 'nrwal_offshore.yaml')} with Outputs(gen_fpath, 'a') as f: f.time_index = pd_date_range('20100101', '20110101', @@ -227,7 +221,7 @@ def test_nrwal_constant_eq_output_request(): f._add_dset('cf_profile', np.random.random(f.shape), np.uint32, attrs={'scale_factor': 1000}, chunks=(None, 10)) - f._add_dset(MetaKeyName.FIXED_CHARGE_RATE, + f._add_dset('fixed_charge_rate', 0.09 * np.ones(f.shape[1], dtype=np.float32), np.float32, attrs={'scale_factor': 1}, chunks=None) @@ -261,9 +255,8 @@ def test_nrwal_cli(runner, clear_loggers): offshore_config = os.path.join(td, 'offshore.json') onshore_config = os.path.join(td, 'onshore.json') sam_configs = {'onshore': onshore_config, - MetaKeyName.OFFSHORE: offshore_config} - nrwal_configs = {MetaKeyName.OFFSHORE: - os.path.join(td, 'nrwal_offshore.yaml')} + "offshore": offshore_config} + nrwal_configs = {"offshore": os.path.join(td, 'nrwal_offshore.yaml')} with Outputs(gen_fpath, 'a') as f: f.time_index = pd_date_range('20100101', '20110101', @@ -271,7 +264,7 @@ def test_nrwal_cli(runner, clear_loggers): f._add_dset('cf_profile', np.random.random(f.shape), np.uint32, attrs={'scale_factor': 1000}, chunks=(None, 10)) - f._add_dset(MetaKeyName.FIXED_CHARGE_RATE, + f._add_dset('fixed_charge_rate', 0.09 * np.ones(f.shape[1], dtype=np.float32), np.float32, attrs={'scale_factor': 1}, chunks=None) @@ -280,15 +273,14 @@ def test_nrwal_cli(runner, clear_loggers): original_dsets = [d for d in f.dsets if d not in ('meta', 'time_index')] meta_raw = f.meta - lcoe_raw = f[MetaKeyName.LCOE_FCR] + lcoe_raw = f['lcoe_fcr'] cf_mean_raw = f['cf_mean'] cf_profile_raw = f['cf_profile'] mask = meta_raw.offshore == 1 - output_request = [MetaKeyName.FIXED_CHARGE_RATE, 'depth', - 'total_losses', + output_request = ['fixed_charge_rate', 'depth', 'total_losses', 'array', 'export', 'gcf_adjustment', - MetaKeyName.LCOE_FCR, 'cf_mean', ] + 'lcoe_fcr', 'cf_mean', ] config = { "execution_control": { @@ -317,14 +309,14 @@ def test_nrwal_cli(runner, clear_loggers): with Outputs(gen_fpath, 'r') as f: meta_new = f.meta - lcoe_new = f[MetaKeyName.LCOE_FCR] + lcoe_new = f['lcoe_fcr'] losses = f['total_losses'] gcf_adjustment = f['gcf_adjustment'] assert np.allclose(cf_mean_raw, f['cf_mean_raw']) assert np.allclose(cf_profile_raw, f['cf_profile_raw']) cf_mean_new = f['cf_mean'] cf_profile_new = f['cf_profile'] - fcr = f[MetaKeyName.FIXED_CHARGE_RATE] + fcr = f['fixed_charge_rate'] depth = f['depth'] for key in [d for d in original_dsets if d in f]: @@ -352,7 +344,7 @@ def test_nrwal_cli(runner, clear_loggers): # make sure the second offshore compute gives same results as first with Outputs(gen_fpath, 'r') as f: - assert np.allclose(lcoe_new, f[MetaKeyName.LCOE_FCR]) + assert np.allclose(lcoe_new, f['lcoe_fcr']) assert np.allclose(cf_mean_new, f['cf_mean']) assert np.allclose(cf_profile_new, f['cf_profile']) assert np.allclose(cf_mean_raw, f['cf_mean_raw']) @@ -406,9 +398,8 @@ def test_nrwal_cli_csv(runner, clear_loggers): offshore_config = os.path.join(td, 'offshore.json') onshore_config = os.path.join(td, 'onshore.json') sam_configs = {'onshore': onshore_config, - MetaKeyName.OFFSHORE: offshore_config} - nrwal_configs = {MetaKeyName.OFFSHORE: - os.path.join(td, 'nrwal_offshore.yaml')} + "offshore": offshore_config} + nrwal_configs = {"offshore": os.path.join(td, 'nrwal_offshore.yaml')} with Outputs(gen_fpath, 'a') as f: f.time_index = pd_date_range('20100101', '20110101', @@ -419,14 +410,14 @@ def test_nrwal_cli_csv(runner, clear_loggers): f._add_dset('cf_mean_raw', np.random.random(f.shape[1]), np.uint32, attrs={'scale_factor': 1000}, chunks=None) - f._add_dset(MetaKeyName.FIXED_CHARGE_RATE, + f._add_dset('fixed_charge_rate', 0.09 * np.ones(f.shape[1], dtype=np.float32), np.float32, attrs={'scale_factor': 1}, chunks=None) - output_request = [MetaKeyName.FIXED_CHARGE_RATE, 'depth', + output_request = ['fixed_charge_rate', 'depth', 'total_losses', 'array', 'export', 'gcf_adjustment', - MetaKeyName.LCOE_FCR, 'cf_mean', ] + 'lcoe_fcr', 'cf_mean', ] config = { "execution_control": { diff --git a/tests/test_sam.py b/tests/test_sam.py index 3cad13a41..7f94f3c1e 100644 --- a/tests/test_sam.py +++ b/tests/test_sam.py @@ -21,7 +21,7 @@ ) from reV.SAM.generation import PvWattsv5, PvWattsv7, PvWattsv8 from reV.SAM.version_checker import PySamVersionChecker -from reV.utilities import MetaKeyName +from reV.utilities import ResourceMetaField from reV.utilities.exceptions import InputError, PySAMVersionWarning @@ -97,7 +97,7 @@ def test_PV_lat_tilt(res, site_index): # get SAM inputs from project_points based on the current site site = res_df.name config, inputs = pp[site] - inputs['tilt'] = MetaKeyName.LATITUDE + inputs['tilt'] = ResourceMetaField.LATITUDE # iterate through requested sites. with warnings.catch_warnings(): warnings.simplefilter("ignore") @@ -107,7 +107,7 @@ def test_PV_lat_tilt(res, site_index): break pass - assert sim.sam_sys_inputs['tilt'] == meta[MetaKeyName.LATITUDE] + assert sim.sam_sys_inputs['tilt'] == meta[ResourceMetaField.LATITUDE] @pytest.mark.parametrize('dt', ('1h', '30min', '5min')) diff --git a/tests/test_supply_curve_tech_mapping.py b/tests/test_supply_curve_tech_mapping.py index c795142a5..150473cd7 100644 --- a/tests/test_supply_curve_tech_mapping.py +++ b/tests/test_supply_curve_tech_mapping.py @@ -12,10 +12,9 @@ import pytest from reV import TESTDATADIR -from reV.handlers.exclusions import ExclusionLayers +from reV.handlers.exclusions import ExclusionLayers, LATITUDE, LONGITUDE from reV.handlers.outputs import Outputs from reV.supply_curve.tech_mapping import TechMapping -from reV.utilities import MetaKeyName EXCL = os.path.join(TESTDATADIR, 'ri_exclusions/ri_exclusions.h5') RES = os.path.join(TESTDATADIR, 'nsrdb/ri_100_nsrdb_2012.h5') @@ -45,8 +44,8 @@ def plot_tech_mapping(dist_margin=1.05): import matplotlib.pyplot as plt with h5py.File(EXCL, 'r') as f: - lats = f[MetaKeyName.LATITUDE][...].flatten() - lons = f[MetaKeyName.LONGITUDE][...].flatten() + lats = f[LATITUDE][...].flatten() + lons = f[LONGITUDE][...].flatten() ind_truth = f[TM_DSET][...].flatten() with Outputs(GEN) as fgen: @@ -55,8 +54,8 @@ def plot_tech_mapping(dist_margin=1.05): ind_test = TechMapping.run(EXCL, RES, dset=None, max_workers=2, dist_margin=dist_margin) - df = pd.DataFrame({MetaKeyName.LATITUDE: lats, - MetaKeyName.LONGITUDE: lons, + df = pd.DataFrame({LATITUDE: lats, + LONGITUDE: lons, TM_DSET: ind_truth, 'test': ind_test.flatten()}) @@ -67,31 +66,31 @@ def plot_tech_mapping(dist_margin=1.05): for i, ind in enumerate(df[TM_DSET].unique()): if ind != -1: mask = df[TM_DSET] == ind - axs.scatter(df.loc[mask, MetaKeyName.LONGITUDE], - df.loc[mask, MetaKeyName.LATITUDE], + axs.scatter(df.loc[mask, LONGITUDE], + df.loc[mask, LATITUDE], c=colors[i], s=0.001) elif ind == -1: mask = df[TM_DSET] == ind - axs.scatter(df.loc[mask, MetaKeyName.LONGITUDE], - df.loc[mask, MetaKeyName.LATITUDE], + axs.scatter(df.loc[mask, LONGITUDE], + df.loc[mask, LATITUDE], c='r', s=0.001) for ind in df[TM_DSET].unique(): if ind != -1: - axs.scatter(gen_meta.loc[ind, MetaKeyName.LONGITUDE], - gen_meta.loc[ind, MetaKeyName.LATITUDE], + axs.scatter(gen_meta.loc[ind, LONGITUDE], + gen_meta.loc[ind, LATITUDE], c='w', s=1) for ind in df['test'].unique(): if ind != -1: - axs.scatter(gen_meta.loc[ind, MetaKeyName.LONGITUDE], - gen_meta.loc[ind, MetaKeyName.LATITUDE], + axs.scatter(gen_meta.loc[ind, LONGITUDE], + gen_meta.loc[ind, LATITUDE], c='r', s=1) mask = df[TM_DSET].values != df['test'] - axs.scatter(df.loc[mask, MetaKeyName.LONGITUDE], - df.loc[mask, MetaKeyName.LATITUDE], + axs.scatter(df.loc[mask, LONGITUDE], + df.loc[mask, LATITUDE], c='k', s=1) axs.axis('equal') From 2a45169d1f7d293ce0f7c4b0d8091eb6b39a369a Mon Sep 17 00:00:00 2001 From: ppinchuk Date: Mon, 20 May 2024 15:30:14 -0600 Subject: [PATCH 24/61] Linter fixes --- reV/SAM/econ.py | 11 +++++----- reV/SAM/generation.py | 13 +++++------ reV/bespoke/place_turbines.py | 3 ++- reV/config/project_points.py | 1 - reV/generation/base.py | 5 +++-- reV/qa_qc/qa_qc.py | 7 +++--- reV/rep_profiles/rep_profiles.py | 5 +++-- reV/supply_curve/points.py | 12 +++++----- reV/utilities/__init__.py | 1 + tests/test_bespoke.py | 8 +++---- tests/test_econ_lcoe.py | 3 +-- tests/test_econ_of_scale.py | 29 +++++++++++-------------- tests/test_gen_pv.py | 12 ++++------ tests/test_supply_curve_tech_mapping.py | 2 +- 14 files changed, 52 insertions(+), 60 deletions(-) diff --git a/reV/SAM/econ.py b/reV/SAM/econ.py index 04a7a4fc2..cd834689a 100644 --- a/reV/SAM/econ.py +++ b/reV/SAM/econ.py @@ -502,13 +502,12 @@ def _windbos(inputs): """ outputs = {} - if (inputs is not None and - 'total_installed_cost' in inputs and - isinstance(inputs['total_installed_cost'], str) and - inputs['total_installed_cost'].lower() == 'windbos'): + if (inputs is not None + and 'total_installed_cost' in inputs + and isinstance(inputs['total_installed_cost'], str) + and inputs['total_installed_cost'].lower() == 'windbos'): wb = WindBos(inputs) - inputs['total_installed_cost'] = \ - wb.total_installed_cost + inputs['total_installed_cost'] = wb.total_installed_cost outputs = wb.output return inputs, outputs diff --git a/reV/SAM/generation.py b/reV/SAM/generation.py index 56d74e056..d2c45b1b5 100644 --- a/reV/SAM/generation.py +++ b/reV/SAM/generation.py @@ -313,8 +313,7 @@ def cf_profile(self): 1D numpy array of capacity factor profile. Datatype is float32 and array length is 8760*time_interval. """ - return (self.gen_profile() / - self.sam_sys_inputs['system_capacity']) + return self.gen_profile() / self.sam_sys_inputs['system_capacity'] def annual_energy(self): """Get annual energy generation value in kWh from SAM. @@ -796,8 +795,8 @@ def set_resource_data(self, resource, meta): if var != "time_index": # ensure that resource array length is multiple of 8760 arr = self.ensure_res_len(arr, time_index) - n_roll = int(self._meta[ResourceMetaField.TIMEZONE] * - self.time_interval) + n_roll = int(self._meta[ResourceMetaField.TIMEZONE] + * self.time_interval) arr = np.roll(arr, n_roll) if var in irrad_vars and np.min(arr) < 0: @@ -1102,8 +1101,7 @@ def cf_profile(self): Datatype is float32 and array length is 8760*time_interval. PV CF is calculated as AC power / DC nameplate. """ - return (self.gen_profile() / - self.sam_sys_inputs['system_capacity']) + return self.gen_profile() / self.sam_sys_inputs['system_capacity'] def cf_profile_ac(self): """Get hourly AC capacity factor (frac) profile in local timezone. @@ -1324,8 +1322,7 @@ def cf_profile(self): 1D numpy array of capacity factor profile. Datatype is float32 and array length is 8760*time_interval. """ - x = np.abs(self.gen_profile() / - self.sam_sys_inputs['system_capacity']) + x = np.abs(self.gen_profile() / self.sam_sys_inputs['system_capacity']) return x @staticmethod diff --git a/reV/bespoke/place_turbines.py b/reV/bespoke/place_turbines.py index 3be797231..b2bd2ac83 100644 --- a/reV/bespoke/place_turbines.py +++ b/reV/bespoke/place_turbines.py @@ -394,7 +394,8 @@ def capacity(self): def convex_hull(self): """This is the convex hull of the turbine locations""" turbines = MultiPoint([Point(x, y) - for x,y in zip(self.turbine_x, self.turbine_y)]) + for x, y in zip(self.turbine_x, + self.turbine_y)]) return turbines.convex_hull @property diff --git a/reV/config/project_points.py b/reV/config/project_points.py index 1a5d5df53..ea117d773 100644 --- a/reV/config/project_points.py +++ b/reV/config/project_points.py @@ -20,7 +20,6 @@ from reV.config.curtailment import Curtailment from reV.config.sam_config import SAMConfig -from reV.utilities import MetaKeyName from reV.utilities.exceptions import ConfigError, ConfigWarning from reV.utilities import SiteDataField diff --git a/reV/generation/base.py b/reV/generation/base.py index a7df97743..4461aef48 100644 --- a/reV/generation/base.py +++ b/reV/generation/base.py @@ -820,8 +820,9 @@ def _parse_site_data(self, inp): "dataframe, but received: {}".format(inp) ) - if (ResourceMetaField.GID not in site_data - and site_data.index.name != ResourceMetaField.GID): + gid_not_in_site_data = ResourceMetaField.GID not in site_data + index_name_not_gid = site_data.index.name != ResourceMetaField.GID + if gid_not_in_site_data and index_name_not_gid: # require gid as column label or index raise KeyError('Site data input must have ' f'{ResourceMetaField.GID} column to match ' diff --git a/reV/qa_qc/qa_qc.py b/reV/qa_qc/qa_qc.py index 4facc053b..fde799a08 100644 --- a/reV/qa_qc/qa_qc.py +++ b/reV/qa_qc/qa_qc.py @@ -105,9 +105,10 @@ def create_scatter_plots( if file.endswith(".csv"): summary_csv = os.path.join(self.out_dir, file) summary = pd.read_csv(summary_csv) - if (MetaKeyName.GID in summary - and MetaKeyName.LATITUDE in summary - and MetaKeyName.LONGITUDE in summary): + has_right_cols = (MetaKeyName.GID in summary + and MetaKeyName.LATITUDE in summary + and MetaKeyName.LONGITUDE in summary) + if has_right_cols: self._scatter_plot(summary_csv, self.out_dir, plot_type=plot_type, cmap=cmap, **kwargs) diff --git a/reV/rep_profiles/rep_profiles.py b/reV/rep_profiles/rep_profiles.py index 2f071b4aa..b687208a8 100644 --- a/reV/rep_profiles/rep_profiles.py +++ b/reV/rep_profiles/rep_profiles.py @@ -478,8 +478,9 @@ def _run_rep_methods(self): """Run the representative profile methods to find the meanoid/medianoid profile and find the profiles most similar.""" - if (self.weights is not None and - (len(self.weights) != self.source_profiles.shape[1])): + weights_not_none = self.weights is not None + shapes_mismatch = len(self.weights) != self.source_profiles.shape[1] + if weights_not_none and shapes_mismatch: e = ('Weights column "{}" resulted in {} weight scalars ' 'which doesnt match gid column which yields ' 'profiles with shape {}.' diff --git a/reV/supply_curve/points.py b/reV/supply_curve/points.py index bd1f9768b..4b47c30ef 100644 --- a/reV/supply_curve/points.py +++ b/reV/supply_curve/points.py @@ -1127,8 +1127,8 @@ def h5(self): def country(self): """Get the SC point country based on the resource meta data.""" country = None - if (ResourceMetaField.COUNTRY in self.h5.meta - and self.county is not None): + county_not_none = self.county is not None + if ResourceMetaField.COUNTRY in self.h5.meta and county_not_none: # make sure country and county are coincident counties = self.h5.meta.loc[self.h5_gid_set, ResourceMetaField.COUNTY].values @@ -1189,8 +1189,8 @@ def elevation(self): def timezone(self): """Get the SC point timezone based on the resource meta data.""" timezone = None - if (ResourceMetaField.TIMEZONE in self.h5.meta - and self.county is not None): + county_not_none = self.county is not None + if ResourceMetaField.TIMEZONE in self.h5.meta and county_not_none: # make sure timezone flag and county are coincident counties = self.h5.meta.loc[self.h5_gid_set, ResourceMetaField.COUNTY].values @@ -1211,8 +1211,8 @@ def offshore(self): """Get the SC point offshore flag based on the resource meta data (if offshore column is present).""" offshore = None - if (ResourceMetaField.OFFSHORE in self.h5.meta - and self.county is not None): + county_not_none = self.county is not None + if ResourceMetaField.OFFSHORE in self.h5.meta and county_not_none: # make sure offshore flag and county are coincident counties = self.h5.meta.loc[self.h5_gid_set, ResourceMetaField.COUNTY].values diff --git a/reV/utilities/__init__.py b/reV/utilities/__init__.py index f15a98f2a..630f30b03 100644 --- a/reV/utilities/__init__.py +++ b/reV/utilities/__init__.py @@ -9,6 +9,7 @@ class SiteDataField(str, Enum): + """An enumerated map to site data column names.""" GID = "gid" CONFIG = "config" diff --git a/tests/test_bespoke.py b/tests/test_bespoke.py index c8ed7d6ee..cee89b786 100644 --- a/tests/test_bespoke.py +++ b/tests/test_bespoke.py @@ -696,8 +696,8 @@ def test_collect_bespoke(): with Resource(h5_file) as fout: meta = fout.meta - assert all(meta[MetaKeyName.GID].values == - sorted(meta[MetaKeyName.GID].values)) + assert all(meta[MetaKeyName.GID].values + == sorted(meta[MetaKeyName.GID].values)) ti = fout.time_index assert len(ti) == 8760 assert "time_index-2012" in fout @@ -799,8 +799,8 @@ def test_bespoke_supply_curve(): test_ind = np.where(sc_full[MetaKeyName.SC_GID] == sc_gid)[0] assert len(test_ind) == 1 test_row = sc_full.iloc[test_ind] - assert (test_row['total_lcoe'].values[0] > - inp_row[MetaKeyName.MEAN_LCOE]) + assert (test_row['total_lcoe'].values[0] + > inp_row[MetaKeyName.MEAN_LCOE]) fpath_baseline = os.path.join(TESTDATADIR, "sc_out/sc_full_lc.csv") sc_baseline = pd.read_csv(fpath_baseline) diff --git a/tests/test_econ_lcoe.py b/tests/test_econ_lcoe.py index 722e624ab..4a84023de 100644 --- a/tests/test_econ_lcoe.py +++ b/tests/test_econ_lcoe.py @@ -153,8 +153,7 @@ def test_append_multi_node(node): TESTDATADIR, 'config/nsrdb_sitedata_atb2020_capcostmults_subset.csv') econ = Econ(points, sam_files, cf_file, - output_request=('lcoe_fcr', - 'capital_cost'), + output_request=('lcoe_fcr', 'capital_cost'), sites_per_worker=25, append=True, site_data=site_data) econ.run(max_workers=1) diff --git a/tests/test_econ_of_scale.py b/tests/test_econ_of_scale.py index 4f163d14d..fcdb77e6d 100644 --- a/tests/test_econ_of_scale.py +++ b/tests/test_econ_of_scale.py @@ -148,10 +148,9 @@ def test_econ_of_scale_baseline(): with Resource(GEN) as res: cf = res["cf_mean-means"] - lcoe = (1000 * (data['fixed_charge_rate'] * - data['capital_cost'] + lcoe = (1000 * (data['fixed_charge_rate'] * data['capital_cost'] + data['fixed_operating_cost']) - / (cf * data['system_capacity'] * 8760)) + / (cf * data['system_capacity'] * 8760)) with h5py.File(gen_temp, "a") as res: res["lcoe_fcr-means"][...] = lcoe @@ -246,18 +245,16 @@ def test_sc_agg_econ_scale(): # of the mean lcoe values from baseline assert np.allclose(sc_df[MetaKeyName.RAW_LCOE], base_df[MetaKeyName.MEAN_LCOE]) - assert all(sc_df[MetaKeyName.MEAN_LCOE] < - base_df[MetaKeyName.MEAN_LCOE]) + assert all(sc_df[MetaKeyName.MEAN_LCOE] + < base_df[MetaKeyName.MEAN_LCOE]) - aep = ((sc_df['mean_fixed_charge_rate'] * - sc_df['mean_capital_cost'] - + sc_df['mean_fixed_operating_cost']) / - sc_df[MetaKeyName.RAW_LCOE]) + aep = ((sc_df['mean_fixed_charge_rate'] * sc_df['mean_capital_cost'] + + sc_df['mean_fixed_operating_cost']) + / sc_df[MetaKeyName.RAW_LCOE]) - true_raw_lcoe = ((data['fixed_charge_rate'] * - data['capital_cost'] + true_raw_lcoe = ((data['fixed_charge_rate'] * data['capital_cost'] + data['fixed_operating_cost']) - / aep + data['variable_operating_cost']) + / aep + data['variable_operating_cost']) eval_inputs = {k: sc_df[k].values.flatten() for k in sc_df.columns} # pylint: disable=eval-used @@ -278,11 +275,11 @@ def test_sc_agg_econ_scale(): assert all(sc_df[MetaKeyName.MEAN_LCOE].diff()[1:] < 0) for i in sc_df.index.values: if sc_df.loc[i, 'scalars'] < 1: - assert (sc_df.loc[i, MetaKeyName.MEAN_LCOE] < - sc_df.loc[i, MetaKeyName.RAW_LCOE]) + assert (sc_df.loc[i, MetaKeyName.MEAN_LCOE] + < sc_df.loc[i, MetaKeyName.RAW_LCOE]) else: - assert (sc_df.loc[i, MetaKeyName.MEAN_LCOE] >= - sc_df.loc[i, MetaKeyName.RAW_LCOE]) + assert (sc_df.loc[i, MetaKeyName.MEAN_LCOE] + >= sc_df.loc[i, MetaKeyName.RAW_LCOE]) def execute_pytest(capture="all", flags="-rapP"): diff --git a/tests/test_gen_pv.py b/tests/test_gen_pv.py index b9d358417..36699fab7 100644 --- a/tests/test_gen_pv.py +++ b/tests/test_gen_pv.py @@ -574,10 +574,8 @@ def test_gen_pv_site_data(): site_data=site_data) test.run(max_workers=1) - assert all(test.out['cf_mean'][0:2] > - baseline.out['cf_mean'][0:2]) - assert np.allclose(test.out['cf_mean'][2:], - baseline.out['cf_mean'][2:]) + assert all(test.out['cf_mean'][0:2] > baseline.out['cf_mean'][0:2]) + assert np.allclose(test.out['cf_mean'][2:], baseline.out['cf_mean'][2:]) assert np.allclose(test.out['losses'][0:2], np.ones(2)) assert np.allclose(test.out['losses'][2:], 14.07566 * np.ones(3)) @@ -803,14 +801,12 @@ def test_irrad_bias_correct(): bias_correct=bc_df) gen.run(max_workers=1) - assert (gen_base.out['cf_mean'][0] == - gen.out['cf_mean'][0]).all() + assert (gen_base.out['cf_mean'][0] == gen.out['cf_mean'][0]).all() assert (gen_base.out['ghi_mean'][0] == gen.out['ghi_mean'][0]).all() assert np.allclose(gen_base.out['cf_profile'][:, 0], gen.out['cf_profile'][:, 0]) - assert (gen_base.out['cf_mean'][1:] < - gen.out['cf_mean'][1:]).all() + assert (gen_base.out['cf_mean'][1:] < gen.out['cf_mean'][1:]).all() assert (gen_base.out['ghi_mean'][1:] < gen.out['ghi_mean'][1:]).all() mask = (gen_base.out['cf_profile'][:, 1:] <= gen.out['cf_profile'][:, 1:]) assert (mask.sum() / mask.size) > 0.99 diff --git a/tests/test_supply_curve_tech_mapping.py b/tests/test_supply_curve_tech_mapping.py index 3cf4c745d..c100aac77 100644 --- a/tests/test_supply_curve_tech_mapping.py +++ b/tests/test_supply_curve_tech_mapping.py @@ -15,7 +15,7 @@ from reV.handlers.exclusions import ExclusionLayers, LATITUDE, LONGITUDE from reV.handlers.outputs import Outputs from reV.supply_curve.tech_mapping import TechMapping -from reV.utilities import MetaKeyName + EXCL = os.path.join(TESTDATADIR, 'ri_exclusions/ri_exclusions.h5') RES = os.path.join(TESTDATADIR, 'nsrdb/ri_100_nsrdb_2012.h5') From bb53ab1a83e600a8cd183d3912945037a50b24b3 Mon Sep 17 00:00:00 2001 From: ppinchuk Date: Mon, 20 May 2024 15:40:12 -0600 Subject: [PATCH 25/61] Fix tests --- reV/econ/economies_of_scale.py | 1 + reV/utilities/__init__.py | 1 + tests/test_curtailment.py | 3 +++ tests/test_econ_windbos.py | 2 +- tests/test_gen_pv.py | 2 +- tests/test_gen_time_scale.py | 2 +- tests/test_gen_wave.py | 2 +- tests/test_nrwal.py | 4 ++-- 8 files changed, 11 insertions(+), 6 deletions(-) diff --git a/reV/econ/economies_of_scale.py b/reV/econ/economies_of_scale.py index 697366515..f5f385b07 100644 --- a/reV/econ/economies_of_scale.py +++ b/reV/econ/economies_of_scale.py @@ -7,6 +7,7 @@ import logging import re +import numpy as np import pandas as pd from rex.utilities.utilities import check_eval_str diff --git a/reV/utilities/__init__.py b/reV/utilities/__init__.py index 630f30b03..882f5c795 100644 --- a/reV/utilities/__init__.py +++ b/reV/utilities/__init__.py @@ -64,6 +64,7 @@ class MetaKeyName(str, Enum): SC_ROW_IND = 'sc_row_ind' SC_COL_IND = 'sc_col_ind' CAPACITY_AC = 'capacity_ac' + CAPITAL_COST = 'capital_cost' SC_POINT_CAPITAL_COST = 'sc_point_capital_cost' SC_POINT_FIXED_OPERATING_COST = 'sc_point_fixed_operating_cost' SC_POINT_ANNUAL_ENERGY = 'sc_point_annual_energy' diff --git a/tests/test_curtailment.py b/tests/test_curtailment.py index 1a578d220..57343fb06 100644 --- a/tests/test_curtailment.py +++ b/tests/test_curtailment.py @@ -285,6 +285,9 @@ def test_eqn_curtailment(plot=False): nc_res = non_curtailed_res[0] c_mask = (c_res.windspeed == 0) & (nc_res.windspeed > 0) + temperature = nc_res['temperature'].values + wind_speed = nc_res['windspeed'].values + eval_mask = eval(c_eqn) # All curtailed windspeeds should satisfy the eqn eval but maybe not the diff --git a/tests/test_econ_windbos.py b/tests/test_econ_windbos.py index 93aea522e..6f758213a 100644 --- a/tests/test_econ_windbos.py +++ b/tests/test_econ_windbos.py @@ -247,7 +247,7 @@ def test_run_gen_econ(points=slice(0, 10), year=2012, max_workers=1): points, sam_files, res_file, - output_request=('cf_mean',), + output_request=('cf_mean', 'cf_profile'), sites_per_worker=3, ) gen.run(max_workers=max_workers, out_fpath=cf_file) diff --git a/tests/test_gen_pv.py b/tests/test_gen_pv.py index 36699fab7..e4a0b216f 100644 --- a/tests/test_gen_pv.py +++ b/tests/test_gen_pv.py @@ -244,7 +244,7 @@ def test_multi_file_nsrdb_2018(model): res_file = TESTDATADIR + "/nsrdb/nsrdb_*{}.h5".format(2018) # run reV 2.0 generation gen = Gen(model, points, sam_files, res_file, - output_request=('cf_mean', ), + output_request=('cf_mean', 'cf_profile'), sites_per_worker=3) gen.run(max_workers=max_workers) diff --git a/tests/test_gen_time_scale.py b/tests/test_gen_time_scale.py index 13fb0c01f..c9be4e984 100644 --- a/tests/test_gen_time_scale.py +++ b/tests/test_gen_time_scale.py @@ -32,7 +32,7 @@ def test_time_index_step(): # run reV 2.0 generation gen = Gen('pvwattsv5', slice(0, None), sam_input, res_file, - output_request=('cf_mean', ), + output_request=('cf_mean', 'cf_profile'), sites_per_worker=100) gen.run(max_workers=1) gen_outs = gen.out['cf_profile'].astype(np.int32) diff --git a/tests/test_gen_wave.py b/tests/test_gen_wave.py index 54d816f79..8647f8476 100644 --- a/tests/test_gen_wave.py +++ b/tests/test_gen_wave.py @@ -38,7 +38,7 @@ def test_mhkwave(): sam_files = TESTDATADIR + '/SAM/mhkwave_default.json' res_file = TESTDATADIR + '/wave/ri_wave_2010.h5' points = slice(0, 100) - output_request = ('cf_mean', ) + output_request = ('cf_mean', 'cf_profile') test = Gen('mhkwave', points, sam_files, res_file, sites_per_worker=3, output_request=output_request) diff --git a/tests/test_nrwal.py b/tests/test_nrwal.py index 6bb06c6db..612399569 100644 --- a/tests/test_nrwal.py +++ b/tests/test_nrwal.py @@ -70,7 +70,7 @@ def test_nrwal(): mask = meta_raw.offshore == 1 output_request = ['fixed_charge_rate', 'depth', - 'total_losses', + 'total_losses', 'cf_profile', 'array', 'export', 'gcf_adjustment', 'lcoe_fcr', 'cf_mean', ] @@ -347,7 +347,7 @@ def test_nrwal_cli(runner, clear_loggers): output_request = ['fixed_charge_rate', 'depth', 'total_losses', 'array', 'export', 'gcf_adjustment', - 'lcoe_fcr', 'cf_mean', ] + 'lcoe_fcr', 'cf_mean', 'cf_profile'] config = { "execution_control": { From 27e4e34318cfba12babb29daa8d92163d4bbcfb2 Mon Sep 17 00:00:00 2001 From: ppinchuk Date: Mon, 20 May 2024 23:03:04 -0600 Subject: [PATCH 26/61] Fix tests --- reV/econ/economies_of_scale.py | 1 + reV/rep_profiles/rep_profiles.py | 7 ++++--- reV/utilities/__init__.py | 2 ++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/reV/econ/economies_of_scale.py b/reV/econ/economies_of_scale.py index f5f385b07..022b262eb 100644 --- a/reV/econ/economies_of_scale.py +++ b/reV/econ/economies_of_scale.py @@ -7,6 +7,7 @@ import logging import re +# pylint: disable=unused-import import numpy as np import pandas as pd from rex.utilities.utilities import check_eval_str diff --git a/reV/rep_profiles/rep_profiles.py b/reV/rep_profiles/rep_profiles.py index b687208a8..234174e39 100644 --- a/reV/rep_profiles/rep_profiles.py +++ b/reV/rep_profiles/rep_profiles.py @@ -478,9 +478,10 @@ def _run_rep_methods(self): """Run the representative profile methods to find the meanoid/medianoid profile and find the profiles most similar.""" - weights_not_none = self.weights is not None - shapes_mismatch = len(self.weights) != self.source_profiles.shape[1] - if weights_not_none and shapes_mismatch: + num_profiles = self.source_profiles.shape[1] + bad_data_shape = (self.weights is not None + and len(self.weights) != num_profiles) + if bad_data_shape: e = ('Weights column "{}" resulted in {} weight scalars ' 'which doesnt match gid column which yields ' 'profiles with shape {}.' diff --git a/reV/utilities/__init__.py b/reV/utilities/__init__.py index 882f5c795..4dc5d63d3 100644 --- a/reV/utilities/__init__.py +++ b/reV/utilities/__init__.py @@ -65,6 +65,8 @@ class MetaKeyName(str, Enum): SC_COL_IND = 'sc_col_ind' CAPACITY_AC = 'capacity_ac' CAPITAL_COST = 'capital_cost' + FIXED_OPERATING_COST = 'fixed_operating_cost' + FIXED_CHARGE_RATE = 'fixed_charge_rate' SC_POINT_CAPITAL_COST = 'sc_point_capital_cost' SC_POINT_FIXED_OPERATING_COST = 'sc_point_fixed_operating_cost' SC_POINT_ANNUAL_ENERGY = 'sc_point_annual_energy' From bca098b7dfc72d620e0bae8398a55a7d55a1b519 Mon Sep 17 00:00:00 2001 From: ppinchuk Date: Tue, 21 May 2024 10:07:47 -0600 Subject: [PATCH 27/61] Add `variable_operating_cost` enum value --- reV/utilities/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/reV/utilities/__init__.py b/reV/utilities/__init__.py index 4dc5d63d3..a18fd96d7 100644 --- a/reV/utilities/__init__.py +++ b/reV/utilities/__init__.py @@ -66,6 +66,7 @@ class MetaKeyName(str, Enum): CAPACITY_AC = 'capacity_ac' CAPITAL_COST = 'capital_cost' FIXED_OPERATING_COST = 'fixed_operating_cost' + VARIABLE_OPERATING_COST = 'variable_operating_cost' FIXED_CHARGE_RATE = 'fixed_charge_rate' SC_POINT_CAPITAL_COST = 'sc_point_capital_cost' SC_POINT_FIXED_OPERATING_COST = 'sc_point_fixed_operating_cost' From a55a768d4dc1b974e89a3a3cc999ec6ba8cad25e Mon Sep 17 00:00:00 2001 From: ppinchuk Date: Tue, 21 May 2024 10:56:30 -0600 Subject: [PATCH 28/61] Fix test --- tests/test_supply_curve_sc_aggregation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_supply_curve_sc_aggregation.py b/tests/test_supply_curve_sc_aggregation.py index dba4e0af4..3b47a9579 100644 --- a/tests/test_supply_curve_sc_aggregation.py +++ b/tests/test_supply_curve_sc_aggregation.py @@ -515,11 +515,11 @@ def test_recalc_lcoe(cap_cost_scale): cc_dset = MetaKeyName.SC_POINT_CAPITAL_COST else: cc_dset = MetaKeyName.SCALED_SC_POINT_CAPITAL_COST - lcoe = lcoe_fcr(summary[MetaKeyName.MEAN_FIXED_CHARGE_RATE], + lcoe = lcoe_fcr(summary['mean_fixed_charge_rate'], summary[cc_dset], summary[MetaKeyName.SC_POINT_FIXED_OPERATING_COST], summary[MetaKeyName.SC_POINT_ANNUAL_ENERGY], - summary[MetaKeyName.MEAN_VARIABLE_OPERATING_COST]) + summary['mean_variable_operating_cost']) assert np.allclose(lcoe, summary[MetaKeyName.MEAN_LCOE]) From 5c05c47605a486f77d5550a8c9adb874621d5684 Mon Sep 17 00:00:00 2001 From: ppinchuk Date: Tue, 21 May 2024 13:29:08 -0600 Subject: [PATCH 29/61] Fix reVX test (hopefully) --- reV/supply_curve/aggregation.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/reV/supply_curve/aggregation.py b/reV/supply_curve/aggregation.py index 23fd0e388..bd09048a2 100644 --- a/reV/supply_curve/aggregation.py +++ b/reV/supply_curve/aggregation.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- """reV aggregation framework.""" -import contextlib import logging import os from abc import ABC, abstractmethod @@ -948,9 +947,11 @@ def save_agg_to_h5(h5_fpath, out_fpath, aggregation): """ agg_out = aggregation.copy() meta = agg_out.pop("meta").reset_index() - with contextlib.suppress(ValueError, TypeError): - for c in meta.columns: + for c in meta.columns: + try: meta[c] = pd.to_numeric(meta[c]) + except (ValueError, TypeError): + pass dsets = [] shapes = {} From 4866bcad1de03a1c8876a1fd43898d32db3dbebb Mon Sep 17 00:00:00 2001 From: "Brandon N. Benton" Date: Mon, 27 May 2024 15:55:47 -0600 Subject: [PATCH 30/61] MetaKeyName -> SupplyCurveField. more descriptive and match other enum conventions. --- reV/SAM/generation.py | 2 +- reV/bespoke/bespoke.py | 62 ++++++----- reV/econ/econ.py | 2 +- reV/generation/base.py | 2 +- reV/generation/generation.py | 2 +- reV/handlers/outputs.py | 4 +- reV/hybrids/hybrids.py | 20 ++-- reV/nrwal/nrwal.py | 4 +- reV/qa_qc/cli_qa_qc.py | 4 +- reV/qa_qc/qa_qc.py | 14 +-- reV/qa_qc/summary.py | 48 ++++----- reV/rep_profiles/rep_profiles.py | 22 ++-- reV/supply_curve/aggregation.py | 18 ++-- reV/supply_curve/competitive_wind_farms.py | 38 +++---- reV/supply_curve/extent.py | 6 +- reV/supply_curve/points.py | 92 ++++++++-------- reV/supply_curve/sc_aggregation.py | 18 ++-- reV/supply_curve/supply_curve.py | 102 +++++++++--------- reV/utilities/__init__.py | 6 +- reV/utilities/pytest_utils.py | 3 +- tests/test_bespoke.py | 90 ++++++++-------- tests/test_econ_of_scale.py | 42 ++++---- tests/test_handlers_transmission.py | 8 +- tests/test_hybrids.py | 44 ++++---- tests/test_rep_profiles.py | 84 +++++++-------- tests/test_supply_curve_aggregation.py | 14 +-- .../test_supply_curve_aggregation_friction.py | 16 +-- tests/test_supply_curve_compute.py | 70 ++++++------ tests/test_supply_curve_points.py | 22 ++-- tests/test_supply_curve_sc_aggregation.py | 78 +++++++------- tests/test_supply_curve_vpd.py | 12 +-- tests/test_supply_curve_wind_dirs.py | 20 ++-- 32 files changed, 487 insertions(+), 482 deletions(-) diff --git a/reV/SAM/generation.py b/reV/SAM/generation.py index d2c45b1b5..52ced392d 100644 --- a/reV/SAM/generation.py +++ b/reV/SAM/generation.py @@ -1008,7 +1008,7 @@ def set_latitude_tilt_az(sam_sys_inputs, meta): sam_sys_inputs : dict Site-agnostic SAM system model inputs arguments. If for a pv simulation the "tilt" parameter was originally not - present or set to 'lat' or MetaKeyName.LATITUDE, the tilt will be + present or set to 'lat' or SupplyCurveField.LATITUDE, the tilt will be set to the absolute value of the latitude found in meta and the azimuth will be 180 if lat>0, 0 if lat<0. """ diff --git a/reV/bespoke/bespoke.py b/reV/bespoke/bespoke.py index 19322d35c..f885c2221 100644 --- a/reV/bespoke/bespoke.py +++ b/reV/bespoke/bespoke.py @@ -36,8 +36,12 @@ from reV.supply_curve.extent import SupplyCurveExtent from reV.supply_curve.points import AggregationSupplyCurvePoint as AggSCPoint from reV.supply_curve.points import SupplyCurvePoint -from reV.utilities import (MetaKeyName, ModuleName, ResourceMetaField, - log_versions) +from reV.utilities import ( + ModuleName, + ResourceMetaField, + SupplyCurveField, + log_versions, +) from reV.utilities.exceptions import EmptySupplyCurvePointError, FileInputError logger = logging.getLogger(__name__) @@ -536,9 +540,9 @@ def _parse_prior_run(self): # {meta_column: sam_sys_input_key} required = { - MetaKeyName.CAPACITY: "system_capacity", - MetaKeyName.TURBINE_X_COORDS: "wind_farm_xCoordinates", - MetaKeyName.TURBINE_Y_COORDS: "wind_farm_yCoordinates", + SupplyCurveField.CAPACITY: "system_capacity", + SupplyCurveField.TURBINE_X_COORDS: "wind_farm_xCoordinates", + SupplyCurveField.TURBINE_Y_COORDS: "wind_farm_yCoordinates", } if self._prior_meta: @@ -811,22 +815,22 @@ def meta(self): self._meta = pd.DataFrame( { - MetaKeyName.SC_POINT_GID: self.sc_point.gid, - MetaKeyName.SC_ROW_IND: row_ind, - MetaKeyName.SC_COL_IND: col_ind, - MetaKeyName.GID: self.sc_point.gid, - MetaKeyName.LATITUDE: self.sc_point.latitude, - MetaKeyName.LONGITUDE: self.sc_point.longitude, - MetaKeyName.TIMEZONE: self.sc_point.timezone, - MetaKeyName.COUNTRY: self.sc_point.country, - MetaKeyName.STATE: self.sc_point.state, - MetaKeyName.COUNTY: self.sc_point.county, - MetaKeyName.ELEVATION: self.sc_point.elevation, - MetaKeyName.OFFSHORE: self.sc_point.offshore, - MetaKeyName.RES_GIDS: res_gids, - MetaKeyName.GID_COUNTS: gid_counts, - MetaKeyName.N_GIDS: self.sc_point.n_gids, - MetaKeyName.AREA_SQ_KM: self.sc_point.area, + SupplyCurveField.SC_POINT_GID: self.sc_point.gid, + SupplyCurveField.SC_ROW_IND: row_ind, + SupplyCurveField.SC_COL_IND: col_ind, + SupplyCurveField.GID: self.sc_point.gid, + SupplyCurveField.LATITUDE: self.sc_point.latitude, + SupplyCurveField.LONGITUDE: self.sc_point.longitude, + SupplyCurveField.TIMEZONE: self.sc_point.timezone, + SupplyCurveField.COUNTRY: self.sc_point.country, + SupplyCurveField.STATE: self.sc_point.state, + SupplyCurveField.COUNTY: self.sc_point.county, + SupplyCurveField.ELEVATION: self.sc_point.elevation, + SupplyCurveField.OFFSHORE: self.sc_point.offshore, + SupplyCurveField.RES_GIDS: res_gids, + SupplyCurveField.GID_COUNTS: gid_counts, + SupplyCurveField.N_GIDS: self.sc_point.n_gids, + SupplyCurveField.AREA_SQ_KM: self.sc_point.area, }, index=[self.sc_point.gid], ) @@ -1047,7 +1051,7 @@ def recalc_lcoe(self): my_mean_lcoe = lcoe_fcr(fcr, cap_cost, foc, aep, voc) self._outputs["lcoe_fcr-means"] = my_mean_lcoe - self._meta[MetaKeyName.MEAN_LCOE] = my_mean_lcoe + self._meta[SupplyCurveField.MEAN_LCOE] = my_mean_lcoe def get_lcoe_kwargs(self): """Get a namespace of arguments for calculating LCOE based on the @@ -1224,11 +1228,11 @@ def run_wind_plant_ts(self): # copy dataset outputs to meta data for supply curve table summary if "cf_mean-means" in self.outputs: - self._meta.loc[:, MetaKeyName.MEAN_CF] = self.outputs[ + self._meta.loc[:, SupplyCurveField.MEAN_CF] = self.outputs[ "cf_mean-means" ] if "lcoe_fcr-means" in self.outputs: - self._meta.loc[:, MetaKeyName.MEAN_LCOE] = self.outputs[ + self._meta.loc[:, SupplyCurveField.MEAN_LCOE] = self.outputs[ "lcoe_fcr-means" ] self.recalc_lcoe() @@ -1311,7 +1315,7 @@ def run_plant_optimization(self): # copy dataset outputs to meta data for supply curve table summary # convert SAM system capacity in kW to reV supply curve cap in MW - self._meta[MetaKeyName.CAPACITY] = ( + self._meta[SupplyCurveField.CAPACITY] = ( self.outputs["system_capacity"] / 1e3 ) @@ -1319,12 +1323,12 @@ def run_plant_optimization(self): baseline_cost = self.plant_optimizer.capital_cost_per_kw( capacity_mw=self._baseline_cap_mw ) - self._meta[MetaKeyName.EOS_MULT] = ( + self._meta[SupplyCurveField.EOS_MULT] = ( self.plant_optimizer.capital_cost / self.plant_optimizer.capacity / baseline_cost ) - self._meta[MetaKeyName.REG_MULT] = self.sam_sys_inputs.get( + self._meta[SupplyCurveField.REG_MULT] = self.sam_sys_inputs.get( "capital_cost_multiplier", 1 ) @@ -1923,7 +1927,7 @@ def _parse_points(points, sam_configs): Slice or list specifying project points, string pointing to a project points csv, or a fully instantiated PointsControl object. Can also be a single site integer value. Points csv should have - `MetaKeyName.GID` and 'config' column, the config maps to the + `SupplyCurveField.GID` and 'config' column, the config maps to the sam_configs dict keys. sam_configs : dict | str | SAMConfig SAM input configuration ID(s) and file path(s). Keys are the SAM @@ -2003,7 +2007,7 @@ def _get_prior_meta(self, gid): meta = None if self._prior_meta is not None: - mask = self._prior_meta[MetaKeyName.GID] == gid + mask = self._prior_meta[SupplyCurveField.GID] == gid if any(mask): meta = self._prior_meta[mask] diff --git a/reV/econ/econ.py b/reV/econ/econ.py index 7dd10b4c6..a77ddc172 100644 --- a/reV/econ/econ.py +++ b/reV/econ/econ.py @@ -352,7 +352,7 @@ def _run_single_worker(pc, econ_fun, output_request, **kwargs): pc : reV.config.project_points.PointsControl Iterable points control object from reV config module. Must have project_points with df property with all relevant - site-specific inputs and a `MetaKeyName.GID` column. By passing + site-specific inputs and a `SupplyCurveField.GID` column. By passing site-specific inputs in this dataframe, which was split using points_control, only the data relevant to the current sites is passed. diff --git a/reV/generation/base.py b/reV/generation/base.py index 4461aef48..f45ee25bc 100644 --- a/reV/generation/base.py +++ b/reV/generation/base.py @@ -302,7 +302,7 @@ def meta(self): Meta data df for sites in project points. Column names are meta data variables, rows are different sites. The row index does not indicate the site number if the project points are - non-sequential or do not start from 0, so a `MetaKeyName.GID` + non-sequential or do not start from 0, so a `SupplyCurveField.GID` column is added. """ return self._meta diff --git a/reV/generation/generation.py b/reV/generation/generation.py index 76c4cd9f4..985009755 100644 --- a/reV/generation/generation.py +++ b/reV/generation/generation.py @@ -443,7 +443,7 @@ def meta(self): Meta data df for sites in project points. Column names are meta data variables, rows are different sites. The row index does not indicate the site number if the project points are - non-sequential or do not start from 0, so a `MetaKeyName.GID` + non-sequential or do not start from 0, so a `SupplyCurveField.GID` column is added. """ if self._meta is None: diff --git a/reV/handlers/outputs.py b/reV/handlers/outputs.py index 34c91dd46..b9322cfaa 100644 --- a/reV/handlers/outputs.py +++ b/reV/handlers/outputs.py @@ -29,8 +29,8 @@ class Outputs(rexOutputs): >>> import pandas as pd >>> import numpy as np >>> - >>> meta = pd.DataFrame({MetaKeyName.LATITUDE: np.ones(100), - >>> MetaKeyName.LONGITUDE: np.ones(100)}) + >>> meta = pd.DataFrame({SupplyCurveField.LATITUDE: np.ones(100), + >>> SupplyCurveField.LONGITUDE: np.ones(100)}) >>> >>> time_index = pd.date_range('20210101', '20220101', freq='1h', >>> closed='right') diff --git a/reV/hybrids/hybrids.py b/reV/hybrids/hybrids.py index c63f793d0..2071a0e51 100644 --- a/reV/hybrids/hybrids.py +++ b/reV/hybrids/hybrids.py @@ -17,7 +17,7 @@ from reV.handlers.outputs import Outputs from reV.hybrids.hybrid_methods import HYBRID_METHODS -from reV.utilities import MetaKeyName +from reV.utilities import SupplyCurveField from reV.utilities.exceptions import ( FileInputError, InputError, @@ -27,17 +27,17 @@ logger = logging.getLogger(__name__) -MERGE_COLUMN = MetaKeyName.SC_POINT_GID +MERGE_COLUMN = SupplyCurveField.SC_POINT_GID PROFILE_DSET_REGEX = 'rep_profiles_[0-9]+$' SOLAR_PREFIX = 'solar_' WIND_PREFIX = 'wind_' NON_DUPLICATE_COLS = { - MetaKeyName.LATITUDE, MetaKeyName.LONGITUDE, - MetaKeyName.COUNTRY, MetaKeyName.STATE, MetaKeyName.COUNTY, - MetaKeyName.ELEVATION, MetaKeyName.TIMEZONE, MetaKeyName.SC_POINT_GID, - MetaKeyName.SC_ROW_IND, MetaKeyName.SC_COL_IND + SupplyCurveField.LATITUDE, SupplyCurveField.LONGITUDE, + SupplyCurveField.COUNTRY, SupplyCurveField.STATE, SupplyCurveField.COUNTY, + SupplyCurveField.ELEVATION, SupplyCurveField.TIMEZONE, SupplyCurveField.SC_POINT_GID, + SupplyCurveField.SC_ROW_IND, SupplyCurveField.SC_COL_IND } -DROPPED_COLUMNS = [MetaKeyName.GID] +DROPPED_COLUMNS = [SupplyCurveField.GID] DEFAULT_FILL_VALUES = {'solar_capacity': 0, 'wind_capacity': 0, 'solar_mean_cf': 0, 'wind_mean_cf': 0} OUTPUT_PROFILE_NAMES = ['hybrid_profile', @@ -699,7 +699,7 @@ def _format_meta_post_merge(self): self._propagate_duplicate_cols(duplicate_cols) self._drop_cols(duplicate_cols) self._hybrid_meta.rename(self.__col_name_map, inplace=True, axis=1) - self._hybrid_meta.index.name = MetaKeyName.GID + self._hybrid_meta.index.name = SupplyCurveField.GID def _propagate_duplicate_cols(self, duplicate_cols): """Fill missing column values from outer merge.""" @@ -744,8 +744,8 @@ def _column_sorting_key(self, c): def _verify_lat_long_match_post_merge(self): """Verify that all the lat/lon values match post merge.""" - lat = self._verify_col_match_post_merge(col_name=MetaKeyName.LATITUDE) - lon = self._verify_col_match_post_merge(col_name=MetaKeyName.LONGITUDE) + lat = self._verify_col_match_post_merge(col_name=SupplyCurveField.LATITUDE) + lon = self._verify_col_match_post_merge(col_name=SupplyCurveField.LONGITUDE) if not lat or not lon: msg = ( "Detected mismatched coordinate values (latitude or " diff --git a/reV/nrwal/nrwal.py b/reV/nrwal/nrwal.py index 9a38e20f0..827753cb1 100644 --- a/reV/nrwal/nrwal.py +++ b/reV/nrwal/nrwal.py @@ -19,7 +19,7 @@ from reV.generation.generation import Gen from reV.handlers.outputs import Outputs -from reV.utilities import log_versions, MetaKeyName, SiteDataField +from reV.utilities import SiteDataField, SupplyCurveField, log_versions from reV.utilities.exceptions import ( DataShapeError, OffshoreWindInputError, @@ -36,7 +36,7 @@ class RevNrwal: """Columns from the `site_data` table to join to the output meta data""" def __init__(self, gen_fpath, site_data, sam_files, nrwal_configs, - output_request, save_raw=True, meta_gid_col=MetaKeyName.GID, + output_request, save_raw=True, meta_gid_col=SupplyCurveField.GID, site_meta_cols=None): """Framework to handle reV-NRWAL analysis. diff --git a/reV/qa_qc/cli_qa_qc.py b/reV/qa_qc/cli_qa_qc.py index ce566400b..c628bb473 100644 --- a/reV/qa_qc/cli_qa_qc.py +++ b/reV/qa_qc/cli_qa_qc.py @@ -19,7 +19,7 @@ SummarizeSupplyCurve, SupplyCurvePlot, ) -from reV.utilities import ModuleName, MetaKeyName +from reV.utilities import ModuleName, SupplyCurveField logger = logging.getLogger(__name__) @@ -194,7 +194,7 @@ def supply_curve_table(ctx, sc_table, columns): show_default=True, help=(" plot_type of plot to create 'plot' or 'plotly', by " "default 'plot'")) -@click.option('--lcoe', '-lcoe', type=STR, default=MetaKeyName.MEAN_LCOE, +@click.option('--lcoe', '-lcoe', type=STR, default=SupplyCurveField.MEAN_LCOE, help="LCOE value to plot, by default %(default)s") @click.pass_context def supply_curve_plot(ctx, sc_table, plot_type, lcoe): diff --git a/reV/qa_qc/qa_qc.py b/reV/qa_qc/qa_qc.py index fde799a08..739df9c63 100644 --- a/reV/qa_qc/qa_qc.py +++ b/reV/qa_qc/qa_qc.py @@ -19,7 +19,7 @@ SupplyCurvePlot, ) from reV.supply_curve.exclusions import ExclusionMaskFromDict -from reV.utilities import ModuleName, MetaKeyName, log_versions +from reV.utilities import ModuleName, SupplyCurveField, log_versions from reV.utilities.exceptions import PipelineError logger = logging.getLogger(__name__) @@ -105,9 +105,9 @@ def create_scatter_plots( if file.endswith(".csv"): summary_csv = os.path.join(self.out_dir, file) summary = pd.read_csv(summary_csv) - has_right_cols = (MetaKeyName.GID in summary - and MetaKeyName.LATITUDE in summary - and MetaKeyName.LONGITUDE in summary) + has_right_cols = (SupplyCurveField.GID in summary + and SupplyCurveField.LATITUDE in summary + and SupplyCurveField.LONGITUDE in summary) if has_right_cols: self._scatter_plot(summary_csv, self.out_dir, plot_type=plot_type, cmap=cmap, @@ -181,7 +181,7 @@ def h5( @classmethod def supply_curve(cls, sc_table, out_dir, columns=None, - lcoe=MetaKeyName.MEAN_LCOE, plot_type='plotly', + lcoe=SupplyCurveField.MEAN_LCOE, plot_type='plotly', cmap='viridis', sc_plot_kwargs=None, scatter_plot_kwargs=None): """ @@ -197,7 +197,7 @@ def supply_curve(cls, sc_table, out_dir, columns=None, Column(s) to summarize, if None summarize all numeric columns, by default None lcoe : str, optional - LCOE value to plot, by default :obj:`MetaKeyName.MEAN_LCOE` + LCOE value to plot, by default :obj:`SupplyCurveField.MEAN_LCOE` plot_type : str, optional plot_type of plot to create 'plot' or 'plotly', by default 'plotly' cmap : str, optional @@ -345,7 +345,7 @@ def __init__(self, module_name, config, out_root): self._default_plot_type = "plotly" self._default_cmap = "viridis" self._default_plot_step = 100 - self._default_lcoe = MetaKeyName.MEAN_LCOE + self._default_lcoe = SupplyCurveField.MEAN_LCOE self._default_area_filter_kernel = 'queen' @property diff --git a/reV/qa_qc/summary.py b/reV/qa_qc/summary.py index 0e56b75e0..380889474 100644 --- a/reV/qa_qc/summary.py +++ b/reV/qa_qc/summary.py @@ -12,7 +12,7 @@ from rex import Resource from rex.utilities import SpawnProcessPool, parse_table -from reV.utilities import MetaKeyName +from reV.utilities import SupplyCurveField logger = logging.getLogger(__name__) @@ -188,7 +188,7 @@ def summarize_dset( summary = pd.concat(summary) - summary.index.name = MetaKeyName.GID + summary.index.name = SupplyCurveField.GID else: summary = self._compute_ds_summary( @@ -216,9 +216,9 @@ def summarize_means(self, out_path=None): """ with Resource(self.h5_file, group=self._group) as f: meta = f.meta - if MetaKeyName.GID not in meta: - if meta.index.name != MetaKeyName.GID: - meta.index.name = MetaKeyName.GID + if SupplyCurveField.GID not in meta: + if meta.index.name != SupplyCurveField.GID: + meta.index.name = SupplyCurveField.GID meta = meta.reset_index() @@ -486,7 +486,7 @@ def _check_value(df, values, scatter=True): values = [values] if scatter: - values += [MetaKeyName.LATITUDE, MetaKeyName.LONGITUDE] + values += [SupplyCurveField.LATITUDE, SupplyCurveField.LONGITUDE] for value in values: if value not in df: @@ -550,8 +550,8 @@ def scatter_plot(self, value, cmap="viridis", out_path=None, **kwargs): Additional kwargs for plotting.dataframes.df_scatter """ self._check_value(self.summary, value) - mplt.df_scatter(self.summary, x=MetaKeyName.LONGITUDE, - y=MetaKeyName.LATITUDE, c=value, colormap=cmap, + mplt.df_scatter(self.summary, x=SupplyCurveField.LONGITUDE, + y=SupplyCurveField.LATITUDE, c=value, colormap=cmap, filename=out_path, **kwargs) def scatter_plotly(self, value, cmap="Viridis", out_path=None, **kwargs): @@ -572,8 +572,8 @@ def scatter_plotly(self, value, cmap="Viridis", out_path=None, **kwargs): Additional kwargs for plotly.express.scatter """ self._check_value(self.summary, value) - fig = px.scatter(self.summary, x=MetaKeyName.LONGITUDE, - y=MetaKeyName.LATITUDE, color=value, + fig = px.scatter(self.summary, x=SupplyCurveField.LONGITUDE, + y=SupplyCurveField.LATITUDE, color=value, color_continuous_scale=cmap, **kwargs) fig.update_layout(font=dict(family="Arial", size=18, color="black")) @@ -582,7 +582,7 @@ def scatter_plotly(self, value, cmap="Viridis", out_path=None, **kwargs): fig.show() - def _extract_sc_data(self, lcoe=MetaKeyName.MEAN_LCOE): + def _extract_sc_data(self, lcoe=SupplyCurveField.MEAN_LCOE): """ Extract supply curve data @@ -590,17 +590,17 @@ def _extract_sc_data(self, lcoe=MetaKeyName.MEAN_LCOE): ---------- lcoe : str, optional LCOE value to use for supply curve, - by default :obj:`MetaKeyName.MEAN_LCOE` + by default :obj:`SupplyCurveField.MEAN_LCOE` Returns ------- sc_df : pandas.DataFrame Supply curve data """ - values = [MetaKeyName.CAPACITY, lcoe] + values = [SupplyCurveField.CAPACITY, lcoe] self._check_value(self.summary, values, scatter=False) sc_df = self.summary[values].sort_values(lcoe) - sc_df['cumulative_capacity'] = sc_df[MetaKeyName.CAPACITY].cumsum() + sc_df['cumulative_capacity'] = sc_df[SupplyCurveField.CAPACITY].cumsum() return sc_df @@ -783,7 +783,7 @@ def columns(self): """ return list(self.sc_table.columns) - def _extract_sc_data(self, lcoe=MetaKeyName.MEAN_LCOE): + def _extract_sc_data(self, lcoe=SupplyCurveField.MEAN_LCOE): """ Extract supply curve data @@ -791,21 +791,21 @@ def _extract_sc_data(self, lcoe=MetaKeyName.MEAN_LCOE): ---------- lcoe : str, optional LCOE value to use for supply curve, - by default :obj:`MetaKeyName.MEAN_LCOE` + by default :obj:`SupplyCurveField.MEAN_LCOE` Returns ------- sc_df : pandas.DataFrame Supply curve data """ - values = [MetaKeyName.CAPACITY, lcoe] + values = [SupplyCurveField.CAPACITY, lcoe] self._check_value(self.sc_table, values, scatter=False) sc_df = self.sc_table[values].sort_values(lcoe) - sc_df['cumulative_capacity'] = sc_df[MetaKeyName.CAPACITY].cumsum() + sc_df['cumulative_capacity'] = sc_df[SupplyCurveField.CAPACITY].cumsum() return sc_df - def supply_curve_plot(self, lcoe=MetaKeyName.MEAN_LCOE, out_path=None, + def supply_curve_plot(self, lcoe=SupplyCurveField.MEAN_LCOE, out_path=None, **kwargs): """ Plot supply curve (cumulative capacity vs lcoe) using seaborn.scatter @@ -813,7 +813,7 @@ def supply_curve_plot(self, lcoe=MetaKeyName.MEAN_LCOE, out_path=None, Parameters ---------- lcoe : str, optional - LCOE value to plot, by default :obj:`MetaKeyName.MEAN_LCOE` + LCOE value to plot, by default :obj:`SupplyCurveField.MEAN_LCOE` out_path : str, optional File path to save plot to, by default None kwargs : dict @@ -824,7 +824,7 @@ def supply_curve_plot(self, lcoe=MetaKeyName.MEAN_LCOE, out_path=None, sc_df, x="cumulative_capacity", y=lcoe, filename=out_path, **kwargs ) - def supply_curve_plotly(self, lcoe=MetaKeyName.MEAN_LCOE, out_path=None, + def supply_curve_plotly(self, lcoe=SupplyCurveField.MEAN_LCOE, out_path=None, **kwargs): """ Plot supply curve (cumulative capacity vs lcoe) using plotly @@ -832,7 +832,7 @@ def supply_curve_plotly(self, lcoe=MetaKeyName.MEAN_LCOE, out_path=None, Parameters ---------- lcoe : str, optional - LCOE value to plot, by default MetaKeyName.MEAN_LCOE + LCOE value to plot, by default SupplyCurveField.MEAN_LCOE out_path : str, optional File path to save plot to, can be a .html or static image, by default None @@ -850,7 +850,7 @@ def supply_curve_plotly(self, lcoe=MetaKeyName.MEAN_LCOE, out_path=None, @classmethod def plot(cls, sc_table, out_dir, plot_type='plotly', - lcoe=MetaKeyName.MEAN_LCOE, **kwargs): + lcoe=SupplyCurveField.MEAN_LCOE, **kwargs): """ Create supply curve plot from supply curve table using lcoe value and save to out_dir @@ -864,7 +864,7 @@ def plot(cls, sc_table, out_dir, plot_type='plotly', plot_type : str, optional plot_type of plot to create 'plot' or 'plotly', by default 'plotly' lcoe : str, optional - LCOE value to plot, by default :obj:`MetaKeyName.MEAN_LCOE` + LCOE value to plot, by default :obj:`SupplyCurveField.MEAN_LCOE` kwargs : dict Additional plotting kwargs """ diff --git a/reV/rep_profiles/rep_profiles.py b/reV/rep_profiles/rep_profiles.py index 234174e39..b9e0994b5 100644 --- a/reV/rep_profiles/rep_profiles.py +++ b/reV/rep_profiles/rep_profiles.py @@ -23,7 +23,7 @@ from scipy import stats from reV.handlers.outputs import Outputs -from reV.utilities import MetaKeyName, log_versions +from reV.utilities import SupplyCurveField, log_versions from reV.utilities.exceptions import DataShapeError, FileInputError logger = logging.getLogger(__name__) @@ -315,12 +315,12 @@ def run( class RegionRepProfile: """Framework to handle rep profile for one resource region""" - RES_GID_COL = MetaKeyName.RES_GIDS - GEN_GID_COL = MetaKeyName.GEN_GIDS + RES_GID_COL = SupplyCurveField.RES_GIDS + GEN_GID_COL = SupplyCurveField.GEN_GIDS def __init__(self, gen_fpath, rev_summary, cf_dset='cf_profile', rep_method='meanoid', err_method='rmse', - weight=MetaKeyName.GID_COUNTS, + weight=SupplyCurveField.GID_COUNTS, n_profiles=1): """ Parameters @@ -394,8 +394,8 @@ def _init_profiles_weights(self): with Resource(self._gen_fpath) as res: meta = res.meta - assert MetaKeyName.GID in meta - source_res_gids = meta[MetaKeyName.GID].values + assert SupplyCurveField.GID in meta + source_res_gids = meta[SupplyCurveField.GID].values msg = ('Resource gids from "gid" column in meta data from "{}" ' 'must be sorted! reV generation should always be run with ' 'sequential project points.'.format(self._gen_fpath)) @@ -546,7 +546,7 @@ def get_region_rep_profile(cls, gen_fpath, rev_summary, cf_dset='cf_profile', rep_method='meanoid', err_method='rmse', - weight=MetaKeyName.GID_COUNTS, + weight=SupplyCurveField.GID_COUNTS, n_profiles=1): """Class method for parallelization of rep profile calc. @@ -604,7 +604,7 @@ class RepProfilesBase(ABC): def __init__(self, gen_fpath, rev_summary, reg_cols=None, cf_dset='cf_profile', rep_method='meanoid', - err_method='rmse', weight=MetaKeyName.GID_COUNTS, + err_method='rmse', weight=SupplyCurveField.GID_COUNTS, n_profiles=1): """ Parameters @@ -953,7 +953,7 @@ class RepProfiles(RepProfilesBase): def __init__(self, gen_fpath, rev_summary, reg_cols, cf_dset='cf_profile', rep_method='meanoid', err_method='rmse', - weight=MetaKeyName.GID_COUNTS, + weight=SupplyCurveField.GID_COUNTS, n_profiles=1, aggregate_profiles=False): """ReV rep profiles class. @@ -1046,7 +1046,7 @@ def __init__(self, gen_fpath, rev_summary, reg_cols, *equally* to the meanoid profile unless these weights are specified. - By default, :obj:`MetaKeyName.GID_COUNTS`. + By default, :obj:`SupplyCurveField.GID_COUNTS`. n_profiles : int, optional Number of representative profiles to save to the output file. By default, ``1``. @@ -1111,7 +1111,7 @@ def _set_meta(self): else: self._meta = self._rev_summary.groupby(self._reg_cols) self._meta = ( - self._meta[MetaKeyName.TIMEZONE] + self._meta[SupplyCurveField.TIMEZONE] .apply(lambda x: stats.mode(x, keepdims=True).mode[0]) ) self._meta = self._meta.reset_index() diff --git a/reV/supply_curve/aggregation.py b/reV/supply_curve/aggregation.py index bd09048a2..629c80d0b 100644 --- a/reV/supply_curve/aggregation.py +++ b/reV/supply_curve/aggregation.py @@ -18,7 +18,7 @@ from reV.supply_curve.extent import SupplyCurveExtent from reV.supply_curve.points import AggregationSupplyCurvePoint from reV.supply_curve.tech_mapping import TechMapping -from reV.utilities import MetaKeyName, log_versions +from reV.utilities import SupplyCurveField, log_versions from reV.utilities.exceptions import ( EmptySupplyCurvePointError, FileInputError, @@ -472,17 +472,17 @@ def _parse_gen_index(gen_fpath): logger.error(msg) raise FileInputError(msg) - if MetaKeyName.GID in gen_index: + if SupplyCurveField.GID in gen_index: gen_index = gen_index.rename( - columns={MetaKeyName.GID: MetaKeyName.RES_GIDS} + columns={SupplyCurveField.GID: SupplyCurveField.RES_GIDS} ) - gen_index[MetaKeyName.GEN_GIDS] = gen_index.index - gen_index = gen_index[[MetaKeyName.RES_GIDS, MetaKeyName.GEN_GIDS]] - gen_index = gen_index.set_index(keys=MetaKeyName.RES_GIDS) + gen_index[SupplyCurveField.GEN_GIDS] = gen_index.index + gen_index = gen_index[[SupplyCurveField.RES_GIDS, SupplyCurveField.GEN_GIDS]] + gen_index = gen_index.set_index(keys=SupplyCurveField.RES_GIDS) gen_index = gen_index.reindex( range(int(gen_index.index.max() + 1)) ) - gen_index = gen_index[MetaKeyName.GEN_GIDS].values + gen_index = gen_index[SupplyCurveField.GEN_GIDS].values gen_index[np.isnan(gen_index)] = -1 gen_index = gen_index.astype(np.int32) else: @@ -920,9 +920,9 @@ def aggregate( for k, v in agg.items(): if k == "meta": v = pd.concat(v, axis=1).T - v = v.sort_values(MetaKeyName.SC_POINT_GID) + v = v.sort_values(SupplyCurveField.SC_POINT_GID) v = v.reset_index(drop=True) - v.index.name = MetaKeyName.SC_GID + v.index.name = SupplyCurveField.SC_GID agg[k] = v else: v = np.dstack(v)[0] diff --git a/reV/supply_curve/competitive_wind_farms.py b/reV/supply_curve/competitive_wind_farms.py index 863d5cdfd..556b73630 100644 --- a/reV/supply_curve/competitive_wind_farms.py +++ b/reV/supply_curve/competitive_wind_farms.py @@ -7,7 +7,7 @@ import numpy as np from rex.utilities.utilities import parse_table -from reV.utilities import MetaKeyName +from reV.utilities import SupplyCurveField logger = logging.getLogger(__name__) @@ -85,14 +85,14 @@ def __getitem__(self, keys): if not isinstance(keys, tuple): msg = ("{} must be a tuple of form (source, gid) where source is: " "{}, '{}', or 'upwind', 'downwind'" - .format(keys, MetaKeyName.SC_GID, MetaKeyName.SC_POINT_GID)) + .format(keys, SupplyCurveField.SC_GID, SupplyCurveField.SC_POINT_GID)) logger.error(msg) raise ValueError(msg) source, gid = keys - if source == MetaKeyName.SC_POINT_GID: + if source == SupplyCurveField.SC_POINT_GID: out = self.map_sc_gid_to_sc_point_gid(gid) - elif source == MetaKeyName.SC_GID: + elif source == SupplyCurveField.SC_GID: out = self.map_sc_point_gid_to_sc_gid(gid) elif source == "upwind": out = self.map_upwind(gid) @@ -100,8 +100,8 @@ def __getitem__(self, keys): out = self.map_downwind(gid) else: msg = ("{} must be: {}, {}, or 'upwind', " - "'downwind'".format(source, MetaKeyName.SC_GID, - MetaKeyName.SC_POINT_GID)) + "'downwind'".format(source, SupplyCurveField.SC_GID, + SupplyCurveField.SC_POINT_GID)) logger.error(msg) raise ValueError(msg) @@ -191,7 +191,7 @@ def _parse_wind_dirs(cls, wind_dirs): """ wind_dirs = cls._parse_table(wind_dirs) - wind_dirs = wind_dirs.set_index(MetaKeyName.SC_POINT_GID) + wind_dirs = wind_dirs.set_index(SupplyCurveField.SC_POINT_GID) columns = [c for c in wind_dirs if c.endswith(('_gid', '_pr'))] wind_dirs = wind_dirs[columns] @@ -221,22 +221,22 @@ def _parse_sc_points(cls, sc_points, offshore=False): Mask array to mask excluded sc_point_gids """ sc_points = cls._parse_table(sc_points) - if MetaKeyName.OFFSHORE in sc_points and not offshore: + if SupplyCurveField.OFFSHORE in sc_points and not offshore: logger.debug('Not including offshore supply curve points in ' 'CompetitiveWindFarm') - mask = sc_points[MetaKeyName.OFFSHORE] == 0 + mask = sc_points[SupplyCurveField.OFFSHORE] == 0 sc_points = sc_points.loc[mask] - mask = np.ones(int(1 + sc_points[MetaKeyName.SC_POINT_GID].max()), + mask = np.ones(int(1 + sc_points[SupplyCurveField.SC_POINT_GID].max()), dtype=bool) - sc_points = sc_points[[MetaKeyName.SC_GID, MetaKeyName.SC_POINT_GID]] - sc_gids = sc_points.set_index(MetaKeyName.SC_GID) + sc_points = sc_points[[SupplyCurveField.SC_GID, SupplyCurveField.SC_POINT_GID]] + sc_gids = sc_points.set_index(SupplyCurveField.SC_GID) sc_gids = {k: int(v[0]) for k, v in sc_gids.iterrows()} - groups = sc_points.groupby(MetaKeyName.SC_POINT_GID) - sc_point_gids = groups[MetaKeyName.SC_GID].unique().to_frame() - sc_point_gids = {int(k): v[MetaKeyName.SC_GID] + groups = sc_points.groupby(SupplyCurveField.SC_POINT_GID) + sc_point_gids = groups[SupplyCurveField.SC_GID].unique().to_frame() + sc_point_gids = {int(k): v[SupplyCurveField.SC_GID] for k, v in sc_point_gids.iterrows()} return sc_gids, sc_point_gids, mask @@ -431,13 +431,13 @@ def remove_noncompetitive_farm( wind farms """ sc_points = self._parse_table(sc_points) - if MetaKeyName.OFFSHORE in sc_points and not self._offshore: - mask = sc_points[MetaKeyName.OFFSHORE] == 0 + if SupplyCurveField.OFFSHORE in sc_points and not self._offshore: + mask = sc_points[SupplyCurveField.OFFSHORE] == 0 sc_points = sc_points.loc[mask] sc_points = sc_points.sort_values(sort_on) - sc_point_gids = sc_points[MetaKeyName.SC_POINT_GID].values.astype(int) + sc_point_gids = sc_points[SupplyCurveField.SC_POINT_GID].values.astype(int) for i in range(len(sc_points)): gid = sc_point_gids[i] @@ -452,7 +452,7 @@ def remove_noncompetitive_farm( self.exclude_sc_point_gid(n) sc_gids = self.sc_gids - mask = sc_points[MetaKeyName.SC_GID].isin(sc_gids) + mask = sc_points[SupplyCurveField.SC_GID].isin(sc_gids) return sc_points.loc[mask].reset_index(drop=True) diff --git a/reV/supply_curve/extent.py b/reV/supply_curve/extent.py index 229797f69..9154e2cb1 100644 --- a/reV/supply_curve/extent.py +++ b/reV/supply_curve/extent.py @@ -9,8 +9,8 @@ import pandas as pd from rex.utilities.utilities import get_chunk_ranges -from reV.handlers.exclusions import ExclusionLayers, LATITUDE, LONGITUDE -from reV.utilities import MetaKeyName +from reV.handlers.exclusions import LATITUDE, LONGITUDE, ExclusionLayers +from reV.utilities import SupplyCurveField from reV.utilities.exceptions import SupplyCurveError, SupplyCurveInputError logger = logging.getLogger(__name__) @@ -391,7 +391,7 @@ def points(self): } ) - self._points.index.name = MetaKeyName.GID # sc_point_gid + self._points.index.name = SupplyCurveField.GID # sc_point_gid return self._points diff --git a/reV/supply_curve/points.py b/reV/supply_curve/points.py index 4b47c30ef..c042784ac 100644 --- a/reV/supply_curve/points.py +++ b/reV/supply_curve/points.py @@ -14,9 +14,9 @@ from reV.econ.economies_of_scale import EconomiesOfScale from reV.econ.utilities import lcoe_fcr -from reV.handlers.exclusions import ExclusionLayers, LATITUDE, LONGITUDE +from reV.handlers.exclusions import LATITUDE, LONGITUDE, ExclusionLayers from reV.supply_curve.exclusions import ExclusionMask, ExclusionMaskFromDict -from reV.utilities import MetaKeyName, ResourceMetaField +from reV.utilities import ResourceMetaField, SupplyCurveField from reV.utilities.exceptions import ( DataShapeError, EmptySupplyCurvePointError, @@ -1256,18 +1256,18 @@ def summary(self): pandas.Series List of supply curve point's meta data """ - meta = {MetaKeyName.SC_POINT_GID: self.sc_point_gid, - MetaKeyName.SOURCE_GIDS: self.h5_gid_set, - MetaKeyName.GID_COUNTS: self.gid_counts, - MetaKeyName.N_GIDS: self.n_gids, - MetaKeyName.AREA_SQ_KM: self.area, - MetaKeyName.LATITUDE: self.latitude, - MetaKeyName.LONGITUDE: self.longitude, - MetaKeyName.COUNTRY: self.country, - MetaKeyName.STATE: self.state, - MetaKeyName.COUNTY: self.county, - MetaKeyName.ELEVATION: self.elevation, - MetaKeyName.TIMEZONE: self.timezone, + meta = {SupplyCurveField.SC_POINT_GID: self.sc_point_gid, + SupplyCurveField.SOURCE_GIDS: self.h5_gid_set, + SupplyCurveField.GID_COUNTS: self.gid_counts, + SupplyCurveField.N_GIDS: self.n_gids, + SupplyCurveField.AREA_SQ_KM: self.area, + SupplyCurveField.LATITUDE: self.latitude, + SupplyCurveField.LONGITUDE: self.longitude, + SupplyCurveField.COUNTRY: self.country, + SupplyCurveField.STATE: self.state, + SupplyCurveField.COUNTY: self.county, + SupplyCurveField.ELEVATION: self.elevation, + SupplyCurveField.TIMEZONE: self.timezone, } meta = pd.Series(meta) @@ -2168,37 +2168,37 @@ def point_summary(self, args=None): """ ARGS = { - MetaKeyName.LATITUDE: self.latitude, - MetaKeyName.LONGITUDE: self.longitude, - MetaKeyName.TIMEZONE: self.timezone, - MetaKeyName.COUNTRY: self.country, - MetaKeyName.STATE: self.state, - MetaKeyName.COUNTY: self.county, - MetaKeyName.ELEVATION: self.elevation, - MetaKeyName.RES_GIDS: self.res_gid_set, - MetaKeyName.GEN_GIDS: self.gen_gid_set, - MetaKeyName.GID_COUNTS: self.gid_counts, - MetaKeyName.N_GIDS: self.n_gids, - MetaKeyName.MEAN_CF: self.mean_cf, - MetaKeyName.MEAN_LCOE: self.mean_lcoe, - MetaKeyName.MEAN_RES: self.mean_res, - MetaKeyName.CAPACITY: self.capacity, - MetaKeyName.AREA_SQ_KM: self.area} - - extra_atts = [MetaKeyName.CAPACITY_AC, - MetaKeyName.OFFSHORE, - MetaKeyName.SC_POINT_CAPITAL_COST, - MetaKeyName.SC_POINT_FIXED_OPERATING_COST, - MetaKeyName.SC_POINT_ANNUAL_ENERGY, - MetaKeyName.SC_POINT_ANNUAL_ENERGY_AC] + SupplyCurveField.LATITUDE: self.latitude, + SupplyCurveField.LONGITUDE: self.longitude, + SupplyCurveField.TIMEZONE: self.timezone, + SupplyCurveField.COUNTRY: self.country, + SupplyCurveField.STATE: self.state, + SupplyCurveField.COUNTY: self.county, + SupplyCurveField.ELEVATION: self.elevation, + SupplyCurveField.RES_GIDS: self.res_gid_set, + SupplyCurveField.GEN_GIDS: self.gen_gid_set, + SupplyCurveField.GID_COUNTS: self.gid_counts, + SupplyCurveField.N_GIDS: self.n_gids, + SupplyCurveField.MEAN_CF: self.mean_cf, + SupplyCurveField.MEAN_LCOE: self.mean_lcoe, + SupplyCurveField.MEAN_RES: self.mean_res, + SupplyCurveField.CAPACITY: self.capacity, + SupplyCurveField.AREA_SQ_KM: self.area} + + extra_atts = [SupplyCurveField.CAPACITY_AC, + SupplyCurveField.OFFSHORE, + SupplyCurveField.SC_POINT_CAPITAL_COST, + SupplyCurveField.SC_POINT_FIXED_OPERATING_COST, + SupplyCurveField.SC_POINT_ANNUAL_ENERGY, + SupplyCurveField.SC_POINT_ANNUAL_ENERGY_AC] for attr in extra_atts: value = getattr(self, attr) if value is not None: ARGS[attr] = value if self._friction_layer is not None: - ARGS[MetaKeyName.MEAN_FRICTION] = self.mean_friction - ARGS[MetaKeyName.MEAN_LCOE_FRICTION] = self.mean_lcoe_friction + ARGS[SupplyCurveField.MEAN_FRICTION] = self.mean_friction + ARGS[SupplyCurveField.MEAN_LCOE_FRICTION] = self.mean_lcoe_friction if self._h5_dsets is not None: for dset, data in self.mean_h5_dsets_data.items(): @@ -2242,14 +2242,14 @@ def economies_of_scale(cap_cost_scale, summary): """ eos = EconomiesOfScale(cap_cost_scale, summary) - summary[MetaKeyName.RAW_LCOE] = eos.raw_lcoe - summary[MetaKeyName.MEAN_LCOE] = eos.scaled_lcoe - summary[MetaKeyName.CAPITAL_COST_SCALAR] = eos.capital_cost_scalar - summary[MetaKeyName.SCALED_CAPITAL_COST] = eos.scaled_capital_cost - if MetaKeyName.SC_POINT_CAPITAL_COST in summary: - scaled_costs = (summary[MetaKeyName.SC_POINT_CAPITAL_COST] + summary[SupplyCurveField.RAW_LCOE] = eos.raw_lcoe + summary[SupplyCurveField.MEAN_LCOE] = eos.scaled_lcoe + summary[SupplyCurveField.CAPITAL_COST_SCALAR] = eos.capital_cost_scalar + summary[SupplyCurveField.SCALED_CAPITAL_COST] = eos.scaled_capital_cost + if SupplyCurveField.SC_POINT_CAPITAL_COST in summary: + scaled_costs = (summary[SupplyCurveField.SC_POINT_CAPITAL_COST] * eos.capital_cost_scalar) - summary[MetaKeyName.SCALED_SC_POINT_CAPITAL_COST] = scaled_costs + summary[SupplyCurveField.SCALED_SC_POINT_CAPITAL_COST] = scaled_costs return summary diff --git a/reV/supply_curve/sc_aggregation.py b/reV/supply_curve/sc_aggregation.py index 45e464379..59b297ba8 100644 --- a/reV/supply_curve/sc_aggregation.py +++ b/reV/supply_curve/sc_aggregation.py @@ -28,7 +28,7 @@ from reV.supply_curve.exclusions import FrictionMask from reV.supply_curve.extent import SupplyCurveExtent from reV.supply_curve.points import GenerationSupplyCurvePoint -from reV.utilities import MetaKeyName, log_versions +from reV.utilities import SupplyCurveField, log_versions from reV.utilities.exceptions import ( EmptySupplyCurvePointError, FileInputError, @@ -170,14 +170,14 @@ def _parse_power_density(self): if self._pdf.endswith(".csv"): self._power_density = pd.read_csv(self._pdf) - if (MetaKeyName.GID in self._power_density + if (SupplyCurveField.GID in self._power_density and 'power_density' in self._power_density): self._power_density = \ - self._power_density.set_index(MetaKeyName.GID) + self._power_density.set_index(SupplyCurveField.GID) else: msg = ('Variable power density file must include "{}" ' 'and "power_density" columns, but received: {}' - .format(MetaKeyName.GID, + .format(SupplyCurveField.GID, self._power_density.columns.values)) logger.error(msg) raise FileInputError(msg) @@ -1151,10 +1151,10 @@ def run_serial( except EmptySupplyCurvePointError: logger.debug("SC point {} is empty".format(gid)) else: - pointsum[MetaKeyName.SC_POINT_GID] = gid - pointsum[MetaKeyName.SC_ROW_IND] = \ + pointsum[SupplyCurveField.SC_POINT_GID] = gid + pointsum[SupplyCurveField.SC_ROW_IND] = \ points.loc[gid, 'row_ind'] - pointsum[MetaKeyName.SC_COL_IND] = \ + pointsum[SupplyCurveField.SC_COL_IND] = \ points.loc[gid, 'col_ind'] pointsum['res_class'] = ri @@ -1335,11 +1335,11 @@ def _summary_to_df(summary): Summary of the SC points. """ summary = pd.DataFrame(summary) - sort_by = [x for x in (MetaKeyName.SC_POINT_GID, 'res_class') + sort_by = [x for x in (SupplyCurveField.SC_POINT_GID, 'res_class') if x in summary] summary = summary.sort_values(sort_by) summary = summary.reset_index(drop=True) - summary.index.name = MetaKeyName.SC_GID + summary.index.name = SupplyCurveField.SC_GID return summary diff --git a/reV/supply_curve/supply_curve.py b/reV/supply_curve/supply_curve.py index 816cafe6d..9af7b85af 100644 --- a/reV/supply_curve/supply_curve.py +++ b/reV/supply_curve/supply_curve.py @@ -18,7 +18,7 @@ from reV.handlers.transmission import TransmissionCosts as TC from reV.handlers.transmission import TransmissionFeatures as TF from reV.supply_curve.competitive_wind_farms import CompetitiveWindFarms -from reV.utilities import MetaKeyName, log_versions +from reV.utilities import SupplyCurveField, log_versions from reV.utilities.exceptions import SupplyCurveError, SupplyCurveInputError logger = logging.getLogger(__name__) @@ -28,7 +28,7 @@ class SupplyCurve: """SupplyCurve""" def __init__(self, sc_points, trans_table, sc_features=None, - sc_capacity_col=MetaKeyName.CAPACITY): + sc_capacity_col=SupplyCurveField.CAPACITY): """ReV LCOT calculation and SupplyCurve sorting class. ``reV`` supply curve computes the transmission costs associated @@ -181,7 +181,7 @@ def _parse_sc_points(sc_points, sc_features=None): if isinstance(sc_points, str) and sc_points.endswith(".h5"): with Resource(sc_points) as res: sc_points = res.meta - sc_points.index.name = MetaKeyName.SC_GID + sc_points.index.name = SupplyCurveField.SC_GID sc_points = sc_points.reset_index() else: sc_points = parse_table(sc_points) @@ -288,7 +288,7 @@ def _parse_trans_table(trans_table): trans_table = trans_table.rename(columns={"dist_mi": "dist_km"}) trans_table["dist_km"] *= 1.60934 - drop_cols = [MetaKeyName.SC_GID, 'cap_left', MetaKeyName.SC_POINT_GID] + drop_cols = [SupplyCurveField.SC_GID, 'cap_left', SupplyCurveField.SC_POINT_GID] drop_cols = [c for c in drop_cols if c in trans_table] if drop_cols: trans_table = trans_table.drop(columns=drop_cols) @@ -297,7 +297,7 @@ def _parse_trans_table(trans_table): @staticmethod def _map_trans_capacity(trans_sc_table, - sc_capacity_col=MetaKeyName.CAPACITY): + sc_capacity_col=SupplyCurveField.CAPACITY): """ Map SC gids to transmission features based on capacity. For any SC gids with capacity > the maximum transmission feature capacity, map @@ -424,7 +424,7 @@ def _check_sub_trans_lines(cls, features): return line_gids[~test].tolist() @classmethod - def _check_substation_conns(cls, trans_table, sc_cols=MetaKeyName.SC_GID): + def _check_substation_conns(cls, trans_table, sc_cols=SupplyCurveField.SC_GID): """ Run checks on substation transmission features to make sure that every sc point connecting to a substation can also connect to its @@ -437,7 +437,7 @@ def _check_substation_conns(cls, trans_table, sc_cols=MetaKeyName.SC_GID): (should already be merged with SC points). sc_cols : str | list, optional Column(s) in trans_table with unique supply curve id, - by default MetaKeyName.SC_GID + by default SupplyCurveField.SC_GID """ missing = {} for sc_point, sc_table in trans_table.groupby(sc_cols): @@ -468,8 +468,8 @@ def _check_sc_trans_table(cls, sc_points, trans_table): Table mapping supply curve points to transmission features (should already be merged with SC points). """ - sc_gids = set(sc_points[MetaKeyName.SC_GID].unique()) - trans_sc_gids = set(trans_table[MetaKeyName.SC_GID].unique()) + sc_gids = set(sc_points[SupplyCurveField.SC_GID].unique()) + trans_sc_gids = set(trans_table[SupplyCurveField.SC_GID].unique()) missing = sorted(list(sc_gids - trans_sc_gids)) if any(missing): msg = ( @@ -506,11 +506,11 @@ def _check_sc_trans_table(cls, sc_points, trans_table): @classmethod def _merge_sc_trans_tables(cls, sc_points, trans_table, - sc_cols=(MetaKeyName.SC_GID, - MetaKeyName.CAPACITY, - MetaKeyName.MEAN_CF, - MetaKeyName.MEAN_LCOE), - sc_capacity_col=MetaKeyName.CAPACITY): + sc_cols=(SupplyCurveField.SC_GID, + SupplyCurveField.CAPACITY, + SupplyCurveField.MEAN_CF, + SupplyCurveField.MEAN_LCOE), + sc_capacity_col=SupplyCurveField.CAPACITY): """ Merge the supply curve table with the transmission features table. @@ -525,7 +525,7 @@ def _merge_sc_trans_tables(cls, sc_points, trans_table, sc_cols : tuple | list, optional List of column from sc_points to transfer into the trans table, If the `sc_capacity_col` is not included, it will get added. - by default (MetaKeyName.SC_GID, 'capacity', 'mean_cf', 'mean_lcoe') + by default (SupplyCurveField.SC_GID, 'capacity', 'mean_cf', 'mean_lcoe') sc_capacity_col : str, optional Name of capacity column in `trans_sc_table`. The values in this column determine the size of transmission lines built. @@ -576,8 +576,8 @@ def _merge_sc_trans_tables(cls, sc_points, trans_table, if isinstance(sc_cols, tuple): sc_cols = list(sc_cols) - if MetaKeyName.MEAN_LCOE_FRICTION in sc_points: - sc_cols.append(MetaKeyName.MEAN_LCOE_FRICTION) + if SupplyCurveField.MEAN_LCOE_FRICTION in sc_points: + sc_cols.append(SupplyCurveField.MEAN_LCOE_FRICTION) if "transmission_multiplier" in sc_points: sc_cols.append("transmission_multiplier") @@ -592,9 +592,9 @@ def _merge_sc_trans_tables(cls, sc_points, trans_table, @classmethod def _map_tables(cls, sc_points, trans_table, - sc_cols=(MetaKeyName.SC_GID, MetaKeyName.CAPACITY, - MetaKeyName.MEAN_CF, MetaKeyName.MEAN_LCOE), - sc_capacity_col=MetaKeyName.CAPACITY): + sc_cols=(SupplyCurveField.SC_GID, SupplyCurveField.CAPACITY, + SupplyCurveField.MEAN_CF, SupplyCurveField.MEAN_LCOE), + sc_capacity_col=SupplyCurveField.CAPACITY): """ Map supply curve points to transmission features @@ -609,8 +609,8 @@ def _map_tables(cls, sc_points, trans_table, sc_cols : tuple | list, optional List of column from sc_points to transfer into the trans table, If the `sc_capacity_col` is not included, it will get added. - by default (MetaKeyName.SC_GID, MetaKeyName.CAPACITY, - MetaKeyName.MEAN_CF, MetaKeyName.MEAN_LCOE) + by default (SupplyCurveField.SC_GID, SupplyCurveField.CAPACITY, + SupplyCurveField.MEAN_CF, SupplyCurveField.MEAN_LCOE) sc_capacity_col : str, optional Name of capacity column in `trans_sc_table`. The values in this column determine the size of transmission lines built. @@ -639,7 +639,7 @@ def _map_tables(cls, sc_points, trans_table, trans_sc_table = \ trans_sc_table.sort_values( - [MetaKeyName.SC_GID, 'trans_gid']).reset_index(drop=True) + [SupplyCurveField.SC_GID, 'trans_gid']).reset_index(drop=True) cls._check_sc_trans_table(sc_points, trans_sc_table) @@ -682,7 +682,7 @@ def _create_handler(trans_table, trans_costs=None, avail_cap_frac=1): return trans_features @staticmethod - def _parse_sc_gids(trans_table, gid_key=MetaKeyName.SC_GID): + def _parse_sc_gids(trans_table, gid_key=SupplyCurveField.SC_GID): """Extract unique sc gids, make bool mask from tranmission table Parameters @@ -709,7 +709,7 @@ def _parse_sc_gids(trans_table, gid_key=MetaKeyName.SC_GID): @staticmethod def _get_capacity(sc_gid, sc_table, connectable=True, - sc_capacity_col=MetaKeyName.CAPACITY): + sc_capacity_col=SupplyCurveField.CAPACITY): """ Get capacity of supply curve point @@ -758,7 +758,7 @@ def _get_capacity(sc_gid, sc_table, connectable=True, def _compute_trans_cap_cost(cls, trans_table, trans_costs=None, avail_cap_frac=1, max_workers=None, connectable=True, line_limited=False, - sc_capacity_col=MetaKeyName.CAPACITY): + sc_capacity_col=SupplyCurveField.CAPACITY): """ Compute levelized cost of transmission for all combinations of supply curve points and tranmission features in trans_table @@ -822,7 +822,7 @@ def _compute_trans_cap_cost(cls, trans_table, trans_costs=None, max_workers = os.cpu_count() logger.info('Computing LCOT costs for all possible connections...') - groups = trans_table.groupby(MetaKeyName.SC_GID) + groups = trans_table.groupby(SupplyCurveField.SC_GID) if max_workers > 1: loggers = [__name__, "reV.handlers.transmission", "reV"] with SpawnProcessPool( @@ -928,31 +928,31 @@ def compute_total_lcoe( self._trans_table["trans_cap_cost_per_mw"] = cost cost *= self._trans_table[self._sc_capacity_col] - cost /= self._trans_table[MetaKeyName.CAPACITY] # align with "mean_cf" + cost /= self._trans_table[SupplyCurveField.CAPACITY] # align with "mean_cf" if 'reinforcement_cost_per_mw' in self._trans_table: logger.info("'reinforcement_cost_per_mw' column found in " "transmission table. Adding reinforcement costs " "to total LCOE.") - cf_mean_arr = self._trans_table[MetaKeyName.MEAN_CF].values + cf_mean_arr = self._trans_table[SupplyCurveField.MEAN_CF].values lcot = (cost * fcr) / (cf_mean_arr * 8760) - lcoe = lcot + self._trans_table[MetaKeyName.MEAN_LCOE] + lcoe = lcot + self._trans_table[SupplyCurveField.MEAN_LCOE] self._trans_table['lcot_no_reinforcement'] = lcot self._trans_table['lcoe_no_reinforcement'] = lcoe r_cost = (self._trans_table['reinforcement_cost_per_mw'] .values.copy()) r_cost *= self._trans_table[self._sc_capacity_col] # align with "mean_cf" - r_cost /= self._trans_table[MetaKeyName.CAPACITY] + r_cost /= self._trans_table[SupplyCurveField.CAPACITY] cost += r_cost # $/MW - cf_mean_arr = self._trans_table[MetaKeyName.MEAN_CF].values + cf_mean_arr = self._trans_table[SupplyCurveField.MEAN_CF].values lcot = (cost * fcr) / (cf_mean_arr * 8760) self._trans_table['lcot'] = lcot self._trans_table['total_lcoe'] = ( self._trans_table['lcot'] - + self._trans_table[MetaKeyName.MEAN_LCOE]) + + self._trans_table[SupplyCurveField.MEAN_LCOE]) if consider_friction: self._calculate_total_lcoe_friction() @@ -961,11 +961,11 @@ def _calculate_total_lcoe_friction(self): """Look for site mean LCOE with friction in the trans table and if found make a total LCOE column with friction.""" - if MetaKeyName.MEAN_LCOE_FRICTION in self._trans_table: + if SupplyCurveField.MEAN_LCOE_FRICTION in self._trans_table: lcoe_friction = ( self._trans_table['lcot'] - + self._trans_table[MetaKeyName.MEAN_LCOE_FRICTION]) - self._trans_table[MetaKeyName.TOTAL_LCOE_FRICTION] = lcoe_friction + + self._trans_table[SupplyCurveField.MEAN_LCOE_FRICTION]) + self._trans_table[SupplyCurveField.TOTAL_LCOE_FRICTION] = lcoe_friction logger.info('Found mean LCOE with friction. Adding key ' '"total_lcoe_friction" to trans table.') @@ -1001,7 +1001,7 @@ def _exclude_noncompetitive_wind_farms( for n in exclude_gids: check = comp_wind_dirs.exclude_sc_point_gid(n) if check: - sc_gids = comp_wind_dirs[MetaKeyName.SC_GID, n] + sc_gids = comp_wind_dirs[SupplyCurveField.SC_GID, n] for sc_id in sc_gids: if self._mask[sc_id]: logger.debug( @@ -1119,7 +1119,7 @@ def _full_sort( conn_lists = {k: deepcopy(init_list) for k in columns} - trans_sc_gids = trans_table[MetaKeyName.SC_GID].values.astype(int) + trans_sc_gids = trans_table[SupplyCurveField.SC_GID].values.astype(int) # syntax is final_key: source_key (source from trans_table) all_cols = {k: k for k in columns} @@ -1158,7 +1158,7 @@ def _full_sort( conn_lists[col_name][sc_gid] = data_arr[i] if total_lcoe_fric is not None: - conn_lists[MetaKeyName.TOTAL_LCOE_FRICTION][sc_gid] = ( + conn_lists[SupplyCurveField.TOTAL_LCOE_FRICTION][sc_gid] = ( total_lcoe_fric[i] ) @@ -1180,12 +1180,12 @@ def _full_sort( index = range(0, int(1 + np.max(self._sc_gids))) connections = pd.DataFrame(conn_lists, index=index) - connections.index.name = MetaKeyName.SC_GID + connections.index.name = SupplyCurveField.SC_GID connections = connections.dropna(subset=[sort_on]) connections = connections[columns].reset_index() - sc_gids = self._sc_points[MetaKeyName.SC_GID].values - connected = connections[MetaKeyName.SC_GID].values + sc_gids = self._sc_points[SupplyCurveField.SC_GID].values + connected = connections[SupplyCurveField.SC_GID].values logger.debug('Connected gids {} out of total supply curve gids {}' .format(len(connected), len(sc_gids))) unconnected = ~np.isin(sc_gids, connected) @@ -1202,7 +1202,7 @@ def _full_sort( warn(msg) supply_curve = self._sc_points.merge( - connections, on=MetaKeyName.SC_GID) + connections, on=SupplyCurveField.SC_GID) return supply_curve.reset_index(drop=True) @@ -1221,12 +1221,12 @@ def _adjust_output_columns(self, columns, consider_friction): # These are essentially should-be-defaults that are not # backwards-compatible, so have to explicitly check for them extra_cols = ['ba_str', 'poi_lat', 'poi_lon', 'reinforcement_poi_lat', - 'reinforcement_poi_lon', MetaKeyName.EOS_MULT, - MetaKeyName.REG_MULT, + 'reinforcement_poi_lon', SupplyCurveField.EOS_MULT, + SupplyCurveField.REG_MULT, 'reinforcement_cost_per_mw', 'reinforcement_dist_km', - 'n_parallel_trans', MetaKeyName.TOTAL_LCOE_FRICTION] + 'n_parallel_trans', SupplyCurveField.TOTAL_LCOE_FRICTION] if not consider_friction: - extra_cols -= {MetaKeyName.TOTAL_LCOE_FRICTION} + extra_cols -= {SupplyCurveField.TOTAL_LCOE_FRICTION} extra_cols = [ col @@ -1344,9 +1344,9 @@ def full_sort( trans_table = trans_table.loc[~pos].sort_values([sort_on, "trans_gid"]) total_lcoe_fric = None - if consider_friction and MetaKeyName.MEAN_LCOE_FRICTION in trans_table: + if consider_friction and SupplyCurveField.MEAN_LCOE_FRICTION in trans_table: total_lcoe_fric = \ - trans_table[MetaKeyName.TOTAL_LCOE_FRICTION].values + trans_table[SupplyCurveField.TOTAL_LCOE_FRICTION].values comp_wind_dirs = None if wind_dirs is not None: @@ -1473,14 +1473,14 @@ def simple_sort( sort_on = self._determine_sort_on(sort_on) connections = trans_table.sort_values([sort_on, 'trans_gid']) - connections = connections.groupby(MetaKeyName.SC_GID).first() + connections = connections.groupby(SupplyCurveField.SC_GID).first() rename = {'trans_gid': 'trans_gid', 'category': 'trans_type'} connections = connections.rename(columns=rename) connections = connections[columns].reset_index() supply_curve = self._sc_points.merge(connections, - on=MetaKeyName.SC_GID) + on=SupplyCurveField.SC_GID) if wind_dirs is not None: supply_curve = CompetitiveWindFarms.run( wind_dirs, diff --git a/reV/utilities/__init__.py b/reV/utilities/__init__.py index a18fd96d7..7640a92d4 100644 --- a/reV/utilities/__init__.py +++ b/reV/utilities/__init__.py @@ -10,6 +10,7 @@ class SiteDataField(str, Enum): """An enumerated map to site data column names.""" + GID = "gid" CONFIG = "config" @@ -32,14 +33,15 @@ class ResourceMetaField(str, Enum): OFFSHORE = 'offshore' -class MetaKeyName(str, Enum): - """An enumerated map to summary/meta keys. +class SupplyCurveField(str, Enum): + """An enumerated map to supply curve summary/meta keys. Each output name should match the name of a key in meth:`AggregationSupplyCurvePoint.summary` or meth:`GenerationSupplyCurvePoint.point_summary` or meth:`BespokeSinglePlant.meta` """ + SC_POINT_GID = 'sc_point_gid' SOURCE_GIDS = 'source_gids' SC_GID = 'sc_gid' diff --git a/reV/utilities/pytest_utils.py b/reV/utilities/pytest_utils.py index 5ef3a332e..d49ec5d6e 100644 --- a/reV/utilities/pytest_utils.py +++ b/reV/utilities/pytest_utils.py @@ -7,9 +7,8 @@ import pandas as pd from packaging import version from rex.outputs import Outputs as RexOutputs -from reV.utilities import ResourceMetaField -from reV.utilities import MetaKeyName +from reV.utilities import ResourceMetaField, SupplyCurveField def pd_date_range(*args, **kwargs): diff --git a/tests/test_bespoke.py b/tests/test_bespoke.py index cee89b786..f32841016 100644 --- a/tests/test_bespoke.py +++ b/tests/test_bespoke.py @@ -25,7 +25,7 @@ from reV.SAM.generation import WindPower from reV.supply_curve.supply_curve import SupplyCurve from reV.supply_curve.tech_mapping import TechMapping -from reV.utilities import MetaKeyName, ModuleName, SiteDataField +from reV.utilities import ModuleName, SiteDataField, SupplyCurveField pytest.importorskip("shapely") @@ -331,7 +331,7 @@ def test_bespoke_points(): for gid in pp.gids: assert pp[gid][0] == "default" - points = pd.DataFrame({MetaKeyName.GID: [33, 34, 35]}) + points = pd.DataFrame({SupplyCurveField.GID: [33, 34, 35]}) pp = BespokeWindPlants._parse_points(points, {'default': SAM}) assert len(pp) == 3 assert SiteDataField.CONFIG in pp.df.columns @@ -381,8 +381,8 @@ def test_single(gid=33): assert (TURB_RATING * bsp.meta['n_turbines'].values[0] == out['system_capacity']) - x_coords = json.loads(bsp.meta[MetaKeyName.TURBINE_X_COORDS].values[0]) - y_coords = json.loads(bsp.meta[MetaKeyName.TURBINE_Y_COORDS].values[0]) + x_coords = json.loads(bsp.meta[SupplyCurveField.TURBINE_X_COORDS].values[0]) + y_coords = json.loads(bsp.meta[SupplyCurveField.TURBINE_Y_COORDS].values[0]) assert bsp.meta['n_turbines'].values[0] == len(x_coords) assert bsp.meta['n_turbines'].values[0] == len(y_coords) @@ -487,9 +487,9 @@ def test_extra_outputs(gid=33): assert "lcoe_fcr-2013" in out assert "lcoe_fcr-means" in out - assert MetaKeyName.CAPACITY in bsp.meta - assert MetaKeyName.MEAN_CF in bsp.meta - assert MetaKeyName.MEAN_LCOE in bsp.meta + assert SupplyCurveField.CAPACITY in bsp.meta + assert SupplyCurveField.MEAN_CF in bsp.meta + assert SupplyCurveField.MEAN_LCOE in bsp.meta assert "pct_slope" in bsp.meta assert "reeds_region" in bsp.meta @@ -527,17 +527,17 @@ def test_extra_outputs(gid=33): assert "lcoe_fcr-2013" in out assert "lcoe_fcr-means" in out - assert MetaKeyName.CAPACITY in bsp.meta - assert MetaKeyName.MEAN_CF in bsp.meta - assert MetaKeyName.MEAN_LCOE in bsp.meta + assert SupplyCurveField.CAPACITY in bsp.meta + assert SupplyCurveField.MEAN_CF in bsp.meta + assert SupplyCurveField.MEAN_LCOE in bsp.meta assert "pct_slope" in bsp.meta assert "reeds_region" in bsp.meta assert "padus" in bsp.meta - assert MetaKeyName.EOS_MULT in bsp.meta - assert MetaKeyName.REG_MULT in bsp.meta - assert np.allclose(bsp.meta[MetaKeyName.REG_MULT], 1) + assert SupplyCurveField.EOS_MULT in bsp.meta + assert SupplyCurveField.REG_MULT in bsp.meta + assert np.allclose(bsp.meta[SupplyCurveField.REG_MULT], 1) n_turbs = round(test_eos_cap / TURB_RATING) test_eos_cap_kw = n_turbs * TURB_RATING @@ -546,7 +546,7 @@ def test_extra_outputs(gid=33): eos_mult = (bsp.plant_optimizer.capital_cost / bsp.plant_optimizer.capacity / (baseline_cost / test_eos_cap_kw)) - assert np.allclose(bsp.meta[MetaKeyName.EOS_MULT], eos_mult) + assert np.allclose(bsp.meta[SupplyCurveField.EOS_MULT], eos_mult) bsp.close() @@ -621,12 +621,12 @@ def test_bespoke(): with Resource(out_fpath_truth) as f: meta = f.meta assert len(meta) <= len(points) - assert MetaKeyName.SC_POINT_GID in meta - assert MetaKeyName.TURBINE_X_COORDS in meta - assert MetaKeyName.TURBINE_Y_COORDS in meta + assert SupplyCurveField.SC_POINT_GID in meta + assert SupplyCurveField.TURBINE_X_COORDS in meta + assert SupplyCurveField.TURBINE_Y_COORDS in meta assert 'possible_x_coords' in meta assert 'possible_y_coords' in meta - assert MetaKeyName.RES_GIDS in meta + assert SupplyCurveField.RES_GIDS in meta dsets_1d = ( "system_capacity", @@ -696,8 +696,8 @@ def test_collect_bespoke(): with Resource(h5_file) as fout: meta = fout.meta - assert all(meta[MetaKeyName.GID].values - == sorted(meta[MetaKeyName.GID].values)) + assert all(meta[SupplyCurveField.GID].values + == sorted(meta[SupplyCurveField.GID].values)) ti = fout.time_index assert len(ti) == 8760 assert "time_index-2012" in fout @@ -706,11 +706,11 @@ def test_collect_bespoke(): for fp in source_fps: with Resource(fp) as source: - assert all(np.isin(source.meta[MetaKeyName.GID].values, - meta[MetaKeyName.GID].values)) + assert all(np.isin(source.meta[SupplyCurveField.GID].values, + meta[SupplyCurveField.GID].values)) for isource, gid in enumerate( - source.meta[MetaKeyName.GID].values): - iout = np.where(meta[MetaKeyName.GID].values == gid)[0] + source.meta[SupplyCurveField.GID].values): + iout = np.where(meta[SupplyCurveField.GID].values == gid)[0] truth = source['cf_profile-2012', :, isource].flatten() test = data[:, iout].flatten() assert np.allclose(truth, test) @@ -779,7 +779,7 @@ def test_bespoke_supply_curve(): del f["meta"] with Outputs(bespoke_sc_fp, mode="a") as f: bespoke_meta = normal_sc_points.copy() - bespoke_meta = bespoke_meta.drop(MetaKeyName.SC_GID, axis=1) + bespoke_meta = bespoke_meta.drop(SupplyCurveField.SC_GID, axis=1) f.meta = bespoke_meta # this is basically copied from test_supply_curve_compute.py @@ -791,16 +791,16 @@ def test_bespoke_supply_curve(): sc = SupplyCurve(bespoke_sc_fp, trans_tables) sc_full = sc.full_sort(fcr=0.1, avail_cap_frac=0.1) - assert all(gid in sc_full[MetaKeyName.SC_GID] - for gid in normal_sc_points[MetaKeyName.SC_GID]) + assert all(gid in sc_full[SupplyCurveField.SC_GID] + for gid in normal_sc_points[SupplyCurveField.SC_GID]) for _, inp_row in normal_sc_points.iterrows(): - sc_gid = inp_row[MetaKeyName.SC_GID] - assert sc_gid in sc_full[MetaKeyName.SC_GID] - test_ind = np.where(sc_full[MetaKeyName.SC_GID] == sc_gid)[0] + sc_gid = inp_row[SupplyCurveField.SC_GID] + assert sc_gid in sc_full[SupplyCurveField.SC_GID] + test_ind = np.where(sc_full[SupplyCurveField.SC_GID] == sc_gid)[0] assert len(test_ind) == 1 test_row = sc_full.iloc[test_ind] assert (test_row['total_lcoe'].values[0] - > inp_row[MetaKeyName.MEAN_LCOE]) + > inp_row[SupplyCurveField.MEAN_LCOE]) fpath_baseline = os.path.join(TESTDATADIR, "sc_out/sc_full_lc.csv") sc_baseline = pd.read_csv(fpath_baseline) @@ -1232,9 +1232,9 @@ def test_bespoke_prior_run(): meta2 = f2.meta data2 = {k: f2[k] for k in f2.dsets} - cols = [MetaKeyName.TURBINE_X_COORDS, MetaKeyName.TURBINE_Y_COORDS, - MetaKeyName.CAPACITY, MetaKeyName.N_GIDS, - MetaKeyName.GID_COUNTS, MetaKeyName.RES_GIDS] + cols = [SupplyCurveField.TURBINE_X_COORDS, SupplyCurveField.TURBINE_Y_COORDS, + SupplyCurveField.CAPACITY, SupplyCurveField.N_GIDS, + SupplyCurveField.GID_COUNTS, SupplyCurveField.RES_GIDS] pd.testing.assert_frame_equal(meta1[cols], meta2[cols]) # multi-year means should not match the 2nd run with 2013 only. @@ -1273,7 +1273,7 @@ def test_gid_map(): SiteDataField.CONFIG: ['default'], 'extra_unused_data': [42]}) - gid_map = pd.DataFrame({MetaKeyName.GID: [3, 4, 13, 12, 11, 10, 9]}) + gid_map = pd.DataFrame({SupplyCurveField.GID: [3, 4, 13, 12, 11, 10, 9]}) new_gid = 50 gid_map["gid_map"] = new_gid fp_gid_map = os.path.join(td, "gid_map.csv") @@ -1333,8 +1333,8 @@ def test_gid_map(): with Resource(res_fp_2013) as f3: ws = f3[f"windspeed_{hh}m", :, new_gid] - cols = [MetaKeyName.N_GIDS, MetaKeyName.GID_COUNTS, - MetaKeyName.RES_GIDS] + cols = [SupplyCurveField.N_GIDS, SupplyCurveField.GID_COUNTS, + SupplyCurveField.RES_GIDS] pd.testing.assert_frame_equal(meta1[cols], meta2[cols]) assert not np.allclose(data1["cf_mean-2013"], data2["cf_mean-2013"]) @@ -1388,7 +1388,7 @@ def test_bespoke_bias_correct(): # intentionally leaving out WTK gid 13 which only has 5 included 90m # pixels in order to check that this is dynamically patched. - bias_correct = pd.DataFrame({MetaKeyName.GID: [3, 4, 12, 11, 10, 9]}) + bias_correct = pd.DataFrame({SupplyCurveField.GID: [3, 4, 12, 11, 10, 9]}) bias_correct['method'] = 'lin_ws' bias_correct['scalar'] = 0.5 fp_bc = os.path.join(td, 'bc.csv') @@ -1444,8 +1444,8 @@ def test_bespoke_bias_correct(): meta2 = f2.meta data2 = {k: f2[k] for k in f2.dsets} - cols = [MetaKeyName.N_GIDS, MetaKeyName.GID_COUNTS, - MetaKeyName.RES_GIDS] + cols = [SupplyCurveField.N_GIDS, SupplyCurveField.GID_COUNTS, + SupplyCurveField.RES_GIDS] pd.testing.assert_frame_equal(meta1[cols], meta2[cols]) assert data1["cf_mean-2013"] * 0.5 > data2["cf_mean-2013"] @@ -1523,12 +1523,12 @@ def test_cli(runner, clear_loggers): with Resource(out_fpath) as f: meta = f.meta assert len(meta) == 2 - assert MetaKeyName.SC_POINT_GID in meta - assert MetaKeyName.TURBINE_X_COORDS in meta - assert MetaKeyName.TURBINE_Y_COORDS in meta + assert SupplyCurveField.SC_POINT_GID in meta + assert SupplyCurveField.TURBINE_X_COORDS in meta + assert SupplyCurveField.TURBINE_Y_COORDS in meta assert 'possible_x_coords' in meta assert 'possible_y_coords' in meta - assert MetaKeyName.RES_GIDS in meta + assert SupplyCurveField.RES_GIDS in meta dsets_1d = ( "system_capacity", @@ -1584,7 +1584,7 @@ def test_bespoke_5min_sample(): # hack techmap because 5min data only has 10 wind resource pixels with h5py.File(excl_fp, 'a') as excl_file: arr = np.random.choice(10, - size=excl_file[MetaKeyName.LATITUDE].shape) + size=excl_file[SupplyCurveField.LATITUDE].shape) excl_file.create_dataset(name=tm_dset, data=arr) bsp = BespokeWindPlants( diff --git a/tests/test_econ_of_scale.py b/tests/test_econ_of_scale.py index fcdb77e6d..1779ed4f3 100644 --- a/tests/test_econ_of_scale.py +++ b/tests/test_econ_of_scale.py @@ -17,7 +17,7 @@ from reV.econ.economies_of_scale import EconomiesOfScale from reV.generation.generation import Gen from reV.supply_curve.sc_aggregation import SupplyCurveAggregation -from reV.utilities import MetaKeyName +from reV.utilities import SupplyCurveField EXCL = os.path.join(TESTDATADIR, "ri_exclusions/ri_exclusions.h5") GEN = os.path.join(TESTDATADIR, "gen_out/ri_my_pv_gen.h5") @@ -90,7 +90,7 @@ def test_lcoe_calc_simple(): true_lcoe = (data["fcr"] * data["capital_cost"] + data["foc"]) / ( data["aep"] / 1000 ) - data[MetaKeyName.MEAN_LCOE] = true_lcoe + data[SupplyCurveField.MEAN_LCOE] = true_lcoe eos = EconomiesOfScale(eqn, data) assert eos.raw_capital_cost == eos.scaled_capital_cost @@ -186,11 +186,11 @@ def test_econ_of_scale_baseline(): base_df = pd.read_csv(out_fp_base + ".csv") sc_df = pd.read_csv(out_fp_sc + ".csv") - assert np.allclose(base_df[MetaKeyName.MEAN_LCOE], - sc_df[MetaKeyName.MEAN_LCOE]) - assert (sc_df[MetaKeyName.CAPITAL_COST_SCALAR] == 1).all() + assert np.allclose(base_df[SupplyCurveField.MEAN_LCOE], + sc_df[SupplyCurveField.MEAN_LCOE]) + assert (sc_df[SupplyCurveField.CAPITAL_COST_SCALAR] == 1).all() assert np.allclose(sc_df['mean_capital_cost'], - sc_df[MetaKeyName.SCALED_CAPITAL_COST]) + sc_df[SupplyCurveField.SCALED_CAPITAL_COST]) def test_sc_agg_econ_scale(): @@ -243,14 +243,14 @@ def test_sc_agg_econ_scale(): # check that econ of scale saved the raw lcoe and that it reduced all # of the mean lcoe values from baseline - assert np.allclose(sc_df[MetaKeyName.RAW_LCOE], - base_df[MetaKeyName.MEAN_LCOE]) - assert all(sc_df[MetaKeyName.MEAN_LCOE] - < base_df[MetaKeyName.MEAN_LCOE]) + assert np.allclose(sc_df[SupplyCurveField.RAW_LCOE], + base_df[SupplyCurveField.MEAN_LCOE]) + assert all(sc_df[SupplyCurveField.MEAN_LCOE] + < base_df[SupplyCurveField.MEAN_LCOE]) aep = ((sc_df['mean_fixed_charge_rate'] * sc_df['mean_capital_cost'] + sc_df['mean_fixed_operating_cost']) - / sc_df[MetaKeyName.RAW_LCOE]) + / sc_df[SupplyCurveField.RAW_LCOE]) true_raw_lcoe = ((data['fixed_charge_rate'] * data['capital_cost'] + data['fixed_operating_cost']) @@ -265,21 +265,21 @@ def test_sc_agg_econ_scale(): + data["fixed_operating_cost"] ) / aep + data["variable_operating_cost"] - assert np.allclose(scalars, sc_df[MetaKeyName.CAPITAL_COST_SCALAR]) + assert np.allclose(scalars, sc_df[SupplyCurveField.CAPITAL_COST_SCALAR]) assert np.allclose(scalars * sc_df['mean_capital_cost'], - sc_df[MetaKeyName.SCALED_CAPITAL_COST]) + sc_df[SupplyCurveField.SCALED_CAPITAL_COST]) - assert np.allclose(true_scaled_lcoe, sc_df[MetaKeyName.MEAN_LCOE]) - assert np.allclose(true_raw_lcoe, sc_df[MetaKeyName.RAW_LCOE]) - sc_df = sc_df.sort_values(MetaKeyName.CAPACITY) - assert all(sc_df[MetaKeyName.MEAN_LCOE].diff()[1:] < 0) + assert np.allclose(true_scaled_lcoe, sc_df[SupplyCurveField.MEAN_LCOE]) + assert np.allclose(true_raw_lcoe, sc_df[SupplyCurveField.RAW_LCOE]) + sc_df = sc_df.sort_values(SupplyCurveField.CAPACITY) + assert all(sc_df[SupplyCurveField.MEAN_LCOE].diff()[1:] < 0) for i in sc_df.index.values: if sc_df.loc[i, 'scalars'] < 1: - assert (sc_df.loc[i, MetaKeyName.MEAN_LCOE] - < sc_df.loc[i, MetaKeyName.RAW_LCOE]) + assert (sc_df.loc[i, SupplyCurveField.MEAN_LCOE] + < sc_df.loc[i, SupplyCurveField.RAW_LCOE]) else: - assert (sc_df.loc[i, MetaKeyName.MEAN_LCOE] - >= sc_df.loc[i, MetaKeyName.RAW_LCOE]) + assert (sc_df.loc[i, SupplyCurveField.MEAN_LCOE] + >= sc_df.loc[i, SupplyCurveField.RAW_LCOE]) def execute_pytest(capture="all", flags="-rapP"): diff --git a/tests/test_handlers_transmission.py b/tests/test_handlers_transmission.py index b3b32428e..c72501453 100644 --- a/tests/test_handlers_transmission.py +++ b/tests/test_handlers_transmission.py @@ -10,7 +10,7 @@ from reV import TESTDATADIR from reV.handlers.transmission import TransmissionFeatures as TF -from reV.utilities import MetaKeyName +from reV.utilities import SupplyCurveField TRANS_COSTS_1 = { "line_tie_in_cost": 200, @@ -86,7 +86,7 @@ def trans_table(): return trans_table -@pytest.mark.parametrize(('i', 'trans_costs', 'distance', MetaKeyName.GID), +@pytest.mark.parametrize(('i', 'trans_costs', 'distance', SupplyCurveField.GID), ((1, TRANS_COSTS_1, 0, 43300), (2, TRANS_COSTS_2, 0, 43300), (1, TRANS_COSTS_1, 100, 43300), @@ -116,8 +116,8 @@ def test_cost_calculation(i, trans_costs, distance, gid, trans_table): assert true_cost == trans_cost -@pytest.mark.parametrize(('trans_costs', MetaKeyName.CAPACITY, - MetaKeyName.GID), +@pytest.mark.parametrize(('trans_costs', SupplyCurveField.CAPACITY, + SupplyCurveField.GID), ((TRANS_COSTS_1, 350, 43300), (TRANS_COSTS_2, 350, 43300), (TRANS_COSTS_1, 100, 43300), diff --git a/tests/test_hybrids.py b/tests/test_hybrids.py index 8cdd5378e..9999373c9 100644 --- a/tests/test_hybrids.py +++ b/tests/test_hybrids.py @@ -13,7 +13,7 @@ from reV.cli import main from reV.hybrids import HYBRID_METHODS, Hybridization from reV.hybrids.hybrids import MERGE_COLUMN, OUTPUT_PROFILE_NAMES, HybridsData -from reV.utilities import ModuleName, MetaKeyName +from reV.utilities import ModuleName, SupplyCurveField from reV.utilities.exceptions import FileInputError, InputError, OutputWarning SOLAR_FPATH = os.path.join( @@ -29,9 +29,9 @@ TESTDATADIR, "rep_profiles_out", "rep_profiles_solar_multiple.h5" ) with Resource(SOLAR_FPATH) as res: - SOLAR_SCPGIDS = set(res.meta[MetaKeyName.SC_POINT_GID]) + SOLAR_SCPGIDS = set(res.meta[SupplyCurveField.SC_POINT_GID]) with Resource(WIND_FPATH) as res: - WIND_SCPGIDS = set(res.meta[MetaKeyName.SC_POINT_GID]) + WIND_SCPGIDS = set(res.meta[SupplyCurveField.SC_POINT_GID]) def test_hybridization_profile_output_single_resource(): @@ -41,10 +41,10 @@ def test_hybridization_profile_output_single_resource(): with Resource(SOLAR_FPATH) as res: solar_idx = np.where( - res.meta[MetaKeyName.SC_POINT_GID] == sc_point_gid + res.meta[SupplyCurveField.SC_POINT_GID] == sc_point_gid )[0][0] - solar_cap = res.meta.loc[solar_idx, MetaKeyName.CAPACITY] + solar_cap = res.meta.loc[solar_idx, SupplyCurveField.CAPACITY] solar_test_profile = res['rep_profiles_0', :, solar_idx] weighted_solar = solar_cap * solar_test_profile @@ -57,7 +57,7 @@ def test_hybridization_profile_output_single_resource(): hwp, ) = h.profiles.values() h_meta = h.hybrid_meta - h_idx = np.where(h_meta[MetaKeyName.SC_POINT_GID] == sc_point_gid)[0][0] + h_idx = np.where(h_meta[SupplyCurveField.SC_POINT_GID] == sc_point_gid)[0][0] assert np.allclose(hp[:, h_idx], weighted_solar) assert np.allclose(hsp[:, h_idx], weighted_solar) @@ -72,10 +72,10 @@ def test_hybridization_profile_output_with_ratio_none(): with Resource(SOLAR_FPATH) as res: solar_idx = np.where( - res.meta[MetaKeyName.SC_POINT_GID] == sc_point_gid + res.meta[SupplyCurveField.SC_POINT_GID] == sc_point_gid )[0][0] - solar_cap = res.meta.loc[solar_idx, MetaKeyName.CAPACITY] + solar_cap = res.meta.loc[solar_idx, SupplyCurveField.CAPACITY] solar_test_profile = res['rep_profiles_0', :, solar_idx] weighted_solar = solar_cap * solar_test_profile @@ -94,7 +94,7 @@ def test_hybridization_profile_output_with_ratio_none(): hwp, ) = h.profiles.values() h_meta = h.hybrid_meta - h_idx = np.where(h_meta[MetaKeyName.SC_POINT_GID] == sc_point_gid)[0][0] + h_idx = np.where(h_meta[SupplyCurveField.SC_POINT_GID] == sc_point_gid)[0][0] assert np.allclose(hp[:, h_idx], weighted_solar) assert np.allclose(hsp[:, h_idx], weighted_solar) @@ -108,16 +108,16 @@ def test_hybridization_profile_output(): with Resource(SOLAR_FPATH) as res: solar_idx = np.where( - res.meta[MetaKeyName.SC_POINT_GID] == common_sc_point_gid + res.meta[SupplyCurveField.SC_POINT_GID] == common_sc_point_gid )[0][0] - solar_cap = res.meta.loc[solar_idx, MetaKeyName.CAPACITY] + solar_cap = res.meta.loc[solar_idx, SupplyCurveField.CAPACITY] solar_test_profile = res['rep_profiles_0', :, solar_idx] with Resource(WIND_FPATH) as res: wind_idx = np.where( - res.meta[MetaKeyName.SC_POINT_GID] == common_sc_point_gid + res.meta[SupplyCurveField.SC_POINT_GID] == common_sc_point_gid )[0][0] - wind_cap = res.meta.loc[wind_idx, MetaKeyName.CAPACITY] + wind_cap = res.meta.loc[wind_idx, SupplyCurveField.CAPACITY] wind_test_profile = res['rep_profiles_0', :, wind_idx] weighted_solar = solar_cap * solar_test_profile @@ -132,7 +132,7 @@ def test_hybridization_profile_output(): ) = h.profiles.values() h_meta = h.hybrid_meta h_idx = np.where( - h_meta[MetaKeyName.SC_POINT_GID] == common_sc_point_gid + h_meta[SupplyCurveField.SC_POINT_GID] == common_sc_point_gid )[0][0] assert np.allclose(hp[:, h_idx], weighted_solar + weighted_wind) @@ -191,7 +191,7 @@ def test_meta_hybridization(input_combination, expected_shape, overlap): ) h.run() assert h.hybrid_meta.shape == expected_shape - assert set(h.hybrid_meta[MetaKeyName.SC_POINT_GID]) == overlap + assert set(h.hybrid_meta[SupplyCurveField.SC_POINT_GID]) == overlap def test_limits_and_ratios_output_values(): @@ -270,7 +270,7 @@ def test_ratios_input(ratio_cols, ratio_bounds, bounds): <= h.hybrid_meta[ratio_denominator] ) - if MetaKeyName.CAPACITY in ratio: + if SupplyCurveField.CAPACITY in ratio: max_solar_capacities = h.hybrid_meta['hybrid_solar_capacity'] max_solar_capacities = max_solar_capacities.values.reshape(1, -1) assert np.all( @@ -413,7 +413,7 @@ def test_hybrid_col_additional_method(): """Test that function decorated with 'hybrid_col' adds to hybrid meta.""" def some_new_hybrid_func(h): - return h.hybrid_meta[MetaKeyName.ELEVATION] * 1000 + return h.hybrid_meta[SupplyCurveField.ELEVATION] * 1000 HYBRID_METHODS["scaled_elevation"] = some_new_hybrid_func @@ -423,7 +423,7 @@ def some_new_hybrid_func(h): assert "scaled_elevation" in HYBRID_METHODS assert "scaled_elevation" in h.hybrid_meta.columns assert np.allclose( - h.hybrid_meta[MetaKeyName.ELEVATION] * 1000, + h.hybrid_meta[SupplyCurveField.ELEVATION] * 1000, h.hybrid_meta["scaled_elevation"], ) @@ -820,10 +820,10 @@ def make_test_file( half_n_rows = n_rows // 2 meta.iloc[-half_n_rows:] = meta.iloc[:half_n_rows].values if duplicate_coord_values: - lat = meta[MetaKeyName.LATITUDE].iloc[-1] - meta.loc[0, MetaKeyName.LATITUDE] = lat - lon = meta[MetaKeyName.LATITUDE].iloc[-1] - meta.loc[0, MetaKeyName.LATITUDE] = lon + lat = meta[SupplyCurveField.LATITUDE].iloc[-1] + meta.loc[0, SupplyCurveField.LATITUDE] = lat + lon = meta[SupplyCurveField.LATITUDE].iloc[-1] + meta.loc[0, SupplyCurveField.LATITUDE] = lon shapes['meta'] = len(meta) for d in dset_names: shapes[d] = (len(res.time_index[t_slice]), len(meta)) diff --git a/tests/test_rep_profiles.py b/tests/test_rep_profiles.py index 372d0914a..fe4b58ec5 100644 --- a/tests/test_rep_profiles.py +++ b/tests/test_rep_profiles.py @@ -17,7 +17,7 @@ RepProfiles, RepresentativeMethods, ) -from reV.utilities import MetaKeyName +from reV.utilities import SupplyCurveField GEN_FPATH = os.path.join(TESTDATADIR, "gen_out/gen_ri_pv_2012_x000.h5") @@ -25,8 +25,8 @@ def test_rep_region_interval(): """Test the rep profile with a weird interval of gids""" sites = np.arange(40) * 2 - rev_summary = pd.DataFrame({MetaKeyName.GEN_GIDS: sites, - MetaKeyName.RES_GIDS: sites}) + rev_summary = pd.DataFrame({SupplyCurveField.GEN_GIDS: sites, + SupplyCurveField.RES_GIDS: sites}) r = RegionRepProfile(GEN_FPATH, rev_summary, weight=None) assert r.i_reps[0] == 14 @@ -34,8 +34,8 @@ def test_rep_region_interval(): def test_rep_methods(): """Test integrated rep methods against baseline rep profile result""" sites = np.arange(100) - rev_summary = pd.DataFrame({MetaKeyName.GEN_GIDS: sites, - MetaKeyName.RES_GIDS: sites}) + rev_summary = pd.DataFrame({SupplyCurveField.GEN_GIDS: sites, + SupplyCurveField.RES_GIDS: sites}) r = RegionRepProfile( GEN_FPATH, @@ -86,8 +86,8 @@ def test_rep_methods(): def test_meanoid(): """Test the simple meanoid method""" sites = np.arange(100) - rev_summary = pd.DataFrame({MetaKeyName.GEN_GIDS: sites, - MetaKeyName.RES_GIDS: sites}) + rev_summary = pd.DataFrame({SupplyCurveField.GEN_GIDS: sites, + SupplyCurveField.RES_GIDS: sites}) r = RegionRepProfile(GEN_FPATH, rev_summary, weight=None) meanoid = RepresentativeMethods.meanoid(r.source_profiles) @@ -102,19 +102,19 @@ def test_weighted_meanoid(): """Test a meanoid weighted by gid_counts vs. a non-weighted meanoid.""" sites = np.arange(100) - rev_summary = pd.DataFrame({MetaKeyName.GEN_GIDS: sites, - MetaKeyName.RES_GIDS: sites, - MetaKeyName.GID_COUNTS: [1] * 50 + [0] * 50}) + rev_summary = pd.DataFrame({SupplyCurveField.GEN_GIDS: sites, + SupplyCurveField.RES_GIDS: sites, + SupplyCurveField.GID_COUNTS: [1] * 50 + [0] * 50}) r = RegionRepProfile(GEN_FPATH, rev_summary) - weights = r._get_region_attr(r._rev_summary, MetaKeyName.GID_COUNTS) + weights = r._get_region_attr(r._rev_summary, SupplyCurveField.GID_COUNTS) w_meanoid = RepresentativeMethods.meanoid( r.source_profiles, weights=weights ) sites = np.arange(50) - rev_summary = pd.DataFrame({MetaKeyName.GEN_GIDS: sites, - MetaKeyName.RES_GIDS: sites}) + rev_summary = pd.DataFrame({SupplyCurveField.GEN_GIDS: sites, + SupplyCurveField.RES_GIDS: sites}) r = RegionRepProfile(GEN_FPATH, rev_summary, weight=None) meanoid = RepresentativeMethods.meanoid(r.source_profiles, weights=None) @@ -130,8 +130,8 @@ def test_integrated(): zeros = np.zeros((100,)) regions = (["r0"] * 7) + (["r1"] * 33) + (["r2"] * 60) timezone = np.random.choice([-4, -5, -6, -7], 100) - rev_summary = pd.DataFrame({MetaKeyName.GEN_GIDS: sites, - MetaKeyName.RES_GIDS: sites, + rev_summary = pd.DataFrame({SupplyCurveField.GEN_GIDS: sites, + SupplyCurveField.RES_GIDS: sites, 'res_class': zeros, 'weight': ones, 'region': regions, @@ -156,12 +156,12 @@ def test_sc_points(): """Test rep profiles for each SC point.""" sites = np.arange(10) timezone = np.random.choice([-4, -5, -6, -7], 10) - rev_summary = pd.DataFrame({MetaKeyName.SC_GID: sites, - MetaKeyName.GEN_GIDS: sites, - MetaKeyName.RES_GIDS: sites, + rev_summary = pd.DataFrame({SupplyCurveField.SC_GID: sites, + SupplyCurveField.GEN_GIDS: sites, + SupplyCurveField.RES_GIDS: sites, 'timezone': timezone}) - rp = RepProfiles(GEN_FPATH, rev_summary, MetaKeyName.SC_GID, weight=None) + rp = RepProfiles(GEN_FPATH, rev_summary, SupplyCurveField.SC_GID, weight=None) rp.run(max_workers=1) with Resource(GEN_FPATH) as res: @@ -180,28 +180,28 @@ def test_agg_profile(): res_gids = [json.dumps(x) for x in res_gids] gid_counts = [json.dumps(x) for x in gid_counts] timezone = np.random.choice([-4, -5, -6, -7], 4) - rev_summary = pd.DataFrame({MetaKeyName.SC_GID: np.arange(4), - MetaKeyName.GEN_GIDS: gen_gids, - MetaKeyName.RES_GIDS: res_gids, - MetaKeyName.GID_COUNTS: gid_counts, - MetaKeyName.TIMEZONE: timezone}) + rev_summary = pd.DataFrame({SupplyCurveField.SC_GID: np.arange(4), + SupplyCurveField.GEN_GIDS: gen_gids, + SupplyCurveField.RES_GIDS: res_gids, + SupplyCurveField.GID_COUNTS: gid_counts, + SupplyCurveField.TIMEZONE: timezone}) - rp = RepProfiles(GEN_FPATH, rev_summary, MetaKeyName.SC_GID, + rp = RepProfiles(GEN_FPATH, rev_summary, SupplyCurveField.SC_GID, cf_dset='cf_profile', err_method=None) rp.run(scaled_precision=False, max_workers=1) for index in rev_summary.index: - gen_gids = json.loads(rev_summary.loc[index, MetaKeyName.GEN_GIDS]) - res_gids = json.loads(rev_summary.loc[index, MetaKeyName.RES_GIDS]) + gen_gids = json.loads(rev_summary.loc[index, SupplyCurveField.GEN_GIDS]) + res_gids = json.loads(rev_summary.loc[index, SupplyCurveField.RES_GIDS]) weights = np.array( - json.loads(rev_summary.loc[index, MetaKeyName.GID_COUNTS])) + json.loads(rev_summary.loc[index, SupplyCurveField.GID_COUNTS])) with Resource(GEN_FPATH) as res: meta = res.meta raw_profiles = [] for gid in res_gids: - iloc = np.where(meta[MetaKeyName.GID] == gid)[0][0] + iloc = np.where(meta[SupplyCurveField.GID] == gid)[0][0] prof = np.expand_dims(res['cf_profile', :, iloc], 1) raw_profiles.append(prof) @@ -218,8 +218,8 @@ def test_agg_profile(): assert np.allclose(rp.profiles[0][:, index], truth) - passthrough_cols = [MetaKeyName.GEN_GIDS, MetaKeyName.RES_GIDS, - MetaKeyName.GID_COUNTS] + passthrough_cols = [SupplyCurveField.GEN_GIDS, SupplyCurveField.RES_GIDS, + SupplyCurveField.GID_COUNTS] for col in passthrough_cols: assert col in rp.meta @@ -236,8 +236,8 @@ def test_many_regions(use_weights): region1 = (["r0"] * 7) + (["r1"] * 33) + (["r2"] * 60) region2 = (["a0"] * 20) + (["b1"] * 10) + (["c2"] * 20) + (["d3"] * 50) timezone = np.random.choice([-4, -5, -6, -7], 100) - rev_summary = pd.DataFrame({MetaKeyName.GEN_GIDS: sites, - MetaKeyName.RES_GIDS: sites, + rev_summary = pd.DataFrame({SupplyCurveField.GEN_GIDS: sites, + SupplyCurveField.RES_GIDS: sites, 'res_class': zeros, 'region1': region1, 'region2': region2, @@ -274,13 +274,13 @@ def test_many_regions_with_list_weights(): region1 = (["r0"] * 7) + (["r1"] * 33) + (["r2"] * 60) region2 = (["a0"] * 20) + (["b1"] * 10) + (["c2"] * 20) + (["d3"] * 50) timezone = np.random.choice([-4, -5, -6, -7], 100) - rev_summary = pd.DataFrame({MetaKeyName.GEN_GIDS: sites, - MetaKeyName.RES_GIDS: sites, + rev_summary = pd.DataFrame({SupplyCurveField.GEN_GIDS: sites, + SupplyCurveField.RES_GIDS: sites, 'res_class': zeros, 'region1': region1, 'region2': region2, 'weights': weights, - MetaKeyName.TIMEZONE: timezone}) + SupplyCurveField.TIMEZONE: timezone}) reg_cols = ['region1', 'region2'] rp = RepProfiles(GEN_FPATH, rev_summary, reg_cols, weight='weights') rp.run() @@ -302,11 +302,11 @@ def test_write_to_file(): zeros = np.zeros((100,)) regions = (["r0"] * 7) + (["r1"] * 33) + (["r2"] * 60) timezone = np.random.choice([-4, -5, -6, -7], 100) - rev_summary = pd.DataFrame({MetaKeyName.GEN_GIDS: sites, - MetaKeyName.RES_GIDS: sites, + rev_summary = pd.DataFrame({SupplyCurveField.GEN_GIDS: sites, + SupplyCurveField.RES_GIDS: sites, 'res_class': zeros, 'region': regions, - MetaKeyName.TIMEZONE: timezone}) + SupplyCurveField.TIMEZONE: timezone}) fout = os.path.join(td, 'temp_rep_profiles.h5') rp = RepProfiles(GEN_FPATH, rev_summary, 'region', n_profiles=3, weight=None) @@ -334,11 +334,11 @@ def test_file_options(): zeros = np.zeros((100,)) regions = (["r0"] * 7) + (["r1"] * 33) + (["r2"] * 60) timezone = np.random.choice([-4, -5, -6, -7], 100) - rev_summary = pd.DataFrame({MetaKeyName.GEN_GIDS: sites, - MetaKeyName.RES_GIDS: sites, + rev_summary = pd.DataFrame({SupplyCurveField.GEN_GIDS: sites, + SupplyCurveField.RES_GIDS: sites, 'res_class': zeros, 'region': regions, - MetaKeyName.TIMEZONE: timezone}) + SupplyCurveField.TIMEZONE: timezone}) fout = os.path.join(td, 'temp_rep_profiles.h5') rp = RepProfiles(GEN_FPATH, rev_summary, 'region', n_profiles=3, weight=None) diff --git a/tests/test_supply_curve_aggregation.py b/tests/test_supply_curve_aggregation.py index 7771cbeb5..72687c5b2 100644 --- a/tests/test_supply_curve_aggregation.py +++ b/tests/test_supply_curve_aggregation.py @@ -12,7 +12,7 @@ from reV import TESTDATADIR from reV.supply_curve.aggregation import Aggregation -from reV.utilities import MetaKeyName +from reV.utilities import SupplyCurveField EXCL = os.path.join(TESTDATADIR, 'ri_exclusions/ri_exclusions.h5') RES = os.path.join(TESTDATADIR, 'nsrdb/ri_100_nsrdb_2012.h5') @@ -47,7 +47,7 @@ def check_agg(agg_out, baseline_h5): truth = f[dset] if dset == 'meta': truth = truth.set_index('sc_gid') - for c in [MetaKeyName.SOURCE_GIDS, MetaKeyName.GID_COUNTS]: + for c in [SupplyCurveField.SOURCE_GIDS, SupplyCurveField.GID_COUNTS]: test[c] = test[c].astype(str) truth = truth.fillna('none') @@ -100,9 +100,9 @@ def test_gid_counts(excl_dict): excl_dict=excl_dict, max_workers=1) for i, row in agg_out['meta'].iterrows(): - n_gids = row[MetaKeyName.N_GIDS] - gid_counts = np.sum(row[MetaKeyName.GID_COUNTS]) - area = row[MetaKeyName.AREA_SQ_KM] + n_gids = row[SupplyCurveField.N_GIDS] + gid_counts = np.sum(row[SupplyCurveField.GID_COUNTS]) + area = row[SupplyCurveField.AREA_SQ_KM] msg = ('For sc_gid {}: the sum of gid_counts ({}), does not match ' 'n_gids ({})'.format(i, n_gids, gid_counts)) @@ -142,8 +142,8 @@ def test_mean_wind_dirs(excl_dict): for i, row in out_meta.iterrows(): test = mean_wind_dirs[:, i] - gids = row[MetaKeyName.SOURCE_GIDS] - fracs = row[MetaKeyName.GID_COUNTS] / row[MetaKeyName.N_GIDS] + gids = row[SupplyCurveField.SOURCE_GIDS] + fracs = row[SupplyCurveField.GID_COUNTS] / row[SupplyCurveField.N_GIDS] truth = compute_mean_wind_dirs(RES, DSET, gids, fracs) diff --git a/tests/test_supply_curve_aggregation_friction.py b/tests/test_supply_curve_aggregation_friction.py index a270899d6..0298e1b5a 100644 --- a/tests/test_supply_curve_aggregation_friction.py +++ b/tests/test_supply_curve_aggregation_friction.py @@ -16,7 +16,7 @@ from reV.supply_curve.exclusions import ExclusionMaskFromDict, FrictionMask from reV.supply_curve.extent import SupplyCurveExtent from reV.supply_curve.sc_aggregation import SupplyCurveAggregation -from reV.utilities import MetaKeyName +from reV.utilities import SupplyCurveField EXCL_FPATH = os.path.join(TESTDATADIR, "ri_exclusions/ri_exclusions.h5") FRICTION_FPATH = os.path.join(TESTDATADIR, "ri_exclusions/ri_friction.h5") @@ -64,7 +64,7 @@ def test_friction_mask(): assert diff < 0.0001, m -@pytest.mark.parametrize(MetaKeyName.GID, [100, 114, 130, 181]) +@pytest.mark.parametrize(SupplyCurveField.GID, [100, 114, 130, 181]) def test_agg_friction(gid): """Test SC Aggregation with friction by checking friction factors and LCOE against a hand calc.""" @@ -99,16 +99,16 @@ def test_agg_friction(gid): gid ) assert np.isclose( - s[MetaKeyName.MEAN_FRICTION].values[0], mean_friction + s[SupplyCurveField.MEAN_FRICTION].values[0], mean_friction ), m m = ("SC point gid {} does not match mean LCOE with friction hand calc" .format(gid)) - assert np.isclose(s[MetaKeyName.MEAN_FRICTION].values[0], + assert np.isclose(s[SupplyCurveField.MEAN_FRICTION].values[0], mean_friction), m m = ('SC point gid {} does not match mean LCOE with friction hand calc' .format(gid)) - assert np.allclose(s[MetaKeyName.MEAN_LCOE_FRICTION], - s[MetaKeyName.MEAN_LCOE] * mean_friction), m + assert np.allclose(s[SupplyCurveField.MEAN_LCOE_FRICTION], + s[SupplyCurveField.MEAN_LCOE] * mean_friction), m # pylint: disable=no-member @@ -134,8 +134,8 @@ def make_friction_file(): f[FRICTION_DSET][...] = data for d in f: - if d not in [FRICTION_DSET, MetaKeyName.LATITUDE, - MetaKeyName.LONGITUDE]: + if d not in [FRICTION_DSET, SupplyCurveField.LATITUDE, + SupplyCurveField.LONGITUDE]: del f[d] with h5py.File(FRICTION_FPATH, "r") as f: diff --git a/tests/test_supply_curve_compute.py b/tests/test_supply_curve_compute.py index 483d78e39..c63a28ad7 100644 --- a/tests/test_supply_curve_compute.py +++ b/tests/test_supply_curve_compute.py @@ -14,7 +14,7 @@ from reV import TESTDATADIR from reV.supply_curve.supply_curve import SupplyCurve -from reV.utilities import MetaKeyName +from reV.utilities import SupplyCurveField from reV.utilities.exceptions import SupplyCurveInputError TRANS_COSTS_1 = { @@ -155,13 +155,13 @@ def test_integrated_sc_full_friction(): transmission_costs=tcosts, avail_cap_frac=avail_cap_frac, columns=SC_FULL_COLUMNS, - sort_on=MetaKeyName.TOTAL_LCOE_FRICTION) + sort_on=SupplyCurveField.TOTAL_LCOE_FRICTION) sc_full = pd.read_csv(sc_full) - assert MetaKeyName.MEAN_LCOE_FRICTION in sc_full - assert MetaKeyName.TOTAL_LCOE_FRICTION in sc_full - test = sc_full[MetaKeyName.MEAN_LCOE_FRICTION] + sc_full['lcot'] - assert np.allclose(test, sc_full[MetaKeyName.TOTAL_LCOE_FRICTION]) + assert SupplyCurveField.MEAN_LCOE_FRICTION in sc_full + assert SupplyCurveField.TOTAL_LCOE_FRICTION in sc_full + test = sc_full[SupplyCurveField.MEAN_LCOE_FRICTION] + sc_full['lcot'] + assert np.allclose(test, sc_full[SupplyCurveField.TOTAL_LCOE_FRICTION]) fpath_baseline = os.path.join( TESTDATADIR, "sc_out/sc_full_out_friction.csv" @@ -178,12 +178,12 @@ def test_integrated_sc_simple_friction(): out_fpath = os.path.join(td, "sc") sc_simple = sc.run(out_fpath, fixed_charge_rate=0.1, simple=True, transmission_costs=tcosts, - sort_on=MetaKeyName.TOTAL_LCOE_FRICTION) + sort_on=SupplyCurveField.TOTAL_LCOE_FRICTION) sc_simple = pd.read_csv(sc_simple) - assert MetaKeyName.MEAN_LCOE_FRICTION in sc_simple - assert MetaKeyName.TOTAL_LCOE_FRICTION in sc_simple - test = sc_simple[MetaKeyName.MEAN_LCOE_FRICTION] + sc_simple['lcot'] - assert np.allclose(test, sc_simple[MetaKeyName.TOTAL_LCOE_FRICTION]) + assert SupplyCurveField.MEAN_LCOE_FRICTION in sc_simple + assert SupplyCurveField.TOTAL_LCOE_FRICTION in sc_simple + test = sc_simple[SupplyCurveField.MEAN_LCOE_FRICTION] + sc_simple['lcot'] + assert np.allclose(test, sc_simple[SupplyCurveField.TOTAL_LCOE_FRICTION]) fpath_baseline = os.path.join( TESTDATADIR, "sc_out/sc_simple_out_friction.csv" @@ -193,7 +193,7 @@ def test_integrated_sc_simple_friction(): def test_sc_warning1(): """Run the full SC test with missing connections and verify warning.""" - mask = TRANS_TABLE[MetaKeyName.SC_POINT_GID].isin(list(range(10))) + mask = TRANS_TABLE[SupplyCurveField.SC_POINT_GID].isin(list(range(10))) trans_table = TRANS_TABLE[~mask] tcosts = TRANS_COSTS_1.copy() avail_cap_frac = tcosts.pop("available_capacity", 1) @@ -279,7 +279,7 @@ def test_parallel(): assert_frame_equal(sc_full_parallel, sc_full_serial) -def verify_trans_cap(sc_table, trans_tables, cap_col=MetaKeyName.CAPACITY): +def verify_trans_cap(sc_table, trans_tables, cap_col=SupplyCurveField.CAPACITY): """ Verify that sc_points are connected to features in the correct capacity bins @@ -300,7 +300,7 @@ def verify_trans_cap(sc_table, trans_tables, cap_col=MetaKeyName.CAPACITY): test = sc_table.merge(trans_features, on='trans_gid', how='left') mask = test[cap_col] > test['max_cap'] - cols = [MetaKeyName.SC_GID, 'trans_gid', cap_col, 'max_cap'] + cols = [SupplyCurveField.SC_GID, 'trans_gid', cap_col, 'max_cap'] msg = ("SC points connected to transmission features with " "max_cap < sc_cap:\n{}" .format(test.loc[mask, cols])) @@ -425,33 +425,33 @@ def test_multi_parallel_trans(): sc = SupplyCurve(SC_POINTS, trans_tables) sc_2 = sc.simple_sort(fcr=0.1, columns=columns) - assert not (set(SC_POINTS[MetaKeyName.SC_GID]) - - set(sc_1[MetaKeyName.SC_GID])) - assert not (set(SC_POINTS[MetaKeyName.SC_GID]) - - set(sc_2[MetaKeyName.SC_GID])) - assert not (set(SC_POINTS[MetaKeyName.SC_POINT_GID]) - - set(sc_1[MetaKeyName.SC_POINT_GID])) - assert not (set(SC_POINTS[MetaKeyName.SC_POINT_GID]) - - set(sc_2[MetaKeyName.SC_POINT_GID])) - assert not (set(sc_1[MetaKeyName.SC_POINT_GID]) - - set(SC_POINTS[MetaKeyName.SC_POINT_GID])) - assert not (set(sc_2[MetaKeyName.SC_POINT_GID]) - - set(SC_POINTS[MetaKeyName.SC_POINT_GID])) + assert not (set(SC_POINTS[SupplyCurveField.SC_GID]) + - set(sc_1[SupplyCurveField.SC_GID])) + assert not (set(SC_POINTS[SupplyCurveField.SC_GID]) + - set(sc_2[SupplyCurveField.SC_GID])) + assert not (set(SC_POINTS[SupplyCurveField.SC_POINT_GID]) + - set(sc_1[SupplyCurveField.SC_POINT_GID])) + assert not (set(SC_POINTS[SupplyCurveField.SC_POINT_GID]) + - set(sc_2[SupplyCurveField.SC_POINT_GID])) + assert not (set(sc_1[SupplyCurveField.SC_POINT_GID]) + - set(SC_POINTS[SupplyCurveField.SC_POINT_GID])) + assert not (set(sc_2[SupplyCurveField.SC_POINT_GID]) + - set(SC_POINTS[SupplyCurveField.SC_POINT_GID])) assert (sc_2.n_parallel_trans > 1).any() mask_2 = sc_2["n_parallel_trans"] > 1 - for gid in sc_2.loc[mask_2, MetaKeyName.SC_GID]: - nx_1 = sc_1.loc[(sc_1[MetaKeyName.SC_GID] == gid), + for gid in sc_2.loc[mask_2, SupplyCurveField.SC_GID]: + nx_1 = sc_1.loc[(sc_1[SupplyCurveField.SC_GID] == gid), 'n_parallel_trans'].values[0] - nx_2 = sc_2.loc[(sc_2[MetaKeyName.SC_GID] == gid), + nx_2 = sc_2.loc[(sc_2[SupplyCurveField.SC_GID] == gid), 'n_parallel_trans'].values[0] assert nx_2 >= nx_1 if nx_1 != nx_2: - lcot_1 = sc_1.loc[(sc_1[MetaKeyName.SC_GID] == gid), + lcot_1 = sc_1.loc[(sc_1[SupplyCurveField.SC_GID] == gid), 'lcot'].values[0] - lcot_2 = sc_2.loc[(sc_2[MetaKeyName.SC_GID] == gid), + lcot_2 = sc_2.loc[(sc_2[SupplyCurveField.SC_GID] == gid), 'lcot'].values[0] assert lcot_2 > lcot_1 @@ -578,8 +578,8 @@ def test_least_cost_full_pass_through(): Test the full supply curve sorting passes through variables correctly """ check_cols = {'poi_lat', 'poi_lon', 'reinforcement_poi_lat', - 'reinforcement_poi_lon', MetaKeyName.EOS_MULT, - MetaKeyName.REG_MULT, + 'reinforcement_poi_lon', SupplyCurveField.EOS_MULT, + SupplyCurveField.REG_MULT, 'reinforcement_cost_per_mw', 'reinforcement_dist_km'} with tempfile.TemporaryDirectory() as td: trans_tables = [] @@ -616,8 +616,8 @@ def test_least_cost_simple_pass_through(): Test the simple supply curve sorting passes through variables correctly """ check_cols = {'poi_lat', 'poi_lon', 'reinforcement_poi_lat', - 'reinforcement_poi_lon', MetaKeyName.EOS_MULT, - MetaKeyName.REG_MULT, + 'reinforcement_poi_lon', SupplyCurveField.EOS_MULT, + SupplyCurveField.REG_MULT, 'reinforcement_cost_per_mw', 'reinforcement_dist_km'} with tempfile.TemporaryDirectory() as td: trans_tables = [] diff --git a/tests/test_supply_curve_points.py b/tests/test_supply_curve_points.py index 6390e63cc..727856f43 100644 --- a/tests/test_supply_curve_points.py +++ b/tests/test_supply_curve_points.py @@ -19,7 +19,7 @@ SupplyCurvePoint, ) from reV.supply_curve.sc_aggregation import SupplyCurveAggregation -from reV.utilities import MetaKeyName +from reV.utilities import SupplyCurveField F_EXCL = os.path.join(TESTDATADIR, "ri_exclusions/ri_exclusions.h5") F_GEN = os.path.join(TESTDATADIR, "gen_out/gen_ri_pv_2012_x000.h5") @@ -68,7 +68,7 @@ def test_slicer(gids, resolution): @pytest.mark.parametrize( - (MetaKeyName.GID, "resolution", "excl_dict", "time_series"), + (SupplyCurveField.GID, "resolution", "excl_dict", "time_series"), [ (37, 64, None, None), (37, 64, EXCL_DICT, None), @@ -109,7 +109,7 @@ def test_weighted_means(gid, resolution, excl_dict, time_series): @pytest.mark.parametrize( - (MetaKeyName.GID, "resolution", "excl_dict", "time_series"), + (SupplyCurveField.GID, "resolution", "excl_dict", "time_series"), [ (37, 64, None, None), (37, 64, EXCL_DICT, None), @@ -164,16 +164,16 @@ def plot_all_sc_points(resolution=64): for gid in range(len(sc)): excl_meta = sc.get_excl_points("meta", gid) axs.scatter( - excl_meta[MetaKeyName.LONGITUDE], - excl_meta[MetaKeyName.LATITUDE], + excl_meta[SupplyCurveField.LONGITUDE], + excl_meta[SupplyCurveField.LATITUDE], c=colors[gid], s=0.01, ) with Outputs(F_GEN) as f: axs.scatter( - f.meta[MetaKeyName.LONGITUDE], - f.meta[MetaKeyName.LATITUDE], + f.meta[SupplyCurveField.LONGITUDE], + f.meta[SupplyCurveField.LATITUDE], c="k", s=25, ) @@ -208,16 +208,16 @@ def plot_single_gen_sc_point(gid=2, resolution=64): if gen_gid != -1: mask = sc._gen_gids == gen_gid axs.scatter( - excl_meta.loc[mask, MetaKeyName.LONGITUDE], - excl_meta.loc[mask, MetaKeyName.LATITUDE], + excl_meta.loc[mask, SupplyCurveField.LONGITUDE], + excl_meta.loc[mask, SupplyCurveField.LATITUDE], marker="s", c=colors[i], s=1, ) axs.scatter( - sc.gen.meta.loc[gen_gid, MetaKeyName.LONGITUDE], - sc.gen.meta.loc[gen_gid, MetaKeyName.LATITUDE], + sc.gen.meta.loc[gen_gid, SupplyCurveField.LONGITUDE], + sc.gen.meta.loc[gen_gid, SupplyCurveField.LATITUDE], c="k", s=100, ) diff --git a/tests/test_supply_curve_sc_aggregation.py b/tests/test_supply_curve_sc_aggregation.py index 3b47a9579..6ccbf5821 100644 --- a/tests/test_supply_curve_sc_aggregation.py +++ b/tests/test_supply_curve_sc_aggregation.py @@ -26,7 +26,7 @@ SupplyCurveAggregation, _warn_about_large_datasets, ) -from reV.utilities import MetaKeyName, ModuleName +from reV.utilities import ModuleName, SupplyCurveField EXCL = os.path.join(TESTDATADIR, 'ri_exclusions/ri_exclusions.h5') RES = os.path.join(TESTDATADIR, 'nsrdb/ri_100_nsrdb_2012.h5') @@ -67,12 +67,12 @@ def test_agg_extent(resolution=64): summary = sca.summarize(GEN) all_res_gids = [] - for gids in summary[MetaKeyName.RES_GIDS]: + for gids in summary[SupplyCurveField.RES_GIDS]: all_res_gids += gids - assert MetaKeyName.SC_COL_IND in summary - assert MetaKeyName.SC_ROW_IND in summary - assert MetaKeyName.GEN_GIDS in summary + assert SupplyCurveField.SC_COL_IND in summary + assert SupplyCurveField.SC_ROW_IND in summary + assert SupplyCurveField.GEN_GIDS in summary assert len(set(all_res_gids)) == 177 @@ -120,8 +120,8 @@ def test_agg_summary(): ) else: - for c in [MetaKeyName.RES_GIDS, MetaKeyName.GEN_GIDS, - MetaKeyName.GID_COUNTS]: + for c in [SupplyCurveField.RES_GIDS, SupplyCurveField.GEN_GIDS, + SupplyCurveField.GID_COUNTS]: summary[c] = summary[c].astype(str) s_baseline = pd.read_csv(AGG_BASELINE, index_col=0) @@ -183,7 +183,7 @@ def test_multi_file_excl(): shutil.copy(EXCL, excl_temp_2) with h5py.File(excl_temp_1, 'a') as f: - shape = f[MetaKeyName.LATITUDE].shape + shape = f[SupplyCurveField.LATITUDE].shape attrs = dict(f['ri_srtm_slope'].attrs) data = np.ones(shape) test_dset = "excl_test" @@ -206,8 +206,8 @@ def test_multi_file_excl(): summary = summary.fillna("None") s_baseline = s_baseline.fillna("None") - assert np.allclose(summary[MetaKeyName.AREA_SQ_KM] * 2, - s_baseline[MetaKeyName.AREA_SQ_KM]) + assert np.allclose(summary[SupplyCurveField.AREA_SQ_KM] * 2, + s_baseline[SupplyCurveField.AREA_SQ_KM]) @pytest.mark.parametrize("pre_extract", (True, False)) @@ -233,8 +233,8 @@ def test_pre_extract_inclusions(pre_extract): ) else: - for c in [MetaKeyName.RES_GIDS, MetaKeyName.GEN_GIDS, - MetaKeyName.GID_COUNTS]: + for c in [SupplyCurveField.RES_GIDS, SupplyCurveField.GEN_GIDS, + SupplyCurveField.GID_COUNTS]: summary[c] = summary[c].astype(str) s_baseline = pd.read_csv(AGG_BASELINE, index_col=0) @@ -292,13 +292,13 @@ def test_agg_extra_dsets(): for dset in h5_dsets: assert "mean_{}".format(dset) in summary.columns - check = summary['mean_lcoe_fcr-2012'] == summary[MetaKeyName.MEAN_LCOE] + check = summary['mean_lcoe_fcr-2012'] == summary[SupplyCurveField.MEAN_LCOE] assert not any(check) - check = summary['mean_lcoe_fcr-2013'] == summary[MetaKeyName.MEAN_LCOE] + check = summary['mean_lcoe_fcr-2013'] == summary[SupplyCurveField.MEAN_LCOE] assert not any(check) avg = (summary['mean_lcoe_fcr-2012'] + summary['mean_lcoe_fcr-2013']) / 2 - assert np.allclose(avg.values, summary[MetaKeyName.MEAN_LCOE].values) + assert np.allclose(avg.values, summary[SupplyCurveField.MEAN_LCOE].values) def test_agg_extra_2D_dsets(): @@ -351,7 +351,7 @@ def test_agg_scalar_excl(): ) summary_with_weights = sca.summarize(GEN, max_workers=1) - dsets = [MetaKeyName.AREA_SQ_KM, MetaKeyName.CAPACITY] + dsets = [SupplyCurveField.AREA_SQ_KM, SupplyCurveField.CAPACITY] for dset in dsets: diff = summary_base[dset].values / summary_with_weights[dset].values msg = ("Fractional exclusions failed for {} which has values {} and {}" @@ -360,8 +360,8 @@ def test_agg_scalar_excl(): assert all(diff == 2), msg for i in summary_base.index: - counts_full = summary_base.loc[i, MetaKeyName.GID_COUNTS] - counts_half = summary_with_weights.loc[i, MetaKeyName.GID_COUNTS] + counts_full = summary_base.loc[i, SupplyCurveField.GID_COUNTS] + counts_half = summary_with_weights.loc[i, SupplyCurveField.GID_COUNTS] for j, counts in enumerate(counts_full): msg = ("GID counts for fractional exclusions failed for index {}!" @@ -391,7 +391,7 @@ def test_data_layer_methods(): for i in summary.index.values: # Check categorical data layers - counts = summary.loc[i, MetaKeyName.GID_COUNTS] + counts = summary.loc[i, SupplyCurveField.GID_COUNTS] rr = summary.loc[i, 'reeds_region'] assert isinstance(rr, str) rr = json.loads(rr) @@ -412,7 +412,7 @@ def test_data_layer_methods(): raise RuntimeError(e) # Check min/mean/max of the same data layer - n = summary.loc[i, MetaKeyName.N_GIDS] + n = summary.loc[i, SupplyCurveField.N_GIDS] slope_mean = summary.loc[i, 'pct_slope_mean'] slope_max = summary.loc[i, 'pct_slope_max'] slope_min = summary.loc[i, 'pct_slope_min'] @@ -429,10 +429,10 @@ def test_recalc_lcoe(cap_cost_scale): """Test supply curve aggregation with the re-calculation of lcoe using the multi-year mean capacity factor""" - data = {MetaKeyName.CAPITAL_COST: 34900000, - MetaKeyName.FIXED_OPERATING_COST: 280000, - MetaKeyName.FIXED_CHARGE_RATE: 0.09606382995843887, - MetaKeyName.VARIABLE_OPERATING_COST: 0, + data = {SupplyCurveField.CAPITAL_COST: 34900000, + SupplyCurveField.FIXED_OPERATING_COST: 280000, + SupplyCurveField.FIXED_CHARGE_RATE: 0.09606382995843887, + SupplyCurveField.VARIABLE_OPERATING_COST: 0, 'system_capacity': 20000} annual_cf = [0.24, 0.26, 0.37, 0.15] annual_lcoe = [] @@ -449,11 +449,11 @@ def test_recalc_lcoe(cap_cost_scale): arr = np.full(res["meta"].shape, v) res.create_dataset(k, res["meta"].shape, data=arr) for year, cf in zip(years, annual_cf): - lcoe = lcoe_fcr(data[MetaKeyName.FIXED_CHARGE_RATE], - data[MetaKeyName.CAPITAL_COST], - data[MetaKeyName.FIXED_OPERATING_COST], + lcoe = lcoe_fcr(data[SupplyCurveField.FIXED_CHARGE_RATE], + data[SupplyCurveField.CAPITAL_COST], + data[SupplyCurveField.FIXED_OPERATING_COST], data['system_capacity'] * cf * 8760, - data[MetaKeyName.VARIABLE_OPERATING_COST]) + data[SupplyCurveField.VARIABLE_OPERATING_COST]) cf_arr = np.full(res['meta'].shape, cf) lcoe_arr = np.full(res['meta'].shape, lcoe) annual_lcoe.append(lcoe) @@ -474,10 +474,10 @@ def test_recalc_lcoe(cap_cost_scale): "lcoe_fcr-means", res["meta"].shape, data=lcoe_arr ) - h5_dsets = [MetaKeyName.CAPITAL_COST, - MetaKeyName.FIXED_OPERATING_COST, - MetaKeyName.FIXED_CHARGE_RATE, - MetaKeyName.VARIABLE_OPERATING_COST, + h5_dsets = [SupplyCurveField.CAPITAL_COST, + SupplyCurveField.FIXED_OPERATING_COST, + SupplyCurveField.FIXED_CHARGE_RATE, + SupplyCurveField.VARIABLE_OPERATING_COST, 'system_capacity'] base = SupplyCurveAggregation( @@ -508,19 +508,19 @@ def test_recalc_lcoe(cap_cost_scale): ) summary = sca.summarize(gen_temp, max_workers=1) - assert not np.allclose(summary_base[MetaKeyName.MEAN_LCOE], - summary[MetaKeyName.MEAN_LCOE]) + assert not np.allclose(summary_base[SupplyCurveField.MEAN_LCOE], + summary[SupplyCurveField.MEAN_LCOE]) if cap_cost_scale == '1': - cc_dset = MetaKeyName.SC_POINT_CAPITAL_COST + cc_dset = SupplyCurveField.SC_POINT_CAPITAL_COST else: - cc_dset = MetaKeyName.SCALED_SC_POINT_CAPITAL_COST + cc_dset = SupplyCurveField.SCALED_SC_POINT_CAPITAL_COST lcoe = lcoe_fcr(summary['mean_fixed_charge_rate'], summary[cc_dset], - summary[MetaKeyName.SC_POINT_FIXED_OPERATING_COST], - summary[MetaKeyName.SC_POINT_ANNUAL_ENERGY], + summary[SupplyCurveField.SC_POINT_FIXED_OPERATING_COST], + summary[SupplyCurveField.SC_POINT_ANNUAL_ENERGY], summary['mean_variable_operating_cost']) - assert np.allclose(lcoe, summary[MetaKeyName.MEAN_LCOE]) + assert np.allclose(lcoe, summary[SupplyCurveField.MEAN_LCOE]) @pytest.mark.parametrize("tm_dset", ("techmap_ri", "techmap_ri_new")) diff --git a/tests/test_supply_curve_vpd.py b/tests/test_supply_curve_vpd.py index 43a018fb2..b0a35326c 100644 --- a/tests/test_supply_curve_vpd.py +++ b/tests/test_supply_curve_vpd.py @@ -11,7 +11,7 @@ from reV import TESTDATADIR from reV.supply_curve.sc_aggregation import SupplyCurveAggregation -from reV.utilities import MetaKeyName +from reV.utilities import SupplyCurveField from reV.utilities.exceptions import FileInputError EXCL = os.path.join(TESTDATADIR, 'ri_exclusions/ri_exclusions.h5') @@ -43,10 +43,10 @@ def test_vpd(): vpd = pd.read_csv(FVPD, index_col=0) for i in summary.index: - capacity = summary.loc[i, MetaKeyName.CAPACITY] - area = summary.loc[i, MetaKeyName.AREA_SQ_KM] + capacity = summary.loc[i, SupplyCurveField.CAPACITY] + area = summary.loc[i, SupplyCurveField.AREA_SQ_KM] res_gids = np.array(summary.loc[i, 'res_gids']) - gid_counts = np.array(summary.loc[i, MetaKeyName.GID_COUNTS]) + gid_counts = np.array(summary.loc[i, SupplyCurveField.GID_COUNTS]) vpd_per_gid = vpd.loc[res_gids, 'power_density'].values truth = area * (vpd_per_gid * gid_counts).sum() / gid_counts.sum() @@ -79,8 +79,8 @@ def test_vpd_fractional_excl(): summary_2 = sca_2.summarize(GEN, max_workers=1) for i in summary_1.index: - cap_full = summary_1.loc[i, MetaKeyName.CAPACITY] - cap_half = summary_2.loc[i, MetaKeyName.CAPACITY] + cap_full = summary_1.loc[i, SupplyCurveField.CAPACITY] + cap_half = summary_2.loc[i, SupplyCurveField.CAPACITY] msg = ('Variable power density for fractional exclusions failed! ' 'Index {} has cap full {} and cap half {}' diff --git a/tests/test_supply_curve_wind_dirs.py b/tests/test_supply_curve_wind_dirs.py index 33962344c..badfabe46 100644 --- a/tests/test_supply_curve_wind_dirs.py +++ b/tests/test_supply_curve_wind_dirs.py @@ -10,7 +10,7 @@ from reV import TESTDATADIR from reV.supply_curve.supply_curve import CompetitiveWindFarms, SupplyCurve -from reV.utilities import MetaKeyName +from reV.utilities import SupplyCurveField TRANS_COSTS = {'line_tie_in_cost': 200, 'line_cost': 1000, 'station_tie_in_cost': 50, 'center_tie_in_cost': 10, @@ -31,7 +31,7 @@ def test_competitive_wind_dirs(downwind): sc_points = CompetitiveWindFarms.run(WIND_DIRS, SC_POINTS, n_dirs=2, - sort_on=MetaKeyName.MEAN_LCOE, + sort_on=SupplyCurveField.MEAN_LCOE, downwind=downwind) if downwind: @@ -108,11 +108,11 @@ def test_upwind_exclusion(): 'sc_full_upwind.csv') sc_out = pd.read_csv(sc_out).sort_values('total_lcoe') - sc_point_gids = sc_out[MetaKeyName.SC_POINT_GID].values.tolist() + sc_point_gids = sc_out[SupplyCurveField.SC_POINT_GID].values.tolist() for _, row in sc_out.iterrows(): - sc_gid = row[MetaKeyName.SC_GID] - sc_point_gids.remove(row[MetaKeyName.SC_POINT_GID]) - sc_point_gid = cwf[MetaKeyName.SC_POINT_GID, sc_gid] + sc_gid = row[SupplyCurveField.SC_GID] + sc_point_gids.remove(row[SupplyCurveField.SC_POINT_GID]) + sc_point_gid = cwf[SupplyCurveField.SC_POINT_GID, sc_gid] for gid in cwf['upwind', sc_point_gid]: msg = 'Upwind gid {} was not excluded!'.format(gid) assert gid not in sc_point_gids, msg @@ -128,11 +128,11 @@ def test_upwind_downwind_exclusion(): 'sc_full_downwind.csv') sc_out = pd.read_csv(sc_out).sort_values('total_lcoe') - sc_point_gids = sc_out[MetaKeyName.SC_POINT_GID].values.tolist() + sc_point_gids = sc_out[SupplyCurveField.SC_POINT_GID].values.tolist() for _, row in sc_out.iterrows(): - sc_gid = row[MetaKeyName.SC_GID] - sc_point_gids.remove(row[MetaKeyName.SC_POINT_GID]) - sc_point_gid = cwf[MetaKeyName.SC_POINT_GID, sc_gid] + sc_gid = row[SupplyCurveField.SC_GID] + sc_point_gids.remove(row[SupplyCurveField.SC_POINT_GID]) + sc_point_gid = cwf[SupplyCurveField.SC_POINT_GID, sc_gid] for gid in cwf['upwind', sc_point_gid]: msg = 'Upwind gid {} was not excluded!'.format(gid) assert gid not in sc_point_gids, msg From 49da781be3f2db641445cf412892a9f508c9a850 Mon Sep 17 00:00:00 2001 From: "Brandon N. Benton" Date: Mon, 27 May 2024 20:39:55 -0600 Subject: [PATCH 31/61] some enum class methods for mapping between sc fields, res fields, and site fields. Dictionary of "old" names used to rename test data to match current sc names. bespoke tests mostly passing. --- reV/SAM/SAM.py | 17 +- reV/SAM/generation.py | 241 +++++++++++++----------- reV/bespoke/bespoke.py | 6 +- reV/config/project_points.py | 42 +++-- reV/supply_curve/points.py | 175 ++++++++++------- reV/utilities/__init__.py | 224 ++++++++++++++++------ tests/test_bespoke.py | 352 +++++++++++++++++++++++------------ 7 files changed, 669 insertions(+), 388 deletions(-) diff --git a/reV/SAM/SAM.py b/reV/SAM/SAM.py index 7496538e0..06c9bd78d 100644 --- a/reV/SAM/SAM.py +++ b/reV/SAM/SAM.py @@ -28,6 +28,7 @@ ) from rex.utilities.utilities import check_res_file +from reV.utilities import ResourceMetaField from reV.utilities.exceptions import ( ResourceError, SAMExecutionError, @@ -713,7 +714,7 @@ def ensure_res_len(arr, time_index): freq = pd.infer_freq(time_index[:s]) msg = "frequencies do not match before and after 2/29" - assert freq == pd.infer_freq(time_index[s + 1:]), msg + assert freq == pd.infer_freq(time_index[s + 1 :]), msg else: freq = pd.infer_freq(time_index) else: @@ -796,11 +797,10 @@ def _parse_meta(meta): location. Should include values for latitude, longitude, elevation, and timezone. Can be None for econ runs. """ - if isinstance(meta, pd.DataFrame): msg = ( - "Meta data must only be for a single site but received: {}" - .format(meta) + "Meta data must only be for a single site but received: " + f"{meta}" ) assert len(meta) == 1, msg meta = meta.iloc[0] @@ -861,7 +861,9 @@ def outputs_to_utc_arr(self): if self._is_hourly(output): n_roll = int( - -1 * self.meta["timezone"] * self.time_interval + -1 + * self.meta[ResourceMetaField.TIMEZONE] + * self.time_interval ) output = np.roll(output, n_roll) @@ -889,8 +891,9 @@ def collect_outputs(self, output_lookup): bad_requests.append(req) if any(bad_requests): - msg = ('Could not retrieve outputs "{}" from PySAM object "{}".' - .format(bad_requests, self.pysam)) + msg = 'Could not retrieve outputs "{}" from PySAM object "{}".'.format( + bad_requests, self.pysam + ) logger.error(msg) raise SAMExecutionError(msg) diff --git a/reV/SAM/generation.py b/reV/SAM/generation.py index 52ced392d..ee975ffed 100644 --- a/reV/SAM/generation.py +++ b/reV/SAM/generation.py @@ -4,6 +4,7 @@ Wraps the NREL-PySAM pvwattsv5, windpower, and tcsmolensalt modules with additional reV features. """ + import copy import logging import os @@ -40,7 +41,7 @@ ) from reV.SAM.econ import LCOE, SingleOwner from reV.SAM.SAM import RevPySam -from reV.utilities import ResourceMetaField +from reV.utilities import ResourceMetaField, SupplyCurveField from reV.utilities.curtailment import curtail from reV.utilities.exceptions import ( InputError, @@ -261,26 +262,35 @@ def tz_elev_check(sam_sys_inputs, site_sys_inputs, meta): """ if meta is not None: + meta = meta.rename( + SupplyCurveField.map_to(ResourceMetaField), axis=1 + ) if sam_sys_inputs is not None: if ResourceMetaField.ELEVATION in sam_sys_inputs: - meta[ResourceMetaField.ELEVATION] = \ - sam_sys_inputs[ResourceMetaField.ELEVATION] + meta[ResourceMetaField.ELEVATION] = sam_sys_inputs[ + ResourceMetaField.ELEVATION + ] if ResourceMetaField.TIMEZONE in sam_sys_inputs: - meta[ResourceMetaField.TIMEZONE] = \ - int(sam_sys_inputs[ResourceMetaField.TIMEZONE]) + meta[ResourceMetaField.TIMEZONE] = int( + sam_sys_inputs[ResourceMetaField.TIMEZONE] + ) # site-specific inputs take priority over generic system inputs if site_sys_inputs is not None: if ResourceMetaField.ELEVATION in site_sys_inputs: - meta[ResourceMetaField.ELEVATION] = \ - site_sys_inputs[ResourceMetaField.ELEVATION] + meta[ResourceMetaField.ELEVATION] = site_sys_inputs[ + ResourceMetaField.ELEVATION + ] if ResourceMetaField.TIMEZONE in site_sys_inputs: - meta[ResourceMetaField.TIMEZONE] = \ - int(site_sys_inputs[ResourceMetaField.TIMEZONE]) + meta[ResourceMetaField.TIMEZONE] = int( + site_sys_inputs[ResourceMetaField.TIMEZONE] + ) if ResourceMetaField.TIMEZONE not in meta: - msg = ('Need timezone input to run SAM gen. Not found in ' - 'resource meta or technology json input config.') + msg = ( + "Need timezone input to run SAM gen. Not found in " + "resource meta or technology json input config." + ) raise SAMExecutionError(msg) return meta @@ -313,7 +323,7 @@ def cf_profile(self): 1D numpy array of capacity factor profile. Datatype is float32 and array length is 8760*time_interval. """ - return self.gen_profile() / self.sam_sys_inputs['system_capacity'] + return self.gen_profile() / self.sam_sys_inputs["system_capacity"] def annual_energy(self): """Get annual energy generation value in kWh from SAM. @@ -375,14 +385,22 @@ def run_gen_and_econ(self): lcoe_out_reqs = None so_out_reqs = None - lcoe_vars = ('lcoe_fcr', 'fixed_charge_rate', - 'capital_cost', - 'fixed_operating_cost', - 'variable_operating_cost') - so_vars = ('ppa_price', 'lcoe_real', 'lcoe_nom', - 'project_return_aftertax_npv', 'flip_actual_irr', - 'gross_revenue') - if 'lcoe_fcr' in self.output_request: + lcoe_vars = ( + "lcoe_fcr", + "fixed_charge_rate", + "capital_cost", + "fixed_operating_cost", + "variable_operating_cost", + ) + so_vars = ( + "ppa_price", + "lcoe_real", + "lcoe_nom", + "project_return_aftertax_npv", + "flip_actual_irr", + "gross_revenue", + ) + if "lcoe_fcr" in self.output_request: lcoe_out_reqs = [r for r in self.output_request if r in lcoe_vars] self.output_request = [ r for r in self.output_request if r not in lcoe_out_reqs @@ -562,14 +580,18 @@ def reV_run( class AbstractSamGenerationFromWeatherFile(AbstractSamGeneration, ABC): """Base class for running sam generation with a weather file on disk.""" - WF_META_DROP_COLS = {ResourceMetaField.LATITUDE, - ResourceMetaField.LONGITUDE, - ResourceMetaField.ELEVATION, - ResourceMetaField.TIMEZONE, - ResourceMetaField.COUNTRY, - ResourceMetaField.STATE, - ResourceMetaField.COUNTY, - 'urban', 'population', 'landcover'} + WF_META_DROP_COLS = { + ResourceMetaField.LATITUDE, + ResourceMetaField.LONGITUDE, + ResourceMetaField.ELEVATION, + ResourceMetaField.TIMEZONE, + ResourceMetaField.COUNTRY, + ResourceMetaField.STATE, + ResourceMetaField.COUNTY, + "urban", + "population", + "landcover", + } @property @abstractmethod @@ -640,24 +662,22 @@ def _create_pysam_wfile(self, resource, meta): # ------- Process metadata m = pd.DataFrame(meta).T timezone = m[ResourceMetaField.TIMEZONE] - m['Source'] = 'NSRDB' - m['Location ID'] = meta.name - m['City'] = '-' - m['State'] = m['state'].apply( - lambda x: '-' if x == 'None' else x) - m['Country'] = m['country'].apply( - lambda x: '-' if x == 'None' else x) - m['Latitude'] = m[ResourceMetaField.LATITUDE] - m['Longitude'] = m[ResourceMetaField.LONGITUDE] - m['Time Zone'] = timezone - m['Elevation'] = m[ResourceMetaField.ELEVATION] - m['Local Time Zone'] = timezone - m['Dew Point Units'] = 'c' - m['DHI Units'] = 'w/m2' - m['DNI Units'] = 'w/m2' - m['Temperature Units'] = 'c' - m['Pressure Units'] = 'mbar' - m['Wind Speed'] = 'm/s' + m["Source"] = "NSRDB" + m["Location ID"] = meta.name + m["City"] = "-" + m["State"] = m["state"].apply(lambda x: "-" if x == "None" else x) + m["Country"] = m["country"].apply(lambda x: "-" if x == "None" else x) + m["Latitude"] = m[ResourceMetaField.LATITUDE] + m["Longitude"] = m[ResourceMetaField.LONGITUDE] + m["Time Zone"] = timezone + m["Elevation"] = m[ResourceMetaField.ELEVATION] + m["Local Time Zone"] = timezone + m["Dew Point Units"] = "c" + m["DHI Units"] = "w/m2" + m["DNI Units"] = "w/m2" + m["Temperature Units"] = "c" + m["Pressure Units"] = "mbar" + m["Wind Speed"] = "m/s" keep_cols = [c for c in m.columns if c not in self.WF_META_DROP_COLS] m[keep_cols].to_csv(fname, index=False, mode="w") @@ -795,26 +815,28 @@ def set_resource_data(self, resource, meta): if var != "time_index": # ensure that resource array length is multiple of 8760 arr = self.ensure_res_len(arr, time_index) - n_roll = int(self._meta[ResourceMetaField.TIMEZONE] - * self.time_interval) + n_roll = int( + self._meta[ResourceMetaField.TIMEZONE] * self.time_interval + ) arr = np.roll(arr, n_roll) if var in irrad_vars and np.min(arr) < 0: - warn('Solar irradiance variable "{}" has a minimum ' - 'value of {}. Truncating to zero.' - .format(var, np.min(arr)), SAMInputWarning) + warn( + 'Solar irradiance variable "{}" has a minimum ' + "value of {}. Truncating to zero.".format( + var, np.min(arr) + ), + SAMInputWarning, + ) arr = np.where(arr < 0, 0, arr) resource[var] = arr.tolist() - resource['lat'] = meta[ResourceMetaField.LATITUDE] - resource['lon'] = meta[ResourceMetaField.LONGITUDE] - resource['tz'] = meta[ResourceMetaField.TIMEZONE] + resource["lat"] = meta[ResourceMetaField.LATITUDE] + resource["lon"] = meta[ResourceMetaField.LONGITUDE] + resource["tz"] = meta[ResourceMetaField.TIMEZONE] - if ResourceMetaField.ELEVATION in meta: - resource['elev'] = meta[ResourceMetaField.ELEVATION] - else: - resource["elev"] = 0.0 + resource["elev"] = meta.get(ResourceMetaField.ELEVATION, 0.0) time_index = self.ensure_res_len(time_index, time_index) resource["minute"] = time_index.minute @@ -977,10 +999,12 @@ def set_resource_data(self, resource, meta): respectively. """ - bad_location_input = ((meta[ResourceMetaField.LATITUDE] < -90) - | (meta[ResourceMetaField.LATITUDE] > 90) - | (meta[ResourceMetaField.LONGITUDE] < -180) - | (meta[ResourceMetaField.LONGITUDE] > 180)) + bad_location_input = ( + (meta[ResourceMetaField.LATITUDE] < -90) + | (meta[ResourceMetaField.LATITUDE] > 90) + | (meta[ResourceMetaField.LONGITUDE] < -180) + | (meta[ResourceMetaField.LONGITUDE] > 180) + ) if bad_location_input.any(): raise ValueError( "Detected latitude/longitude values outside of " @@ -1008,7 +1032,7 @@ def set_latitude_tilt_az(sam_sys_inputs, meta): sam_sys_inputs : dict Site-agnostic SAM system model inputs arguments. If for a pv simulation the "tilt" parameter was originally not - present or set to 'lat' or SupplyCurveField.LATITUDE, the tilt will be + present or set to 'lat' or MetaKeyName.LATITUDE, the tilt will be set to the absolute value of the latitude found in meta and the azimuth will be 180 if lat>0, 0 if lat<0. """ @@ -1023,15 +1047,15 @@ def set_latitude_tilt_az(sam_sys_inputs, meta): elif ( sam_sys_inputs["tilt"] == "lat" or sam_sys_inputs["tilt"] == ResourceMetaField.LATITUDE + ) or ( + sam_sys_inputs["tilt"] == "lat" + or sam_sys_inputs["tilt"] == ResourceMetaField.LATITUDE ): set_tilt = True - elif (sam_sys_inputs['tilt'] == 'lat' - or sam_sys_inputs['tilt'] == ResourceMetaField.LATITUDE): - set_tilt = True if set_tilt: # set tilt to abs(latitude) - sam_sys_inputs['tilt'] = np.abs(meta[ResourceMetaField.LATITUDE]) + sam_sys_inputs["tilt"] = np.abs(meta[ResourceMetaField.LATITUDE]) if meta[ResourceMetaField.LATITUDE] > 0: # above the equator, az = 180 sam_sys_inputs["azimuth"] = 180 @@ -1101,7 +1125,7 @@ def cf_profile(self): Datatype is float32 and array length is 8760*time_interval. PV CF is calculated as AC power / DC nameplate. """ - return self.gen_profile() / self.sam_sys_inputs['system_capacity'] + return self.gen_profile() / self.sam_sys_inputs["system_capacity"] def cf_profile_ac(self): """Get hourly AC capacity factor (frac) profile in local timezone. @@ -1209,10 +1233,9 @@ def collect_outputs(self, output_lookup=None): class PvWattsv5(AbstractSamPv): - """Photovoltaic (PV) generation with pvwattsv5. - """ + """Photovoltaic (PV) generation with pvwattsv5.""" - MODULE = 'pvwattsv5' + MODULE = "pvwattsv5" PYSAM = PySamPv5 @staticmethod @@ -1227,10 +1250,9 @@ def default(): class PvWattsv7(AbstractSamPv): - """Photovoltaic (PV) generation with pvwattsv7. - """ + """Photovoltaic (PV) generation with pvwattsv7.""" - MODULE = 'pvwattsv7' + MODULE = "pvwattsv7" PYSAM = PySamPv7 @staticmethod @@ -1245,10 +1267,9 @@ def default(): class PvWattsv8(AbstractSamPv): - """Photovoltaic (PV) generation with pvwattsv8. - """ + """Photovoltaic (PV) generation with pvwattsv8.""" - MODULE = 'pvwattsv8' + MODULE = "pvwattsv8" PYSAM = PySamPv8 @staticmethod @@ -1305,10 +1326,9 @@ def default(): class TcsMoltenSalt(AbstractSamSolar): - """Concentrated Solar Power (CSP) generation with tower molten salt - """ + """Concentrated Solar Power (CSP) generation with tower molten salt""" - MODULE = 'tcsmolten_salt' + MODULE = "tcsmolten_salt" PYSAM = PySamCSP def cf_profile(self): @@ -1322,7 +1342,7 @@ def cf_profile(self): 1D numpy array of capacity factor profile. Datatype is float32 and array length is 8760*time_interval. """ - x = np.abs(self.gen_profile() / self.sam_sys_inputs['system_capacity']) + x = np.abs(self.gen_profile() / self.sam_sys_inputs["system_capacity"]) return x @staticmethod @@ -1341,7 +1361,7 @@ class SolarWaterHeat(AbstractSamGenerationFromWeatherFile): Solar Water Heater generation """ - MODULE = 'solarwaterheat' + MODULE = "solarwaterheat" PYSAM = PySamSwh PYSAM_WEATHER_TAG = "solar_resource_file" @@ -1361,7 +1381,7 @@ class LinearDirectSteam(AbstractSamGenerationFromWeatherFile): Process heat linear Fresnel direct steam generation """ - MODULE = 'lineardirectsteam' + MODULE = "lineardirectsteam" PYSAM = PySamLds PYSAM_WEATHER_TAG = "file_name" @@ -1397,7 +1417,7 @@ class TroughPhysicalHeat(AbstractSamGenerationFromWeatherFile): Trough Physical Process Heat generation """ - MODULE = 'troughphysicalheat' + MODULE = "troughphysicalheat" PYSAM = PySamTpph PYSAM_WEATHER_TAG = "file_name" @@ -1693,8 +1713,10 @@ def _set_resource_temperature(self, resource): ) ) if len(val) > 1: - msg = ("Found multiple values for 'temperature' for site {}: {}" - .format(self.site, val)) + msg = ( + "Found multiple values for 'temperature' for site " + "{}: {}".format(self.site, val) + ) logger.error(msg) raise InputError(msg) @@ -1742,8 +1764,8 @@ def _set_egs_plant_design_temperature(self): msg = ( "EGS plant design temperature ({}C) is lower than resource " "temperature ({}C) by more than a factor of {}. Increasing " - "EGS plant design temperature to match resource temperature" - .format( + "EGS plant design temperature to match resource " + "temperature".format( egs_plant_design_temp, resource_temp, self.MAX_RT_TO_EGS_RATIO, @@ -1759,18 +1781,18 @@ def _set_nameplate_to_match_resource_potential(self, resource): if "nameplate" in self.sam_sys_inputs: msg = ( 'Found "nameplate" input in config! Resource potential ' - "from input data will be ignored. Nameplate capacity is {}" - .format( - self.sam_sys_inputs["nameplate"] - ) + "from input data will be ignored. Nameplate capacity is " + "{}".format(self.sam_sys_inputs["nameplate"]) ) logger.info(msg) return val = set(resource["potential_MW"].unique()) if len(val) > 1: - msg = ('Found multiple values for "potential_MW" for site {}: {}' - .format(self.site, val)) + msg = ( + 'Found multiple values for "potential_MW" for site ' + "{}: {}".format(self.site, val) + ) logger.error(msg) raise InputError(msg) @@ -1797,8 +1819,9 @@ def _set_resource_potential_to_match_gross_output(self): self.sam_sys_inputs["resource_potential"] = -1 return - gross_gen = (self.pysam.Outputs.gross_output - * self._RESOURCE_POTENTIAL_MULT) + gross_gen = ( + self.pysam.Outputs.gross_output * self._RESOURCE_POTENTIAL_MULT + ) if "resource_potential" in self.sam_sys_inputs: msg = ( 'Setting "resource_potential" is not allowed! Updating ' @@ -1820,7 +1843,8 @@ def _set_resource_potential_to_match_gross_output(self): self.sam_sys_inputs["prod_and_inj_wells_to_drill"] = ( self.pysam.Outputs.num_wells_getem_output - ncw - + self.pysam.Outputs.num_wells_getem_inj) + + self.pysam.Outputs.num_wells_getem_inj + ) self["ui_calculations_only"] = 0 def _set_costs(self): @@ -2047,7 +2071,7 @@ def __init__(self, *args, **kwargs): class WindPower(AbstractSamWind): """Class for Wind generation from SAM""" - MODULE = 'windpower' + MODULE = "windpower" PYSAM = PySamWindPower def set_resource_data(self, resource, meta): @@ -2099,7 +2123,7 @@ def set_resource_data(self, resource, meta): if "rh" in resource: # set relative humidity for icing. - rh = self.ensure_res_len(resource['rh'].values, time_index) + rh = self.ensure_res_len(resource["rh"].values, time_index) n_roll = int(meta[ResourceMetaField.TIMEZONE] * self.time_interval) rh = np.roll(rh, n_roll, axis=0) data_dict["rh"] = rh.tolist() @@ -2112,10 +2136,10 @@ def set_resource_data(self, resource, meta): temp = np.roll(temp, n_roll, axis=0) data_dict["data"] = temp.tolist() - data_dict['lat'] = float(meta[ResourceMetaField.LATITUDE]) - data_dict['lon'] = float(meta[ResourceMetaField.LONGITUDE]) - data_dict['tz'] = int(meta[ResourceMetaField.TIMEZONE]) - data_dict['elev'] = float(meta[ResourceMetaField.ELEVATION]) + data_dict["lat"] = float(meta[ResourceMetaField.LATITUDE]) + data_dict["lon"] = float(meta[ResourceMetaField.LONGITUDE]) + data_dict["tz"] = int(meta[ResourceMetaField.TIMEZONE]) + data_dict["elev"] = float(meta[ResourceMetaField.ELEVATION]) time_index = self.ensure_res_len(time_index, time_index) data_dict["minute"] = time_index.minute.tolist() @@ -2245,10 +2269,9 @@ def set_resource_data(self, ws_edges, wd_edges, wind_dist): class MhkWave(AbstractSamGeneration): - """Class for Wave generation from SAM - """ + """Class for Wave generation from SAM""" - MODULE = 'mhkwave' + MODULE = "mhkwave" PYSAM = PySamMhkWave def set_resource_data(self, resource, meta): @@ -2299,9 +2322,9 @@ def set_resource_data(self, resource, meta): n_roll = int(meta[ResourceMetaField.TIMEZONE] * self.time_interval) data_dict[var] = np.roll(arr, n_roll, axis=0).tolist() - data_dict['lat'] = meta[ResourceMetaField.LATITUDE] - data_dict['lon'] = meta[ResourceMetaField.LONGITUDE] - data_dict['tz'] = meta[ResourceMetaField.TIMEZONE] + data_dict["lat"] = meta[ResourceMetaField.LATITUDE] + data_dict["lon"] = meta[ResourceMetaField.LONGITUDE] + data_dict["tz"] = meta[ResourceMetaField.TIMEZONE] time_index = self.ensure_res_len(time_index, time_index) data_dict["minute"] = time_index.minute diff --git a/reV/bespoke/bespoke.py b/reV/bespoke/bespoke.py index f885c2221..a921b9639 100644 --- a/reV/bespoke/bespoke.py +++ b/reV/bespoke/bespoke.py @@ -585,7 +585,7 @@ def _parse_gid_map(gid_map): if isinstance(gid_map, str): if gid_map.endswith(".csv"): gid_map = pd.read_csv(gid_map).to_dict() - err_msg = f'Need {ResourceMetaField.GID} in gid_map column' + err_msg = f"Need {ResourceMetaField.GID} in gid_map column" assert ResourceMetaField.GID in gid_map, err_msg assert "gid_map" in gid_map, 'Need "gid_map" in gid_map column' gid_map = { @@ -1278,8 +1278,8 @@ def run_plant_optimization(self): pxc = json.dumps(pxc) pyc = json.dumps(pyc) - self._meta["turbine_x_coords"] = txc - self._meta["turbine_y_coords"] = tyc + self._meta[SupplyCurveField.TURBINE_X_COORDS] = txc + self._meta[SupplyCurveField.TURBINE_Y_COORDS] = tyc self._meta["possible_x_coords"] = pxc self._meta["possible_y_coords"] = pyc diff --git a/reV/config/project_points.py b/reV/config/project_points.py index ea117d773..c4b2ac637 100644 --- a/reV/config/project_points.py +++ b/reV/config/project_points.py @@ -20,8 +20,8 @@ from reV.config.curtailment import Curtailment from reV.config.sam_config import SAMConfig +from reV.utilities import SiteDataField, SupplyCurveField from reV.utilities.exceptions import ConfigError, ConfigWarning -from reV.utilities import SiteDataField logger = logging.getLogger(__name__) @@ -288,7 +288,7 @@ def __getitem__(self, site): names (keys) and values. """ - site_bool = (self.df[SiteDataField.GID] == site) + site_bool = self.df[SiteDataField.GID] == site try: config_id = self.df.loc[site_bool, SiteDataField.CONFIG].values[0] except (KeyError, IndexError) as ex: @@ -614,15 +614,18 @@ def _parse_points(cls, points, res_file=None): raise ValueError( "Cannot parse Project points data from {}".format(type(points)) ) - + df = df.rename(SupplyCurveField.map_to(SiteDataField), axis=1) if SiteDataField.GID not in df.columns: - raise KeyError('Project points data must contain ' - f'{SiteDataField.GID} column.') + raise KeyError( + "Project points data must contain " + f"{SiteDataField.GID} column." + ) # pylint: disable=no-member if SiteDataField.CONFIG not in df.columns: - df = cls._parse_sites(points[SiteDataField.GID].values, - res_file=res_file) + df = cls._parse_sites( + df[SiteDataField.GID].values, res_file=res_file + ) gids = df[SiteDataField.GID].values if not np.array_equal(np.sort(gids), gids): @@ -633,7 +636,7 @@ def _parse_points(cls, points, res_file=None): ) logger.warning(msg) warn(msg) - df['points_order'] = df.index.values + df["points_order"] = df.index.values df = df.sort_values(SiteDataField.GID).reset_index(drop=True) return df @@ -725,8 +728,10 @@ def index(self, gid): Row index of gid in the project points dataframe. """ if gid not in self._df[SiteDataField.GID].values: - e = ('Requested resource gid {} is not present in the project ' - 'points dataframe. Cannot return row index.'.format(gid)) + e = ( + "Requested resource gid {} is not present in the project " + "points dataframe. Cannot return row index.".format(gid) + ) logger.error(e) raise ConfigError(e) @@ -807,9 +812,15 @@ def join_df(self, df2, key=SiteDataField.GID): """ # ensure df2 doesnt have any duplicate columns for suffix reasons. df2_cols = [c for c in df2.columns if c not in self._df or c == key] - self._df = pd.merge(self._df, df2[df2_cols], how='left', - left_on=SiteDataField.GID, right_on=key, - copy=False, validate='1:1') + self._df = pd.merge( + self._df, + df2[df2_cols], + how="left", + left_on=SiteDataField.GID, + right_on=key, + copy=False, + validate="1:1", + ) def get_sites_from_config(self, config): """Get a site list that corresponds to a config key. @@ -825,8 +836,9 @@ def get_sites_from_config(self, config): List of sites associated with the requested configuration ID. If the configuration ID is not recognized, an empty list is returned. """ - sites = self.df.loc[(self.df[SiteDataField.CONFIG] == config), - SiteDataField.GID].values + sites = self.df.loc[ + (self.df[SiteDataField.CONFIG] == config), SiteDataField.GID + ].values return list(sites) diff --git a/reV/supply_curve/points.py b/reV/supply_curve/points.py index c042784ac..d9620d9d4 100644 --- a/reV/supply_curve/points.py +++ b/reV/supply_curve/points.py @@ -2,6 +2,7 @@ """ reV supply curve points frameworks. """ + import logging from abc import ABC from warnings import warn @@ -1130,16 +1131,19 @@ def country(self): county_not_none = self.county is not None if ResourceMetaField.COUNTRY in self.h5.meta and county_not_none: # make sure country and county are coincident - counties = self.h5.meta.loc[self.h5_gid_set, - ResourceMetaField.COUNTY].values + counties = self.h5.meta.loc[ + self.h5_gid_set, ResourceMetaField.COUNTY + ].values iloc = np.where(counties == self.county)[0][0] - country = self.h5.meta.loc[self.h5_gid_set, - ResourceMetaField.COUNTRY].values + country = self.h5.meta.loc[ + self.h5_gid_set, ResourceMetaField.COUNTRY + ].values country = country[iloc] elif ResourceMetaField.COUNTRY in self.h5.meta: - country = self.h5.meta.loc[self.h5_gid_set, - ResourceMetaField.COUNTRY].mode() + country = self.h5.meta.loc[ + self.h5_gid_set, ResourceMetaField.COUNTRY + ].mode() country = country.values[0] return country @@ -1150,16 +1154,19 @@ def state(self): state = None if ResourceMetaField.STATE in self.h5.meta and self.county is not None: # make sure state and county are coincident - counties = self.h5.meta.loc[self.h5_gid_set, - ResourceMetaField.COUNTY].values + counties = self.h5.meta.loc[ + self.h5_gid_set, ResourceMetaField.COUNTY + ].values iloc = np.where(counties == self.county)[0][0] - state = self.h5.meta.loc[self.h5_gid_set, - ResourceMetaField.STATE].values + state = self.h5.meta.loc[ + self.h5_gid_set, ResourceMetaField.STATE + ].values state = state[iloc] elif ResourceMetaField.STATE in self.h5.meta: - state = self.h5.meta.loc[self.h5_gid_set, - ResourceMetaField.STATE].mode() + state = self.h5.meta.loc[ + self.h5_gid_set, ResourceMetaField.STATE + ].mode() state = state.values[0] return state @@ -1169,8 +1176,9 @@ def county(self): """Get the SC point county based on the resource meta data.""" county = None if ResourceMetaField.COUNTY in self.h5.meta: - county = self.h5.meta.loc[self.h5_gid_set, - ResourceMetaField.COUNTY].mode() + county = self.h5.meta.loc[ + self.h5_gid_set, ResourceMetaField.COUNTY + ].mode() county = county.values[0] return county @@ -1180,8 +1188,9 @@ def elevation(self): """Get the SC point elevation based on the resource meta data.""" elevation = None if ResourceMetaField.ELEVATION in self.h5.meta: - elevation = self.h5.meta.loc[self.h5_gid_set, - ResourceMetaField.ELEVATION].mean() + elevation = self.h5.meta.loc[ + self.h5_gid_set, ResourceMetaField.ELEVATION + ].mean() return elevation @@ -1192,16 +1201,19 @@ def timezone(self): county_not_none = self.county is not None if ResourceMetaField.TIMEZONE in self.h5.meta and county_not_none: # make sure timezone flag and county are coincident - counties = self.h5.meta.loc[self.h5_gid_set, - ResourceMetaField.COUNTY].values + counties = self.h5.meta.loc[ + self.h5_gid_set, ResourceMetaField.COUNTY + ].values iloc = np.where(counties == self.county)[0][0] - timezone = self.h5.meta.loc[self.h5_gid_set, - ResourceMetaField.TIMEZONE].values + timezone = self.h5.meta.loc[ + self.h5_gid_set, ResourceMetaField.TIMEZONE + ].values timezone = timezone[iloc] elif ResourceMetaField.TIMEZONE in self.h5.meta: - timezone = self.h5.meta.loc[self.h5_gid_set, - ResourceMetaField.TIMEZONE].mode() + timezone = self.h5.meta.loc[ + self.h5_gid_set, ResourceMetaField.TIMEZONE + ].mode() timezone = timezone.values[0] return timezone @@ -1214,16 +1226,19 @@ def offshore(self): county_not_none = self.county is not None if ResourceMetaField.OFFSHORE in self.h5.meta and county_not_none: # make sure offshore flag and county are coincident - counties = self.h5.meta.loc[self.h5_gid_set, - ResourceMetaField.COUNTY].values + counties = self.h5.meta.loc[ + self.h5_gid_set, ResourceMetaField.COUNTY + ].values iloc = np.where(counties == self.county)[0][0] - offshore = self.h5.meta.loc[self.h5_gid_set, - ResourceMetaField.OFFSHORE].values + offshore = self.h5.meta.loc[ + self.h5_gid_set, ResourceMetaField.OFFSHORE + ].values offshore = offshore[iloc] elif ResourceMetaField.OFFSHORE in self.h5.meta: - offshore = self.h5.meta.loc[self.h5_gid_set, - ResourceMetaField.OFFSHORE].mode() + offshore = self.h5.meta.loc[ + self.h5_gid_set, ResourceMetaField.OFFSHORE + ].mode() offshore = offshore.values[0] return offshore @@ -1256,19 +1271,20 @@ def summary(self): pandas.Series List of supply curve point's meta data """ - meta = {SupplyCurveField.SC_POINT_GID: self.sc_point_gid, - SupplyCurveField.SOURCE_GIDS: self.h5_gid_set, - SupplyCurveField.GID_COUNTS: self.gid_counts, - SupplyCurveField.N_GIDS: self.n_gids, - SupplyCurveField.AREA_SQ_KM: self.area, - SupplyCurveField.LATITUDE: self.latitude, - SupplyCurveField.LONGITUDE: self.longitude, - SupplyCurveField.COUNTRY: self.country, - SupplyCurveField.STATE: self.state, - SupplyCurveField.COUNTY: self.county, - SupplyCurveField.ELEVATION: self.elevation, - SupplyCurveField.TIMEZONE: self.timezone, - } + meta = { + SupplyCurveField.SC_POINT_GID: self.sc_point_gid, + SupplyCurveField.SOURCE_GIDS: self.h5_gid_set, + SupplyCurveField.GID_COUNTS: self.gid_counts, + SupplyCurveField.N_GIDS: self.n_gids, + SupplyCurveField.AREA_SQ_KM: self.area, + SupplyCurveField.LATITUDE: self.latitude, + SupplyCurveField.LONGITUDE: self.longitude, + SupplyCurveField.COUNTRY: self.country, + SupplyCurveField.STATE: self.state, + SupplyCurveField.COUNTY: self.county, + SupplyCurveField.ELEVATION: self.elevation, + SupplyCurveField.TIMEZONE: self.timezone, + } meta = pd.Series(meta) return meta @@ -1722,24 +1738,31 @@ def mean_lcoe(self): # year CF, but the output should be identical to the original LCOE and # so is not consequential). if self._recalc_lcoe: - required = ('fixed_charge_rate', - 'capital_cost', - 'fixed_operating_cost', - 'variable_operating_cost', - 'system_capacity') - if (self.mean_h5_dsets_data is not None - and all(k in self.mean_h5_dsets_data for k in required)): - aep = (self.mean_h5_dsets_data['system_capacity'] - * self.mean_cf * 8760) + required = ( + "fixed_charge_rate", + "capital_cost", + "fixed_operating_cost", + "variable_operating_cost", + "system_capacity", + ) + if self.mean_h5_dsets_data is not None and all( + k in self.mean_h5_dsets_data for k in required + ): + aep = ( + self.mean_h5_dsets_data["system_capacity"] + * self.mean_cf + * 8760 + ) # Note the AEP computation uses the SAM config # `system_capacity`, so no need to scale `capital_cost` # or `fixed_operating_cost` by anything mean_lcoe = lcoe_fcr( - self.mean_h5_dsets_data['fixed_charge_rate'], - self.mean_h5_dsets_data['capital_cost'], - self.mean_h5_dsets_data['fixed_operating_cost'], - aep, self.mean_h5_dsets_data[ - 'variable_operating_cost']) + self.mean_h5_dsets_data["fixed_charge_rate"], + self.mean_h5_dsets_data["capital_cost"], + self.mean_h5_dsets_data["fixed_operating_cost"], + aep, + self.mean_h5_dsets_data["variable_operating_cost"], + ) # alternative if lcoe was not able to be re-calculated from # multi year mean CF @@ -1977,8 +2000,9 @@ def sc_point_capital_cost(self): return None cap_cost_per_mw = ( - self.mean_h5_dsets_data['capital_cost'] - / self.mean_h5_dsets_data['system_capacity']) + self.mean_h5_dsets_data["capital_cost"] + / self.mean_h5_dsets_data["system_capacity"] + ) return cap_cost_per_mw * self.capacity @property @@ -1999,14 +2023,14 @@ def sc_point_fixed_operating_cost(self): if self.mean_h5_dsets_data is None: return None - required = ('fixed_operating_cost', - 'system_capacity') + required = ("fixed_operating_cost", "system_capacity") if not all(k in self.mean_h5_dsets_data for k in required): return None fixed_cost_per_mw = ( - self.mean_h5_dsets_data['fixed_operating_cost'] - / self.mean_h5_dsets_data['system_capacity']) + self.mean_h5_dsets_data["fixed_operating_cost"] + / self.mean_h5_dsets_data["system_capacity"] + ) return fixed_cost_per_mw * self.capacity @property @@ -2183,14 +2207,17 @@ def point_summary(self, args=None): SupplyCurveField.MEAN_LCOE: self.mean_lcoe, SupplyCurveField.MEAN_RES: self.mean_res, SupplyCurveField.CAPACITY: self.capacity, - SupplyCurveField.AREA_SQ_KM: self.area} - - extra_atts = [SupplyCurveField.CAPACITY_AC, - SupplyCurveField.OFFSHORE, - SupplyCurveField.SC_POINT_CAPITAL_COST, - SupplyCurveField.SC_POINT_FIXED_OPERATING_COST, - SupplyCurveField.SC_POINT_ANNUAL_ENERGY, - SupplyCurveField.SC_POINT_ANNUAL_ENERGY_AC] + SupplyCurveField.AREA_SQ_KM: self.area, + } + + extra_atts = [ + SupplyCurveField.CAPACITY_AC, + SupplyCurveField.OFFSHORE, + SupplyCurveField.SC_POINT_CAPITAL_COST, + SupplyCurveField.SC_POINT_FIXED_OPERATING_COST, + SupplyCurveField.SC_POINT_ANNUAL_ENERGY, + SupplyCurveField.SC_POINT_ANNUAL_ENERGY_AC, + ] for attr in extra_atts: value = getattr(self, attr) if value is not None: @@ -2247,9 +2274,13 @@ def economies_of_scale(cap_cost_scale, summary): summary[SupplyCurveField.CAPITAL_COST_SCALAR] = eos.capital_cost_scalar summary[SupplyCurveField.SCALED_CAPITAL_COST] = eos.scaled_capital_cost if SupplyCurveField.SC_POINT_CAPITAL_COST in summary: - scaled_costs = (summary[SupplyCurveField.SC_POINT_CAPITAL_COST] - * eos.capital_cost_scalar) - summary[SupplyCurveField.SCALED_SC_POINT_CAPITAL_COST] = scaled_costs + scaled_costs = ( + summary[SupplyCurveField.SC_POINT_CAPITAL_COST] + * eos.capital_cost_scalar + ) + summary[SupplyCurveField.SCALED_SC_POINT_CAPITAL_COST] = ( + scaled_costs + ) return summary diff --git a/reV/utilities/__init__.py b/reV/utilities/__init__.py index 7640a92d4..3ce30d6b2 100644 --- a/reV/utilities/__init__.py +++ b/reV/utilities/__init__.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- """reV utilities.""" -from enum import Enum + +from enum import Enum, EnumMeta import PySAM from rex.utilities.loggers import log_versions as rex_log_versions @@ -8,14 +9,39 @@ from reV.version import __version__ -class SiteDataField(str, Enum): +class FieldEnum(str, Enum): + """Base Field enum with some mapping methods.""" + + @classmethod + def map_to(cls, other): + """Return a rename map from this enum to another.""" + return { + cls[mem]: other[mem] + for mem in cls.__members__ + if mem in other.__members__ + } + + @classmethod + def map_from(cls, other): + """Return a rename map from a dictionary of name / member pairs (e.g. + 'sc_gid': 'SC_GID') to this enum.""" + return {name: cls[mem] for name, mem in other.items()} + + def __str__(self): + return self.value + + def __format__(self, format_spec): + return str.__format__(self.value, format_spec) + + +class SiteDataField(FieldEnum): """An enumerated map to site data column names.""" GID = "gid" CONFIG = "config" -class ResourceMetaField(str, Enum): +class ResourceMetaField(FieldEnum): """An enumerated map to resource meta column names. Each output name should match the name of a key the resource file @@ -30,10 +56,40 @@ class ResourceMetaField(str, Enum): COUNTY = "county" STATE = "state" COUNTRY = "country" - OFFSHORE = 'offshore' + OFFSHORE = "offshore" + + +# Dictionary of "old" supply curve field names. Used to rename legacy data to +# match current naming conventions + + +OldSupplyCurveField = { + "sc_point_gid": "SC_POINT_GID", + "source_gids": "SOURCE_GIDS", + "sc_gid": "SC_GID", + "gid_counts": "GID_COUNTS", + "gid": "GID", + "n_gids": "N_GIDS", + "res_gids": "RES_GIDS", + "gen_gids": "GEN_GIDS", + "area_sq_km": "AREA_SQ_KM", + "latitude": "LATITUDE", + "longitude": "LONGITUDE", + "elevation": "ELEVATION", + "timezone": "TIMEZONE", + "county": "COUNTY", + "state": "STATE", + "country": "COUNTRY", + "mean_lcoe": "MEAN_LCOE", + "mean_res": "MEAN_RES", + "capacity": "CAPACITY", + "sc_row_ind": "SC_ROW_IND", + "sc_col_ind": "SC_COL_IND", + "mean_cf": "MEAN_CF", +} -class SupplyCurveField(str, Enum): +class OriginalSupplyCurveField(FieldEnum): """An enumerated map to supply curve summary/meta keys. Each output name should match the name of a key in @@ -42,55 +98,103 @@ class SupplyCurveField(str, Enum): meth:`BespokeSinglePlant.meta` """ - SC_POINT_GID = 'sc_point_gid' - SOURCE_GIDS = 'source_gids' - SC_GID = 'sc_gid' - GID_COUNTS = 'gid_counts' - GID = 'gid' - N_GIDS = 'n_gids' - RES_GIDS = 'res_gids' - GEN_GIDS = 'gen_gids' - AREA_SQ_KM = 'area_sq_km' - LATITUDE = 'latitude' - LONGITUDE = 'longitude' - ELEVATION = 'elevation' - TIMEZONE = 'timezone' + SC_POINT_GID = "sc_point_gid" + SOURCE_GIDS = "source_gids" + SC_GID = "sc_gid" + GID_COUNTS = "gid_counts" + GID = "gid" + N_GIDS = "n_gids" + RES_GIDS = "res_gids" + GEN_GIDS = "gen_gids" + AREA_SQ_KM = "area_sq_km" + LATITUDE = "latitude" + LONGITUDE = "longitude" + ELEVATION = "elevation" + TIMEZONE = "timezone" COUNTY = "county" STATE = "state" COUNTRY = "country" - MEAN_CF = 'mean_cf' - MEAN_LCOE = 'mean_lcoe' - MEAN_RES = 'mean_res' - CAPACITY = 'capacity' - OFFSHORE = 'offshore' - SC_ROW_IND = 'sc_row_ind' - SC_COL_IND = 'sc_col_ind' - CAPACITY_AC = 'capacity_ac' - CAPITAL_COST = 'capital_cost' - FIXED_OPERATING_COST = 'fixed_operating_cost' - VARIABLE_OPERATING_COST = 'variable_operating_cost' - FIXED_CHARGE_RATE = 'fixed_charge_rate' - SC_POINT_CAPITAL_COST = 'sc_point_capital_cost' - SC_POINT_FIXED_OPERATING_COST = 'sc_point_fixed_operating_cost' - SC_POINT_ANNUAL_ENERGY = 'sc_point_annual_energy' - SC_POINT_ANNUAL_ENERGY_AC = 'sc_point_annual_energy_ac' - MEAN_FRICTION = 'mean_friction' - MEAN_LCOE_FRICTION = 'mean_lcoe_friction' - TOTAL_LCOE_FRICTION = 'total_lcoe_friction' - RAW_LCOE = 'raw_lcoe' - CAPITAL_COST_SCALAR = 'capital_cost_scalar' - SCALED_CAPITAL_COST = 'scaled_capital_cost' - SCALED_SC_POINT_CAPITAL_COST = 'scaled_sc_point_capital_cost' - TURBINE_X_COORDS = 'turbine_x_coords' - TURBINE_Y_COORDS = 'turbine_y_coords' + MEAN_CF = "mean_cf" + MEAN_LCOE = "mean_lcoe" + MEAN_RES = "mean_res" + CAPACITY = "capacity" + OFFSHORE = "offshore" + SC_ROW_IND = "sc_row_ind" + SC_COL_IND = "sc_col_ind" + CAPACITY_AC = "capacity_ac" + CAPITAL_COST = "capital_cost" + FIXED_OPERATING_COST = "fixed_operating_cost" + VARIABLE_OPERATING_COST = "variable_operating_cost" + FIXED_CHARGE_RATE = "fixed_charge_rate" + SC_POINT_CAPITAL_COST = "sc_point_capital_cost" + SC_POINT_FIXED_OPERATING_COST = "sc_point_fixed_operating_cost" + SC_POINT_ANNUAL_ENERGY = "sc_point_annual_energy" + SC_POINT_ANNUAL_ENERGY_AC = "sc_point_annual_energy_ac" + MEAN_FRICTION = "mean_friction" + MEAN_LCOE_FRICTION = "mean_lcoe_friction" + TOTAL_LCOE_FRICTION = "total_lcoe_friction" + RAW_LCOE = "raw_lcoe" + CAPITAL_COST_SCALAR = "capital_cost_scalar" + SCALED_CAPITAL_COST = "scaled_capital_cost" + SCALED_SC_POINT_CAPITAL_COST = "scaled_sc_point_capital_cost" + TURBINE_X_COORDS = "turbine_x_coords" + TURBINE_Y_COORDS = "turbine_y_coords" EOS_MULT = "eos_mult" REG_MULT = "reg_mult" - def __str__(self): - return self.value - def __format__(self, format_spec): - return str.__format__(self.value, format_spec) +class SupplyCurveField(FieldEnum): + """An enumerated map to supply curve summary/meta keys. + + Each output name should match the name of a key in + meth:`AggregationSupplyCurvePoint.summary` or + meth:`GenerationSupplyCurvePoint.point_summary` or + meth:`BespokeSinglePlant.meta` + """ + + SC_POINT_GID = "sc_point_gid_m" + SOURCE_GIDS = "source_gids_m" + SC_GID = "sc_gid_m" + GID_COUNTS = "gid_counts_m" + GID = "gid_m" + N_GIDS = "n_gids_m" + RES_GIDS = "res_gids_m" + GEN_GIDS = "gen_gids_m" + AREA_SQ_KM = "area_sq_km_m" + LATITUDE = "latitude_m" + LONGITUDE = "longitude_m" + ELEVATION = "elevation_m" + TIMEZONE = "timezone_m" + COUNTY = "county_m" + STATE = "state_m" + COUNTRY = "country_m" + MEAN_CF = "mean_cf_m" + MEAN_LCOE = "mean_lcoe_m" + MEAN_RES = "mean_res_m" + CAPACITY = "capacity_m" + OFFSHORE = "offshore_m" + SC_ROW_IND = "sc_row_ind_m" + SC_COL_IND = "sc_col_ind_m" + CAPACITY_AC = "capacity_ac_m" + CAPITAL_COST = "capital_cost_m" + FIXED_OPERATING_COST = "fixed_operating_cost_m" + VARIABLE_OPERATING_COST = "variable_operating_cost_m" + FIXED_CHARGE_RATE = "fixed_charge_rate_m" + SC_POINT_CAPITAL_COST = "sc_point_capital_cost_m" + SC_POINT_FIXED_OPERATING_COST = "sc_point_fixed_operating_cost_m" + SC_POINT_ANNUAL_ENERGY = "sc_point_annual_energy_m" + SC_POINT_ANNUAL_ENERGY_AC = "sc_point_annual_energy_ac_m" + MEAN_FRICTION = "mean_friction_m" + MEAN_LCOE_FRICTION = "mean_lcoe_friction_m" + TOTAL_LCOE_FRICTION = "total_lcoe_friction_m" + RAW_LCOE = "raw_lcoe_m" + CAPITAL_COST_SCALAR = "capital_cost_scalar_m" + SCALED_CAPITAL_COST = "scaled_capital_cost_m" + SCALED_SC_POINT_CAPITAL_COST = "scaled_sc_point_capital_cost_m" + TURBINE_X_COORDS = "turbine_x_coords_m" + TURBINE_Y_COORDS = "turbine_y_coords_m" + EOS_MULT = "eos_mult_m" + REG_MULT = "reg_mult_m" class ModuleName(str, Enum): @@ -107,17 +211,17 @@ class ModuleName(str, Enum): click name conversions: https://tinyurl.com/4rehbsvf """ - BESPOKE = 'bespoke' - COLLECT = 'collect' - ECON = 'econ' - GENERATION = 'generation' - HYBRIDS = 'hybrids' - MULTI_YEAR = 'multi-year' - NRWAL = 'nrwal' - QA_QC = 'qa-qc' - REP_PROFILES = 'rep-profiles' - SUPPLY_CURVE = 'supply-curve' - SUPPLY_CURVE_AGGREGATION = 'supply-curve-aggregation' + BESPOKE = "bespoke" + COLLECT = "collect" + ECON = "econ" + GENERATION = "generation" + HYBRIDS = "hybrids" + MULTI_YEAR = "multi-year" + NRWAL = "nrwal" + QA_QC = "qa-qc" + REP_PROFILES = "rep-profiles" + SUPPLY_CURVE = "supply-curve" + SUPPLY_CURVE_AGGREGATION = "supply-curve-aggregation" def __str__(self): return self.value @@ -148,6 +252,6 @@ def log_versions(logger): logger : logging.Logger Logger object to log memory message to. """ - logger.info('Running with reV version {}'.format(__version__)) + logger.info("Running with reV version {}".format(__version__)) rex_log_versions(logger) - logger.debug('- PySAM version {}'.format(PySAM.__version__)) + logger.debug("- PySAM version {}".format(PySAM.__version__)) diff --git a/tests/test_bespoke.py b/tests/test_bespoke.py index f32841016..d435f2b67 100644 --- a/tests/test_bespoke.py +++ b/tests/test_bespoke.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- """reV bespoke wind plant optimization tests""" + import copy import json import os @@ -25,7 +26,12 @@ from reV.SAM.generation import WindPower from reV.supply_curve.supply_curve import SupplyCurve from reV.supply_curve.tech_mapping import TechMapping -from reV.utilities import ModuleName, SiteDataField, SupplyCurveField +from reV.utilities import ( + ModuleName, + OldSupplyCurveField, + SiteDataField, + SupplyCurveField, +) pytest.importorskip("shapely") @@ -82,7 +88,7 @@ def test_turbine_placement(gid=33): """Test turbine placement with zero available area.""" - output_request = ('system_capacity', 'cf_mean', 'cf_profile') + output_request = ("system_capacity", "cf_mean", "cf_profile") with tempfile.TemporaryDirectory() as td: res_fp = os.path.join(td, "ri_100_wtk_{}.h5") excl_fp = os.path.join(td, "ri_exclusions.h5") @@ -173,8 +179,7 @@ def test_turbine_placement(gid=33): def test_zero_area(gid=33): """Test turbine placement with zero available area.""" - output_request = ('system_capacity', 'cf_mean', - 'cf_profile') + output_request = ("system_capacity", "cf_mean", "cf_profile") objective_function = ( "(0.0975 * capital_cost + fixed_operating_cost) " @@ -224,8 +229,7 @@ def test_zero_area(gid=33): def test_correct_turb_location(gid=33): """Test turbine location is reported correctly.""" - output_request = ('system_capacity', 'cf_mean', - 'cf_profile') + output_request = ("system_capacity", "cf_mean", "cf_profile") objective_function = ( "(0.0975 * capital_cost + fixed_operating_cost) " @@ -324,15 +328,19 @@ def test_packing_algorithm(gid=33): def test_bespoke_points(): """Test the bespoke points input options""" # pylint: disable=W0612 - points = pd.DataFrame({SiteDataField.GID: [33, 34, 35], - SiteDataField.CONFIG: ['default'] * 3}) - pp = BespokeWindPlants._parse_points(points, {'default': SAM}) + points = pd.DataFrame( + { + SiteDataField.GID: [33, 34, 35], + SiteDataField.CONFIG: ["default"] * 3, + } + ) + pp = BespokeWindPlants._parse_points(points, {"default": SAM}) assert len(pp) == 3 for gid in pp.gids: assert pp[gid][0] == "default" points = pd.DataFrame({SupplyCurveField.GID: [33, 34, 35]}) - pp = BespokeWindPlants._parse_points(points, {'default': SAM}) + pp = BespokeWindPlants._parse_points(points, {"default": SAM}) assert len(pp) == 3 assert SiteDataField.CONFIG in pp.df.columns for gid in pp.gids: @@ -341,8 +349,7 @@ def test_bespoke_points(): def test_single(gid=33): """Test a single wind plant bespoke optimization run""" - output_request = ('system_capacity', 'cf_mean', - 'cf_profile') + output_request = ("system_capacity", "cf_mean", "cf_profile") with tempfile.TemporaryDirectory() as td: res_fp = os.path.join(td, "ri_100_wtk_{}.h5") excl_fp = os.path.join(td, "ri_exclusions.h5") @@ -379,12 +386,18 @@ def test_single(gid=33): assert "annual_energy-2013" in out assert "annual_energy-means" in out - assert (TURB_RATING * bsp.meta['n_turbines'].values[0] - == out['system_capacity']) - x_coords = json.loads(bsp.meta[SupplyCurveField.TURBINE_X_COORDS].values[0]) - y_coords = json.loads(bsp.meta[SupplyCurveField.TURBINE_Y_COORDS].values[0]) - assert bsp.meta['n_turbines'].values[0] == len(x_coords) - assert bsp.meta['n_turbines'].values[0] == len(y_coords) + assert ( + TURB_RATING * bsp.meta["n_turbines"].values[0] + == out["system_capacity"] + ) + x_coords = json.loads( + bsp.meta[SupplyCurveField.TURBINE_X_COORDS].values[0] + ) + y_coords = json.loads( + bsp.meta[SupplyCurveField.TURBINE_Y_COORDS].values[0] + ) + assert bsp.meta["n_turbines"].values[0] == len(x_coords) + assert bsp.meta["n_turbines"].values[0] == len(y_coords) for y in (2012, 2013): cf = out[f"cf_profile-{y}"] @@ -394,11 +407,11 @@ def test_single(gid=33): # simple windpower obj for comparison wp_sam_config = bsp.sam_sys_inputs - wp_sam_config['wind_farm_wake_model'] = 0 - wp_sam_config['wake_int_loss'] = 0 - wp_sam_config['wind_farrm_xCoordinates'] = [0] - wp_sam_config['wind_farrm_yCoordinates'] = [0] - wp_sam_config['system_capacity'] = TURB_RATING + wp_sam_config["wind_farm_wake_model"] = 0 + wp_sam_config["wake_int_loss"] = 0 + wp_sam_config["wind_farm_xCoordinates"] = [0] + wp_sam_config["wind_farm_yCoordinates"] = [0] + wp_sam_config["system_capacity"] = TURB_RATING res_df = bsp.res_df[(bsp.res_df.index.year == 2012)].copy() wp = WindPower( res_df, bsp.meta, wp_sam_config, output_request=bsp._out_req @@ -425,8 +438,7 @@ def test_single(gid=33): def test_extra_outputs(gid=33): """Test running bespoke single farm optimization with lcoe requests""" - output_request = ('system_capacity', 'cf_mean', - 'cf_profile', 'lcoe_fcr') + output_request = ("system_capacity", "cf_mean", "cf_profile", "lcoe_fcr") objective_function = ( "(fixed_charge_rate * capital_cost + fixed_operating_cost) " @@ -541,11 +553,16 @@ def test_extra_outputs(gid=33): n_turbs = round(test_eos_cap / TURB_RATING) test_eos_cap_kw = n_turbs * TURB_RATING - baseline_cost = (140 * test_eos_cap_kw - * np.exp(-test_eos_cap_kw / 1E5 * 0.1 + (1 - 0.1))) - eos_mult = (bsp.plant_optimizer.capital_cost - / bsp.plant_optimizer.capacity - / (baseline_cost / test_eos_cap_kw)) + baseline_cost = ( + 140 + * test_eos_cap_kw + * np.exp(-test_eos_cap_kw / 1e5 * 0.1 + (1 - 0.1)) + ) + eos_mult = ( + bsp.plant_optimizer.capital_cost + / bsp.plant_optimizer.capacity + / (baseline_cost / test_eos_cap_kw) + ) assert np.allclose(bsp.meta[SupplyCurveField.EOS_MULT], eos_mult) bsp.close() @@ -553,11 +570,16 @@ def test_extra_outputs(gid=33): def test_bespoke(): """Test bespoke optimization with multiple plants, parallel processing, and - file output. """ - output_request = ('system_capacity', 'cf_mean', - 'cf_profile', - 'extra_unused_data', 'winddirection', 'windspeed', - 'ws_mean') + file output.""" + output_request = ( + "system_capacity", + "cf_mean", + "cf_profile", + "extra_unused_data", + "winddirection", + "windspeed", + "ws_mean", + ) with tempfile.TemporaryDirectory() as td: out_fpath_request = os.path.join(td, "wind") @@ -569,13 +591,20 @@ def test_bespoke(): shutil.copy(RES.format(2013), res_fp.format(2013)) res_fp = res_fp.format("*") # both 33 and 35 are included, 37 is fully excluded - points = pd.DataFrame({SiteDataField.GID: [33, 35], - SiteDataField.CONFIG: ['default'] * 2, - 'extra_unused_data': [0, 42]}) + points = pd.DataFrame( + { + SiteDataField.GID: [33, 35], + SiteDataField.CONFIG: ["default"] * 2, + "extra_unused_data": [0, 42], + } + ) fully_excluded_points = pd.DataFrame( - {SiteDataField.GID: [37], - SiteDataField.CONFIG: ['default'], - 'extra_unused_data': [0]}) + { + SiteDataField.GID: [37], + SiteDataField.CONFIG: ["default"], + "extra_unused_data": [0], + } + ) TechMapping.run(excl_fp, RES.format(2012), dset=TM_DSET, max_workers=1) @@ -624,8 +653,8 @@ def test_bespoke(): assert SupplyCurveField.SC_POINT_GID in meta assert SupplyCurveField.TURBINE_X_COORDS in meta assert SupplyCurveField.TURBINE_Y_COORDS in meta - assert 'possible_x_coords' in meta - assert 'possible_y_coords' in meta + assert "possible_x_coords" in meta + assert "possible_y_coords" in meta assert SupplyCurveField.RES_GIDS in meta dsets_1d = ( @@ -696,8 +725,10 @@ def test_collect_bespoke(): with Resource(h5_file) as fout: meta = fout.meta - assert all(meta[SupplyCurveField.GID].values - == sorted(meta[SupplyCurveField.GID].values)) + assert all( + meta[SupplyCurveField.GID].values + == sorted(meta[SupplyCurveField.GID].values) + ) ti = fout.time_index assert len(ti) == 8760 assert "time_index-2012" in fout @@ -706,20 +737,29 @@ def test_collect_bespoke(): for fp in source_fps: with Resource(fp) as source: - assert all(np.isin(source.meta[SupplyCurveField.GID].values, - meta[SupplyCurveField.GID].values)) + src_meta = source.meta.rename( + SupplyCurveField.map_from(OldSupplyCurveField), axis=1 + ) + assert all( + np.isin( + src_meta[SupplyCurveField.GID].values, + meta[SupplyCurveField.GID].values, + ) + ) for isource, gid in enumerate( - source.meta[SupplyCurveField.GID].values): - iout = np.where(meta[SupplyCurveField.GID].values == gid)[0] - truth = source['cf_profile-2012', :, isource].flatten() + src_meta[SupplyCurveField.GID].values + ): + iout = np.where(meta[SupplyCurveField.GID].values == gid)[ + 0 + ] + truth = source["cf_profile-2012", :, isource].flatten() test = data[:, iout].flatten() assert np.allclose(truth, test) def test_consistent_eval_namespace(gid=33): """Test that all the same variables are available for every eval.""" - output_request = ('system_capacity', 'cf_mean', - 'cf_profile') + output_request = ("system_capacity", "cf_mean", "cf_profile") cap_cost_fun = "2000" foc_fun = "0" voc_fun = "0" @@ -771,6 +811,9 @@ def test_bespoke_supply_curve(): normal_path = os.path.join(TESTDATADIR, "sc_out/baseline_agg_summary.csv") normal_sc_points = pd.read_csv(normal_path) + normal_sc_points = normal_sc_points.rename( + SupplyCurveField.map_from(OldSupplyCurveField), axis=1 + ) with tempfile.TemporaryDirectory() as td: bespoke_sc_fp = os.path.join(td, "bespoke_out.h5") @@ -791,16 +834,20 @@ def test_bespoke_supply_curve(): sc = SupplyCurve(bespoke_sc_fp, trans_tables) sc_full = sc.full_sort(fcr=0.1, avail_cap_frac=0.1) - assert all(gid in sc_full[SupplyCurveField.SC_GID] - for gid in normal_sc_points[SupplyCurveField.SC_GID]) + assert all( + gid in sc_full[SupplyCurveField.SC_GID] + for gid in normal_sc_points[SupplyCurveField.SC_GID] + ) for _, inp_row in normal_sc_points.iterrows(): sc_gid = inp_row[SupplyCurveField.SC_GID] assert sc_gid in sc_full[SupplyCurveField.SC_GID] test_ind = np.where(sc_full[SupplyCurveField.SC_GID] == sc_gid)[0] assert len(test_ind) == 1 test_row = sc_full.iloc[test_ind] - assert (test_row['total_lcoe'].values[0] - > inp_row[SupplyCurveField.MEAN_LCOE]) + assert ( + test_row["total_lcoe"].values[0] + > inp_row[SupplyCurveField.MEAN_LCOE] + ) fpath_baseline = os.path.join(TESTDATADIR, "sc_out/sc_full_lc.csv") sc_baseline = pd.read_csv(fpath_baseline) @@ -810,8 +857,7 @@ def test_bespoke_supply_curve(): @pytest.mark.parametrize("wlm", [2, 100]) def test_wake_loss_multiplier(wlm): """Test wake loss multiplier.""" - output_request = ('system_capacity', 'cf_mean', - 'cf_profile') + output_request = ("system_capacity", "cf_mean", "cf_profile") with tempfile.TemporaryDirectory() as td: res_fp = os.path.join(td, "ri_100_wtk_{}.h5") excl_fp = os.path.join(td, "ri_exclusions.h5") @@ -887,8 +933,7 @@ def test_wake_loss_multiplier(wlm): def test_bespoke_wind_plant_with_power_curve_losses(): """Test bespoke ``wind_plant`` with power curve losses.""" - output_request = ('system_capacity', 'cf_mean', - 'cf_profile') + output_request = ("system_capacity", "cf_mean", "cf_profile") with tempfile.TemporaryDirectory() as td: res_fp = os.path.join(td, "ri_100_wtk_{}.h5") excl_fp = os.path.join(td, "ri_exclusions.h5") @@ -961,8 +1006,7 @@ def test_bespoke_wind_plant_with_power_curve_losses(): def test_bespoke_run_with_power_curve_losses(): """Test bespoke run with power curve losses.""" - output_request = ('system_capacity', 'cf_mean', - 'cf_profile') + output_request = ("system_capacity", "cf_mean", "cf_profile") with tempfile.TemporaryDirectory() as td: res_fp = os.path.join(td, "ri_100_wtk_{}.h5") excl_fp = os.path.join(td, "ri_exclusions.h5") @@ -1027,8 +1071,7 @@ def test_bespoke_run_with_power_curve_losses(): def test_bespoke_run_with_scheduled_losses(): """Test bespoke run with scheduled losses.""" - output_request = ('system_capacity', 'cf_mean', - 'cf_profile') + output_request = ("system_capacity", "cf_mean", "cf_profile") with tempfile.TemporaryDirectory() as td: res_fp = os.path.join(td, "ri_100_wtk_{}.h5") excl_fp = os.path.join(td, "ri_exclusions.h5") @@ -1058,16 +1101,25 @@ def test_bespoke_run_with_scheduled_losses(): bsp.close() sam_inputs = copy.deepcopy(SAM_SYS_INPUTS) - sam_inputs[ScheduledLossesMixin.OUTAGE_CONFIG_KEY] = [{ - 'name': 'Environmental', - 'count': 115, - 'duration': 2, - 'percentage_of_capacity_lost': 100, - 'allowed_months': ['April', 'May', 'June', 'July', 'August', - 'September', 'October']}] - sam_inputs['hourly'] = [0] * 8760 # only needed for testing - output_request = ('system_capacity', 'cf_mean', - 'cf_profile', 'hourly') + sam_inputs[ScheduledLossesMixin.OUTAGE_CONFIG_KEY] = [ + { + "name": "Environmental", + "count": 115, + "duration": 2, + "percentage_of_capacity_lost": 100, + "allowed_months": [ + "April", + "May", + "June", + "July", + "August", + "September", + "October", + ], + } + ] + sam_inputs["hourly"] = [0] * 8760 # only needed for testing + output_request = ("system_capacity", "cf_mean", "cf_profile", "hourly") bsp = BespokeSinglePlant( 33, @@ -1104,8 +1156,7 @@ def test_bespoke_run_with_scheduled_losses(): def test_bespoke_aep_is_zero_if_no_turbines_placed(): """Test that bespoke aep output is zero if no turbines placed.""" - output_request = ('system_capacity', 'cf_mean', - 'cf_profile') + output_request = ("system_capacity", "cf_mean", "cf_profile") objective_function = "aep" @@ -1153,11 +1204,15 @@ def test_bespoke_prior_run(): single vertical level (e.g., with Sup3rCC data) """ sam_sys_inputs = copy.deepcopy(SAM_SYS_INPUTS) - sam_sys_inputs['fixed_charge_rate'] = 0.096 - sam_configs = {'default': sam_sys_inputs} - output_request = ('system_capacity', 'cf_mean', - 'cf_profile', 'extra_unused_data', - 'lcoe_fcr') + sam_sys_inputs["fixed_charge_rate"] = 0.096 + sam_configs = {"default": sam_sys_inputs} + output_request = ( + "system_capacity", + "cf_mean", + "cf_profile", + "extra_unused_data", + "lcoe_fcr", + ) with tempfile.TemporaryDirectory() as td: out_fpath1 = os.path.join(td, "bespoke_out2.h5") out_fpath2 = os.path.join(td, "bespoke_out1.h5") @@ -1178,9 +1233,13 @@ def test_bespoke_prior_run(): res_fp_2013 = res_fp.format("2013") # gids 33 and 35 are included, 37 is fully excluded - points = pd.DataFrame({SiteDataField.GID: [33], - SiteDataField.CONFIG: ['default'], - 'extra_unused_data': [42]}) + points = pd.DataFrame( + { + SiteDataField.GID: [33], + SiteDataField.CONFIG: ["default"], + "extra_unused_data": [42], + } + ) TechMapping.run(excl_fp, RES.format(2012), dset=TM_DSET, max_workers=1) @@ -1232,9 +1291,14 @@ def test_bespoke_prior_run(): meta2 = f2.meta data2 = {k: f2[k] for k in f2.dsets} - cols = [SupplyCurveField.TURBINE_X_COORDS, SupplyCurveField.TURBINE_Y_COORDS, - SupplyCurveField.CAPACITY, SupplyCurveField.N_GIDS, - SupplyCurveField.GID_COUNTS, SupplyCurveField.RES_GIDS] + cols = [ + SupplyCurveField.TURBINE_X_COORDS, + SupplyCurveField.TURBINE_Y_COORDS, + SupplyCurveField.CAPACITY, + SupplyCurveField.N_GIDS, + SupplyCurveField.GID_COUNTS, + SupplyCurveField.RES_GIDS, + ] pd.testing.assert_frame_equal(meta1[cols], meta2[cols]) # multi-year means should not match the 2nd run with 2013 only. @@ -1255,9 +1319,14 @@ def test_gid_map(): new resource data files for example so you can run forecasted resource with the same spatial configuration.""" - output_request = ('system_capacity', 'cf_mean', - 'cf_profile', 'extra_unused_data', - 'winddirection', 'ws_mean') + output_request = ( + "system_capacity", + "cf_mean", + "cf_profile", + "extra_unused_data", + "winddirection", + "ws_mean", + ) with tempfile.TemporaryDirectory() as td: out_fpath1 = os.path.join(td, "bespoke_out2.h5") out_fpath2 = os.path.join(td, "bespoke_out1.h5") @@ -1269,11 +1338,17 @@ def test_gid_map(): res_fp_2013 = res_fp.format("2013") # gids 33 and 35 are included, 37 is fully excluded - points = pd.DataFrame({SiteDataField.GID: [33], - SiteDataField.CONFIG: ['default'], - 'extra_unused_data': [42]}) + points = pd.DataFrame( + { + SiteDataField.GID: [33], + SiteDataField.CONFIG: ["default"], + "extra_unused_data": [42], + } + ) - gid_map = pd.DataFrame({SupplyCurveField.GID: [3, 4, 13, 12, 11, 10, 9]}) + gid_map = pd.DataFrame( + {SupplyCurveField.GID: [3, 4, 13, 12, 11, 10, 9]} + ) new_gid = 50 gid_map["gid_map"] = new_gid fp_gid_map = os.path.join(td, "gid_map.csv") @@ -1333,8 +1408,11 @@ def test_gid_map(): with Resource(res_fp_2013) as f3: ws = f3[f"windspeed_{hh}m", :, new_gid] - cols = [SupplyCurveField.N_GIDS, SupplyCurveField.GID_COUNTS, - SupplyCurveField.RES_GIDS] + cols = [ + SupplyCurveField.N_GIDS, + SupplyCurveField.GID_COUNTS, + SupplyCurveField.RES_GIDS, + ] pd.testing.assert_frame_equal(meta1[cols], meta2[cols]) assert not np.allclose(data1["cf_mean-2013"], data2["cf_mean-2013"]) @@ -1369,8 +1447,13 @@ def test_gid_map(): def test_bespoke_bias_correct(): """Test bespoke run with bias correction on windspeed data.""" - output_request = ('system_capacity', 'cf_mean', - 'cf_profile', 'extra_unused_data', 'ws_mean') + output_request = ( + "system_capacity", + "cf_mean", + "cf_profile", + "extra_unused_data", + "ws_mean", + ) with tempfile.TemporaryDirectory() as td: out_fpath1 = os.path.join(td, "bespoke_out2.h5") out_fpath2 = os.path.join(td, "bespoke_out1.h5") @@ -1382,16 +1465,22 @@ def test_bespoke_bias_correct(): res_fp_2013 = res_fp.format("2013") # gids 33 and 35 are included, 37 is fully excluded - points = pd.DataFrame({SiteDataField.GID: [33], - SiteDataField.CONFIG: ['default'], - 'extra_unused_data': [42]}) + points = pd.DataFrame( + { + SiteDataField.GID: [33], + SiteDataField.CONFIG: ["default"], + "extra_unused_data": [42], + } + ) # intentionally leaving out WTK gid 13 which only has 5 included 90m # pixels in order to check that this is dynamically patched. - bias_correct = pd.DataFrame({SupplyCurveField.GID: [3, 4, 12, 11, 10, 9]}) - bias_correct['method'] = 'lin_ws' - bias_correct['scalar'] = 0.5 - fp_bc = os.path.join(td, 'bc.csv') + bias_correct = pd.DataFrame( + {SupplyCurveField.GID: [3, 4, 12, 11, 10, 9]} + ) + bias_correct["method"] = "lin_ws" + bias_correct["scalar"] = 0.5 + fp_bc = os.path.join(td, "bc.csv") bias_correct.to_csv(fp_bc) TechMapping.run(excl_fp, RES.format(2013), dset=TM_DSET, max_workers=1) @@ -1444,8 +1533,11 @@ def test_bespoke_bias_correct(): meta2 = f2.meta data2 = {k: f2[k] for k in f2.dsets} - cols = [SupplyCurveField.N_GIDS, SupplyCurveField.GID_COUNTS, - SupplyCurveField.RES_GIDS] + cols = [ + SupplyCurveField.N_GIDS, + SupplyCurveField.GID_COUNTS, + SupplyCurveField.RES_GIDS, + ] pd.testing.assert_frame_equal(meta1[cols], meta2[cols]) assert data1["cf_mean-2013"] * 0.5 > data2["cf_mean-2013"] @@ -1454,9 +1546,14 @@ def test_bespoke_bias_correct(): def test_cli(runner, clear_loggers): """Test bespoke CLI""" - output_request = ('system_capacity', 'cf_mean', - 'cf_profile', 'winddirection', 'windspeed', - 'ws_mean') + output_request = ( + "system_capacity", + "cf_mean", + "cf_profile", + "winddirection", + "windspeed", + "ws_mean", + ) with tempfile.TemporaryDirectory() as td: dirname = os.path.basename(td) @@ -1526,8 +1623,8 @@ def test_cli(runner, clear_loggers): assert SupplyCurveField.SC_POINT_GID in meta assert SupplyCurveField.TURBINE_X_COORDS in meta assert SupplyCurveField.TURBINE_Y_COORDS in meta - assert 'possible_x_coords' in meta - assert 'possible_y_coords' in meta + assert "possible_x_coords" in meta + assert "possible_y_coords" in meta assert SupplyCurveField.RES_GIDS in meta dsets_1d = ( @@ -1563,10 +1660,16 @@ def test_cli(runner, clear_loggers): def test_bespoke_5min_sample(): """Sample a 5min resource dataset for 60min outputs in bespoke""" - output_request = ('system_capacity', 'cf_mean', - 'cf_profile', 'extra_unused_data', - 'winddirection', 'windspeed', 'ws_mean') - tm_dset = 'test_wtk_5min' + output_request = ( + "system_capacity", + "cf_mean", + "cf_profile", + "extra_unused_data", + "winddirection", + "windspeed", + "ws_mean", + ) + tm_dset = "test_wtk_5min" with tempfile.TemporaryDirectory() as td: out_fpath = os.path.join(td, "wind_bespoke.h5") @@ -1574,17 +1677,22 @@ def test_bespoke_5min_sample(): shutil.copy(EXCL, excl_fp) res_fp = os.path.join(TESTDATADIR, "wtk/wtk_2010_*m.h5") - points = pd.DataFrame({SiteDataField.GID: [33, 35], - SiteDataField.CONFIG: ['default'] * 2, - 'extra_unused_data': [0, 42]}) + points = pd.DataFrame( + { + SiteDataField.GID: [33, 35], + SiteDataField.CONFIG: ["default"] * 2, + "extra_unused_data": [0, 42], + } + ) sam_sys_inputs = copy.deepcopy(SAM_SYS_INPUTS) sam_sys_inputs["time_index_step"] = 12 sam_configs = {"default": sam_sys_inputs} # hack techmap because 5min data only has 10 wind resource pixels - with h5py.File(excl_fp, 'a') as excl_file: - arr = np.random.choice(10, - size=excl_file[SupplyCurveField.LATITUDE].shape) + with h5py.File(excl_fp, "a") as excl_file: + arr = np.random.choice( + 10, size=excl_file[SupplyCurveField.LATITUDE].shape + ) excl_file.create_dataset(name=tm_dset, data=arr) bsp = BespokeWindPlants( From a8ab4c8e47f1e26cf180509235bb2b15ea10bc0c Mon Sep 17 00:00:00 2001 From: "Brandon N. Benton" Date: Tue, 28 May 2024 19:02:32 -0600 Subject: [PATCH 32/61] all bespoke tests passing. exclusion file in `test_bespoke_5min_sample` is fixed so we access with 'latitude'. --- reV/bespoke/bespoke.py | 8 +++- reV/generation/generation.py | 93 ++++++++++++++++++++++-------------- tests/test_bespoke.py | 4 +- 3 files changed, 65 insertions(+), 40 deletions(-) diff --git a/reV/bespoke/bespoke.py b/reV/bespoke/bespoke.py index a921b9639..64a04c26a 100644 --- a/reV/bespoke/bespoke.py +++ b/reV/bespoke/bespoke.py @@ -584,13 +584,17 @@ def _parse_gid_map(gid_map): if isinstance(gid_map, str): if gid_map.endswith(".csv"): - gid_map = pd.read_csv(gid_map).to_dict() + gid_map = ( + pd.read_csv(gid_map) + .rename(SupplyCurveField.map_to(ResourceMetaField), axis=1) + .to_dict() + ) err_msg = f"Need {ResourceMetaField.GID} in gid_map column" assert ResourceMetaField.GID in gid_map, err_msg assert "gid_map" in gid_map, 'Need "gid_map" in gid_map column' gid_map = { gid_map[ResourceMetaField.GID][i]: gid_map["gid_map"][i] - for i in gid_map[ResourceMetaField.GID].keys() + for i in gid_map[ResourceMetaField.GID] } elif gid_map.endswith(".json"): diff --git a/reV/generation/generation.py b/reV/generation/generation.py index 985009755..5e9a2e274 100644 --- a/reV/generation/generation.py +++ b/reV/generation/generation.py @@ -30,7 +30,7 @@ TroughPhysicalHeat, WindPower, ) -from reV.utilities import ModuleName, ResourceMetaField +from reV.utilities import ModuleName, ResourceMetaField, SupplyCurveField from reV.utilities.exceptions import ( ConfigError, InputError, @@ -41,16 +41,16 @@ ATTR_DIR = os.path.dirname(os.path.realpath(__file__)) -ATTR_DIR = os.path.join(ATTR_DIR, 'output_attributes') -with open(os.path.join(ATTR_DIR, 'other.json')) as f: +ATTR_DIR = os.path.join(ATTR_DIR, "output_attributes") +with open(os.path.join(ATTR_DIR, "other.json")) as f: OTHER_ATTRS = json.load(f) -with open(os.path.join(ATTR_DIR, 'generation.json')) as f: +with open(os.path.join(ATTR_DIR, "generation.json")) as f: GEN_ATTRS = json.load(f) -with open(os.path.join(ATTR_DIR, 'linear_fresnel.json')) as f: +with open(os.path.join(ATTR_DIR, "linear_fresnel.json")) as f: LIN_ATTRS = json.load(f) -with open(os.path.join(ATTR_DIR, 'solar_water_heat.json')) as f: +with open(os.path.join(ATTR_DIR, "solar_water_heat.json")) as f: SWH_ATTRS = json.load(f) -with open(os.path.join(ATTR_DIR, 'trough_heat.json')) as f: +with open(os.path.join(ATTR_DIR, "trough_heat.json")) as f: TPPH_ATTRS = json.load(f) @@ -83,13 +83,24 @@ class Gen(BaseGen): OUT_ATTRS.update(TPPH_ATTRS) OUT_ATTRS.update(BaseGen.ECON_ATTRS) - def __init__(self, technology, project_points, sam_files, resource_file, - low_res_resource_file=None, - output_request=('cf_mean',), - site_data=None, curtailment=None, gid_map=None, - drop_leap=False, sites_per_worker=None, - memory_utilization_limit=0.4, scale_outputs=True, - write_mapped_gids=False, bias_correct=None): + def __init__( + self, + technology, + project_points, + sam_files, + resource_file, + low_res_resource_file=None, + output_request=("cf_mean",), + site_data=None, + curtailment=None, + gid_map=None, + drop_leap=False, + sites_per_worker=None, + memory_utilization_limit=0.4, + scale_outputs=True, + write_mapped_gids=False, + bias_correct=None, + ): """ReV generation analysis class. ``reV`` generation analysis runs SAM simulations by piping in @@ -479,7 +490,7 @@ def meta(self): self._meta.loc[:, ResourceMetaField.GID] = sites self._meta.index = self.project_points.sites self._meta.index.name = ResourceMetaField.GID - self._meta.loc[:, 'reV_tech'] = self.project_points.tech + self._meta.loc[:, "reV_tech"] = self.project_points.tech return self._meta @@ -569,8 +580,7 @@ def handle_lifetime_index(self, ti): array_vars = [ var for var, attrs in GEN_ATTRS.items() if attrs["type"] == "array" ] - valid_vars = ['gen_profile', 'cf_profile', - 'cf_profile_ac'] + valid_vars = ["gen_profile", "cf_profile", "cf_profile_ac"] invalid_vars = set(array_vars) - set(valid_vars) invalid_requests = [ var for var in self.output_request if var in invalid_vars @@ -746,14 +756,15 @@ def _parse_gid_map(self, gid_map): if isinstance(gid_map, str): if gid_map.endswith(".csv"): gid_map = pd.read_csv(gid_map).to_dict() - msg = f'Need {ResourceMetaField.GID} in gid_map column' + msg = f"Need {ResourceMetaField.GID} in gid_map column" assert ResourceMetaField.GID in gid_map, msg - assert 'gid_map' in gid_map, 'Need "gid_map" in gid_map column' + assert "gid_map" in gid_map, 'Need "gid_map" in gid_map column' gid_map = { - gid_map[ResourceMetaField.GID][i]: gid_map['gid_map'][i] - for i in gid_map[ResourceMetaField.GID].keys()} + gid_map[ResourceMetaField.GID][i]: gid_map["gid_map"][i] + for i in gid_map[ResourceMetaField.GID].keys() + } - elif gid_map.endswith('.json'): + elif gid_map.endswith(".json"): with open(gid_map) as f: gid_map = json.load(f) @@ -816,14 +827,20 @@ def _parse_nn_map(self): if "*" in self.res_file or "*" in self.lr_res_file: handler_class = MultiFileResource - with handler_class(self.res_file) as hr_res, \ - handler_class(self.lr_res_file) as lr_res: - logger.info('Making nearest neighbor map for multi ' - 'resolution resource data...') - nn_d, nn_map = MultiResolutionResource.make_nn_map(hr_res, - lr_res) - logger.info('Done making nearest neighbor map for multi ' - 'resolution resource data!') + with handler_class(self.res_file) as hr_res, handler_class( + self.lr_res_file + ) as lr_res: + logger.info( + "Making nearest neighbor map for multi " + "resolution resource data..." + ) + nn_d, nn_map = MultiResolutionResource.make_nn_map( + hr_res, lr_res + ) + logger.info( + "Done making nearest neighbor map for multi " + "resolution resource data!" + ) logger.info( "Made nearest neighbor mapping between nominal-" @@ -886,7 +903,9 @@ def _parse_bc(bias_correct): return bias_correct if isinstance(bias_correct, str): - bias_correct = pd.read_csv(bias_correct) + bias_correct = pd.read_csv(bias_correct).rename( + SupplyCurveField.map_to(ResourceMetaField), axis=1 + ) msg = ( "Bias correction data must be a filepath to csv or a dataframe " @@ -894,10 +913,14 @@ def _parse_bc(bias_correct): ) assert isinstance(bias_correct, pd.DataFrame), msg - msg = ('Bias correction table must have {!r} column but only found: ' - '{}'.format(ResourceMetaField.GID, list(bias_correct.columns))) - assert (ResourceMetaField.GID in bias_correct - or bias_correct.index.name == ResourceMetaField.GID), msg + msg = ( + "Bias correction table must have {!r} column but only found: " + "{}".format(ResourceMetaField.GID, list(bias_correct.columns)) + ) + assert ( + ResourceMetaField.GID in bias_correct + or bias_correct.index.name == ResourceMetaField.GID + ), msg if bias_correct.index.name != ResourceMetaField.GID: bias_correct = bias_correct.set_index(ResourceMetaField.GID) diff --git a/tests/test_bespoke.py b/tests/test_bespoke.py index d435f2b67..113ca1152 100644 --- a/tests/test_bespoke.py +++ b/tests/test_bespoke.py @@ -1690,9 +1690,7 @@ def test_bespoke_5min_sample(): # hack techmap because 5min data only has 10 wind resource pixels with h5py.File(excl_fp, "a") as excl_file: - arr = np.random.choice( - 10, size=excl_file[SupplyCurveField.LATITUDE].shape - ) + arr = np.random.choice(10, size=excl_file["latitude"].shape) excl_file.create_dataset(name=tm_dset, data=arr) bsp = BespokeWindPlants( From 51c12d801ad89a23d95132aeaf7f6a45e0ed2b13 Mon Sep 17 00:00:00 2001 From: "Brandon N. Benton" Date: Tue, 28 May 2024 20:20:01 -0600 Subject: [PATCH 33/61] need conditional axis selection for field renaming in tz_elev_check. single site vs dataframe --- reV/SAM/generation.py | 5 +++-- tests/test_curtailment.py | 27 +++++++++++++++++---------- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/reV/SAM/generation.py b/reV/SAM/generation.py index ee975ffed..d8eaf3f59 100644 --- a/reV/SAM/generation.py +++ b/reV/SAM/generation.py @@ -257,13 +257,14 @@ def tz_elev_check(sam_sys_inputs, site_sys_inputs, meta): Returns ------- meta : pd.DataFrame | pd.Series - Datafram or series for a single site. Will include "timezone" + Dataframe or series for a single site. Will include "timezone" and "elevation" from the sam and site system inputs if found. """ if meta is not None: + axis = 0 if isinstance(meta, pd.core.series.Series) else 1 meta = meta.rename( - SupplyCurveField.map_to(ResourceMetaField), axis=1 + SupplyCurveField.map_to(ResourceMetaField), axis=axis ) if sam_sys_inputs is not None: if ResourceMetaField.ELEVATION in sam_sys_inputs: diff --git a/tests/test_curtailment.py b/tests/test_curtailment.py index 57343fb06..51bc865ab 100644 --- a/tests/test_curtailment.py +++ b/tests/test_curtailment.py @@ -5,6 +5,7 @@ @author: gbuster """ + import os from copy import deepcopy @@ -63,10 +64,16 @@ def test_cf_curtailment(year, site): points = slice(site, site + 1) # run reV 2.0 generation - gen = Gen('windpower', points, sam_files, res_file, - output_request=('cf_profile',), - curtailment=curtailment, - sites_per_worker=50, scale_outputs=True) + gen = Gen( + "windpower", + points, + sam_files, + res_file, + output_request=("cf_profile",), + curtailment=curtailment, + sites_per_worker=50, + scale_outputs=True, + ) gen.run(max_workers=1) results, check_curtailment = test_res_curtailment(year, site=site) results["cf_profile"] = gen.out["cf_profile"].flatten() @@ -83,8 +90,6 @@ def test_cf_curtailment(year, site): ) assert np.sum(check) == 0, msg - return results - @pytest.mark.parametrize("year", ["2012", "2013"]) def test_curtailment_res_mean(year): @@ -200,8 +205,10 @@ def test_res_curtailment(year, site): sza = SolarPosition( non_curtailed_res.time_index, - non_curtailed_res.meta[[ResourceMetaField.LATITUDE, - ResourceMetaField.LONGITUDE]].values).zenith + non_curtailed_res.meta[ + [ResourceMetaField.LATITUDE, ResourceMetaField.LONGITUDE] + ].values, + ).zenith ti = non_curtailed_res.time_index @@ -285,8 +292,8 @@ def test_eqn_curtailment(plot=False): nc_res = non_curtailed_res[0] c_mask = (c_res.windspeed == 0) & (nc_res.windspeed > 0) - temperature = nc_res['temperature'].values - wind_speed = nc_res['windspeed'].values + temperature = nc_res["temperature"].values + wind_speed = nc_res["windspeed"].values eval_mask = eval(c_eqn) From d3142b78642a798277bc39e6361540270c1e051d Mon Sep 17 00:00:00 2001 From: Paul Pinchuk Date: Wed, 29 May 2024 11:11:41 -0600 Subject: [PATCH 34/61] Don't use Enum in pytest parametrize --- tests/test_handlers_transmission.py | 2 +- tests/test_supply_curve_aggregation_friction.py | 2 +- tests/test_supply_curve_points.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_handlers_transmission.py b/tests/test_handlers_transmission.py index c72501453..8fc68a80d 100644 --- a/tests/test_handlers_transmission.py +++ b/tests/test_handlers_transmission.py @@ -86,7 +86,7 @@ def trans_table(): return trans_table -@pytest.mark.parametrize(('i', 'trans_costs', 'distance', SupplyCurveField.GID), +@pytest.mark.parametrize(('i', 'trans_costs', 'distance', 'gid'), ((1, TRANS_COSTS_1, 0, 43300), (2, TRANS_COSTS_2, 0, 43300), (1, TRANS_COSTS_1, 100, 43300), diff --git a/tests/test_supply_curve_aggregation_friction.py b/tests/test_supply_curve_aggregation_friction.py index 0298e1b5a..47751be4f 100644 --- a/tests/test_supply_curve_aggregation_friction.py +++ b/tests/test_supply_curve_aggregation_friction.py @@ -64,7 +64,7 @@ def test_friction_mask(): assert diff < 0.0001, m -@pytest.mark.parametrize(SupplyCurveField.GID, [100, 114, 130, 181]) +@pytest.mark.parametrize("gid", [100, 114, 130, 181]) def test_agg_friction(gid): """Test SC Aggregation with friction by checking friction factors and LCOE against a hand calc.""" diff --git a/tests/test_supply_curve_points.py b/tests/test_supply_curve_points.py index 727856f43..a6842aa92 100644 --- a/tests/test_supply_curve_points.py +++ b/tests/test_supply_curve_points.py @@ -68,7 +68,7 @@ def test_slicer(gids, resolution): @pytest.mark.parametrize( - (SupplyCurveField.GID, "resolution", "excl_dict", "time_series"), + ("gid", "resolution", "excl_dict", "time_series"), [ (37, 64, None, None), (37, 64, EXCL_DICT, None), @@ -109,7 +109,7 @@ def test_weighted_means(gid, resolution, excl_dict, time_series): @pytest.mark.parametrize( - (SupplyCurveField.GID, "resolution", "excl_dict", "time_series"), + ("gid", "resolution", "excl_dict", "time_series"), [ (37, 64, None, None), (37, 64, EXCL_DICT, None), From ce68fec19d073aa542fc171792dfb429832dcb7e Mon Sep 17 00:00:00 2001 From: Paul Pinchuk Date: Wed, 29 May 2024 11:12:46 -0600 Subject: [PATCH 35/61] Rename meta columns when reading data --- tests/test_hybrids.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/test_hybrids.py b/tests/test_hybrids.py index 9999373c9..b7b27c281 100644 --- a/tests/test_hybrids.py +++ b/tests/test_hybrids.py @@ -13,7 +13,7 @@ from reV.cli import main from reV.hybrids import HYBRID_METHODS, Hybridization from reV.hybrids.hybrids import MERGE_COLUMN, OUTPUT_PROFILE_NAMES, HybridsData -from reV.utilities import ModuleName, SupplyCurveField +from reV.utilities import ModuleName, SupplyCurveField, OldSupplyCurveField from reV.utilities.exceptions import FileInputError, InputError, OutputWarning SOLAR_FPATH = os.path.join( @@ -29,9 +29,15 @@ TESTDATADIR, "rep_profiles_out", "rep_profiles_solar_multiple.h5" ) with Resource(SOLAR_FPATH) as res: - SOLAR_SCPGIDS = set(res.meta[SupplyCurveField.SC_POINT_GID]) + meta = res.meta.rename( + columns=SupplyCurveField.map_from(OldSupplyCurveField) + ) + SOLAR_SCPGIDS = set(meta[SupplyCurveField.SC_POINT_GID]) with Resource(WIND_FPATH) as res: - WIND_SCPGIDS = set(res.meta[SupplyCurveField.SC_POINT_GID]) + meta = res.meta.rename( + columns=SupplyCurveField.map_from(OldSupplyCurveField) + ) + WIND_SCPGIDS = set(meta[SupplyCurveField.SC_POINT_GID]) def test_hybridization_profile_output_single_resource(): @@ -57,7 +63,9 @@ def test_hybridization_profile_output_single_resource(): hwp, ) = h.profiles.values() h_meta = h.hybrid_meta - h_idx = np.where(h_meta[SupplyCurveField.SC_POINT_GID] == sc_point_gid)[0][0] + h_idx = np.where( + h_meta[SupplyCurveField.SC_POINT_GID] == sc_point_gid + )[0][0] assert np.allclose(hp[:, h_idx], weighted_solar) assert np.allclose(hsp[:, h_idx], weighted_solar) From 36d00ef18c6a172cef6564103baf2b45034fc23a Mon Sep 17 00:00:00 2001 From: Paul Pinchuk Date: Wed, 29 May 2024 11:14:47 -0600 Subject: [PATCH 36/61] Don't use enum in pytest parametrize pt 2 --- tests/test_handlers_transmission.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_handlers_transmission.py b/tests/test_handlers_transmission.py index 8fc68a80d..eade43156 100644 --- a/tests/test_handlers_transmission.py +++ b/tests/test_handlers_transmission.py @@ -116,8 +116,7 @@ def test_cost_calculation(i, trans_costs, distance, gid, trans_table): assert true_cost == trans_cost -@pytest.mark.parametrize(('trans_costs', SupplyCurveField.CAPACITY, - SupplyCurveField.GID), +@pytest.mark.parametrize(('trans_costs', "capacity", "gid"), ((TRANS_COSTS_1, 350, 43300), (TRANS_COSTS_2, 350, 43300), (TRANS_COSTS_1, 100, 43300), From 3b8deff95823bffa984d5ac59e0dd8346e96fe3a Mon Sep 17 00:00:00 2001 From: "Brandon N. Benton" Date: Wed, 29 May 2024 11:42:55 -0600 Subject: [PATCH 37/61] using `.name.lower` to match fixed sc point attributes --- reV/econ/economies_of_scale.py | 81 ++++++++++++++++++++++----------- reV/supply_curve/aggregation.py | 14 ++++-- reV/supply_curve/points.py | 2 +- reV/utilities/__init__.py | 1 + 4 files changed, 67 insertions(+), 31 deletions(-) diff --git a/reV/econ/economies_of_scale.py b/reV/econ/economies_of_scale.py index 022b262eb..38f0e75ed 100644 --- a/reV/econ/economies_of_scale.py +++ b/reV/econ/economies_of_scale.py @@ -3,16 +3,20 @@ reV module for calculating economies of scale where larger power plants will have reduced capital cost. """ + import copy import logging import re # pylint: disable=unused-import -import numpy as np import pandas as pd from rex.utilities.utilities import check_eval_str from reV.econ.utilities import lcoe_fcr +from reV.utilities import ( + OldSupplyCurveField, + SupplyCurveField, +) logger = logging.getLogger(__name__) @@ -59,23 +63,32 @@ def _preflight(self): check_eval_str(str(self._eqn)) if isinstance(self._data, pd.DataFrame): - self._data = {k: self._data[k].values.flatten() - for k in self._data.columns} + self._data = { + k: self._data[k].values.flatten() for k in self._data.columns + } if not isinstance(self._data, dict): - e = ('Cannot evaluate EconomiesOfScale with data input of type: {}' - .format(type(self._data))) + e = ( + "Cannot evaluate EconomiesOfScale with data input of type: " + "{}".format(type(self._data)) + ) + logger.error(e) raise TypeError(e) missing = [name for name in self.vars if name not in self._data] if any(missing): - e = ('Cannot evaluate EconomiesOfScale, missing data for variables' - ': {} for equation: {}'.format(missing, self._eqn)) + e = ( + "Cannot evaluate EconomiesOfScale, missing data for variables" + ": {} for equation: {}".format(missing, self._eqn) + ) logger.error(e) raise KeyError(e) + rename_map = SupplyCurveField.map_from(OldSupplyCurveField) + self._data = {rename_map.get(k, k): v for k, v in self._data.items()} + @staticmethod def is_num(s): """Check if a string is a number""" @@ -89,7 +102,7 @@ def is_num(s): @staticmethod def is_method(s): """Check if a string is a numpy/pandas or python builtin method""" - return bool(s.startswith(('np.', 'pd.')) or s in dir(__builtins__)) + return bool(s.startswith(("np.", "pd.")) or s in dir(__builtins__)) @property def vars(self): @@ -105,8 +118,8 @@ def vars(self): """ var_names = [] if self._eqn is not None: - delimiters = ('*', '/', '+', '-', ' ', '(', ')', '[', ']', ',') - regex_pattern = '|'.join(map(re.escape, delimiters)) + delimiters = ("*", "/", "+", "-", " ", "(", ")", "[", "]", ",") + regex_pattern = "|".join(map(re.escape, delimiters)) var_names = [] for sub in re.split(regex_pattern, str(self._eqn)): if sub and not self.is_num(sub) and not self.is_method(sub): @@ -160,9 +173,10 @@ def _get_prioritized_keys(input_dict, key_list): break if out is None: - e = ('Could not find requested key list ({}) in the input ' - 'dictionary keys: {}' - .format(key_list, list(input_dict.keys()))) + e = ( + "Could not find requested key list ({}) in the input " + "dictionary keys: {}".format(key_list, list(input_dict.keys())) + ) logger.error(e) raise KeyError(e) @@ -190,7 +204,10 @@ def raw_capital_cost(self): out : float | np.ndarray Unscaled (raw) capital_cost found in the data input arg. """ - key_list = ['capital_cost', 'mean_capital_cost'] + key_list = [ + SupplyCurveField.CAPITAL_COST, + "mean_capital_cost", + ] return self._get_prioritized_keys(self._data, key_list) @property @@ -217,7 +234,7 @@ def system_capacity(self): ------- out : float | np.ndarray """ - key_list = ['system_capacity', 'mean_system_capacity'] + key_list = ["system_capacity", "mean_system_capacity"] return self._get_prioritized_keys(self._data, key_list) @property @@ -229,9 +246,12 @@ def fcr(self): out : float | np.ndarray Fixed charge rate from input data arg """ - key_list = ['fixed_charge_rate', - 'mean_fixed_charge_rate', - 'fcr', 'mean_fcr'] + key_list = [ + SupplyCurveField.FIXED_CHARGE_RATE, + "mean_fixed_charge_rate", + "fcr", + "mean_fcr", + ] return self._get_prioritized_keys(self._data, key_list) @property @@ -243,9 +263,12 @@ def foc(self): out : float | np.ndarray Fixed operating cost from input data arg """ - key_list = ['fixed_operating_cost', - 'mean_fixed_operating_cost', - 'foc', 'mean_foc'] + key_list = [ + SupplyCurveField.FIXED_OPERATING_COST, + "mean_fixed_operating_cost", + "foc", + "mean_foc", + ] return self._get_prioritized_keys(self._data, key_list) @property @@ -257,9 +280,12 @@ def voc(self): out : float | np.ndarray Variable operating cost from input data arg """ - key_list = ['variable_operating_cost', - 'mean_variable_operating_cost', - 'voc', 'mean_voc'] + key_list = [ + SupplyCurveField.VARIABLE_OPERATING_COST, + "mean_variable_operating_cost", + "voc", + "mean_voc", + ] return self._get_prioritized_keys(self._data, key_list) @property @@ -285,7 +311,7 @@ def raw_lcoe(self): ------- lcoe : float | np.ndarray """ - key_list = ["raw_lcoe", "mean_lcoe"] + key_list = [SupplyCurveField.RAW_LCOE, SupplyCurveField.MEAN_LCOE] return copy.deepcopy(self._get_prioritized_keys(self._data, key_list)) @property @@ -301,5 +327,6 @@ def scaled_lcoe(self): LCOE calculated with the scaled capital cost based on the EconomiesOfScale input equation. """ - return lcoe_fcr(self.fcr, self.scaled_capital_cost, self.foc, - self.aep, self.voc) + return lcoe_fcr( + self.fcr, self.scaled_capital_cost, self.foc, self.aep, self.voc + ) diff --git a/reV/supply_curve/aggregation.py b/reV/supply_curve/aggregation.py index 629c80d0b..0d7850bef 100644 --- a/reV/supply_curve/aggregation.py +++ b/reV/supply_curve/aggregation.py @@ -18,7 +18,7 @@ from reV.supply_curve.extent import SupplyCurveExtent from reV.supply_curve.points import AggregationSupplyCurvePoint from reV.supply_curve.tech_mapping import TechMapping -from reV.utilities import SupplyCurveField, log_versions +from reV.utilities import ResourceMetaField, SupplyCurveField, log_versions from reV.utilities.exceptions import ( EmptySupplyCurvePointError, FileInputError, @@ -472,12 +472,17 @@ def _parse_gen_index(gen_fpath): logger.error(msg) raise FileInputError(msg) + gen_index = gen_index.rename( + ResourceMetaField.map_to(SupplyCurveField), axis=1 + ) if SupplyCurveField.GID in gen_index: gen_index = gen_index.rename( columns={SupplyCurveField.GID: SupplyCurveField.RES_GIDS} ) gen_index[SupplyCurveField.GEN_GIDS] = gen_index.index - gen_index = gen_index[[SupplyCurveField.RES_GIDS, SupplyCurveField.GEN_GIDS]] + gen_index = gen_index[ + [SupplyCurveField.RES_GIDS, SupplyCurveField.GEN_GIDS] + ] gen_index = gen_index.set_index(keys=SupplyCurveField.RES_GIDS) gen_index = gen_index.reindex( range(int(gen_index.index.max() + 1)) @@ -696,7 +701,10 @@ def run_serial( "area_filter_kernel": area_filter_kernel, "min_area": min_area, } - dsets = (*agg_dset, "meta",) + dsets = ( + *agg_dset, + "meta", + ) agg_out = {ds: [] for ds in dsets} with AggFileHandler(excl_fpath, h5_fpath, **file_kwargs) as fh: n_finished = 0 diff --git a/reV/supply_curve/points.py b/reV/supply_curve/points.py index d9620d9d4..093f31b9b 100644 --- a/reV/supply_curve/points.py +++ b/reV/supply_curve/points.py @@ -2219,7 +2219,7 @@ def point_summary(self, args=None): SupplyCurveField.SC_POINT_ANNUAL_ENERGY_AC, ] for attr in extra_atts: - value = getattr(self, attr) + value = getattr(self, attr.name.lower()) if value is not None: ARGS[attr] = value diff --git a/reV/utilities/__init__.py b/reV/utilities/__init__.py index 3ce30d6b2..deeea147c 100644 --- a/reV/utilities/__init__.py +++ b/reV/utilities/__init__.py @@ -86,6 +86,7 @@ class ResourceMetaField(FieldEnum): "sc_row_ind": "SC_ROW_IND", "sc_col_ind": "SC_COL_IND", "mean_cf": "MEAN_CF", + "capital_cost": "CAPITAL_COST", } From 4687ed62afb2a69aebb9bec4851adda590af3dd4 Mon Sep 17 00:00:00 2001 From: Paul Pinchuk Date: Wed, 29 May 2024 11:47:43 -0600 Subject: [PATCH 38/61] Use supply curve field names for lookup --- reV/econ/economies_of_scale.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/reV/econ/economies_of_scale.py b/reV/econ/economies_of_scale.py index 022b262eb..bb723f5f2 100644 --- a/reV/econ/economies_of_scale.py +++ b/reV/econ/economies_of_scale.py @@ -13,6 +13,7 @@ from rex.utilities.utilities import check_eval_str from reV.econ.utilities import lcoe_fcr +from reV.utilities import SupplyCurveField logger = logging.getLogger(__name__) @@ -190,7 +191,7 @@ def raw_capital_cost(self): out : float | np.ndarray Unscaled (raw) capital_cost found in the data input arg. """ - key_list = ['capital_cost', 'mean_capital_cost'] + key_list = [SupplyCurveField.CAPITAL_COST, 'mean_capital_cost'] return self._get_prioritized_keys(self._data, key_list) @property @@ -229,7 +230,7 @@ def fcr(self): out : float | np.ndarray Fixed charge rate from input data arg """ - key_list = ['fixed_charge_rate', + key_list = [SupplyCurveField.FIXED_CHARGE_RATE, 'mean_fixed_charge_rate', 'fcr', 'mean_fcr'] return self._get_prioritized_keys(self._data, key_list) @@ -243,7 +244,7 @@ def foc(self): out : float | np.ndarray Fixed operating cost from input data arg """ - key_list = ['fixed_operating_cost', + key_list = [SupplyCurveField.FIXED_OPERATING_COST, 'mean_fixed_operating_cost', 'foc', 'mean_foc'] return self._get_prioritized_keys(self._data, key_list) @@ -257,7 +258,7 @@ def voc(self): out : float | np.ndarray Variable operating cost from input data arg """ - key_list = ['variable_operating_cost', + key_list = [SupplyCurveField.VARIABLE_OPERATING_COST, 'mean_variable_operating_cost', 'voc', 'mean_voc'] return self._get_prioritized_keys(self._data, key_list) @@ -285,7 +286,7 @@ def raw_lcoe(self): ------- lcoe : float | np.ndarray """ - key_list = ["raw_lcoe", "mean_lcoe"] + key_list = [SupplyCurveField.RAW_LCOE, SupplyCurveField.MEAN_LCOE] return copy.deepcopy(self._get_prioritized_keys(self._data, key_list)) @property From f604e810b2b9c79ae70e5fe6f7d74e2c7c96d749 Mon Sep 17 00:00:00 2001 From: Paul Pinchuk Date: Wed, 29 May 2024 11:50:06 -0600 Subject: [PATCH 39/61] `extra_atts` now agnostic to changes in field values --- reV/supply_curve/points.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/reV/supply_curve/points.py b/reV/supply_curve/points.py index d9620d9d4..a5c667193 100644 --- a/reV/supply_curve/points.py +++ b/reV/supply_curve/points.py @@ -2210,16 +2210,21 @@ def point_summary(self, args=None): SupplyCurveField.AREA_SQ_KM: self.area, } - extra_atts = [ - SupplyCurveField.CAPACITY_AC, - SupplyCurveField.OFFSHORE, - SupplyCurveField.SC_POINT_CAPITAL_COST, - SupplyCurveField.SC_POINT_FIXED_OPERATING_COST, - SupplyCurveField.SC_POINT_ANNUAL_ENERGY, - SupplyCurveField.SC_POINT_ANNUAL_ENERGY_AC, - ] - for attr in extra_atts: - value = getattr(self, attr) + extra_atts = { + SupplyCurveField.CAPACITY_AC: self.capacity_ac, + SupplyCurveField.OFFSHORE: self.offshore, + SupplyCurveField.SC_POINT_CAPITAL_COST: self.sc_point_capital_cost, + SupplyCurveField.SC_POINT_FIXED_OPERATING_COST: ( + self.sc_point_fixed_operating_cost + ), + SupplyCurveField.SC_POINT_ANNUAL_ENERGY: ( + self.sc_point_annual_energy + ), + SupplyCurveField.SC_POINT_ANNUAL_ENERGY_AC: ( + self.sc_point_annual_energy_ac + ), + } + for attr, value in extra_atts.items(): if value is not None: ARGS[attr] = value From 073f22769cd9b0e1fa722038184acb4a3cfa4d6e Mon Sep 17 00:00:00 2001 From: Paul Pinchuk Date: Wed, 29 May 2024 11:50:27 -0600 Subject: [PATCH 40/61] Use `ResourceMetaField` when looking at gen file meta columns --- reV/supply_curve/aggregation.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/reV/supply_curve/aggregation.py b/reV/supply_curve/aggregation.py index 629c80d0b..a4f7d53d2 100644 --- a/reV/supply_curve/aggregation.py +++ b/reV/supply_curve/aggregation.py @@ -18,7 +18,7 @@ from reV.supply_curve.extent import SupplyCurveExtent from reV.supply_curve.points import AggregationSupplyCurvePoint from reV.supply_curve.tech_mapping import TechMapping -from reV.utilities import SupplyCurveField, log_versions +from reV.utilities import ResourceMetaField, SupplyCurveField, log_versions from reV.utilities.exceptions import ( EmptySupplyCurvePointError, FileInputError, @@ -472,12 +472,14 @@ def _parse_gen_index(gen_fpath): logger.error(msg) raise FileInputError(msg) - if SupplyCurveField.GID in gen_index: + if ResourceMetaField.GID in gen_index: gen_index = gen_index.rename( - columns={SupplyCurveField.GID: SupplyCurveField.RES_GIDS} + columns={ResourceMetaField.GID: SupplyCurveField.RES_GIDS} ) gen_index[SupplyCurveField.GEN_GIDS] = gen_index.index - gen_index = gen_index[[SupplyCurveField.RES_GIDS, SupplyCurveField.GEN_GIDS]] + gen_index = gen_index[ + [SupplyCurveField.RES_GIDS, SupplyCurveField.GEN_GIDS] + ] gen_index = gen_index.set_index(keys=SupplyCurveField.RES_GIDS) gen_index = gen_index.reindex( range(int(gen_index.index.max() + 1)) From 078bebc75b5a6af1d578dc582cfe83910815c782 Mon Sep 17 00:00:00 2001 From: Paul Pinchuk Date: Wed, 29 May 2024 11:50:32 -0600 Subject: [PATCH 41/61] Fix tests --- tests/test_econ_of_scale.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/test_econ_of_scale.py b/tests/test_econ_of_scale.py index 1779ed4f3..9532aa49e 100644 --- a/tests/test_econ_of_scale.py +++ b/tests/test_econ_of_scale.py @@ -81,48 +81,47 @@ def test_lcoe_calc_simple(): # from pvwattsv7 defaults data = { "aep": 35188456.00, - "capital_cost": 53455000.00, + SupplyCurveField.CAPITAL_COST: 53455000.00, "foc": 360000.00, "voc": 0, "fcr": 0.096, } - true_lcoe = (data["fcr"] * data["capital_cost"] + data["foc"]) / ( - data["aep"] / 1000 - ) + true_lcoe = (data["fcr"] * data[SupplyCurveField.CAPITAL_COST] + + data["foc"]) / (data["aep"] / 1000) data[SupplyCurveField.MEAN_LCOE] = true_lcoe eos = EconomiesOfScale(eqn, data) assert eos.raw_capital_cost == eos.scaled_capital_cost - assert eos.raw_capital_cost == data["capital_cost"] + assert eos.raw_capital_cost == data[SupplyCurveField.CAPITAL_COST] assert np.allclose(eos.raw_lcoe, true_lcoe, rtol=0.001) assert np.allclose(eos.scaled_lcoe, true_lcoe, rtol=0.001) eqn = 1 eos = EconomiesOfScale(eqn, data) assert eos.raw_capital_cost == eos.scaled_capital_cost - assert eos.raw_capital_cost == data["capital_cost"] + assert eos.raw_capital_cost == data[SupplyCurveField.CAPITAL_COST] assert np.allclose(eos.raw_lcoe, true_lcoe, rtol=0.001) assert np.allclose(eos.scaled_lcoe, true_lcoe, rtol=0.001) eqn = 2 - true_scaled = ((data['fcr'] * eqn * data['capital_cost'] + true_scaled = ((data['fcr'] * eqn * data[SupplyCurveField.CAPITAL_COST] + data['foc']) / (data['aep'] / 1000)) eos = EconomiesOfScale(eqn, data) assert eqn * eos.raw_capital_cost == eos.scaled_capital_cost - assert eos.raw_capital_cost == data["capital_cost"] + assert eos.raw_capital_cost == data[SupplyCurveField.CAPITAL_COST] assert np.allclose(eos.raw_lcoe, true_lcoe, rtol=0.001) assert np.allclose(eos.scaled_lcoe, true_scaled, rtol=0.001) data['system_capacity'] = 2 eqn = '1 / system_capacity' - true_scaled = ((data['fcr'] * 0.5 * data['capital_cost'] + true_scaled = ((data['fcr'] * 0.5 * data[SupplyCurveField.CAPITAL_COST] + data['foc']) / (data['aep'] / 1000)) eos = EconomiesOfScale(eqn, data) assert 0.5 * eos.raw_capital_cost == eos.scaled_capital_cost - assert eos.raw_capital_cost == data["capital_cost"] + assert eos.raw_capital_cost == data[SupplyCurveField.CAPITAL_COST] assert np.allclose(eos.raw_lcoe, true_lcoe, rtol=0.001) assert np.allclose(eos.scaled_lcoe, true_scaled, rtol=0.001) @@ -212,7 +211,7 @@ def test_sc_agg_econ_scale(): res.create_dataset(k, res["meta"].shape, data=arr) res[k].attrs["scale_factor"] = 1.0 - eqn = "2 * np.multiply(1000, capacity) ** -0.3" + eqn = f"2 * np.multiply(1000, {SupplyCurveField.CAPACITY}) ** -0.3" out_fp_base = os.path.join(td, "base") base = SupplyCurveAggregation( EXCL, @@ -265,7 +264,8 @@ def test_sc_agg_econ_scale(): + data["fixed_operating_cost"] ) / aep + data["variable_operating_cost"] - assert np.allclose(scalars, sc_df[SupplyCurveField.CAPITAL_COST_SCALAR]) + assert np.allclose(scalars, + sc_df[SupplyCurveField.CAPITAL_COST_SCALAR]) assert np.allclose(scalars * sc_df['mean_capital_cost'], sc_df[SupplyCurveField.SCALED_CAPITAL_COST]) From a11bc331923f54528743681e3f288b48ce5d26c1 Mon Sep 17 00:00:00 2001 From: Paul Pinchuk Date: Wed, 29 May 2024 12:14:23 -0600 Subject: [PATCH 42/61] Linter fix --- tests/test_hybrids.py | 4 +++- tests/test_rep_profiles.py | 13 +++++++++---- tests/test_supply_curve_aggregation.py | 3 ++- tests/test_supply_curve_compute.py | 9 ++++++--- tests/test_supply_curve_sc_aggregation.py | 6 ++++-- 5 files changed, 24 insertions(+), 11 deletions(-) diff --git a/tests/test_hybrids.py b/tests/test_hybrids.py index b7b27c281..46eabdae3 100644 --- a/tests/test_hybrids.py +++ b/tests/test_hybrids.py @@ -102,7 +102,9 @@ def test_hybridization_profile_output_with_ratio_none(): hwp, ) = h.profiles.values() h_meta = h.hybrid_meta - h_idx = np.where(h_meta[SupplyCurveField.SC_POINT_GID] == sc_point_gid)[0][0] + h_idx = np.where( + h_meta[SupplyCurveField.SC_POINT_GID] == sc_point_gid + )[0][0] assert np.allclose(hp[:, h_idx], weighted_solar) assert np.allclose(hsp[:, h_idx], weighted_solar) diff --git a/tests/test_rep_profiles.py b/tests/test_rep_profiles.py index fe4b58ec5..bf779e041 100644 --- a/tests/test_rep_profiles.py +++ b/tests/test_rep_profiles.py @@ -104,7 +104,9 @@ def test_weighted_meanoid(): sites = np.arange(100) rev_summary = pd.DataFrame({SupplyCurveField.GEN_GIDS: sites, SupplyCurveField.RES_GIDS: sites, - SupplyCurveField.GID_COUNTS: [1] * 50 + [0] * 50}) + SupplyCurveField.GID_COUNTS: ( + [1] * 50 + [0] * 50 + )}) r = RegionRepProfile(GEN_FPATH, rev_summary) weights = r._get_region_attr(r._rev_summary, SupplyCurveField.GID_COUNTS) @@ -161,7 +163,8 @@ def test_sc_points(): SupplyCurveField.RES_GIDS: sites, 'timezone': timezone}) - rp = RepProfiles(GEN_FPATH, rev_summary, SupplyCurveField.SC_GID, weight=None) + rp = RepProfiles(GEN_FPATH, rev_summary, + SupplyCurveField.SC_GID, weight=None) rp.run(max_workers=1) with Resource(GEN_FPATH) as res: @@ -191,8 +194,10 @@ def test_agg_profile(): rp.run(scaled_precision=False, max_workers=1) for index in rev_summary.index: - gen_gids = json.loads(rev_summary.loc[index, SupplyCurveField.GEN_GIDS]) - res_gids = json.loads(rev_summary.loc[index, SupplyCurveField.RES_GIDS]) + gen_gids = json.loads(rev_summary.loc[index, + SupplyCurveField.GEN_GIDS]) + res_gids = json.loads(rev_summary.loc[index, + SupplyCurveField.RES_GIDS]) weights = np.array( json.loads(rev_summary.loc[index, SupplyCurveField.GID_COUNTS])) diff --git a/tests/test_supply_curve_aggregation.py b/tests/test_supply_curve_aggregation.py index 72687c5b2..fd26a7e02 100644 --- a/tests/test_supply_curve_aggregation.py +++ b/tests/test_supply_curve_aggregation.py @@ -47,7 +47,8 @@ def check_agg(agg_out, baseline_h5): truth = f[dset] if dset == 'meta': truth = truth.set_index('sc_gid') - for c in [SupplyCurveField.SOURCE_GIDS, SupplyCurveField.GID_COUNTS]: + for c in [SupplyCurveField.SOURCE_GIDS, + SupplyCurveField.GID_COUNTS]: test[c] = test[c].astype(str) truth = truth.fillna('none') diff --git a/tests/test_supply_curve_compute.py b/tests/test_supply_curve_compute.py index c63a28ad7..526a28760 100644 --- a/tests/test_supply_curve_compute.py +++ b/tests/test_supply_curve_compute.py @@ -182,8 +182,10 @@ def test_integrated_sc_simple_friction(): sc_simple = pd.read_csv(sc_simple) assert SupplyCurveField.MEAN_LCOE_FRICTION in sc_simple assert SupplyCurveField.TOTAL_LCOE_FRICTION in sc_simple - test = sc_simple[SupplyCurveField.MEAN_LCOE_FRICTION] + sc_simple['lcot'] - assert np.allclose(test, sc_simple[SupplyCurveField.TOTAL_LCOE_FRICTION]) + test = (sc_simple[SupplyCurveField.MEAN_LCOE_FRICTION] + + sc_simple['lcot']) + assert np.allclose(test, + sc_simple[SupplyCurveField.TOTAL_LCOE_FRICTION]) fpath_baseline = os.path.join( TESTDATADIR, "sc_out/sc_simple_out_friction.csv" @@ -279,7 +281,8 @@ def test_parallel(): assert_frame_equal(sc_full_parallel, sc_full_serial) -def verify_trans_cap(sc_table, trans_tables, cap_col=SupplyCurveField.CAPACITY): +def verify_trans_cap(sc_table, trans_tables, + cap_col=SupplyCurveField.CAPACITY): """ Verify that sc_points are connected to features in the correct capacity bins diff --git a/tests/test_supply_curve_sc_aggregation.py b/tests/test_supply_curve_sc_aggregation.py index 6ccbf5821..c38c8d663 100644 --- a/tests/test_supply_curve_sc_aggregation.py +++ b/tests/test_supply_curve_sc_aggregation.py @@ -292,9 +292,11 @@ def test_agg_extra_dsets(): for dset in h5_dsets: assert "mean_{}".format(dset) in summary.columns - check = summary['mean_lcoe_fcr-2012'] == summary[SupplyCurveField.MEAN_LCOE] + check = (summary['mean_lcoe_fcr-2012'] + == summary[SupplyCurveField.MEAN_LCOE]) assert not any(check) - check = summary['mean_lcoe_fcr-2013'] == summary[SupplyCurveField.MEAN_LCOE] + check = (summary['mean_lcoe_fcr-2013'] + == summary[SupplyCurveField.MEAN_LCOE]) assert not any(check) avg = (summary['mean_lcoe_fcr-2012'] + summary['mean_lcoe_fcr-2013']) / 2 From fa71fd6e771f40719c93b6eaf1de389db4c5c134 Mon Sep 17 00:00:00 2001 From: "Brandon N. Benton" Date: Wed, 29 May 2024 12:16:12 -0600 Subject: [PATCH 43/61] map_from_legacy method with internal access to old name dict for base enum class. --- reV/utilities/__init__.py | 61 ++++++++++++++++++++++----------------- tests/test_bespoke.py | 5 ++-- tests/test_hybrids.py | 38 +++++++++++------------- 3 files changed, 53 insertions(+), 51 deletions(-) diff --git a/reV/utilities/__init__.py b/reV/utilities/__init__.py index deeea147c..6145ead0b 100644 --- a/reV/utilities/__init__.py +++ b/reV/utilities/__init__.py @@ -8,10 +8,38 @@ from reV.version import __version__ +OldSupplyCurveField = { + "sc_point_gid": "SC_POINT_GID", + "source_gids": "SOURCE_GIDS", + "sc_gid": "SC_GID", + "gid_counts": "GID_COUNTS", + "gid": "GID", + "n_gids": "N_GIDS", + "res_gids": "RES_GIDS", + "gen_gids": "GEN_GIDS", + "area_sq_km": "AREA_SQ_KM", + "latitude": "LATITUDE", + "longitude": "LONGITUDE", + "elevation": "ELEVATION", + "timezone": "TIMEZONE", + "county": "COUNTY", + "state": "STATE", + "country": "COUNTRY", + "mean_lcoe": "MEAN_LCOE", + "mean_res": "MEAN_RES", + "capacity": "CAPACITY", + "sc_row_ind": "SC_ROW_IND", + "sc_col_ind": "SC_COL_IND", + "mean_cf": "MEAN_CF", + "capital_cost": "CAPITAL_COST", +} + class FieldEnum(str, Enum): """Base Field enum with some mapping methods.""" + _OLD_NAMES = OldSupplyCurveField + @classmethod def map_to(cls, other): """Return a rename map from this enum to another.""" @@ -27,6 +55,12 @@ def map_from(cls, other): 'sc_gid': 'SC_GID') to this enum.""" return {name: cls[mem] for name, mem in other.items()} + @classmethod + def map_from_legacy(cls): + """Return a rename map from a dictionary of name / member pairs (e.g. + 'sc_gid': 'SC_GID') to this enum.""" + return {name: cls[mem] for name, mem in cls._OLD_NAMES.items()} + def __str__(self): return self.value @@ -63,33 +97,6 @@ class ResourceMetaField(FieldEnum): # match current naming conventions -OldSupplyCurveField = { - "sc_point_gid": "SC_POINT_GID", - "source_gids": "SOURCE_GIDS", - "sc_gid": "SC_GID", - "gid_counts": "GID_COUNTS", - "gid": "GID", - "n_gids": "N_GIDS", - "res_gids": "RES_GIDS", - "gen_gids": "GEN_GIDS", - "area_sq_km": "AREA_SQ_KM", - "latitude": "LATITUDE", - "longitude": "LONGITUDE", - "elevation": "ELEVATION", - "timezone": "TIMEZONE", - "county": "COUNTY", - "state": "STATE", - "country": "COUNTRY", - "mean_lcoe": "MEAN_LCOE", - "mean_res": "MEAN_RES", - "capacity": "CAPACITY", - "sc_row_ind": "SC_ROW_IND", - "sc_col_ind": "SC_COL_IND", - "mean_cf": "MEAN_CF", - "capital_cost": "CAPITAL_COST", -} - - class OriginalSupplyCurveField(FieldEnum): """An enumerated map to supply curve summary/meta keys. diff --git a/tests/test_bespoke.py b/tests/test_bespoke.py index 113ca1152..2673f0ae3 100644 --- a/tests/test_bespoke.py +++ b/tests/test_bespoke.py @@ -28,7 +28,6 @@ from reV.supply_curve.tech_mapping import TechMapping from reV.utilities import ( ModuleName, - OldSupplyCurveField, SiteDataField, SupplyCurveField, ) @@ -738,7 +737,7 @@ def test_collect_bespoke(): for fp in source_fps: with Resource(fp) as source: src_meta = source.meta.rename( - SupplyCurveField.map_from(OldSupplyCurveField), axis=1 + SupplyCurveField.map_from_legacy(), axis=1 ) assert all( np.isin( @@ -812,7 +811,7 @@ def test_bespoke_supply_curve(): normal_path = os.path.join(TESTDATADIR, "sc_out/baseline_agg_summary.csv") normal_sc_points = pd.read_csv(normal_path) normal_sc_points = normal_sc_points.rename( - SupplyCurveField.map_from(OldSupplyCurveField), axis=1 + SupplyCurveField.map_from_legacy(), axis=1 ) with tempfile.TemporaryDirectory() as td: diff --git a/tests/test_hybrids.py b/tests/test_hybrids.py index 46eabdae3..9ba672845 100644 --- a/tests/test_hybrids.py +++ b/tests/test_hybrids.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -"""reV hybrids tests. -""" +"""reV hybrids tests.""" + import json import os import tempfile @@ -13,7 +13,7 @@ from reV.cli import main from reV.hybrids import HYBRID_METHODS, Hybridization from reV.hybrids.hybrids import MERGE_COLUMN, OUTPUT_PROFILE_NAMES, HybridsData -from reV.utilities import ModuleName, SupplyCurveField, OldSupplyCurveField +from reV.utilities import ModuleName, SupplyCurveField from reV.utilities.exceptions import FileInputError, InputError, OutputWarning SOLAR_FPATH = os.path.join( @@ -29,14 +29,10 @@ TESTDATADIR, "rep_profiles_out", "rep_profiles_solar_multiple.h5" ) with Resource(SOLAR_FPATH) as res: - meta = res.meta.rename( - columns=SupplyCurveField.map_from(OldSupplyCurveField) - ) + meta = res.meta.rename(columns=SupplyCurveField.map_from_legacy()) SOLAR_SCPGIDS = set(meta[SupplyCurveField.SC_POINT_GID]) with Resource(WIND_FPATH) as res: - meta = res.meta.rename( - columns=SupplyCurveField.map_from(OldSupplyCurveField) - ) + meta = res.meta.rename(columns=SupplyCurveField.map_from_legacy()) WIND_SCPGIDS = set(meta[SupplyCurveField.SC_POINT_GID]) @@ -51,7 +47,7 @@ def test_hybridization_profile_output_single_resource(): )[0][0] solar_cap = res.meta.loc[solar_idx, SupplyCurveField.CAPACITY] - solar_test_profile = res['rep_profiles_0', :, solar_idx] + solar_test_profile = res["rep_profiles_0", :, solar_idx] weighted_solar = solar_cap * solar_test_profile @@ -63,9 +59,9 @@ def test_hybridization_profile_output_single_resource(): hwp, ) = h.profiles.values() h_meta = h.hybrid_meta - h_idx = np.where( - h_meta[SupplyCurveField.SC_POINT_GID] == sc_point_gid - )[0][0] + h_idx = np.where(h_meta[SupplyCurveField.SC_POINT_GID] == sc_point_gid)[0][ + 0 + ] assert np.allclose(hp[:, h_idx], weighted_solar) assert np.allclose(hsp[:, h_idx], weighted_solar) @@ -84,7 +80,7 @@ def test_hybridization_profile_output_with_ratio_none(): )[0][0] solar_cap = res.meta.loc[solar_idx, SupplyCurveField.CAPACITY] - solar_test_profile = res['rep_profiles_0', :, solar_idx] + solar_test_profile = res["rep_profiles_0", :, solar_idx] weighted_solar = solar_cap * solar_test_profile @@ -102,9 +98,9 @@ def test_hybridization_profile_output_with_ratio_none(): hwp, ) = h.profiles.values() h_meta = h.hybrid_meta - h_idx = np.where( - h_meta[SupplyCurveField.SC_POINT_GID] == sc_point_gid - )[0][0] + h_idx = np.where(h_meta[SupplyCurveField.SC_POINT_GID] == sc_point_gid)[0][ + 0 + ] assert np.allclose(hp[:, h_idx], weighted_solar) assert np.allclose(hsp[:, h_idx], weighted_solar) @@ -121,14 +117,14 @@ def test_hybridization_profile_output(): res.meta[SupplyCurveField.SC_POINT_GID] == common_sc_point_gid )[0][0] solar_cap = res.meta.loc[solar_idx, SupplyCurveField.CAPACITY] - solar_test_profile = res['rep_profiles_0', :, solar_idx] + solar_test_profile = res["rep_profiles_0", :, solar_idx] with Resource(WIND_FPATH) as res: wind_idx = np.where( res.meta[SupplyCurveField.SC_POINT_GID] == common_sc_point_gid )[0][0] wind_cap = res.meta.loc[wind_idx, SupplyCurveField.CAPACITY] - wind_test_profile = res['rep_profiles_0', :, wind_idx] + wind_test_profile = res["rep_profiles_0", :, wind_idx] weighted_solar = solar_cap * solar_test_profile weighted_wind = wind_cap * wind_test_profile @@ -281,7 +277,7 @@ def test_ratios_input(ratio_cols, ratio_bounds, bounds): ) if SupplyCurveField.CAPACITY in ratio: - max_solar_capacities = h.hybrid_meta['hybrid_solar_capacity'] + max_solar_capacities = h.hybrid_meta["hybrid_solar_capacity"] max_solar_capacities = max_solar_capacities.values.reshape(1, -1) assert np.all( h.profiles["hybrid_solar_profile"] <= max_solar_capacities @@ -834,7 +830,7 @@ def make_test_file( meta.loc[0, SupplyCurveField.LATITUDE] = lat lon = meta[SupplyCurveField.LATITUDE].iloc[-1] meta.loc[0, SupplyCurveField.LATITUDE] = lon - shapes['meta'] = len(meta) + shapes["meta"] = len(meta) for d in dset_names: shapes[d] = (len(res.time_index[t_slice]), len(meta)) From 123162c662b88bb2be01ee02f90e481c3c2e7193 Mon Sep 17 00:00:00 2001 From: "Brandon N. Benton" Date: Wed, 29 May 2024 12:52:46 -0600 Subject: [PATCH 44/61] re: private legacy name map within enums - not fancy but this works. --- reV/utilities/__init__.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/reV/utilities/__init__.py b/reV/utilities/__init__.py index 6145ead0b..a37981440 100644 --- a/reV/utilities/__init__.py +++ b/reV/utilities/__init__.py @@ -38,8 +38,6 @@ class FieldEnum(str, Enum): """Base Field enum with some mapping methods.""" - _OLD_NAMES = OldSupplyCurveField - @classmethod def map_to(cls, other): """Return a rename map from this enum to another.""" @@ -57,9 +55,9 @@ def map_from(cls, other): @classmethod def map_from_legacy(cls): - """Return a rename map from a dictionary of name / member pairs (e.g. - 'sc_gid': 'SC_GID') to this enum.""" - return {name: cls[mem] for name, mem in cls._OLD_NAMES.items()} + """Return a dictionary -> this enum map using the dictionary of legacy + names""" + return cls.map_from(OldSupplyCurveField) def __str__(self): return self.value From 7dd1090aa4204fc73cb6a733d6e9944bac22f36d Mon Sep 17 00:00:00 2001 From: Paul Pinchuk Date: Wed, 29 May 2024 14:30:28 -0600 Subject: [PATCH 45/61] Use `ResourceMetaField` for accessing gen fields --- reV/rep_profiles/rep_profiles.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/reV/rep_profiles/rep_profiles.py b/reV/rep_profiles/rep_profiles.py index b9e0994b5..9b12ee726 100644 --- a/reV/rep_profiles/rep_profiles.py +++ b/reV/rep_profiles/rep_profiles.py @@ -23,7 +23,7 @@ from scipy import stats from reV.handlers.outputs import Outputs -from reV.utilities import SupplyCurveField, log_versions +from reV.utilities import ResourceMetaField, SupplyCurveField, log_versions from reV.utilities.exceptions import DataShapeError, FileInputError logger = logging.getLogger(__name__) @@ -394,8 +394,8 @@ def _init_profiles_weights(self): with Resource(self._gen_fpath) as res: meta = res.meta - assert SupplyCurveField.GID in meta - source_res_gids = meta[SupplyCurveField.GID].values + assert ResourceMetaField.GID in meta + source_res_gids = meta[ResourceMetaField.GID].values msg = ('Resource gids from "gid" column in meta data from "{}" ' 'must be sorted! reV generation should always be run with ' 'sequential project points.'.format(self._gen_fpath)) From 3c226162b25876cd0c5312837933364bf2afada7 Mon Sep 17 00:00:00 2001 From: Paul Pinchuk Date: Wed, 29 May 2024 14:30:34 -0600 Subject: [PATCH 46/61] Fix tests --- tests/test_rep_profiles.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_rep_profiles.py b/tests/test_rep_profiles.py index bf779e041..f621764b1 100644 --- a/tests/test_rep_profiles.py +++ b/tests/test_rep_profiles.py @@ -17,7 +17,7 @@ RepProfiles, RepresentativeMethods, ) -from reV.utilities import SupplyCurveField +from reV.utilities import ResourceMetaField, SupplyCurveField GEN_FPATH = os.path.join(TESTDATADIR, "gen_out/gen_ri_pv_2012_x000.h5") @@ -137,7 +137,7 @@ def test_integrated(): 'res_class': zeros, 'weight': ones, 'region': regions, - 'timezone': timezone}) + SupplyCurveField.TIMEZONE: timezone}) rp = RepProfiles(GEN_FPATH, rev_summary, 'region', weight='weight') rp.run(max_workers=1) p1, m1 = rp.profiles, rp.meta @@ -161,7 +161,7 @@ def test_sc_points(): rev_summary = pd.DataFrame({SupplyCurveField.SC_GID: sites, SupplyCurveField.GEN_GIDS: sites, SupplyCurveField.RES_GIDS: sites, - 'timezone': timezone}) + SupplyCurveField.TIMEZONE: timezone}) rp = RepProfiles(GEN_FPATH, rev_summary, SupplyCurveField.SC_GID, weight=None) @@ -206,7 +206,7 @@ def test_agg_profile(): raw_profiles = [] for gid in res_gids: - iloc = np.where(meta[SupplyCurveField.GID] == gid)[0][0] + iloc = np.where(meta[ResourceMetaField.GID] == gid)[0][0] prof = np.expand_dims(res['cf_profile', :, iloc], 1) raw_profiles.append(prof) @@ -247,7 +247,7 @@ def test_many_regions(use_weights): 'region1': region1, 'region2': region2, 'weight': sites + 1, - 'timezone': timezone}) + SupplyCurveField.TIMEZONE: timezone}) reg_cols = ['region1', 'region2'] if use_weights: rp = RepProfiles(GEN_FPATH, rev_summary, reg_cols, weight="weight") From 5f8ffb07aacee8ed33c155288971876dbd382c27 Mon Sep 17 00:00:00 2001 From: Paul Pinchuk Date: Wed, 29 May 2024 16:30:24 -0600 Subject: [PATCH 47/61] Use `SupplyCurveField` enum --- reV/hybrids/hybrid_methods.py | 58 +++++++++++++++++------------------ reV/hybrids/hybrids.py | 26 ++++++++++------ 2 files changed, 45 insertions(+), 39 deletions(-) diff --git a/reV/hybrids/hybrid_methods.py b/reV/hybrids/hybrid_methods.py index de743cfe1..5b4b1c0c2 100644 --- a/reV/hybrids/hybrid_methods.py +++ b/reV/hybrids/hybrid_methods.py @@ -3,18 +3,12 @@ @author: ppinchuk """ +from reV.utilities import SupplyCurveField def aggregate_solar_capacity(h): """Compute the total solar capcity allowed in hybridization. - Note - ---- - No limiting is done on the ratio of wind to solar. This method - checks for an existing 'hybrid_solar_capacity'. If one does not exist, - it is assumed that there is no limit on the solar to wind capacity - ratio and the solar capacity is copied into this new column. - Parameters ---------- h : `reV.hybrids.Hybridization` @@ -26,27 +20,25 @@ def aggregate_solar_capacity(h): ------- data : Series | None A series of data containing the capacity allowed in the hybrid - capacity sum, or `None` if 'hybrid_solar_capacity' already exists. + capacity sum, or `None` if 'hybrid_solar_capacity' already + exists. Notes ----- - + No limiting is done on the ratio of wind to solar. This method + checks for an existing 'hybrid_solar_capacity'. If one does not + exist, it is assumed that there is no limit on the solar to wind + capacity ratio and the solar capacity is copied into this new + column. """ - if 'hybrid_solar_capacity' in h.hybrid_meta: + if f'hybrid_solar_{SupplyCurveField.CAPACITY}' in h.hybrid_meta: return None - return h.hybrid_meta['solar_capacity'] + return h.hybrid_meta[f'solar_{SupplyCurveField.CAPACITY}'] def aggregate_wind_capacity(h): """Compute the total wind capcity allowed in hybridization. - Note - ---- - No limiting is done on the ratio of wind to solar. This method - checks for an existing 'hybrid_wind_capacity'. If one does not exist, - it is assumed that there is no limit on the solar to wind capacity - ratio and the wind capacity is copied into this new column. - Parameters ---------- h : `reV.hybrids.Hybridization` @@ -58,15 +50,19 @@ def aggregate_wind_capacity(h): ------- data : Series | None A series of data containing the capacity allowed in the hybrid - capacity sum, or `None` if 'hybrid_solar_capacity' already exists. + capacity sum, or `None` if 'hybrid_solar_capacity' already + exists. Notes ----- - + No limiting is done on the ratio of wind to solar. This method + checks for an existing 'hybrid_wind_capacity'. If one does not + exist, it is assumed that there is no limit on the solar to wind + capacity ratio and the wind capacity is copied into this new column. """ - if 'hybrid_wind_capacity' in h.hybrid_meta: + if f'hybrid_wind_{SupplyCurveField.CAPACITY}' in h.hybrid_meta: return None - return h.hybrid_meta['wind_capacity'] + return h.hybrid_meta[f'wind_{SupplyCurveField.CAPACITY}'] def aggregate_capacity(h): @@ -85,8 +81,8 @@ def aggregate_capacity(h): A series of data containing the aggregated capacity, or `None` if the capacity columns are missing. """ - - sc, wc = 'hybrid_solar_capacity', 'hybrid_wind_capacity' + sc = f'hybrid_solar_{SupplyCurveField.CAPACITY}' + wc = f'hybrid_wind_{SupplyCurveField.CAPACITY}' missing_solar_cap = sc not in h.hybrid_meta.columns missing_wind_cap = wc not in h.hybrid_meta.columns if missing_solar_cap or missing_wind_cap: @@ -113,8 +109,10 @@ def aggregate_capacity_factor(h): if the capacity and/or mean_cf columns are missing. """ - sc, wc = 'hybrid_solar_capacity', 'hybrid_wind_capacity' - scf, wcf = 'solar_mean_cf', 'wind_mean_cf' + sc = f'hybrid_solar_{SupplyCurveField.CAPACITY}' + wc = f'hybrid_wind_{SupplyCurveField.CAPACITY}' + scf = f'solar_{SupplyCurveField.MEAN_CF}' + wcf = f'wind_{SupplyCurveField.MEAN_CF}' missing_solar_cap = sc not in h.hybrid_meta.columns missing_wind_cap = wc not in h.hybrid_meta.columns missing_solar_mean_cf = scf not in h.hybrid_meta.columns @@ -132,8 +130,8 @@ def aggregate_capacity_factor(h): HYBRID_METHODS = { - 'hybrid_solar_capacity': aggregate_solar_capacity, - 'hybrid_wind_capacity': aggregate_wind_capacity, - 'hybrid_capacity': aggregate_capacity, - 'hybrid_mean_cf': aggregate_capacity_factor + f'hybrid_solar_{SupplyCurveField.CAPACITY}': aggregate_solar_capacity, + f'hybrid_wind_{SupplyCurveField.CAPACITY}': aggregate_wind_capacity, + f'hybrid_{SupplyCurveField.CAPACITY}': aggregate_capacity, + f'hybrid_{SupplyCurveField.MEAN_CF}': aggregate_capacity_factor } diff --git a/reV/hybrids/hybrids.py b/reV/hybrids/hybrids.py index 2071a0e51..a579408be 100644 --- a/reV/hybrids/hybrids.py +++ b/reV/hybrids/hybrids.py @@ -34,12 +34,15 @@ NON_DUPLICATE_COLS = { SupplyCurveField.LATITUDE, SupplyCurveField.LONGITUDE, SupplyCurveField.COUNTRY, SupplyCurveField.STATE, SupplyCurveField.COUNTY, - SupplyCurveField.ELEVATION, SupplyCurveField.TIMEZONE, SupplyCurveField.SC_POINT_GID, - SupplyCurveField.SC_ROW_IND, SupplyCurveField.SC_COL_IND + SupplyCurveField.ELEVATION, SupplyCurveField.TIMEZONE, + SupplyCurveField.SC_POINT_GID, SupplyCurveField.SC_ROW_IND, + SupplyCurveField.SC_COL_IND } DROPPED_COLUMNS = [SupplyCurveField.GID] -DEFAULT_FILL_VALUES = {'solar_capacity': 0, 'wind_capacity': 0, - 'solar_mean_cf': 0, 'wind_mean_cf': 0} +DEFAULT_FILL_VALUES = {f'solar_{SupplyCurveField.CAPACITY}': 0, + f'wind_{SupplyCurveField.CAPACITY}': 0, + f'solar_{SupplyCurveField.MEAN_CF}': 0, + f'wind_{SupplyCurveField.MEAN_CF}': 0} OUTPUT_PROFILE_NAMES = ['hybrid_profile', 'hybrid_solar_profile', 'hybrid_wind_profile'] @@ -637,7 +640,7 @@ def hybridize(self): """Combine the solar and wind metas and run hybridize methods.""" self._format_meta_pre_merge() self._merge_solar_wind_meta() - self._verify_lat_long_match_post_merge() + self._verify_lat_lon_match_post_merge() self._format_meta_post_merge() self._fillna_meta_cols() self._apply_limits() @@ -742,10 +745,14 @@ def _column_sorting_key(self, c): first_index = -1 return first_index, self._hybrid_meta.columns.get_loc(c) - def _verify_lat_long_match_post_merge(self): + def _verify_lat_lon_match_post_merge(self): """Verify that all the lat/lon values match post merge.""" - lat = self._verify_col_match_post_merge(col_name=SupplyCurveField.LATITUDE) - lon = self._verify_col_match_post_merge(col_name=SupplyCurveField.LONGITUDE) + lat = self._verify_col_match_post_merge( + col_name=ColNameFormatter.fmt(SupplyCurveField.LATITUDE) + ) + lon = self._verify_col_match_post_merge( + col_name=ColNameFormatter.fmt(SupplyCurveField.LONGITUDE) + ) if not lat or not lon: msg = ( "Detected mismatched coordinate values (latitude or " @@ -1189,7 +1196,8 @@ def _compute_hybridized_profile_components(self): def __rep_profile_hybridization_params(self): """Zip the rep profile hybridization parameters.""" - cap_col_names = ["hybrid_solar_capacity", "hybrid_wind_capacity"] + cap_col_names = [f"hybrid_solar_{SupplyCurveField.CAPACITY}", + f"hybrid_wind_{SupplyCurveField.CAPACITY}"] idx_maps = [ self.meta_hybridizer.solar_profile_indices_map, self.meta_hybridizer.wind_profile_indices_map, From 7c72c621f42c7fd3d809d9cacde147311b0e7df7 Mon Sep 17 00:00:00 2001 From: Paul Pinchuk Date: Wed, 29 May 2024 16:30:29 -0600 Subject: [PATCH 48/61] Fix tests --- tests/test_hybrids.py | 342 +++++++++++++++++++++++++----------------- 1 file changed, 205 insertions(+), 137 deletions(-) diff --git a/tests/test_hybrids.py b/tests/test_hybrids.py index 9ba672845..297ec3d4a 100644 --- a/tests/test_hybrids.py +++ b/tests/test_hybrids.py @@ -3,6 +3,7 @@ import json import os +import shutil import tempfile import numpy as np @@ -29,19 +30,63 @@ TESTDATADIR, "rep_profiles_out", "rep_profiles_solar_multiple.h5" ) with Resource(SOLAR_FPATH) as res: - meta = res.meta.rename(columns=SupplyCurveField.map_from_legacy()) - SOLAR_SCPGIDS = set(meta[SupplyCurveField.SC_POINT_GID]) + SOLAR_SCPGIDS = set(res.meta["sc_point_gid"]) with Resource(WIND_FPATH) as res: - meta = res.meta.rename(columns=SupplyCurveField.map_from_legacy()) - WIND_SCPGIDS = set(meta[SupplyCurveField.SC_POINT_GID]) + WIND_SCPGIDS = set(res.meta["sc_point_gid"]) -def test_hybridization_profile_output_single_resource(): +def _fix_meta(fp): + with Outputs(fp, mode="a") as out: + meta = out.meta + del out._h5['meta'] + out._meta = None + out.meta = meta.rename(columns=SupplyCurveField.map_from_legacy()) + + +@pytest.fixture(scope="module") +def module_td(): + with tempfile.TemporaryDirectory() as td: + yield td + + +@pytest.fixture(scope="module") +def solar_fpath(module_td): + new_fp = os.path.join(module_td, "solar.h5") + shutil.copy(SOLAR_FPATH, new_fp) + _fix_meta(new_fp) + yield new_fp + + +@pytest.fixture(scope="module") +def wind_fpath(module_td): + new_fp = os.path.join(module_td, "wind.h5") + shutil.copy(WIND_FPATH, new_fp) + _fix_meta(new_fp) + yield new_fp + + +@pytest.fixture(scope="module") +def solar_fpath_30_min(module_td): + new_fp = os.path.join(module_td, "solar_30min.h5") + shutil.copy(SOLAR_FPATH_30_MIN, new_fp) + _fix_meta(new_fp) + yield new_fp + + +@pytest.fixture(scope="module") +def solar_fpath_mult(module_td): + new_fp = os.path.join(module_td, "solar_mult.h5") + shutil.copy(SOLAR_FPATH_MULT, new_fp) + _fix_meta(new_fp) + yield new_fp + + +def test_hybridization_profile_output_single_resource(solar_fpath, wind_fpath): """Test that the hybridization calculation is correct (1 resource).""" sc_point_gid = 40005 - with Resource(SOLAR_FPATH) as res: + with Resource(solar_fpath) as res: solar_idx = np.where( res.meta[SupplyCurveField.SC_POINT_GID] == sc_point_gid )[0][0] @@ -51,13 +96,9 @@ def test_hybridization_profile_output_single_resource(): weighted_solar = solar_cap * solar_test_profile - h = Hybridization(SOLAR_FPATH, WIND_FPATH, allow_solar_only=True) + h = Hybridization(solar_fpath, wind_fpath, allow_solar_only=True) h.run() - ( - hp, - hsp, - hwp, - ) = h.profiles.values() + hp, hsp, hwp = h.profiles.values() h_meta = h.hybrid_meta h_idx = np.where(h_meta[SupplyCurveField.SC_POINT_GID] == sc_point_gid)[0][ 0 @@ -69,12 +110,13 @@ def test_hybridization_profile_output_single_resource(): assert np.allclose(hwp[:, h_idx], 0) -def test_hybridization_profile_output_with_ratio_none(): +def test_hybridization_profile_output_with_ratio_none(solar_fpath, wind_fpath): """Test that the hybridization calculation is correct (1 resource).""" sc_point_gid = 40005 - with Resource(SOLAR_FPATH) as res: + with Resource(solar_fpath) as res: + solar_idx = np.where( res.meta[SupplyCurveField.SC_POINT_GID] == sc_point_gid )[0][0] @@ -85,18 +127,14 @@ def test_hybridization_profile_output_with_ratio_none(): weighted_solar = solar_cap * solar_test_profile h = Hybridization( - SOLAR_FPATH, - WIND_FPATH, + solar_fpath, + wind_fpath, allow_solar_only=True, ratio=None, ratio_bounds=None, ) h.run() - ( - hp, - hsp, - hwp, - ) = h.profiles.values() + hp, hsp, hwp = h.profiles.values() h_meta = h.hybrid_meta h_idx = np.where(h_meta[SupplyCurveField.SC_POINT_GID] == sc_point_gid)[0][ 0 @@ -108,18 +146,18 @@ def test_hybridization_profile_output_with_ratio_none(): assert np.allclose(hwp[:, h_idx], 0) -def test_hybridization_profile_output(): +def test_hybridization_profile_output(solar_fpath, wind_fpath): """Test that the hybridization calculation is correct.""" common_sc_point_gid = 38883 - with Resource(SOLAR_FPATH) as res: + with Resource(solar_fpath) as res: solar_idx = np.where( res.meta[SupplyCurveField.SC_POINT_GID] == common_sc_point_gid )[0][0] solar_cap = res.meta.loc[solar_idx, SupplyCurveField.CAPACITY] solar_test_profile = res["rep_profiles_0", :, solar_idx] - with Resource(WIND_FPATH) as res: + with Resource(wind_fpath) as res: wind_idx = np.where( res.meta[SupplyCurveField.SC_POINT_GID] == common_sc_point_gid )[0][0] @@ -129,7 +167,7 @@ def test_hybridization_profile_output(): weighted_solar = solar_cap * solar_test_profile weighted_wind = wind_cap * wind_test_profile - h = Hybridization(SOLAR_FPATH, WIND_FPATH) + h = Hybridization(solar_fpath, wind_fpath) h.run() ( hp, @@ -146,12 +184,14 @@ def test_hybridization_profile_output(): assert np.allclose(hwp[:, h_idx], weighted_wind) -@pytest.mark.parametrize( - "input_files", - [(SOLAR_FPATH, WIND_FPATH), (SOLAR_FPATH_30_MIN, WIND_FPATH)], -) -def test_hybridization_output_shapes(input_files): +@pytest.mark.parametrize("half_hour", [True, False]) +def test_hybridization_output_shapes(half_hour, solar_fpath, + solar_fpath_30_min, wind_fpath): """Test that the output shapes are as expected.""" + if half_hour: + input_files = solar_fpath_30_min, wind_fpath + else: + input_files = solar_fpath, wind_fpath sfp, wfp = input_files h = Hybridization(sfp, wfp) @@ -185,13 +225,14 @@ def test_hybridization_output_shapes(input_files): ((True, True), (147, 73), SOLAR_SCPGIDS | WIND_SCPGIDS), ], ) -def test_meta_hybridization(input_combination, expected_shape, overlap): +def test_meta_hybridization(input_combination, expected_shape, overlap, + solar_fpath, wind_fpath): """Test that the meta is hybridized properly.""" allow_solar_only, allow_wind_only = input_combination h = Hybridization( - SOLAR_FPATH, - WIND_FPATH, + solar_fpath, + wind_fpath, allow_solar_only=allow_solar_only, allow_wind_only=allow_wind_only, ) @@ -200,19 +241,20 @@ def test_meta_hybridization(input_combination, expected_shape, overlap): assert set(h.hybrid_meta[SupplyCurveField.SC_POINT_GID]) == overlap -def test_limits_and_ratios_output_values(): +def test_limits_and_ratios_output_values(solar_fpath, wind_fpath): """Test that limits and ratios are properly applied in succession.""" - limits = {"solar_capacity": 50, "wind_capacity": 0.5} - ratio_numerator = "solar_capacity" - ratio_denominator = "wind_capacity" + limits = {f"solar_{SupplyCurveField.CAPACITY}": 50, + f"wind_{SupplyCurveField.CAPACITY}": 0.5} + ratio_numerator = f"solar_{SupplyCurveField.CAPACITY}" + ratio_denominator = f"wind_{SupplyCurveField.CAPACITY}" ratio = "{}/{}".format(ratio_numerator, ratio_denominator) ratio_bounds = (0.3, 3.6) bounds = (0.3 - 1e6, 3.6 + 1e6) h = Hybridization( - SOLAR_FPATH, - WIND_FPATH, + solar_fpath, + wind_fpath, limits=limits, ratio=ratio, ratio_bounds=ratio_bounds, @@ -232,15 +274,19 @@ def test_limits_and_ratios_output_values(): h.hybrid_meta["hybrid_{}".format(ratio_denominator)] <= h.hybrid_meta[ratio_denominator] ) - assert np.all(h.hybrid_meta["solar_capacity"] <= limits["solar_capacity"]) - assert np.all(h.hybrid_meta["wind_capacity"] <= limits["wind_capacity"]) + assert np.all(h.hybrid_meta[f"solar_{SupplyCurveField.CAPACITY}"] + <= limits[f"solar_{SupplyCurveField.CAPACITY}"]) + assert np.all(h.hybrid_meta[f"wind_{SupplyCurveField.CAPACITY}"] + <= limits[f"wind_{SupplyCurveField.CAPACITY}"]) @pytest.mark.parametrize( "ratio_cols", [ - ("solar_capacity", "wind_capacity"), - ("solar_area_sq_km", "wind_area_sq_km"), + (f"solar_{SupplyCurveField.CAPACITY}", + f"wind_{SupplyCurveField.CAPACITY}"), + (f"solar_{SupplyCurveField.AREA_SQ_KM}", + f"wind_{SupplyCurveField.AREA_SQ_KM}"), ], ) @pytest.mark.parametrize( @@ -252,12 +298,13 @@ def test_limits_and_ratios_output_values(): ((0.3, 3.6), (0.3 - 1e6, 3.6 + 1e6)), ], ) -def test_ratios_input(ratio_cols, ratio_bounds, bounds): +def test_ratios_input(ratio_cols, ratio_bounds, bounds, solar_fpath, + wind_fpath): """Test that the hybrid meta limits the ratio columns correctly.""" ratio_numerator, ratio_denominator = ratio_cols ratio = "{}/{}".format(ratio_numerator, ratio_denominator) h = Hybridization( - SOLAR_FPATH, WIND_FPATH, ratio=ratio, ratio_bounds=ratio_bounds + solar_fpath, wind_fpath, ratio=ratio, ratio_bounds=ratio_bounds ) h.run() @@ -277,19 +324,21 @@ def test_ratios_input(ratio_cols, ratio_bounds, bounds): ) if SupplyCurveField.CAPACITY in ratio: - max_solar_capacities = h.hybrid_meta["hybrid_solar_capacity"] + col = f"hybrid_solar_{SupplyCurveField.CAPACITY}" + max_solar_capacities = h.hybrid_meta[col] max_solar_capacities = max_solar_capacities.values.reshape(1, -1) assert np.all( h.profiles["hybrid_solar_profile"] <= max_solar_capacities ) - max_wind_capacities = h.hybrid_meta["hybrid_wind_capacity"] + col = f"hybrid_wind_{SupplyCurveField.CAPACITY}" + max_wind_capacities = h.hybrid_meta[col] max_wind_capacities = max_wind_capacities.values.reshape(1, -1) assert np.all(h.profiles["hybrid_wind_profile"] <= max_wind_capacities) -def test_rep_profile_idx_map(): +def test_rep_profile_idx_map(solar_fpath, wind_fpath): """Test that rep profile index mappings are correct shape.""" - h = Hybridization(SOLAR_FPATH, WIND_FPATH, allow_wind_only=True) + h = Hybridization(solar_fpath, wind_fpath, allow_wind_only=True) for h_idxs, r_idxs in ( h.meta_hybridizer.solar_profile_indices_map, @@ -312,55 +361,65 @@ def test_rep_profile_idx_map(): assert r_idxs.size == shape -def test_limits_values(): +def test_limits_values(solar_fpath, wind_fpath): """Test that column values are properly limited on user input.""" - limits = {"solar_capacity": 100, "wind_capacity": 0.5} + limits = {f"solar_{SupplyCurveField.CAPACITY}": 100, + f"wind_{SupplyCurveField.CAPACITY}": 0.5} - h = Hybridization(SOLAR_FPATH, WIND_FPATH, limits=limits) + h = Hybridization(solar_fpath, wind_fpath, limits=limits) h.run() - assert np.all(h.hybrid_meta["solar_capacity"] <= limits["solar_capacity"]) - assert np.all(h.hybrid_meta["wind_capacity"] <= limits["wind_capacity"]) + assert np.all(h.hybrid_meta[f"solar_{SupplyCurveField.CAPACITY}"] + <= limits[f"solar_{SupplyCurveField.CAPACITY}"]) + assert np.all(h.hybrid_meta[f"wind_{SupplyCurveField.CAPACITY}"] + <= limits[f"wind_{SupplyCurveField.CAPACITY}"]) -def test_invalid_limits_column_name(): +def test_invalid_limits_column_name(solar_fpath, wind_fpath): """Test invalid inputs for limits columns.""" - test_limits = {"un_prefixed_col": 0, "wind_capacity": 10} + test_limits = {"un_prefixed_col": 0, + f"wind_{SupplyCurveField.CAPACITY}": 10} with pytest.raises(InputError) as excinfo: - Hybridization(SOLAR_FPATH, WIND_FPATH, limits=test_limits) + Hybridization(solar_fpath, wind_fpath, limits=test_limits) assert "Input limits column" in str(excinfo.value) assert "does not start with a valid prefix" in str(excinfo.value) -def test_fillna_values(): +def test_fillna_values(solar_fpath, wind_fpath): """Test that N/A values are filled properly based on user input.""" - fill_vals = {"solar_n_gids": 0, "wind_capacity": -1} + fill_vals = {f"solar_{SupplyCurveField.N_GIDS}": 0, + f"wind_{SupplyCurveField.CAPACITY}": -1} h = Hybridization( - SOLAR_FPATH, - WIND_FPATH, + solar_fpath, + wind_fpath, allow_solar_only=True, allow_wind_only=True, fillna=fill_vals, ) h.run() - assert not np.any(h.hybrid_meta["solar_n_gids"].isna()) - assert not np.any(h.hybrid_meta["wind_capacity"].isna()) - assert np.any(h.hybrid_meta["solar_n_gids"].values == 0) - assert np.any(h.hybrid_meta["wind_capacity"].values == -1) + assert not np.any(h.hybrid_meta[f"solar_{SupplyCurveField.N_GIDS}"].isna()) + assert not np.any( + h.hybrid_meta[f"wind_{SupplyCurveField.CAPACITY}"].isna() + ) + assert np.any(h.hybrid_meta[f"solar_{SupplyCurveField.N_GIDS}"].values + == 0) + assert np.any(h.hybrid_meta[f"wind_{SupplyCurveField.CAPACITY}"].values + == -1) -def test_invalid_fillna_column_name(): +def test_invalid_fillna_column_name(solar_fpath, wind_fpath): """Test invalid inputs for fillna columns.""" - test_fillna = {"un_prefixed_col": 0, "wind_capacity": 10} + test_fillna = {"un_prefixed_col": 0, + f"wind_{SupplyCurveField.CAPACITY}": 10} with pytest.raises(InputError) as excinfo: - Hybridization(SOLAR_FPATH, WIND_FPATH, fillna=test_fillna) + Hybridization(solar_fpath, wind_fpath, fillna=test_fillna) assert "Input fillna column" in str(excinfo.value) assert "does not start with a valid prefix" in str(excinfo.value) @@ -375,28 +434,30 @@ def test_invalid_fillna_column_name(): ((True, True), (True, True)), ], ) -def test_all_allow_solar_allow_wind_combinations(input_combination, na_vals): +def test_all_allow_solar_allow_wind_combinations(input_combination, na_vals, + solar_fpath, wind_fpath): """Test that "allow_x_only" options perform the intended merges.""" allow_solar_only, allow_wind_only = input_combination h = Hybridization( - SOLAR_FPATH, - WIND_FPATH, + solar_fpath, + wind_fpath, allow_solar_only=allow_solar_only, allow_wind_only=allow_wind_only, ) h.run() - for col_name, should_have_na_vals in zip( - ["solar_sc_gid", "wind_sc_gid"], na_vals - ): + cols = [f"solar_{SupplyCurveField.SC_GID}", + f"wind_{SupplyCurveField.SC_GID}"] + for col_name, should_have_na_vals in zip(cols, na_vals): if should_have_na_vals: assert np.any(h.hybrid_meta[col_name].isna()) else: assert not np.any(h.hybrid_meta[col_name].isna()) -def test_warning_for_improper_data_output_from_hybrid_method(): +def test_warning_for_improper_data_output_from_hybrid_method(solar_fpath, + wind_fpath): """Test that hybrid function with incorrect output throws warning.""" def some_new_hybrid_func(__): @@ -405,7 +466,7 @@ def some_new_hybrid_func(__): HYBRID_METHODS["scaled_elevation"] = some_new_hybrid_func with pytest.warns(OutputWarning) as records: - h = Hybridization(SOLAR_FPATH, WIND_FPATH) + h = Hybridization(solar_fpath, wind_fpath) h.run() messages = [r.message.args[0] for r in records] @@ -415,7 +476,7 @@ def some_new_hybrid_func(__): HYBRID_METHODS.pop("scaled_elevation") -def test_hybrid_col_additional_method(): +def test_hybrid_col_additional_method(solar_fpath, wind_fpath): """Test that function decorated with 'hybrid_col' adds to hybrid meta.""" def some_new_hybrid_func(h): @@ -423,7 +484,7 @@ def some_new_hybrid_func(h): HYBRID_METHODS["scaled_elevation"] = some_new_hybrid_func - h = Hybridization(SOLAR_FPATH, WIND_FPATH) + h = Hybridization(solar_fpath, wind_fpath) h.run() assert "scaled_elevation" in HYBRID_METHODS @@ -436,27 +497,28 @@ def some_new_hybrid_func(h): HYBRID_METHODS.pop("scaled_elevation") -def test_duplicate_lat_long_values(): +def test_duplicate_lat_long_values(solar_fpath, wind_fpath, module_td): """Test duplicate lat/long values corresponding to unique merge column.""" - with tempfile.TemporaryDirectory() as td: - fout_solar = os.path.join(td, "rep_profiles_solar.h5") - make_test_file(SOLAR_FPATH, fout_solar, duplicate_coord_values=True) + fout_solar = os.path.join(module_td, "rep_profiles_solar.h5") + make_test_file(solar_fpath, fout_solar, duplicate_coord_values=True) - with pytest.raises(FileInputError) as excinfo: - h = Hybridization(fout_solar, WIND_FPATH) - h.run() + with pytest.raises(FileInputError) as excinfo: + h = Hybridization(fout_solar, wind_fpath) + h.run() - assert "Detected mismatched coordinate values" in str(excinfo.value) + assert "Detected mismatched coordinate values" in str(excinfo.value) -def test_invalid_ratio_bounds_length_input(): +def test_invalid_ratio_bounds_length_input(solar_fpath, wind_fpath): """Test improper ratios input.""" - ratio = "solar_capacity/wind_capacity" + ratio = ( + f"solar_{SupplyCurveField.CAPACITY}/wind_{SupplyCurveField.CAPACITY}" + ) with pytest.raises(InputError) as excinfo: Hybridization( - SOLAR_FPATH, WIND_FPATH, ratio=ratio, ratio_bounds=(1, 2, 3) + solar_fpath, wind_fpath, ratio=ratio, ratio_bounds=(1, 2, 3) ) msg = ( @@ -466,13 +528,13 @@ def test_invalid_ratio_bounds_length_input(): assert msg in str(excinfo.value) -def test_ratio_column_missing(): +def test_ratio_column_missing(solar_fpath, wind_fpath): """Test missing ratio column.""" - ratio = "solar_col_dne/wind_capacity" + ratio = f"solar_col_dne/wind_{SupplyCurveField.CAPACITY}" with pytest.raises(FileInputError) as excinfo: Hybridization( - SOLAR_FPATH, WIND_FPATH, ratio=ratio, ratio_bounds=(1, 1) + solar_fpath, wind_fpath, ratio=ratio, ratio_bounds=(1, 1) ) assert "Input ratios column" in str(excinfo.value) @@ -480,12 +542,12 @@ def test_ratio_column_missing(): @pytest.mark.parametrize("ratio", [None, ("solar_capacity", "wind_capacity")]) -def test_ratio_not_string(ratio): +def test_ratio_not_string(ratio, solar_fpath, wind_fpath): """Test ratio input is not string.""" with pytest.raises(InputError) as excinfo: Hybridization( - SOLAR_FPATH, WIND_FPATH, ratio=ratio, ratio_bounds=(1, 1) + solar_fpath, wind_fpath, ratio=ratio, ratio_bounds=(1, 1) ) assert "Ratio input type " in str(excinfo.value) @@ -495,12 +557,12 @@ def test_ratio_not_string(ratio): @pytest.mark.parametrize( "ratio", ["solar_capacity", "solar_capacity/wind_capacity/solar_capacity"] ) -def test_invalid_ratio_format(ratio): +def test_invalid_ratio_format(ratio, solar_fpath, wind_fpath): """Test ratio input is not string.""" with pytest.raises(InputError) as excinfo: Hybridization( - SOLAR_FPATH, WIND_FPATH, ratio=ratio, ratio_bounds=(1, 1) + solar_fpath, wind_fpath, ratio=ratio, ratio_bounds=(1, 1) ) long_msg = ( @@ -511,27 +573,27 @@ def test_invalid_ratio_format(ratio): assert long_msg in str(excinfo.value) -def test_invalid_ratio_column_name(): +def test_invalid_ratio_column_name(solar_fpath, wind_fpath): """Test invalid inputs for ratio columns.""" - ratio = "un_prefixed_col/wind_capacity" + ratio = f"un_prefixed_col/wind_{SupplyCurveField.CAPACITY}" with pytest.raises(InputError) as excinfo: Hybridization( - SOLAR_FPATH, WIND_FPATH, ratio=ratio, ratio_bounds=(1, 1) + solar_fpath, wind_fpath, ratio=ratio, ratio_bounds=(1, 1) ) assert "Input ratios column" in str(excinfo.value) assert "does not start with a valid prefix" in str(excinfo.value) -def test_no_overlap_in_merge_column_values(): +def test_no_overlap_in_merge_column_values(solar_fpath, wind_fpath): """Test duplicate values in merge column.""" with tempfile.TemporaryDirectory() as td: fout_solar = os.path.join(td, "rep_profiles_solar.h5") fout_wind = os.path.join(td, "rep_profiles_wind.h5") - make_test_file(SOLAR_FPATH, fout_solar, p_slice=slice(0, 3)) - make_test_file(WIND_FPATH, fout_wind, p_slice=slice(90, 100)) + make_test_file(solar_fpath, fout_solar, p_slice=slice(0, 3)) + make_test_file(wind_fpath, fout_wind, p_slice=slice(90, 100)) with pytest.raises(FileInputError) as excinfo: Hybridization(fout_solar, fout_wind) @@ -539,39 +601,39 @@ def test_no_overlap_in_merge_column_values(): assert "No overlap detected in the values" in str(excinfo.value) -def test_duplicate_merge_column_values(): +def test_duplicate_merge_column_values(solar_fpath, wind_fpath): """Test duplicate values in merge column.""" with tempfile.TemporaryDirectory() as td: fout_solar = os.path.join(td, "rep_profiles_solar.h5") - make_test_file(SOLAR_FPATH, fout_solar, duplicate_rows=True) + make_test_file(solar_fpath, fout_solar, duplicate_rows=True) with pytest.raises(FileInputError) as excinfo: - Hybridization(fout_solar, WIND_FPATH) + Hybridization(fout_solar, wind_fpath) assert "Duplicate" in str(excinfo.value) -def test_merge_columns_missing(): +def test_merge_columns_missing(solar_fpath, wind_fpath): """Test missing merge column.""" with tempfile.TemporaryDirectory() as td: fout_solar = os.path.join(td, "rep_profiles_solar.h5") - make_test_file(SOLAR_FPATH, fout_solar, drop_cols=[MERGE_COLUMN]) + make_test_file(solar_fpath, fout_solar, drop_cols=[MERGE_COLUMN]) with pytest.raises(FileInputError) as excinfo: - Hybridization(fout_solar, WIND_FPATH) + Hybridization(fout_solar, wind_fpath) msg = "Cannot hybridize: merge column" assert msg in str(excinfo.value) assert "missing" in str(excinfo.value) -def test_invalid_num_profiles(): +def test_invalid_num_profiles(solar_fpath_mult, wind_fpath): """Test input files with an invalid number of profiles (>1).""" with pytest.raises(FileInputError) as excinfo: - Hybridization(SOLAR_FPATH_MULT, WIND_FPATH) + Hybridization(solar_fpath_mult, wind_fpath) msg = ( "This module is not intended for hybridization of " @@ -581,14 +643,14 @@ def test_invalid_num_profiles(): assert msg in str(excinfo.value) -def test_invalid_time_index_overlap(): +def test_invalid_time_index_overlap(solar_fpath, wind_fpath): """Test input files with an invalid time index overlap.""" with tempfile.TemporaryDirectory() as td: fout_solar = os.path.join(td, "rep_profiles_solar.h5") fout_wind = os.path.join(td, "rep_profiles_wind.h5") - make_test_file(SOLAR_FPATH, fout_solar, t_slice=slice(0, 1500)) - make_test_file(WIND_FPATH, fout_wind, t_slice=slice(1000, 3000)) + make_test_file(solar_fpath, fout_solar, t_slice=slice(0, 1500)) + make_test_file(wind_fpath, fout_wind, t_slice=slice(1000, 3000)) with pytest.raises(FileInputError) as excinfo: Hybridization(fout_solar, fout_wind) @@ -600,24 +662,24 @@ def test_invalid_time_index_overlap(): assert msg in str(excinfo.value) -def test_valid_time_index_overlap(): +def test_valid_time_index_overlap(solar_fpath_30_min, wind_fpath): """Test input files with a valid time index overlap.""" - h = Hybridization(SOLAR_FPATH_30_MIN, WIND_FPATH) + h = Hybridization(solar_fpath_30_min, wind_fpath) - with Resource(SOLAR_FPATH_30_MIN) as res: + with Resource(solar_fpath_30_min) as res: assert np.all(res.time_index == h.solar_time_index) - with Resource(WIND_FPATH) as res: + with Resource(wind_fpath) as res: assert np.all(res.time_index == h.wind_time_index) assert len(res.time_index) == len(h.hybrid_time_index) -def test_write_to_file(): +def test_write_to_file(solar_fpath, wind_fpath): """Test hybrid rep profiles with file write.""" with tempfile.TemporaryDirectory() as td: fout = os.path.join(td, "temp_hybrid_profiles.h5") - h = Hybridization(SOLAR_FPATH, WIND_FPATH) + h = Hybridization(solar_fpath, wind_fpath) h.run(fout=fout) with Resource(fout) as res: @@ -634,13 +696,13 @@ def test_write_to_file(): assert "rep_profiles_0" not in disk_dsets -def test_hybrids_data_content(): +def test_hybrids_data_content(solar_fpath, wind_fpath): """Test HybridsData class content.""" fv = -999 - h_data = HybridsData(SOLAR_FPATH, WIND_FPATH) + h_data = HybridsData(solar_fpath, wind_fpath) - with Resource(SOLAR_FPATH) as sr, Resource(WIND_FPATH) as wr: + with Resource(solar_fpath) as sr, Resource(wind_fpath) as wr: assert np.all(h_data.solar_meta.fillna(fv) == sr.meta.fillna(fv)) assert np.all(h_data.wind_meta.fillna(fv) == wr.meta.fillna(fv)) assert np.all(h_data.solar_time_index == sr.time_index) @@ -649,35 +711,40 @@ def test_hybrids_data_content(): assert np.all(h_data.hybrid_time_index == hyb_idx) -def test_hybrids_data_contains_col(): +def test_hybrids_data_contains_col(solar_fpath, wind_fpath): """Test the 'contains_col' method of HybridsData for accuracy.""" - h_data = HybridsData(SOLAR_FPATH, WIND_FPATH) + h_data = HybridsData(solar_fpath, wind_fpath) assert h_data.contains_col("trans_capacity") assert h_data.contains_col("dist_mi") assert h_data.contains_col("dist_km") assert not h_data.contains_col("dne_col_for_test") -@pytest.mark.parametrize( - "input_files", - [(SOLAR_FPATH, WIND_FPATH), (SOLAR_FPATH_30_MIN, WIND_FPATH)], -) +@pytest.mark.parametrize("half_hour", [True, False]) @pytest.mark.parametrize( "ratio", - ["solar_capacity/wind_capacity", "solar_area_sq_km/wind_area_sq_km"], + [f"solar_{SupplyCurveField.CAPACITY}/wind_{SupplyCurveField.CAPACITY}", + f"solar_{SupplyCurveField.AREA_SQ_KM}" + f"/wind_{SupplyCurveField.AREA_SQ_KM}"], ) @pytest.mark.parametrize("ratio_bounds", [None, (0.5, 1.5), (0.3, 3.6)]) @pytest.mark.parametrize("input_combination", [(False, False), (True, True)]) def test_hybrids_cli_from_config( - runner, input_files, ratio, ratio_bounds, input_combination, clear_loggers + runner, half_hour, ratio, ratio_bounds, input_combination, clear_loggers, + solar_fpath, solar_fpath_30_min, wind_fpath ): """Test hybrids cli from config""" fv = -999 - sfp, wfp = input_files allow_solar_only, allow_wind_only = input_combination - fill_vals = {"solar_n_gids": 0, "wind_capacity": -1} - limits = {"solar_capacity": 100} + fill_vals = {f"solar_{SupplyCurveField.N_GIDS}": 0, + f"wind_{SupplyCurveField.CAPACITY}": -1} + limits = {f"solar_{SupplyCurveField.CAPACITY}": 100} + + if half_hour: + sfp, wfp = solar_fpath_30_min, wind_fpath + else: + sfp, wfp = solar_fpath, wind_fpath with tempfile.TemporaryDirectory() as td: config = { @@ -747,13 +814,14 @@ def test_hybrids_cli_from_config( os.path.join(TESTDATADIR, "rep_profiles_out", "rep_profiles_dne.h5"), ], ) -def test_hybrids_cli_bad_fpath_input(runner, bad_fpath, clear_loggers): +def test_hybrids_cli_bad_fpath_input(runner, bad_fpath, clear_loggers, + wind_fpath): """Test cli when filepath input is ambiguous or invalid.""" with tempfile.TemporaryDirectory() as td: config = { "solar_fpath": bad_fpath, - "wind_fpath": WIND_FPATH, + "wind_fpath": wind_fpath, "log_directory": td, "execution_control": { "nodes": 1, @@ -828,8 +896,8 @@ def make_test_file( if duplicate_coord_values: lat = meta[SupplyCurveField.LATITUDE].iloc[-1] meta.loc[0, SupplyCurveField.LATITUDE] = lat - lon = meta[SupplyCurveField.LATITUDE].iloc[-1] - meta.loc[0, SupplyCurveField.LATITUDE] = lon + lon = meta[SupplyCurveField.LONGITUDE].iloc[-1] + meta.loc[0, SupplyCurveField.LONGITUDE] = lon shapes["meta"] = len(meta) for d in dset_names: shapes[d] = (len(res.time_index[t_slice]), len(meta)) From da09ca73e1d89d37b23166f58bc08bb18a4a3a0d Mon Sep 17 00:00:00 2001 From: Paul Pinchuk Date: Wed, 29 May 2024 16:33:18 -0600 Subject: [PATCH 49/61] Fix NRWAL tests --- reV/nrwal/nrwal.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/reV/nrwal/nrwal.py b/reV/nrwal/nrwal.py index 827753cb1..4ee73f356 100644 --- a/reV/nrwal/nrwal.py +++ b/reV/nrwal/nrwal.py @@ -19,7 +19,7 @@ from reV.generation.generation import Gen from reV.handlers.outputs import Outputs -from reV.utilities import SiteDataField, SupplyCurveField, log_versions +from reV.utilities import SiteDataField, ResourceMetaField, log_versions from reV.utilities.exceptions import ( DataShapeError, OffshoreWindInputError, @@ -36,7 +36,8 @@ class RevNrwal: """Columns from the `site_data` table to join to the output meta data""" def __init__(self, gen_fpath, site_data, sam_files, nrwal_configs, - output_request, save_raw=True, meta_gid_col=SupplyCurveField.GID, + output_request, save_raw=True, + meta_gid_col=ResourceMetaField.GID, site_meta_cols=None): """Framework to handle reV-NRWAL analysis. From 96f7eff15e01a2ebd9fb659efe19a17dd17fd8c8 Mon Sep 17 00:00:00 2001 From: Paul Pinchuk Date: Wed, 29 May 2024 16:35:23 -0600 Subject: [PATCH 50/61] Fix QA/QC tests --- reV/qa_qc/summary.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/reV/qa_qc/summary.py b/reV/qa_qc/summary.py index 380889474..acda6a409 100644 --- a/reV/qa_qc/summary.py +++ b/reV/qa_qc/summary.py @@ -12,7 +12,7 @@ from rex import Resource from rex.utilities import SpawnProcessPool, parse_table -from reV.utilities import SupplyCurveField +from reV.utilities import SupplyCurveField, ResourceMetaField logger = logging.getLogger(__name__) @@ -188,7 +188,7 @@ def summarize_dset( summary = pd.concat(summary) - summary.index.name = SupplyCurveField.GID + summary.index.name = ResourceMetaField.GID else: summary = self._compute_ds_summary( @@ -216,9 +216,9 @@ def summarize_means(self, out_path=None): """ with Resource(self.h5_file, group=self._group) as f: meta = f.meta - if SupplyCurveField.GID not in meta: - if meta.index.name != SupplyCurveField.GID: - meta.index.name = SupplyCurveField.GID + if ResourceMetaField.GID not in meta: + if meta.index.name != ResourceMetaField.GID: + meta.index.name = ResourceMetaField.GID meta = meta.reset_index() From 8e7f3980c84ac27bdf843591b4c0f521bda4d5b4 Mon Sep 17 00:00:00 2001 From: Paul Pinchuk Date: Wed, 29 May 2024 16:39:10 -0600 Subject: [PATCH 51/61] Fix tests --- tests/test_supply_curve_aggregation.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_supply_curve_aggregation.py b/tests/test_supply_curve_aggregation.py index fd26a7e02..06d455d5c 100644 --- a/tests/test_supply_curve_aggregation.py +++ b/tests/test_supply_curve_aggregation.py @@ -46,7 +46,10 @@ def check_agg(agg_out, baseline_h5): for dset, test in agg_out.items(): truth = f[dset] if dset == 'meta': - truth = truth.set_index('sc_gid') + truth = truth.rename( + columns=SupplyCurveField.map_from_legacy() + ) + truth = truth.set_index(SupplyCurveField.SC_GID) for c in [SupplyCurveField.SOURCE_GIDS, SupplyCurveField.GID_COUNTS]: test[c] = test[c].astype(str) From 6fcaefc17cb37cc3e2bfcc1d0b8023db4a311bb9 Mon Sep 17 00:00:00 2001 From: Paul Pinchuk Date: Wed, 29 May 2024 16:56:29 -0600 Subject: [PATCH 52/61] Fix tests --- reV/utilities/__init__.py | 2 ++ tests/test_supply_curve_compute.py | 18 +++++++++++------- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/reV/utilities/__init__.py b/reV/utilities/__init__.py index a37981440..d7cd649f5 100644 --- a/reV/utilities/__init__.py +++ b/reV/utilities/__init__.py @@ -32,6 +32,8 @@ "sc_col_ind": "SC_COL_IND", "mean_cf": "MEAN_CF", "capital_cost": "CAPITAL_COST", + "mean_lcoe_friction": "MEAN_LCOE_FRICTION", + "total_lcoe_friction": "TOTAL_LCOE_FRICTION", } diff --git a/tests/test_supply_curve_compute.py b/tests/test_supply_curve_compute.py index 526a28760..eeed68126 100644 --- a/tests/test_supply_curve_compute.py +++ b/tests/test_supply_curve_compute.py @@ -37,16 +37,17 @@ } path = os.path.join(TESTDATADIR, "sc_out/baseline_agg_summary.csv") -SC_POINTS = pd.read_csv(path) +LEGACY_SC_COL_MAP = SupplyCurveField.map_from_legacy() +SC_POINTS = pd.read_csv(path).rename(columns=LEGACY_SC_COL_MAP) path = os.path.join(TESTDATADIR, "sc_out/baseline_agg_summary_friction.csv") -SC_POINTS_FRICTION = pd.read_csv(path) +SC_POINTS_FRICTION = pd.read_csv(path).rename(columns=LEGACY_SC_COL_MAP) path = os.path.join(TESTDATADIR, "trans_tables/ri_transmission_table.csv") -TRANS_TABLE = pd.read_csv(path) +TRANS_TABLE = pd.read_csv(path).rename(columns=LEGACY_SC_COL_MAP) path = os.path.join(TESTDATADIR, "trans_tables/transmission_multipliers.csv") -MULTIPLIERS = pd.read_csv(path) +MULTIPLIERS = pd.read_csv(path).rename(columns=LEGACY_SC_COL_MAP) SC_FULL_COLUMNS = ( "trans_gid", @@ -67,6 +68,7 @@ def baseline_verify(sc_full, fpath_baseline): if os.path.exists(fpath_baseline): baseline = pd.read_csv(fpath_baseline) + baseline = baseline.rename(columns=LEGACY_SC_COL_MAP) # double check useful for when tables are changing # but lcoe should be the same check = np.allclose(baseline["total_lcoe"], sc_full["total_lcoe"]) @@ -681,11 +683,13 @@ def test_least_cost_simple_with_ac_capacity_column(): trans_tables.append(out_fp) sc = SC_POINTS.copy() - sc["capacity_ac"] = sc["capacity"] / 1.02 + sc[SupplyCurveField.CAPACITY_AC] = sc[SupplyCurveField.CAPACITY] / 1.02 - sc = SupplyCurve(sc, trans_tables, sc_capacity_col="capacity_ac") + sc = SupplyCurve(sc, trans_tables, + sc_capacity_col=SupplyCurveField.CAPACITY_AC) sc_simple_ac_cap = sc.simple_sort(fcr=0.1) - verify_trans_cap(sc_simple_ac_cap, trans_tables, cap_col="capacity_ac") + verify_trans_cap(sc_simple_ac_cap, trans_tables, + cap_col=SupplyCurveField.CAPACITY_AC) assert np.allclose( sc_simple["trans_cap_cost_per_mw"] * 1.02, From 549a5a1b48d6473a5ab0e49ab8e507b9f8f68734 Mon Sep 17 00:00:00 2001 From: Paul Pinchuk Date: Wed, 29 May 2024 17:08:44 -0600 Subject: [PATCH 53/61] Fix tests --- tests/test_supply_curve_sc_aggregation.py | 50 +++++++++++++---------- 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/tests/test_supply_curve_sc_aggregation.py b/tests/test_supply_curve_sc_aggregation.py index c38c8d663..f8d49ac15 100644 --- a/tests/test_supply_curve_sc_aggregation.py +++ b/tests/test_supply_curve_sc_aggregation.py @@ -26,6 +26,7 @@ SupplyCurveAggregation, _warn_about_large_datasets, ) +from reV.handlers.exclusions import LATITUDE from reV.utilities import ModuleName, SupplyCurveField EXCL = os.path.join(TESTDATADIR, 'ri_exclusions/ri_exclusions.h5') @@ -42,13 +43,12 @@ "reeds_region": {"dset": "ri_reeds_regions", "method": "mode"}, "padus": {"dset": "ri_padus", "method": "mode"}, } - EXCL_DICT = { "ri_srtm_slope": {"inclusion_range": (None, 5), "exclude_nodata": True}, "ri_padus": {"exclude_values": [1], "exclude_nodata": True}, } - RTOL = 0.001 +LEGACY_SC_COL_MAP = SupplyCurveField.map_from_legacy() def test_agg_extent(resolution=64): @@ -124,7 +124,9 @@ def test_agg_summary(): SupplyCurveField.GID_COUNTS]: summary[c] = summary[c].astype(str) - s_baseline = pd.read_csv(AGG_BASELINE, index_col=0) + s_baseline = pd.read_csv(AGG_BASELINE) + s_baseline = s_baseline.rename(columns=LEGACY_SC_COL_MAP) + s_baseline = s_baseline.set_index(s_baseline.columns[0]) summary = summary.fillna("None") s_baseline = s_baseline.fillna("None") @@ -160,8 +162,9 @@ def test_agg_summary_solar_ac(pd): ) summary = sca.summarize(gen, max_workers=1) - assert "capacity_ac" in summary - assert np.allclose(summary["capacity"] / 1.3, summary["capacity_ac"]) + assert SupplyCurveField.CAPACITY_AC in summary + assert np.allclose(summary[SupplyCurveField.CAPACITY] / 1.3, + summary[SupplyCurveField.CAPACITY_AC]) def test_multi_file_excl(): @@ -183,7 +186,7 @@ def test_multi_file_excl(): shutil.copy(EXCL, excl_temp_2) with h5py.File(excl_temp_1, 'a') as f: - shape = f[SupplyCurveField.LATITUDE].shape + shape = f[LATITUDE].shape attrs = dict(f['ri_srtm_slope'].attrs) data = np.ones(shape) test_dset = "excl_test" @@ -201,7 +204,9 @@ def test_multi_file_excl(): ) summary = sca.summarize(GEN) - s_baseline = pd.read_csv(AGG_BASELINE, index_col=0) + s_baseline = pd.read_csv(AGG_BASELINE) + s_baseline = s_baseline.rename(columns=LEGACY_SC_COL_MAP) + s_baseline = s_baseline.set_index(s_baseline.columns[0]) summary = summary.fillna("None") s_baseline = s_baseline.fillna("None") @@ -237,7 +242,9 @@ def test_pre_extract_inclusions(pre_extract): SupplyCurveField.GID_COUNTS]: summary[c] = summary[c].astype(str) - s_baseline = pd.read_csv(AGG_BASELINE, index_col=0) + s_baseline = pd.read_csv(AGG_BASELINE) + s_baseline = s_baseline.rename(columns=LEGACY_SC_COL_MAP) + s_baseline = s_baseline.set_index(s_baseline.columns[0]) summary = summary.fillna("None") s_baseline = s_baseline.fillna("None") @@ -425,16 +432,17 @@ def test_data_layer_methods(): @pytest.mark.parametrize( - "cap_cost_scale", ["1", "2 * np.multiply(1000, capacity) ** -0.3"] + "cap_cost_scale", + ["1", f"2 * np.multiply(1000, {SupplyCurveField.CAPACITY}) ** -0.3"] ) def test_recalc_lcoe(cap_cost_scale): """Test supply curve aggregation with the re-calculation of lcoe using the multi-year mean capacity factor""" - data = {SupplyCurveField.CAPITAL_COST: 34900000, - SupplyCurveField.FIXED_OPERATING_COST: 280000, - SupplyCurveField.FIXED_CHARGE_RATE: 0.09606382995843887, - SupplyCurveField.VARIABLE_OPERATING_COST: 0, + data = {"capital_cost": 34900000, + "fixed_operating_cost": 280000, + "fixed_charge_rate": 0.09606382995843887, + "variable_operating_cost": 0, 'system_capacity': 20000} annual_cf = [0.24, 0.26, 0.37, 0.15] annual_lcoe = [] @@ -451,11 +459,11 @@ def test_recalc_lcoe(cap_cost_scale): arr = np.full(res["meta"].shape, v) res.create_dataset(k, res["meta"].shape, data=arr) for year, cf in zip(years, annual_cf): - lcoe = lcoe_fcr(data[SupplyCurveField.FIXED_CHARGE_RATE], - data[SupplyCurveField.CAPITAL_COST], - data[SupplyCurveField.FIXED_OPERATING_COST], + lcoe = lcoe_fcr(data["fixed_charge_rate"], + data["capital_cost"], + data["fixed_operating_cost"], data['system_capacity'] * cf * 8760, - data[SupplyCurveField.VARIABLE_OPERATING_COST]) + data["variable_operating_cost"]) cf_arr = np.full(res['meta'].shape, cf) lcoe_arr = np.full(res['meta'].shape, lcoe) annual_lcoe.append(lcoe) @@ -476,10 +484,10 @@ def test_recalc_lcoe(cap_cost_scale): "lcoe_fcr-means", res["meta"].shape, data=lcoe_arr ) - h5_dsets = [SupplyCurveField.CAPITAL_COST, - SupplyCurveField.FIXED_OPERATING_COST, - SupplyCurveField.FIXED_CHARGE_RATE, - SupplyCurveField.VARIABLE_OPERATING_COST, + h5_dsets = ["capital_cost", + "fixed_operating_cost", + "fixed_charge_rate", + "variable_operating_cost", 'system_capacity'] base = SupplyCurveAggregation( From d338e0eb4920f1344684ca42537fd18a475ecb5d Mon Sep 17 00:00:00 2001 From: Paul Pinchuk Date: Wed, 29 May 2024 17:14:16 -0600 Subject: [PATCH 54/61] Fix tests --- tests/test_supply_curve_vpd.py | 59 ++++++++++++++++++++++------------ 1 file changed, 38 insertions(+), 21 deletions(-) diff --git a/tests/test_supply_curve_vpd.py b/tests/test_supply_curve_vpd.py index b0a35326c..6b8e334c2 100644 --- a/tests/test_supply_curve_vpd.py +++ b/tests/test_supply_curve_vpd.py @@ -4,6 +4,7 @@ @author: gbuster """ import os +import tempfile import numpy as np import pandas as pd @@ -34,18 +35,24 @@ def test_vpd(): """Test variable power density""" + vpd = pd.read_csv(FVPD) + vpd = vpd.rename(columns=SupplyCurveField.map_from_legacy()) + vpd = vpd.set_index(vpd.columns[0]) + + with tempfile.TemporaryDirectory() as td: + tmp_path = os.path.join(td, "vpd.csv") + vpd.to_csv(tmp_path) + sca = SupplyCurveAggregation(EXCL, TM_DSET, excl_dict=EXCL_DICT, + res_class_dset=RES_CLASS_DSET, + res_class_bins=RES_CLASS_BINS, + data_layers=DATA_LAYERS, + power_density=tmp_path) + summary = sca.summarize(GEN, max_workers=1) - sca = SupplyCurveAggregation(EXCL, TM_DSET, excl_dict=EXCL_DICT, - res_class_dset=RES_CLASS_DSET, - res_class_bins=RES_CLASS_BINS, - data_layers=DATA_LAYERS, power_density=FVPD) - summary = sca.summarize(GEN, max_workers=1) - - vpd = pd.read_csv(FVPD, index_col=0) for i in summary.index: capacity = summary.loc[i, SupplyCurveField.CAPACITY] area = summary.loc[i, SupplyCurveField.AREA_SQ_KM] - res_gids = np.array(summary.loc[i, 'res_gids']) + res_gids = np.array(summary.loc[i, SupplyCurveField.RES_GIDS]) gid_counts = np.array(summary.loc[i, SupplyCurveField.GID_COUNTS]) vpd_per_gid = vpd.loc[res_gids, 'power_density'].values truth = area * (vpd_per_gid * gid_counts).sum() / gid_counts.sum() @@ -62,21 +69,31 @@ def test_vpd_fractional_excl(): gids_subset = list(range(0, 20)) excl_dict_1 = {'ri_padus': {'exclude_values': [1]}} - sca_1 = SupplyCurveAggregation(EXCL, TM_DSET, excl_dict=excl_dict_1, - res_class_dset=RES_CLASS_DSET, - res_class_bins=RES_CLASS_BINS, - data_layers=DATA_LAYERS, power_density=FVPD, - gids=gids_subset) - summary_1 = sca_1.summarize(GEN, max_workers=1) - excl_dict_2 = {'ri_padus': {'exclude_values': [1], 'weight': 0.5}} - sca_2 = SupplyCurveAggregation(EXCL, TM_DSET, excl_dict=excl_dict_2, - res_class_dset=RES_CLASS_DSET, - res_class_bins=RES_CLASS_BINS, - data_layers=DATA_LAYERS, power_density=FVPD, - gids=gids_subset) - summary_2 = sca_2.summarize(GEN, max_workers=1) + + vpd = pd.read_csv(FVPD) + vpd = vpd.rename(columns=SupplyCurveField.map_from_legacy()) + vpd = vpd.set_index(vpd.columns[0]) + + with tempfile.TemporaryDirectory() as td: + tmp_path = os.path.join(td, "vpd.csv") + vpd.to_csv(tmp_path) + + sca_1 = SupplyCurveAggregation(EXCL, TM_DSET, excl_dict=excl_dict_1, + res_class_dset=RES_CLASS_DSET, + res_class_bins=RES_CLASS_BINS, + data_layers=DATA_LAYERS, power_density=tmp_path, + gids=gids_subset) + summary_1 = sca_1.summarize(GEN, max_workers=1) + + sca_2 = SupplyCurveAggregation(EXCL, TM_DSET, excl_dict=excl_dict_2, + res_class_dset=RES_CLASS_DSET, + res_class_bins=RES_CLASS_BINS, + data_layers=DATA_LAYERS, + power_density=tmp_path, + gids=gids_subset) + summary_2 = sca_2.summarize(GEN, max_workers=1) for i in summary_1.index: cap_full = summary_1.loc[i, SupplyCurveField.CAPACITY] From bd71b0e41cae988bf35585a4afcbcf7527223387 Mon Sep 17 00:00:00 2001 From: Paul Pinchuk Date: Wed, 29 May 2024 17:24:46 -0600 Subject: [PATCH 55/61] Performn column rename when parsing table --- reV/supply_curve/competitive_wind_farms.py | 12 ++++++++++-- reV/supply_curve/supply_curve.py | 9 +++++++-- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/reV/supply_curve/competitive_wind_farms.py b/reV/supply_curve/competitive_wind_farms.py index 556b73630..cd5709374 100644 --- a/reV/supply_curve/competitive_wind_farms.py +++ b/reV/supply_curve/competitive_wind_farms.py @@ -190,6 +190,8 @@ def _parse_wind_dirs(cls, wind_dirs): cardinal direction for each sc point gid """ wind_dirs = cls._parse_table(wind_dirs) + wind_dirs = wind_dirs.rename( + columns=SupplyCurveField.map_from_legacy()) wind_dirs = wind_dirs.set_index(SupplyCurveField.SC_POINT_GID) columns = [c for c in wind_dirs if c.endswith(('_gid', '_pr'))] @@ -221,6 +223,8 @@ def _parse_sc_points(cls, sc_points, offshore=False): Mask array to mask excluded sc_point_gids """ sc_points = cls._parse_table(sc_points) + sc_points = sc_points.rename( + columns=SupplyCurveField.map_from_legacy()) if SupplyCurveField.OFFSHORE in sc_points and not offshore: logger.debug('Not including offshore supply curve points in ' 'CompetitiveWindFarm') @@ -230,7 +234,8 @@ def _parse_sc_points(cls, sc_points, offshore=False): mask = np.ones(int(1 + sc_points[SupplyCurveField.SC_POINT_GID].max()), dtype=bool) - sc_points = sc_points[[SupplyCurveField.SC_GID, SupplyCurveField.SC_POINT_GID]] + sc_points = sc_points[[SupplyCurveField.SC_GID, + SupplyCurveField.SC_POINT_GID]] sc_gids = sc_points.set_index(SupplyCurveField.SC_GID) sc_gids = {k: int(v[0]) for k, v in sc_gids.iterrows()} @@ -431,13 +436,16 @@ def remove_noncompetitive_farm( wind farms """ sc_points = self._parse_table(sc_points) + sc_points = sc_points.rename( + columns=SupplyCurveField.map_from_legacy()) if SupplyCurveField.OFFSHORE in sc_points and not self._offshore: mask = sc_points[SupplyCurveField.OFFSHORE] == 0 sc_points = sc_points.loc[mask] sc_points = sc_points.sort_values(sort_on) - sc_point_gids = sc_points[SupplyCurveField.SC_POINT_GID].values.astype(int) + sc_point_gids = sc_points[SupplyCurveField.SC_POINT_GID].values + sc_point_gids = sc_point_gids.astype(int) for i in range(len(sc_points)): gid = sc_point_gids[i] diff --git a/reV/supply_curve/supply_curve.py b/reV/supply_curve/supply_curve.py index 9af7b85af..e34f05e74 100644 --- a/reV/supply_curve/supply_curve.py +++ b/reV/supply_curve/supply_curve.py @@ -185,6 +185,8 @@ def _parse_sc_points(sc_points, sc_features=None): sc_points = sc_points.reset_index() else: sc_points = parse_table(sc_points) + sc_points = sc_points.rename( + columns=SupplyCurveField.map_from_legacy()) logger.debug( "Supply curve points table imported with columns: {}".format( @@ -194,6 +196,8 @@ def _parse_sc_points(sc_points, sc_features=None): if sc_features is not None: sc_features = parse_table(sc_features) + sc_features = sc_features.rename( + columns=SupplyCurveField.map_from_legacy()) merge_cols = [c for c in sc_features if c in sc_points] sc_points = sc_points.merge(sc_features, on=merge_cols, how="left") logger.debug( @@ -288,12 +292,13 @@ def _parse_trans_table(trans_table): trans_table = trans_table.rename(columns={"dist_mi": "dist_km"}) trans_table["dist_km"] *= 1.60934 - drop_cols = [SupplyCurveField.SC_GID, 'cap_left', SupplyCurveField.SC_POINT_GID] + drop_cols = [SupplyCurveField.SC_GID, 'cap_left', + SupplyCurveField.SC_POINT_GID] drop_cols = [c for c in drop_cols if c in trans_table] if drop_cols: trans_table = trans_table.drop(columns=drop_cols) - return trans_table + return trans_table.rename(columns=SupplyCurveField.map_from_legacy()) @staticmethod def _map_trans_capacity(trans_sc_table, From f2b81363c744e4e1898e3cd8051ed82a27a9e180 Mon Sep 17 00:00:00 2001 From: Paul Pinchuk Date: Wed, 29 May 2024 17:24:50 -0600 Subject: [PATCH 56/61] Fix tests --- tests/test_supply_curve_wind_dirs.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/test_supply_curve_wind_dirs.py b/tests/test_supply_curve_wind_dirs.py index badfabe46..1dff451c7 100644 --- a/tests/test_supply_curve_wind_dirs.py +++ b/tests/test_supply_curve_wind_dirs.py @@ -45,9 +45,12 @@ def test_competitive_wind_dirs(downwind): sc_points.to_csv(baseline, index=False) else: baseline = pd.read_csv(baseline) + baseline = baseline.rename(columns=SupplyCurveField.map_from_legacy()) - sc_points = sc_points.sort_values(by="sc_gid").reset_index(drop=True) - baseline = baseline.sort_values(by="sc_gid").reset_index(drop=True) + sc_points = sc_points.sort_values(by=SupplyCurveField.SC_GID) + sc_points = sc_points.reset_index(drop=True) + baseline = baseline.sort_values(by=SupplyCurveField.SC_GID) + baseline = baseline.reset_index(drop=True) assert_frame_equal(sc_points, baseline, check_dtype=False) @@ -72,6 +75,7 @@ def test_sc_full_wind_dirs(downwind): sc_out.to_csv(baseline, index=False) else: baseline = pd.read_csv(baseline) + baseline = baseline.rename(columns=SupplyCurveField.map_from_legacy()) assert_frame_equal(sc_out, baseline, check_dtype=False) @@ -94,6 +98,7 @@ def test_sc_simple_wind_dirs(downwind): sc_out.to_csv(baseline, index=False) else: baseline = pd.read_csv(baseline) + baseline = baseline.rename(columns=SupplyCurveField.map_from_legacy()) assert_frame_equal(sc_out, baseline, check_dtype=False) @@ -107,6 +112,7 @@ def test_upwind_exclusion(): sc_out = os.path.join(TESTDATADIR, 'comp_wind_farms', 'sc_full_upwind.csv') sc_out = pd.read_csv(sc_out).sort_values('total_lcoe') + sc_out = sc_out.rename(columns=SupplyCurveField.map_from_legacy()) sc_point_gids = sc_out[SupplyCurveField.SC_POINT_GID].values.tolist() for _, row in sc_out.iterrows(): @@ -127,6 +133,7 @@ def test_upwind_downwind_exclusion(): sc_out = os.path.join(TESTDATADIR, 'comp_wind_farms', 'sc_full_downwind.csv') sc_out = pd.read_csv(sc_out).sort_values('total_lcoe') + sc_out = sc_out.rename(columns=SupplyCurveField.map_from_legacy()) sc_point_gids = sc_out[SupplyCurveField.SC_POINT_GID].values.tolist() for _, row in sc_out.iterrows(): From 14c794bb86aeae08a9e3083ad59f7dad5b06637f Mon Sep 17 00:00:00 2001 From: Paul Pinchuk Date: Wed, 29 May 2024 17:26:32 -0600 Subject: [PATCH 57/61] Fix test --- tests/test_bespoke.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/test_bespoke.py b/tests/test_bespoke.py index 2673f0ae3..85ada355c 100644 --- a/tests/test_bespoke.py +++ b/tests/test_bespoke.py @@ -723,7 +723,7 @@ def test_collect_bespoke(): collector.collect("cf_profile-2012") with Resource(h5_file) as fout: - meta = fout.meta + meta = fout.meta.rename(columns=SupplyCurveField.map_from_legacy()) assert all( meta[SupplyCurveField.GID].values == sorted(meta[SupplyCurveField.GID].values) @@ -737,8 +737,7 @@ def test_collect_bespoke(): for fp in source_fps: with Resource(fp) as source: src_meta = source.meta.rename( - SupplyCurveField.map_from_legacy(), axis=1 - ) + columns=SupplyCurveField.map_from_legacy()) assert all( np.isin( src_meta[SupplyCurveField.GID].values, From dddf821a734cb0c2fce6e36ae0cb895c1776777c Mon Sep 17 00:00:00 2001 From: ppinchuk Date: Thu, 30 May 2024 11:03:32 -0600 Subject: [PATCH 58/61] Updated handling of aliases --- reV/utilities/__init__.py | 174 ++++++++++++++++++-------------------- 1 file changed, 80 insertions(+), 94 deletions(-) diff --git a/reV/utilities/__init__.py b/reV/utilities/__init__.py index d7cd649f5..97afa5256 100644 --- a/reV/utilities/__init__.py +++ b/reV/utilities/__init__.py @@ -1,48 +1,47 @@ # -*- coding: utf-8 -*- """reV utilities.""" - -from enum import Enum, EnumMeta +from enum import Enum import PySAM from rex.utilities.loggers import log_versions as rex_log_versions from reV.version import __version__ -OldSupplyCurveField = { - "sc_point_gid": "SC_POINT_GID", - "source_gids": "SOURCE_GIDS", - "sc_gid": "SC_GID", - "gid_counts": "GID_COUNTS", - "gid": "GID", - "n_gids": "N_GIDS", - "res_gids": "RES_GIDS", - "gen_gids": "GEN_GIDS", - "area_sq_km": "AREA_SQ_KM", - "latitude": "LATITUDE", - "longitude": "LONGITUDE", - "elevation": "ELEVATION", - "timezone": "TIMEZONE", - "county": "COUNTY", - "state": "STATE", - "country": "COUNTRY", - "mean_lcoe": "MEAN_LCOE", - "mean_res": "MEAN_RES", - "capacity": "CAPACITY", - "sc_row_ind": "SC_ROW_IND", - "sc_col_ind": "SC_COL_IND", - "mean_cf": "MEAN_CF", - "capital_cost": "CAPITAL_COST", - "mean_lcoe_friction": "MEAN_LCOE_FRICTION", - "total_lcoe_friction": "TOTAL_LCOE_FRICTION", -} - class FieldEnum(str, Enum): """Base Field enum with some mapping methods.""" @classmethod def map_to(cls, other): - """Return a rename map from this enum to another.""" + """Return a rename map from this enum to another. + + Mapping is performed on matching enum names. In other words, if + both enums have a `ONE` attribute, this will be mapped from one + enum to another. + + Parameters + ---------- + other : :class:`Enum` + ``Enum`` subclass with ``__members__`` attribute. + + Returns + ------- + dict + Dictionary mapping matching values from one enum to another. + + Examples + -------- + >>> class Test1(FieldEnum): + >>> ONE = "one_x" + >>> TWO = "two" + >>> + >>> class Test2(Enum): + >>> ONE = "one_y" + >>> THREE = "three" + >>> + >>> Test1.map_to(Test2) + {: } + """ return { cls[mem]: other[mem] for mem in cls.__members__ @@ -51,15 +50,32 @@ def map_to(cls, other): @classmethod def map_from(cls, other): - """Return a rename map from a dictionary of name / member pairs (e.g. - 'sc_gid': 'SC_GID') to this enum.""" - return {name: cls[mem] for name, mem in other.items()} + """Map from a dictionary of name / member pairs to this enum. - @classmethod - def map_from_legacy(cls): - """Return a dictionary -> this enum map using the dictionary of legacy - names""" - return cls.map_from(OldSupplyCurveField) + Parameters + ---------- + other : dict + Dictionary mapping key values (typically old aliases) to + enum values. For example, ``{'sc_gid': 'SC_GID'}`` would + return a dictionary that maps ``'sc_gid'`` to the ``SC_GID`` + member of this enum. + + Returns + ------- + dict + Mapping of input dictionary keys to member values of this + enum. + + Examples + -------- + >>> class Test(FieldEnum): + >>> ONE = "one_x" + >>> TWO = "two_y" + >>> + >>> Test.map_from({1: "ONE", 2: "TWO"}) + {1: , 2: } + """ + return {name: cls[mem] for name, mem in other.items()} def __str__(self): return self.value @@ -93,11 +109,7 @@ class ResourceMetaField(FieldEnum): OFFSHORE = "offshore" -# Dictionary of "old" supply curve field names. Used to rename legacy data to -# match current naming conventions - - -class OriginalSupplyCurveField(FieldEnum): +class SupplyCurveField(FieldEnum): """An enumerated map to supply curve summary/meta keys. Each output name should match the name of a key in @@ -151,58 +163,32 @@ class OriginalSupplyCurveField(FieldEnum): REG_MULT = "reg_mult" -class SupplyCurveField(FieldEnum): - """An enumerated map to supply curve summary/meta keys. + @classmethod + def map_from_legacy(cls): + """Map of legacy names to current values. - Each output name should match the name of a key in - meth:`AggregationSupplyCurvePoint.summary` or - meth:`GenerationSupplyCurvePoint.point_summary` or - meth:`BespokeSinglePlant.meta` - """ + Returns + ------- + dict + Dictionary that maps legacy supply curve column names to + members of this enum. + """ + legacy_map = {} + for current_field, old_field in cls.map_to(_LegacySCAliases).items(): + aliases = old_field.value + if isinstance(aliases, str): + aliases = [aliases] + legacy_map.update({alias: current_field for alias in aliases}) - SC_POINT_GID = "sc_point_gid_m" - SOURCE_GIDS = "source_gids_m" - SC_GID = "sc_gid_m" - GID_COUNTS = "gid_counts_m" - GID = "gid_m" - N_GIDS = "n_gids_m" - RES_GIDS = "res_gids_m" - GEN_GIDS = "gen_gids_m" - AREA_SQ_KM = "area_sq_km_m" - LATITUDE = "latitude_m" - LONGITUDE = "longitude_m" - ELEVATION = "elevation_m" - TIMEZONE = "timezone_m" - COUNTY = "county_m" - STATE = "state_m" - COUNTRY = "country_m" - MEAN_CF = "mean_cf_m" - MEAN_LCOE = "mean_lcoe_m" - MEAN_RES = "mean_res_m" - CAPACITY = "capacity_m" - OFFSHORE = "offshore_m" - SC_ROW_IND = "sc_row_ind_m" - SC_COL_IND = "sc_col_ind_m" - CAPACITY_AC = "capacity_ac_m" - CAPITAL_COST = "capital_cost_m" - FIXED_OPERATING_COST = "fixed_operating_cost_m" - VARIABLE_OPERATING_COST = "variable_operating_cost_m" - FIXED_CHARGE_RATE = "fixed_charge_rate_m" - SC_POINT_CAPITAL_COST = "sc_point_capital_cost_m" - SC_POINT_FIXED_OPERATING_COST = "sc_point_fixed_operating_cost_m" - SC_POINT_ANNUAL_ENERGY = "sc_point_annual_energy_m" - SC_POINT_ANNUAL_ENERGY_AC = "sc_point_annual_energy_ac_m" - MEAN_FRICTION = "mean_friction_m" - MEAN_LCOE_FRICTION = "mean_lcoe_friction_m" - TOTAL_LCOE_FRICTION = "total_lcoe_friction_m" - RAW_LCOE = "raw_lcoe_m" - CAPITAL_COST_SCALAR = "capital_cost_scalar_m" - SCALED_CAPITAL_COST = "scaled_capital_cost_m" - SCALED_SC_POINT_CAPITAL_COST = "scaled_sc_point_capital_cost_m" - TURBINE_X_COORDS = "turbine_x_coords_m" - TURBINE_Y_COORDS = "turbine_y_coords_m" - EOS_MULT = "eos_mult_m" - REG_MULT = "reg_mult_m" + return legacy_map + + +class _LegacySCAliases(Enum): + """Legacy supply curve column names. + + Enum values can be either a single string or an iterable of string + values where each string value represents a previously known alias. + """ class ModuleName(str, Enum): From f6f677e36afd984539a871dda6de61ab21efada8 Mon Sep 17 00:00:00 2001 From: ppinchuk Date: Thu, 30 May 2024 11:04:40 -0600 Subject: [PATCH 59/61] Bump version --- reV/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reV/version.py b/reV/version.py index 756b3ea19..b657e3872 100644 --- a/reV/version.py +++ b/reV/version.py @@ -2,4 +2,4 @@ reV Version number """ -__version__ = "0.8.8" +__version__ = "0.8.9" From 2cecd2bb580be9d824af0c9513da63da5a19eee8 Mon Sep 17 00:00:00 2001 From: ppinchuk Date: Thu, 30 May 2024 11:12:22 -0600 Subject: [PATCH 60/61] Linter fixes --- reV/SAM/SAM.py | 5 ++-- reV/econ/econ.py | 8 +++---- reV/qa_qc/summary.py | 12 ++++++---- reV/supply_curve/competitive_wind_farms.py | 3 ++- reV/supply_curve/supply_curve.py | 27 ++++++++++++++-------- reV/utilities/pytest_utils.py | 2 +- 6 files changed, 34 insertions(+), 23 deletions(-) diff --git a/reV/SAM/SAM.py b/reV/SAM/SAM.py index 06c9bd78d..3df805588 100644 --- a/reV/SAM/SAM.py +++ b/reV/SAM/SAM.py @@ -891,9 +891,8 @@ def collect_outputs(self, output_lookup): bad_requests.append(req) if any(bad_requests): - msg = 'Could not retrieve outputs "{}" from PySAM object "{}".'.format( - bad_requests, self.pysam - ) + msg = ('Could not retrieve outputs "{}" from PySAM object "{}".' + .format(bad_requests, self.pysam)) logger.error(msg) raise SAMExecutionError(msg) diff --git a/reV/econ/econ.py b/reV/econ/econ.py index a77ddc172..1987657a5 100644 --- a/reV/econ/econ.py +++ b/reV/econ/econ.py @@ -352,10 +352,10 @@ def _run_single_worker(pc, econ_fun, output_request, **kwargs): pc : reV.config.project_points.PointsControl Iterable points control object from reV config module. Must have project_points with df property with all relevant - site-specific inputs and a `SupplyCurveField.GID` column. By passing - site-specific inputs in this dataframe, which was split using - points_control, only the data relevant to the current sites is - passed. + site-specific inputs and a `SupplyCurveField.GID` column. + By passing site-specific inputs in this dataframe, which + was split using points_control, only the data relevant to + the current sites is passed. econ_fun : method reV_run() method from one of the econ modules (SingleOwner, SAM_LCOE, WindBos). diff --git a/reV/qa_qc/summary.py b/reV/qa_qc/summary.py index acda6a409..386caf4cb 100644 --- a/reV/qa_qc/summary.py +++ b/reV/qa_qc/summary.py @@ -600,7 +600,9 @@ def _extract_sc_data(self, lcoe=SupplyCurveField.MEAN_LCOE): values = [SupplyCurveField.CAPACITY, lcoe] self._check_value(self.summary, values, scatter=False) sc_df = self.summary[values].sort_values(lcoe) - sc_df['cumulative_capacity'] = sc_df[SupplyCurveField.CAPACITY].cumsum() + sc_df['cumulative_capacity'] = ( + sc_df[SupplyCurveField.CAPACITY].cumsum() + ) return sc_df @@ -801,7 +803,9 @@ def _extract_sc_data(self, lcoe=SupplyCurveField.MEAN_LCOE): values = [SupplyCurveField.CAPACITY, lcoe] self._check_value(self.sc_table, values, scatter=False) sc_df = self.sc_table[values].sort_values(lcoe) - sc_df['cumulative_capacity'] = sc_df[SupplyCurveField.CAPACITY].cumsum() + sc_df['cumulative_capacity'] = ( + sc_df[SupplyCurveField.CAPACITY].cumsum() + ) return sc_df @@ -824,8 +828,8 @@ def supply_curve_plot(self, lcoe=SupplyCurveField.MEAN_LCOE, out_path=None, sc_df, x="cumulative_capacity", y=lcoe, filename=out_path, **kwargs ) - def supply_curve_plotly(self, lcoe=SupplyCurveField.MEAN_LCOE, out_path=None, - **kwargs): + def supply_curve_plotly(self, lcoe=SupplyCurveField.MEAN_LCOE, + out_path=None, **kwargs): """ Plot supply curve (cumulative capacity vs lcoe) using plotly diff --git a/reV/supply_curve/competitive_wind_farms.py b/reV/supply_curve/competitive_wind_farms.py index cd5709374..d2b3cb373 100644 --- a/reV/supply_curve/competitive_wind_farms.py +++ b/reV/supply_curve/competitive_wind_farms.py @@ -85,7 +85,8 @@ def __getitem__(self, keys): if not isinstance(keys, tuple): msg = ("{} must be a tuple of form (source, gid) where source is: " "{}, '{}', or 'upwind', 'downwind'" - .format(keys, SupplyCurveField.SC_GID, SupplyCurveField.SC_POINT_GID)) + .format(keys, SupplyCurveField.SC_GID, + SupplyCurveField.SC_POINT_GID)) logger.error(msg) raise ValueError(msg) diff --git a/reV/supply_curve/supply_curve.py b/reV/supply_curve/supply_curve.py index e34f05e74..ecff874bf 100644 --- a/reV/supply_curve/supply_curve.py +++ b/reV/supply_curve/supply_curve.py @@ -429,7 +429,8 @@ def _check_sub_trans_lines(cls, features): return line_gids[~test].tolist() @classmethod - def _check_substation_conns(cls, trans_table, sc_cols=SupplyCurveField.SC_GID): + def _check_substation_conns(cls, trans_table, + sc_cols=SupplyCurveField.SC_GID): """ Run checks on substation transmission features to make sure that every sc point connecting to a substation can also connect to its @@ -530,7 +531,8 @@ def _merge_sc_trans_tables(cls, sc_points, trans_table, sc_cols : tuple | list, optional List of column from sc_points to transfer into the trans table, If the `sc_capacity_col` is not included, it will get added. - by default (SupplyCurveField.SC_GID, 'capacity', 'mean_cf', 'mean_lcoe') + by default (SupplyCurveField.SC_GID, 'capacity', 'mean_cf', + 'mean_lcoe') sc_capacity_col : str, optional Name of capacity column in `trans_sc_table`. The values in this column determine the size of transmission lines built. @@ -597,8 +599,10 @@ def _merge_sc_trans_tables(cls, sc_points, trans_table, @classmethod def _map_tables(cls, sc_points, trans_table, - sc_cols=(SupplyCurveField.SC_GID, SupplyCurveField.CAPACITY, - SupplyCurveField.MEAN_CF, SupplyCurveField.MEAN_LCOE), + sc_cols=(SupplyCurveField.SC_GID, + SupplyCurveField.CAPACITY, + SupplyCurveField.MEAN_CF, + SupplyCurveField.MEAN_LCOE), sc_capacity_col=SupplyCurveField.CAPACITY): """ Map supply curve points to transmission features @@ -933,7 +937,8 @@ def compute_total_lcoe( self._trans_table["trans_cap_cost_per_mw"] = cost cost *= self._trans_table[self._sc_capacity_col] - cost /= self._trans_table[SupplyCurveField.CAPACITY] # align with "mean_cf" + # align with "mean_cf" + cost /= self._trans_table[SupplyCurveField.CAPACITY] if 'reinforcement_cost_per_mw' in self._trans_table: logger.info("'reinforcement_cost_per_mw' column found in " @@ -970,7 +975,9 @@ def _calculate_total_lcoe_friction(self): lcoe_friction = ( self._trans_table['lcot'] + self._trans_table[SupplyCurveField.MEAN_LCOE_FRICTION]) - self._trans_table[SupplyCurveField.TOTAL_LCOE_FRICTION] = lcoe_friction + self._trans_table[SupplyCurveField.TOTAL_LCOE_FRICTION] = ( + lcoe_friction + ) logger.info('Found mean LCOE with friction. Adding key ' '"total_lcoe_friction" to trans table.') @@ -1163,9 +1170,8 @@ def _full_sort( conn_lists[col_name][sc_gid] = data_arr[i] if total_lcoe_fric is not None: - conn_lists[SupplyCurveField.TOTAL_LCOE_FRICTION][sc_gid] = ( - total_lcoe_fric[i] - ) + col_name = SupplyCurveField.TOTAL_LCOE_FRICTION + conn_lists[col_name][sc_gid] = total_lcoe_fric[i] current_prog = connected // (len(self) / 100) if current_prog > progress: @@ -1349,7 +1355,8 @@ def full_sort( trans_table = trans_table.loc[~pos].sort_values([sort_on, "trans_gid"]) total_lcoe_fric = None - if consider_friction and SupplyCurveField.MEAN_LCOE_FRICTION in trans_table: + col_in_table = SupplyCurveField.MEAN_LCOE_FRICTION in trans_table + if consider_friction and col_in_table: total_lcoe_fric = \ trans_table[SupplyCurveField.TOTAL_LCOE_FRICTION].values diff --git a/reV/utilities/pytest_utils.py b/reV/utilities/pytest_utils.py index d49ec5d6e..94e183830 100644 --- a/reV/utilities/pytest_utils.py +++ b/reV/utilities/pytest_utils.py @@ -8,7 +8,7 @@ from packaging import version from rex.outputs import Outputs as RexOutputs -from reV.utilities import ResourceMetaField, SupplyCurveField +from reV.utilities import ResourceMetaField def pd_date_range(*args, **kwargs): From b267dac2fbcc237601c278150fe45841c36d7632 Mon Sep 17 00:00:00 2001 From: ppinchuk Date: Thu, 30 May 2024 11:16:38 -0600 Subject: [PATCH 61/61] Linter fix --- reV/SAM/SAM.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reV/SAM/SAM.py b/reV/SAM/SAM.py index 3df805588..9ae66fbdd 100644 --- a/reV/SAM/SAM.py +++ b/reV/SAM/SAM.py @@ -714,7 +714,7 @@ def ensure_res_len(arr, time_index): freq = pd.infer_freq(time_index[:s]) msg = "frequencies do not match before and after 2/29" - assert freq == pd.infer_freq(time_index[s + 1 :]), msg + assert freq == pd.infer_freq(time_index[s + 1:]), msg else: freq = pd.infer_freq(time_index) else: