Repository: climate Updated Branches: refs/heads/master 77463b926 -> 58d6cdf16
CLIMATE-927 - RCMES script for running multiple evaluations Project: http://git-wip-us.apache.org/repos/asf/climate/repo Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/fb02c749 Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/fb02c749 Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/fb02c749 Branch: refs/heads/master Commit: fb02c74925b977abd2c03f19ea0ccd9b15faec23 Parents: 77463b9 Author: Alex <ago...@users.noreply.github.com> Authored: Thu Sep 28 23:11:51 2017 -0700 Committer: Alex <ago...@users.noreply.github.com> Committed: Thu Sep 28 23:11:51 2017 -0700 ---------------------------------------------------------------------- RCMES/CORDEX/cordex.py | 56 ++++++++++++++++++ RCMES/CORDEX/metadata_extractor.py | 35 +++++++++-- RCMES/CORDEX/templates/CORDEX.yaml.template | 8 +-- RCMES/metrics_and_plots.py | 2 +- RCMES/run_RCMES.py | 24 +++++--- ocw/plotter.py | 74 ++++++++++++------------ 6 files changed, 143 insertions(+), 56 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/climate/blob/fb02c749/RCMES/CORDEX/cordex.py ---------------------------------------------------------------------- diff --git a/RCMES/CORDEX/cordex.py b/RCMES/CORDEX/cordex.py new file mode 100644 index 0000000..dc80be4 --- /dev/null +++ b/RCMES/CORDEX/cordex.py @@ -0,0 +1,56 @@ +import os +import subprocess +import jinja2 +from metadata_extractor import CORDEXMetadataExtractor, obs4MIPSMetadataExtractor + +# These should be modified. TODO: domains can also be made into separate group +# CORDEX domain +domain = 'NAM-44' + +# The output directory +workdir = '/home/goodman/data_processing/CORDEX/analysis' + +# Location of osb4Mips files +obs_dir = '/proj3/data/obs4mips' + +# Location of CORDEX files +models_dir = '/proj3/data/CORDEX/{domain}/*'.format(domain=domain) + +# Extract metadata from model and obs files, pairing up files with the same +# variables for separate evaluations +obs_extractor = obs4MIPSMetadataExtractor(obs_dir) +models_extractor = CORDEXMetadataExtractor(models_dir) +groups = obs_extractor.group(models_extractor, 'variable') + +# Configuration file template, to be rendered repeatedly for each evaluation +# run +env = jinja2.Environment(loader=jinja2.FileSystemLoader('./templates'), + trim_blocks=True, lstrip_blocks=True) +t = env.get_template('CORDEX.yaml.template') + +# Each group represents a single evaluation. Repeat the evaluation for +# three seasons: Summer, Winter, and Annual. +seasons = ['annual', 'winter', 'summer'] +for group in groups: + obs_info, models_info = group + instrument = obs_info['instrument'] + variable = obs_info['variable'] + for season in seasons: + configfile_basename = '_'.join([domain, instrument, variable, season]) + '.yaml' + configfile_path = os.path.join(workdir, domain, instrument, + variable, season) + if not os.path.exists(configfile_path): + os.makedirs(configfile_path) + configfile_path = os.path.join(configfile_path, configfile_basename) + with open(configfile_path, 'w') as configfile: + configfile.write(t.render(obs_info=obs_info, models_info=models_info, + season=season, output_dir=workdir)) + + # TODO: Do this in parallel. Will change this once this approach + # is well tested. + code = subprocess.call(['python', '../run_RCMES.py', configfile_path]) + errored = [] + if code: + errored.append(configfile_path) + +print("All runs done. The following ended with an error: {}".format(errored)) http://git-wip-us.apache.org/repos/asf/climate/blob/fb02c749/RCMES/CORDEX/metadata_extractor.py ---------------------------------------------------------------------- diff --git a/RCMES/CORDEX/metadata_extractor.py b/RCMES/CORDEX/metadata_extractor.py index cb18946..b97d890 100644 --- a/RCMES/CORDEX/metadata_extractor.py +++ b/RCMES/CORDEX/metadata_extractor.py @@ -84,6 +84,13 @@ class MetadataExtractor(object): """ return self.get_field('variable') + @property + def field_filters(self): + """ + Override this to filter out specific characters contained in a field. + """ + return dict() + def query(self, **kwargs): """ Narrow down the list of files by field names. @@ -95,7 +102,8 @@ class MetadataExtractor(object): data = self.data for field, value in kwargs.items(): value = value if isinstance(value, list) else [value] - data = [meta for meta in data if meta[field] in value] + data = [meta for meta in data + if self._match_filter(meta, field) in value] return data def group(self, extractor, field): @@ -116,7 +124,7 @@ class MetadataExtractor(object): groups = [] for meta in results: - val = meta[field] + val = self._match_filter(meta, field) kwargs.update({field: val}) match = extractor.query(**kwargs) groups.append((meta, match)) @@ -151,6 +159,16 @@ class MetadataExtractor(object): pattern = '_'.join(base[:len(self.fields)] + ['*.nc']) return pattern + def _match_filter(self, meta, field): + """ + Filter (ignore) certain character patterns when matching a field. + """ + val = meta[field] + if field in self.field_filters: + for pattern in self.field_filters[field]: + val = val.replace(pattern, '') + return val + def _extract(self): """ Do the actual metadata extraction from the list of filename given @@ -183,6 +201,13 @@ class obs4MIPSMetadataExtractor(MetadataExtractor): fields = ['variable', 'instrument', 'processing_level', 'version'] return fields + @property + def field_filters(self): + """ + Field filters for CALIPSO + """ + return dict(variable=['calipso', 'Lidarsr532']) + def filter_filename(self, fname): """ CALIPSO files have odd naming conventions, so we will use @@ -190,8 +215,6 @@ class obs4MIPSMetadataExtractor(MetadataExtractor): """ fname = os.path.basename(fname) fname = fname.replace('_obs4MIPs_', '_') - fname = fname.replace('calipso', '') - fname = fname.replace('Lidarsr532', '') return fname def get_pattern(self, fname): @@ -202,8 +225,8 @@ class obs4MIPSMetadataExtractor(MetadataExtractor): offset = -2 if len(base) != 5 else -1 pattern = '_'.join(base[:offset] + ['*.nc']) return pattern - - + + class CORDEXMetadataExtractor(MetadataExtractor): @property def models(self): http://git-wip-us.apache.org/repos/asf/climate/blob/fb02c749/RCMES/CORDEX/templates/CORDEX.yaml.template ---------------------------------------------------------------------- diff --git a/RCMES/CORDEX/templates/CORDEX.yaml.template b/RCMES/CORDEX/templates/CORDEX.yaml.template index daf6ec5..c0a1994 100644 --- a/RCMES/CORDEX/templates/CORDEX.yaml.template +++ b/RCMES/CORDEX/templates/CORDEX.yaml.template @@ -1,6 +1,6 @@ {% set domain = models_info[0].domain %} {% set instrument = obs_info.instrument %} -{% set variable = obs_info.variable %} +{% set variable = models_info[0].variable %} {% set basename = [variable, instrument, domain, season]|join('_') %} workdir: {{ [output_dir, domain, instrument, variable, season]|join('/') }} output_netcdf_filename: {{ basename }}.nc @@ -23,7 +23,7 @@ time: average_each_year: True space: - boundary_type: {{ domain[:3] }} + boundary_type: CORDEX {{ domain[:3] }} regrid: regrid_on_reference: True @@ -32,12 +32,12 @@ datasets: - loader_name: local_split name: {{ instrument }} file_path: {{ obs_info.filename }} - variable_name: {{ variable }} + variable_name: {{ obs_info.variable }} {% for model_info in models_info %} - loader_name: local_split name: {{ model_info.model }} file_path: {{ model_info.filename }} - variable_name: {{ variable }} + variable_name: {{ model_info.variable }} lat_name: lat lon_name: lon {% endfor %} http://git-wip-us.apache.org/repos/asf/climate/blob/fb02c749/RCMES/metrics_and_plots.py ---------------------------------------------------------------------- diff --git a/RCMES/metrics_and_plots.py b/RCMES/metrics_and_plots.py index 78c2f14..0eb1f02 100644 --- a/RCMES/metrics_and_plots.py +++ b/RCMES/metrics_and_plots.py @@ -59,7 +59,7 @@ def Map_plot_bias_of_multiyear_climatology(obs_dataset, obs_name, model_datasets string_list = list(string.ascii_lowercase) nmodels = len(model_datasets) - row, column = plotter._best_grid_shape((row, column), nmodels + 1) + row, column = plotter._best_grid_shape(nmodels + 1, (row, column)) ax = fig.add_subplot(row,column,1) if map_projection == 'npstere': m = Basemap(ax=ax, projection ='npstere', boundinglat=lat_min, lon_0=0, http://git-wip-us.apache.org/repos/asf/climate/blob/fb02c749/RCMES/run_RCMES.py ---------------------------------------------------------------------- diff --git a/RCMES/run_RCMES.py b/RCMES/run_RCMES.py index 7641299..f9ec2c3 100644 --- a/RCMES/run_RCMES.py +++ b/RCMES/run_RCMES.py @@ -25,6 +25,11 @@ from datetime import datetime from glob import glob from getpass import getpass import numpy as np + +# Need these lines to run RCMES through SSH without X11 +import matplotlib +matplotlib.use('Agg') + import ocw.utils as utils import ocw.dataset_processor as dsp from ocw.dataset import Bounds @@ -68,7 +73,7 @@ time_info = config['time'] temporal_resolution = time_info['temporal_resolution'] # Read time info -maximum_overlap_period = space_info.get('maximum_overlap_period', False) +maximum_overlap_period = time_info.get('maximum_overlap_period', False) if not maximum_overlap_period: start_time = datetime.strptime(time_info['start_time'].strftime('%Y%m%d'),'%Y%m%d') end_time = datetime.strptime(time_info['end_time'].strftime('%Y%m%d'),'%Y%m%d') @@ -86,7 +91,8 @@ if not 'boundary_type' in space_info: else: domain = space_info['boundary_type'] if domain.startswith('CORDEX '): - domain = domain.replace('CORDEX ', '').lower() + domain = domain.replace('CORDEX', '').lower() + domain = domain.replace(' ', '') min_lat, max_lat, min_lon, max_lon = utils.CORDEX_boundary(domain) # Additional arguments for the DatasetLoader @@ -110,9 +116,9 @@ multiplying_factor = np.ones(len(datasets)) multiplying_factor[0] = fact names = [dataset.name for dataset in datasets] for i, dataset in enumerate(datasets): - if temporal_resolution == 'daily' or temporal_resolution == 'monthly': - datasets[i] = dsp.normalize_dataset_datetimes(dataset, - temporal_resolution) + res = dataset.temporal_resolution() + if res == 'daily' or res == 'monthly': + datasets[i] = dsp.normalize_dataset_datetimes(dataset, res) if multiplying_factor[i] != 1: datasets[i].values *= multiplying_factor[i] @@ -148,7 +154,7 @@ else: for i, dataset in enumerate(datasets): datasets[i] = dsp.subset(dataset, bounds) if dataset.temporal_resolution() != temporal_resolution: - datasets[i] = dsp.temporal_rebin(dataset, temporal_resolution) + datasets[i] = dsp.temporal_rebin(datasets[i], temporal_resolution) # Temporally subset both observation and model datasets # for the user specified season @@ -269,8 +275,10 @@ if nmetrics > 0: file_name = workdir+plot_info['file_name'] print('metrics {0}/{1}: {2}'.format(imetric, nmetrics, metrics_name)) + default_shape = (int(np.ceil(np.sqrt(ntarget + 2))), + int(np.ceil(np.sqrt(ntarget + 2)))) if metrics_name == 'Map_plot_bias_of_multiyear_climatology': - row, column = plot_info.get('subplots_array', (1, 1)) + row, column = plot_info.get('subplots_array', default_shape) if 'map_projection' in plot_info.keys(): Map_plot_bias_of_multiyear_climatology( reference_dataset, reference_name, target_datasets, target_names, @@ -287,7 +295,7 @@ if nmetrics > 0: elif config['use_subregions']: if (metrics_name == 'Timeseries_plot_subregion_interannual_variability' and average_each_year): - row, column = plot_info.get('subplots_array', (1, 1)) + row, column = plot_info.get('subplots_array', default_shape) Time_series_subregion( reference_subregion_mean, reference_name, target_subregion_mean, target_names, False, file_name, row, column, http://git-wip-us.apache.org/repos/asf/climate/blob/fb02c749/ocw/plotter.py ---------------------------------------------------------------------- diff --git a/ocw/plotter.py b/ocw/plotter.py index 4896a64..7f9b092 100755 --- a/ocw/plotter.py +++ b/ocw/plotter.py @@ -80,12 +80,12 @@ def _nice_intervals(data, nlevs): else: mnlvl = mn mxlvl = mx - + # hack to make generated intervals from mpl the same for all versions autolimit_mode = mpl.rcParams.get('axes.autolimit_mode') if autolimit_mode: mpl.rc('axes', autolimit_mode='round_numbers') - + locator = mpl.ticker.MaxNLocator(nlevs) clevs = locator.tick_values(mnlvl, mxlvl) if autolimit_mode: @@ -122,14 +122,14 @@ def _best_grid_shape(nplots, oldshape): # rows and columns for gridshape, automatically # correct it so that it fits only as many plots # as needed - while diff >= ncols: - nrows -= 1 + while diff >= nrows: + ncols -= 1 size = nrows * ncols diff = size - nplots # Don't forget to remove unnecessary columns too - if nrows == 1: - ncols = nplots + if ncols == 1: + nrows = nplots newshape = nrows, ncols return newshape @@ -1096,24 +1096,24 @@ def draw_histogram(dataset_array, data_names, fname, fmt='png', nbins=10): def fill_US_states_with_color(regions, fname, fmt='png', ptitle='', colors=False, values=None, region_names=None): - - ''' Fill the States over the contiguous US with colors - - :param regions: The list of subregions(lists of US States) to be filled + + ''' Fill the States over the contiguous US with colors + + :param regions: The list of subregions(lists of US States) to be filled with different colors. - :type regions: :class:`list` + :type regions: :class:`list` :param fname: The filename of the plot. :type fname: :mod:`string` :param fmt: (Optional) filetype for the output. :type fmt: :mod:`string` :param ptitle: (Optional) plot title. :type ptitle: :mod:`string` - :param colors: (Optional) : If True, each region will be filled + :param colors: (Optional) : If True, each region will be filled with different colors without using values - :type colors: :class:`bool` - :param values: (Optional) : If colors==False, color for each region scales - an associated element in values - :type values: :class:`numpy.ndarray` + :type colors: :class:`bool` + :param values: (Optional) : If colors==False, color for each region scales + an associated element in values + :type values: :class:`numpy.ndarray` ''' nregion = len(regions) @@ -1140,7 +1140,7 @@ def fill_US_states_with_color(regions, fname, fmt='png', ptitle='', lons=np.empty(0) for shape in shapes: patches.append(Polygon(np.array(shape), True)) - + lons = np.append(lons, shape[:,0]) lats = np.append(lats, shape[:,1]) if colors: @@ -1164,13 +1164,13 @@ def draw_plot_to_compare_trends(obs_data, ens_data, model_data, fname, fmt='png', ptitle='', data_labels=None, xlabel='', ylabel=''): - ''' Fill the States over the contiguous US with colors - + ''' Fill the States over the contiguous US with colors + :param obs_data: An array of observed trend and standard errors for regions :type obs_data: :class:'numpy.ndarray' - :param ens_data: An array of trend and standard errors from a multi-model ensemble for regions + :param ens_data: An array of trend and standard errors from a multi-model ensemble for regions :type ens_data: : class:'numpy.ndarray' - :param model_data: An array of trends from models for regions + :param model_data: An array of trends from models for regions :type model_data: : class:'numpy.ndarray' :param fname: The filename of the plot. :type fname: :mod:`string` @@ -1182,36 +1182,36 @@ def draw_plot_to_compare_trends(obs_data, ens_data, model_data, :type data_labels: :mod:`list` :param xlabel: (Optional) a label for x-axis :type xlabel: :mod:`string` - :param ylabel: (Optional) a label for y-axis + :param ylabel: (Optional) a label for y-axis :type ylabel: :mod:`string` ''' - nregions = obs_data.shape[1] + nregions = obs_data.shape[1] # Set up the figure fig = plt.figure() fig.set_size_inches((8.5, 11.)) fig.dpi = 300 ax = fig.add_subplot(111) - - b_plot = ax.boxplot(model_data, widths=np.repeat(0.2, nregions), positions=np.arange(nregions)+1.3) + + b_plot = ax.boxplot(model_data, widths=np.repeat(0.2, nregions), positions=np.arange(nregions)+1.3) plt.setp(b_plot['medians'], color='black') plt.setp(b_plot['whiskers'], color='black') plt.setp(b_plot['boxes'], color='black') plt.setp(b_plot['fliers'], color='black') - ax.errorbar(np.arange(nregions)+0.8, obs_data[0,:], yerr=obs_data[1,:], - fmt='o', color='r', ecolor='r') - ax.errorbar(np.arange(nregions)+1., ens_data[0,:], yerr=ens_data[1,:], - fmt='o', color='b', ecolor='b') + ax.errorbar(np.arange(nregions)+0.8, obs_data[0,:], yerr=obs_data[1,:], + fmt='o', color='r', ecolor='r') + ax.errorbar(np.arange(nregions)+1., ens_data[0,:], yerr=ens_data[1,:], + fmt='o', color='b', ecolor='b') ax.set_xticks(np.arange(nregions)+1) ax.set_xlim([0, nregions+1]) - + if data_labels: ax.set_xticklabels(data_labels) - fig.savefig('%s.%s' % (fname, fmt), bbox_inches='tight') + fig.savefig('%s.%s' % (fname, fmt), bbox_inches='tight') -def draw_precipitation_JPDF (plot_data, plot_level, x_ticks, x_names,y_ticks,y_names, - output_file, title=None, diff_plot=False, cmap = cm.BrBG, - cbar_ticks=[0.01, 0.10, 0.5, 2, 5, 25], +def draw_precipitation_JPDF (plot_data, plot_level, x_ticks, x_names,y_ticks,y_names, + output_file, title=None, diff_plot=False, cmap = cm.BrBG, + cbar_ticks=[0.01, 0.10, 0.5, 2, 5, 25], cbar_label=['0.01', '0.10', '0.5', '2', '5', '25']): ''' :param plot_data: a numpy array of data to plot (dimY, dimX) @@ -1219,11 +1219,11 @@ def draw_precipitation_JPDF (plot_data, plot_level, x_ticks, x_names,y_ticks,y_n :param plot_level: levels to plot :type plot_level: :class:'numpy.ndarray' :param x_ticks: x values where tick makrs are located - :type x_ticks: :class:'numpy.ndarray' + :type x_ticks: :class:'numpy.ndarray' :param x_names: labels for the ticks on x-axis (dimX) :type x_names: :class:'list' :param y_ticks: y values where tick makrs are located - :type y_ticks: :class:'numpy.ndarray' + :type y_ticks: :class:'numpy.ndarray' :param y_names: labels for the ticks on y-axis (dimY) :type y_names: :class:'list' :param output_file: name of output png file @@ -1246,7 +1246,7 @@ def draw_precipitation_JPDF (plot_data, plot_level, x_ticks, x_names,y_ticks,y_n dimY, dimX = plot_data.shape plot_data2 = np.zeros([dimY,dimX]) # sectioned array for plotting - # fontsize + # fontsize rcParams['axes.labelsize'] = 8 rcParams['xtick.labelsize'] = 8 rcParams['ytick.labelsize'] = 8