This is an automated email from the ASF dual-hosted git repository. huikyole pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/climate.git
commit 40dd07f7eb8dd4af35a4100fe6996be028319704 Author: huikyole <huikyo....@jpl.nasa.gov> AuthorDate: Wed Aug 7 21:08:37 2019 -0700 CLIMATE-966 - Jupyter Notebook to run cordex.py on AWS --- RCMES/CORDEX/CORDEX.ipynb | 166 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 166 insertions(+) diff --git a/RCMES/CORDEX/CORDEX.ipynb b/RCMES/CORDEX/CORDEX.ipynb new file mode 100644 index 0000000..9574260 --- /dev/null +++ b/RCMES/CORDEX/CORDEX.ipynb @@ -0,0 +1,166 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import sys\n", + "import subprocess\n", + "import jinja2\n", + "import requests\n", + "from metadata_extractor import CORDEXMetadataExtractor, obs4MIPSMetadataExtractor\n", + "from tqdm import tqdm_notebook as tqdm\n", + "from glob import glob\n", + "from IPython.display import Markdown, Image" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "domain should be one of these three: 'AFR-44', 'EUR-11', 'NAM-44'\n", + "* AFR-44: CORDEX Africa RCMs at 44 km resolution\n", + "* EUR-11: CORDEX Europe RCMs at 11 km resolution\n", + "* NAM-44: CORDEX North America RCMs at 44 km resolution" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "domain = 'NAM-44'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# The output directory\n", + "cwd = os.getcwd()\n", + "workdir = cwd +'/evaluation_result'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Location of obs4Mips files\n", + "obs_dir = '/mnt/efs/obs4mips'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Location of CORDEX files\n", + "models_dir = '/mnt/efs/'+domain+'/*'.format(domain=domain)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Extract metadata from model and obs files, pairing up files with the same\n", + "# variables for separate evaluations\n", + "obs_extractor = obs4MIPSMetadataExtractor(obs_dir)\n", + "models_extractor = CORDEXMetadataExtractor(models_dir)\n", + "groups = obs_extractor.group(models_extractor, 'variable')\n", + "\n", + "# Configuration file template, to be rendered repeatedly for each evaluation\n", + "# run\n", + "env = jinja2.Environment(loader=jinja2.FileSystemLoader('./templates'),\n", + " trim_blocks=True, lstrip_blocks=True)\n", + "t = env.get_template('CORDEX.yaml.template')\n", + "\n", + "# Each group represents a single evaluation. Repeat the evaluation for\n", + "# three seasons: Summer, Winter, and Annual.\n", + "seasons = ['annual', 'winter', 'summer']\n", + "errored = []\n", + "for group in tqdm(groups, desc='variable loop'):\n", + " obs_info, models_info = group\n", + " instrument = obs_info['instrument']\n", + " variable = obs_info['variable']\n", + " for season in tqdm(seasons, desc='season loop'):\n", + " configfile_basename = '_'.join([domain, instrument, variable, season]) + '.yaml'\n", + " configfile_path = os.path.join(workdir, domain, instrument,\n", + " variable, season)\n", + " if not os.path.exists(configfile_path):\n", + " os.makedirs(configfile_path)\n", + " configfile_path = os.path.join(configfile_path, configfile_basename)\n", + " with open(configfile_path, 'w') as configfile:\n", + " configfile.write(t.render(obs_info=obs_info, models_info=models_info,\n", + " season=season, output_dir=workdir))\n", + "\n", + " # TODO: Do this in parallel. Will change this once this approach\n", + " # is well tested.\n", + " code = subprocess.call([sys.executable, '../run_RCMES.py', configfile_path])\n", + " if code:\n", + " errored.append(configfile_path)\n", + "print(\"All runs done. The following ended with an error: {}\".format(errored))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Check the evaluation result or download the processed obs4mips and model output." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "display(Markdown('Evaluation results'))\n", + "ip_address = (requests.get('http://169.254.169.254/latest/meta-data/public-ipv4').content).decode('utf-8')\n", + "for obs_info in obs_extractor.data:\n", + " inst_name = obs_info['instrument']\n", + " var_name = obs_info['variable']\n", + " display(Markdown('Instrument: '+inst_name+'& Variable: '+var_name))\n", + " for season in seasons:\n", + " os.chdir(workdir+'/'+domain+'/'+inst_name+'/'+var_name+'/'+season)\n", + " png_files = glob('*.png')\n", + " for png_file in png_files: \n", + " display(Image(os.getcwd()+'/'+png_file))\n", + " nc_file = glob('*.nc')[0]\n", + " display(Markdown('['+nc_file+'](http://'+ip_address+':8888'+os.getcwd()+'/'+nc_file+')'))\n", + "os.chdir(cwd) \n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}