CLIMATE-966 - Jupyter Notebook to run cordex.py on AWS
diff --git a/RCMES/CORDEX/CORDEX.ipynb b/RCMES/CORDEX/CORDEX.ipynb
new file mode 100644
index 0000000..9574260
--- /dev/null
+++ b/RCMES/CORDEX/CORDEX.ipynb
@@ -0,0 +1,166 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "import sys\n",
+ "import subprocess\n",
+ "import jinja2\n",
+ "import requests\n",
+ "from metadata_extractor import CORDEXMetadataExtractor, obs4MIPSMetadataExtractor\n",
+ "from tqdm import tqdm_notebook as tqdm\n",
+ "from glob import glob\n",
+ "from IPython.display import Markdown, Image"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "domain should be one of these three: 'AFR-44', 'EUR-11', 'NAM-44'\n",
+ "* AFR-44: CORDEX Africa RCMs at 44 km resolution\n",
+ "* EUR-11: CORDEX Europe RCMs at 11 km resolution\n",
+ "* NAM-44: CORDEX North America RCMs at 44 km resolution"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "domain = 'NAM-44'"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# The output directory\n",
+ "cwd = os.getcwd()\n",
+ "workdir = cwd +'/evaluation_result'"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Location of obs4Mips files\n",
+ "obs_dir = '/mnt/efs/obs4mips'"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Location of CORDEX files\n",
+ "models_dir = '/mnt/efs/'+domain+'/*'.format(domain=domain)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Extract metadata from model and obs files, pairing up files with the same\n",
+ "# variables for separate evaluations\n",
+ "obs_extractor = obs4MIPSMetadataExtractor(obs_dir)\n",
+ "models_extractor = CORDEXMetadataExtractor(models_dir)\n",
+ "groups = obs_extractor.group(models_extractor, 'variable')\n",
+ "\n",
+ "# Configuration file template, to be rendered repeatedly for each evaluation\n",
+ "# run\n",
+ "env = jinja2.Environment(loader=jinja2.FileSystemLoader('./templates'),\n",
+ " trim_blocks=True, lstrip_blocks=True)\n",
+ "t = env.get_template('CORDEX.yaml.template')\n",
+ "\n",
+ "# Each group represents a single evaluation. Repeat the evaluation for\n",
+ "# three seasons: Summer, Winter, and Annual.\n",
+ "seasons = ['annual', 'winter', 'summer']\n",
+ "errored = []\n",
+ "for group in tqdm(groups, desc='variable loop'):\n",
+ " obs_info, models_info = group\n",
+ " instrument = obs_info['instrument']\n",
+ " variable = obs_info['variable']\n",
+ " for season in tqdm(seasons, desc='season loop'):\n",
+ " configfile_basename = '_'.join([domain, instrument, variable, season]) + '.yaml'\n",
+ " configfile_path = os.path.join(workdir, domain, instrument,\n",
+ " variable, season)\n",
+ " if not os.path.exists(configfile_path):\n",
+ " os.makedirs(configfile_path)\n",
+ " configfile_path = os.path.join(configfile_path, configfile_basename)\n",
+ " with open(configfile_path, 'w') as configfile:\n",
+ " configfile.write(t.render(obs_info=obs_info, models_info=models_info,\n",
+ " season=season, output_dir=workdir))\n",
+ "\n",
+ " # TODO: Do this in parallel. Will change this once this approach\n",
+ " # is well tested.\n",
+ " code = subprocess.call([sys.executable, '../run_RCMES.py', configfile_path])\n",
+ " if code:\n",
+ " errored.append(configfile_path)\n",
+ "print(\"All runs done. The following ended with an error: {}\".format(errored))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Check the evaluation result or download the processed obs4mips and model output."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "display(Markdown('Evaluation results'))\n",
+ "ip_address = (requests.get('http://169.254.169.254/latest/meta-data/public-ipv4').content).decode('utf-8')\n",
+ "for obs_info in obs_extractor.data:\n",
+ " inst_name = obs_info['instrument']\n",
+ " var_name = obs_info['variable']\n",
+ " display(Markdown('Instrument: '+inst_name+'& Variable: '+var_name))\n",
+ " for season in seasons:\n",
+ " os.chdir(workdir+'/'+domain+'/'+inst_name+'/'+var_name+'/'+season)\n",
+ " png_files = glob('*.png')\n",
+ " for png_file in png_files: \n",
+ " display(Image(os.getcwd()+'/'+png_file))\n",
+ " nc_file = glob('*.nc')[0]\n",
+ " display(Markdown('['+nc_file+'](http://'+ip_address+':8888'+os.getcwd()+'/'+nc_file+')'))\n",
+ "os.chdir(cwd) \n"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.7.3"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
