Merge branch 'CLIMATE-970'
diff --git a/RCMES/CORDEX/CORDEX.ipynb b/RCMES/CORDEX/CORDEX.ipynb
new file mode 100644
index 0000000..2cef98e
--- /dev/null
+++ b/RCMES/CORDEX/CORDEX.ipynb
@@ -0,0 +1,166 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "import sys\n",
+ "import subprocess\n",
+ "import jinja2\n",
+ "import requests\n",
+ "from metadata_extractor import CORDEXMetadataExtractor, obs4MIPSMetadataExtractor\n",
+ "from tqdm import tqdm_notebook as tqdm\n",
+ "from glob import glob\n",
+ "from IPython.display import Markdown, Image, FileLink"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "domain should be one of these three: 'AFR-44', 'EUR-11', 'NAM-44'\n",
+ "* AFR-44: CORDEX Africa RCMs at 44 km resolution\n",
+ "* EUR-11: CORDEX Europe RCMs at 11 km resolution\n",
+ "* NAM-44: CORDEX North America RCMs at 44 km resolution"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "domain = 'NAM-44'"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# The output directory\n",
+ "cwd = os.getcwd()\n",
+ "workdir = cwd +'/evaluation_result'"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Location of obs4Mips files\n",
+ "obs_dir = '/mnt/efs/obs4mips'"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Location of CORDEX files\n",
+ "models_dir = '/mnt/efs/'+domain+'/*'.format(domain=domain)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Extract metadata from model and obs files, pairing up files with the same\n",
+ "# variables for separate evaluations\n",
+ "obs_extractor = obs4MIPSMetadataExtractor(obs_dir)\n",
+ "models_extractor = CORDEXMetadataExtractor(models_dir)\n",
+ "groups = obs_extractor.group(models_extractor, 'variable')\n",
+ "\n",
+ "# Configuration file template, to be rendered repeatedly for each evaluation\n",
+ "# run\n",
+ "env = jinja2.Environment(loader=jinja2.FileSystemLoader('./templates'),\n",
+ " trim_blocks=True, lstrip_blocks=True)\n",
+ "t = env.get_template('CORDEX.yaml.template')\n",
+ "\n",
+ "# Each group represents a single evaluation. Repeat the evaluation for\n",
+ "# three seasons: Summer, Winter, and Annual.\n",
+ "seasons = ['annual', 'winter', 'summer']\n",
+ "errored = []\n",
+ "for group in tqdm(groups, desc='variable loop'):\n",
+ " obs_info, models_info = group\n",
+ " instrument = obs_info['instrument']\n",
+ " variable = obs_info['variable']\n",
+ " for season in tqdm(seasons, desc='season loop'):\n",
+ " configfile_basename = '_'.join([domain, instrument, variable, season]) + '.yaml'\n",
+ " configfile_path = os.path.join(workdir, domain, instrument,\n",
+ " variable, season)\n",
+ " if not os.path.exists(configfile_path):\n",
+ " os.makedirs(configfile_path)\n",
+ " configfile_path = os.path.join(configfile_path, configfile_basename)\n",
+ " with open(configfile_path, 'w') as configfile:\n",
+ " configfile.write(t.render(obs_info=obs_info, models_info=models_info,\n",
+ " season=season, output_dir=workdir))\n",
+ "\n",
+ " # TODO: Do this in parallel. Will change this once this approach\n",
+ " # is well tested.\n",
+ " code = subprocess.call([sys.executable, '../run_RCMES.py', configfile_path])\n",
+ " if code:\n",
+ " errored.append(configfile_path)\n",
+ "print(\"All runs done. The following ended with an error: {}\".format(errored))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Check the evaluation result or download the processed obs4mips and model output."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "display(Markdown('Evaluation results'))\n",
+ "ip_address = (requests.get('http://169.254.169.254/latest/meta-data/public-ipv4').content).decode('utf-8')\n",
+ "for obs_info in obs_extractor.data:\n",
+ " inst_name = obs_info['instrument']\n",
+ " var_name = obs_info['variable']\n",
+ " display(Markdown('Instrument: '+inst_name+'& Variable: '+var_name))\n",
+ " for season in seasons:\n",
+ " savedir = os.path.join('evaluation_result', domain, inst_name, var_name, season)\n",
+ " png_files = glob(os.path.join(savedir, '*.png'))\n",
+ " for png_file in png_files: \n",
+ " display(Image(png_file))\n",
+ " nc_file = glob(os.path.join(savedir, '*.nc'))[0]\n",
+ " display(FileLink(nc_file))\n",
+ "os.chdir(cwd) \n"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.7.3"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/RCMES/statistical_downscaling/run_statistical_downscaling.py b/RCMES/statistical_downscaling/run_statistical_downscaling.py
index 18b39fe..832c283 100644
--- a/RCMES/statistical_downscaling/run_statistical_downscaling.py
+++ b/RCMES/statistical_downscaling/run_statistical_downscaling.py
@@ -91,7 +91,7 @@
config_file = str(sys.argv[1])
-print 'Reading the configuration file ', config_file
+print('Reading the configuration file ', config_file)
config = yaml.load(open(config_file))
diff --git a/dmitri_example.ipynb b/dmitri_example.ipynb
new file mode 100644
index 0000000..b9876e5
--- /dev/null
+++ b/dmitri_example.ipynb
@@ -0,0 +1,61 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "- Example code for first commit to OCW repo (-Dmitri Kalashnikov)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Hello World!\n",
+ "Hello World!\n",
+ "Hello World!\n",
+ "Hello World!\n",
+ "Hello World!\n"
+ ]
+ }
+ ],
+ "source": [
+ "for k in range(5):\n",
+ " print('Hello World!')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.7.1"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/docs/source/ocw/overview.rst b/docs/source/ocw/overview.rst
index 13f4aba..702f538 100644
--- a/docs/source/ocw/overview.rst
+++ b/docs/source/ocw/overview.rst
@@ -68,7 +68,7 @@
>>> import ocw.metrics
>>> # Load 2 datasets
>>> bias = ocw.metrics.Bias()
->>> print bias.run(dataset1, dataset2)
+>>> print(bias.run(dataset1, dataset2))
While this can be useful for one-off situations, it's far more likely that you'll need to run a number of metrics over a number of datasets. This is where running metrics within an evaluation comes in (covered in greater detail below).
@@ -141,8 +141,8 @@
>>>
>>> new_eval = eval.Evaluation(ref_dataset, target_datasets, metrics)
>>> new_eval.run()
->>> print new_eval.results
->>> print new_eval.unary_results
+>>> print(new_eval.results)
+>>> print(new_eval.unary_results)
First, we load the datasets to process and perform any necessary manipulations (which are omitted for brevity). Then, we load the metrics that we want to run (namely, ``Bias`` and ``TemporalStdDev``). We then load our evaluation object::
diff --git a/examples/simple_model_tstd.py b/examples/simple_model_tstd.py
index 75cb5c0..854ee03 100644
--- a/examples/simple_model_tstd.py
+++ b/examples/simple_model_tstd.py
@@ -6,7 +6,9 @@
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
diff --git a/obs4MIPs/Toolbox/CMORresources.py b/obs4MIPs/Toolbox/CMORresources.py
index d20f971..4aab0fc 100644
--- a/obs4MIPs/Toolbox/CMORresources.py
+++ b/obs4MIPs/Toolbox/CMORresources.py
@@ -15,6 +15,8 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import print_function
+
import netCDF4
import re
import pdb
@@ -34,7 +36,7 @@
f=open( inpath + '/' + table, 'r')
if( f == None ):
- print "Table file %s does not exist " % (inpath + "/" + table )
+ print("Table file {} does not exist ".format(inpath + "/" + table))
lines = f.readlines()
diff --git a/obs4MIPs/Toolbox/ESGFexcel.py b/obs4MIPs/Toolbox/ESGFexcel.py
index 92fd4df..e559e2a 100644
--- a/obs4MIPs/Toolbox/ESGFexcel.py
+++ b/obs4MIPs/Toolbox/ESGFexcel.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import print_function
import pdb
import xlrd
@@ -39,8 +40,8 @@
if( os.path.isfile(self.xcl) ):
wb=xlrd.open_workbook(self.xcl)
else:
- print "****** Could not find "+self.xcl+" file ****"
- print "****** Please check excel file name ****"
+ print("****** Could not find {} file ****".format(self.xcl))
+ print("****** Please check excel file name ****")
raise NameError(self.xcl)
sheet=wb.sheet_by_name('Resources')
@@ -55,7 +56,7 @@
for i in arange(sheet.nrows-1) + 1] ] )
pdb.set_trace()
self.ReadXCL()
- print self.resources.keys()
+ print(self.resources.keys())
def ReadXCL(self):
'''
@@ -64,14 +65,14 @@
try:
import xlrd
except:
- print "****** Could not find xlrd Python Package ****"
- print "****** Please install xlrd package to read excel files ****"
+ print("****** Could not find xlrd Python Package ****")
+ print("****** Please install xlrd package to read excel files ****")
if( os.path.isfile(self.xcl) ):
wb=xlrd.open_workbook(self.xcl)
else:
- print "****** Could not find "+self.xcl+" file ****"
- print "****** Please check excel file name ****"
+ print("****** Could not find {} file ****".format(self.xcl))
+ print("****** Please check excel file name ****")
raise NameError(self.xcl)
sheet=wb.sheet_by_name('Variables')
diff --git a/obs4MIPs/Toolbox/ESGFresources.py b/obs4MIPs/Toolbox/ESGFresources.py
index ddbfe60..57378ad 100644
--- a/obs4MIPs/Toolbox/ESGFresources.py
+++ b/obs4MIPs/Toolbox/ESGFresources.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import print_function
import pdb
import shutil
@@ -83,14 +84,14 @@
try:
import xlrd
except:
- print "****** Could not find xlrd Python Package ****"
- print "****** Please install xlrd package to read excel files ****"
+ print("****** Could not find xlrd Python Package ****")
+ print("****** Please install xlrd package to read excel files ****")
if( os.path.isfile(self.xcl) ):
wb=xlrd.open_workbook(self.xcl)
else:
- print "****** Could not find "+self.xcl+" file ****"
- print "****** Please check excel file name ****"
+ print("****** Could not find {} file ****".format(self.xcl))
+ print("****** Please check excel file name ****")
raise NameError(self.xcl)
sheet=wb.sheet_by_name('Variables')
@@ -184,7 +185,7 @@
if files.endswith(".nc"):
filetimestamp = files.split('_')[-1].strip(".nc")
file = os.path.join(r,files)
- print file
+ print(file)
# -----------------
# Delete attributes
# ------------------
@@ -192,7 +193,7 @@
DelGlbAttributes=eval(rc['DelGlbAttributes'].\
replace('\\','\''))
for attribute in DelGlbAttributes:
- print "Deleting attribute: %s" % attribute
+ print("Deleting attribute: {}".format(attribute))
Attr.GlbDel(attribute)
# -----------------
# set attributes
@@ -200,7 +201,7 @@
SetGlbAttributes=eval(rc['SetGlbAttributes'].\
replace('\\','\''))
for (attribute,Value) in SetGlbAttributes:
- print "Assigning attribute (%s,%s)" % (attribute,Value)
+ print("Assigning attribute ({},{})".format(attribute, Value))
Attr.GlbSet(attribute,Value)
Attr.close()
@@ -221,8 +222,8 @@
# -----------
# Move files
# -----------
- print file
- print newfilename
+ print(file)
+ print(newfilename)
os.rename(file,newfilename)
diff --git a/obs4MIPs/factory/formats.py b/obs4MIPs/factory/formats.py
index ced7d74..ed5091c 100644
--- a/obs4MIPs/factory/formats.py
+++ b/obs4MIPs/factory/formats.py
@@ -19,6 +19,8 @@
#
# Select the right handler depending on the file format
+from __future__ import print_function
+
import os
import pdb
import sys
@@ -220,7 +222,7 @@
for file in self.flist:
filename=file.strip()
if( not os.path.exists(filename) ):
- print "File %s does not exist in filelist" % filename
+ print("File {} does not exist in filelist".format(filename))
# --------------------------------------------------------
# Extract General information from first file in the list
@@ -252,7 +254,7 @@
# Concatenate following files
# ---------------------------
for filename in self.flist[ 1: ]:
- print "reading %s" % filename.strip()
+ print("reading {}".format(filename.strip()))
f = cdms2.open( filename.strip(), 'r' )
data2 = f(self.vartoread)[:]
data = numpy.concatenate((data,data2), axis=0)
diff --git a/obs4MIPs/obs4MIPs_process.py b/obs4MIPs/obs4MIPs_process.py
index 3370100..3a03c92 100755
--- a/obs4MIPs/obs4MIPs_process.py
+++ b/obs4MIPs/obs4MIPs_process.py
@@ -17,6 +17,8 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import print_function
+
import cdms2
import cdtime
import cmor
@@ -62,12 +64,12 @@
except:
tmplFile = rc['file_template'].format(year)
if( not os.path.isfile( tmplFile) ) :
- print "**** Warning %s not found\n" % ( tmplFile )
+ print("**** Warning {} not found\n".format(tmplFile))
continue
files= os.popen( "ls " + tmplFile).readlines()
if( files == [] ):
- print "No file found: Check your resource file"
+ print("No file found: Check your resource file")
return -1
# ------------------------------------------------
# Get the right handler to manage this file format
@@ -137,13 +139,13 @@
variable=aVariable[j]
Handler.open(fnm, variable=variable)
rc['cvrt_original_var'] = aVariable[j]
- print "Working on variable %s " % variable
+ print("Working on variable {} ".format(variable))
except:
if( aVariable[j] != 'equation' ) :
- print "Variable %s can't open" % variable
+ print("Variable {} can't open".format(variable))
continue
else:
- print "Executing %s " % eval(rc['equation'])[j]
+ print("Executing {} ".format(eval(rc['equation'])[j]))
# pdb.set_trace()
rc['cvrt_original_units'] = eval(rc['original_units'])[j]
@@ -405,19 +407,19 @@
'''
Describe program synopsis.
'''
- print
- print "*************************"
- print message
- print "*************************"
- print
- print
- print "obs4MIPS_process.py [-h] -r resource"
- print " resource: File containing Global attributes"
- print ""
- print "obs4MIPS will convert an input data file into CMIP5 format using "
- print "CMOR. A directory path will be creating using CMOR by default or "
- print "using a template provided in the resource file."
- print
+ print()
+ print("*************************")
+ print(message)
+ print("*************************")
+ print()
+ print()
+ print("obs4MIPS_process.py [-h] -r resource")
+ print(" resource: File containing Global attributes")
+ print("")
+ print("obs4MIPS will convert an input data file into CMIP5 format using ")
+ print("CMOR. A directory path will be creating using CMOR by default or ")
+ print("using a template provided in the resource file.")
+ print()
# ********************************************************************
#
