obs4MIPs/Toolbox/ESGFresources.py

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.


import pdb
import shutil
import os,sys
from numpy import arange
from Toolbox.CMORresources import CMORAttributes

#  ********************************************************************
#         ESGFresources()
#
#  ********************************************************************
class ESGFresources:
    '''
    Create dictionary based on a file using key=value standard      
    '''
    def __init__(self,rcFile):
        '''
        Read CMIP5 Table and convert into a dictionary
        strip quote double quote, leading and trailing white space
        from key and value.
        '''
        f=open( rcFile, 'r')
        lines=f.readlines()
        f.close()

        self.resources=dict([(key.strip(), value.strip(),) 
                             for line in lines if line != '\n' \
                                 and line[0] != "#" 
                             for (key, value) in
                             [line.strip().\
                                  replace("'","").\
                                  replace("\"","").\
                                  rstrip().\
                                  split("=")]])

        for key in self.resources.keys():
            self.resources[key]=self.resources[key].replace("\\","\"")
            
        self.xcl=None
        try:
            self.xcl=self.resources['excel_file']
            rc=self.ReadXCL()
        except:
            pass

    def __getitem__(self,key):
        '''
        Retreive item from resource dictionary
        '''
        return self.resources[key]

    def __setitem__(self,key,value):
        '''
        '''
        self.resources[key]=value

    def __delete__(self,key):
        '''
        '''
        del self.resources[key]

    def ReadXCL(self):
        '''
        Read Excel Table and fill rc variable related field.
        '''
        try:
           import xlrd
        except:
           print "****** Could not find xlrd Python Package ****"
           print "****** Please install xlrd package to read excel files ****"

        if( os.path.isfile(self.xcl) ):
		wb=xlrd.open_workbook(self.xcl)
        else:
           print "****** Could not find "+self.xcl+" file ****"
           print "****** Please check excel file name ****"
           raise NameError(self.xcl)

        sheet=wb.sheet_by_name('Variables')

        self.resources['cmor_var']    = [ sheet.row( i )[ 0 ].value.\
                                          encode('ascii','ignore') \
                                          for i in arange(sheet.nrows-2) + 2 ]

        self.resources['original_var'] = [ sheet.row( i )[ 1 ].value.\
                                           encode('ascii','ignore') \
                                           for i in arange(sheet.nrows-2) + 2 ]

        self.resources['original_units']=[ sheet.row( i )[ 2 ].value.\
                                           encode('ascii','ignore') \
                                           for i in arange(sheet.nrows-2) + 2 ]

        self.resources['level']         =[ sheet.row( i )[ 3 ].value.\
                                           encode('ascii','ignore') \
                                           for i in arange(sheet.nrows-2) + 2]
                                    
        self.resources['equation']     =[ sheet.row( i )[ 6 ].value.\
                                           encode('ascii','ignore') \
                                           for i in arange(sheet.nrows-2) + 2 ]

        # -----------------------------------------------------------------
        # Make sure it is a string. The main program will call eval on it.
        # -----------------------------------------------------------------
        self.resources['cmor_var']       = str(self.resources['cmor_var'] )
        self.resources['original_var']   = str(self.resources['original_var'])
        self.resources['original_units'] =str(self.resources['original_units'])
        self.resources['equation']       =str(self.resources['equation'])
        self.resources['level']          =str(self.resources['level'])
        return 1


#  ********************************************************************
#      movefiles()                                                     
#
#  rename files in new obs4MIPS path
#  ********************************************************************
def movefiles(rc):  
    '''
    Change CMIP5 directory struture to obs4MIPS directory structure.
    Create new path and rename files following with new convention 
    Remove old directory tree.
    '''
    # ----------------------
    # New path for obs4MIPS
    # ----------------------
    path = rc['institute_id'      ]    + '/' + \
           rc['instrument'        ]    + '/' + \
           rc['project_id'        ]    + '/' + \
           rc['product'           ]    + '/' + \
           rc['modeling_realm'    ]    + '/' + \
           rc['cvrt_cmor_var'     ]    + '/' + \
           rc['frequency'         ]    + '/' + \
           rc['data_structure'    ]    + '/' + \
           rc['institute_id'      ]    + '/' + \
           rc['instrument'        ]    
 
   # -------------------------------------------------------------
   # For TRMM data set version_directory is set to 'false' so that
   # no version will be created in the directory
   # -------------------------------------------------------------
    try:
           rc['version_directory'] == 'false'
           pass
    except:
           path +=  '/V' + rc['processing_version'] + '/'

   # -----------------------------
   # Make sure the directory exist 
   # ------------------------------
    try:
        os.makedirs(path)
    except:
        None

    # -----------------
    # Rename all files
    # -----------------
    cmorpath = rc['project_id'  ]       + '/' + \
               rc['product'     ]       + '/' + \
               rc['institute_id']       + '/' 

    # -----------------
    # Manage attributes
    # -----------------
    for r,d,f in os.walk(cmorpath):
        for files in f:
            if files.endswith(".nc"):
                filetimestamp = files.split('_')[-1].strip(".nc")
                file = os.path.join(r,files)
                print file
                # -----------------
                # Delete attributes
                # ------------------
                Attr=CMORAttributes( file )
                DelGlbAttributes=eval(rc['DelGlbAttributes'].\
                                      replace('\\','\''))
                for attribute in DelGlbAttributes:
                    print "Deleting attribute: %s" % attribute
                    Attr.GlbDel(attribute)
                # -----------------
                # set attributes
                # ------------------
                SetGlbAttributes=eval(rc['SetGlbAttributes'].\
                                      replace('\\','\''))
                for (attribute,Value) in SetGlbAttributes:
                    print "Assigning attribute (%s,%s)" % (attribute,Value)
                    Attr.GlbSet(attribute,Value)
                Attr.close()

                source = ''
                if( rc['source_fn'] == 'SYNOPTIC' ):
                    source = rc['SYNOPTIC']+"z_"
                elif( rc['source_fn'] != '' ):
                    source = rc['source_fn'].split('_')[0] + '_'
                
                newfilename = rc['cvrt_cmor_var'     ] + '_' + \
                              rc['instrument'        ] + '-' + \
                              source                         + \
                              rc['processing_level'  ] + '_v'+ \
                              rc['processing_version'] + '_' + \
                              filetimestamp            + '.nc'
                
                newfilename = os.path.join(path,newfilename)
                # -----------
                # Move files
                # -----------
                print file
                print newfilename
                os.rename(file,newfilename)

                    
    # ----------------
    # Remove cmor path
    # ----------------
    shutil.rmtree( rc['project_id'] )
    return 0