src/main/vm/src/scripts/ncl/.svn/text-base/wget-download.sh.svn-base

#!/bin/bash
#
# ESG Federation download script
#
# Template version
version=0.4.3
# Generated by Gateway: ESG-NCAR
#
# Script generated user OpenID: https://www.earthsystemgrid.org/myopenid/ahart
#

CACHE_FILE=.md5_results

##############################################################################
#
# Your download selection includes data secured using ESG
# certificate-based security.  In order to access the download URLs
# you must first obtain a credentials file from your home Gateway's
# MyProxy server.
#
# If you don't already have a myproxy client you can download the
# MyProxyLogon Java client from
#   http://www.earthsystemgrid.org//webstart/myProxyLogon/MyProxyLogon-ESG.jar
#
# Then execute it as follows:
#  $ java -jar MyProxyLogon-ESG.jar -h vetswebprod.ucar.edu -p 7512 -u <username>
#
# Further information is available at
#   http://www.earthsystemgrid.org//help/download-help.htm
#
##############################################################################

##############################################################################
#
# Script defaults
#

# ESG_HOME should point to the directory containing ESG credentials.
#   Default is $HOME/.esg.
ESG_HOME=${ESG_HOME:-$HOME/.esg}
ESG_CREDENTIALS=${X509_USER_PROXY:-$ESG_HOME/credentials.pem}
ESG_CERT_DIR=${X509_CERT_DIR:-$ESG_HOME/certificates}
COOKIE_JAR=$ESG_HOME/cookies
CERT_EXPIRATION_WARNING=$((60 * 60 * 1))	#One hour (in seconds)

# Configure checking of server SSL certificates.
#   Disabling server certificate checking can resolve problems with myproxy
#   servers being out of sync with datanodes.
CHECK_SERVER_CERT=${CHECK_SERVER_CERT:-Yes}



usage() {
    echo "Usage: $(basename $0) [flags]"
    echo "Flags is one of:"
    sed -n '/^while getopts/,/^done/  s/^\([^)]*\)[^#]*#\(.*$\)/\1 \2/p' $0
}
#defaults
debug=0
clean_work=1

#parse flags
while getopts ':c:pdvqo:' OPT; do
    case $OPT in
    	c) ESG_CREDENTIALS="$OPTARG";;	#<cert> : use this certificate for authentication.
        p) clean_work=0;;       #	: preserve data that failed checksum
        o) output="$OPTARG";;   #<file>	: Write output for DML in the given file
        d) debug=1;;            #	: display debug information
        v) verbose=1;;          #       : be more verbose
        q) quiet=1;;            #	: be less verbose
        \?) echo "Unknown option '$OPTARG'" >&2 && usage && exit 1;;
        \:) echo "Missing parameter for flag '$OPTARG'" >&2 && usage && exit 1;;
    esac
done
shift $(($OPTIND - 1))

if [[ "$output" ]]; then
    #check and prepare the file
    if [[ -f "$output" ]]; then
        read -p "Overwrite existing file $output? (y/N) " answ
        case $answ in y|Y|yes|Yes);; *) echo "Aborting then..."; exit 0;; esac
    fi
    : > "$output" || { echo "Can't write file $output"; break; }
fi

##############################################################################

# Retrieve ESG credentials (not done yet)
get_credentials() {
    cat <<EOF
Your download selection includes data secured using ESG
certificate-based security.  In order to access the download URLs
you must first obtain a credentials file from your home Gateway's
MyProxy server at vetswebprod.ucar.edu:7512

If you don't already have a myproxy client you can download the
MyProxyLogon Java client from
  http://www.earthsystemgrid.org//webstart/myProxyLogon/MyProxyLogon-ESG.jar

Then execute it as follows:
 $ java -jar MyProxyLogon-ESG.jar -u <username> -h vetswebprod.ucar.edu -p 7512
Further information is available at
  http://www.earthsystemgrid.org//help/download-help.htm

EOF
	exit 1
}

# check the certificate validity
check_cert() {
    #chek openssl and certificate
    if (which openssl &>/dev/null); then
        if ! openssl x509 -checkend 0 -noout -in $ESG_CERT; then
            echo "The Certificate has expired, please renew."
            return 1
        else
            if ! openssl x509 -checkend $CERT_EXPIRATION_WARNING -noout -in $ESG_CERT; then
                echo "The certificate expires in less than $((CERT_EXPIRATION_WARNING / 60 / 60)) hour(s), please renew."
                return 2
            fi
        fi
    fi
}

#
# Detect ESG credentials
#
find_credentials() {

    if [[ -f "$ESG_CREDENTIALS" ]]; then
    	# file found, proceed.
    	ESG_CERT="$ESG_CREDENTIALS"
    	ESG_KEY="$ESG_CREDENTIALS"
    elif [[ -f "$X509_USER_CERT" && -f "$X509_USER_KEY" ]]; then
    	# second try, use these certificates.
        ESG_CERT="$X509_USER_CERT"
        ESG_KEY="$X509_USER_KEY"
    else
	    # If credentials are not present exit
    	echo "No ESG Credentials found in $ESG_CREDENTIALS" >&2
    	    get_credentials
	fi


    #chek openssl and certificate
    if (which openssl &>/dev/null); then
    	if ( openssl version | grep 'OpenSSL 1\.0' ); then
        	echo '** WARNING: ESGF Host certificate checking might not be compatible with OpenSSL 1.0+'
    	fi
        check_cert || { (($?==1)); exit 1; }
    fi
    
    if [[ $CHECK_SERVER_CERT == "Yes" ]]; then
    	[[ -d "$ESG_CERT_DIR" ]] || { echo "CA certs not found. Aborting."; exit 1; }
    	PKI_WGET_OPTS="--ca-directory=$ESG_CERT_DIR"
    fi

    #some wget version complain if there's no file present
    [[ -f $COOKIE_JAR ]] || touch $COOKIE_JAR

    PKI_WGET_OPTS="$PKI_WGET_OPTS --certificate=$ESG_CERT --private-key=$ESG_KEY --save-cookies=$COOKIE_JAR --load-cookies=$COOKIE_JAR"

}

check_chksum() {
    local file="$1"
    local chk_type=$2
    local chk_value=$3
    local local_chksum

    case $chk_type in
        md5) local_chksum=$(md5sum $file | cut -f1 -d" ");;
        *) echo "Can't verify checksum." && return 0;;
    esac

    #verify
    ((debug)) && echo "local:$local_chksum vs remote:$chk_value"
    diff -q <(echo $local_chksum) <(echo $chk_value) >/dev/null
}

download() {
    wget="wget -c $PKI_WGET_OPTS"
    ((quiet)) && wget="$wget -q" || { ((!verbose)) && wget="$wget -nv"; }
    
    while read line
    do
        # read csv here document into proper variables
        eval $(awk -F "' '" '{$0=substr($0,2,length($0)-2); $3=tolower($3); print "file=\""$1"\";url=\""$2"\";chksum_type=\""$3"\";chksum=\""$4"\""}' <(echo $line) )

        #Process the file
        echo -n "$file ..."

        #are we just writing a file?
        if [ "$output" ]; then
            echo "$file - $url" >> $output
            echo ""
            continue
        fi

        while : ; do
                #if we have the file, check if it's already processed.
                [ -f $file ] && cached="$(grep $file $CACHE_FILE)" || unset cached

                #check it wasn't modified
                if [[ -n "$cached" && "$(stat -c %Y $file)" == $(echo "$cached" | cut -d ' ' -f2) ]]; then
                    echo "Already downloaded and verified"
                    break
                fi

                # (if we had the file size, we could check before trying to complete)
                echo "Downloading"
                $wget -O "$file" $url || { failed=1; break; }

                #check if file is there
                if [[ -f $file ]]; then
                        ((debug)) && echo file found
                        if ! check_chksum "$file" $chksum_type $chksum; then
                                echo "  $chksum_type failed!"
                                if ((clean_work)); then
                                        rm $file
                                        #try again
                                        echo -n "  re-downloading..."
                                        continue
                                else
                                        echo "  don't use -p or remove manually."
                                fi
                        else
                                echo "  $chksum_type ok. done!"
                                echo $file $(stat -c %Y $file) $chksum >> $CACHE_FILE
                        fi
                fi
                #done!
                break
        done
        
        if ((failed)); then
            echo "download failed"
            # most common failure is certificate expiration, so check this
            check_cert
            unset failed
        fi
        
    done <<EOF--dataset.file.url.chksum_type.chksum
'ncl_ncarg-6.1.0.Linux_Debian_x86_64_gcc432.tar.gz' 'http://www.earthsystemgrid.org//download/fileTokenDownload.htm?fileAccessPointId=425ef002-12f9-4087-8df0-dfd60304469a&authzToken=3648d4cf-d81c-474d-9779-bd0d6f91b832&gateway=ESG-NCAR' '' ''
'ncl_ncarg-6.1.0.Linux_Debian_x86_64_gcc445.tar.gz' 'http://www.earthsystemgrid.org//download/fileTokenDownload.htm?fileAccessPointId=afb26ae5-6a93-4801-b0fb-7bad630e922f&authzToken=c866c5c8-ea30-4682-b1ce-1040e2fa377c&gateway=ESG-NCAR' '' ''
EOF--dataset.file.url.chksum_type.chksum

}


#
# MAIN
#
echo "Running $(basename $0) version: $version"

find_credentials
#do we have old results? Create the file if not
[ ! -f $CACHE_FILE ] && echo "#filename mtime checksum" > $CACHE_FILE

download

#remove duplicates (if any)
{ rm $CACHE_FILE && tac | awk '!x[$1]++' | tac > $CACHE_FILE; } < $CACHE_FILE