| #!/usr/bin/python |
| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, |
| # software distributed under the License is distributed on an |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| # KIND, either express or implied. See the License for the |
| # specific language governing permissions and limitations |
| # under the License. |
| |
| # Version @VERSION@ |
| # |
| # A plugin for executing script needed by cloud stack |
| from __future__ import with_statement |
| |
| from copy import copy |
| from datetime import datetime |
| from httplib import * |
| from string import join |
| |
| import os |
| import sys |
| import time |
| import hashlib |
| import base64 |
| import hmac |
| import traceback |
| import urllib2 |
| |
| import XenAPIPlugin |
| sys.path.extend(["/opt/xensource/sm/"]) |
| import util |
| |
| NULL = 'null' |
| |
| # Value conversion utility functions ... |
| |
| |
| def to_none(value): |
| return value if value is not None and value.strip() != NULL else None |
| |
| |
| def to_bool(value): |
| return True if to_none(value) in ['true', 'True', None] else False |
| |
| |
| def to_integer(value, default): |
| return int(value) if to_none(value) is not None else default |
| |
| |
| def optional_str_value(value, default): |
| return value if is_not_blank(value) else default |
| |
| |
| def is_not_blank(value): |
| return True if to_none(value) is not None and value.strip != '' else False |
| |
| |
| def get_optional_key(map, key, default=''): |
| return map[key] if key in map else default |
| |
| |
| def log(message): |
| util.SMlog('#### VMOPS %s ####' % message) |
| |
| |
| def echo(fn): |
| def wrapped(*v, **k): |
| name = fn.__name__ |
| log("enter %s ####" % name) |
| res = fn(*v, **k) |
| log("exit %s with result %s" % name, res) |
| return res |
| return wrapped |
| |
| |
| def require_str_value(value, error_message): |
| |
| if is_not_blank(value): |
| return value |
| |
| raise ValueError(error_message) |
| |
| |
| def retry(max_attempts, fn): |
| |
| attempts = 1 |
| while attempts <= max_attempts: |
| log("Attempting execution {0}/{1} of {2}". |
| format(attempts, max_attempts, fn.__name__)) |
| try: |
| return fn() |
| except: |
| if (attempts >= max_attempts): |
| raise |
| attempts = attempts + 1 |
| |
| |
| def compute_md5(filename, buffer_size=8192): |
| |
| hasher = hashlib.md5() |
| |
| with open(filename, 'rb') as file: |
| data = file.read(buffer_size) |
| while data != "": |
| hasher.update(data) |
| data = file.read(buffer_size) |
| |
| return base64.encodestring(hasher.digest())[:-1] |
| |
| |
| class S3Client(object): |
| |
| DEFAULT_END_POINT = 's3.amazonaws.com' |
| DEFAULT_CONNECTION_TIMEOUT = 50000 |
| DEFAULT_SOCKET_TIMEOUT = 50000 |
| DEFAULT_MAX_ERROR_RETRY = 3 |
| |
| HEADER_CONTENT_MD5 = 'Content-MD5' |
| HEADER_CONTENT_TYPE = 'Content-Type' |
| HEADER_CONTENT_LENGTH = 'Content-Length' |
| |
| def __init__(self, access_key, secret_key, end_point=None, |
| https_flag=None, connection_timeout=None, socket_timeout=None, |
| max_error_retry=None): |
| |
| self.access_key = require_str_value( |
| access_key, 'An access key must be specified.') |
| self.secret_key = require_str_value( |
| secret_key, 'A secret key must be specified.') |
| self.end_point = optional_str_value(end_point, self.DEFAULT_END_POINT) |
| self.https_flag = to_bool(https_flag) |
| self.connection_timeout = to_integer( |
| connection_timeout, self.DEFAULT_CONNECTION_TIMEOUT) |
| self.socket_timeout = to_integer( |
| socket_timeout, self.DEFAULT_SOCKET_TIMEOUT) |
| self.max_error_retry = to_integer( |
| max_error_retry, self.DEFAULT_MAX_ERROR_RETRY) |
| |
| def build_canocialized_resource(self, bucket, key): |
| |
| return '/{bucket}/{key}'.format(bucket=bucket, key=key) |
| |
| def noop_send_body(): |
| pass |
| |
| def noop_read(response): |
| return response.read() |
| |
| def do_operation( |
| self, method, bucket, key, input_headers={}, |
| fn_send_body=noop_send_body, fn_read=noop_read): |
| |
| headers = copy(input_headers) |
| headers['Expect'] = '100-continue' |
| |
| uri = self.build_canocialized_resource(bucket, key) |
| signature, request_date = self.sign_request(method, uri, headers) |
| headers['Authorization'] = "AWS {0}:{1}".format( |
| self.access_key, signature) |
| headers['Date'] = request_date |
| |
| connection = HTTPSConnection(self.end_point) \ |
| if self.https_flag else HTTPConnection(self.end_point) |
| connection.timeout = self.socket_timeout |
| |
| def perform_request(): |
| |
| connection.request(method, uri, fn_send_body(), headers) |
| response = connection.getresponse() |
| log("Sent {0} request to {1} {2} with headers {3}. \ |
| Got response status {4}: {5}". |
| format(method, self.end_point, uri, headers, |
| response.status, response.reason)) |
| return fn_read(response) |
| |
| try: |
| return retry(self.max_error_retry, perform_request) |
| finally: |
| connection.close() |
| |
| ''' |
| See http://bit.ly/MMC5de for more information regarding the creation of |
| AWS authorization tokens and header signing |
| ''' |
| def sign_request(self, operation, canocialized_resource, headers): |
| |
| request_date = datetime.utcnow( |
| ).strftime('%a, %d %b %Y %H:%M:%S +0000') |
| |
| content_hash = get_optional_key(headers, self.HEADER_CONTENT_MD5) |
| content_type = get_optional_key(headers, self.HEADER_CONTENT_TYPE) |
| |
| string_to_sign = join( |
| [operation, content_hash, content_type, request_date, |
| canocialized_resource], '\n') |
| |
| signature = base64.encodestring( |
| hmac.new(self.secret_key, string_to_sign.encode('utf8'), |
| hashlib.sha1).digest())[:-1] |
| |
| return signature, request_date |
| |
| def put(self, bucket, key, src_filename): |
| |
| headers = { |
| self.HEADER_CONTENT_MD5: compute_md5(src_filename), |
| self.HEADER_CONTENT_TYPE: 'application/octet-stream', |
| self.HEADER_CONTENT_LENGTH: os.stat(src_filename).st_size, |
| } |
| |
| def send_body(): |
| return open(src_filename, 'rb') |
| |
| self.do_operation('PUT', bucket, key, headers, send_body) |
| |
| def get(self, bucket, key, target_filename): |
| |
| def read(response): |
| |
| with open(target_filename, 'wb') as file: |
| while True: |
| block = response.read(8192) |
| if not block: |
| break |
| file.write(block) |
| |
| return self.do_operation('GET', bucket, key, fn_read=read) |
| |
| def delete(self, bucket, key): |
| |
| return self.do_operation('DELETE', bucket, key) |
| |
| |
| def parseArguments(args): |
| |
| # The keys in the args map will correspond to the properties defined on |
| # the com.cloud.utils.S3Utils#ClientOptions interface |
| client = S3Client( |
| args['accessKey'], args['secretKey'], args['endPoint'], |
| args['isHttps'], args['connectionTimeout'], args['socketTimeout']) |
| |
| operation = args['operation'] |
| bucket = args['bucket'] |
| key = args['key'] |
| filename = args['filename'] |
| |
| if is_blank(operation): |
| raise ValueError('An operation must be specified.') |
| |
| if is_blank(bucket): |
| raise ValueError('A bucket must be specified.') |
| |
| if is_blank(key): |
| raise ValueError('A value must be specified.') |
| |
| if is_blank(filename): |
| raise ValueError('A filename must be specified.') |
| |
| return client, operation, bucket, key, filename |
| |
| |
| @echo |
| def s3(session, args): |
| |
| client, operation, bucket, key, filename = parseArguments(args) |
| |
| try: |
| |
| if operation == 'put': |
| client.put(bucket, key, filename) |
| elif operation == 'get': |
| client.get(bucket, key, filename) |
| elif operation == 'delete': |
| client.delete(bucket, key, filename) |
| else: |
| raise RuntimeError( |
| "S3 plugin does not support operation {0}.".format(operation)) |
| |
| return 'true' |
| |
| except: |
| log("Operation {0} on file {1} from/in bucket {2} key {3}.".format( |
| operation, filename, bucket, key)) |
| log(traceback.format_exc()) |
| return 'false' |
| |
| if __name__ == "__main__": |
| XenAPIPlugin.dispatch({"s3": s3}) |
