contrib/scrap-ec2-sizes.py

#!/usr/bin/env python
#
#  Licensed to the Apache Software Foundation (ASF) under one
#  or more contributor license agreements.  See the NOTICE file
#  distributed with this work for additional information
#  regarding copyright ownership.  The ASF licenses this file
#  to you under the Apache License, Version 2.0 (the
#  "License"); you may not use this file except in compliance
#  with the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing,
#  software distributed under the License is distributed on an
#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
#  KIND, either express or implied.  See the License for the
#  specific language governing permissions and limitations
#  under the License.
"""
This script downloads and parses AWS EC2 from https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonEC2/current/index.json.
It writes a Python module with constants about EC2's sizes and regions.

Use it as following:
    $ python contrib/scrap-ec2-sizes.py > libcloud/compute/constants.py
"""

import re
import os
import json

import requests
import ijson

FILEPATH = os.environ.get('TMP_JSON', '/tmp/ec.json')
URL = "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonEC2/current/index.json"
IGNORED_FIELDS = ['locationType', 'operatingSystem']
REG_STORAGE = re.compile('(\d+) x ([0-9,]+)')
REG_BANDWIDTH = re.compile('\D*(\d+)\D*')
#  From <https://aws.amazon.com/marketplace/help/200777880>
REGION_DETAILS = {
    # America
    'US East (N. Virginia)': {
        'id': 'us-east-1',
        'endpoint': 'ec2.us-east-1.amazonaws.com',
        'api_name': 'ec2_us_east',
        'country': 'USA',
        'signature_version': '2',
    },
    'US East (Ohio)': {
        'id': 'us-east-2',
        'endpoint': 'ec2.us-east-2.amazonaws.com',
        'api_name': 'ec2_us_east_ohio',
        'country': 'USA',
        'signature_version': '4',
    },
    'US West (N. California)': {
        'id': 'us-west-1',
        'endpoint': 'ec2.us-west-1.amazonaws.com',
        'api_name': 'ec2_us_west',
        'country': 'USA',
        'signature_version': '2',
    },
    'US West (Oregon)': {
        'id': 'us-west-2',
        'endpoint': 'ec2.us-west-2.amazonaws.com',
        'api_name': 'ec2_us_west_oregon',
        'country': 'US',
        'signature_version': '2',
    },
    'Canada (Central)': {
        'id': 'ca-central-1',
        'endpoint': 'ec2.ca-central-1.amazonaws.com',
        'api_name': 'ec2_ca_central_1',
        'country': 'Canada',
        'signature_version': '4',
    },
    'South America (Sao Paulo)': {
        'id': 'sa-east-1',
        'endpoint': 'ec2.sa-east-1.amazonaws.com',
        'api_name': 'ec2_sa_east',
        'country': 'Brazil',
        'signature_version': '2',
    },
    'AWS GovCloud (US)': {
        'id': 'us-gov-west-1',
        'endpoint': 'ec2.us-gov-west-1.amazonaws.com',
        'api_name': 'ec2_us_govwest',
        'country': 'US',
        'signature_version': '2',
    },
    # EU
    'eu-west-1': {
        'id': 'eu-west-1',
        'endpoint': 'ec2.eu-west-1.amazonaws.com',
        'api_name': 'ec2_eu_west',
        'country': 'Ireland',
        'signature_version': '2',
    },
    'EU (Ireland)': {  # Duplicate from AWS' JSON
        'id': 'eu-west-1',
        'endpoint': 'ec2.eu-west-1.amazonaws.com',
        'api_name': 'ec2_eu_west',
        'country': 'Ireland',
        'signature_version': '2',
    },
    'EU (London)': {
        'id': 'eu-west-2',
        'endpoint': 'ec2.eu-west-2.amazonaws.com',
        'api_name': 'ec2_eu_west_london',
        'country': 'United Kingdom',
        'signature_version': '4',
    },
    'EU (Paris)': {
        'id': 'eu-west-3',
        'endpoint': 'ec2.eu-west-3.amazonaws.com',
        'api_name': 'ec2_eu_west_paris',
        'country': 'France',
        'signature_version': '4',
    },
    'EU (Frankfurt)': {
        'id': 'eu-central-1',
        'endpoint': 'ec2.eu-central-1.amazonaws.com',
        'api_name': 'ec2_eu_central',
        'country': 'Frankfurt',
        'signature_version': '4',
    },
    # Asia
    'Asia Pacific (Mumbai)': {
        'id': 'ap-south-1',
        'endpoint': 'ec2.ap-south-1.amazonaws.com',
        'api_name': 'ec2_ap_south_1',
        'country': 'India',
        'signature_version': '4',
    },
    'Asia Pacific (Singapore)': {
        'id': 'ap-southeast-1',
        'endpoint': 'ec2.ap-southeast-1.amazonaws.com',
        'api_name': 'ec2_ap_southeast',
        'country': 'Singapore',
        'signature_version': '2',
    },
    'Asia Pacific (Sydney)': {
        'id': 'ap-southeast-2',
        'endpoint': 'ec2.ap-southeast-2.amazonaws.com',
        'api_name': 'ec2_ap_southeast_2',
        'country': 'Australia',
        'signature_version': '2',
    },
    'Asia Pacific (Tokyo)': {
        'id': 'ap-northeast-1',
        'endpoint': 'ec2.ap-northeast-1.amazonaws.com',
        'api_name': 'ec2_ap_northeast',
        'country': 'Japan',
        'signature_version': '2',
    },
    'Asia Pacific (Seoul)': {
        'id': 'ap-northeast-2',
        'endpoint': 'ec2.ap-northeast-2.amazonaws.com',
        'api_name': 'ec2_ap_northeast',
        'country': 'South Korea',
        'signature_version': '4',
    },
    'Asia Pacific (Osaka-Local)': {
        'id': 'ap-northeast-3',
        'endpoint': 'ec2.ap-northeast-3.amazonaws.com',
        'api_name': 'ec2_ap_northeast',
        'country': 'Japan',
        'signature_version': '4',
    },
    # Not in JSON
    'China (Beijing)': {
        'id': 'cn-north-1',
        'endpoint': 'ec2.cn-north-1.amazonaws.com.cn',
        'api_name': 'ec2_cn_north',
        'country': 'China',
        'signature_version': '4',
    },
    'China (Ningxia)': {
        'id': 'cn-northwest-1',
        'endpoint': 'ec2.cn-northwest-1.amazonaws.com.cn',
        'api_name': 'ec2_cn_northwest',
        'country': 'China',
        'signature_version': '4',
    },
}


def download_json():
    response = requests.get(URL, stream=True)
    try:
        return open(FILEPATH, 'r')
    except IOError:
        with open(FILEPATH, 'wb') as fo:
            for chunk in response.iter_content(chunk_size=2**20):
                if chunk:
                    fo.write(chunk)
    return open(FILEPATH, 'r')


def get_json():
    try:
        return open(FILEPATH, 'r')
    except IOError:
        return download_json()


def filter_extras(extras):
    return {
        key: extras[key] for key in extras
        if key not in [
            'capacitystatus', 'ebsOptimized', 'operation', 'licenseModel',
            'preInstalledSw', 'tenancy', 'usagetype'
        ]
    }


def parse():
    # Set vars
    sizes = {}
    regions = {r['id']: r for r in REGION_DETAILS.values()}
    for region_id in regions:
        regions[region_id]['instance_types'] = []
    # Parse
    json_file = get_json()
    products_data = ijson.items(json_file, 'products')
    products_data = next(products_data)
    for sku in products_data:
        if products_data[sku]['productFamily'] != "Compute Instance":
            continue
        location = products_data[sku]['attributes'].pop('location')
        if location not in REGION_DETAILS:
            continue
        # Get region & size ID
        region_id = REGION_DETAILS[location]['id']
        instance_type = products_data[sku]['attributes']['instanceType']
        # Add size to region
        if instance_type not in regions[region_id]['instance_types']:
            regions[region_id]['instance_types'].append(instance_type)
        # Parse sizes
        if instance_type not in sizes:
            for field in IGNORED_FIELDS:
                products_data[sku]['attributes'].pop(field, None)
            # Compute RAM
            ram = int(float(products_data[sku]['attributes']['memory'].split()[0]
                      .replace(',', '')) * 1024)
            # Compute bandwdith
            bw_match = REG_BANDWIDTH.match(products_data[sku]['attributes']['networkPerformance'])
            if bw_match is not None:
                bandwidth = int(bw_match.groups()[0])
            else:
                bandwidth = None
            sizes[instance_type] = {
                'id': instance_type,
                'name': instance_type,
                'ram': ram,
                'bandwidth': bandwidth,
                'extra': filter_extras(products_data[sku]['attributes']),
            }
            if products_data[sku]['attributes'].get('storage') != "EBS only":
                disk_number, disk_size = REG_STORAGE.match(
                    products_data[sku]['attributes']['storage']).groups()
                disk_number, disk_size = int(disk_number), int(disk_size.replace(',', ''))
                sizes[instance_type]['disk'] = disk_number * disk_size
            else:
                sizes[instance_type]['disk'] = 0
            products_data[sku]['attributes']
    # Sort
    for region in regions:
        regions[region]['instance_types'] = sorted(regions[region]['instance_types'])
    return sizes, regions


def dump():
    sizes, regions = parse()
    print("# File generated by script")
    print("INSTANCE_TYPES = " + json.dumps(sizes, indent=4, sort_keys=True).replace('null', 'None'))
    print("REGION_DETAILS = " + json.dumps(regions, indent=4, sort_keys=True))


if __name__ == '__main__':
    dump()