bigtop-packages/src/charm/giraph/layer-giraph/reactive/giraph.py

# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os

from glob import glob
from jujubigdata import utils
from path import Path

from charms.reactive import is_state, when, when_not, set_state
from charms.layer.apache_bigtop_base import Bigtop, get_package_version
from charmhelpers import fetch
from charmhelpers.core import hookenv


def get_good_jars(dir, prefix=True):
    """
    Walk a directory (non-recursively) and return a list of (good) jars.

    Some jars included in giraph have classes that are incompatible with yarn
    nodemanagers. Filter these out when constructing a list of good
    jars. If omitting the entire .jar is too coarse, the jar will need to be
    reconstructed with the offending .class removed.

    param: str dir: Directory to walk
    param: bool prefix: When true, prepend the directory to each jar entry
    """
    # Known incompatible jars:
    # - hive-exec-0.11.0 protobuf class
    #   java.lang.VerifyError: ... overrides final method getUnknownFields
    # - jersey-*-1.17
    #   Conflicts with hadoop-2.7.3 jersey-*-1.9.jar, manifesting as:
    #   com.sun.jersey.api.container: No WebApplication provider is present
    bad_jars = ['hive-exec-0.11.0.jar',
                'jersey-core-1.17.jar',
                'jersey-json-1.17.jar']
    good_jars = []
    for file in os.listdir(dir):
        if file.endswith('.jar') and file not in bad_jars:
            good_jars.append(Path(dir / file) if prefix else file)

    return good_jars


@when('bigtop.available')
def report_status():
    """Set juju status based on the deployment topology."""
    giraph_joined = is_state('giraph.joined')
    giraph_installed = is_state('giraph.installed')
    if not giraph_joined:
        hookenv.status_set('blocked',
                           'waiting for relation to a giraph host')
    elif giraph_installed:
        hookenv.status_set('active',
                           'ready')


@when('bigtop.available', 'giraph.joined')
@when_not('giraph.installed')
def install_giraph(giraph):
    """Install giraph when prerequisite states are present."""
    hookenv.status_set('maintenance', 'installing giraph')
    bigtop = Bigtop()
    bigtop.render_site_yaml(
        roles=[
            'giraph-client',
        ],
    )
    bigtop.trigger_puppet()

    # Put down the -doc subpackage so we get giraph-examples
    fetch.apt_install('giraph-doc')

    giraph_home = Path('/usr/lib/giraph')
    giraph_docdir = Path('/usr/share/doc/giraph')
    giraph_libdir = Path(giraph_home / 'lib')
    giraph_examples = glob('{}/giraph-examples-*.jar'.format(giraph_docdir))

    # Gather a list of all the giraph jars (needed for -libjars)
    giraph_jars = giraph_examples
    giraph_jars.extend(get_good_jars(giraph_home, prefix=True))
    giraph_jars.extend(get_good_jars(giraph_libdir, prefix=True))

    # Update environment with appropriate giraph bits. HADOOP_CLASSPATH can
    # use wildcards (and it should for readability), but GIRAPH_JARS, which
    # is intended to be used as 'hadoop jar -libjars $GIRAPH_JARS', needs to
    # be a comma-separate list of jars.
    with utils.environment_edit_in_place('/etc/environment') as env:
        cur_cp = env['HADOOP_CLASSPATH'] if 'HADOOP_CLASSPATH' in env else ""
        env['GIRAPH_HOME'] = giraph_home
        env['HADOOP_CLASSPATH'] = "{examples}/*:{home}/*:{libs}/*:{cp}".format(
            examples=giraph_docdir,
            home=giraph_home,
            libs=giraph_libdir,
            cp=cur_cp
        )
        env['GIRAPH_JARS'] = ','.join(j for j in giraph_jars)

    set_state('giraph.installed')
    report_status()
    # set app version string for juju status output
    giraph_version = get_package_version('giraph') or 'unknown'
    hookenv.application_version_set(giraph_version)