ambari-server/src/main/java/org/apache/ambari/server/stack/ExtensionModule.java

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.ambari.server.stack;

import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.state.ConfigHelper;
import org.apache.ambari.server.state.ExtensionInfo;
import org.apache.ambari.server.state.PropertyDependencyInfo;
import org.apache.ambari.server.state.PropertyInfo;
import org.apache.ambari.server.state.RepositoryInfo;
import org.apache.ambari.server.state.ServiceInfo;
import org.apache.ambari.server.state.stack.ExtensionMetainfoXml;
import org.apache.ambari.server.state.stack.RepositoryXml;
import org.apache.ambari.server.state.stack.ServiceMetainfoXml;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * Extension module which provides all functionality related to parsing and fully
 * resolving extensions from the extension definition.
 *
 * An extension version is like a stack version but it contains custom services.  Linking an extension
 * version to the current stack version allows the cluster to install the custom services contained in
 * the extension version.
 *
 * <p>
 * Each extension node is identified by name and version, contains service
 * child nodes and may extend a single parent extension.
 * </p>
 *
 * <p>
 * Resolution of a extension is a depth first traversal up the inheritance chain where each extension node
 * calls resolve on its parent before resolving itself.  After the parent resolve call returns, all
 * ancestors in the inheritance tree are fully resolved.  The act of resolving the extension includes
 * resolution of the services children of the extension as well as merging of other extension
 * state with the fully resolved parent.
 * </p>
 *
 * <p>
 * Because a service may explicitly extend another service in a extension outside of the inheritance tree,
 * service child node resolution involves a depth first resolution of the extension associated with the
 * services explicit parent, if any.  This follows the same steps defined above fore extension node
 * resolution.  After the services explicit parent is fully resolved, the services state is merged
 * with it's parent.
 * </p>
 *
 * <p>
 * If a cycle in a extension definition is detected, an exception is thrown from the resolve call.
 * </p>
 *
 */
public class ExtensionModule extends BaseModule<ExtensionModule, ExtensionInfo> implements Validable {

  /**
   * Context which provides access to external functionality
   */
  private StackContext stackContext;

  /**
   * Map of child configuration modules keyed by configuration type
   */
  private Map<String, ConfigurationModule> configurationModules = new HashMap<String, ConfigurationModule>();

  /**
   * Map of child service modules keyed by service name
   */
  private Map<String, ServiceModule> serviceModules = new HashMap<String, ServiceModule>();

  /**
   * Corresponding ExtensionInfo instance
   */
  private ExtensionInfo extensionInfo;

  /**
   * Encapsulates IO operations on extension directory
   */
  private ExtensionDirectory extensionDirectory;

  /**
   * Extension id which is in the form extensionName:extensionVersion
   */
  private String id;

  /**
   * validity flag
   */
  protected boolean valid = true;

  /**
   * Logger
   */
  private final static Logger LOG = LoggerFactory.getLogger(ExtensionModule.class);

  /**
   * Constructor.
   * @param extensionDirectory  represents extension directory
   * @param extensionContext    general extension context
   */
  public ExtensionModule(ExtensionDirectory extensionDirectory, StackContext stackContext) {
    this.extensionDirectory = extensionDirectory;
    this.stackContext = stackContext;
    this.extensionInfo = new ExtensionInfo();
    populateExtensionInfo();
  }

  public Map<String, ServiceModule> getServiceModules() {
	  return serviceModules;
  }

  /**
   * Fully resolve the extension. See extension resolution description in the class documentation.
   * If the extension has a parent, this extension will be merged against its fully resolved parent
   * if one is specified. Merging applies to all extension state including child service and
   * configuration modules.  Services may extend a service in another version in the
   * same extension hierarchy or may explicitly extend a service in a different
   * hierarchy.
   *
   * @param parentModule   not used.  Each extension determines its own parent since extensions don't
   *                       have containing modules
   * @param allStacks      all stacks modules contained in the stack definition
   * @param commonServices all common services
   * @param extensions     all extensions
   *
   * @throws AmbariException if an exception occurs during extension resolution
   */
  @Override
  public void resolve(
      ExtensionModule parentModule, Map<String, StackModule> allStacks, Map<String, ServiceModule> commonServices, Map<String, ExtensionModule> extensions)
      throws AmbariException {
    moduleState = ModuleState.VISITED;
    checkExtensionName(allStacks);

    String parentVersion = extensionInfo.getParentExtensionVersion();
    mergeServicesWithExplicitParent(allStacks, commonServices, extensions);
    // merge with parent version of same extension definition
    if (parentVersion != null) {
      mergeExtensionWithParent(parentVersion, allStacks, commonServices, extensions);
    }
    moduleState = ModuleState.RESOLVED;
  }

  @Override
  public ExtensionInfo getModuleInfo() {
    return extensionInfo;
  }

  @Override
  public boolean isDeleted() {
    return false;
  }

  @Override
  public String getId() {
    return id;
  }

  @Override
  public void finalizeModule() {
    finalizeChildModules(serviceModules.values());
    finalizeChildModules(configurationModules.values());
  }

  /**
   * Get the associated extension directory.
   *
   * @return associated extension directory
   */
  public ExtensionDirectory getExtensionDirectory() {
    return extensionDirectory;
  }

  /**
   * Merge the extension with its parent.
   *
   * @param allStacks      all stacks in stack definition
   * @param commonServices all common services specified in the stack definition
   * @param parentVersion  version of the extensions parent
   *
   * @throws AmbariException if an exception occurs merging with the parent
   */
  private void mergeExtensionWithParent(
      String parentVersion, Map<String, StackModule> allStacks, Map<String, ServiceModule> commonServices, Map<String, ExtensionModule> extensions)
      throws AmbariException {

    String parentExtensionKey = extensionInfo.getName() + StackManager.PATH_DELIMITER + parentVersion;
    ExtensionModule parentExtension = extensions.get(parentExtensionKey);

    if (parentExtension == null) {
      throw new AmbariException("Extension '" + extensionInfo.getName() + ":" + extensionInfo.getVersion() +
          "' specifies a parent that doesn't exist");
    }

    resolveExtension(parentExtension, allStacks, commonServices, extensions);
    /*mergeConfigurations(parentStack, allStacks, commonServices);
    mergeRoleCommandOrder(parentStack);*/

    /*if (extensionInfo.getStackHooksFolder() == null) {
      extensionInfo.setStackHooksFolder(parentStack.getModuleInfo().getStackHooksFolder());
    }

    if (extensionInfo.getKerberosDescriptorFileLocation() == null) {
      extensionInfo.setKerberosDescriptorFileLocation(parentStack.getModuleInfo().getKerberosDescriptorFileLocation());
    }

    if (extensionInfo.getWidgetsDescriptorFileLocation() == null) {
      extensionInfo.setWidgetsDescriptorFileLocation(parentStack.getModuleInfo().getWidgetsDescriptorFileLocation());
    }*/

    mergeServicesWithParent(parentExtension, allStacks, commonServices, extensions);
  }

  /**
   * Merge child services with parent extension.
   *
   * @param parentExtension    parent extension module
   * @param allStacks          all stacks in stack definition
   * @param commonServices     all common services
   * @param extensions         all extensions
   *
   * @throws AmbariException if an exception occurs merging the child services with the parent extension
   */
  private void mergeServicesWithParent(
      ExtensionModule parentExtension, Map<String, StackModule> allStacks, Map<String, ServiceModule> commonServices, Map<String, ExtensionModule> extensions)
      throws AmbariException {
    extensionInfo.getServices().clear();

    LOG.info("***Merging extension services with parent: " + parentExtension.getId());

    Collection<ServiceModule> mergedModules = mergeChildModules(
        allStacks, commonServices, extensions, serviceModules, parentExtension.serviceModules);
    for (ServiceModule module : mergedModules) {
      serviceModules.put(module.getId(), module);
      extensionInfo.getServices().add(module.getModuleInfo());
    }
  }

  /**
   * Merge services with their explicitly specified parent if one has been specified.
   * @param allStacks      all stacks in stack definition
   * @param commonServices all common services specified in the stack definition
   *
   * @throws AmbariException if an exception occurs while merging child services with their explicit parents
   */
  private void mergeServicesWithExplicitParent(
        Map<String, StackModule> allStacks, Map<String, ServiceModule> commonServices, Map<String, ExtensionModule> extensions) throws AmbariException {
    for (ServiceModule service : serviceModules.values()) {
      ServiceInfo serviceInfo = service.getModuleInfo();
      String parent = serviceInfo.getParent();
      if (parent != null) {
        mergeServiceWithExplicitParent(service, parent, allStacks, commonServices, extensions);
      }
    }
  }

  /**
   * Merge a service with its explicitly specified parent.
   * @param service          the service to merge
   * @param parent           the explicitly specified parent service
   * @param allStacks        all stacks specified in the stack definition
   * @param commonServices   all common services specified in the stack definition
   *
   * @throws AmbariException if an exception occurs merging a service with its explicit parent
   */
  private void mergeServiceWithExplicitParent(
      ServiceModule service, String parent, Map<String, StackModule> allStacks,
      Map<String, ServiceModule> commonServices, Map<String, ExtensionModule> extensions)
      throws AmbariException {
    if(isCommonServiceParent(parent)) {
      LOG.info("merging with common service: " + service.getModuleInfo().getName());
      mergeServiceWithCommonServiceParent(service, parent, allStacks, commonServices, extensions);
      LOG.info("display name: " + service.getModuleInfo().getDisplayName());
    } else {
      throw new AmbariException("The service '" + service.getModuleInfo().getName() + "' in extension '" + extensionInfo.getName() + ":"
          + extensionInfo.getVersion() + "' extends an invalid parent: '" + parent + "'");
    }
  }

  /**
   * @param allStacks        all stacks specified in the stack definition
   *
   * @throws AmbariException if the extension name is the same as any of the stacks
   */
  private void checkExtensionName(Map<String, StackModule> allStacks)
      throws AmbariException {

    String name = extensionInfo.getName();
    for (StackModule stack : allStacks.values()) {
      String stackName = stack.getModuleInfo().getName();
      if (name.equals(stackName)) {
        throw new AmbariException("The extension '" + name + "' has a name which matches a stack name");
      }
    }
  }

  /**
   * Check if parent is common service
   * @param parent  Parent string
   * @return true: if parent is common service, false otherwise
   */
  private boolean isCommonServiceParent(String parent) {
    return parent != null
        && !parent.isEmpty()
        && parent.split(StackManager.PATH_DELIMITER)[0].equalsIgnoreCase(StackManager.COMMON_SERVICES);
  }

  /**
   * Merge a service with its explicitly specified common service as parent.
   * Parent: common-services/<serviceName>/<serviceVersion>
   * Common Services Lookup Key: <serviceName>/<serviceVersion>
   * Example:
   *  Parent: common-services/HDFS/2.1.0.2.0
   *  Key: HDFS/2.1.0.2.0
   *
   * @param service          the service to merge
   * @param parent           the explicitly specified common service as parent
   * @param allStacks        all stacks specified in the stack definition
   * @param commonServices   all common services specified in the stack definition
   * @throws AmbariException
   */
  private void mergeServiceWithCommonServiceParent(
      ServiceModule service, String parent, Map<String, StackModule> allStacks,
      Map<String, ServiceModule> commonServices, Map<String, ExtensionModule> extensions)
      throws AmbariException {
    ServiceInfo serviceInfo = service.getModuleInfo();
    String[] parentToks = parent.split(StackManager.PATH_DELIMITER);
    if(parentToks.length != 3 || !parentToks[0].equalsIgnoreCase(StackManager.COMMON_SERVICES)) {
      throw new AmbariException("The service '" + serviceInfo.getName() + "' in extension '" + extensionInfo.getName() + ":"
          + extensionInfo.getVersion() + "' extends an invalid parent: '" + parent + "'");
    }

    String baseServiceKey = parentToks[1] + StackManager.PATH_DELIMITER + parentToks[2];
    ServiceModule baseService = commonServices.get(baseServiceKey);
    if (baseService == null) {
      setValid(false);
      extensionInfo.setValid(false);
      String error = "The service '" + serviceInfo.getName() + "' in extension '" + extensionInfo.getName() + ":"
          + extensionInfo.getVersion() + "' extends a non-existent service: '" + parent + "'";
      addError(error);
      extensionInfo.addError(error);
    } else {
      if (baseService.isValid()) {
        service.resolveExplicit(baseService, allStacks, commonServices, extensions);
      } else {
        setValid(false);
        extensionInfo.setValid(false);
        addErrors(baseService.getErrors());
        extensionInfo.addErrors(baseService.getErrors());
      }
    }
  }

  /**
   * Populate the extension module and info from the extension definition.
   */
  private void populateExtensionInfo() {
    extensionInfo.setName(extensionDirectory.getExtensionDirName());
    extensionInfo.setVersion(extensionDirectory.getName());

    id = String.format("%s:%s", extensionInfo.getName(), extensionInfo.getVersion());

    LOG.debug("Adding new extension to known extensions"
        + ", extensionName = " + extensionInfo.getName()
        + ", extensionVersion = " + extensionInfo.getVersion());


    //todo: give additional thought on handling missing metainfo.xml
    ExtensionMetainfoXml emx = extensionDirectory.getMetaInfoFile();
    if (emx != null) {
      if (!emx.isValid()) {
        extensionInfo.setValid(false);
        extensionInfo.addErrors(emx.getErrors());
      }
      extensionInfo.setParentExtensionVersion(emx.getExtends());
      extensionInfo.setStacks(emx.getStacks());
      extensionInfo.setExtensions(emx.getExtensions());
    }

    try {
      // Read the service for this extension
      populateServices();
      if (!extensionInfo.isValid()) {
        setValid(false);
        addErrors(extensionInfo.getErrors());
      }

      //todo: shouldn't blindly catch Exception, re-evaluate this.
    } catch (Exception e) {
      String error = "Exception caught while populating services for extension: " +
          extensionInfo.getName() + "-" + extensionInfo.getVersion();
      setValid(false);
      extensionInfo.setValid(false);
      addError(error);
      extensionInfo.addError(error);
      LOG.error(error);
    }
  }

  /**
   * Populate the child services.
   */
  private void populateServices()throws AmbariException {
    for (ServiceDirectory serviceDir : extensionDirectory.getServiceDirectories()) {
      populateService(serviceDir);
    }
  }

  /**
   * Populate a child service.
   *
   * @param serviceDirectory the child service directory
   */
  private void populateService(ServiceDirectory serviceDirectory)  {
    Collection<ServiceModule> serviceModules = new ArrayList<ServiceModule>();
    // unfortunately, we allow multiple services to be specified in the same metainfo.xml,
    // so we can't move the unmarshal logic into ServiceModule
    ServiceMetainfoXml metaInfoXml = serviceDirectory.getMetaInfoFile();
    if (!metaInfoXml.isValid()){
      extensionInfo.setValid(metaInfoXml.isValid());
      setValid(metaInfoXml.isValid());
      extensionInfo.addErrors(metaInfoXml.getErrors());
      addErrors(metaInfoXml.getErrors());
      return;
    }
    List<ServiceInfo> serviceInfos = metaInfoXml.getServices();

    for (ServiceInfo serviceInfo : serviceInfos) {
      ServiceModule serviceModule = new ServiceModule(stackContext, serviceInfo, serviceDirectory);
      serviceModules.add(serviceModule);
      if (!serviceModule.isValid()){
        extensionInfo.setValid(false);
        setValid(false);
        extensionInfo.addErrors(serviceModule.getErrors());
        addErrors(serviceModule.getErrors());
      }
    }
    addServices(serviceModules);
  }

  /**
   * Resolve another extension module.
   *
   * @param parentExtension    extension module to be resolved
   * @param allStacks          all stack modules in stack definition
   * @param commonServices     all common services specified in the stack definition
   * @param extensions         all extensions
   * @throws AmbariException if unable to resolve the extension
   */
  private void resolveExtension(
          ExtensionModule parentExtension, Map<String, StackModule> allStacks, Map<String, ServiceModule> commonServices, Map<String, ExtensionModule> extensions)
          throws AmbariException {
    if (parentExtension.getModuleState() == ModuleState.INIT) {
	  parentExtension.resolve(null, allStacks, commonServices, extensions);
    } else if (parentExtension.getModuleState() == ModuleState.VISITED) {
      //todo: provide more information to user about cycle
      throw new AmbariException("Cycle detected while parsing extension definition");
    }
    if (!parentExtension.isValid() || (parentExtension.getModuleInfo() != null && !parentExtension.getModuleInfo().isValid())) {
      setValid(parentExtension.isValid());
      extensionInfo.setValid(parentExtension.extensionInfo.isValid());
      addErrors(parentExtension.getErrors());
      extensionInfo.addErrors(parentExtension.getErrors());
    }
  }

  /**
   * Add a child service module to the extension.
   *
   * @param service  service module to add
   */
  private void addService(ServiceModule service) {
    ServiceInfo serviceInfo = service.getModuleInfo();
    Object previousValue = serviceModules.put(service.getId(), service);
    if (previousValue == null) {
      extensionInfo.getServices().add(serviceInfo);
    }
  }

  /**
   * Add child service modules to the extension.
   *
   * @param services  collection of service modules to add
   */
  private void addServices(Collection<ServiceModule> services) {
    for (ServiceModule service : services) {
      addService(service);
    }
  }

  @Override
  public boolean isValid() {
    return valid;
  }

  @Override
  public void setValid(boolean valid) {
    this.valid = valid;
  }

  private Set<String> errorSet = new HashSet<String>();

  @Override
  public Collection getErrors() {
    return errorSet;
  }

  @Override
  public void addError(String error) {
    errorSet.add(error);
  }

  @Override
  public void addErrors(Collection<String> errors) {
    this.errorSet.addAll(errors);
  }

}