common/scala/src/main/scala/org/apache/openwhisk/core/entity/ExecManifest.scala

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.openwhisk.core.entity

import pureconfig._
import pureconfig.generic.auto._

import scala.util.{Failure, Success, Try}
import spray.json._
import spray.json.DefaultJsonProtocol._
import org.apache.openwhisk.core.{ConfigKeys, WhiskConfig}
import org.apache.openwhisk.core.entity.Attachments._
import org.apache.openwhisk.core.entity.Attachments.Attached._
import fastparse._
import NoWhitespace._

import scala.concurrent.duration.{Duration, FiniteDuration}

/**
 * Reads manifest of supported runtimes from configuration file and stores
 * a representation of each runtime which is used for serialization and
 * deserialization. This singleton must be initialized.
 */
protected[core] object ExecManifest {

  /**
   * Required properties to initialize this singleton via WhiskConfig.
   */
  protected[core] def requiredProperties = Map(WhiskConfig.runtimesManifest -> null)

  /**
   * Reads runtimes manifest from WhiskConfig and initializes the
   * singleton Runtime instance.
   *
   * @param config a valid configuration
   * @param manifestOverride an optional inline manifest (used for testing)
   * @return the manifest if initialized successfully, or an failure
   */
  protected[core] def initialize(config: WhiskConfig, manifestOverride: Option[String] = None): Try[Runtimes] = {
    val rmc = loadConfigOrThrow[RuntimeManifestConfig](ConfigKeys.runtimes)
    val mf = Try(manifestOverride.getOrElse(config.runtimesManifest).parseJson.asJsObject).flatMap(runtimes(_, rmc))
    mf.foreach(m => manifest = Some(m))
    mf
  }

  /**
   * Gets existing runtime manifests.
   *
   * @return singleton Runtimes instance previous initialized from WhiskConfig
   * @throws IllegalStateException if singleton was not previously initialized
   */
  @throws[IllegalStateException]
  protected[core] def runtimesManifest: Runtimes = {
    manifest.getOrElse {
      throw new IllegalStateException("Runtimes manifest is not initialized.")
    }
  }

  private var manifest: Option[Runtimes] = None

  /**
   * @param config a configuration object as JSON
   * @return Runtimes instance
   */
  protected[entity] def runtimes(config: JsObject, runtimeManifestConfig: RuntimeManifestConfig): Try[Runtimes] = Try {
    val runtimes = config.fields
      .get("runtimes")
      .map(_.convertTo[Map[String, Set[RuntimeManifest]]].map {
        case (name, versions) =>
          RuntimeFamily(name, versions.map { mf =>
            val img = ImageName(mf.image.name, mf.image.registry, mf.image.prefix, mf.image.tag)
            mf.copy(image = img)
          })
      }.toSet)

    val blackbox = config.fields
      .get("blackboxes")
      .map(_.convertTo[Set[ImageName]].map { image =>
        ImageName(image.name, image.registry, image.prefix, image.tag)
      })

    val bypassPullForLocalImages = runtimeManifestConfig.bypassPullForLocalImages
      .flatMap {
        case true  => runtimeManifestConfig.localImagePrefix
        case false => None
      }

    Runtimes(runtimes.getOrElse(Set.empty), blackbox.getOrElse(Set.empty), bypassPullForLocalImages)
  }

  /**
   * Misc options related to runtime manifests.
   *
   * @param bypassPullForLocalImages if true, allow images with a prefix that matches localImagePrefix
   *                                 to skip docker pull on invoker even if the image is not part of the blackbox set;
   *                                 this is useful for testing with local images that aren't published to the runtimes registry
   * @param localImagePrefix         image prefix for bypassPullForLocalImages
   */
  protected[core] case class RuntimeManifestConfig(bypassPullForLocalImages: Option[Boolean] = None,
                                                   localImagePrefix: Option[String] = None)

  /**
   * A runtime manifest describes the "exec" runtime support.
   *
   * @param kind            the name of the kind e.g., nodejs:6
   * @param deprecated      true iff the runtime is deprecated (allows get/delete but not create/update/invoke)
   * @param default         true iff the runtime is the default kind for its family (nodejs:default -> nodejs:6)
   * @param attached        true iff the source is an attachments (not inlined source)
   * @param requireMain     true iff main entry point is not optional
   * @param sentinelledLogs true iff the runtime generates stdout/stderr log sentinels after an activation
   * @param image           optional image name, otherwise inferred via fixed mapping (remove colons and append 'action')
   * @param stemCells       optional list of stemCells to be initialized by invoker per kind
   */
  protected[core] case class RuntimeManifest(kind: String,
                                             image: ImageName,
                                             deprecated: Option[Boolean] = None,
                                             default: Option[Boolean] = None,
                                             attached: Option[Attached] = None,
                                             requireMain: Option[Boolean] = None,
                                             sentinelledLogs: Option[Boolean] = None,
                                             stemCells: Option[List[StemCell]] = None)

  /**
   * A stemcell configuration read from the manifest for a container image to be initialized by the container pool.
   *
   * @param initialCount  the initial number of stemcell containers to create
   * @param memory the max memory this stemcell will allocate
   * @param reactive the reactive prewarming prewarmed config, which is disabled by default
   */
  protected[entity] case class StemCell(initialCount: Int,
                                        memory: ByteSize,
                                        reactive: Option[ReactivePrewarmingConfig] = None) {
    require(initialCount > 0, "initialCount must be positive")
  }

  /**
   * A stemcell's ReactivePrewarmingConfig configuration
   *
   * @param minCount the max number of stemcell containers to exist
   * @param maxCount  the max number of stemcell containers to create
   * @param ttl time to live of the prewarmed container
   * @param threshold the executed activation number of cold start in previous one minute
   * @param increment increase per increment prewarmed number under per threshold activations
   */
  protected[core] case class ReactivePrewarmingConfig(minCount: Int,
                                                      maxCount: Int,
                                                      ttl: FiniteDuration,
                                                      threshold: Int,
                                                      increment: Int) {
    require(
      minCount >= 0 && minCount <= maxCount,
      "minCount must be be greater than 0 and less than or equal to maxCount")
    require(maxCount > 0, "maxCount must be positive")
    require(ttl.toMillis > 0, "ttl must be positive")
    require(threshold > 0, "threshold must be positive")
    require(increment > 0 && increment <= maxCount, "increment must be positive and less than or equal to maxCount")
  }

  /**
   * An image name for an action refers to the container image canonically as
   * "prefix/name[:tag]" e.g., "openwhisk/python3action:latest".
   */
  protected[core] case class ImageName(name: String,
                                       registry: Option[String] = None,
                                       prefix: Option[String] = None,
                                       tag: Option[String] = None) {

    /**
     * The actual name of the image for an action kind resolved by registry setting.
     */
    def resolveImageName(systemRegistry: Option[String] = None): String = {
      val r = Option(registry.getOrElse(systemRegistry.getOrElse((""))))
        .filter(_.nonEmpty)
        .map { reg =>
          if (reg.endsWith("/")) reg else reg + "/"
        }
        .getOrElse("")
      val p = prefix.filter(_.nonEmpty).map(_ + "/").getOrElse("")
      val t = tag.filter(_.nonEmpty).map(":" + _).getOrElse("")
      r + p + name + t
    }

    /**
     * Overrides equals to allow match on undefined tag or when tag is latest
     * in this or that.
     */
    override def equals(that: Any) = that match {
      case ImageName(n, r, p, t) =>
        name == n && registry == r && p == prefix && (t == tag || {
          val thisTag = tag.getOrElse(ImageName.defaultImageTag)
          val thatTag = t.getOrElse(ImageName.defaultImageTag)
          thisTag == thatTag
        })

      case _ => false
    }
  }

  protected[core] object ImageName {
    private val defaultImageTag = "latest"

    // docker image name grammar, taken from: https://github.com/docker/distribution/blob/master/reference/reference.go
    //
    // Grammar
    //
    // reference                       := name [ ":" tag ] [ "@" digest ]
    // name                            := [domain '/'] path-component ['/' path-component]*
    // domain                          := domain-component ['.' domain-component]* [':' port-number]
    // domain-component                := /([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9-]*[a-zA-Z0-9])/
    // port-number                     := /[0-9]+/
    // path-component                  := alpha-numeric [separator alpha-numeric]*
    // alpha-numeric                   := /[a-z0-9]+/
    // separator                       := /[_.]|__|[-]*/
    //
    // tag                             := /[\w][\w.-]{0,127}/
    //
    // digest                          := digest-algorithm ":" digest-hex
    // digest-algorithm                := digest-algorithm-component [ digest-algorithm-separator digest-algorithm-component ]*
    // digest-algorithm-separator      := /[+.-_]/
    // digest-algorithm-component      := /[A-Za-z][A-Za-z0-9]*/
    // digest-hex                      := /[0-9a-fA-F]{32,}/ ; At least 128 bit digest value
    private def lowercaseLetters[_: P] = P(CharIn("a-z"))
    private def uppercaseLetters[_: P] = P(CharIn("a-Z"))
    private def letters[_: P] = P(lowercaseLetters | uppercaseLetters)
    private def digits[_: P] = P(CharIn("0-9"))

    private def alphaNumeric[_: P] = P(lowercaseLetters | digits)
    private def alphaNumericWithUpper[_: P] = P(letters | digits)
    private def word[_: P] = P(alphaNumericWithUpper | "_")

    private def digestHex[_: P] = P(digits | CharIn("a-fA-F")).rep(32)
    private def digestAlgorithmComponent[_: P] = P(letters ~ alphaNumericWithUpper.rep)
    private def digestAlgorithmSeperator[_: P] = P("+" | "." | "-" | "_")
    private def digestAlgorithm[_: P] = P(digestAlgorithmComponent.rep(min = 1, sep = digestAlgorithmSeperator))
    private def digest[_: P] = P(digestAlgorithm ~ ":" ~ digestHex)

    private def tag[_: P] = P(word ~ (word | "." | "-").rep(max = 127))

    private def separator[_: P] = P("_" | "." | "__" | "-".rep)
    private def pathComponent[_: P] = P(alphaNumeric.rep(min = 1, sep = separator))
    private def portNumber[_: P] = P(digits.rep(1))
    // FIXME: this is not correct yet. It accepts "-" as the beginning and end of a domain
    private def domainComponent[_: P] = P(alphaNumericWithUpper | "-").rep
    private def domain[_: P] =
      P((domainComponent
        .rep(min = 2, sep = ".") ~ (":" ~ portNumber).?) | (domainComponent.rep(min = 1, sep = ".") ~ ":" ~ portNumber))
    private def name[_: P] = P((domain.! ~ "/").? ~ pathComponent.!.rep(min = 1, sep = "/"))

    private def reference[_: P] = P(Start ~ name ~ (":" ~ tag.!).? ~ ("@" ~ digest.!).? ~ End)

    /**
     * Constructs an ImageName from a string. This method checks that the image name conforms
     * to the Docker naming. As a result, failure to deserialize a string will throw an exception
     * which fails the Try. Callers could use this to short-circuit operations (CRUD or activation).
     * Internal container names use the proper constructor directly.
     */
    def fromString(s: String): Try[ImageName] = {
      parse(s, reference(_)) match {
        case Parsed.Success((registry, imagePathParts, imageTag, _), _) =>
          // imagePathParts has at least one element per the parser above
          val prefix = (imagePathParts.dropRight(1)).mkString("/")
          val imageName = imagePathParts.last

          Success(ImageName(imageName, registry, if (prefix.nonEmpty) Some(prefix) else None, imageTag))
        case Parsed.Failure(_, _, _) =>
          Failure(DeserializationException("could not parse image name"))
      }
    }
  }

  /**
   * A runtime family manifest is a collection of runtimes grouped by a family (e.g., swift with versions swift:2 and swift:3).
   *
   * @param name runtime family
   * @version set of runtime manifests
   */
  protected[entity] case class RuntimeFamily(name: String, versions: Set[RuntimeManifest])

  /**
   * A collection of runtime families.
   *
   * @param runtimes                 set of supported runtime families
   * @param blackboxImages           set of blackbox container images
   * @param bypassPullForLocalImages container image prefix that is exempted from docker pull operations
   */
  protected[core] case class Runtimes(runtimes: Set[RuntimeFamily],
                                      blackboxImages: Set[ImageName],
                                      bypassPullForLocalImages: Option[String]) {

    val knownContainerRuntimes: Set[String] = runtimes.flatMap(_.versions.map(_.kind))

    val manifests: Map[String, RuntimeManifest] = {
      runtimes.flatMap {
        _.versions.map { m =>
          m.kind -> m
        }
      }.toMap
    }

    def skipDockerPull(image: ImageName): Boolean = {
      blackboxImages.contains(image) ||
      image.prefix.flatMap(p => bypassPullForLocalImages.map(_ == p)).getOrElse(false)
    }

    def toJson: JsObject = {
      runtimes
        .map { family =>
          family.name -> family.versions.map {
            case rt =>
              JsObject(
                "kind" -> rt.kind.toJson,
                "image" -> rt.image.resolveImageName().toJson,
                "deprecated" -> rt.deprecated.getOrElse(false).toJson,
                "default" -> rt.default.getOrElse(false).toJson,
                "attached" -> rt.attached.isDefined.toJson,
                "requireMain" -> rt.requireMain.getOrElse(false).toJson)
          }
        }
        .toMap
        .toJson
        .asJsObject
    }

    def resolveDefaultRuntime(kind: String): Option[RuntimeManifest] = {
      kind match {
        case defaultSplitter(family) => defaultRuntimes.get(family).flatMap(manifests.get(_))
        case _                       => manifests.get(kind)
      }
    }

    /**
     * Collects all runtimes for which there is a stemcell configuration defined
     *
     * @return list of runtime manifests with stemcell configurations
     */
    def stemcells: Map[RuntimeManifest, List[StemCell]] = {
      manifests
        .flatMap {
          case (_, m) => m.stemCells.map(m -> _)
        }
        .filter(_._2.nonEmpty)
    }

    private val defaultRuntimes: Map[String, String] = {
      runtimes.map { family =>
        family.versions.filter(_.default.exists(identity)).toList match {
          case Nil if family.versions.size == 1 => family.name -> family.versions.head.kind
          case Nil                              => throw new IllegalArgumentException(s"${family.name} has multiple versions, but no default.")
          case d :: Nil                         => family.name -> d.kind
          case ds =>
            throw new IllegalArgumentException(s"Found more than one default for ${family.name}: ${ds.mkString(",")}.")
        }
      }.toMap
    }

    private val defaultSplitter = "([a-z0-9]+):default".r
  }

  protected[entity] implicit val imageNameSerdes: RootJsonFormat[ImageName] = jsonFormat4(ImageName.apply)

  protected[entity] implicit val ttlSerdes: RootJsonFormat[FiniteDuration] = new RootJsonFormat[FiniteDuration] {
    override def write(finiteDuration: FiniteDuration): JsValue = JsString(finiteDuration.toString)

    override def read(value: JsValue): FiniteDuration = value match {
      case JsString(s) =>
        val duration = Duration(s)
        FiniteDuration(duration.length, duration.unit)
      case _ =>
        deserializationError("time unit not supported. Only milliseconds, seconds, minutes, hours, days are supported")
    }
  }

  protected[entity] implicit val reactivePrewarmingConfigSerdes: RootJsonFormat[ReactivePrewarmingConfig] = jsonFormat5(
    ReactivePrewarmingConfig.apply)

  protected[entity] implicit val stemCellSerdes = new RootJsonFormat[StemCell] {
    import org.apache.openwhisk.core.entity.size.serdes
    val defaultSerdes = jsonFormat3(StemCell.apply)
    override def read(value: JsValue): StemCell = {
      val fields = value.asJsObject.fields
      val initialCount: Option[Int] =
        fields
          .get("initialCount")
          .orElse(fields.get("count"))
          .map(_.convertTo[Int])
      val memory: Option[ByteSize] = fields.get("memory").map(_.convertTo[ByteSize])
      val config = fields.get("reactive").map(_.convertTo[ReactivePrewarmingConfig])

      (initialCount, memory) match {
        case (Some(c), Some(m)) => StemCell(c, m, config)
        case (Some(c), None) =>
          throw new IllegalArgumentException(s"memory is required, just provide initialCount: ${c}")
        case (None, Some(m)) =>
          throw new IllegalArgumentException(s"initialCount is required, just provide memory: ${m.toString}")
        case _ => throw new IllegalArgumentException("both initialCount and memory are required")
      }
    }

    override def write(s: StemCell) = defaultSerdes.write(s)
  }

  protected[entity] implicit val runtimeManifestSerdes: RootJsonFormat[RuntimeManifest] = jsonFormat8(RuntimeManifest)

}