ci/publish/Jenkinsfile

// -*- mode: groovy -*-

// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

// Jenkins pipeline
// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/

//mxnet libraries
mx_scala_pub = 'lib/**, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, config.mk, scala-package/pom.xml, scala-package/**/pom.xml, scala-package/**/target/**, scala-package/*/target/repo/**'

// timeout in minutes
max_time = 120

node('restricted-utility') {
  // Loading the utilities requires a node context unfortunately
  checkout scm
  utils = load('ci/Jenkinsfile_utils.groovy')
}
utils.assign_node_labels(utility: 'restricted-utility', linux_cpu: 'restricted-mxnetlinux-cpu', linux_gpu: 'restricted-mxnetlinux-gpu', linux_gpu_p3: 'restricted-mxnetlinux-gpu-p3', windows_cpu: 'restricted-mxnetwindows-cpu', windows_gpu: 'restricted-mxnetwindows-gpu')

// CPU and GPU. OSX nodes are not currently supported by Jenkins
def nodeMap = ['cpu': NODE_LINUX_CPU, 'gpu': NODE_LINUX_GPU_P3]
def scalaOSMap = ['cpu': 'linux-x86_64-cpu', 'gpu': 'linux-x86_64-gpu']
def scalaVariantMap = ['cpu': 'cpu', 'gpu': 'cu92']

def wrapStep(nodeToRun, workspaceName, step) {
  return {
    node(nodeToRun) {
      ws("workspace/${workspaceName}") {
        timeout(time: max_time, unit: 'MINUTES') {
          step()
        }
      }
    }
  }
}

def toBuild = [:]
def labels = ['cpu', 'gpu']
for (x in labels) {
  def label = x // Required due to language
  toBuild["Scala Build ${label}"] = wrapStep(nodeMap['cpu'], "build-scala-${label}") {
    withEnv(["MAVEN_PUBLISH_OS_TYPE=${scalaOSMap[label]}", "mxnet_variant=${scalaVariantMap[label]}"]) {
      utils.init_git()
      utils.docker_run("publish.ubuntu1404_cpu", 'publish_scala_build', false, '500m', 'MAVEN_PUBLISH_OS_TYPE mxnet_variant')
      utils.pack_lib("scala_${label}", mx_scala_pub, false)
    }
  }
}

def toTest = [:]
def systems = ['ubuntu1604', 'ubuntu1804', 'centos7']
for (x in labels) {
  def label = x // Required due to language
  for (y in systems) {
    def system = y // Required due to language
    toTest["Scala Test ${system} ${label}"] = wrapStep(nodeMap[label], "test-scala-${system}-${label}") {
      withEnv(["mxnet_variant=${scalaVariantMap[label]}"]) {
        utils.unpack_and_init("scala_${label}", mx_scala_pub, false)
        utils.docker_run("publish.test.${system}_${label}", 'publish_scala_test', label == 'gpu', '500m', 'mxnet_variant')
      }
    }
  }
}

def toDeploy = [:]
for (x in labels) {
  def label = x // Required due to language
  toDeploy["Scala Deploy ${label}"] = wrapStep(nodeMap[label], "deploy-scala-${label}") {
    withEnv(["MAVEN_PUBLISH_OS_TYPE=${scalaOSMap[label]}", "mxnet_variant=${scalaVariantMap[label]}"]) {
      utils.unpack_and_init("scala_${label}", mx_scala_pub, false)
      utils.docker_run("publish.ubuntu1604_${label}", 'publish_scala_deploy', label == 'gpu' ? true : false, '500m', 'MAVEN_PUBLISH_OS_TYPE MAVEN_PUBLISH_SECRET_ENDPOINT_URL MAVEN_PUBLISH_SECRET_NAME_CREDENTIALS MAVEN_PUBLISH_SECRET_NAME_GPG DOCKERHUB_SECRET_ENDPOINT_REGION mxnet_variant')
    }
  }
}

utils.main_wrapper(
core_logic: {
  stage('Build Packages') {
    parallel toBuild
  }
  stage('Test Packages') {
    parallel toTest
  }
  stage('Deploy Packages') {
    parallel toDeploy
  }
}
,
failure_handler: {
  if (currentBuild.result == "FAILURE") {
    emailext body: 'Generating the nightly maven has failed. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[NIGHTLY MAVEN FAILED] Build ${BUILD_NUMBER}', to: '${EMAIL}'
  }
}
)