website/build.gradle

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * License); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an AS IS BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// Define common lifecycle tasks and artifact types
apply plugin: "base"

// Publish website to asf-git branch.
apply plugin: 'org.ajoberstar.grgit'

def dockerImageTag = 'beam-website'
def dockerWorkDir = "/repo"
def buildDir = "${project.rootDir}/build/website"
def buildContentDir = "${project.rootDir}/build/website/generated-content"
def repoContentDir = "${project.rootDir}/website/generated-content"
def commitedChanges = false
def gitboxUrl = project.findProperty('gitPublishRemote') ?: 'https://gitbox.apache.org/repos/asf/beam.git'

def shell = { cmd ->
  println cmd
  exec {
    executable 'sh'
    args '-c', cmd
  } 
}


def envdir = "${buildDir}/gradleenv"

task setupVirtualenv {
  doLast {
    exec {
      commandLine 'virtualenv', "${envdir}"
    }
    exec {
      executable 'sh'
      args '-c', ". ${envdir}/bin/activate && pip install beautifulsoup4"
    }
  }
  outputs.dirs(envdir)
}

task buildDockerImage(type: Exec) {
  inputs.files 'Gemfile', 'Gemfile.lock'
  commandLine 'docker', 'build', '-t', dockerImageTag, '.'
}

task createDockerContainer(type: Exec) {
  dependsOn buildDockerImage
  standardOutput = new ByteArrayOutputStream()
  ext.containerId = {
    return standardOutput.toString().trim()
  }
  commandLine '/bin/bash', '-c',
    "docker create -v $project.rootDir:$dockerWorkDir -u \$(id -u):\$(id -g) $dockerImageTag"
}

task startDockerContainer(type: Exec) {
  dependsOn createDockerContainer
  ext.containerId = {
    return createDockerContainer.containerId()
  }
  commandLine 'docker', 'start',
    "${->createDockerContainer.containerId()}" // Lazily evaluate containerId.
}

task stopAndRemoveDockerContainer(type: Exec) {
  commandLine 'docker', 'rm', '-f', "${->createDockerContainer.containerId()}"
}

task setupBuildDir(type: Copy) {
  from('.') {
    include 'Gemfile*'
    include 'Rakefile'
  }
  into buildDir
}

task cleanWebsite(type: Delete) {
  delete buildDir
}
clean.dependsOn cleanWebsite

task buildWebsite(type: Exec) {
  def baseurlFlag = project.findProperty('githubPullRequestId') ? "--baseurl=/${project.findProperty('githubPullRequestId')}" : ''
  dependsOn startDockerContainer, setupBuildDir
  finalizedBy stopAndRemoveDockerContainer
  inputs.files 'Gemfile.lock', '_config.yml'
  inputs.dir 'src'
  inputs.property 'baseurl', baseurlFlag
  outputs.dir "$buildDir/.sass-cache"
  outputs.dir buildContentDir
  commandLine 'docker', 'exec',
    "${->startDockerContainer.containerId()}", '/bin/bash', '-c',
    """cd $dockerWorkDir/build/website && \
      bundle exec jekyll build \
      --config $dockerWorkDir/website/_config.yml \
      --incremental ${baseurlFlag} \
      --source $dockerWorkDir/website/src
      """
}
build.dependsOn buildWebsite

task testWebsite(type: Exec) {
  dependsOn startDockerContainer, buildWebsite
  finalizedBy stopAndRemoveDockerContainer

  inputs.files "$buildDir/Rakefile"
  inputs.dir buildContentDir
  commandLine 'docker', 'exec',
    "${->startDockerContainer.containerId()}", '/bin/bash', '-c',
    """cd $dockerWorkDir/build/website && \
      bundle exec rake test"""
}

check.dependsOn testWebsite

task preCommit {
  dependsOn testWebsite
}

// Creates a new commit on asf-site branch
task commitWebsite << {
  assert file("${buildContentDir}/index.html").exists()
  // Generated javadoc and pydoc content is not built or stored in this repo.
  assert !file("${buildContentDir}/documentation/sdks/javadoc").exists()
  assert !file("${buildContentDir}/documentation/sdks/pydoc").exists()

  def git = grgit.open()
  // get the latest commit on master
  def latestCommit = grgit.log(maxCommits: 1)[0].abbreviatedId

  shell "git fetch --force origin +asf-site:asf-site"
  git.checkout(branch: 'asf-site')

  // Delete the previous content.
  git.remove(patterns: [ 'website/generated-content' ])
  assert !file("${repoContentDir}/index.html").exists()
  delete repoContentDir

  // Copy the built content and add it.
  copy {
    from buildContentDir
    into repoContentDir
  }
  assert file("${repoContentDir}/index.html").exists()
  git.add(patterns: ['website/generated-content'])

  def currentDate = new Date().format('yyyy/MM/dd HH:mm:ss')
  String message = "Publishing website ${currentDate} at commit ${latestCommit}"
  if (git.status().isClean()) {
    println 'No changes to commit'
  } else {
    println 'Creating commit for changes'
    commitedChanges = true
    git.commit(message: message)
  }
}

/*
 * Pushes the asf-site branch commits.
 *
 * This requires write access to the asf-site branch and can be run on
 * Jenkins executors with the git-websites label.
 *
 * For more details on publishing, see:
 * https://www.apache.org/dev/project-site.html
 * https://github.com/apache/infrastructure-puppet/blob/deployment/modules/gitwcsub/files/config/gitwcsub.cfg
 *
 * You can test this locally with a forked repository by manually adding the
 * website-publish remote pointing to your forked repository, for example:
 *   git remote add website-publish git@github.com:${GITUSER}/beam.git
 * because the remote is only added if it doesn't exist. The remote needs
 * to be added before every execution of the publishing.
 */
task publishWebsite << {
  def git = grgit.open()
  git.checkout(branch: 'asf-site')
  if (!commitedChanges) {
    println 'No changes to push'
    return
  }

  // Because git.push() fails to authenticate, run git push directly.
  shell "git push ${gitboxUrl} asf-site"
}

commitWebsite.dependsOn testWebsite
publishWebsite.dependsOn commitWebsite

/*
 * Stages a pull request on GCS
 * For example:
 *   ./gradlew :beam-website:stageWebsite -PgithubPullRequestId=${ghprbPullId} -PwebsiteBucket=foo
 */
task stageWebsite << {
  assert project.hasProperty('githubPullRequestId')
  assert githubPullRequestId.isInteger()

  def gcs_bucket = project.findProperty('websiteBucket') ?: 'apache-beam-website-pull-requests'
  def gcs_path = "gs://${gcs_bucket}/${githubPullRequestId}"

  // Remove current site if it exists.
  shell "gsutil -m rm -r -f ${gcs_path} || true"

  // Fixup the links to index.html files
  shell "cd ${buildDir} && ln -s generated-content content || true"
  shell ". ${envdir}/bin/activate && cd ${buildDir} && " +
        "python ${project.rootDir}/website/.jenkins/append_index_html_to_internal_links.py"

  // Copy the build website to GCS
  shell "gsutil -m cp -r ${buildContentDir} ${gcs_path}"

  println "Website published to http://${gcs_bucket}." +
      "storage.googleapis.com/${githubPullRequestId}/index.html"
}

stageWebsite.dependsOn setupVirtualenv
stageWebsite.dependsOn buildWebsite