website/build.gradle

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * License); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an AS IS BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

plugins {
  id 'base'                  // Define common lifecycle tasks and artifact types
  id 'org.ajoberstar.grgit'  // Publish website to asf-git branch.
}

apply plugin: org.apache.beam.gradle.BeamModulePlugin
applyPythonNature()

def dockerImageTag = 'beam-website'
def dockerWorkDir = "/opt"
def buildDir = "${project.rootDir}/build/website"
def dockerSourceDir = "$dockerWorkDir/website/www"
def dockerBuildDir = "$dockerWorkDir/build/website"

def commitedChanges = false
def gitboxUrl = project.findProperty('gitPublishRemote') ?: 'https://gitbox.apache.org/repos/asf/beam.git'

def shell = { cmd ->
  println cmd
  exec {
    executable 'sh'
    args '-c', cmd
  }
}


def envdir = "${buildDir}/gradleenv"

task setupVirtualenvWithDependencies {
  doLast {
    exec {
      commandLine "python${project.ext.pythonVersion}", '-m', 'venv', "${envdir}"
    }
    exec {
      executable 'sh'
      args '-c', ". ${envdir}/bin/activate && pip install beautifulsoup4"
    }
  }
  outputs.dirs(envdir)
}

task buildDockerImage(type: Exec) {
  commandLine 'docker', 'build', '-t', dockerImageTag, '.'
}

task createDockerContainer(type: Exec) {
  dependsOn buildDockerImage
  standardOutput = new ByteArrayOutputStream()
  ext.containerId = {
    return standardOutput.toString().trim()
  }
  gradle.taskGraph.whenReady {
    def extraOptions = ''
    if (gradle.taskGraph.hasTask(":${project.name}:serveWebsite")) {
      // Publish port 1313 where Hugo serves website from
      extraOptions = "--publish 127.0.0.1:1313:1313"
    }

    if (System.getenv().getOrDefault('NODE_NAME', '').startsWith('websites')) {
      // Jenkins websites node: run as root. Files written to /repo will get
      // "jenkins" user ownership. BEAM-9737
    } else {
      // Otherwise: run as current user, so as to not require sudo to clean up
      // build/ directory.
      extraOptions += " -u \$(id -u):\$(id -g)"
    }
    commandLine '/bin/bash', '-c',
      "docker create -v $project.rootDir:$dockerWorkDir $extraOptions $dockerImageTag sh -c 'trap \"exit 0\" INT; while true; do sleep 30; done;'"
  }
}

task startDockerContainer(type: Exec) {
  dependsOn createDockerContainer
  ext.containerId = {
    return createDockerContainer.containerId()
  }
  commandLine 'docker', 'start',
    "${->createDockerContainer.containerId()}" // Lazily evaluate containerId.
}

// Clone Docsy submodule which is our Hugo theme
task initGitSubmodules(type: Exec) {
    commandLine 'docker', 'exec',
                "${->startDockerContainer.containerId()}", 'git',  'submodule', 'update', '--init',  '--recursive'
}

// Install yarn dependencies
task installDependencies(type: Exec) {
    commandLine 'docker', 'exec', '--workdir', "$dockerSourceDir",
                "${->startDockerContainer.containerId()}", 'yarn', 'install'
}

// Run build_code_samples.sh to fetch Beam project content
// which is used by code_sample shortcodes to inject snippets into codeblocks
task buildCodeSamples(type: Exec) {
  commandLine 'docker', 'exec', '--workdir', "$dockerSourceDir",
              "${->startDockerContainer.containerId()}", 'yarn', 'build_code_samples'
}

task setupDockerContainer(type: Exec) {
  dependsOn startDockerContainer
  finalizedBy initGitSubmodules, installDependencies, buildCodeSamples
  ext.containerId = {
    return startDockerContainer.containerId()
  }

  // Create the config to point to a GitHub or Colab blob in the repo, e.g. apache/beam/blob/master
  commandLine 'docker', 'exec',
    "${->startDockerContainer.containerId()}", '/bin/bash', '-c',
    """echo '[params]\n  branch_repo = "${getBranchRepo()}"' > /tmp/_config_branch_repo.toml"""
}

def performanceLooksOutDir = "$dockerSourceDir/site/static/images/performance/looks"
def gcpCredsInContainer = "/tmp/gcp-key.json"

task copyGcpCredsToContainer(type: Exec) {
  dependsOn setupDockerContainer

  def gcpCredsHost = providers.gradleProperty("gcpCredsFile").orNull

  onlyIf { gcpCredsHost != null }

  doFirst {
    commandLine 'docker', 'cp',
      gcpCredsHost,
      "${->startDockerContainer.containerId()}:${gcpCredsInContainer}"
  }
}

task downloadPerformanceLooks(type: Exec) {
  dependsOn setupDockerContainer, copyGcpCredsToContainer

  def gcpCredsHost = providers.gradleProperty("gcpCredsFile").orNull

  onlyIf { gcpCredsHost != null }

  doFirst {
    // The website container is started as the host uid, which has no
    // /etc/passwd entry. gcloud storage calls pwd.getpwuid and crashes,
    // so the build publishes without the plots. Root is required only to
    // add that uid. gcloud then runs as the repo owner.
    commandLine 'docker', 'exec', '-u', '0',
      "${-> setupDockerContainer.containerId()}", '/bin/bash', '-lc',
      """
      set -u

      SRC="gs://public_looker_explores_us_a3853f40"
      OUT_DIR="${performanceLooksOutDir}"
      HOST_UID="\$(stat -c '%u' /opt/website)"
      HOST_GID="\$(stat -c '%g' /opt/website)"

      if ! getent passwd "\$HOST_UID" >/dev/null 2>&1; then
        echo "beamweb:x:\${HOST_UID}:\${HOST_GID}:beamweb:/tmp:/bin/bash" >> /etc/passwd
      fi

      export HOME=/tmp
      export CLOUDSDK_CONFIG=/tmp/gcloud
      export GOOGLE_APPLICATION_CREDENTIALS="${gcpCredsInContainer}"

      mkdir -p "\$OUT_DIR"
      # mkdir above runs as root. The leaf directory must be writable by the host uid.
      chown "\$HOST_UID:\$HOST_GID" "\$OUT_DIR"

      echo "[performance_looks] Syncing looks..."

      if gosu "\$HOST_UID:\$HOST_GID" gcloud auth activate-service-account --key-file="\$GOOGLE_APPLICATION_CREDENTIALS" --no-user-output-enabled \
        && gosu "\$HOST_UID:\$HOST_GID" gcloud storage rsync -r "\$SRC" "\$OUT_DIR"; then
        echo "[performance_looks] Download completed successfully"
      else
        echo "[performance_looks][WARNING] Failed to sync looks from GCS"
        echo "[performance_looks][WARNING] Website build will continue without updated images"
      fi
    """
  }
}

task stopAndRemoveDockerContainer(type: Exec) {
  commandLine 'docker', 'rm', '-f', "${->createDockerContainer.containerId()}"
}

task cleanWebsite(type: Delete) {
  delete buildDir
}
clean.dependsOn cleanWebsite

class BuildTaskConfiguration {
  String name
  boolean useTestConfig = false
  String baseUrl = ''
  String dockerWorkDir = ''
}
def createBuildTask = {
  BuildTaskConfiguration config = it as BuildTaskConfiguration
  task "build${config.name}Website" (type:Exec) {
    dependsOn setupDockerContainer
    dependsOn downloadPerformanceLooks
    finalizedBy stopAndRemoveDockerContainer

    def configs = "$dockerSourceDir/site/config.toml"
    def baseUrlFlag = config.baseUrl ? "--baseURL /${config.baseUrl}" : ""
    commandLine 'docker', 'exec',
      "${->setupDockerContainer.containerId()}", '/bin/bash', '-c',
      """cd $dockerSourceDir && \
        yarn build \
        -d $dockerBuildDir/generated-${config.name.toLowerCase()}-content \
        --config $configs \
        $baseUrlFlag
        """
  }
}

// task buildLocalWebsite
createBuildTask(
  name:'Local',
)
task buildWebsite(dependsOn:buildLocalWebsite)
build.dependsOn buildWebsite

// task buildGcsWebsite
createBuildTask(
  name:'Gcs',
  baseUrl: getBaseUrl(),
)

// task buildApacheWebsite
createBuildTask(
  name:'Apache',
)

/**
 * Use the pull request ID in the URL path, or the git branch when building locally.
 * This allows staging multiple work trees without clobbering eachother, i.e.
 * for shared GCS staging, or for locally comparing the differences between
 * branches.
 */
def getBaseUrl() {
  def pullRequestId = System.getenv('ghprbPullId')?.trim()
  if (pullRequestId) { return pullRequestId }

  // grgit is null if building outside of git (i.e. from source archive)
  def buildContext = grgit ? grgit.branch.current().getName() : 'archive'
  return "${System.getProperty('user.name')}-${buildContext}"
}

/**
 * Gets the branch repository where the new files are located. This is used to point
 * all the GitHub and Colab files to the pull request's branch when testing, while
 * pointing them to the apache/beam master branch on production. This is done so tests
 * and staged versions work even when the files don't exist in master yet.
 */
def getBranchRepo() {
  // Jenkins stores the GitHub author in $ghprbPullAuthorLogin
  def author = System.env.ghprbPullAuthorLogin
  if (author == null) {
    // If the author is not defined, it's most probably running locally.
    if (grgit != null) {
      // If on git, try to infer the author from the remote URL
      for (remote in grgit.remote.list()) {
        if (remote.getName() == 'origin') {
          // remote.url = 'git@github.com:author/beam.git'
          author = remote.url.split(':')[1].split('/')[0]
          break
        }
      }
    }
  }
  // Jenkins stores the branch in both of the following environment variables.
  def branch = System.env.ghprbSourceBranch ?: System.env.GIT_BRANCH
  if (branch == null && grgit) {
    branch = grgit.branch.current().getName()
  }

  // Return the author's branch repo, otherwise default to the master repo.
  if (author && branch) {
    return "${author}/beam/blob/${branch}"
  }
  return "apache/beam/blob/master"
}

def buildContentDir(name) {
  "${project.rootDir}/build/website/generated-${name.toLowerCase()}-content"
}

task serveWebsite(type: Exec) {
  dependsOn setupDockerContainer
  finalizedBy stopAndRemoveDockerContainer
  commandLine 'docker', 'exec',
    "${->setupDockerContainer.containerId()}", '/bin/bash', '-c',
    """cd $dockerSourceDir && \
      yarn develop \
      --bind="0.0.0.0" \
      --config $dockerSourceDir/site/config.toml,/tmp/_config_branch_repo.toml
      """
}

task testWebsite(type: Exec) {
  // dependsOn setupDockerContainer, 'buildWebsite'
  finalizedBy stopAndRemoveDockerContainer

  commandLine 'docker', 'exec',
    "${->setupDockerContainer.containerId()}", '/bin/bash', '-c',
    "$dockerSourceDir/check-links.sh $dockerBuildDir/generated-local-content"
}

testWebsite.dependsOn 'buildLocalWebsite'

task preCommit {
  dependsOn testWebsite
}

// Creates a new commit on asf-site branch
task commitWebsite {
  doLast {
    assert grgit : "Cannot commit website outside of git repository"
    assert file("${buildContentDir('apache')}/index.html").exists()
    // Generated javadoc and pydoc content is not built or stored in this repo.
    assert !file("${buildContentDir('apache')}/documentation/sdks/javadoc").exists()
    assert !file("${buildContentDir('apache')}/documentation/sdks/pydoc").exists()

    def git = grgit.open(currentDir: project.rootDir)
    // get the latest commit on master
    def latestCommit = grgit.log(maxCommits: 1)[0].abbreviatedId

    shell "git fetch --force origin +asf-site:asf-site"
    shell "git stash"
    shell "git checkout asf-site"
    git.checkout(branch: 'asf-site')

    // Delete the previous content. These are asf-site branch paths.
    git.remove(patterns: [ 'website/generated-content' ])
    def repoContentDir = "${project.rootDir}/website/generated-content"
    assert !file("${repoContentDir}/index.html").exists()
    delete repoContentDir

    // Copy the built content and add it.
    copy {
      from buildContentDir('apache')
      into repoContentDir
    }
    assert file("${repoContentDir}/index.html").exists()
    git.add(patterns: ['website/generated-content'])

    def currentDate = new Date().format('yyyy/MM/dd HH:mm:ss')
    String message = "Publishing website ${currentDate} at commit ${latestCommit}"
    if (!git.status().staged.getAllChanges()) {
      println 'No changes to commit'
    } else {
      println 'Creating commit for changes'
      commitedChanges = true
      git.commit(message: message)
    }
  }
}

/*
 * Pushes the asf-site branch commits.
 *
 * This requires write access to the asf-site branch and can be run on
 * Jenkins executors with the git-websites label.
 *
 * For more details on publishing, see:
 * https://www.apache.org/dev/project-site.html
 * https://github.com/apache/infrastructure-puppet/blob/deployment/modules/gitwcsub/files/config/gitwcsub.cfg
 *
 * You can test this locally with a forked repository by manually adding the
 * website-publish remote pointing to your forked repository, for example:
 *   git remote add website-publish git@github.com:${GITUSER}/beam.git
 * because the remote is only added if it doesn't exist. The remote needs
 * to be added before every execution of the publishing.
 */
task publishWebsite {
  doLast {
    assert grgit : "Cannot publish website outside of git repository"

    def git = grgit.open(currentDir: project.rootDir)
    git.checkout(branch: 'asf-site')
    if (!commitedChanges) {
      println 'No changes to push'
      return
    }

    // Because git.push() fails to authenticate, run git push directly.
    shell "git push ${gitboxUrl} asf-site"
  }
}

commitWebsite.dependsOn buildApacheWebsite
publishWebsite.dependsOn commitWebsite

/*
 * Stages a pull request on GCS
 * For example:
 *   ./gradlew :website:stageWebsite -PwebsiteBucket=foo
 */
task stageWebsite {
  doLast {
    def baseUrl = getBaseUrl()
    assert baseUrl : 'Website staging requires a valid baseUrl'
    def gcs_bucket = project.findProperty('websiteBucket') ?: 'apache-beam-website-pull-requests'
    def gcs_path = "gs://${gcs_bucket}/${baseUrl}"

    // Fixup the links to index.html files
    shell ". ${envdir}/bin/activate && python append_index_html_to_internal_links.py ${buildContentDir('gcs')}"

    // Copy the build website to GCS
    shell "gcloud storage rsync -r --delete-unmatched-destination-objects ${buildContentDir('gcs')} ${gcs_path}"

    println "Website published to http://${gcs_bucket}." +
        "storage.googleapis.com/${baseUrl}/index.html"
  }
}

stageWebsite.dependsOn setupVirtualenvWithDependencies
stageWebsite.dependsOn buildGcsWebsite