blob: 34a51603bc95f57f4902318a5a24dec3f36b4f45 [file] [log] [blame]
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.util.regex.Matcher
import java.util.regex.Pattern
import org.apache.commons.codec.digest.DigestUtils
description = 'Solr Docker image'
apply plugin: 'base'
// Solr Docker inputs
def dockerImageSolrDist = "${ -> propertyOrEnvOrDefault("solr.docker.dist", "SOLR_DOCKER_DIST", 'full') }"
def isDistSlim = {String dist -> dist.toLowerCase(Locale.ROOT) == "slim"}
def isImageSlim = { -> isDistSlim(dockerImageSolrDist) }
def distToSuffix = {String dist -> isDistSlim(dist) ? "-slim" : ""}
def dockerImageDistSuffix = "${ -> distToSuffix(dockerImageSolrDist)}"
def dockerImageTagSuffix = "${ -> propertyOrEnvOrDefault("solr.docker.imageTagSuffix", "SOLR_DOCKER_IMAGE_TAG_SUFFIX", '')}"
def dockerImageRepo = "${ -> propertyOrEnvOrDefault("solr.docker.imageRepo", "SOLR_DOCKER_IMAGE_REPO", "apache/solr") }"
def dockerImageTag = "${ -> propertyOrEnvOrDefault("solr.docker.imageTag", "SOLR_DOCKER_IMAGE_TAG", project.version + dockerImageDistSuffix) }"
def dockerImageName = "${ -> propertyOrEnvOrDefault("solr.docker.imageName", "SOLR_DOCKER_IMAGE_NAME", "${dockerImageRepo}:${dockerImageTag}${dockerImageTagSuffix}") }"
def baseDockerImage = "${ -> propertyOrEnvOrDefault("solr.docker.baseImage", "SOLR_DOCKER_BASE_IMAGE", 'eclipse-temurin:17-jre-jammy') }"
def officialDockerImageName = {String dist -> "${ -> propertyOrEnvOrDefault("solr.docker.imageName", "SOLR_DOCKER_IMAGE_NAME", "${dockerImageRepo}-official:${dockerImageTag}${distToSuffix(dist)}${dockerImageTagSuffix}") }" }
def releaseGpgFingerprint = "${ -> propertyOrDefault('signing.gnupg.keyName',propertyOrDefault('signing.keyId','')) }"
// Build directory locations
def imageIdFile = "$buildDir/image-id"
def smokeTestOfficial = "$buildDir/smoke-check-official"
def imageIdFileOfficial = { String dist -> "$smokeTestOfficial/$dist/image-id" }
configurations {
packaging {
canBeResolved = true
}
packagingLocal {
canBeResolved = true
}
packagingOfficial {
canBeResolved = true
}
solrFullTgz {
canBeConsumed = false
canBeResolved = true
}
solrSlimTgz {
canBeConsumed = false
canBeResolved = true
}
solrFullTgzSignature {
canBeConsumed = false
canBeResolved = true
}
solrSlimTgzSignature {
canBeConsumed = false
canBeResolved = true
}
dockerImage {
canBeResolved = true
}
dockerOfficialFullSmokeCheckImage {
canBeConsumed = false
canBeResolved = true
}
dockerOfficialSlimSmokeCheckImage {
canBeConsumed = false
canBeResolved = true
}
}
ext {
dockerfilesDirPath = "${buildDir}/dockerfiles"
packagingDir = file("${buildDir}/packaging")
}
dependencies {
packaging files(packagingDir) {
builtBy 'assemblePackaging'
}
packagingLocal files("${dockerfilesDirPath}/Dockerfile.local") {
builtBy 'createDockerfileLocal'
}
packagingOfficial files("${dockerfilesDirPath}/Dockerfile.official-full") {
builtBy 'createDockerfileOfficialFull'
}
packagingOfficial files("${dockerfilesDirPath}/Dockerfile.official-slim") {
builtBy 'createDockerfileOfficialSlim'
}
solrFullTgz project(path: ":solr:packaging", configuration: "solrFullTgz")
solrSlimTgz project(path: ":solr:packaging", configuration: "solrSlimTgz")
solrFullTgzSignature project(path: ":solr:packaging", configuration: 'solrFullTgzSignature')
solrSlimTgzSignature project(path: ":solr:packaging", configuration: 'solrSlimTgzSignature')
dockerImage files(imageIdFile) {
builtBy 'dockerBuild'
}
dockerOfficialFullSmokeCheckImage files(imageIdFileOfficial("Full")) {
builtBy 'testBuildDockerfileOfficialFull'
}
dockerOfficialSlimSmokeCheckImage files(imageIdFileOfficial("Slim")) {
builtBy 'testBuildDockerfileOfficialSlim'
}
}
// We're using commons-codec for computing checksums.
buildscript {
repositories {
mavenCentral()
}
dependencies {
classpath "commons-codec:commons-codec:${scriptDepVersions['commons-codec']}"
}
}
def checksum = { file ->
return new DigestUtils(DigestUtils.sha512Digest).digestAsHex(file).trim()
}
task assemblePackaging(type: Sync) {
description = 'Assemble docker scripts and Dockerfile for Solr Packaging'
dependsOn configurations.packagingLocal
from(projectDir, {
include "scripts/**"
include "README.md"
})
from(configurations.packagingLocal, {
include 'Dockerfile.local'
rename {
'Dockerfile'
}
})
into packagingDir
}
task dockerBuild() {
group = 'Docker'
description = 'Build Solr docker image'
// Ensure that the docker image is rebuilt on build-arg changes or changes in the docker context
inputs.properties([
baseDockerImage: baseDockerImage
])
var solrTgzConfiguration = isImageSlim() ? configurations.solrSlimTgz : configurations.solrFullTgz
inputs.files(solrTgzConfiguration)
inputs.property("isSlimImage", isImageSlim())
dependsOn(solrTgzConfiguration)
doLast {
exec {
standardInput = solrTgzConfiguration.singleFile.newDataInputStream()
commandLine "docker", "build",
"-f", "solr-${ -> project.version }${dockerImageDistSuffix}/docker/Dockerfile",
"--iidfile", imageIdFile,
"--build-arg", "BASE_IMAGE=${ -> inputs.properties.baseDockerImage}",
"-"
}
}
// Print information on the image after it has been created
doLast {
def dockerImageId = file(imageIdFile).text
project.logger.lifecycle("Solr Docker Image Created")
project.logger.lifecycle("\tID: \t${ -> dockerImageId }")
project.logger.lifecycle("\tBase Image: \t${ -> baseDockerImage }")
project.logger.lifecycle("\tSolr Version: \t${ -> project.version }")
project.logger.lifecycle("\tSolr Distribution: \t${isImageSlim() ? "Slim" : "Full"}")
}
outputs.files(imageIdFile)
}
task dockerTag(dependsOn: tasks.dockerBuild) {
group = 'Docker'
description = 'Tag Solr docker image'
def dockerImageIdFile = file(imageIdFile)
// Ensure that the docker image is re-tagged if the image ID or desired tag changes
inputs.properties([
dockerImageName: dockerImageName,
])
inputs.file(dockerImageIdFile)
doLast {
def dockerImageId = dockerImageIdFile.text
exec {
commandLine "docker", "tag", dockerImageId, inputs.properties.dockerImageName
}
// Print information on the image after it has been created
project.logger.lifecycle("Solr Docker Image Tagged")
project.logger.lifecycle("\tID: \t$dockerImageId")
project.logger.lifecycle("\tTag: \t$dockerImageName")
}
}
task testDocker(type: TestDockerImageTask, dependsOn: tasks.dockerBuild) {
group = 'Docker'
description = 'Test Solr docker image built from the local Dockerfile'
idFile = tasks.dockerBuild.outputs.files.singleFile
outputDir = file("$buildDir/test-results")
// include/exclude options are designed for people who know their customizations will break some tests
testCasesInclude.value(Arrays.asList(propertyOrEnvOrDefault("solr.docker.tests.include", "SOLR_DOCKER_TESTS_INCLUDE", ",").split(",")))
var excludeCases = new HashSet(Arrays.asList(propertyOrEnvOrDefault("solr.docker.tests.exclude", "SOLR_DOCKER_TESTS_EXCLUDE", ",").split(",")))
if (isImageSlim()) {
// The slim image does not contain the prometheus-exporter.
excludeCases.add("prometheus-exporter-standalone")
excludeCases.add("prometheus-exporter-cloud")
}
testCasesExclude.value(excludeCases)
}
task dockerPush(dependsOn: tasks.dockerTag) {
group = 'Docker'
description = 'Push Solr docker image'
// Ensure that the docker image is re-pushed if the image ID or tag changes
inputs.properties([
dockerImageName: dockerImageName,
])
inputs.file(imageIdFile)
// We don't want to push a docker image unless the tests have passed
mustRunAfter tasks.testDocker
doLast {
exec {
commandLine "docker", "push", dockerImageName
}
// Print information on the image after it has been created
project.logger.lifecycle("Solr Docker Image Pushed: \t$dockerImageName")
}
}
// One task to build and tag a Solr docker image
task docker {
dependsOn tasks.dockerBuild, tasks.dockerTag
}
ext {
// Filters/Patterns for re-use in multiple "template" related tasks
commentFilter = { line ->
if (line.startsWith('#-#')) {
return null;
}
return line;
}
propReplacePattern = Pattern.compile('_REPLACE_((_|\\p{Upper})+)_')
}
task createBodySnippetDockerfile(type: Copy) {
from 'templates/Dockerfile.body.template'
into "$buildDir/snippets/"
outputs.file("$buildDir/snippets/Dockerfile.body.snippet")
rename { name -> name.replace("template","snippet") }
filteringCharset 'UTF-8'
// NOTE: The only "templating" the Dockerfile.body supports is removing comments.
//
// Any situation where it feel appropriate to add variable substitution should be reconsidered, and probably
// implemented as either a build-arg or as a variable expanded in the header snippets (or both)
filter( commentFilter )
}
ext {
// NOTE: 'props' are variables that will be replaced in the respective templates,
// and they must consist solely of characters matching the regex '(_|\\p{Upper})+'.
// They may only be used in ARG and FROM lines, via the syntax: '_REPLACE_FOO_'
// where 'FOO' is the key used in 'props' (NOTE the leading and trailing literal '_' characters)
dfLocalDetails = [
name: 'Local',
template: 'local',
desc: 'Dockerfile used to create local Solr docker images directly from Solr release tgz file',
// NOTE: There should be no reason for Dockerfile.local to include unique values
//
// Values identical in both Dockerfiles should use consistent names in both templates and
// be defined in the task creation.
props: [:]
]
dfOfficialFullDetails = [
name: 'OfficialFull',
template: 'official',
desc: 'Dockerfile used to create official Full Solr docker images published to hub.docker.io',
props: [
// NOTE: Only include values here that are distinct and unique to the Official Dockerfiles
//
// Values identical in both Dockerfiles should use consistent names in both templates and
// be defined in the task creation
// NOTE: SHA is lazy computed...
'SOLR_TGZ_SHA': "${ -> checksum(configurations.solrFullTgz.singleFile) }",
'RELEASE_MANAGER_GPG_FINGERPRINT': "${ -> releaseGpgFingerprint}",
'SOLR_DIST': ""
]
]
dfOfficialSlimDetails = [
name: 'OfficialSlim',
template: 'official',
desc: 'Dockerfile used to create official Slim Solr docker images published to hub.docker.io',
props: [
// NOTE: Only include values here that are distinct and unique to the Official Dockerfiles
//
// Values identical in both Dockerfiles should use consistent names in both templates and
// be defined in the task creation
// NOTE: SHA is lazy computed...
'SOLR_TGZ_SHA': "${ -> checksum(configurations.solrSlimTgz.singleFile) }",
'RELEASE_MANAGER_GPG_FINGERPRINT': "${ -> releaseGpgFingerprint}",
'SOLR_DIST': "-slim"
]
]
}
/*
This section creates the following gradle tasks:
- createDockerfileOfficialFull
- createDockerfileOfficialSlim
- createDockerfileLocal
Both will create a self-standing Dockerfile that can be used to build a Solr image.
These are templated using a unique header file for each Dockerfile and the same body template for the logic on installing Solr.
These templates can be found in the templates/ directory.
The snippets of each section (header and body) are saved to build/snippets after they are templated and before they are combined.
The final Dockerfiles are merely the snippet headers combined with the snippet body.
*/
[ dfLocalDetails, dfOfficialFullDetails, dfOfficialSlimDetails ].each{ details ->
def fileName = "Dockerfile.${ -> details.name.replaceAll("(.)([A-Z])", "\$1-\$2").toLowerCase(Locale.ROOT) }"
def outFile = file("$dockerfilesDirPath/${fileName}")
tasks.create("createDockerfile${details.name}", Copy) {
description "Creates ${details.desc}"
def props = [
// Values defined here should be common (and consistent) across both Dockerfiles
'BASE_IMAGE': baseDockerImage,
'SOLR_VERSION': "${ -> project.version}",
* : details.props
]
dependsOn tasks.createBodySnippetDockerfile
inputs.properties(props)
inputs.file("$buildDir/snippets/Dockerfile.body.snippet")
outputs.file(outFile)
from "templates/Dockerfile.${details.template}.header.template"
into "$buildDir/snippets/"
rename { fileName + ".header.snippet" }
filteringCharset 'UTF-8'
filter( commentFilter )
filter( { line ->
if ( line.startsWith("FROM ") || line.startsWith("ARG ") ) {
Matcher matcher = project.ext.propReplacePattern.matcher(line);
StringBuilder sb = new StringBuilder();
if (matcher.find()) {
String key = matcher.group(1);
if (null == key || key.isEmpty() || ( ! props.containsKey(key) ) ) {
throw new GradleException("Line contains invalid REPLACE variable (" + key + "): " + line);
}
matcher.appendReplacement(sb, props.get(key) );
}
matcher.appendTail(sb);
return sb.toString();
}
return line;
})
doLast {
outFile.withWriter('UTF-8') { writer ->
files("$buildDir/snippets/${fileName}.header.snippet",
"$buildDir/snippets/Dockerfile.body.snippet").each { snippet ->
snippet.withReader('UTF-8') { reader ->
writer << reader
}
}
}
}
}
}
assemblePackaging.dependsOn tasks.createDockerfileLocal
tasks.createDockerfileOfficialFull.dependsOn configurations.solrFullTgz // to lazy compute SHA
tasks.createDockerfileOfficialSlim.dependsOn configurations.solrSlimTgz // to lazy compute SHA
tasks.createDockerfileOfficialSlim.mustRunAfter tasks.createDockerfileOfficialFull // Eliminate a weird warning
task createDockerfileOfficial {
dependsOn tasks.createDockerfileOfficialFull
dependsOn tasks.createDockerfileOfficialSlim
}
// sanity check...
if (''.equals(releaseGpgFingerprint)) {
gradle.taskGraph.whenReady { graph ->
if ( graph.hasTask(tasks.createDockerfileOfficialFull) || graph.hasTask(tasks.createDockerfileOfficialSlim) ) {
throw new GradleException("No GPG keyName found, please see help/publishing.txt (GPG key is neccessary to create Dockerfile.official)")
}
}
}
[ 'Full', 'Slim' ].each { variant ->
def lowerVariant = variant.toLowerCase(Locale.ROOT)
tasks.create("testBuildDockerfileOfficial${variant}", Copy) {
description = "Test 'docker build' works with our generated Dockerfile.official-${lowerVariant} using Mocked URLs"
dependsOn tasks.named("createDockerfileOfficial${variant}")
def mockHttpdHome = file("$smokeTestOfficial/$lowerVariant/mock-httpd-home");
inputs.file("$dockerfilesDirPath/Dockerfile.official-$lowerVariant")
outputs.dir(mockHttpdHome)
outputs.file(imageIdFileOfficial(variant))
dependsOn configurations.named("solr${variant}Tgz")
dependsOn configurations.named("solr${variant}TgzSignature")
from configurations.named("solr${variant}Tgz")
from configurations.named("solr${variant}TgzSignature")
into mockHttpdHome
doLast {
// A file to record the container ID of our mock httpd
def mockServerIdFile = file("$smokeTestOfficial/$lowerVariant/dockerfile-mock-artifact-server-cid.txt")
// if we encounter any problems running our test, we'll fill this in and use it to suppress any
// other exceptions we encounter on cleanup...
def mainException = null;
// TODO: setup a little 'suppressOrThrow(Exception)' closure for reuse below....
try {
// run an httpd server to host our artifacts
logger.lifecycle('Running mock HTTPD server our testing...');
exec {
commandLine 'docker', 'run',
'--cidfile', mockServerIdFile,
'--rm',
'-d',
'-v', "${mockHttpdHome.absoluteFile}:/data/${-> project.version}",
'-w', '/data',
'-p', '9876:9876',
'python:3-alpine', 'python', '-m', 'http.server', '9876'
}
try {
def mockServerId = mockServerIdFile.text
def mockServerIpStdOut = new ByteArrayOutputStream()
exec {
commandLine 'docker', 'inspect', "--format={{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}", mockServerId
standardOutput = mockServerIpStdOut
}
def mockServerIp = mockServerIpStdOut.toString().trim()
// *NOW* we can actually run our docker build command...
logger.lifecycle('Running docker build on Dockerfile.official...');
def downloadHost = "mock-downloads.apache.org"
// We are not signing, so do not use an apache.org host, as that will check the
// GPG signature, which will not be created.
if (!project(":solr:distribution").ext.withSignedArtifacts) {
downloadHost = "mock-downloads.test.org"
}
exec {
standardInput = file("${dockerfilesDirPath}/Dockerfile.official-${lowerVariant}").newDataInputStream()
commandLine 'docker', 'build',
'--add-host', "${downloadHost}:${mockServerIp}",
'--no-cache', // force fresh downloads from our current network
"--iidfile", imageIdFileOfficial(variant),
'--build-arg', "SOLR_DOWNLOAD_SERVER=http://${downloadHost}:9876",
'--tag', officialDockerImageName(variant),
'-'
}
def officialDockerImageId = file(imageIdFileOfficial(variant)).text
// Print information on the image after it has been created
project.logger.lifecycle("\"Official $variant\" Solr Docker Image Tagged")
project.logger.lifecycle("\tID: \t$officialDockerImageId")
project.logger.lifecycle("\tTag: \t${officialDockerImageName(variant)}")
} finally {
// Try to shut down our mock httpd server....
if (mockServerIdFile.exists()) {
def mockServerId = mockServerIdFile.text
try {
exec { commandLine 'docker', 'stop', mockServerId }
} catch (Exception e) {
logger.error("Unable to stop docker container ${mockServerId}", e)
if (null != mainException) {
mainException.addSuppressed(e);
} else {
mainException = e;
throw e;
}
} finally {
project.delete(mockServerIdFile)
}
}
}
} catch (Exception e) {
mainException = e
throw e;
}
}
}
}
task testBuildDockerfileOfficial {
dependsOn tasks.testBuildDockerfileOfficialFull
dependsOn tasks.testBuildDockerfileOfficialSlim
}
task testDockerfileOfficialFull(type: TestDockerImageTask, dependsOn: configurations.dockerOfficialFullSmokeCheckImage) {
description = 'Smoke Test Full Solr docker image built from the official Dockerfile'
idFile = file(imageIdFileOfficial("Full"))
outputDir = file("$smokeTestOfficial/full/test-results")
// This test does not respect the include/exclude properties that `testDocker` does.
// All docker tests will be run, no matter the properties specified.
testCasesInclude.empty()
testCasesExclude.empty()
}
task testDockerfileOfficialSlim(type: TestDockerImageTask, dependsOn: configurations.dockerOfficialSlimSmokeCheckImage) {
description = 'Smoke Test Slim Solr docker image built from the official Slim Dockerfile'
idFile = file(imageIdFileOfficial("Slim"))
outputDir = file("$smokeTestOfficial/slim/test-results")
// This test does not respect the include/exclude properties that `testDocker` does.
// All docker tests will be run, no matter the properties specified.
testCasesInclude.empty()
testCasesExclude = ['prometheus-exporter-cloud', 'prometheus-exporter-standalone']
}
task testDockerfileOfficial {
dependsOn tasks.testDockerfileOfficialFull
dependsOn tasks.testDockerfileOfficialSlim
}
// Re-usable class for running tests...
abstract class TestDockerImageTask extends DefaultTask {
// Ensure that the docker image is re-tested if the image ID changes or the test files change
@InputFile
abstract public RegularFileProperty getIdFile()
// Ensure that the docker image is re-tested if the shared test library changes
@InputFile
File sharedTestLibraryFile = project.file("tests/shared.sh")
@InputDirectory
File sourceDir = project.file("tests/cases")
@Input
SetProperty<String> testCasesInclude = project.objects.setProperty(String)
@Input
SetProperty<String> testCasesExclude = project.objects.setProperty(String)
@OutputDirectory
abstract public DirectoryProperty getOutputDir()
@Inject
abstract public WorkerExecutor getWorkerExecutor();
public static interface SingleTestParameters extends WorkParameters {
// NOTE: we explicitly don't use DirectoryProperty here because the way WorkerExecutor serializes the params
// causes weird "wrapped" objects to come back that don't work when you try to call `.getAsFile()` or `.getAsFile().getPath()`
Property<String> getTestName();
Property<String> getTestDir();
Property<String> getWorkDir();
Property<String> getImageId();
}
public abstract static class SingleTestAction implements WorkAction<SingleTestParameters> {
@Inject
abstract public ExecOperations getExec();
@Override
public void execute() {
def testCaseName = getParameters().getTestName().get()
def testCaseDir = getParameters().getTestDir().get()
def testCaseWorkDir = getParameters().getWorkDir().get()
def testCaseBuildDir = "${testCaseWorkDir}/build_dir"
System.out.println("Starting Solr Docker test: ${testCaseName}")
def res = getExec().exec {
// we'll handle it ourselves so we can report the details of which test failed
ignoreExitValue true
environment "DEBUG", "true"
environment "TEST_DIR", testCaseDir
environment "BUILD_DIR", testCaseBuildDir
standardOutput new FileOutputStream("${testCaseWorkDir}/test.std.out")
errorOutput new FileOutputStream("${testCaseWorkDir}/test.err.out")
commandLine "bash", "${testCaseDir}/test.sh", getParameters().getImageId().get()
}
def ev = res.getExitValue()
if (0 != ev) {
throw new GradleException("Docker test failure=${ev}: Test: ${testCaseName} Output: ${testCaseWorkDir}");
} else {
System.out.println("Completed Solr Docker test: ${testCaseName}")
}
}
}
@TaskAction
public void runTests() {
// no easy way to control the amount of parallelization of the work queue (independent of `org.gradle.workers.max`)
// so the best we can do (simply) is a boolean setting that controls if we `await()` after each `submit()`
def workQueue = getWorkerExecutor().noIsolation();
def runConcurrentTests = project.propertyOrEnvOrDefault("solr.docker.tests.concurrent", "SOLR_DOCKER_TESTS_CONCURRENT", "false").toBoolean()
def imageId = idFile.getAsFile().get().text
// Print information on the image before it is tested
logger.lifecycle("Testing Solr Image ID: $imageId ${ -> runConcurrentTests ? '(concurrently)' : '(sequentially)'}")
def includes = testCasesInclude.get()
def excludes = testCasesExclude.get()
// "Run" each of the test cases
sourceDir.eachFile { file ->
def testName = file.getName()
def outDir = outputDir.get()
def testCaseWorkDir = outDir.dir(testName)
// If specific tests are specified, only run those. Otherwise run all that are not ignored.
def runTest = !includes.isEmpty() ? includes.contains(testName) : !excludes.contains(testName)
if (runTest) {
project.mkdir testCaseWorkDir
def paramSetup = { params ->
params.getTestName().set(testName)
params.getTestDir().set(file.getPath())
params.getWorkDir().set(testCaseWorkDir.getAsFile().getPath())
params.getImageId().set(imageId)
}
workQueue.submit(SingleTestAction.class, paramSetup);
if (! runConcurrentTests) {
workQueue.await();
}
}
}
}
}