| /* |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * "License"); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| import groovy.json.JsonOutput |
| import java.net.URI |
| |
| buildscript { |
| repositories { |
| mavenCentral() |
| } |
| } |
| |
| apply plugin: 'java' |
| apply plugin: 'groovy' |
| |
| def PACKAGES_GROUP = 'package' |
| |
| final String VERBOSE = "verbose" |
| final String DEPS_BUILD = "buildwithdeps" |
| final boolean buildDependencies = false || System.getProperty(DEPS_BUILD) == 'true' |
| final String BOM = System.getProperty('bomfile') ?: "$rootDir/bigtop.bom" |
| def final config = new ConfigSlurper().parse(new URL("file:$BOM")) |
| |
| config.bigtop.builddir = projectDir.absolutePath + "/build" |
| config.bigtop.outputdir = projectDir.absolutePath + "/output" |
| config.bigtop.distdir = projectDir.absolutePath + "/dist" |
| config.bigtop.dldir = projectDir.absolutePath + "/dl" |
| |
| def bomVersions = [] |
| |
| def final BASE_DIR = projectDir.absolutePath |
| def final REPO_DIR = "$BASE_DIR/bigtop-repos" |
| def final BUILD_DIR = config.bigtop.builddir |
| def final OUTPUT_DIR = config.bigtop.outputdir |
| def final DIST_DIR = config.bigtop.distdir |
| def final DL_DIR = config.bigtop.dldir |
| |
| // List of tasks derived from the BOM file is used to preserve the natural order |
| def bomTasks = [] |
| |
| // Package building and logic around it |
| |
| def touchTargetFile = { fileName -> |
| // to comply with make build |
| ant.touch(file:fileName,mkdirs:true) |
| } |
| def ifExists = { url -> |
| if (url == null) return |
| URLConnection uCon = new URL(url).openConnection() |
| try { |
| return (uCon as HttpURLConnection).responseCode == 200 |
| } catch(Exception e) { |
| return false |
| } |
| } |
| def safeDelete = { fileName -> |
| exec { |
| workingDir '.' |
| commandLine 'rm', '-rf', fileName |
| } |
| } |
| def getDate() { |
| new Date().format('E, dd MMM yyyy HH:mm:ss Z') |
| } |
| /** |
| * To avoid breaking the compat with existing packages let's use the old style names |
| */ |
| def toOldStyleName = { newname -> |
| newname.toUpperCase().replaceAll("\\-", "_") |
| } |
| def setDefaults = { comp -> |
| // The closure parameter is a component in the config.bigtop.components |
| if (!comp.pkg) { |
| comp.pkg = comp.name |
| } |
| if (!comp.version.pkg) { |
| comp.version.pkg = comp.version.base |
| } |
| } |
| def devNull = new OutputStream() { |
| @Override |
| public void write(int b) {} |
| } |
| def nativePackaging = { |
| def result = exec { |
| commandLine "/bin/bash", "-c", |
| """dpkg-query -S /bin/sh && exit 1 |
| rpm -qf /bin/sh && exit 2 |
| exit 0 |
| """ |
| ignoreExitValue true |
| errorOutput devNull |
| standardOutput devNull |
| } |
| [false, [pkg:"deb", repo:"apt"], [pkg:"rpm", repo:"yum"]][result.getExitValue()] |
| }.call() |
| |
| def generateDistributionsFile = { file -> |
| def writer = new File(file); |
| writer.text ="""Origin: Bigtop |
| Label: Bigtop |
| Suite: stable |
| Codename: bigtop |
| Version: ${config.bigtop.version} |
| Architectures: amd64 ppc64el arm64 source |
| Components: contrib |
| Description: Bigtop |
| """; |
| } |
| |
| task "packages-help" (description: "All package build related tasks information", group: PACKAGES_GROUP) doLast { |
| config.bigtop.components.each { label, comp -> |
| println (comp.name + "\n\t[" + tasks.findAll { alltask -> alltask.name.startsWith(comp.name)}*.name.join(", ") + "]") |
| } |
| } |
| |
| task "bom-json" (description: "List the components of the stack in json format") doLast { |
| def componentObjects = config.bigtop.components.sort().collect { |
| setDefaults(it.value) |
| [ |
| name: [ |
| project: it.value.name, |
| pkg: it.value.pkg, |
| relNotes: it.value.relNotes, |
| packaging: it.value.packaging, |
| ], |
| tarball: [ |
| destination: it.value.tarball.destination, |
| source: it.value.tarball.source, |
| ], |
| url: [ |
| site: it.value.url.site, |
| archive: it.value.url.archive, |
| ], |
| version: [ |
| base: it.value.version.base, |
| pkg: it.value.version.pkg, |
| release: it.value.version.release, |
| ], |
| git: [ |
| repo: it.value.git.repo, |
| ref: it.value.git.ref, |
| dir: it.value.git.dir, |
| ] |
| ] |
| } |
| def fullDefinition = [ |
| version: config.bigtop.version, |
| components: componentObjects |
| ] |
| def json = JsonOutput.toJson(fullDefinition) |
| println JsonOutput.prettyPrint(json) |
| } |
| |
| task "all-components" (description: "List the components of the stack") doLast { |
| println "${project.name} ${config.bigtop.version} stack includes the following components" |
| config.bigtop.components.sort().each { label, comp -> |
| println sprintf('\t%1$s %2$s%3$s', comp.name.padRight(20), comp.version.base.padLeft(15), |
| comp.packaging ? " (${comp.packaging.toUpperCase()} only)" : '') |
| } |
| } |
| |
| def genTasks = { target -> |
| Task t = task "${target}-download" (dependsOn: "${target}_vardefines", |
| description: "Download $target artifacts", |
| group: PACKAGES_GROUP) doLast { |
| |
| def final TARBALL_SRC = config.bigtop.components[target].tarball.source |
| def final TARBALL_DST = config.bigtop.components[target].tarball.destination |
| def final DOWNLOAD_DST = config.bigtop.components[target].downloaddst |
| def final DOWNLOAD_URL = config.bigtop.components[target].downloadurl |
| |
| def final GIT_REPO = project.hasProperty('git_repo') ? project.property('git_repo') : config.bigtop.components[target].git.repo |
| def final GIT_REF = project.hasProperty('git_ref') ? project.property('git_ref') : config.bigtop.components[target].git.ref |
| def final GIT_DIR = project.hasProperty('git_dir') ? project.property('git_dir') : config.bigtop.components[target].git.dir |
| def final GIT_COMMIT_HASH = project.hasProperty('git_commit_hash') ? project.property('git_commit_hash') : config.bigtop.components[target].git.commit_hash |
| def final GIT_USER_NAME = config.bigtop.components[target].git.user ?: config.bigtop.git.user |
| def final GIT_ACCESS_TOKEN = config.bigtop.components[target].git.token ?: config.bigtop.git.token |
| |
| if (!DOWNLOAD_URL && !(GIT_REPO && GIT_REF)) |
| return |
| |
| mkdir(DL_DIR) |
| if (TARBALL_DST?.isEmpty() || new File(DOWNLOAD_DST)?.exists() || new File(config.bigtop.components[target].targetdl)?.exists()) { |
| println "\tFile $DOWNLOAD_DST appears to be already downloaded. Exiting..." |
| return |
| } |
| if (GIT_REPO && GIT_REF) { |
| def dir = GIT_DIR |
| if (dir == null || dir.isEmpty()) { |
| dir = TARBALL_SRC.substring(0, TARBALL_SRC.lastIndexOf(".t")) |
| } |
| delete("${DL_DIR}/${dir}") |
| def depth = "--depth 1" |
| if ( GIT_COMMIT_HASH ) { |
| depth = "" |
| } |
| if (GIT_USER_NAME && GIT_ACCESS_TOKEN) { |
| def uri = new URI(GIT_REPO) |
| if (!GIT_REPO.toLowerCase().startsWith("http")) { |
| println("\tGit credentials supported only with 'http' or 'https' schemes"); |
| return |
| } |
| uri = new URI(uri.getScheme(), |
| "${GIT_USER_NAME}:${GIT_ACCESS_TOKEN}".toString(), // userInfo |
| uri.getHost(), uri.getPort(), uri.getPath(), uri.getQuery(), |
| uri.getFragment()) |
| exec { |
| workingDir DL_DIR |
| commandLine "git clone ${depth} --branch $GIT_REF ${uri} ${dir}".split() |
| errorOutput devNull |
| standardOutput devNull |
| } |
| } else { |
| exec { |
| workingDir DL_DIR |
| commandLine "git clone ${depth} --branch $GIT_REF ${GIT_REPO} ${dir}".split() |
| errorOutput devNull |
| standardOutput devNull |
| } |
| } |
| exec { |
| workingDir "$DL_DIR/${dir}" |
| commandLine "git submodule update --init".split() |
| errorOutput devNull |
| standardOutput devNull |
| } |
| if ( GIT_COMMIT_HASH ) { |
| exec { |
| workingDir "$DL_DIR/${dir}" |
| commandLine "git reset $GIT_COMMIT_HASH --hard".split() |
| errorOutput devNull |
| standardOutput devNull |
| } |
| } |
| delete("${DL_DIR}/${dir}/.git") |
| exec { |
| workingDir DL_DIR |
| commandLine "tar -czf ${TARBALL_DST} ${dir}".split() |
| } |
| delete("${DL_DIR}/${dir}") |
| } |
| else { |
| download { |
| src DOWNLOAD_URL |
| dest DOWNLOAD_DST |
| retries 3 |
| } |
| } |
| touchTargetFile(config.bigtop.components[target].targetdl) |
| } |
| task "${target}-tar" (dependsOn: ["${target}_vardefines", "${target}-download"], |
| description: "Preparing a tarball for $target artifacts", |
| group: PACKAGES_GROUP) doLast { |
| if (new File(config.bigtop.components[target].targettar)?.exists()) { |
| println "\tNothing to do. Exiting..." |
| return |
| } |
| def final TAR_DIR = config.bigtop.components[target].tardir |
| def final TARBALL_SRC = config.bigtop.components[target].tarball.source ?: "" |
| def final TARBALL_DST = config.bigtop.components[target].tarball.destination |
| def final DOWNLOAD_DST = config.bigtop.components[target].downloaddst ?: "" |
| def final SEED_TAR = config.bigtop.components[target].seedtar |
| |
| safeDelete(TAR_DIR); mkdir(TAR_DIR) |
| |
| if (TARBALL_SRC.isEmpty() || TARBALL_SRC.endsWith('.zip')) { |
| if (TARBALL_SRC.isEmpty()) { |
| // if no tar file needed (i.e. bigtop-utils) |
| // create some contents to pack later |
| copy { |
| from 'LICENSE' |
| into TAR_DIR |
| } |
| } else { |
| // i.e. a ZIP file |
| exec { |
| workingDir TAR_DIR |
| commandLine "unzip $DOWNLOAD_DST".split(' ') |
| } |
| def unpacked = new File(TAR_DIR) |
| // check wether we have to move contents one level up |
| if (unpacked.list().size() == 1) { |
| def TOP_LEVEL_DIR = unpacked.list()[0] |
| new File("$TAR_DIR/$TOP_LEVEL_DIR").list({ d, f -> |
| (f != "." && f != "..")}).each { f -> |
| new File("$TAR_DIR/$TOP_LEVEL_DIR/$f").renameTo("$TAR_DIR/$f") |
| } |
| safeDelete(TOP_LEVEL_DIR) |
| } |
| } |
| // create SEED_TAR |
| exec { |
| workingDir "$TAR_DIR/.." |
| commandLine "tar -czf $SEED_TAR ${new File("$TAR_DIR/..").list().join(' ')}".split(' ') |
| } |
| } else { |
| println "Copy $DOWNLOAD_DST to $SEED_TAR" |
| copy { |
| from DOWNLOAD_DST |
| into config.bigtop.builddir + "/$target/tar/" |
| rename TARBALL_DST, SEED_TAR |
| } |
| } |
| touchTargetFile(config.bigtop.components[target].targettar) |
| } |
| |
| // Keeping the reference to deb task to be used later for correct sequencing |
| Task tdeb = task "$target-deb"(dependsOn: "${target}-sdeb", |
| description: "Building DEB for $target artifacts", |
| group: PACKAGES_GROUP) doLast { |
| if (new File(config.bigtop.components[target].targetdeb)?.exists()) { |
| println "\tNothing to do. Exiting..." |
| return |
| } |
| def final BIGTOP_BUILD_STAMP = System.getenv('BIGTOP_BUILD_STAMP') ?: |
| config.bigtop.components[target].version.release |
| def final PKG_NAME = config.bigtop.components[target].pkg |
| def final PKG_VERSION = config.bigtop.components[target].version.pkg |
| def final PKG_OUTPUT_DIR = config.bigtop.components[target].outputdir |
| def final BASE_VERSION = config.bigtop.components[target].version.base |
| def final SRCDEB = "${PKG_NAME}_$PKG_VERSION-${BIGTOP_BUILD_STAMP}.dsc" |
| def final HADOOP_VERSION = config.bigtop.components["hadoop"].version.pkg |
| |
| exec { |
| workingDir PKG_OUTPUT_DIR |
| commandLine "dpkg-source -x $SRCDEB".split(' ') |
| } |
| // Order of debuild parameters is important; hence specifying explicitely rather |
| // than in an array of args |
| def command = """debuild \ |
| --preserve-envvar PATH \ |
| --preserve-envvar MAVEN3_HOME \ |
| --preserve-envvar MAVEN_OPTS \ |
| --preserve-envvar JAVA_HOME \ |
| --preserve-envvar BIGTOP_JDK \ |
| --set-envvar=HADOOP_VERSION=$HADOOP_VERSION \ |
| --set-envvar=${toOldStyleName(target)}_BASE_VERSION=$BASE_VERSION \ |
| --set-envvar=${toOldStyleName(target)}_VERSION=$PKG_VERSION \ |
| --set-envvar=${toOldStyleName(target)}_RELEASE=$BIGTOP_BUILD_STAMP \ |
| -uc -us -b |
| """ |
| exec { |
| workingDir "$PKG_OUTPUT_DIR/$PKG_NAME-$PKG_VERSION" |
| commandLine command.split(' ') |
| } |
| exec { |
| workingDir "$PKG_OUTPUT_DIR" |
| commandLine 'rm','-rf',"$PKG_NAME-$PKG_VERSION" |
| } |
| touchTargetFile(config.bigtop.components[target].targetdeb) |
| } |
| // Guarantee that tasks are ran in the order set by BOM file dependencies section |
| if (buildDependencies) |
| config.bigtop.dependencies.each { dependsOn, dependees -> |
| if (dependees.contains(tdeb.name-"-deb")) tdeb.dependsOn "$dependsOn-deb" |
| } |
| // If dependencies aren't explicitly set, the default build order is defined |
| // by the sequence of the components in the BOM file |
| if (!config.bigtop.dependencies && bomTasks.size() > 0) |
| tdeb.mustRunAfter "${bomTasks.get(bomTasks.size() - 1)}-deb".toLowerCase() |
| task "$target-sdeb" (dependsOn: ["${target}_vardefines", "${target}-tar"], |
| description: "Building SDEB for $target artifacts", |
| group: PACKAGES_GROUP |
| ) doLast { |
| if (new File(config.bigtop.components[target].targetsdeb)?.exists()) { |
| println "\tNothing to do. Exiting..." |
| return |
| } |
| def final BIGTOP_BUILD_STAMP = System.getenv('BIGTOP_BUILD_STAMP') ?: |
| config.bigtop.components[target].version.release |
| def final TARBALL_SRC = config.bigtop.components[target].tarball.source |
| def final TARBALL_DST = config.bigtop.components[target].tarball.destination ?: TARBALL_SRC |
| def final PKG_BUILD_DIR = config.bigtop.components[target].builddir |
| def final NAME = config.bigtop.components[target].name |
| def final PKG_NAME = config.bigtop.components[target].pkg |
| def final SEED_TAR = config.bigtop.components[target].seedtar |
| def final PKG_VERSION = config.bigtop.components[target].version.pkg |
| def final PKG_OUTPUT_DIR = config.bigtop.components[target].outputdir |
| delete ("$PKG_BUILD_DIR/deb") |
| def final DEB_BLD_DIR = "$PKG_BUILD_DIR/deb/$NAME-${PKG_VERSION}" |
| def final DEB_PKG_DIR = "$PKG_BUILD_DIR/deb/$PKG_NAME-${PKG_VERSION}-${BIGTOP_BUILD_STAMP}" |
| mkdir (DEB_BLD_DIR) |
| copy { |
| from SEED_TAR |
| into "$PKG_BUILD_DIR/deb/" |
| rename TARBALL_DST, "${PKG_NAME}_${PKG_VERSION}.orig.tar.gz" |
| } |
| exec { |
| workingDir DEB_BLD_DIR |
| commandLine "tar --strip-components 1 -xf $DEB_BLD_DIR/../${PKG_NAME}_${PKG_VERSION}.orig.tar.gz".split(' ') |
| } |
| copy { |
| from fileTree("${BASE_DIR}/bigtop-packages/src/deb/$NAME") { |
| include '**/*' |
| } |
| into "$DEB_BLD_DIR/debian" |
| } |
| copy { |
| from "${BASE_DIR}/bigtop-packages/src/templates/init.d.tmpl" |
| into "$DEB_BLD_DIR/debian" |
| } |
| copy { |
| from "${BASE_DIR}/bigtop-packages/src/scripts" |
| into "$DEB_BLD_DIR/debian" |
| } |
| copy { |
| from "${BASE_DIR}/bigtop-packages/src/extensions" |
| into "$DEB_BLD_DIR/debian" |
| } |
| copy { |
| from fileTree("$BASE_DIR/bigtop-packages/src/common/$NAME") { |
| include '**/*' |
| } |
| into "$DEB_BLD_DIR/debian" |
| } |
| // Prepeare bom file with all the versions |
| def bomWriter = new File("$DEB_BLD_DIR/debian/bigtop.bom").newWriter() |
| bomVersions.each { bomWriter << "$it\n"} |
| bomWriter.close() |
| // Create changelog |
| def changelog = new File("$DEB_BLD_DIR/debian/changelog").newWriter() |
| changelog << "$PKG_NAME ($PKG_VERSION-$BIGTOP_BUILD_STAMP) stable; urgency=low\n" |
| changelog << " Clean build\n" |
| changelog << " -- Bigtop <dev@bigtop.apache.org> ${getDate()}\n" |
| changelog.close() |
| |
| // Move patches and create "series" |
| def patches = new File( "$DEB_BLD_DIR/debian").list({ d, f -> f ==~ /patch.*diff/}).sort() |
| if (patches.size() > 0) { |
| mkdir("$DEB_BLD_DIR/debian/patches") |
| def seriesWriter = new File("$DEB_BLD_DIR/debian/patches/series").newWriter() |
| patches.each { f -> |
| def res = new File("$DEB_BLD_DIR/debian/$f").renameTo( "$DEB_BLD_DIR/debian/patches/$f") |
| seriesWriter << "$f\n" |
| } |
| seriesWriter.close() |
| } |
| // Deleting obsolete files |
| delete fileTree(dir: "$DEB_BLD_DIR/debian", includes: ['*.ex', '*.EX', '*.~']) |
| // Creating source package |
| exec { |
| workingDir DEB_BLD_DIR |
| commandLine "dpkg-buildpackage -uc -us -sa -S".split(' ') |
| } |
| mkdir(PKG_OUTPUT_DIR) |
| copy { |
| from fileTree(dir: "$DEB_PKG_DIR/..", includes: ['*.dsc', '*.diff.gz', '*.debian.tar.gz', '*.debian.tar.xz', "*_source.changes", "*.orig.tar.gz" ]) |
| into PKG_OUTPUT_DIR |
| } |
| safeDelete(DEB_BLD_DIR) |
| touchTargetFile(config.bigtop.components[target].targetsdeb) |
| } |
| |
| // Keeping the reference to task to be used later for correct sequencing |
| Task trpm = task "$target-rpm" (dependsOn: ["${target}-srpm"], |
| description: "Building RPM for $target artifacts", |
| group: PACKAGES_GROUP) doLast { |
| if (new File(config.bigtop.components[target].targetrpm)?.exists()) { |
| println "\tNothing to do. Exiting..." |
| return |
| } |
| def final BIGTOP_BASE_VERSION = "${config.bigtop.base_version}" |
| |
| def final PKG_NAME_SUFFIX = config.bigtop.components[target].rpm_pkg_suffix |
| def RPM_PKG_NAME_SUFFIX = PKG_NAME_SUFFIX |
| def GRADLE_PKG_NAME_SUFFIX = PKG_NAME_SUFFIX |
| if (!project.hasProperty("pkgSuffix") || !PKG_NAME_SUFFIX) { |
| RPM_PKG_NAME_SUFFIX = "%{nil}" |
| GRADLE_PKG_NAME_SUFFIX = "" |
| } |
| |
| def final BIGTOP_BUILD_STAMP = System.getenv('BIGTOP_BUILD_STAMP') ?: |
| config.bigtop.components[target].version.release |
| def final PKG_BUILD_DIR = config.bigtop.components[target].builddir |
| def final NAME = config.bigtop.components[target].name |
| def final PKG_NAME = config.bigtop.components[target].pkg |
| def final PKG_OUTPUT_DIR = config.bigtop.components[target].outputdir |
| def final PKG_VERSION = config.bigtop.components[target].version.pkg |
| def final BASE_VERSION = config.bigtop.components[target].version.base |
| def final HADOOP_VERSION = config.bigtop.components["hadoop"].version.pkg |
| def RELEASE_DIST = "rpmbuild --eval '%{?dist}' 2>/dev/null".execute().text.trim().replaceAll("'",'') |
| def SRCRPM = "$PKG_OUTPUT_DIR/${PKG_NAME}${GRADLE_PKG_NAME_SUFFIX}-${PKG_VERSION}-$BIGTOP_BUILD_STAMP${RELEASE_DIST}.src.rpm" |
| |
| def final DO_MAVEN_DEPLOY = project.findProperty('doMavenDeploy') == 'true' |
| def final MAVEN_DEPLOY_SOURCE = project.findProperty('mavenDeploySource') == 'true' |
| def final MAVEN_REPO_ID = project.hasProperty('mavenRepoId') ? project.property('mavenRepoId') : 'default' |
| def final MAVEN_REPO_URI = project.hasProperty('mavenRepoUri') ? project.property('mavenRepoUri') : null |
| |
| def final PARENT_DIR = project.hasProperty("parentDir") ? project.property('parentDir') : "%{nil}" |
| def FULL_PARENT_DIR = "${PARENT_DIR}" |
| if (PARENT_DIR != "%{nil}") { |
| FULL_PARENT_DIR = "${PARENT_DIR}/${BIGTOP_BASE_VERSION}" |
| } |
| |
| def command = [ |
| '--define', "_topdir $PKG_BUILD_DIR/rpm/", |
| '--define', "${NAME}_base_version $BASE_VERSION", |
| '--define', "hadoop_version ${HADOOP_VERSION}", |
| '--define', "${NAME}_version ${PKG_VERSION}", |
| '--define', "${NAME}_release ${BIGTOP_BUILD_STAMP}%{?dist}", |
| '--rebuild', SRCRPM, |
| '--define', "do_maven_deploy ${DO_MAVEN_DEPLOY}", |
| '--define', "maven_deploy_source ${MAVEN_DEPLOY_SOURCE}", |
| "--define", "parent_dir ${FULL_PARENT_DIR}", |
| "--define", "bigtop_base_version ${BIGTOP_BASE_VERSION}", |
| "--define", "pkg_name_suffix ${RPM_PKG_NAME_SUFFIX}", |
| ] |
| |
| if (MAVEN_REPO_URI != null) { |
| command.addAll('--define', "maven_repo_uri ${MAVEN_REPO_URI}") |
| command.addAll('--define', "maven_repo_id ${MAVEN_REPO_ID}") |
| } |
| |
| exec { |
| workingDir BASE_DIR |
| executable 'rpmbuild' |
| environment QA_RPATHS: 0x0002 |
| args command |
| } |
| copy { |
| from fileTree("$PKG_BUILD_DIR/rpm/RPMS") { |
| include '**/*' |
| } |
| into PKG_OUTPUT_DIR |
| } |
| touchTargetFile(config.bigtop.components[target].targetrpm) |
| } |
| // Guarantee that tasks are ran in the order set by BOM file dependencies section |
| if (buildDependencies) |
| config.bigtop.dependencies.each { dependsOn, dependees -> |
| if (dependees.contains(trpm.name - "-rpm")) trpm.dependsOn "$dependsOn-rpm" |
| } |
| // If dependencies aren't explicitly set, the default build order is defined |
| // by the sequence of the components in the BOM file |
| if (!config.bigtop.dependencies && bomTasks.size() > 0) |
| trpm.mustRunAfter "${bomTasks.get(bomTasks.size() - 1)}-rpm".toLowerCase() |
| task "$target-srpm" (dependsOn: ["${target}_vardefines" , "${target}-tar"], |
| description: "Building SRPM for $target artifacts", |
| group: PACKAGES_GROUP) doLast { |
| if (new File(config.bigtop.components[target].targetsrpm)?.exists()) { |
| println "\tNothing to do. Exiting..." |
| return |
| } |
| def final BIGTOP_BASE_VERSION = "${config.bigtop.base_version}" |
| |
| def final PKG_NAME_SUFFIX = config.bigtop.components[target].rpm_pkg_suffix |
| def RPM_PKG_NAME_SUFFIX = PKG_NAME_SUFFIX |
| def GRADLE_PKG_NAME_SUFFIX = PKG_NAME_SUFFIX |
| if (!project.hasProperty("pkgSuffix") || !PKG_NAME_SUFFIX) { |
| RPM_PKG_NAME_SUFFIX = "%{nil}" |
| GRADLE_PKG_NAME_SUFFIX = "" |
| } |
| def final BIGTOP_BUILD_STAMP = System.getenv('BIGTOP_BUILD_STAMP') ?: |
| config.bigtop.components[target].version.release |
| def final NAME = config.bigtop.components[target].name |
| def final PKG_NAME = config.bigtop.components[target].pkg |
| def final PKG_NAME_FOR_PKG = NAME.replaceAll("-", "_") |
| def final PKG_BUILD_DIR = config.bigtop.components[target].builddir |
| def final SEED_TAR = config.bigtop.components[target].seedtar |
| def final PKG_VERSION = config.bigtop.components[target].version.pkg |
| def final BASE_VERSION = config.bigtop.components[target].version.base |
| def final PKG_OUTPUT_DIR = config.bigtop.components[target].outputdir |
| safeDelete ("$PKG_BUILD_DIR/rpm") |
| ['INSTALL','SOURCES','BUILD','SRPMS','RPMS'].each { rpmdir -> |
| mkdir("$PKG_BUILD_DIR/rpm/$rpmdir") |
| } |
| copy { |
| from fileTree("${BASE_DIR}/bigtop-packages/src/rpm/$NAME") { |
| include '**/*' |
| } |
| into "$PKG_BUILD_DIR/rpm" |
| } |
| copy { |
| from SEED_TAR |
| into "$PKG_BUILD_DIR/rpm/SOURCES" |
| } |
| copy { |
| from "${BASE_DIR}/bigtop-packages/src/templates/init.d.tmpl" |
| into "$PKG_BUILD_DIR/rpm/SOURCES" |
| } |
| copy { |
| from "${BASE_DIR}/bigtop-packages/src/extensions" |
| into "$PKG_BUILD_DIR/rpm/SOURCES" |
| include "*" |
| } |
| copy { |
| from "${BASE_DIR}/bigtop-packages/src/scripts" |
| into "$PKG_BUILD_DIR/rpm/SOURCES" |
| include "*" |
| } |
| copy { |
| from fileTree("$BASE_DIR/bigtop-packages/src/common/$NAME") { |
| include '**/*' |
| } |
| into "$PKG_BUILD_DIR/rpm/SOURCES" |
| } |
| // Writing bigtop.bom files with all the versions |
| def bomWriter = new File("$PKG_BUILD_DIR/rpm/SOURCES/bigtop.bom").newWriter() |
| bomVersions.each { bomWriter << "$it\n"} |
| bomWriter.close() |
| |
| def specFileName = "${PKG_BUILD_DIR}/rpm/SPECS/${NAME}.spec" |
| def specTmpFileName = "${PKG_BUILD_DIR}/rpm/SPECS/tmp_${NAME}.spec" |
| def specFile = new File(specFileName) |
| def specWriter = new File(specTmpFileName).newWriter() |
| def patches = new File("${PKG_BUILD_DIR}/rpm/SOURCES").list({ d, f -> f ==~ /patch.*diff/}).sort() |
| specFile.eachLine { line -> |
| if (line.startsWith("#BIGTOP_PATCH_FILES")) { |
| def patchNum = 0 |
| patches.each { p -> |
| specWriter << "Patch$patchNum: $p\n" |
| patchNum++ |
| } |
| } else { |
| if (line.startsWith("#BIGTOP_PATCH_COMMANDS")) { |
| def patchNum = 0 |
| patches.each { p -> |
| specWriter << "%patch$patchNum -p1\n" |
| patchNum++ |
| } |
| } else { |
| specWriter << "$line\n" |
| } |
| } |
| } |
| specWriter.close() |
| specFile.delete() |
| new File(specTmpFileName).renameTo(specFileName) |
| |
| def command = [ |
| '--define', "_topdir $PKG_BUILD_DIR/rpm/", |
| '--define', "${PKG_NAME_FOR_PKG}_base_version $BASE_VERSION", |
| '--define', "${PKG_NAME_FOR_PKG}_version ${PKG_VERSION}", |
| '--define', "${PKG_NAME_FOR_PKG}_release ${BIGTOP_BUILD_STAMP}%{?dist}", |
| '-bs', '--nodeps', "--buildroot=${PKG_BUILD_DIR}/rpm/INSTALL", |
| specFileName, |
| '--define', "pkg_name_suffix ${RPM_PKG_NAME_SUFFIX}", |
| ] |
| exec { |
| workingDir BASE_DIR |
| executable 'rpmbuild' |
| args command |
| } |
| mkdir(PKG_OUTPUT_DIR) |
| def RELEASE_DIST = "rpmbuild --eval '%{?dist}' 2>/dev/null".execute().text.trim().replaceAll("'",'') |
| copy { |
| from "$PKG_BUILD_DIR/rpm/SRPMS/${PKG_NAME}${GRADLE_PKG_NAME_SUFFIX}-${PKG_VERSION}-${BIGTOP_BUILD_STAMP}${RELEASE_DIST}.src.rpm" |
| into PKG_OUTPUT_DIR |
| } |
| touchTargetFile(config.bigtop.components[target].targetsrpm) |
| } |
| if (nativePackaging) { |
| def ptype = nativePackaging.pkg |
| task "$target-pkg" (dependsOn: "$target-$ptype", |
| description: "Invoking a native binary packaging target $ptype", |
| group: PACKAGES_GROUP) doLast { |
| } |
| task "$target-spkg" (dependsOn: "$target-s$ptype", |
| description: "Invoking a native binary packaging target s$ptype", |
| group: PACKAGES_GROUP) doLast { |
| } |
| } |
| task "$target-pkg-ind" ( |
| description: "Invoking a native binary packaging for $target in Docker. Usage: \$ ./gradlew " + |
| "-POS=[centos-7|fedora-35|debian-10|ubuntu-18.04] " + |
| "-Pprefix=[trunk|1.4.0|1.3.0|1.2.1|...] $target-pkg-ind " + |
| "-Pnexus=[true|false]" + |
| "[-Ppreferred-java-version=[8|11]]" + |
| "[-Pmvn-cache-volume=[true|false]]", |
| group: PACKAGES_GROUP) doLast { |
| def _prefix = project.hasProperty("prefix") ? prefix : "trunk" |
| def _OS = project.hasProperty("OS") ? OS : "centos-7" |
| if (_OS == "fedora-35" || _OS == "ubuntu-22.04") { |
| logger.warn(_OS + ' requires privileged mode, pass `-Pdocker-run-option="--privileged"` to gradle option') |
| } |
| def _target_pkg = "$target-pkg" |
| def _preferred_java_version = project.hasProperty("preferred-java-version") ? this.'preferred-java-version' : "8" |
| def _mvn_cache_volume = project.hasProperty("mvn-cache-volume") ? this.'mvn-cache-volume' : "" |
| def _docker_run_option = project.hasProperty("docker-run-option") ? this.'docker-run-option' : "" |
| def additionalConfigKeys = ['git_repo', 'git_ref', 'git_dir', 'git_commit_hash', 'base_version'] |
| additionalConfigKeys.each { key -> |
| if (project.hasProperty(key)) { |
| _target_pkg += " -P${key}=" + project.property(key) |
| } |
| } |
| def command = [ |
| 'bash', '-x', |
| './bigtop-ci/build.sh', |
| '--prefix', _prefix, |
| '--os', _OS, |
| '--target', _target_pkg, |
| '--docker-run-option', _docker_run_option, |
| ] |
| if ( project.hasProperty("nexus") ) { |
| command.add('--nexus') |
| } |
| if ( _preferred_java_version.toInteger() > 8 ) { |
| command.add('--preferred-java-version') |
| command.add( _preferred_java_version) |
| } |
| if ( project.hasProperty("mvn-cache-volume") ) { |
| command.add('--mvn-cache-volume') |
| command.add(_mvn_cache_volume) |
| } |
| |
| println "Building $_prefix $_target_pkg on $_OS in Docker...\n" |
| |
| exec { |
| workingDir BASE_DIR |
| commandLine command |
| } |
| } |
| task "$target-version" (description: "Show version of $target component", group: PACKAGES_GROUP) doLast { |
| println "Base: ${config.bigtop.components[target].version.base}" |
| } |
| task "${target}_vardefines" doLast { |
| if (project.hasProperty('base_version')) { |
| def BASE_VERSION = project.property("base_version") |
| if (config.bigtop.components[target].tarball.source) { |
| config.bigtop.components[target].tarball.source = config.bigtop.components[target].tarball.source |
| .replaceAll(config.bigtop.components[target].version.base, BASE_VERSION) |
| } |
| config.bigtop.components[target].tarball.destination = config.bigtop.components[target].tarball.destination |
| .replaceAll(config.bigtop.components[target].version.base, BASE_VERSION) |
| |
| config.bigtop.components[target].version.base = BASE_VERSION |
| config.bigtop.components[target].version.pkg = BASE_VERSION |
| } |
| |
| setDefaults(config.bigtop.components[target]) |
| |
| config.bigtop.components[target].package.release = '1' |
| |
| config.bigtop.components[target].builddir = config.bigtop.builddir + "/$target" |
| config.bigtop.components[target].outputdir = config.bigtop.outputdir + "/$target" |
| config.bigtop.components[target].srcdir = config.bigtop.builddir + "/source" |
| config.bigtop.components[target].tardir = config.bigtop.builddir + "/$target/tar/${target}-${config.bigtop.components[target].version.base}" |
| config.bigtop.components[target].seedtar = config.bigtop.builddir + "/$target/tar/" + config.bigtop.components[target].tarball.destination |
| |
| config.bigtop.components[target].downloadurl = |
| (!config.bigtop.components[target].url.isEmpty() && |
| !config.bigtop.components[target].url.site.isEmpty() && |
| !config.bigtop.components[target].tarball.source.isEmpty()) ? |
| config.bigtop.components[target].url.site + '/' + config.bigtop.components[target].tarball.source : null |
| if (!config.bigtop.components[target].tarball.destination.isEmpty()) |
| config.bigtop.components[target].downloaddst = DL_DIR + '/' + config.bigtop.components[target].tarball.destination |
| |
| // test that the download url will return http 200. If it does not, use the ARCHIVE url instead of the MIRROR SITE url |
| if (!config.bigtop.components[target].url.isEmpty() && !ifExists(config.bigtop.components[target].downloadurl)) { |
| config.bigtop.components[target].downloadurl = config.bigtop.components[target].url.archive + '/' + config.bigtop.components[target].tarball.source |
| } |
| |
| config.bigtop.components[target].targetdl = config.bigtop.components[target].builddir + '/.download' |
| config.bigtop.components[target].targettar = config.bigtop.components[target].builddir + '/.tar' |
| config.bigtop.components[target].targetsrpm = config.bigtop.components[target].builddir + '/.srpm' |
| config.bigtop.components[target].targetrpm = config.bigtop.components[target].builddir + '/.rpm' |
| config.bigtop.components[target].targetsdeb = config.bigtop.components[target].builddir + '/.sdeb' |
| config.bigtop.components[target].targetdeb = config.bigtop.components[target].builddir + '/.deb' |
| config.bigtop.components[target].targetrelnotes = config.bigtop.components[target].builddir + '/.relnotes' |
| |
| if (System.getProperty(VERBOSE)) { |
| println "$target properties (explcit and derived)" |
| config.bigtop.components[target].toProperties(target).each {k ,v -> |
| println "$k\t= $v" |
| } |
| } |
| } |
| |
| task "$target-info" (dependsOn: "${target}_vardefines", |
| description: "Info about $target component build", |
| group: PACKAGES_GROUP) doLast { |
| println "Info for package $target" |
| println " Will download from URL: ${config.bigtop.components[target].downloadurl}" |
| println " To destination file: ${config.bigtop.components[target].downloaddst}" |
| println " Then unpack into ${config.bigtop.components[target].srcdir}" |
| println " And create a seed tarball ${config.bigtop.components[target].seedtar}" |
| |
| println "Version: " + config.bigtop.components[target].version.base |
| //TODO more about stamping |
| } |
| task "$target-relnotes" (description: "Preparing release notes for $target. No yet implemented!!!", group: PACKAGES_GROUP) doLast { |
| } |
| task "$target-clean" (dependsOn: "${target}_vardefines", |
| description: "Removing $target component build and output directories", |
| group: PACKAGES_GROUP) doLast { |
| safeDelete(config.bigtop.components[target].builddir) |
| safeDelete(config.bigtop.components[target].outputdir) |
| } |
| task "$target-help" (description: "List of available tasks for $target", group: PACKAGES_GROUP) doLast { |
| println (target + "\n\t[" + tasks.findAll { alltask -> alltask.name.startsWith(target)}*.name.join(", ") + "]") |
| } |
| |
| bomTasks.add(target) |
| } |
| |
| // Let's enforce some of the configuration requirements |
| private void doValidateBOM(config) { |
| assert config.bigtop.version |
| assert config.bigtop.stack.jdk |
| assert config.bigtop.stack.scala |
| assert config.bigtop.apache.size() != 0 |
| } |
| |
| // We need to make sure that all dynamic tasks are available for invocation |
| project.afterEvaluate { |
| doValidateBOM(config) |
| config.bigtop.components.each { component_label, comp -> |
| assert component_label == comp.name |
| if (!comp.packaging || comp.packaging.equalsIgnoreCase(nativePackaging.pkg)) { |
| genTasks(comp.name) |
| } |
| } |
| // Versions need to be preserved for more than just component: |
| // - there are JDK version requirement |
| // - possibly more in the future |
| bomVersions = config.bigtop.components.collect { |
| "${toOldStyleName(it.value.name)}_VERSION=${it.value.version.base}" |
| } |
| bomVersions += config.bigtop.stack.collect { // Version of the stack have different syntax |
| "${toOldStyleName(it.key)}_VERSION=${it.value.version_base}" |
| } |
| if (System.getProperty(VERBOSE))println "BIGTOP_BOM_VERSIONS:\n${bomVersions.join(' ')}" |
| // Putting all targets of different types into one common target |
| task "srpm" (dependsOn: tasks.findAll { alltask -> alltask.name.endsWith("-srpm")}*.name, |
| description: "Build all SRPM packages for the stack components", |
| group: PACKAGES_GROUP |
| ) doLast { } |
| task "rpm" (dependsOn: tasks.findAll { alltask -> alltask.name.endsWith("-rpm")}*.name, |
| description: "Build all RPM packages for the stack", |
| group: PACKAGES_GROUP |
| ) doLast { } |
| task "sdeb" (dependsOn: tasks.findAll { alltask -> alltask.name.endsWith("-sdeb")}*.name, |
| description: "Build all SDEB packages for the stack components", |
| group: PACKAGES_GROUP |
| ) doLast { } |
| task "deb" (dependsOn: tasks.findAll { alltask -> alltask.name.endsWith("-deb")}*.name, |
| description: "Build all DEB packages for the stack components", |
| group: PACKAGES_GROUP |
| ) doLast { } |
| task "pkgs" (dependsOn: tasks.findAll { alltask -> alltask.name.endsWith("-pkg")}*.name, |
| description: "Build all native packages for the stack components", |
| group: PACKAGES_GROUP |
| ) doLast { } |
| task "pkgs-ind" (dependsOn: tasks.findAll { alltask -> alltask.name.endsWith("-pkg-ind")}*.name, |
| description: "Build all native packages for the stack components inside Docker", |
| group: PACKAGES_GROUP |
| ) doLast { } |
| |
| task allclean (dependsOn: [clean, tasks.findAll { alltask -> alltask.name.endsWith("-clean")}*.name], |
| description: "Removing $BUILD_DIR, $OUTPUT_DIR, and $DIST_DIR.\n\t\t" + |
| "Cleaning all components' build and output directories.", |
| group: PACKAGES_GROUP) doLast { |
| safeDelete(BUILD_DIR) |
| safeDelete(OUTPUT_DIR) |
| safeDelete(DIST_DIR) |
| } |
| task realclean (dependsOn: allclean, |
| description: "Removing $BUILD_DIR, $OUTPUT_DIR, $DIST_DIR, and $DL_DIR", |
| group: PACKAGES_GROUP) doLast { |
| delete (DL_DIR) |
| } |
| } |
| |
| task "apt" ( |
| description: "Creating APT repository", |
| group: PACKAGES_GROUP) doLast { |
| |
| delete ( "$OUTPUT_DIR/apt") |
| mkdir ("$OUTPUT_DIR/apt/conf") |
| |
| generateDistributionsFile("$OUTPUT_DIR/apt/conf/distributions"); |
| |
| fileTree (OUTPUT_DIR) { |
| include "*/*.changes" |
| }.each { changeFile -> |
| exec { |
| workingDir BUILD_DIR |
| commandLine "reprepro -Vb $OUTPUT_DIR/apt include bigtop $changeFile".split(' ') |
| } |
| } |
| } |
| |
| task "yum" ( |
| description: "Creating YUM repository", |
| group: PACKAGES_GROUP) doLast { |
| delete ( "$OUTPUT_DIR/repodata") |
| exec { |
| workingDir BUILD_DIR |
| commandLine "createrepo -o $OUTPUT_DIR $OUTPUT_DIR".split(' ') |
| } |
| } |
| |
| if (nativePackaging) { |
| task "repo" (dependsOn: nativePackaging.repo, |
| description: "Invoking a native repository target ${nativePackaging.repo}", |
| group: PACKAGES_GROUP) doLast { } |
| } |
| |
| task "repo-ind" ( |
| description: "Invoking a native repository in Docker. Usage: \$ ./gradlew " + |
| "-POS=[centos-7|fedora-35|debian-10|ubuntu-18.04] " + |
| "-Pprefix=[trunk|1.4.0|1.3.0|1.2.1|...] repo-ind", |
| group: PACKAGES_GROUP) doLast { |
| def _prefix = project.hasProperty("prefix") ? prefix : "trunk" |
| def _OS = project.hasProperty("OS") ? OS : "centos-7" |
| if (_OS == "fedora-35") { |
| logger.warn('fedora-35 requires privileged mode, pass `-Pdocker-run-option="--privileged"` to gradle option') |
| } |
| def _docker_run_option = project.hasProperty("docker-run-option") ? this.'docker-run-option' : "" |
| def command = [ |
| 'bash', '-x', |
| './bigtop-ci/build.sh', |
| '--prefix', _prefix, |
| '--os', _OS, |
| '--target', 'repo', |
| '--docker-run-option', _docker_run_option |
| ] |
| exec { |
| workingDir BASE_DIR |
| commandLine command |
| } |
| } |