blob: b560ba2ec1d7f5ec1c3b557c9c9ea5e265142816 [file] [log] [blame]
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
buildscript {
repositories {
mavenCentral()
}
}
plugins {
id "de.undercouch.download" version "2.0.0"
id "org.nosphere.apache.rat" version "0.2.0"
}
// Generated by http://patorjk.com/software/taag/#p=display&f=Bloody&t=Apache%20Bigtop
def final ASCII_bigtop = '''
:aass..
=XS22nai,>__. .
=n-- +!!!""^--
.vX> .
.)e<o;.
._v2`-{S> .
..<de~..;)Sa,
.._aoX}:===>=-?Xo>, . .
. .__aaoZe!`=><i=s+s;~*XXos,,_ . . .
...........______=iisaaoXXZY!"~._v(=d=:nc-1s,-~?SX#Xouass,,_____.:..........
=XXoXXXSXXXXXXXXZUZXX21?!"^-.._au*`=u2` ]X>.+*a>,.-"!Y1XSSX##ZZXXXXXXXXXXXoXXc
.{XXXXXX2*?!!"!"^~--- ...__aa2!^- =dX( .+XXc. ~!1nas,,.---~~^""!"!!!?YSXXXXX2+
-"YSXXXo=. ._=sssaaav1!!~- ._aXXe` )SXo>. -~"?Yoouass_s,, _vXXXX2}~
-{XXZoai%%*XXSSSX>.. .<uXXX2~ . {XXXXs,. . .=dXXXZX2lii%uXXXXe-
.<XXXXX%- -<XXXXX1|==%vdXXXXXo;:. ._vXXXXXXos_=i|*XXXXX> -<XXXXX`
=SXXXZc ..nXXX2> ---=2XXX2^-"|||}"--~{ZXXX1-- .:XXXXo; . )XXXX2`
. =XXXXZc nXXXX> =XXXXe..__s=>_...)XXXX1 . .:SXXXo; .)XXXX2..
<SXXXXc . .nXXXS> =XXXXosummmmBmma,)ZXXX1 :XXXX2; )XXXXX.
.<XXXXX( :nXXXS; <XXXXXm#mmmWmmmmmoZXXX1 ..3XXXo;. . )XXXXX;
. nXXXXX;. :XXXXX; .=XXXXXmmmBmmmWmB#XXXXX1 . .nXXXX> :XXXXXc .
. =oZXXXe; .<XXXX2` . )XXXXZmBmBmWmmmW#2XXXX1 . .vXXXXc vXXXXo;
+Y3S2Xz__...vXXXXe . .)ZXXXZmmWmBmmBBm#XXXXXo.. {XXXXz:.___vSS2Y1=
. ---+"""*!!*Y1s|=_==uXSSSXZUXUXUXUXUXS2XX2n|_=||%Y*??!"""^~---
.--- - ---------------- - - -.
.o. oooo oooooooooo. o8o .
.888. `888 `888' `Y8b `"' .o8
.8"888. oo.ooooo. .oooo. .ooooo. 888 .oo. .ooooo. 888 888oooo .oooooooo.o888oo .ooooo. oo.ooooo.
.8' `888. 888' `88b`P )88b d88' `"Y8 888P"Y88b d88' `88b 888oooo888'`888 888' `88b 888 d88' `88b 888' `88b
.88ooo8888. 888 888 .oP"888 888 888 888 888ooo888 888 `88b 888 888 888 888 888 888 888 888
.8' `888. 888 888d8( 888 888 .o8 888 888 888 .o 888 .88P 888 `88bod8P' 888 .888 888 888 888
o88o o8888o 888bod8P'`Y888""8o`Y8bod8P'o888o o888o`Y8bod8P' o888bood8P' o888o`8oooooo. "888"`Y8bod8P' 888bod8P'
888 d" YD 888
o888o "Y88888P' o888o
'''
def final langLevel = "1.7"
rat {
excludes = [
".git/**",
".idea/**",
"**/.gitignore",
/* Test data with rigid structure and/or binary */
"bigtop-tests/test-artifacts/**/resources/**",
"bigtop-tests/smoke-tests/alluxio/datafile",
"bigtop-tests/smoke-tests/hive/passwd.ql",
"bigtop-tests/smoke-tests/kite/sandwiches.csv",
"bigtop-tests/smoke-tests/phoenix/*.csv",
"bigtop-tests/smoke-tests/phoenix/*.sql",
"bigtop-tests/smoke-tests/ignite-hadoop/*.data",
"bigtop-tests/smoke-tests/tajo/table1/*.csv",
"**/target/**",
"**/build/**",
"**/.gradle/**",
"**/*.iml",
".gradle",
"output/**",
"MAINTAINERS.txt",
"buildSrc/build/**",
"bigtop-data-generators/**/input_data/**",
"docker/**/*.yaml",
"bigtop-deploy/puppet/**/*.yaml",
"bigtop-deploy/puppet/modules/hadoop/files/*/id_hdfsuser*",
"bigtop-deploy/puppet/modules/bigtop-util/Gemfile.lock",
"bigtop-deploy/puppet/modules/zeppelin/templates/interpreter.json",
/* Debian package files that have a rigid structure
(* matches package name) */
"bigtop-packages/src/deb/*/*.dirs",
"bigtop-packages/src/deb/*/dirs",
"bigtop-packages/src/deb/*/*.install",
"bigtop-packages/src/deb/*/*.include",
"bigtop-packages/src/deb/**/include-binaries",
"bigtop-packages/src/deb/**/format",
"bigtop-packages/src/deb/*/*.manpages",
"bigtop-packages/src/deb/*/*.docs",
"bigtop-packages/src/deb/*/docs",
"bigtop-packages/src/deb/*/compat",
"bigtop-packages/src/deb/*/source/format",
"bigtop-packages/src/deb/*/*.lintian-overrides",
"bigtop-packages/src/deb/*/shlibs.local",
"bigtop-packages/src/**/*.diff",
"bigtop-packages/src/common/*/*.json",
"bigtop-packages/src/common/**/*.default",
"bigtop-repos/apt/distributions",
/* Juju charm files with rigid structure */
"bigtop-packages/src/charm/**/wheelhouse.txt",
"bigtop-packages/src/charm/**/*.json",
"bigtop-packages/src/charm/**/*.svg",
"bigtop-packages/src/charm/**/*.yaml",
"bigtop-deploy/juju/**/*.yaml",
"bigtop-tests/cloud-weather-report/**/*.yaml",
/* Misc individual files */
"src/site/resources/bigtop.rdf",
"src/site/resources/images/bigtop-logo.ai",
/* File with FreeBSD documentation license */
"src/site/resources/css/freebsd_docbook.css",
"test/MANIFEST.txt",
"bigtop-deploy/puppet/modules/hadoop/templates/taskcontroller.cfg",
"bigtop-deploy/puppet/modules/hadoop/templates/container-executor.cfg",
"bigtop-deploy/puppet/README.md",
"bigtop-deploy/puppet/modules/qfs/README.md",
"bigtop-deploy/puppet/config/site.csv.example",
"dl/**",
"bigtop-deploy/puppet/modules/gpdb/templates/**",
"provisioner/docker/config/hiera.yaml",
"provisioner/docker/config/hieradata/PLACEHOLDER",
"provisioner/docker/config/hosts",
]
}
allprojects {
apply plugin: 'java'
apply plugin: 'maven'
group = 'org.apache.bigtop'
version = '1.2.0-SNAPSHOT'
description = """Bigtop"""
sourceCompatibility = langLevel
targetCompatibility = langLevel
repositories {
maven { url "http://repository.apache.org/snapshots" }
maven { url "http://repo.maven.apache.org/maven2" }
mavenCentral()
}
}
subprojects {
apply plugin: 'groovy'
}
project(':itest-common') {
description = """iTest: system and integration testing in the cloud"""
dependencies {
compile group: 'org.codehaus.groovy', name: 'groovy-all', version:'2.1.8'
compile group: 'junit', name: 'junit', version:'4.11'
compile group: 'commons-logging', name: 'commons-logging', version:'1.1'
compile group: 'org.apache.ant', name: 'ant', version:'1.8.2'
compile group: 'org.apache.ant', name: 'ant-junit', version:'1.8.2'
}
task packageITest(type: Jar) {
from sourceSets.main.output
//classifier = 'tests'
}
artifacts.archives packageITest
test.doFirst {
if (System.getenv('JAVA_HOME') == null)
throw new GradleException("JAVA_HOME is required to test ${project.name}")
}
test {
exclude '**/Dummy*'
}
}
/**
* checkSmokeTestProjects() configures smoke-tests:<component>:test task only
* in if the project has smoke.tests properties set.
* This is done to avoid running cluster smoke tests during the normal life-cycle
* of the project development
*/
def checkClusterTestProjects = { property ->
def suiteName = property.replaceAll('.', '-')
FileTree fTree = fileTree(dir: "bigtop-tests/$suiteName", include: '*/build.gradle')
fTree.each() { smokeProject ->
def parent = smokeProject.getParentFile().name
project (":bigtop-tests:$suiteName:$parent") {
test {
onlyIf {project.hasProperty(suiteName)}
}
}
}
}
def TESTARTIFACTS_GROUP = 'test artifacts'
def DEVENV_GROUP = 'development tools'
def DEPLOY_GROUP = 'deployment'
def DOCKERBUILD_GROUP = 'docker build'
// All packaging logic is separated into its own build module
apply from: 'packages.gradle'
// Wrapping all release logic to simplify the release process
apply from: 'release.gradle'
task toolchain(type:Exec,
description: 'Setup dev. env via toolchain; Requires: Puppet, sudo',
group: DEVENV_GROUP) {
def command = [
'sudo', 'puppet', 'apply', '-d',
"--modulepath=${projectDir.absolutePath}:/etc/puppet/modules", '-e',
'include bigtop_toolchain::installer'
]
workingDir '.'
commandLine command
}
task "toolchain-puppetmodules"(type:Exec,
description: 'Setup puppet modules via toolchain; Requires: Puppet, sudo',
group: DEVENV_GROUP) {
def command = [
'sudo', 'puppet', 'apply', '-d',
"--modulepath=${projectDir.absolutePath}", '-e',
'include bigtop_toolchain::puppet-modules'
]
workingDir '.'
commandLine command
}
task "toolchain-devtools"(type:Exec,
description: 'Setup additional dev. tools like Groovy SDK via toolchain; Requires: Puppet, sudo',
group: DEVENV_GROUP) {
def command = [
'sudo', 'puppet', 'apply', '-d',
"--modulepath=${projectDir.absolutePath}", '-e',
'include bigtop_toolchain::development-tools'
]
workingDir '.'
commandLine command
}
task "docker-provisioner"(type:Exec,
description: 'Provision a Bigtop cluster on Docker container(s). Default to CentOS and 1 node.\nCreate a 3 node cluster:\n $ ./gradlew -Pnum_instances=3 docker-provisioner\nRun smoke tests on 3 node cluster:\n $ ./gradlew -Pnum_instances=3 -Prun_smoke_tests=true docker-provisioner\nSpecify configuration file (located in bigtop-deploy/vm/vagrant-puppet-docker/):\n $ ./gradlew -Pconfig=vagrantconfig_debian.yaml docker-provisioner',
group: DEPLOY_GROUP) {
def _config = project.hasProperty("config") ? config : "vagrantconfig.yaml"
def _num_instances = project.hasProperty("num_instances") ? num_instances : 1
def command = [
'./docker-hadoop.sh',
'-C', _config,
'--create', _num_instances,
]
if ( project.hasProperty("run_smoke_tests") && run_smoke_tests ) {
command.add("--smoke-tests")
}
workingDir 'bigtop-deploy/vm/vagrant-puppet-docker'
commandLine command
}
task "docker-provisioner-ssh"(type:Exec,
description: 'Show ssh command to get in to the provisioned container',
group: DEPLOY_GROUP) {
workingDir 'bigtop-deploy/vm/vagrant-puppet-docker'
def message = "To ssh in to the container, do: \n(cd " + workingDir + " && vagrant ssh <CONTAIER_NAME>)"
def command = [
'/bin/echo',
message
]
commandLine command
}
task "docker-provisioner-status"(type:Exec,
description: 'Show status of Bigtop Docker cluster',
group: DEPLOY_GROUP) {
def command = [
'vagrant',
'status',
]
workingDir 'bigtop-deploy/vm/vagrant-puppet-docker'
commandLine command
}
task "docker-provisioner-destroy"(type:Exec,
description: 'Destroy provisioned Bigtop Docker cluster',
group: DEPLOY_GROUP) {
def command = [
'./docker-hadoop.sh',
'--destroy'
]
workingDir 'bigtop-deploy/vm/vagrant-puppet-docker'
commandLine command
}
task installTopLevel(type:Exec) {
workingDir "."
commandLine 'mvn clean install -f pom.xml'.split(" ")
}
task installiTest(dependsOn: installTopLevel, type:Exec) {
workingDir "."
commandLine 'mvn clean install -f bigtop-test-framework/pom.xml -DskipTests'.split(" ")
}
task installTestArtifacts(dependsOn: installiTest, type:Exec) {
workingDir "."
commandLine 'mvn clean install -f bigtop-tests/test-artifacts/pom.xml'.split(" ")
}
task installConf(type:Exec) {
workingDir "."
commandLine 'mvn clean install -f bigtop-tests/test-execution/conf/pom.xml'.split(" ")
}
task installCommon(type:Exec) {
workingDir "."
commandLine 'mvn clean install -f bigtop-tests/test-execution/common/pom.xml'.split(" ")
}
task installAllLocalArtifacts (
description: "Prepare and locally install all test artifacts",
group: TESTARTIFACTS_GROUP) {
}
/**
* Allows user to specify which artifacts to install by dynamically generating tasks.
*/
def artifactToInstall = {
def final BASE_DIR = projectDir.absolutePath
def final TEST_DIR = "$BASE_DIR/bigtop-tests/test-artifacts"
def project = new XmlSlurper().parse("$TEST_DIR/pom.xml")
project.modules.module.each { artifact ->
task "install-${artifact}" (description: "Installs ${artifact} artifact with Maven",
group: TESTARTIFACTS_GROUP,
dependsOn: installiTest
) << {
def final PATH = "${TEST_DIR}/$artifact/pom.xml"
def final WRAPPER = "mvn clean install -f " + PATH
exec {
workingDir '.'
commandLine WRAPPER.split(" ")
}
}
}
}
def _NEXUS_URL = project.hasProperty("NEXUS_URL") ? NEXUS_URL : "http://localhost:8081"
def generate_nexus_tasks = { name, url, snapshot ->
task "configure-nexus-${name}"(type: Exec) {
def req = [ data : [ id: name, name: name, repoType: "proxy", repoPolicy : snapshot, browseable: true, "indexable": true, "notFoundCacheTTL": 1440,
"artifactMaxAge": 1440, "metadataMaxAge": 1440, "itemMaxAge": 1440, "provider": "maven2",
"providerRole": "org.sonatype.nexus.proxy.repository.Repository", "downloadRemoteIndexes": true,
"autoBlockActive": true, "fileTypeValidation": true, "exposed": true, "checksumPolicy": "WARN",
"remoteStorage": ["remoteStorageUrl": url, "authentication": null,
"connectionSettings": null]]]
def root = new groovy.json.JsonBuilder(req)
def p = root.toString()
def _NEXUS_USERPASS = project.hasProperty("NEXUS_USERPASS") ? NEXUS_USERPASS : "admin:admin123"
File.createTempFile("temp",".tmp").with {
deleteOnExit()
write root.toString()
workingDir '.'
commandLine "curl", "-o", "/dev/null", "-X", "POST", "-d", "@$absolutePath",
"--header", "Content-Type: application/json", "-u", "$_NEXUS_USERPASS", "$_NEXUS_URL/service/local/repositories"
}
}
}
def repos = [ [ "name": "conjars", "url" : "http://conjars.org", 'snapshot':'RELEASE'],
[ "name": "repository.jboss.org", "url": "http://repository.jboss.org/nexus/content/groups/public/", 'snapshot': 'RELEASE'],
[ "name": "apache.snapshots.https", "url": "https://repository.apache.org/content/repositories/snapshots", "snapshot": 'SNAPSHOT'],
[ "name": "apache.snapshots", "url": "https://repository.apache.org/content/repositories/snapshots", "snapshot": 'SNAPSHOT'],
[ "name": "maven2-repository.atlassian", "url": "https://maven.atlassian.com/repository/public", "snapshot": 'RELASE']]
repos.each { r->
generate_nexus_tasks( r.name, r.url, r.snapshot)
}
task "configure-nexus"(dependsOn: tasks.findAll { alltask -> alltask.name.startsWith("configure-nexus-")}*.name,
description: 'configure build to use a sonatype nexus server.\n' +
' Use -PNEXUS_URL=http://server:8081 to overwrite default server http://localhost:8081\n' +
' Use -PNEXUS_USERPASS=user:passwd to overwrite default username passwd admin:admin123') << {
def m2Dir = System.getProperty("user.home") + "/.m2"
mkdir(m2Dir)
def writer = new File(m2Dir + "/settings.xml")
def writeMirrorLine = { name ->
writer.append("<mirror><name>$name</name><url>$_NEXUS_URL/content/repositories/$name/</url><mirrorOf>$name</mirrorOf></mirror>")
}
writer.text = "<settings><mirrors>"
writeMirrorLine( "central")
repos.each{ r->
writeMirrorLine( r.name)
}
writer.append("</mirrors></settings>")
}
task "bigtop-slaves"(type:Exec,
description: 'Build bigtop/slaves images.\n' +
'Usage:\n $ ./gradlew -POS=[centos-6|centos-7|fedora-20|debian-8|ubuntu-14.04|opensuse-13.2] -Pprefix=STRING_TO_PREFIX bigtop-slaves\n' +
'Example:\n $ ./gradlew -POS=debian-8 -Pprefix=1.0.0 bigtop-slaves\n' +
'The built image name: bigtop/slaves:1.0.0-debian-8',
group: DOCKERBUILD_GROUP) {
def _prefix = project.hasProperty("prefix") ? prefix : "trunk"
def _OS = project.hasProperty("OS") ? OS : "centos-7"
def command = [
'docker', 'build',
'-t', 'bigtop/slaves:' + _prefix + '-' + _OS,
'-f', "docker/bigtop-slaves/" + _OS + "/Dockerfile",
'--force-rm', '--no-cache', '--pull', '--rm',
'.'
]
workingDir '.'
commandLine command
}
project.afterEvaluate {
checkClusterTestProjects("smoke.tests")
artifactToInstall(dependsOn: [installTopLevel, installCommon, installConf, installiTest])
}
installAllLocalArtifacts.dependsOn installTopLevel, installCommon, installConf, installiTest, installTestArtifacts
help.doFirst {
println ASCII_bigtop
}
repositories {
maven { url "http://repository.apache.org/snapshots" }
maven { url "http://repo.maven.apache.org/maven2" }
}