blob: ee31831d2299385af1cd5d2debebac94e6689869 [file] [log] [blame]
// Copyright (C) 2014-2015 LinkedIn Corp. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
// this file except in compliance with the License. You may obtain a copy of the
// License at http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed
// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
// CONDITIONS OF ANY KIND, either express or implied.
buildscript {
repositories {
maven {
url "https://plugins.gradle.org/m2/"
}
}
dependencies {
classpath 'gradle.plugin.org.inferred:gradle-processors:1.1.2'
}
}
apply plugin: 'org.inferred.processors'
apply plugin: 'idea'
idea.project {
languageLevel = JavaVersion.VERSION_1_7
}
ext.build_script_dir = "${projectDir.path}/build_script"
ext.isDefaultEnvironment = !project.hasProperty('overrideBuildEnvironment')
File getEnvironmentScript()
{
final File env = file(isDefaultEnvironment ? 'defaultEnvironment.gradle' : project.overrideBuildEnvironment)
assert env.isFile() : "The environment script [$env] does not exists or is not a file."
return env
}
apply from: environmentScript
ext.publishToMaven = project.hasProperty('publishToMaven')
if (ext.publishToMaven) {
plugins.apply('maven')
// Workaround for a bug in gradle's "maven" plugin. See https://discuss.gradle.org/t/error-in-parallel-build/7215/3
project.setProperty("org.gradle.parallel", "false")
}
ext.signArtifacts = !project.hasProperty('doNotSignArtifacts')
if (!project.hasProperty('group') || project.group.length() == 0) {
project.group = 'com.linkedin.gobblin'
}
if (!project.hasProperty('artifactRepository') || project.artifactRepository.length() == 0) {
project.artifactRepository = "https://oss.sonatype.org/service/local/staging/deploy/maven2/"
}
if (!project.hasProperty('artifactSnapshotRepository') || project.artifactSnapshotRepository.length() == 0) {
project.artifactSnapshotRepository = "https://oss.sonatype.org/content/repositories/snapshots/"
}
if (!project.hasProperty('version') || project.version == 'unspecified') {
exec {
commandLine 'git', 'fetch', '-t', 'https://github.com/linkedin/gobblin.git', 'master'
}
def versionOut = new ByteArrayOutputStream()
exec {
commandLine 'git', 'describe', '--tags', '--always'
standardOutput versionOut
}
def tagStr = versionOut.toString().trim()
println 'Using latest tag for version: ' + tagStr
if (tagStr.startsWith("gobblin_")) {
project.version = tagStr.substring(8)
}
else {
project.version = tagStr
}
if (!project.hasProperty('useHadoop2')) {
project.version = project.version + "-hadoop1"
}
}
println "name=" + project.name + " group=" + project.group
println "project.version=" + project.version
if (!project.hasProperty('hadoopVersion')) {
if (project.hasProperty('useHadoop2')) {
ext.hadoopVersion = '2.3.0'
} else {
ext.hadoopVersion = '1.2.1'
}
}
if (!project.hasProperty('hiveVersion')) {
ext.hiveVersion = '1.2.1'
}
if (!project.hasProperty('pegasusVersion')) {
ext.pegasusVersion = '1.15.9'
}
if (!project.hasProperty('bytemanVersion')) {
ext.bytemanVersion = '2.2.1'
}
ext.avroVersion = '1.7.7'
ext.dropwizardMetricsVersion = '3.1.0'
ext.externalDependency = [
"antlrRuntime": "org.antlr:antlr-runtime:3.5.2",
"avro": "org.apache.avro:avro:" + avroVersion,
"avroMapredH1": "org.apache.avro:avro-mapred:" + avroVersion + ":hadoop1",
"avroMapredH2": "org.apache.avro:avro-mapred:" + avroVersion + ":hadoop2",
"commonsCli": "commons-cli:commons-cli:1.3.1",
"commonsCodec": "commons-codec:commons-codec:1.10",
"commonsDbcp": "commons-dbcp:commons-dbcp:1.4",
"commonsEmail": "org.apache.commons:commons-email:1.4",
"commonsLang": "commons-lang:commons-lang:2.6",
"commonsLang3": "org.apache.commons:commons-lang3:3.4",
"commonsConfiguration": "commons-configuration:commons-configuration:1.10",
"commonsIo": "commons-io:commons-io:2.4",
"commonsMath": "org.apache.commons:commons-math3:3.5",
"commonsHttpClient": "commons-httpclient:commons-httpclient:3.1",
"commonsCompress":"org.apache.commons:commons-compress:1.10",
"datanucleusCore": "org.datanucleus:datanucleus-core:4.1.2",
"datanucleusRdbms": "org.datanucleus:datanucleus-rdbms:4.1.2",
"guava": "com.google.guava:guava:15.0",
"gson": "com.google.code.gson:gson:2.3.1",
"findBugs": "com.google.code.findbugs:jsr305:1.3.9",
"hadoop": "org.apache.hadoop:hadoop-core:"+hadoopVersion,
"hadoopCommon": "org.apache.hadoop:hadoop-common:"+hadoopVersion,
"hadoopClientCore": "org.apache.hadoop:hadoop-mapreduce-client-core:"+hadoopVersion,
"hadoopClientCommon": "org.apache.hadoop:hadoop-mapreduce-client-common:"+hadoopVersion,
"hadoopHdfs": "org.apache.hadoop:hadoop-hdfs:"+hadoopVersion,
"hadoopAuth": "org.apache.hadoop:hadoop-auth:"+hadoopVersion,
"hadoopYarnApi": "org.apache.hadoop:hadoop-yarn-api:"+hadoopVersion,
"hadoopYarnCommon": "org.apache.hadoop:hadoop-yarn-common:"+hadoopVersion,
"hadoopYarnClient": "org.apache.hadoop:hadoop-yarn-client:"+hadoopVersion,
"hadoopYarnMiniCluster": "org.apache.hadoop:hadoop-minicluster:"+hadoopVersion,
"hadoopAnnotations": "org.apache.hadoop:hadoop-annotations:"+hadoopVersion,
"hadoopAws": "org.apache.hadoop:hadoop-aws:2.6.0",
"hiveService": "org.apache.hive:hive-service:"+hiveVersion,
"hiveJdbc": "org.apache.hive:hive-jdbc:"+hiveVersion,
"hiveMetastore": "org.apache.hive:hive-metastore:"+hiveVersion,
"hiveExec": "org.apache.hive:hive-exec:" + hiveVersion + ":core",
"hiveSerDe": "org.apache.hive:hive-serde:"+hiveVersion,
"httpclient": "org.apache.httpcomponents:httpclient:4.5",
"httpcore": "org.apache.httpcomponents:httpcore:4.4.1",
"kafka": "org.apache.kafka:kafka_2.11:0.8.2.1",
"kafkaTest": "org.apache.kafka:kafka_2.11:0.8.2.1:test",
"kafkaClient": "org.apache.kafka:kafka-clients:0.8.2.1",
"quartz": "org.quartz-scheduler:quartz:2.2.1",
"testng": "org.testng:testng:6.9.6",
"jacksonCore": "org.codehaus.jackson:jackson-core-asl:1.9.13",
"jacksonMapper": "org.codehaus.jackson:jackson-mapper-asl:1.9.13",
"jasypt": "org.jasypt:jasypt:1.9.2",
"slf4j": "org.slf4j:slf4j-api:1.7.12",
"log4j": "log4j:log4j:1.2.17",
"log4jextras": "log4j:apache-log4j-extras:1.2.17",
"slf4jLog4j": "org.slf4j:slf4j-log4j12:1.7.12",
"jodaTime": "joda-time:joda-time:2.9",
"metricsCore": "io.dropwizard.metrics:metrics-core:" + dropwizardMetricsVersion,
"metricsJvm": "io.dropwizard.metrics:metrics-jvm:" + dropwizardMetricsVersion,
"metricsGraphite": "io.dropwizard.metrics:metrics-graphite:" + dropwizardMetricsVersion,
"jsch": "com.jcraft:jsch:0.1.53",
"jdo2": "javax.jdo:jdo2-api:2.1",
"azkaban": "com.linkedin.azkaban:azkaban:2.5.0",
"commonsVfs": "org.apache.commons:commons-vfs2:2.0",
"mysqlConnector": "mysql:mysql-connector-java:5.1.36",
"javaxInject": "javax.inject:javax.inject:1",
"guice": "com.google.inject:guice:3.0",
"derby": "org.apache.derby:derby:10.11.1.1",
"mockito": "org.mockito:mockito-core:1.10.19",
"salesforceWsc": "com.force.api:force-wsc:29.0.0",
"salesforcePartner": "com.force.api:force-partner-api:29.0.0",
"scala": "org.scala-lang:scala-library:2.11.6",
"influxdbJava": "org.influxdb:influxdb-java:1.5",
"lombok":"org.projectlombok:lombok:1.16.4",
"mockRunnerJdbc":"com.mockrunner:mockrunner-jdbc:1.0.8",
"xerces":"xerces:xercesImpl:2.11.0",
"typesafeConfig": "com.typesafe:config:1.2.1",
"byteman": "org.jboss.byteman:byteman:"+bytemanVersion,
"bytemanBmunit": "org.jboss.byteman:byteman-bmunit:"+bytemanVersion,
"bcpgJdk15on": "org.bouncycastle:bcpg-jdk15on:1.52",
"bcprovJdk15on": "org.bouncycastle:bcprov-jdk15on:1.52",
"calciteCore": "org.apache.calcite:calcite-core:1.2.0-incubating",
"calciteAvatica": "org.apache.calcite:calcite-avatica:1.2.0-incubating",
"jhyde": "org.pentaho:pentaho-aggdesigner-algorithm:5.1.5-jhyde",
"curatorFramework": "org.apache.curator:curator-framework:2.8.0",
"curatorTest": "org.apache.curator:curator-test:2.8.0",
"hamcrest": "org.hamcrest:hamcrest-all:1.3",
"joptSimple": "net.sf.jopt-simple:jopt-simple:4.9",
"pegasus" : [
"data" : "com.linkedin.pegasus:data:"+pegasusVersion,
"generator" : "com.linkedin.pegasus:generator:"+pegasusVersion,
"restliClient" : "com.linkedin.pegasus:restli-client:"+pegasusVersion,
"restliServer" : "com.linkedin.pegasus:restli-server:"+pegasusVersion,
"restliTools" : "com.linkedin.pegasus:restli-tools:"+pegasusVersion,
"pegasusCommon" : "com.linkedin.pegasus:pegasus-common:"+pegasusVersion,
"restliCommon" : "com.linkedin.pegasus:restli-common:"+pegasusVersion,
"r2" : "com.linkedin.pegasus:r2:"+pegasusVersion,
"d2" : "com.linkedin.pegasus:d2:"+pegasusVersion,
"restliNettyStandalone" : "com.linkedin.pegasus:restli-netty-standalone:"+pegasusVersion
]
];
if (!isDefaultEnvironment)
{
ext.externalDependency.each { overrideDepKey, overrideDepValue ->
if (externalDependency[overrideDepKey] != null)
{
externalDependency[overrideDepKey] = overrideDepValue
}
}
}
task wrapper(type: Wrapper) { gradleVersion = '1.12' }
import javax.tools.ToolProvider
task javadocTarball(type: Tar) {
baseName = "gobblin-javadoc-all"
destinationDir = new File(project.buildDir, baseName)
compression = Compression.GZIP
extension = 'tgz'
description = "Generates a tar-ball with all javadocs to ${destinationDir}/${archiveName}"
}
javadocTarball << {
def indexFile = new File(destinationDir, "index.md")
def version = rootProject.ext.javadocVersion
indexFile << """----
layout: page
title: Gobblin Javadoc packages ${version}
permalink: /javadoc/${version}/
----
"""
rootProject.ext.javadocPackages.each {
indexFile << "* [${it}](${it})\n"
}
}
//Javadoc initialization for subprojects
ext.javadocVersion = null != project.version ? project.version.toString() : "latest"
if (ext.javadocVersion.indexOf('-') > 0) {
// Remove any "-" addons from the version
ext.javadocVersion = javadocVersion.substring(0, javadocVersion.indexOf('-'))
}
ext.javadocPackages = new HashSet<String>()
subprojects.each{Project pr ->
if (file(pr.projectDir.absolutePath + "/src/main/java").exists()) {
rootProject.ext.javadocPackages += pr.name
}
}
subprojects {
plugins.withType(JavaPlugin) {
//Sometimes generating javadocs can lead to OOM. This may needs to be increased.
javadoc {
options.JFlags = ["-Xmx256m"]
}
rootProject.tasks.javadocTarball.dependsOn project.tasks.javadoc
if ( rootProject.ext.javadocPackages.contains(project.name)) {
rootProject.tasks.javadocTarball.into(project.name){from(fileTree(dir: "${project.buildDir}/docs/javadoc/"))}
}
}
}
subprojects {
plugins.withType(JavaPlugin) {
plugins.apply('idea')
plugins.apply('eclipse')
plugins.apply('maven')
// Disabling the FindBugs plugin for now because FindBugs 2.x is not compatible with Java8
// apply plugin: 'findbugs'
sourceCompatibility = JavaVersion.VERSION_1_7
/* findbugs {
ignoreFailures = true
effort = "max"
excludeFilter = file(rootProject.projectDir.path + "/ligradle/findbugs/findbugsExclude.xml")
}
*/
test {
if (project.hasProperty("printTestOutput")) {
testLogging.showStandardStreams = true
}
}
configurations {
compile
dependencies {
if (project.hasProperty('useHadoop2')) {
compile externalDependency.hadoopCommon
compile externalDependency.hadoopClientCore
compile externalDependency.hadoopAnnotations
if (project.name.equals('gobblin-runtime') || project.name.equals('gobblin-test')) {
compile externalDependency.hadoopClientCommon
}
} else {
compile externalDependency.hadoop
}
compile(externalDependency.guava) {
force = true
}
// Required to add JDK's tool jar, which is required to run byteman tests.
compile (files(((URLClassLoader) ToolProvider.getSystemToolClassLoader()).getURLs()))
}
}
if (isDefaultEnvironment) {
task sourcesJar(type: Jar, dependsOn: classes) {
from sourceSets.main.allSource
classifier = 'sources'
}
task javadocJar(type: Jar) {
from javadoc
classifier = 'javadoc'
}
artifacts { archives sourcesJar, javadocJar }
}
// Publishing of maven artifacts for subprojects
if (rootProject.ext.publishToMaven) {
plugins.apply('maven')
if (rootProject.ext.signArtifacts) {
plugins.apply('signing')
}
project.version = rootProject.version
project.group = rootProject.group
uploadArchives {
repositories {
mavenDeployer {
beforeDeployment { MavenDeployment deployment ->
if (rootProject.ext.signArtifacts) {
signing.signPom(deployment)
}
}
repository(url: rootProject.artifactRepository) {
authentication(userName: ossrhUsername, password: ossrhPassword)
}
snapshotRepository(url: rootProject.artifactSnapshotRepository) {
authentication(userName: ossrhUsername, password: ossrhPassword)
}
pom.project {
name "${project.name}"
packaging 'jar'
// optionally artifactId can be defined here
description 'Gobblin Ingestion Framework'
url 'https://github.com/linkedin/gobblin/'
scm {
connection 'scm:git:git@github.com:linkedin/gobblin.git'
developerConnection 'scm:git:git@github.com:linkedin/gobblin.git'
url 'git@github.com:linkedin/gobblin.git'
}
licenses {
license {
name 'The Apache License, Version 2.0'
url 'http://www.apache.org/licenses/LICENSE-2.0.txt'
}
}
developers {
developer {
name 'Abhishek Tiwari'
organization 'LinkedIn'
}
developer {
name 'Chavdar Botev'
organization 'LinkedIn'
}
developer {
name 'Issac Buenrostro'
organization 'LinkedIn'
}
developer {
name 'Min Tu'
organization 'LinkedIn'
}
developer {
name 'Narasimha Veeramreddy'
organization 'LinkedIn'
}
developer {
name 'Pradhan Cadabam'
organization 'LinkedIn'
}
developer {
name 'Sahil Takiar'
organization 'LinkedIn'
}
developer {
name 'Shirshanka Das'
organization 'LinkedIn'
}
developer {
name 'Yinan Li'
organization 'LinkedIn'
}
developer {
name 'Ying Dai'
organization 'LinkedIn'
}
developer {
name 'Ziyang Liu'
organization 'LinkedIn'
}
}
}
}
}
}
if (rootProject.ext.signArtifacts) {
signing {
sign configurations.archives
}
}
}
// Configure the IDEA plugin to (1) add the codegen as source dirs and (2) work around
// an apparent bug in the plugin which doesn't set the outputDir/testOutputDir as documented
idea.project {
languageLevel = JavaVersion.VERSION_1_7
}
idea.module {
// Gradle docs claim the two settings below are the default, but
// the actual defaults appear to be "out/production/$MODULE_NAME"
// and "out/test/$MODULE_NAME". Changing it so IDEA and gradle share
// the class output directory.
outputDir = sourceSets.main.output.classesDir
testOutputDir = sourceSets.test.output.classesDir
}
// Add standard javadoc repositories so we can reference classes in them using @link
tasks.javadoc.options.links "http://typesafehub.github.io/config/latest/api/",
"https://docs.oracle.com/javase/7/docs/api/",
"http://docs.guava-libraries.googlecode.com/git-history/v15.0/javadoc/",
"http://hadoop.apache.org/docs/r${rootProject.ext.hadoopVersion}/api/",
"https://hive.apache.org/javadocs/r${rootProject.ext.hiveVersion}/api/",
"http://avro.apache.org/docs/${avroVersion}/api/java/",
"https://dropwizard.github.io/metrics/${dropwizardMetricsVersion}/apidocs/"
rootProject.ext.javadocPackages.each {
tasks.javadoc.options.linksOffline "http://linkedin.github.io/gobblin/javadoc/${javadocVersion}/${it}/",
"${rootProject.buildDir}/${it}/docs/javadoc/"
}
afterEvaluate {
// add the standard pegasus dependencies wherever the plugin is used
if (project.plugins.hasPlugin('pegasus')) {
dependencies {
dataTemplateCompile externalDependency.pegasus.data
restClientCompile externalDependency.pegasus.restliClient,externalDependency.pegasus.restliCommon,externalDependency.pegasus.restliTools
}
}
}
}
}
//Turn off javadoc lint for Java 8+
if (JavaVersion.current().isJava8Compatible()) {
allprojects {
tasks.withType(Javadoc) {
options.addStringOption('Xdoclint:none', '-quiet')
}
}
}