blob: 02d069e05ff013860960498e9e61d51b33148e7d [file] [log] [blame]
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
plugins {
id "org.nosphere.apache.rat" version "0.3.0"
}
apply plugin: 'java'
apply plugin: 'eclipse'
apply plugin: 'idea'
apply plugin: 'checkstyle'
apply plugin: 'jacoco'
apply plugin: 'findbugs'
apply plugin: 'maven-publish'
apply from: 'gradle/sqoop-version-gen.gradle'
apply from: 'gradle/sqoop-package.gradle'
group = 'org.apache.sqoopgradle'
sourceCompatibility = javaSourceCompatibilityVersion
repositories {
mavenCentral()
maven {
url 'http://conjars.org/repo/'
}
maven {
url 'http://maven.cubrid.org/'
}
// DB2 driver repository
maven {
url 'https://artifacts.alfresco.com/nexus/content/repositories/public/'
}
// Oracle driver repository
maven {
url 'http://www.datanucleus.org/downloads/maven2/'
}
}
sourceSets {
main {
java {
srcDirs = ['src/java']
}
resources {
srcDirs = ['src/java']
}
}
test {
java {
srcDirs = ['src/test']
exclude 'aop/'
}
resources {
srcDirs = ['src/test', 'testdata/hcatalog/conf', 'conf']
exclude 'aop/'
}
}
aop {
java {
srcDirs = ['src/test/aop']
}
resources {
srcDirs = ['src/test/aop']
}
}
}
configurations {
common.extendsFrom(redist)
compile.extendsFrom(common)
}
compileJava.dependsOn(SqoopVersionFileGen)
compileJava {
options.compilerArgs += '-proc:none'
options.compilerArgs += '-Xlint:all'
}
compileTestJava {
options.compilerArgs += '-proc:none'
options.compilerArgs += '-Xlint:all'
}
configurations.all {
resolutionStrategy {
force group: 'org.apache.avro', name: 'avro', version: avroVersion
force group: 'org.apache.avro', name: 'avro-mapred', version: avroVersion
force group: 'com.google.guava', name: 'guava', version: guavaVersion
force group: 'com.google.protobuf', name: 'protobuf-java', version: hiveProtobufVersion
}
exclude group: 'org.apache.hadoop', module: 'avro'
}
def forkEveryDefault = Integer.valueOf(System.getProperty("forkEvery.default", "0"))
def ignoreTestFailures = Boolean.getBoolean("ignoreTestFailures")
dependencies {
compile group: 'org.apache.hadoop', name: 'hadoop-aws', version: hadoopVersion
compile (group: 'org.apache.hadoop', name: 'hadoop-common', version: hadoopVersion) {
exclude group: "asm", module: "asm"
}
compile (group: 'org.apache.hadoop', name: 'hadoop-hdfs', version: hadoopVersion) {
exclude group: "asm", module: "asm"
}
compile (group: 'org.apache.hadoop', name: 'hadoop-mapreduce-client-common', version: hadoopVersion) {
exclude group: "asm", module: "asm"
}
compile (group: 'org.apache.hadoop', name: 'hadoop-mapreduce-client-core', version: hadoopVersion) {
exclude group: "asm", module: "asm"
}
compile group: 'org.apache.hbase', name: 'hbase-hadoop-compat', version: hbaseVersion
aopCompile group: 'org.aspectj', name: 'aspectjtools', version: aspectjVersion
aopCompile group: 'org.aspectj', name: 'aspectjrt', version: aspectjVersion
aopCompile group: 'org.apache.hadoop', name: 'hadoop-common', version: hadoopVersion
common group: 'org.apache.accumulo', name: 'accumulo-core', version: accumuloVersion
common group: 'org.apache.accumulo', name: 'accumulo-minicluster', version: accumuloVersion
common group: 'org.eclipse.jetty', name: 'jetty-runner', version: jettyVersion
common group: 'org.apache.hbase', name: 'hbase-hadoop2-compat', version: hbaseVersion
common group: 'org.apache.hbase', name: 'hbase-server', version: hbaseVersion
common group: 'org.apache.hbase', name: 'hbase-client', version: hbaseVersion
common group: 'org.apache.hbase', name: 'hbase-common', version: hbaseVersion
common(group: 'org.apache.hive.hcatalog', name: 'hive-hcatalog-core', version: hcatalogVersion) {
exclude group: 'org.apache.avro', module: 'avro'
exclude group: 'org.apache.hive', module: 'hive-exec'
}
common(group: 'org.apache.hive', name: 'hive-exec', version: hcatalogVersion, classifier: 'core') {
exclude group : 'org.apache.calcite', module :'calcite-core'
}
// Kryo and calcite are dependencies of hive-exec:core
common group: 'com.esotericsoftware', name: 'kryo', version: kryoVersion
common group: 'org.apache.calcite', name: 'calcite-core', version: calciteVersion
common(group: 'org.apache.hive', name: 'hive-jdbc', version: hcatalogVersion) {
exclude group: 'org.apache.avro', module: 'avro'
exclude group: 'asm', module: 'asm'
}
common group: 'commons-cli', name: 'commons-cli', version: commonscliVersion
common group: 'commons-logging', name: 'commons-logging', version: commonsloggingVersion
common group: 'commons-net', name: 'commons-net', version: commonsnetVersion
common group: 'log4j', name: 'log4j', version: log4jVersion
common group: 'org.postgresql', name: 'postgresql', version: postgresqlVersion
common group: 'org.apache.parquet', name: 'parquet-hadoop-bundle', version: parquetVersion
testCompile group: 'com.h2database', name: 'h2', version: h2Version
testCompile group: 'org.apache.hbase', name: 'hbase-server', version: hbaseVersion, classifier: 'tests'
testCompile group: 'org.apache.hbase', name: 'hbase-hadoop2-compat', version: hbaseVersion, classifier: 'tests'
testCompile group: 'org.apache.hbase', name: 'hbase-hadoop-compat', version: hbaseVersion, classifier: 'tests'
testCompile (group: 'org.apache.hadoop', name: 'hadoop-minikdc', version: hadoopVersion) {
exclude group: 'org.apache.directory.api', module: 'api-ldap-schema-data'
}
testCompile group: 'junit', name: 'junit', version: junitVersion
testCompile group: 'org.assertj', name: 'assertj-core', version: assertjVersion
testCompile group: 'org.mockito', name: 'mockito-core', version: mockitoallVersion
testCompile group: 'com.github.stefanbirkner', name: 'system-rules', version: systemRulesVersion
testCompile group: 'org.apache.zookeeper', name: 'zookeeper', version: zookeeperVersion, ext: 'jar'
// JDBC drivers
testCompile group: 'mysql', name: 'mysql-connector-java', version: mySqlConnectorVersion
testCompile group: 'cubrid', name: 'cubrid-jdbc', version: cubridJdbcDriverVersion
testCompile group: 'com.microsoft.sqlserver', name: 'mssql-jdbc', version: msSqlJdbcDriverVersion
testCompile group: 'com.ibm.db2.jcc', name: 'db2jcc4', version: db2JdbcDriverVersion
testCompile group: 'oracle', name: 'ojdbc6', version: oracleJdbcDriverVersion
}
task unitTest (type: Test) {
description 'Run unit tests which do not need proprietary JDBC driver'
useJUnit {
includeCategories 'org.apache.sqoop.testcategories.sqooptest.UnitTest'
excludeCategories 'org.apache.sqoop.testcategories.thirdpartytest.ThirdPartyTest'
}
}
task integrationTest(type: Test) {
description 'Run integration tests which do not need a docker container or an external database/service'
useJUnit {
includeCategories 'org.apache.sqoop.testcategories.sqooptest.IntegrationTest'
excludeCategories 'org.apache.sqoop.testcategories.sqooptest.ManualTest'
excludeCategories 'org.apache.sqoop.testcategories.KerberizedTest'
excludeCategories 'org.apache.sqoop.testcategories.thirdpartytest.ThirdPartyTest'
}
}
task thirdPartyTest (type: Test) {
description 'Run Third-party Tests - you need to specify -Dsqoop.thirdparty.lib.dir where the Third party driver ' +
'jars reside (relative to the project directory)'
useJUnit {
includeCategories 'org.apache.sqoop.testcategories.thirdpartytest.ThirdPartyTest'
excludeCategories 'org.apache.sqoop.testcategories.sqooptest.ManualTest'
excludeCategories 'org.apache.sqoop.testcategories.KerberizedTest'
}
systemProperty "ms.datatype.test.data.file.export" ,"DatatypeTestData-export-lite.txt"
systemProperty "ms.datatype.test.data.file.import" ,"DatatypeTestData-import-lite.txt"
systemProperty "ms.datatype.test.data.file.delim" ,","
systemProperty "ms.datatype.test.hdfsprefix" ,"file:///"
}
task postgreSqlTest (type: Test) {
description 'Run Postgresql third party tests.'
useJUnit {
includeCategories 'org.apache.sqoop.testcategories.thirdpartytest.PostgresqlTest'
excludeCategories 'org.apache.sqoop.testcategories.sqooptest.ManualTest'
excludeCategories 'org.apache.sqoop.testcategories.KerberizedTest'
}
}
task mySqlTest (type: Test) {
description 'Run MySql third party tests.'
useJUnit {
includeCategories 'org.apache.sqoop.testcategories.thirdpartytest.MysqlTest'
excludeCategories 'org.apache.sqoop.testcategories.sqooptest.ManualTest'
excludeCategories 'org.apache.sqoop.testcategories.KerberizedTest'
}
}
task cubridTest (type: Test) {
description 'Run Cubrid third party tests.'
useJUnit {
includeCategories 'org.apache.sqoop.testcategories.thirdpartytest.CubridTest'
excludeCategories 'org.apache.sqoop.testcategories.sqooptest.ManualTest'
excludeCategories 'org.apache.sqoop.testcategories.KerberizedTest'
}
}
task db2Test (type: Test) {
description 'Run DB2 third party tests.'
useJUnit {
includeCategories 'org.apache.sqoop.testcategories.thirdpartytest.Db2Test'
excludeCategories 'org.apache.sqoop.testcategories.sqooptest.ManualTest'
excludeCategories 'org.apache.sqoop.testcategories.KerberizedTest'
}
}
task mainFrameTest (type: Test) {
description 'Run Mainframe third party tests.'
useJUnit {
includeCategories 'org.apache.sqoop.testcategories.thirdpartytest.MainFrameTest'
excludeCategories 'org.apache.sqoop.testcategories.sqooptest.ManualTest'
excludeCategories 'org.apache.sqoop.testcategories.KerberizedTest'
}
}
task oracleTest (type: Test) {
description 'Run Oracle third party tests.'
useJUnit {
includeCategories 'org.apache.sqoop.testcategories.thirdpartytest.OracleTest'
excludeCategories 'org.apache.sqoop.testcategories.sqooptest.ManualTest'
excludeCategories 'org.apache.sqoop.testcategories.KerberizedTest'
}
}
task oracleXeTest (type: Test) {
description 'Run Oracle third party tests which can be run with Oracle Express Edition too.'
useJUnit {
includeCategories 'org.apache.sqoop.testcategories.thirdpartytest.OracleTest'
excludeCategories 'org.apache.sqoop.testcategories.thirdpartytest.OracleEeTest'
excludeCategories 'org.apache.sqoop.testcategories.sqooptest.ManualTest'
excludeCategories 'org.apache.sqoop.testcategories.KerberizedTest'
}
}
task oracleEeTest (type: Test) {
description 'Run Oracle third party tests which require Oracle Enterprise Edition.'
useJUnit {
includeCategories 'org.apache.sqoop.testcategories.thirdpartytest.OracleEeTest'
excludeCategories 'org.apache.sqoop.testcategories.sqooptest.ManualTest'
excludeCategories 'org.apache.sqoop.testcategories.KerberizedTest'
}
}
task sqlServerTest (type: Test) {
description 'Run MS SqlServer third party tests.'
useJUnit {
includeCategories 'org.apache.sqoop.testcategories.thirdpartytest.SqlServerTest'
excludeCategories 'org.apache.sqoop.testcategories.sqooptest.ManualTest'
excludeCategories 'org.apache.sqoop.testcategories.KerberizedTest'
}
systemProperty "ms.datatype.test.data.file.export" ,"DatatypeTestData-export-lite.txt"
systemProperty "ms.datatype.test.data.file.import" ,"DatatypeTestData-import-lite.txt"
systemProperty "ms.datatype.test.data.file.delim" ,","
systemProperty "ms.datatype.test.hdfsprefix" ,"file:///"
}
test {
description 'Run tests that do not need external JDBC driver and/or a docker container. ' +
'This is the same as running unitTest, integrationTest and kerberizedTest.'
useJUnit {
includeCategories 'org.apache.sqoop.testcategories.sqooptest.UnitTest'
includeCategories 'org.apache.sqoop.testcategories.sqooptest.IntegrationTest'
excludeCategories 'org.apache.sqoop.testcategories.KerberizedTest'
excludeCategories 'org.apache.sqoop.testcategories.thirdpartytest.ThirdPartyTest'
excludeCategories 'org.apache.sqoop.testcategories.sqooptest.ManualTest'
}
}
task s3Test(type: Test) {
description 'Run S3 tests'
useJUnit {
includeCategories 'org.apache.sqoop.testcategories.thirdpartytest.S3Test'
}
}
task allTest (type: Test){
description 'Run all Sqoop tests'
useJUnit {
includeCategories 'org.apache.sqoop.testcategories.sqooptest.UnitTest'
includeCategories 'org.apache.sqoop.testcategories.sqooptest.IntegrationTest'
includeCategories 'org.apache.sqoop.testcategories.thirdpartytest.ThirdPartyTest'
excludeCategories 'org.apache.sqoop.testcategories.KerberizedTest'
excludeCategories 'org.apache.sqoop.testcategories.sqooptest.ManualTest'
}
}
def testBuildDir = "$buildDir/test/"
def testBuildDirData ="$testBuildDir/data/"
tasks.withType(Test) {
doFirst {
project.mkdir(testBuildDir)
project.mkdir(testBuildDirData)
}
testLogging {
events 'passed', 'skipped'
}
workingDir = testBuildDirData
systemProperties(System.getProperties())
systemProperty "test.data.dir", "$projectDir/testdata"
systemProperty 'test.build.data', "$testBuildDir/data"
systemProperty "hadoop.root.logger", "DEBUG,console"
systemProperty "user.dir", workingDir
systemProperty "sqoop.src.dir", "$testBuildDir/data"
systemProperty "hadoop.tmp.dir", "$testBuildDir/hadoop"
systemProperty "fs.default.name", ""
systemProperty "hadoop.log.dir", "$testBuildDir/logs"
systemProperty "hive.home", "$projectDir/testdata/hive"
systemProperty "sqoop.throwOnError", ""
minHeapSize = "512m"
maxHeapSize = "8G"
jvmArgs "-da:org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge"
jacoco{
excludes = ["**/SqoopVersion*"]
}
forkEvery forkEveryDefault
ignoreFailures ignoreTestFailures
}
task kerberizedTest (type: Test){
description 'Run Kerberized Test'
// A Gradle test task with forkEvery 1 and includeCategories performs poorly because it starts a new JVM even for the filtered tests.
// To work around this performance problem we need to add every kerberized test in a separate include field here.
include '**/TestKerberosAuthenticator*'
include '**/HBaseKerberizedConnectivityTest*'
include '**/TestHiveMiniCluster*'
include '**/TestHiveServer2TextImport*'
useJUnit {
includeCategories 'org.apache.sqoop.testcategories.KerberizedTest'
}
forkEvery 1
ignoreFailures ignoreTestFailures
}
test.finalizedBy(kerberizedTest)
allTest.finalizedBy(kerberizedTest)
tasks.withType(Checkstyle) {
reports {
xml.enabled false
html.enabled true
html.stylesheet resources.text.fromFile('config/checkstyle/checkstyle-noframes.xsl')
}
ignoreFailures = true
}
tasks.withType(FindBugs) {
reports {
xml.enabled false
html.enabled true
}
ignoreFailures = false
}
findbugs {
sourceSets = []
}
checkstyle {
sourceSets = []
toolVersion checkstyleVersion
}
jacocoTestReport {
description "Generates a coverage report of the unit test results under $buildDir/jacocoHtml/test/index.html"
executionData test, kerberizedTest
reports {
xml.enabled true
csv.enabled false
html.destination file("$buildDir/jacocoHtml/test")
}
}
task jacocoThirdPartyReport(type: JacocoReport){
description "Generates a coverage report of the thirdparty test results under $buildDir/jacocoHtml/3rd/index.html"
sourceSets sourceSets.main
executionData thirdPartyTest
reports {
xml.enabled true
csv.enabled false
html.destination file("$buildDir/jacocoHtml/3rd")
}
}
task jacocoCompositeReport(type: JacocoReport){
description "Generates a composite coverage report of test and thirdparty test results under $buildDir/jacocoHtml/index.html"
sourceSets sourceSets.main
executionData test, kerberizedTest, thirdPartyTest
reports {
xml.enabled true
csv.enabled false
html.destination file("$buildDir/jacocoHtml/")
}
}
javadoc {
source = sourceSets.main.allJava
include 'com/cloudera/sqoop/lib/**'
include 'org/apache/sqoop/lib/**'
destinationDir = reporting.file("${project.docsDir}/api")
}
task faultInjectionJar(type: Jar) {
description 'Create the test jar'
appendix = 'test'
from sourceSets.aop.output
}
def archiveName = jar.baseName + "-" + jar.version
def binArtifactName = archiveName + ".bin__hadoop-" + hadoopVersion
def distDir = "$buildDir/$binArtifactName"
rat {
inputDir = distDir
reportDir = project.file(buildDir)
excludes = ['**/docs/**', '**/testdata/**', '**/.idea/**', '**/test-dir/**', '**/.settings/**', '**/.gradle/**','**/gradle/**' ,'**/gradle/build/**', '*.iml', '*.iws', '*.ipr', 'derby.log', '**/gradlew**' ]
failOnError = true
xmlOutput = false
htmlOutput = true
plainOutput = false
}
rat.dependsOn('packageDist')
def scripts = ['export', 'list-databases', 'metastore', 'create-hive-table', 'help',
'import-mainframe', 'list-tables', 'version',
'eval', 'import', 'job', 'merge',
'import-all-tables', 'codegen']
task runSqoopHelp(type: JavaExec, dependsOn: 'classes') {
main = 'org.apache.sqoop.Sqoop'
jvmArgs = ['-Dhadoop.security.log.file=./build/security-audit.log']
classpath = sourceSets.main.runtimeClasspath + configurations.runtime
args = ['help']
standardOutput = new ByteArrayOutputStream()
ext.helpOut = {
standardOutput.toString()
}
}
task createAllStartScripts(dependsOn: 'runSqoopHelp' ) doLast{
// Placeholder task with validation
// We call Sqoop help to verify that all tasks that we create scripts for are present in Sqoop help and vice-versa
// We currently depend on the format of the help output to parse the relevant information out
def matcher = (tasks.runSqoopHelp.helpOut() =~/(?m)^\s\s([a-z-]+)\s+\w+/)
def matches = new String[matcher.getCount()]
for (def i=0; i < matcher.getCount(); i++) {
matches[i] = matcher[i][1]
}
if (matches.sort() != scripts.sort())
throw new GradleException('Error: script list in gradle build script and result of Sqoop help are not the same, but should be')
}
scripts.each() { scriptName ->
def t = tasks.create(name: scriptName + 'StartScript', type: CreateStartScripts) {
unixStartScriptGenerator.template = resources.text.fromFile('gradle/customUnixStartScript.txt')
windowsStartScriptGenerator.template = resources.text.fromFile('gradle/customWindowsStartScript.txt')
outputDir = file("$buildDir/bin")
applicationName = scriptName
mainClassName = 'com.cloudera.sqoop.Sqoop'
classpath = project.configurations.runtime
doLast {
file("$buildDir/bin/"+ scriptName).renameTo(file("$buildDir/bin/"+"sqoop-"+scriptName))
file("$buildDir/bin/"+ scriptName +".bat").renameTo(file("$buildDir/bin/"+"sqoop-"+scriptName+".bat"))
}
}
t.dependsOn('compileJava')
createAllStartScripts.dependsOn(t)
}
task checkVersion doLast {
if (version.contains('SNAPSHOT'))
throw new GradleException('Error: cannot release a snapshot. Set -Pversion')
}
task release(dependsOn: ['checkVersion', 'tar', 'rat']) {
doLast {
println 'Release complete'
println "Binary tar: $buildDir/$binArtifactName-${version}.tar.gz"
println "Documentation: $buildDir/docs"
println "Release notes: $buildDir/docs/sqoop-$version" + ".releasenotes.html"
println "Release audit report: $buildDir/index.html"
}
}
task relnotes(type: Exec) {
workingDir 'src/scripts'
if (!version.contains('SNAPSHOT')) {
commandLine "python", "relnotes.py", "$buildDir/docs", "$projectDir", "$oldHash\\..HEAD", "$version", "$oldVersion"
} else {
commandLine "true" //noop
println "Will not run releasenotes for SNAPSHOT version."
}
}
task realDocs(type: Exec, dependsOn: ['relnotes', 'javadoc']) {
workingDir projectDir
commandLine "make", "-C", "$projectDir/src/docs", "BUILDROOT=$buildDir", "VERSION=$version"
}
task docs(dependsOn: ['realDocs', 'relnotes', 'javadoc']) {
//placeholder
}
def $mvnRepo = System.getProperty("mvnRepo", "snapshots")
publishing {
repositories {
maven {
name "cloudera." + $mvnRepo + ".repo"
url "https://repository.cloudera.com/content/repositories/"+$mvnRepo
credentials {
username 'username'
password 'password'
}
}
}
}
publishing {
publications {
sqoop(MavenPublication) {
groupId group
version version
from components.java
artifact sourceJar {
classifier "sources"
}
}
sqoopTest(MavenPublication) {
groupId group
version version
artifactId 'sqoop-test'
artifact testJar
artifact sourceTestJar
pom.withXml {
def dependencies = asNode().appendNode('dependencies')
configurations.testRuntime.getResolvedConfiguration().getFirstLevelModuleDependencies().each {
def dependency = dependencies.appendNode('dependency')
dependency.appendNode('groupId', it.moduleGroup)
dependency.appendNode('artifactId', it.moduleName)
dependency.appendNode('version', it.moduleVersion)
}
}
}
}
}
wrapper {
gradleVersion = '4.9'
}