blob: ee82712b40a83374ab0ae0d351641c939cb22daa [file] [log] [blame]
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import groovy.transform.Memoized
import java.util.regex.Matcher
import java.util.regex.Pattern
buildscript {
repositories {
gradlePluginPortal()
}
dependencies {
classpath 'gradle.plugin.com.github.jengelman.gradle.plugins:shadow:7.0.0'
classpath 'com.palantir.baseline:gradle-baseline-java:4.0.0'
classpath 'com.palantir.gradle.gitversion:gradle-git-version:0.12.3'
classpath 'com.diffplug.spotless:spotless-plugin-gradle:6.8.0'
classpath 'gradle.plugin.org.inferred:gradle-processors:3.3.0'
classpath 'me.champeau.jmh:jmh-gradle-plugin:0.6.6'
classpath "com.github.alisiikh:gradle-scalastyle-plugin:3.4.1"
classpath 'com.palantir.gradle.revapi:gradle-revapi:1.7.0'
classpath 'com.gorylenko.gradle-git-properties:gradle-git-properties:2.4.1'
}
}
plugins {
id 'nebula.dependency-recommender' version '11.0.0'
}
try {
// apply these plugins in a try-catch block so that we can handle cases without .git directory
apply plugin: 'com.palantir.git-version'
} catch (Exception e) {
project.logger.error(e.getMessage())
}
if (JavaVersion.current() == JavaVersion.VERSION_1_8) {
project.ext.jdkVersion = '8'
} else if (JavaVersion.current() == JavaVersion.VERSION_11) {
project.ext.jdkVersion = '11'
} else {
throw new GradleException("This build must be run with JDK 8 or 11 but was executed with JDK " + JavaVersion.current())
}
apply plugin: 'com.gorylenko.gradle-git-properties'
// git properties file for the root project for adding to the source tarball
gitProperties {
gitPropertiesName = 'iceberg-build.properties'
gitPropertiesResourceDir = file("${rootDir}/build")
extProperty = 'gitProps'
failOnNoGitDirectory = true
keys = ['git.branch', 'git.build.version', 'git.closest.tag.name','git.commit.id.abbrev', 'git.commit.id',
'git.commit.message.short', 'git.commit.time', 'git.tags']
}
generateGitProperties.outputs.upToDateWhen { false }
if (file("${rootDir}/iceberg-build.properties").exists()) {
tasks.register('buildInfo', Exec) {
project.logger.info('Using build info from iceberg-build.properties')
commandLine 'cp', "${rootDir}/iceberg-build.properties", 'build/iceberg-build.properties'
}
} else {
tasks.register('buildInfo') {
project.logger.info('Generating iceberg-build.properties from git')
dependsOn generateGitProperties
}
}
dependencyRecommendations {
propertiesFile file: file('versions.props')
}
def projectVersion = getProjectVersion()
allprojects {
group = "org.apache.iceberg"
version = projectVersion
repositories {
mavenCentral()
mavenLocal()
}
}
subprojects {
apply plugin: 'nebula.dependency-recommender'
apply plugin: 'java-library'
configurations {
testImplementation.extendsFrom compileOnly
compileClasspath {
// do not exclude Guava so the bundle project can reference classes.
if (project.name != 'iceberg-bundled-guava') {
exclude group: 'com.google.guava', module: 'guava'
}
// contains a copy of Guava
exclude group: 'org.apache.spark', module: 'spark-network-common_2.12'
}
all {
exclude group: 'org.slf4j', module: 'slf4j-log4j12'
exclude group: 'org.mortbay.jetty'
exclude group: 'com.sun.jersey'
exclude group: 'com.sun.jersey.contribs'
exclude group: 'org.pentaho', module: 'pentaho-aggdesigner-algorithm'
}
testArtifacts
}
compileJava {
options.encoding = "UTF-8"
}
compileTestJava {
options.encoding = "UTF-8"
}
javadoc {
options.encoding = 'UTF-8'
}
sourceCompatibility = '1.8'
targetCompatibility = '1.8'
dependencies {
implementation 'org.slf4j:slf4j-api'
implementation 'com.github.stephenc.findbugs:findbugs-annotations'
testImplementation 'org.junit.vintage:junit-vintage-engine'
testImplementation 'org.junit.jupiter:junit-jupiter-engine'
testImplementation 'org.junit.jupiter:junit-jupiter'
testImplementation 'org.slf4j:slf4j-simple'
testImplementation 'org.mockito:mockito-core'
testImplementation 'org.mockito:mockito-inline'
testImplementation 'org.assertj:assertj-core'
}
test {
def logDir = "${rootDir}/build/testlogs"
def logFile = "${logDir}/${project.name}.log"
mkdir("${logDir}")
delete("${logFile}")
def buildLog = new File(logFile)
addTestOutputListener(new TestOutputListener() {
def lastDescriptor
@Override
void onOutput(TestDescriptor testDescriptor, TestOutputEvent testOutputEvent) {
if (lastDescriptor != testDescriptor) {
buildLog << "--------\n- Test log for: "<< testDescriptor << "\n--------\n"
lastDescriptor = testDescriptor
}
buildLog << testOutputEvent.destination << " " << testOutputEvent.message
}
})
maxHeapSize = "1500m"
testLogging {
events "failed"
exceptionFormat "full"
}
}
}
project(':iceberg-bundled-guava') {
apply plugin: 'com.github.johnrengelman.shadow'
tasks.jar.dependsOn tasks.shadowJar
dependencies {
compileOnly('com.google.guava:guava') {
exclude group: 'com.google.code.findbugs'
// may be LGPL - use ALv2 findbugs-annotations instead
exclude group: 'com.google.errorprone'
exclude group: 'com.google.j2objc'
}
}
shadowJar {
classifier null
configurations = [project.configurations.compileClasspath]
zip64 true
// include the LICENSE and NOTICE files for the shaded Jar
from(projectDir) {
include 'LICENSE'
include 'NOTICE'
}
dependencies {
exclude(dependency('com.github.stephenc.findbugs:findbugs-annotations'))
exclude(dependency('org.slf4j:slf4j-api'))
exclude(dependency('org.checkerframework:checker-qual'))
}
relocate 'com.google.common', 'org.apache.iceberg.relocated.com.google.common'
minimize()
}
jar {
archiveClassifier = 'empty'
}
}
project(':iceberg-api') {
apply plugin: 'com.palantir.revapi'
dependencies {
implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
compileOnly "com.google.errorprone:error_prone_annotations"
testImplementation "org.apache.avro:avro"
}
tasks.processTestResources.dependsOn rootProject.tasks.buildInfo
sourceSets {
test {
resources {
srcDir "${rootDir}/build"
}
}
}
}
project(':iceberg-common') {
dependencies {
implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
}
}
project(':iceberg-core') {
test {
useJUnitPlatform()
}
dependencies {
api project(':iceberg-api')
implementation project(':iceberg-common')
implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
implementation("org.apache.avro:avro") {
exclude group: 'org.tukaani' // xz compression is not supported
}
implementation 'io.airlift:aircompressor'
implementation 'org.apache.httpcomponents.client5:httpclient5'
implementation "com.fasterxml.jackson.core:jackson-databind"
implementation "com.fasterxml.jackson.core:jackson-core"
implementation "com.github.ben-manes.caffeine:caffeine"
implementation "org.roaringbitmap:RoaringBitmap"
compileOnly("org.apache.hadoop:hadoop-client") {
exclude group: 'org.apache.avro', module: 'avro'
exclude group: 'org.slf4j', module: 'slf4j-log4j12'
}
testImplementation 'org.mock-server:mockserver-netty'
testImplementation 'org.mock-server:mockserver-client-java'
testImplementation "org.xerial:sqlite-jdbc"
testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
}
}
project(':iceberg-data') {
dependencies {
implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
api project(':iceberg-api')
implementation project(':iceberg-core')
compileOnly project(':iceberg-parquet')
compileOnly project(':iceberg-orc')
compileOnly("org.apache.hadoop:hadoop-common") {
exclude group: 'commons-beanutils'
exclude group: 'org.apache.avro', module: 'avro'
exclude group: 'org.slf4j', module: 'slf4j-log4j12'
}
implementation("org.apache.orc:orc-core::nohive") {
exclude group: 'org.apache.hadoop'
exclude group: 'commons-lang'
// These artifacts are shaded and included in the orc-core fat jar
exclude group: 'com.google.protobuf', module: 'protobuf-java'
exclude group: 'org.apache.hive', module: 'hive-storage-api'
}
implementation("org.apache.parquet:parquet-avro") {
exclude group: 'org.apache.avro', module: 'avro'
// already shaded by Parquet
exclude group: 'it.unimi.dsi'
exclude group: 'org.codehaus.jackson'
}
compileOnly "org.apache.avro:avro"
testImplementation("org.apache.hadoop:hadoop-client") {
exclude group: 'org.apache.avro', module: 'avro'
exclude group: 'org.slf4j', module: 'slf4j-log4j12'
}
testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts')
}
test {
// Only for TestSplitScan as of Gradle 5.0+
maxHeapSize '1500m'
}
}
project(':iceberg-aliyun') {
dependencies {
implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
api project(':iceberg-api')
implementation project(':iceberg-core')
implementation project(':iceberg-common')
compileOnly 'com.aliyun.oss:aliyun-sdk-oss'
compileOnly 'javax.xml.bind:jaxb-api'
compileOnly 'javax.activation:activation'
compileOnly 'org.glassfish.jaxb:jaxb-runtime'
compileOnly("org.apache.hadoop:hadoop-common") {
exclude group: 'org.apache.avro', module: 'avro'
exclude group: 'org.slf4j', module: 'slf4j-log4j12'
exclude group: 'javax.servlet', module: 'servlet-api'
exclude group: 'com.google.code.gson', module: 'gson'
}
testImplementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-xml'
testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
testImplementation 'org.springframework:spring-web'
testImplementation('org.springframework.boot:spring-boot-starter-jetty') {
exclude module: 'logback-classic'
exclude group: 'org.eclipse.jetty.websocket', module: 'javax-websocket-server-impl'
exclude group: 'org.eclipse.jetty.websocket', module: 'websocket-server'
}
testImplementation('org.springframework.boot:spring-boot-starter-web') {
exclude module: 'logback-classic'
exclude module: 'spring-boot-starter-logging'
}
}
}
project(':iceberg-aws') {
dependencies {
implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
api project(':iceberg-api')
implementation project(':iceberg-common')
implementation project(':iceberg-core')
compileOnly 'software.amazon.awssdk:url-connection-client'
compileOnly 'software.amazon.awssdk:apache-client'
compileOnly 'software.amazon.awssdk:s3'
compileOnly 'software.amazon.awssdk:kms'
compileOnly 'software.amazon.awssdk:glue'
compileOnly 'software.amazon.awssdk:sts'
compileOnly 'software.amazon.awssdk:dynamodb'
compileOnly 'software.amazon.awssdk:lakeformation'
compileOnly("org.apache.hadoop:hadoop-common") {
exclude group: 'org.apache.avro', module: 'avro'
exclude group: 'org.slf4j', module: 'slf4j-log4j12'
exclude group: 'javax.servlet', module: 'servlet-api'
exclude group: 'com.google.code.gson', module: 'gson'
}
testImplementation 'software.amazon.awssdk:iam'
testImplementation 'software.amazon.awssdk:s3control'
testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
testImplementation("com.adobe.testing:s3mock-junit4") {
exclude module: "spring-boot-starter-logging"
exclude module: "logback-classic"
exclude group: 'junit'
}
}
sourceSets {
integration {
java.srcDir "$projectDir/src/integration/java"
resources.srcDir "$projectDir/src/integration/resources"
compileClasspath += main.output + test.output
runtimeClasspath += main.output + test.output
}
}
configurations {
integrationImplementation.extendsFrom testImplementation
integrationRuntime.extendsFrom testRuntimeOnly
}
task integrationTest(type: Test) {
testClassesDirs = sourceSets.integration.output.classesDirs
classpath = sourceSets.integration.runtimeClasspath
}
}
project(':iceberg-gcp') {
dependencies {
implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
api project(':iceberg-api')
implementation project(':iceberg-common')
implementation project(':iceberg-core')
implementation platform('com.google.cloud:libraries-bom')
implementation 'com.google.cloud:google-cloud-storage'
testImplementation 'com.google.cloud:google-cloud-nio'
testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
testImplementation("org.apache.hadoop:hadoop-common") {
exclude group: 'org.apache.avro', module: 'avro'
exclude group: 'org.slf4j', module: 'slf4j-log4j12'
exclude group: 'javax.servlet', module: 'servlet-api'
exclude group: 'com.google.code.gson', module: 'gson'
}
}
}
project(':iceberg-hive-metastore') {
dependencies {
implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
implementation project(':iceberg-core')
api project(':iceberg-api')
implementation project(':iceberg-common')
implementation "com.github.ben-manes.caffeine:caffeine"
compileOnly "org.apache.avro:avro"
compileOnly("org.apache.hive:hive-metastore") {
exclude group: 'org.apache.avro', module: 'avro'
exclude group: 'org.slf4j', module: 'slf4j-log4j12'
exclude group: 'org.pentaho' // missing dependency
exclude group: 'org.apache.hbase'
exclude group: 'org.apache.logging.log4j'
exclude group: 'co.cask.tephra'
exclude group: 'com.google.code.findbugs', module: 'jsr305'
exclude group: 'org.eclipse.jetty.aggregate', module: 'jetty-all'
exclude group: 'org.eclipse.jetty.orbit', module: 'javax.servlet'
exclude group: 'org.apache.parquet', module: 'parquet-hadoop-bundle'
exclude group: 'com.tdunning', module: 'json'
exclude group: 'javax.transaction', module: 'transaction-api'
exclude group: 'com.zaxxer', module: 'HikariCP'
}
// By default, hive-exec is a fat/uber jar and it exports a guava library
// that's really old. We use the core classifier to be able to override our guava
// version. Luckily, hive-exec seems to work okay so far with this version of guava
// See: https://github.com/apache/hive/blob/master/ql/pom.xml#L911 for more context.
testImplementation("org.apache.hive:hive-exec::core") {
exclude group: 'org.apache.avro', module: 'avro'
exclude group: 'org.slf4j', module: 'slf4j-log4j12'
exclude group: 'org.pentaho' // missing dependency
exclude group: 'org.apache.hive', module: 'hive-llap-tez'
exclude group: 'org.apache.logging.log4j'
exclude group: 'com.google.protobuf', module: 'protobuf-java'
exclude group: 'org.apache.calcite'
exclude group: 'org.apache.calcite.avatica'
exclude group: 'com.google.code.findbugs', module: 'jsr305'
}
testImplementation("org.apache.hive:hive-metastore") {
exclude group: 'org.apache.avro', module: 'avro'
exclude group: 'org.slf4j', module: 'slf4j-log4j12'
exclude group: 'org.pentaho' // missing dependency
exclude group: 'org.apache.hbase'
exclude group: 'org.apache.logging.log4j'
exclude group: 'co.cask.tephra'
exclude group: 'com.google.code.findbugs', module: 'jsr305'
exclude group: 'org.eclipse.jetty.aggregate', module: 'jetty-all'
exclude group: 'org.eclipse.jetty.orbit', module: 'javax.servlet'
exclude group: 'org.apache.parquet', module: 'parquet-hadoop-bundle'
exclude group: 'com.tdunning', module: 'json'
exclude group: 'javax.transaction', module: 'transaction-api'
exclude group: 'com.zaxxer', module: 'HikariCP'
}
compileOnly("org.apache.hadoop:hadoop-client") {
exclude group: 'org.apache.avro', module: 'avro'
exclude group: 'org.slf4j', module: 'slf4j-log4j12'
}
testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
}
}
project(':iceberg-orc') {
dependencies {
implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
api project(':iceberg-api')
implementation project(':iceberg-common')
implementation project(':iceberg-core')
implementation("org.apache.avro:avro") {
exclude group: 'org.tukaani' // xz compression is not supported
}
implementation("org.apache.orc:orc-core::nohive") {
exclude group: 'org.apache.hadoop'
exclude group: 'commons-lang'
// These artifacts are shaded and included in the orc-core fat jar
exclude group: 'com.google.protobuf', module: 'protobuf-java'
exclude group: 'org.apache.hive', module: 'hive-storage-api'
}
compileOnly("org.apache.hadoop:hadoop-common") {
exclude group: 'commons-beanutils'
exclude group: 'org.apache.avro', module: 'avro'
exclude group: 'org.slf4j', module: 'slf4j-log4j12'
}
compileOnly("org.apache.hadoop:hadoop-client") {
exclude group: 'org.apache.avro', module: 'avro'
}
testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
testImplementation project(':iceberg-common')
}
}
project(':iceberg-parquet') {
dependencies {
implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
api project(':iceberg-api')
implementation project(':iceberg-core')
implementation project(':iceberg-common')
implementation("org.apache.parquet:parquet-avro") {
exclude group: 'org.apache.avro', module: 'avro'
// already shaded by Parquet
exclude group: 'it.unimi.dsi'
exclude group: 'org.codehaus.jackson'
}
compileOnly "org.apache.avro:avro"
compileOnly("org.apache.hadoop:hadoop-client") {
exclude group: 'org.apache.avro', module: 'avro'
}
testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts')
}
}
project(':iceberg-arrow') {
dependencies {
implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
api project(':iceberg-api')
implementation project(':iceberg-core')
implementation project(':iceberg-parquet')
implementation("org.apache.arrow:arrow-vector") {
exclude group: 'io.netty', module: 'netty-buffer'
exclude group: 'io.netty', module: 'netty-common'
exclude group: 'com.google.code.findbugs', module: 'jsr305'
}
implementation("org.apache.arrow:arrow-memory-netty") {
exclude group: 'com.google.code.findbugs', module: 'jsr305'
exclude group: 'io.netty', module: 'netty-common'
exclude group: 'io.netty', module: 'netty-buffer'
}
runtimeOnly("io.netty:netty-buffer")
implementation("org.apache.parquet:parquet-avro") {
exclude group: 'org.apache.avro', module: 'avro'
// already shaded by Parquet
exclude group: 'it.unimi.dsi'
exclude group: 'org.codehaus.jackson'
}
testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts')
// To run ArrowReaderTest test cases, :netty-common is needed.
// We import :netty-common through :arrow-memory-netty
// so that the same version as used by the :arrow-memory-netty module is picked.
testImplementation("org.apache.arrow:arrow-memory-netty")
testImplementation("org.apache.hadoop:hadoop-common")
testImplementation("org.apache.hadoop:hadoop-mapreduce-client-core")
}
}
project(':iceberg-pig') {
dependencies {
implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
api project(':iceberg-api')
implementation project(':iceberg-common')
implementation project(':iceberg-core')
implementation project(':iceberg-parquet')
implementation("org.apache.parquet:parquet-avro") {
exclude group: 'org.apache.avro', module: 'avro'
// already shaded by Parquet
exclude group: 'it.unimi.dsi'
exclude group: 'org.codehaus.jackson'
}
compileOnly("org.apache.pig:pig") {
exclude group: "junit", module: "junit"
}
compileOnly("org.apache.hadoop:hadoop-mapreduce-client-core")
compileOnly("org.apache.hadoop:hadoop-client") {
exclude group: 'org.apache.avro', module: 'avro'
}
testImplementation("org.apache.hadoop:hadoop-minicluster") {
exclude group: 'org.apache.avro', module: 'avro'
}
}
}
project(':iceberg-nessie') {
test {
useJUnitPlatform()
}
dependencies {
api project(':iceberg-api')
implementation project(':iceberg-common')
implementation project(':iceberg-core')
implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
implementation "org.projectnessie:nessie-client"
implementation "com.fasterxml.jackson.core:jackson-databind"
implementation "com.fasterxml.jackson.core:jackson-core"
testImplementation "org.projectnessie:nessie-jaxrs-testextension"
// Need to "pull in" el-api explicitly :(
testImplementation "jakarta.el:jakarta.el-api"
compileOnly "org.apache.hadoop:hadoop-common"
testImplementation "org.apache.avro:avro"
testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts')
// Only there to prevent "warning: unknown enum constant SchemaType.OBJECT" compile messages
compileOnly "org.eclipse.microprofile.openapi:microprofile-openapi-api:3.0"
testCompileOnly "org.eclipse.microprofile.openapi:microprofile-openapi-api:3.0"
}
}
project(':iceberg-dell') {
dependencies {
implementation project(':iceberg-core')
implementation project(':iceberg-common')
implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
compileOnly 'com.emc.ecs:object-client-bundle'
testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
testImplementation "javax.xml.bind:jaxb-api"
testImplementation "javax.activation:activation"
testImplementation "org.glassfish.jaxb:jaxb-runtime"
}
}
@Memoized
boolean versionFileExists() {
return file('version.txt').exists()
}
@Memoized
String getVersionFromFile() {
return file('version.txt').text.trim()
}
String getProjectVersion() {
if (versionFileExists()) {
return getVersionFromFile()
}
try {
// we're fetching the version from the latest tag (prefixed with 'apache-iceberg-'),
// which can look like this: '0.13.0-2-g805400f0.dirty' but we're only interested in the MAJOR.MINOR.PATCH part
String version = gitVersion(prefix: 'apache-iceberg-')
Pattern pattern = Pattern.compile("^([0-9]+)\\.([0-9]+)\\.([0-9]+)(.*)?\$")
Matcher matcher = pattern.matcher(version)
if (matcher.matches()) {
// bump the MINOR version and always set the PATCH version to 0
return matcher.group(1) + "." + (Integer.valueOf(matcher.group(2)) + 1) + ".0-SNAPSHOT"
}
return version
} catch (Exception e) {
throw new Exception("Neither version.txt nor git version exists: " + e.getMessage(), e)
}
}
String getJavadocVersion() {
if (versionFileExists()) {
return getVersionFromFile()
}
try {
// use the branch name in place of version in Javadoc
return versionDetails().branchName
} catch (NullPointerException e) {
throw new Exception("Neither version.txt nor git version exists")
}
}
apply from: 'jmh.gradle'
apply from: 'baseline.gradle'
apply from: 'deploy.gradle'
apply from: 'tasks.gradle'