| /* |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * "License"); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| /** DSL documentation |
| The structure of this configuration DSL is pretty self-explanatory. The main |
| section is 'components' that contains the description of all things included |
| into the stack. The following words are considered terminals and shouldn't be |
| modified as it will affect the parsing behavior: |
| - bigtop |
| - version |
| - stack |
| - version_base |
| - apache |
| - pkg |
| - tarball, source, destinations |
| - url, download_path, site, archive |
| Also, the following are base constants and if renamed will affect the some of |
| the stack's definitions |
| - APACHE_MIRROR |
| - APACHE_ARCHIVE |
| |
| bigtop { // *the name should be change: the parsing code depends on it* |
| version = "STACK-VERSION" // *required* |
| stack { // *required* Fundamental properties of the Stack: JDK, SDK, GDK, etc |
| 'jdk' { version = '1.8'; version_base = version } |
| 'scala' { version = '2.10.4'; version_base = version } |
| } |
| apache { // *required* These shoudn't be modified unless ASF Infra demands changes |
| APACHE_MIRROR = "https://apache.osuosl.org" |
| APACHE_ARCHIVE = "https://archive.apache.org/dist" |
| } |
| git { // *optional* This is a global setting to access protected git repositories, |
| // can be specified per component as well |
| user = "john_doe" |
| token = "john's access token" |
| } |
| components { *required; preserve the name* if empty, nothing will be built |
| 'label' { // label *SHOULD* be the same as the name; otherwise some tasks will fail |
| name = 'component1' // *required* the name of the component |
| // 'pkg' value is optional and will be set to that of 'name' i.e. [pkg := name] |
| pkg = name // *optional* and will be set to the 'name' value |
| // 'base' is required; [pkg := base ]; [release := 1 ] |
| version { base = 'x.y.z'; pkg = base; release = 1 } |
| tarball { |
| source = "apache-component1-${version.base}.tar.gz" |
| // It is advised to use different destination filenames to avoid |
| // clashes when working with git repos and downloading the artifacts |
| // from the branches with the same names. |
| destination = source |
| } |
| url { // *optional* |
| download_path = "/component1/component1-${version.base}" |
| site = "${apache.APACHE_MIRROR}/${download_path}" |
| archive = "${apache.APACHE_ARCHIVE}/${download_path}" |
| } |
| git { |
| // Setting the info to access a git repository. Ref is any valid git reference. |
| // If git repo information is provided, the *url* element above will be ignored. |
| repo = "https://github.com/apache/bigtop.git" |
| ref = "branch-name" |
| // *dir* defines the name of the top-level folder inside of the tar-ball archive. |
| // if set to null, the directory name will be set to tar-ball.dist without |
| // the .tar* suffix |
| // This setting is important to allow build to locate unpacked source code |
| dir = "${name}-${version.base}-src" |
| // *optional* |
| // You can setup repo-specific user credentials overriding any global settings |
| user = "john_doe" |
| token = "john's access token" |
| } |
| } |
| } |
| } |
| End of DSL Documentation */ |
| |
| bigtop { |
| /** Base Configuration of the mirror and archives */ |
| version = "1.6.0-SNAPSHOT" |
| stack { |
| 'jdk' { version = "1." + ( System.getenv('BIGTOP_JDK') ?: "8" ); version_base = version } |
| 'scala' { version = '2.12.13'; version_base = version } |
| } |
| apache { |
| APACHE_MIRROR = "https://apache.osuosl.org" |
| APACHE_ARCHIVE = "https://archive.apache.org/dist" |
| } |
| /** End of Base Configuration */ |
| |
| /** Dependencies section defines the order in which packages should be built. |
| The syntax is as follows: |
| dependsOn := [list of dependents] |
| So, below phoenix ought to be build after the hbase has been built |
| |
| To build a component with its dependencies, run build with -Dbuildwithdeps=true |
| */ |
| dependencies = [ |
| 'bigtop-utils':['ambari', 'bigtop-groovy', 'bigtop-jsvc', |
| 'elasticsearch', 'flink', 'logstash', 'zookeeper'], |
| 'bigtop-groovy':['hadoop'], |
| 'bigtop-jsvc':['hadoop'], |
| zookeeper:['hadoop', 'hbase', 'kafka'], |
| hadoop:['hbase', 'hive', 'tez', 'sqoop', |
| 'oozie', 'solr', 'spark', |
| 'phoenix', 'alluxio', 'ycsb', 'zeppelin' |
| ], |
| hbase:['phoenix', 'ycsb', 'hive'], |
| hive:['oozie', 'zeppelin'], |
| spark:['livy', 'oozie', 'zeppelin'], |
| ] |
| |
| components { |
| 'zookeeper' { |
| name = 'zookeeper' |
| pkg = name |
| version { |
| base = '3.4.14' |
| pkg = base |
| release = 1 |
| } |
| tarball { |
| source = "zookeeper-${version.base}.tar.gz" |
| destination = source |
| } |
| url { |
| download_path = "/zookeeper/zookeeper-${version.base}" |
| site = "${apache.APACHE_MIRROR}/${download_path}" |
| archive = "${apache.APACHE_ARCHIVE}/${download_path}" |
| } |
| } |
| 'hadoop' { |
| name = 'hadoop' |
| relNotes = 'Apache Hadoop' |
| version { base = '3.2.2'; pkg = base; release = 1 } |
| tarball { destination = "${name}-${version.base}.tar.gz" |
| source = "${name}-${version.base}-src.tar.gz" } |
| url { download_path = "/$name/common/$name-${version.base}" |
| site = "${apache.APACHE_MIRROR}/${download_path}" |
| archive = "${apache.APACHE_ARCHIVE}/${download_path}" } |
| } |
| 'hbase' { |
| name = 'hbase' |
| relNotes = 'Apache HBase' |
| version { base = '2.2.6'; pkg = base; release = 1 } |
| tarball { destination = "${name}-${version.base}.tar.gz" |
| source = "${name}-${version.base}-src.tar.gz" } |
| url { download_path = "/$name/${version.base}/" |
| site = "${apache.APACHE_MIRROR}/${download_path}" |
| archive = "${apache.APACHE_ARCHIVE}/${download_path}" } |
| } |
| |
| 'hive' { |
| name = 'hive' |
| relNotes = 'Apache Hive' |
| version { base = '3.1.2'; pkg = base; release = 1 } |
| tarball { destination = "apache-${name}-${version.base}-src.tar.gz" |
| source = destination } |
| url { download_path = "/$name/$name-${version.base}/" |
| site = "${apache.APACHE_MIRROR}/${download_path}" |
| archive = "${apache.APACHE_ARCHIVE}/${download_path}" } |
| } |
| 'tez' { |
| name = 'tez' |
| relNotes = 'Apache TEZ' |
| version { base = '0.10.0'; pkg = base; release = 1 } |
| tarball { destination = "apache-${name}-${version.base}-src.tar.gz" |
| source = destination } |
| url { download_path = "/$name/${version.base}/" |
| site = "${apache.APACHE_MIRROR}/${download_path}" |
| archive = "${apache.APACHE_ARCHIVE}/${download_path}" } |
| } |
| 'sqoop' { |
| name = 'sqoop' |
| relNotes = 'Apache Sqoop v1' |
| version { base = '1.4.7'; pkg = base; release = 1 } |
| tarball { destination = "${name}-${version.base}.tar.gz" |
| source = destination } |
| url { download_path = "/$name/${version.base}/" |
| site = "${apache.APACHE_MIRROR}/${download_path}" |
| archive = "${apache.APACHE_ARCHIVE}/${download_path}" } |
| } |
| 'oozie' { |
| name = 'oozie' |
| relNotes = 'Apache Oozie' |
| version { base = '5.2.1'; pkg = base; release = 1 } |
| tarball { destination = "$name-${version.base}.tar.gz" |
| source = destination } |
| url { download_path = "/$name/${version.base}/" |
| site = "${apache.APACHE_MIRROR}/${download_path}" |
| archive = "${apache.APACHE_ARCHIVE}/${download_path}" } |
| } |
| 'solr' { |
| name = 'solr' |
| relNotes = 'Apache Solr' |
| version { base = '8.7.0'; pkg = base; release = 1 } |
| tarball { destination = "$name-${version.base}-src.tgz" |
| source = destination } |
| url { download_path = "lucene/$name/${version.base}" |
| site = "${apache.APACHE_MIRROR}/${download_path}" |
| archive = "${apache.APACHE_ARCHIVE}/${download_path}" } |
| } |
| 'spark' { |
| name = 'spark' |
| pkg = 'spark-core' |
| relNotes = 'Apache Spark' |
| /* |
| * Remember to fetch the new LICENSE-binary and licenses-binary files |
| * when upgrading spark version. |
| * See comments in [bigtop-packages/src/common/spark/install_spark.sh] for details. |
| */ |
| version { base = '3.0.1'; pkg = base; release = 1 } |
| tarball { destination = "$name-${version.base}.tar.gz" |
| source = "$name-${version.base}.tgz" } |
| url { download_path = "/$name/$name-${version.base}" |
| site = "${apache.APACHE_MIRROR}/${download_path}" |
| archive = "${apache.APACHE_ARCHIVE}/${download_path}" } |
| } |
| |
| 'flink' { |
| name = 'flink' |
| relNotes = 'Apache Flink' |
| version { base = '1.11.3'; pkg = base; release = 1 } |
| tarball { destination = "$name-${version.base}.tar.gz" |
| source = "$name-${version.base}-src.tgz" } |
| url { download_path = "/$name/$name-${version.base}" |
| site = "${apache.APACHE_MIRROR}/${download_path}" |
| archive = "${apache.APACHE_ARCHIVE}/${download_path}" } |
| } |
| |
| |
| 'phoenix' { |
| name = 'phoenix' |
| relNotes = 'Apache Phoenix: A SQL skin over HBase' |
| version { base = "5.1.0"; pkg = base; release = 1 } |
| tarball { destination = "$name-${version.base}-src.tar.gz" |
| source = destination } |
| url { download_path = "/$name/$name-${version.base}" |
| site = "${apache.APACHE_MIRROR}/${download_path}" |
| archive = "${apache.APACHE_ARCHIVE}/${download_path}" } |
| } |
| 'bigtop-groovy' { |
| name = 'bigtop-groovy' |
| relNotes = "Groovy: a dynamic language for the Java platform" |
| version { base = '2.5.4'; pkg = '2.5.4'; release = 1 } |
| tarball { destination = "$name-${version.base}.tar.gz"; |
| source = "apache-groovy-binary-${version.base}.zip" } |
| url { download_path = "/groovy/${version.base}/distribution/" |
| site = "${apache.APACHE_MIRROR}/${download_path}" |
| archive = "${apache.APACHE_ARCHIVE}/${download_path}" } |
| } |
| 'bigtop-utils' { |
| name = "bigtop-utils" |
| relNotes = "Service package for Apache Bigtop runtime" |
| version { base = bigtop.version; pkg = base-"-SNAPSHOT"; release = 1 } |
| tarball { destination = "bigtop-utils-${version.base}.tar.gz" } |
| } |
| 'bigtop-jsvc' { |
| name = "bigtop-jsvc" |
| relNotes = "Apache Common Daemon (jsvc) service package" |
| version { base = '1.0.15'; pkg = base; release = 1 } |
| tarball { source = "commons-daemon-${version.base}-native-src.tar.gz" |
| destination = "commons-daemon-${version.base}.tar.gz" } |
| url { download_path = "/commons/daemon/source" |
| site = "${apache.APACHE_MIRROR}/${download_path}" |
| archive = "${apache.APACHE_ARCHIVE}/${download_path}" } |
| } |
| 'alluxio' { |
| name = "alluxio" |
| pkg = "alluxio" |
| relNotes = "Alluxio: a memory-centric distributed file system" |
| version { base = '2.4.1'; pkg = base; release = 1 } |
| tarball { destination = "$name-${version.base}.tar.gz" |
| source = "v${version.base}.tar.gz" } |
| url { site = "https://github.com/Alluxio/alluxio/archive" |
| archive = site } |
| } |
| 'kafka' { |
| name = 'kafka' |
| relNotes = 'Apache Kafka' |
| version { base = '2.4.1'; pkg = base; release = 1 } |
| tarball { destination = "$name-${version.base}.tar.gz" |
| source = "$name-${version.base}-src.tgz" } |
| url { download_path = "/$name/${version.base}/" |
| site = "${apache.APACHE_MIRROR}/${download_path}" |
| archive = "${apache.APACHE_ARCHIVE}/${download_path}" } |
| } |
| 'ycsb' { |
| name = 'ycsb' |
| relNotes = 'Yahoo! Cloud Serving Benchmark' |
| version { base = '0.17.0'; pkg = base; release = 1 } |
| tarball { destination = "$name-${version.base}.tar.gz" |
| source = "${version.base}.tar.gz" } |
| url { site = "https://github.com/brianfrankcooper/YCSB/archive" |
| archive = site } |
| } |
| 'zeppelin' { |
| name = 'zeppelin' |
| relNotes = 'Apache Zeppelin' |
| version { base = '0.9.0'; pkg = base; release = 1 } |
| tarball { source = "$name-${version.base}.tgz" |
| destination = "$name-${version.base}.tar.gz" } |
| url { download_path = "/$name/$name-${version.base}/" |
| site = "${apache.APACHE_MIRROR}/${download_path}" |
| archive = "${apache.APACHE_ARCHIVE}/${download_path}" } |
| } |
| 'gpdb' { |
| name = 'gpdb' |
| relNotes = 'GreenPlum' |
| version { base = '5.28.5'; pkg = '5.28.5'; release = 1 } |
| tarball { destination = "$name-${version.base}.tar.gz" |
| source = "${version.base}.tar.gz" } |
| url { site = "https://github.com/greenplum-db/gpdb/archive/" |
| archive = site } |
| } |
| 'ambari' { |
| name = 'ambari' |
| relNotes = 'Apache Ambari' |
| version { base = '2.7.5'; pkg = '2.7.5.0'; release = 1 } |
| tarball { destination = "apache-$name-${version.base}-src.tar.gz" |
| source = destination } |
| url { download_path = "/$name/$name-${version.base}/" |
| site = "${apache.APACHE_MIRROR}/${download_path}" |
| archive = "${apache.APACHE_ARCHIVE}/${download_path}" } |
| } |
| 'bigtop-ambari-mpack' { |
| name = 'bigtop-ambari-mpack' |
| relNotes = 'Bigtop Mpack' |
| version { base = '2.6.1'; pkg = '2.6.1.0'; release = 1 } |
| tarball { destination = "apache-ambari-${version.base}-src.tar.gz" |
| source = destination } |
| url { download_path = "/ambari/ambari-${version.base}/" |
| site = "${apache.APACHE_MIRROR}/${download_path}" |
| archive = "${apache.APACHE_ARCHIVE}/${download_path}" } |
| } |
| 'livy' { |
| name = 'livy' |
| relNotes = 'Apache Livy' |
| version { base = '0.7.1'; pkg = base; release = 1 } |
| tarball { destination = "${name}-${version.base}.tar.gz" |
| source = "apache-livy-${version.base}-incubating-src.zip" } |
| url { download_path = "incubator/livy/${version.base}-incubating/" |
| site = "${apache.APACHE_MIRROR}/${download_path}" |
| archive = "${apache.APACHE_ARCHIVE}/${download_path}" } |
| } |
| 'elasticsearch' { |
| name = 'elasticsearch' |
| relNotes = 'Search and Analytics engine' |
| version { base = '5.6.14'; pkg = base; release = 1 } |
| tarball { destination = "$name-${version.base}.tar.gz" |
| source = "v${version.base}.zip" } |
| url { site = "https://github.com/elastic/elasticsearch/archive" |
| archive = site } |
| } |
| 'logstash' { |
| name = 'logstash' |
| relNotes = 'Data processing pipeline' |
| version { base = '5.4.1'; pkg = base; release = 1 } |
| tarball { destination = "$name-${version.base}.tar.gz" |
| source = "v${version.base}.zip" } |
| url { site = "https://github.com/elastic/logstash/archive" |
| archive = site } |
| } |
| 'kibana' { |
| name = 'kibana' |
| relNotes = 'A browser-based analytics and search dashboard' |
| version { base = '5.4.1'; pkg = base; release = 1 } |
| tarball { destination = "$name-${version.base}.tar.gz" |
| source = "v${version.base}.zip" } |
| url { site = "https://github.com/elastic/kibana/archive" |
| archive = site } |
| } |
| } |
| } |