| /* |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * "License"); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, |
| * software distributed under the License is distributed on an |
| * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| * KIND, either express or implied. See the License for the |
| * specific language governing permissions and limitations |
| * under the License. |
| */ |
| |
| apply plugin: 'eclipse' |
| apply plugin: 'idea' |
| apply plugin: 'java' |
| |
| defaultTasks 'distTar' |
| |
| task wrapper(type: Wrapper) { |
| description = "Updates gradlew and supporting files." |
| gradleVersion = '2.3' |
| } |
| |
| version = "$SAMZA_VERSION" |
| |
| repositories { |
| mavenLocal() |
| mavenCentral() |
| maven { url "https://repository.apache.org/content/groups/public" } |
| } |
| |
| |
| idea { |
| module { |
| sourceDirs += file('src/main/java') |
| testSourceDirs += file('src/test/java') |
| } |
| } |
| |
| // a configuration for dependencies that need exploding into package |
| configurations { |
| explode |
| } |
| |
| dependencies { |
| compile(group: 'org.codehaus.jackson', name: 'jackson-jaxrs', version: '1.9.13') |
| compile(group: 'org.slf4j', name: 'slf4j-api', version: "$SLF4J_VERSION") |
| compile(group: 'org.slf4j', name: 'slf4j-log4j12', version: "$SLF4J_VERSION") |
| compile(group: 'org.schwering', name: 'irclib', version: '1.10') |
| compile(group: 'org.apache.samza', name: 'samza-api', version: "$SAMZA_VERSION") |
| compile(group: 'org.apache.samza', name: 'samza-kv_2.11', version: "$SAMZA_VERSION") |
| |
| compile(group: 'org.apache.samza', name: 'samza-test_2.11', version: "$SAMZA_VERSION") |
| compile(group: 'org.apache.samza', name: 'samza-kafka_2.11', version: "$SAMZA_VERSION") |
| compile(group: 'org.apache.samza', name: 'samza-kv-rocksdb_2.11', version: "$SAMZA_VERSION") |
| compile(group: 'org.apache.samza', name: 'samza-azure_2.11', version: "$SAMZA_VERSION") |
| compile(group: 'org.apache.samza', name: 'samza-aws_2.11', version: "$SAMZA_VERSION") |
| testCompile(group: 'junit', name: 'junit', version: "4.12") |
| explode (group: 'org.apache.samza', name: 'samza-shell', ext: 'tgz', classifier: 'dist', version: "$SAMZA_VERSION") |
| runtime(group: 'org.apache.samza', name: 'samza-core_2.11', version: "$SAMZA_VERSION") |
| runtime(group: 'org.apache.samza', name: 'samza-log4j_2.11', version: "$SAMZA_VERSION") |
| |
| runtime(group: 'org.apache.samza', name: 'samza-shell', version: "$SAMZA_VERSION") |
| runtime(group: 'org.apache.samza', name: 'samza-yarn_2.11', version: "$SAMZA_VERSION") |
| runtime(group: 'org.apache.kafka', name: 'kafka_2.11', version: "$KAFKA_VERSION") |
| runtime(group: 'org.apache.hadoop', name: 'hadoop-hdfs', version: "$HADOOP_VERSION") |
| } |
| |
| // make the samza distribution .tgz file |
| task distTar(dependsOn: build, type: Tar) { |
| destinationDir(new File(project.buildDir, "/distributions")) |
| compression(Compression.GZIP) |
| classifier('dist') |
| extension('tar.gz') |
| into("config") { |
| from("src/main/config") { |
| include "wikipedia-feed.properties" |
| include "wikipedia-parser.properties" |
| include "wikipedia-stats.properties" |
| include "wikipedia-application.properties" |
| |
| // expand the Maven tokens with Gradle equivalents. Also change 'target' (Maven) to 'build/distributions' (Gradle) |
| filter { String line -> |
| line.replaceAll('[\$][{]basedir[}]', project.projectDir.toString()).replaceAll('[\$][{]project.artifactId[}]', project.name.toString()).replaceAll('/target/', '/build/distributions/').replaceAll('[\$][{]pom.version[}]', version) |
| } |
| } |
| } |
| |
| into("bin") { |
| from { |
| configurations.explode.collect { tarTree(it) } |
| } |
| } |
| |
| into("lib") { |
| from configurations.runtime |
| from configurations.runtime.artifacts.files |
| from("src/main/resources/") { |
| include "log4j.xml" |
| } |
| } |
| } |
| |
| // install everything |
| task installGrid(type: Exec) { |
| workingDir(project.projectDir) |
| commandLine("bin/grid", "install", "all") |
| outputs.upToDateWhen { |
| ["kafka", "zookeeper", "yarn"].every { |
| (new File(project.projectDir, "deploy/" + it)).exists() |
| } |
| } |
| } |
| |
| // update the Samza job |
| task deployHelloSamza(dependsOn: [distTar, installGrid], type: Sync) { |
| into(new File(project.projectDir, "/deploy/samza")) |
| from(tarTree(distTar.archivePath)) |
| } |
| |
| |
| // run everything |
| task startGrid(type: Exec) { |
| workingDir(project.projectDir) |
| commandLine("bin/grid", "start", "all") |
| outputs.upToDateWhen { |
| // use running zookeeper as proxy |
| File zookeeperPidFile = new File("/tmp/zookeeper/zookeeper_server.pid") |
| zookeeperPidFile.exists() && |
| "kill -0 ${zookeeperPidFile.text}".execute().waitFor() == 0 |
| } |
| } |
| |
| // stop everything |
| task stopGrid(type: Exec) { |
| workingDir(project.projectDir) |
| commandLine("bin/grid", "stop", "all") |
| } |
| |
| // |
| // Samza helpers |
| // |
| |
| // helper task to run Samza jobs |
| class SamzaTask extends DefaultTask { |
| String configFile; |
| |
| @TaskAction |
| def startSamza() { |
| project.exec { |
| workingDir(project.projectDir) |
| commandLine("deploy/samza/bin/run-job.sh", |
| "--config-factory=org.apache.samza.config.factories.PropertiesConfigFactory", |
| "--config-path=file://${project.projectDir}/deploy/samza/config/${configFile}") |
| } |
| } |
| } |
| |
| // helper to run the Samza job to retrieve Wikipedia data |
| task runWikiFeed(dependsOn: [startGrid, deployHelloSamza], type: SamzaTask) { |
| configFile("wikipedia-feed.properties") |
| } |
| |
| // helper to run the Samza job process Wikipedia data |
| task runWikiParser(dependsOn: [startGrid, deployHelloSamza], type: SamzaTask) { |
| configFile("wikipedia-parser.properties") |
| } |
| |
| // helper to run the Samza job to summarize stats on Wikipedia edits |
| task runWikiStats(dependsOn: [startGrid, deployHelloSamza], type: SamzaTask) { |
| configFile("wikipedia-stats.properties") |
| } |
| |
| |
| // |
| // Kafka helpers |
| // |
| |
| // show all Kafka topics |
| task listKafkaTopics(type: Exec) { |
| workingDir(project.projectDir) |
| commandLine("deploy/kafka/bin/kafka-topics.sh", |
| "--zookeeper", "localhost:2181", |
| "--list") |
| } |
| |
| // helper task to monitor a Kafka topic |
| class KafkaDumpTask extends DefaultTask { |
| String topic; |
| |
| @TaskAction |
| def dumpTopic() { |
| project.exec { |
| workingDir(project.projectDir) |
| commandLine("deploy/kafka/bin/kafka-console-consumer.sh", |
| "--zookeeper", "localhost:2181", |
| "--topic", "${topic}") |
| } |
| } |
| } |
| |
| // helper to dump the wikipedia-raw topic |
| task dumpWikiRaw(dependsOn: startGrid, type: KafkaDumpTask) { |
| topic("wikipedia-raw") |
| } |
| |
| // helper to dump the wikipedia-edits topic |
| task dumpWikiEdits(dependsOn: startGrid, type: KafkaDumpTask) { |
| topic("wikipedia-edits") |
| } |
| |
| // helper to dump the wikipedia-stats topic |
| task dumpWikiStats(dependsOn: startGrid, type: KafkaDumpTask) { |
| topic("wikipedia-stats") |
| } |