| { |
| "spark": { |
| "log.level": "WARN", |
| "checkpoint.dir": "hdfs://localhost/test/griffin/cp", |
| "batch.interval": "2s", |
| "process.interval": "10s", |
| "init.clear": true, |
| "config": { |
| "spark.master": "local[*]", |
| "spark.task.maxFailures": 5, |
| "spark.streaming.kafkaMaxRatePerPartition": 1000, |
| "spark.streaming.concurrentJobs": 4, |
| "spark.yarn.maxAppAttempts": 5, |
| "spark.yarn.am.attemptFailuresValidityInterval": "1h", |
| "spark.yarn.max.executor.failures": 120, |
| "spark.yarn.executor.failuresValidityInterval": "1h", |
| "spark.hadoop.fs.hdfs.impl.disable.cache": true |
| } |
| }, |
| "sinks": [ |
| { |
| "name": "consoleSink", |
| "type": "CONSOLE", |
| "config": { |
| "max.log.lines": 100 |
| } |
| }, |
| { |
| "name": "hdfsSink", |
| "type": "HDFS", |
| "config": { |
| "path": "hdfs://localhost/griffin/streaming/persist", |
| "max.persist.lines": 10000, |
| "max.lines.per.file": 10000 |
| } |
| }, |
| { |
| "name": "elasticSink", |
| "type": "ELASTICSEARCH", |
| "config": { |
| "method": "post", |
| "api": "http://localhost:9200/griffin/accuracy" |
| } |
| } |
| ], |
| "griffin.checkpoint": [ |
| { |
| "type": "zk", |
| "config": { |
| "hosts": "localhost:2181", |
| "namespace": "griffin/infocache", |
| "lock.path": "lock", |
| "mode": "persist", |
| "init.clear": true, |
| "close.clear": false |
| } |
| } |
| ] |
| } |
