| { |
| "spark": { |
| "log.level": "WARN", |
| "checkpoint.dir": "hdfs:///griffin/checkpoint/${JOB_NAME}", |
| "init.clear": true, |
| "batch.interval": "1m", |
| "process.interval": "5m", |
| "config": { |
| "spark.default.parallelism": 4, |
| "spark.task.maxFailures": 5, |
| "spark.streaming.kafkaMaxRatePerPartition": 1000, |
| "spark.streaming.concurrentJobs": 4, |
| "spark.yarn.maxAppAttempts": 5, |
| "spark.yarn.am.attemptFailuresValidityInterval": "1h", |
| "spark.yarn.max.executor.failures": 120, |
| "spark.yarn.executor.failuresValidityInterval": "1h", |
| "spark.hadoop.fs.hdfs.impl.disable.cache": true |
| } |
| }, |
| "sinks": [ |
| { |
| "type": "CONSOLE", |
| "config": { |
| "max.log.lines": 100 |
| } |
| }, |
| { |
| "type": "HDFS", |
| "config": { |
| "path": "hdfs:///griffin/persist", |
| "max.persist.lines": 10000, |
| "max.lines.per.file": 10000 |
| } |
| }, |
| { |
| "type": "ELASTICSEARCH", |
| "config": { |
| "method": "post", |
| "api": "http://es:9200/griffin/accuracy" |
| } |
| } |
| ], |
| "griffin.checkpoint": [ |
| { |
| "type": "zk", |
| "config": { |
| "hosts": "zk:2181", |
| "namespace": "griffin/infocache", |
| "lock.path": "lock", |
| "mode": "persist", |
| "init.clear": true, |
| "close.clear": false |
| } |
| } |
| ] |
| } |
