| { |
| "name": "prof_streaming", |
| "process.type": "streaming", |
| "data.sources": [ |
| { |
| "name": "source", |
| "baseline": true, |
| "connector": { |
| "type": "kafka", |
| "version": "0.8", |
| "dataframe.name": "kafka", |
| "config": { |
| "kafka.config": { |
| "bootstrap.servers": "10.147.177.107:9092", |
| "group.id": "group1", |
| "auto.offset.reset": "smallest", |
| "auto.commit.enable": "false" |
| }, |
| "topics": "sss", |
| "key.type": "java.lang.String", |
| "value.type": "java.lang.String" |
| }, |
| "pre.proc": [ |
| { |
| "dsl.type": "df-ops", |
| "in.dataframe.name": "kafka", |
| "out.dataframe.name": "out1", |
| "rule": "from_json" |
| }, |
| { |
| "dsl.type": "spark-sql", |
| "in.dataframe.name": "out1", |
| "out.datafrmae.name": "out3", |
| "rule": "select name, age from out1" |
| } |
| ] |
| }, |
| "checkpoint": { |
| "file.path": "hdfs://localhost/griffin/streaming/dump/source", |
| "info.path": "source", |
| "ready.time.interval": "10s", |
| "ready.time.delay": "0", |
| "time.range": [ |
| "0", |
| "0" |
| ] |
| } |
| } |
| ], |
| "evaluate.rule": { |
| "rules": [ |
| { |
| "dsl.type": "griffin-dsl", |
| "dq.type": "profiling", |
| "out.dataframe.name": "prof", |
| "rule": "select count(name) as `cnt`, max(age) as `max`, min(age) as `min` from source", |
| "out": [ |
| { |
| "type": "metric", |
| "name": "prof" |
| } |
| ] |
| }, |
| { |
| "dsl.type": "griffin-dsl", |
| "dq.type": "profiling", |
| "out.dataframe.name": "grp", |
| "rule": "select name, count(*) as `cnt` from source group by name", |
| "out": [ |
| { |
| "type": "metric", |
| "name": "name_group", |
| "flatten": "array" |
| } |
| ] |
| } |
| ] |
| }, |
| "sinks": [ |
| "elasticSink" |
| ] |
| } |