| { |
| "name": "timeliness_streaming", |
| "process.type": "streaming", |
| "data.sources": [ |
| { |
| "name": "source", |
| "connector": { |
| "type": "kafka", |
| "version": "0.8", |
| "dataframe.name": "this", |
| "config": { |
| "kafka.config": { |
| "bootstrap.servers": "10.149.247.156:9092", |
| "group.id": "group1", |
| "auto.offset.reset": "smallest", |
| "auto.commit.enable": "false" |
| }, |
| "topics": "fff", |
| "key.type": "java.lang.String", |
| "value.type": "java.lang.String" |
| }, |
| "pre.proc": [ |
| { |
| "dsl.type": "df-ops", |
| "in.dataframe.name": "this", |
| "out.dataframe.name": "s1", |
| "rule": "from_json" |
| }, |
| { |
| "dsl.type": "spark-sql", |
| "out.dataframe.name": "this", |
| "rule": "select name, age from s1" |
| } |
| ] |
| }, |
| "checkpoint": { |
| "file.path": "hdfs://localhost/griffin/streaming/dump/source", |
| "info.path": "source", |
| "ready.time.interval": "10s", |
| "ready.time.delay": "0", |
| "time.range": [ |
| "0", |
| "0" |
| ] |
| } |
| } |
| ], |
| "evaluate.rule": { |
| "rules": [ |
| { |
| "dsl.type": "griffin-dsl", |
| "dq.type": "timeliness", |
| "out.dataframe.name": "timeliness", |
| "rule": "ts, end_ts", |
| "details": { |
| "source": "source", |
| "latency": "latency", |
| "total": "total", |
| "avg": "avg", |
| "threshold": "1h", |
| "step": "step", |
| "count": "cnt", |
| "step.size": "5m", |
| "percentile": "percentile", |
| "percentile.values": [ |
| 0.2, |
| 0.5, |
| 0.8 |
| ] |
| }, |
| "out": [ |
| { |
| "type": "metric", |
| "name": "timeliness" |
| }, |
| { |
| "type": "record", |
| "name": "lateRecords" |
| } |
| ] |
| } |
| ] |
| }, |
| "sinks": [ |
| "CONSOLE", |
| "ELASTICSEARCH" |
| ] |
| } |