measure/src/test/resources/_accuracy-streaming-griffindsl.json

{
  "name": "accu_streaming",
  "process.type": "streaming",
  "data.sources": [
    {
      "name": "source",
      "baseline": true,
      "connector": {
        "type": "kafka",
        "version": "0.8",
        "dataframe.name": "this",
        "config": {
          "kafka.config": {
            "bootstrap.servers": "10.147.177.107:9092",
            "group.id": "group1",
            "auto.offset.reset": "smallest",
            "auto.commit.enable": "false"
          },
          "topics": "sss",
          "key.type": "java.lang.String",
          "value.type": "java.lang.String"
        },
        "pre.proc": [
          {
            "dsl.type": "df-ops",
            "in.dataframe.name": "this",
            "out.dataframe.name": "s1",
            "rule": "from_json"
          },
          {
            "dsl.type": "spark-sql",
            "out.dataframe.name": "this",
            "rule": "select name, age from s1"
          }
        ]
      },
      "checkpoint": {
        "type": "parquet",
        "file.path": "hdfs://localhost/griffin/streaming/dump/source",
        "info.path": "source",
        "ready.time.interval": "10s",
        "ready.time.delay": "0",
        "time.range": [
          "-2m",
          "0"
        ],
        "updatable": true
      }
    },
    {
      "name": "target",
      "connector": {
        "type": "kafka",
        "version": "0.8",
        "dataframe.name": "this",
        "config": {
          "kafka.config": {
            "bootstrap.servers": "10.147.177.107:9092",
            "group.id": "group1",
            "auto.offset.reset": "smallest",
            "auto.commit.enable": "false"
          },
          "topics": "ttt",
          "key.type": "java.lang.String",
          "value.type": "java.lang.String"
        },
        "pre.proc": [
          {
            "dsl.type": "df-ops",
            "in.dataframe.name": "this",
            "out.dataframe.name": "t1",
            "rule": "from_json"
          },
          {
            "dsl.type": "spark-sql",
            "out.dataframe.name": "this",
            "rule": "select name, age from t1"
          }
        ]
      },
      "checkpoint": {
        "type": "parquet",
        "file.path": "hdfs://localhost/griffin/streaming/dump/target",
        "info.path": "target",
        "ready.time.interval": "10s",
        "ready.time.delay": "0",
        "time.range": [
          "-2m",
          "0"
        ]
      }
    }
  ],
  "evaluate.rule": {
    "rules": [
      {
        "dsl.type": "griffin-dsl",
        "dq.type": "accuracy",
        "out.dataframe.name": "accu",
        "rule": "source.name = target.name and source.age = target.age",
        "details": {
          "source": "source",
          "target": "target",
          "miss": "miss_count",
          "total": "total_count",
          "matched": "matched_count"
        },
        "out": [
          {
            "type": "metric",
            "name": "accu"
          },
          {
            "type": "record",
            "name": "missRecords"
          }
        ]
      }
    ]
  },
  "sinks": [
    "CONSOLE",
    "ELASTICSEARCH"
  ]
}