)]}' { "log": [ { "commit": "e293406f5756a9d375a1e123f32dbbdd72934130", "tree": "ded9195f695b3f81fb75841bbe6297092b4ad7de", "parents": [ "51bb5ac2b3e054392f9a8b04052f33957123f11c" ], "author": { "name": "William Guo", "email": "guoyp@apache.org", "time": "Wed Jul 03 17:15:13 2024 +0800" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Wed Jul 03 17:15:13 2024 +0800" }, "message": "Griffin 2.0.0 arch (#654)\n\n* new proposal for data quality tool\r\n\r\n* repolish\r\n\r\n* enterprise job scheduler might retry or stop the downstream scheduler based on standardized result\r\n\r\n* illustrate data quality etl phase, append after business phase\r\n\r\n* elaborate integrate with business workflow\r\n\r\n* init DQDiagrams\r\n\r\n* add metric storage service in DQDiagrams\r\n\r\n* triggered on demand\r\n\r\n* add more metrics\r\n\r\n* update arch diagram\r\n\r\n* init two table diff result set\r\n\r\n* data platform upgrades data quality checking pipeline\r\n\r\n* typo\r\n\r\n* update it as data quality constrains" }, { "commit": "51bb5ac2b3e054392f9a8b04052f33957123f11c", "tree": "0b87163379c1d10a9be06ccbec8cc0f70457fdc9", "parents": [ "1e20391fac1a82de6065c1ee929fd96d63159973" ], "author": { "name": "dependabot[bot]", "email": "49699333+dependabot[bot]@users.noreply.github.com", "time": "Mon Jun 24 10:27:37 2024 +0800" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Mon Jun 24 10:27:37 2024 +0800" }, "message": "Bump com.fasterxml.jackson.core:jackson-databind in /service (#655)\n\nBumps [com.fasterxml.jackson.core:jackson-databind](https://github.com/FasterXML/jackson) from 2.9.9.3 to 2.12.7.1.\r\n- [Commits](https://github.com/FasterXML/jackson/commits)\r\n\r\n---\r\nupdated-dependencies:\r\n- dependency-name: com.fasterxml.jackson.core:jackson-databind\r\n dependency-type: direct:production\r\n...\r\n\r\nSigned-off-by: dependabot[bot] \u003csupport@github.com\u003e\r\nCo-authored-by: dependabot[bot] \u003c49699333+dependabot[bot]@users.noreply.github.com\u003e" }, { "commit": "1e20391fac1a82de6065c1ee929fd96d63159973", "tree": "7d6755d5dd8f6c563deeb851c9426027f86a0a8d", "parents": [ "1140318e246120ad6286c263ded865135a165652" ], "author": { "name": "dependabot[bot]", "email": "49699333+dependabot[bot]@users.noreply.github.com", "time": "Sun Jun 23 10:05:11 2024 +0800" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Sun Jun 23 10:05:11 2024 +0800" }, "message": "--- (#652)\n\nupdated-dependencies:\r\n- dependency-name: requests\r\n dependency-type: direct:production\r\n...\r\n\r\nSigned-off-by: dependabot[bot] \u003csupport@github.com\u003e\r\nCo-authored-by: dependabot[bot] \u003c49699333+dependabot[bot]@users.noreply.github.com\u003e" }, { "commit": "1140318e246120ad6286c263ded865135a165652", "tree": "51864c609788abbf438a5c095bfe50c932b1463c", "parents": [ "6e84ee5d3048a9691c9826a5471f52b9da0237cb" ], "author": { "name": "dependabot[bot]", "email": "49699333+dependabot[bot]@users.noreply.github.com", "time": "Sun Jun 23 10:04:55 2024 +0800" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Sun Jun 23 10:04:55 2024 +0800" }, "message": "Bump urllib3 from 1.26.3 to 1.26.19 in /dev (#653)\n\nBumps [urllib3](https://github.com/urllib3/urllib3) from 1.26.3 to 1.26.19.\r\n- [Release notes](https://github.com/urllib3/urllib3/releases)\r\n- [Changelog](https://github.com/urllib3/urllib3/blob/1.26.19/CHANGES.rst)\r\n- [Commits](https://github.com/urllib3/urllib3/compare/1.26.3...1.26.19)\r\n\r\n---\r\nupdated-dependencies:\r\n- dependency-name: urllib3\r\n dependency-type: direct:production\r\n...\r\n\r\nSigned-off-by: dependabot[bot] \u003csupport@github.com\u003e\r\nCo-authored-by: dependabot[bot] \u003c49699333+dependabot[bot]@users.noreply.github.com\u003e" }, { "commit": "6e84ee5d3048a9691c9826a5471f52b9da0237cb", "tree": "1ffaf0b6583cd2562a9c86613e436bcf532dc883", "parents": [ "9f3197cb2869f5d15a2d3b965650e22ce254f3aa" ], "author": { "name": "dependabot[bot]", "email": "49699333+dependabot[bot]@users.noreply.github.com", "time": "Fri Apr 12 11:00:55 2024 +0800" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Fri Apr 12 11:00:55 2024 +0800" }, "message": "Bump idna from 2.10 to 3.7 in /dev (#647)\n\nBumps [idna](https://github.com/kjd/idna) from 2.10 to 3.7.\r\n- [Release notes](https://github.com/kjd/idna/releases)\r\n- [Changelog](https://github.com/kjd/idna/blob/master/HISTORY.rst)\r\n- [Commits](https://github.com/kjd/idna/compare/v2.10...v3.7)\r\n\r\n---\r\nupdated-dependencies:\r\n- dependency-name: idna\r\n dependency-type: direct:production\r\n...\r\n\r\nSigned-off-by: dependabot[bot] \u003csupport@github.com\u003e\r\nCo-authored-by: dependabot[bot] \u003c49699333+dependabot[bot]@users.noreply.github.com\u003e" }, { "commit": "9f3197cb2869f5d15a2d3b965650e22ce254f3aa", "tree": "5b64ff48aa24047c2d1491240c48f36cf47d2105", "parents": [ "3c0b5ddcbfa95ae3311f927fb98c0d3fde36027e" ], "author": { "name": "Yu Xiaoyu", "email": "34530004+XiaoyuBD@users.noreply.github.com", "time": "Tue May 09 10:05:30 2023 +0800" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Tue May 09 10:05:30 2023 +0800" }, "message": "JDBC Connector SQL mode (#628)\n\n* jdbc sql mode\r\n\r\n* add ut\r\n\r\n* add description for jdbc connector config" }, { "commit": "3c0b5ddcbfa95ae3311f927fb98c0d3fde36027e", "tree": "bdc9661025658d13f1fb2e8f4557b495c8737fcf", "parents": [ "74b3a25dc7bc226c886fa56030646e066c11d9cb", "f4b25ccd90fdbcedd6057f9956e4b563443963a7" ], "author": { "name": "Chitral Verma", "email": "chitralverma@gmail.com", "time": "Tue Sep 06 05:53:42 2022 +0200" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Tue Sep 06 05:53:42 2022 +0200" }, "message": "Merge pull request #606 from apache/dependabot/maven/service/org.postgresql-postgresql-42.4.1\n\nBump postgresql from 42.3.1 to 42.4.1 in /service" }, { "commit": "74b3a25dc7bc226c886fa56030646e066c11d9cb", "tree": "8cdeac2aca33f73a458ac630286153c50c869e94", "parents": [ "bb6ab36cf6008d72fec3cfba29e7dfaccf5d966c", "9310ce52fdaa88983430f554384da11f7d8b84b4" ], "author": { "name": "Chitral Verma", "email": "chitralverma@gmail.com", "time": "Tue Sep 06 05:53:09 2022 +0200" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Tue Sep 06 05:53:09 2022 +0200" }, "message": "Merge pull request #607 from apache/dependabot/maven/measure/org.postgresql-postgresql-42.4.1\n\nBump postgresql from 42.3.1 to 42.4.1 in /measure" }, { "commit": "9310ce52fdaa88983430f554384da11f7d8b84b4", "tree": "8cdeac2aca33f73a458ac630286153c50c869e94", "parents": [ "bb6ab36cf6008d72fec3cfba29e7dfaccf5d966c" ], "author": { "name": "dependabot[bot]", "email": "49699333+dependabot[bot]@users.noreply.github.com", "time": "Sat Aug 06 05:55:06 2022 +0000" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Sat Aug 06 05:55:06 2022 +0000" }, "message": "Bump postgresql from 42.3.1 to 42.4.1 in /measure\n\nBumps [postgresql](https://github.com/pgjdbc/pgjdbc) from 42.3.1 to 42.4.1.\n- [Release notes](https://github.com/pgjdbc/pgjdbc/releases)\n- [Changelog](https://github.com/pgjdbc/pgjdbc/blob/master/CHANGELOG.md)\n- [Commits](https://github.com/pgjdbc/pgjdbc/compare/REL42.3.1...REL42.4.1)\n\n---\nupdated-dependencies:\n- dependency-name: org.postgresql:postgresql\n dependency-type: direct:production\n...\n\nSigned-off-by: dependabot[bot] \u003csupport@github.com\u003e" }, { "commit": "f4b25ccd90fdbcedd6057f9956e4b563443963a7", "tree": "d1f29ddd9c84a249c6680d6419391719bd4e0130", "parents": [ "bb6ab36cf6008d72fec3cfba29e7dfaccf5d966c" ], "author": { "name": "dependabot[bot]", "email": "49699333+dependabot[bot]@users.noreply.github.com", "time": "Sat Aug 06 05:53:24 2022 +0000" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Sat Aug 06 05:53:24 2022 +0000" }, "message": "Bump postgresql from 42.3.1 to 42.4.1 in /service\n\nBumps [postgresql](https://github.com/pgjdbc/pgjdbc) from 42.3.1 to 42.4.1.\n- [Release notes](https://github.com/pgjdbc/pgjdbc/releases)\n- [Changelog](https://github.com/pgjdbc/pgjdbc/blob/master/CHANGELOG.md)\n- [Commits](https://github.com/pgjdbc/pgjdbc/compare/REL42.3.1...REL42.4.1)\n\n---\nupdated-dependencies:\n- dependency-name: org.postgresql:postgresql\n dependency-type: direct:production\n...\n\nSigned-off-by: dependabot[bot] \u003csupport@github.com\u003e" }, { "commit": "bb6ab36cf6008d72fec3cfba29e7dfaccf5d966c", "tree": "057f1d3e56f091f761758639a3796c843de68386", "parents": [ "bc40c97974fd98272c75b731a4a588b1746af111" ], "author": { "name": "lipzhu", "email": "lipzhu@ebay.com", "time": "Mon Jan 24 10:51:04 2022 +0530" }, "committer": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Mon Jan 24 10:51:04 2022 +0530" }, "message": "[GRIFFIN-362] Add postgresql and oracle driver into dependencies\n\n**What changes were proposed in this pull request?**\n1. Add Oracle and postgresql JDBC driver into dependencies in measure module due user report\nhttps://issues.apache.org/jira/browse/GRIFFIN-362\n2. Update postgresql jdbc driver version to the latest in service module.\n\n**Does this PR introduce any user-facing change?**\nNo.\n\nHow was this patch tested?\nUnit Tests\n\nCloses #597 from lipzhu/GRIFFIN-362.\n\nAuthored-by: lipzhu \u003clipzhu@ebay.com\u003e\nSigned-off-by: chitralverma \u003cchitralverma@gmail.com\u003e\n" }, { "commit": "bc40c97974fd98272c75b731a4a588b1746af111", "tree": "7343f856b2c01a68bcd294684d7d5e713c3fe64b", "parents": [ "1681e740ca733f6ad3f88e14dec123c8f3973a74" ], "author": { "name": "Lipeng Zhu", "email": "lipzhu@ebay.com", "time": "Mon Jan 24 10:36:22 2022 +0530" }, "committer": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Mon Jan 24 10:36:22 2022 +0530" }, "message": "[GRIFFIN-369] Bug fix for avro format in Spark 2.3.x environment\n\n**What changes were proposed in this pull request?**\nBuilt in Avro format is released in Spark 2.4.0,https://issues.apache.org/jira/browse/SPARK-24768\nFor Griffin, we still need to convert the Avro to com.databricks.spark.avro in Spark 2.3.x environment.\n\n**Does this PR introduce any user-facing change?**\nNo.\n\n**How was this patch tested?**\nUnit Tests\n\nCloses #598 from lipzhu/GRIFFIN-369.\n\nLead-authored-by: Lipeng Zhu \u003clipzhu@ebay.com\u003e\nCo-authored-by: Chitral Verma \u003cchitralverma@gmail.com\u003e\nCo-authored-by: lipzhu \u003clipzhu@ebay.com\u003e\nSigned-off-by: chitralverma \u003cchitralverma@gmail.com\u003e\n" }, { "commit": "1681e740ca733f6ad3f88e14dec123c8f3973a74", "tree": "aac023367d035cff38db4b8ae02a9af85e34cacd", "parents": [ "ecb4d2b11223d52e7f47d11f5cf242a8a3fa429c" ], "author": { "name": "Lipeng Zhu", "email": "lipzhu@icloud.com", "time": "Thu Jan 20 14:52:53 2022 +0800" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Thu Jan 20 14:52:53 2022 +0800" }, "message": "[GRIFFIN-367] For task GRIFFIN-367, update local deploy document. (#596)\n\n* For task GRIFFIN-367, update local deploy document.\r\n\r\n* Update doc." }, { "commit": "ecb4d2b11223d52e7f47d11f5cf242a8a3fa429c", "tree": "55d2e10141a0af5675f6c1cafac234ed9536a5c2", "parents": [ "4a7bbf4cf06a8ee83ad86b63f20a04e74efee8df" ], "author": { "name": "Chitral Verma", "email": "chitralverma@gmail.com", "time": "Mon Oct 04 20:42:14 2021 +0530" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Mon Oct 04 23:12:14 2021 +0800" }, "message": "[GRIFFIN-365] Measure Enhancements and Stability fixes (#593)\n\n* [GRIFFIN-365] Update pom.xml with scapegoat and other changes\r\n\r\n* [GRIFFIN-365] Remove ban on elasticsearch-spark dependency\r\n\r\n* [GRIFFIN-365] Measure enhancements\r\n\r\n* [GRIFFIN-365] Fix test cases\r\n\r\n* [GRIFFIN-365] Updates to documentation and fix for breaking tests\r\n\r\n* [GRIFFIN-365] Revert elasticsearch changes\r\n\r\n* Update schema_conformance.md\r\n\r\n* Update sparksql.md" }, { "commit": "4a7bbf4cf06a8ee83ad86b63f20a04e74efee8df", "tree": "b3c04348350bdb5cfd5e68e589d2b00d7612b441", "parents": [ "8044a4fb73d8da950643d49eb3273dae8d035b25", "ab1bffe7957db0c407210d69049d9afa7a57bc7f" ], "author": { "name": "William Guo", "email": "guoyp@apache.org", "time": "Wed Jul 07 19:29:49 2021 +0800" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Wed Jul 07 19:29:49 2021 +0800" }, "message": "Merge pull request #590 from chitralverma/improve-mergepr-script\n\n[GRIFFIN-360] Improvements to merge_pr.py" }, { "commit": "8044a4fb73d8da950643d49eb3273dae8d035b25", "tree": "b16fb72016c99c08eabbcb339c76189076b91aac", "parents": [ "7a50813643137b843eafa796e4dd7588f04da0dc", "c1cae820b72b8a6929d8bb2dbbb431d2e62e67c4" ], "author": { "name": "William Guo", "email": "guoyp@apache.org", "time": "Wed Jul 07 19:29:20 2021 +0800" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Wed Jul 07 19:29:20 2021 +0800" }, "message": "Merge pull request #583 from chitralverma/check-stale-pr-and-issues\n\n[GRIFFIN-347] Setup automated workflows for greetings and stale checks" }, { "commit": "7a50813643137b843eafa796e4dd7588f04da0dc", "tree": "e1b43858da4343d7a91c0b8c02708ad9438c5982", "parents": [ "076c8d08abf9f825dde22213e35e1b24f50cb1f2", "490829160f0e819acc8993a5a39fdf1a68f8c851" ], "author": { "name": "William Guo", "email": "guoyp@apache.org", "time": "Mon Jul 05 20:57:43 2021 +0800" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Mon Jul 05 20:57:43 2021 +0800" }, "message": "Merge pull request #591 from chitralverma/fix-measures\n\n[GRIFFIN-358] Rewrite the Rule/ Measure implementations" }, { "commit": "490829160f0e819acc8993a5a39fdf1a68f8c851", "tree": "e1b43858da4343d7a91c0b8c02708ad9438c5982", "parents": [ "444c956db0c491fc49fd5bc91cfd9a0b04b7cbd6" ], "author": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Wed Jun 30 12:16:34 2021 +0530" }, "committer": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Wed Jun 30 12:16:34 2021 +0530" }, "message": "[GRIFFIN-358] Fix import\n" }, { "commit": "444c956db0c491fc49fd5bc91cfd9a0b04b7cbd6", "tree": "b766c4f7ea2bd7a1f0f250262a9e344a90d237bd", "parents": [ "e1a6c03a6013ac212dd9ecb09c0a305bf5350007" ], "author": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Sat Jun 12 04:38:22 2021 +0530" }, "committer": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Sat Jun 12 04:38:22 2021 +0530" }, "message": "[GRIFFIN-358] Added documentation for SchemaConformanceMeasure\n" }, { "commit": "e1a6c03a6013ac212dd9ecb09c0a305bf5350007", "tree": "08780aa19615e2a2cb9476776737e7b5d871d3b5", "parents": [ "e3803d23b4bd1744ba272a5985250244e686d1f9" ], "author": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Sat Jun 12 03:34:44 2021 +0530" }, "committer": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Sat Jun 12 04:37:53 2021 +0530" }, "message": "[GRIFFIN-358] Changed Metric output format and fixed test cases\n" }, { "commit": "e3803d23b4bd1744ba272a5985250244e686d1f9", "tree": "9e614ee746e4d6f35068c8a903dff008ca4eb5a9", "parents": [ "c2a173f649006ec28297bb1b4a9725d97eac0caa" ], "author": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Sat Jun 12 00:35:43 2021 +0530" }, "committer": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Sat Jun 12 00:35:43 2021 +0530" }, "message": "[GRIFFIN-358] Added SchemaConformance measure\n" }, { "commit": "c2a173f649006ec28297bb1b4a9725d97eac0caa", "tree": "7593d29b97e5ff37b524bb7dd17959477f1f5370", "parents": [ "0a08b859ea5e467fd2ed15a3c3e638624451f0ec" ], "author": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Fri Jun 11 20:55:39 2021 +0530" }, "committer": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Fri Jun 11 21:01:15 2021 +0530" }, "message": "[GRIFFIN-358] Error handling and code formatting changes\n" }, { "commit": "0a08b859ea5e467fd2ed15a3c3e638624451f0ec", "tree": "3aa7776a376c390ef4fb036942a685d7fec41e44", "parents": [ "3586a30caec02eaa248964d839a604cef1e0adfd" ], "author": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Fri Jun 11 11:25:42 2021 +0530" }, "committer": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Fri Jun 11 11:25:42 2021 +0530" }, "message": "[GRIFFIN-358] Added sampling option to ProfilingMeasure\n" }, { "commit": "3586a30caec02eaa248964d839a604cef1e0adfd", "tree": "42039eb3209d4372a3b701d1d6037c22dd481f7b", "parents": [ "7943855ac46bc47c97dfd673e8d2a3f5f0ec51ba" ], "author": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Fri Jun 04 10:49:43 2021 +0530" }, "committer": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Fri Jun 04 10:49:43 2021 +0530" }, "message": "[GRIFFIN-358] Fixed breaking test case\n" }, { "commit": "7943855ac46bc47c97dfd673e8d2a3f5f0ec51ba", "tree": "583d593df50b1ff00f5225427c4f60aed49df69d", "parents": [ "cb25879055da5fe930ea9ac039b2080f1da4cb03" ], "author": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Fri Jun 04 08:56:54 2021 +0530" }, "committer": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Fri Jun 04 09:24:37 2021 +0530" }, "message": "[GRIFFIN-358] Added code documentation for all new measures.\n" }, { "commit": "cb25879055da5fe930ea9ac039b2080f1da4cb03", "tree": "abf5504a1c1cc7c5ec5ad75ac78ffba0be7d4b50", "parents": [ "e6a3f6b7126fdac96a1ab9f3513ff06fd2ef0444" ], "author": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Thu Jun 03 03:39:03 2021 +0530" }, "committer": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Thu Jun 03 04:38:51 2021 +0530" }, "message": "[GRIFFIN-358] Added parallelization to MeasureExecutor\n" }, { "commit": "e6a3f6b7126fdac96a1ab9f3513ff06fd2ef0444", "tree": "b05321e012024778fcade6574ad69957013ad016", "parents": [ "74f0d68484b04eeb0915a379e683fc3a2ee6456f" ], "author": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Sun May 30 00:07:30 2021 +0530" }, "committer": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Sun May 30 00:20:43 2021 +0530" }, "message": "[GRIFFIN-358] Changes structure of Measure\n" }, { "commit": "74f0d68484b04eeb0915a379e683fc3a2ee6456f", "tree": "1a2d95d098e4b19676b54196d7dd7339aff595f8", "parents": [ "3acd96ba985be12edc87f0e2d778e07f93cb2fcf" ], "author": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Sat May 29 23:52:49 2021 +0530" }, "committer": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Sat May 29 23:52:49 2021 +0530" }, "message": "[GRIFFIN-358] Added test cases for Data pre proc\n" }, { "commit": "3acd96ba985be12edc87f0e2d778e07f93cb2fcf", "tree": "f2c49e942a4adaeddffb7b88b4263fc8197c0068", "parents": [ "c676998bd5344f5d269fcd04786d87ca6533c0a5" ], "author": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Sat May 29 02:24:40 2021 +0530" }, "committer": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Sat May 29 02:24:40 2021 +0530" }, "message": "[GRIFFIN-358] Updated Configurations for pre proc and batch all measures\n" }, { "commit": "c676998bd5344f5d269fcd04786d87ca6533c0a5", "tree": "d5c5d32727009dd36d7d4a6235b684ca4f9a7327", "parents": [ "86555b54e83d53e033244e4cee00d0c48c05e7dd" ], "author": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Sat May 29 02:16:26 2021 +0530" }, "committer": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Sat May 29 02:16:26 2021 +0530" }, "message": "[GRIFFIN-358] Allow users to run old \"evaluate.rule\" configs as well\n" }, { "commit": "86555b54e83d53e033244e4cee00d0c48c05e7dd", "tree": "0ddaa6100aa840040f4739e7a443d4069f4978ff", "parents": [ "b1f1de133b19a0347b593167b4373be70224c09d" ], "author": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Sat May 29 01:27:28 2021 +0530" }, "committer": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Sat May 29 01:27:28 2021 +0530" }, "message": "[GRIFFIN-358] Added accuracy measure configuration guide.\n" }, { "commit": "b1f1de133b19a0347b593167b4373be70224c09d", "tree": "dfa4b37f1a01d30a55054b9066c7cdfd4ea4ea8a", "parents": [ "89e38a83e49cd97a6041dc068a19013dea68b8b1" ], "author": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Fri May 28 07:35:21 2021 +0530" }, "committer": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Fri May 28 07:44:30 2021 +0530" }, "message": "[GRIFFIN-358] Changed \u0027target\u0027 to \u0027ref\u0027 to clear terminology\n" }, { "commit": "89e38a83e49cd97a6041dc068a19013dea68b8b1", "tree": "e10643ed3cfaad9c886d5c2cf533c45a4dae779a", "parents": [ "15bcfa44969252865e47c34733a7b4e2a0584664" ], "author": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Mon May 03 03:03:52 2021 +0530" }, "committer": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Mon May 03 03:03:52 2021 +0530" }, "message": "[GRIFFIN-358] Added profiling measure configuration guide.\n" }, { "commit": "15bcfa44969252865e47c34733a7b4e2a0584664", "tree": "38a0b44b4eefeffd1d8b0bf3ce2a85c3dae399bf", "parents": [ "aaa1bf3baad389871a7828549866789c62ea76ad" ], "author": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Sun May 02 23:09:43 2021 +0530" }, "committer": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Mon May 03 02:21:57 2021 +0530" }, "message": "[GRIFFIN-358] Added measure configuration guide for duplication and sparkSql measures.\n" }, { "commit": "aaa1bf3baad389871a7828549866789c62ea76ad", "tree": "2659de87daacb9c3cca40f395aa710f3abbb4c61", "parents": [ "25ed0d1d0f9f1e4d66b7f0d64cd4e7e3cf81b40e" ], "author": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Sun May 02 22:34:07 2021 +0530" }, "committer": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Mon May 03 02:21:27 2021 +0530" }, "message": "[GRIFFIN-358] Update Duplication Measure to exclude null values\n" }, { "commit": "25ed0d1d0f9f1e4d66b7f0d64cd4e7e3cf81b40e", "tree": "3265e279a48d1e50dcca61b0c4a51737218da07e", "parents": [ "739d3f5cbf0f7c0cd528119370b7bd751c47d592" ], "author": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Sun May 02 18:33:55 2021 +0530" }, "committer": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Mon May 03 02:21:26 2021 +0530" }, "message": "[GRIFFIN-358] Added general documentation for new dimensions/ measures and completeness measure configuration guide.\n" }, { "commit": "c1cae820b72b8a6929d8bb2dbbb431d2e62e67c4", "tree": "0baa34f4a5cf306c33f4c5c76e3eaa35367b1954", "parents": [ "7c8f23e7719829290c4da1f0618be72a77e94906" ], "author": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Wed Apr 21 22:30:45 2021 +0530" }, "committer": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Wed Apr 21 22:36:01 2021 +0530" }, "message": "[GRIFFIN-347] Removed automation for issues as that\u0027s handled by Jira\n" }, { "commit": "7c8f23e7719829290c4da1f0618be72a77e94906", "tree": "e98bda1acde4a6cd546d78f63e25d9ca99959a4b", "parents": [ "293980e96e462ceaa1d0c2164f252573738186dd", "076c8d08abf9f825dde22213e35e1b24f50cb1f2" ], "author": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Wed Apr 21 20:26:48 2021 +0530" }, "committer": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Wed Apr 21 20:26:48 2021 +0530" }, "message": "[GRIFFIN-347] Updated with master\n" }, { "commit": "ab1bffe7957db0c407210d69049d9afa7a57bc7f", "tree": "9aef99256bc6499c8957700eb4349e9e7c4943ab", "parents": [ "076c8d08abf9f825dde22213e35e1b24f50cb1f2" ], "author": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Wed Apr 21 20:11:27 2021 +0530" }, "committer": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Wed Apr 21 20:11:27 2021 +0530" }, "message": "[GRIFFIN-360] Improvements to merge_pr.py\n" }, { "commit": "739d3f5cbf0f7c0cd528119370b7bd751c47d592", "tree": "fbaf77d8c1467609338c524c301ed8f6030c2f0f", "parents": [ "64a00d106c5bc795a5d0cb72f1814f7d53cd7538" ], "author": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Wed Apr 07 15:46:59 2021 +0530" }, "committer": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Wed Apr 07 20:14:40 2021 +0530" }, "message": "[GRIFFIN-358] Fixed breaking test cases\n" }, { "commit": "64a00d106c5bc795a5d0cb72f1814f7d53cd7538", "tree": "98ce0b914b3ef299a14884b068620e2d2beaf46f", "parents": [ "889ca5d5f62a56967f39fa09fb2ea4f7cbaa2178" ], "author": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Mon Apr 05 22:36:35 2021 +0530" }, "committer": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Mon Apr 05 22:36:35 2021 +0530" }, "message": "[GRIFFIN-358] Fixed formatting\n" }, { "commit": "889ca5d5f62a56967f39fa09fb2ea4f7cbaa2178", "tree": "03bffa582b9a6d848be185924cd5cbe82952e657", "parents": [ "8cd2190c704df7e6c4c6a45688a2f900ba16b4db" ], "author": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Mon Apr 05 17:07:00 2021 +0530" }, "committer": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Mon Apr 05 17:07:00 2021 +0530" }, "message": "[GRIFFIN-358] Added ProfilingMeasureTest\n" }, { "commit": "8cd2190c704df7e6c4c6a45688a2f900ba16b4db", "tree": "b3d611791a2b70ef04c028c38270400c6ff84f05", "parents": [ "8c13b30d6d8a35fb2b851b1c0ef2ed71ada5b49b" ], "author": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Mon Apr 05 10:56:41 2021 +0530" }, "committer": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Mon Apr 05 10:56:41 2021 +0530" }, "message": "[GRIFFIN-358] Added DuplicationMeasureTest\n" }, { "commit": "8c13b30d6d8a35fb2b851b1c0ef2ed71ada5b49b", "tree": "5d8028b55f5420efffb7bb10fe74bdec19c9e0dd", "parents": [ "1cbf6df29694daea0236029ec97489e19e738013" ], "author": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Sun Apr 04 23:07:48 2021 +0530" }, "committer": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Sun Apr 04 23:07:48 2021 +0530" }, "message": "[GRIFFIN-358] Added SparkSqlMeasureTest\n" }, { "commit": "1cbf6df29694daea0236029ec97489e19e738013", "tree": "1371f0907ceb8a2a96952d2ab7a2cc9501c7513d", "parents": [ "2c1cbea747abccccef4b2755e3bbf1b24bfffe64" ], "author": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Sun Apr 04 22:34:07 2021 +0530" }, "committer": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Sun Apr 04 22:34:07 2021 +0530" }, "message": "[GRIFFIN-358] Added AccuracyMeasureTest\n" }, { "commit": "2c1cbea747abccccef4b2755e3bbf1b24bfffe64", "tree": "a8b66d3d8568bdd12a68c3d860a8f43d6603d042", "parents": [ "42c3c173e412aca0cdc0a77bf88424c4a80ad372" ], "author": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Sun Apr 04 16:55:18 2021 +0530" }, "committer": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Sun Apr 04 16:59:50 2021 +0530" }, "message": "[GRIFFIN-358] Added CompletenessMeasureTest\n" }, { "commit": "42c3c173e412aca0cdc0a77bf88424c4a80ad372", "tree": "d36d38f0608669cdd78027aac66735a9a51a1591", "parents": [ "e66a5cd5fcde0151790edeb2ba1fcc1ff593e12d" ], "author": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Sat Apr 03 22:29:24 2021 +0530" }, "committer": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Sat Apr 03 22:29:55 2021 +0530" }, "message": "[GRIFFIN-358] Merge Measure constants\n" }, { "commit": "e66a5cd5fcde0151790edeb2ba1fcc1ff593e12d", "tree": "7ccb4d7543bbd8e51e632302c12a22db56746837", "parents": [ "095864e06cbfff112b66f2d7a441da4c07feec53" ], "author": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Sat Apr 03 21:20:19 2021 +0530" }, "committer": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Sat Apr 03 22:16:27 2021 +0530" }, "message": "[GRIFFIN-358] New Accuracy Measure\n" }, { "commit": "095864e06cbfff112b66f2d7a441da4c07feec53", "tree": "5a93b5234c88c60a399338720c25fe7ac96c88eb", "parents": [ "063e16c50d5be65addaaabd85e0aaabf91580749" ], "author": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Tue Mar 30 00:07:34 2021 +0530" }, "committer": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Tue Mar 30 00:07:41 2021 +0530" }, "message": "[GRIFFIN-358] Changes to Metric Flush process\n" }, { "commit": "063e16c50d5be65addaaabd85e0aaabf91580749", "tree": "900d731c8f8638a8b24fe0cb14708527cbd39d8b", "parents": [ "3f565f07d950c7c0bf69ec27b1365822c20ed5fe" ], "author": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Mon Mar 29 23:08:25 2021 +0530" }, "committer": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Mon Mar 29 23:11:56 2021 +0530" }, "message": "[GRIFFIN-358] New Duplication (Distinctness, Uniqueness) Measure\n" }, { "commit": "3f565f07d950c7c0bf69ec27b1365822c20ed5fe", "tree": "43edd7c96720708c38a105adadde6a61c504e225", "parents": [ "43737545ba18a99bffebbcb41b57739606e6ea38" ], "author": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Sun Mar 28 23:17:25 2021 +0530" }, "committer": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Sun Mar 28 23:17:25 2021 +0530" }, "message": "[GRIFFIN-358] New SparkSQL Measure\n" }, { "commit": "43737545ba18a99bffebbcb41b57739606e6ea38", "tree": "80668b1a5b4c20592f49422a06728f84ba32d6ae", "parents": [ "b66f796b4fc146207717f12f4df71e13bc11399f" ], "author": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Sun Mar 28 21:19:58 2021 +0530" }, "committer": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Sun Mar 28 22:31:36 2021 +0530" }, "message": "[GRIFFIN-358] New Profiling Measure\n" }, { "commit": "b66f796b4fc146207717f12f4df71e13bc11399f", "tree": "c32ba7f3bea1dcbab56995f0ea3922d392295fc0", "parents": [ "6745b98aa356bdda0978a1ed50150727fc7cc4b9" ], "author": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Wed Mar 24 19:17:35 2021 +0530" }, "committer": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Sun Mar 28 21:19:14 2021 +0530" }, "message": "[GRIFFIN-358] Rewrite new measure hierarchy and new completeness measure\n" }, { "commit": "6745b98aa356bdda0978a1ed50150727fc7cc4b9", "tree": "712498d75eb25f994025dc446477340de521e9ac", "parents": [ "076c8d08abf9f825dde22213e35e1b24f50cb1f2" ], "author": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Sun Mar 21 10:36:08 2021 +0530" }, "committer": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Mon Mar 22 17:53:26 2021 +0530" }, "message": "[GRIFFIN-358] Rewrite dataset preprocessing as SQL Queries\n" }, { "commit": "076c8d08abf9f825dde22213e35e1b24f50cb1f2", "tree": "61afa11b313cd867f2d2cf69c4e8b7b6e0e05526", "parents": [ "6977d1dbb5997122d4458ce9e3ac0423dd62be7d" ], "author": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Tue Mar 09 14:33:02 2021 +0530" }, "committer": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Tue Mar 09 14:33:02 2021 +0530" }, "message": "[GRIFFIN-345] Support cross-version compilation for Scala and Spark dependencies\n\n**What changes were proposed in this pull request?**\n\n_This PR affects only the measure module._\n\nIn newer environments specially clouds, Griffin measure module may face compatibility issues due the old Scala and Spark versions. To remedy this following topics are covered in this ticket,\n\n- Cross-compilation across scala major versions (2.11, 2.12)\n- Update Spark Version (2.4+)\n- Create maven profiles to build different scala and spark versions\n- Changes to build strategy\n\nThis process is also done is apache spark to build for different versions of Scala and Hadoop.\n\n**Does this PR introduce any user-facing change?**\nNo\n\n**How was this patch tested?**\nVia maven build process.\n\nCloses #589 from chitralverma/cross-version-build.\n\nAuthored-by: chitralverma \u003cchitralverma@gmail.com\u003e\nSigned-off-by: chitralverma \u003cchitralverma@gmail.com\u003e\n" }, { "commit": "6977d1dbb5997122d4458ce9e3ac0423dd62be7d", "tree": "0a8961c6de98eef920e6c80170f8fe75e410fe9f", "parents": [ "399472722ba7c1c912ee03e970e202df97bfc9ff" ], "author": { "name": "yuxiaoyu", "email": "yuxiaoyu@bytedance.com", "time": "Mon Dec 07 11:59:14 2020 +0800" }, "committer": { "name": "wankunde", "email": "wankunde@163.com", "time": "Mon Dec 07 11:59:14 2020 +0800" }, "message": "Support http remote conf\n\nIn our production practice, many Griffin jobs run on yarn in cluster mode. We upload different conf files to the http file server and we also provide services that generate specific configurations based on different HTTP URLs.\nSo we supports setting HTTP URLs as conf in submitting Griffin jobs in this PR. There is no effect on JSON or File conf mode. And it works well in our production environment for a long time.\n\nAuthor: yuxiaoyu \u003cyuxiaoyu@bytedance.com\u003e\n\nCloses #587 from XiaoyuBD/support_http_url_conf.\n" }, { "commit": "399472722ba7c1c912ee03e970e202df97bfc9ff", "tree": "8b5667f26c6bfc39ac1e7d67645baeb419b3f556", "parents": [ "d43398b3f98507913805b12e905581895553f9fb" ], "author": { "name": "Eugene", "email": "liujin@apache.org", "time": "Wed Dec 02 03:14:01 2020 -0700" }, "committer": { "name": "William Guo", "email": "guoyp@apache.org", "time": "Wed Dec 02 03:14:01 2020 -0700" }, "message": "Fix doc format glitches\n\nAuthor: Eugene \u003cliujin@apache.org\u003e\n\nCloses #588 from toyboxman/doc-pr.\n" }, { "commit": "d43398b3f98507913805b12e905581895553f9fb", "tree": "5c46c38a25be20a9ce9b7d20c99466f3169691a1", "parents": [ "53521d5ac690074b88861fc4166d13a6a248e4af" ], "author": { "name": "Yuepeng", "email": "yueguo@mbox-4427435.lvs02.dev.ebayc3.com", "time": "Sun Nov 08 23:10:43 2020 -0700" }, "committer": { "name": "Yuepeng", "email": "yueguo@mbox-4427435.lvs02.dev.ebayc3.com", "time": "Sun Nov 08 23:10:43 2020 -0700" }, "message": "[maven-release-plugin] prepare for next development iteration\n" }, { "commit": "53521d5ac690074b88861fc4166d13a6a248e4af", "tree": "e9525d011b290fa8d83b19577388b6e77e862ee7", "parents": [ "469d47d589ba3e5e9257c42f9106388f529ba02c" ], "author": { "name": "Yuepeng", "email": "yueguo@mbox-4427435.lvs02.dev.ebayc3.com", "time": "Sun Nov 08 23:09:19 2020 -0700" }, "committer": { "name": "Yuepeng", "email": "yueguo@mbox-4427435.lvs02.dev.ebayc3.com", "time": "Sun Nov 08 23:09:19 2020 -0700" }, "message": "[maven-release-plugin] prepare release griffin-0.6.0\n" }, { "commit": "469d47d589ba3e5e9257c42f9106388f529ba02c", "tree": "d398254a2ff28a741e8e603e83cd1b1701be2b0d", "parents": [ "0a87ef026e38c0fdcfbf897dfc5f324f509582d8" ], "author": { "name": "William Guo", "email": "guoyp@apache.org", "time": "Sun Nov 08 13:56:32 2020 +0530" }, "committer": { "name": "Chitral Verma", "email": "chitralverma@gmail.com", "time": "Sun Nov 08 13:56:32 2020 +0530" }, "message": "Change connectors to connector for datasource\n\nCloses #586 from guoyuepeng/change_connectors_to_connector_for_datasource.\n\nLead-authored-by: William Guo \u003cguoyp@apache.org\u003e\nCo-authored-by: deyiyao \u003cdeyiyao@ebay.com\u003e\nCo-authored-by: ahutsunshine \u003cahutsunshine@gmail.com\u003e\nSigned-off-by: Chitral Verma \u003cchitralverma@gmail.com\u003e\n" }, { "commit": "0a87ef026e38c0fdcfbf897dfc5f324f509582d8", "tree": "4c07efc60ed0499b682ef5fa95f65cd55c87ced8", "parents": [ "ef6466def71b154805779c1007a010c633577d78" ], "author": { "name": "William Guo", "email": "guoyp@apache.org", "time": "Mon Oct 26 17:35:16 2020 +0800" }, "committer": { "name": "William Guo", "email": "guoyp@apache.org", "time": "Mon Oct 26 17:35:16 2020 +0800" }, "message": "update angular cli version for release issue\n\nAuthor: William Guo \u003cguoyp@apache.org\u003e\n\nCloses #585 from guoyuepeng/update_angular_cli_verion_for_release.\n" }, { "commit": "ef6466def71b154805779c1007a010c633577d78", "tree": "912a85bd5900347d793e56bef937c99373c42de5", "parents": [ "921e1d0d50dde198e0a80d7bbde962121dbba4e8" ], "author": { "name": "William Guo", "email": "guoyp@apache.org", "time": "Mon Oct 26 11:14:23 2020 +0800" }, "committer": { "name": "William Guo", "email": "guoyp@apache.org", "time": "Mon Oct 26 11:14:23 2020 +0800" }, "message": "compaliancy fix\n\nAuthor: William Guo \u003cguoyp@apache.org\u003e\n\nCloses #584 from guoyuepeng/compaliancy_fix_before_release_0.7.0.\n" }, { "commit": "293980e96e462ceaa1d0c2164f252573738186dd", "tree": "2491fd91f1476fbdfc951296095356ea99f323cc", "parents": [ "921e1d0d50dde198e0a80d7bbde962121dbba4e8" ], "author": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Mon Sep 21 10:35:53 2020 +0530" }, "committer": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Mon Sep 21 10:35:53 2020 +0530" }, "message": "For task GRIFFIN-347, Add workflows\n" }, { "commit": "921e1d0d50dde198e0a80d7bbde962121dbba4e8", "tree": "d2ec1c16841ed65935d0369597295f7af318eb74", "parents": [ "dd887db62af4c3ee738f30259ec60f88d46fff06" ], "author": { "name": "ambition119", "email": "1269223860@qq.com", "time": "Tue Sep 15 09:31:34 2020 +0800" }, "committer": { "name": "William Guo", "email": "guoyp@apache.org", "time": "Tue Sep 15 09:31:34 2020 +0800" }, "message": "[GRIFFIN-SERVICE] use service tar.gz deploy and griffin.sh start\n\n If we use` java -jar` start service ,then It is inconvenient to modify the configuration file. Modifying the configuration file requires recompiling the jar. And it is inconvenient to stop the service.\n This PR provides `tar.gz` installation method and shell startup method，and it has been deployed and run in our production environment.\n\n1. ls -al service-0.6.0-SNAPSHOT\n bin\n config\n lib\n\n2. cd service-0.6.0-SNAPSHOT/\n\n3. ./bin/griffin.sh start\n\n4. jps\n 17860 GriffinWebApplication\n\n5. ./bin/griffin.sh stop\n stopping 17860 of service ...\n\n If start service, we can access http://${ip}:8080\n![image](https://user-images.githubusercontent.com/20353043/92120728-ffc00480-ee2b-11ea-9e9b-8a02d208bf4b.png)\n\nAuthor: ambition119 \u003c1269223860@qq.com\u003e\n\nCloses #582 from ambition119/service.tar.gz.\n" }, { "commit": "dd887db62af4c3ee738f30259ec60f88d46fff06", "tree": "aa84e3ddd69ea6f6186dd50884180ddf77a9e37c", "parents": [ "1aa8995af89b723b6fab6f0922afa1916a2f8084" ], "author": { "name": "wankunde", "email": "wankunde@163.com", "time": "Mon Aug 17 13:11:10 2020 +0530" }, "committer": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Mon Aug 17 13:11:10 2020 +0530" }, "message": "[GRIFFIN-339] Import griffin tool for debug and run user jobs\n\nWith Griffin tool, user can run dq jobs in command line.\nThis is helpful for user to debug and run user dq jobs.\n\nCloses #581 from wankunde/measure_tools.\n\nAuthored-by: wankunde \u003cwankunde@163.com\u003e\nSigned-off-by: chitralverma \u003cchitralverma@gmail.com\u003e\n" }, { "commit": "1aa8995af89b723b6fab6f0922afa1916a2f8084", "tree": "9e012dd92494749eb2efee9613006459d1f4514b", "parents": [ "7123269b5cc69806499bffc0757da63eaa1b9306" ], "author": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Mon Aug 10 10:49:42 2020 +0800" }, "committer": { "name": "William Guo", "email": "guoyp@apache.org", "time": "Mon Aug 10 10:49:42 2020 +0800" }, "message": "[GRIFFIN-305] Standardize sink hierarchy\n\n**What changes were proposed in this pull request?**\nCurrently, the implementation of `Sinks` in Griffin poses the below issues. This PR aims at fixing these issues.\n- `Sinks` are based on the recursive MultiSink class which is a sink itself but the underlying implementation is that of a `Seq` which causes ambiguity and isn\u0027t much useful. This has been removed.\n- Some unused code like `SinkContext` has been removed.\n- Data is converted from the performant DataFrame to RDD while persisting in both streaming and batch pipelines. A new method `sinkBatchRecords` has been added to allow operations directly on DataFrame for batch pipelines. Streaming will still use the old implementation which will be replaced with structured streaming.\n- Refactored the methods of `Sink` like changed `start`/ `finish` to `open`/ `close` and `jobName` was incorrectly passed as `metricName`.\n- Presently, only one instance of a sink with a given type can be defined in the env config. This will not allow the cases where you want to configure multiple sinks of same type like HDFS or JDBC. Added sink `name` to env config which is used to define the sink that should be used in the job config also.\n- Updated all sinks as per the changes above. With some additional changes to ConsoleSink\n\n**Does this PR introduce any user-facing change?**\nYes. As mentioned above, the sink config has changed in env and job configs.\n\n**How was this patch tested?**\nGriffin test suite and additional unit test cases\n\nAuthor: chitralverma \u003cchitralverma@gmail.com\u003e\n\nCloses #575 from chitralverma/standardize-sink-hierarchy.\n" }, { "commit": "7123269b5cc69806499bffc0757da63eaa1b9306", "tree": "cdc288a672b512242c83e2cc8c205382950107f8", "parents": [ "35a1714e60184e316de702d1f0bbec3f0f78023b" ], "author": { "name": "Eugene", "email": "liujin@apache.org", "time": "Sun Jun 21 05:13:49 2020 -0700" }, "committer": { "name": "Eugene", "email": "liujin@apache.org", "time": "Sun Jun 21 05:13:49 2020 -0700" }, "message": "Fix Unit Test Issue In Measure Test Case\n\n[GRIFFIN-329] Measure unit test cases fail on the condition of no docker image\n\nThe unit test case tries to download a ES docker image and run the following cases. If the downloading fails, some cases will abort due to exceptions. In the revision, a new flag is introduced in execution, unless the docker image is avaiable always, some cases will be excluded.\n\nAuthor: Eugene \u003cliujin@apache.org\u003e\n\nCloses #580 from toyboxman/Fix.\n" }, { "commit": "35a1714e60184e316de702d1f0bbec3f0f78023b", "tree": "a23a5d2f3aeb4a01595da282e4816ed26d0ea938", "parents": [ "491a168da33c2811e9945f1e037a2df5ddc57ccc" ], "author": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Thu Jun 04 19:01:34 2020 +0800" }, "committer": { "name": "William Guo", "email": "guoyp@apache.org", "time": "Thu Jun 04 19:01:34 2020 +0800" }, "message": "[GRIFFIN-326] New Data Connector for Elasticsearch\n\n**What changes were proposed in this pull request?**\n\nThis ticket proposes the following changes,\n- Deprecate the current implementation in favour of the direct implementation in the official [elasticsearch-hadoop](https://github.com/elastic/elasticsearch-hadoop/tree/master/spark/sql-20) library.\n- This library is built on DataSource API built on spark 2.2.x+ and thus brings support for filter pushdowns, column pruning, unified read and write and additional optimizations.\n- Many configuration options are available for ES connectivity, [check here](https://github.com/elastic/elasticsearch-hadoop/blob/master/mr/src/main/java/org/elasticsearch/hadoop/cfg/ConfigurationOptions.java).\n- Any filters can be applied as expressions directly on the data frame and are pushed automatically to the source.\n\n**Does this PR introduce any user-facing change?**\nYes. As mentioned above, the old connector has been deprecated and config structure for Elasticsearch data connector has changed now.\n\n**How was this patch tested?**\nGriffin test suite and additional unit test cases\n\nAuthor: chitralverma \u003cchitralverma@gmail.com\u003e\n\nCloses #569 from chitralverma/new-elastic-search-connector.\n" }, { "commit": "491a168da33c2811e9945f1e037a2df5ddc57ccc", "tree": "a9786a0106b4280ca166e0fa1016f568e15217a7", "parents": [ "3b77ccc018655b64ea12af5807f18f19dc642a17" ], "author": { "name": "DongfangLu", "email": "ayludongfang@163.com", "time": "Wed May 13 18:50:54 2020 +0800" }, "committer": { "name": "William Guo", "email": "guoyp@apache.org", "time": "Wed May 13 18:50:54 2020 +0800" }, "message": "Upgrade UI packages for jquery\n\nUpgrade jquery\n\nAuthor: DongfangLu \u003cayludongfang@163.com\u003e\n\nCloses #571 from ludongfang/ui_package_upgrade.\n" }, { "commit": "3b77ccc018655b64ea12af5807f18f19dc642a17", "tree": "409f2f1e9ec2ae541fc10defaf603b13918d1431", "parents": [ "24c270e016a9c7c42ec5f137672dadc7f7ef13f6" ], "author": { "name": "Yu", "email": "yu.liu003@gmail.com", "time": "Wed Mar 11 23:15:19 2020 +0800" }, "committer": { "name": "William Guo", "email": "guoyp@apache.org", "time": "Wed Mar 11 23:15:19 2020 +0800" }, "message": "[GRIFFIN-316] Fix job exception handling\n\n**What changes were proposed in this pull request?**\n\nCurrently we are using Try instance to represent the results of a DQ job, whether succeeded or failed. But as we are only wrapping the Boolean result by applying \"Try\" at the most outside level, the underlying failure would not be able to caught and it would always return \"Success\" even if exception got.\n\nThis is to modify all the underlying execute/doExecute methods of a DQ job, by handling exception with \"Try\" instances so that it could be passed properly to users when things get wrong.\n\n**Does this PR introduce any user-facing change?**\nNo.\n\n**How was this patch tested?**\nGriffin test suite.\n\nAuthor: Yu \u003cyu.liu003@gmail.com\u003e\n\nCloses #562 from PnPie/exception_catch.\n" }, { "commit": "24c270e016a9c7c42ec5f137672dadc7f7ef13f6", "tree": "b0ddd21ef9b89d22cfee5ea6a28156746400fda8", "parents": [ "23950ecd5f3c0f57d096639ff4185655aeec6df7" ], "author": { "name": "yuxiaoyu", "email": "yuxiaoyu@bytedance.com", "time": "Tue Feb 11 20:15:53 2020 +0800" }, "committer": { "name": "wankunde", "email": "wankunde@163.com", "time": "Tue Feb 11 20:15:53 2020 +0800" }, "message": "optimize get metric maps in \u0027MetricWriteStep\u0027\n\n**Why/What changes？**\nIn \u0027MetricWriteStep.getMetricMaps()\u0027 the dataframe was transformed to json rdd, and then collect, and then transformed to Seq[Map].\nIt\u0027s not elegant and hard to understand. More optimized way is to collect it first, and then transform it to Seq[Map] directly.\n\nWe have test it with our DQ cases. It works well.\n\nAuthor: yuxiaoyu \u003cyuxiaoyu@bytedance.com\u003e\n\nCloses #566 from XiaoyuBD/optimizeMetricWriteGetMaps.\n" }, { "commit": "23950ecd5f3c0f57d096639ff4185655aeec6df7", "tree": "75578f298146d24c74c20577ed4f4b6144501cf8", "parents": [ "727e9617c9f408f252035246b469788354fe1a69" ], "author": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Mon Feb 10 17:09:29 2020 +0800" }, "committer": { "name": "wankunde", "email": "wankunde@163.com", "time": "Mon Feb 10 17:09:29 2020 +0800" }, "message": "[GRIFFIN-323] Refactor configuration Data Source Connector\n\n**What changes were proposed in this pull request?**\n\nThis ticket proposes the following changes,\n\n- remove \u0027version\u0027 from \u0027DataConnectorParam\u0027 as it is not being used anywhere in the codebase.\n- change \u0027connectors\u0027 from array type to a single JSON object. Since a data source named X may only be of one type (hive, file etc), the connector field should not be an array.\n- rename connectors to connector\n- update existing config files and documentation for reference\n\n**Does this PR introduce any user-facing change?**\nYes. As mentioned above, the config structure has changed now.\n\n**How was this patch tested?**\nGriffin test suite.\n\nAuthor: chitralverma \u003cchitralverma@gmail.com\u003e\n\nCloses #568 from chitralverma/refactor-data-connector-config.\n" }, { "commit": "727e9617c9f408f252035246b469788354fe1a69", "tree": "46fa56fd4d3e0059618cf4d870067240fbaee15a", "parents": [ "6784dc047c1a2904b4f752a6e9538194f67a1d5a" ], "author": { "name": "yuxiaoyu", "email": "yuxiaoyu@bytedance.com", "time": "Sat Feb 08 16:15:03 2020 +0800" }, "committer": { "name": "wankunde", "email": "wankunde@163.com", "time": "Sat Feb 08 16:15:03 2020 +0800" }, "message": "[GRIFFIN-322] Add SQL mode for ES connector\n\nAs [GRIFFIN-322](https://issues.apache.org/jira/projects/GRIFFIN/issues/GRIFFIN-322?filter\u003dallopenissues) , we want add sql mode for es connector.\n\n**The sql mode would more effective and user-friendly.**\n\nCurrent mode config:\n{ \"class\": \"org.apache.griffin.measure.datasource.connector.batch.ElasticSearchGriffinDataConnector\",\n \"index\": \"index-xxx\",\n \"type\": \"metric\",\n \"host\": \"xxxxxxxxxx\",\n \"port\": \"xxxx\",\n \"fields\": [\"col_a\", \"col_b\", \"col_c\"],\n \"size\": 100}\n\nSQL mode config:\n{ \"class\": \"org.apache.griffin.measure.datasource.connector.batch.ElasticSearchGriffinDataConnector\",\n \"sql.mode\": true,\n \"host\": \"xxxxxxxxxx\",\n \"port\": \"xxxx\",\n \"sql\": \"select col_a, col_b, col_c from index-xx limit 100\"}\n\nCompared with current mode, SQL mode could support other types except number type.\n\nAuthor: yuxiaoyu \u003cyuxiaoyu@bytedance.com\u003e\n\nCloses #567 from XiaoyuBD/enrichEsConnectorAddSqlMode.\n" }, { "commit": "6784dc047c1a2904b4f752a6e9538194f67a1d5a", "tree": "f3047e0ce04a0c1a52aefa20727a8b082ce6b7d1", "parents": [ "15304da715a74bef00d337b7fff4fc526f87b1bf" ], "author": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Thu Jan 16 15:20:21 2020 +0800" }, "committer": { "name": "wankunde", "email": "wankunde@163.com", "time": "Thu Jan 16 15:20:21 2020 +0800" }, "message": "[GRIFFIN-317] Define guidelines for Griffin Project Improvement Proposals (GPIP)\n\n**What changes were proposed in this pull request?**\n\nTaking inspiration from Apache Spark, this ticket aims to define guidelines for Griffin Project Improvement Proposals (GPIP).\n\nThe purpose of a GPIP is to inform and involve the user community in major improvements to the Apache Griffin codebase throughout the development process, to increase the likelihood that user needs are met.\n\nA GPIP aims to discuss the design and implementation of major features and changes in a collaborative manner. These major features must not be small/ incremental/ wide-scoped, as these features can be resolved by normal Jira process.\n\n**Does this PR introduce any user-facing change?**\nNo\n\n**How was this patch tested?**\nNot Applicable\n\nAuthor: chitralverma \u003cchitralverma@gmail.com\u003e\n\nCloses #563 from chitralverma/griffin-pip-template.\n" }, { "commit": "15304da715a74bef00d337b7fff4fc526f87b1bf", "tree": "dd09a9774227d50f7a6352b55c8f0d01b3d6aec4", "parents": [ "fa11527ea9a694c7d778d3ba2ea007db3dbf21a8" ], "author": { "name": "neveljkovic", "email": "nveljkovic@plume.com", "time": "Mon Jan 06 17:17:36 2020 +0800" }, "committer": { "name": "wankunde", "email": "wankunde@163.com", "time": "Mon Jan 06 17:17:36 2020 +0800" }, "message": "[GRIFFIN-318] Replace all YYYY with yyyy in all user guides and examples\n\nhttps://issues.apache.org/jira/browse/GRIFFIN-318\nReplace YYYY with yyyy and DD with dd\n\nAuthor: neveljkovic \u003cnveljkovic@plume.com\u003e\n\nCloses #565 from neveljkovic/GRIFFIN-318.\n" }, { "commit": "fa11527ea9a694c7d778d3ba2ea007db3dbf21a8", "tree": "a520e2d0ea5171bf2f7c604908073b00e41a9c1a", "parents": [ "ee82cbd989fcbb73e9043595f6e0dc3e03aa992f" ], "author": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Mon Jan 06 17:12:03 2020 +0800" }, "committer": { "name": "wankunde", "email": "wankunde@163.com", "time": "Mon Jan 06 17:12:03 2020 +0800" }, "message": "[GRIFFIN-319] Deprecate old Data Connectors\n\n**What changes were proposed in this pull request?**\n\nThis ticket aims to inform users of the deprecated data source connectors.\n\nDeprecated connectors:\n\n- MySqlDataConnector in favour of JDBCBasedDataConnector\n- AvroBatchDataConnector in favour of FileBasedDataConnector\n- TextDirBatchDataConnector in favour of FileBasedDataConnector\n\nThe documentation is also updated corresponding to the new connectors for reference.\n\n**Does this PR introduce any user-facing change?**\nNo\n\n**How was this patch tested?**\nNot Applicable\n\nAuthor: chitralverma \u003cchitralverma@gmail.com\u003e\n\nCloses #564 from chitralverma/deprecate-old-data-connectors.\n" }, { "commit": "ee82cbd989fcbb73e9043595f6e0dc3e03aa992f", "tree": "f2a61e9cc2f9aa9f79ecfa3422cab52a242a37e3", "parents": [ "7f718d3c2c73e4a0fd87afbea4e9165d56e4258c" ], "author": { "name": "tusharpatil20", "email": "tushargp20@gmail.com", "time": "Tue Dec 31 10:28:00 2019 +0800" }, "committer": { "name": "wankunde", "email": "wankunde@163.com", "time": "Tue Dec 31 10:28:00 2019 +0800" }, "message": "[GRIFFIN-315] Adding JDBC based data connector\n\n**What changes were proposed in this pull request?**\n\nJDBC based data connector to read data from different JDBC based data sources.\n\n**Does this PR introduce any user-facing change?**\nNo\n\n**How was this patch tested?**\nGriffin test suite.\n\nAuthor: tusharpatil20 \u003ctushargp20@gmail.com\u003e\n\nCloses #561 from tusharpatil20/JDBCBased-source-connector.\n" }, { "commit": "7f718d3c2c73e4a0fd87afbea4e9165d56e4258c", "tree": "f8b62ff5563bb7f1d7d51ec85a488b7f99453681", "parents": [ "ed47c2b2232e07adea0a8919786ce835a15f2191" ], "author": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Wed Dec 25 10:39:11 2019 +0800" }, "committer": { "name": "wankunde", "email": "wankunde@163.com", "time": "Wed Dec 25 10:39:11 2019 +0800" }, "message": "[GRIFFIN-312] Code Style Standardization\n\n**What changes were proposed in this pull request?**\n\nThis PR targets the following,\n- fix the various warnings during build and in source code,\n- perform code formatting as per a standard style,\n- fix scalastyle integration\n\nUsing the [code format from spark code style can be automatically imposed on measure module. Link: https://github.com/apache/spark/blob/master/dev/.scalafmt.conf\n\nSince ScalaStyle targets scala source code only, it should be a part of the measure module only. Current misconfiguration is also suppressing the formatting errors\n\nScalafmt is used for code formatting.\n\n**Does this PR introduce any user-facing change?**\nNo\n\n**How was this patch tested?**\nGriffin test suite.\n\nAuthor: chitralverma \u003cchitralverma@gmail.com\u003e\n\nCloses #560 from chitralverma/code-style-standardization.\n" }, { "commit": "ed47c2b2232e07adea0a8919786ce835a15f2191", "tree": "e1bd5044c5a713290c09b72a11110057abe8e5df", "parents": [ "a12729b32764a6d76359de359e15da71e06fda8f" ], "author": { "name": "tusharpatil", "email": "tus", "time": "Mon Dec 16 21:22:20 2019 +0800" }, "committer": { "name": "wankunde", "email": "wankunde@163.com", "time": "Mon Dec 16 21:22:20 2019 +0800" }, "message": "Enum based configs\n\n**What changes were proposed in this pull request?**\n\nAll the predefined: `DQTypes, DSLTypes, FlattenType, OutputType, ProcessType, SinkType and WriteMode` are compared with config using a regex-based approach. This will make unnecessary overhead in terms of execution time and maintainability.\n\nThis PR uses a predefined enum based approach than regex-based to provide the same functionality.\n\n**Does this PR introduce any user-facing change?**\nNo\n\n**How was this patch tested?**\nGriffin test suite.\n\nAuthor: tusharpatil \u003ctus\u003e\nAuthor: tusharpatil20 \u003c35911427+tusharpatil20@users.noreply.github.com\u003e\n\nCloses #558 from tusharpatil20/enum-based-configs.\n" }, { "commit": "a12729b32764a6d76359de359e15da71e06fda8f", "tree": "50c78f18a495bbb88b12a8f0cd1967f80299a480", "parents": [ "8ef73f1e8c90330d22eb7d370f5e0f3faa1b22aa" ], "author": { "name": "wankunde", "email": "wankunde@163.com", "time": "Fri Dec 13 21:25:14 2019 +0800" }, "committer": { "name": "William Guo", "email": "guoyp@apache.org", "time": "Fri Dec 13 21:25:14 2019 +0800" }, "message": "[GRIFFIN-310] Unified scala code style and enable scala code style checking by default\n\nGriffin has more and more contributors, so we need a unified code style and enable code style by default.\n\nAuthor: wankunde \u003cwankunde@163.com\u003e\n\nCloses #559 from wankunde/scalastyle.\n" }, { "commit": "8ef73f1e8c90330d22eb7d370f5e0f3faa1b22aa", "tree": "0175adb53ecb8bec12066af181894c4634281665", "parents": [ "cef0a0ec9b9265798bc4e99c1903aa8b68741a04" ], "author": { "name": "wankunde", "email": "wankunde@163.com", "time": "Sat Nov 30 13:08:12 2019 +0800" }, "committer": { "name": "wankunde", "email": "wankunde@163.com", "time": "Sat Nov 30 13:08:12 2019 +0800" }, "message": "[GRIFFIN-301] Update custom data connector to have the same parameters as build-in data connector\n\nNow custom data connectors have different parameters with build-in data connector, which will confuse the user.\n\nFor example : https://issues.apache.org/jira/browse/GRIFFIN-300\n\nAuthor: wankunde \u003cwankunde@163.com\u003e\n\nCloses #556 from wankunde/custom_data_connector.\n" }, { "commit": "cef0a0ec9b9265798bc4e99c1903aa8b68741a04", "tree": "9c4d8ef0b661666a7825eb6e939f30245bc26d61", "parents": [ "6d9e9a7fdac0c2b7833cacbf081f9d5a93c59773" ], "author": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Sat Nov 30 13:01:30 2019 +0800" }, "committer": { "name": "wankunde", "email": "wankunde@163.com", "time": "Sat Nov 30 13:01:30 2019 +0800" }, "message": "[GRIFFIN-304] Eliminate older contexts\n\n**What changes were proposed in this pull request?**\n\nAs SparkSession is a direct replacement for SparkContext, SQLContext and HiveContext, there is no need to pass/ instantiate them. If any of the oder contexts are needed, they can be derived from SparkSession.\n\nThis issue aims to eliminate dependency on older Contexts in favour of SparkSession.\n\n**Does this PR introduce any user-facing change?**\nNo\n\n**How was this patch tested?**\nGriffin test suite.\n\nAuthor: chitralverma \u003cchitralverma@gmail.com\u003e\n\nCloses #557 from chitralverma/eliminate-older-contexts.\n" }, { "commit": "6d9e9a7fdac0c2b7833cacbf081f9d5a93c59773", "tree": "6abff794159ee2cb990a2efc9e70aafdeb7afb1c", "parents": [ "48f304dc316ed380fef2c5c7891f7ec41e0ddd30" ], "author": { "name": "wankunde", "email": "wankunde@163.com", "time": "Mon Nov 25 19:32:00 2019 +0800" }, "committer": { "name": "wankunde", "email": "wankunde@163.com", "time": "Mon Nov 25 19:32:00 2019 +0800" }, "message": "Bug fix for reflecting a custom sink object\n\nBug fix for reflecting a custom sink object.\n\nAuthor: wankunde \u003cwankunde@163.com\u003e\n\nCloses #551 from wankunde/custom_sink.\n" }, { "commit": "48f304dc316ed380fef2c5c7891f7ec41e0ddd30", "tree": "0c528cd4c8a6a4cedf48c8485de6d110a9d50479", "parents": [ "350663f61842ede34f06224aec92e38d1bff92ee" ], "author": { "name": "chitralverma", "email": "chitralverma@gmail.com", "time": "Thu Nov 21 09:26:44 2019 +0800" }, "committer": { "name": "William Guo", "email": "guoyp@apache.org", "time": "Thu Nov 21 09:26:44 2019 +0800" }, "message": "[GRIFFIN-297] Allow support for additional file based data sources\n\n**What changes were proposed in this pull request?**\n\nThe PR extends the current support beyond just Avro and Text for various file based data sources (Parquet, ORC, etc).\n\n - Allows users to specify additional file based data sources like Parquet, CSV, TSV, ORC etc.\n - Allows data to be read directly from stand-alone files as well as directories present in both local/ distributed file systems.\n - Allows users to specify schema directly through options (useful for CSV/ TSV types).\n\nA sample config looks like,\n\n```\n{\n \"name\": \"source\",\n \"baseline\": true,\n \"connectors\": [\n {\n \"type\": \"file\",\n \"version\": \"1.7\",\n \"config\": {\n \"format\": \"parquet\",\n \"options\": {\n \"k1\": \"v1\",\n \"k2\": \"v2\"\n },\n \"paths\": [\n \"/home/chitral/path/to/source/\",\n \"/home/chitral/path/to/test.parquet\"\n ]\n }\n }\n ]\n}\n\n```\n**Does this PR introduce any user-facing change?**\nNo\n\n**How was this patch tested?**\nGriffin test suite. Some additional unit test has also been added.\n\nAuthor: chitralverma \u003cchitralverma@gmail.com\u003e\n\nCloses #555 from chitralverma/allow_file_based_batch_connectors.\n" }, { "commit": "350663f61842ede34f06224aec92e38d1bff92ee", "tree": "e90548fb931a23d8e1a7713f96d1c233944653bf", "parents": [ "2f3c146579424eac43117ab06a2addf995596ac6" ], "author": { "name": "wankunde", "email": "wankunde@163.com", "time": "Sat Nov 16 11:11:34 2019 +0800" }, "committer": { "name": "William Guo", "email": "guoyp@apache.org", "time": "Sat Nov 16 11:11:34 2019 +0800" }, "message": "[GRIFFIN-298] add CompletenessExpr2DQSteps test case\n\nAdd some test case for CompletenessExpr2DQSteps transform， in addition some small code optimization.\n\nAuthor: wankunde \u003cwankunde@163.com\u003e\n\nCloses #550 from wankunde/CompletenessExpr2DQSteps.\n" }, { "commit": "2f3c146579424eac43117ab06a2addf995596ac6", "tree": "d2c5d8aad25fb23f544ee1215cf89df2e8b2b1fc", "parents": [ "71a2d79bb61fba46b4466b626596a431a992cfbc" ], "author": { "name": "wankunde", "email": "wankunde@163.com", "time": "Thu Nov 14 09:07:33 2019 +0800" }, "committer": { "name": "William Guo", "email": "guoyp@apache.org", "time": "Thu Nov 14 09:07:33 2019 +0800" }, "message": "[GRIFFIN-299] Add oracle jdk8 support in travis build phase\n\nAs Ubuntu Xenial has become the default Travis CI build environment, we may build fail. \n\nThe workaround is to either add `dist: trusty` to your .travis.yml file or use `openjdk8`.\n\nAuthor: wankunde \u003cwankunde@163.com\u003e\n\nCloses #552 from wankunde/travis.\n" }, { "commit": "71a2d79bb61fba46b4466b626596a431a992cfbc", "tree": "3730e1332ea5c75a57af47db07f27294ddacc697", "parents": [ "0119a19c93d0607ecfda2c5e029169e689c2bfd0" ], "author": { "name": "wankunde", "email": "wankunde@163.com", "time": "Fri Nov 01 21:56:28 2019 +0800" }, "committer": { "name": "William Guo", "email": "guoyp@apache.org", "time": "Fri Nov 01 21:56:28 2019 +0800" }, "message": "[GRIFFIN-295] Limit the memory used by test case\n\nThe container memory size is 3G in travis, but out test cases always uses more than 3G memory, so `Cannot allocate memory` will be thrown.\n\n```\nJava HotSpot(TM) 64-Bit Server VM warning: INFO: os::commit_memory(0x00000000fe980000, 23592960, 0) failed; error\u003d\u0027Cannot allocate memory\u0027 (errno\u003d12)\n#\n# There is insufficient memory for the Java Runtime Environment to continue.\n# Native memory allocation (mmap) failed to map 23592960 bytes for committing reserved memory.\n# An error report file with more information is saved as:\n# /home/travis/build/apache/griffin/measure/hs_err_pid11948.log\n# [ timer expired, abort... ]\n```\n\nThere are two kind of programs in our tests, the maven main program and the tests run by maven-surefire-plugin and scalatest-maven-plugin.\nIf the memory is unlimited, test cases will occupy as much memory as possible especially spark jobs.\n\nSpark jobs will not free the memory until a full GC occurs , even if we have stopped the spark context .so we need to limit the momery used by test cases.\n\nWe can limit the maven memory used by set export MAVEN_OPTS\u003d\" -Xmx1024m -XX:ReservedCodeCacheSize\u003d128m\" , and we can limit the memory used by spark job tests by configuring the maven-surefire-plugin and scalatest-maven-plugin.\n\nFor example:\nBefore we limit the memory used, maven program occupy 1.5G memory and spark job occupy 1.8G memory.\n\u003cimg width\u003d\"1153\" alt\u003d\"1\" src\u003d\"https://user-images.githubusercontent.com/3626747/67956554-40108e00-fc2f-11e9-83de-d0840fb42cb7.png\"\u003e\n\u003cimg width\u003d\"1150\" alt\u003d\"2\" src\u003d\"https://user-images.githubusercontent.com/3626747/67956567-46066f00-fc2f-11e9-8a73-6d141be28e70.png\"\u003e\n\nAfter we limit the memory used, maven program occupy 1G memory and spark job occupy 1G memory.\n\u003cimg width\u003d\"1142\" alt\u003d\"3\" src\u003d\"https://user-images.githubusercontent.com/3626747/67956579-4999f600-fc2f-11e9-9cd4-9032966ca923.png\"\u003e\n\u003cimg width\u003d\"1139\" alt\u003d\"4\" src\u003d\"https://user-images.githubusercontent.com/3626747/67956586-4dc61380-fc2f-11e9-800b-1d26d637a479.png\"\u003e\n\nAuthor: wankunde \u003cwankunde@163.com\u003e\n\nCloses #546 from wankunde/testcase_memory_limit.\n" }, { "commit": "0119a19c93d0607ecfda2c5e029169e689c2bfd0", "tree": "e0880be419634fdab74589d05a367274613a71b3", "parents": [ "30eb3dd9796ac10b30e7a21dc6c46926d8a56738" ], "author": { "name": "Zhao Li", "email": "mrlzbebop@gmail.com", "time": "Fri Nov 01 21:36:02 2019 +0800" }, "committer": { "name": "William Guo", "email": "guoyp@apache.org", "time": "Fri Nov 01 21:36:02 2019 +0800" }, "message": "[GRIFFIN-294] bugfix for completness enumeration wrong sql\n\nif there is only \u0027hive_none\u0027 in enumeration values list, it will generate wrong sql. Update code to fix this bug\n\nAuthor: Zhao Li \u003cmrlzbebop@gmail.com\u003e\nAuthor: Zhao \u003cLittleZhao@users.noreply.github.com\u003e\n\nCloses #545 from LittleZhao/griffin-294.\n" }, { "commit": "30eb3dd9796ac10b30e7a21dc6c46926d8a56738", "tree": "5d26e8ffdf2fe57720fb3f8cf8b207fb8ee2a536", "parents": [ "36b447440e9ddbb9101fd9607777f45f056d8624" ], "author": { "name": "ahutsunshine", "email": "ahutsunshine@gmail.com", "time": "Thu Oct 24 19:30:35 2019 +0800" }, "committer": { "name": "William Guo", "email": "guoyp@apache.org", "time": "Thu Oct 24 19:30:35 2019 +0800" }, "message": "fix tests failure\n\n1. fix hive connect failure\n2.improve test run time\n\nAuthor: ahutsunshine \u003cahutsunshine@gmail.com\u003e\n\nCloses #543 from ahutsunshine/master.\n" }, { "commit": "36b447440e9ddbb9101fd9607777f45f056d8624", "tree": "1f921b874518e8bfbe24ab8b326d4ed16bd9b97d", "parents": [ "e9dda8b53aadfc127854a89ac797c761b95de92a" ], "author": { "name": "William Guo", "email": "guoyp@apache.org", "time": "Tue Oct 22 21:30:00 2019 +0800" }, "committer": { "name": "William Guo", "email": "guoyp@apache.org", "time": "Tue Oct 22 21:30:00 2019 +0800" }, "message": "remove -Xms500m -Xmx1g -XX:MaxPermSize\u003d256m temporarily\n\nAuthor: William Guo \u003cguoyp@apache.org\u003e\n\nCloses #544 from guoyuepeng/fix_maven_memory_opt.\n" }, { "commit": "e9dda8b53aadfc127854a89ac797c761b95de92a", "tree": "2577e3a11f2294d597c3ef0ff27475fab40b9ac8", "parents": [ "72b0447d38ebd7aec23ad1494967716e84b4c367" ], "author": { "name": "neveljkovic", "email": "nveljkovic@plume.com", "time": "Mon Oct 14 20:14:56 2019 +0800" }, "committer": { "name": "William Guo", "email": "guoyp@apache.org", "time": "Mon Oct 14 20:14:56 2019 +0800" }, "message": "[GRIFFIN-293][SERVICE] livy.need.queue\u003dtrue\n\nThis is how we fixed issue described in https://issues.apache.org/jira/browse/GRIFFIN-293\nSolution is deployed to our servers and works OK.\n\nAuthor: neveljkovic \u003cnveljkovic@plume.com\u003e\n\nCloses #541 from neveljkovic/griffin-293.\n" }, { "commit": "72b0447d38ebd7aec23ad1494967716e84b4c367", "tree": "88de882b43ff352b6bd41b0bfedabbeaf29ef376", "parents": [ "4f3caa57b9dc8470f631aa62bc2994fb61bf089c" ], "author": { "name": "‘Zhao", "email": "mrlzbebop@gmail.com", "time": "Thu Oct 10 23:13:27 2019 +0800" }, "committer": { "name": "William Guo", "email": "guoyp@apache.org", "time": "Thu Oct 10 23:13:27 2019 +0800" }, "message": "[GRIFFIN-289] New feature for griffin COMPLETENESS dq type\n\nAs describing in GRIFFIN-289, add two new ways to check \u0027incompleteness\u0027 record: regular expression and enumeration.\n\nAdd \u0027error.confs\u0027 in dq json file. Each json object in \u0027error.confs\u0027 list means one column configuration.\n\nIf do not have \u0027error.confs\u0027, using old \u0027incompleteness\u0027 process, which is compatible for existing json file.\n\nAdd ut for the new json format.\n\nAuthor: ‘Zhao \u003cmrlzbebop@gmail.com\u003e\nAuthor: Zhao Li \u003cmrlzbebop@gmail.com\u003e\n\nCloses #538 from LittleZhao/griffin-289.\n" }, { "commit": "4f3caa57b9dc8470f631aa62bc2994fb61bf089c", "tree": "93cbca66acf7513437fb3bf9bf7355e0e534fa51", "parents": [ "ad2a2b10ce24ddce94af37021cc3daeb75f39a50" ], "author": { "name": "jasonliaoxiaoge", "email": "181276056@qq.com", "time": "Thu Sep 26 22:53:27 2019 +0800" }, "committer": { "name": "William Guo", "email": "guoyp@apache.org", "time": "Thu Sep 26 22:53:27 2019 +0800" }, "message": "add placeholder for cron expression\n\nadd placeholder for cron expression, cause java quartz is a little difference from crontab in linux\n\nAuthor: jasonliaoxiaoge \u003c181276056@qq.com\u003e\n\nCloses #503 from jasonliaoxiaoge/master.\n" }, { "commit": "ad2a2b10ce24ddce94af37021cc3daeb75f39a50", "tree": "62b13c62c9e547f3ab303adeecdadacfc32a6795", "parents": [ "b121dd92cceefaf438792150afd34f2a0f047674" ], "author": { "name": "wankunde", "email": "wankunde@163.com", "time": "Wed Sep 18 07:29:05 2019 +0800" }, "committer": { "name": "William Guo", "email": "guoyp@apache.org", "time": "Wed Sep 18 07:29:05 2019 +0800" }, "message": "[GRIFFIN-290] Fix bug for submitting job to livy\n\nWhen griffin submit multiple DQ jobs to livy, the http parameter `name` is always griffin.\nSo livy will reject them.\n\njob request :\n\n```\n[owner: null, request: [proxyUser: None, file: hdfs://nameservice-standby/user/kun.wan/measure-0.6.0-SNAPSHOT.jar,\nargs: {\n\"spark\" :\n\nUnknown macro: { \"log.level\" }\n,\n\"sinks\" : [\n\nUnknown macro: { .... }\n],\n\"griffin.checkpoint\" : [ ]\n},{\n\"measure.type\" : \"griffin\",\n\"id\" : 5202,\n\"name\" : \"spu_null_check\",\n\"owner\" : \"test\",\n\"description\" : \"check null value for store and category\",\n\"deleted\" : false,\n\"timestamp\" : 1568195100000,\n\"dq.type\" : \"PROFILING\",\n\"sinks\" : [ \"ELASTICSEARCH\", \"HDFS\" ],\n\"process.type\" : \"BATCH\",\n\"rule.description\" :\n\n,\n\"data.sources\" : [\n\nUnknown macro: { .... }\n],\n\"evaluate.rule\" :\n\n,\n\"measure.type\" : \"griffin\"\n},raw,raw, driverMemory: 1g, executorMemory: 6g, executorCores: 2, numExecutors: 6, queue: root.users.kun_dot_wan, name: griffin]]\n```\n\nlivy Response :\n\n```\n400 Bad Request\n[Date:\"Thu, 12 Sep 2019 10:00:00 GMT\", Content-Type:\"application/json;charset\u003dutf-8\", Content-Length:\"47\", Server:\"Jetty(9.3.24.v20180605)\"]\n{\"msg\":\"Duplicate session name: Some(griffin)\"}\n```\n\nAuthor: wankunde \u003cwankunde@163.com\u003e\n\nCloses #534 from wankunde/livy_bug.\n" }, { "commit": "b121dd92cceefaf438792150afd34f2a0f047674", "tree": "aacb6681a8b0e2586d3015bab24c76618e8c21cd", "parents": [ "805b09605f83d00639e8f87edde6b80e04ca5529" ], "author": { "name": "wankunde", "email": "wankunde@163.com", "time": "Sat Sep 14 15:45:39 2019 +0800" }, "committer": { "name": "William Guo", "email": "guoyp@apache.org", "time": "Sat Sep 14 15:45:39 2019 +0800" }, "message": "[GRIFFIN-291] Relocate HttpClient code in measure jar using shade plugin\n\nNow projects use different version of httpclient , and very easy to have conflicts with other components.\n\nWe can shade the httpclient sources into measure jar and the conflicts disappear.\n\nAuthor: wankunde \u003cwankunde@163.com\u003e\n\nCloses #535 from wankunde/httpclient.\n" }, { "commit": "805b09605f83d00639e8f87edde6b80e04ca5529", "tree": "5fb00625a461966747b94159cf320841e33738cd", "parents": [ "e039434610aa1794f43e20e089f6362ecf015163" ], "author": { "name": "wankunde", "email": "wankunde@163.com", "time": "Fri Sep 13 22:17:03 2019 +0800" }, "committer": { "name": "William Guo", "email": "guoyp@apache.org", "time": "Fri Sep 13 22:17:03 2019 +0800" }, "message": "[GRIFFIN-288] optimize hdfs sink\n\nWhen we sink records to hdfs , it may be OOM if the result is huge.\n\n```\n19/09/06 18:52:39 INFO LineBufferedStream: 19/09/06 18:52:39 ERROR sink.HdfsSink: Java heap space\n19/09/06 18:52:39 INFO LineBufferedStream: java.lang.OutOfMemoryError: Java heap space\n19/09/06 18:52:39 INFO LineBufferedStream: at java.util.Arrays.copyOf(Arrays.java:3332)\n19/09/06 18:52:39 INFO LineBufferedStream: at java.lang.AbstractStringBuilder.ensureCapacityInternal(AbstractStringBuilder.java:124)\n19/09/06 18:52:39 INFO LineBufferedStream: at java.lang.AbstractStringBuilder.append(AbstractStringBuilder.java:448)\n19/09/06 18:52:39 INFO LineBufferedStream: at java.lang.StringBuilder.append(StringBuilder.java:136)\n19/09/06 18:52:39 INFO LineBufferedStream: at scala.collection.mutable.StringBuilder.append(StringBuilder.scala:200)\n19/09/06 18:52:39 INFO LineBufferedStream: at scala.collection.TraversableOnce$$anonfun$addString$1.apply(TraversableOnce.scala:364)\n19/09/06 18:52:39 INFO LineBufferedStream: at scala.collection.IndexedSeqOptimized$class.foreach(IndexedSeqOptimized.scala:33)\n19/09/06 18:52:39 INFO LineBufferedStream: at scala.collection.mutable.WrappedArray.foreach(WrappedArray.scala:35)\n19/09/06 18:52:39 INFO LineBufferedStream: at scala.collection.TraversableOnce$class.addString(TraversableOnce.scala:357)\n19/09/06 18:52:39 INFO LineBufferedStream: at scala.collection.AbstractTraversable.addString(Traversable.scala:104)\n19/09/06 18:52:39 INFO LineBufferedStream: at scala.collection.TraversableOnce$class.mkString(TraversableOnce.scala:323)\n19/09/06 18:52:39 INFO LineBufferedStream: at scala.collection.AbstractTraversable.mkString(Traversable.scala:104)\n19/09/06 18:52:39 INFO LineBufferedStream: at scala.collection.TraversableOnce$class.mkString(TraversableOnce.scala:325)\n19/09/06 18:52:39 INFO LineBufferedStream: at scala.collection.AbstractTraversable.mkString(Traversable.scala:104)\n19/09/06 18:52:39 INFO LineBufferedStream: at org.apache.griffin.measure.sink.HdfsSink.org$apache$griffin$measure$sink$HdfsSink$$sinkRecords2Hdfs(HdfsSink.scala:191)\n19/09/06 18:52:39 INFO LineBufferedStream: at org.apache.griffin.measure.sink.HdfsSink.sinkRecords(HdfsSink.scala:133)\n19/09/06 18:52:39 INFO LineBufferedStream: at org.apache.griffin.measure.sink.MultiSinks$$anonfun$sinkRecords$1.apply(MultiSinks.scala:63)\n19/09/06 18:52:39 INFO LineBufferedStream: at org.apache.griffin.measure.sink.MultiSinks$$anonfun$sinkRecords$1.apply(MultiSinks.scala:61)\n19/09/06 18:52:39 INFO LineBufferedStream: at scala.collection.immutable.List.foreach(List.scala:392)\n19/09/06 18:52:39 INFO LineBufferedStream: at org.apache.griffin.measure.sink.MultiSinks.sinkRecords(MultiSinks.scala:61)\n19/09/06 18:52:39 INFO LineBufferedStream: at org.apache.griffin.measure.step.write.RecordWriteStep.execute(RecordWriteStep.scala:49)\n19/09/06 18:52:39 INFO LineBufferedStream: at org.apache.griffin.measure.step.transform.SparkSqlTransformStep.doExecute(SparkSqlTransformStep.scala:40)\n19/09/06 18:52:39 INFO LineBufferedStream: at org.apache.griffin.measure.step.transform.TransformStep$class.execute(TransformStep.scala:72)\n19/09/06 18:52:39 INFO LineBufferedStream: at org.apache.griffin.measure.step.transform.SparkSqlTransformStep.execute(SparkSqlTransformStep.scala:27)\n19/09/06 18:52:39 INFO LineBufferedStream: at org.apache.griffin.measure.step.transform.TransformStep$$anonfun$2$$anonfun$apply$1.apply$mcV$sp(TransformStep.scala:51)\n19/09/06 18:52:39 INFO LineBufferedStream: at org.apache.griffin.measure.step.transform.TransformStep$$anonfun$2$$anonfun$apply$1.apply(TransformStep.scala:50)\n19/09/06 18:52:39 INFO LineBufferedStream: at org.apache.griffin.measure.step.transform.TransformStep$$anonfun$2$$anonfun$apply$1.apply(TransformStep.scala:50)\n19/09/06 18:52:39 INFO LineBufferedStream: at scala.concurrent.impl.Future$PromiseCompletingRunnable.liftedTree1$1(Future.scala:24)\n19/09/06 18:52:39 INFO LineBufferedStream: at scala.concurrent.impl.Future$PromiseCompletingRunnable.run(Future.scala:24)\n19/09/06 18:52:39 INFO LineBufferedStream: at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\n19/09/06 18:52:39 INFO LineBufferedStream: at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\n19/09/06 18:52:39 INFO LineBufferedStream: at java.lang.Thread.run(Thread.java:748)\n```\n\nAuthor: wankunde \u003cwankunde@163.com\u003e\n\nCloses #533 from wankunde/hdfssink.\n" }, { "commit": "e039434610aa1794f43e20e089f6362ecf015163", "tree": "38cfc8e4f9459e370d7139276bdba0347252a799", "parents": [ "16d43040e41302a786a7d9ac6cd5e8260373b0ca" ], "author": { "name": "wankunde", "email": "wankunde@163.com", "time": "Mon Sep 09 08:28:19 2019 +0800" }, "committer": { "name": "William Guo", "email": "guoyp@apache.org", "time": "Mon Sep 09 08:28:19 2019 +0800" }, "message": "[GRIFFIN-286] Remove spark-testing-base dependency jar\n\nNow we use spark-testing-base jar to test spark job in measure module, but this jar maybe conflict with the spark version(CDH spark version,spark AE) or scala version(few scala version with specified spark version).\n\nSo I suggest removing the dependency of this package.\n\nAuthor: wankunde \u003cwankunde@163.com\u003e\n\nCloses #531 from wankunde/remoteSparkTestBase.\n" }, { "commit": "16d43040e41302a786a7d9ac6cd5e8260373b0ca", "tree": "5417ecb16742ec96fb2c04fc9ecb304d6aebb670", "parents": [ "043dd8d24f958715f23198a269aeef9b5c1f95fb" ], "author": { "name": "Simon George", "email": "simongeorge@rentalcars.com", "time": "Wed Sep 04 21:38:54 2019 +0800" }, "committer": { "name": "William Guo", "email": "guoyp@apache.org", "time": "Wed Sep 04 21:38:54 2019 +0800" }, "message": "Made UI tests run without errors\n\nThese changes allow the UI tests to execute without errors when run using \"npm test\"\n\nAuthor: Simon George \u003csimongeorge@rentalcars.com\u003e\n\nCloses #529 from simegeorge/fix-ui-tests.\n" }, { "commit": "043dd8d24f958715f23198a269aeef9b5c1f95fb", "tree": "eb8348ccad444adc88893ad6fd094e534ebde7b5", "parents": [ "5447cf440b7053bc7c63e4fe179a846084ea76e2" ], "author": { "name": "Johnnie", "email": "joohnnie.z@gmail.com", "time": "Tue Sep 03 07:40:33 2019 +0800" }, "committer": { "name": "William Guo", "email": "guoyp@apache.org", "time": "Tue Sep 03 07:40:33 2019 +0800" }, "message": "[GRIFFIN-279] Upgrade Spring boot to 2.1.7.RELEASE\n\nAs spring boot 1.x is end of life from Aug 1st 2019, it would be great to migrate to 2.1.x.\n\nBelow is the announcement\n\nhttps://spring.io/blog/2018/07/30/spring-boot-1-x-eol-aug-1st-2019\n\nMigrate Guide is\n\nhttps://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.0-Migration-Guide\n\nAuthor: Johnnie \u003cjoohnnie.z@gmail.com\u003e\n\nCloses #528 from joohnnie/GRIFFIN-279.\n" }, { "commit": "5447cf440b7053bc7c63e4fe179a846084ea76e2", "tree": "8ceeb6277906f054b05ef7535d08e55bb1fed9f7", "parents": [ "efc5a0454f9610e0545ffe0519d1657d81a22767", "c469ea0a5228f75ffc9445510ae48ab51e0ad7df" ], "author": { "name": "Lionel Liu", "email": "bhlx3lyx7@163.com", "time": "Thu Aug 29 22:13:46 2019 +0800" }, "committer": { "name": "GitHub", "email": "noreply@github.com", "time": "Thu Aug 29 22:13:46 2019 +0800" }, "message": "Merge pull request #530 from aleksgor/GRIFFIN-AGORSHKOV\n\nAdd Mysql, Cassandra and Elasticsearch connectors." } ], "next": "efc5a0454f9610e0545ffe0519d1657d81a22767" }