cherry pick #3656 #3659 #3632 to 0.14 (#3661)
* fix(gitlab): modify pipeline (#3659)
closes #3658
* fix(tapd): add incremental support (#3632)
closes #3631
* fix(tapd): modify incremental
closes #3631
diff --git a/plugins/gitlab/e2e/raw_tables/_raw_gitlab_api_pipeline.csv b/plugins/gitlab/e2e/raw_tables/_raw_gitlab_api_pipeline.csv
index da64513..87381da 100644
--- a/plugins/gitlab/e2e/raw_tables/_raw_gitlab_api_pipeline.csv
+++ b/plugins/gitlab/e2e/raw_tables/_raw_gitlab_api_pipeline.csv
@@ -1,8 +1,8 @@
id,params,data,url,input,created_at
-152,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":457474837,""iid"":1,""project_id"":12345678,""sha"":""b1b82852d48b516a18e56c5bab0ebf54b8f4ccfd"",""ref"":""renovate/pin-dependencies"",""status"":""failed"",""source"":""external"",""created_at"":""2022-01-27T10:07:09.429Z"",""updated_at"":""2022-01-27T10:07:09.553Z"",""web_url"":""https://gitlab.com/merico-dev/ee/charts/-/pipelines/457474837""}",https://gitlab.com/api/v4/projects/12345678/pipelines?page=1&per_page=100&sort=asc&with_stats=true,null,2022-06-16 15:15:02.849
+152,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":457474837,""iid"":1,""project_id"":12345678,""sha"":""b1b82852d48b516a18e56c5bab0ebf54b8f4ccfd"",""ref"":""renovate/pin-dependencies"",""status"":""running"",""source"":""external"",""created_at"":""2022-01-27T10:07:09.429Z"",""updated_at"":null,""web_url"":""https://gitlab.com/merico-dev/ee/charts/-/pipelines/457474837""}",https://gitlab.com/api/v4/projects/12345678/pipelines?page=1&per_page=100&sort=asc&with_stats=true,null,2022-06-16 15:15:02.849
153,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":457474996,""iid"":2,""project_id"":12345678,""sha"":""739ab912e6e1ed27cecd8a2d00bfd6fa52afd90a"",""ref"":""renovate/jest-monorepo"",""status"":""failed"",""source"":""external"",""created_at"":""2022-01-27T10:07:18.884Z"",""updated_at"":""2022-01-27T10:07:19.043Z"",""web_url"":""https://gitlab.com/merico-dev/ee/charts/-/pipelines/457474996""}",https://gitlab.com/api/v4/projects/12345678/pipelines?page=1&per_page=100&sort=asc&with_stats=true,null,2022-06-16 15:15:02.849
154,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":457475160,""iid"":3,""project_id"":12345678,""sha"":""44d127e0ab7dbc4bc259b55929c9d00b62fc3bf4"",""ref"":""renovate/lodash-monorepo"",""status"":""failed"",""source"":""external"",""created_at"":""2022-01-27T10:07:26.435Z"",""updated_at"":""2022-01-27T10:07:26.638Z"",""web_url"":""https://gitlab.com/merico-dev/ee/charts/-/pipelines/457475160""}",https://gitlab.com/api/v4/projects/12345678/pipelines?page=1&per_page=100&sort=asc&with_stats=true,null,2022-06-16 15:15:02.849
-155,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":457475337,""iid"":4,""project_id"":12345678,""sha"":""10a6464b6bd2cf4b59b8ac37ce1466e013f5a20d"",""ref"":""renovate/shx-0.x"",""status"":""failed"",""source"":""external"",""created_at"":""2022-01-27T10:07:36.502Z"",""updated_at"":""2022-01-27T10:07:37.545Z"",""web_url"":""https://gitlab.com/merico-dev/ee/charts/-/pipelines/457475337""}",https://gitlab.com/api/v4/projects/12345678/pipelines?page=1&per_page=100&sort=asc&with_stats=true,null,2022-06-16 15:15:02.849
+155,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":457475337,""iid"":4,""project_id"":12345678,""sha"":""10a6464b6bd2cf4b59b8ac37ce1466e013f5a20d"",""ref"":""renovate/shx-0.x"",""status"":""running"",""source"":""external"",""created_at"":""2022-01-27T10:07:36.502Z"",""updated_at"":null,""web_url"":""https://gitlab.com/merico-dev/ee/charts/-/pipelines/457475337""}",https://gitlab.com/api/v4/projects/12345678/pipelines?page=1&per_page=100&sort=asc&with_stats=true,null,2022-06-16 15:15:02.849
156,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":485811050,""iid"":5,""project_id"":12345678,""sha"":""c791ea6949d6b4aadf79b15ba666cb690c6527ac"",""ref"":""EE-7121"",""status"":""failed"",""source"":""push"",""created_at"":""2022-03-07T06:26:42.109Z"",""updated_at"":""2022-03-07T06:26:42.109Z"",""web_url"":""https://gitlab.com/merico-dev/ee/charts/-/pipelines/485811050""}",https://gitlab.com/api/v4/projects/12345678/pipelines?page=1&per_page=100&sort=asc&with_stats=true,null,2022-06-16 15:15:02.849
157,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":485811059,""iid"":6,""project_id"":12345678,""sha"":""c791ea6949d6b4aadf79b15ba666cb690c6527ac"",""ref"":""refs/merge-requests/74/head"",""status"":""failed"",""source"":""merge_request_event"",""created_at"":""2022-03-07T06:26:43.784Z"",""updated_at"":""2022-03-07T06:26:43.784Z"",""web_url"":""https://gitlab.com/merico-dev/ee/charts/-/pipelines/485811059""}",https://gitlab.com/api/v4/projects/12345678/pipelines?page=1&per_page=100&sort=asc&with_stats=true,null,2022-06-16 15:15:02.849
158,"{""ConnectionId"":1,""ProjectId"":12345678}","{""id"":485813816,""iid"":7,""project_id"":12345678,""sha"":""ecc7c0b2874c812ed882c9effbbda26e0abc7110"",""ref"":""EE-7121"",""status"":""failed"",""source"":""push"",""created_at"":""2022-03-07T06:33:56.824Z"",""updated_at"":""2022-03-07T06:33:56.824Z"",""web_url"":""https://gitlab.com/merico-dev/ee/charts/-/pipelines/485813816""}",https://gitlab.com/api/v4/projects/12345678/pipelines?page=1&per_page=100&sort=asc&with_stats=true,null,2022-06-16 15:15:02.849
diff --git a/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_pipelines.csv b/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_pipelines.csv
index 57ac49b..4256089 100644
--- a/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_pipelines.csv
+++ b/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_pipelines.csv
@@ -1,8 +1,8 @@
connection_id,gitlab_id,gitlab_created_at,project_id,status,web_url,duration,started_at,finished_at,coverage,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
-1,457474837,2022-01-27T10:07:09.429+00:00,12345678,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/457474837,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,152,
+1,457474837,2022-01-27T10:07:09.429+00:00,12345678,running,https://gitlab.com/merico-dev/ee/charts/-/pipelines/457474837,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,152,
1,457474996,2022-01-27T10:07:18.884+00:00,12345678,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/457474996,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,153,
1,457475160,2022-01-27T10:07:26.435+00:00,12345678,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/457475160,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,154,
-1,457475337,2022-01-27T10:07:36.502+00:00,12345678,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/457475337,1,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,155,
+1,457475337,2022-01-27T10:07:36.502+00:00,12345678,running,https://gitlab.com/merico-dev/ee/charts/-/pipelines/457475337,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,155,
1,485811050,2022-03-07T06:26:42.109+00:00,12345678,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485811050,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,156,
1,485811059,2022-03-07T06:26:43.784+00:00,12345678,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485811059,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,157,
1,485813816,2022-03-07T06:33:56.824+00:00,12345678,failed,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485813816,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,158,
diff --git a/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_project_commits.csv b/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_project_commits.csv
deleted file mode 100644
index 719c4f8..0000000
--- a/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_project_commits.csv
+++ /dev/null
@@ -1,38 +0,0 @@
-connection_id,gitlab_project_id,commit_sha,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
-1,12345678,05976c113b02fb71d5c43682bd2f9d96a3454efa,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,34,
-1,12345678,083864201849f6707a34d24e61c43638696f95a6,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,6,
-1,12345678,08973065d33c6fd2fb1f592b7a1e0fbe2ee72edd,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,11,
-1,12345678,0a4574d52b5964c37bfcb2952f6c6a8a9c6d5bcf,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,7,
-1,12345678,0dbb8bfdbbf7373656fa3eb036bdcbb25ae162da,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,26,
-1,12345678,1cba71b355d2bd4e1350602cfb9b98cce5a5cb4d,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,8,
-1,12345678,1f16955d45cec904c8968bb27689d91a71e9a2f2,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,12,
-1,12345678,211795deaaaaddb23b797261cdcd148e6083d5b2,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,21,
-1,12345678,25f2b34099a3e1867fcf73a54e50f78f3c8d8f75,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,9,
-1,12345678,382084b42697577d3a6adf71ce73d4b5ddd22977,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,2,
-1,12345678,3f04e0a61d0c4d2dd736a6bcaa3a06826269a533,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,3,
-1,12345678,45bed79fcdc6ba3444245f55660f839db50d79c8,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,33,
-1,12345678,48295660f8bd0c0a024849d4407830face57d4bc,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,37,
-1,12345678,4f0d2f7b1a47d21b01a68e59c12d6f8885ab230e,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,24,
-1,12345678,553f7613b178301da4075202afe91ebda5d97ab5,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,16,
-1,12345678,5a141ad0c28960b02c40e6c476d8df3cc76f58c4,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,31,
-1,12345678,5b6a2be437e0c66ba6276030a61a10583c3f83fa,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,22,
-1,12345678,5d6c369200c633b06d85c9375f61fef297761ef1,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,19,
-1,12345678,645a724b558a7fa3885fa233d06f68f833d893fc,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,27,
-1,12345678,64abdcdd9ee9c821f56faf93c5bb0ebc57f575e4,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,28,
-1,12345678,6ebf519f25328e49a004f5a574c8b69ba7420be1,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,15,
-1,12345678,81b12c995b2fc12e20fd05113cf36dbd57b6e7b0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,20,
-1,12345678,846deac12a1263e4631ced45abbc5e157b23c1be,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,30,
-1,12345678,8a40765e605966842b05b5d8cdf2dd6890556d09,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,14,
-1,12345678,8a96911c5350f7575215f02f0242f7e27ad3d1f4,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,10,
-1,12345678,91e5666bdfe90e5ada3e65aaa9240b2b0b1a072f,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,17,
-1,12345678,93d13cf14a470995ec35186292bf165e1152dc2a,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,32,
-1,12345678,9a80506e158547d87e7ad2902c0196deff5b5c07,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,35,
-1,12345678,9e5ec04b0a8273b73936ea150275919b0602c260,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,13,
-1,12345678,9ef8183140bae8c39a440d9c7eb47f3bc7dcab75,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,18,
-1,12345678,a09f2d4ae88e78d7424fba1ae6d0c036b677271d,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,36,
-1,12345678,abbe0ab2c7bb1dc2cfaa3ef3062f378fb908ba71,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,5,
-1,12345678,ad25fcda15e310a7eb34b3e38ff9b962b54f0e03,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,1,
-1,12345678,bb6dfec8dcdacb5b6b37dbd8eb1aa6f9e8c06138,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,23,
-1,12345678,c4d406c077af582837d3611a46eded70151f93fb,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,29,
-1,12345678,e01d4f03811cd0da9949848731236e0aa261cf54,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,4,
-1,12345678,e24fecacb1a2c85f904bef57ffd5598ca2df023c,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,25,
diff --git a/plugins/gitlab/e2e/snapshot_tables/cicd_pipelines.csv b/plugins/gitlab/e2e/snapshot_tables/cicd_pipelines.csv
index e8ead06..f6cb8d0 100644
--- a/plugins/gitlab/e2e/snapshot_tables/cicd_pipelines.csv
+++ b/plugins/gitlab/e2e/snapshot_tables/cicd_pipelines.csv
@@ -1,21 +1,21 @@
-id,name,result,status,type,duration_sec,created_date,finished_date,environment
-gitlab:GitlabPipeline:1:457474837,gitlab:GitlabProject:1:12345678,FAILURE,DONE,,0,2022-01-27T10:07:09.429+00:00,2022-01-27T10:07:09.553+00:00,
-gitlab:GitlabPipeline:1:457474996,gitlab:GitlabProject:1:12345678,FAILURE,DONE,,0,2022-01-27T10:07:18.884+00:00,2022-01-27T10:07:19.043+00:00,
-gitlab:GitlabPipeline:1:457475160,gitlab:GitlabProject:1:12345678,FAILURE,DONE,,0,2022-01-27T10:07:26.435+00:00,2022-01-27T10:07:26.638+00:00,
-gitlab:GitlabPipeline:1:457475337,gitlab:GitlabProject:1:12345678,FAILURE,DONE,,1,2022-01-27T10:07:36.502+00:00,2022-01-27T10:07:37.545+00:00,
-gitlab:GitlabPipeline:1:485811050,gitlab:GitlabProject:1:12345678,FAILURE,DONE,,0,2022-03-07T06:26:42.109+00:00,2022-03-07T06:26:42.109+00:00,
-gitlab:GitlabPipeline:1:485811059,gitlab:GitlabProject:1:12345678,FAILURE,DONE,,0,2022-03-07T06:26:43.784+00:00,2022-03-07T06:26:43.784+00:00,
-gitlab:GitlabPipeline:1:485813816,gitlab:GitlabProject:1:12345678,FAILURE,DONE,,0,2022-03-07T06:33:56.824+00:00,2022-03-07T06:33:56.824+00:00,
-gitlab:GitlabPipeline:1:485813830,gitlab:GitlabProject:1:12345678,FAILURE,DONE,,0,2022-03-07T06:33:58.889+00:00,2022-03-07T06:33:58.889+00:00,
-gitlab:GitlabPipeline:1:485814501,gitlab:GitlabProject:1:12345678,FAILURE,DONE,,0,2022-03-07T06:35:28.111+00:00,2022-03-07T06:35:28.111+00:00,
-gitlab:GitlabPipeline:1:485814516,gitlab:GitlabProject:1:12345678,FAILURE,DONE,,0,2022-03-07T06:35:31.255+00:00,2022-03-07T06:35:31.255+00:00,
-gitlab:GitlabPipeline:1:485814871,gitlab:GitlabProject:1:12345678,FAILURE,DONE,,42,2022-03-07T06:36:50.020+00:00,2022-03-07T06:37:32.103+00:00,
-gitlab:GitlabPipeline:1:485817670,gitlab:GitlabProject:1:12345678,FAILURE,DONE,,1956,2022-03-07T06:45:09.471+00:00,2022-03-07T07:17:46.305+00:00,
-gitlab:GitlabPipeline:1:485837602,gitlab:GitlabProject:1:12345678,FAILURE,DONE,,434,2022-03-07T07:20:45.859+00:00,2022-03-07T07:28:00.277+00:00,
-gitlab:GitlabPipeline:1:485842553,gitlab:GitlabProject:1:12345678,FAILURE,DONE,,287,2022-03-07T07:30:47.018+00:00,2022-03-07T07:35:34.998+00:00,
-gitlab:GitlabPipeline:1:485845850,gitlab:GitlabProject:1:12345678,FAILURE,DONE,,419,2022-03-07T07:38:58.611+00:00,2022-03-07T07:45:58.412+00:00,
-gitlab:GitlabPipeline:1:485852752,gitlab:GitlabProject:1:12345678,FAILURE,DONE,,319,2022-03-07T07:46:09.385+00:00,2022-03-07T07:51:28.709+00:00,
-gitlab:GitlabPipeline:1:485865876,gitlab:GitlabProject:1:12345678,FAILURE,DONE,,480,2022-03-07T08:04:56.406+00:00,2022-03-07T08:12:56.453+00:00,
-gitlab:GitlabPipeline:1:485877118,gitlab:GitlabProject:1:12345678,FAILURE,DONE,,289,2022-03-07T08:22:48.943+00:00,2022-03-07T08:27:38.364+00:00,
-gitlab:GitlabPipeline:1:485905167,gitlab:GitlabProject:1:12345678,FAILURE,DONE,,687,2022-03-07T09:02:09.994+00:00,2022-03-07T09:13:37.013+00:00,
-gitlab:GitlabPipeline:1:485932863,gitlab:GitlabProject:1:12345678,SUCCESS,DONE,,398,2022-03-07T09:34:57.476+00:00,2022-03-07T09:41:36.267+00:00,
+id,name,result,status,type,duration_sec,created_date,finished_date,environment,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+gitlab:GitlabPipeline:1:457474837,gitlab:GitlabProject:1:12345678,SUCCESS,IN_PROGRESS,,0,2022-01-27T10:07:09.429+00:00,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,152,
+gitlab:GitlabPipeline:1:457474996,gitlab:GitlabProject:1:12345678,FAILURE,DONE,,0,2022-01-27T10:07:18.884+00:00,2022-01-27T10:07:19.043+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,153,
+gitlab:GitlabPipeline:1:457475160,gitlab:GitlabProject:1:12345678,FAILURE,DONE,,0,2022-01-27T10:07:26.435+00:00,2022-01-27T10:07:26.638+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,154,
+gitlab:GitlabPipeline:1:457475337,gitlab:GitlabProject:1:12345678,SUCCESS,IN_PROGRESS,,0,2022-01-27T10:07:36.502+00:00,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,155,
+gitlab:GitlabPipeline:1:485811050,gitlab:GitlabProject:1:12345678,FAILURE,DONE,,0,2022-03-07T06:26:42.109+00:00,2022-03-07T06:26:42.109+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,156,
+gitlab:GitlabPipeline:1:485811059,gitlab:GitlabProject:1:12345678,FAILURE,DONE,,0,2022-03-07T06:26:43.784+00:00,2022-03-07T06:26:43.784+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,157,
+gitlab:GitlabPipeline:1:485813816,gitlab:GitlabProject:1:12345678,FAILURE,DONE,,0,2022-03-07T06:33:56.824+00:00,2022-03-07T06:33:56.824+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,158,
+gitlab:GitlabPipeline:1:485813830,gitlab:GitlabProject:1:12345678,FAILURE,DONE,,0,2022-03-07T06:33:58.889+00:00,2022-03-07T06:33:58.889+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,159,
+gitlab:GitlabPipeline:1:485814501,gitlab:GitlabProject:1:12345678,FAILURE,DONE,,0,2022-03-07T06:35:28.111+00:00,2022-03-07T06:35:28.111+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,160,
+gitlab:GitlabPipeline:1:485814516,gitlab:GitlabProject:1:12345678,FAILURE,DONE,,0,2022-03-07T06:35:31.255+00:00,2022-03-07T06:35:31.255+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,161,
+gitlab:GitlabPipeline:1:485814871,gitlab:GitlabProject:1:12345678,FAILURE,DONE,,42,2022-03-07T06:36:50.020+00:00,2022-03-07T06:37:32.103+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,162,
+gitlab:GitlabPipeline:1:485817670,gitlab:GitlabProject:1:12345678,FAILURE,DONE,,1956,2022-03-07T06:45:09.471+00:00,2022-03-07T07:17:46.305+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,163,
+gitlab:GitlabPipeline:1:485837602,gitlab:GitlabProject:1:12345678,FAILURE,DONE,,434,2022-03-07T07:20:45.859+00:00,2022-03-07T07:28:00.277+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,164,
+gitlab:GitlabPipeline:1:485842553,gitlab:GitlabProject:1:12345678,FAILURE,DONE,,287,2022-03-07T07:30:47.018+00:00,2022-03-07T07:35:34.998+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,165,
+gitlab:GitlabPipeline:1:485845850,gitlab:GitlabProject:1:12345678,FAILURE,DONE,,419,2022-03-07T07:38:58.611+00:00,2022-03-07T07:45:58.412+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,166,
+gitlab:GitlabPipeline:1:485852752,gitlab:GitlabProject:1:12345678,FAILURE,DONE,,319,2022-03-07T07:46:09.385+00:00,2022-03-07T07:51:28.709+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,167,
+gitlab:GitlabPipeline:1:485865876,gitlab:GitlabProject:1:12345678,FAILURE,DONE,,480,2022-03-07T08:04:56.406+00:00,2022-03-07T08:12:56.453+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,168,
+gitlab:GitlabPipeline:1:485877118,gitlab:GitlabProject:1:12345678,FAILURE,DONE,,289,2022-03-07T08:22:48.943+00:00,2022-03-07T08:27:38.364+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,169,
+gitlab:GitlabPipeline:1:485905167,gitlab:GitlabProject:1:12345678,FAILURE,DONE,,687,2022-03-07T09:02:09.994+00:00,2022-03-07T09:13:37.013+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,170,
+gitlab:GitlabPipeline:1:485932863,gitlab:GitlabProject:1:12345678,SUCCESS,DONE,,398,2022-03-07T09:34:57.476+00:00,2022-03-07T09:41:36.267+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,171,
diff --git a/plugins/gitlab/tasks/pipeline_convertor.go b/plugins/gitlab/tasks/pipeline_convertor.go
index 366f5a0..7b49852 100644
--- a/plugins/gitlab/tasks/pipeline_convertor.go
+++ b/plugins/gitlab/tasks/pipeline_convertor.go
@@ -92,15 +92,13 @@
}
// rebuild the FinishedDate and DurationSec by Status
- finishedAt := time.Now()
if domainPipeline.Status != devops.DONE {
domainPipeline.FinishedDate = nil
- } else if gitlabPipeline.GitlabUpdatedAt != nil {
- finishedAt = *gitlabPipeline.GitlabUpdatedAt
+ domainPipeline.DurationSec = 0
+ } else if domainPipeline.FinishedDate != nil {
+ durationTime := domainPipeline.FinishedDate.Sub(createdAt)
+ domainPipeline.DurationSec = uint64(durationTime.Seconds())
}
- durationTime := finishedAt.Sub(createdAt)
-
- domainPipeline.DurationSec = uint64(durationTime.Seconds())
return []interface{}{
domainPipeline,
diff --git a/plugins/gitlab/tasks/pipeline_extractor.go b/plugins/gitlab/tasks/pipeline_extractor.go
index 9f4529b..325183c 100644
--- a/plugins/gitlab/tasks/pipeline_extractor.go
+++ b/plugins/gitlab/tasks/pipeline_extractor.go
@@ -74,8 +74,9 @@
return nil, err
}
- duration := int(gitlabApiPipeline.UpdatedAt.ToTime().Sub(gitlabApiPipeline.CreatedAt.ToTime()).Seconds())
- gitlabApiPipeline.Duration = duration
+ if gitlabApiPipeline.UpdatedAt != nil && gitlabApiPipeline.CreatedAt != nil {
+ gitlabApiPipeline.Duration = int(gitlabApiPipeline.UpdatedAt.ToTime().Sub(gitlabApiPipeline.CreatedAt.ToTime()).Seconds())
+ }
gitlabPipeline := &models.GitlabPipeline{
GitlabId: gitlabApiPipeline.Id,
ProjectId: data.Options.ProjectId,
diff --git a/plugins/tapd/tasks/bug_commit_collector.go b/plugins/tapd/tasks/bug_commit_collector.go
index 1c0ac01..5129db2 100644
--- a/plugins/tapd/tasks/bug_commit_collector.go
+++ b/plugins/tapd/tasks/bug_commit_collector.go
@@ -47,7 +47,6 @@
db := taskCtx.GetDal()
logger := taskCtx.GetLogger()
logger.Info("collect issueCommits")
- num := 0
since := data.Since
incremental := false
if since == nil {
@@ -103,11 +102,6 @@
var data struct {
Stories []json.RawMessage `json:"data"`
}
- if len(data.Stories) > 0 {
- fmt.Println(len(data.Stories))
- num += len(data.Stories)
- fmt.Printf("num is %d", num)
- }
err := helper.UnmarshalResponse(res, &data)
return data.Stories, err
},
diff --git a/plugins/tapd/tasks/story_bug_collector.go b/plugins/tapd/tasks/story_bug_collector.go
index 764299b..a702f41 100644
--- a/plugins/tapd/tasks/story_bug_collector.go
+++ b/plugins/tapd/tasks/story_bug_collector.go
@@ -20,13 +20,12 @@
import (
"fmt"
"github.com/apache/incubator-devlake/errors"
- "net/url"
- "reflect"
-
"github.com/apache/incubator-devlake/plugins/core"
"github.com/apache/incubator-devlake/plugins/core/dal"
"github.com/apache/incubator-devlake/plugins/helper"
"github.com/apache/incubator-devlake/plugins/tapd/models"
+ "net/url"
+ "reflect"
)
const RAW_STORY_BUG_TABLE = "tapd_api_story_bugs"
@@ -40,6 +39,7 @@
logger.Info("collect storyBugs")
clauses := []dal.Clause{
+ dal.Select("id"),
dal.From(&models.TapdStory{}),
dal.Where("connection_id = ? and workspace_id = ?", data.Options.ConnectionId, data.Options.WorkspaceId),
}
@@ -76,7 +76,7 @@
var CollectStoryBugMeta = core.SubTaskMeta{
Name: "collectStoryBugs",
EntryPoint: CollectStoryBugs,
- EnabledByDefault: true,
+ EnabledByDefault: false,
Description: "collect Tapd storyBugs",
DomainTypes: []string{core.DOMAIN_TYPE_TICKET},
}
diff --git a/plugins/tapd/tasks/story_bug_extractor.go b/plugins/tapd/tasks/story_bug_extractor.go
index 55e5f94..3da00ce 100644
--- a/plugins/tapd/tasks/story_bug_extractor.go
+++ b/plugins/tapd/tasks/story_bug_extractor.go
@@ -30,7 +30,7 @@
var ExtractStoryBugsMeta = core.SubTaskMeta{
Name: "extractStoryBugs",
EntryPoint: ExtractStoryBugs,
- EnabledByDefault: true,
+ EnabledByDefault: false,
Description: "Extract raw company data into tool layer table _tool_tapd_story_bugs",
DomainTypes: []string{core.DOMAIN_TYPE_TICKET},
}
diff --git a/plugins/tapd/tasks/story_commit_collector.go b/plugins/tapd/tasks/story_commit_collector.go
index e6a440b..6dd5026 100644
--- a/plugins/tapd/tasks/story_commit_collector.go
+++ b/plugins/tapd/tasks/story_commit_collector.go
@@ -47,7 +47,6 @@
db := taskCtx.GetDal()
logger := taskCtx.GetLogger()
logger.Info("collect issueCommits")
- num := 0
since := data.Since
incremental := false
if since == nil {
@@ -103,11 +102,6 @@
var data struct {
Stories []json.RawMessage `json:"data"`
}
- if len(data.Stories) > 0 {
- fmt.Println(len(data.Stories))
- num += len(data.Stories)
- fmt.Printf("num is %d", num)
- }
err := helper.UnmarshalResponse(res, &data)
return data.Stories, err
},
diff --git a/plugins/tapd/tasks/task_commit_collector.go b/plugins/tapd/tasks/task_commit_collector.go
index d171cdf..cdd6e29 100644
--- a/plugins/tapd/tasks/task_commit_collector.go
+++ b/plugins/tapd/tasks/task_commit_collector.go
@@ -47,7 +47,6 @@
db := taskCtx.GetDal()
logger := taskCtx.GetLogger()
logger.Info("collect issueCommits")
- num := 0
since := data.Since
incremental := false
if since == nil {
@@ -103,11 +102,6 @@
var data struct {
Stories []json.RawMessage `json:"data"`
}
- if len(data.Stories) > 0 {
- fmt.Println(len(data.Stories))
- num += len(data.Stories)
- fmt.Printf("num is %d", num)
- }
err := helper.UnmarshalResponse(res, &data)
return data.Stories, err
},
