[bug][customize] Import csv issues API handle issues in multiple boards (#6978) (#7142)
* fix(customize): csv import should handle issues belong to multiple boards
* fix(customize): use more practical csv file for update test
diff --git a/backend/plugins/customize/e2e/import_issues_test.go b/backend/plugins/customize/e2e/import_issues_test.go
index 849a0cf..8ed8780 100644
--- a/backend/plugins/customize/e2e/import_issues_test.go
+++ b/backend/plugins/customize/e2e/import_issues_test.go
@@ -89,10 +89,21 @@
t.Fatal(err1)
}
defer issueFile.Close()
- err = svc.ImportIssue(`{"ConnectionId":1,"Owner":"thenicetgp","Repo":"lake"}`, issueFile)
+ err = svc.ImportIssue("csv-board", issueFile)
if err != nil {
t.Fatal(err)
}
+
+ issueFile2, err2 := os.Open("raw_tables/issues_input2.csv")
+ if err2 != nil {
+ t.Fatal(err2)
+ }
+ defer issueFile2.Close()
+ err = svc.ImportIssue("csv-board2", issueFile2)
+ if err != nil {
+ t.Fatal(err)
+ }
+
dataflowTester.VerifyTableWithRawData(
ticket.Issue{},
"snapshot_tables/issues_output.csv",
diff --git a/backend/plugins/customize/e2e/raw_tables/issue_repo_commits.csv b/backend/plugins/customize/e2e/raw_tables/issue_repo_commits.csv
index 79f3dcd..b8a1314 100644
--- a/backend/plugins/customize/e2e/raw_tables/issue_repo_commits.csv
+++ b/backend/plugins/customize/e2e/raw_tables/issue_repo_commits.csv
@@ -1,13 +1,25 @@
-"created_at","updated_at","_raw_data_params","_raw_data_table","_raw_data_id","_raw_data_remark","issue_id","repo_url","commit_sha","host","namespace","repo_name"
-"2023-03-30 14:06:06.831","2023-03-30 14:06:06.831","{""ConnectionId"":1,""BoardId"":8}","_raw_jira_api_remotelinks",1,"","jira:JiraIssue:1:10063","https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8748a066cbaf67b15e86f2c636f9931347e987cf","8748a066cbaf67b15e86f2c636f9931347e987cf","example.com","PROJECTNAME","ui_jira"
-"2023-03-30 14:06:06.831","2023-03-30 14:06:06.831","{""ConnectionId"":1,""BoardId"":8}","_raw_jira_api_remotelinks",2,"","jira:JiraIssue:1:10064","https://bitbucket.org/mynamespace/incubator-devlake/commits/abc0892edaee00dd7ee268dbee71620407a29bca","abc0892edaee00dd7ee268dbee71620407a29bca","bitbucket.org","mynamespace","incubator-devlake"
-"2023-03-30 14:06:06.831","2023-03-30 14:06:06.831","{""ConnectionId"":1,""BoardId"":8}","_raw_jira_api_remotelinks",3,"","jira:JiraIssue:1:10064","https://github.com/apache/incubator-devlake/commit/e6bde456807818c5c78d7b265964d6d48b653af6","e6bde456807818c5c78d7b265964d6d48b653af6","github.com","apache","incubator-devlake"
-"2023-03-30 14:06:06.831","2023-03-30 14:06:06.831","{""ConnectionId"":1,""BoardId"":8}","_raw_jira_api_remotelinks",4,"","jira:JiraIssue:1:10065","https://gitlab.com/namespace1/namespace2/myrepo/-/commit/8f91020bcf684c6ad07adfafa3d8a2f826686c42","8f91020bcf684c6ad07adfafa3d8a2f826686c42","gitlab.com","namespace1/namespace2","murepo"
-"2023-03-30 14:06:06.831","2023-03-30 14:06:06.831","{""ConnectionId"":1,""BoardId"":8}","_raw_jira_api_remotelinks",5,"","jira:JiraIssue:1:10066","https://gitlab.com/meri.co/vdev.co/-/commit/0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef","0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef","gitlab.com","meri.co","vdev.co"
-"2023-03-30 14:06:06.831","2023-03-30 14:06:06.831","{""ConnectionId"":1,""BoardId"":8}","_raw_jira_api_remotelinks",13,"","jira:JiraIssue:1:10139","https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8993c04249e9d549e8950daec86717548c53c423","8993c04249e9d549e8950daec86717548c53c423","example.com","PROJECTNAME","ui_jira"
-"2023-03-30 14:06:06.831","2023-03-30 14:06:06.831","{""ConnectionId"":1,""BoardId"":8}","_raw_jira_api_remotelinks",19,"","jira:JiraIssue:1:10145","https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/07aa2ebed68e286dc51a7e0082031196a6135f74","07aa2ebed68e286dc51a7e0082031196a6135f74","example.com","PROJECTNAME","ui_jira"
-"2023-03-30 14:06:06.831","2023-03-30 14:06:06.831","{""ConnectionId"":1,""BoardId"":8}","_raw_jira_api_remotelinks",21,"","jira:JiraIssue:1:10145","https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8993c04249e9d549e8950daec86717548c53c423","ef5ab26111744f65f5191b247767a473c70d6c95","example.com","PROJECTNAME","ui_jira"
-"2023-03-30 14:06:06.831","2023-03-30 14:06:06.831","{""ConnectionId"":1,""BoardId"":8}","_raw_jira_api_remotelinks",20,"","jira:JiraIssue:1:10145","https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d70d6687e06304d9b6e0cb32b3f8c0f0928400f7","d70d6687e06304d9b6e0cb32b3f8c0f0928400f7","example.com","PROJECTNAME","ui_jira"
-"2023-03-30 14:06:06.831","2023-03-30 14:06:06.831","{""ConnectionId"":1,""BoardId"":8}","_raw_jira_api_remotelinks",36,"","jira:JiraIssue:1:10159","https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d28785ff09229ac9e3c6734f0c97466ab00eb4da","d28785ff09229ac9e3c6734f0c97466ab00eb4da","example.com","PROJECTNAME","ui_jira"
-"2023-03-30 14:06:06.831","2023-03-30 14:06:06.831","{""ConnectionId"":1,""BoardId"":8}","_raw_jira_api_remotelinks",38,"","jira:JiraIssue:1:10202","https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/0ab12c4d4064003602edceed900d1456b6209894","0ab12c4d4064003602edceed900d1456b6209894","example.com","PROJECTNAME","ui_jira"
-"2023-03-30 14:06:06.831","2023-03-30 14:06:06.831","{""ConnectionId"":1,""BoardId"":8}","_raw_jira_api_remotelinks",39,"","jira:JiraIssue:1:10203","https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/980e9fe7bc3e22a0409f7241a024eaf9c53680dd","980e9fe7bc3e22a0409f7241a024eaf9c53680dd","example.com","PROJECTNAME","ui_jira"
+issue_id,repo_url,commit_sha,host,namespace,repo_name
+csv:1,https://example.com:8080/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8748a066cbaf67b15e86f2c636f9931347e987cf,8748a066cbaf67b15e86f2c636f9931347e987cf,example.com,PROJECTNAME,ui_jira
+csv:10,https://bitbucket.org/mynamespace/incubator-devlake/commits/abc0892edaee00dd7ee268dbee71620407a29bca,abc0892edaee00dd7ee268dbee71620407a29bca,bitbucket.org,mynamespace,incubator-devlake
+csv:11,https://github.com/apache/incubator-devlake/commit/e6bde456807818c5c78d7b265964d6d48b653af6,e6bde456807818c5c78d7b265964d6d48b653af6,github.com,apache,incubator-devlake
+csv:12,https://gitlab.com/namespace1/namespace2/myrepo/-/commit/8f91020bcf684c6ad07adfafa3d8a2f826686c42,8f91020bcf684c6ad07adfafa3d8a2f826686c42,gitlab.com,namespace1/namespace2,murepo
+csv:13,https://gitlab.com/meri.co/vdev.co/-/commit/0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,gitlab.com,meri.co,vdev.co
+csv:13,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8993c04249e9d549e8950daec86717548c53c423,8993c04249e9d549e8950daec86717548c53c423,example.com,PROJECTNAME,ui_jira
+csv:13,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/07aa2ebed68e286dc51a7e0082031196a6135f74,07aa2ebed68e286dc51a7e0082031196a6135f74,example.com,PROJECTNAME,ui_jira
+csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8993c04249e9d549e8950daec86717548c53c423,ef5ab26111744f65f5191b247767a473c70d6c95,example.com,PROJECTNAME,ui_jira
+csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,example.com,PROJECTNAME,ui_jira
+csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d28785ff09229ac9e3c6734f0c97466ab00eb4da,d28785ff09229ac9e3c6734f0c97466ab00eb4da,example.com,PROJECTNAME,ui_jira
+csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/0ab12c4d4064003602edceed900d1456b6209894,0ab12c4d4064003602edceed900d1456b6209894,example.com,PROJECTNAME,ui_jira
+csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/980e9fe7bc3e22a0409f7241a024eaf9c53680dd,980e9fe7bc3e22a0409f7241a024eaf9c53680dd,example.com,PROJECTNAME,ui_jira
+jira:JiraIssue:1:10063,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8748a066cbaf67b15e86f2c636f9931347e987cf,8748a066cbaf67b15e86f2c636f9931347e987cf,example.com,PROJECTNAME,ui_jira
+jira:JiraIssue:1:10064,https://bitbucket.org/mynamespace/incubator-devlake/commits/abc0892edaee00dd7ee268dbee71620407a29bca,abc0892edaee00dd7ee268dbee71620407a29bca,bitbucket.org,mynamespace,incubator-devlake
+jira:JiraIssue:1:10064,https://github.com/apache/incubator-devlake/commit/e6bde456807818c5c78d7b265964d6d48b653af6,e6bde456807818c5c78d7b265964d6d48b653af6,github.com,apache,incubator-devlake
+jira:JiraIssue:1:10065,https://gitlab.com/namespace1/namespace2/myrepo/-/commit/8f91020bcf684c6ad07adfafa3d8a2f826686c42,8f91020bcf684c6ad07adfafa3d8a2f826686c42,gitlab.com,namespace1/namespace2,murepo
+jira:JiraIssue:1:10066,https://gitlab.com/meri.co/vdev.co/-/commit/0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,gitlab.com,meri.co,vdev.co
+jira:JiraIssue:1:10139,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8993c04249e9d549e8950daec86717548c53c423,8993c04249e9d549e8950daec86717548c53c423,example.com,PROJECTNAME,ui_jira
+jira:JiraIssue:1:10145,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/07aa2ebed68e286dc51a7e0082031196a6135f74,07aa2ebed68e286dc51a7e0082031196a6135f74,example.com,PROJECTNAME,ui_jira
+jira:JiraIssue:1:10145,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8993c04249e9d549e8950daec86717548c53c423,ef5ab26111744f65f5191b247767a473c70d6c95,example.com,PROJECTNAME,ui_jira
+jira:JiraIssue:1:10145,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,example.com,PROJECTNAME,ui_jira
+jira:JiraIssue:1:10159,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d28785ff09229ac9e3c6734f0c97466ab00eb4da,d28785ff09229ac9e3c6734f0c97466ab00eb4da,example.com,PROJECTNAME,ui_jira
+jira:JiraIssue:1:10202,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/0ab12c4d4064003602edceed900d1456b6209894,0ab12c4d4064003602edceed900d1456b6209894,example.com,PROJECTNAME,ui_jira
+jira:JiraIssue:1:10203,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/980e9fe7bc3e22a0409f7241a024eaf9c53680dd,980e9fe7bc3e22a0409f7241a024eaf9c53680dd,example.com,PROJECTNAME,ui_jira
diff --git a/backend/plugins/customize/e2e/raw_tables/issues_commits.csv b/backend/plugins/customize/e2e/raw_tables/issues_commits.csv
index 11586e3..3d91488 100644
--- a/backend/plugins/customize/e2e/raw_tables/issues_commits.csv
+++ b/backend/plugins/customize/e2e/raw_tables/issues_commits.csv
@@ -1,13 +1,25 @@
-"created_at","updated_at","_raw_data_params","_raw_data_table","_raw_data_id","_raw_data_remark","issue_id","commit_sha"
-"2022-10-25 08:38:12.588","2022-10-25 08:38:12.588","{""ConnectionId"":1,""BoardId"":8}","_raw_jira_api_remotelinks",1,"","jira:JiraIssue:1:10063","8748a066cbaf67b15e86f2c636f9931347e987cf"
-"2022-10-25 08:38:12.588","2022-10-25 08:38:12.588","{""ConnectionId"":1,""BoardId"":9}","_raw_jira_api_remotelinks",2,"","jira:JiraIssue:1:10064","abc0892edaee00dd7ee268dbee71620407a29bca"
-"2022-10-25 08:38:12.588","2022-10-25 08:38:12.588","{""ConnectionId"":1,""BoardId"":8}","_raw_jira_api_remotelinks",3,"","jira:JiraIssue:1:10064","e6bde456807818c5c78d7b265964d6d48b653af6"
-"2022-10-25 08:38:12.588","2022-10-25 08:38:12.588","{""ConnectionId"":1,""BoardId"":8}","_raw_jira_api_remotelinks",4,"","jira:JiraIssue:1:10065","8f91020bcf684c6ad07adfafa3d8a2f826686c42"
-"2022-10-25 08:38:12.588","2022-10-25 08:38:12.588","{""ConnectionId"":1,""BoardId"":8}","_raw_jira_api_remotelinks",5,"","jira:JiraIssue:1:10066","0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef"
-"2022-10-25 08:38:12.588","2022-10-25 08:38:12.588","{""ConnectionId"":2,""BoardId"":8}","_raw_jira_api_remotelinks",13,"","jira:JiraIssue:1:10139","8993c04249e9d549e8950daec86717548c53c423"
-"2022-10-25 08:38:12.588","2022-10-25 08:38:12.588","{""ConnectionId"":1,""BoardId"":8}","_raw_jira_api_remotelinks",19,"","jira:JiraIssue:1:10145","07aa2ebed68e286dc51a7e0082031196a6135f74"
-"2022-10-25 08:38:12.588","2022-10-25 08:38:12.588","{""ConnectionId"":1,""BoardId"":8}","_raw_jira_api_remotelinks",20,"","jira:JiraIssue:1:10145","d70d6687e06304d9b6e0cb32b3f8c0f0928400f7"
-"2022-10-25 08:38:12.588","2022-10-25 08:38:12.588","{""ConnectionId"":11,""BoardId"":18}","_raw_jira_api_remotelinks",21,"","jira:JiraIssue:1:10145","ef5ab26111744f65f5191b247767a473c70d6c95"
-"2022-10-25 08:38:12.588","2022-10-25 08:38:12.588","{""ConnectionId"":1,""BoardId"":8}","_raw_jira_api_remotelinks",36,"","jira:JiraIssue:1:10159","d28785ff09229ac9e3c6734f0c97466ab00eb4da"
-"2022-10-25 08:38:12.588","2022-10-25 08:38:12.588","{""ConnectionId"":1,""BoardId"":8}","_raw_jira_api_remotelinks",38,"","jira:JiraIssue:1:10202","0ab12c4d4064003602edceed900d1456b6209894"
-"2022-10-25 08:38:12.588","2022-10-25 08:38:12.588","{""ConnectionId"":1,""BoardId"":8}","_raw_jira_api_remotelinks",39,"","jira:JiraIssue:1:10203","980e9fe7bc3e22a0409f7241a024eaf9c53680dd"
+issue_id,commit_sha
+csv:1,8748a066cbaf67b15e86f2c636f9931347e987cf
+csv:10,abc0892edaee00dd7ee268dbee71620407a29bca
+csv:11,e6bde456807818c5c78d7b265964d6d48b653af6
+csv:12,8f91020bcf684c6ad07adfafa3d8a2f826686c42
+csv:13,0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef
+csv:13,8993c04249e9d549e8950daec86717548c53c423
+csv:13,07aa2ebed68e286dc51a7e0082031196a6135f74
+csv:14,ef5ab26111744f65f5191b247767a473c70d6c95
+csv:14,d70d6687e06304d9b6e0cb32b3f8c0f0928400f7
+csv:14,d28785ff09229ac9e3c6734f0c97466ab00eb4da
+csv:14,0ab12c4d4064003602edceed900d1456b6209894
+csv:14,980e9fe7bc3e22a0409f7241a024eaf9c53680dd
+jira:JiraIssue:1:10063,8748a066cbaf67b15e86f2c636f9931347e987cf
+jira:JiraIssue:1:10064,abc0892edaee00dd7ee268dbee71620407a29bca
+jira:JiraIssue:1:10064,e6bde456807818c5c78d7b265964d6d48b653af6
+jira:JiraIssue:1:10065,8f91020bcf684c6ad07adfafa3d8a2f826686c42
+jira:JiraIssue:1:10066,0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef
+jira:JiraIssue:1:10139,8993c04249e9d549e8950daec86717548c53c423
+jira:JiraIssue:1:10145,07aa2ebed68e286dc51a7e0082031196a6135f74
+jira:JiraIssue:1:10145,d70d6687e06304d9b6e0cb32b3f8c0f0928400f7
+jira:JiraIssue:1:10145,ef5ab26111744f65f5191b247767a473c70d6c95
+jira:JiraIssue:1:10159,d28785ff09229ac9e3c6734f0c97466ab00eb4da
+jira:JiraIssue:1:10202,0ab12c4d4064003602edceed900d1456b6209894
+jira:JiraIssue:1:10203,980e9fe7bc3e22a0409f7241a024eaf9c53680dd
diff --git a/backend/plugins/customize/e2e/raw_tables/issues_input.csv b/backend/plugins/customize/e2e/raw_tables/issues_input.csv
index c55b064..ac05acd 100644
--- a/backend/plugins/customize/e2e/raw_tables/issues_input.csv
+++ b/backend/plugins/customize/e2e/raw_tables/issues_input.csv
@@ -1,5 +1,7 @@
-"id","created_at","updated_at","_raw_data_params","_raw_data_table","_raw_data_id","_raw_data_remark","url","icon_url","issue_key","title","description","epic_key","type","status","original_status","story_point","resolution_date","created_date","updated_date","parent_issue_id","priority","original_estimate_minutes","time_spent_minutes","time_remaining_minutes","creator_id","creator_name","assignee_id","assignee_name","severity","component","lead_time_minutes","original_project","original_type","x_int","x_time","x_varchar","x_float","labels"
-"bitbucket:BitbucketIssue:1:1","2022-09-15 15:27:56.395","2022-09-15 15:27:56.395","{""ConnectionId"":1,""Owner"":""thenicetgp"",""Repo"":""lake""}","_raw_bitbucket_api_issues",60,"","https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/1","",1,"issue test","bitbucket issues test for devlake","","issue","TODO","new",0,NULL,"2022-07-17 07:15:55.959+00:00","2022-07-17 09:11:42.656+00:00","","major",0,0,0,"bitbucket:BitbucketAccount:1:62abf394192edb006fa0e8cf","tgp","bitbucket:BitbucketAccount:1:62abf394192edb006fa0e8cf","tgp","","",NULL,NULL,NULL,10,"2022-09-15 15:27:56","world",8,NULL
-"bitbucket:BitbucketIssue:1:10","2022-09-15 15:27:56.395","2022-09-15 15:27:56.395","{""ConnectionId"":1,""Owner"":""thenicetgp"",""Repo"":""lake""}","_raw_bitbucket_api_issues",52,"","https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/10","",10,"issue test007","issue test007","","issue","TODO","new",0,NULL,"2022-08-12 13:43:00.783+00:00","2022-08-12 13:43:00.783+00:00","","trivial",0,0,0,"bitbucket:BitbucketAccount:1:62abf394192edb006fa0e8cf","tgp","bitbucket:BitbucketAccount:1:62abf394192edb006fa0e8cf","tgp","","",NULL,NULL,NULL,30,"2022-09-15 15:27:56","abc",2456790,"hello worlds"
-"bitbucket:BitbucketIssue:1:13","2022-09-15 15:27:56.395","2022-09-15 15:27:56.395","{""ConnectionId"":1,""Owner"":""thenicetgp"",""Repo"":""lake""}","_raw_bitbucket_api_issues",50,"","https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/13","",13,"issue test010","issue test010","","issue","TODO","new",0,NULL,"2022-08-12 13:44:46.508+00:00","2022-08-12 13:44:46.508+00:00","","critical",0,0,0,"bitbucket:BitbucketAccount:1:62abf394192edb006fa0e8cf","tgp","","","","",NULL,NULL,NULL,1,"2022-09-15 15:27:56",NULL,0.00014,NULL
-"bitbucket:BitbucketIssue:1:14","2022-09-15 15:27:56.395","2022-09-15 15:27:56.395","{""ConnectionId"":1,""Owner"":""thenicetgp"",""Repo"":""lake""}","_raw_bitbucket_api_issues",49,"","https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/14","",14,"issue test011","issue test011","","issue","TODO","new",0,NULL,"2022-08-12 13:45:12.810+00:00","2022-08-12 13:45:12.810+00:00","","blocker",0,0,0,"bitbucket:BitbucketAccount:1:62abf394192edb006fa0e8cf","tgp","bitbucket:BitbucketAccount:1:62abf394192edb006fa0e8cf","tgp","","",NULL,NULL,NULL,41534568464351,"2022-09-15 15:27:56",NULL,NULL,"label1,label2,label3"
+id,url,issue_key,title,original_type,original_status,created_date,resolution_date,story_point,priority,severity,original_estimate_minutes,time_spent_minutes,component,epic_key,creator_name,assignee_name,x_int,x_time,x_varchar,x_float,labels
+csv:1,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/1,1,issue test,BUG,new,2022-07-17 07:15:55.959+00:00,NULL,0,major,,0,0,,,tgp,tgp,10,2022-09-15 15:27:56,world,8,NULL
+csv:10,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/10,10,issue test007,BUG,new,2022-08-12 13:43:00.783+00:00,NULL,0,trivial,,0,0,,,tgp,tgp,30,2022-09-15 15:27:56,abc,24590,hello worlds
+csv:11,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/11,11,issue test011,REQUIREMENT,new,2022-08-10 13:44:46.508+00:00,NULL,0,major,,0,0,,,tgp,,1,2022-09-15 15:27:56,NULL,0.00014,NULL
+csv:12,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/12,12,issue test012,REQUIREMENT,new,2022-08-11 13:44:46.508+00:00,NULL,0,major,,0,0,,,tgp,,1,2022-09-15 15:27:56,NULL,0.00014,NULL
+csv:13,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/13,13,issue test013,REQUIREMENT,new,2022-08-12 13:44:46.508+00:00,NULL,0,critical,,0,0,,,tgp,,1,2022-09-15 15:27:56,NULL,0.00014,NULL
+csv:14,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/14,14,issue test014,INCIDENT,new,2022-08-12 13:45:12.810+00:00,NULL,0,blocker,,0,0,,,tgp,tgp,41534568464351,2022-09-15 15:27:56,NULL,NULL,"label1,label2,label3"
diff --git a/backend/plugins/customize/e2e/raw_tables/issues_input2.csv b/backend/plugins/customize/e2e/raw_tables/issues_input2.csv
new file mode 100644
index 0000000..215bc65
--- /dev/null
+++ b/backend/plugins/customize/e2e/raw_tables/issues_input2.csv
@@ -0,0 +1,3 @@
+id,url,issue_key,title,original_type,original_status,created_date,resolution_date,story_point,priority,severity,original_estimate_minutes,time_spent_minutes,component,epic_key,creator_name,assignee_name,x_int,x_time,x_varchar,x_float,labels
+csv:1,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/1,1,issue test,BUG,new,2022-07-17 07:15:55.959+00:00,NULL,0,major,,0,0,,,tgp,tgp,10,2022-09-15 15:27:56,world,8,NULL
+csv:10,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/10,10,issue test007,BUG,new,2022-08-12 13:43:00.783+00:00,NULL,0,trivial,,0,0,,,tgp,tgp,30,2022-09-15 15:27:56,abc,24590,hello worlds
diff --git a/backend/plugins/customize/e2e/snapshot_tables/_tool_customized_fields.csv b/backend/plugins/customize/e2e/snapshot_tables/_tool_customized_fields.csv
deleted file mode 100644
index 6d8ab78..0000000
--- a/backend/plugins/customize/e2e/snapshot_tables/_tool_customized_fields.csv
+++ /dev/null
@@ -1,5 +0,0 @@
-tb_name,column_name,display_name,data_type
-issues,x_float,test column x_float,float
-issues,x_int,test column x_int,bigint
-issues,x_text,test column x_text,text
-issues,x_time,test column x_time,timestamp
diff --git a/backend/plugins/customize/e2e/snapshot_tables/board_issues.csv b/backend/plugins/customize/e2e/snapshot_tables/board_issues.csv
index 5c1c86d..dac0662 100644
--- a/backend/plugins/customize/e2e/snapshot_tables/board_issues.csv
+++ b/backend/plugins/customize/e2e/snapshot_tables/board_issues.csv
@@ -1,5 +1,9 @@
board_id,issue_id,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
-"{""ConnectionId"":1,""Owner"":""thenicetgp"",""Repo"":""lake""}",bitbucket:BitbucketIssue:1:1,,,0,
-"{""ConnectionId"":1,""Owner"":""thenicetgp"",""Repo"":""lake""}",bitbucket:BitbucketIssue:1:10,,,0,
-"{""ConnectionId"":1,""Owner"":""thenicetgp"",""Repo"":""lake""}",bitbucket:BitbucketIssue:1:13,,,0,
-"{""ConnectionId"":1,""Owner"":""thenicetgp"",""Repo"":""lake""}",bitbucket:BitbucketIssue:1:14,,,0,
+csv-board,csv:1,,,0,
+csv-board,csv:10,,,0,
+csv-board,csv:11,,,0,
+csv-board,csv:12,,,0,
+csv-board,csv:13,,,0,
+csv-board,csv:14,,,0,
+csv-board2,csv:1,,,0,
+csv-board2,csv:10,,,0,
diff --git a/backend/plugins/customize/e2e/snapshot_tables/issue_commits.csv b/backend/plugins/customize/e2e/snapshot_tables/issue_commits.csv
index 98b01ac..97b875a 100644
--- a/backend/plugins/customize/e2e/snapshot_tables/issue_commits.csv
+++ b/backend/plugins/customize/e2e/snapshot_tables/issue_commits.csv
@@ -1,13 +1,25 @@
issue_id,commit_sha,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
-jira:JiraIssue:1:10063,8748a066cbaf67b15e86f2c636f9931347e987cf,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_remotelinks,1,
-jira:JiraIssue:1:10064,abc0892edaee00dd7ee268dbee71620407a29bca,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_remotelinks,2,
-jira:JiraIssue:1:10064,e6bde456807818c5c78d7b265964d6d48b653af6,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_remotelinks,3,
-jira:JiraIssue:1:10065,8f91020bcf684c6ad07adfafa3d8a2f826686c42,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_remotelinks,4,
-jira:JiraIssue:1:10066,0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_remotelinks,5,
-jira:JiraIssue:1:10139,8993c04249e9d549e8950daec86717548c53c423,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_remotelinks,13,
-jira:JiraIssue:1:10145,07aa2ebed68e286dc51a7e0082031196a6135f74,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_remotelinks,19,
-jira:JiraIssue:1:10145,d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_remotelinks,20,
-jira:JiraIssue:1:10145,ef5ab26111744f65f5191b247767a473c70d6c95,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_remotelinks,21,
-jira:JiraIssue:1:10159,d28785ff09229ac9e3c6734f0c97466ab00eb4da,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_remotelinks,36,
-jira:JiraIssue:1:10202,0ab12c4d4064003602edceed900d1456b6209894,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_remotelinks,38,
-jira:JiraIssue:1:10203,980e9fe7bc3e22a0409f7241a024eaf9c53680dd,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_remotelinks,39,
+csv:1,8748a066cbaf67b15e86f2c636f9931347e987cf,"{""ConnectionId"":1,""BoardId"":8}",,,
+csv:10,abc0892edaee00dd7ee268dbee71620407a29bca,"{""ConnectionId"":1,""BoardId"":8}",,,
+csv:11,e6bde456807818c5c78d7b265964d6d48b653af6,"{""ConnectionId"":1,""BoardId"":8}",,,
+csv:12,8f91020bcf684c6ad07adfafa3d8a2f826686c42,"{""ConnectionId"":1,""BoardId"":8}",,,
+csv:13,07aa2ebed68e286dc51a7e0082031196a6135f74,"{""ConnectionId"":1,""BoardId"":8}",,,
+csv:13,0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,"{""ConnectionId"":1,""BoardId"":8}",,,
+csv:13,8993c04249e9d549e8950daec86717548c53c423,"{""ConnectionId"":1,""BoardId"":8}",,,
+csv:14,0ab12c4d4064003602edceed900d1456b6209894,"{""ConnectionId"":1,""BoardId"":8}",,,
+csv:14,980e9fe7bc3e22a0409f7241a024eaf9c53680dd,"{""ConnectionId"":1,""BoardId"":8}",,,
+csv:14,d28785ff09229ac9e3c6734f0c97466ab00eb4da,"{""ConnectionId"":1,""BoardId"":8}",,,
+csv:14,d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,"{""ConnectionId"":1,""BoardId"":8}",,,
+csv:14,ef5ab26111744f65f5191b247767a473c70d6c95,"{""ConnectionId"":1,""BoardId"":8}",,,
+jira:JiraIssue:1:10063,8748a066cbaf67b15e86f2c636f9931347e987cf,"{""ConnectionId"":1,""BoardId"":8}",,,
+jira:JiraIssue:1:10064,abc0892edaee00dd7ee268dbee71620407a29bca,"{""ConnectionId"":1,""BoardId"":8}",,,
+jira:JiraIssue:1:10064,e6bde456807818c5c78d7b265964d6d48b653af6,"{""ConnectionId"":1,""BoardId"":8}",,,
+jira:JiraIssue:1:10065,8f91020bcf684c6ad07adfafa3d8a2f826686c42,"{""ConnectionId"":1,""BoardId"":8}",,,
+jira:JiraIssue:1:10066,0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,"{""ConnectionId"":1,""BoardId"":8}",,,
+jira:JiraIssue:1:10139,8993c04249e9d549e8950daec86717548c53c423,"{""ConnectionId"":1,""BoardId"":8}",,,
+jira:JiraIssue:1:10145,07aa2ebed68e286dc51a7e0082031196a6135f74,"{""ConnectionId"":1,""BoardId"":8}",,,
+jira:JiraIssue:1:10145,d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,"{""ConnectionId"":1,""BoardId"":8}",,,
+jira:JiraIssue:1:10145,ef5ab26111744f65f5191b247767a473c70d6c95,"{""ConnectionId"":1,""BoardId"":8}",,,
+jira:JiraIssue:1:10159,d28785ff09229ac9e3c6734f0c97466ab00eb4da,"{""ConnectionId"":1,""BoardId"":8}",,,
+jira:JiraIssue:1:10202,0ab12c4d4064003602edceed900d1456b6209894,"{""ConnectionId"":1,""BoardId"":8}",,,
+jira:JiraIssue:1:10203,980e9fe7bc3e22a0409f7241a024eaf9c53680dd,"{""ConnectionId"":1,""BoardId"":8}",,,
diff --git a/backend/plugins/customize/e2e/snapshot_tables/issue_labels.csv b/backend/plugins/customize/e2e/snapshot_tables/issue_labels.csv
index 7b7191d..d7523c0 100644
--- a/backend/plugins/customize/e2e/snapshot_tables/issue_labels.csv
+++ b/backend/plugins/customize/e2e/snapshot_tables/issue_labels.csv
@@ -1,5 +1,5 @@
issue_id,label_name,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
-bitbucket:BitbucketIssue:1:10,hello worlds,"{""ConnectionId"":1,""Owner"":""thenicetgp"",""Repo"":""lake""}",,0,
-bitbucket:BitbucketIssue:1:14,label1,"{""ConnectionId"":1,""Owner"":""thenicetgp"",""Repo"":""lake""}",,0,
-bitbucket:BitbucketIssue:1:14,label2,"{""ConnectionId"":1,""Owner"":""thenicetgp"",""Repo"":""lake""}",,0,
-bitbucket:BitbucketIssue:1:14,label3,"{""ConnectionId"":1,""Owner"":""thenicetgp"",""Repo"":""lake""}",,0,
+csv:10,hello worlds,csv-board2,,0,
+csv:14,label1,csv-board,,0,
+csv:14,label2,csv-board,,0,
+csv:14,label3,csv-board,,0,
diff --git a/backend/plugins/customize/e2e/snapshot_tables/issue_repo_commits.csv b/backend/plugins/customize/e2e/snapshot_tables/issue_repo_commits.csv
index e871954..311a936 100644
--- a/backend/plugins/customize/e2e/snapshot_tables/issue_repo_commits.csv
+++ b/backend/plugins/customize/e2e/snapshot_tables/issue_repo_commits.csv
@@ -1,13 +1,25 @@
issue_id,repo_url,commit_sha,host,namespace,repo_name,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
-jira:JiraIssue:1:10063,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8748a066cbaf67b15e86f2c636f9931347e987cf,8748a066cbaf67b15e86f2c636f9931347e987cf,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_remotelinks,1,
-jira:JiraIssue:1:10064,https://bitbucket.org/mynamespace/incubator-devlake/commits/abc0892edaee00dd7ee268dbee71620407a29bca,abc0892edaee00dd7ee268dbee71620407a29bca,bitbucket.org,mynamespace,incubator-devlake,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_remotelinks,2,
-jira:JiraIssue:1:10064,https://github.com/apache/incubator-devlake/commit/e6bde456807818c5c78d7b265964d6d48b653af6,e6bde456807818c5c78d7b265964d6d48b653af6,github.com,apache,incubator-devlake,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_remotelinks,3,
-jira:JiraIssue:1:10065,https://gitlab.com/namespace1/namespace2/myrepo/-/commit/8f91020bcf684c6ad07adfafa3d8a2f826686c42,8f91020bcf684c6ad07adfafa3d8a2f826686c42,gitlab.com,namespace1/namespace2,murepo,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_remotelinks,4,
-jira:JiraIssue:1:10066,https://gitlab.com/meri.co/vdev.co/-/commit/0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,gitlab.com,meri.co,vdev.co,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_remotelinks,5,
-jira:JiraIssue:1:10139,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8993c04249e9d549e8950daec86717548c53c423,8993c04249e9d549e8950daec86717548c53c423,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_remotelinks,13,
-jira:JiraIssue:1:10145,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/07aa2ebed68e286dc51a7e0082031196a6135f74,07aa2ebed68e286dc51a7e0082031196a6135f74,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_remotelinks,19,
-jira:JiraIssue:1:10145,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8993c04249e9d549e8950daec86717548c53c423,ef5ab26111744f65f5191b247767a473c70d6c95,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_remotelinks,21,
-jira:JiraIssue:1:10145,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_remotelinks,20,
-jira:JiraIssue:1:10159,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d28785ff09229ac9e3c6734f0c97466ab00eb4da,d28785ff09229ac9e3c6734f0c97466ab00eb4da,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_remotelinks,36,
-jira:JiraIssue:1:10202,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/0ab12c4d4064003602edceed900d1456b6209894,0ab12c4d4064003602edceed900d1456b6209894,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_remotelinks,38,
-jira:JiraIssue:1:10203,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/980e9fe7bc3e22a0409f7241a024eaf9c53680dd,980e9fe7bc3e22a0409f7241a024eaf9c53680dd,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_remotelinks,39,
+csv:1,https://example.com:8080/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8748a066cbaf67b15e86f2c636f9931347e987cf,8748a066cbaf67b15e86f2c636f9931347e987cf,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",,,
+csv:10,https://bitbucket.org/mynamespace/incubator-devlake/commits/abc0892edaee00dd7ee268dbee71620407a29bca,abc0892edaee00dd7ee268dbee71620407a29bca,bitbucket.org,mynamespace,incubator-devlake,"{""ConnectionId"":1,""BoardId"":8}",,,
+csv:11,https://github.com/apache/incubator-devlake/commit/e6bde456807818c5c78d7b265964d6d48b653af6,e6bde456807818c5c78d7b265964d6d48b653af6,github.com,apache,incubator-devlake,"{""ConnectionId"":1,""BoardId"":8}",,,
+csv:12,https://gitlab.com/namespace1/namespace2/myrepo/-/commit/8f91020bcf684c6ad07adfafa3d8a2f826686c42,8f91020bcf684c6ad07adfafa3d8a2f826686c42,gitlab.com,namespace1/namespace2,murepo,"{""ConnectionId"":1,""BoardId"":8}",,,
+csv:13,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/07aa2ebed68e286dc51a7e0082031196a6135f74,07aa2ebed68e286dc51a7e0082031196a6135f74,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",,,
+csv:13,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8993c04249e9d549e8950daec86717548c53c423,8993c04249e9d549e8950daec86717548c53c423,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",,,
+csv:13,https://gitlab.com/meri.co/vdev.co/-/commit/0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,gitlab.com,meri.co,vdev.co,"{""ConnectionId"":1,""BoardId"":8}",,,
+csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/0ab12c4d4064003602edceed900d1456b6209894,0ab12c4d4064003602edceed900d1456b6209894,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",,,
+csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8993c04249e9d549e8950daec86717548c53c423,ef5ab26111744f65f5191b247767a473c70d6c95,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",,,
+csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/980e9fe7bc3e22a0409f7241a024eaf9c53680dd,980e9fe7bc3e22a0409f7241a024eaf9c53680dd,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",,,
+csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d28785ff09229ac9e3c6734f0c97466ab00eb4da,d28785ff09229ac9e3c6734f0c97466ab00eb4da,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",,,
+csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",,,
+jira:JiraIssue:1:10063,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8748a066cbaf67b15e86f2c636f9931347e987cf,8748a066cbaf67b15e86f2c636f9931347e987cf,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",,,
+jira:JiraIssue:1:10064,https://bitbucket.org/mynamespace/incubator-devlake/commits/abc0892edaee00dd7ee268dbee71620407a29bca,abc0892edaee00dd7ee268dbee71620407a29bca,bitbucket.org,mynamespace,incubator-devlake,"{""ConnectionId"":1,""BoardId"":8}",,,
+jira:JiraIssue:1:10064,https://github.com/apache/incubator-devlake/commit/e6bde456807818c5c78d7b265964d6d48b653af6,e6bde456807818c5c78d7b265964d6d48b653af6,github.com,apache,incubator-devlake,"{""ConnectionId"":1,""BoardId"":8}",,,
+jira:JiraIssue:1:10065,https://gitlab.com/namespace1/namespace2/myrepo/-/commit/8f91020bcf684c6ad07adfafa3d8a2f826686c42,8f91020bcf684c6ad07adfafa3d8a2f826686c42,gitlab.com,namespace1/namespace2,murepo,"{""ConnectionId"":1,""BoardId"":8}",,,
+jira:JiraIssue:1:10066,https://gitlab.com/meri.co/vdev.co/-/commit/0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,gitlab.com,meri.co,vdev.co,"{""ConnectionId"":1,""BoardId"":8}",,,
+jira:JiraIssue:1:10139,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8993c04249e9d549e8950daec86717548c53c423,8993c04249e9d549e8950daec86717548c53c423,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",,,
+jira:JiraIssue:1:10145,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/07aa2ebed68e286dc51a7e0082031196a6135f74,07aa2ebed68e286dc51a7e0082031196a6135f74,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",,,
+jira:JiraIssue:1:10145,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8993c04249e9d549e8950daec86717548c53c423,ef5ab26111744f65f5191b247767a473c70d6c95,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",,,
+jira:JiraIssue:1:10145,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",,,
+jira:JiraIssue:1:10159,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d28785ff09229ac9e3c6734f0c97466ab00eb4da,d28785ff09229ac9e3c6734f0c97466ab00eb4da,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",,,
+jira:JiraIssue:1:10202,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/0ab12c4d4064003602edceed900d1456b6209894,0ab12c4d4064003602edceed900d1456b6209894,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",,,
+jira:JiraIssue:1:10203,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/980e9fe7bc3e22a0409f7241a024eaf9c53680dd,980e9fe7bc3e22a0409f7241a024eaf9c53680dd,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}",,,
diff --git a/backend/plugins/customize/e2e/snapshot_tables/issues.csv b/backend/plugins/customize/e2e/snapshot_tables/issues.csv
deleted file mode 100644
index 954eb21..0000000
--- a/backend/plugins/customize/e2e/snapshot_tables/issues.csv
+++ /dev/null
@@ -1,31 +0,0 @@
-id,x_test,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
-jira:JiraIssue:1:10063,2020-06-12T08:13:13.360+0800,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1701,
-jira:JiraIssue:1:10064,2020-06-12T08:15:36.123+0800,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1702,
-jira:JiraIssue:1:10065,2020-06-12T08:15:41.600+0800,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1703,
-jira:JiraIssue:1:10066,2020-06-12T08:15:46.144+0800,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1704,
-jira:JiraIssue:1:10067,2020-06-12T08:16:44.157+0800,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1705,
-jira:JiraIssue:1:10068,2020-06-12T08:17:26.986+0800,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1706,
-jira:JiraIssue:1:10070,2020-06-12T08:18:58.050+0800,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1707,
-jira:JiraIssue:1:10071,2020-06-12T08:19:17.336+0800,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1708,
-jira:JiraIssue:1:10072,2020-06-12T08:19:24.637+0800,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1709,
-jira:JiraIssue:1:10076,2020-06-12T08:24:25.922+0800,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1710,
-jira:JiraIssue:1:10077,2020-06-12T08:24:39.624+0800,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1711,
-jira:JiraIssue:1:10078,2020-06-12T08:24:49.017+0800,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1712,
-jira:JiraIssue:1:10079,2020-06-12T08:24:56.048+0800,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1713,
-jira:JiraIssue:1:10081,2020-06-12T08:28:00.241+0800,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1714,
-jira:JiraIssue:1:10082,2020-06-12T08:29:43.677+0800,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1715,
-jira:JiraIssue:1:10085,2020-06-12T08:33:57.204+0800,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1716,
-jira:JiraIssue:1:10086,2020-06-12T08:35:15.489+0800,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1717,
-jira:JiraIssue:1:10087,2020-06-12T08:40:54.210+0800,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1718,
-jira:JiraIssue:1:10088,2020-06-12T08:41:01.683+0800,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1719,
-jira:JiraIssue:1:10089,2020-06-12T08:41:16.622+0800,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1720,
-jira:JiraIssue:1:10090,2020-06-12T08:41:36.317+0800,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1721,
-jira:JiraIssue:1:10091,2020-06-12T08:48:29.035+0800,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1722,
-jira:JiraIssue:1:10092,2020-06-12T08:48:39.803+0800,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1723,
-jira:JiraIssue:1:10093,2020-06-12T08:48:46.751+0800,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1724,
-jira:JiraIssue:1:10094,2020-06-12T08:48:53.279+0800,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1725,
-jira:JiraIssue:1:10095,2020-06-12T08:50:45.674+0800,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1726,
-jira:JiraIssue:1:10096,2020-06-12T08:51:57.807+0800,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1727,
-jira:JiraIssue:1:10097,2020-06-12T08:52:04.767+0800,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1728,
-jira:JiraIssue:1:10098,2020-06-12T08:52:12.678+0800,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1729,
-jira:JiraIssue:1:10099,2020-06-12T08:52:23.456+0800,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1730,
diff --git a/backend/plugins/customize/e2e/snapshot_tables/issues_csv.csv b/backend/plugins/customize/e2e/snapshot_tables/issues_csv.csv
deleted file mode 100644
index 4f930ee..0000000
--- a/backend/plugins/customize/e2e/snapshot_tables/issues_csv.csv
+++ /dev/null
@@ -1,4 +0,0 @@
-"id","created_at","updated_at","_raw_data_params","_raw_data_table","_raw_data_id","_raw_data_remark","url","icon_url","issue_key","title","description","epic_key","type","status","original_status","story_point","resolution_date","created_date","updated_date","parent_issue_id","priority","original_estimate_minutes","time_spent_minutes","time_remaining_minutes","creator_id","creator_name","assignee_id","assignee_name","severity","component","lead_time_minutes","original_project","original_type","x_int","x_time","x_varchar","x_float"
-"abc","2023-02-07 14:55:19.650","2023-02-07 14:55:19.650","","",0,"","","","","","","","","","",0,NULL,NULL,NULL,"","",0,0,0,"","","","","","",0,"","",20,"2022-09-15 15:27:56","hello",123.5
-"bitbucket:BitbucketIssue:1:1","2022-09-15 15:27:56.395","2022-09-15 15:27:56.395","{""ConnectionId"":1,""Owner"":""thenicetgp"",""Repo"":""lake""}","_raw_bitbucket_api_issues",60,"","https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/1","",1,"issue test","bitbucket issues test for devlake","","issue","TODO","new",0,NULL,"2022-07-17 07:15:55.959","2022-07-17 09:11:42.656","","major",0,0,0,"bitbucket:BitbucketAccount:1:62abf394192edb006fa0e8cf","tgp","bitbucket:BitbucketAccount:1:62abf394192edb006fa0e8cf","tgp","","",NULL,NULL,NULL,10,"2022-09-15 15:27:56","world",8
-"bitbucket:BitbucketIssue:1:10","2022-09-15 15:27:56.395","2022-09-15 15:27:56.395","{""ConnectionId"":1,""Owner"":""thenicetgp"",""Repo"":""lake""}","_raw_bitbucket_api_issues",52,"","https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/10","",10,"issue test007","issue test007","","issue","TODO","new",0,NULL,"2022-08-12 13:43:00.783","2022-08-12 13:43:00.783","","trivial",0,0,0,"bitbucket:BitbucketAccount:1:62abf394192edb006fa0e8cf","tgp","bitbucket:BitbucketAccount:1:62abf394192edb006fa0e8cf","tgp","","",NULL,NULL,NULL,30,"2022-09-15 15:27:56","abc",2456790
diff --git a/backend/plugins/customize/e2e/snapshot_tables/issues_output.csv b/backend/plugins/customize/e2e/snapshot_tables/issues_output.csv
index d64c1e8..4e3528e 100644
--- a/backend/plugins/customize/e2e/snapshot_tables/issues_output.csv
+++ b/backend/plugins/customize/e2e/snapshot_tables/issues_output.csv
@@ -1,5 +1,7 @@
id,url,icon_url,issue_key,title,description,epic_key,type,original_type,status,original_status,story_point,resolution_date,created_date,updated_date,lead_time_minutes,parent_issue_id,priority,original_estimate_minutes,time_spent_minutes,time_remaining_minutes,creator_id,creator_name,assignee_id,assignee_name,severity,component,original_project,x_varchar,x_text,x_time,x_float,x_int,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
-bitbucket:BitbucketIssue:1:1,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/1,,1,issue test,bitbucket issues test for devlake,,issue,,TODO,new,0,,2022-07-17T07:15:55.959+00:00,2022-07-17T09:11:42.656+00:00,,,major,0,0,0,bitbucket:BitbucketAccount:1:62abf394192edb006fa0e8cf,tgp,bitbucket:BitbucketAccount:1:62abf394192edb006fa0e8cf,tgp,,,,world,,2022-09-15T15:27:56.000+00:00,8,10,"{""ConnectionId"":1,""Owner"":""thenicetgp"",""Repo"":""lake""}",_raw_bitbucket_api_issues,60,
-bitbucket:BitbucketIssue:1:10,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/10,,10,issue test007,issue test007,,issue,,TODO,new,0,,2022-08-12T13:43:00.783+00:00,2022-08-12T13:43:00.783+00:00,,,trivial,0,0,0,bitbucket:BitbucketAccount:1:62abf394192edb006fa0e8cf,tgp,bitbucket:BitbucketAccount:1:62abf394192edb006fa0e8cf,tgp,,,,abc,,2022-09-15T15:27:56.000+00:00,2.45679e+06,30,"{""ConnectionId"":1,""Owner"":""thenicetgp"",""Repo"":""lake""}",_raw_bitbucket_api_issues,52,
-bitbucket:BitbucketIssue:1:13,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/13,,13,issue test010,issue test010,,issue,,TODO,new,0,,2022-08-12T13:44:46.508+00:00,2022-08-12T13:44:46.508+00:00,,,critical,0,0,0,bitbucket:BitbucketAccount:1:62abf394192edb006fa0e8cf,tgp,,,,,,,,2022-09-15T15:27:56.000+00:00,0.00014,1,"{""ConnectionId"":1,""Owner"":""thenicetgp"",""Repo"":""lake""}",_raw_bitbucket_api_issues,50,
-bitbucket:BitbucketIssue:1:14,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/14,,14,issue test011,issue test011,,issue,,TODO,new,0,,2022-08-12T13:45:12.810+00:00,2022-08-12T13:45:12.810+00:00,,,blocker,0,0,0,bitbucket:BitbucketAccount:1:62abf394192edb006fa0e8cf,tgp,bitbucket:BitbucketAccount:1:62abf394192edb006fa0e8cf,tgp,,,,,,2022-09-15T15:27:56.000+00:00,,41534568464351,"{""ConnectionId"":1,""Owner"":""thenicetgp"",""Repo"":""lake""}",_raw_bitbucket_api_issues,49,
+csv:1,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/1,,1,issue test,,,,BUG,,new,0,,2022-07-17T07:15:55.959+00:00,,,,major,0,0,,,tgp,,tgp,,,,world,,2022-09-15T15:27:56.000+00:00,8,10,csv-board2,,,
+csv:10,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/10,,10,issue test007,,,,BUG,,new,0,,2022-08-12T13:43:00.783+00:00,,,,trivial,0,0,,,tgp,,tgp,,,,abc,,2022-09-15T15:27:56.000+00:00,24590,30,csv-board2,,,
+csv:11,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/11,,11,issue test011,,,,REQUIREMENT,,new,0,,2022-08-10T13:44:46.508+00:00,,,,major,0,0,,,tgp,,,,,,,,2022-09-15T15:27:56.000+00:00,0.00014,1,csv-board,,,
+csv:12,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/12,,12,issue test012,,,,REQUIREMENT,,new,0,,2022-08-11T13:44:46.508+00:00,,,,major,0,0,,,tgp,,,,,,,,2022-09-15T15:27:56.000+00:00,0.00014,1,csv-board,,,
+csv:13,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/13,,13,issue test013,,,,REQUIREMENT,,new,0,,2022-08-12T13:44:46.508+00:00,,,,critical,0,0,,,tgp,,,,,,,,2022-09-15T15:27:56.000+00:00,0.00014,1,csv-board,,,
+csv:14,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/14,,14,issue test014,,,,INCIDENT,,new,0,,2022-08-12T13:45:12.810+00:00,,,,blocker,0,0,,,tgp,,tgp,,,,,,2022-09-15T15:27:56.000+00:00,,41534568464351,csv-board,,,
diff --git a/backend/plugins/customize/service/service.go b/backend/plugins/customize/service/service.go
index 911c939..a5187c2 100644
--- a/backend/plugins/customize/service/service.go
+++ b/backend/plugins/customize/service/service.go
@@ -85,6 +85,8 @@
}
return result, nil
}
+
+// checkField checks if the field exist in table
func (s *Service) checkField(table, field string) (bool, errors.Error) {
if table == "" {
return false, errors.Default.New("empty table name")
@@ -143,24 +145,44 @@
return s.dal.Delete(&models.CustomizedField{}, dal.Where("tb_name = ? AND column_name = ?", table, field))
}
+// getCustomizedFields returns all the customized fields definitions of the table
func (s *Service) getCustomizedFields(table string) ([]models.CustomizedField, errors.Error) {
var result []models.CustomizedField
err := s.dal.All(&result, dal.Where("tb_name = ?", table))
return result, err
}
+// ImportIssue import csv file to the table `issues`, and create relations to boards
+// issue could exist in multiple boards, so we should only delete an old records when it doesn't belong to another board
func (s *Service) ImportIssue(boardId string, file io.ReadCloser) errors.Error {
- err := s.dal.Delete(&ticket.Issue{}, dal.Where("_raw_data_params = ?", boardId))
+ err := s.dal.Delete(
+ &ticket.Issue{},
+ dal.Where("id IN (SELECT issue_id FROM board_issues WHERE board_id=? AND issue_id NOT IN (SELECT issue_id FROM board_issues WHERE board_id!=?))", boardId, boardId),
+ )
if err != nil {
return err
}
- err = s.dal.Delete(&ticket.BoardIssue{}, dal.Where("board_id = ?", boardId))
+
+ err = s.dal.Delete(
+ &ticket.IssueLabel{},
+ dal.Where("issue_id IN (SELECT issue_id FROM board_issues WHERE board_id=? AND issue_id NOT IN (SELECT issue_id FROM board_issues WHERE board_id!=?))", boardId, boardId),
+ )
if err != nil {
return err
}
+
+ err = s.dal.Delete(
+ &ticket.BoardIssue{},
+ dal.Where("board_id = ?", boardId),
+ )
+ if err != nil {
+ return err
+ }
+
return s.importCSV(file, boardId, s.issueHandlerFactory(boardId))
}
+// SaveBoard make sure the board exists in table `boards`
func (s *Service) SaveBoard(boardId, boardName string) errors.Error {
return s.dal.CreateOrUpdate(&ticket.Board{
DomainEntity: domainlayer.DomainEntity{
@@ -171,40 +193,39 @@
})
}
-func (s *Service) ImportIssueCommit(rawDataParams string, file io.ReadCloser) errors.Error {
- err := s.dal.Delete(&crossdomain.IssueCommit{}, dal.Where("_raw_data_params = ?", rawDataParams))
+// ImportIssueCommit imports csv file into the table `issue_commits`
+func (s *Service) ImportIssueCommit(boardId string, file io.ReadCloser) errors.Error {
+ err := s.dal.Delete(
+ &crossdomain.IssueCommit{},
+ dal.Where("issue_id IN (SELECT issue_id FROM board_issues WHERE board_id=? AND issue_id NOT IN (SELECT issue_id FROM board_issues WHERE board_id!=?))", boardId, boardId),
+ )
if err != nil {
return err
}
- return s.importCSV(file, rawDataParams, s.issueCommitHandler)
+ return s.importCSV(file, boardId, s.issueCommitHandler)
}
// ImportIssueRepoCommit imports data to the table `issue_repo_commits` and `issue_commits`
-func (s *Service) ImportIssueRepoCommit(rawDataParams string, file io.ReadCloser) errors.Error {
- fields := make(map[string]struct{})
- // get all fields of the table `issue_repo_commit`
- columns, err := s.dal.GetColumns(&crossdomain.IssueCommit{}, func(columnMeta dal.ColumnMeta) bool {
- return true
- })
- if err != nil {
- return err
- }
- for _, column := range columns {
- fields[column.Name()] = struct{}{}
- }
+func (s *Service) ImportIssueRepoCommit(boardId string, file io.ReadCloser) errors.Error {
// delete old records of the table `issue_repo_commit` and `issue_commit`
- err = s.dal.Delete(&crossdomain.IssueRepoCommit{}, dal.Where("_raw_data_params = ?", rawDataParams))
+ err := s.dal.Delete(
+ &crossdomain.IssueRepoCommit{},
+ dal.Where("issue_id IN (SELECT issue_id FROM board_issues WHERE board_id=? AND issue_id NOT IN (SELECT issue_id FROM board_issues WHERE board_id!=?))", boardId, boardId),
+ )
if err != nil {
return err
}
- err = s.dal.Delete(&crossdomain.IssueCommit{}, dal.Where("_raw_data_params = ?", rawDataParams))
+ err = s.dal.Delete(
+ &crossdomain.IssueCommit{},
+ dal.Where("issue_id IN (SELECT issue_id FROM board_issues WHERE board_id=? AND issue_id NOT IN (SELECT issue_id FROM board_issues WHERE board_id!=?))", boardId, boardId),
+ )
if err != nil {
return err
}
- return s.importCSV(file, rawDataParams, s.issueRepoCommitHandlerFactory(fields))
+ return s.importCSV(file, boardId, s.issueRepoCommitHandler)
}
-// importCSV imports the csv file to the database,
+// importCSV extract records from csv file, and save them to DB using recordHandler
// the rawDataParams is used to identify the data source,
// the recordHandler is used to handle the record, it should return an error if the record is invalid
// the `created_at` and `updated_at` will be set to the current time
@@ -238,6 +259,7 @@
}
}
+// issueHandlerFactory returns a handler that save record into `issues`, `board_issues` and `issue_labels` table
func (s *Service) issueHandlerFactory(boardId string) func(record map[string]interface{}) errors.Error {
return func(record map[string]interface{}) errors.Error {
var err errors.Error
@@ -294,25 +316,22 @@
}
}
+// issueCommitHandler save record into `issue_commits` table
func (s *Service) issueCommitHandler(record map[string]interface{}) errors.Error {
return s.dal.CreateWithMap(&crossdomain.IssueCommit{}, record)
}
// issueRepoCommitHandlerFactory returns a handler that will populate the `issue_commits` and `issue_repo_commits` table
// ths issueCommitsFields is used to filter the fields that should be inserted into the `issue_commits` table
-func (s *Service) issueRepoCommitHandlerFactory(issueCommitsFields map[string]struct{}) func(record map[string]interface{}) errors.Error {
- return func(record map[string]interface{}) errors.Error {
- err := s.dal.CreateWithMap(&crossdomain.IssueRepoCommit{}, record)
- if err != nil {
- return err
- }
- for head := range record {
- if _, exists := issueCommitsFields[head]; exists {
- continue
- } else {
- delete(record, head)
- }
- }
- return s.dal.CreateWithMap(&crossdomain.IssueCommit{}, record)
+func (s *Service) issueRepoCommitHandler(record map[string]interface{}) errors.Error {
+ err := s.dal.CreateWithMap(&crossdomain.IssueRepoCommit{}, record)
+ if err != nil {
+ return err
}
+ // remove fields that not in table `issue_commits`
+ delete(record, "host")
+ delete(record, "namespace")
+ delete(record, "repo_name")
+ delete(record, "repo_url")
+ return s.dal.CreateWithMap(&crossdomain.IssueCommit{}, record)
}
