fix: Fix azure devops migration from 0.17 to 0.18 (#5999)
Cherry-pick of #5957
* fix: Fix AzDO connection migration 0.17 -> 0.18
In 0.17, all connection fields are encrypted.
In 0.18 only `token` is, so we need a migration of AzDo connection to decrypt all fields but `token`.
* fix: Allow setting entities=None to ScopeConfig
Allow setting `entities` field to `None` in `ScopeConfig` and fallback to default value.
* feat: Add RenameColumn migration op
* fix: Add missing AzDO migrations from 0.17 to 0.18
Now that migrations have been made manual, we need to add all the migrations for
all that changed from 0.17 to 0.18 that were automatic before...
Those are all required migrations collected with `git diff v0.17.0 v0.18.0-beta7 -- backend/python/plugins/azuredevops/azuredevops/models.py`:
* Rename GitRepository.transformation_rule_id -> scope_config_id
* Add GitRepository.provider
* Rename Job.startTime -> start_time
* Rename Job.finishTime -> start_time
---------
Co-authored-by: Camille Teruel <camille.teruel@meri.co>
diff --git a/backend/python/plugins/azuredevops/azuredevops/models.py b/backend/python/plugins/azuredevops/azuredevops/models.py
index d77d9e6..63765e5 100644
--- a/backend/python/plugins/azuredevops/azuredevops/models.py
+++ b/backend/python/plugins/azuredevops/azuredevops/models.py
@@ -158,7 +158,11 @@
@migration(20230802000001, name="rename startTime/finishTime to start_time/finish_time")
def rename_starttime_and_finishtime_for_job(b: MigrationScriptBuilder):
- b.execute(f'ALTER TABLE _tool_azuredevops_jobs RENAME COLUMN startTime TO start_time', Dialect.MYSQL, ignore_error=True)
- b.execute(f'ALTER TABLE _tool_azuredevops_jobs RENAME COLUMN finishTime TO finish_time', Dialect.MYSQL, ignore_error=True)
- b.execute(f'ALTER TABLE _tool_azuredevops_jobs RENAME COLUMN `startTime` TO start_time', Dialect.POSTGRESQL, ignore_error=True)
- b.execute(f'ALTER TABLE _tool_azuredevops_jobs RENAME COLUMN `finishTime` TO finish_time', Dialect.POSTGRESQL, ignore_error=True)
\ No newline at end of file
+ b.rename_column('_tool_azuredevops_jobs', 'startTime', 'start_time')
+ b.rename_column('_tool_azuredevops_jobs', 'finishTime', 'finish_time')
+
+
+@migration(20230825150421, name="add missing migrations from 0.17 to 0.18")
+def add_missing_migrations_0_17_to_0_18(b: MigrationScriptBuilder):
+ b.rename_column('_tool_azuredevops_gitrepositories', 'transformation_rule_id', 'scope_config_id')
+ b.add_column('_tool_azuredevops_gitrepositories', 'provider', 'varchar(255)')
\ No newline at end of file
diff --git a/backend/python/pydevlake/pydevlake/model.py b/backend/python/pydevlake/pydevlake/model.py
index 4b7f9bf..dc801e3 100644
--- a/backend/python/pydevlake/pydevlake/model.py
+++ b/backend/python/pydevlake/pydevlake/model.py
@@ -87,6 +87,12 @@
name: str = Field(default="default")
domain_types: list[DomainType] = Field(default=list(DomainType), alias="entities")
+ @validator('domain_types', pre=True, always=True)
+ def set_default_domain_types(cls, v):
+ if v is None:
+ return list(DomainType)
+ return v
+
class RawModel(SQLModel):
id: int = Field(primary_key=True)
diff --git a/backend/server/services/remote/models/migration.go b/backend/server/services/remote/models/migration.go
index 129b66c..0d92d60 100644
--- a/backend/server/services/remote/models/migration.go
+++ b/backend/server/services/remote/models/migration.go
@@ -97,6 +97,27 @@
var _ Operation = (*DropTableOperation)(nil)
+type RenameColumnOperation struct {
+ Table string `json:"table"`
+ OldName string `json:"old_name"`
+ NewName string `json:"new_name"`
+}
+
+func (o RenameColumnOperation) Execute(dal dal.Dal) errors.Error {
+ if !dal.HasColumn(o.Table, o.OldName) {
+ return nil
+ }
+ if dal.HasColumn(o.Table, o.NewName) {
+ err := dal.DropColumns(o.Table, o.NewName)
+ if err != nil {
+ return err
+ }
+ }
+ return dal.RenameColumn(o.Table, o.OldName, o.NewName)
+}
+
+var _ Operation = (*RenameTableOperation)(nil)
+
type RenameTableOperation struct {
OldName string `json:"old_name"`
NewName string `json:"new_name"`
@@ -155,10 +176,12 @@
operation = &DropColumnOperation{}
case "drop_table":
operation = &DropTableOperation{}
+ case "rename_column":
+ operation = &RenameColumnOperation{}
case "rename_table":
operation = &RenameTableOperation{}
default:
- return errors.BadInput.New("unsupported operation type")
+ return errors.BadInput.New("unsupported operation type: " + operationType)
}
err = json.Unmarshal(operationRaw, operation)
if err != nil {
diff --git a/backend/server/services/remote/models/migrationscripts/azuredevops/azuredevops_20230714_decrypt_azdo_fields.go b/backend/server/services/remote/models/migrationscripts/azuredevops/azuredevops_20230714_decrypt_azdo_fields.go
new file mode 100644
index 0000000..d02cb10
--- /dev/null
+++ b/backend/server/services/remote/models/migrationscripts/azuredevops/azuredevops_20230714_decrypt_azdo_fields.go
@@ -0,0 +1,96 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package azuredevops
+
+import (
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/models/migrationscripts/archived"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/migrationhelper"
+)
+
+var _ plugin.MigrationScript = (*DecryptConnectionFields)(nil)
+
+type azureDevopsConnection20230825 struct {
+ archived.Model
+ Name string
+ Token string
+ Proxy *string
+ Organization *string
+}
+
+type DecryptConnectionFields struct{}
+
+func (script *DecryptConnectionFields) Up(basicRes context.BasicRes) errors.Error {
+ encryptionSecret := basicRes.GetConfig(plugin.EncodeKeyEnvStr)
+ if encryptionSecret == "" {
+ return errors.BadInput.New("invalid encryptionSecret")
+ }
+
+ err := migrationhelper.TransformColumns(
+ basicRes,
+ script,
+ "_tool_azuredevops_azuredevopsconnections",
+ []string{"name", "proxy", "organization"},
+ func(src *azureDevopsConnection20230825) (*azureDevopsConnection20230825, errors.Error) {
+ encName := src.Name
+ name, err := plugin.Decrypt(encryptionSecret, encName)
+ if err != nil {
+ return src, nil
+ }
+ src.Name = name
+
+ if src.Proxy != nil {
+ encProxy := *src.Proxy
+ decProxy, err := plugin.Decrypt(encryptionSecret, encProxy)
+ if err != nil {
+ return src, nil
+ }
+ if decProxy == "" {
+ src.Proxy = nil
+ } else {
+ src.Proxy = &decProxy
+ }
+ }
+
+ if src.Organization != nil {
+ encOrg := *src.Organization
+ decOrg, err := plugin.Decrypt(encryptionSecret, encOrg)
+ if err != nil {
+ return src, nil
+ }
+ if decOrg == "" {
+ src.Organization = nil
+ } else {
+ src.Organization = &decOrg
+ }
+ }
+ return src, nil
+ },
+ )
+ return err
+}
+
+func (*DecryptConnectionFields) Version() uint64 {
+ return 20230825090504
+}
+
+func (script *DecryptConnectionFields) Name() string {
+ return "Decrypt Azure DevOps connection fields"
+}
diff --git a/backend/server/services/remote/models/migrationscripts/register.go b/backend/server/services/remote/models/migrationscripts/register.go
index af1e9f9..15565c3 100644
--- a/backend/server/services/remote/models/migrationscripts/register.go
+++ b/backend/server/services/remote/models/migrationscripts/register.go
@@ -25,6 +25,7 @@
var allMigrations = map[string][]plugin.MigrationScript{
"azuredevops": {
&azuredevops.AddRawDataForScope{},
+ &azuredevops.DecryptConnectionFields{},
},
}
