feat: add scopes support for q_dev plugin
diff --git a/backend/plugins/q_dev/api/blueprint_v200.go b/backend/plugins/q_dev/api/blueprint_v200.go index e3b845c..54e9dc0 100644 --- a/backend/plugins/q_dev/api/blueprint_v200.go +++ b/backend/plugins/q_dev/api/blueprint_v200.go
@@ -56,7 +56,7 @@ func makeDataSourcePipelinePlanV200( subtaskMetas []plugin.SubTaskMeta, - scopeDetails []*srvhelper.ScopeDetail[models.QDevS3Slice, srvhelper.NoScopeConfig], + scopeDetails []*srvhelper.ScopeDetail[models.QDevS3Slice, models.QDevScopeConfig], connection *models.QDevConnection, ) (coreModels.PipelinePlan, errors.Error) { plan := make(coreModels.PipelinePlan, len(scopeDetails)) @@ -86,7 +86,7 @@ } func makeScopesV200( - scopeDetails []*srvhelper.ScopeDetail[models.QDevS3Slice, srvhelper.NoScopeConfig], + scopeDetails []*srvhelper.ScopeDetail[models.QDevS3Slice, models.QDevScopeConfig], connection *models.QDevConnection, ) ([]plugin.Scope, errors.Error) { scopes := make([]plugin.Scope, 0)
diff --git a/backend/plugins/q_dev/api/init.go b/backend/plugins/q_dev/api/init.go index 3bb6745..3a3dc2f 100644 --- a/backend/plugins/q_dev/api/init.go +++ b/backend/plugins/q_dev/api/init.go
@@ -21,7 +21,6 @@ "github.com/apache/incubator-devlake/core/context" "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" - "github.com/apache/incubator-devlake/helpers/srvhelper" "github.com/apache/incubator-devlake/plugins/q_dev/models" "github.com/go-playground/validator/v10" ) @@ -29,7 +28,7 @@ var vld *validator.Validate var connectionHelper *api.ConnectionApiHelper var basicRes context.BasicRes -var dsHelper *api.DsHelper[models.QDevConnection, models.QDevS3Slice, srvhelper.NoScopeConfig] +var dsHelper *api.DsHelper[models.QDevConnection, models.QDevS3Slice, models.QDevScopeConfig] func Init(br context.BasicRes, p plugin.PluginMeta) { basicRes = br @@ -41,13 +40,13 @@ ) dsHelper = api.NewDataSourceHelper[ - models.QDevConnection, models.QDevS3Slice, srvhelper.NoScopeConfig, + models.QDevConnection, models.QDevS3Slice, models.QDevScopeConfig, ]( basicRes, p.Name(), []string{"prefix", "basePath", "name"}, func(c models.QDevConnection) models.QDevConnection { return c.Sanitize() }, func(s models.QDevS3Slice) models.QDevS3Slice { return s.Sanitize() }, - nil, + func(sc models.QDevScopeConfig) models.QDevScopeConfig { return sc }, ) }
diff --git a/backend/plugins/q_dev/api/scopes.go b/backend/plugins/q_dev/api/scopes.go new file mode 100644 index 0000000..2b78a7b --- /dev/null +++ b/backend/plugins/q_dev/api/scopes.go
@@ -0,0 +1,36 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" +) + +// GetScopes returns available S3 slices for a connection +func GetScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + // Use the existing GetScopeList function + return GetScopeList(input) +} + +// GetScopeConfigs returns empty list since we don't have scope configs yet +func GetScopeConfigs(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return &plugin.ApiResourceOutput{ + Body: []interface{}{}, + }, nil +}
diff --git a/backend/plugins/q_dev/impl/impl.go b/backend/plugins/q_dev/impl/impl.go index 8011821..46bb166 100644 --- a/backend/plugins/q_dev/impl/impl.go +++ b/backend/plugins/q_dev/impl/impl.go
@@ -57,6 +57,7 @@ &models.QDevUserData{}, &models.QDevS3FileMeta{}, &models.QDevS3Slice{}, + &models.QDevScopeConfig{}, } } @@ -77,7 +78,7 @@ } func (p QDev) ScopeConfig() dal.Tabler { - return nil + return &models.QDevScopeConfig{} } func (p QDev) SubTaskMetas() []plugin.SubTaskMeta { @@ -150,7 +151,7 @@ "POST": api.TestExistingConnection, }, "connections/:connectionId/scopes": { - "GET": api.GetScopeList, + "GET": api.GetScopes, "PUT": api.PutScopes, }, "connections/:connectionId/scopes/:scopeId": { @@ -158,8 +159,8 @@ "PATCH": api.PatchScope, "DELETE": api.DeleteScope, }, - "connections/:connectionId/scopes/:scopeId/latest-sync-state": { - "GET": api.GetScopeLatestSyncState, + "connections/:connectionId/scope-configs": { + "GET": api.GetScopeConfigs, }, } }
diff --git a/backend/plugins/q_dev/models/migrationscripts/20260112_add_scope_config.go b/backend/plugins/q_dev/models/migrationscripts/20260112_add_scope_config.go new file mode 100644 index 0000000..4d59650 --- /dev/null +++ b/backend/plugins/q_dev/models/migrationscripts/20260112_add_scope_config.go
@@ -0,0 +1,49 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" + "github.com/apache/incubator-devlake/helpers/migrationhelper" +) + +type addQDevScopeConfig struct{} + +type qdevScopeConfig20260112 struct { + archived.ScopeConfig + ProcessingEnabled bool `json:"processingEnabled" gorm:"type:boolean"` + FilePattern string `json:"filePattern" gorm:"type:varchar(255)"` +} + +func (qdevScopeConfig20260112) TableName() string { + return "_tool_q_dev_scope_configs" +} + +func (u *addQDevScopeConfig) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables(basicRes, &qdevScopeConfig20260112{}) +} + +func (*addQDevScopeConfig) Version() uint64 { + return 20260112230000 +} + +func (*addQDevScopeConfig) Name() string { + return "Add Q Developer scope config table" +}
diff --git a/backend/plugins/q_dev/models/migrationscripts/register.go b/backend/plugins/q_dev/models/migrationscripts/register.go index 86971e5..695ffed 100644 --- a/backend/plugins/q_dev/models/migrationscripts/register.go +++ b/backend/plugins/q_dev/models/migrationscripts/register.go
@@ -31,5 +31,6 @@ new(addS3SliceTable), new(addScopeConfigIdToS3Slice), new(addScopeIdFields), + new(addQDevScopeConfig), } }
diff --git a/backend/plugins/q_dev/models/scope_config.go b/backend/plugins/q_dev/models/scope_config.go new file mode 100644 index 0000000..f7da821 --- /dev/null +++ b/backend/plugins/q_dev/models/scope_config.go
@@ -0,0 +1,47 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/plugin" +) + +type QDevScopeConfig struct { + common.ScopeConfig `mapstructure:",squash" json:",inline"` + + // Processing options + ProcessingEnabled bool `json:"processingEnabled" mapstructure:"processingEnabled"` + + // File filtering + FilePattern string `json:"filePattern" mapstructure:"filePattern"` +} + +func (QDevScopeConfig) TableName() string { + return "_tool_q_dev_scope_configs" +} + +func (c QDevScopeConfig) ScopeConfigId() uint64 { + return c.ID +} + +func (c QDevScopeConfig) ScopeConfigConnectionId() uint64 { + return c.ConnectionId +} + +var _ plugin.ToolLayerScopeConfig = (*QDevScopeConfig)(nil)
diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index b56720d..52d1449 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml
@@ -32,31 +32,31 @@ --collation-server=utf8mb4_bin --skip-log-bin - postgres: - image: postgres:14.2 - volumes: - - postgres-storage:/var/lib/postgresql - restart: always - ports: - - 5432:5432 - environment: - POSTGRES_DB: lake - POSTGRES_USER: merico - POSTGRES_PASSWORD: merico - TZ: UTC + # postgres: + # image: postgres:14.2 + # volumes: + # - postgres-storage:/var/lib/postgresql + # restart: always + # ports: + # - 5432:5432 + # environment: + # POSTGRES_DB: lake + # POSTGRES_USER: merico + # POSTGRES_PASSWORD: merico + # TZ: UTC - postgres17: - image: postgres:17.2 - volumes: - - postgres17-storage:/var/lib/postgresql - restart: always - ports: - - 5432:5432 - environment: - POSTGRES_DB: lake - POSTGRES_USER: merico - POSTGRES_PASSWORD: merico - TZ: UTC + # postgres17: + # image: postgres:17.2 + # volumes: + # - postgres17-storage:/var/lib/postgresql + # restart: always + # ports: + # - 5432:5432 + # environment: + # POSTGRES_DB: lake + # POSTGRES_USER: merico + # POSTGRES_PASSWORD: merico + # TZ: UTC grafana: image: devlake.docker.scarf.sh/apache/devlake-dashboard:latest @@ -142,6 +142,6 @@ volumes: mysql-storage: grafana-storage: - postgres-storage: - postgres17-storage: + # postgres-storage: + # postgres17-storage: devlake-log: \ No newline at end of file