[horus] Removal of horus infrastructure (#450)
diff --git a/app/horus/base/config/config.go b/app/horus/base/config/config.go
deleted file mode 100644
index fd55396..0000000
--- a/app/horus/base/config/config.go
+++ /dev/null
@@ -1,91 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more
-// contributor license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright ownership.
-// The ASF licenses this file to You under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance with
-// the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package config
-
-type Config struct {
- Address string `yaml:"address"`
- KubeTimeSecond int64 `yaml:"kubeTimeSecond"`
- Mysql *MysqlConfiguration `yaml:"mysql"`
- DingTalk *DingTalkConfiguration `yaml:"dingTalk"`
- Slack *SlackConfiguration `yaml:"slack"`
- KubeMultiple map[string]string `yaml:"kubeMultiple"`
- PromMultiple map[string]string `yaml:"promMultiple"`
- NodeRecovery *RecoveryConfiguration `yaml:"nodeRecovery"`
- CustomModular *ModularConfiguration `yaml:"customModular"`
- NodeDownTime *DowntimeConfiguration `yaml:"nodeDownTime"`
- PodStagnationCleaner *CleanerConfiguration `yaml:"podStagnationCleaner"`
-}
-
-type MysqlConfiguration struct {
- Name string `yaml:"name"`
- Address string `yaml:"address"`
- Debug bool `yaml:"debug"`
-}
-
-type DingTalkConfiguration struct {
- WebhookUrl string `yaml:"webhookUrl"`
- Title string `yaml:"title"`
- AtMobiles []string `yaml:"atMobiles"`
-}
-
-type SlackConfiguration struct {
- WebhookUrl string `yaml:"webhookUrl"`
- Title string `yaml:"title"`
-}
-
-type RecoveryConfiguration struct {
- DayNumber int `yaml:"dayNumber"`
- IntervalSecond int `yaml:"intervalSecond"`
- PromQueryTimeSecond int64 `yaml:"promQueryTimeSecond"`
- DingTalk *DingTalkConfiguration `yaml:"dingTalk"`
- Slack *SlackConfiguration `yaml:"slack"`
-}
-
-type ModularConfiguration struct {
- Enabled bool `yaml:"enabled"`
- DailyLimit map[string]int `yaml:"dailyLimit"`
- AbnormalityQL map[string]string `yaml:"abnormalityQL"`
- RecoveryQL map[string]string `yaml:"recoveryQL"`
- IntervalSecond int `yaml:"intervalSecond"`
- PromQueryTimeSecond int64 `yaml:"promQueryTimeSecond"`
- KubeMultiple map[string]string `yaml:"kubeMultiple"`
- DingTalk *DingTalkConfiguration `yaml:"dingTalk"`
- Slack *SlackConfiguration `yaml:"slack"`
-}
-
-type DowntimeConfiguration struct {
- Enabled bool `yaml:"enabled"`
- IntervalSecond int `yaml:"intervalSecond"`
- PromQueryTimeSecond int64 `yaml:"promQueryTimeSecond"`
- KubeMultiple map[string]string `yaml:"kubeMultiple"`
- AbnormalityQL []string `yaml:"abnormalityQL"`
- AbnormalInfoSystemQL string `yaml:"abnormalInfoSystemQL"`
- AbnormalRecoveryQL []string `yaml:"abnormalRecoveryQL"`
- AllSystemUser string `yaml:"allSystemUser"`
- AllSystemPassword string `yaml:"allSystemPassword"`
- DingTalk *DingTalkConfiguration `yaml:"dingTalk"`
- Slack *SlackConfiguration `yaml:"slack"`
-}
-
-type CleanerConfiguration struct {
- Enabled bool `yaml:"enabled"`
- IntervalSecond int `yaml:"intervalSecond"`
- DoubleSecond int `yaml:"doubleSecond"`
- LabelSelector string `yaml:"labelSelector"`
- FieldSelector string `yaml:"fieldSelector"`
- KubeMultiple map[string]string `yaml:"kubeMultiple"`
- DingTalk *DingTalkConfiguration `yaml:"dingTalk"`
-}
diff --git a/app/horus/base/config/parse.go b/app/horus/base/config/parse.go
deleted file mode 100644
index 8e65762..0000000
--- a/app/horus/base/config/parse.go
+++ /dev/null
@@ -1,44 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more
-// contributor license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright ownership.
-// The ASF licenses this file to You under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance with
-// the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package config
-
-import (
- "fmt"
- "gopkg.in/yaml.v2"
- "os"
-)
-
-func Load(in string) (*Config, error) {
- cfg := &Config{}
- err := yaml.Unmarshal([]byte(in), cfg)
- if err != nil {
- return nil, err
- }
- return cfg, err
-}
-
-func LoadFile(name string) (*Config, error) {
- bytes, err := os.ReadFile(name)
- if err != nil {
- return nil, err
- }
- cfg, err := Load(string(bytes))
- if err != nil {
- fmt.Printf("parsing yaml file err:%v", err)
- return nil, err
- }
- return cfg, err
-}
diff --git a/app/horus/base/db/db.go b/app/horus/base/db/db.go
deleted file mode 100644
index 0408b0a..0000000
--- a/app/horus/base/db/db.go
+++ /dev/null
@@ -1,244 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more
-// contributor license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright ownership.
-// The ASF licenses this file to You under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance with
-// the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package db
-
-import (
- "fmt"
- "github.com/apache/dubbo-kubernetes/app/horus/base/config"
- _ "github.com/go-sql-driver/mysql"
- "time"
- "xorm.io/xorm"
- xlog "xorm.io/xorm/log"
-)
-
-type NodeDataInfo struct {
- Id int64 `json:"id"`
- NodeName string `json:"node_name" xorm:"node_name"`
- NodeIP string `json:"node_ip" xorm:"node_ip"`
- Sn string `json:"sn"`
- ClusterName string `json:"cluster_name" xorm:"cluster_name"`
- ModuleName string `json:"module_name" xorm:"module_name"`
- Reason string `json:"reason"`
- Restart int `json:"restart"`
- Repair int `json:"repair"`
- RepairTicketUrl string `json:"repair_ticket_url" xorm:"repair_ticket_url"`
- FirstDate string `json:"first_date" xorm:"first_date"`
- CreateTime time.Time `json:"create_time" xorm:"create_time created"`
- UpdateTime time.Time `json:"update_time" xorm:"update_time updated"`
- RecoveryMark int64 `json:"recovery_mark" xorm:"recovery_mark"`
- RecoveryQL string `json:"recovery_ql" xorm:"recovery_ql"`
- DownTimeRecoveryMark int64 `json:"downtime_recovery_mark" xorm:"downtime_recovery_mark"`
- DownTimeRecoveryQL []string `json:"downtime_recovery_ql" xorm:"downtime_recovery_ql"`
-}
-
-type PodDataInfo struct {
- Id int64 `json:"id"`
- PodName string `json:"pod_name" xorm:"pod_name"`
- PodIP string `json:"pod_ip" xorm:"pod_ip"`
- Sn string `json:"sn"`
- NodeName string `json:"node_name" xorm:"node_name"`
- ClusterName string `json:"cluster_name" xorm:"cluster_name"`
- ModuleName string `json:"module_name" xorm:"module_name"`
- Reason string `json:"reason"`
- FirstDate string `json:"first_date" xorm:"first_date"`
- CreateTime time.Time `json:"create_time" xorm:"create_time created"`
- UpdateTime time.Time `json:"update_time" xorm:"update_time updated"`
-}
-
-var (
- db *xorm.Engine
-)
-
-func InitDataBase(mc *config.MysqlConfiguration) error {
- data, err := xorm.NewEngine("mysql", mc.Address)
- if err != nil {
- fmt.Printf("Unable to connect to mysql server:\n addr:%s\n err:%v\n", mc.Address, err)
- }
- data.Logger().SetLevel(xlog.LOG_INFO)
- data.ShowSQL(mc.Debug)
- db = data
- return nil
-}
-
-func (n *NodeDataInfo) Add() (int64, error) {
- row, err := db.Insert(n)
- return row, err
-}
-
-func (n *NodeDataInfo) Get() (*NodeDataInfo, error) {
- exist, err := db.Get(n)
- if err != nil {
- return nil, err
- }
- if !exist {
- return nil, nil
- }
- return n, nil
-}
-
-func (n *NodeDataInfo) Update() (bool, error) {
- firstDate := time.Now().Format("2006-01-02 15:04:05")
- n.FirstDate = firstDate
-
- row, err := db.Where(fmt.Sprintf("id=%d", n.Id)).Update(n)
- if err != nil {
- return false, err
- }
- if row > 0 {
- return true, nil
- }
- return false, nil
-}
-
-func (n *NodeDataInfo) Check() (bool, error) {
- exist, err := db.Exist(n)
- return exist, err
-}
-
-func (n *NodeDataInfo) AddOrGet() (int64, error) {
- exist, _ := n.Check()
- if exist {
- return n.Id, nil
- }
- row, err := n.Add()
- return row, err
-}
-
-func (n *NodeDataInfo) RecoveryMarker() (bool, error) {
- n.RecoveryMark = 1
- return n.Update()
-}
-
-func (n *NodeDataInfo) DownTimeRecoveryMarker() (bool, error) {
- n.DownTimeRecoveryMark = 1
- return n.Update()
-}
-
-func (n *NodeDataInfo) RestartMarker() (bool, error) {
- n.Restart = 1
- return n.Update()
-}
-
-func GetNode() ([]NodeDataInfo, error) {
- var ndi []NodeDataInfo
- session := db.Where(fmt.Sprintf("id>%d", 0))
- err := session.Find(&ndi)
- return ndi, err
-}
-
-func GetNodeByName(nodeName, moduleName string) (*NodeDataInfo, error) {
- ndi := NodeDataInfo{
- NodeName: nodeName,
- ModuleName: moduleName,
- }
- return ndi.Get()
-}
-
-func GetRecoveryNodeDataInfoDate(day int) ([]NodeDataInfo, error) {
- var ndi []NodeDataInfo
- session := db.Where(fmt.Sprintf("recovery_mark = 0 AND first_date > DATE_SUB(CURDATE(), INTERVAL %d DAY)", day))
- err := session.Find(&ndi)
- return ndi, err
-}
-
-func GetDownTimeRecoveryNodeDataInfoDate(day int) ([]NodeDataInfo, error) {
- var ndi []NodeDataInfo
- session := db.Where(fmt.Sprintf("downtime_recovery_mark = 0 AND first_date > DATE_SUB(CURDATE(), INTERVAL %d DAY)", day))
- err := session.Find(&ndi)
- return ndi, err
-}
-
-func GetRestartNodeDataInfoDate() ([]NodeDataInfo, error) {
- var ndi []NodeDataInfo
- session := db.Where("restart = 0 and repair = 0 and module_name = ?", "nodeDown")
- err := session.Find(&ndi)
- return ndi, err
-}
-
-func GetDailyLimitNodeDataInfoDate(day, module, cluster string) ([]NodeDataInfo, error) {
- var ndi []NodeDataInfo
- session := db.Where("DATE(first_date) = ? AND module_name = ? AND cluster_name = ?", day, module, cluster)
- err := session.Find(&ndi)
- return ndi, err
-}
-
-func (p *PodDataInfo) Add() (int64, error) {
- row, err := db.Insert(p)
- return row, err
-}
-
-func (p *PodDataInfo) Get() (*PodDataInfo, error) {
- exist, err := db.Get(p)
- if err != nil {
- return nil, err
- }
- if !exist {
- return nil, nil
- }
- return p, nil
-}
-
-func (p *PodDataInfo) Update() (bool, error) {
- firstDate := time.Now().Format("2006-01-02 15:04:05")
- p.FirstDate = firstDate
-
- row, err := db.Where(fmt.Sprintf("id=%d", p.Id)).Update(p)
- if err != nil {
- return false, err
- }
- if row > 0 {
- return true, nil
- }
- return false, nil
-}
-
-func (p *PodDataInfo) Check() (bool, error) {
- exist, err := db.Exist(p)
- return exist, err
-}
-
-func (p *PodDataInfo) AddOrGet() (int64, error) {
- exist, _ := p.Check()
- if exist {
- return p.Id, nil
- }
- row, err := p.Add()
- return row, err
-}
-
-func GetPod() ([]PodDataInfo, error) {
- var pdi []PodDataInfo
- session := db.Where(fmt.Sprintf("id>%d", 0))
- err := session.Find(&pdi)
- return pdi, err
-}
-
-func GetPodByName(podName, moduleName string) (*PodDataInfo, error) {
- pdi := PodDataInfo{
- PodName: podName,
- ModuleName: moduleName,
- }
- return pdi.Get()
-}
-
-func GetLimitPodDataInfo(limit, offset int, orderColumn, where string, args ...interface{}) ([]PodDataInfo, error) {
- var pdi []PodDataInfo
- err := db.Where(where, args...).Limit(limit, offset).Desc(orderColumn).Find(pdi)
- if err != nil {
- return nil, err
- }
- return pdi, nil
-}
diff --git a/app/horus/base/db/db_test.go b/app/horus/base/db/db_test.go
deleted file mode 100644
index 3fcf922..0000000
--- a/app/horus/base/db/db_test.go
+++ /dev/null
@@ -1,51 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more
-// contributor license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright ownership.
-// The ASF licenses this file to You under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance with
-// the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package db_test
-
-import (
- "github.com/apache/dubbo-kubernetes/app/horus/base/config"
- "github.com/apache/dubbo-kubernetes/app/horus/base/db"
- "testing"
- "time"
-)
-
-func TestDataBase(t *testing.T) {
- mc := &config.MysqlConfiguration{
- Address: "root:root@tcp(127.0.0.1:3306)/horus?charset=utf8&parseTime=True",
- Debug: true,
- }
-
- err := db.InitDataBase(mc)
- if err != nil {
- t.Fatalf("Failed to initialize database: %v", err)
- return
- }
-
- today := time.Now().Format("2006-01-02 15:04:05")
- data := db.NodeDataInfo{
- NodeName: "db-test",
- NodeIP: "1.1.1.1",
- Sn: "123",
- ClusterName: "beijing01",
- ModuleName: "model",
- Reason: "time wrong",
- Restart: 0,
- Repair: 0,
- FirstDate: today,
- }
- id, err := data.Add()
- t.Logf("test.add id:%v err:%v", id, err)
-}
diff --git a/app/horus/base/db/table.sql b/app/horus/base/db/table.sql
deleted file mode 100644
index bd61301..0000000
--- a/app/horus/base/db/table.sql
+++ /dev/null
@@ -1,51 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more
-// contributor license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright ownership.
-// The ASF licenses this file to You under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance with
-// the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-CREATE TABLE `node_data_info` (
- `id` int(10) UNSIGNED NOT NULL AUTO_INCREMENT,
- `node_name` varchar(255) CHARACTER SET utf8mb3 COLLATE utf8mb3_general_ci NOT NULL COMMENT '节点名',
- `node_ip` varchar(255) CHARACTER SET utf8mb3 COLLATE utf8mb3_general_ci NOT NULL COMMENT '节点IP',
- `sn` varchar(255) CHARACTER SET utf8mb3 COLLATE utf8mb3_general_ci NOT NULL COMMENT '序列号',
- `cluster_name` varchar(255) CHARACTER SET utf8mb3 COLLATE utf8mb3_general_ci NULL DEFAULT NULL COMMENT '集群名',
- `module_name` varchar(255) CHARACTER SET utf8mb3 COLLATE utf8mb3_general_ci NOT NULL COMMENT '模块名',
- `reason` varchar(255) CHARACTER SET utf8mb3 COLLATE utf8mb3_general_ci NULL DEFAULT NULL COMMENT '维护原因',
- `restart` int(10) UNSIGNED NULL DEFAULT NULL COMMENT '重启标志',
- `repair` int(10) UNSIGNED NULL DEFAULT NULL COMMENT '修复标志',
- `repair_ticket_url` varchar(255) CHARACTER SET utf8mb3 COLLATE utf8mb3_general_ci NULL DEFAULT NULL COMMENT '修复单跳转 URL',
- `first_date` datetime(0) NULL DEFAULT NULL COMMENT '最早开始维护时间',
- `create_time` datetime(0) NULL DEFAULT CURRENT_TIMESTAMP(0) COMMENT '创建时间',
- `update_time` datetime(0) NULL DEFAULT CURRENT_TIMESTAMP(0) ON UPDATE CURRENT_TIMESTAMP(0) COMMENT '更新时间',
- PRIMARY KEY (`id`) USING BTREE,
- UNIQUE INDEX `idx_unique_node_module` (`node_name`, `module_name`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=3 CHARACTER SET=utf8mb3 COLLATE=utf8mb3_general_ci ROW_FORMAT=Dynamic;
-
-CREATE TABLE `pod_data_info` (
- `id` int unsigned NOT NULL AUTO_INCREMENT,
- `pod_name` varchar(255) CHARACTER SET utf8mb3 COLLATE utf8mb3_general_ci NOT NULL COMMENT 'Pod 名',
- `pod_ip` varchar(255) CHARACTER SET utf8mb3 COLLATE utf8mb3_general_ci DEFAULT NULL COMMENT 'PodIP',
- `sn` varchar(255) CHARACTER SET utf8mb3 COLLATE utf8mb3_general_ci NOT NULL COMMENT '序列号',
- `node_name` varchar(255) CHARACTER SET utf8mb3 COLLATE utf8mb3_general_ci NOT NULL COMMENT '节点名',
- `cluster_name` varchar(255) CHARACTER SET utf8mb3 COLLATE utf8mb3_general_ci DEFAULT NULL COMMENT '集群名',
- `module_name` varchar(255) CHARACTER SET utf8mb3 COLLATE utf8mb3_general_ci NOT NULL COMMENT '模块名',
- `reason` varchar(255) CHARACTER SET utf8mb3 COLLATE utf8mb3_general_ci DEFAULT NULL COMMENT '维护原因',
- `restart` int DEFAULT NULL COMMENT '重启标志',
- `repair` int DEFAULT NULL COMMENT '修复标志',
- `repair_ticket_url` varchar(255) CHARACTER SET utf8mb3 COLLATE utf8mb3_general_ci DEFAULT NULL COMMENT '修复单跳转 URL',
- `first_date` datetime DEFAULT NULL COMMENT '最早开始维护时间',
- `create_time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
- `update_time` datetime DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间',
- PRIMARY KEY (`id`) USING BTREE,
- UNIQUE KEY `idxUniqueName` (`pod_name`, `module_name`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8mb3 ROW_FORMAT=DYNAMIC;
\ No newline at end of file
diff --git a/app/horus/cmd/main.go b/app/horus/cmd/main.go
deleted file mode 100644
index 997044c..0000000
--- a/app/horus/cmd/main.go
+++ /dev/null
@@ -1,193 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more
-// contributor license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright ownership.
-// The ASF licenses this file to You under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance with
-// the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package main
-
-import (
- "context"
- "flag"
- "github.com/apache/dubbo-kubernetes/app/horus/base/config"
- "github.com/apache/dubbo-kubernetes/app/horus/base/db"
- "github.com/apache/dubbo-kubernetes/app/horus/core/horuser"
- "github.com/apache/dubbo-kubernetes/app/horus/core/ticker"
- "github.com/prometheus/client_golang/prometheus"
- "github.com/prometheus/client_golang/prometheus/promhttp"
- "k8s.io/klog"
- "net/http"
- "os"
- "os/signal"
- "sync"
- "syscall"
-)
-
-var (
- address string
- configFile string
-)
-
-func main() {
- flag.StringVar(&configFile, "configFile", "../../manifests/horus/horus.yaml", "horus config file")
- flag.StringVar(&address, "address", "0.0.0.0:38089", "horus address")
- klog.InitFlags(flag.CommandLine)
- flag.Parse()
-
- c, err := config.LoadFile(configFile)
- if err != nil {
- klog.Errorf("load config file failed err:%+v", err)
- return
- } else {
- klog.Infof("load config file success.")
- }
-
- err = db.InitDataBase(c.Mysql)
- if err != nil {
- klog.Errorf("horus db initial failed err:%v", err)
- return
- } else {
- klog.Infof("horus db initial success.")
- }
- horus := horuser.NewHoruser(c)
- prometheus.MustRegister(horus)
- group, stopChan := setupStopChanWithContext()
- ctx, cancel := context.WithCancel(context.Background())
- group.Add(func() error {
- for {
- select {
- case <-stopChan:
- cancel()
- return nil
- case <-ctx.Done():
- return nil
- }
- }
- })
- group.Add(func() error {
- http.Handle("/metrics", promhttp.Handler())
- srv := http.Server{Addr: c.Address}
- err := srv.ListenAndServe()
- if err != nil {
- klog.Errorf("horus metrics err:%v", err)
- return err
- }
- return nil
- })
- group.Add(func() error {
- klog.Info("horus ticker manager start success.")
- err := ticker.Manager(ctx)
- if err != nil {
- klog.Errorf("horus ticker manager start failed err:%v", err)
- return err
- }
- return nil
- })
- group.Add(func() error {
- if c.CustomModular.Enabled {
- klog.Info("horus node recovery manager start success.")
- err := horus.RecoveryManager(ctx)
- if err != nil {
- klog.Errorf("horus node recovery manager start failed err:%v", err)
- return err
- }
- }
- return nil
- })
- group.Add(func() error {
- if c.CustomModular.Enabled {
- klog.Info("horus node customize modular manager start success.")
- err := horus.CustomizeModularManager(ctx)
- if err != nil {
- klog.Errorf("horus node customize modular manager start failed err:%v", err)
- return err
- }
- }
- return nil
- })
- group.Add(func() error {
- if c.NodeDownTime.Enabled {
- klog.Info("horus node downtime manager start success.")
- err := horus.DownTimeManager(ctx)
- if err != nil {
- klog.Errorf("horus node downtime manager start failed err:%v", err)
- return err
- }
- }
- return nil
- })
- group.Add(func() error {
- if c.NodeDownTime.Enabled {
- klog.Info("horus node downtime restart manager start success.")
- err := horus.DowntimeRestartManager(ctx)
- if err != nil {
- klog.Errorf("horus node downtime restart manager start failed err:%v", err)
- return err
- }
- }
- return nil
- })
- group.Add(func() error {
- if c.NodeDownTime.Enabled {
- klog.Info("horus node downtime recovery manager start success.")
- err := horus.DownTimeRecoveryManager(ctx)
- if err != nil {
- klog.Errorf("horus node downtime recovery manager start failed err:%v", err)
- return err
- }
- }
- return nil
- })
- group.Add(func() error {
- if c.PodStagnationCleaner.Enabled {
- klog.Info("horus pod stagnation clean manager start success.")
- err := horus.PodStagnationCleanManager(ctx)
- if err != nil {
- klog.Errorf("horus pod stagnation clean manager start failed err:%v", err)
- return err
- }
- }
- return nil
- })
- group.Wait()
-}
-
-type WaitGroup struct {
- wg sync.WaitGroup
-}
-
-func (g *WaitGroup) Add(f func() error) {
- g.wg.Add(1)
- go func() {
- defer g.wg.Done()
- _ = f()
- }()
-}
-
-func (g *WaitGroup) Wait() {
- g.wg.Wait()
-}
-
-func setupStopChanWithContext() (*WaitGroup, <-chan struct{}) {
- stopChan := make(chan struct{})
- SignalChan := make(chan os.Signal, 1)
- signal.Notify(SignalChan, syscall.SIGTERM, syscall.SIGQUIT)
- g := &WaitGroup{}
- g.Add(func() error {
- select {
- case <-SignalChan:
- close(SignalChan)
- }
- return nil
- })
- return g, stopChan
-}
diff --git a/app/horus/core/alerter/dingtalk.go b/app/horus/core/alerter/dingtalk.go
deleted file mode 100644
index 2a13d53..0000000
--- a/app/horus/core/alerter/dingtalk.go
+++ /dev/null
@@ -1,71 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more
-// contributor license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright ownership.
-// The ASF licenses this file to You under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance with
-// the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package alerter
-
-import (
- "bytes"
- "encoding/json"
- "fmt"
- "github.com/apache/dubbo-kubernetes/app/horus/base/config"
- "k8s.io/klog/v2"
- "net/http"
-)
-
-type T struct {
- At struct {
- AtMobiles []string `json:"atMobiles"`
- AtUserIds []string `json:"atUserIds"`
- IsAtAll bool `json:"isAtAll"`
- } `json:"at"`
- Text struct {
- Content string `json:"content"`
- } `json:"text"`
- Msgtype string `json:"msgtype"`
-}
-
-type content struct {
- Content string `json:"content"`
-}
-
-type at struct {
- AtMobiles []string `json:"atMobiles"`
-}
-
-type Message struct {
- MsgType string `json:"msgtype"`
- Text content `json:"text"`
- At at `json:"at"`
-}
-
-func DingTalkSend(dk *config.DingTalkConfiguration, msg string) {
- dtm := Message{MsgType: "text"}
- dtm.Text.Content = fmt.Sprint(dk.Title, msg)
- dtm.At.AtMobiles = dk.AtMobiles
- bs, err := json.Marshal(dtm)
- if err != nil {
- klog.Errorf("dingTalk json marshal err:%v\n dtm:%v\n", err, dtm)
- return
- }
- res, err := http.Post(dk.WebhookUrl, "application/json", bytes.NewBuffer(bs))
- if err != nil {
- klog.Errorf("send dingTalk err:%v\n msg:%v\n", err, msg)
- }
- if res != nil && res.StatusCode != 200 {
- klog.Errorf("send dingTalk status code err:%v\n code:%v\n msg:%v\n", err, res.StatusCode, msg)
- return
- }
- klog.Infof("send dingTalk success code:%v\n msg:%v\n", res.StatusCode, msg)
-}
diff --git a/app/horus/core/alerter/dingtalk_test.go b/app/horus/core/alerter/dingtalk_test.go
deleted file mode 100644
index 698cdfc..0000000
--- a/app/horus/core/alerter/dingtalk_test.go
+++ /dev/null
@@ -1,31 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more
-// contributor license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright ownership.
-// The ASF licenses this file to You under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance with
-// the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package alerter_test
-
-import (
- "github.com/apache/dubbo-kubernetes/app/horus/base/config"
- "github.com/apache/dubbo-kubernetes/app/horus/core/alerter"
- "testing"
-)
-
-func TestDingTalkSend(t *testing.T) {
- im := &config.DingTalkConfiguration{
- WebhookUrl: "https://oapi.dingtalk.com/robot/send?access_token=aa2f3f74d7a2504653ca89b7a673707ba1d04b6d9d320c3572e5464d2f81471x",
- Title: "",
- AtMobiles: nil,
- }
- alerter.DingTalkSend(im, "test")
-}
diff --git a/app/horus/core/alerter/slack.go b/app/horus/core/alerter/slack.go
deleted file mode 100644
index 0a2db18..0000000
--- a/app/horus/core/alerter/slack.go
+++ /dev/null
@@ -1,43 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more
-// contributor license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright ownership.
-// The ASF licenses this file to You under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance with
-// the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package alerter
-
-import (
- "bytes"
- "encoding/json"
- "fmt"
- "github.com/apache/dubbo-kubernetes/app/horus/base/config"
- "k8s.io/klog/v2"
- "net/http"
-)
-
-type Text struct {
- Text string `json:"text"`
-}
-
-func SlackSend(sk *config.SlackConfiguration, channel string) {
- skm := Text{Text: "text"}
- skm.Text = fmt.Sprintf("%s"+"%v", sk.Title, channel)
- bs, err := json.Marshal(skm)
- if err != nil {
- klog.Errorf("slack json marshal err:%v\n skm:%v\n", err, skm)
- }
- res, err := http.Post(sk.WebhookUrl, "application/json", bytes.NewBuffer(bs))
- if res.StatusCode != 200 {
- klog.Errorf("send slack status code err:%v\n code:%v\n channel:%v\n", err, res.StatusCode, channel)
- return
- }
-}
diff --git a/app/horus/core/alerter/slack_test.go b/app/horus/core/alerter/slack_test.go
deleted file mode 100644
index 8e8378b..0000000
--- a/app/horus/core/alerter/slack_test.go
+++ /dev/null
@@ -1,30 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more
-// contributor license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright ownership.
-// The ASF licenses this file to You under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance with
-// the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package alerter_test
-
-import (
- "github.com/apache/dubbo-kubernetes/app/horus/base/config"
- "github.com/apache/dubbo-kubernetes/app/horus/core/alerter"
- "testing"
-)
-
-func TestSlackSend(t *testing.T) {
- im := &config.SlackConfiguration{
- WebhookUrl: "https://hooks.slack.com/services/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",
- Title: "",
- }
- alerter.SlackSend(im, "test")
-}
diff --git a/app/horus/core/horuser/horuser.go b/app/horus/core/horuser/horuser.go
deleted file mode 100644
index 9a45fc8..0000000
--- a/app/horus/core/horuser/horuser.go
+++ /dev/null
@@ -1,71 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more
-// contributor license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright ownership.
-// The ASF licenses this file to You under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance with
-// the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package horuser
-
-import (
- "context"
- "github.com/apache/dubbo-kubernetes/app/horus/base/config"
- clientset "k8s.io/client-go/kubernetes"
- "k8s.io/client-go/rest"
- "k8s.io/client-go/tools/clientcmd"
- "k8s.io/klog/v2"
- "time"
-)
-
-type Horuser struct {
- cc *config.Config
- kubeClientMap map[string]*clientset.Clientset
-}
-
-func NewHoruser(c *config.Config) *Horuser {
- hr := &Horuser{
- cc: c,
- kubeClientMap: map[string]*clientset.Clientset{},
- }
- i := 1
- n := len(c.KubeMultiple)
- for clusterName, km := range c.KubeMultiple {
- kcfg, err := k8sBuildConfig(km)
- if err != nil {
- klog.Errorf("NewHoruser k8sBuildConfig err:%v\n name:%v\n", err, clusterName)
- }
- km := clientset.NewForConfigOrDie(kcfg)
- hr.kubeClientMap[clusterName] = km
- klog.Info("NewHoruser k8sBuildConfig success.")
- klog.Infof("[KubeMultipleCluster:%v\n] Count:[%d/%d]", clusterName, n, i)
- i++
- }
- return hr
-}
-
-func k8sBuildConfig(kubeconfig string) (*rest.Config, error) {
- if kubeconfig != "" {
- cfg, err := clientcmd.BuildConfigFromFlags("", kubeconfig)
- if err != nil {
- return nil, err
- }
- return cfg, err
- }
- cfg, err := rest.InClusterConfig()
- if err != nil {
- return nil, err
- }
- return cfg, err
-}
-
-func (h *Horuser) GetK8sContext() (context.Context, context.CancelFunc) {
- return context.WithTimeout(context.Background(), time.Duration(h.cc.KubeTimeSecond)*time.Second)
-}
diff --git a/app/horus/core/horuser/metrics.go b/app/horus/core/horuser/metrics.go
deleted file mode 100644
index 977cd2d..0000000
--- a/app/horus/core/horuser/metrics.go
+++ /dev/null
@@ -1,196 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more
-// contributor license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright ownership.
-// The ASF licenses this file to You under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance with
-// the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package horuser
-
-import (
- "fmt"
- "github.com/apache/dubbo-kubernetes/app/horus/base/db"
- "github.com/prometheus/client_golang/prometheus"
- "k8s.io/klog/v2"
- "strings"
-)
-
-var (
- FeatureInfo = prometheus.NewDesc(
- "horus_feature_info",
- "Indicates the enabled status of specific features in different clusters, with each feature represented by its name, enabled state, and associated cluster name.",
- []string{
- "feature",
- "enabled",
- "cluster_name",
- },
- nil)
-
- MultipleInfo = prometheus.NewDesc(
- "horus_multiple_info",
- "Tracks the Prometheus multiple addresses associated with different clusters, providing visibility into the Prometheus endpoints used by each cluster.",
- []string{
- "cluster_name",
- "prometheus_multiple_address",
- },
- nil)
-
- NodeInfo = prometheus.NewDesc(
- "horus_node_info",
- "Provides detailed information about the nodes within a cluster, including node identity, operational state, maintenance details, and recovery status.",
- []string{
- "node_name",
- "node_ip",
- "sn",
- "cluster_name",
- "module_name",
- "reason",
- "restart",
- "repair",
- "repair_ticket_url",
- "first_date",
- "create_time",
- "update_time",
- "recovery_ql",
- "recovery_mark",
- },
- nil)
-
- PodStagnativeInfo = prometheus.NewDesc(
- "pod_stagnative_info",
- "pod_stagnative_info",
- []string{
- "pod_name",
- "pod_ip",
- "node_name",
- "cluster_name",
- "module_name",
- "reason",
- "first_date",
- "create_time",
- "update_time",
- },
- nil)
-)
-
-func (h *Horuser) Collect(ch chan<- prometheus.Metric) {
- kFunc := func(m map[string]string) string {
- s := []string{}
- for k := range m {
- s = append(s, k)
- }
- return strings.Join(s, ",")
- }
- info := map[string]string{}
- buttons := map[bool]string{true: "Open", false: "Close"}
-
- modularKey := fmt.Sprintf("custom modular,%s", buttons[h.cc.CustomModular.Enabled])
- info[modularKey] = kFunc(h.cc.CustomModular.KubeMultiple)
- downtimeKey := fmt.Sprintf("node downtime,%s", buttons[h.cc.NodeDownTime.Enabled])
- info[downtimeKey] = kFunc(h.cc.NodeDownTime.KubeMultiple)
-
- nodeFunc := func() {
- nodes, err := db.GetNode()
- if err != nil {
- klog.Errorf("horus metrics collect db get node err:%v", err)
- return
- }
- if len(nodes) == 0 {
- klog.Infof("horus metrics collect db zero err:%v", err)
- return
- }
- for _, v := range nodes {
- v := v
- ct := v.CreateTime.Local().Format("2006-01-02 15:04:05")
- ut := v.UpdateTime.Local().Format("2006-01-02 15:04:05")
- p := prometheus.MustNewConstMetric(NodeInfo,
- prometheus.GaugeValue, 1,
- v.NodeName,
- v.NodeIP,
- v.Sn,
- v.ClusterName,
- v.ModuleName,
- v.Reason,
- fmt.Sprintf("%d", v.Restart),
- fmt.Sprintf("%d", v.Repair),
- v.RepairTicketUrl,
- v.FirstDate,
- ct,
- ut,
- v.RecoveryQL,
- fmt.Sprintf("%d", v.RecoveryMark),
- )
- ch <- p
- }
- }
- nodeFunc()
-
- podFunc := func() {
- pods, err := db.GetPod()
- if err != nil {
- klog.Errorf("horus metrics collect db get pod err:%v", err)
- return
- }
- klog.Info("horus metrics collect db get pod success.")
- if len(pods) == 0 {
- klog.Errorf("horus metrics collect db zero err:%v", err)
- return
- }
- for _, v := range pods {
- v := v
- ct := v.CreateTime.Local().Format("2006-01-02 15:04:05")
- ut := v.UpdateTime.Local().Format("2006-01-02 15:04:05")
- p := prometheus.MustNewConstMetric(PodStagnativeInfo,
- prometheus.GaugeValue, 1,
- v.PodName,
- v.PodIP,
- v.NodeName,
- v.ClusterName,
- v.ModuleName,
- v.Reason,
- v.FirstDate,
- ct,
- ut,
- )
- ch <- p
- }
- }
- podFunc()
-
- for k, clusterName := range info {
- s := strings.Split(k, ",")
- feature, enabled := s[0], s[1]
- p := prometheus.MustNewConstMetric(
- FeatureInfo,
- prometheus.GaugeValue, 1,
- feature,
- enabled,
- clusterName,
- )
- ch <- p
- }
-
- for clusterName, address := range h.cc.PromMultiple {
- p := prometheus.MustNewConstMetric(
- MultipleInfo,
- prometheus.GaugeValue, 1,
- clusterName,
- address,
- )
- ch <- p
- }
-}
-
-func (h *Horuser) Describe(ch chan<- *prometheus.Desc) {
- ch <- FeatureInfo
- ch <- MultipleInfo
- ch <- NodeInfo
-}
diff --git a/app/horus/core/horuser/node_cordon.go b/app/horus/core/horuser/node_cordon.go
deleted file mode 100644
index 212de17..0000000
--- a/app/horus/core/horuser/node_cordon.go
+++ /dev/null
@@ -1,58 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more
-// contributor license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright ownership.
-// The ASF licenses this file to You under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance with
-// the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package horuser
-
-import (
- v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
- "k8s.io/klog/v2"
-)
-
-func (h *Horuser) Cordon(nodeName, clusterName, moduleName string) (err error) {
- kubeClient := h.kubeClientMap[clusterName]
- if kubeClient == nil {
- klog.Error("node Cordon kubeClient by clusterName empty.")
- klog.Infof("nodeName:%v\n,clusterName:%v\n", nodeName, clusterName)
- return err
- }
-
- ctxFirst, cancelFirst := h.GetK8sContext()
- defer cancelFirst()
- node, err := kubeClient.CoreV1().Nodes().Get(ctxFirst, nodeName, v1.GetOptions{})
- if err != nil {
- klog.Errorf("node Cordon get err:%v", err)
- klog.Infof("nodeName:%v\n clusterName:%v\n", nodeName, clusterName)
- return err
- }
- annotations := node.Annotations
- if annotations == nil {
- annotations = map[string]string{}
- }
- annotations["dubbo.io/disable-by"] = "horus"
-
- node.Spec.Unschedulable = true
-
- ctxSecond, cancelSecond := h.GetK8sContext()
- defer cancelSecond()
- node, err = kubeClient.CoreV1().Nodes().Update(ctxSecond, node, v1.UpdateOptions{})
- if err != nil {
- klog.Errorf("node Cordon update err:%v", err)
- klog.Infof("nodeName:%v\n clusterName:%v\n", nodeName, clusterName)
- } else {
- klog.Info("node Cordon success.")
- klog.Infof("nodeName:%v\n clusterName:%v\n", nodeName, clusterName)
- }
- return err
-}
diff --git a/app/horus/core/horuser/node_downtime.go b/app/horus/core/horuser/node_downtime.go
deleted file mode 100644
index 6572243..0000000
--- a/app/horus/core/horuser/node_downtime.go
+++ /dev/null
@@ -1,191 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more
-// contributor license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright ownership.
-// The ASF licenses this file to You under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance with
-// the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package horuser
-
-import (
- "context"
- "fmt"
- "github.com/apache/dubbo-kubernetes/app/horus/base/db"
- "github.com/apache/dubbo-kubernetes/app/horus/core/alerter"
- metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
- "k8s.io/apimachinery/pkg/util/wait"
- "k8s.io/klog"
- "strings"
- "sync"
- "time"
-)
-
-const (
- NODE_DOWN = "nodeDown"
- NODE_DOWN_REASON = "downtime"
-)
-
-func (h *Horuser) DownTimeManager(ctx context.Context) error {
- go wait.UntilWithContext(ctx, h.DownTimeCheck, time.Duration(h.cc.NodeDownTime.IntervalSecond)*time.Second)
- <-ctx.Done()
- return nil
-}
-
-func (h *Horuser) DownTimeCheck(ctx context.Context) {
- var wg sync.WaitGroup
-
- for clusterName, addr := range h.cc.PromMultiple {
- clusterName := clusterName
- if _, exist := h.cc.NodeDownTime.KubeMultiple[clusterName]; !exist {
- klog.Info("DownTimeCheck config disable.")
- klog.Infof("clusterName:%v\n", clusterName)
- continue
- }
- addr := addr
- wg.Add(1)
- go func() {
- defer wg.Done()
- h.DownTimeNodes(clusterName, addr)
- }()
- }
- wg.Wait()
-}
-
-func (h *Horuser) DownTimeNodes(clusterName, addr string) {
- kubeClient := h.kubeClientMap[clusterName]
- if kubeClient == nil {
- klog.Errorf("DownTimeNodes kubeClient by clusterName empty.")
- return
- }
- ctxFirst, cancelFirst := h.GetK8sContext()
- defer cancelFirst()
-
- klog.Info("DownTimeNodes Query Start.")
- klog.Infof("clusterName:%v\n", clusterName)
-
- nodeDownTimeRes := make(map[string]int)
- aq := len(h.cc.NodeDownTime.AbnormalityQL)
-
- for _, ql := range h.cc.NodeDownTime.AbnormalityQL {
- ql := ql
- res, err := h.InstantQuery(addr, ql, clusterName, h.cc.NodeDownTime.PromQueryTimeSecond)
- if err != nil {
- klog.Errorf("downtimeNodes InstantQuery err:%v", err)
- klog.Infof("clusterName:%v\n", clusterName)
- continue
- }
-
- for _, v := range res {
- v := v
- nodeName := string(v.Metric["node"])
- if nodeName == "" {
- klog.Error("downtimeNodes InstantQuery nodeName empty.")
- klog.Infof("clusterName:%v\n metric:%v\n", clusterName, v.Metric)
- continue
- }
- nodeDownTimeRes[nodeName]++
-
- }
- }
-
- WithDownNodeIPs := make(map[string]string)
-
- for node, count := range nodeDownTimeRes {
- if count < aq {
- klog.Error("downtimeNodes not reach threshold.")
- klog.Infof("clusterName:%v nodeName:%v threshold:%v count:%v", clusterName, node, aq, count)
- continue
- }
- abnormalInfoSystemQL := fmt.Sprintf(h.cc.NodeDownTime.AbnormalInfoSystemQL, node)
-
- res, err := h.InstantQuery(addr, abnormalInfoSystemQL, clusterName, h.cc.NodeDownTime.PromQueryTimeSecond)
- if len(res) == 0 {
- klog.Errorf("no results returned for query:%s", abnormalInfoSystemQL)
- continue
- }
- if err != nil {
- klog.Errorf("downtimeNodes InstantQuery NodeName To IPs empty err:%v", err)
- klog.Infof("clusterName:%v\n AbnormalInfoSystemQL:%v, err:%v", clusterName, abnormalInfoSystemQL, err)
- continue
- }
- instanceIP := ""
- for _, v := range res {
- instanceIP += string(v.Metric["instance"])
- }
- WithDownNodeIPs[node] = instanceIP
- }
-
- msg := fmt.Sprintf("\n【%s】\n【集群：%v】\n【已达到宕机临界点：%v】", h.cc.NodeDownTime.DingTalk.Title, clusterName, len(WithDownNodeIPs))
-
- newfound := 0
-
- for nodeName, _ := range WithDownNodeIPs {
- firstDate := time.Now().Format("2006-01-02")
- err := h.Cordon(nodeName, clusterName, NODE_DOWN)
- if err != nil {
- klog.Errorf("Cordon node err:%v", err)
- return
- }
- klog.Info("Cordon node success.")
-
- klog.Infof("clusterName:%v\n nodeName:%v\n", clusterName, nodeName)
-
- node, err := kubeClient.CoreV1().Nodes().Get(ctxFirst, nodeName, metav1.GetOptions{})
- nodeIP, err := func() (string, error) {
- for _, address := range node.Status.Addresses {
- if address.Type == "InternalIP" {
- return address.Address, nil
- }
- }
- return "", nil
- }()
-
- abnormalRecoveryQL := []string{}
- for _, ql := range h.cc.NodeDownTime.AbnormalRecoveryQL {
- count := strings.Count(ql, "%s")
-
- p := make([]interface{}, count)
- for i := 0; i < count; i++ {
- p[i] = nodeName
- }
-
- query := fmt.Sprintf(ql, p...)
- abnormalRecoveryQL = append(abnormalRecoveryQL, query)
- }
-
- write := db.NodeDataInfo{
- NodeName: nodeName,
- NodeIP: nodeIP,
- ClusterName: clusterName,
- ModuleName: NODE_DOWN,
- DownTimeRecoveryQL: abnormalRecoveryQL,
- }
- exist, _ := write.Check()
- if exist {
- continue
- }
- newfound++
- if newfound > 0 {
- klog.Infof("DownTimeNodes get WithDownNodeIPs【集群:%v】【节点:%v】【IP:%v】", clusterName, nodeName, nodeIP)
- alerter.DingTalkSend(h.cc.NodeDownTime.DingTalk, msg)
- }
- msg += fmt.Sprintf("node:%v ip:%v", nodeName, nodeIP)
- write.Reason = NODE_DOWN_REASON
- write.FirstDate = firstDate
- _, err = write.Add()
- if err != nil {
- klog.Errorf("DownTimeNodes abnormal cordonNode AddOrGet err:%v", err)
- klog.Infof("clusterName:%v nodeName:%v", clusterName, nodeName)
- }
- klog.Info("DownTimeNodes abnormal cordonNode AddOrGet success.")
- klog.Infof("clusterName:%v nodeName:%v", clusterName, nodeName)
- }
-}
diff --git a/app/horus/core/horuser/node_drain.go b/app/horus/core/horuser/node_drain.go
deleted file mode 100644
index b1dcab2..0000000
--- a/app/horus/core/horuser/node_drain.go
+++ /dev/null
@@ -1,66 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more
-// contributor license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright ownership.
-// The ASF licenses this file to You under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance with
-// the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package horuser
-
-import (
- "fmt"
- v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
- "k8s.io/klog/v2"
-)
-
-func (h *Horuser) Drain(nodeName, clusterName string) (err error) {
- kubeClient := h.kubeClientMap[clusterName]
- if kubeClient == nil {
- klog.Error("node Drain kubeClient by clusterName empty.")
- klog.Infof("nodeName:%v\n,clusterName:%v\n", nodeName, clusterName)
- return err
- }
-
- ctxFirst, cancelFirst := h.GetK8sContext()
- defer cancelFirst()
- listOpts := v1.ListOptions{FieldSelector: fmt.Sprintf("spec.nodeName=%s", nodeName)}
- pod, err := kubeClient.CoreV1().Pods("").List(ctxFirst, listOpts)
- if err != nil {
- klog.Errorf("node Drain err:%v", err)
- klog.Infof("nodeName:%v\n clusterName:%v\n", nodeName, clusterName)
- return err
- }
- if len(pod.Items) == 0 {
- klog.Error("Unable to find pod on node..")
- klog.Infof("nodeName:%v,clusterName:%v\n", nodeName, clusterName)
- }
- count := len(pod.Items)
- for items, pods := range pod.Items {
- ds := false
- for _, owner := range pods.OwnerReferences {
- if owner.Kind == "DaemonSet" {
- ds = true
- break
- }
- }
- if ds {
- continue
- }
- klog.Errorf("node Drain evict pod result items:%d count:%v nodeName:%v\n clusterName:%v\n podName:%v\n podNamespace:%v\n", items+1, count, nodeName, clusterName, pods.Name, pods.Namespace)
-
- err = h.Evict(pods.Name, pods.Namespace, clusterName)
- if err != nil {
- klog.Errorf("node Drain evict pod err:%v items:%d count:%v nodeName:%v\n clusterName:%v\n podName:%v\n podNamespace:%v\n", err, items+1, count, nodeName, clusterName, pods.Name, pods.Namespace)
- return err
- }
- }
- return nil
-}
diff --git a/app/horus/core/horuser/node_modular.go b/app/horus/core/horuser/node_modular.go
deleted file mode 100644
index 162fbfc..0000000
--- a/app/horus/core/horuser/node_modular.go
+++ /dev/null
@@ -1,135 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more
-// contributor license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright ownership.
-// The ASF licenses this file to You under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance with
-// the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package horuser
-
-import (
- "context"
- "fmt"
- "github.com/apache/dubbo-kubernetes/app/horus/base/db"
- "github.com/apache/dubbo-kubernetes/app/horus/core/alerter"
- "k8s.io/apimachinery/pkg/util/wait"
- "k8s.io/klog/v2"
- "sync"
- "time"
-)
-
-func (h *Horuser) CustomizeModularManager(ctx context.Context) error {
- go wait.UntilWithContext(ctx, h.CustomizeModular, time.Duration(h.cc.CustomModular.IntervalSecond)*time.Second)
- <-ctx.Done()
- return nil
-}
-
-func (h *Horuser) CustomizeModular(ctx context.Context) {
- var wg sync.WaitGroup
- for clusterName, addr := range h.cc.PromMultiple {
- if _, exists := h.cc.CustomModular.KubeMultiple[clusterName]; !exists {
- klog.Info("CustomizeModular config disable.")
- klog.Infof("clusterName: %v\n", clusterName)
- continue
- }
- wg.Add(1)
- go func(clusterName, addr string) {
- defer wg.Done()
- h.CustomizeModularOnCluster(clusterName, addr)
- }(clusterName, addr)
- }
- wg.Wait()
-}
-
-func (h *Horuser) CustomizeModularOnCluster(clusterName, addr string) {
- klog.Infof("CustomizeModularOnCluster Start clusterName:%v", clusterName)
- for moduleName, abnormalityQL := range h.cc.CustomModular.AbnormalityQL {
- ql := abnormalityQL
- vecs, err := h.InstantQuery(addr, ql, clusterName, h.cc.CustomModular.PromQueryTimeSecond)
- if err != nil {
- klog.Errorf("CustomizeModularOnCluster InstantQuery err:%v", err)
- klog.Infof("clusterName:%v abnormalityQL: %v", clusterName, ql)
- return
- }
- count := len(vecs)
- for index, vec := range vecs {
- vec := vec
- labelMap := vec.Metric
- nodeName := string(labelMap["node"])
- if nodeName == "" {
- klog.Warningf("CustomizeModularOnCluster empty.")
- klog.Infof("clusterName:%v moduleName:%v index:%d", clusterName, moduleName, index)
- continue
- }
- ip := string(labelMap["instance"])
- value := vec.Value.String()
- klog.Infof("CustomizeModularOnCluster Query result clusterName:%v moduleName:%v %d nodeName:%v value:%v count:%v",
- clusterName, moduleName, index+1, nodeName, value, count)
- h.CustomizeModularNodes(clusterName, moduleName, nodeName, ip)
- }
- }
-}
-
-func (h *Horuser) CustomizeModularNodes(clusterName, moduleName, nodeName, ip string) {
- today := time.Now().Format("2006-01-02")
-
- recoveryQL := fmt.Sprintf(h.cc.CustomModular.RecoveryQL[moduleName], nodeName)
- data, err := db.GetDailyLimitNodeDataInfoDate(today, moduleName, clusterName)
- if err != nil {
- klog.Errorf("CustomizeModularNodes GetDailyLimitNodeDataInfoDate err:%v", err)
- return
- }
- err = h.Cordon(nodeName, clusterName, moduleName)
-
- write := db.NodeDataInfo{
- NodeName: nodeName,
- NodeIP: ip,
- ClusterName: clusterName,
- ModuleName: moduleName,
- Reason: moduleName,
- FirstDate: today,
- RecoveryQL: recoveryQL,
- }
-
- pass, _ := write.Check()
- if pass {
- klog.Infof("CustomizeModularNodes already existing clusterName:%v\n nodeName:%v\n moduleName:%v\n", clusterName, nodeName, moduleName)
- return
- }
-
- _, err = write.AddOrGet()
- if err != nil {
- klog.Errorf("CustomizeModularNodes AddOrGet err:%v", err)
- klog.Infof("moduleName:%v nodeName:%v\n", moduleName, write.NodeName)
- }
-
- res := "Success"
- if err != nil {
- res = fmt.Sprintf("result failed:%v", err)
- klog.Errorf("Cordon failed:%v", res)
- }
- msg := fmt.Sprintf("\n【集群:%v】\n【发现 %s 达到禁止调度条件】\n【禁止调度节点:%v】\n 【处理结果: %v】\n 【今日操作次数:%v】\n",
- clusterName, moduleName, nodeName, res, len(data)+1)
- alerter.DingTalkSend(h.cc.CustomModular.DingTalk, msg)
- alerter.SlackSend(h.cc.CustomModular.Slack, msg)
-
- dailyLimit := h.cc.CustomModular.DailyLimit[moduleName]
- if len(data) > dailyLimit {
- msg := fmt.Sprintf("\n【日期:%v】\n【集群:%v\n】\n【今日 Cordon 节点数: %v】\n【已达到今日上限: %v】\n【节点:%v】",
- data, clusterName, len(data), dailyLimit, nodeName)
- alerter.DingTalkSend(h.cc.CustomModular.DingTalk, msg)
- alerter.SlackSend(h.cc.CustomModular.Slack, msg)
- return
- }
-
- klog.Infof("CustomizeModularNodes AddOrGet success.")
- klog.Infof("moduleName:%v nodeName:%v\n", moduleName, write.NodeName)
-}
diff --git a/app/horus/core/horuser/node_recovery.go b/app/horus/core/horuser/node_recovery.go
deleted file mode 100644
index 5343f0e..0000000
--- a/app/horus/core/horuser/node_recovery.go
+++ /dev/null
@@ -1,183 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more
-// contributor license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright ownership.
-// The ASF licenses this file to You under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance with
-// the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package horuser
-
-import (
- "context"
- "fmt"
- "github.com/apache/dubbo-kubernetes/app/horus/base/db"
- "github.com/apache/dubbo-kubernetes/app/horus/core/alerter"
- "github.com/gammazero/workerpool"
- "k8s.io/apimachinery/pkg/util/wait"
- "k8s.io/klog/v2"
- "time"
-)
-
-func (h *Horuser) RecoveryManager(ctx context.Context) error {
- go wait.UntilWithContext(ctx, h.recoveryCheck, time.Duration(h.cc.NodeRecovery.IntervalSecond)*time.Second)
- <-ctx.Done()
- return nil
-}
-
-func (h *Horuser) DownTimeRecoveryManager(ctx context.Context) error {
- go wait.UntilWithContext(ctx, h.downTimeRecoveryCheck, time.Duration(h.cc.NodeRecovery.IntervalSecond)*time.Second)
- <-ctx.Done()
- return nil
-}
-
-func (h *Horuser) recoveryCheck(ctx context.Context) {
- data, err := db.GetRecoveryNodeDataInfoDate(h.cc.NodeRecovery.DayNumber)
- if err != nil {
- klog.Errorf("recovery check GetRecoveryNodeDataInfoDate err:%v", err)
- return
- }
- if len(data) == 0 {
- klog.Info("recovery check GetRecoveryNodeDataInfoDate zero.")
- return
- }
- wp := workerpool.New(50)
- for _, d := range data {
- d := d
- wp.Submit(func() {
- h.recoveryNodes(d)
-
- })
-
- }
- wp.StopWait()
-}
-
-func (h *Horuser) downTimeRecoveryCheck(ctx context.Context) {
- data, err := db.GetDownTimeRecoveryNodeDataInfoDate(h.cc.NodeRecovery.DayNumber)
- if err != nil {
- klog.Errorf("recovery check GetDownTimeRecoveryNodeDataInfoDate err:%v", err)
- return
- }
- if len(data) == 0 {
- klog.Info("recovery check GetDownTimeRecoveryNodeDataInfoDate zero.")
- return
- }
- wp := workerpool.New(50)
- for _, d := range data {
- d := d
- wp.Submit(func() {
- h.downTimeRecoveryNodes(d)
- })
- }
- wp.StopWait()
-}
-
-func (h *Horuser) recoveryNodes(n db.NodeDataInfo) {
- promAddr := h.cc.PromMultiple[n.ClusterName]
- if promAddr == "" {
- klog.Error("recoveryNodes promAddr by clusterName empty.")
- klog.Infof("clusterName:%v nodeName:%v", n.ClusterName, n.NodeName)
- return
- }
- vecs, err := h.InstantQuery(promAddr, n.RecoveryQL, n.ClusterName, h.cc.NodeRecovery.PromQueryTimeSecond)
- if err != nil {
- klog.Errorf("recoveryNodes InstantQuery err:%v", err)
- klog.Infof("recoveryQL:%v", n.RecoveryQL)
- return
- }
- if len(vecs) != 1 {
- klog.Infof("Expected 1 result, but got:%d", len(vecs))
- return
- }
- if err != nil {
- klog.Errorf("recoveryNodes InstantQuery err:%v", err)
- klog.Infof("recoveryQL:%v", n.RecoveryQL)
- return
- }
- klog.Info("recoveryNodes InstantQuery success.")
-
- err = h.UnCordon(n.NodeName, n.ClusterName)
- res := "Success"
- if err != nil {
- res = fmt.Sprintf("result failed:%v", err)
- }
- msg := fmt.Sprintf("\n【集群: %v】\n【封锁节点恢复调度】\n【已恢复调度节点: %v】\n【处理结果：%v】\n【日期: %v】\n", n.ClusterName, n.NodeName, res, n.CreateTime)
- alerter.DingTalkSend(h.cc.NodeRecovery.DingTalk, msg)
- alerter.SlackSend(h.cc.CustomModular.Slack, msg)
-
- success, err := n.RecoveryMarker()
- if err != nil {
- klog.Errorf("RecoveryMarker result failed err:%v", err)
- return
- }
- klog.Infof("RecoveryMarker result success:%v", success)
-}
-
-func (h *Horuser) downTimeRecoveryNodes(n db.NodeDataInfo) {
- promAddr := h.cc.PromMultiple[n.ClusterName]
- if promAddr == "" {
- klog.Error("downTimeRecoveryNodes promAddr by clusterName empty.")
- klog.Infof("clusterName:%v nodeName:%v", n.ClusterName, n.NodeName)
- return
- }
- rq := len(h.cc.NodeDownTime.AbnormalRecoveryQL)
- counter := 0
- for _, ql := range n.DownTimeRecoveryQL {
- vecs, err := h.InstantQuery(promAddr, ql, n.ClusterName, h.cc.NodeRecovery.PromQueryTimeSecond)
- if err != nil {
- klog.Errorf("downTimeRecoveryNodes InstantQuery err: %v", err)
- klog.Infof("DownTimeRecoveryQL: %v", ql)
- continue
- }
- if len(vecs) == 0 {
- klog.Infof("No results for query: %v", ql)
- continue
- } else {
- klog.Infof("Query successful for: %v", ql)
- counter++
- klog.Infof("Counter: %v", counter)
- }
-
- if counter != rq {
- klog.Infof("Expected %d results, but got: %d", rq, counter)
- continue
- }
-
- if counter > rq {
- klog.Error("downTimeRecoveryNodes did not reach threshold.")
- continue
- }
-
- if counter == rq {
- klog.Info("Reaching the downtime recovery threshold.")
- err = h.UnCordon(n.NodeName, n.ClusterName)
- res := "Success"
- if err != nil {
- res = fmt.Sprintf("result failed: %v", err)
- }
-
- msg := fmt.Sprintf("\n【集群: %v】\n【封锁宕机节点恢复调度】\n【已恢复调度节点: %v】\n【处理结果：%v】\n【日期: %v】\n",
- n.ClusterName, n.NodeName, res, n.CreateTime)
-
- alerter.DingTalkSend(h.cc.NodeDownTime.DingTalk, msg)
- alerter.SlackSend(h.cc.NodeDownTime.Slack, msg)
-
- success, err := n.DownTimeRecoveryMarker()
- if err != nil {
- klog.Errorf("DownTimeRecoveryMarker result failed: %v", err)
- return
- }
- klog.Infof("DownTimeRecoveryMarker result success: %v", success)
-
- klog.Info("recoveryNodes InstantQuery success.")
- }
- }
-}
diff --git a/app/horus/core/horuser/node_restart.go b/app/horus/core/horuser/node_restart.go
deleted file mode 100644
index 5c81ed9..0000000
--- a/app/horus/core/horuser/node_restart.go
+++ /dev/null
@@ -1,117 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more
-// contributor license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright ownership.
-// The ASF licenses this file to You under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance with
-// the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package horuser
-
-import (
- "context"
- "fmt"
- "github.com/apache/dubbo-kubernetes/app/horus/base/db"
- "github.com/apache/dubbo-kubernetes/app/horus/core/alerter"
- "github.com/gammazero/workerpool"
- "k8s.io/apimachinery/pkg/util/wait"
- "k8s.io/klog/v2"
- "os/exec"
- "strings"
- "time"
-)
-
-func (h *Horuser) DowntimeRestartManager(ctx context.Context) error {
- go wait.UntilWithContext(ctx, h.RestartOrRepair, time.Duration(h.cc.NodeDownTime.IntervalSecond)*time.Second)
- <-ctx.Done()
- return nil
-}
-
-func (h *Horuser) RestartOrRepair(ctx context.Context) {
- nodes, err := db.GetRestartNodeDataInfoDate()
- if err != nil {
- klog.Errorf("RestartOrRepair GetRestartNodeDataInfoDate err:%v", err)
- return
- }
- if len(nodes) == 0 {
- klog.Info("RestartOrRepair to zero.")
- }
- klog.Infof("RestartOrRepair Count:%v", len(nodes))
- wp := workerpool.New(10)
- for _, n := range nodes {
- n := n
- wp.Submit(func() {
- h.TryRestart(n)
- })
- }
-}
-
-func (h *Horuser) TryRestart(node db.NodeDataInfo) {
- err := h.Drain(node.NodeName, node.ClusterName)
- if err != nil {
- klog.Errorf("Drain node err:%v", err)
- return
- }
- klog.Info("Drain node success.")
- klog.Infof("clusterName:%v\n nodeName:%v\n", node.ClusterName, node.NodeName)
-
- success, err := node.RestartMarker()
- if err != nil {
- klog.Errorf("Error getting RestartMarker for nodeName:%v: err:%v", node.NodeName, err)
- return
- }
- klog.Infof("RestartMarker result success:%v", success)
-
- if success {
- msg := fmt.Sprintf("\n【宕机节点等待腾空后重启】\n【节点:%v】\n【日期:%v】\n【集群:%v】\n", node.NodeName, node.FirstDate, node.ClusterName)
- alerter.DingTalkSend(h.cc.NodeDownTime.DingTalk, msg)
-
- cmd := exec.Command("/bin/bash", "core/horuser/restart.sh", node.NodeIP, h.cc.NodeDownTime.AllSystemUser, h.cc.NodeDownTime.AllSystemPassword)
- output, err := cmd.CombinedOutput()
- if err != nil {
- klog.Errorf("Failed restart for Output:%v node:%v: err:%v", string(output), node.NodeName, err)
- return
- }
- klog.Infof("Successfully restarted node %v.", node.NodeName)
-
- rq := len(h.cc.NodeDownTime.AbnormalRecoveryQL)
- ql := strings.Join(h.cc.NodeDownTime.AbnormalRecoveryQL, " or ")
- vecs, err := h.InstantQuery(h.cc.PromMultiple[node.ClusterName], ql, node.ClusterName, h.cc.NodeDownTime.PromQueryTimeSecond)
- if err != nil {
- klog.Errorf("Failed to query Prometheus for recovery threshold after restart: %v", err)
- return
- }
-
- if len(vecs) == rq {
- klog.Infof("Node %v has reached recovery threshold after restart.", node.NodeName)
-
- err = h.UnCordon(node.NodeName, node.ClusterName)
- if err != nil {
- klog.Errorf("Uncordon node failed after restart: %v", err)
- return
- }
-
- msg = fmt.Sprintf("\n【集群: %v】\n【宕机节点已恢复】\n【恢复节点: %v】\n【处理结果：成功】\n【日期: %v】\n", node.ClusterName, node.NodeName, node.FirstDate)
- alerter.DingTalkSend(h.cc.NodeDownTime.DingTalk, msg)
- alerter.SlackSend(h.cc.NodeDownTime.Slack, msg)
-
- } else {
- klog.Infof("Node %v has not reached recovery threshold after restart.", node.NodeName)
- }
-
- } else {
- klog.Infof("RestartMarker did not success for node %v", node.NodeName)
- }
-
- if node.Restart > 2 {
- klog.Error("The node has already been rebooted more than twice.")
- return
- }
-}
diff --git a/app/horus/core/horuser/node_uncordon.go b/app/horus/core/horuser/node_uncordon.go
deleted file mode 100644
index b6f9cc1..0000000
--- a/app/horus/core/horuser/node_uncordon.go
+++ /dev/null
@@ -1,53 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more
-// contributor license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright ownership.
-// The ASF licenses this file to You under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance with
-// the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package horuser
-
-import (
- v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
- "k8s.io/klog/v2"
-)
-
-func (h *Horuser) UnCordon(nodeName, clusterName string) (err error) {
- kubeClient := h.kubeClientMap[clusterName]
- if kubeClient == nil {
- klog.Error("node UnCordon kubeClient by clusterName empty.")
- klog.Infof("nodeName:%v\n,clusterName:%v\n", nodeName, clusterName)
- return err
- }
-
- ctxFirst, cancelFirst := h.GetK8sContext()
- defer cancelFirst()
- node, err := kubeClient.CoreV1().Nodes().Get(ctxFirst, nodeName, v1.GetOptions{})
- if err != nil {
- klog.Errorf("node UnCordon get err:%v", err)
- klog.Infof("nodeName:%v\n clusterName:%v\n", nodeName, clusterName)
- return err
- }
-
- node.Spec.Unschedulable = false
-
- ctxSecond, cancelSecond := h.GetK8sContext()
- defer cancelSecond()
- node, err = kubeClient.CoreV1().Nodes().Update(ctxSecond, node, v1.UpdateOptions{})
- if err != nil {
- klog.Errorf("node UnCordon update err:%v", err)
- klog.Infof("nodeName:%v\n clusterName:%v\n", nodeName, clusterName)
- return err
- }
- klog.Info("node UnCordon success.")
- klog.Infof("nodeName:%v\n clusterName:%v\n", nodeName, clusterName)
- return nil
-}
diff --git a/app/horus/core/horuser/pod_evict.go b/app/horus/core/horuser/pod_evict.go
deleted file mode 100644
index 573d227..0000000
--- a/app/horus/core/horuser/pod_evict.go
+++ /dev/null
@@ -1,56 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more
-// contributor license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright ownership.
-// The ASF licenses this file to You under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance with
-// the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package horuser
-
-import (
- v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
- "k8s.io/klog/v2"
-)
-
-func (h *Horuser) Evict(podName, podNamespace, clusterName string) (err error) {
- kubeClient := h.kubeClientMap[clusterName]
- if kubeClient == nil {
- klog.Error("pod Evict kubeClient by clusterName empty.")
- klog.Infof("podName:%v\n podNamespace:%v\n clusterName:%v\n", podName, podNamespace, clusterName)
- return err
- }
-
- ctxFirst, cancelFirst := h.GetK8sContext()
- defer cancelFirst()
- _, err = kubeClient.CoreV1().Pods(podNamespace).Get(ctxFirst, podName, v1.GetOptions{})
- if err != nil {
- klog.Errorf("pod Evict get err:%v", err)
- klog.Infof("clusterName:%v\n podName:%v\n podNamespace:%v\n", clusterName, podName, podNamespace)
- return err
- }
- ctxSecond, cancelSecond := h.GetK8sContext()
- defer cancelSecond()
- var gracePeriodSeconds int64 = -1
- propagationPolicy := v1.DeletePropagationBackground
- err = kubeClient.CoreV1().Pods(podNamespace).Delete(ctxSecond, podName, v1.DeleteOptions{
- GracePeriodSeconds: &gracePeriodSeconds,
- PropagationPolicy: &propagationPolicy,
- })
- if err != nil {
- klog.Errorf("pod Evict delete failed err:%v", err)
- klog.Infof("clusterName:%v\n podName:%v\n podNamespace:%v\n", clusterName, podName, podNamespace)
- return err
- }
- klog.Info("pod Evict delete success.")
- klog.Infof("clusterName:%v\n podName:%v\n podNamespace:%v\n", clusterName, podName, podNamespace)
- return nil
-
-}
diff --git a/app/horus/core/horuser/pod_remove.go b/app/horus/core/horuser/pod_remove.go
deleted file mode 100644
index 91251bb..0000000
--- a/app/horus/core/horuser/pod_remove.go
+++ /dev/null
@@ -1,86 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more
-// contributor license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright ownership.
-// The ASF licenses this file to You under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance with
-// the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package horuser
-
-import (
- "context"
- "encoding/json"
- corev1 "k8s.io/api/core/v1"
- v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
- "k8s.io/apimachinery/pkg/types"
- "k8s.io/klog/v2"
-)
-
-type RemoveJsonValue struct {
- Op string `json:"op"`
- Path string `json:"path"`
-}
-
-func (h *Horuser) Finalizer(clusterName, podName, podNamespace string) error {
- kubeClient := h.kubeClientMap[clusterName]
- if kubeClient == nil {
- klog.Error("Finalizer kubeClient by clusterName empty.")
- klog.Infof("clusterName:%v\n podName:%v\n", clusterName, podName)
- return nil
- }
- finalizer := RemoveJsonValue{
- Op: "remove",
- Path: "/metadata/finalizers",
- }
- var payload []interface{}
- payload = append(payload, finalizer)
- data, _ := json.Marshal(payload)
- ctx, cancel := h.GetK8sContext()
- defer cancel()
- _, err := kubeClient.CoreV1().Pods(podNamespace).Patch(ctx, podName, types.JSONPatchType, data, v1.PatchOptions{})
- return err
-}
-
-func (h *Horuser) Terminating(clusterName string, oldPod *corev1.Pod) bool {
- kubeClient := h.kubeClientMap[clusterName]
- if kubeClient == nil {
- return false
- }
- newPod, _ := kubeClient.CoreV1().Pods(oldPod.Namespace).Get(context.Background(), oldPod.Name, v1.GetOptions{})
- if newPod == nil {
- return false
- }
- if newPod.UID != oldPod.UID {
- return false
- }
- if newPod.DeletionTimestamp.IsZero() {
- return false
- }
- return true
-}
-
-func (h *Horuser) Retrieve(clusterName, fieldSelector string) ([]corev1.Pod, error) {
- kubeClient := h.kubeClientMap[clusterName]
- if kubeClient == nil {
- klog.Error("Retrieve kubeClient by clusterName empty.")
- klog.Infof("clusterName:%v\n", clusterName)
- return nil, nil
- }
- ctx, cancel := h.GetK8sContext()
- defer cancel()
- list := v1.ListOptions{FieldSelector: fieldSelector}
- pods, err := kubeClient.CoreV1().Pods("").List(ctx, list)
- if err != nil {
- klog.Errorf("Retrieve list pod err:%v", err)
- klog.Infof("clusterName:%v\n fieldSelector:%v", clusterName, fieldSelector)
- }
- return pods.Items, err
-}
diff --git a/app/horus/core/horuser/pod_stagnation.go b/app/horus/core/horuser/pod_stagnation.go
deleted file mode 100644
index 0f648c1..0000000
--- a/app/horus/core/horuser/pod_stagnation.go
+++ /dev/null
@@ -1,129 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more
-// contributor license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright ownership.
-// The ASF licenses this file to You under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance with
-// the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package horuser
-
-import (
- "context"
- "fmt"
- "github.com/apache/dubbo-kubernetes/app/horus/base/db"
- "github.com/apache/dubbo-kubernetes/app/horus/core/alerter"
- "github.com/gammazero/workerpool"
- corev1 "k8s.io/api/core/v1"
- "k8s.io/apimachinery/pkg/util/wait"
- "k8s.io/klog/v2"
- "sync"
- "time"
-)
-
-const (
- ModuleName = "podStagnationCleaner"
- Reason = "StagnationCleanup"
-)
-
-func (h *Horuser) PodStagnationCleanManager(ctx context.Context) error {
- go wait.UntilWithContext(ctx, h.PodStagnationClean, time.Duration(h.cc.PodStagnationCleaner.IntervalSecond)*time.Second)
- <-ctx.Done()
- return nil
-}
-
-func (h *Horuser) PodStagnationClean(ctx context.Context) {
- var wg sync.WaitGroup
- for cn := range h.cc.PodStagnationCleaner.KubeMultiple {
- cn := cn
- wg.Add(1)
- go func() {
- defer wg.Done()
- h.PodsOnCluster(cn)
- }()
- }
- wg.Wait()
-}
-
-func (h *Horuser) PodsOnCluster(clusterName string) {
- pods, err := h.Retrieve(clusterName, h.cc.PodStagnationCleaner.FieldSelector)
- if err != nil {
- klog.Errorf("Failed to retrieve pods on err:%v", err)
- klog.Infof("clusterName:%v\n", clusterName)
- return
- }
- count := len(pods)
- if count == 0 {
- klog.Infof("PodsOnCluster no abnomal clusterName:%v\n", clusterName)
- return
- }
- wp := workerpool.New(10)
- for index, pod := range pods {
- pod := pod
- if pod.Status.Phase == corev1.PodRunning {
- continue
- }
- msg := fmt.Sprintf("\n【集群：%v】\n【停滞：%d/%d】\n【PodName:%v】\n【Namespace:%v】\n【Phase:%v】\n【节点：%v】\n", clusterName, index+1, count, pod.Name, pod.Namespace, pod.Status.Phase, pod.Spec.NodeName)
- klog.Infof(msg)
-
- wp.Submit(func() {
- h.PodSingle(pod, clusterName)
- })
- }
- wp.StopWait()
-}
-
-func (h *Horuser) PodSingle(pod corev1.Pod, clusterName string) {
- var err error
- if !pod.DeletionTimestamp.IsZero() {
- if len(pod.Finalizers) > 0 {
- time.Sleep(time.Duration(h.cc.PodStagnationCleaner.DoubleSecond) * time.Second)
- if !h.Terminating(clusterName, &pod) {
- klog.Infof("Pod %s is still terminating skipping.", pod.Name)
- return
- }
- err := h.Finalizer(clusterName, pod.Name, pod.Namespace)
- if err != nil {
- klog.Errorf("Failed to patch finalizer for pod %s: %v", pod.Name, err)
- return
- }
- klog.Infof("Successfully patched finalizer for pod %s", pod.Name)
- }
- return
- }
-
- if len(pod.Finalizers) == 0 && pod.Name != "" {
- err := h.Evict(pod.Name, pod.Namespace, clusterName)
- if err != nil {
- klog.Errorf("Failed to evict pod %s err:%v", pod.Name, err)
- return
- }
- klog.Infof("Evicted pod %s successfully", pod.Name)
- }
- res := "Success"
- if err != nil {
- res = fmt.Sprintf("result failed:%v", err)
- }
- today := time.Now().Format("2006-01-02")
- msg := fmt.Sprintf("\n【集群：%v】\n【Pod：%v】\n【Namespace：%v】\n【清除 finalizer:%v】\n", clusterName, pod.Name, pod.Namespace, res)
- alerter.DingTalkSend(h.cc.PodStagnationCleaner.DingTalk, msg)
- write := db.PodDataInfo{
- PodName: pod.Name,
- PodIP: pod.Status.PodIP,
- NodeName: pod.Spec.NodeName,
- ClusterName: clusterName,
- ModuleName: ModuleName,
- Reason: Reason,
- FirstDate: today,
- }
- _, err = write.AddOrGet()
- klog.Errorf("write AddOrGet err:%v", err)
- return
-}
diff --git a/app/horus/core/horuser/query.go b/app/horus/core/horuser/query.go
deleted file mode 100644
index 3147cdd..0000000
--- a/app/horus/core/horuser/query.go
+++ /dev/null
@@ -1,56 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more
-// contributor license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright ownership.
-// The ASF licenses this file to You under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance with
-// the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package horuser
-
-import (
- "context"
- apiV1 "github.com/prometheus/client_golang/api"
- prometheusV1 "github.com/prometheus/client_golang/api/prometheus/v1"
- "github.com/prometheus/common/model"
- "k8s.io/klog/v2"
- "time"
-)
-
-func (h *Horuser) InstantQuery(address, ql, clusterName string, timeWindowsSecond int64) (model.Vector, error) {
- client, err := apiV1.NewClient(apiV1.Config{Address: address})
- if err != nil {
- klog.Errorf("InstantQuery creating NewClient err:%v", err)
- return nil, err
- }
- promClient := h.cc.PromMultiple[clusterName]
- if promClient == "" && address == "" {
- klog.Error("PromMultiple empty.")
- klog.Infof("clusterName:%v\n ql:%v", clusterName, ql)
- return nil, err
- }
-
- apiV1 := prometheusV1.NewAPI(client)
- ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeWindowsSecond)*time.Second)
- defer cancel()
-
- result, _, err := apiV1.Query(ctx, ql, time.Now())
- if err != nil {
- klog.Errorf("prometheus Query error:%v url:%+v ql:%v", err, address, ql)
- return nil, err
- }
-
- vector, has := result.(model.Vector)
- if !has {
- klog.Errorf("prometheus Query result vector error %+v", result)
- return nil, err
- }
- return vector, nil
-}
diff --git a/app/horus/core/horuser/restart.sh b/app/horus/core/horuser/restart.sh
deleted file mode 100644
index 1523b68..0000000
--- a/app/horus/core/horuser/restart.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-host_computer=$1
-host_username=$2
-host_password=$3
-
-if [ $# -lt 3 ]; then
- echo "ERROR: The expected value is parameter 3, but there are only $#: input parameters."
- exit 1
-fi
-
-for i in $host_computer; do
- sshpass -p "$host_password" ssh "$host_username"@$i -o "StrictHostKeyChecking=no" "echo $host_password | sudo -S reboot"
-done
diff --git a/app/horus/core/ticker/ticker.go b/app/horus/core/ticker/ticker.go
deleted file mode 100644
index eb90edf..0000000
--- a/app/horus/core/ticker/ticker.go
+++ /dev/null
@@ -1,34 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more
-// contributor license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright ownership.
-// The ASF licenses this file to You under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance with
-// the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package ticker
-
-import (
- "context"
- "k8s.io/apimachinery/pkg/util/wait"
- "k8s.io/klog/v2"
- "time"
-)
-
-func Manager(ctx context.Context) error {
- go wait.UntilWithContext(ctx, tickerFunc, 30*time.Second)
- <-ctx.Done()
- klog.Infof("ticker Manager exit receive quit signal")
- return nil
-}
-
-func tickerFunc(ctx context.Context) {
- klog.V(2).Infof("tickerFunc Running")
-}
diff --git a/manifests/horus/horus.yaml b/manifests/horus/horus.yaml
deleted file mode 100644
index c6f48ee..0000000
--- a/manifests/horus/horus.yaml
+++ /dev/null
@@ -1,107 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-address: 0.0.0.0:38089
-kubeTimeSecond: 5
-
-mysql:
- name: horus
- address: "root:root@tcp(127.0.0.1:3306)/horus?charset=utf8&parseTime=True"
- debug: false
-
-kubeMultiple:
- cluster: config.1
-
-promMultiple:
- cluster: http://192.168.15.133:31974
-
-nodeRecovery:
- dayNumber: 1
- intervalSecond: 15
- promQueryTimeSecond: 60
- dingTalk:
- webhookUrl: "https://oapi.dingtalk.com/robot/send?access_token=37f8891e60e524013275cc01efafdb5976b81ef7269ce271b769bcd025826c12"
- title: "自定义通知"
- atMobiles:
- - 15000000000
- slack:
- webhookUrl: "https://hooks.slack.com/services/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"
- title: "自定义通知"
-
-customModular:
- enabled: false
- dailyLimit:
- node_cpu: 1
- abnormalityQL:
- node_cpu: |-
- 100 - (avg by (node) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 15
- recoveryQL:
- node_cpu: |-
- 100 - (avg by (node) (rate(node_cpu_seconds_total{mode="idle",node="%s"}[5m])) * 100) < 10
- intervalSecond: 15
- promQueryTimeSecond: 60
- kubeMultiple:
- cluster: config.1
- dingTalk:
- webhookUrl: "https://oapi.dingtalk.com/robot/send?access_token=37f8891e60e524013275cc01efafdb5976b81ef7269ce271b769bcd025826c12"
- title: "自定义通知"
- atMobiles:
- - 15000000000
- slack:
- webhookUrl: "https://hooks.slack.com/services/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"
- title: "自定义通知"
-
-nodeDownTime:
- enabled: true
- intervalSecond: 15
- promQueryTimeSecond: 60
- abnormalityQL:
- - 100 - (avg by (node) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 15
- - node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"} * 100 < 16
- # - avg(node_memory_MemFree_bytes) by (node) < 20
- abnormalInfoSystemQL:
- node_os_info{node="%s"}
- allSystemUser: "zxj"
- allSystemPassword: "1"
- abnormalRecoveryQL:
- - 100 - (avg by (node) (rate(node_cpu_seconds_total{mode="idle",node="%s"}[5m])) * 100) < 15
- - node_filesystem_avail_bytes{mountpoint="/",node="%s"} / node_filesystem_size_bytes{mountpoint="/",node="%s"} * 100 > 15
- # - (avg by (node) (node_memory_MemFree_bytes{node="%s"} / node_memory_MemTotal_bytes{node="%s"} )) * 100 > 50
- kubeMultiple:
- cluster: config.1
- dingTalk:
- webhookUrl: "https://oapi.dingtalk.com/robot/send?access_token=37f8891e60e524013275cc01efafdb5976b81ef7269ce271b769bcd025826c12"
- title: "自定义通知"
- atMobiles:
- - 15000000000
- slack:
- webhookUrl: "https://hooks.slack.com/services/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"
- title: "自定义通知"
-
-podStagnationCleaner:
- enabled: false
- intervalSecond: 15
- doubleSecond: 60
- fieldSelector: "status.phase!=Running"
- kubeMultiple:
- cluster: config.1
- dingTalk:
- webhookUrl: "https://oapi.dingtalk.com/robot/send?access_token=37f8891e60e524013275cc01efafdb5976b81ef7269ce271b769bcd025826c12"
- title: "自定义通知"
- atMobiles:
- - 15000000000
- slack:
- webhookUrl: "https://hooks.slack.com/services/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"
- title: "自定义通知"
\ No newline at end of file
