init new empty branch feature.
diff --git a/admin/admin.go b/admin/admin.go
deleted file mode 100644
index 06908f4..0000000
--- a/admin/admin.go
+++ /dev/null
@@ -1,210 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package admin
-
-import (
- "context"
- "fmt"
- "sync"
- "time"
-
- "github.com/apache/rocketmq-client-go/v2/internal"
- "github.com/apache/rocketmq-client-go/v2/internal/remote"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/rlog"
-)
-
-type Admin interface {
- CreateTopic(ctx context.Context, opts ...OptionCreate) error
- DeleteTopic(ctx context.Context, opts ...OptionDelete) error
- //TODO
- //TopicList(ctx context.Context, mq *primitive.MessageQueue) (*remote.RemotingCommand, error)
- //GetBrokerClusterInfo(ctx context.Context) (*remote.RemotingCommand, error)
- Close() error
-}
-
-// TODO: move outdated context to ctx
-type adminOptions struct {
- internal.ClientOptions
-}
-
-type AdminOption func(options *adminOptions)
-
-func defaultAdminOptions() *adminOptions {
- opts := &adminOptions{
- ClientOptions: internal.DefaultClientOptions(),
- }
- opts.GroupName = "TOOLS_ADMIN"
- opts.InstanceName = time.Now().String()
- return opts
-}
-
-// WithResolver nameserver resolver to fetch nameserver addr
-func WithResolver(resolver primitive.NsResolver) AdminOption {
- return func(options *adminOptions) {
- options.Resolver = resolver
- }
-}
-
-type admin struct {
- cli internal.RMQClient
-
- opts *adminOptions
-
- closeOnce sync.Once
-}
-
-// NewAdmin initialize admin
-func NewAdmin(opts ...AdminOption) (Admin, error) {
- defaultOpts := defaultAdminOptions()
- for _, opt := range opts {
- opt(defaultOpts)
- }
- namesrv, err := internal.NewNamesrv(defaultOpts.Resolver)
- defaultOpts.Namesrv = namesrv
- if err != nil {
- return nil, err
- }
-
- cli := internal.GetOrNewRocketMQClient(defaultOpts.ClientOptions, nil)
- if cli == nil {
- return nil, fmt.Errorf("GetOrNewRocketMQClient faild")
- }
- defaultOpts.Namesrv = cli.GetNameSrv()
- //log.Printf("Client: %#v", namesrv.srvs)
- return &admin{
- cli: cli,
- opts: defaultOpts,
- }, nil
-}
-
-// CreateTopic create topic.
-// TODO: another implementation like sarama, without brokerAddr as input
-func (a *admin) CreateTopic(ctx context.Context, opts ...OptionCreate) error {
- cfg := defaultTopicConfigCreate()
- for _, apply := range opts {
- apply(&cfg)
- }
-
- request := &internal.CreateTopicRequestHeader{
- Topic: cfg.Topic,
- DefaultTopic: cfg.DefaultTopic,
- ReadQueueNums: cfg.ReadQueueNums,
- WriteQueueNums: cfg.WriteQueueNums,
- Perm: cfg.Perm,
- TopicFilterType: cfg.TopicFilterType,
- TopicSysFlag: cfg.TopicSysFlag,
- Order: cfg.Order,
- }
-
- cmd := remote.NewRemotingCommand(internal.ReqCreateTopic, request, nil)
- _, err := a.cli.InvokeSync(ctx, cfg.BrokerAddr, cmd, 5*time.Second)
- if err != nil {
- rlog.Error("create topic error", map[string]interface{}{
- rlog.LogKeyTopic: cfg.Topic,
- rlog.LogKeyBroker: cfg.BrokerAddr,
- rlog.LogKeyUnderlayError: err,
- })
- } else {
- rlog.Info("create topic success", map[string]interface{}{
- rlog.LogKeyTopic: cfg.Topic,
- rlog.LogKeyBroker: cfg.BrokerAddr,
- })
- }
- return err
-}
-
-// DeleteTopicInBroker delete topic in broker.
-func (a *admin) deleteTopicInBroker(ctx context.Context, topic string, brokerAddr string) (*remote.RemotingCommand, error) {
- request := &internal.DeleteTopicRequestHeader{
- Topic: topic,
- }
-
- cmd := remote.NewRemotingCommand(internal.ReqDeleteTopicInBroker, request, nil)
- return a.cli.InvokeSync(ctx, brokerAddr, cmd, 5*time.Second)
-}
-
-// DeleteTopicInNameServer delete topic in nameserver.
-func (a *admin) deleteTopicInNameServer(ctx context.Context, topic string, nameSrvAddr string) (*remote.RemotingCommand, error) {
- request := &internal.DeleteTopicRequestHeader{
- Topic: topic,
- }
-
- cmd := remote.NewRemotingCommand(internal.ReqDeleteTopicInNameSrv, request, nil)
- return a.cli.InvokeSync(ctx, nameSrvAddr, cmd, 5*time.Second)
-}
-
-// DeleteTopic delete topic in both broker and nameserver.
-func (a *admin) DeleteTopic(ctx context.Context, opts ...OptionDelete) error {
- cfg := defaultTopicConfigDelete()
- for _, apply := range opts {
- apply(&cfg)
- }
- //delete topic in broker
- if cfg.BrokerAddr == "" {
- a.cli.GetNameSrv().UpdateTopicRouteInfo(cfg.Topic)
- cfg.BrokerAddr = a.cli.GetNameSrv().FindBrokerAddrByTopic(cfg.Topic)
- }
-
- if _, err := a.deleteTopicInBroker(ctx, cfg.Topic, cfg.BrokerAddr); err != nil {
- rlog.Error("delete topic in broker error", map[string]interface{}{
- rlog.LogKeyTopic: cfg.Topic,
- rlog.LogKeyBroker: cfg.BrokerAddr,
- rlog.LogKeyUnderlayError: err,
- })
- return err
- }
-
- //delete topic in nameserver
- if len(cfg.NameSrvAddr) == 0 {
- a.cli.GetNameSrv().UpdateTopicRouteInfo(cfg.Topic)
- cfg.NameSrvAddr = a.cli.GetNameSrv().AddrList()
- _, _, err := a.cli.GetNameSrv().UpdateTopicRouteInfo(cfg.Topic)
- if err != nil {
- rlog.Error("delete topic in nameserver error", map[string]interface{}{
- rlog.LogKeyTopic: cfg.Topic,
- rlog.LogKeyUnderlayError: err,
- })
- }
- cfg.NameSrvAddr = a.cli.GetNameSrv().AddrList()
- }
-
- for _, nameSrvAddr := range cfg.NameSrvAddr {
- if _, err := a.deleteTopicInNameServer(ctx, cfg.Topic, nameSrvAddr); err != nil {
- rlog.Error("delete topic in nameserver error", map[string]interface{}{
- "nameServer": nameSrvAddr,
- rlog.LogKeyTopic: cfg.Topic,
- rlog.LogKeyUnderlayError: err,
- })
- return err
- }
- }
- rlog.Info("delete topic success", map[string]interface{}{
- "nameServer": cfg.NameSrvAddr,
- rlog.LogKeyTopic: cfg.Topic,
- rlog.LogKeyBroker: cfg.BrokerAddr,
- })
- return nil
-}
-
-func (a *admin) Close() error {
- a.closeOnce.Do(func() {
- a.cli.Shutdown()
- })
- return nil
-}
diff --git a/admin/option.go b/admin/option.go
deleted file mode 100644
index d5a648e..0000000
--- a/admin/option.go
+++ /dev/null
@@ -1,131 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package admin
-
-func defaultTopicConfigCreate() TopicConfigCreate {
- opts := TopicConfigCreate{
- DefaultTopic: "defaultTopic",
- ReadQueueNums: 8,
- WriteQueueNums: 8,
- Perm: 6,
- TopicFilterType: "SINGLE_TAG",
- TopicSysFlag: 0,
- Order: false,
- }
- return opts
-}
-
-type TopicConfigCreate struct {
- Topic string
- BrokerAddr string
- DefaultTopic string
- ReadQueueNums int
- WriteQueueNums int
- Perm int
- TopicFilterType string
- TopicSysFlag int
- Order bool
-}
-
-type OptionCreate func(*TopicConfigCreate)
-
-func WithTopicCreate(Topic string) OptionCreate {
- return func(opts *TopicConfigCreate) {
- opts.Topic = Topic
- }
-}
-
-func WithBrokerAddrCreate(BrokerAddr string) OptionCreate {
- return func(opts *TopicConfigCreate) {
- opts.BrokerAddr = BrokerAddr
- }
-}
-
-func WithReadQueueNums(ReadQueueNums int) OptionCreate {
- return func(opts *TopicConfigCreate) {
- opts.ReadQueueNums = ReadQueueNums
- }
-}
-
-func WithWriteQueueNums(WriteQueueNums int) OptionCreate {
- return func(opts *TopicConfigCreate) {
- opts.WriteQueueNums = WriteQueueNums
- }
-}
-
-func WithPerm(Perm int) OptionCreate {
- return func(opts *TopicConfigCreate) {
- opts.Perm = Perm
- }
-}
-
-func WithTopicFilterType(TopicFilterType string) OptionCreate {
- return func(opts *TopicConfigCreate) {
- opts.TopicFilterType = TopicFilterType
- }
-}
-
-func WithTopicSysFlag(TopicSysFlag int) OptionCreate {
- return func(opts *TopicConfigCreate) {
- opts.TopicSysFlag = TopicSysFlag
- }
-}
-
-func WithOrder(Order bool) OptionCreate {
- return func(opts *TopicConfigCreate) {
- opts.Order = Order
- }
-}
-
-func defaultTopicConfigDelete() TopicConfigDelete {
- opts := TopicConfigDelete{}
- return opts
-}
-
-type TopicConfigDelete struct {
- Topic string
- ClusterName string
- NameSrvAddr []string
- BrokerAddr string
-}
-
-type OptionDelete func(*TopicConfigDelete)
-
-func WithTopicDelete(Topic string) OptionDelete {
- return func(opts *TopicConfigDelete) {
- opts.Topic = Topic
- }
-}
-
-func WithBrokerAddrDelete(BrokerAddr string) OptionDelete {
- return func(opts *TopicConfigDelete) {
- opts.BrokerAddr = BrokerAddr
- }
-}
-
-func WithClusterName(ClusterName string) OptionDelete {
- return func(opts *TopicConfigDelete) {
- opts.ClusterName = ClusterName
- }
-}
-
-func WithNameSrvAddr(NameSrvAddr []string) OptionDelete {
- return func(opts *TopicConfigDelete) {
- opts.NameSrvAddr = NameSrvAddr
- }
-}
diff --git a/api.go b/api.go
deleted file mode 100644
index 31f58d5..0000000
--- a/api.go
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package rocketmq
-
-import (
- "context"
- "github.com/apache/rocketmq-client-go/v2/errors"
-
- "github.com/apache/rocketmq-client-go/v2/consumer"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/producer"
-)
-
-type Producer interface {
- Start() error
- Shutdown() error
- SendSync(ctx context.Context, mq ...*primitive.Message) (*primitive.SendResult, error)
- SendAsync(ctx context.Context, mq func(ctx context.Context, result *primitive.SendResult, err error),
- msg ...*primitive.Message) error
- SendOneWay(ctx context.Context, mq ...*primitive.Message) error
-}
-
-func NewProducer(opts ...producer.Option) (Producer, error) {
- return producer.NewDefaultProducer(opts...)
-}
-
-type TransactionProducer interface {
- Start() error
- Shutdown() error
- SendMessageInTransaction(ctx context.Context, mq *primitive.Message) (*primitive.TransactionSendResult, error)
-}
-
-func NewTransactionProducer(listener primitive.TransactionListener, opts ...producer.Option) (TransactionProducer, error) {
- return producer.NewTransactionProducer(listener, opts...)
-}
-
-type PushConsumer interface {
- // Start the PullConsumer for consuming message
- Start() error
-
- // Shutdown the PullConsumer, all offset of MessageQueue will be sync to broker before process exit
- Shutdown() error
- // Subscribe a topic for consuming
- Subscribe(topic string, selector consumer.MessageSelector,
- f func(context.Context, ...*primitive.MessageExt) (consumer.ConsumeResult, error)) error
-
- // Unsubscribe a topic
- Unsubscribe(topic string) error
-}
-
-func NewPushConsumer(opts ...consumer.Option) (PushConsumer, error) {
- return consumer.NewPushConsumer(opts...)
-}
-
-type PullConsumer interface {
- // Start the PullConsumer for consuming message
- Start() error
-
- // Shutdown the PullConsumer, all offset of MessageQueue will be commit to broker before process exit
- Shutdown() error
-
- // Subscribe a topic for consuming
- Subscribe(topic string, selector consumer.MessageSelector) error
-
- // Unsubscribe a topic
- Unsubscribe(topic string) error
-
- // MessageQueues get MessageQueue list about for a given topic. This method will issue a remote call to the server
- // if it does not already have any MessageQueue about the given topic.
- MessageQueues(topic string) []primitive.MessageQueue
-
- // Pull message for the topic specified. It is an error to not have subscribed to any topics before pull for message
- //
- // Specified numbers of messages is returned if message greater that numbers, and the offset will auto forward.
- // It means that if you meeting messages consuming failed, you should process failed messages by yourself.
- Pull(ctx context.Context, topic string, numbers int) (*primitive.PullResult, error)
-
- // Pull message for the topic specified from a specified MessageQueue and offset. It is an error to not have
- // subscribed to any topics before pull for message. the method will not affect the offset recorded
- //
- // Specified numbers of messages is returned.
- PullFrom(ctx context.Context, mq primitive.MessageQueue, offset int64, numbers int) (*primitive.PullResult, error)
-
- // Lookup offset for the given message queue by timestamp. The returned offset for the message queue is the
- // earliest offset whose timestamp is greater than or equal to the given timestamp in the corresponding message
- // queue.
- //
- // Timestamp must be millisecond level, if you want to lookup the earliest offset of the mq, you could set the
- // timestamp 0, and if you want to the latest offset the mq, you could set the timestamp math.MaxInt64.
- Lookup(ctx context.Context, mq primitive.MessageQueue, timestamp int64) (int64, error)
-
- // Commit the offset of specified mqs to broker, if auto-commit is disable, you must commit the offset manually.
- Commit(ctx context.Context, mqs ...primitive.MessageQueue) (int64, error)
-
- // CommittedOffset return the offset of specified Message
- CommittedOffset(mq primitive.MessageQueue) (int64, error)
-
- // Seek set offset of the mq, if you wanna re-consuming your message form one position, the method may help you.
- // if you want re-consuming from one time, you cloud Lookup() then seek it.
- Seek(mq primitive.MessageQueue, offset int64) error
-
- // Pause consuming for specified MessageQueues, after pause, client will not fetch any message from the specified
- // message queues
- //
- // Note that this method does not affect message queue subscription. In particular, it does not cause a group
- // rebalance.
- //
- // if a MessageQueue belong a topic that has not been subscribed, an error will be returned
- //Pause(mqs ...primitive.MessageQueue) error
-
- // Resume specified message queues which have been paused with Pause, if a MessageQueue that not paused,
- // it will be ignored. if not subscribed, an error will be returned
- //Resume(mqs ...primitive.MessageQueue) error
-}
-
-// The PullConsumer has not implemented completely, if you want have an experience of PullConsumer, you could use
-// consumer.NewPullConsumer(...), but it may changed in the future.
-//
-// The PullConsumer will be supported in next release
-func NewPullConsumer(opts ...consumer.Option) (PullConsumer, error) {
- return nil, errors.ErrPullConsumer
-}
diff --git a/benchmark/consumer.go b/benchmark/consumer.go
deleted file mode 100644
index 907a1e7..0000000
--- a/benchmark/consumer.go
+++ /dev/null
@@ -1,277 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package main
-
-import (
- "context"
- "flag"
- "fmt"
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/consumer"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/rlog"
- "os"
- "os/signal"
- "sync"
- "sync/atomic"
- "syscall"
- "time"
-)
-
-type statiBenchmarkConsumerSnapshot struct {
- receiveMessageTotal int64
- born2ConsumerTotalRT int64
- store2ConsumerTotalRT int64
- born2ConsumerMaxRT int64
- store2ConsumerMaxRT int64
- createdAt time.Time
- next *statiBenchmarkConsumerSnapshot
-}
-
-type consumeSnapshots struct {
- sync.RWMutex
- head, tail, cur *statiBenchmarkConsumerSnapshot
- len int
-}
-
-func (s *consumeSnapshots) takeSnapshot() {
- b := s.cur
- sn := new(statiBenchmarkConsumerSnapshot)
- sn.receiveMessageTotal = atomic.LoadInt64(&b.receiveMessageTotal)
- sn.born2ConsumerMaxRT = atomic.LoadInt64(&b.born2ConsumerMaxRT)
- sn.born2ConsumerTotalRT = atomic.LoadInt64(&b.born2ConsumerTotalRT)
- sn.store2ConsumerMaxRT = atomic.LoadInt64(&b.store2ConsumerMaxRT)
- sn.store2ConsumerTotalRT = atomic.LoadInt64(&b.store2ConsumerTotalRT)
- sn.createdAt = time.Now()
-
- s.Lock()
- if s.tail != nil {
- s.tail.next = sn
- }
- s.tail = sn
- if s.head == nil {
- s.head = s.tail
- }
-
- s.len++
- if s.len > 10 {
- s.head = s.head.next
- s.len--
- }
- s.Unlock()
-}
-
-func (s *consumeSnapshots) printStati() {
- s.RLock()
- if s.len < 10 {
- s.RUnlock()
- return
- }
-
- f, l := s.head, s.tail
- respSucCount := float64(l.receiveMessageTotal - f.receiveMessageTotal)
- consumeTps := respSucCount / l.createdAt.Sub(f.createdAt).Seconds()
- avgB2CRT := float64(l.born2ConsumerTotalRT-f.born2ConsumerTotalRT) / respSucCount
- avgS2CRT := float64(l.store2ConsumerTotalRT-f.store2ConsumerTotalRT) / respSucCount
- s.RUnlock()
-
- rlog.Info("Benchmark Consumer Snapshot", map[string]interface{}{
- "consumeTPS": int64(consumeTps),
- "average(B2C)RT": avgB2CRT,
- "average(S2C)RT": avgS2CRT,
- "max(B2C)RT": l.born2ConsumerMaxRT,
- "max(S2C)RT": l.store2ConsumerMaxRT,
- })
-}
-
-type consumerBenchmark struct {
- topic string
- groupPrefix string
- nameSrv string
- isPrefixEnable bool
- filterType string
- expression string
- testMinutes int
- instanceCount int
-
- flags *flag.FlagSet
-
- groupID string
-}
-
-func init() {
- c := &consumerBenchmark{}
- flags := flag.NewFlagSet("consumer", flag.ExitOnError)
- c.flags = flags
-
- flags.StringVar(&c.topic, "t", "BenchmarkTest", "topic")
- flags.StringVar(&c.groupPrefix, "g", "benchmark_consumer", "group prefix")
- flags.StringVar(&c.nameSrv, "n", "", "namesrv address list, separated by comma")
- flags.BoolVar(&c.isPrefixEnable, "p", true, "group prefix is enable")
- flags.StringVar(&c.filterType, "f", "", "filter type,options:TAG|SQL92, or empty")
- flags.StringVar(&c.expression, "e", "*", "expression")
- flags.IntVar(&c.testMinutes, "m", 10, "test minutes")
- flags.IntVar(&c.instanceCount, "i", 1, "instance count")
-
- registerCommand("consumer", c)
-}
-
-func (bc *consumerBenchmark) consumeMsg(stati *statiBenchmarkConsumerSnapshot, exit chan struct{}) {
- c, err := rocketmq.NewPushConsumer(
- consumer.WithGroupName(bc.groupID),
- consumer.WithNameServer([]string{bc.nameSrv}),
- )
- if err != nil {
- panic("new push consumer error:" + err.Error())
- }
-
- selector := consumer.MessageSelector{}
- err = c.Subscribe(bc.topic, selector, func(ctx context.Context,
- msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
- for _, msg := range msgs {
- atomic.AddInt64(&stati.receiveMessageTotal, 1)
- now := time.Now().UnixNano() / int64(time.Millisecond)
- b2cRT := now - msg.BornTimestamp
- atomic.AddInt64(&stati.born2ConsumerTotalRT, b2cRT)
- s2cRT := now - msg.StoreTimestamp
- atomic.AddInt64(&stati.store2ConsumerTotalRT, s2cRT)
-
- for {
- old := atomic.LoadInt64(&stati.born2ConsumerMaxRT)
- if old >= b2cRT || atomic.CompareAndSwapInt64(&stati.born2ConsumerMaxRT, old, b2cRT) {
- break
- }
- }
-
- for {
- old := atomic.LoadInt64(&stati.store2ConsumerMaxRT)
- if old >= s2cRT || atomic.CompareAndSwapInt64(&stati.store2ConsumerMaxRT, old, s2cRT) {
- break
- }
- }
- }
- return consumer.ConsumeSuccess, nil
- })
-
- rlog.Info("Test Start", nil)
- c.Start()
- select {
- case <-exit:
- c.Shutdown()
- return
- }
-}
-
-func (bc *consumerBenchmark) run(args []string) {
- bc.flags.Parse(args)
- if bc.topic == "" {
- rlog.Error("Empty Topic", nil)
- bc.usage()
- return
- }
-
- if bc.groupPrefix == "" {
- rlog.Error("Empty Group Prefix", nil)
- bc.usage()
- return
- }
-
- if bc.nameSrv == "" {
- rlog.Error("Empty Nameserver", nil)
- bc.usage()
- return
- }
-
- if bc.testMinutes <= 0 {
- rlog.Error("Test Time Must Be Positive Integer", nil)
- bc.usage()
- return
- }
-
- if bc.instanceCount <= 0 {
- rlog.Error("Thread Count Must Be Positive Integer", nil)
- bc.usage()
- return
- }
-
- bc.groupID = bc.groupPrefix
- if bc.isPrefixEnable {
- bc.groupID += fmt.Sprintf("_%d", time.Now().UnixNano()/int64(time.Millisecond)%100)
- }
-
- stati := statiBenchmarkConsumerSnapshot{}
- snapshots := consumeSnapshots{cur: &stati}
- exitChan := make(chan struct{})
-
- wg := sync.WaitGroup{}
-
- wg.Add(1)
- go func() {
- bc.consumeMsg(&stati, exitChan)
- wg.Done()
- }()
-
- // snapshot
- wg.Add(1)
- go func() {
- defer wg.Done()
- ticker := time.NewTicker(time.Second)
- for {
- select {
- case <-ticker.C:
- snapshots.takeSnapshot()
- case <-exitChan:
- ticker.Stop()
- return
- }
- }
- }()
-
- // print statistic
- wg.Add(1)
- go func() {
- defer wg.Done()
- ticker := time.NewTicker(time.Second * 10)
- for {
- select {
- case <-ticker.C:
- snapshots.printStati()
- case <-exitChan:
- ticker.Stop()
- return
- }
- }
- }()
-
- signalChan := make(chan os.Signal, 1)
- signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM)
- select {
- case <-time.Tick(time.Minute * time.Duration(bc.testMinutes)):
- case <-signalChan:
- }
-
- close(exitChan)
- wg.Wait()
- snapshots.takeSnapshot()
- snapshots.printStati()
- rlog.Info("Test Done", nil)
-}
-
-func (bc *consumerBenchmark) usage() {
- bc.flags.Usage()
-}
diff --git a/benchmark/main.go b/benchmark/main.go
deleted file mode 100644
index 79eca8f..0000000
--- a/benchmark/main.go
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package main
-
-import (
- "fmt"
- "github.com/apache/rocketmq-client-go/v2/rlog"
- "os"
-)
-
-type command interface {
- usage()
- run(args []string)
-}
-
-var (
- cmds = map[string]command{}
-)
-
-func registerCommand(name string, cmd command) {
- if cmd == nil {
- panic("empty command")
- }
-
- _, ok := cmds[name]
- if ok {
- panic(fmt.Sprintf("%s command existed", name))
- }
-
- cmds[name] = cmd
-}
-
-func usage() {
- rlog.Info("Command", map[string]interface{}{
- "name": os.Args[0],
- })
- for _, cmd := range cmds {
- cmd.usage()
- }
-}
-
-// go run *.go [command name] [command args]
-func main() {
- if len(os.Args) < 2 {
- rlog.Error("Lack Command Name", nil)
- usage()
- return
- }
-
- name := os.Args[1]
- cmd, ok := cmds[name]
- if !ok {
- rlog.Error("Command Isn't Supported", map[string]interface{}{
- "command": name,
- })
- usage()
- return
- }
-
- cmd.run(os.Args[2:])
-}
diff --git a/benchmark/message.go b/benchmark/message.go
deleted file mode 100644
index d5690fe..0000000
--- a/benchmark/message.go
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package main
-
-import "strings"
-
-var (
- longText = ""
- longTextLen = 0
-)
-
-func init() {
- longText = strings.Repeat("0123456789", 100)
- longTextLen = len(longText)
-}
-
-func buildMsg(size int) string {
- return longText[:size]
-}
diff --git a/benchmark/producer.go b/benchmark/producer.go
deleted file mode 100644
index 7516352..0000000
--- a/benchmark/producer.go
+++ /dev/null
@@ -1,290 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package main
-
-import (
- "context"
- "flag"
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/producer"
- "github.com/apache/rocketmq-client-go/v2/rlog"
- "os"
- "os/signal"
- "sync"
- "sync/atomic"
- "syscall"
- "time"
-)
-
-type statiBenchmarkProducerSnapshot struct {
- sendRequestSuccessCount int64
- sendRequestFailedCount int64
- receiveResponseSuccessCount int64
- receiveResponseFailedCount int64
- sendMessageSuccessTimeTotal int64
- sendMessageMaxRT int64
- createdAt time.Time
- next *statiBenchmarkProducerSnapshot
-}
-
-type produceSnapshots struct {
- sync.RWMutex
- head, tail, cur *statiBenchmarkProducerSnapshot
- len int
-}
-
-func (s *produceSnapshots) takeSnapshot() {
- b := s.cur
- sn := new(statiBenchmarkProducerSnapshot)
- sn.sendRequestSuccessCount = atomic.LoadInt64(&b.sendRequestSuccessCount)
- sn.sendRequestFailedCount = atomic.LoadInt64(&b.sendRequestFailedCount)
- sn.receiveResponseSuccessCount = atomic.LoadInt64(&b.receiveResponseSuccessCount)
- sn.receiveResponseFailedCount = atomic.LoadInt64(&b.receiveResponseFailedCount)
- sn.sendMessageSuccessTimeTotal = atomic.LoadInt64(&b.sendMessageSuccessTimeTotal)
- sn.sendMessageMaxRT = atomic.LoadInt64(&b.sendMessageMaxRT)
- sn.createdAt = time.Now()
-
- s.Lock()
- if s.tail != nil {
- s.tail.next = sn
- }
- s.tail = sn
- if s.head == nil {
- s.head = s.tail
- }
-
- s.len++
- if s.len > 10 {
- s.head = s.head.next
- s.len--
- }
- s.Unlock()
-}
-
-func (s *produceSnapshots) printStati() {
- s.RLock()
- if s.len < 10 {
- s.RUnlock()
- return
- }
-
- f, l := s.head, s.tail
- respSucCount := float64(l.receiveResponseSuccessCount - f.receiveResponseSuccessCount)
- sendTps := respSucCount / l.createdAt.Sub(f.createdAt).Seconds()
- avgRT := float64(l.sendMessageSuccessTimeTotal-f.sendMessageSuccessTimeTotal) / respSucCount
- maxRT := atomic.LoadInt64(&s.cur.sendMessageMaxRT)
- s.RUnlock()
-
- rlog.Info("Benchmark Producer Snapshot", map[string]interface{}{
- "sendTps": int64(sendTps),
- "maxRt": maxRT,
- "averageRt": avgRT,
- "sendFailed": l.sendRequestFailedCount,
- "responseFailed": l.receiveResponseFailedCount,
- "total": l.receiveResponseSuccessCount,
- })
-}
-
-type producerBenchmark struct {
- topic string
- nameSrv string
- groupID string
- instanceCount int
- testMinutes int
- bodySize int
-
- flags *flag.FlagSet
-}
-
-func init() {
- p := &producerBenchmark{}
- flags := flag.NewFlagSet("producer", flag.ExitOnError)
- p.flags = flags
-
- flags.StringVar(&p.topic, "t", "", "topic name")
- flags.StringVar(&p.nameSrv, "n", "", "nameserver address")
- flags.StringVar(&p.groupID, "g", "", "group id")
- flags.IntVar(&p.instanceCount, "i", 1, "instance count")
- flags.IntVar(&p.testMinutes, "m", 10, "test minutes")
- flags.IntVar(&p.bodySize, "s", 32, "body size")
-
- registerCommand("producer", p)
-}
-
-func (bp *producerBenchmark) produceMsg(stati *statiBenchmarkProducerSnapshot, exit chan struct{}) {
- p, err := rocketmq.NewProducer(
- producer.WithNameServer([]string{bp.nameSrv}),
- producer.WithRetry(2),
- )
-
- if err != nil {
- rlog.Error("New Producer Error", map[string]interface{}{
- rlog.LogKeyUnderlayError: err.Error(),
- })
- return
- }
-
- err = p.Start()
-
- defer p.Shutdown()
-
- topic, tag := bp.topic, "benchmark-producer"
- msgStr := buildMsg(bp.bodySize)
-
-AGAIN:
- select {
- case <-exit:
- return
- default:
- }
-
- now := time.Now()
- r, err := p.SendSync(context.Background(), primitive.NewMessage(topic, []byte(msgStr)))
-
- if err != nil {
- rlog.Error("Send Message Error", map[string]interface{}{
- rlog.LogKeyUnderlayError: err.Error(),
- })
- goto AGAIN
- }
-
- if r.Status == primitive.SendOK {
- atomic.AddInt64(&stati.receiveResponseSuccessCount, 1)
- atomic.AddInt64(&stati.sendRequestSuccessCount, 1)
- currentRT := int64(time.Since(now) / time.Millisecond)
- atomic.AddInt64(&stati.sendMessageSuccessTimeTotal, currentRT)
- prevRT := atomic.LoadInt64(&stati.sendMessageMaxRT)
- for currentRT > prevRT {
- if atomic.CompareAndSwapInt64(&stati.sendMessageMaxRT, prevRT, currentRT) {
- break
- }
- prevRT = atomic.LoadInt64(&stati.sendMessageMaxRT)
- }
- goto AGAIN
- }
- rlog.Error("Send Message Error", map[string]interface{}{
- "topic": topic,
- "tag": tag,
- rlog.LogKeyUnderlayError: err.Error(),
- })
- goto AGAIN
-}
-
-func (bp *producerBenchmark) run(args []string) {
- bp.flags.Parse(args)
-
- if bp.topic == "" {
- rlog.Error("Empty Topic", nil)
- bp.flags.Usage()
- return
- }
-
- if bp.groupID == "" {
- rlog.Error("Empty Group Id", nil)
- bp.flags.Usage()
- return
- }
-
- if bp.nameSrv == "" {
- rlog.Error("Empty Nameserver", nil)
- bp.flags.Usage()
- return
- }
- if bp.instanceCount <= 0 {
- rlog.Error("Instance Count Must Be Positive Integer", nil)
- bp.flags.Usage()
- return
- }
- if bp.testMinutes <= 0 {
- rlog.Error("Test Time Must Be Positive Integer", nil)
- bp.flags.Usage()
- return
- }
- if bp.bodySize <= 0 {
- rlog.Error("Body Size Must Be Positive Integer", nil)
- bp.flags.Usage()
- return
- }
-
- stati := statiBenchmarkProducerSnapshot{}
- snapshots := produceSnapshots{cur: &stati}
- exitChan := make(chan struct{})
- wg := sync.WaitGroup{}
-
- for i := 0; i < bp.instanceCount; i++ {
- i := i
- go func() {
- wg.Add(1)
- bp.produceMsg(&stati, exitChan)
- rlog.Info("Producer Done and Exit", map[string]interface{}{
- "id": i,
- })
- wg.Done()
- }()
- }
-
- // snapshot
- wg.Add(1)
- go func() {
- defer wg.Done()
- ticker := time.NewTicker(time.Second)
- for {
- select {
- case <-ticker.C:
- snapshots.takeSnapshot()
- case <-exitChan:
- ticker.Stop()
- return
- }
- }
- }()
-
- // print statistic
- wg.Add(1)
- go func() {
- defer wg.Done()
- ticker := time.NewTicker(time.Second * 10)
- for {
- select {
- case <-ticker.C:
- snapshots.printStati()
- case <-exitChan:
- ticker.Stop()
- return
- }
- }
- }()
-
- signalChan := make(chan os.Signal, 1)
- signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM)
- select {
- case <-time.Tick(time.Minute * time.Duration(bp.testMinutes)):
- case <-signalChan:
- }
-
- close(exitChan)
- wg.Wait()
- snapshots.takeSnapshot()
- snapshots.printStati()
- rlog.Info("Test Done", nil)
-}
-
-func (bp *producerBenchmark) usage() {
- bp.flags.Usage()
-}
diff --git a/benchmark/stable.go b/benchmark/stable.go
deleted file mode 100644
index 2659bc5..0000000
--- a/benchmark/stable.go
+++ /dev/null
@@ -1,280 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package main
-
-import (
- "flag"
- "github.com/apache/rocketmq-client-go/v2/errors"
- "github.com/apache/rocketmq-client-go/v2/rlog"
- "os"
- "os/signal"
- "syscall"
- "time"
-)
-
-type stableTest struct {
- nameSrv string
- topic string
- groupID string
- opIntervalSec int
- testMin int
-
- op func()
-
- flags *flag.FlagSet
-}
-
-func (st *stableTest) buildFlags(name string) {
- flags := flag.NewFlagSet(name, flag.ExitOnError)
- flags.StringVar(&st.topic, "t", "stable-test", "topic name")
- flags.StringVar(&st.nameSrv, "n", "", "nameserver address")
- flags.StringVar(&st.groupID, "g", "stable-test", "group id")
- flags.IntVar(&st.testMin, "m", 10, "test minutes")
- flags.IntVar(&st.opIntervalSec, "s", 1, "operation interval[produce/consume]")
-
- st.flags = flags
-}
-
-func (st *stableTest) checkFlag() error {
- if st.topic == "" {
- return errors.ErrEmptyTopic
- }
-
- if st.nameSrv == "" {
- return errors.ErrEmptyNameSrv
- }
-
- if st.groupID == "" {
- return errors.ErrEmptyGroupID
- }
-
- if st.testMin <= 0 {
- return errors.ErrTestMin
- }
-
- if st.opIntervalSec <= 0 {
- return errors.ErrOperationInterval
- }
-
- return nil
-}
-
-func (st *stableTest) run() {
- opTicker := time.NewTicker(time.Duration(st.opIntervalSec) * time.Second)
- closeChan := time.Tick(time.Duration(st.testMin) * time.Minute)
-
- signalChan := make(chan os.Signal, 1)
- signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM)
- for {
- select {
- case <-signalChan:
- opTicker.Stop()
- rlog.Info("Test Done", nil)
- return
- case <-closeChan:
- opTicker.Stop()
- rlog.Info("Test Done", nil)
- return
- case <-opTicker.C:
- st.op()
- }
- }
-}
-
-type stableTestProducer struct {
- *stableTest
- bodySize int
-
- //p rocketmq.Producer
-}
-
-func (stp *stableTestProducer) buildFlags(name string) {
- stp.stableTest.buildFlags(name)
- stp.flags.IntVar(&stp.bodySize, "b", 32, "body size")
-}
-
-func (stp *stableTestProducer) checkFlag() error {
- err := stp.stableTest.checkFlag()
- if err != nil {
- return err
- }
- if stp.bodySize <= 0 {
- return errors.ErrMessageBody
- }
-
- return nil
-}
-
-func (stp *stableTestProducer) usage() {
- stp.flags.Usage()
-}
-
-func (stp *stableTestProducer) run(args []string) {
- err := stp.flags.Parse(args)
- if err != nil {
- rlog.Info("Parse Args Error", map[string]interface{}{
- "args": args,
- rlog.LogKeyUnderlayError: err.Error(),
- })
- stp.usage()
- return
- }
-
- err = stp.checkFlag()
- if err != nil {
- rlog.Error("Check Flag Error", map[string]interface{}{
- rlog.LogKeyUnderlayError: err.Error(),
- })
- stp.usage()
- return
- }
-
- //p, err := rocketmq.NewProducer(&rocketmq.ProducerConfig{
- // ClientConfig: rocketmq.ClientConfig{GroupID: stp.groupID, NameServer: stp.nameSrv},
- //})
- //if err != nil {
- // fmt.Printf("new consumer error:%s\n", err)
- // return
- //}
- //
- //err = p.Start()
- //if err != nil {
- // fmt.Printf("start consumer error:%s\n", err)
- // return
- //}
- //defer p.Shutdown()
- //
- //stp.p = p
- stp.stableTest.run()
-}
-
-func (stp *stableTestProducer) sendMessage() {
- //r, err := stp.p.SendMessageSync(&rocketmq.Message{Topic: stp.topic, Body: buildMsg(stp.bodySize)})
- //if err == nil {
- // fmt.Printf("send result:%+v\n", r)
- // return
- //}
- //fmt.Printf("send message error:%s", err)
-}
-
-type stableTestConsumer struct {
- *stableTest
- expression string
-
- //c rocketmq.PullConsumer
- offsets map[int]int64
-}
-
-func (stc *stableTestConsumer) buildFlags(name string) {
- stc.stableTest.buildFlags(name)
- stc.flags.StringVar(&stc.expression, "e", "*", "expression")
-}
-
-func (stc *stableTestConsumer) checkFlag() error {
- err := stc.stableTest.checkFlag()
- if err != nil {
- return err
- }
-
- if stc.expression == "" {
- return errors.ErrEmptyExpression
- }
- return nil
-}
-
-func (stc *stableTestConsumer) usage() {
- stc.flags.Usage()
-}
-
-func (stc *stableTestConsumer) run(args []string) {
- err := stc.flags.Parse(args)
- if err != nil {
- rlog.Error("Parse Args Error", map[string]interface{}{
- "args": args,
- rlog.LogKeyUnderlayError: err.Error(),
- })
- stc.usage()
- return
- }
-
- err = stc.checkFlag()
- if err != nil {
- rlog.Error("Check Flag Error", map[string]interface{}{
- rlog.LogKeyUnderlayError: err.Error(),
- })
- stc.usage()
- return
- }
- //
- //c, err := rocketmq.NewPullConsumer(&rocketmq.PullConsumerConfig{
- // ClientConfig: rocketmq.ClientConfig{GroupID: stc.groupID, NameServer: stc.nameSrv},
- //})
- //if err != nil {
- // fmt.Printf("new pull consumer error:%s\n", err)
- // return
- //}
- //
- //err = c.Start()
- //if err != nil {
- // fmt.Printf("start consumer error:%s\n", err)
- // return
- //}
- //defer c.Shutdown()
- //
- //stc.c = c
- stc.stableTest.run()
-}
-
-func (stc *stableTestConsumer) pullMessage() {
- //mqs := stc.c.FetchSubscriptionMessageQueues(stc.topic)
- //
- //for _, mq := range mqs {
- // offset := stc.offsets[mq.ID]
- // pr := stc.c.Pull(mq, stc.expression, offset, 32)
- //fmt.Printf("pull from %s, offset:%d, count:%+v\n", mq.String(), offset, len(pr.Messages))
- //
- //switch pr.Status {
- //case rocketmq.PullNoNewMsg:
- // stc.offsets[mq.ID] = 0 // pull from the begin
- //case rocketmq.PullFound:
- // fallthrough
- //case rocketmq.PullNoMatchedMsg:
- // fallthrough
- //case rocketmq.PullOffsetIllegal:
- // stc.offsets[mq.ID] = pr.NextBeginOffset
- //case rocketmq.PullBrokerTimeout:
- // fmt.Println("broker timeout occur")
- //}
- //}
-}
-
-func init() {
- // consumer
- name := "stableTestProducer"
- p := &stableTestProducer{stableTest: &stableTest{}}
- p.buildFlags(name)
- p.op = p.sendMessage
- registerCommand(name, p)
-
- // consumer
- name = "stableTestConsumer"
- c := &stableTestConsumer{stableTest: &stableTest{}, offsets: map[int]int64{}}
- c.buildFlags(name)
- c.op = c.pullMessage
- registerCommand(name, c)
-}
diff --git a/config.go b/config.go
deleted file mode 100644
index 8f1592c..0000000
--- a/config.go
+++ /dev/null
@@ -1,17 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-package rocketmq
diff --git a/consumer/consumer.go b/consumer/consumer.go
deleted file mode 100644
index 8056b22..0000000
--- a/consumer/consumer.go
+++ /dev/null
@@ -1,1063 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package consumer
-
-import (
- "context"
- "fmt"
- "github.com/apache/rocketmq-client-go/v2/errors"
- "sort"
- "strconv"
- "strings"
- "sync"
- "sync/atomic"
- "time"
-
- jsoniter "github.com/json-iterator/go"
-
- "github.com/tidwall/gjson"
-
- "github.com/apache/rocketmq-client-go/v2/internal"
- "github.com/apache/rocketmq-client-go/v2/internal/remote"
- "github.com/apache/rocketmq-client-go/v2/internal/utils"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/rlog"
-)
-
-const (
- // Delay some time when exception error
- _PullDelayTimeWhenError = 3 * time.Second
-
- // Flow control interval
- _PullDelayTimeWhenFlowControl = 50 * time.Millisecond
-
- // Delay some time when suspend pull service
- _PullDelayTimeWhenSuspend = 30 * time.Second
-
- // Long polling mode, the Consumer connection max suspend time
- _BrokerSuspendMaxTime = 20 * time.Second
-
- // Long polling mode, the Consumer connection timeout (must greater than _BrokerSuspendMaxTime)
- _ConsumerTimeoutWhenSuspend = 30 * time.Second
-
- // Offset persistent interval for consumer
- _PersistConsumerOffsetInterval = 5 * time.Second
-)
-
-type ConsumeType string
-
-const (
- _PullConsume = ConsumeType("CONSUME_ACTIVELY")
- _PushConsume = ConsumeType("CONSUME_PASSIVELY")
-
- _SubAll = "*"
-)
-
-// Message model defines the way how messages are delivered to each consumer clients.
-// </p>
-//
-// RocketMQ supports two message models: clustering and broadcasting. If clustering is set, consumer clients with
-// the same {@link #ConsumerGroup} would only consume shards of the messages subscribed, which achieves load
-// balances; Conversely, if the broadcasting is set, each consumer client will consume all subscribed messages
-// separately.
-// </p>
-//
-// This field defaults to clustering.
-type MessageModel int
-
-const (
- BroadCasting MessageModel = iota
- Clustering
-)
-
-func (mode MessageModel) String() string {
- switch mode {
- case BroadCasting:
- return "BroadCasting"
- case Clustering:
- return "Clustering"
- default:
- return "Unknown"
- }
-}
-
-// Consuming point on consumer booting.
-// </p>
-//
-// There are three consuming points:
-// <ul>
-// <li>
-// <code>CONSUME_FROM_LAST_OFFSET</code>: consumer clients pick up where it stopped previously.
-// If it were a newly booting up consumer client, according aging of the consumer group, there are two
-// cases:
-// <ol>
-// <li>
-// if the consumer group is created so recently that the earliest message being subscribed has yet
-// expired, which means the consumer group represents a lately launched business, consuming will
-// start from the very beginning;
-// </li>
-// <li>
-// if the earliest message being subscribed has expired, consuming will start from the latest
-// messages, meaning messages born prior to the booting timestamp would be ignored.
-// </li>
-// </ol>
-// </li>
-// <li>
-// <code>CONSUME_FROM_FIRST_OFFSET</code>: Consumer client will start from earliest messages available.
-// </li>
-// <li>
-// <code>CONSUME_FROM_TIMESTAMP</code>: Consumer client will start from specified timestamp, which means
-// messages born prior to {@link #consumeTimestamp} will be ignored
-// </li>
-// </ul>
-type ConsumeFromWhere int
-
-const (
- ConsumeFromLastOffset ConsumeFromWhere = iota
- ConsumeFromFirstOffset
- ConsumeFromTimestamp
-)
-
-type ExpressionType string
-
-const (
- /**
- * <ul>
- * Keywords:
- * <li>{@code AND, OR, NOT, BETWEEN, IN, TRUE, FALSE, IS, NULL}</li>
- * </ul>
- * <p/>
- * <ul>
- * Data type:
- * <li>Boolean, like: TRUE, FALSE</li>
- * <li>String, like: 'abc'</li>
- * <li>Decimal, like: 123</li>
- * <li>Float number, like: 3.1415</li>
- * </ul>
- * <p/>
- * <ul>
- * Grammar:
- * <li>{@code AND, OR}</li>
- * <li>{@code >, >=, <, <=, =}</li>
- * <li>{@code BETWEEN A AND B}, equals to {@code >=A AND <=B}</li>
- * <li>{@code NOT BETWEEN A AND B}, equals to {@code >B OR <A}</li>
- * <li>{@code IN ('a', 'b')}, equals to {@code ='a' OR ='b'}, this operation only support String type.</li>
- * <li>{@code IS NULL}, {@code IS NOT NULL}, check parameter whether is null, or not.</li>
- * <li>{@code =TRUE}, {@code =FALSE}, check parameter whether is true, or false.</li>
- * </ul>
- * <p/>
- * <p>
- * Example:
- * (a > 10 AND a < 100) OR (b IS NOT NULL AND b=TRUE)
- * </p>
- */
- SQL92 = ExpressionType("SQL92")
-
- /**
- * Only support or operation such as
- * "tag1 || tag2 || tag3", <br>
- * If null or * expression, meaning subscribe all.
- */
- TAG = ExpressionType("TAG")
-)
-
-func IsTagType(exp string) bool {
- if exp == "" || exp == "TAG" {
- return true
- }
- return false
-}
-
-type MessageSelector struct {
- Type ExpressionType
- Expression string
-}
-
-type ConsumeResult int
-
-const (
- ConsumeSuccess ConsumeResult = iota
- ConsumeRetryLater
- Commit
- Rollback
- SuspendCurrentQueueAMoment
-)
-
-type ConsumeResultHolder struct {
- ConsumeResult
-}
-
-type ConsumerReturn int
-
-const (
- SuccessReturn ConsumerReturn = iota
- ExceptionReturn
- NullReturn
- TimeoutReturn
- FailedReturn
-)
-
-type PullRequest struct {
- consumerGroup string
- mq *primitive.MessageQueue
- pq *processQueue
- nextOffset int64
- lockedFirst bool
-}
-
-func (pr *PullRequest) String() string {
- return fmt.Sprintf("[ConsumerGroup: %s, Topic: %s, MessageQueue: %d]",
- pr.consumerGroup, pr.mq.Topic, pr.mq.QueueId)
-}
-
-type defaultConsumer struct {
- /**
- * Consumers of the same role is required to have exactly same subscriptions and consumerGroup to correctly achieve
- * load balance. It's required and needs to be globally unique.
- * </p>
- *
- * See <a href="http://rocketmq.apache.org/docs/core-concept/">here</a> for further discussion.
- */
- consumerGroup string
- model MessageModel
- allocate func(string, string, []*primitive.MessageQueue, []string) []*primitive.MessageQueue
- unitMode bool
- consumeOrderly bool
- fromWhere ConsumeFromWhere
- consumerStartTimestamp int64
-
- cType ConsumeType
- client internal.RMQClient
- mqChanged func(topic string, mqAll, mqDivided []*primitive.MessageQueue)
- state int32
- pause bool
- once sync.Once
- option consumerOptions
- // key: primitive.MessageQueue
- // value: *processQueue
- processQueueTable sync.Map
-
- // key: topic(string)
- // value: map[int]*primitive.MessageQueue
- topicSubscribeInfoTable sync.Map
-
- // key: topic
- // value: *SubscriptionData
- subscriptionDataTable sync.Map
- storage OffsetStore
- // chan for push consumer
- prCh chan PullRequest
-
- pullFromWhichNodeTable sync.Map
-
- stat *StatsManager
-}
-
-func (dc *defaultConsumer) start() error {
- if dc.model == Clustering {
- // set retry topic
- retryTopic := internal.GetRetryTopic(dc.consumerGroup)
- sub := buildSubscriptionData(retryTopic, MessageSelector{TAG, _SubAll})
- dc.subscriptionDataTable.Store(retryTopic, sub)
- }
-
- if dc.model == Clustering {
- dc.option.ChangeInstanceNameToPID()
- dc.storage = NewRemoteOffsetStore(dc.consumerGroup, dc.client, dc.client.GetNameSrv())
- } else {
- dc.storage = NewLocalFileOffsetStore(dc.consumerGroup, dc.client.ClientID())
- }
-
- dc.client.Start()
- atomic.StoreInt32(&dc.state, int32(internal.StateRunning))
- dc.consumerStartTimestamp = time.Now().UnixNano() / int64(time.Millisecond)
- dc.stat = NewStatsManager()
- return nil
-}
-
-func (dc *defaultConsumer) shutdown() error {
- atomic.StoreInt32(&dc.state, int32(internal.StateShutdown))
-
- mqs := make([]*primitive.MessageQueue, 0)
- dc.processQueueTable.Range(func(key, value interface{}) bool {
- k := key.(primitive.MessageQueue)
- pq := value.(*processQueue)
- pq.WithDropped(true)
- // close msg channel using RWMutex to make sure no data was writing
- pq.mutex.Lock()
- close(pq.msgCh)
- pq.mutex.Unlock()
- mqs = append(mqs, &k)
- return true
- })
- dc.stat.ShutDownStat()
- dc.storage.persist(mqs)
- dc.client.Shutdown()
- return nil
-}
-
-func (dc *defaultConsumer) persistConsumerOffset() error {
- err := dc.makeSureStateOK()
- if err != nil {
- return err
- }
- mqs := make([]*primitive.MessageQueue, 0)
- dc.processQueueTable.Range(func(key, value interface{}) bool {
- k := key.(primitive.MessageQueue)
- mqs = append(mqs, &k)
- return true
- })
- dc.storage.persist(mqs)
- return nil
-}
-
-func (dc *defaultConsumer) updateOffset(queue *primitive.MessageQueue, offset int64) error {
- dc.storage.update(queue, offset, false)
- return nil
-}
-
-func (dc *defaultConsumer) subscriptionAutomatically(topic string) {
- _, exist := dc.subscriptionDataTable.Load(topic)
- if !exist {
- s := MessageSelector{
- Expression: _SubAll,
- }
- dc.subscriptionDataTable.Store(topic, buildSubscriptionData(topic, s))
- }
-}
-
-func (dc *defaultConsumer) updateTopicSubscribeInfo(topic string, mqs []*primitive.MessageQueue) {
- _, exist := dc.subscriptionDataTable.Load(topic)
- if exist {
- dc.topicSubscribeInfoTable.Store(topic, mqs)
- }
-}
-
-func (dc *defaultConsumer) isSubscribeTopicNeedUpdate(topic string) bool {
- _, exist := dc.subscriptionDataTable.Load(topic)
- if !exist {
- return false
- }
- _, exist = dc.topicSubscribeInfoTable.Load(topic)
- return !exist
-}
-
-func (dc *defaultConsumer) doBalance() {
- dc.subscriptionDataTable.Range(func(key, value interface{}) bool {
- topic := key.(string)
- v, exist := dc.topicSubscribeInfoTable.Load(topic)
- if !exist {
- rlog.Warning("do balance in group failed, the topic does not exist", map[string]interface{}{
- rlog.LogKeyConsumerGroup: dc.consumerGroup,
- rlog.LogKeyTopic: topic,
- })
- return true
- }
- mqs := v.([]*primitive.MessageQueue)
- switch dc.model {
- case BroadCasting:
- changed := dc.updateProcessQueueTable(topic, mqs)
- if changed {
- dc.mqChanged(topic, mqs, mqs)
- rlog.Debug("MessageQueue changed", map[string]interface{}{
- rlog.LogKeyConsumerGroup: dc.consumerGroup,
- rlog.LogKeyTopic: topic,
- rlog.LogKeyMessageQueue: fmt.Sprintf("%v", mqs),
- })
- }
- case Clustering:
- cidAll := dc.findConsumerList(topic)
- if cidAll == nil {
- rlog.Warning("do balance in group failed, get consumer id list failed", map[string]interface{}{
- rlog.LogKeyConsumerGroup: dc.consumerGroup,
- rlog.LogKeyTopic: topic,
- })
- return true
- }
- mqAll := make([]*primitive.MessageQueue, len(mqs))
- copy(mqAll, mqs)
- sort.Strings(cidAll)
- sort.SliceStable(mqAll, func(i, j int) bool {
- v := strings.Compare(mqAll[i].Topic, mqAll[j].Topic)
- if v != 0 {
- return v < 0
- }
-
- v = strings.Compare(mqAll[i].BrokerName, mqAll[j].BrokerName)
- if v != 0 {
- return v < 0
- }
- return (mqAll[i].QueueId - mqAll[j].QueueId) < 0
- })
- allocateResult := dc.allocate(dc.consumerGroup, dc.client.ClientID(), mqAll, cidAll)
- changed := dc.updateProcessQueueTable(topic, allocateResult)
- if changed {
- dc.mqChanged(topic, mqAll, allocateResult)
- rlog.Debug("MessageQueue do balance done", map[string]interface{}{
- rlog.LogKeyConsumerGroup: dc.consumerGroup,
- rlog.LogKeyTopic: topic,
- "clientID": dc.client.ClientID(),
- "mqAllSize": len(mqAll),
- "cidAllSize": len(cidAll),
- "rebalanceResultSize": len(allocateResult),
- "rebalanceResultSet": allocateResult,
- })
- }
- }
- return true
- })
-}
-
-func (dc *defaultConsumer) SubscriptionDataList() []*internal.SubscriptionData {
- result := make([]*internal.SubscriptionData, 0)
- dc.subscriptionDataTable.Range(func(key, value interface{}) bool {
- result = append(result, value.(*internal.SubscriptionData))
- return true
- })
- return result
-}
-
-func (dc *defaultConsumer) makeSureStateOK() error {
- if atomic.LoadInt32(&dc.state) != int32(internal.StateRunning) {
- return fmt.Errorf("state not running, actually: %v", dc.state)
- }
- return nil
-}
-
-type lockBatchRequestBody struct {
- ConsumerGroup string `json:"consumerGroup"`
- ClientId string `json:"clientId"`
- MQs []*primitive.MessageQueue `json:"mqSet"`
-}
-
-func (dc *defaultConsumer) lock(mq *primitive.MessageQueue) bool {
- brokerResult := dc.client.GetNameSrv().FindBrokerAddressInSubscribe(mq.BrokerName, internal.MasterId, true)
-
- if brokerResult == nil {
- return false
- }
-
- body := &lockBatchRequestBody{
- ConsumerGroup: dc.consumerGroup,
- ClientId: dc.client.ClientID(),
- MQs: []*primitive.MessageQueue{mq},
- }
- lockedMQ := dc.doLock(brokerResult.BrokerAddr, body)
- var lockOK bool
- for idx := range lockedMQ {
- _mq := lockedMQ[idx]
- v, exist := dc.processQueueTable.Load(_mq)
- if exist {
- pq := v.(*processQueue)
- pq.WithLock(true)
- pq.UpdateLastConsumeTime()
- pq.UpdateLastLockTime()
- }
- if _mq == *mq {
- lockOK = true
- }
- }
- fields := map[string]interface{}{
- "lockOK": lockOK,
- rlog.LogKeyConsumerGroup: dc.consumerGroup,
- rlog.LogKeyMessageQueue: mq.String(),
- }
- if lockOK {
- rlog.Debug("lock MessageQueue", fields)
- } else {
- rlog.Info("lock MessageQueue", fields)
- }
- return lockOK
-}
-
-func (dc *defaultConsumer) unlock(mq *primitive.MessageQueue, oneway bool) {
- brokerResult := dc.client.GetNameSrv().FindBrokerAddressInSubscribe(mq.BrokerName, internal.MasterId, true)
-
- if brokerResult == nil {
- return
- }
-
- body := &lockBatchRequestBody{
- ConsumerGroup: dc.consumerGroup,
- ClientId: dc.client.ClientID(),
- MQs: []*primitive.MessageQueue{mq},
- }
- dc.doUnlock(brokerResult.BrokerAddr, body, oneway)
- rlog.Info("unlock MessageQueue", map[string]interface{}{
- rlog.LogKeyConsumerGroup: dc.consumerGroup,
- "clientID": dc.client.ClientID(),
- rlog.LogKeyMessageQueue: mq.String(),
- })
-}
-
-func (dc *defaultConsumer) lockAll() {
- mqMapSet := dc.buildProcessQueueTableByBrokerName()
- for broker, mqs := range mqMapSet {
- if len(mqs) == 0 {
- continue
- }
- brokerResult := dc.client.GetNameSrv().FindBrokerAddressInSubscribe(broker, internal.MasterId, true)
- if brokerResult == nil {
- continue
- }
- body := &lockBatchRequestBody{
- ConsumerGroup: dc.consumerGroup,
- ClientId: dc.client.ClientID(),
- MQs: mqs,
- }
- lockedMQ := dc.doLock(brokerResult.BrokerAddr, body)
- set := make(map[primitive.MessageQueue]bool)
- for idx := range lockedMQ {
- _mq := lockedMQ[idx]
- v, exist := dc.processQueueTable.Load(_mq)
- if exist {
- pq := v.(*processQueue)
- pq.WithLock(true)
- pq.UpdateLastConsumeTime()
- }
- set[_mq] = true
- }
- for idx := range mqs {
- _mq := mqs[idx]
- if !set[*_mq] {
- v, exist := dc.processQueueTable.Load(_mq)
- if exist {
- pq := v.(*processQueue)
- pq.WithLock(false)
- pq.UpdateLastLockTime()
- rlog.Info("lock MessageQueue", map[string]interface{}{
- "lockOK": false,
- rlog.LogKeyConsumerGroup: dc.consumerGroup,
- rlog.LogKeyMessageQueue: _mq.String(),
- })
- }
- }
- }
- }
-}
-
-func (dc *defaultConsumer) unlockAll(oneway bool) {
- mqMapSet := dc.buildProcessQueueTableByBrokerName()
- for broker, mqs := range mqMapSet {
- if len(mqs) == 0 {
- continue
- }
- brokerResult := dc.client.GetNameSrv().FindBrokerAddressInSubscribe(broker, internal.MasterId, true)
- if brokerResult == nil {
- continue
- }
- body := &lockBatchRequestBody{
- ConsumerGroup: dc.consumerGroup,
- ClientId: dc.client.ClientID(),
- MQs: mqs,
- }
- dc.doUnlock(brokerResult.BrokerAddr, body, oneway)
- for idx := range mqs {
- _mq := mqs[idx]
- v, exist := dc.processQueueTable.Load(_mq)
- if exist {
- rlog.Info("lock MessageQueue", map[string]interface{}{
- "lockOK": false,
- rlog.LogKeyConsumerGroup: dc.consumerGroup,
- rlog.LogKeyMessageQueue: _mq.String(),
- })
- v.(*processQueue).WithLock(false)
- }
- }
- }
-}
-
-func (dc *defaultConsumer) doLock(addr string, body *lockBatchRequestBody) []primitive.MessageQueue {
- data, _ := jsoniter.Marshal(body)
- request := remote.NewRemotingCommand(internal.ReqLockBatchMQ, nil, data)
- response, err := dc.client.InvokeSync(context.Background(), addr, request, 1*time.Second)
- if err != nil {
- rlog.Error("lock MessageQueue to broker invoke error", map[string]interface{}{
- rlog.LogKeyBroker: addr,
- rlog.LogKeyUnderlayError: err,
- })
- return nil
- }
- lockOKMQSet := struct {
- MQs []primitive.MessageQueue `json:"lockOKMQSet"`
- }{}
- if len(response.Body) == 0 {
- return nil
- }
- err = jsoniter.Unmarshal(response.Body, &lockOKMQSet)
- if err != nil {
- rlog.Error("Unmarshal lock mq body error", map[string]interface{}{
- rlog.LogKeyUnderlayError: err,
- })
- return nil
- }
- return lockOKMQSet.MQs
-}
-
-func (dc *defaultConsumer) doUnlock(addr string, body *lockBatchRequestBody, oneway bool) {
- data, _ := jsoniter.Marshal(body)
- request := remote.NewRemotingCommand(internal.ReqUnlockBatchMQ, nil, data)
- if oneway {
- err := dc.client.InvokeOneWay(context.Background(), addr, request, 3*time.Second)
- if err != nil {
- rlog.Error("lock MessageQueue to broker invoke oneway error", map[string]interface{}{
- rlog.LogKeyBroker: addr,
- rlog.LogKeyUnderlayError: err,
- })
- }
- } else {
- response, err := dc.client.InvokeSync(context.Background(), addr, request, 1*time.Second)
- rlog.Error("lock MessageQueue to broker invoke error", map[string]interface{}{
- rlog.LogKeyBroker: addr,
- rlog.LogKeyUnderlayError: err,
- })
- if response.Code != internal.ResSuccess {
- // TODO error
- }
- }
-}
-
-func (dc *defaultConsumer) buildProcessQueueTableByBrokerName() map[string][]*primitive.MessageQueue {
- result := make(map[string][]*primitive.MessageQueue, 0)
-
- dc.processQueueTable.Range(func(key, value interface{}) bool {
- mq := key.(primitive.MessageQueue)
- mqs, exist := result[mq.BrokerName]
- if !exist {
- mqs = make([]*primitive.MessageQueue, 0)
- }
- mqs = append(mqs, &mq)
- result[mq.BrokerName] = mqs
- return true
- })
-
- return result
-}
-
-func (dc *defaultConsumer) updateProcessQueueTable(topic string, mqs []*primitive.MessageQueue) bool {
- var changed bool
- mqSet := make(map[primitive.MessageQueue]bool)
- for idx := range mqs {
- mqSet[*mqs[idx]] = true
- }
- dc.processQueueTable.Range(func(key, value interface{}) bool {
- mq := key.(primitive.MessageQueue)
- pq := value.(*processQueue)
- if mq.Topic == topic {
- if !mqSet[mq] {
- pq.WithDropped(true)
- if dc.removeUnnecessaryMessageQueue(&mq, pq) {
- dc.processQueueTable.Delete(key)
- changed = true
- rlog.Debug("remove unnecessary mq when updateProcessQueueTable", map[string]interface{}{
- rlog.LogKeyConsumerGroup: dc.consumerGroup,
- rlog.LogKeyMessageQueue: mq.String(),
- })
- }
- } else if pq.isPullExpired() && dc.cType == _PushConsume {
- pq.WithDropped(true)
- if dc.removeUnnecessaryMessageQueue(&mq, pq) {
- dc.processQueueTable.Delete(key)
- changed = true
- rlog.Debug("remove unnecessary mq because pull was paused, prepare to fix it", map[string]interface{}{
- rlog.LogKeyConsumerGroup: dc.consumerGroup,
- rlog.LogKeyMessageQueue: mq.String(),
- })
- }
- }
- }
- return true
- })
-
- if dc.cType == _PushConsume {
- for item := range mqSet {
- // BUG: the mq will send to channel, if not copy once, the next iter will modify the mq in the channel.
- mq := item
- _, exist := dc.processQueueTable.Load(mq)
- if exist {
- continue
- }
- if dc.consumeOrderly && !dc.lock(&mq) {
- rlog.Warning("do defaultConsumer, add a new mq failed, because lock failed", map[string]interface{}{
- rlog.LogKeyConsumerGroup: dc.consumerGroup,
- rlog.LogKeyMessageQueue: mq.String(),
- })
- continue
- }
- dc.storage.remove(&mq)
- nextOffset := dc.computePullFromWhere(&mq)
- if nextOffset >= 0 {
- _, exist := dc.processQueueTable.Load(mq)
- if exist {
- rlog.Debug("do defaultConsumer, mq already exist", map[string]interface{}{
- rlog.LogKeyConsumerGroup: dc.consumerGroup,
- rlog.LogKeyMessageQueue: mq.String(),
- })
- } else {
- rlog.Debug("do defaultConsumer, add a new mq", map[string]interface{}{
- rlog.LogKeyConsumerGroup: dc.consumerGroup,
- rlog.LogKeyMessageQueue: mq.String(),
- })
- pq := newProcessQueue(dc.consumeOrderly)
- dc.processQueueTable.Store(mq, pq)
- pr := PullRequest{
- consumerGroup: dc.consumerGroup,
- mq: &mq,
- pq: pq,
- nextOffset: nextOffset,
- }
- dc.prCh <- pr
- changed = true
- }
- } else {
- rlog.Warning("do defaultConsumer, add a new mq failed", map[string]interface{}{
- rlog.LogKeyConsumerGroup: dc.consumerGroup,
- rlog.LogKeyMessageQueue: mq.String(),
- })
- }
- }
- }
-
- return changed
-}
-
-func (dc *defaultConsumer) removeUnnecessaryMessageQueue(mq *primitive.MessageQueue, pq *processQueue) bool {
- dc.storage.persist([]*primitive.MessageQueue{mq})
- dc.storage.remove(mq)
- return true
-}
-
-func (dc *defaultConsumer) computePullFromWhere(mq *primitive.MessageQueue) int64 {
- if dc.cType == _PullConsume {
- return 0
- }
- var result = int64(-1)
- lastOffset := dc.storage.read(mq, _ReadFromStore)
- if lastOffset >= 0 {
- result = lastOffset
- } else {
- switch dc.option.FromWhere {
- case ConsumeFromLastOffset:
- if lastOffset == -1 {
- if strings.HasPrefix(mq.Topic, internal.RetryGroupTopicPrefix) {
- result = 0
- } else {
- lastOffset, err := dc.queryMaxOffset(mq)
- if err == nil {
- result = lastOffset
- } else {
- rlog.Warning("query max offset error", map[string]interface{}{
- rlog.LogKeyMessageQueue: mq,
- rlog.LogKeyUnderlayError: err,
- })
- }
- }
- } else {
- result = -1
- }
- case ConsumeFromFirstOffset:
- if lastOffset == -1 {
- result = 0
- }
- case ConsumeFromTimestamp:
- if lastOffset == -1 {
- if strings.HasPrefix(mq.Topic, internal.RetryGroupTopicPrefix) {
- lastOffset, err := dc.queryMaxOffset(mq)
- if err == nil {
- result = lastOffset
- } else {
- result = -1
- rlog.Warning("query max offset error", map[string]interface{}{
- rlog.LogKeyMessageQueue: mq,
- rlog.LogKeyUnderlayError: err,
- })
- }
- } else {
- t, err := time.Parse("20060102150405", dc.option.ConsumeTimestamp)
- if err != nil {
- result = -1
- } else {
- lastOffset, err := dc.searchOffsetByTimestamp(mq, t.Unix()*1000)
- if err != nil {
- result = -1
- } else {
- result = lastOffset
- }
- }
- }
- }
- default:
- }
- }
- return result
-}
-
-func (dc *defaultConsumer) pullInner(ctx context.Context, queue *primitive.MessageQueue, data *internal.SubscriptionData,
- offset int64, numbers int, sysFlag int32, commitOffsetValue int64) (*primitive.PullResult, error) {
-
- brokerResult := dc.tryFindBroker(queue)
- if brokerResult == nil {
- rlog.Warning("no broker found for mq", map[string]interface{}{
- rlog.LogKeyMessageQueue: queue,
- })
- return nil, errors.ErrBrokerNotFound
- }
-
- if brokerResult.Slave {
- sysFlag = clearCommitOffsetFlag(sysFlag)
- }
-
- if (data.ExpType == string(TAG)) && brokerResult.BrokerVersion < internal.V4_1_0 {
- return nil, fmt.Errorf("the broker [%s, %v] does not upgrade to support for filter message by %v",
- queue.BrokerName, brokerResult.BrokerVersion, data.ExpType)
- }
-
- pullRequest := &internal.PullMessageRequestHeader{
- ConsumerGroup: dc.consumerGroup,
- Topic: queue.Topic,
- QueueId: int32(queue.QueueId),
- QueueOffset: offset,
- MaxMsgNums: int32(numbers),
- SysFlag: sysFlag,
- CommitOffset: commitOffsetValue,
- // TODO: 和java对齐
- SuspendTimeoutMillis: _BrokerSuspendMaxTime,
- SubExpression: data.SubString,
- // TODO: add subversion
- ExpressionType: string(data.ExpType),
- }
-
- if data.ExpType == string(TAG) {
- pullRequest.SubVersion = 0
- } else {
- pullRequest.SubVersion = data.SubVersion
- }
-
- // TODO: add computPullFromWhichFilterServer
-
- return dc.client.PullMessage(context.Background(), brokerResult.BrokerAddr, pullRequest)
-}
-
-func (dc *defaultConsumer) processPullResult(mq *primitive.MessageQueue, result *primitive.PullResult, data *internal.SubscriptionData) {
-
- dc.updatePullFromWhichNode(mq, result.SuggestWhichBrokerId)
-
- switch result.Status {
- case primitive.PullFound:
- result.SetMessageExts(primitive.DecodeMessage(result.GetBody()))
- msgs := result.GetMessageExts()
-
- // filter message according to tags
- msgListFilterAgain := msgs
- if data.Tags.Len() > 0 && data.ClassFilterMode {
- msgListFilterAgain = make([]*primitive.MessageExt, 0)
- for _, msg := range msgs {
- _, exist := data.Tags.Contains(msg.GetTags())
- if exist {
- msgListFilterAgain = append(msgListFilterAgain, msg)
- }
- }
- }
-
- // TODO: add filter message hook
- for _, msg := range msgListFilterAgain {
- traFlag, _ := strconv.ParseBool(msg.GetProperty(primitive.PropertyTransactionPrepared))
- if traFlag {
- msg.TransactionId = msg.GetProperty(primitive.PropertyUniqueClientMessageIdKeyIndex)
- }
-
- msg.WithProperty(primitive.PropertyMinOffset, strconv.FormatInt(result.MinOffset, 10))
- msg.WithProperty(primitive.PropertyMaxOffset, strconv.FormatInt(result.MaxOffset, 10))
- }
-
- result.SetMessageExts(msgListFilterAgain)
- }
-}
-
-func (dc *defaultConsumer) findConsumerList(topic string) []string {
- brokerAddr := dc.client.GetNameSrv().FindBrokerAddrByTopic(topic)
- if brokerAddr == "" {
- dc.client.GetNameSrv().UpdateTopicRouteInfo(topic)
- brokerAddr = dc.client.GetNameSrv().FindBrokerAddrByTopic(topic)
- }
-
- if brokerAddr != "" {
- req := &internal.GetConsumerListRequestHeader{
- ConsumerGroup: dc.consumerGroup,
- }
- cmd := remote.NewRemotingCommand(internal.ReqGetConsumerListByGroup, req, nil)
- res, err := dc.client.InvokeSync(context.Background(), brokerAddr, cmd, 3*time.Second) // TODO 超时机制有问题
- if err != nil {
- rlog.Error("get consumer list of group from broker error", map[string]interface{}{
- rlog.LogKeyConsumerGroup: dc.consumerGroup,
- rlog.LogKeyBroker: brokerAddr,
- rlog.LogKeyUnderlayError: err,
- })
- return nil
- }
- result := gjson.ParseBytes(res.Body)
- list := make([]string, 0)
- arr := result.Get("consumerIdList").Array()
- for idx := range arr {
- list = append(list, arr[idx].String())
- }
- return list
- }
- return nil
-}
-
-func (dc *defaultConsumer) sendBack(msg *primitive.MessageExt, level int) error {
- return nil
-}
-
-// QueryMaxOffset with specific queueId and topic
-func (dc *defaultConsumer) queryMaxOffset(mq *primitive.MessageQueue) (int64, error) {
- brokerAddr := dc.client.GetNameSrv().FindBrokerAddrByName(mq.BrokerName)
- if brokerAddr == "" {
- dc.client.GetNameSrv().UpdateTopicRouteInfo(mq.Topic)
- brokerAddr = dc.client.GetNameSrv().FindBrokerAddrByName(mq.BrokerName)
- }
- if brokerAddr == "" {
- return -1, fmt.Errorf("the broker [%s] does not exist", mq.BrokerName)
- }
-
- request := &internal.GetMaxOffsetRequestHeader{
- Topic: mq.Topic,
- QueueId: mq.QueueId,
- }
-
- cmd := remote.NewRemotingCommand(internal.ReqGetMaxOffset, request, nil)
- response, err := dc.client.InvokeSync(context.Background(), brokerAddr, cmd, 3*time.Second)
- if err != nil {
- return -1, err
- }
-
- return strconv.ParseInt(response.ExtFields["offset"], 10, 64)
-}
-
-func (dc *defaultConsumer) queryOffset(mq *primitive.MessageQueue) int64 {
- return dc.storage.read(mq, _ReadMemoryThenStore)
-}
-
-// SearchOffsetByTimestamp with specific queueId and topic
-func (dc *defaultConsumer) searchOffsetByTimestamp(mq *primitive.MessageQueue, timestamp int64) (int64, error) {
- brokerAddr := dc.client.GetNameSrv().FindBrokerAddrByName(mq.BrokerName)
- if brokerAddr == "" {
- dc.client.GetNameSrv().UpdateTopicRouteInfo(mq.Topic)
- brokerAddr = dc.client.GetNameSrv().FindBrokerAddrByName(mq.BrokerName)
- }
- if brokerAddr == "" {
- return -1, fmt.Errorf("the broker [%s] does not exist", mq.BrokerName)
- }
-
- request := &internal.SearchOffsetRequestHeader{
- Topic: mq.Topic,
- QueueId: mq.QueueId,
- Timestamp: timestamp,
- }
-
- cmd := remote.NewRemotingCommand(internal.ReqSearchOffsetByTimestamp, request, nil)
- response, err := dc.client.InvokeSync(context.Background(), brokerAddr, cmd, 3*time.Second)
- if err != nil {
- return -1, err
- }
-
- return strconv.ParseInt(response.ExtFields["offset"], 10, 64)
-}
-
-func buildSubscriptionData(topic string, selector MessageSelector) *internal.SubscriptionData {
- subData := &internal.SubscriptionData{
- Topic: topic,
- SubString: selector.Expression,
- ExpType: string(selector.Type),
- }
- subData.SubVersion = time.Now().UnixNano()
-
- if selector.Type != "" && selector.Type != TAG {
- return subData
- }
-
- if selector.Expression == "" || selector.Expression == _SubAll {
- subData.ExpType = string(TAG)
- subData.SubString = _SubAll
- } else {
- tags := strings.Split(selector.Expression, "||")
- subData.Tags = utils.NewSet()
- subData.Codes = utils.NewSet()
- for idx := range tags {
- trimString := strings.Trim(tags[idx], " ")
- if trimString != "" {
- if _, ok := subData.Tags.Contains(trimString); !ok {
- subData.Tags.AddKV(trimString, trimString)
- }
- hCode := utils.HashString(trimString)
- v := strconv.Itoa(hCode)
- if _, ok := subData.Codes.Contains(v); !ok {
- subData.Codes.AddKV(v, v)
- }
- }
- }
- }
- return subData
-}
-
-func buildSysFlag(commitOffset, suspend, subscription, classFilter bool) int32 {
- var flag int32 = 0
- if commitOffset {
- flag |= 0x1 << 0
- }
-
- if suspend {
- flag |= 0x1 << 1
- }
-
- if subscription {
- flag |= 0x1 << 2
- }
-
- if classFilter {
- flag |= 0x1 << 3
- }
-
- return flag
-}
-
-func clearCommitOffsetFlag(sysFlag int32) int32 {
- return sysFlag & (^0x1 << 0)
-}
-
-func (dc *defaultConsumer) tryFindBroker(mq *primitive.MessageQueue) *internal.FindBrokerResult {
- result := dc.client.GetNameSrv().FindBrokerAddressInSubscribe(mq.BrokerName, dc.recalculatePullFromWhichNode(mq), false)
- if result != nil {
- return result
- }
- dc.client.GetNameSrv().UpdateTopicRouteInfo(mq.Topic)
- return dc.client.GetNameSrv().FindBrokerAddressInSubscribe(mq.BrokerName, dc.recalculatePullFromWhichNode(mq), false)
-}
-
-func (dc *defaultConsumer) updatePullFromWhichNode(mq *primitive.MessageQueue, brokerId int64) {
- dc.pullFromWhichNodeTable.Store(*mq, brokerId)
-}
-
-func (dc *defaultConsumer) recalculatePullFromWhichNode(mq *primitive.MessageQueue) int64 {
- v, exist := dc.pullFromWhichNodeTable.Load(*mq)
- if exist {
- return v.(int64)
- }
- return internal.MasterId
-}
diff --git a/consumer/consumer_test.go b/consumer/consumer_test.go
deleted file mode 100644
index 12ccd18..0000000
--- a/consumer/consumer_test.go
+++ /dev/null
@@ -1,171 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package consumer
-
-import (
- "sync"
- "testing"
- "time"
-
- "github.com/golang/mock/gomock"
- . "github.com/smartystreets/goconvey/convey"
- "github.com/stretchr/testify/assert"
-
- "github.com/apache/rocketmq-client-go/v2/internal"
- "github.com/apache/rocketmq-client-go/v2/internal/remote"
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-func TestParseTimestamp(t *testing.T) {
- layout := "20060102150405"
- timestamp, err := time.ParseInLocation(layout, "20190430193409", time.UTC)
- assert.Nil(t, err)
- assert.Equal(t, int64(1556652849), timestamp.Unix())
-}
-
-func TestDoRebalance(t *testing.T) {
- Convey("Given a defaultConsumer", t, func() {
- dc := &defaultConsumer{
- model: Clustering,
- }
-
- topic := "test"
- broker := "127.0.0.1:8889"
- clientID := "clientID"
- mqs := []*primitive.MessageQueue{
- {
- Topic: topic,
- BrokerName: "",
- QueueId: 0,
- },
- {
- Topic: topic,
- BrokerName: "",
- QueueId: 1,
- },
- }
- dc.topicSubscribeInfoTable.Store(topic, mqs)
- sub := &internal.SubscriptionData{}
- dc.subscriptionDataTable.Store(topic, sub)
-
- ctrl := gomock.NewController(t)
- defer ctrl.Finish()
- namesrvCli := internal.NewMockNamesrvs(ctrl)
- namesrvCli.EXPECT().FindBrokerAddrByTopic(gomock.Any()).Return(broker)
-
- rmqCli := internal.NewMockRMQClient(ctrl)
- rmqCli.EXPECT().InvokeSync(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).
- Return(&remote.RemotingCommand{
- Body: []byte("{\"consumerIdList\": [\"a1\", \"a2\", \"a3\"] }"),
- }, nil)
- rmqCli.EXPECT().ClientID().Return(clientID)
- rmqCli.SetNameSrv(namesrvCli)
-
- dc.client = rmqCli
-
- var wg sync.WaitGroup
- wg.Add(1)
- dc.allocate = func(cg string, clientID string, mqAll []*primitive.MessageQueue, cidAll []string) []*primitive.MessageQueue {
- assert.Equal(t, cidAll, []string{"a1", "a2", "a3"})
- wg.Done()
- return nil
- }
-
- dc.doBalance()
-
- wg.Wait()
- })
-}
-
-func TestComputePullFromWhere(t *testing.T) {
- Convey("Given a defaultConsumer", t, func() {
- dc := &defaultConsumer{
- model: Clustering,
- cType: _PushConsume,
- }
-
- ctrl := gomock.NewController(t)
- defer ctrl.Finish()
-
- offsetStore := NewMockOffsetStore(ctrl)
- dc.storage = offsetStore
-
- mq := &primitive.MessageQueue{
- Topic: "test",
- }
-
- namesrvCli := internal.NewMockNamesrvs(ctrl)
-
- rmqCli := internal.NewMockRMQClient(ctrl)
- dc.client = rmqCli
- rmqCli.SetNameSrv(namesrvCli)
-
- Convey("get effective offset", func() {
- offsetStore.EXPECT().read(gomock.Any(), gomock.Any()).Return(int64(10))
- res := dc.computePullFromWhere(mq)
- assert.Equal(t, int64(10), res)
- })
-
- Convey("ConsumeFromLastOffset for normal topic", func() {
- offsetStore.EXPECT().read(gomock.Any(), gomock.Any()).Return(int64(-1))
- dc.option.FromWhere = ConsumeFromLastOffset
-
- broker := "a"
- namesrvCli.EXPECT().FindBrokerAddrByName(gomock.Any()).Return(broker)
-
- rmqCli.EXPECT().InvokeSync(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).
- Return(&remote.RemotingCommand{
- ExtFields: map[string]string{
- "offset": "20",
- },
- }, nil)
-
- res := dc.computePullFromWhere(mq)
- assert.Equal(t, int64(20), res)
- })
-
- Convey("ConsumeFromFirstOffset for normal topic", func() {
- offsetStore.EXPECT().read(gomock.Any(), gomock.Any()).Return(int64(-1))
- dc.option.FromWhere = ConsumeFromFirstOffset
-
- res := dc.computePullFromWhere(mq)
- assert.Equal(t, int64(0), res)
- })
-
- Convey("ConsumeFromTimestamp for normal topic", func() {
- offsetStore.EXPECT().read(gomock.Any(), gomock.Any()).Return(int64(-1))
- dc.option.FromWhere = ConsumeFromTimestamp
-
- dc.option.ConsumeTimestamp = "20060102150405"
-
- broker := "a"
- namesrvCli.EXPECT().FindBrokerAddrByName(gomock.Any()).Return(broker)
-
- rmqCli.EXPECT().InvokeSync(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).
- Return(&remote.RemotingCommand{
- ExtFields: map[string]string{
- "offset": "30",
- },
- }, nil)
-
- res := dc.computePullFromWhere(mq)
- assert.Equal(t, int64(30), res)
- })
-
- })
-}
diff --git a/consumer/interceptor.go b/consumer/interceptor.go
deleted file mode 100644
index 05ff94a..0000000
--- a/consumer/interceptor.go
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package consumer
-
-import (
- "context"
- "fmt"
- "time"
-
- "github.com/apache/rocketmq-client-go/v2/internal"
- "github.com/apache/rocketmq-client-go/v2/internal/utils"
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-// WithTrace support rocketmq trace: https://github.com/apache/rocketmq/wiki/RIP-6-Message-Trace.
-func WithTrace(traceCfg *primitive.TraceConfig) Option {
- return func(options *consumerOptions) {
-
- ori := options.Interceptors
- options.Interceptors = make([]primitive.Interceptor, 0)
- options.Interceptors = append(options.Interceptors, newTraceInterceptor(traceCfg))
- options.Interceptors = append(options.Interceptors, ori...)
- }
-}
-
-func newTraceInterceptor(traceCfg *primitive.TraceConfig) primitive.Interceptor {
- dispatcher := internal.NewTraceDispatcher(traceCfg)
- if dispatcher != nil {
- dispatcher.Start()
- }
-
- return func(ctx context.Context, req, reply interface{}, next primitive.Invoker) error {
- if dispatcher == nil {
- return fmt.Errorf("GetOrNewRocketMQClient faild")
- }
- consumerCtx, exist := primitive.GetConsumerCtx(ctx)
- if !exist || len(consumerCtx.Msgs) == 0 {
- return next(ctx, req, reply)
- }
-
- beginT := time.Now()
- // before traceCtx
- traceCx := internal.TraceContext{
- RequestId: primitive.CreateUniqID(),
- TimeStamp: time.Now().UnixNano() / int64(time.Millisecond),
- TraceType: internal.SubBefore,
- GroupName: consumerCtx.ConsumerGroup,
- IsSuccess: true,
- }
- beans := make([]internal.TraceBean, 0)
- for _, msg := range consumerCtx.Msgs {
- if msg == nil {
- continue
- }
- regionID := msg.GetRegionID()
- traceOn := msg.IsTraceOn()
- if traceOn == "false" {
- continue
- }
- bean := internal.TraceBean{
- Topic: msg.Topic,
- MsgId: msg.MsgId,
- Tags: msg.GetTags(),
- Keys: msg.GetKeys(),
- StoreTime: msg.StoreTimestamp,
- BodyLength: int(msg.StoreSize),
- RetryTimes: int(msg.ReconsumeTimes),
- ClientHost: utils.LocalIP,
- StoreHost: utils.LocalIP,
- }
- beans = append(beans, bean)
- traceCx.RegionId = regionID
- }
- if len(beans) > 0 {
- traceCx.TraceBeans = beans
- traceCx.TimeStamp = time.Now().UnixNano() / int64(time.Millisecond)
- dispatcher.Append(traceCx)
- }
-
- err := next(ctx, req, reply)
-
- // after traceCtx
- costTime := time.Since(beginT).Nanoseconds() / int64(time.Millisecond)
- ctxType := consumerCtx.Properties[primitive.PropCtxType]
- afterCtx := internal.TraceContext{
- TimeStamp: time.Now().UnixNano() / int64(time.Millisecond),
-
- TraceType: internal.SubAfter,
- RegionId: traceCx.RegionId,
- GroupName: traceCx.GroupName,
- RequestId: traceCx.RequestId,
- IsSuccess: consumerCtx.Success,
- CostTime: costTime,
- TraceBeans: traceCx.TraceBeans,
- ContextCode: primitive.ConsumeReturnType(ctxType).Ordinal(),
- }
- dispatcher.Append(afterCtx)
- return err
- }
-}
diff --git a/consumer/lock.go b/consumer/lock.go
deleted file mode 100644
index 6fb17cd..0000000
--- a/consumer/lock.go
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package consumer
-
-import (
- "sync"
-
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-type QueueLock struct {
- lockTable sync.Map
-}
-
-func newQueueLock() *QueueLock {
- return &QueueLock{}
-}
-
-func (ql QueueLock) fetchLock(queue primitive.MessageQueue) sync.Locker {
- v, _ := ql.lockTable.LoadOrStore(queue, new(sync.Mutex))
- return v.(*sync.Mutex)
-}
diff --git a/consumer/mock_offset_store.go b/consumer/mock_offset_store.go
deleted file mode 100644
index bac1cdb..0000000
--- a/consumer/mock_offset_store.go
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Code generated by MockGen. DO NOT EDIT.
-// Source: offset_store.go
-
-// Package consumer is a generated GoMock package.
-package consumer
-
-import (
- reflect "reflect"
-
- primitive "github.com/apache/rocketmq-client-go/v2/primitive"
- gomock "github.com/golang/mock/gomock"
-)
-
-// MockOffsetStore is a mock of OffsetStore interface
-type MockOffsetStore struct {
- ctrl *gomock.Controller
- recorder *MockOffsetStoreMockRecorder
-}
-
-// MockOffsetStoreMockRecorder is the mock recorder for MockOffsetStore
-type MockOffsetStoreMockRecorder struct {
- mock *MockOffsetStore
-}
-
-// NewMockOffsetStore creates a new mock instance
-func NewMockOffsetStore(ctrl *gomock.Controller) *MockOffsetStore {
- mock := &MockOffsetStore{ctrl: ctrl}
- mock.recorder = &MockOffsetStoreMockRecorder{mock}
- return mock
-}
-
-// EXPECT returns an object that allows the caller to indicate expected use
-func (m *MockOffsetStore) EXPECT() *MockOffsetStoreMockRecorder {
- return m.recorder
-}
-
-// persist mocks base method
-func (m *MockOffsetStore) persist(mqs []*primitive.MessageQueue) {
- m.ctrl.Call(m, "persist", mqs)
-}
-
-// persist indicates an expected call of persist
-func (mr *MockOffsetStoreMockRecorder) persist(mqs interface{}) *gomock.Call {
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "persist", reflect.TypeOf((*MockOffsetStore)(nil).persist), mqs)
-}
-
-// remove mocks base method
-func (m *MockOffsetStore) remove(mq *primitive.MessageQueue) {
- m.ctrl.Call(m, "remove", mq)
-}
-
-// remove indicates an expected call of remove
-func (mr *MockOffsetStoreMockRecorder) remove(mq interface{}) *gomock.Call {
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "remove", reflect.TypeOf((*MockOffsetStore)(nil).remove), mq)
-}
-
-// read mocks base method
-func (m *MockOffsetStore) read(mq *primitive.MessageQueue, t readType) int64 {
- ret := m.ctrl.Call(m, "read", mq, t)
- ret0, _ := ret[0].(int64)
- return ret0
-}
-
-// read indicates an expected call of read
-func (mr *MockOffsetStoreMockRecorder) read(mq, t interface{}) *gomock.Call {
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "read", reflect.TypeOf((*MockOffsetStore)(nil).read), mq, t)
-}
-
-// update mocks base method
-func (m *MockOffsetStore) update(mq *primitive.MessageQueue, offset int64, increaseOnly bool) {
- m.ctrl.Call(m, "update", mq, offset, increaseOnly)
-}
-
-// update indicates an expected call of update
-func (mr *MockOffsetStoreMockRecorder) update(mq, offset, increaseOnly interface{}) *gomock.Call {
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "update", reflect.TypeOf((*MockOffsetStore)(nil).update), mq, offset, increaseOnly)
-}
diff --git a/consumer/offset_store.go b/consumer/offset_store.go
deleted file mode 100644
index 5ecfd14..0000000
--- a/consumer/offset_store.go
+++ /dev/null
@@ -1,402 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package consumer
-
-import (
- "context"
- "fmt"
- "os"
- "path/filepath"
- "strconv"
- "sync"
- "time"
-
- "github.com/apache/rocketmq-client-go/v2/internal"
- "github.com/apache/rocketmq-client-go/v2/internal/remote"
- "github.com/apache/rocketmq-client-go/v2/internal/utils"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/rlog"
- jsoniter "github.com/json-iterator/go"
-)
-
-type readType int
-
-const (
- _ReadFromMemory readType = iota
- _ReadFromStore
- _ReadMemoryThenStore
-)
-
-var (
- _LocalOffsetStorePath = os.Getenv("rocketmq.client.localOffsetStoreDir")
-)
-
-func init() {
- if _LocalOffsetStorePath == "" {
- _LocalOffsetStorePath = filepath.Join(os.Getenv("HOME"), ".rocketmq_client_go")
- }
-}
-
-//go:generate mockgen -source offset_store.go -destination mock_offset_store.go -self_package github.com/apache/rocketmq-client-go/v2/consumer --package consumer OffsetStore
-type OffsetStore interface {
- persist(mqs []*primitive.MessageQueue)
- remove(mq *primitive.MessageQueue)
- read(mq *primitive.MessageQueue, t readType) int64
- update(mq *primitive.MessageQueue, offset int64, increaseOnly bool)
-}
-
-type OffsetSerializeWrapper struct {
- OffsetTable map[MessageQueueKey]int64 `json:"offsetTable"`
-}
-
-type MessageQueueKey primitive.MessageQueue
-
-func (mq MessageQueueKey) MarshalText() (text []byte, err error) {
- repr := struct {
- Topic string `json:"topic"`
- BrokerName string `json:"brokerName"`
- QueueId int `json:"queueId"`
- }{
- Topic: mq.Topic,
- BrokerName: mq.BrokerName,
- QueueId: mq.QueueId,
- }
- text, err = jsoniter.Marshal(repr)
- return
-}
-
-func (mq *MessageQueueKey) UnmarshalText(text []byte) error {
- repr := struct {
- Topic string `json:"topic"`
- BrokerName string `json:"brokerName"`
- QueueId int `json:"queueId"`
- }{}
- err := jsoniter.Unmarshal(text, &repr)
- if err != nil {
- return err
- }
- mq.Topic = repr.Topic
- mq.QueueId = repr.QueueId
- mq.BrokerName = repr.BrokerName
-
- return nil
-}
-
-type localFileOffsetStore struct {
- group string
- path string
- OffsetTable *sync.Map // concurrent safe , map[MessageQueueKey]int64
- // mutex for offset file
- mutex sync.Mutex
-}
-
-func NewLocalFileOffsetStore(clientID, group string) OffsetStore {
- store := &localFileOffsetStore{
- group: group,
- path: filepath.Join(_LocalOffsetStorePath, clientID, group, "offset.json"),
- OffsetTable: new(sync.Map),
- }
- store.load()
- return store
-}
-
-func (local *localFileOffsetStore) load() {
- local.mutex.Lock()
- defer local.mutex.Unlock()
- data, err := utils.FileReadAll(local.path)
- if os.IsNotExist(err) {
- return
- }
- if err != nil {
- rlog.Info("read from local store error, try to use bak file", map[string]interface{}{
- rlog.LogKeyUnderlayError: err,
- })
- data, err = utils.FileReadAll(filepath.Join(local.path, ".bak"))
- }
- if err != nil {
- rlog.Info("read from local store bak file error", map[string]interface{}{
- rlog.LogKeyUnderlayError: err,
- })
- return
- }
- datas := make(map[MessageQueueKey]int64)
-
- wrapper := OffsetSerializeWrapper{
- OffsetTable: datas,
- }
-
- err = jsoniter.Unmarshal(data, &wrapper)
- if err != nil {
- rlog.Warning("unmarshal local offset error", map[string]interface{}{
- "local_path": local.path,
- rlog.LogKeyUnderlayError: err.Error(),
- })
- return
- }
-
- if datas != nil {
- for k, v := range datas {
- local.OffsetTable.Store(k, v)
- }
- }
-}
-
-func (local *localFileOffsetStore) read(mq *primitive.MessageQueue, t readType) int64 {
- switch t {
- case _ReadFromMemory, _ReadMemoryThenStore:
- off := readFromMemory(local.OffsetTable, mq)
- if off >= 0 || (off == -1 && t == _ReadFromMemory) {
- return off
- }
- fallthrough
- case _ReadFromStore:
- local.load()
- return readFromMemory(local.OffsetTable, mq)
- default:
-
- }
- return -1
-}
-
-func (local *localFileOffsetStore) update(mq *primitive.MessageQueue, offset int64, increaseOnly bool) {
- local.mutex.Lock()
- defer local.mutex.Unlock()
- rlog.Debug("update offset", map[string]interface{}{
- rlog.LogKeyMessageQueue: mq,
- "new_offset": offset,
- })
- key := MessageQueueKey(*mq)
- localOffset, exist := local.OffsetTable.Load(key)
- if !exist {
- local.OffsetTable.Store(key, offset)
- return
- }
- if increaseOnly {
- if localOffset.(int64) < offset {
- local.OffsetTable.Store(key, offset)
- }
- } else {
- local.OffsetTable.Store(key, offset)
- }
-}
-
-func (local *localFileOffsetStore) persist(mqs []*primitive.MessageQueue) {
- if len(mqs) == 0 {
- return
- }
- local.mutex.Lock()
- defer local.mutex.Unlock()
-
- datas := make(map[MessageQueueKey]int64)
- local.OffsetTable.Range(func(key, value interface{}) bool {
- k := key.(MessageQueueKey)
- v := value.(int64)
- datas[k] = v
- return true
- })
-
- wrapper := OffsetSerializeWrapper{
- OffsetTable: datas,
- }
- data, _ := jsoniter.Marshal(wrapper)
- utils.CheckError(fmt.Sprintf("persist offset to %s", local.path), utils.WriteToFile(local.path, data))
-}
-
-func (local *localFileOffsetStore) remove(mq *primitive.MessageQueue) {
- // nothing to do
-}
-
-type remoteBrokerOffsetStore struct {
- group string
- OffsetTable map[primitive.MessageQueue]int64 `json:"OffsetTable"`
- client internal.RMQClient
- namesrv internal.Namesrvs
- mutex sync.RWMutex
-}
-
-func NewRemoteOffsetStore(group string, client internal.RMQClient, namesrv internal.Namesrvs) OffsetStore {
- return &remoteBrokerOffsetStore{
- group: group,
- client: client,
- namesrv: namesrv,
- OffsetTable: make(map[primitive.MessageQueue]int64),
- }
-}
-
-func (r *remoteBrokerOffsetStore) persist(mqs []*primitive.MessageQueue) {
- r.mutex.Lock()
- defer r.mutex.Unlock()
- if len(mqs) == 0 {
- return
- }
-
- used := make(map[primitive.MessageQueue]struct{}, 0)
- for _, mq := range mqs {
- used[*mq] = struct{}{}
- }
-
- for mq, off := range r.OffsetTable {
- if _, ok := used[mq]; !ok {
- delete(r.OffsetTable, mq)
- continue
- }
- err := r.updateConsumeOffsetToBroker(r.group, mq, off)
- if err != nil {
- rlog.Warning("update offset to broker error", map[string]interface{}{
- rlog.LogKeyConsumerGroup: r.group,
- rlog.LogKeyMessageQueue: mq.String(),
- rlog.LogKeyUnderlayError: err.Error(),
- "offset": off,
- })
- } else {
- rlog.Info("update offset to broker success", map[string]interface{}{
- rlog.LogKeyConsumerGroup: r.group,
- rlog.LogKeyMessageQueue: mq.String(),
- "offset": off,
- })
- }
- }
-}
-
-func (r *remoteBrokerOffsetStore) remove(mq *primitive.MessageQueue) {
- r.mutex.Lock()
- defer r.mutex.Unlock()
-
- delete(r.OffsetTable, *mq)
- rlog.Warning("delete mq from offset table", map[string]interface{}{
- rlog.LogKeyConsumerGroup: r.group,
- rlog.LogKeyMessageQueue: mq,
- })
-}
-
-func (r *remoteBrokerOffsetStore) read(mq *primitive.MessageQueue, t readType) int64 {
- r.mutex.RLock()
- switch t {
- case _ReadFromMemory, _ReadMemoryThenStore:
- off, exist := r.OffsetTable[*mq]
- if exist {
- r.mutex.RUnlock()
- return off
- }
- if t == _ReadFromMemory {
- r.mutex.RUnlock()
- return -1
- }
- fallthrough
- case _ReadFromStore:
- off, err := r.fetchConsumeOffsetFromBroker(r.group, mq)
- if err != nil {
- rlog.Error("fetch offset of mq from broker error", map[string]interface{}{
- rlog.LogKeyConsumerGroup: r.group,
- rlog.LogKeyMessageQueue: mq.String(),
- rlog.LogKeyUnderlayError: err,
- })
- r.mutex.RUnlock()
- return -1
- }
- rlog.Warning("fetch offset of mq from broker success", map[string]interface{}{
- rlog.LogKeyConsumerGroup: r.group,
- rlog.LogKeyMessageQueue: mq.String(),
- "offset": off,
- })
- r.mutex.RUnlock()
- r.update(mq, off, true)
- return off
- default:
- }
-
- return -1
-}
-
-func (r *remoteBrokerOffsetStore) update(mq *primitive.MessageQueue, offset int64, increaseOnly bool) {
- r.mutex.Lock()
- defer r.mutex.Unlock()
- localOffset, exist := r.OffsetTable[*mq]
- if !exist {
- r.OffsetTable[*mq] = offset
- return
- }
- if increaseOnly {
- if localOffset < offset {
- r.OffsetTable[*mq] = offset
- }
- } else {
- r.OffsetTable[*mq] = offset
- }
-}
-
-func (r *remoteBrokerOffsetStore) fetchConsumeOffsetFromBroker(group string, mq *primitive.MessageQueue) (int64, error) {
- broker := r.namesrv.FindBrokerAddrByName(mq.BrokerName)
- if broker == "" {
- r.namesrv.UpdateTopicRouteInfo(mq.Topic)
- broker = r.namesrv.FindBrokerAddrByName(mq.BrokerName)
- }
- if broker == "" {
- return int64(-1), fmt.Errorf("broker: %s address not found", mq.BrokerName)
- }
- queryOffsetRequest := &internal.QueryConsumerOffsetRequestHeader{
- ConsumerGroup: group,
- Topic: mq.Topic,
- QueueId: mq.QueueId,
- }
- cmd := remote.NewRemotingCommand(internal.ReqQueryConsumerOffset, queryOffsetRequest, nil)
- res, err := r.client.InvokeSync(context.Background(), broker, cmd, 3*time.Second)
- if err != nil {
- return -1, err
- }
- if res.Code != internal.ResSuccess {
- return -2, fmt.Errorf("broker response code: %d, remarks: %s", res.Code, res.Remark)
- }
-
- off, err := strconv.ParseInt(res.ExtFields["offset"], 10, 64)
-
- if err != nil {
- return -1, err
- }
-
- return off, nil
-}
-
-func (r *remoteBrokerOffsetStore) updateConsumeOffsetToBroker(group string, mq primitive.MessageQueue, off int64) error {
- broker := r.namesrv.FindBrokerAddrByName(mq.BrokerName)
- if broker == "" {
- r.namesrv.UpdateTopicRouteInfo(mq.Topic)
- broker = r.namesrv.FindBrokerAddrByName(mq.BrokerName)
- }
- if broker == "" {
- return fmt.Errorf("broker: %s address not found", mq.BrokerName)
- }
-
- updateOffsetRequest := &internal.UpdateConsumerOffsetRequestHeader{
- ConsumerGroup: group,
- Topic: mq.Topic,
- QueueId: mq.QueueId,
- CommitOffset: off,
- }
- cmd := remote.NewRemotingCommand(internal.ReqUpdateConsumerOffset, updateOffsetRequest, nil)
- return r.client.InvokeOneWay(context.Background(), broker, cmd, 5*time.Second)
-}
-
-func readFromMemory(table *sync.Map, mq *primitive.MessageQueue) int64 {
- localOffset, exist := table.Load(MessageQueueKey(*mq))
- if !exist {
- return -1
- }
-
- return localOffset.(int64)
-}
diff --git a/consumer/offset_store_test.go b/consumer/offset_store_test.go
deleted file mode 100644
index cfa0eaa..0000000
--- a/consumer/offset_store_test.go
+++ /dev/null
@@ -1,243 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package consumer
-
-import (
- "path/filepath"
- "testing"
-
- "github.com/apache/rocketmq-client-go/v2/internal"
- "github.com/apache/rocketmq-client-go/v2/internal/remote"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/golang/mock/gomock"
- . "github.com/smartystreets/goconvey/convey"
-)
-
-func TestNewLocalFileOffsetStore(t *testing.T) {
- Convey("Given some test cases", t, func() {
- type testCase struct {
- clientId string
- group string
- expectedResult *localFileOffsetStore
- }
- cases := []testCase{
- {
- clientId: "",
- group: "testGroup",
- expectedResult: &localFileOffsetStore{
- group: "testGroup",
- path: filepath.Join(_LocalOffsetStorePath, "/testGroup/offset.json"),
- },
- }, {
- clientId: "192.168.24.1@default",
- group: "",
- expectedResult: &localFileOffsetStore{
- group: "",
- path: filepath.Join(_LocalOffsetStorePath, "/192.168.24.1@default/offset.json"),
- },
- }, {
- clientId: "192.168.24.1@default",
- group: "testGroup",
- expectedResult: &localFileOffsetStore{
- group: "testGroup",
- path: filepath.Join(_LocalOffsetStorePath, "/192.168.24.1@default/testGroup/offset.json"),
- },
- },
- }
-
- for _, value := range cases {
- result := NewLocalFileOffsetStore(value.clientId, value.group).(*localFileOffsetStore)
- value.expectedResult.OffsetTable = result.OffsetTable
- So(result, ShouldResemble, value.expectedResult)
- }
- })
-}
-
-func TestLocalFileOffsetStore(t *testing.T) {
- Convey("Given a local store with a starting value", t, func() {
- localStore := NewLocalFileOffsetStore("192.168.24.1@default", "testGroup")
-
- type offsetCase struct {
- queue *primitive.MessageQueue
- setOffset int64
- expectedOffset int64
- }
- mq := &primitive.MessageQueue{
- Topic: "testTopic",
- BrokerName: "default",
- QueueId: 1,
- }
-
- Convey("test update", func() {
- Convey("when increaseOnly is false", func() {
- cases := []offsetCase{
- {
- queue: mq,
- setOffset: 3,
- expectedOffset: 3,
- }, {
- queue: mq,
- setOffset: 1,
- expectedOffset: 1,
- },
- }
- for _, value := range cases {
- localStore.update(value.queue, value.setOffset, false)
- offset := localStore.read(value.queue, _ReadFromMemory)
- So(offset, ShouldEqual, value.expectedOffset)
- }
- })
-
- Convey("when increaseOnly is true", func() {
- localStore.update(mq, 0, false)
-
- cases := []offsetCase{
- {
- queue: mq,
- setOffset: 3,
- expectedOffset: 3,
- }, {
- queue: mq,
- setOffset: 1,
- expectedOffset: 3,
- },
- }
- for _, value := range cases {
- localStore.update(value.queue, value.setOffset, true)
- offset := localStore.read(value.queue, _ReadFromMemory)
- So(offset, ShouldEqual, value.expectedOffset)
- }
- })
- })
-
- Convey("test persist", func() {
- localStore.update(mq, 1, false)
- offset := localStore.read(mq, _ReadFromMemory)
- So(offset, ShouldEqual, 1)
-
- queues := []*primitive.MessageQueue{mq}
- localStore.persist(queues)
- offset = localStore.read(mq, _ReadFromStore)
- So(offset, ShouldEqual, 1)
-
- localStore.(*localFileOffsetStore).OffsetTable.Delete(MessageQueueKey(*mq))
- offset = localStore.read(mq, _ReadMemoryThenStore)
- So(offset, ShouldEqual, 1)
- })
- })
-}
-
-func TestRemoteBrokerOffsetStore(t *testing.T) {
- Convey("Given a remote store with a starting value", t, func() {
- ctrl := gomock.NewController(t)
- defer ctrl.Finish()
-
- namesrv := internal.NewMockNamesrvs(ctrl)
-
- rmqClient := internal.NewMockRMQClient(ctrl)
- remoteStore := NewRemoteOffsetStore("testGroup", rmqClient, namesrv)
-
- type offsetCase struct {
- queue *primitive.MessageQueue
- setOffset int64
- expectedOffset int64
- }
- mq := &primitive.MessageQueue{
- Topic: "testTopic",
- BrokerName: "default",
- QueueId: 1,
- }
-
- Convey("test update", func() {
- Convey("when increaseOnly is false", func() {
- cases := []offsetCase{
- {
- queue: mq,
- setOffset: 3,
- expectedOffset: 3,
- }, {
- queue: mq,
- setOffset: 1,
- expectedOffset: 1,
- },
- }
- for _, value := range cases {
- remoteStore.update(value.queue, value.setOffset, false)
- offset := remoteStore.read(value.queue, _ReadFromMemory)
- So(offset, ShouldEqual, value.expectedOffset)
- }
- })
-
- Convey("when increaseOnly is true", func() {
- remoteStore.update(mq, 0, false)
-
- cases := []offsetCase{
- {
- queue: mq,
- setOffset: 3,
- expectedOffset: 3,
- }, {
- queue: mq,
- setOffset: 1,
- expectedOffset: 3,
- },
- }
- for _, value := range cases {
- remoteStore.update(value.queue, value.setOffset, true)
- offset := remoteStore.read(value.queue, _ReadFromMemory)
- So(offset, ShouldEqual, value.expectedOffset)
- }
- })
- })
-
- Convey("test persist", func() {
- queues := []*primitive.MessageQueue{mq}
-
- namesrv.EXPECT().FindBrokerAddrByName(gomock.Any()).Return("192.168.24.1:10911").MaxTimes(2)
-
- ret := &remote.RemotingCommand{
- Code: internal.ResSuccess,
- ExtFields: map[string]string{
- "offset": "1",
- },
- }
- rmqClient.EXPECT().InvokeSync(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(ret, nil).MaxTimes(2)
-
- remoteStore.persist(queues)
- offset := remoteStore.read(mq, _ReadFromStore)
- So(offset, ShouldEqual, 1)
-
- remoteStore.remove(mq)
- offset = remoteStore.read(mq, _ReadFromMemory)
- So(offset, ShouldEqual, -1)
- offset = remoteStore.read(mq, _ReadMemoryThenStore)
- So(offset, ShouldEqual, 1)
-
- })
-
- Convey("test remove", func() {
- remoteStore.update(mq, 1, false)
- offset := remoteStore.read(mq, _ReadFromMemory)
- So(offset, ShouldEqual, 1)
-
- remoteStore.remove(mq)
- offset = remoteStore.read(mq, _ReadFromMemory)
- So(offset, ShouldEqual, -1)
- })
- })
-}
diff --git a/consumer/option.go b/consumer/option.go
deleted file mode 100644
index 330f6c4..0000000
--- a/consumer/option.go
+++ /dev/null
@@ -1,276 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package consumer
-
-import (
- "time"
-
- "github.com/apache/rocketmq-client-go/v2/internal"
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-type consumerOptions struct {
- internal.ClientOptions
-
- /**
- * Backtracking consumption time with second precision. Time format is
- * 20131223171201<br>
- * Implying Seventeen twelve and 01 seconds on December 23, 2013 year<br>
- * Default backtracking consumption time Half an hour ago.
- */
- ConsumeTimestamp string
-
- // The socket timeout in milliseconds
- ConsumerPullTimeout time.Duration
-
- // Concurrently max span offset.it has no effect on sequential consumption
- ConsumeConcurrentlyMaxSpan int
-
- // Flow control threshold on queue level, each message queue will cache at most 1000 messages by default,
- // Consider the {PullBatchSize}, the instantaneous value may exceed the limit
- PullThresholdForQueue int64
-
- // Limit the cached message size on queue level, each message queue will cache at most 100 MiB messages by default,
- // Consider the {@code pullBatchSize}, the instantaneous value may exceed the limit
- //
- // The size of a message only measured by message body, so it's not accurate
- PullThresholdSizeForQueue int
-
- // Flow control threshold on topic level, default value is -1(Unlimited)
- //
- // The value of {@code pullThresholdForQueue} will be overwrote and calculated based on
- // {@code pullThresholdForTopic} if it is't unlimited
- //
- // For example, if the value of pullThresholdForTopic is 1000 and 10 message queues are assigned to this consumer,
- // then pullThresholdForQueue will be set to 100
- PullThresholdForTopic int
-
- // Limit the cached message size on topic level, default value is -1 MiB(Unlimited)
- //
- // The value of {@code pullThresholdSizeForQueue} will be overwrote and calculated based on
- // {@code pullThresholdSizeForTopic} if it is't unlimited
- //
- // For example, if the value of pullThresholdSizeForTopic is 1000 MiB and 10 message queues are
- // assigned to this consumer, then pullThresholdSizeForQueue will be set to 100 MiB
- PullThresholdSizeForTopic int
-
- // Message pull Interval
- PullInterval time.Duration
-
- // Batch consumption size
- ConsumeMessageBatchMaxSize int
-
- // Batch pull size
- PullBatchSize int32
-
- // Whether update subscription relationship when every pull
- PostSubscriptionWhenPull bool
-
- // Max re-consume times. -1 means 16 times.
- //
- // If messages are re-consumed more than {@link #maxReconsumeTimes} before Success, it's be directed to a deletion
- // queue waiting.
- MaxReconsumeTimes int32
-
- // Suspending pulling time for cases requiring slow pulling like flow-control scenario.
- SuspendCurrentQueueTimeMillis time.Duration
-
- // Maximum amount of time a message may block the consuming thread.
- ConsumeTimeout time.Duration
-
- ConsumerModel MessageModel
- Strategy AllocateStrategy
- ConsumeOrderly bool
- FromWhere ConsumeFromWhere
-
- Interceptors []primitive.Interceptor
- // TODO traceDispatcher
- MaxTimeConsumeContinuously time.Duration
- //
- AutoCommit bool
- RebalanceLockInterval time.Duration
-
- Resolver primitive.NsResolver
-}
-
-func defaultPushConsumerOptions() consumerOptions {
- opts := consumerOptions{
- ClientOptions: internal.DefaultClientOptions(),
- Strategy: AllocateByAveragely,
- MaxTimeConsumeContinuously: time.Duration(60 * time.Second),
- RebalanceLockInterval: 20 * time.Second,
- MaxReconsumeTimes: -1,
- ConsumerModel: Clustering,
- AutoCommit: true,
- Resolver: primitive.NewHttpResolver("DEFAULT"),
- }
- opts.ClientOptions.GroupName = "DEFAULT_CONSUMER"
- return opts
-}
-
-type Option func(*consumerOptions)
-
-func defaultPullConsumerOptions() consumerOptions {
- opts := consumerOptions{
- ClientOptions: internal.DefaultClientOptions(),
- Resolver: primitive.NewHttpResolver("DEFAULT"),
- }
- opts.ClientOptions.GroupName = "DEFAULT_CONSUMER"
- return opts
-}
-
-func WithConsumerModel(m MessageModel) Option {
- return func(options *consumerOptions) {
- options.ConsumerModel = m
- }
-}
-
-func WithConsumeFromWhere(w ConsumeFromWhere) Option {
- return func(options *consumerOptions) {
- options.FromWhere = w
- }
-}
-
-func WithConsumerOrder(order bool) Option {
- return func(options *consumerOptions) {
- options.ConsumeOrderly = order
- }
-}
-
-func WithConsumeMessageBatchMaxSize(consumeMessageBatchMaxSize int) Option {
- return func(options *consumerOptions) {
- options.ConsumeMessageBatchMaxSize = consumeMessageBatchMaxSize
- }
-}
-
-// WithChainConsumerInterceptor returns a ConsumerOption that specifies the chained interceptor for consumer.
-// The first interceptor will be the outer most, while the last interceptor will be the inner most wrapper
-// around the real call.
-func WithInterceptor(fs ...primitive.Interceptor) Option {
- return func(options *consumerOptions) {
- options.Interceptors = append(options.Interceptors, fs...)
- }
-}
-
-// WithGroupName set group name address
-func WithGroupName(group string) Option {
- return func(opts *consumerOptions) {
- if group == "" {
- return
- }
- opts.GroupName = group
- }
-}
-
-func WithInstance(name string) Option {
- return func(options *consumerOptions) {
- options.InstanceName = name
- }
-}
-
-// WithNamespace set the namespace of consumer
-func WithNamespace(namespace string) Option {
- return func(opts *consumerOptions) {
- opts.Namespace = namespace
- }
-}
-
-func WithVIPChannel(enable bool) Option {
- return func(opts *consumerOptions) {
- opts.VIPChannelEnabled = enable
- }
-}
-
-// WithRetry return a Option that specifies the retry times when send failed.
-// TODO: use retry middleware instead
-func WithRetry(retries int) Option {
- return func(opts *consumerOptions) {
- opts.RetryTimes = retries
- }
-}
-
-func WithCredentials(c primitive.Credentials) Option {
- return func(options *consumerOptions) {
- options.ClientOptions.Credentials = c
- }
-}
-
-// WithMaxReconsumeTimes set MaxReconsumeTimes of options, if message reconsume greater than MaxReconsumeTimes, it will
-// be sent to retry or dlq topic. more info reference by examples/consumer/retry.
-func WithMaxReconsumeTimes(times int32) Option {
- return func(opts *consumerOptions) {
- opts.MaxReconsumeTimes = times
- }
-}
-
-func WithStrategy(strategy AllocateStrategy) Option {
- return func(opts *consumerOptions) {
- opts.Strategy = strategy
- }
-}
-
-func WithPullBatchSize(batchSize int32) Option {
- return func(options *consumerOptions) {
- options.PullBatchSize = batchSize
- }
-}
-
-func WithRebalanceLockInterval(interval time.Duration) Option {
- return func(options *consumerOptions) {
- options.RebalanceLockInterval = interval
- }
-}
-
-func WithAutoCommit(auto bool) Option {
- return func(options *consumerOptions) {
- options.AutoCommit = auto
- }
-}
-
-func WithSuspendCurrentQueueTimeMillis(suspendT time.Duration) Option {
- return func(options *consumerOptions) {
- options.SuspendCurrentQueueTimeMillis = suspendT
- }
-}
-
-func WithPullInterval(interval time.Duration) Option {
- return func(options *consumerOptions) {
- options.PullInterval = interval
- }
-}
-
-// WithNsResolver set nameserver resolver to fetch nameserver addr
-func WithNsResolver(resolver primitive.NsResolver) Option {
- return func(options *consumerOptions) {
- options.Resolver = resolver
- }
-}
-
-// WithNameServer set NameServer address, only support one NameServer cluster in alpha2
-func WithNameServer(nameServers primitive.NamesrvAddr) Option {
- return func(options *consumerOptions) {
- options.Resolver = primitive.NewPassthroughResolver(nameServers)
- }
-}
-
-// WithNameServerDomain set NameServer domain
-func WithNameServerDomain(nameServerUrl string) Option {
- return func(opts *consumerOptions) {
- opts.Resolver = primitive.NewHttpResolver("DEFAULT", nameServerUrl)
- }
-}
diff --git a/consumer/process_queue.go b/consumer/process_queue.go
deleted file mode 100644
index 76a9236..0000000
--- a/consumer/process_queue.go
+++ /dev/null
@@ -1,397 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package consumer
-
-import (
- "strconv"
- "sync"
- "sync/atomic"
- "time"
-
- "github.com/emirpasic/gods/maps/treemap"
- "github.com/emirpasic/gods/utils"
- gods_util "github.com/emirpasic/gods/utils"
- uatomic "go.uber.org/atomic"
-
- "github.com/apache/rocketmq-client-go/v2/internal"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/rlog"
-)
-
-const (
- _RebalanceLockMaxTime = 30 * time.Second
- _RebalanceInterval = 20 * time.Second
- _PullMaxIdleTime = 120 * time.Second
-)
-
-type processQueue struct {
- cachedMsgCount int64
- cachedMsgSize int64
- tryUnlockTimes int64
- queueOffsetMax int64
- msgAccCnt int64
- msgCache *treemap.Map
- mutex sync.RWMutex
- consumeLock sync.Mutex
- consumingMsgOrderlyTreeMap *treemap.Map
- dropped *uatomic.Bool
- lastPullTime atomic.Value
- lastConsumeTime atomic.Value
- locked *uatomic.Bool
- lastLockTime atomic.Value
- consuming bool
- lockConsume sync.Mutex
- msgCh chan []*primitive.MessageExt
- order bool
-}
-
-func newProcessQueue(order bool) *processQueue {
- consumingMsgOrderlyTreeMap := treemap.NewWith(gods_util.Int64Comparator)
-
- lastConsumeTime := atomic.Value{}
- lastConsumeTime.Store(time.Now())
-
- lastLockTime := atomic.Value{}
- lastLockTime.Store(time.Now())
-
- lastPullTime := atomic.Value{}
- lastPullTime.Store(time.Now())
-
- pq := &processQueue{
- msgCache: treemap.NewWith(utils.Int64Comparator),
- lastPullTime: lastPullTime,
- lastConsumeTime: lastConsumeTime,
- lastLockTime: lastLockTime,
- msgCh: make(chan []*primitive.MessageExt, 32),
- consumingMsgOrderlyTreeMap: consumingMsgOrderlyTreeMap,
- order: order,
- locked: uatomic.NewBool(false),
- dropped: uatomic.NewBool(false),
- }
- return pq
-}
-
-func (pq *processQueue) putMessage(messages ...*primitive.MessageExt) {
- if len(messages) == 0 {
- return
- }
- pq.mutex.Lock()
- if pq.IsDroppd() {
- pq.mutex.Unlock()
- return
- }
- if !pq.order {
- pq.msgCh <- messages
- }
- validMessageCount := 0
- for idx := range messages {
- msg := messages[idx]
- _, found := pq.msgCache.Get(msg.QueueOffset)
- if found {
- continue
- }
- _, found = pq.consumingMsgOrderlyTreeMap.Get(msg.QueueOffset)
- if found {
- continue
- }
- pq.msgCache.Put(msg.QueueOffset, msg)
- validMessageCount++
- pq.queueOffsetMax = msg.QueueOffset
- atomic.AddInt64(&pq.cachedMsgSize, int64(len(msg.Body)))
- }
- pq.mutex.Unlock()
-
- atomic.AddInt64(&pq.cachedMsgCount, int64(validMessageCount))
-
- if pq.msgCache.Size() > 0 && !pq.consuming {
- pq.consuming = true
- }
-
- msg := messages[len(messages)-1]
- maxOffset, err := strconv.ParseInt(msg.GetProperty(primitive.PropertyMaxOffset), 10, 64)
- if err != nil {
- acc := maxOffset - msg.QueueOffset
- if acc > 0 {
- pq.msgAccCnt = acc
- }
- }
-}
-
-func (pq *processQueue) WithLock(lock bool) {
- pq.locked.Store(lock)
-}
-
-func (pq *processQueue) IsLock() bool {
- return pq.locked.Load()
-}
-
-func (pq *processQueue) WithDropped(dropped bool) {
- pq.dropped.Store(dropped)
-}
-
-func (pq *processQueue) IsDroppd() bool {
- return pq.dropped.Load()
-}
-
-func (pq *processQueue) UpdateLastConsumeTime() {
- pq.lastConsumeTime.Store(time.Now())
-}
-
-func (pq *processQueue) LastConsumeTime() time.Time {
- return pq.lastConsumeTime.Load().(time.Time)
-}
-
-func (pq *processQueue) UpdateLastLockTime() {
- pq.lastLockTime.Store(time.Now())
-}
-
-func (pq *processQueue) LastLockTime() time.Time {
- return pq.lastLockTime.Load().(time.Time)
-}
-
-func (pq *processQueue) LastPullTime() time.Time {
- return pq.lastPullTime.Load().(time.Time)
-}
-
-func (pq *processQueue) UpdateLastPullTime() {
- pq.lastPullTime.Store(time.Now())
-}
-
-func (pq *processQueue) makeMessageToCosumeAgain(messages ...*primitive.MessageExt) {
- pq.mutex.Lock()
- for _, msg := range messages {
- pq.consumingMsgOrderlyTreeMap.Remove(msg.QueueOffset)
- pq.msgCache.Put(msg.QueueOffset, msg)
- }
-
- pq.mutex.Unlock()
-}
-
-func (pq *processQueue) removeMessage(messages ...*primitive.MessageExt) int64 {
- result := int64(-1)
- pq.mutex.Lock()
- pq.UpdateLastConsumeTime()
- if !pq.msgCache.Empty() {
- result = pq.queueOffsetMax + 1
- removedCount := 0
- for idx := range messages {
- msg := messages[idx]
- _, found := pq.msgCache.Get(msg.QueueOffset)
- if !found {
- continue
- }
- pq.msgCache.Remove(msg.QueueOffset)
- removedCount++
- atomic.AddInt64(&pq.cachedMsgSize, int64(-len(msg.Body)))
- }
- atomic.AddInt64(&pq.cachedMsgCount, int64(-removedCount))
- }
- if !pq.msgCache.Empty() {
- first, _ := pq.msgCache.Min()
- result = first.(int64)
- }
- pq.mutex.Unlock()
- return result
-}
-
-func (pq *processQueue) isLockExpired() bool {
- return time.Now().Sub(pq.LastLockTime()) > _RebalanceLockMaxTime
-}
-
-func (pq *processQueue) isPullExpired() bool {
- return time.Now().Sub(pq.LastPullTime()) > _PullMaxIdleTime
-}
-
-func (pq *processQueue) cleanExpiredMsg(consumer defaultConsumer) {
- if consumer.option.ConsumeOrderly {
- return
- }
- var loop = 16
- if pq.msgCache.Size() < 16 {
- loop = pq.msgCache.Size()
- }
-
- for i := 0; i < loop; i++ {
- pq.mutex.RLock()
- if pq.msgCache.Empty() {
- pq.mutex.RLock()
- return
- }
- _, firstValue := pq.msgCache.Min()
- msg := firstValue.(*primitive.MessageExt)
- startTime := msg.GetProperty(primitive.PropertyConsumeStartTime)
- if startTime != "" {
- st, err := strconv.ParseInt(startTime, 10, 64)
- if err != nil {
- rlog.Warning("parse message start consume time error", map[string]interface{}{
- "time": startTime,
- rlog.LogKeyUnderlayError: err,
- })
- continue
- }
- if time.Now().Unix()-st <= int64(consumer.option.ConsumeTimeout) {
- pq.mutex.RLock()
- return
- }
- }
- pq.mutex.RLock()
-
- err := consumer.sendBack(msg, 3)
- if err != nil {
- rlog.Error("send message back to broker error when clean expired messages", map[string]interface{}{
- rlog.LogKeyUnderlayError: err,
- })
- continue
- }
- pq.removeMessage(msg)
- }
-}
-
-func (pq *processQueue) getMaxSpan() int {
- pq.mutex.RLock()
- defer pq.mutex.RUnlock()
- if pq.msgCache.Size() == 0 {
- return 0
- }
- firstKey, _ := pq.msgCache.Min()
- lastKey, _ := pq.msgCache.Max()
- return int(lastKey.(int64) - firstKey.(int64))
-}
-
-func (pq *processQueue) getMessages() []*primitive.MessageExt {
- return <-pq.msgCh
-}
-
-func (pq *processQueue) takeMessages(number int) []*primitive.MessageExt {
- for pq.msgCache.Empty() {
- time.Sleep(10 * time.Millisecond)
- }
- result := make([]*primitive.MessageExt, number)
- i := 0
- pq.mutex.Lock()
- for ; i < number; i++ {
- k, v := pq.msgCache.Min()
- if v == nil {
- break
- }
- result[i] = v.(*primitive.MessageExt)
- pq.consumingMsgOrderlyTreeMap.Put(k, v)
- pq.msgCache.Remove(k)
- }
- pq.mutex.Unlock()
- return result[:i]
-}
-
-func (pq *processQueue) Min() int64 {
- if pq.msgCache.Empty() {
- return -1
- }
- k, _ := pq.msgCache.Min()
- if k != nil {
- return k.(int64)
- }
- return -1
-}
-
-func (pq *processQueue) Max() int64 {
- if pq.msgCache.Empty() {
- return -1
- }
- k, _ := pq.msgCache.Max()
- if k != nil {
- return k.(int64)
- }
- return -1
-}
-
-func (pq *processQueue) MinOrderlyCache() int64 {
- if pq.consumingMsgOrderlyTreeMap.Empty() {
- return -1
- }
- k, _ := pq.consumingMsgOrderlyTreeMap.Min()
- if k != nil {
- return k.(int64)
- }
- return -1
-}
-
-func (pq *processQueue) MaxOrderlyCache() int64 {
- if pq.consumingMsgOrderlyTreeMap.Empty() {
- return -1
- }
- k, _ := pq.consumingMsgOrderlyTreeMap.Max()
- if k != nil {
- return k.(int64)
- }
- return -1
-}
-
-func (pq *processQueue) clear() {
- pq.mutex.Lock()
- defer pq.mutex.Unlock()
- pq.msgCache.Clear()
- pq.cachedMsgCount = 0
- pq.cachedMsgSize = 0
- pq.queueOffsetMax = 0
-}
-
-func (pq *processQueue) commit() int64 {
- pq.mutex.Lock()
- defer pq.mutex.Unlock()
-
- var offset int64
- iter, _ := pq.consumingMsgOrderlyTreeMap.Max()
- if iter != nil {
- offset = iter.(int64)
- }
- pq.cachedMsgCount -= int64(pq.consumingMsgOrderlyTreeMap.Size())
- pq.consumingMsgOrderlyTreeMap.Each(func(key interface{}, value interface{}) {
- msg := value.(*primitive.MessageExt)
- pq.cachedMsgSize -= int64(len(msg.Body))
- })
- pq.consumingMsgOrderlyTreeMap.Clear()
- return offset + 1
-}
-
-func (pq *processQueue) currentInfo() internal.ProcessQueueInfo {
- pq.mutex.RLock()
- defer pq.mutex.RUnlock()
- info := internal.ProcessQueueInfo{
- Locked: pq.locked.Load(),
- TryUnlockTimes: pq.tryUnlockTimes,
- LastLockTimestamp: pq.LastLockTime().UnixNano() / int64(time.Millisecond),
- Dropped: pq.dropped.Load(),
- LastPullTimestamp: pq.LastPullTime().UnixNano() / int64(time.Millisecond),
- LastConsumeTimestamp: pq.LastConsumeTime().UnixNano() / int64(time.Millisecond),
- }
-
- if !pq.msgCache.Empty() {
- info.CachedMsgMinOffset = pq.Min()
- info.CachedMsgMaxOffset = pq.Max()
- info.CachedMsgCount = pq.msgCache.Size()
- info.CachedMsgSizeInMiB = pq.cachedMsgSize / int64(1024*1024)
- }
-
- if !pq.consumingMsgOrderlyTreeMap.Empty() {
- info.TransactionMsgMinOffset = pq.MinOrderlyCache()
- info.TransactionMsgMaxOffset = pq.MaxOrderlyCache()
- info.TransactionMsgCount = pq.consumingMsgOrderlyTreeMap.Size()
- }
-
- return info
-}
diff --git a/consumer/pull_consumer.go b/consumer/pull_consumer.go
deleted file mode 100644
index 874973b..0000000
--- a/consumer/pull_consumer.go
+++ /dev/null
@@ -1,256 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package consumer
-
-import (
- "context"
- "fmt"
- errors2 "github.com/apache/rocketmq-client-go/v2/errors"
- "sync"
- "sync/atomic"
-
- "github.com/pkg/errors"
-
- "github.com/apache/rocketmq-client-go/v2/internal"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/rlog"
-)
-
-type PullConsumer interface {
- // Start
- Start()
-
- // Shutdown refuse all new pull operation, finish all submitted.
- Shutdown()
-
- // Pull pull message of topic, selector indicate which queue to pull.
- Pull(ctx context.Context, topic string, selector MessageSelector, numbers int) (*primitive.PullResult, error)
-
- // PullFrom pull messages of queue from the offset to offset + numbers
- PullFrom(ctx context.Context, queue *primitive.MessageQueue, offset int64, numbers int) (*primitive.PullResult, error)
-
- // updateOffset update offset of queue in mem
- UpdateOffset(queue *primitive.MessageQueue, offset int64) error
-
- // PersistOffset persist all offset in mem.
- PersistOffset(ctx context.Context) error
-
- // CurrentOffset return the current offset of queue in mem.
- CurrentOffset(queue *primitive.MessageQueue) (int64, error)
-}
-
-var (
- queueCounterTable sync.Map
-)
-
-type defaultPullConsumer struct {
- *defaultConsumer
-
- option consumerOptions
- client internal.RMQClient
- GroupName string
- Model MessageModel
- UnitMode bool
-
- interceptor primitive.Interceptor
-}
-
-func NewPullConsumer(options ...Option) (*defaultPullConsumer, error) {
- defaultOpts := defaultPullConsumerOptions()
- for _, apply := range options {
- apply(&defaultOpts)
- }
-
- srvs, err := internal.NewNamesrv(defaultOpts.Resolver)
- if err != nil {
- return nil, errors.Wrap(err, "new Namesrv failed.")
- }
-
- defaultOpts.Namesrv = srvs
- dc := &defaultConsumer{
- client: internal.GetOrNewRocketMQClient(defaultOpts.ClientOptions, nil),
- consumerGroup: defaultOpts.GroupName,
- cType: _PullConsume,
- state: int32(internal.StateCreateJust),
- prCh: make(chan PullRequest, 4),
- model: defaultOpts.ConsumerModel,
- option: defaultOpts,
- }
- if dc.client == nil {
- return nil, fmt.Errorf("GetOrNewRocketMQClient faild")
- }
- defaultOpts.Namesrv = dc.client.GetNameSrv()
-
- c := &defaultPullConsumer{
- defaultConsumer: dc,
- }
- return c, nil
-}
-
-func (c *defaultPullConsumer) Start() error {
- atomic.StoreInt32(&c.state, int32(internal.StateRunning))
-
- var err error
- c.once.Do(func() {
- err = c.start()
- if err != nil {
- return
- }
- })
-
- return err
-}
-
-func (c *defaultPullConsumer) Pull(ctx context.Context, topic string, selector MessageSelector, numbers int) (*primitive.PullResult, error) {
- mq := c.getNextQueueOf(topic)
- if mq == nil {
- return nil, fmt.Errorf("prepard to pull topic: %s, but no queue is founded", topic)
- }
-
- data := buildSubscriptionData(mq.Topic, selector)
- result, err := c.pull(context.Background(), mq, data, c.nextOffsetOf(mq), numbers)
-
- if err != nil {
- return nil, err
- }
-
- c.processPullResult(mq, result, data)
- return result, nil
-}
-
-func (c *defaultPullConsumer) getNextQueueOf(topic string) *primitive.MessageQueue {
- queues, err := c.defaultConsumer.client.GetNameSrv().FetchSubscribeMessageQueues(topic)
- if err != nil && len(queues) > 0 {
- rlog.Error("get next mq error", map[string]interface{}{
- rlog.LogKeyTopic: topic,
- rlog.LogKeyUnderlayError: err.Error(),
- })
- return nil
- }
- var index int64
- v, exist := queueCounterTable.Load(topic)
- if !exist {
- index = -1
- queueCounterTable.Store(topic, int64(0))
- } else {
- index = v.(int64)
- }
-
- return queues[int(atomic.AddInt64(&index, 1))%len(queues)]
-}
-
-// SubscribeWithChan ack manually
-func (c *defaultPullConsumer) SubscribeWithChan(topic, selector MessageSelector) (chan *primitive.Message, error) {
- return nil, nil
-}
-
-// SubscribeWithFunc ack automatic
-func (c *defaultPullConsumer) SubscribeWithFunc(topic, selector MessageSelector,
- f func(msg *primitive.Message) ConsumeResult) error {
- return nil
-}
-
-func (c *defaultPullConsumer) ACK(msg *primitive.Message, result ConsumeResult) {
-
-}
-
-func (dc *defaultConsumer) checkPull(ctx context.Context, mq *primitive.MessageQueue, offset int64, numbers int) error {
- err := dc.makeSureStateOK()
- if err != nil {
- return err
- }
-
- if mq == nil {
- return errors2.ErrMQEmpty
- }
-
- if offset < 0 {
- return errors2.ErrOffset
- }
-
- if numbers <= 0 {
- return errors2.ErrNumbers
- }
- return nil
-}
-
-// TODO: add timeout limit
-// TODO: add hook
-func (c *defaultPullConsumer) pull(ctx context.Context, mq *primitive.MessageQueue, data *internal.SubscriptionData,
- offset int64, numbers int) (*primitive.PullResult, error) {
-
- if err := c.checkPull(ctx, mq, offset, numbers); err != nil {
- return nil, err
- }
-
- c.subscriptionAutomatically(mq.Topic)
-
- sysFlag := buildSysFlag(false, true, true, false)
-
- pullResp, err := c.pullInner(ctx, mq, data, offset, numbers, sysFlag, 0)
- if err != nil {
- return nil, err
- }
- c.processPullResult(mq, pullResp, data)
-
- return pullResp, err
-}
-
-func (c *defaultPullConsumer) makeSureStateOK() error {
- if atomic.LoadInt32(&c.state) != int32(internal.StateRunning) {
- return fmt.Errorf("the consumer state is [%d], not running", c.state)
- }
- return nil
-}
-
-func (c *defaultPullConsumer) nextOffsetOf(queue *primitive.MessageQueue) int64 {
- return c.computePullFromWhere(queue)
-}
-
-// PullFrom pull messages of queue from the offset to offset + numbers
-func (c *defaultPullConsumer) PullFrom(ctx context.Context, queue *primitive.MessageQueue, offset int64, numbers int) (*primitive.PullResult, error) {
- if err := c.checkPull(ctx, queue, offset, numbers); err != nil {
- return nil, err
- }
-
- selector := MessageSelector{}
- data := buildSubscriptionData(queue.Topic, selector)
-
- return c.pull(ctx, queue, data, offset, numbers)
-}
-
-// updateOffset update offset of queue in mem
-func (c *defaultPullConsumer) UpdateOffset(queue *primitive.MessageQueue, offset int64) error {
- return c.updateOffset(queue, offset)
-}
-
-// PersistOffset persist all offset in mem.
-func (c *defaultPullConsumer) PersistOffset(ctx context.Context) error {
- return c.persistConsumerOffset()
-}
-
-// CurrentOffset return the current offset of queue in mem.
-func (c *defaultPullConsumer) CurrentOffset(queue *primitive.MessageQueue) (int64, error) {
- v := c.queryOffset(queue)
- return v, nil
-}
-
-// Shutdown close defaultConsumer, refuse new request.
-func (c *defaultPullConsumer) Shutdown() error {
- return c.defaultConsumer.shutdown()
-}
diff --git a/consumer/pull_consumer_test.go b/consumer/pull_consumer_test.go
deleted file mode 100644
index f7bc454..0000000
--- a/consumer/pull_consumer_test.go
+++ /dev/null
@@ -1,18 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package consumer
diff --git a/consumer/push_consumer.go b/consumer/push_consumer.go
deleted file mode 100644
index 8642aa4..0000000
--- a/consumer/push_consumer.go
+++ /dev/null
@@ -1,1254 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package consumer
-
-import (
- "context"
- "fmt"
- errors2 "github.com/apache/rocketmq-client-go/v2/errors"
- "math"
- "strconv"
- "strings"
- "sync"
- "sync/atomic"
- "time"
-
- "github.com/pkg/errors"
-
- "github.com/apache/rocketmq-client-go/v2/internal"
- "github.com/apache/rocketmq-client-go/v2/internal/remote"
- "github.com/apache/rocketmq-client-go/v2/internal/utils"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/rlog"
-)
-
-// In most scenarios, this is the mostly recommended usage to consume messages.
-//
-// Technically speaking, this push client is virtually a wrapper of the underlying pull service. Specifically, on
-// arrival of messages pulled from brokers, it roughly invokes the registered callback handler to feed the messages.
-//
-// See quick start/Consumer in the example module for a typical usage.
-//
-// <strong>Thread Safety:</strong> After initialization, the instance can be regarded as thread-safe.
-
-const (
- Mb = 1024 * 1024
-)
-
-type PushConsumerCallback struct {
- topic string
- f func(context.Context, ...*primitive.MessageExt) (ConsumeResult, error)
-}
-
-func (callback PushConsumerCallback) UniqueID() string {
- return callback.topic
-}
-
-type pushConsumer struct {
- *defaultConsumer
- queueFlowControlTimes int
- queueMaxSpanFlowControlTimes int
- consumeFunc utils.Set
- submitToConsume func(*processQueue, *primitive.MessageQueue)
- subscribedTopic map[string]string
- interceptor primitive.Interceptor
- queueLock *QueueLock
- done chan struct{}
- closeOnce sync.Once
-}
-
-func NewPushConsumer(opts ...Option) (*pushConsumer, error) {
- defaultOpts := defaultPushConsumerOptions()
- for _, apply := range opts {
- apply(&defaultOpts)
- }
- srvs, err := internal.NewNamesrv(defaultOpts.Resolver)
- if err != nil {
- return nil, errors.Wrap(err, "new Namesrv failed.")
- }
- if !defaultOpts.Credentials.IsEmpty() {
- srvs.SetCredentials(defaultOpts.Credentials)
- }
- defaultOpts.Namesrv = srvs
-
- if defaultOpts.Namespace != "" {
- defaultOpts.GroupName = defaultOpts.Namespace + "%" + defaultOpts.GroupName
- }
-
- dc := &defaultConsumer{
- client: internal.GetOrNewRocketMQClient(defaultOpts.ClientOptions, nil),
- consumerGroup: defaultOpts.GroupName,
- cType: _PushConsume,
- state: int32(internal.StateCreateJust),
- prCh: make(chan PullRequest, 4),
- model: defaultOpts.ConsumerModel,
- consumeOrderly: defaultOpts.ConsumeOrderly,
- fromWhere: defaultOpts.FromWhere,
- allocate: defaultOpts.Strategy,
- option: defaultOpts,
- }
- if dc.client == nil {
- return nil, fmt.Errorf("GetOrNewRocketMQClient faild")
- }
- defaultOpts.Namesrv = dc.client.GetNameSrv()
-
- p := &pushConsumer{
- defaultConsumer: dc,
- subscribedTopic: make(map[string]string, 0),
- queueLock: newQueueLock(),
- done: make(chan struct{}, 1),
- consumeFunc: utils.NewSet(),
- }
- dc.mqChanged = p.messageQueueChanged
- if p.consumeOrderly {
- p.submitToConsume = p.consumeMessageOrderly
- } else {
- p.submitToConsume = p.consumeMessageCurrently
- }
-
- p.interceptor = primitive.ChainInterceptors(p.option.Interceptors...)
-
- return p, nil
-}
-
-func (pc *pushConsumer) Start() error {
- var err error
- pc.once.Do(func() {
- rlog.Info("the consumer start beginning", map[string]interface{}{
- rlog.LogKeyConsumerGroup: pc.consumerGroup,
- "messageModel": pc.model,
- "unitMode": pc.unitMode,
- })
- atomic.StoreInt32(&pc.state, int32(internal.StateStartFailed))
- pc.validate()
-
- err = pc.client.RegisterConsumer(pc.consumerGroup, pc)
- if err != nil {
- rlog.Error("the consumer group has been created, specify another one", map[string]interface{}{
- rlog.LogKeyConsumerGroup: pc.consumerGroup,
- })
- err = errors2.ErrCreated
- return
- }
-
- err = pc.defaultConsumer.start()
- if err != nil {
- return
- }
-
- go func() {
- // todo start clean msg expired
- for {
- select {
- case pr := <-pc.prCh:
- go func() {
- pc.pullMessage(&pr)
- }()
- case <-pc.done:
- rlog.Info("push consumer close pullConsumer listener.", map[string]interface{}{
- rlog.LogKeyConsumerGroup: pc.consumerGroup,
- })
- return
- }
- }
- }()
-
- go primitive.WithRecover(func() {
- // initial lock.
- if !pc.consumeOrderly {
- return
- }
-
- time.Sleep(1000 * time.Millisecond)
- pc.lockAll()
-
- lockTicker := time.NewTicker(pc.option.RebalanceLockInterval)
- defer lockTicker.Stop()
- for {
- select {
- case <-lockTicker.C:
- pc.lockAll()
- case <-pc.done:
- rlog.Info("push consumer close tick.", map[string]interface{}{
- rlog.LogKeyConsumerGroup: pc.consumerGroup,
- })
- return
- }
- }
- })
- })
-
- if err != nil {
- return err
- }
-
- pc.client.UpdateTopicRouteInfo()
- for k := range pc.subscribedTopic {
- _, exist := pc.topicSubscribeInfoTable.Load(k)
- if !exist {
- pc.client.Shutdown()
- return fmt.Errorf("the topic=%s route info not found, it may not exist", k)
- }
- }
- pc.client.CheckClientInBroker()
- pc.client.SendHeartbeatToAllBrokerWithLock()
- pc.client.RebalanceImmediately()
-
- return err
-}
-
-func (pc *pushConsumer) Shutdown() error {
- var err error
- pc.closeOnce.Do(func() {
- close(pc.done)
-
- pc.client.UnregisterConsumer(pc.consumerGroup)
- err = pc.defaultConsumer.shutdown()
- })
-
- return err
-}
-
-func (pc *pushConsumer) Subscribe(topic string, selector MessageSelector,
- f func(context.Context, ...*primitive.MessageExt) (ConsumeResult, error)) error {
- if atomic.LoadInt32(&pc.state) == int32(internal.StateStartFailed) ||
- atomic.LoadInt32(&pc.state) == int32(internal.StateShutdown) {
- return errors2.ErrStartTopic
- }
-
- if pc.option.Namespace != "" {
- topic = pc.option.Namespace + "%" + topic
- }
- data := buildSubscriptionData(topic, selector)
- pc.subscriptionDataTable.Store(topic, data)
- pc.subscribedTopic[topic] = ""
-
- pc.consumeFunc.Add(&PushConsumerCallback{
- f: f,
- topic: topic,
- })
- return nil
-}
-
-func (pc *pushConsumer) Unsubscribe(topic string) error {
- if pc.option.Namespace != "" {
- topic = pc.option.Namespace + "%" + topic
- }
- pc.subscriptionDataTable.Delete(topic)
- return nil
-}
-
-func (pc *pushConsumer) Rebalance() {
- pc.defaultConsumer.doBalance()
-}
-
-func (pc *pushConsumer) PersistConsumerOffset() error {
- return pc.defaultConsumer.persistConsumerOffset()
-}
-
-func (pc *pushConsumer) UpdateTopicSubscribeInfo(topic string, mqs []*primitive.MessageQueue) {
- pc.defaultConsumer.updateTopicSubscribeInfo(topic, mqs)
-}
-
-func (pc *pushConsumer) IsSubscribeTopicNeedUpdate(topic string) bool {
- return pc.defaultConsumer.isSubscribeTopicNeedUpdate(topic)
-}
-
-func (pc *pushConsumer) SubscriptionDataList() []*internal.SubscriptionData {
- return pc.defaultConsumer.SubscriptionDataList()
-}
-
-func (pc *pushConsumer) IsUnitMode() bool {
- return pc.unitMode
-}
-
-func (pc *pushConsumer) GetcType() string {
- return string(pc.cType)
-}
-
-func (pc *pushConsumer) GetModel() string {
- return pc.model.String()
-}
-
-func (pc *pushConsumer) GetWhere() string {
- switch pc.fromWhere {
- case ConsumeFromLastOffset:
- return "CONSUME_FROM_LAST_OFFSET"
- case ConsumeFromFirstOffset:
- return "CONSUME_FROM_FIRST_OFFSET"
- case ConsumeFromTimestamp:
- return "CONSUME_FROM_TIMESTAMP"
- default:
- return "UNKOWN"
- }
-
-}
-
-func (pc *pushConsumer) ConsumeMessageDirectly(msg *primitive.MessageExt, brokerName string) *internal.ConsumeMessageDirectlyResult {
- var msgs = []*primitive.MessageExt{msg}
- var mq = &primitive.MessageQueue{
- Topic: msg.Topic,
- BrokerName: brokerName,
- QueueId: msg.Queue.QueueId,
- }
-
- beginTime := time.Now()
- pc.resetRetryAndNamespace(msgs)
- var result ConsumeResult
-
- var err error
- msgCtx := &primitive.ConsumeMessageContext{
- Properties: make(map[string]string),
- ConsumerGroup: pc.consumerGroup,
- MQ: mq,
- Msgs: msgs,
- }
- ctx := context.Background()
- ctx = primitive.WithConsumerCtx(ctx, msgCtx)
- ctx = primitive.WithMethod(ctx, primitive.ConsumerPush)
- concurrentCtx := primitive.NewConsumeConcurrentlyContext()
- concurrentCtx.MQ = *mq
- ctx = primitive.WithConcurrentlyCtx(ctx, concurrentCtx)
-
- result, err = pc.consumeInner(ctx, msgs)
-
- consumeRT := time.Now().Sub(beginTime)
-
- res := &internal.ConsumeMessageDirectlyResult{
- Order: false,
- AutoCommit: true,
- SpentTimeMills: int64(consumeRT / time.Millisecond),
- }
-
- if err != nil {
- msgCtx.Properties[primitive.PropCtxType] = string(primitive.ExceptionReturn)
- res.ConsumeResult = internal.ThrowException
- res.Remark = err.Error()
- } else if result == ConsumeSuccess {
- msgCtx.Properties[primitive.PropCtxType] = string(primitive.SuccessReturn)
- res.ConsumeResult = internal.ConsumeSuccess
- } else if result == ConsumeRetryLater {
- msgCtx.Properties[primitive.PropCtxType] = string(primitive.FailedReturn)
- res.ConsumeResult = internal.ConsumeRetryLater
- }
-
- pc.stat.increaseConsumeRT(pc.consumerGroup, mq.Topic, int64(consumeRT/time.Millisecond))
-
- return res
-}
-
-func (pc *pushConsumer) GetConsumerRunningInfo() *internal.ConsumerRunningInfo {
- info := internal.NewConsumerRunningInfo()
-
- pc.subscriptionDataTable.Range(func(key, value interface{}) bool {
- topic := key.(string)
- info.SubscriptionData[value.(*internal.SubscriptionData)] = true
- status := internal.ConsumeStatus{
- PullRT: pc.stat.getPullRT(topic, pc.consumerGroup).avgpt,
- PullTPS: pc.stat.getPullTPS(topic, pc.consumerGroup).tps,
- ConsumeRT: pc.stat.getConsumeRT(topic, pc.consumerGroup).avgpt,
- ConsumeOKTPS: pc.stat.getConsumeOKTPS(topic, pc.consumerGroup).tps,
- ConsumeFailedTPS: pc.stat.getConsumeFailedTPS(topic, pc.consumerGroup).tps,
- ConsumeFailedMsgs: pc.stat.topicAndGroupConsumeFailedTPS.getStatsDataInHour(topic + "@" + pc.consumerGroup).sum,
- }
- info.StatusTable[topic] = status
- return true
- })
-
- pc.processQueueTable.Range(func(key, value interface{}) bool {
- mq := key.(primitive.MessageQueue)
- pq := value.(*processQueue)
- pInfo := pq.currentInfo()
- pInfo.CommitOffset = pc.storage.read(&mq, _ReadMemoryThenStore)
- info.MQTable[mq] = pInfo
- return true
- })
-
- nsAddr := ""
- for _, value := range pc.client.GetNameSrv().AddrList() {
- nsAddr += fmt.Sprintf("%s;", value)
- }
- info.Properties[internal.PropNameServerAddr] = nsAddr
- info.Properties[internal.PropConsumeType] = string(pc.cType)
- info.Properties[internal.PropConsumeOrderly] = strconv.FormatBool(pc.consumeOrderly)
- info.Properties[internal.PropThreadPoolCoreSize] = "-1"
- info.Properties[internal.PropConsumerStartTimestamp] = strconv.FormatInt(pc.consumerStartTimestamp, 10)
- return info
-}
-
-func (pc *pushConsumer) messageQueueChanged(topic string, mqAll, mqDivided []*primitive.MessageQueue) {
- v, exit := pc.subscriptionDataTable.Load(topic)
- if !exit {
- return
- }
- data := v.(*internal.SubscriptionData)
- newVersion := time.Now().UnixNano()
- rlog.Info("the MessageQueue changed, version also updated", map[string]interface{}{
- rlog.LogKeyValueChangedFrom: data.SubVersion,
- rlog.LogKeyValueChangedTo: newVersion,
- })
- data.SubVersion = newVersion
-
- // TODO: optimize
- count := 0
- pc.processQueueTable.Range(func(key, value interface{}) bool {
- count++
- return true
- })
- if count > 0 {
- if pc.option.PullThresholdForTopic != -1 {
- newVal := pc.option.PullThresholdForTopic / count
- if newVal == 0 {
- newVal = 1
- }
- rlog.Info("The PullThresholdForTopic is changed", map[string]interface{}{
- rlog.LogKeyValueChangedFrom: pc.option.PullThresholdForTopic,
- rlog.LogKeyValueChangedTo: newVal,
- })
- pc.option.PullThresholdForTopic = newVal
- }
-
- if pc.option.PullThresholdSizeForTopic != -1 {
- newVal := pc.option.PullThresholdSizeForTopic / count
- if newVal == 0 {
- newVal = 1
- }
- rlog.Info("The PullThresholdSizeForTopic is changed", map[string]interface{}{
- rlog.LogKeyValueChangedFrom: pc.option.PullThresholdSizeForTopic,
- rlog.LogKeyValueChangedTo: newVal,
- })
- }
- }
- pc.client.SendHeartbeatToAllBrokerWithLock()
-}
-
-func (pc *pushConsumer) validate() {
- internal.ValidateGroup(pc.consumerGroup)
-
- if pc.consumerGroup == internal.DefaultConsumerGroup {
- // TODO FQA
- rlog.Error(fmt.Sprintf("consumerGroup can't equal [%s], please specify another one.", internal.DefaultConsumerGroup), nil)
- }
-
- if len(pc.subscribedTopic) == 0 {
- rlog.Error("number of subscribed topics is 0.", nil)
- }
-
- if pc.option.ConsumeConcurrentlyMaxSpan < 1 || pc.option.ConsumeConcurrentlyMaxSpan > 65535 {
- if pc.option.ConsumeConcurrentlyMaxSpan == 0 {
- pc.option.ConsumeConcurrentlyMaxSpan = 1000
- } else {
- rlog.Error("option.ConsumeConcurrentlyMaxSpan out of range [1, 65535]", nil)
- }
- }
-
- if pc.option.PullThresholdForQueue < 1 || pc.option.PullThresholdForQueue > 65535 {
- if pc.option.PullThresholdForQueue == 0 {
- pc.option.PullThresholdForQueue = 1024
- } else {
- rlog.Error("option.PullThresholdForQueue out of range [1, 65535]", nil)
- }
- }
-
- if pc.option.PullThresholdForTopic < 1 || pc.option.PullThresholdForTopic > 6553500 {
- if pc.option.PullThresholdForTopic == 0 {
- pc.option.PullThresholdForTopic = 102400
- } else {
- rlog.Error("option.PullThresholdForTopic out of range [1, 6553500]", nil)
- }
- }
-
- if pc.option.PullThresholdSizeForQueue < 1 || pc.option.PullThresholdSizeForQueue > 1024 {
- if pc.option.PullThresholdSizeForQueue == 0 {
- pc.option.PullThresholdSizeForQueue = 512
- } else {
- rlog.Error("option.PullThresholdSizeForQueue out of range [1, 1024]", nil)
- }
- }
-
- if pc.option.PullThresholdSizeForTopic < 1 || pc.option.PullThresholdSizeForTopic > 102400 {
- if pc.option.PullThresholdSizeForTopic == 0 {
- pc.option.PullThresholdSizeForTopic = 51200
- } else {
- rlog.Error("option.PullThresholdSizeForTopic out of range [1, 102400]", nil)
- }
- }
-
- if pc.option.PullInterval < 0 || pc.option.PullInterval > 65535*time.Millisecond {
- rlog.Error("option.PullInterval out of range [0, 65535]", nil)
- }
-
- if pc.option.ConsumeMessageBatchMaxSize < 1 || pc.option.ConsumeMessageBatchMaxSize > 1024 {
- if pc.option.ConsumeMessageBatchMaxSize == 0 {
- pc.option.ConsumeMessageBatchMaxSize = 1
- } else {
- rlog.Error("option.ConsumeMessageBatchMaxSize out of range [1, 1024]", nil)
- }
- }
-
- if pc.option.PullBatchSize < 1 || pc.option.PullBatchSize > 1024 {
- if pc.option.PullBatchSize == 0 {
- pc.option.PullBatchSize = 32
- } else {
- rlog.Error("option.PullBatchSize out of range [1, 1024]", nil)
- }
- }
-}
-
-func (pc *pushConsumer) pullMessage(request *PullRequest) {
- rlog.Debug("start a new Pull Message task for PullRequest", map[string]interface{}{
- rlog.LogKeyPullRequest: request.String(),
- })
- var sleepTime time.Duration
- pq := request.pq
- go primitive.WithRecover(func() {
- for {
- select {
- case <-pc.done:
- rlog.Info("push consumer close pullMessage.", map[string]interface{}{
- rlog.LogKeyConsumerGroup: pc.consumerGroup,
- })
- return
- default:
- pc.submitToConsume(request.pq, request.mq)
- if request.pq.IsDroppd() {
- rlog.Info("push consumer quit pullMessage for dropped queue.", map[string]interface{}{
- rlog.LogKeyConsumerGroup: pc.consumerGroup,
- })
- return
- }
- }
- }
- })
-
- for {
- NEXT:
- select {
- case <-pc.done:
- rlog.Info("push consumer close message handle.", map[string]interface{}{
- rlog.LogKeyConsumerGroup: pc.consumerGroup,
- })
- return
- default:
- }
-
- if pq.IsDroppd() {
- rlog.Debug("the request was dropped, so stop task", map[string]interface{}{
- rlog.LogKeyPullRequest: request.String(),
- })
- return
- }
- if sleepTime > 0 {
- rlog.Debug(fmt.Sprintf("pull MessageQueue: %d sleep %d ms for mq: %v", request.mq.QueueId, sleepTime/time.Millisecond, request.mq), nil)
- time.Sleep(sleepTime)
- }
- // reset time
- sleepTime = pc.option.PullInterval
- pq.lastPullTime.Store(time.Now())
- err := pc.makeSureStateOK()
- if err != nil {
- rlog.Warning("consumer state error", map[string]interface{}{
- rlog.LogKeyUnderlayError: err.Error(),
- })
- sleepTime = _PullDelayTimeWhenError
- goto NEXT
- }
-
- if pc.pause {
- rlog.Debug(fmt.Sprintf("consumer [%s] of [%s] was paused, execute pull request [%s] later",
- pc.option.InstanceName, pc.consumerGroup, request.String()), nil)
- sleepTime = _PullDelayTimeWhenSuspend
- goto NEXT
- }
-
- cachedMessageSizeInMiB := int(pq.cachedMsgSize / Mb)
- if pq.cachedMsgCount > pc.option.PullThresholdForQueue {
- if pc.queueFlowControlTimes%1000 == 0 {
- rlog.Warning("the cached message count exceeds the threshold, so do flow control", map[string]interface{}{
- "PullThresholdForQueue": pc.option.PullThresholdForQueue,
- "minOffset": pq.Min(),
- "maxOffset": pq.Max(),
- "count": pq.cachedMsgCount,
- "size(MiB)": cachedMessageSizeInMiB,
- "flowControlTimes": pc.queueFlowControlTimes,
- rlog.LogKeyPullRequest: request.String(),
- })
- }
- pc.queueFlowControlTimes++
- sleepTime = _PullDelayTimeWhenFlowControl
- goto NEXT
- }
-
- if cachedMessageSizeInMiB > pc.option.PullThresholdSizeForQueue {
- if pc.queueFlowControlTimes%1000 == 0 {
- rlog.Warning("the cached message size exceeds the threshold, so do flow control", map[string]interface{}{
- "PullThresholdSizeForQueue": pc.option.PullThresholdSizeForQueue,
- "minOffset": pq.Min(),
- "maxOffset": pq.Max(),
- "count": pq.cachedMsgCount,
- "size(MiB)": cachedMessageSizeInMiB,
- "flowControlTimes": pc.queueFlowControlTimes,
- rlog.LogKeyPullRequest: request.String(),
- })
- }
- pc.queueFlowControlTimes++
- sleepTime = _PullDelayTimeWhenFlowControl
- goto NEXT
- }
-
- if !pc.consumeOrderly {
- if pq.getMaxSpan() > pc.option.ConsumeConcurrentlyMaxSpan {
- if pc.queueMaxSpanFlowControlTimes%1000 == 0 {
- rlog.Warning("the queue's messages span too long, so do flow control", map[string]interface{}{
- "ConsumeConcurrentlyMaxSpan": pc.option.ConsumeConcurrentlyMaxSpan,
- "minOffset": pq.Min(),
- "maxOffset": pq.Max(),
- "maxSpan": pq.getMaxSpan(),
- "flowControlTimes": pc.queueFlowControlTimes,
- rlog.LogKeyPullRequest: request.String(),
- })
- }
- sleepTime = _PullDelayTimeWhenFlowControl
- goto NEXT
- }
- } else {
- if pq.IsLock() {
- if !request.lockedFirst {
- offset := pc.computePullFromWhere(request.mq)
- brokerBusy := offset < request.nextOffset
- rlog.Info("the first time to pull message, so fix offset from broker, offset maybe changed", map[string]interface{}{
- rlog.LogKeyPullRequest: request.String(),
- rlog.LogKeyValueChangedFrom: request.nextOffset,
- rlog.LogKeyValueChangedTo: offset,
- "brokerBusy": brokerBusy,
- })
- if brokerBusy {
- rlog.Info("[NOTIFY_ME] the first time to pull message, but pull request offset larger than "+
- "broker consume offset", map[string]interface{}{"offset": offset})
- }
- request.lockedFirst = true
- request.nextOffset = offset
- }
- } else {
- rlog.Info("pull message later because not locked in broker", map[string]interface{}{
- rlog.LogKeyPullRequest: request.String(),
- })
- sleepTime = _PullDelayTimeWhenError
- goto NEXT
- }
- }
-
- v, exist := pc.subscriptionDataTable.Load(request.mq.Topic)
- if !exist {
- rlog.Info("find the consumer's subscription failed", map[string]interface{}{
- rlog.LogKeyPullRequest: request.String(),
- })
- sleepTime = _PullDelayTimeWhenError
- goto NEXT
- }
- beginTime := time.Now()
- var (
- commitOffsetEnable bool
- commitOffsetValue int64
- subExpression string
- )
-
- if pc.model == Clustering {
- commitOffsetValue = pc.storage.read(request.mq, _ReadFromMemory)
- if commitOffsetValue > 0 {
- commitOffsetEnable = true
- }
- }
-
- sd := v.(*internal.SubscriptionData)
- classFilter := sd.ClassFilterMode
- if pc.option.PostSubscriptionWhenPull && classFilter {
- subExpression = sd.SubString
- }
-
- sysFlag := buildSysFlag(commitOffsetEnable, true, subExpression != "", classFilter)
-
- pullRequest := &internal.PullMessageRequestHeader{
- ConsumerGroup: pc.consumerGroup,
- Topic: request.mq.Topic,
- QueueId: int32(request.mq.QueueId),
- QueueOffset: request.nextOffset,
- MaxMsgNums: pc.option.PullBatchSize,
- SysFlag: sysFlag,
- CommitOffset: commitOffsetValue,
- SubExpression: _SubAll,
- ExpressionType: string(TAG),
- SuspendTimeoutMillis: 20 * time.Second,
- }
- //
- //if data.ExpType == string(TAG) {
- // pullRequest.SubVersion = 0
- //} else {
- // pullRequest.SubVersion = data.SubVersion
- //}
-
- brokerResult := pc.defaultConsumer.tryFindBroker(request.mq)
- if brokerResult == nil {
- rlog.Warning("no broker found for mq", map[string]interface{}{
- rlog.LogKeyPullRequest: request.mq.String(),
- })
- sleepTime = _PullDelayTimeWhenError
- goto NEXT
- }
-
- if brokerResult.Slave {
- pullRequest.SysFlag = clearCommitOffsetFlag(pullRequest.SysFlag)
- }
-
- result, err := pc.client.PullMessage(context.Background(), brokerResult.BrokerAddr, pullRequest)
- if err != nil {
- rlog.Warning("pull message from broker error", map[string]interface{}{
- rlog.LogKeyBroker: brokerResult.BrokerAddr,
- rlog.LogKeyUnderlayError: err.Error(),
- })
- sleepTime = _PullDelayTimeWhenError
- goto NEXT
- }
-
- if result.Status == primitive.PullBrokerTimeout {
- rlog.Warning("pull broker timeout", map[string]interface{}{
- rlog.LogKeyBroker: brokerResult.BrokerAddr,
- })
- sleepTime = _PullDelayTimeWhenError
- goto NEXT
- }
-
- switch result.Status {
- case primitive.PullFound:
- rlog.Debug(fmt.Sprintf("Topic: %s, QueueId: %d found messages.", request.mq.Topic, request.mq.QueueId), nil)
- prevRequestOffset := request.nextOffset
- request.nextOffset = result.NextBeginOffset
-
- rt := time.Now().Sub(beginTime) / time.Millisecond
- pc.stat.increasePullRT(pc.consumerGroup, request.mq.Topic, int64(rt))
-
- pc.processPullResult(request.mq, result, sd)
-
- msgFounded := result.GetMessageExts()
- firstMsgOffset := int64(math.MaxInt64)
- if msgFounded != nil && len(msgFounded) != 0 {
- firstMsgOffset = msgFounded[0].QueueOffset
- pc.stat.increasePullTPS(pc.consumerGroup, request.mq.Topic, len(msgFounded))
- pq.putMessage(msgFounded...)
- }
- if result.NextBeginOffset < prevRequestOffset || firstMsgOffset < prevRequestOffset {
- rlog.Warning("[BUG] pull message result maybe data wrong", map[string]interface{}{
- "nextBeginOffset": result.NextBeginOffset,
- "firstMsgOffset": firstMsgOffset,
- "prevRequestOffset": prevRequestOffset,
- })
- }
- case primitive.PullNoNewMsg, primitive.PullNoMsgMatched:
- request.nextOffset = result.NextBeginOffset
- pc.correctTagsOffset(request)
- case primitive.PullOffsetIllegal:
- rlog.Warning("the pull request offset illegal", map[string]interface{}{
- rlog.LogKeyPullRequest: request.String(),
- "result": result.String(),
- })
- request.nextOffset = result.NextBeginOffset
- pq.WithDropped(true)
- time.Sleep(10 * time.Second)
- pc.storage.update(request.mq, request.nextOffset, false)
- pc.storage.persist([]*primitive.MessageQueue{request.mq})
- pc.processQueueTable.Delete(*request.mq)
- rlog.Warning(fmt.Sprintf("fix the pull request offset: %s", request.String()), nil)
- default:
- rlog.Warning(fmt.Sprintf("unknown pull status: %v", result.Status), nil)
- sleepTime = _PullDelayTimeWhenError
- }
- }
-}
-
-func (pc *pushConsumer) correctTagsOffset(pr *PullRequest) {
- if pr.pq.cachedMsgCount <= 0 {
- pc.storage.update(pr.mq, pr.nextOffset, true)
- }
-}
-
-func (pc *pushConsumer) sendMessageBack(brokerName string, msg *primitive.MessageExt, delayLevel int) bool {
- var brokerAddr string
- if len(brokerName) != 0 {
- brokerAddr = pc.defaultConsumer.client.GetNameSrv().FindBrokerAddrByName(brokerName)
- } else {
- brokerAddr = msg.StoreHost
- }
- _, err := pc.client.InvokeSync(context.Background(), brokerAddr, pc.buildSendBackRequest(msg, delayLevel), 3*time.Second)
- if err != nil {
- return false
- }
- return true
-}
-
-func (pc *pushConsumer) buildSendBackRequest(msg *primitive.MessageExt, delayLevel int) *remote.RemotingCommand {
- req := &internal.ConsumerSendMsgBackRequestHeader{
- Group: pc.consumerGroup,
- OriginTopic: msg.Topic,
- Offset: msg.CommitLogOffset,
- DelayLevel: delayLevel,
- OriginMsgId: msg.MsgId,
- MaxReconsumeTimes: pc.getMaxReconsumeTimes(),
- }
-
- return remote.NewRemotingCommand(internal.ReqConsumerSendMsgBack, req, msg.Body)
-}
-
-func (pc *pushConsumer) suspend() {
- pc.pause = true
- rlog.Info(fmt.Sprintf("suspend consumer: %s", pc.consumerGroup), nil)
-}
-
-func (pc *pushConsumer) resume() {
- pc.pause = false
- pc.doBalance()
- rlog.Info(fmt.Sprintf("resume consumer: %s", pc.consumerGroup), nil)
-}
-
-func (pc *pushConsumer) ResetOffset(topic string, table map[primitive.MessageQueue]int64) {
- //topic := cmd.ExtFields["topic"]
- //group := cmd.ExtFields["group"]
- //if topic == "" || group == "" {
- // rlog.Warning("received reset offset command from: %s, but missing params.", from)
- // return
- //}
- //t, err := strconv.ParseInt(cmd.ExtFields["timestamp"], 10, 64)
- //if err != nil {
- // rlog.Warning("received reset offset command from: %s, but parse time error: %s", err.Error())
- // return
- //}
- //rlog.Infof("invoke reset offset operation from broker. brokerAddr=%s, topic=%s, group=%s, timestamp=%v",
- // from, topic, group, t)
- //
- //offsetTable := make(map[MessageQueue]int64, 0)
- //err = json.Unmarshal(cmd.Body, &offsetTable)
- //if err != nil {
- // rlog.Warning("received reset offset command from: %s, but parse offset table: %s", err.Error())
- // return
- //}
- //v, exist := c.consumerMap.Load(group)
- //if !exist {
- // rlog.Infof("[reset-offset] consumer dose not exist. group=%s", group)
- // return
- //}
- pc.suspend()
- defer pc.resume()
-
- pc.processQueueTable.Range(func(key, value interface{}) bool {
- mq := key.(primitive.MessageQueue)
- pq := value.(*processQueue)
- if _, ok := table[mq]; ok && mq.Topic == topic {
- pq.WithDropped(true)
- pq.clear()
- }
- return true
- })
- time.Sleep(10 * time.Second)
- v, exist := pc.topicSubscribeInfoTable.Load(topic)
- if !exist {
- return
- }
- queuesOfTopic := v.([]*primitive.MessageQueue)
- for _, k := range queuesOfTopic {
- if _, ok := table[*k]; ok {
- pc.storage.update(k, table[*k], false)
- v, exist := pc.processQueueTable.Load(k)
- if !exist {
- continue
- }
- pq := v.(*processQueue)
- pc.removeUnnecessaryMessageQueue(k, pq)
- pc.processQueueTable.Delete(k)
- }
- }
-}
-
-func (pc *pushConsumer) removeUnnecessaryMessageQueue(mq *primitive.MessageQueue, pq *processQueue) bool {
- pc.defaultConsumer.removeUnnecessaryMessageQueue(mq, pq)
- if !pc.consumeOrderly || Clustering != pc.model {
- return true
- }
- // TODO orderly
- return true
-}
-
-func (pc *pushConsumer) consumeInner(ctx context.Context, subMsgs []*primitive.MessageExt) (ConsumeResult, error) {
- if len(subMsgs) == 0 {
- return ConsumeRetryLater, errors.New("msg list empty")
- }
-
- f, exist := pc.consumeFunc.Contains(subMsgs[0].Topic)
-
- // fix lost retry message
- if !exist && strings.HasPrefix(subMsgs[0].Topic, internal.RetryGroupTopicPrefix) {
- f, exist = pc.consumeFunc.Contains(subMsgs[0].GetProperty(primitive.PropertyRetryTopic))
- }
-
- if !exist {
- return ConsumeRetryLater, fmt.Errorf("the consume callback missing for topic: %s", subMsgs[0].Topic)
- }
-
- callback, ok := f.(*PushConsumerCallback)
- if !ok {
- return ConsumeRetryLater, fmt.Errorf("the consume callback assert failed for topic: %s", subMsgs[0].Topic)
- }
- if pc.interceptor == nil {
- return callback.f(ctx, subMsgs...)
- } else {
- var container ConsumeResultHolder
- err := pc.interceptor(ctx, subMsgs, &container, func(ctx context.Context, req, reply interface{}) error {
- msgs := req.([]*primitive.MessageExt)
- r, e := callback.f(ctx, msgs...)
-
- realReply := reply.(*ConsumeResultHolder)
- realReply.ConsumeResult = r
-
- msgCtx, _ := primitive.GetConsumerCtx(ctx)
- msgCtx.Success = realReply.ConsumeResult == ConsumeSuccess
- if realReply.ConsumeResult == ConsumeSuccess {
- msgCtx.Properties[primitive.PropCtxType] = string(primitive.SuccessReturn)
- } else {
- msgCtx.Properties[primitive.PropCtxType] = string(primitive.FailedReturn)
- }
- return e
- })
- return container.ConsumeResult, err
- }
-}
-
-// resetRetryAndNamespace modify retry message.
-func (pc *pushConsumer) resetRetryAndNamespace(subMsgs []*primitive.MessageExt) {
- groupTopic := internal.RetryGroupTopicPrefix + pc.consumerGroup
- beginTime := time.Now()
- for idx := range subMsgs {
- msg := subMsgs[idx]
- retryTopic := msg.GetProperty(primitive.PropertyRetryTopic)
- if retryTopic == "" && groupTopic == msg.Topic {
- msg.Topic = retryTopic
- }
- subMsgs[idx].WithProperty(primitive.PropertyConsumeStartTime, strconv.FormatInt(
- beginTime.UnixNano()/int64(time.Millisecond), 10))
- }
-}
-
-func (pc *pushConsumer) consumeMessageCurrently(pq *processQueue, mq *primitive.MessageQueue) {
- msgs := pq.getMessages()
- if msgs == nil {
- return
- }
- for count := 0; count < len(msgs); count++ {
- var subMsgs []*primitive.MessageExt
- if count+pc.option.ConsumeMessageBatchMaxSize > len(msgs) {
- subMsgs = msgs[count:]
- count = len(msgs)
- } else {
- next := count + pc.option.ConsumeMessageBatchMaxSize
- subMsgs = msgs[count:next]
- count = next - 1
- }
- go primitive.WithRecover(func() {
- RETRY:
- if pq.IsDroppd() {
- rlog.Info("the message queue not be able to consume, because it was dropped", map[string]interface{}{
- rlog.LogKeyMessageQueue: mq.String(),
- rlog.LogKeyConsumerGroup: pc.consumerGroup,
- })
- return
- }
-
- beginTime := time.Now()
- pc.resetRetryAndNamespace(subMsgs)
- var result ConsumeResult
-
- var err error
- msgCtx := &primitive.ConsumeMessageContext{
- Properties: make(map[string]string),
- ConsumerGroup: pc.consumerGroup,
- MQ: mq,
- Msgs: subMsgs,
- }
- ctx := context.Background()
- ctx = primitive.WithConsumerCtx(ctx, msgCtx)
- ctx = primitive.WithMethod(ctx, primitive.ConsumerPush)
- concurrentCtx := primitive.NewConsumeConcurrentlyContext()
- concurrentCtx.MQ = *mq
- ctx = primitive.WithConcurrentlyCtx(ctx, concurrentCtx)
-
- result, err = pc.consumeInner(ctx, subMsgs)
-
- consumeRT := time.Now().Sub(beginTime)
- if err != nil {
- msgCtx.Properties[primitive.PropCtxType] = string(primitive.ExceptionReturn)
- } else if consumeRT >= pc.option.ConsumeTimeout {
- msgCtx.Properties[primitive.PropCtxType] = string(primitive.TimeoutReturn)
- } else if result == ConsumeSuccess {
- msgCtx.Properties[primitive.PropCtxType] = string(primitive.SuccessReturn)
- } else if result == ConsumeRetryLater {
- msgCtx.Properties[primitive.PropCtxType] = string(primitive.FailedReturn)
- }
-
- pc.stat.increaseConsumeRT(pc.consumerGroup, mq.Topic, int64(consumeRT/time.Millisecond))
-
- if !pq.IsDroppd() {
- msgBackFailed := make([]*primitive.MessageExt, 0)
- if result == ConsumeSuccess {
- pc.stat.increaseConsumeOKTPS(pc.consumerGroup, mq.Topic, len(subMsgs))
- } else {
- pc.stat.increaseConsumeFailedTPS(pc.consumerGroup, mq.Topic, len(subMsgs))
- if pc.model == BroadCasting {
- for i := 0; i < len(subMsgs); i++ {
- rlog.Warning("BROADCASTING, the message consume failed, drop it", map[string]interface{}{
- "message": subMsgs[i],
- })
- }
- } else {
- for i := 0; i < len(subMsgs); i++ {
- msg := subMsgs[i]
- if !pc.sendMessageBack(mq.BrokerName, msg, concurrentCtx.DelayLevelWhenNextConsume) {
- msg.ReconsumeTimes += 1
- msgBackFailed = append(msgBackFailed, msg)
- }
- }
- }
- }
-
- offset := pq.removeMessage(subMsgs...)
-
- if offset >= 0 && !pq.IsDroppd() {
- pc.storage.update(mq, int64(offset), true)
- }
- if len(msgBackFailed) > 0 {
- subMsgs = msgBackFailed
- time.Sleep(5 * time.Second)
- goto RETRY
- }
- } else {
- rlog.Warning("processQueue is dropped without process consume result.", map[string]interface{}{
- rlog.LogKeyMessageQueue: mq,
- "message": subMsgs,
- })
- }
- })
- }
-}
-
-func (pc *pushConsumer) consumeMessageOrderly(pq *processQueue, mq *primitive.MessageQueue) {
- if pq.IsDroppd() {
- rlog.Warning("the message queue not be able to consume, because it's dropped.", map[string]interface{}{
- rlog.LogKeyMessageQueue: mq.String(),
- })
- return
- }
-
- lock := pc.queueLock.fetchLock(*mq)
- lock.Lock()
- defer lock.Unlock()
- if pc.model == BroadCasting || (pq.IsLock() && !pq.isLockExpired()) {
- beginTime := time.Now()
-
- continueConsume := true
- for continueConsume {
- if pq.IsDroppd() {
- rlog.Warning("the message queue not be able to consume, because it's dropped.", map[string]interface{}{
- rlog.LogKeyMessageQueue: mq.String(),
- })
- break
- }
- if pc.model == Clustering {
- if !pq.IsLock() {
- rlog.Warning("the message queue not locked, so consume later", map[string]interface{}{
- rlog.LogKeyMessageQueue: mq.String(),
- })
- pc.tryLockLaterAndReconsume(mq, 10)
- return
- }
- if pq.isLockExpired() {
- rlog.Warning("the message queue lock expired, so consume later", map[string]interface{}{
- rlog.LogKeyMessageQueue: mq.String(),
- })
- pc.tryLockLaterAndReconsume(mq, 10)
- return
- }
- }
- interval := time.Now().Sub(beginTime)
- if interval > pc.option.MaxTimeConsumeContinuously {
- time.Sleep(10 * time.Millisecond)
- return
- }
- batchSize := pc.option.ConsumeMessageBatchMaxSize
- msgs := pq.takeMessages(batchSize)
-
- pc.resetRetryAndNamespace(msgs)
-
- if len(msgs) == 0 {
- continueConsume = false
- break
- }
-
- // TODO: add message consumer hook
- beginTime = time.Now()
-
- ctx := context.Background()
- msgCtx := &primitive.ConsumeMessageContext{
- Properties: make(map[string]string),
- ConsumerGroup: pc.consumerGroup,
- MQ: mq,
- Msgs: msgs,
- }
- ctx = primitive.WithConsumerCtx(ctx, msgCtx)
- ctx = primitive.WithMethod(ctx, primitive.ConsumerPush)
-
- orderlyCtx := primitive.NewConsumeOrderlyContext()
- orderlyCtx.MQ = *mq
- ctx = primitive.WithOrderlyCtx(ctx, orderlyCtx)
-
- pq.lockConsume.Lock()
- result, _ := pc.consumeInner(ctx, msgs)
- pq.lockConsume.Unlock()
-
- if result == Rollback || result == SuspendCurrentQueueAMoment {
- rlog.Warning("consumeMessage Orderly return not OK", map[string]interface{}{
- rlog.LogKeyConsumerGroup: pc.consumerGroup,
- "messages": msgs,
- rlog.LogKeyMessageQueue: mq,
- })
- }
-
- // jsut put consumeResult in consumerMessageCtx
- //interval = time.Now().Sub(beginTime)
- //consumeReult := SuccessReturn
- //if interval > pc.option.ConsumeTimeout {
- // consumeReult = TimeoutReturn
- //} else if SuspendCurrentQueueAMoment == result {
- // consumeReult = FailedReturn
- //} else if ConsumeSuccess == result {
- // consumeReult = SuccessReturn
- //}
-
- // process result
- commitOffset := int64(-1)
- if pc.option.AutoCommit {
- switch result {
- case Commit, Rollback:
- rlog.Warning("the message queue consume result is illegal, we think you want to ack these message: %v", map[string]interface{}{
- rlog.LogKeyMessageQueue: mq,
- })
- case ConsumeSuccess:
- commitOffset = pq.commit()
- case SuspendCurrentQueueAMoment:
- if pc.checkReconsumeTimes(msgs) {
- pq.makeMessageToCosumeAgain(msgs...)
- time.Sleep(time.Duration(orderlyCtx.SuspendCurrentQueueTimeMillis) * time.Millisecond)
- continueConsume = false
- } else {
- commitOffset = pq.commit()
- }
- default:
- }
- } else {
- switch result {
- case ConsumeSuccess:
- case Commit:
- commitOffset = pq.commit()
- case Rollback:
- // pq.rollback
- time.Sleep(time.Duration(orderlyCtx.SuspendCurrentQueueTimeMillis) * time.Millisecond)
- continueConsume = false
- case SuspendCurrentQueueAMoment:
- if pc.checkReconsumeTimes(msgs) {
- time.Sleep(time.Duration(orderlyCtx.SuspendCurrentQueueTimeMillis) * time.Millisecond)
- continueConsume = false
- }
- default:
- }
- }
- if commitOffset > 0 && !pq.IsDroppd() {
- _ = pc.updateOffset(mq, commitOffset)
- }
- }
- } else {
- if pq.IsDroppd() {
- rlog.Warning("the message queue not be able to consume, because it's dropped.", map[string]interface{}{
- rlog.LogKeyMessageQueue: mq.String(),
- })
- }
- pc.tryLockLaterAndReconsume(mq, 100)
- }
-}
-
-func (pc *pushConsumer) checkReconsumeTimes(msgs []*primitive.MessageExt) bool {
- suspend := false
- if len(msgs) != 0 {
- maxReconsumeTimes := pc.getOrderlyMaxReconsumeTimes()
- for _, msg := range msgs {
- if msg.ReconsumeTimes > maxReconsumeTimes {
- rlog.Warning(fmt.Sprintf("msg will be send to retry topic due to ReconsumeTimes > %d, \n", maxReconsumeTimes), nil)
- msg.WithProperty("RECONSUME_TIME", strconv.Itoa(int(msg.ReconsumeTimes)))
- if !pc.sendMessageBack("", msg, -1) {
- suspend = true
- msg.ReconsumeTimes += 1
- }
- } else {
- suspend = true
- msg.ReconsumeTimes += 1
- }
- }
- }
- return suspend
-}
-
-func (pc *pushConsumer) getOrderlyMaxReconsumeTimes() int32 {
- if pc.option.MaxReconsumeTimes == -1 {
- return math.MaxInt32
- } else {
- return pc.option.MaxReconsumeTimes
- }
-}
-
-func (pc *pushConsumer) getMaxReconsumeTimes() int32 {
- if pc.option.MaxReconsumeTimes == -1 {
- return 16
- } else {
- return pc.option.MaxReconsumeTimes
- }
-}
-
-func (pc *pushConsumer) tryLockLaterAndReconsume(mq *primitive.MessageQueue, delay int64) {
- time.Sleep(time.Duration(delay) * time.Millisecond)
- if pc.lock(mq) == true {
- pc.submitConsumeRequestLater(10)
- } else {
- pc.submitConsumeRequestLater(3000)
- }
-}
-
-func (pc *pushConsumer) submitConsumeRequestLater(suspendTimeMillis int64) {
- if suspendTimeMillis == -1 {
- suspendTimeMillis = int64(pc.option.SuspendCurrentQueueTimeMillis / time.Millisecond)
- }
- if suspendTimeMillis < 10 {
- suspendTimeMillis = 10
- } else if suspendTimeMillis > 30000 {
- suspendTimeMillis = 30000
- }
- time.Sleep(time.Duration(suspendTimeMillis) * time.Millisecond)
-}
diff --git a/consumer/push_consumer_test.go b/consumer/push_consumer_test.go
deleted file mode 100644
index 78bc1f7..0000000
--- a/consumer/push_consumer_test.go
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package consumer
-
-import (
- "context"
- "github.com/apache/rocketmq-client-go/v2/rlog"
- "testing"
-
- "github.com/apache/rocketmq-client-go/v2/internal"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/golang/mock/gomock"
- . "github.com/smartystreets/goconvey/convey"
-)
-
-func mockB4Start(c *pushConsumer) {
- c.topicSubscribeInfoTable.Store("TopicTest", []*primitive.MessageQueue{})
-}
-
-func TestStart(t *testing.T) {
- Convey("test Start method", t, func() {
- c, _ := NewPushConsumer(
- WithGroupName("testGroup"),
- WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- WithConsumerModel(BroadCasting),
- )
-
- ctrl := gomock.NewController(t)
- defer ctrl.Finish()
-
- client := internal.NewMockRMQClient(ctrl)
- c.client = client
-
- err := c.Subscribe("TopicTest", MessageSelector{}, func(ctx context.Context,
- msgs ...*primitive.MessageExt) (ConsumeResult, error) {
- rlog.Info("Subscribe Callback", map[string]interface{}{
- "msgs": msgs,
- })
- return ConsumeSuccess, nil
- })
-
- _, exists := c.subscriptionDataTable.Load("TopicTest")
- So(exists, ShouldBeTrue)
-
- err = c.Unsubscribe("TopicTest")
- So(err, ShouldBeNil)
- _, exists = c.subscriptionDataTable.Load("TopicTest")
- So(exists, ShouldBeFalse)
-
- err = c.Subscribe("TopicTest", MessageSelector{}, func(ctx context.Context,
- msgs ...*primitive.MessageExt) (ConsumeResult, error) {
- rlog.Info("Subscribe Callback", map[string]interface{}{
- "msgs": msgs,
- })
- return ConsumeSuccess, nil
- })
-
- _, exists = c.subscriptionDataTable.Load("TopicTest")
- So(exists, ShouldBeTrue)
-
- client.EXPECT().ClientID().Return("127.0.0.1@DEFAULT")
- client.EXPECT().Start().Return()
- client.EXPECT().RegisterConsumer(gomock.Any(), gomock.Any()).Return(nil)
- client.EXPECT().UpdateTopicRouteInfo().AnyTimes().Return()
-
- Convey("test topic route info not found", func() {
- client.EXPECT().Shutdown().Return()
- err = c.Start()
- So(err.Error(), ShouldContainSubstring, "route info not found")
- })
-
- Convey("test topic route info found", func() {
- client.EXPECT().RebalanceImmediately().Return()
- client.EXPECT().CheckClientInBroker().Return()
- client.EXPECT().SendHeartbeatToAllBrokerWithLock().Return()
- mockB4Start(c)
- err = c.Start()
- So(err, ShouldBeNil)
- })
- })
-}
diff --git a/consumer/statistics.go b/consumer/statistics.go
deleted file mode 100644
index e9d5d79..0000000
--- a/consumer/statistics.go
+++ /dev/null
@@ -1,485 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package consumer
-
-import (
- "container/list"
- "fmt"
- "sync"
- "sync/atomic"
- "time"
-
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/rlog"
-)
-
-type StatsManager struct {
- startOnce sync.Once
- closeOnce sync.Once
- topicAndGroupConsumeOKTPS *statsItemSet
- topicAndGroupConsumeRT *statsItemSet
- topicAndGroupConsumeFailedTPS *statsItemSet
- topicAndGroupPullTPS *statsItemSet
- topicAndGroupPullRT *statsItemSet
-}
-
-func NewStatsManager() *StatsManager {
- mgr := &StatsManager{}
- mgr.topicAndGroupConsumeOKTPS = newStatsItemSet("CONSUME_OK_TPS")
- mgr.topicAndGroupConsumeRT = newStatsItemSet("CONSUME_RT")
- mgr.topicAndGroupConsumeFailedTPS = newStatsItemSet("CONSUME_FAILED_TPS")
- mgr.topicAndGroupPullTPS = newStatsItemSet("PULL_TPS")
- mgr.topicAndGroupPullRT = newStatsItemSet("PULL_RT")
- return mgr
-}
-
-type ConsumeStatus struct {
- PullRT float64
- PullTPS float64
- ConsumeRT float64
- ConsumeOKTPS float64
- ConsumeFailedTPS float64
- ConsumeFailedMsgs int64
-}
-
-func (mgr *StatsManager) increasePullRT(group, topic string, rt int64) {
- mgr.topicAndGroupPullRT.addValue(topic+"@"+group, rt, 1)
-}
-
-func (mgr *StatsManager) increasePullTPS(group, topic string, msgs int) {
- mgr.topicAndGroupPullTPS.addValue(topic+"@"+group, int64(msgs), 1)
-}
-
-func (mgr *StatsManager) increaseConsumeRT(group, topic string, rt int64) {
- mgr.topicAndGroupConsumeRT.addValue(topic+"@"+group, rt, 1)
-}
-
-func (mgr *StatsManager) increaseConsumeOKTPS(group, topic string, msgs int) {
- mgr.topicAndGroupConsumeOKTPS.addValue(topic+"@"+group, int64(msgs), 1)
-}
-
-func (mgr *StatsManager) increaseConsumeFailedTPS(group, topic string, msgs int) {
- mgr.topicAndGroupConsumeFailedTPS.addValue(topic+"@"+group, int64(msgs), 1)
-}
-
-func (mgr *StatsManager) GetConsumeStatus(group, topic string) ConsumeStatus {
- cs := ConsumeStatus{}
- ss := mgr.getPullRT(group, topic)
- cs.PullTPS = ss.tps
-
- ss = mgr.getPullTPS(group, topic)
- cs.PullTPS = ss.tps
-
- ss = mgr.getConsumeRT(group, topic)
- cs.ConsumeRT = ss.avgpt
-
- ss = mgr.getConsumeOKTPS(group, topic)
- cs.ConsumeOKTPS = ss.tps
-
- ss = mgr.getConsumeFailedTPS(group, topic)
-
- cs.ConsumeFailedTPS = ss.tps
-
- ss = mgr.topicAndGroupConsumeFailedTPS.getStatsDataInHour(topic + "@" + group)
- cs.ConsumeFailedMsgs = ss.sum
- return cs
-}
-
-func (mgr *StatsManager) ShutDownStat() {
- mgr.closeOnce.Do(func() {
- close(mgr.topicAndGroupConsumeOKTPS.closed)
- close(mgr.topicAndGroupConsumeRT.closed)
- close(mgr.topicAndGroupConsumeFailedTPS.closed)
- close(mgr.topicAndGroupPullTPS.closed)
- close(mgr.topicAndGroupPullRT.closed)
- })
-}
-
-func (mgr *StatsManager) getPullRT(group, topic string) statsSnapshot {
- return mgr.topicAndGroupPullRT.getStatsDataInMinute(topic + "@" + group)
-}
-
-func (mgr *StatsManager) getPullTPS(group, topic string) statsSnapshot {
- return mgr.topicAndGroupPullTPS.getStatsDataInMinute(topic + "@" + group)
-}
-
-func (mgr *StatsManager) getConsumeRT(group, topic string) statsSnapshot {
- ss := mgr.topicAndGroupPullRT.getStatsDataInMinute(topic + "@" + group)
- if ss.sum == 0 {
- return mgr.topicAndGroupConsumeRT.getStatsDataInHour(topic + "@" + group)
- }
- return ss
-}
-
-func (mgr *StatsManager) getConsumeOKTPS(group, topic string) statsSnapshot {
- return mgr.topicAndGroupConsumeOKTPS.getStatsDataInMinute(topic + "@" + group)
-}
-
-func (mgr *StatsManager) getConsumeFailedTPS(group, topic string) statsSnapshot {
- return mgr.topicAndGroupConsumeFailedTPS.getStatsDataInMinute(topic + "@" + group)
-}
-
-var csListLock sync.Mutex
-
-func computeStatsData(csList *list.List) statsSnapshot {
- csListLock.Lock()
- defer csListLock.Unlock()
- tps, avgpt, sum := 0.0, 0.0, int64(0)
- if csList.Len() > 0 {
- first := csList.Front().Value.(callSnapshot)
- last := csList.Back().Value.(callSnapshot)
- sum = last.value - first.value
- tps = float64(sum*1000.0) / float64(last.timestamp-first.timestamp)
- timesDiff := last.time - first.time
- if timesDiff > 0 {
- avgpt = float64(sum*1.0) / float64(timesDiff)
- }
- }
- return statsSnapshot{
- tps: tps,
- avgpt: avgpt,
- sum: sum,
- }
-}
-
-type statsItemSet struct {
- statsName string
- statsItemTable sync.Map
- closed chan struct{}
-}
-
-func newStatsItemSet(statsName string) *statsItemSet {
- sis := &statsItemSet{
- statsName: statsName,
- closed: make(chan struct{}),
- }
- sis.init()
- return sis
-}
-
-func (sis *statsItemSet) init() {
- go primitive.WithRecover(func() {
- ticker := time.NewTicker(10 * time.Second)
- defer ticker.Stop()
- for {
- select {
- case <-sis.closed:
- return
- case <-ticker.C:
- sis.samplingInSeconds()
- }
- }
- })
-
- go primitive.WithRecover(func() {
- ticker := time.NewTicker(10 * time.Minute)
- defer ticker.Stop()
- for {
- select {
- case <-sis.closed:
- return
- case <-ticker.C:
- sis.samplingInMinutes()
- }
- }
- })
-
- go primitive.WithRecover(func() {
- ticker := time.NewTicker(time.Hour)
- defer ticker.Stop()
- for {
- select {
- case <-sis.closed:
- return
- case <-ticker.C:
- sis.samplingInHour()
- }
- }
- })
-
- go primitive.WithRecover(func() {
- time.Sleep(nextMinutesTime().Sub(time.Now()))
- ticker := time.NewTicker(time.Minute)
- defer ticker.Stop()
- for {
- select {
- case <-sis.closed:
- return
- case <-ticker.C:
- sis.printAtMinutes()
- }
- }
- })
-
- go primitive.WithRecover(func() {
- time.Sleep(nextHourTime().Sub(time.Now()))
- ticker := time.NewTicker(time.Hour)
- defer ticker.Stop()
- for {
- select {
- case <-sis.closed:
- return
- case <-ticker.C:
- sis.printAtHour()
- }
- }
- })
-
- go primitive.WithRecover(func() {
- time.Sleep(nextMonthTime().Sub(time.Now()))
- ticker := time.NewTicker(24 * time.Hour)
- defer ticker.Stop()
- for {
- select {
- case <-sis.closed:
- return
- case <-ticker.C:
- sis.printAtDay()
- }
- }
- })
-}
-
-func (sis *statsItemSet) samplingInSeconds() {
- sis.statsItemTable.Range(func(key, value interface{}) bool {
- si := value.(*statsItem)
- si.samplingInSeconds()
- return true
- })
-}
-
-func (sis *statsItemSet) samplingInMinutes() {
- sis.statsItemTable.Range(func(key, value interface{}) bool {
- si := value.(*statsItem)
- si.samplingInMinutes()
- return true
- })
-}
-
-func (sis *statsItemSet) samplingInHour() {
- sis.statsItemTable.Range(func(key, value interface{}) bool {
- si := value.(*statsItem)
- si.samplingInHour()
- return true
- })
-}
-
-func (sis *statsItemSet) printAtMinutes() {
- sis.statsItemTable.Range(func(key, value interface{}) bool {
- si := value.(*statsItem)
- si.printAtMinutes()
- return true
- })
-}
-
-func (sis *statsItemSet) printAtHour() {
- sis.statsItemTable.Range(func(key, value interface{}) bool {
- si := value.(*statsItem)
- si.printAtHour()
- return true
- })
-}
-
-func (sis *statsItemSet) printAtDay() {
- sis.statsItemTable.Range(func(key, value interface{}) bool {
- si := value.(*statsItem)
- si.printAtDay()
- return true
- })
-}
-
-func (sis *statsItemSet) addValue(key string, incValue, incTimes int64) {
- si := sis.getAndCreateStateItem(key)
- atomic.AddInt64(&si.value, incValue)
- atomic.AddInt64(&si.times, incTimes)
-}
-
-func (sis *statsItemSet) getAndCreateStateItem(key string) *statsItem {
- if val, ok := sis.statsItemTable.Load(key); ok {
- return val.(*statsItem)
- } else {
- si := newStatsItem(sis.statsName, key)
- sis.statsItemTable.Store(key, si)
- return si
- }
-}
-
-func (sis *statsItemSet) getStatsDataInMinute(key string) statsSnapshot {
- if val, ok := sis.statsItemTable.Load(key); ok {
- si := val.(*statsItem)
- return si.getStatsDataInMinute()
- }
- return statsSnapshot{}
-}
-
-func (sis *statsItemSet) getStatsDataInHour(key string) statsSnapshot {
- if val, ok := sis.statsItemTable.Load(key); ok {
- si := val.(*statsItem)
- return si.getStatsDataInHour()
- }
- return statsSnapshot{}
-}
-
-func (sis *statsItemSet) getStatsDataInDay(key string) statsSnapshot {
- if val, ok := sis.statsItemTable.Load(key); ok {
- si := val.(*statsItem)
- return si.getStatsDataInDay()
- }
- return statsSnapshot{}
-}
-
-func (sis *statsItemSet) getStatsItem(key string) *statsItem {
- val, _ := sis.statsItemTable.Load(key)
- return val.(*statsItem)
-}
-
-type statsItem struct {
- value int64
- times int64
- csListMinute *list.List
- csListHour *list.List
- csListDay *list.List
- statsName string
- statsKey string
- csListMinuteLock sync.Mutex
- csListHourLock sync.Mutex
- csListDayLock sync.Mutex
-}
-
-func (si *statsItem) getStatsDataInMinute() statsSnapshot {
- return computeStatsData(si.csListMinute)
-}
-
-func (si *statsItem) getStatsDataInHour() statsSnapshot {
- return computeStatsData(si.csListHour)
-}
-
-func (si *statsItem) getStatsDataInDay() statsSnapshot {
- return computeStatsData(si.csListDay)
-}
-
-func newStatsItem(statsName, statsKey string) *statsItem {
- return &statsItem{
- statsName: statsName,
- statsKey: statsKey,
- csListMinute: list.New(),
- csListHour: list.New(),
- csListDay: list.New(),
- }
-}
-
-func (si *statsItem) samplingInSeconds() {
- si.csListMinuteLock.Lock()
- defer si.csListMinuteLock.Unlock()
- si.csListMinute.PushBack(callSnapshot{
- timestamp: time.Now().Unix() * 1000,
- time: atomic.LoadInt64(&si.times),
- value: atomic.LoadInt64(&si.value),
- })
- if si.csListMinute.Len() > 7 {
- si.csListMinute.Remove(si.csListMinute.Front())
- }
-}
-
-func (si *statsItem) samplingInMinutes() {
- si.csListHourLock.Lock()
- defer si.csListHourLock.Unlock()
- si.csListHour.PushBack(callSnapshot{
- timestamp: time.Now().Unix() * 1000,
- time: atomic.LoadInt64(&si.times),
- value: atomic.LoadInt64(&si.value),
- })
- if si.csListHour.Len() > 7 {
- si.csListHour.Remove(si.csListHour.Front())
- }
-}
-
-func (si *statsItem) samplingInHour() {
- si.csListDayLock.Lock()
- defer si.csListDayLock.Unlock()
- si.csListDay.PushBack(callSnapshot{
- timestamp: time.Now().Unix() * 1000,
- time: atomic.LoadInt64(&si.times),
- value: atomic.LoadInt64(&si.value),
- })
- if si.csListDay.Len() > 25 {
- si.csListHour.Remove(si.csListDay.Front())
- }
-}
-
-func (si *statsItem) printAtMinutes() {
- ss := computeStatsData(si.csListMinute)
- rlog.Info("Stats In One Minute, SUM: %d TPS: AVGPT: %.2f", map[string]interface{}{
- "statsName": si.statsName,
- "statsKey": si.statsKey,
- "SUM": ss.sum,
- "TPS": fmt.Sprintf("%.2f", ss.tps),
- "AVGPT": ss.avgpt,
- })
-}
-
-func (si *statsItem) printAtHour() {
- ss := computeStatsData(si.csListHour)
- rlog.Info("Stats In One Hour, SUM: %d TPS: AVGPT: %.2f", map[string]interface{}{
- "statsName": si.statsName,
- "statsKey": si.statsKey,
- "SUM": ss.sum,
- "TPS": fmt.Sprintf("%.2f", ss.tps),
- "AVGPT": ss.avgpt,
- })
-}
-
-func (si *statsItem) printAtDay() {
- ss := computeStatsData(si.csListDay)
- rlog.Info("Stats In One Day, SUM: %d TPS: AVGPT: %.2f", map[string]interface{}{
- "statsName": si.statsName,
- "statsKey": si.statsKey,
- "SUM": ss.sum,
- "TPS": fmt.Sprintf("%.2f", ss.tps),
- "AVGPT": ss.avgpt,
- })
-}
-
-func nextMinutesTime() time.Time {
- now := time.Now()
- m, _ := time.ParseDuration("1m")
- return now.Add(m)
-}
-
-func nextHourTime() time.Time {
- now := time.Now()
- m, _ := time.ParseDuration("1h")
- return now.Add(m)
-}
-
-func nextMonthTime() time.Time {
- now := time.Now()
- return now.AddDate(0, 1, 0)
-}
-
-type callSnapshot struct {
- timestamp int64
- time int64
- value int64
-}
-
-type statsSnapshot struct {
- sum int64
- tps float64
- avgpt float64
-}
diff --git a/consumer/statistics_test.go b/consumer/statistics_test.go
deleted file mode 100644
index 4836141..0000000
--- a/consumer/statistics_test.go
+++ /dev/null
@@ -1,214 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package consumer
-
-import (
- "testing"
- "time"
-)
-
-func almostEqual(a, b float64) bool {
- diff := abs(a - b)
- return diff/a < 0.01
-}
-
-func abs(a float64) float64 {
- if a > 0 {
- return a
- }
- return -a
-}
-
-func TestNextMinuteTime(t *testing.T) {
- nextMinute := nextMinutesTime()
- minuteElapse := nextMinute.Sub(time.Now()).Minutes()
- if !almostEqual(minuteElapse, 1.0) {
- t.Errorf("wrong next one minute. want=%f, got=%f", 1.0, minuteElapse)
- }
-}
-
-func TestNextHourTime(t *testing.T) {
- nextHour := nextHourTime()
- hourElapse := nextHour.Sub(time.Now()).Hours()
- if !almostEqual(hourElapse, 1.0) {
- t.Errorf("wrong next one hour. want=%f, got=%f", 1.0, hourElapse)
- }
-}
-
-func TestIncreasePullRTGetPullRT(t *testing.T) {
- mgr := NewStatsManager()
- mgr.ShutDownStat()
-
- tests := []struct {
- RT int64
- ExpectSum int64
- }{
- {1, 0},
- {1, 1},
- {1, 2},
- {1, 3},
- {1, 4},
- {1, 5},
- {1, 6},
- {1, 6},
- }
- for _, tt := range tests {
- mgr.increasePullRT("rocketmq", "default", tt.RT)
- mgr.topicAndGroupPullRT.samplingInSeconds()
- snapshot := mgr.getPullRT("rocketmq", "default")
- if snapshot.sum != tt.ExpectSum {
- t.Errorf("wrong Pull RT sum. want=%d, got=%d", tt.ExpectSum, snapshot.sum)
- }
- }
-}
-
-//func TestIncreaseConsumeRTGetConsumeRT(t *testing.T) {
-// ShutDownStat()
-// tests := []struct {
-// RT int64
-// ExpectSum int64
-// }{
-// {1, 0},
-// {1, 1},
-// {1, 2},
-// {1, 3},
-// {1, 4},
-// {1, 5},
-// {1, 6},
-// {1, 6},
-// }
-// for _, tt := range tests {
-// increaseConsumeRT("rocketmq", "default", tt.RT)
-// topicAndGroupConsumeRT.samplingInMinutes()
-// snapshot := getConsumeRT("rocketmq", "default")
-// if snapshot.sum != tt.ExpectSum {
-// t.Errorf("wrong consume RT sum. want=%d, got=%d", tt.ExpectSum, snapshot.sum)
-// }
-// }
-//}
-
-func TestIncreasePullTPSGetPullTPS(t *testing.T) {
- mgr := NewStatsManager()
- mgr.ShutDownStat()
- tests := []struct {
- RT int
- ExpectSum int64
- }{
- {1, 0},
- {1, 1},
- {1, 2},
- {1, 3},
- {1, 4},
- {1, 5},
- {1, 6},
- {1, 6},
- }
- for _, tt := range tests {
- mgr.increasePullTPS("rocketmq", "default", tt.RT)
- mgr.topicAndGroupPullTPS.samplingInSeconds()
- snapshot := mgr.getPullTPS("rocketmq", "default")
- if snapshot.sum != tt.ExpectSum {
- t.Errorf("wrong Pull TPS sum. want=%d, got=%d", tt.ExpectSum, snapshot.sum)
- }
- }
-}
-
-func TestIncreaseConsumeOKTPSGetConsumeOKTPS(t *testing.T) {
- mgr := NewStatsManager()
- mgr.ShutDownStat()
- tests := []struct {
- RT int
- ExpectSum int64
- }{
- {1, 0},
- {1, 1},
- {1, 2},
- {1, 3},
- {1, 4},
- {1, 5},
- {1, 6},
- {1, 6},
- }
- for _, tt := range tests {
- mgr.increaseConsumeOKTPS("rocketmq", "default", tt.RT)
- mgr.topicAndGroupConsumeOKTPS.samplingInSeconds()
- snapshot := mgr.getConsumeOKTPS("rocketmq", "default")
- if snapshot.sum != tt.ExpectSum {
- t.Errorf("wrong Consume OK TPS sum. want=%d, got=%d", tt.ExpectSum, snapshot.sum)
- }
- }
-}
-
-func TestIncreaseConsumeFailedTPSGetConsumeFailedTPS(t *testing.T) {
- mgr := NewStatsManager()
- mgr.ShutDownStat()
- tests := []struct {
- RT int
- ExpectSum int64
- }{
- {1, 0},
- {1, 1},
- {1, 2},
- {1, 3},
- {1, 4},
- {1, 5},
- {1, 6},
- {1, 6},
- }
- for _, tt := range tests {
- mgr.increaseConsumeFailedTPS("rocketmq", "default", tt.RT)
- mgr.topicAndGroupConsumeFailedTPS.samplingInSeconds()
- snapshot := mgr.getConsumeFailedTPS("rocketmq", "default")
- if snapshot.sum != tt.ExpectSum {
- t.Errorf("wrong Consume Failed TPS sum. want=%d, got=%d", tt.ExpectSum, snapshot.sum)
- }
- }
-}
-
-func TestGetConsumeStatus(t *testing.T) {
- mgr := NewStatsManager()
- mgr.ShutDownStat()
- group, topic := "rocketmq", "default"
-
- tests := []struct {
- RT int
- ExpectFailMessage int64
- }{
- {1, 0},
- {1, 1},
- {1, 2},
- {1, 3},
- {1, 4},
- }
- for _, tt := range tests {
- mgr.increasePullRT(group, topic, int64(tt.RT))
- mgr.increasePullTPS(group, topic, tt.RT)
- mgr.increaseConsumeRT(group, topic, int64(tt.RT))
- mgr.increaseConsumeOKTPS(group, topic, tt.RT)
- mgr.increaseConsumeFailedTPS(group, topic, tt.RT)
- mgr.topicAndGroupPullRT.samplingInSeconds()
- mgr.topicAndGroupPullTPS.samplingInSeconds()
- mgr.topicAndGroupConsumeRT.samplingInMinutes()
- mgr.topicAndGroupConsumeOKTPS.samplingInSeconds()
- mgr.topicAndGroupConsumeFailedTPS.samplingInMinutes()
- status := mgr.GetConsumeStatus(group, topic)
- if status.ConsumeFailedMsgs != tt.ExpectFailMessage {
- t.Errorf("wrong ConsumeFailedMsg. want=0, got=%d", status.ConsumeFailedMsgs)
- }
- }
-}
diff --git a/consumer/strategy.go b/consumer/strategy.go
deleted file mode 100644
index 4a07928..0000000
--- a/consumer/strategy.go
+++ /dev/null
@@ -1,247 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package consumer
-
-import (
- "strings"
-
- "github.com/stathat/consistent"
-
- "github.com/apache/rocketmq-client-go/v2/internal/utils"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/rlog"
-)
-
-// Strategy Algorithm for message allocating between consumers
-// An allocate strategy proxy for based on machine room nearside priority. An actual allocate strategy can be
-// specified.
-//
-// If any consumer is alive in a machine room, the message queue of the broker which is deployed in the same machine
-// should only be allocated to those. Otherwise, those message queues can be shared along all consumers since there are
-// no alive consumer to monopolize them.
-//
-// Average Hashing queue algorithm
-// Cycle average Hashing queue algorithm
-// Use Message QueueID specified
-// Computer room Hashing queue algorithm, such as Alipay logic room
-// Consistent Hashing queue algorithm
-
-type AllocateStrategy func(string, string, []*primitive.MessageQueue, []string) []*primitive.MessageQueue
-
-func AllocateByAveragely(consumerGroup, currentCID string, mqAll []*primitive.MessageQueue,
- cidAll []string) []*primitive.MessageQueue {
- if currentCID == "" || len(mqAll) == 0 || len(cidAll) == 0 {
- return nil
- }
-
- var (
- find bool
- index int
- )
- for idx := range cidAll {
- if cidAll[idx] == currentCID {
- find = true
- index = idx
- break
- }
- }
- if !find {
- rlog.Warning("[BUG] ConsumerId not in cidAll", map[string]interface{}{
- rlog.LogKeyConsumerGroup: consumerGroup,
- "consumerId": currentCID,
- "cidAll": cidAll,
- })
- return nil
- }
-
- mqSize := len(mqAll)
- cidSize := len(cidAll)
- mod := mqSize % cidSize
-
- var averageSize int
- if mqSize <= cidSize {
- averageSize = 1
- } else {
- if mod > 0 && index < mod {
- averageSize = mqSize/cidSize + 1
- } else {
- averageSize = mqSize / cidSize
- }
- }
-
- var startIndex int
- if mod > 0 && index < mod {
- startIndex = index * averageSize
- } else {
- startIndex = index*averageSize + mod
- }
-
- num := utils.MinInt(averageSize, mqSize-startIndex)
- result := make([]*primitive.MessageQueue, 0)
- for i := 0; i < num; i++ {
- result = append(result, mqAll[(startIndex+i)%mqSize])
- }
- return result
-}
-
-func AllocateByAveragelyCircle(consumerGroup, currentCID string, mqAll []*primitive.MessageQueue,
- cidAll []string) []*primitive.MessageQueue {
- if currentCID == "" || len(mqAll) == 0 || len(cidAll) == 0 {
- return nil
- }
-
- var (
- find bool
- index int
- )
- for idx := range cidAll {
- if cidAll[idx] == currentCID {
- find = true
- index = idx
- break
- }
- }
- if !find {
- rlog.Warning("[BUG] ConsumerId not in cidAll", map[string]interface{}{
- rlog.LogKeyConsumerGroup: consumerGroup,
- "consumerId": currentCID,
- "cidAll": cidAll,
- })
- return nil
- }
-
- result := make([]*primitive.MessageQueue, 0)
- for i := index; i < len(mqAll); i++ {
- if i%len(cidAll) == index {
- result = append(result, mqAll[i])
- }
- }
- return result
-}
-
-// TODO
-func AllocateByMachineNearby(consumerGroup, currentCID string, mqAll []*primitive.MessageQueue,
- cidAll []string) []*primitive.MessageQueue {
- return AllocateByAveragely(consumerGroup, currentCID, mqAll, cidAll)
-}
-
-func AllocateByConfig(list []*primitive.MessageQueue) AllocateStrategy {
- return func(consumerGroup, currentCID string, mqAll []*primitive.MessageQueue, cidAll []string) []*primitive.MessageQueue {
- return list
- }
-}
-
-func AllocateByMachineRoom(consumeridcs []string) AllocateStrategy {
- return func(consumerGroup, currentCID string, mqAll []*primitive.MessageQueue, cidAll []string) []*primitive.MessageQueue {
- if currentCID == "" || len(mqAll) == 0 || len(cidAll) == 0 {
- return nil
- }
-
- var (
- find bool
- index int
- )
- for idx := range cidAll {
- if cidAll[idx] == currentCID {
- find = true
- index = idx
- break
- }
- }
- if !find {
- rlog.Warning("[BUG] ConsumerId not in cidAll", map[string]interface{}{
- rlog.LogKeyConsumerGroup: consumerGroup,
- "consumerId": currentCID,
- "cidAll": cidAll,
- })
- return nil
- }
-
- var premqAll []*primitive.MessageQueue
- for _, mq := range mqAll {
- temp := strings.Split(mq.BrokerName, "@")
- if len(temp) == 2 {
- for _, idc := range consumeridcs {
- if idc == temp[0] {
- premqAll = append(premqAll, mq)
- }
- }
- }
- }
-
- mod := len(premqAll) / len(cidAll)
- rem := len(premqAll) % len(cidAll)
- startIndex := mod * index
- endIndex := startIndex + mod
-
- result := make([]*primitive.MessageQueue, 0)
- for i := startIndex; i < endIndex; i++ {
- result = append(result, mqAll[i])
- }
- if rem > index {
- result = append(result, premqAll[index+mod*len(cidAll)])
- }
- return result
- }
-}
-
-func AllocateByConsistentHash(virtualNodeCnt int) AllocateStrategy {
- return func(consumerGroup, currentCID string, mqAll []*primitive.MessageQueue, cidAll []string) []*primitive.MessageQueue {
- if currentCID == "" || len(mqAll) == 0 || len(cidAll) == 0 {
- return nil
- }
-
- var (
- find bool
- )
- for idx := range cidAll {
- if cidAll[idx] == currentCID {
- find = true
- break
- }
- }
- if !find {
- rlog.Warning("[BUG] ConsumerId not in cidAll", map[string]interface{}{
- rlog.LogKeyConsumerGroup: consumerGroup,
- "consumerId": currentCID,
- "cidAll": cidAll,
- })
- return nil
- }
-
- c := consistent.New()
- c.NumberOfReplicas = virtualNodeCnt
- for _, cid := range cidAll {
- c.Add(cid)
- }
-
- result := make([]*primitive.MessageQueue, 0)
- for _, mq := range mqAll {
- clientNode, err := c.Get(mq.String())
- if err != nil {
- rlog.Warning("[BUG] AllocateByConsistentHash err: %s", map[string]interface{}{
- rlog.LogKeyUnderlayError: err,
- })
- }
- if currentCID == clientNode {
- result = append(result, mq)
- }
- }
- return result
- }
-}
diff --git a/consumer/strategy_test.go b/consumer/strategy_test.go
deleted file mode 100644
index d521b4b..0000000
--- a/consumer/strategy_test.go
+++ /dev/null
@@ -1,487 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package consumer
-
-import (
- "github.com/apache/rocketmq-client-go/v2/rlog"
- "testing"
-
- "github.com/apache/rocketmq-client-go/v2/primitive"
- . "github.com/smartystreets/goconvey/convey"
-)
-
-func TestAllocateByAveragely(t *testing.T) {
- Convey("Given message queues with a starting value", t, func() {
- queues := []*primitive.MessageQueue{
- {
- QueueId: 0,
- },
- {
- QueueId: 1,
- },
- {
- QueueId: 2,
- },
- {
- QueueId: 3,
- },
- {
- QueueId: 4,
- },
- {
- QueueId: 5,
- },
- }
-
- Convey("When params is empty", func() {
- result := AllocateByAveragely("testGroup", "", queues, []string{"192.168.24.1@default"})
- So(result, ShouldBeNil)
-
- result = AllocateByAveragely("testGroup", "192.168.24.1@default", nil, []string{"192.168.24.1@default"})
- So(result, ShouldBeNil)
-
- result = AllocateByAveragely("testGroup", "192.168.24.1@default", queues, nil)
- So(result, ShouldBeNil)
- })
-
- type testCase struct {
- currentCid string
- mqAll []*primitive.MessageQueue
- cidAll []string
- expectedQueue []*primitive.MessageQueue
- }
- cases := []testCase{
- {
- currentCid: "192.168.24.1@default",
- mqAll: queues,
- cidAll: []string{"192.168.24.1@default", "192.168.24.2@default"},
- expectedQueue: []*primitive.MessageQueue{
- {
- QueueId: 0,
- },
- {
- QueueId: 1,
- },
- {
- QueueId: 2,
- },
- },
- },
- {
- currentCid: "192.168.24.2@default",
- mqAll: queues,
- cidAll: []string{"192.168.24.1@default", "192.168.24.2@default", "192.168.24.3@default"},
- expectedQueue: []*primitive.MessageQueue{
- {
- QueueId: 2,
- },
- {
- QueueId: 3,
- },
- },
- },
- {
- currentCid: "192.168.24.2@default",
- mqAll: queues,
- cidAll: []string{"192.168.24.1@default", "192.168.24.2@default", "192.168.24.3@default", "192.168.24.4@default"},
- expectedQueue: []*primitive.MessageQueue{
- {
- QueueId: 2,
- },
- {
- QueueId: 3,
- },
- },
- },
- {
- currentCid: "192.168.24.4@default",
- mqAll: queues,
- cidAll: []string{"192.168.24.1@default", "192.168.24.2@default", "192.168.24.3@default", "192.168.24.4@default"},
- expectedQueue: []*primitive.MessageQueue{
- {
- QueueId: 5,
- },
- },
- },
- {
- currentCid: "192.168.24.7@default",
- mqAll: queues,
- cidAll: []string{"192.168.24.1@default", "192.168.24.2@default", "192.168.24.3@default", "192.168.24.4@default", "192.168.24.5@default", "192.168.24.6@default", "192.168.24.7@default"},
- expectedQueue: []*primitive.MessageQueue{},
- },
- }
-
- Convey("the result of AllocateByAveragely should be deep equal expectedQueue", func() {
- for _, value := range cases {
- result := AllocateByAveragely("testGroup", value.currentCid, value.mqAll, value.cidAll)
- So(result, ShouldResemble, value.expectedQueue)
- }
- })
- })
-}
-
-func TestAllocateByAveragelyCircle(t *testing.T) {
- Convey("Given message queues with a starting value", t, func() {
- queues := []*primitive.MessageQueue{
- {
- QueueId: 0,
- },
- {
- QueueId: 1,
- },
- {
- QueueId: 2,
- },
- {
- QueueId: 3,
- },
- {
- QueueId: 4,
- },
- {
- QueueId: 5,
- },
- }
-
- Convey("When params is empty", func() {
- result := AllocateByAveragelyCircle("testGroup", "", queues, []string{"192.168.24.1@default"})
- So(result, ShouldBeNil)
-
- result = AllocateByAveragelyCircle("testGroup", "192.168.24.1@default", nil, []string{"192.168.24.1@default"})
- So(result, ShouldBeNil)
-
- result = AllocateByAveragelyCircle("testGroup", "192.168.24.1@default", queues, nil)
- So(result, ShouldBeNil)
- })
-
- type testCase struct {
- currentCid string
- mqAll []*primitive.MessageQueue
- cidAll []string
- expectedQueue []*primitive.MessageQueue
- }
- cases := []testCase{
- {
- currentCid: "192.168.24.1@default",
- mqAll: queues,
- cidAll: []string{"192.168.24.1@default", "192.168.24.2@default"},
- expectedQueue: []*primitive.MessageQueue{
- {
- QueueId: 0,
- },
- {
- QueueId: 2,
- },
- {
- QueueId: 4,
- },
- },
- },
- {
- currentCid: "192.168.24.2@default",
- mqAll: queues,
- cidAll: []string{"192.168.24.1@default", "192.168.24.2@default", "192.168.24.3@default"},
- expectedQueue: []*primitive.MessageQueue{
- {
- QueueId: 1,
- },
- {
- QueueId: 4,
- },
- },
- },
- {
- currentCid: "192.168.24.2@default",
- mqAll: queues,
- cidAll: []string{"192.168.24.1@default", "192.168.24.2@default", "192.168.24.3@default", "192.168.24.4@default"},
- expectedQueue: []*primitive.MessageQueue{
- {
- QueueId: 1,
- },
- {
- QueueId: 5,
- },
- },
- },
- {
- currentCid: "192.168.24.4@default",
- mqAll: queues,
- cidAll: []string{"192.168.24.1@default", "192.168.24.2@default", "192.168.24.3@default", "192.168.24.4@default"},
- expectedQueue: []*primitive.MessageQueue{
- {
- QueueId: 3,
- },
- },
- },
- {
- currentCid: "192.168.24.7@default",
- mqAll: queues,
- cidAll: []string{"192.168.24.1@default", "192.168.24.2@default", "192.168.24.3@default", "192.168.24.4@default", "192.168.24.5@default", "192.168.24.6@default", "192.168.24.7@default"},
- expectedQueue: []*primitive.MessageQueue{},
- },
- }
-
- Convey("the result of AllocateByAveragelyCircle should be deep equal expectedQueue", func() {
- for _, value := range cases {
- result := AllocateByAveragelyCircle("testGroup", value.currentCid, value.mqAll, value.cidAll)
- So(result, ShouldResemble, value.expectedQueue)
- }
- })
- })
-}
-
-func TestAllocateByConfig(t *testing.T) {
- Convey("Given message queues with a starting value", t, func() {
- queues := []*primitive.MessageQueue{
- {
- QueueId: 0,
- },
- {
- QueueId: 1,
- },
- {
- QueueId: 2,
- },
- {
- QueueId: 3,
- },
- {
- QueueId: 4,
- },
- {
- QueueId: 5,
- },
- }
-
- strategy := AllocateByConfig(queues)
- result := strategy("testGroup", "192.168.24.1@default", queues, []string{"192.168.24.1@default", "192.168.24.2@default"})
- So(result, ShouldResemble, queues)
- })
-}
-
-func TestAllocateByMachineRoom(t *testing.T) {
- Convey("Given some consumer IDCs with a starting value", t, func() {
- idcs := []string{"192.168.24.1", "192.168.24.2"}
- strategy := AllocateByMachineRoom(idcs)
-
- queues := []*primitive.MessageQueue{
- {
- QueueId: 0,
- BrokerName: "192.168.24.1@defaultName",
- },
- {
- QueueId: 1,
- BrokerName: "192.168.24.1@defaultName",
- },
- {
- QueueId: 2,
- BrokerName: "192.168.24.1@defaultName",
- },
- {
- QueueId: 3,
- BrokerName: "192.168.24.2@defaultName",
- },
- {
- QueueId: 4,
- BrokerName: "192.168.24.2@defaultName",
- },
- {
- QueueId: 5,
- BrokerName: "192.168.24.3@defaultName",
- },
- }
-
- Convey("When params is empty", func() {
- result := strategy("testGroup", "", queues, []string{"192.168.24.1@default"})
- So(result, ShouldBeNil)
-
- result = strategy("testGroup", "192.168.24.1@default", nil, []string{"192.168.24.1@default"})
- So(result, ShouldBeNil)
-
- result = strategy("testGroup", "192.168.24.1@default", queues, nil)
- So(result, ShouldBeNil)
- })
-
- type testCase struct {
- currentCid string
- mqAll []*primitive.MessageQueue
- cidAll []string
- expectedQueue []*primitive.MessageQueue
- }
- cases := []testCase{
- {
- currentCid: "192.168.24.1@default",
- mqAll: queues,
- cidAll: []string{"192.168.24.1@default", "192.168.24.2@default"},
- expectedQueue: []*primitive.MessageQueue{
- {
- QueueId: 0,
- BrokerName: "192.168.24.1@defaultName",
- },
- {
- QueueId: 1,
- BrokerName: "192.168.24.1@defaultName",
- },
- {
- QueueId: 4,
- BrokerName: "192.168.24.2@defaultName",
- },
- },
- },
- {
- currentCid: "192.168.24.2@default",
- mqAll: queues,
- cidAll: []string{"192.168.24.1@default", "192.168.24.2@default", "192.168.24.3@default"},
- expectedQueue: []*primitive.MessageQueue{
- {
- QueueId: 1,
- BrokerName: "192.168.24.1@defaultName",
- },
- {
- QueueId: 4,
- BrokerName: "192.168.24.2@defaultName",
- },
- },
- },
- {
- currentCid: "192.168.24.2@default",
- mqAll: queues,
- cidAll: []string{"192.168.24.1@default", "192.168.24.2@default", "192.168.24.3@default", "192.168.24.4@default"},
- expectedQueue: []*primitive.MessageQueue{
- {
- QueueId: 1,
- BrokerName: "192.168.24.1@defaultName",
- },
- },
- },
- {
- currentCid: "192.168.24.4@default",
- mqAll: queues,
- cidAll: []string{"192.168.24.1@default", "192.168.24.2@default", "192.168.24.3@default", "192.168.24.4@default"},
- expectedQueue: []*primitive.MessageQueue{
- {
- QueueId: 3,
- BrokerName: "192.168.24.2@defaultName",
- },
- },
- },
- {
- currentCid: "192.168.24.7@default",
- mqAll: queues,
- cidAll: []string{"192.168.24.1@default", "192.168.24.2@default", "192.168.24.3@default", "192.168.24.4@default", "192.168.24.5@default", "192.168.24.6@default", "192.168.24.7@default"},
- expectedQueue: []*primitive.MessageQueue{},
- },
- }
-
- Convey("the result of AllocateByMachineRoom should be deep equal expectedQueue", func() {
- for _, value := range cases {
- result := strategy("testGroup", value.currentCid, value.mqAll, value.cidAll)
- So(result, ShouldResemble, value.expectedQueue)
- }
- })
- })
-}
-
-func TestAllocateByConsistentHash(t *testing.T) {
- Convey("Given virtualNodeCnt with a starting value", t, func() {
- virtualNodeCnt := 10
- strategy := AllocateByConsistentHash(virtualNodeCnt)
-
- queues := []*primitive.MessageQueue{
- {
- QueueId: 0,
- BrokerName: "192.168.24.1@defaultName",
- },
- {
- QueueId: 1,
- BrokerName: "192.168.24.1@defaultName",
- },
- {
- QueueId: 2,
- BrokerName: "192.168.24.1@defaultName",
- },
- {
- QueueId: 3,
- BrokerName: "192.168.24.2@defaultName",
- },
- {
- QueueId: 4,
- BrokerName: "192.168.24.2@defaultName",
- },
- {
- QueueId: 5,
- BrokerName: "192.168.24.3@defaultName",
- },
- }
-
- Convey("When params is empty", func() {
- result := strategy("testGroup", "", queues, []string{"192.168.24.1@default"})
- So(result, ShouldBeNil)
-
- result = strategy("testGroup", "192.168.24.1@default", nil, []string{"192.168.24.1@default"})
- So(result, ShouldBeNil)
-
- result = strategy("testGroup", "192.168.24.1@default", queues, nil)
- So(result, ShouldBeNil)
- })
-
- type testCase struct {
- currentCid string
- mqAll []*primitive.MessageQueue
- cidAll []string
- }
- cases := []testCase{
- {
- currentCid: "192.168.24.1@default",
- mqAll: queues,
- cidAll: []string{"192.168.24.1@default", "192.168.24.2@default", "192.168.24.3@default"},
- },
- {
- currentCid: "192.168.24.2@default",
- mqAll: queues,
- cidAll: []string{"192.168.24.1@default", "192.168.24.2@default", "192.168.24.3@default"},
- },
- {
- currentCid: "192.168.24.3@default",
- mqAll: queues,
- cidAll: []string{"192.168.24.1@default", "192.168.24.2@default", "192.168.24.3@default"},
- },
- {
- currentCid: "192.168.24.1@default",
- mqAll: queues,
- cidAll: []string{"192.168.24.1@default", "192.168.24.2@default"},
- },
- {
- currentCid: "192.168.24.2@default",
- mqAll: queues,
- cidAll: []string{"192.168.24.1@default", "192.168.24.2@default"},
- },
- }
-
- Convey("observe the result of AllocateByMachineRoom", func() {
- for _, value := range cases {
- result := strategy("testGroup", value.currentCid, value.mqAll, value.cidAll)
- rlog.Info("Result Of AllocateByMachineRoom", map[string]interface{}{
- "currentCid": value.currentCid,
- "cidAll": value.cidAll,
- "allocateResult": result,
- })
- }
- })
- })
-}
diff --git a/docs/Introduction.md b/docs/Introduction.md
deleted file mode 100644
index 011603a..0000000
--- a/docs/Introduction.md
+++ /dev/null
@@ -1,130 +0,0 @@
-## How to use
-
-### go mod
-```
-require (
- github.com/apache/rocketmq-client-go/v2 v2.1.0-rc3
-)
-```
-
-### Set Logger
-Go Client define the `Logger` interface for log output, user can specify implementation of private.
-in default, client use `logrus`.
-```
-rlog.SetLogger(Logger)
-```
-
-### Send message
-#### Interface
-```
-Producer interface {
- Start() error
- Shutdown() error
- SendSync(context.Context, *primitive.Message) (*internal.SendResult, error)
- SendOneWay(context.Context, *primitive.Message) error
-}
-```
-
-#### Examples
-- create a new `Producer` instance
-```
-p, err := rocketmq.NewProducer(
- producer.WithNameServer(endPoint),
- //producer.WithNsResolver(primitive.NewPassthroughResolver(endPoint)),
- producer.WithRetry(2),
- producer.WithGroupName("GID_xxxxxx"),
- )
-```
-
-- start the producer
-```go
-err := p.Start()
-```
-
-- send message with sync
-```
-result, err := p.SendSync(context.Background(), &primitive.Message{
- Topic: "test",
- Body: []byte("Hello RocketMQ Go Client!"),
-})
-
-// do something with result
-```
-
-- or send message with oneway
-```
-err := p.SendOneWay(context.Background(), &primitive.Message{
- Topic: "test",
- Body: []byte("Hello RocketMQ Go Client!"),
-})
-```
-Full examples: [producer](../examples/producer)
-
-### Consume Message
-now only support `PushConsumer`
-
-#### Interface
-```
-PushConsumer interface {
- // Start the PullConsumer for consuming message
- Start() error
-
- // Shutdown the PullConsumer, all offset of MessageQueue will be sync to broker before process exit
- Shutdown() error
- // Subscribe a topic for consuming
- Subscribe(topic string, selector consumer.MessageSelector,
- f func(context.Context, ...*primitive.MessageExt) (consumer.ConsumeResult, error)) error
-}
-```
-
-#### Usage
-- Create a `PushConsumer` instance
-```
-c, err := rocketmq.NewPushConsumer(
- consumer.WithNameServer(endPoint),
- consumer.WithConsumerModel(consumer.Clustering),
- consumer.WithGroupName("GID_XXXXXX"),
- )
-```
-
-- Subscribe a topic(only support one topic now), and define your consuming function
-```
-err := c.Subscribe("test", consumer.MessageSelector{}, func(ctx *consumer.ConsumeMessageContext,
- msgs []*primitive.MessageExt) (consumer.ConsumeResult, error) {
- rlog.Info("Subscribe Callback", map[string]interface{}{
- "msgs": msgs,
- })
- return consumer.ConsumeSuccess, nil
-})
-```
-- start the consumer(**NOTE: MUST after subscribe**)
-```
-err = c.Start()
-```
-
-Full examples: [consumer](../examples/consumer)
-
-
-### Admin: Topic Operation
-
-#### Examples
-- create topic
-```
-testAdmin, err := admin.NewAdmin(admin.WithResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})))
-err = testAdmin.CreateTopic(
- context.Background(),
- admin.WithTopicCreate("newTopic"),
- admin.WithBrokerAddrCreate("127.0.0.1:10911"),
-)
-```
-
-- delete topic
-`ClusterName` not supported yet
-```
-err = testAdmin.DeleteTopic(
- context.Background(),
- admin.WithTopicDelete("newTopic"),
- //admin.WithBrokerAddrDelete("127.0.0.1:10911"), //optional
- //admin.WithNameSrvAddr(nameSrvAddr), //optional
-)
-```
\ No newline at end of file
diff --git a/docs/client-design.gliffy b/docs/client-design.gliffy
deleted file mode 100644
index 418f4d8..0000000
--- a/docs/client-design.gliffy
+++ /dev/null
@@ -1 +0,0 @@
-{"contentType":"application/gliffy+json","version":"1.1","metadata":{"title":"untitled","revision":0,"exportBorder":false},"embeddedResources":{"index":0,"resources":[]},"stage":{"objects":[{"x":31,"y":44,"rotation":0,"id":57,"uid":"com.gliffy.shape.basic.basic_v1.default.text","width":150,"height":27,"lockAspectRatio":false,"lockShape":false,"order":57,"graphic":{"type":"Text","Text":{"tid":null,"valign":"middle","overflow":"none","vposition":"none","hposition":"none","html":"<p style=\"text-align:center;\"><span style=\"font-size: 24px; font-family: Arial; color: rgb(106, 168, 79); white-space: pre-wrap; font-weight: bold; text-decoration: none; line-height: 27px;\">Architecture</span></p>","paddingLeft":2,"paddingRight":2,"paddingBottom":2,"paddingTop":2}},"children":null,"linkMap":[]},{"x":0,"y":10,"rotation":0,"id":51,"uid":"com.gliffy.shape.uml.uml_v1.default.simple_class","width":710,"height":360,"lockAspectRatio":false,"lockShape":false,"order":0,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.rectangle.basic_v1","strokeWidth":1,"strokeColor":"#000000","fillColor":"#FFFFFF","gradient":false,"dropShadow":false,"state":0,"shadowX":0,"shadowY":0,"opacity":1}},"children":[],"linkMap":[]},{"x":420,"y":187,"rotation":0,"id":44,"uid":"com.gliffy.shape.basic.basic_v1.default.line","width":100,"height":100,"lockAspectRatio":false,"lockShape":false,"order":25,"graphic":{"type":"Line","Line":{"strokeWidth":2,"strokeColor":"#000000","fillColor":"none","dashStyle":null,"startArrow":0,"endArrow":1,"startArrowRotation":"auto","endArrowRotation":"auto","ortho":false,"interpolationType":"linear","cornerRadius":null,"controlPath":[[-3,-2],[37,-2]],"lockSegments":{}}},"children":[],"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":11,"px":1,"py":0.5}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":13,"px":0,"py":0.5}}},"linkMap":[]},{"x":104,"y":171,"rotation":0,"id":42,"uid":"com.gliffy.shape.basic.basic_v1.default.text","width":150,"height":28,"lockAspectRatio":false,"lockShape":false,"order":24,"graphic":{"type":"Text","Text":{"tid":null,"valign":"middle","overflow":"none","vposition":"none","hposition":"none","html":"<p style=\"text-align:center;\"><span style=\"font-size: 12px; font-family: Arial; white-space: pre-wrap; font-weight: bold; text-decoration: none; line-height: 14px; color: rgb(0, 0, 0);\">1. API wrapper\n</span></p><p style=\"text-align:center;\"><span style=\"font-size: 12px; font-family: Arial; white-space: pre-wrap; font-weight: bold; text-decoration: none; line-height: 14px; color: rgb(0, 0, 0);\">2. connections manager</span></p>","paddingLeft":2,"paddingRight":2,"paddingBottom":2,"paddingTop":2}},"children":null,"linkMap":[]},{"x":451,"y":61,"rotation":0,"id":37,"uid":"com.gliffy.shape.basic.basic_v1.default.text","width":150,"height":14,"lockAspectRatio":false,"lockShape":false,"order":23,"graphic":{"type":"Text","Text":{"tid":null,"valign":"middle","overflow":"none","vposition":"none","hposition":"none","html":"<p style=\"text-align:center;\"><span style=\"font-size: 12px; font-family: Arial; white-space: pre-wrap; font-weight: bold; text-decoration: none; line-height: 14px; color: rgb(0, 0, 0);\">3. offset storage</span></p>","paddingLeft":2,"paddingRight":2,"paddingBottom":2,"paddingTop":2}},"children":null,"linkMap":[]},{"x":433,"y":46,"rotation":0,"id":36,"uid":"com.gliffy.shape.basic.basic_v1.default.text","width":150,"height":14,"lockAspectRatio":false,"lockShape":false,"order":22,"graphic":{"type":"Text","Text":{"tid":null,"valign":"middle","overflow":"none","vposition":"none","hposition":"none","html":"<p style=\"text-align:center;\"><span style=\"font-size: 12px; font-family: Arial; white-space: pre-wrap; font-weight: bold; text-decoration: none; line-height: 14px; color: rgb(0, 0, 0);\">2. balance</span></p>","paddingLeft":2,"paddingRight":2,"paddingBottom":2,"paddingTop":2}},"children":null,"linkMap":[]},{"x":471,"y":30,"rotation":0,"id":35,"uid":"com.gliffy.shape.basic.basic_v1.default.text","width":180,"height":14,"lockAspectRatio":false,"lockShape":false,"order":21,"graphic":{"type":"Text","Text":{"tid":null,"valign":"middle","overflow":"none","vposition":"none","hposition":"none","html":"<p style=\"text-align:center;\"><span style=\"font-size: 12px; font-family: Arial; white-space: pre-wrap; font-weight: bold; text-decoration: none; line-height: 14px; color: rgb(0, 0, 0);\">1. push/pull(push based pull)</span></p>","paddingLeft":2,"paddingRight":2,"paddingBottom":2,"paddingTop":2}},"children":null,"linkMap":[]},{"x":252,"y":61,"rotation":0,"id":28,"uid":"com.gliffy.shape.basic.basic_v1.default.text","width":150,"height":14,"lockAspectRatio":false,"lockShape":false,"order":20,"graphic":{"type":"Text","Text":{"tid":null,"valign":"middle","overflow":"none","vposition":"none","hposition":"none","html":"<p style=\"text-align:center;\"><span style=\"font-size: 12px; font-family: Arial; white-space: pre-wrap; font-weight: bold; text-decoration: none; line-height: 14px; color: rgb(0, 0, 0);\">2. muti-producers</span></p>","paddingLeft":2,"paddingRight":2,"paddingBottom":2,"paddingTop":2}},"children":null,"linkMap":[]},{"x":251,"y":42,"rotation":0,"id":27,"uid":"com.gliffy.shape.basic.basic_v1.default.text","width":150,"height":14,"lockAspectRatio":false,"lockShape":false,"order":19,"graphic":{"type":"Text","Text":{"tid":null,"valign":"middle","overflow":"none","vposition":"none","hposition":"none","html":"<p style=\"text-align:center;\"><span style=\"font-size: 12px; font-family: Arial; white-space: pre-wrap; font-weight: bold; text-decoration: none; line-height: 14px; color: rgb(0, 0, 0);\">1. queue selector</span></p>","paddingLeft":2,"paddingRight":2,"paddingBottom":2,"paddingTop":2}},"children":null,"linkMap":[]},{"x":154,"y":272,"rotation":0,"id":21,"uid":"com.gliffy.shape.uml.uml_v1.default.message","width":100,"height":100,"lockAspectRatio":false,"lockShape":false,"order":18,"graphic":{"type":"Line","Line":{"strokeWidth":1,"strokeColor":"#000000","fillColor":"none","dashStyle":null,"startArrow":0,"endArrow":2,"startArrowRotation":"auto","endArrowRotation":"auto","ortho":true,"interpolationType":"linear","cornerRadius":null,"controlPath":[[278,-52],[278,-45.33333333333334],[278,-38.66666666666666],[278,-32]],"lockSegments":{}}},"children":null,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":7,"px":0.5,"py":1}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":0,"px":0.5,"py":0}}},"linkMap":[]},{"x":154,"y":272,"rotation":0,"id":20,"uid":"com.gliffy.shape.uml.uml_v1.default.message","width":100,"height":100,"lockAspectRatio":false,"lockShape":false,"order":17,"graphic":{"type":"Line","Line":{"strokeWidth":1,"strokeColor":"#000000","fillColor":"none","dashStyle":null,"startArrow":0,"endArrow":2,"startArrowRotation":"auto","endArrowRotation":"auto","ortho":true,"interpolationType":"linear","cornerRadius":null,"controlPath":[[388,-152],[388,-137],[352.5584412271571,-137],[352.5584412271571,-122]],"lockSegments":{}}},"children":null,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":17,"px":0.5,"py":1}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":7,"px":0.7071067811865476,"py":0}}},"linkMap":[]},{"x":144,"y":262,"rotation":0,"id":19,"uid":"com.gliffy.shape.uml.uml_v1.default.message","width":100,"height":100,"lockAspectRatio":false,"lockShape":false,"order":16,"graphic":{"type":"Line","Line":{"strokeWidth":1,"strokeColor":"#000000","fillColor":"none","dashStyle":null,"startArrow":0,"endArrow":2,"startArrowRotation":"auto","endArrowRotation":"auto","ortho":true,"interpolationType":"linear","cornerRadius":null,"controlPath":[[183,-142],[183,-127],[213.44155877284288,-127],[213.44155877284288,-112]],"lockSegments":{}}},"children":null,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":15,"px":0.5,"py":1}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":7,"px":0.2928932188134524,"py":0}}},"linkMap":[]},{"x":472,"y":80,"rotation":0,"id":17,"uid":"com.gliffy.shape.uml.uml_v1.default.simple_class","width":140,"height":40,"lockAspectRatio":false,"lockShape":false,"order":14,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.rectangle.basic_v1","strokeWidth":2,"strokeColor":"#000000","fillColor":"#FFFFFF","gradient":false,"dropShadow":false,"state":0,"shadowX":0,"shadowY":0,"opacity":1}},"children":[{"x":1.9999999999999998,"y":0,"rotation":0,"id":18,"uid":null,"width":135.99999999999997,"height":14,"lockAspectRatio":false,"lockShape":false,"order":"auto","graphic":{"type":"Text","Text":{"tid":null,"valign":"middle","overflow":"none","vposition":"none","hposition":"none","html":"<p style=\"text-align:center;\"><span style=\"font-size: 12px; font-family: Arial; white-space: pre-wrap; font-weight: bold; text-decoration: none; line-height: 14px; color: rgb(0, 0, 0);\">consumer</span></p>","paddingLeft":2,"paddingRight":2,"paddingBottom":2,"paddingTop":2}},"children":null}],"linkMap":[]},{"x":252,"y":80,"rotation":0,"id":15,"uid":"com.gliffy.shape.uml.uml_v1.default.simple_class","width":150,"height":40,"lockAspectRatio":false,"lockShape":false,"order":12,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.rectangle.basic_v1","strokeWidth":2,"strokeColor":"#000000","fillColor":"#FFFFFF","gradient":false,"dropShadow":false,"state":0,"shadowX":0,"shadowY":0,"opacity":1}},"children":[{"x":2.142857142857143,"y":0,"rotation":0,"id":16,"uid":null,"width":145.71428571428572,"height":14,"lockAspectRatio":false,"lockShape":false,"order":"auto","graphic":{"type":"Text","Text":{"tid":null,"valign":"middle","overflow":"none","vposition":"none","hposition":"none","html":"<p style=\"text-align:center;\"><span style=\"font-size: 12px; font-family: Arial; white-space: pre-wrap; font-weight: bold; text-decoration: none; line-height: 14px; color: rgb(0, 0, 0);\">producer</span></p>","paddingLeft":2,"paddingRight":2,"paddingBottom":2,"paddingTop":2}},"children":null}],"linkMap":[]},{"x":457,"y":165,"rotation":0,"id":13,"uid":"com.gliffy.shape.uml.uml_v1.default.simple_class","width":140,"height":40,"lockAspectRatio":false,"lockShape":false,"order":10,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.rectangle.basic_v1","strokeWidth":2,"strokeColor":"#000000","fillColor":"#FFFFFF","gradient":false,"dropShadow":false,"state":0,"shadowX":0,"shadowY":0,"opacity":1}},"children":[{"x":1.9999999999999998,"y":0,"rotation":0,"id":14,"uid":null,"width":135.99999999999997,"height":14,"lockAspectRatio":false,"lockShape":false,"order":"auto","graphic":{"type":"Text","Text":{"tid":null,"valign":"middle","overflow":"none","vposition":"none","hposition":"none","html":"<p style=\"text-align:center;\"><span style=\"font-size: 12px; font-family: Arial; white-space: pre-wrap; font-weight: bold; text-decoration: none; line-height: 14px; color: rgb(0, 0, 0);\">route</span></p>","paddingLeft":2,"paddingRight":2,"paddingBottom":2,"paddingTop":2}},"children":null}],"linkMap":[]},{"x":267,"y":165,"rotation":0,"id":11,"uid":"com.gliffy.shape.uml.uml_v1.default.simple_class","width":150,"height":40,"lockAspectRatio":false,"lockShape":false,"order":8,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.rectangle.basic_v1","strokeWidth":2,"strokeColor":"#000000","fillColor":"#FFFFFF","gradient":false,"dropShadow":false,"state":0,"shadowX":0,"shadowY":0,"opacity":1}},"children":[{"x":2.142857142857143,"y":0,"rotation":0,"id":12,"uid":null,"width":145.71428571428572,"height":14,"lockAspectRatio":false,"lockShape":false,"order":"auto","graphic":{"type":"Text","Text":{"tid":null,"valign":"middle","overflow":"none","vposition":"none","hposition":"none","html":"<p style=\"text-align:center;\"><span style=\"font-size: 12px; font-family: Arial; white-space: pre-wrap; font-weight: bold; text-decoration: none; line-height: 14px; color: rgb(0, 0, 0);\">manager</span></p>","paddingLeft":2,"paddingRight":2,"paddingBottom":2,"paddingTop":2}},"children":null}],"linkMap":[]},{"x":237,"y":248,"rotation":0,"id":9,"uid":"com.gliffy.shape.basic.basic_v1.default.text","width":150,"height":14,"lockAspectRatio":false,"lockShape":false,"order":7,"graphic":{"type":"Text","Text":{"tid":null,"valign":"middle","overflow":"none","vposition":"none","hposition":"none","html":"<p style=\"text-align:center;\"><span style=\"font-size: 12px; font-family: Arial; white-space: pre-wrap; font-weight: bold; text-decoration: none; line-height: 14px; color: rgb(0, 0, 0);\">remote</span></p>","paddingLeft":2,"paddingRight":2,"paddingBottom":2,"paddingTop":2}},"children":null,"linkMap":[]},{"x":252.00000000000003,"y":150,"rotation":0,"id":7,"uid":"com.gliffy.shape.uml.uml_v1.default.simple_class","width":359.99999999999994,"height":70,"lockAspectRatio":false,"lockShape":false,"order":6,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.rectangle.basic_v1","strokeWidth":2,"strokeColor":"#000000","fillColor":"#FFFFFF","gradient":false,"dropShadow":true,"state":0,"shadowX":4,"shadowY":4,"opacity":1}},"children":[],"linkMap":[]},{"x":342,"y":299,"rotation":0,"id":5,"uid":"com.gliffy.shape.uml.uml_v1.default.simple_class","width":180,"height":40,"lockAspectRatio":false,"lockShape":false,"order":4,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.rectangle.basic_v1","strokeWidth":2,"strokeColor":"#000000","fillColor":"#FFFFFF","gradient":false,"dropShadow":false,"state":0,"shadowX":0,"shadowY":0,"opacity":1}},"children":[{"x":2.5714285714285716,"y":0,"rotation":0,"id":6,"uid":null,"width":174.8571428571429,"height":14,"lockAspectRatio":false,"lockShape":false,"order":"auto","graphic":{"type":"Text","Text":{"tid":null,"valign":"middle","overflow":"none","vposition":"none","hposition":"none","html":"<p style=\"text-align:center;\"><span style=\"font-size: 12px; font-family: Arial; white-space: pre-wrap; font-weight: bold; text-decoration: none; line-height: 14px; color: rgb(0, 0, 0);\">codec</span></p>","paddingLeft":2,"paddingRight":2,"paddingBottom":2,"paddingTop":2}},"children":null}],"linkMap":[]},{"x":342,"y":248,"rotation":0,"id":2,"uid":"com.gliffy.shape.uml.uml_v1.default.simple_class","width":180,"height":40,"lockAspectRatio":false,"lockShape":false,"order":2,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.rectangle.basic_v1","strokeWidth":2,"strokeColor":"#000000","fillColor":"#FFFFFF","gradient":false,"dropShadow":false,"state":0,"shadowX":0,"shadowY":0,"opacity":1}},"children":[{"x":2.5714285714285716,"y":0,"rotation":0,"id":4,"uid":null,"width":174.8571428571429,"height":14,"lockAspectRatio":false,"lockShape":false,"order":"auto","graphic":{"type":"Text","Text":{"tid":null,"valign":"middle","overflow":"none","vposition":"none","hposition":"none","html":"<p style=\"text-align:center;\"><span style=\"font-size: 12px; font-family: Arial; white-space: pre-wrap; font-weight: bold; text-decoration: none; line-height: 14px; color: rgb(0, 0, 0);\">client</span></p>","paddingLeft":2,"paddingRight":2,"paddingBottom":2,"paddingTop":2}},"children":null}],"linkMap":[]},{"x":287,"y":240,"rotation":0,"id":0,"uid":"com.gliffy.shape.uml.uml_v1.default.simple_class","width":290,"height":110,"lockAspectRatio":false,"lockShape":false,"order":1,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.rectangle.basic_v1","strokeWidth":2,"strokeColor":"#000000","fillColor":"#FFFFFF","gradient":false,"dropShadow":true,"state":0,"shadowX":4,"shadowY":4,"opacity":1}},"children":[],"linkMap":[]}],"background":"#FFFFFF","width":710,"height":370,"maxWidth":5000,"maxHeight":5000,"nodeIndex":59,"autoFit":true,"exportBorder":false,"gridOn":true,"snapToGrid":true,"drawingGuidesOn":true,"pageBreaksOn":false,"printGridOn":false,"printPaper":"LETTER","printShrinkToFit":false,"printPortrait":true,"shapeStyles":{},"lineStyles":{"global":{"endArrow":1}},"textStyles":{},"themeData":null}}
\ No newline at end of file
diff --git a/docs/feature.md b/docs/feature.md
deleted file mode 100644
index eed3e17..0000000
--- a/docs/feature.md
+++ /dev/null
@@ -1,86 +0,0 @@
-# Feature
-
-## Producer
-
-### MessageType
-- [x] NormalMessage
-- [ ] TransactionMessage
-- [ ] DelayMessage
-
-### SendWith
-- [x] Sync
-- [ ] Async
-- [x] OneWay
-
-### Other
-- [ ] Config
-- [ ] MessageId Generate
-- [ ] CompressMsg
-- [ ] LoadBalance
-- [ ] DefaultTopic
-- [ ] VipChannel
-- [ ] Retry
-- [ ] Hook
-- [ ] CheckRequestQueue
-- [ ] MQFaultStrategy
-
-## Consumer
-
-### ReceiveType
-- [x] Push
-- [ ] Pull
-
-### ConsumingType
-- [x] Concurrently
-- [ ] Orderly
-
-### MessageModel
-- [x] CLUSTERING
-- [x] BROADCASTING
-
-### AllocateMessageQueueStrategy
-- [x] AllocateMessageQueueAveragely
-- [x] AllocateMessageQueueAveragelyByCircle
-- [X] AllocateMessageQueueByConfig
-- [X] AllocateMessageQueueByMachineRoom
-
-### Other
-- [x] Rebalance
-- [x] Flow Control
-- [ ] compress
-- [x] ConsumeFromWhere
-- [ ] Retry(sendMessageBack)
-- [ ] Hook
-
-## Common
-- [ ] PollNameServer
-- [x] Heartbeat
-- [x] UpdateTopicRouteInfoFromNameServer
-- [ ] CleanOfflineBroker
-- [ ] ClearExpiredMessage(form consumer consumeMessageService)
-
-## Remoting
-- [x] API
- - [x] InvokeSync
- - [x] InvokeAsync
- - [x] InvokeOneWay
-- [x] Serialize
- - [x] JSON
- - [x] ROCKETMQ
-- [ ] Other
- - [ ] VIPChannel
- - [ ] RPCHook
-
-## Admin
-
-### Topic/Cluster
-- [x] updateTopic
-- [x] deleteTopic
-- [ ] updateSubGroup
-- [ ] deleteSubGroup
-- [ ] updateBrokerConfig
-- [ ] updateTopicPerm
-- [ ] listTopic
-- [ ] topicRoute
-- [ ] topicStatus
-- [ ] topicClusterList
\ No newline at end of file
diff --git a/docs/images/client-design.png b/docs/images/client-design.png
deleted file mode 100644
index 97b86ef..0000000
--- a/docs/images/client-design.png
+++ /dev/null
Binary files differ
diff --git a/docs/zh/native-design_zh.md b/docs/zh/native-design_zh.md
deleted file mode 100644
index dd407a4..0000000
--- a/docs/zh/native-design_zh.md
+++ /dev/null
@@ -1,114 +0,0 @@
-# RocketMQ Go Client Design Draft
-
-## Architecture
-
-### Overview
-
-
-### Description
-在RocketMQ Java Client的实现里面，代码耦合了大量的admin方面的功能， 其为了尽可能的提高代码复用率，代码的依赖关系较为复杂、接口的设计比
-较重、语义的界限不够清晰。因此，为了避免简单的对Java代码进行翻译，故Go客户端进行了重新的设计，剥离了admin相关的逻辑。整体如上面的图所示，在逻辑层次上，按照请求
-顺序，从下到上总共分为三层：
-- remote层：client网络通信和私有协议层，将到每个到服务端(NameServer或Broker)的连接实体抽象为一个client结构，并在这里实现了网络数据的
-序列/反序列化。
-- 公共层：由于remote层对上层只暴露了`Sync/Async/Oneway`三个接口，所以对于特定的Request/Response、连接的管理、路由等信息的处理和维护、
-其它`producer/consumer`共用的逻辑，均在这里实现。
-- 业务逻辑层：在这里实现各种producer、consumer的语义。所有producer、consumer专有的逻辑，均放到这里实现，如`queue selector`,
-`consume balance`, `offset storage`等。除了基础的数据结构外，producer和consumer之间内部的代码不能进行复用。
-
-
-## 设计目标
-0. 兼容cgo版本已经暴露出去的API
-1. 实现语义清晰、轻量的API接口
-2. 依赖关系清晰简单，设计和实现要正交
-
-## 目录结构
-### 源码
-- producer：producer相关逻辑
-- consumer：consumer相关逻辑
-- common（可改名）：连接管理和路由管理相关的通用逻辑
-- remote：网络通信和序列化
-
-### 其它
-- benchmark：压力测试相关代码
-- core：1.2版本cgo的代码库，该目录下的代码将会被移除，只保留API进行兼容，并会被标记为`Deprecated`
-- docs：文档，包括面向用户和开发者
-- examples：示例代码
-- test：集成测试代码
-
-## API
-
-### remote
-```go
-NewRemotingCommand(code int16, header CustomHeader) *RemotingCommand
-
-// send a request to servers and return until response received.
-SendMessageSync(ctx context.Context, brokerAddrs, brokerName string, request *SendMessageRequest, msgs []*Message) (*SendResult, error)
-
-SendMessageAsync(ctx context.Context, brokerAddrs, brokerName string, request *SendMessageRequest, msgs []*Message, f func(result *SendResult)) error
-
-SendMessageOneWay(ctx context.Context, brokerAddrs string, request *SendMessageRequest, msgs []*Message) (*SendResult, error)
-```
-
-### common
-All struct needed has been defined in codebase.
-
-```go
-// PullMessage with sync
-SendMessage(topic string, msgs *[]Message) error
-
-// SendMessageAsync send message with batch by async
-SendMessageAsync(topic string, msgs *[]Message, f func(result *SendResult)) error
-
-// PullMessage with sync
-PullMessage(ctx context.Context, brokerAddrs string, request *PullMessageRequest) (*PullResult, error)
-
-// PullMessageAsync pull message async
-func PullMessageAsync(ctx context.Context, brokerAddrs string, request *PullMessageRequest, f func(result *PullResult)) error
-
-// QueryMaxOffset with specific queueId and topic
-QueryMaxOffset(topic string, queueId int) error
-
-// QueryConsumerOffset with specific queueId and topic of consumerGroup
-QueryConsumerOffset(consumerGroup, topic string, queue int) (int64, error)
-
-// SearchOffsetByTimestamp with specific queueId and topic
-SearchOffsetByTimestamp(topic string, queue int, timestamp int64) (int64, error)
-
-// UpdateConsumerOffset with specific queueId and topic
-UpdateConsumerOffset(consumerGroup, topic string, queue int, offset int64) error
-```
-
-## Road map
-for more details about features: [feature-list](../feature.md)
-
-### Milestone1(due: 2019.3.10)
-
-#### producer
-- [ ] normal message
-- [ ] order message
-
-#### consumer
-- [ ] normal message with pull/push
-- [ ] order message with pull/push
-- [ ] rebalance
-- [ ] offset manager
-
-#### common
-- [ ] API wrapper
-- [ ] connections manager
-- [ ] route
-
-#### remote
-- [ ] serializer
-- [ ] communication
-- [ ] processor
-- [ ] RPC
-
-### Milestone2 (2019.4.12)
-- Transaction Message
-- ACL
-- Message Tracing
-
-## sub project
-- RocketMQ Administration tools: JVM too heavy for command line tools
\ No newline at end of file
diff --git a/docs/zh/rocketmq-protocol_zh.md b/docs/zh/rocketmq-protocol_zh.md
deleted file mode 100644
index 818d2e1..0000000
--- a/docs/zh/rocketmq-protocol_zh.md
+++ /dev/null
@@ -1,117 +0,0 @@
-# RocketMQ 通信协议
-
-在 RocketMQ 中，`RemotingCommand` 是 RocketMQ 通信的基本对象，Request/Response 最后均被包装成 `RemotingCommand`。一个 `RemotingCommand` 在被序列化后的格式如下：
-
-```
-++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
-+ frame_size | header_length | header_body | body +
-++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
-+ 4bytes | 4bytes | (21 + r_len + e_len) bytes | remain bytes +
-++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
-```
-
-| item | type | description |
-| :-: | :-: | :-: |
-| frame_size | `int32` | 一个 `RemotingCommand` 数据包大小 |
-| header_length | `int32` | 高8位表示数据的序列化方式，余下的表示真实 header 长度 |
-| header_body | `[]byte` | header 的 payload，长度由附带的 `remark` 和 `properties` 决定|
-| body | `[]byte` | 具体 Request/Response 的 payload |
-
-## Header
-
-RocketMQ 的 Header 序列化方式有两种：JSON 和 RocketMQ 私有的序列化方式。JSON 序列化方式不再赘述。具体可以参考 Java `RemotingCommand` 类。
-
-主要介绍 RocketMQ 的私有序列化方式。
-
-在序列化的时候，需要将序列化方式记录进数据包里面，即对 `header_length` 进行编码
-
-```go
-// 编码算法
-
-// 编码后的 header_length
-var header_length int32
-
-// 实际的 header 长度
-var headerDataLen int32
-
-// 序列化方式
-var SerializedType byte
-
-result := make([]byte, 4)
-result[0]|SerializedType
-result[1]|byte((headerDataLen >> 16) & 0xFF)
-result[2]|byte((headerDataLen >> 8) & 0xFF)
-result[3]|byte(headerDataLen & 0xFF)
-binary.Read(result, binary.BigEndian, &header_length)
-
-// 解码算法
-headerDataLen := header_length & 0xFFFFFF
-SerializedType := byte((header_length >> 24) & 0xFF)
-```
-
-### Header Frame
-
-```
-+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
-+ request_code | l_flag | v_flag | opaque | request_flag | r_len | r_body | e_len | e_body +
-+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
-+ 2bytes | 1byte | 2bytes | 4bytes | 4 bytes | 4 bytes | r_len bytes | 4 bytes | e_len bytes +
-+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
-```
-
-| item | type | description |
-| :-: | :-: | :-: |
-| request_code | `int16` | 哪一种 Request 或 ResponseCode，具体类别由 request_flag 决定 |
-| l_flag | `byte` | language 位，用来标识Request来源方的开发语言 |
-| v_flag | `int16` | 版本标记位 |
-| request_flag |`int32`| Header标记位，用来标记该 `RemotingCommand` 的类型和请求方式 |
-| opaque | `int32` | 标识 Request/Response 的 RequestID，Broker 返回的 Response 通过该值和 Client 缓存的 Request 一一对应 |
-| r_len | `int32` | length of remark, remark 是 Request/Response 的附带说明信息，一般在 Response 中用来说明具体的错误原因 |
-| r_body | `[]byte` | payload of remark |
-| e_len | `int32` | length of extended fields，即 properties，一些非标准字段会存储在这里，在 RocketMQ 的各种 feature 中均有广泛应用 |
-| e_body | `int32` | payload of extended fields |
-
-## Body
-
-`body` 是具体的 Request/Response 的数据，在 RocketMQ 中，有许多种 Request/Response。每个类有自己的序列化和反序列方式，由于种类过多，
-这里就不再展开。可以具体参考Java代码中对`CommandCustomHeader`的使用。下面列一些 Client 使用到的 Request 和 Response。
-
-### RequestCode
-
-| item | type | description |
-| :-: | :-: | :-: |
-| SEND_MESSAGE | 10 | 向broker发送消息 |
-| PULL_MESSAGE | 11 | 从broker拉取消息，client的push模式也是通过pull的长轮询来实现的 |
-| TODO... | | |
-
-### ResponseCode
-
-| item | type | description |
-| :-: | :-: | :-: |
-| FLUSH_DISK_TIMEOUT | 10 | broker 存储层刷盘超时 |
-| SLAVE_NOT_AVAILABLE | 11 | slave 节点无法服务 |
-| FLUSH_SLAVE_TIMEOUT | 12 | 数据同步到 slave 超时 |
-| MESSAGE_ILLEGAL | 13 | 消息格式不合格 |
-| SERVICE_NOT_AVAILABLE | 14 | broker 暂时不可用 |
-| VERSION_NOT_SUPPORTED | 15 | 不支持的请求，目前没有看到使用 |
-| NO_PERMISSION | 16 | 对 broker、topic 或 subscription 无访问权限 |
-| TOPIC_EXIST_ALREADY | 18 | topic 已存在，目前没看到使用 |
-| PULL_NOT_FOUND | 19 | 没拉到消息，大多为 offset 错误 |
-| PULL_RETRY_IMMEDIATELY | 20 | 建议 client 立即重新拉取消息 |
-| PULL_OFFSET_MOVED | 21 | offset 太小或太大 |
-| QUERY_NOT_FOUND | 22 | 管理面 Response，TODO |
-| SUBSCRIPTION_PARSE_FAILED | 23 | 订阅数据解析失败 |
-| SUBSCRIPTION_NOT_EXIST | 24 | 订阅不存在 |
-| SUBSCRIPTION_NOT_LATEST | 25 | 订阅数据版本和 request 数据版本不匹配 |
-| SUBSCRIPTION_GROUP_NOT_EXIST | 26 | 订阅组不存在 |
-| FILTER_DATA_NOT_EXIST | 27 | filter 数据不存在 |
-| FILTER_DATA_NOT_LATEST | 28 | filter 数据版本和 request 数据版本不匹配 |
-| TRANSACTION_SHOULD_COMMIT | 200 | 事务 Response，TODO |
-| TRANSACTION_SHOULD_ROLLBACK | 201 | 事务 Response，TODO |
-| TRANSACTION_STATE_UNKNOW | 202 | 事务 Response，TODO | |
-| TRANSACTION_STATE_GROUP_WRONG | 203 | 事务 Response，TODO |
-| NO_BUYER_ID | 204 | 不知道是什么，没看到 broker 端在使用 |
-| NOT_IN_CURRENT_UNIT | 205 | 不知道是什么，没看到 broker 端在使用 |
-| CONSUMER_NOT_ONLINE | 206 | consumer 不在线，控制面 response |
-| CONSUME_MSG_TIMEOUT | 207 | client request 等待 broker 相应超时 |
-| NO_MESSAGE | 208 | 控制面 response，由 client 自己设置，不清楚具体用途 |
diff --git a/errors/errors.go b/errors/errors.go
deleted file mode 100644
index 195984e..0000000
--- a/errors/errors.go
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package errors
-
-import "errors"
-
-var (
- ErrRequestTimeout = errors.New("request timeout")
- ErrMQEmpty = errors.New("MessageQueue is nil")
- ErrOffset = errors.New("offset < 0")
- ErrNumbers = errors.New("numbers < 0")
- ErrEmptyTopic = errors.New("empty topic")
- ErrEmptyNameSrv = errors.New("empty namesrv")
- ErrEmptyGroupID = errors.New("empty group id")
- ErrTestMin = errors.New("test minutes must be positive integer")
- ErrOperationInterval = errors.New("operation interval must be positive integer")
- ErrMessageBody = errors.New("message body size must be positive integer")
- ErrEmptyExpression = errors.New("empty expression")
- ErrCreated = errors.New("consumer group has been created")
- ErrBrokerNotFound = errors.New("broker can not found")
- ErrStartTopic = errors.New("cannot subscribe topic since client either failed to start or has been shutdown.")
- ErrResponse = errors.New("response error")
- ErrCompressLevel = errors.New("unsupported compress level")
- ErrUnknownIP = errors.New("unknown IP address")
- ErrService = errors.New("service close is not running, please check")
- ErrTopicNotExist = errors.New("topic not exist")
- ErrNotExisted = errors.New("not existed")
- ErrNoNameserver = errors.New("nameServerAddrs can't be empty.")
- ErrMultiIP = errors.New("multiple IP addr does not support")
- ErrIllegalIP = errors.New("IP addr error")
- ErrTopicEmpty = errors.New("topic is nil")
- ErrMessageEmpty = errors.New("message is nil")
- ErrNotRunning = errors.New("producer not started")
- ErrPullConsumer = errors.New("pull consumer has not supported")
- ErrMultipleTopics = errors.New("the topic of the messages in one batch should be the same")
-)
diff --git a/examples/admin/topic/main.go b/examples/admin/topic/main.go
deleted file mode 100644
index ef9a536..0000000
--- a/examples/admin/topic/main.go
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
- "context"
- "fmt"
-
- "github.com/apache/rocketmq-client-go/v2/admin"
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-func main() {
- topic := "newOne"
- //clusterName := "DefaultCluster"
- nameSrvAddr := []string{"127.0.0.1:9876"}
- brokerAddr := "127.0.0.1:10911"
-
- testAdmin, err := admin.NewAdmin(admin.WithResolver(primitive.NewPassthroughResolver(nameSrvAddr)))
- if err != nil {
- fmt.Println(err.Error())
- }
-
- //create topic
- err = testAdmin.CreateTopic(
- context.Background(),
- admin.WithTopicCreate(topic),
- admin.WithBrokerAddrCreate(brokerAddr),
- )
- if err != nil {
- fmt.Println("Create topic error:", err.Error())
- }
-
- //deletetopic
- err = testAdmin.DeleteTopic(
- context.Background(),
- admin.WithTopicDelete(topic),
- //admin.WithBrokerAddrDelete(brokerAddr),
- //admin.WithNameSrvAddr(nameSrvAddr),
- )
- if err != nil {
- fmt.Println("Delete topic error:", err.Error())
- }
-
- err = testAdmin.Close()
- if err != nil {
- fmt.Printf("Shutdown admin error: %s", err.Error())
- }
-}
diff --git a/examples/consumer/acl/main.go b/examples/consumer/acl/main.go
deleted file mode 100644
index 96d0dd0..0000000
--- a/examples/consumer/acl/main.go
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
- "context"
- "fmt"
- "os"
- "time"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/consumer"
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-func main() {
- c, err := rocketmq.NewPushConsumer(
- consumer.WithGroupName("testGroup"),
- consumer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- consumer.WithCredentials(primitive.Credentials{
- AccessKey: "RocketMQ",
- SecretKey: "12345678",
- }),
- )
- if err != nil {
- fmt.Println("init consumer error: " + err.Error())
- os.Exit(0)
- }
-
- err = c.Subscribe("test", consumer.MessageSelector{}, func(ctx context.Context,
- msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
- fmt.Printf("subscribe callback: %v \n", msgs)
- return consumer.ConsumeSuccess, nil
- })
- if err != nil {
- fmt.Println(err.Error())
- }
- // Note: start after subscribe
- err = c.Start()
- if err != nil {
- fmt.Println(err.Error())
- os.Exit(-1)
- }
- time.Sleep(time.Hour)
- err = c.Shutdown()
- if err != nil {
- fmt.Printf("Shutdown Consumer error: %s", err.Error())
- }
-}
diff --git a/examples/consumer/broadcast/main.go b/examples/consumer/broadcast/main.go
deleted file mode 100644
index 219ab3e..0000000
--- a/examples/consumer/broadcast/main.go
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
- "context"
- "fmt"
- "os"
- "time"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/consumer"
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-func main() {
- c, _ := rocketmq.NewPushConsumer(
- consumer.WithGroupName("testGroup"),
- consumer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- consumer.WithConsumeFromWhere(consumer.ConsumeFromFirstOffset),
- consumer.WithConsumerModel(consumer.BroadCasting),
- )
- err := c.Subscribe("min", consumer.MessageSelector{}, func(ctx context.Context,
- msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
- fmt.Printf("subscribe callback: %v \n", msgs)
- return consumer.ConsumeSuccess, nil
- })
- if err != nil {
- fmt.Println(err.Error())
- }
- // Note: start after subscribe
- err = c.Start()
- if err != nil {
- fmt.Println(err.Error())
- os.Exit(-1)
- }
- time.Sleep(time.Hour)
- err = c.Shutdown()
- if err != nil {
- fmt.Printf("Shutdown Consumer error: %s", err.Error())
- }
-}
diff --git a/examples/consumer/delay/main.go b/examples/consumer/delay/main.go
deleted file mode 100644
index 1bbe6ed..0000000
--- a/examples/consumer/delay/main.go
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
- "context"
- "fmt"
- "os"
- "time"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/consumer"
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-func main() {
- c, _ := rocketmq.NewPushConsumer(
- consumer.WithGroupName("testGroup"),
- consumer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- )
- err := c.Subscribe("TopicTest", consumer.MessageSelector{}, func(ctx context.Context,
- msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
-
- for _, msg := range msgs {
- t := time.Now().UnixNano()/int64(time.Millisecond) - msg.BornTimestamp
- fmt.Printf("Receive message[msgId=%s] %d ms later\n", msg.MsgId, t)
- }
-
- return consumer.ConsumeSuccess, nil
- })
- if err != nil {
- fmt.Println(err.Error())
- }
- // Note: start after subscribe
- err = c.Start()
- if err != nil {
- fmt.Println(err.Error())
- os.Exit(-1)
- }
- time.Sleep(time.Hour)
- err = c.Shutdown()
- if err != nil {
- fmt.Printf("Shutdown Consumer error: %s", err.Error())
- }
-}
diff --git a/examples/consumer/interceptor/main.go b/examples/consumer/interceptor/main.go
deleted file mode 100644
index 83f43b0..0000000
--- a/examples/consumer/interceptor/main.go
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
- "context"
- "fmt"
- "os"
- "time"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/consumer"
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-func main() {
- c, _ := rocketmq.NewPushConsumer(
- consumer.WithGroupName("testGroup"),
- consumer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- consumer.WithConsumerModel(consumer.Clustering),
- consumer.WithConsumeFromWhere(consumer.ConsumeFromFirstOffset),
- consumer.WithInterceptor(UserFistInterceptor(), UserSecondInterceptor()))
- err := c.Subscribe("TopicTest", consumer.MessageSelector{}, func(ctx context.Context,
- msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
- fmt.Printf("subscribe callback: %v \n", msgs)
- return consumer.ConsumeSuccess, nil
- })
- if err != nil {
- fmt.Println(err.Error())
- }
- // Note: start after subscribe
- err = c.Start()
- if err != nil {
- fmt.Println(err.Error())
- os.Exit(-1)
- }
- time.Sleep(time.Hour)
- err = c.Shutdown()
- if err != nil {
- fmt.Printf("Shutdown Consumer error: %s", err.Error())
- }
-}
-
-func UserFistInterceptor() primitive.Interceptor {
- return func(ctx context.Context, req, reply interface{}, next primitive.Invoker) error {
- msgCtx, _ := primitive.GetConsumerCtx(ctx)
- fmt.Printf("msgCtx: %v, mehtod: %s", msgCtx, primitive.GetMethod(ctx))
-
- msgs := req.([]*primitive.MessageExt)
- fmt.Printf("user first interceptor before invoke: %v\n", msgs)
- e := next(ctx, msgs, reply)
-
- holder := reply.(*consumer.ConsumeResultHolder)
- fmt.Printf("user first interceptor after invoke: %v, result: %v\n", msgs, holder)
- return e
- }
-}
-
-func UserSecondInterceptor() primitive.Interceptor {
- return func(ctx context.Context, req, reply interface{}, next primitive.Invoker) error {
- msgs := req.([]*primitive.MessageExt)
- fmt.Printf("user second interceptor before invoke: %v\n", msgs)
- e := next(ctx, msgs, reply)
- holder := reply.(*consumer.ConsumeResultHolder)
- fmt.Printf("user second interceptor after invoke: %v, result: %v\n", msgs, holder)
- return e
- }
-}
diff --git a/examples/consumer/namespace/main.go b/examples/consumer/namespace/main.go
deleted file mode 100644
index d46f210..0000000
--- a/examples/consumer/namespace/main.go
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
- "context"
- "fmt"
- "os"
- "time"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/consumer"
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-func main() {
- c, err := rocketmq.NewPushConsumer(
- consumer.WithGroupName("testGroup"),
- consumer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- consumer.WithCredentials(primitive.Credentials{
- AccessKey: "RocketMQ",
- SecretKey: "12345678",
- }),
- consumer.WithNamespace("namespace"),
- )
- if err != nil {
- fmt.Println("init consumer error: " + err.Error())
- os.Exit(0)
- }
-
- err = c.Subscribe("test", consumer.MessageSelector{}, func(ctx context.Context,
- msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
- fmt.Printf("subscribe callback: %v \n", msgs)
- return consumer.ConsumeSuccess, nil
- })
- if err != nil {
- fmt.Println(err.Error())
- }
- // Note: start after subscribe
- err = c.Start()
- if err != nil {
- fmt.Println(err.Error())
- os.Exit(-1)
- }
- time.Sleep(time.Hour)
- err = c.Shutdown()
- if err != nil {
- fmt.Printf("Shutdown Consumer error: %s", err.Error())
- }
-}
diff --git a/examples/consumer/orderly/main.go b/examples/consumer/orderly/main.go
deleted file mode 100644
index 0f33e5f..0000000
--- a/examples/consumer/orderly/main.go
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
- "context"
- "fmt"
- "os"
- "time"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/consumer"
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-func main() {
- c, _ := rocketmq.NewPushConsumer(
- consumer.WithGroupName("testGroup"),
- consumer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- consumer.WithConsumerModel(consumer.Clustering),
- consumer.WithConsumeFromWhere(consumer.ConsumeFromFirstOffset),
- consumer.WithConsumerOrder(true),
- )
- err := c.Subscribe("TopicTest", consumer.MessageSelector{}, func(ctx context.Context,
- msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
- orderlyCtx, _ := primitive.GetOrderlyCtx(ctx)
- fmt.Printf("orderly context: %v\n", orderlyCtx)
- fmt.Printf("subscribe orderly callback: %v \n", msgs)
- return consumer.ConsumeSuccess, nil
- })
- if err != nil {
- fmt.Println(err.Error())
- }
- // Note: start after subscribe
- err = c.Start()
- if err != nil {
- fmt.Println(err.Error())
- os.Exit(-1)
- }
- time.Sleep(time.Hour)
- err = c.Shutdown()
- if err != nil {
- fmt.Printf("Shutdown Consumer error: %s", err.Error())
- }
-}
diff --git a/examples/consumer/pull/main.go b/examples/consumer/pull/main.go
deleted file mode 100644
index 5b5819e..0000000
--- a/examples/consumer/pull/main.go
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
- "context"
- "fmt"
- "github.com/apache/rocketmq-client-go/v2/errors"
- "time"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/consumer"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/rlog"
-)
-
-func main() {
- c, err := rocketmq.NewPullConsumer(
- consumer.WithGroupName("testGroup"),
- consumer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- )
- if err != nil {
- rlog.Fatal(fmt.Sprintf("fail to new pullConsumer: %s", err), nil)
- }
- err = c.Start()
- if err != nil {
- rlog.Fatal(fmt.Sprintf("fail to new pullConsumer: %s", err), nil)
- }
-
- ctx := context.Background()
- queue := primitive.MessageQueue{
- Topic: "TopicTest",
- BrokerName: "", // replace with your broker name. otherwise, pull will failed.
- QueueId: 0,
- }
-
- offset := int64(0)
- for {
- resp, err := c.PullFrom(ctx, queue, offset, 10)
- if err != nil {
- if err == errors.ErrRequestTimeout {
- fmt.Printf("timeout \n")
- time.Sleep(1 * time.Second)
- continue
- }
- fmt.Printf("unexpectable err: %v \n", err)
- return
- }
- if resp.Status == primitive.PullFound {
- fmt.Printf("pull message success. nextOffset: %d \n", resp.NextBeginOffset)
- for _, msg := range resp.GetMessageExts() {
- fmt.Printf("pull msg: %v \n", msg)
- }
- }
- offset = resp.NextBeginOffset
- }
-}
diff --git a/examples/consumer/retry/concurrent/main.go b/examples/consumer/retry/concurrent/main.go
deleted file mode 100644
index 49f16d1..0000000
--- a/examples/consumer/retry/concurrent/main.go
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
- "context"
- "fmt"
- "os"
- "time"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/consumer"
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-// use concurrent consumer model, when Subscribe function return consumer.ConsumeRetryLater, the message will be
-// send to RocketMQ retry topic. we could set DelayLevelWhenNextConsume in ConsumeConcurrentlyContext, which used to
-// indicate the delay of message re-send to origin topic from retry topic.
-//
-// in this example, we always set DelayLevelWhenNextConsume=1, means that the message will be sent to origin topic after
-// 1s. in case of the unlimited retry, we will return consumer.ConsumeSuccess after ReconsumeTimes > 5
-func main() {
- c, _ := rocketmq.NewPushConsumer(
- consumer.WithGroupName("testGroup"),
- consumer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- consumer.WithConsumerModel(consumer.Clustering),
- )
-
- // The DelayLevel specify the waiting time that before next reconsume,
- // and it range is from 1 to 18 now.
- //
- // The time of each level is the value of indexing of {level-1} in [1s, 5s, 10s, 30s,
- // 1m, 2m, 3m, 4m, 5m, 6m, 7m, 8m, 9m, 10m, 20m, 30m, 1h, 2h]
- delayLevel := 1
- err := c.Subscribe("TopicTest", consumer.MessageSelector{}, func(ctx context.Context,
- msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
- fmt.Printf("subscribe callback len: %d \n", len(msgs))
-
- concurrentCtx, _ := primitive.GetConcurrentlyCtx(ctx)
- concurrentCtx.DelayLevelWhenNextConsume = delayLevel // only run when return consumer.ConsumeRetryLater
-
- for _, msg := range msgs {
- if msg.ReconsumeTimes > 5 {
- fmt.Printf("msg ReconsumeTimes > 5. msg: %v", msg)
- return consumer.ConsumeSuccess, nil
- } else {
- fmt.Printf("subscribe callback: %v \n", msg)
- }
- }
- return consumer.ConsumeRetryLater, nil
- })
- if err != nil {
- fmt.Println(err.Error())
- }
- // Note: start after subscribe
- err = c.Start()
- if err != nil {
- fmt.Println(err.Error())
- os.Exit(-1)
- }
- time.Sleep(time.Hour)
- err = c.Shutdown()
- if err != nil {
- fmt.Printf("shundown Consumer error: %s", err.Error())
- }
-}
diff --git a/examples/consumer/retry/order/main.go b/examples/consumer/retry/order/main.go
deleted file mode 100644
index e2d13bf..0000000
--- a/examples/consumer/retry/order/main.go
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-/**
- * use orderly consumer model, when Subscribe function return consumer.SuspendCurrentQueueAMoment, it will be re-send to
- * local msg queue for later consume if msg.ReconsumeTimes < MaxReconsumeTimes, otherwise, it will be send to rocketmq
- * DLQ topic, we should manually resolve the msg.
- */
-package main
-
-import (
- "context"
- "fmt"
- "os"
- "time"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/consumer"
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-func main() {
- c, _ := rocketmq.NewPushConsumer(
- consumer.WithGroupName("testGroup"),
- consumer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- consumer.WithConsumerModel(consumer.Clustering),
- consumer.WithConsumeFromWhere(consumer.ConsumeFromFirstOffset),
- consumer.WithConsumerOrder(true),
- consumer.WithMaxReconsumeTimes(5),
- )
-
- err := c.Subscribe("TopicTest", consumer.MessageSelector{}, func(ctx context.Context,
- msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
- orderlyCtx, _ := primitive.GetOrderlyCtx(ctx)
- fmt.Printf("orderly context: %v\n", orderlyCtx)
- fmt.Printf("subscribe orderly callback len: %d \n", len(msgs))
-
- for _, msg := range msgs {
- if msg.ReconsumeTimes > 5 {
- fmt.Printf("msg ReconsumeTimes > 5. msg: %v", msg)
- } else {
- fmt.Printf("subscribe orderly callback: %v \n", msg)
- }
- }
- return consumer.SuspendCurrentQueueAMoment, nil
-
- })
- if err != nil {
- fmt.Println(err.Error())
- }
- // Note: start after subscribe
- err = c.Start()
- if err != nil {
- fmt.Println(err.Error())
- os.Exit(-1)
- }
- time.Sleep(time.Hour)
- err = c.Shutdown()
- if err != nil {
- fmt.Printf("shundown Consumer error: %s", err.Error())
- }
-}
diff --git a/examples/consumer/simple/main.go b/examples/consumer/simple/main.go
deleted file mode 100644
index 70f35b3..0000000
--- a/examples/consumer/simple/main.go
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
- "context"
- "fmt"
- "os"
- "time"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/consumer"
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-func main() {
- c, _ := rocketmq.NewPushConsumer(
- consumer.WithGroupName("testGroup"),
- consumer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- )
- err := c.Subscribe("test", consumer.MessageSelector{}, func(ctx context.Context,
- msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
- for i := range msgs {
- fmt.Printf("subscribe callback: %v \n", msgs[i])
- }
-
- return consumer.ConsumeSuccess, nil
- })
- if err != nil {
- fmt.Println(err.Error())
- }
- // Note: start after subscribe
- err = c.Start()
- if err != nil {
- fmt.Println(err.Error())
- os.Exit(-1)
- }
- time.Sleep(time.Hour)
- err = c.Shutdown()
- if err != nil {
- fmt.Printf("shutdown Consumer error: %s", err.Error())
- }
-}
diff --git a/examples/consumer/strategy/main.go b/examples/consumer/strategy/main.go
deleted file mode 100644
index 269ce11..0000000
--- a/examples/consumer/strategy/main.go
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
- "context"
- "fmt"
- "os"
- "time"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/consumer"
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-func main() {
- c, _ := rocketmq.NewPushConsumer(
- consumer.WithGroupName("testGroup"),
- consumer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- consumer.WithStrategy(consumer.AllocateByAveragely),
- )
- err := c.Subscribe("TopicTest", consumer.MessageSelector{}, func(ctx context.Context,
- msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
- fmt.Printf("subscribe callback: %v \n", msgs)
- return consumer.ConsumeSuccess, nil
- })
- if err != nil {
- fmt.Println(err.Error())
- }
- // Note: start after subscribe
- err = c.Start()
- if err != nil {
- fmt.Println(err.Error())
- os.Exit(-1)
- }
- time.Sleep(time.Hour)
- err = c.Shutdown()
- if err != nil {
- fmt.Printf("shutdown Consumer error: %s", err.Error())
- }
-}
diff --git a/examples/consumer/tag/main.go b/examples/consumer/tag/main.go
deleted file mode 100644
index ec16f51..0000000
--- a/examples/consumer/tag/main.go
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
- "context"
- "fmt"
- "os"
- "time"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/consumer"
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-func main() {
- c, _ := rocketmq.NewPushConsumer(
- consumer.WithGroupName("testGroup"),
- consumer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- )
- selector := consumer.MessageSelector{
- Type: consumer.TAG,
- Expression: "TagA || TagC",
- }
- err := c.Subscribe("TopicTest", selector, func(ctx context.Context,
- msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
- fmt.Printf("subscribe callback: %v \n", msgs)
- return consumer.ConsumeSuccess, nil
- })
- if err != nil {
- fmt.Println(err.Error())
- }
- err = c.Start()
- if err != nil {
- fmt.Println(err.Error())
- os.Exit(-1)
- }
- time.Sleep(time.Hour)
- err = c.Shutdown()
- if err != nil {
- fmt.Printf("shutdown Consumer error: %s", err.Error())
- }
-}
diff --git a/examples/consumer/trace/main.go b/examples/consumer/trace/main.go
deleted file mode 100644
index 97f22ed..0000000
--- a/examples/consumer/trace/main.go
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
- "context"
- "fmt"
- "os"
- "time"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/consumer"
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-func main() {
- namesrvs := []string{"127.0.0.1:9876"}
- traceCfg := &primitive.TraceConfig{
- Access: primitive.Local,
- Resolver: primitive.NewPassthroughResolver(namesrvs),
- }
-
- c, _ := rocketmq.NewPushConsumer(
- consumer.WithGroupName("testGroup"),
- consumer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- consumer.WithTrace(traceCfg),
- )
- err := c.Subscribe("TopicTest", consumer.MessageSelector{}, func(ctx context.Context,
- msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
- fmt.Printf("subscribe callback: %v \n", msgs)
- return consumer.ConsumeSuccess, nil
- })
- if err != nil {
- fmt.Println(err.Error())
- }
- // Note: start after subscribe
- err = c.Start()
- if err != nil {
- fmt.Println(err.Error())
- os.Exit(-1)
-
- }
- time.Sleep(time.Hour)
- err = c.Shutdown()
- if err != nil {
- fmt.Printf("shutdown Consumer error: %s", err.Error())
- }
-}
diff --git a/examples/producer/acl/main.go b/examples/producer/acl/main.go
deleted file mode 100644
index 96881d3..0000000
--- a/examples/producer/acl/main.go
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Package main implements a producer with user custom interceptor.
-package main
-
-import (
- "context"
- "fmt"
- "os"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/producer"
-)
-
-func main() {
- p, err := rocketmq.NewProducer(
- producer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- producer.WithRetry(2),
- producer.WithCredentials(primitive.Credentials{
- AccessKey: "RocketMQ",
- SecretKey: "12345678",
- }),
- )
-
- if err != nil {
- fmt.Println("init producer error: " + err.Error())
- os.Exit(0)
- }
-
- err = p.Start()
- if err != nil {
- fmt.Printf("start producer error: %s", err.Error())
- os.Exit(1)
- }
- for i := 0; i < 100000; i++ {
- res, err := p.SendSync(context.Background(), primitive.NewMessage("test",
- []byte("Hello RocketMQ Go Client!")))
-
- if err != nil {
- fmt.Printf("send message error: %s\n", err)
- } else {
- fmt.Printf("send message success: result=%s\n", res.String())
- }
- }
- err = p.Shutdown()
- if err != nil {
- fmt.Printf("shutdown producer error: %s", err.Error())
- }
-}
diff --git a/examples/producer/async/main.go b/examples/producer/async/main.go
deleted file mode 100644
index 2a5182c..0000000
--- a/examples/producer/async/main.go
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
- "context"
- "fmt"
- "os"
- "sync"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/producer"
-)
-
-// Package main implements a async producer to send message.
-func main() {
- p, _ := rocketmq.NewProducer(
- producer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- producer.WithRetry(2),
- producer.WithQueueSelector(producer.NewManualQueueSelector()))
-
- err := p.Start()
- if err != nil {
- fmt.Printf("start producer error: %s", err.Error())
- os.Exit(1)
- }
- var wg sync.WaitGroup
- for i := 0; i < 10; i++ {
- wg.Add(1)
- err := p.SendAsync(context.Background(),
- func(ctx context.Context, result *primitive.SendResult, e error) {
- if e != nil {
- fmt.Printf("receive message error: %s\n", err)
- } else {
- fmt.Printf("send message success: result=%s\n", result.String())
- }
- wg.Done()
- }, primitive.NewMessage("test", []byte("Hello RocketMQ Go Client!")))
-
- if err != nil {
- fmt.Printf("send message error: %s\n", err)
- }
- }
- wg.Wait()
- err = p.Shutdown()
- if err != nil {
- fmt.Printf("shutdown producer error: %s", err.Error())
- }
-}
diff --git a/examples/producer/batch/main.go b/examples/producer/batch/main.go
deleted file mode 100644
index dc591a1..0000000
--- a/examples/producer/batch/main.go
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
- "context"
- "fmt"
- "os"
- "strconv"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/producer"
-)
-
-func main() {
- p, _ := rocketmq.NewProducer(
- producer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- producer.WithRetry(2),
- )
- err := p.Start()
- if err != nil {
- fmt.Printf("start producer error: %s", err.Error())
- os.Exit(1)
- }
- var msgs []*primitive.Message
- for i := 0; i < 10; i++ {
- msgs = append(msgs, primitive.NewMessage("test",
- []byte("Hello RocketMQ Go Client! num: "+strconv.Itoa(i))))
- }
-
- res, err := p.SendSync(context.Background(), msgs...)
-
- if err != nil {
- fmt.Printf("send message error: %s\n", err)
- } else {
- fmt.Printf("send message success: result=%s\n", res.String())
- }
- err = p.Shutdown()
- if err != nil {
- fmt.Printf("shutdown producer error: %s", err.Error())
- }
-}
diff --git a/examples/producer/delay/main.go b/examples/producer/delay/main.go
deleted file mode 100644
index aadbb70..0000000
--- a/examples/producer/delay/main.go
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
- "context"
- "fmt"
- "os"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/producer"
-)
-
-func main() {
- p, _ := rocketmq.NewProducer(
- producer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- producer.WithRetry(2),
- )
- err := p.Start()
- if err != nil {
- fmt.Printf("start producer error: %s", err.Error())
- os.Exit(1)
- }
- for i := 0; i < 10; i++ {
- msg := primitive.NewMessage("test", []byte("Hello RocketMQ Go Client!"))
- msg.WithDelayTimeLevel(3)
- res, err := p.SendSync(context.Background(), msg)
-
- if err != nil {
- fmt.Printf("send message error: %s\n", err)
- } else {
- fmt.Printf("send message success: result=%s\n", res.String())
- }
- }
- err = p.Shutdown()
- if err != nil {
- fmt.Printf("shutdown producer error: %s", err.Error())
- }
-}
diff --git a/examples/producer/interceptor/main.go b/examples/producer/interceptor/main.go
deleted file mode 100644
index c40f9a0..0000000
--- a/examples/producer/interceptor/main.go
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Package main implements a producer with user custom interceptor.
-package main
-
-import (
- "context"
- "fmt"
- "os"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/producer"
-)
-
-func main() {
- p, _ := rocketmq.NewProducer(
- producer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- producer.WithRetry(2),
- producer.WithInterceptor(UserFirstInterceptor(), UserSecondInterceptor()),
- )
- err := p.Start()
- if err != nil {
- fmt.Printf("start producer error: %s", err.Error())
- os.Exit(1)
- }
- for i := 0; i < 10; i++ {
- res, err := p.SendSync(context.Background(), primitive.NewMessage("test",
- []byte("Hello RocketMQ Go Client!")))
-
- if err != nil {
- fmt.Printf("send message error: %s\n", err)
- } else {
- fmt.Printf("send message success: result=%s\n", res.String())
- }
- }
- err = p.Shutdown()
- if err != nil {
- fmt.Printf("shutdown producer error: %s", err.Error())
- }
-}
-
-func UserFirstInterceptor() primitive.Interceptor {
- return func(ctx context.Context, req, reply interface{}, next primitive.Invoker) error {
- fmt.Printf("user first interceptor before invoke: req:%v\n", req)
- err := next(ctx, req, reply)
- fmt.Printf("user first interceptor after invoke: req: %v, reply: %v \n", req, reply)
- return err
- }
-}
-
-func UserSecondInterceptor() primitive.Interceptor {
- return func(ctx context.Context, req, reply interface{}, next primitive.Invoker) error {
- fmt.Printf("user second interceptor before invoke: req: %v\n", req)
- err := next(ctx, req, reply)
- fmt.Printf("user second interceptor after invoke: req: %v, reply: %v \n", req, reply)
- return err
- }
-}
diff --git a/examples/producer/namespace/main.go b/examples/producer/namespace/main.go
deleted file mode 100644
index 9124284..0000000
--- a/examples/producer/namespace/main.go
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Package main implements a producer with user custom interceptor.
-package main
-
-import (
- "context"
- "fmt"
- "os"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/producer"
-)
-
-func main() {
- p, err := rocketmq.NewProducer(
- producer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- producer.WithRetry(2),
- producer.WithCredentials(primitive.Credentials{
- AccessKey: "RocketMQ",
- SecretKey: "12345678",
- }),
- producer.WithNamespace("namespace"),
- )
-
- if err != nil {
- fmt.Println("init producer error: " + err.Error())
- os.Exit(0)
- }
-
- err = p.Start()
- if err != nil {
- fmt.Printf("start producer error: %s", err.Error())
- os.Exit(1)
- }
- for i := 0; i < 100000; i++ {
- res, err := p.SendSync(context.Background(), primitive.NewMessage("test",
- []byte("Hello RocketMQ Go Client!")))
-
- if err != nil {
- fmt.Printf("send message error: %s\n", err)
- } else {
- fmt.Printf("send message success: result=%s\n", res.String())
- }
- }
- err = p.Shutdown()
- if err != nil {
- fmt.Printf("shutdown producer error: %s", err.Error())
- }
-}
diff --git a/examples/producer/simple/main.go b/examples/producer/simple/main.go
deleted file mode 100644
index 6fd3364..0000000
--- a/examples/producer/simple/main.go
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
- "context"
- "fmt"
- "os"
- "strconv"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/producer"
-)
-
-// Package main implements a simple producer to send message.
-func main() {
- p, _ := rocketmq.NewProducer(
- producer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- producer.WithRetry(2),
- )
- err := p.Start()
- if err != nil {
- fmt.Printf("start producer error: %s", err.Error())
- os.Exit(1)
- }
- topic := "test"
-
- for i := 0; i < 10; i++ {
- msg := &primitive.Message{
- Topic: topic,
- Body: []byte("Hello RocketMQ Go Client! " + strconv.Itoa(i)),
- }
- res, err := p.SendSync(context.Background(), msg)
-
- if err != nil {
- fmt.Printf("send message error: %s\n", err)
- } else {
- fmt.Printf("send message success: result=%s\n", res.String())
- }
- }
- err = p.Shutdown()
- if err != nil {
- fmt.Printf("shutdown producer error: %s", err.Error())
- }
-}
diff --git a/examples/producer/tag/main.go b/examples/producer/tag/main.go
deleted file mode 100644
index 7ce8559..0000000
--- a/examples/producer/tag/main.go
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
- "context"
- "fmt"
- "os"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/producer"
-)
-
-func main() {
- p, _ := rocketmq.NewProducer(
- producer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- producer.WithRetry(2),
- )
- err := p.Start()
- if err != nil {
- fmt.Printf("start producer error: %s", err.Error())
- os.Exit(1)
- }
- tags := []string{"TagA", "TagB", "TagC"}
- for i := 0; i < 3; i++ {
- tag := tags[i%3]
- msg := primitive.NewMessage("test",
- []byte("Hello RocketMQ Go Client!"))
- msg.WithTag(tag)
-
- res, err := p.SendSync(context.Background(), msg)
- if err != nil {
- fmt.Printf("send message error: %s\n", err)
- } else {
- fmt.Printf("send message success: result=%s\n", res.String())
- }
- }
- err = p.Shutdown()
- if err != nil {
- fmt.Printf("shutdown producer error: %s", err.Error())
- }
-}
diff --git a/examples/producer/trace/main.go b/examples/producer/trace/main.go
deleted file mode 100644
index b741704..0000000
--- a/examples/producer/trace/main.go
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
- "context"
- "fmt"
- "os"
- "time"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/producer"
-)
-
-func main() {
- namesrvs := []string{"127.0.0.1:9876"}
- traceCfg := &primitive.TraceConfig{
- Access: primitive.Local,
- Resolver: primitive.NewPassthroughResolver(namesrvs),
- }
-
- p, _ := rocketmq.NewProducer(
- producer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- producer.WithRetry(2),
- producer.WithTrace(traceCfg))
- err := p.Start()
- if err != nil {
- fmt.Printf("start producer error: %s", err.Error())
- os.Exit(1)
- }
- for i := 0; i < 1; i++ {
- res, err := p.SendSync(context.Background(), primitive.NewMessage("test",
- []byte("Hello RocketMQ Go Client!")))
-
- if err != nil {
- fmt.Printf("send message error: %s\n", err)
- } else {
- fmt.Printf("send message success: result=%s\n", res.String())
- }
- }
-
- time.Sleep(10 * time.Second)
-
- err = p.Shutdown()
- if err != nil {
- fmt.Printf("shutdown producer error: %s", err.Error())
- }
-}
diff --git a/examples/producer/transaction/main.go b/examples/producer/transaction/main.go
deleted file mode 100644
index 05b6c52..0000000
--- a/examples/producer/transaction/main.go
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
- "context"
- "fmt"
- "os"
- "strconv"
- "sync"
- "sync/atomic"
- "time"
-
- "github.com/apache/rocketmq-client-go/v2"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/producer"
-)
-
-type DemoListener struct {
- localTrans *sync.Map
- transactionIndex int32
-}
-
-func NewDemoListener() *DemoListener {
- return &DemoListener{
- localTrans: new(sync.Map),
- }
-}
-
-func (dl *DemoListener) ExecuteLocalTransaction(msg *primitive.Message) primitive.LocalTransactionState {
- nextIndex := atomic.AddInt32(&dl.transactionIndex, 1)
- fmt.Printf("nextIndex: %v for transactionID: %v\n", nextIndex, msg.TransactionId)
- status := nextIndex % 3
- dl.localTrans.Store(msg.TransactionId, primitive.LocalTransactionState(status+1))
-
- fmt.Printf("dl")
- return primitive.UnknowState
-}
-
-func (dl *DemoListener) CheckLocalTransaction(msg *primitive.MessageExt) primitive.LocalTransactionState {
- fmt.Printf("%v msg transactionID : %v\n", time.Now(), msg.TransactionId)
- v, existed := dl.localTrans.Load(msg.TransactionId)
- if !existed {
- fmt.Printf("unknow msg: %v, return Commit", msg)
- return primitive.CommitMessageState
- }
- state := v.(primitive.LocalTransactionState)
- switch state {
- case 1:
- fmt.Printf("checkLocalTransaction COMMIT_MESSAGE: %v\n", msg)
- return primitive.CommitMessageState
- case 2:
- fmt.Printf("checkLocalTransaction ROLLBACK_MESSAGE: %v\n", msg)
- return primitive.RollbackMessageState
- case 3:
- fmt.Printf("checkLocalTransaction unknow: %v\n", msg)
- return primitive.UnknowState
- default:
- fmt.Printf("checkLocalTransaction default COMMIT_MESSAGE: %v\n", msg)
- return primitive.CommitMessageState
- }
-}
-
-func main() {
- p, _ := rocketmq.NewTransactionProducer(
- NewDemoListener(),
- producer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- producer.WithRetry(1),
- )
- err := p.Start()
- if err != nil {
- fmt.Printf("start producer error: %s\n", err.Error())
- os.Exit(1)
- }
-
- for i := 0; i < 10; i++ {
- res, err := p.SendMessageInTransaction(context.Background(),
- primitive.NewMessage("TopicTest5", []byte("Hello RocketMQ again "+strconv.Itoa(i))))
-
- if err != nil {
- fmt.Printf("send message error: %s\n", err)
- } else {
- fmt.Printf("send message success: result=%s\n", res.String())
- }
- }
- time.Sleep(5 * time.Minute)
- err = p.Shutdown()
- if err != nil {
- fmt.Printf("shutdown producer error: %s", err.Error())
- }
-}
diff --git a/go.mod b/go.mod
deleted file mode 100644
index 0aac6ba..0000000
--- a/go.mod
+++ /dev/null
@@ -1,17 +0,0 @@
-module github.com/apache/rocketmq-client-go/v2
-
-go 1.13
-
-require (
- github.com/emirpasic/gods v1.12.0
- github.com/golang/mock v1.3.1
- github.com/json-iterator/go v1.1.9
- github.com/pkg/errors v0.8.1
- github.com/sirupsen/logrus v1.4.1
- github.com/smartystreets/goconvey v1.6.4
- github.com/stathat/consistent v1.0.0
- github.com/stretchr/testify v1.3.0
- github.com/tidwall/gjson v1.13.0
- go.uber.org/atomic v1.5.1
- stathat.com/c/consistent v1.0.0 // indirect
-)
diff --git a/go.sum b/go.sum
deleted file mode 100644
index 157f415..0000000
--- a/go.sum
+++ /dev/null
@@ -1,63 +0,0 @@
-github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
-github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg=
-github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o=
-github.com/golang/mock v1.3.1 h1:qGJ6qTW+x6xX/my+8YUVl4WNpX9B7+/l2tRsHGZ7f2s=
-github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
-github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
-github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
-github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns=
-github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
-github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
-github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
-github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
-github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
-github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg=
-github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
-github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
-github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k=
-github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
-github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
-github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
-github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
-github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
-github.com/stathat/consistent v1.0.0 h1:ZFJ1QTRn8npNBKW065raSZ8xfOqhpb8vLOkfp4CcL/U=
-github.com/stathat/consistent v1.0.0/go.mod h1:uajTPbgSygZBJ+V+0mY7meZ8i0XAcZs7AQ6V121XSxw=
-github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
-github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
-github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/tidwall/gjson v1.13.0 h1:3TFY9yxOQShrvmjdM76K+jc66zJeT6D3/VFFYCGQf7M=
-github.com/tidwall/gjson v1.13.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
-github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
-github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
-github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
-github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
-go.uber.org/atomic v1.5.1 h1:rsqfU5vBkVknbhUGbAUwQKR2H4ItV8tjJ+6kJX4cxHM=
-go.uber.org/atomic v1.5.1/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs=
-golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c h1:IGkKhmfzcztjm6gYkykvu/NiS8kaqbCWAEWWAyf8J5U=
-golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-stathat.com/c/consistent v1.0.0 h1:ezyc51EGcRPJUxfHGSgJjWzJdj3NiMU9pNfLNGiXV0c=
-stathat.com/c/consistent v1.0.0/go.mod h1:QkzMWzcbB+yQBL2AttO6sgsQS/JSTapcDISJalmCDS0=
diff --git a/internal/callback.go b/internal/callback.go
deleted file mode 100644
index fea11b4..0000000
--- a/internal/callback.go
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package internal
-
-import (
- "net"
-
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-// remotingClient callback TransactionProducer
-type CheckTransactionStateCallback struct {
- Addr net.Addr
- Msg *primitive.MessageExt
- Header CheckTransactionStateRequestHeader
-}
diff --git a/internal/client.go b/internal/client.go
deleted file mode 100644
index c7f3e58..0000000
--- a/internal/client.go
+++ /dev/null
@@ -1,893 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package internal
-
-import (
- "context"
- "errors"
- "fmt"
- errors2 "github.com/apache/rocketmq-client-go/v2/errors"
- "net"
- "os"
- "sort"
- "strconv"
- "strings"
- "sync"
- "time"
-
- "github.com/apache/rocketmq-client-go/v2/internal/remote"
- "github.com/apache/rocketmq-client-go/v2/internal/utils"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/rlog"
-)
-
-const (
- clientVersion = "v2.1.0"
- defaultTraceRegionID = "DefaultRegion"
-
- // tracing message switch
- _TraceOff = "false"
-
- // Pulling topic information interval from the named server
- _PullNameServerInterval = 30 * time.Second
-
- // Sending heart beat interval to all broker
- _HeartbeatBrokerInterval = 30 * time.Second
-
- // Offset persistent interval for consumer
- _PersistOffsetInterval = 5 * time.Second
-
- // Rebalance interval
- _RebalanceInterval = 20 * time.Second
-)
-
-var (
- ErrServiceState = errors2.ErrService
-
- _VIPChannelEnable = false
-)
-
-func init() {
- if os.Getenv("com.rocketmq.sendMessageWithVIPChannel") != "" {
- value, err := strconv.ParseBool(os.Getenv("com.rocketmq.sendMessageWithVIPChannel"))
- if err == nil {
- _VIPChannelEnable = value
- }
- }
-}
-
-type InnerProducer interface {
- PublishTopicList() []string
- UpdateTopicPublishInfo(topic string, info *TopicPublishInfo)
- IsPublishTopicNeedUpdate(topic string) bool
- IsUnitMode() bool
-}
-
-type InnerConsumer interface {
- PersistConsumerOffset() error
- UpdateTopicSubscribeInfo(topic string, mqs []*primitive.MessageQueue)
- IsSubscribeTopicNeedUpdate(topic string) bool
- SubscriptionDataList() []*SubscriptionData
- Rebalance()
- IsUnitMode() bool
- GetConsumerRunningInfo() *ConsumerRunningInfo
- ConsumeMessageDirectly(msg *primitive.MessageExt, brokerName string) *ConsumeMessageDirectlyResult
- GetcType() string
- GetModel() string
- GetWhere() string
- ResetOffset(topic string, table map[primitive.MessageQueue]int64)
-}
-
-func DefaultClientOptions() ClientOptions {
- opts := ClientOptions{
- InstanceName: "DEFAULT",
- RetryTimes: 3,
- ClientIP: utils.LocalIP,
- }
- return opts
-}
-
-type ClientOptions struct {
- GroupName string
- NameServerAddrs primitive.NamesrvAddr
- Namesrv Namesrvs
- ClientIP string
- InstanceName string
- UnitMode bool
- UnitName string
- VIPChannelEnabled bool
- RetryTimes int
- Interceptors []primitive.Interceptor
- Credentials primitive.Credentials
- Namespace string
- Resolver primitive.NsResolver
-}
-
-func (opt *ClientOptions) ChangeInstanceNameToPID() {
- if opt.InstanceName == "DEFAULT" {
- opt.InstanceName = fmt.Sprintf("%d#%d", os.Getpid(), time.Now().UnixNano())
- }
-}
-
-func (opt *ClientOptions) String() string {
- return fmt.Sprintf("ClientOption [ClientIP=%s, InstanceName=%s, "+
- "UnitMode=%v, UnitName=%s, VIPChannelEnabled=%v]", opt.ClientIP,
- opt.InstanceName, opt.UnitMode, opt.UnitName, opt.VIPChannelEnabled)
-}
-
-//go:generate mockgen -source client.go -destination mock_client.go -self_package github.com/apache/rocketmq-client-go/v2/internal --package internal RMQClient
-type RMQClient interface {
- Start()
- Shutdown()
-
- ClientID() string
-
- RegisterProducer(group string, producer InnerProducer) error
- UnregisterProducer(group string)
- InvokeSync(ctx context.Context, addr string, request *remote.RemotingCommand,
- timeoutMillis time.Duration) (*remote.RemotingCommand, error)
- InvokeAsync(ctx context.Context, addr string, request *remote.RemotingCommand,
- f func(*remote.RemotingCommand, error)) error
- InvokeOneWay(ctx context.Context, addr string, request *remote.RemotingCommand,
- timeoutMillis time.Duration) error
- CheckClientInBroker()
- SendHeartbeatToAllBrokerWithLock()
- UpdateTopicRouteInfo()
-
- ProcessSendResponse(brokerName string, cmd *remote.RemotingCommand, resp *primitive.SendResult, msgs ...*primitive.Message) error
-
- RegisterConsumer(group string, consumer InnerConsumer) error
- UnregisterConsumer(group string)
- PullMessage(ctx context.Context, brokerAddrs string, request *PullMessageRequestHeader) (*primitive.PullResult, error)
- RebalanceImmediately()
- UpdatePublishInfo(topic string, data *TopicRouteData, changed bool)
-
- GetNameSrv() Namesrvs
-}
-
-var _ RMQClient = new(rmqClient)
-
-type rmqClient struct {
- option ClientOptions
- // group -> InnerProducer
- producerMap sync.Map
-
- // group -> InnerConsumer
- consumerMap sync.Map
- once sync.Once
-
- remoteClient remote.RemotingClient
- hbMutex sync.Mutex
- close bool
- rbMutex sync.Mutex
- done chan struct{}
- shutdownOnce sync.Once
-}
-
-func (c *rmqClient) GetNameSrv() Namesrvs {
- return c.option.Namesrv
-}
-
-var clientMap sync.Map
-
-func GetOrNewRocketMQClient(option ClientOptions, callbackCh chan interface{}) RMQClient {
- client := &rmqClient{
- option: option,
- remoteClient: remote.NewRemotingClient(),
- done: make(chan struct{}),
- }
- actual, loaded := clientMap.LoadOrStore(client.ClientID(), client)
-
- if loaded {
- // compare namesrv address
- client = actual.(*rmqClient)
- now := option.Namesrv.(*namesrvs).resolver.Resolve()
- old := client.GetNameSrv().(*namesrvs).resolver.Resolve()
- if len(now) != len(old) {
- rlog.Error("different namesrv option in the same instance", map[string]interface{}{
- "NewNameSrv": now,
- "BeforeNameSrv": old,
- })
- return nil
- }
- sort.Strings(now)
- sort.Strings(old)
- for i := 0; i < len(now); i++ {
- if now[i] != old[i] {
- rlog.Error("different namesrv option in the same instance", map[string]interface{}{
- "NewNameSrv": now,
- "BeforeNameSrv": old,
- })
- return nil
- }
- }
- } else {
- client.remoteClient.RegisterRequestFunc(ReqNotifyConsumerIdsChanged, func(req *remote.RemotingCommand, addr net.Addr) *remote.RemotingCommand {
- rlog.Info("receive broker's notification to consumer group", map[string]interface{}{
- rlog.LogKeyConsumerGroup: req.ExtFields["consumerGroup"],
- })
- client.RebalanceImmediately()
- return nil
- })
- client.remoteClient.RegisterRequestFunc(ReqCheckTransactionState, func(req *remote.RemotingCommand, addr net.Addr) *remote.RemotingCommand {
- header := new(CheckTransactionStateRequestHeader)
- header.Decode(req.ExtFields)
- msgExts := primitive.DecodeMessage(req.Body)
- if len(msgExts) == 0 {
- rlog.Warning("checkTransactionState, decode message failed", nil)
- return nil
- }
- msgExt := msgExts[0]
- // TODO: add namespace support
- transactionID := msgExt.GetProperty(primitive.PropertyUniqueClientMessageIdKeyIndex)
- if len(transactionID) > 0 {
- msgExt.TransactionId = transactionID
- }
- group := msgExt.GetProperty(primitive.PropertyProducerGroup)
- if group == "" {
- rlog.Warning("checkTransactionState, pick producer group failed", nil)
- return nil
- }
- if option.GroupName != group {
- rlog.Warning("producer group is not equal", nil)
- return nil
- }
- callback := &CheckTransactionStateCallback{
- Addr: addr,
- Msg: msgExt,
- Header: *header,
- }
- callbackCh <- callback
- return nil
- })
-
- client.remoteClient.RegisterRequestFunc(ReqGetConsumerRunningInfo, func(req *remote.RemotingCommand, addr net.Addr) *remote.RemotingCommand {
- rlog.Info("receive get consumer running info request...", nil)
- header := new(GetConsumerRunningInfoHeader)
- header.Decode(req.ExtFields)
- val, exist := clientMap.Load(header.clientID)
- res := remote.NewRemotingCommand(ResError, nil, nil)
- if !exist {
- res.Remark = fmt.Sprintf("Can't find specified client instance of: %s", header.clientID)
- } else {
- cli, ok := val.(*rmqClient)
- var runningInfo *ConsumerRunningInfo
- if ok {
- runningInfo = cli.getConsumerRunningInfo(header.consumerGroup)
- }
- if runningInfo != nil {
- res.Code = ResSuccess
- data, err := runningInfo.Encode()
- if err != nil {
- res.Remark = fmt.Sprintf("json marshal error: %s", err.Error())
- } else {
- res.Body = data
- }
- } else {
- res.Remark = "there is unexpected error when get running info, please check log"
- }
- }
- return res
- })
-
- client.remoteClient.RegisterRequestFunc(ReqConsumeMessageDirectly, func(req *remote.RemotingCommand, addr net.Addr) *remote.RemotingCommand {
- rlog.Info("receive consume message directly request...", nil)
- header := new(ConsumeMessageDirectlyHeader)
- header.Decode(req.ExtFields)
- val, exist := clientMap.Load(header.clientID)
- res := remote.NewRemotingCommand(ResError, nil, nil)
- if !exist {
- res.Remark = fmt.Sprintf("Can't find specified client instance of: %s", header.clientID)
- } else {
- cli, ok := val.(*rmqClient)
- msg := primitive.DecodeMessage(req.Body)[0]
- var consumeMessageDirectlyResult *ConsumeMessageDirectlyResult
- if ok {
- consumeMessageDirectlyResult = cli.consumeMessageDirectly(msg, header.consumerGroup, header.brokerName)
- }
- if consumeMessageDirectlyResult != nil {
- res.Code = ResSuccess
- data, err := consumeMessageDirectlyResult.Encode()
- if err != nil {
- res.Remark = fmt.Sprintf("json marshal error: %s", err.Error())
- } else {
- res.Body = data
- }
- } else {
- res.Remark = "there is unexpected error when consume message directly, please check log"
- }
- }
- return res
- })
-
- client.remoteClient.RegisterRequestFunc(ReqResetConsumerOffset, func(req *remote.RemotingCommand, addr net.Addr) *remote.RemotingCommand {
- rlog.Info("receive reset consumer offset request...", map[string]interface{}{
- rlog.LogKeyBroker: addr.String(),
- rlog.LogKeyTopic: req.ExtFields["topic"],
- rlog.LogKeyConsumerGroup: req.ExtFields["group"],
- rlog.LogKeyTimeStamp: req.ExtFields["timestamp"],
- })
- header := new(ResetOffsetHeader)
- header.Decode(req.ExtFields)
-
- body := new(ResetOffsetBody)
- body.Decode(req.Body)
-
- client.resetOffset(header.topic, header.group, body.OffsetTable)
- return nil
- })
- }
- return client
-}
-
-func (c *rmqClient) Start() {
- //ctx, cancel := context.WithCancel(context.Background())
- //c.cancel = cancel
- c.once.Do(func() {
- if !c.option.Credentials.IsEmpty() {
- c.remoteClient.RegisterInterceptor(remote.ACLInterceptor(c.option.Credentials))
- }
- go primitive.WithRecover(func() {
- op := func() {
- c.GetNameSrv().UpdateNameServerAddress()
- }
- time.Sleep(10 * time.Second)
- op()
-
- ticker := time.NewTicker(2 * time.Minute)
- defer ticker.Stop()
- for {
- select {
- case <-ticker.C:
- op()
- case <-c.done:
- rlog.Info("The RMQClient stopping update name server domain info.", map[string]interface{}{
- "clientID": c.ClientID(),
- })
- return
- }
- }
- })
-
- // schedule update route info
- go primitive.WithRecover(func() {
- // delay
- op := func() {
- c.UpdateTopicRouteInfo()
- }
- time.Sleep(10 * time.Millisecond)
- op()
-
- ticker := time.NewTicker(_PullNameServerInterval)
- defer ticker.Stop()
- for {
- select {
- case <-ticker.C:
- op()
- case <-c.done:
- rlog.Info("The RMQClient stopping update topic route info.", map[string]interface{}{
- "clientID": c.ClientID(),
- })
- return
- }
- }
- })
-
- go primitive.WithRecover(func() {
- op := func() {
- c.GetNameSrv().cleanOfflineBroker()
- c.SendHeartbeatToAllBrokerWithLock()
- }
-
- time.Sleep(time.Second)
- op()
-
- ticker := time.NewTicker(_HeartbeatBrokerInterval)
- defer ticker.Stop()
- for {
- select {
- case <-ticker.C:
- op()
- case <-c.done:
- rlog.Info("The RMQClient stopping clean off line broker and heart beat", map[string]interface{}{
- "clientID": c.ClientID(),
- })
- return
- }
- }
- })
-
- // schedule persist offset
- go primitive.WithRecover(func() {
- op := func() {
- c.consumerMap.Range(func(key, value interface{}) bool {
- consumer := value.(InnerConsumer)
- err := consumer.PersistConsumerOffset()
- if err != nil {
- rlog.Error("persist offset failed", map[string]interface{}{
- rlog.LogKeyUnderlayError: err,
- })
- }
- return true
- })
- }
- time.Sleep(10 * time.Second)
- op()
-
- ticker := time.NewTicker(_PersistOffsetInterval)
- defer ticker.Stop()
- for {
- select {
- case <-ticker.C:
- op()
- case <-c.done:
- rlog.Info("The RMQClient stopping persist offset", map[string]interface{}{
- "clientID": c.ClientID(),
- })
- return
- }
- }
- })
-
- go primitive.WithRecover(func() {
- ticker := time.NewTicker(_RebalanceInterval)
- defer ticker.Stop()
- for {
- select {
- case <-ticker.C:
- c.RebalanceImmediately()
- case <-c.done:
- rlog.Info("The RMQClient stopping do rebalance", map[string]interface{}{
- "clientID": c.ClientID(),
- })
- return
- }
- }
- })
- })
-}
-
-func (c *rmqClient) removeClient() {
- rlog.Info("will remove client from clientMap", map[string]interface{}{
- "clientID": c.ClientID(),
- })
- clientMap.Delete(c.ClientID())
-}
-
-func (c *rmqClient) Shutdown() {
- c.shutdownOnce.Do(func() {
- close(c.done)
- c.close = true
- c.remoteClient.ShutDown()
- c.removeClient()
- })
-}
-
-func (c *rmqClient) ClientID() string {
- id := c.option.ClientIP + "@"
- if c.option.InstanceName == "DEFAULT" {
- id += strconv.Itoa(os.Getpid())
- } else {
- id += c.option.InstanceName
- }
- if c.option.UnitName != "" {
- id += "@" + c.option.UnitName
- }
- return id
-}
-
-func (c *rmqClient) InvokeSync(ctx context.Context, addr string, request *remote.RemotingCommand,
- timeoutMillis time.Duration) (*remote.RemotingCommand, error) {
- if c.close {
- return nil, ErrServiceState
- }
- var cancel context.CancelFunc
- ctx, cancel = context.WithTimeout(ctx, timeoutMillis)
- defer cancel()
- return c.remoteClient.InvokeSync(ctx, addr, request)
-}
-
-func (c *rmqClient) InvokeAsync(ctx context.Context, addr string, request *remote.RemotingCommand,
- f func(*remote.RemotingCommand, error)) error {
- if c.close {
- return ErrServiceState
- }
- return c.remoteClient.InvokeAsync(ctx, addr, request, func(future *remote.ResponseFuture) {
- f(future.ResponseCommand, future.Err)
- })
-
-}
-
-func (c *rmqClient) InvokeOneWay(ctx context.Context, addr string, request *remote.RemotingCommand,
- timeoutMillis time.Duration) error {
- if c.close {
- return ErrServiceState
- }
- return c.remoteClient.InvokeOneWay(ctx, addr, request)
-}
-
-func (c *rmqClient) CheckClientInBroker() {
-}
-
-// TODO
-func (c *rmqClient) SendHeartbeatToAllBrokerWithLock() {
- c.hbMutex.Lock()
- defer c.hbMutex.Unlock()
- hbData := NewHeartbeatData(c.ClientID())
-
- c.producerMap.Range(func(key, value interface{}) bool {
- pData := producerData{
- GroupName: key.(string),
- }
- hbData.ProducerDatas.Add(pData)
- return true
- })
-
- c.consumerMap.Range(func(key, value interface{}) bool {
- consumer := value.(InnerConsumer)
- cData := consumerData{
- GroupName: key.(string),
- CType: consumeType(consumer.GetcType()),
- MessageModel: strings.ToUpper(consumer.GetModel()),
- Where: consumer.GetWhere(),
- UnitMode: consumer.IsUnitMode(),
- SubscriptionDatas: consumer.SubscriptionDataList(),
- }
- hbData.ConsumerDatas.Add(cData)
- return true
- })
- if hbData.ProducerDatas.Len() == 0 && hbData.ConsumerDatas.Len() == 0 {
- rlog.Info("sending heartbeat, but no producer and no consumer", nil)
- return
- }
- c.GetNameSrv().(*namesrvs).brokerAddressesMap.Range(func(key, value interface{}) bool {
- brokerName := key.(string)
- data := value.(*BrokerData)
- for id, addr := range data.BrokerAddresses {
- rlog.Debug("try to send heart beat to broker", map[string]interface{}{
- "brokerName": brokerName,
- "brokerId": id,
- "brokerAddr": addr,
- })
- if hbData.ConsumerDatas.Len() == 0 && id != 0 {
- rlog.Debug("notice, will not send heart beat to broker", map[string]interface{}{
- "brokerName": brokerName,
- "brokerId": id,
- "brokerAddr": addr,
- })
- continue
- }
- cmd := remote.NewRemotingCommand(ReqHeartBeat, nil, hbData.encode())
-
- ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
- response, err := c.remoteClient.InvokeSync(ctx, addr, cmd)
- if err != nil {
- cancel()
- rlog.Warning("send heart beat to broker error", map[string]interface{}{
- rlog.LogKeyUnderlayError: err,
- })
- return true
- }
- cancel()
- if response.Code == ResSuccess {
- c.GetNameSrv().(*namesrvs).AddBrokerVersion(brokerName, addr, int32(response.Version))
- rlog.Debug("send heart beat to broker success", map[string]interface{}{
- "brokerName": brokerName,
- "brokerId": id,
- "brokerAddr": addr,
- })
- } else {
- rlog.Warning("send heart beat to broker failed", map[string]interface{}{
- "brokerName": brokerName,
- "brokerId": id,
- "brokerAddr": addr,
- "responseCode": response.Code,
- })
- }
- }
- return true
- })
-}
-
-func (c *rmqClient) UpdateTopicRouteInfo() {
- publishTopicSet := make(map[string]bool, 0)
- c.producerMap.Range(func(key, value interface{}) bool {
- producer := value.(InnerProducer)
- list := producer.PublishTopicList()
- for idx := range list {
- publishTopicSet[list[idx]] = true
- }
- return true
- })
- for topic := range publishTopicSet {
- data, changed, _ := c.GetNameSrv().UpdateTopicRouteInfo(topic)
- c.UpdatePublishInfo(topic, data, changed)
- }
-
- subscribedTopicSet := make(map[string]bool, 0)
- c.consumerMap.Range(func(key, value interface{}) bool {
- consumer := value.(InnerConsumer)
- list := consumer.SubscriptionDataList()
- for idx := range list {
- subscribedTopicSet[list[idx].Topic] = true
- }
- return true
- })
-
- for topic := range subscribedTopicSet {
- data, changed, _ := c.GetNameSrv().UpdateTopicRouteInfo(topic)
- c.updateSubscribeInfo(topic, data, changed)
- }
-}
-
-func (c *rmqClient) ProcessSendResponse(brokerName string, cmd *remote.RemotingCommand, resp *primitive.SendResult, msgs ...*primitive.Message) error {
- var status primitive.SendStatus
- switch cmd.Code {
- case ResFlushDiskTimeout:
- status = primitive.SendFlushDiskTimeout
- case ResFlushSlaveTimeout:
- status = primitive.SendFlushSlaveTimeout
- case ResSlaveNotAvailable:
- status = primitive.SendSlaveNotAvailable
- case ResSuccess:
- status = primitive.SendOK
- default:
- status = primitive.SendUnknownError
- return errors.New(cmd.Remark)
- }
-
- msgIDs := make([]string, 0)
- for i := 0; i < len(msgs); i++ {
- msgIDs = append(msgIDs, msgs[i].GetProperty(primitive.PropertyUniqueClientMessageIdKeyIndex))
- }
- uniqueMsgId := strings.Join(msgIDs, ",")
-
- regionId := cmd.ExtFields[primitive.PropertyMsgRegion]
- trace := cmd.ExtFields[primitive.PropertyTraceSwitch]
-
- if regionId == "" {
- regionId = defaultTraceRegionID
- }
-
- qId, _ := strconv.Atoi(cmd.ExtFields["queueId"])
- off, _ := strconv.ParseInt(cmd.ExtFields["queueOffset"], 10, 64)
-
- resp.Status = status
- resp.MsgID = uniqueMsgId
- resp.OffsetMsgID = cmd.ExtFields["msgId"]
- resp.MessageQueue = &primitive.MessageQueue{
- Topic: msgs[0].Topic,
- BrokerName: brokerName,
- QueueId: qId,
- }
- resp.QueueOffset = off
- resp.TransactionID = cmd.ExtFields["transactionId"]
- resp.RegionID = regionId
- resp.TraceOn = trace != "" && trace != _TraceOff
- return nil
-}
-
-// PullMessage with sync
-func (c *rmqClient) PullMessage(ctx context.Context, brokerAddrs string, request *PullMessageRequestHeader) (*primitive.PullResult, error) {
- cmd := remote.NewRemotingCommand(ReqPullMessage, request, nil)
- var cancel context.CancelFunc
- ctx, cancel = context.WithTimeout(ctx, 30*time.Second)
- defer cancel()
- res, err := c.remoteClient.InvokeSync(ctx, brokerAddrs, cmd)
- if err != nil {
- return nil, err
- }
-
- return c.processPullResponse(res)
-}
-
-func (c *rmqClient) processPullResponse(response *remote.RemotingCommand) (*primitive.PullResult, error) {
-
- pullResult := &primitive.PullResult{}
- switch response.Code {
- case ResSuccess:
- pullResult.Status = primitive.PullFound
- case ResPullNotFound:
- pullResult.Status = primitive.PullNoNewMsg
- case ResPullRetryImmediately:
- pullResult.Status = primitive.PullNoMsgMatched
- case ResPullOffsetMoved:
- pullResult.Status = primitive.PullOffsetIllegal
- default:
- return nil, fmt.Errorf("unknown Response Code: %d, remark: %s", response.Code, response.Remark)
- }
-
- c.decodeCommandCustomHeader(pullResult, response)
- pullResult.SetBody(response.Body)
-
- return pullResult, nil
-}
-
-func (c *rmqClient) decodeCommandCustomHeader(pr *primitive.PullResult, cmd *remote.RemotingCommand) {
- v, exist := cmd.ExtFields["maxOffset"]
- if exist {
- pr.MaxOffset, _ = strconv.ParseInt(v, 10, 64)
- }
-
- v, exist = cmd.ExtFields["minOffset"]
- if exist {
- pr.MinOffset, _ = strconv.ParseInt(v, 10, 64)
- }
-
- v, exist = cmd.ExtFields["nextBeginOffset"]
- if exist {
- pr.NextBeginOffset, _ = strconv.ParseInt(v, 10, 64)
- }
-
- v, exist = cmd.ExtFields["suggestWhichBrokerId"]
- if exist {
- pr.SuggestWhichBrokerId, _ = strconv.ParseInt(v, 10, 64)
- }
-}
-
-func (c *rmqClient) RegisterConsumer(group string, consumer InnerConsumer) error {
- _, exist := c.consumerMap.Load(group)
- if exist {
- rlog.Warning("the consumer group exist already", map[string]interface{}{
- rlog.LogKeyConsumerGroup: group,
- })
- return fmt.Errorf("the consumer group exist already")
- }
- c.consumerMap.Store(group, consumer)
- return nil
-}
-
-func (c *rmqClient) UnregisterConsumer(group string) {
- c.consumerMap.Delete(group)
-}
-
-func (c *rmqClient) RegisterProducer(group string, producer InnerProducer) error {
- _, loaded := c.producerMap.LoadOrStore(group, producer)
- if loaded {
- return fmt.Errorf("the producer group \"%s\" has been created, specify another one", c.option.GroupName)
- }
- return nil
-}
-
-func (c *rmqClient) UnregisterProducer(group string) {
- c.producerMap.Delete(group)
-}
-
-func (c *rmqClient) RebalanceImmediately() {
- c.rbMutex.Lock()
- defer c.rbMutex.Unlock()
- c.consumerMap.Range(func(key, value interface{}) bool {
- consumer := value.(InnerConsumer)
- consumer.Rebalance()
- return true
- })
-}
-
-func (c *rmqClient) UpdatePublishInfo(topic string, data *TopicRouteData, changed bool) {
- if data == nil {
- return
- }
-
- c.producerMap.Range(func(key, value interface{}) bool {
- p := value.(InnerProducer)
- updated := changed
- if !updated {
- updated = p.IsPublishTopicNeedUpdate(topic)
- }
- if updated {
- publishInfo := c.GetNameSrv().(*namesrvs).routeData2PublishInfo(topic, data)
- publishInfo.HaveTopicRouterInfo = true
- p.UpdateTopicPublishInfo(topic, publishInfo)
- }
- return true
- })
-}
-
-func (c *rmqClient) updateSubscribeInfo(topic string, data *TopicRouteData, changed bool) {
- if data == nil {
- return
- }
- c.consumerMap.Range(func(key, value interface{}) bool {
- consumer := value.(InnerConsumer)
- updated := changed
- if !updated {
- updated = consumer.IsSubscribeTopicNeedUpdate(topic)
- }
- if updated {
- consumer.UpdateTopicSubscribeInfo(topic, routeData2SubscribeInfo(topic, data))
- }
-
- return true
- })
-}
-
-func (c *rmqClient) isNeedUpdateSubscribeInfo(topic string) bool {
- var result bool
- c.consumerMap.Range(func(key, value interface{}) bool {
- consumer := value.(InnerConsumer)
- if consumer.IsSubscribeTopicNeedUpdate(topic) {
- result = true
- return false
- }
- return true
- })
- return result
-}
-
-func (c *rmqClient) resetOffset(topic string, group string, offsetTable map[primitive.MessageQueue]int64) {
- consumer, exist := c.consumerMap.Load(group)
- if !exist {
- rlog.Warning("group "+group+" do not exists", nil)
- return
- }
- consumer.(InnerConsumer).ResetOffset(topic, offsetTable)
-}
-
-func (c *rmqClient) getConsumerRunningInfo(group string) *ConsumerRunningInfo {
- consumer, exist := c.consumerMap.Load(group)
- if !exist {
- return nil
- }
- info := consumer.(InnerConsumer).GetConsumerRunningInfo()
- if info != nil {
- info.Properties[PropClientVersion] = clientVersion
- }
- return info
-}
-
-func (c *rmqClient) consumeMessageDirectly(msg *primitive.MessageExt, group string, brokerName string) *ConsumeMessageDirectlyResult {
- consumer, exist := c.consumerMap.Load(group)
- if !exist {
- return nil
- }
- res := consumer.(InnerConsumer).ConsumeMessageDirectly(msg, brokerName)
- return res
-}
-
-func routeData2SubscribeInfo(topic string, data *TopicRouteData) []*primitive.MessageQueue {
- list := make([]*primitive.MessageQueue, 0)
- for idx := range data.QueueDataList {
- qd := data.QueueDataList[idx]
- if queueIsReadable(qd.Perm) {
- for i := 0; i < qd.ReadQueueNums; i++ {
- list = append(list, &primitive.MessageQueue{
- Topic: topic,
- BrokerName: qd.BrokerName,
- QueueId: i,
- })
- }
- }
- }
- return list
-}
-
-func brokerVIPChannel(brokerAddr string) string {
- if !_VIPChannelEnable {
- return brokerAddr
- }
- var brokerAddrNew strings.Builder
- ipAndPort := strings.Split(brokerAddr, ":")
- port, err := strconv.Atoi(ipAndPort[1])
- if err != nil {
- return ""
- }
- brokerAddrNew.WriteString(ipAndPort[0])
- brokerAddrNew.WriteString(":")
- brokerAddrNew.WriteString(strconv.Itoa(port - 2))
- return brokerAddrNew.String()
-}
diff --git a/internal/constants.go b/internal/constants.go
deleted file mode 100644
index e2e911f..0000000
--- a/internal/constants.go
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package internal
-
-const (
- RetryGroupTopicPrefix = "%RETRY%"
- DefaultConsumerGroup = "DEFAULT_CONSUMER"
- ClientInnerProducerGroup = "CLIENT_INNER_PRODUCER"
- SystemTopicPrefix = "rmq_sys_"
-)
-
-func GetRetryTopic(group string) string {
- return RetryGroupTopicPrefix + group
-}
diff --git a/internal/mock_client.go b/internal/mock_client.go
deleted file mode 100644
index c975038..0000000
--- a/internal/mock_client.go
+++ /dev/null
@@ -1,430 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Code generated by MockGen. DO NOT EDIT.
-// Source: client.go
-
-// Package internal is a generated GoMock package.
-package internal
-
-import (
- "context"
- "reflect"
- "time"
-
- "github.com/golang/mock/gomock"
-
- "github.com/apache/rocketmq-client-go/v2/internal/remote"
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-// MockInnerProducer is a mock of InnerProducer interface
-type MockInnerProducer struct {
- ctrl *gomock.Controller
- recorder *MockInnerProducerMockRecorder
-}
-
-// MockInnerProducerMockRecorder is the mock recorder for MockInnerProducer
-type MockInnerProducerMockRecorder struct {
- mock *MockInnerProducer
-}
-
-// NewMockInnerProducer creates a new mock instance
-func NewMockInnerProducer(ctrl *gomock.Controller) *MockInnerProducer {
- mock := &MockInnerProducer{ctrl: ctrl}
- mock.recorder = &MockInnerProducerMockRecorder{mock}
- return mock
-}
-
-// EXPECT returns an object that allows the caller to indicate expected use
-func (m *MockInnerProducer) EXPECT() *MockInnerProducerMockRecorder {
- return m.recorder
-}
-
-// PublishTopicList mocks base method
-func (m *MockInnerProducer) PublishTopicList() []string {
- ret := m.ctrl.Call(m, "PublishTopicList")
- ret0, _ := ret[0].([]string)
- return ret0
-}
-
-// PublishTopicList indicates an expected call of PublishTopicList
-func (mr *MockInnerProducerMockRecorder) PublishTopicList() *gomock.Call {
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PublishTopicList", reflect.TypeOf((*MockInnerProducer)(nil).PublishTopicList))
-}
-
-// UpdateTopicPublishInfo mocks base method
-func (m *MockInnerProducer) UpdateTopicPublishInfo(topic string, info *TopicPublishInfo) {
- m.ctrl.Call(m, "UpdateTopicPublishInfo", topic, info)
-}
-
-// UpdateTopicPublishInfo indicates an expected call of UpdateTopicPublishInfo
-func (mr *MockInnerProducerMockRecorder) UpdateTopicPublishInfo(topic, info interface{}) *gomock.Call {
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateTopicPublishInfo", reflect.TypeOf((*MockInnerProducer)(nil).UpdateTopicPublishInfo), topic, info)
-}
-
-// IsPublishTopicNeedUpdate mocks base method
-func (m *MockInnerProducer) IsPublishTopicNeedUpdate(topic string) bool {
- ret := m.ctrl.Call(m, "IsPublishTopicNeedUpdate", topic)
- ret0, _ := ret[0].(bool)
- return ret0
-}
-
-// IsPublishTopicNeedUpdate indicates an expected call of IsPublishTopicNeedUpdate
-func (mr *MockInnerProducerMockRecorder) IsPublishTopicNeedUpdate(topic interface{}) *gomock.Call {
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsPublishTopicNeedUpdate", reflect.TypeOf((*MockInnerProducer)(nil).IsPublishTopicNeedUpdate), topic)
-}
-
-// IsUnitMode mocks base method
-func (m *MockInnerProducer) IsUnitMode() bool {
- ret := m.ctrl.Call(m, "IsUnitMode")
- ret0, _ := ret[0].(bool)
- return ret0
-}
-
-// IsUnitMode indicates an expected call of IsUnitMode
-func (mr *MockInnerProducerMockRecorder) IsUnitMode() *gomock.Call {
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsUnitMode", reflect.TypeOf((*MockInnerProducer)(nil).IsUnitMode))
-}
-
-// MockInnerConsumer is a mock of InnerConsumer interface
-type MockInnerConsumer struct {
- ctrl *gomock.Controller
- recorder *MockInnerConsumerMockRecorder
-}
-
-// MockInnerConsumerMockRecorder is the mock recorder for MockInnerConsumer
-type MockInnerConsumerMockRecorder struct {
- mock *MockInnerConsumer
-}
-
-// NewMockInnerConsumer creates a new mock instance
-func NewMockInnerConsumer(ctrl *gomock.Controller) *MockInnerConsumer {
- mock := &MockInnerConsumer{ctrl: ctrl}
- mock.recorder = &MockInnerConsumerMockRecorder{mock}
- return mock
-}
-
-// EXPECT returns an object that allows the caller to indicate expected use
-func (m *MockInnerConsumer) EXPECT() *MockInnerConsumerMockRecorder {
- return m.recorder
-}
-
-// PersistConsumerOffset mocks base method
-func (m *MockInnerConsumer) PersistConsumerOffset() error {
- ret := m.ctrl.Call(m, "PersistConsumerOffset")
- ret0, _ := ret[0].(error)
- return ret0
-}
-
-// PersistConsumerOffset indicates an expected call of PersistConsumerOffset
-func (mr *MockInnerConsumerMockRecorder) PersistConsumerOffset() *gomock.Call {
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PersistConsumerOffset", reflect.TypeOf((*MockInnerConsumer)(nil).PersistConsumerOffset))
-}
-
-// UpdateTopicSubscribeInfo mocks base method
-func (m *MockInnerConsumer) UpdateTopicSubscribeInfo(topic string, mqs []*primitive.MessageQueue) {
- m.ctrl.Call(m, "UpdateTopicSubscribeInfo", topic, mqs)
-}
-
-// UpdateTopicSubscribeInfo indicates an expected call of UpdateTopicSubscribeInfo
-func (mr *MockInnerConsumerMockRecorder) UpdateTopicSubscribeInfo(topic, mqs interface{}) *gomock.Call {
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateTopicSubscribeInfo", reflect.TypeOf((*MockInnerConsumer)(nil).UpdateTopicSubscribeInfo), topic, mqs)
-}
-
-// IsSubscribeTopicNeedUpdate mocks base method
-func (m *MockInnerConsumer) IsSubscribeTopicNeedUpdate(topic string) bool {
- ret := m.ctrl.Call(m, "IsSubscribeTopicNeedUpdate", topic)
- ret0, _ := ret[0].(bool)
- return ret0
-}
-
-// IsSubscribeTopicNeedUpdate indicates an expected call of IsSubscribeTopicNeedUpdate
-func (mr *MockInnerConsumerMockRecorder) IsSubscribeTopicNeedUpdate(topic interface{}) *gomock.Call {
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsSubscribeTopicNeedUpdate", reflect.TypeOf((*MockInnerConsumer)(nil).IsSubscribeTopicNeedUpdate), topic)
-}
-
-// SubscriptionDataList mocks base method
-func (m *MockInnerConsumer) SubscriptionDataList() []*SubscriptionData {
- ret := m.ctrl.Call(m, "SubscriptionDataList")
- ret0, _ := ret[0].([]*SubscriptionData)
- return ret0
-}
-
-// SubscriptionDataList indicates an expected call of SubscriptionDataList
-func (mr *MockInnerConsumerMockRecorder) SubscriptionDataList() *gomock.Call {
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SubscriptionDataList", reflect.TypeOf((*MockInnerConsumer)(nil).SubscriptionDataList))
-}
-
-// Rebalance mocks base method
-func (m *MockInnerConsumer) Rebalance() {
- m.ctrl.Call(m, "Rebalance")
-}
-
-// Rebalance indicates an expected call of Rebalance
-func (mr *MockInnerConsumerMockRecorder) Rebalance() *gomock.Call {
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Rebalance", reflect.TypeOf((*MockInnerConsumer)(nil).Rebalance))
-}
-
-// IsUnitMode mocks base method
-func (m *MockInnerConsumer) IsUnitMode() bool {
- ret := m.ctrl.Call(m, "IsUnitMode")
- ret0, _ := ret[0].(bool)
- return ret0
-}
-
-// IsUnitMode indicates an expected call of IsUnitMode
-func (mr *MockInnerConsumerMockRecorder) IsUnitMode() *gomock.Call {
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsUnitMode", reflect.TypeOf((*MockInnerConsumer)(nil).IsUnitMode))
-}
-
-// GetConsumerRunningInfo mocks base method
-func (m *MockInnerConsumer) GetConsumerRunningInfo() *ConsumerRunningInfo {
- ret := m.ctrl.Call(m, "GetConsumerRunningInfo")
- ret0, _ := ret[0].(*ConsumerRunningInfo)
- return ret0
-}
-
-// GetConsumerRunningInfo indicates an expected call of GetConsumerRunningInfo
-func (mr *MockInnerConsumerMockRecorder) GetConsumerRunningInfo() *gomock.Call {
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetConsumerRunningInfo", reflect.TypeOf((*MockInnerConsumer)(nil).GetConsumerRunningInfo))
-}
-
-// MockRMQClient is a mock of RMQClient interface
-type MockRMQClient struct {
- ctrl *gomock.Controller
- recorder *MockRMQClientMockRecorder
- Namesrv *MockNamesrvs
-}
-
-func (m *MockRMQClient) GetNameSrv() Namesrvs {
- return m.Namesrv
-}
-
-func (m *MockRMQClient) SetNameSrv(mockNamesrvs *MockNamesrvs) {
- m.Namesrv = mockNamesrvs
-}
-
-// MockRMQClientMockRecorder is the mock recorder for MockRMQClient
-type MockRMQClientMockRecorder struct {
- mock *MockRMQClient
-}
-
-// NewMockRMQClient creates a new mock instance
-func NewMockRMQClient(ctrl *gomock.Controller) *MockRMQClient {
- mock := &MockRMQClient{ctrl: ctrl}
- mock.recorder = &MockRMQClientMockRecorder{mock}
- return mock
-}
-
-// EXPECT returns an object that allows the caller to indicate expected use
-func (m *MockRMQClient) EXPECT() *MockRMQClientMockRecorder {
- return m.recorder
-}
-
-// Start mocks base method
-func (m *MockRMQClient) Start() {
- m.ctrl.Call(m, "Start")
-}
-
-// Start indicates an expected call of Start
-func (mr *MockRMQClientMockRecorder) Start() *gomock.Call {
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Start", reflect.TypeOf((*MockRMQClient)(nil).Start))
-}
-
-// Shutdown mocks base method
-func (m *MockRMQClient) Shutdown() {
- m.ctrl.Call(m, "Shutdown")
-}
-
-// Shutdown indicates an expected call of Shutdown
-func (mr *MockRMQClientMockRecorder) Shutdown() *gomock.Call {
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Shutdown", reflect.TypeOf((*MockRMQClient)(nil).Shutdown))
-}
-
-// ClientID mocks base method
-func (m *MockRMQClient) ClientID() string {
- ret := m.ctrl.Call(m, "ClientID")
- ret0, _ := ret[0].(string)
- return ret0
-}
-
-// ClientID indicates an expected call of ClientID
-func (mr *MockRMQClientMockRecorder) ClientID() *gomock.Call {
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClientID", reflect.TypeOf((*MockRMQClient)(nil).ClientID))
-}
-
-// RegisterProducer mocks base method
-func (m *MockRMQClient) RegisterProducer(group string, producer InnerProducer) error {
- ret := m.ctrl.Call(m, "RegisterProducer", group, producer)
- ret0, _ := ret[0].(error)
- return ret0
-}
-
-// RegisterProducer indicates an expected call of RegisterProducer
-func (mr *MockRMQClientMockRecorder) RegisterProducer(group, producer interface{}) *gomock.Call {
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RegisterProducer", reflect.TypeOf((*MockRMQClient)(nil).RegisterProducer), group, producer)
-}
-
-// UnregisterProducer mocks base method
-func (m *MockRMQClient) UnregisterProducer(group string) {
- m.ctrl.Call(m, "UnregisterProducer", group)
-}
-
-// UnregisterProducer indicates an expected call of UnregisterProducer
-func (mr *MockRMQClientMockRecorder) UnregisterProducer(group interface{}) *gomock.Call {
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UnregisterProducer", reflect.TypeOf((*MockRMQClient)(nil).UnregisterProducer), group)
-}
-
-// InvokeSync mocks base method
-func (m *MockRMQClient) InvokeSync(ctx context.Context, addr string, request *remote.RemotingCommand, timeoutMillis time.Duration) (*remote.RemotingCommand, error) {
- ret := m.ctrl.Call(m, "InvokeSync", ctx, addr, request, timeoutMillis)
- ret0, _ := ret[0].(*remote.RemotingCommand)
- ret1, _ := ret[1].(error)
- return ret0, ret1
-}
-
-// InvokeSync indicates an expected call of InvokeSync
-func (mr *MockRMQClientMockRecorder) InvokeSync(ctx, addr, request, timeoutMillis interface{}) *gomock.Call {
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InvokeSync", reflect.TypeOf((*MockRMQClient)(nil).InvokeSync), ctx, addr, request, timeoutMillis)
-}
-
-// InvokeAsync mocks base method
-func (m *MockRMQClient) InvokeAsync(ctx context.Context, addr string, request *remote.RemotingCommand, f func(*remote.RemotingCommand, error)) error {
- ret := m.ctrl.Call(m, "InvokeAsync", ctx, addr, request, f)
- ret0, _ := ret[0].(error)
- return ret0
-}
-
-// InvokeAsync indicates an expected call of InvokeAsync
-func (mr *MockRMQClientMockRecorder) InvokeAsync(ctx, addr, request, f interface{}) *gomock.Call {
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InvokeAsync", reflect.TypeOf((*MockRMQClient)(nil).InvokeAsync), ctx, addr, request, f)
-}
-
-// InvokeOneWay mocks base method
-func (m *MockRMQClient) InvokeOneWay(ctx context.Context, addr string, request *remote.RemotingCommand, timeoutMillis time.Duration) error {
- ret := m.ctrl.Call(m, "InvokeOneWay", ctx, addr, request, timeoutMillis)
- ret0, _ := ret[0].(error)
- return ret0
-}
-
-// InvokeOneWay indicates an expected call of InvokeOneWay
-func (mr *MockRMQClientMockRecorder) InvokeOneWay(ctx, addr, request, timeoutMillis interface{}) *gomock.Call {
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InvokeOneWay", reflect.TypeOf((*MockRMQClient)(nil).InvokeOneWay), ctx, addr, request, timeoutMillis)
-}
-
-// CheckClientInBroker mocks base method
-func (m *MockRMQClient) CheckClientInBroker() {
- m.ctrl.Call(m, "CheckClientInBroker")
-}
-
-// CheckClientInBroker indicates an expected call of CheckClientInBroker
-func (mr *MockRMQClientMockRecorder) CheckClientInBroker() *gomock.Call {
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CheckClientInBroker", reflect.TypeOf((*MockRMQClient)(nil).CheckClientInBroker))
-}
-
-// SendHeartbeatToAllBrokerWithLock mocks base method
-func (m *MockRMQClient) SendHeartbeatToAllBrokerWithLock() {
- m.ctrl.Call(m, "SendHeartbeatToAllBrokerWithLock")
-}
-
-// SendHeartbeatToAllBrokerWithLock indicates an expected call of SendHeartbeatToAllBrokerWithLock
-func (mr *MockRMQClientMockRecorder) SendHeartbeatToAllBrokerWithLock() *gomock.Call {
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendHeartbeatToAllBrokerWithLock", reflect.TypeOf((*MockRMQClient)(nil).SendHeartbeatToAllBrokerWithLock))
-}
-
-// UpdateTopicRouteInfo mocks base method
-func (m *MockRMQClient) UpdateTopicRouteInfo() {
- m.ctrl.Call(m, "UpdateTopicRouteInfo")
-}
-
-// UpdateTopicRouteInfo indicates an expected call of UpdateTopicRouteInfo
-func (mr *MockRMQClientMockRecorder) UpdateTopicRouteInfo() *gomock.Call {
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateTopicRouteInfo", reflect.TypeOf((*MockRMQClient)(nil).UpdateTopicRouteInfo))
-}
-
-// ProcessSendResponse mocks base method
-func (m *MockRMQClient) ProcessSendResponse(brokerName string, cmd *remote.RemotingCommand, resp *primitive.SendResult, msgs ...*primitive.Message) error {
- varargs := []interface{}{brokerName, cmd, resp}
- for _, a := range msgs {
- varargs = append(varargs, a)
- }
- ret := m.ctrl.Call(m, "ProcessSendResponse", varargs...)
- ret0, _ := ret[0].(error)
- return ret0
-}
-
-// ProcessSendResponse indicates an expected call of ProcessSendResponse
-func (mr *MockRMQClientMockRecorder) ProcessSendResponse(brokerName, cmd, resp interface{}, msgs ...interface{}) *gomock.Call {
- varargs := append([]interface{}{brokerName, cmd, resp}, msgs...)
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ProcessSendResponse", reflect.TypeOf((*MockRMQClient)(nil).ProcessSendResponse), varargs...)
-}
-
-// RegisterConsumer mocks base method
-func (m *MockRMQClient) RegisterConsumer(group string, consumer InnerConsumer) error {
- ret := m.ctrl.Call(m, "RegisterConsumer", group, consumer)
- ret0, _ := ret[0].(error)
- return ret0
-}
-
-// RegisterConsumer indicates an expected call of RegisterConsumer
-func (mr *MockRMQClientMockRecorder) RegisterConsumer(group, consumer interface{}) *gomock.Call {
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RegisterConsumer", reflect.TypeOf((*MockRMQClient)(nil).RegisterConsumer), group, consumer)
-}
-
-// UnregisterConsumer mocks base method
-func (m *MockRMQClient) UnregisterConsumer(group string) {
- m.ctrl.Call(m, "UnregisterConsumer", group)
-}
-
-// UnregisterConsumer indicates an expected call of UnregisterConsumer
-func (mr *MockRMQClientMockRecorder) UnregisterConsumer(group interface{}) *gomock.Call {
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UnregisterConsumer", reflect.TypeOf((*MockRMQClient)(nil).UnregisterConsumer), group)
-}
-
-// PullMessage mocks base method
-func (m *MockRMQClient) PullMessage(ctx context.Context, brokerAddrs string, request *PullMessageRequestHeader) (*primitive.PullResult, error) {
- ret := m.ctrl.Call(m, "PullMessage", ctx, brokerAddrs, request)
- ret0, _ := ret[0].(*primitive.PullResult)
- ret1, _ := ret[1].(error)
- return ret0, ret1
-}
-
-// PullMessage indicates an expected call of PullMessage
-func (mr *MockRMQClientMockRecorder) PullMessage(ctx, brokerAddrs, request interface{}) *gomock.Call {
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PullMessage", reflect.TypeOf((*MockRMQClient)(nil).PullMessage), ctx, brokerAddrs, request)
-}
-
-// RebalanceImmediately mocks base method
-func (m *MockRMQClient) RebalanceImmediately() {
- m.ctrl.Call(m, "RebalanceImmediately")
-}
-
-// RebalanceImmediately indicates an expected call of RebalanceImmediately
-func (mr *MockRMQClientMockRecorder) RebalanceImmediately() *gomock.Call {
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RebalanceImmediately", reflect.TypeOf((*MockRMQClient)(nil).RebalanceImmediately))
-}
-
-// UpdatePublishInfo mocks base method
-func (m *MockRMQClient) UpdatePublishInfo(topic string, data *TopicRouteData, changed bool) {
- m.ctrl.Call(m, "UpdatePublishInfo", topic, data, changed)
-}
-
-// UpdatePublishInfo indicates an expected call of UpdatePublishInfo
-func (mr *MockRMQClientMockRecorder) UpdatePublishInfo(topic, data, changed interface{}) *gomock.Call {
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdatePublishInfo", reflect.TypeOf((*MockRMQClient)(nil).UpdatePublishInfo), topic, data, changed)
-}
diff --git a/internal/mock_namesrv.go b/internal/mock_namesrv.go
deleted file mode 100644
index 7ce6f97..0000000
--- a/internal/mock_namesrv.go
+++ /dev/null
@@ -1,193 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-// Code generated by MockGen. DO NOT EDIT.
-// Source: namesrv.go
-
-// Package internal is a generated GoMock package.
-package internal
-
-import (
- reflect "reflect"
-
- primitive "github.com/apache/rocketmq-client-go/v2/primitive"
- gomock "github.com/golang/mock/gomock"
-)
-
-// MockNamesrvs is a mock of Namesrvs interface
-type MockNamesrvs struct {
- ctrl *gomock.Controller
- recorder *MockNamesrvsMockRecorder
-}
-
-func (m *MockNamesrvs) UpdateTopicRouteInfoWithDefault(topic string, defaultTopic string, defaultQueueNum int) (*TopicRouteData, bool, error) {
- return m.UpdateTopicRouteInfo(topic)
-}
-
-// MockNamesrvsMockRecorder is the mock recorder for MockNamesrvs
-type MockNamesrvsMockRecorder struct {
- mock *MockNamesrvs
-}
-
-// NewMockNamesrvs creates a new mock instance
-func NewMockNamesrvs(ctrl *gomock.Controller) *MockNamesrvs {
- mock := &MockNamesrvs{ctrl: ctrl}
- mock.recorder = &MockNamesrvsMockRecorder{mock}
- return mock
-}
-
-// EXPECT returns an object that allows the caller to indicate expected use
-func (m *MockNamesrvs) EXPECT() *MockNamesrvsMockRecorder {
- return m.recorder
-}
-
-// UpdateNameServerAddress mocks base method
-func (m *MockNamesrvs) UpdateNameServerAddress() {
- m.ctrl.T.Helper()
- m.ctrl.Call(m, "UpdateNameServerAddress")
-}
-
-// UpdateNameServerAddress indicates an expected call of UpdateNameServerAddress
-func (mr *MockNamesrvsMockRecorder) UpdateNameServerAddress() *gomock.Call {
- mr.mock.ctrl.T.Helper()
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateNameServerAddress", reflect.TypeOf((*MockNamesrvs)(nil).UpdateNameServerAddress))
-}
-
-// AddBroker mocks base method
-func (m *MockNamesrvs) AddBroker(routeData *TopicRouteData) {
- m.ctrl.T.Helper()
- m.ctrl.Call(m, "AddBroker", routeData)
-}
-
-// AddBroker indicates an expected call of AddBroker
-func (mr *MockNamesrvsMockRecorder) AddBroker(routeData interface{}) *gomock.Call {
- mr.mock.ctrl.T.Helper()
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddBroker", reflect.TypeOf((*MockNamesrvs)(nil).AddBroker), routeData)
-}
-
-// cleanOfflineBroker mocks base method
-func (m *MockNamesrvs) cleanOfflineBroker() {
- m.ctrl.T.Helper()
- m.ctrl.Call(m, "cleanOfflineBroker")
-}
-
-// cleanOfflineBroker indicates an expected call of cleanOfflineBroker
-func (mr *MockNamesrvsMockRecorder) cleanOfflineBroker() *gomock.Call {
- mr.mock.ctrl.T.Helper()
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "cleanOfflineBroker", reflect.TypeOf((*MockNamesrvs)(nil).cleanOfflineBroker))
-}
-
-// UpdateTopicRouteInfo mocks base method
-func (m *MockNamesrvs) UpdateTopicRouteInfo(topic string) (*TopicRouteData, bool, error) {
- m.ctrl.T.Helper()
- ret := m.ctrl.Call(m, "UpdateTopicRouteInfo", topic)
- ret0, _ := ret[0].(*TopicRouteData)
- ret1, _ := ret[1].(bool)
- ret2, _ := ret[2].(error)
- return ret0, ret1, ret2
-}
-
-// UpdateTopicRouteInfo indicates an expected call of UpdateTopicRouteInfo
-func (mr *MockNamesrvsMockRecorder) UpdateTopicRouteInfo(topic interface{}) *gomock.Call {
- mr.mock.ctrl.T.Helper()
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateTopicRouteInfo", reflect.TypeOf((*MockNamesrvs)(nil).UpdateTopicRouteInfo), topic)
-}
-
-// FetchPublishMessageQueues mocks base method
-func (m *MockNamesrvs) FetchPublishMessageQueues(topic string) ([]*primitive.MessageQueue, error) {
- m.ctrl.T.Helper()
- ret := m.ctrl.Call(m, "FetchPublishMessageQueues", topic)
- ret0, _ := ret[0].([]*primitive.MessageQueue)
- ret1, _ := ret[1].(error)
- return ret0, ret1
-}
-
-// FetchPublishMessageQueues indicates an expected call of FetchPublishMessageQueues
-func (mr *MockNamesrvsMockRecorder) FetchPublishMessageQueues(topic interface{}) *gomock.Call {
- mr.mock.ctrl.T.Helper()
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FetchPublishMessageQueues", reflect.TypeOf((*MockNamesrvs)(nil).FetchPublishMessageQueues), topic)
-}
-
-// FindBrokerAddrByTopic mocks base method
-func (m *MockNamesrvs) FindBrokerAddrByTopic(topic string) string {
- m.ctrl.T.Helper()
- ret := m.ctrl.Call(m, "FindBrokerAddrByTopic", topic)
- ret0, _ := ret[0].(string)
- return ret0
-}
-
-// FindBrokerAddrByTopic indicates an expected call of FindBrokerAddrByTopic
-func (mr *MockNamesrvsMockRecorder) FindBrokerAddrByTopic(topic interface{}) *gomock.Call {
- mr.mock.ctrl.T.Helper()
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FindBrokerAddrByTopic", reflect.TypeOf((*MockNamesrvs)(nil).FindBrokerAddrByTopic), topic)
-}
-
-// FindBrokerAddrByName mocks base method
-func (m *MockNamesrvs) FindBrokerAddrByName(brokerName string) string {
- m.ctrl.T.Helper()
- ret := m.ctrl.Call(m, "FindBrokerAddrByName", brokerName)
- ret0, _ := ret[0].(string)
- return ret0
-}
-
-// FindBrokerAddrByName indicates an expected call of FindBrokerAddrByName
-func (mr *MockNamesrvsMockRecorder) FindBrokerAddrByName(brokerName interface{}) *gomock.Call {
- mr.mock.ctrl.T.Helper()
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FindBrokerAddrByName", reflect.TypeOf((*MockNamesrvs)(nil).FindBrokerAddrByName), brokerName)
-}
-
-// FindBrokerAddressInSubscribe mocks base method
-func (m *MockNamesrvs) FindBrokerAddressInSubscribe(brokerName string, brokerId int64, onlyThisBroker bool) *FindBrokerResult {
- m.ctrl.T.Helper()
- ret := m.ctrl.Call(m, "FindBrokerAddressInSubscribe", brokerName, brokerId, onlyThisBroker)
- ret0, _ := ret[0].(*FindBrokerResult)
- return ret0
-}
-
-// FindBrokerAddressInSubscribe indicates an expected call of FindBrokerAddressInSubscribe
-func (mr *MockNamesrvsMockRecorder) FindBrokerAddressInSubscribe(brokerName, brokerId, onlyThisBroker interface{}) *gomock.Call {
- mr.mock.ctrl.T.Helper()
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FindBrokerAddressInSubscribe", reflect.TypeOf((*MockNamesrvs)(nil).FindBrokerAddressInSubscribe), brokerName, brokerId, onlyThisBroker)
-}
-
-// FetchSubscribeMessageQueues mocks base method
-func (m *MockNamesrvs) FetchSubscribeMessageQueues(topic string) ([]*primitive.MessageQueue, error) {
- m.ctrl.T.Helper()
- ret := m.ctrl.Call(m, "FetchSubscribeMessageQueues", topic)
- ret0, _ := ret[0].([]*primitive.MessageQueue)
- ret1, _ := ret[1].(error)
- return ret0, ret1
-}
-
-// FetchSubscribeMessageQueues indicates an expected call of FetchSubscribeMessageQueues
-func (mr *MockNamesrvsMockRecorder) FetchSubscribeMessageQueues(topic interface{}) *gomock.Call {
- mr.mock.ctrl.T.Helper()
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FetchSubscribeMessageQueues", reflect.TypeOf((*MockNamesrvs)(nil).FetchSubscribeMessageQueues), topic)
-}
-
-// AddrList mocks base method
-func (m *MockNamesrvs) AddrList() []string {
- m.ctrl.T.Helper()
- ret := m.ctrl.Call(m, "AddrList")
- ret0, _ := ret[0].([]string)
- return ret0
-}
-
-// AddrList indicates an expected call of AddrList
-func (mr *MockNamesrvsMockRecorder) AddrList() *gomock.Call {
- mr.mock.ctrl.T.Helper()
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddrList", reflect.TypeOf((*MockNamesrvs)(nil).AddrList))
-}
diff --git a/internal/model.go b/internal/model.go
deleted file mode 100644
index 7a011d7..0000000
--- a/internal/model.go
+++ /dev/null
@@ -1,406 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package internal
-
-import (
- "bytes"
- "encoding/json"
- "fmt"
- "github.com/tidwall/gjson"
- "sort"
- "strconv"
- "strings"
-
- "github.com/apache/rocketmq-client-go/v2/internal/utils"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/rlog"
- jsoniter "github.com/json-iterator/go"
-)
-
-type FindBrokerResult struct {
- BrokerAddr string
- Slave bool
- BrokerVersion int32
-}
-
-type (
- // groupName of consumer
- consumeType string
-
- ServiceState int32
-)
-
-const (
- StateCreateJust ServiceState = iota
- StateStartFailed
- StateRunning
- StateShutdown
-)
-
-type SubscriptionData struct {
- ClassFilterMode bool `json:"classFilterMode"`
- Topic string `json:"topic"`
- SubString string `json:"subString"`
- Tags utils.Set `json:"tagsSet"`
- Codes utils.Set `json:"codeSet"`
- SubVersion int64 `json:"subVersion"`
- ExpType string `json:"expressionType"`
-}
-
-type producerData struct {
- GroupName string `json:"groupName"`
-}
-
-func (p producerData) UniqueID() string {
- return p.GroupName
-}
-
-type consumerData struct {
- GroupName string `json:"groupName"`
- CType consumeType `json:"consumeType"`
- MessageModel string `json:"messageModel"`
- Where string `json:"consumeFromWhere"`
- SubscriptionDatas []*SubscriptionData `json:"subscriptionDataSet"`
- UnitMode bool `json:"unitMode"`
-}
-
-func (c consumerData) UniqueID() string {
- return c.GroupName
-}
-
-type heartbeatData struct {
- ClientId string `json:"clientID"`
- ProducerDatas utils.Set `json:"producerDataSet"`
- ConsumerDatas utils.Set `json:"consumerDataSet"`
-}
-
-func NewHeartbeatData(clientID string) *heartbeatData {
- return &heartbeatData{
- ClientId: clientID,
- ProducerDatas: utils.NewSet(),
- ConsumerDatas: utils.NewSet(),
- }
-}
-
-func (data *heartbeatData) encode() []byte {
- d, err := jsoniter.Marshal(data)
- if err != nil {
- rlog.Error("marshal heartbeatData error", map[string]interface{}{
- rlog.LogKeyUnderlayError: err,
- })
- return nil
- }
- rlog.Debug("heartbeat: "+string(d), nil)
- return d
-}
-
-const (
- PropNameServerAddr = "PROP_NAMESERVER_ADDR"
- PropThreadPoolCoreSize = "PROP_THREADPOOL_CORE_SIZE"
- PropConsumeOrderly = "PROP_CONSUMEORDERLY"
- PropConsumeType = "PROP_CONSUME_TYPE"
- PropClientVersion = "PROP_CLIENT_VERSION"
- PropConsumerStartTimestamp = "PROP_CONSUMER_START_TIMESTAMP"
-)
-
-type ProcessQueueInfo struct {
- CommitOffset int64 `json:"commitOffset"`
- CachedMsgMinOffset int64 `json:"cachedMsgMinOffset"`
- CachedMsgMaxOffset int64 `json:"cachedMsgMaxOffset"`
- CachedMsgCount int `json:"cachedMsgCount"`
- CachedMsgSizeInMiB int64 `json:"cachedMsgSizeInMiB"`
- TransactionMsgMinOffset int64 `json:"transactionMsgMinOffset"`
- TransactionMsgMaxOffset int64 `json:"transactionMsgMaxOffset"`
- TransactionMsgCount int `json:"transactionMsgCount"`
- Locked bool `json:"locked"`
- TryUnlockTimes int64 `json:"tryUnlockTimes"`
- LastLockTimestamp int64 `json:"lastLockTimestamp"`
- Dropped bool `json:"dropped"`
- LastPullTimestamp int64 `json:"lastPullTimestamp"`
- LastConsumeTimestamp int64 `json:"lastConsumeTimestamp"`
-}
-
-type ConsumeStatus struct {
- PullRT float64 `json:"pullRT"`
- PullTPS float64 `json:"pullTPS"`
- ConsumeRT float64 `json:"consumeRT"`
- ConsumeOKTPS float64 `json:"consumeOKTPS"`
- ConsumeFailedTPS float64 `json:"consumeFailedTPS"`
- ConsumeFailedMsgs int64 `json:"consumeFailedMsgs"`
-}
-
-type ConsumerRunningInfo struct {
- Properties map[string]string
- SubscriptionData map[*SubscriptionData]bool
- MQTable map[primitive.MessageQueue]ProcessQueueInfo
- StatusTable map[string]ConsumeStatus
-}
-
-func (info ConsumerRunningInfo) Encode() ([]byte, error) {
- data, err := json.Marshal(info.Properties)
- if err != nil {
- return nil, err
- }
- jsonData := fmt.Sprintf("{\"%s\":%s", "properties", string(data))
-
- data, err = json.Marshal(info.StatusTable)
- if err != nil {
- return nil, err
- }
- jsonData = fmt.Sprintf("%s,\"%s\":%s", jsonData, "statusTable", string(data))
-
- subs := make([]*SubscriptionData, len(info.SubscriptionData))
- idx := 0
- for k := range info.SubscriptionData {
- subs[idx] = k
- idx++
- }
-
- // make sure test case table
- sort.Slice(subs, func(i, j int) bool {
- sub1 := subs[i]
- sub2 := subs[j]
- if sub1.ClassFilterMode != sub2.ClassFilterMode {
- return sub1.ClassFilterMode == false
- }
- com := strings.Compare(sub1.Topic, sub1.Topic)
- if com != 0 {
- return com > 0
- }
-
- com = strings.Compare(sub1.SubString, sub1.SubString)
- if com != 0 {
- return com > 0
- }
-
- if sub1.SubVersion != sub2.SubVersion {
- return sub1.SubVersion > sub2.SubVersion
- }
-
- com = strings.Compare(sub1.ExpType, sub1.ExpType)
- if com != 0 {
- return com > 0
- }
-
- v1, _ := sub1.Tags.MarshalJSON()
- v2, _ := sub2.Tags.MarshalJSON()
- com = bytes.Compare(v1, v2)
- if com != 0 {
- return com > 0
- }
-
- v1, _ = sub1.Codes.MarshalJSON()
- v2, _ = sub2.Codes.MarshalJSON()
- com = bytes.Compare(v1, v2)
- if com != 0 {
- return com > 0
- }
- return true
- })
-
- data, err = json.Marshal(subs)
- if err != nil {
- return nil, err
- }
- jsonData = fmt.Sprintf("%s,\"%s\":%s", jsonData, "subscriptionSet", string(data))
-
- tableJson := ""
- keys := make([]primitive.MessageQueue, 0)
-
- for k := range info.MQTable {
- keys = append(keys, k)
- }
-
- sort.Slice(keys, func(i, j int) bool {
- q1 := keys[i]
- q2 := keys[j]
- com := strings.Compare(q1.Topic, q2.Topic)
- if com != 0 {
- return com < 0
- }
-
- com = strings.Compare(q1.BrokerName, q2.BrokerName)
- if com != 0 {
- return com < 0
- }
-
- return q1.QueueId < q2.QueueId
- })
-
- for idx := range keys {
- dataK, err := json.Marshal(keys[idx])
- if err != nil {
- return nil, err
- }
- dataV, err := json.Marshal(info.MQTable[keys[idx]])
- tableJson = fmt.Sprintf("%s,%s:%s", tableJson, string(dataK), string(dataV))
- }
- tableJson = strings.TrimLeft(tableJson, ",")
- jsonData = fmt.Sprintf("%s,\"%s\":%s}", jsonData, "mqTable", fmt.Sprintf("{%s}", tableJson))
- return []byte(jsonData), nil
-}
-
-func NewConsumerRunningInfo() *ConsumerRunningInfo {
- return &ConsumerRunningInfo{
- Properties: make(map[string]string),
- SubscriptionData: make(map[*SubscriptionData]bool),
- MQTable: make(map[primitive.MessageQueue]ProcessQueueInfo),
- StatusTable: make(map[string]ConsumeStatus),
- }
-}
-
-type ConsumeMessageDirectlyResult struct {
- Order bool `json:"order"`
- AutoCommit bool `json:"autoCommit"`
- ConsumeResult ConsumeResult `json:"consumeResult"`
- Remark string `json:"remark"`
- SpentTimeMills int64 `json:"spentTimeMills"`
-}
-
-type ConsumeResult int
-
-const (
- ConsumeSuccess ConsumeResult = iota
- ConsumeRetryLater
- Rollback
- Commit
- ThrowException
- ReturnNull
-)
-
-func (result ConsumeMessageDirectlyResult) Encode() ([]byte, error) {
- data, err := json.Marshal(result)
- if err != nil {
- return nil, err
- }
- return data, nil
-}
-
-type ResetOffsetBody struct {
- OffsetTable map[primitive.MessageQueue]int64 `json:"offsetTable"`
-}
-
-// Decode note: the origin implementation for parse json is in gson format.
-// this func should support both gson and fastjson schema.
-func (resetOffsetBody *ResetOffsetBody) Decode(body []byte) {
- validJSON := gjson.ValidBytes(body)
-
- var offsetTable map[primitive.MessageQueue]int64
-
- if validJSON {
- offsetTable = parseGsonFormat(body)
- } else {
- offsetTable = parseFastJsonFormat(body)
- }
-
- resetOffsetBody.OffsetTable = offsetTable
-}
-
-func parseGsonFormat(body []byte) map[primitive.MessageQueue]int64 {
- result := gjson.ParseBytes(body)
-
- rlog.Debug("offset table string "+result.Get("offsetTable").String(), nil)
-
- offsetTable := make(map[primitive.MessageQueue]int64, 0)
-
- offsetStr := result.Get("offsetTable").String()
- if len(offsetStr) <= 2 {
- rlog.Warning("parse reset offset table json get nothing in body", map[string]interface{}{
- "origin json": offsetStr,
- })
- return offsetTable
- }
-
- offsetTableArray := strings.Split(offsetStr, "],[")
-
- for index, v := range offsetTableArray {
- kvArray := strings.Split(v, "},")
-
- var kstr, vstr string
- if index == len(offsetTableArray)-1 {
- vstr = kvArray[1][:len(kvArray[1])-2]
- } else {
- vstr = kvArray[1]
- }
- offset, err := strconv.ParseInt(vstr, 10, 64)
- if err != nil {
- rlog.Error("Unmarshal offset error", map[string]interface{}{
- rlog.LogKeyUnderlayError: err,
- })
- return nil
- }
-
- if index == 0 {
- kstr = kvArray[0][2:len(kvArray[0])] + "}"
- } else {
- kstr = kvArray[0] + "}"
- }
- kObj := new(primitive.MessageQueue)
- err = jsoniter.Unmarshal([]byte(kstr), &kObj)
- if err != nil {
- rlog.Error("Unmarshal message queue error", map[string]interface{}{
- rlog.LogKeyUnderlayError: err,
- })
- return nil
- }
- offsetTable[*kObj] = offset
- }
-
- return offsetTable
-}
-
-func parseFastJsonFormat(body []byte) map[primitive.MessageQueue]int64 {
- offsetTable := make(map[primitive.MessageQueue]int64)
-
- jsonStr := string(body)
- offsetStr := gjson.Get(jsonStr, "offsetTable").String()
-
- if len(offsetStr) <= 2 {
- rlog.Warning("parse reset offset table json get nothing in body", map[string]interface{}{
- "origin json": jsonStr,
- })
- return offsetTable
- }
-
- trimStr := offsetStr[2 : len(offsetStr)-1]
-
- split := strings.Split(trimStr, ",{")
-
- for _, v := range split {
- tuple := strings.Split(v, "}:")
-
- queueStr := "{" + tuple[0] + "}"
-
- var err error
- // ignore err for now
- offset, err := strconv.Atoi(tuple[1])
-
- var queue primitive.MessageQueue
- err = json.Unmarshal([]byte(queueStr), &queue)
-
- if err != nil {
- rlog.Error("parse reset offset table json get nothing in body", map[string]interface{}{
- "origin json": jsonStr,
- })
- }
-
- offsetTable[queue] = int64(offset)
- }
-
- return offsetTable
-}
diff --git a/internal/model_test.go b/internal/model_test.go
deleted file mode 100644
index a653fa4..0000000
--- a/internal/model_test.go
+++ /dev/null
@@ -1,484 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package internal
-
-import (
- "encoding/json"
- "github.com/apache/rocketmq-client-go/v2/rlog"
- "strings"
- "testing"
-
- . "github.com/smartystreets/goconvey/convey"
- "github.com/tidwall/gjson"
-
- "github.com/apache/rocketmq-client-go/v2/internal/utils"
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-func TestHeartbeatData(t *testing.T) {
- Convey("test heatbeat json", t, func() {
-
- Convey("producerData set marshal", func() {
- pData := &producerData{
- GroupName: "group name",
- }
- pData2 := &producerData{
- GroupName: "group name 2",
- }
- set := utils.NewSet()
- set.Add(pData)
- set.Add(pData2)
-
- v, err := json.Marshal(set)
- So(err, ShouldBeNil)
- rlog.Info("Json Producer", map[string]interface{}{
- "result": string(v),
- })
- })
-
- Convey("producer heatbeat", func() {
-
- hbt := NewHeartbeatData("producer client id")
- p1 := &producerData{
- GroupName: "group name",
- }
- p2 := &producerData{
- GroupName: "group name 2",
- }
-
- hbt.ProducerDatas.Add(p1)
- hbt.ProducerDatas.Add(p2)
-
- v, err := json.Marshal(hbt)
- So(err, ShouldBeNil)
- rlog.Info("Json Producer", map[string]interface{}{
- "result": string(v),
- })
- })
-
- Convey("consumer heartbeat", func() {
-
- hbt := NewHeartbeatData("consumer client id")
- c1 := consumerData{
- GroupName: "consumer data 1",
- }
- c2 := consumerData{
- GroupName: "consumer data 2",
- }
- hbt.ConsumerDatas.Add(c1)
- hbt.ConsumerDatas.Add(c2)
-
- v, err := json.Marshal(hbt)
- So(err, ShouldBeNil)
- rlog.Info("Json Consumer", map[string]interface{}{
- "result": string(v),
- })
- })
-
- Convey("producer & consumer heartbeat", func() {
-
- hbt := NewHeartbeatData("consumer client id")
-
- p1 := &producerData{
- GroupName: "group name",
- }
- p2 := &producerData{
- GroupName: "group name 2",
- }
-
- hbt.ProducerDatas.Add(p1)
- hbt.ProducerDatas.Add(p2)
-
- c1 := consumerData{
- GroupName: "consumer data 1",
- }
- c2 := consumerData{
- GroupName: "consumer data 2",
- }
- hbt.ConsumerDatas.Add(c1)
- hbt.ConsumerDatas.Add(c2)
-
- v, err := json.Marshal(hbt)
- So(err, ShouldBeNil)
- rlog.Info("Json Producer and Consumer", map[string]interface{}{
- "result": string(v),
- })
- })
- })
-
-}
-
-func TestConsumerRunningInfo_MarshalJSON(t *testing.T) {
- Convey("test ConsumerRunningInfo MarshalJson", t, func() {
- props := map[string]string{
- "maxReconsumeTimes": "-1",
- "unitMode": "false",
- "adjustThreadPoolNumsThreshold": "100000",
- "consumerGroup": "mq-client-go-test%GID_GO_TEST",
- "messageModel": "CLUSTERING",
- "suspendCurrentQueueTimeMillis": "1000",
- "pullThresholdSizeForTopic": "-1",
- "pullThresholdSizeForQueue": "100",
- "PROP_CLIENT_VERSION": "V4_5_1",
- "consumeConcurrentlyMaxSpan": "2000",
- "postSubscriptionWhenPull": "false",
- "consumeTimestamp": "20191127013617",
- "PROP_CONSUME_TYPE": "CONSUME_PASSIVELY",
- "consumeTimeout": "15",
- "consumeMessageBatchMaxSize": "1",
- "PROP_THREADPOOL_CORE_SIZE": "20",
- "pullInterval": "0",
- "pullThresholdForQueue": "1000",
- "pullThresholdForTopic": "-1",
- "consumeFromWhere": "CONSUME_FROM_FIRST_OFFSET",
- "PROP_NAMESERVER_ADDR": "mq-client-go-test.mq-internet-access.mq-internet.aliyuncs.com:80;",
- "pullBatchSize": "32",
- "consumeThreadMin": "20",
- "PROP_CONSUMER_START_TIMESTAMP": "1574791577504",
- "consumeThreadMax": "20",
- "subscription": "{}",
- "PROP_CONSUMEORDERLY": "false",
- }
- subData := map[*SubscriptionData]bool{
- &SubscriptionData{
- ClassFilterMode: false,
- Codes: utils.NewSet(),
- ExpType: "TAG",
- SubString: "*",
- SubVersion: 1574791579242,
- Tags: utils.NewSet(),
- Topic: "%RETRY%mq-client-go-test%GID_GO_TEST",
- }: true,
- &SubscriptionData{
- ClassFilterMode: true,
- Codes: utils.NewSet(),
- ExpType: "TAG",
- SubString: "*",
- SubVersion: 1574791577523,
- Tags: utils.NewSet(),
- Topic: "mq-client-go-test%go-test",
- }: true,
- }
- statusTable := map[string]ConsumeStatus{
- "%RETRY%mq-client-go-test%GID_GO_TEST": {
- PullRT: 11.11,
- PullTPS: 22.22,
- ConsumeRT: 33.33,
- ConsumeOKTPS: 44.44,
- ConsumeFailedTPS: 55.55,
- ConsumeFailedMsgs: 666,
- },
- "mq-client-go-test%go-test": {
- PullRT: 123,
- PullTPS: 123,
- ConsumeRT: 123,
- ConsumeOKTPS: 123,
- ConsumeFailedTPS: 123,
- ConsumeFailedMsgs: 1234,
- },
- }
- mqTable := map[primitive.MessageQueue]ProcessQueueInfo{
- {
- Topic: "%RETRY%mq-client-go-test%GID_GO_TEST",
- BrokerName: "qd7internet-01",
- QueueId: 0,
- }: {
- CommitOffset: 0,
- CachedMsgMinOffset: 0,
- CachedMsgMaxOffset: 0,
- CachedMsgCount: 0,
- CachedMsgSizeInMiB: 0,
- TransactionMsgMinOffset: 0,
- TransactionMsgMaxOffset: 0,
- TransactionMsgCount: 0,
- Locked: false,
- TryUnlockTimes: 0,
- LastLockTimestamp: 1574791579221,
- Dropped: false,
- LastPullTimestamp: 1574791579242,
- LastConsumeTimestamp: 1574791579221,
- },
- {
- Topic: "%RETRY%mq-client-go-test%GID_GO_TEST",
- BrokerName: "qd7internet-01",
- QueueId: 1,
- }: {
- CommitOffset: 1,
- CachedMsgMinOffset: 2,
- CachedMsgMaxOffset: 3,
- CachedMsgCount: 4,
- CachedMsgSizeInMiB: 5,
- TransactionMsgMinOffset: 6,
- TransactionMsgMaxOffset: 7,
- TransactionMsgCount: 8,
- Locked: true,
- TryUnlockTimes: 9,
- LastLockTimestamp: 1574791579221,
- Dropped: false,
- LastPullTimestamp: 1574791579242,
- LastConsumeTimestamp: 1574791579221,
- },
- }
- info := ConsumerRunningInfo{
- Properties: props,
- SubscriptionData: subData,
- StatusTable: statusTable,
- MQTable: mqTable,
- }
- data, err := info.Encode()
- So(err, ShouldBeNil)
- result := gjson.ParseBytes(data)
- Convey("test Properties fields", func() {
- r1 := result.Get("properties")
- So(r1.Exists(), ShouldBeTrue)
- m := r1.Map()
- So(len(m), ShouldEqual, 27)
-
- So(m["PROP_CLIENT_VERSION"], ShouldNotBeEmpty)
- So(m["PROP_CLIENT_VERSION"].String(), ShouldEqual, "V4_5_1")
-
- So(m["PROP_CONSUME_TYPE"], ShouldNotBeNil)
- So(m["PROP_CONSUME_TYPE"].String(), ShouldEqual, "CONSUME_PASSIVELY")
-
- So(m["PROP_THREADPOOL_CORE_SIZE"], ShouldNotBeNil)
- So(m["PROP_THREADPOOL_CORE_SIZE"].String(), ShouldEqual, "20")
-
- So(m["PROP_NAMESERVER_ADDR"], ShouldNotBeNil)
- So(m["PROP_NAMESERVER_ADDR"].String(), ShouldEqual, "mq-client-go-test.mq-internet-access.mq-internet.aliyuncs.com:80;")
-
- So(m["PROP_CONSUMER_START_TIMESTAMP"], ShouldNotBeNil)
- So(m["PROP_CONSUMER_START_TIMESTAMP"].String(), ShouldEqual, "1574791577504")
-
- So(m["PROP_CONSUMEORDERLY"], ShouldNotBeNil)
- So(m["PROP_CONSUMEORDERLY"].String(), ShouldEqual, "false")
- })
- Convey("test SubscriptionData fields", func() {
- r2 := result.Get("subscriptionSet")
- So(r2.Exists(), ShouldBeTrue)
- arr := r2.Array()
- So(len(arr), ShouldEqual, 2)
-
- m1 := arr[0].Map()
- So(len(m1), ShouldEqual, 7)
- So(m1["classFilterMode"].Bool(), ShouldEqual, false)
- So(len(m1["codes"].Array()), ShouldEqual, 0)
- So(m1["expressionType"].String(), ShouldEqual, "TAG")
- So(m1["subString"].String(), ShouldEqual, "*")
- So(m1["subVersion"].Int(), ShouldEqual, 1574791579242)
- So(len(m1["tags"].Array()), ShouldEqual, 0)
- So(m1["topic"].String(), ShouldEqual, "%RETRY%mq-client-go-test%GID_GO_TEST")
-
- m2 := arr[1].Map()
- So(len(m2), ShouldEqual, 7)
- So(m2["classFilterMode"].Bool(), ShouldEqual, true)
- So(len(m2["codes"].Array()), ShouldEqual, 0)
- So(m2["expressionType"].String(), ShouldEqual, "TAG")
- So(m2["subString"].String(), ShouldEqual, "*")
- So(m2["subVersion"].Int(), ShouldEqual, 1574791577523)
- So(len(m2["tags"].Array()), ShouldEqual, 0)
- So(m2["topic"].String(), ShouldEqual, "mq-client-go-test%go-test")
- })
- Convey("test StatusTable fields", func() {
- r3 := result.Get("statusTable")
- So(r3.Exists(), ShouldBeTrue)
- m := r3.Map()
- So(len(m), ShouldEqual, 2)
-
- status1 := m["mq-client-go-test%go-test"].Map()
- So(len(status1), ShouldEqual, 6)
- So(status1["pullRT"].Float(), ShouldEqual, 123)
- So(status1["pullTPS"].Float(), ShouldEqual, 123)
- So(status1["consumeRT"].Float(), ShouldEqual, 123)
- So(status1["consumeOKTPS"].Float(), ShouldEqual, 123)
- So(status1["consumeFailedTPS"].Float(), ShouldEqual, 123)
- So(status1["consumeFailedMsgs"].Int(), ShouldEqual, 1234)
-
- status2 := m["%RETRY%mq-client-go-test%GID_GO_TEST"].Map()
- So(len(status2), ShouldEqual, 6)
- So(status2["pullRT"].Float(), ShouldEqual, 11.11)
- So(status2["pullTPS"].Float(), ShouldEqual, 22.22)
- So(status2["consumeRT"].Float(), ShouldEqual, 33.33)
- So(status2["consumeOKTPS"].Float(), ShouldEqual, 44.44)
- So(status2["consumeFailedTPS"].Float(), ShouldEqual, 55.55)
- So(status2["consumeFailedMsgs"].Int(), ShouldEqual, 666)
- })
- Convey("test MQTable fields", func() {
- r4 := result.Get("mqTable")
- So(r4.Exists(), ShouldBeTrue)
- objNumbers := strings.Split(r4.String(), "},{")
- So(len(objNumbers), ShouldEqual, 2)
-
- obj1Str := objNumbers[0][1:len(objNumbers[0])] + "}"
- obj1KV := strings.Split(obj1Str, "}:{")
- So(len(obj1KV), ShouldEqual, 2)
-
- obj1 := gjson.Parse("{" + obj1KV[1][0:len(obj1KV[1])])
- So(obj1.Exists(), ShouldBeTrue)
- obj1M := obj1.Map()
- So(len(obj1M), ShouldEqual, 14)
- So(obj1M["commitOffset"].Int(), ShouldEqual, 0)
- So(obj1M["cachedMsgMinOffset"].Int(), ShouldEqual, 0)
- So(obj1M["cachedMsgMaxOffset"].Int(), ShouldEqual, 0)
- So(obj1M["cachedMsgCount"].Int(), ShouldEqual, 0)
- So(obj1M["cachedMsgSizeInMiB"].Int(), ShouldEqual, 0)
- So(obj1M["transactionMsgMinOffset"].Int(), ShouldEqual, 0)
- So(obj1M["transactionMsgMaxOffset"].Int(), ShouldEqual, 0)
- So(obj1M["transactionMsgCount"].Int(), ShouldEqual, 0)
- So(obj1M["locked"].Bool(), ShouldEqual, false)
- So(obj1M["tryUnlockTimes"].Int(), ShouldEqual, 0)
- So(obj1M["lastLockTimestamp"].Int(), ShouldEqual, 1574791579221)
- So(obj1M["dropped"].Bool(), ShouldEqual, false)
- So(obj1M["lastPullTimestamp"].Int(), ShouldEqual, 1574791579242)
- So(obj1M["lastConsumeTimestamp"].Int(), ShouldEqual, 1574791579221)
-
- obj2Str := "{" + objNumbers[1][0:len(objNumbers[1])-1]
- obj2KV := strings.Split(obj2Str, "}:{")
- So(len(obj2KV), ShouldEqual, 2)
- obj2 := gjson.Parse("{" + obj2KV[1][0:len(obj2KV[1])])
- So(obj2.Exists(), ShouldBeTrue)
- obj2M := obj2.Map()
- So(len(obj2M), ShouldEqual, 14)
- So(obj2M["commitOffset"].Int(), ShouldEqual, 1)
- So(obj2M["cachedMsgMinOffset"].Int(), ShouldEqual, 2)
- So(obj2M["cachedMsgMaxOffset"].Int(), ShouldEqual, 3)
- So(obj2M["cachedMsgCount"].Int(), ShouldEqual, 4)
- So(obj2M["cachedMsgSizeInMiB"].Int(), ShouldEqual, 5)
- So(obj2M["transactionMsgMinOffset"].Int(), ShouldEqual, 6)
- So(obj2M["transactionMsgMaxOffset"].Int(), ShouldEqual, 7)
- So(obj2M["transactionMsgCount"].Int(), ShouldEqual, 8)
- So(obj2M["locked"].Bool(), ShouldEqual, true)
- So(obj2M["tryUnlockTimes"].Int(), ShouldEqual, 9)
- So(obj2M["lastLockTimestamp"].Int(), ShouldEqual, 1574791579221)
- So(obj2M["dropped"].Bool(), ShouldEqual, false)
- So(obj2M["lastPullTimestamp"].Int(), ShouldEqual, 1574791579242)
- So(obj2M["lastConsumeTimestamp"].Int(), ShouldEqual, 1574791579221)
- })
- })
-}
-
-func TestConsumeMessageDirectlyResult_MarshalJSON(t *testing.T) {
- Convey("test ConsumeMessageDirectlyResult MarshalJson", t, func() {
- Convey("test consume success", func() {
- consumeMessageDirectlyResult := ConsumeMessageDirectlyResult{
- Order: false,
- AutoCommit: true,
- SpentTimeMills: 2,
- }
- consumeMessageDirectlyResult.ConsumeResult = ConsumeSuccess
- data, err := consumeMessageDirectlyResult.Encode()
- So(err, ShouldBeNil)
- rlog.Info("Json consumeMessageDirectlyResult", map[string]interface{}{
- "result": string(data),
- })
- })
-
- Convey("test consume timeout", func() {
- consumeResult := ConsumeMessageDirectlyResult{
- Order: false,
- AutoCommit: true,
- SpentTimeMills: 2,
- }
- consumeResult.ConsumeResult = ReturnNull
- data, err := consumeResult.Encode()
- So(err, ShouldBeNil)
- rlog.Info("Json consumeMessageDirectlyResult", map[string]interface{}{
- "result": string(data),
- })
- })
-
- Convey("test consume exception", func() {
- consumeResult := ConsumeMessageDirectlyResult{
- Order: false,
- AutoCommit: true,
- SpentTimeMills: 5,
- }
- consumeResult.ConsumeResult = ThrowException
- consumeResult.Remark = "Unknown Exception"
- data, err := consumeResult.Encode()
- So(err, ShouldBeNil)
- rlog.Info("Json consumeMessageDirectlyResult", map[string]interface{}{
- "result": string(data),
- })
- })
- })
-}
-
-func TestRestOffsetBody_MarshalJSON(t *testing.T) {
- Convey("test ResetOffset Body Decode gson json schema", t, func() {
- body := "{\"offsetTable\":[[{\"topic\":\"zx_tst\",\"brokerName\":\"tjwqtst-common-rocketmq-raft0\",\"queueId\":5},23354233],[{\"topic\":\"zx_tst\",\"brokerName\":\"tjwqtst-common-rocketmq-raft0\",\"queueId\":4},23354245],[{\"topic\":\"zx_tst\",\"brokerName\":\"tjwqtst-common-rocketmq-raft0\",\"queueId\":7},23354203],[{\"topic\":\"zx_tst\",\"brokerName\":\"tjwqtst-common-rocketmq-raft0\",\"queueId\":6},23354312],[{\"topic\":\"zx_tst\",\"brokerName\":\"tjwqtst-common-rocketmq-raft0\",\"queueId\":1},23373517],[{\"topic\":\"zx_tst\",\"brokerName\":\"tjwqtst-common-rocketmq-raft0\",\"queueId\":0},23373350],[{\"topic\":\"zx_tst\",\"brokerName\":\"tjwqtst-common-rocketmq-raft0\",\"queueId\":3},23373424],[{\"topic\":\"zx_tst\",\"brokerName\":\"tjwqtst-common-rocketmq-raft0\",\"queueId\":2},23373382]]}"
- resetOffsetBody := new(ResetOffsetBody)
- resetOffsetBody.Decode([]byte(body))
- offsetTable := resetOffsetBody.OffsetTable
- So(offsetTable, ShouldNotBeNil)
- So(len(offsetTable), ShouldEqual, 8)
- messageQueue := primitive.MessageQueue{
- Topic: "zx_tst",
- BrokerName: "tjwqtst-common-rocketmq-raft0",
- QueueId: 5,
- }
- So(offsetTable[messageQueue], ShouldEqual, 23354233)
- })
-
- Convey("test ResetOffset Body Decode fast json schema", t, func() {
- body := "{\"offsetTable\":{{\"brokerName\":\"RaftNode00\",\"queueId\":0,\"topic\":\"topicB\"}:11110,{\"brokerName\":\"RaftNode00\",\"queueId\":1,\"topic\":\"topicB\"}:0,{\"brokerName\":\"RaftNode00\",\"queueId\":2,\"topic\":\"topicB\"}:0,{\"brokerName\":\"RaftNode00\",\"queueId\":3,\"topic\":\"topicB\"}:0}}"
- resetOffsetBody := new(ResetOffsetBody)
- resetOffsetBody.Decode([]byte(body))
- offsetTable := resetOffsetBody.OffsetTable
- So(offsetTable, ShouldNotBeNil)
- So(len(offsetTable), ShouldEqual, 4)
- messageQueue := primitive.MessageQueue{
- Topic: "topicB",
- BrokerName: "RaftNode00",
- QueueId: 0,
- }
- So(offsetTable[messageQueue], ShouldEqual, 11110)
- })
-
- Convey("test ResetOffset Body Decode fast json schema with one item", t, func() {
- body := "{\"offsetTable\":{{\"brokerName\":\"RaftNode00\",\"queueId\":0,\"topic\":\"topicB\"}:11110}}"
- resetOffsetBody := new(ResetOffsetBody)
- resetOffsetBody.Decode([]byte(body))
- offsetTable := resetOffsetBody.OffsetTable
- So(offsetTable, ShouldNotBeNil)
- So(len(offsetTable), ShouldEqual, 1)
- messageQueue := primitive.MessageQueue{
- Topic: "topicB",
- BrokerName: "RaftNode00",
- QueueId: 0,
- }
- So(offsetTable[messageQueue], ShouldEqual, 11110)
- })
-
- Convey("test ResetOffset Body Decode empty fast json ", t, func() {
- body := "{\"offsetTable\":{}}"
- resetOffsetBody := new(ResetOffsetBody)
- resetOffsetBody.Decode([]byte(body))
- offsetTable := resetOffsetBody.OffsetTable
- So(offsetTable, ShouldNotBeNil)
- So(len(offsetTable), ShouldEqual, 0)
- })
-
- Convey("test ResetOffset Body Decode empty gson json ", t, func() {
- body := "{\"offsetTable\":[]}"
- resetOffsetBody := new(ResetOffsetBody)
- resetOffsetBody.Decode([]byte(body))
- offsetTable := resetOffsetBody.OffsetTable
- So(offsetTable, ShouldNotBeNil)
- So(len(offsetTable), ShouldEqual, 0)
- })
-}
diff --git a/internal/mq_version.go b/internal/mq_version.go
deleted file mode 100644
index 4b5c645..0000000
--- a/internal/mq_version.go
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing software
-distributed under the License is distributed on an "AS IS" BASIS
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package internal
-
-const (
- V4_1_0 = 0
-)
diff --git a/internal/namesrv.go b/internal/namesrv.go
deleted file mode 100644
index 96e708a..0000000
--- a/internal/namesrv.go
+++ /dev/null
@@ -1,185 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package internal
-
-import (
- "errors"
- "fmt"
- "regexp"
- "strings"
- "sync"
-
- "github.com/apache/rocketmq-client-go/v2/internal/remote"
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-const (
- DEFAULT_NAMESRV_ADDR = "http://jmenv.tbsite.net:8080/rocketmq/nsaddr"
-)
-
-var (
- ipRegex, _ = regexp.Compile(`^((25[0-5]|2[0-4]\d|((1\d{2})|([1-9]?\d)))\.){3}(25[0-5]|2[0-4]\d|((1\d{2})|([1-9]?\d)))`)
-
- ErrNoNameserver = errors.New("nameServerAddrs can't be empty.")
- ErrMultiIP = errors.New("multiple IP addr does not support")
- ErrIllegalIP = errors.New("IP addr error")
-)
-
-//go:generate mockgen -source namesrv.go -destination mock_namesrv.go -self_package github.com/apache/rocketmq-client-go/v2/internal --package internal Namesrvs
-type Namesrvs interface {
- UpdateNameServerAddress()
-
- AddBroker(routeData *TopicRouteData)
-
- cleanOfflineBroker()
-
- UpdateTopicRouteInfo(topic string) (routeData *TopicRouteData, changed bool, err error)
-
- UpdateTopicRouteInfoWithDefault(topic string, defaultTopic string, defaultQueueNum int) (*TopicRouteData, bool, error)
-
- FetchPublishMessageQueues(topic string) ([]*primitive.MessageQueue, error)
-
- FindBrokerAddrByTopic(topic string) string
-
- FindBrokerAddrByName(brokerName string) string
-
- FindBrokerAddressInSubscribe(brokerName string, brokerId int64, onlyThisBroker bool) *FindBrokerResult
-
- FetchSubscribeMessageQueues(topic string) ([]*primitive.MessageQueue, error)
-
- AddrList() []string
-}
-
-// namesrvs rocketmq namesrv instance.
-type namesrvs struct {
- // namesrv addr list
- srvs []string
-
- // lock for getNameServerAddress in case of update index race condition
- lock sync.Locker
-
- // index indicate the next position for getNameServerAddress
- index int
-
- // brokerName -> *BrokerData
- brokerAddressesMap sync.Map
-
- bundleClient *rmqClient
-
- // brokerName -> map[string]int32: brokerAddr -> version
- brokerVersionMap map[string]map[string]int32
- // lock for broker version read/write
- brokerLock *sync.RWMutex
-
- //subscribeInfoMap sync.Map
- routeDataMap sync.Map
-
- lockNamesrv sync.Mutex
-
- nameSrvClient remote.RemotingClient
-
- resolver primitive.NsResolver
-}
-
-var _ Namesrvs = (*namesrvs)(nil)
-var namesrvMap sync.Map
-
-// NewNamesrv init Namesrv from namesrv addr string.
-// addr primitive.NamesrvAddr
-func GetOrSetNamesrv(clientId string, namesrv *namesrvs) *namesrvs {
- actual, _ := namesrvMap.LoadOrStore(clientId, namesrv)
- return actual.(*namesrvs)
-}
-func GetNamesrv(clientId string) (*namesrvs, error) {
- actual, ok := namesrvMap.Load(clientId)
- if !ok {
- return nil, fmt.Errorf("the namesrv in instanceName [%s] not found", clientId)
- }
- return actual.(*namesrvs), nil
-}
-func NewNamesrv(resolver primitive.NsResolver) (*namesrvs, error) {
- addr := resolver.Resolve()
- if len(addr) == 0 {
- return nil, errors.New("no name server addr found with resolver: " + resolver.Description())
- }
-
- if err := primitive.NamesrvAddr(addr).Check(); err != nil {
- return nil, err
- }
- nameSrvClient := remote.NewRemotingClient()
- return &namesrvs{
- srvs: addr,
- lock: new(sync.Mutex),
- nameSrvClient: nameSrvClient,
- brokerVersionMap: make(map[string]map[string]int32, 0),
- brokerLock: new(sync.RWMutex),
- resolver: resolver,
- }, nil
-}
-
-// getNameServerAddress return namesrv using round-robin strategy.
-func (s *namesrvs) getNameServerAddress() string {
- s.lock.Lock()
- defer s.lock.Unlock()
-
- addr := s.srvs[s.index%len(s.srvs)]
- index := s.index + 1
- if index < 0 {
- index = -index
- }
- index %= len(s.srvs)
- s.index = index
- if strings.HasPrefix(addr, "https") {
- return strings.TrimPrefix(addr, "https://")
- }
- return strings.TrimPrefix(addr, "http://")
-}
-
-func (s *namesrvs) Size() int {
- return len(s.srvs)
-}
-
-func (s *namesrvs) String() string {
- return strings.Join(s.srvs, ";")
-}
-func (s *namesrvs) SetCredentials(credentials primitive.Credentials) {
- s.nameSrvClient.RegisterInterceptor(remote.ACLInterceptor(credentials))
-}
-
-func (s *namesrvs) AddrList() []string {
- return s.srvs
-}
-
-// UpdateNameServerAddress will update srvs.
-// docs: https://rocketmq.apache.org/docs/best-practice-namesvr/
-func (s *namesrvs) UpdateNameServerAddress() {
- s.lock.Lock()
- defer s.lock.Unlock()
-
- srvs := s.resolver.Resolve()
- if len(srvs) == 0 {
- return
- }
-
- updated := primitive.Diff(s.srvs, srvs)
- if !updated {
- return
- }
-
- s.srvs = srvs
-}
diff --git a/internal/namesrv_test.go b/internal/namesrv_test.go
deleted file mode 100644
index b047a07..0000000
--- a/internal/namesrv_test.go
+++ /dev/null
@@ -1,149 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package internal
-
-import (
- "fmt"
- "github.com/apache/rocketmq-client-go/v2/rlog"
- "net"
- "net/http"
- "os"
- "strings"
- "sync"
- "testing"
-
- "github.com/apache/rocketmq-client-go/v2/primitive"
-
- . "github.com/smartystreets/goconvey/convey"
- "github.com/stretchr/testify/assert"
-)
-
-// TestSelector test roundrobin selector in namesrv
-func TestSelector(t *testing.T) {
- srvs := []string{"127.0.0.1:9876", "127.0.0.1:9879", "12.24.123.243:10911", "12.24.123.243:10915"}
- namesrv, err := NewNamesrv(primitive.NewPassthroughResolver(srvs))
- assert.Nil(t, err)
-
- assert.Equal(t, srvs[0], namesrv.getNameServerAddress())
- assert.Equal(t, srvs[1], namesrv.getNameServerAddress())
- assert.Equal(t, srvs[2], namesrv.getNameServerAddress())
- assert.Equal(t, srvs[3], namesrv.getNameServerAddress())
- assert.Equal(t, srvs[0], namesrv.getNameServerAddress())
- assert.Equal(t, srvs[1], namesrv.getNameServerAddress())
- assert.Equal(t, srvs[2], namesrv.getNameServerAddress())
- assert.Equal(t, srvs[3], namesrv.getNameServerAddress())
- assert.Equal(t, srvs[0], namesrv.getNameServerAddress())
-}
-
-func TestGetNamesrv(t *testing.T) {
- Convey("Test GetNamesrv round-robin strategy", t, func() {
- ns := &namesrvs{
- srvs: []string{"192.168.100.1",
- "192.168.100.2",
- "192.168.100.3",
- "192.168.100.4",
- "192.168.100.5",
- },
- lock: new(sync.Mutex),
- }
-
- index1 := ns.index
- IP1 := ns.getNameServerAddress()
-
- index2 := ns.index
- IP2 := ns.getNameServerAddress()
-
- So(index1+1, ShouldEqual, index2)
- So(IP1, ShouldEqual, ns.srvs[index1])
- So(IP2, ShouldEqual, ns.srvs[index2])
- })
-}
-
-func TestUpdateNameServerAddress(t *testing.T) {
- Convey("Test UpdateNameServerAddress method", t, func() {
- srvs := []string{
- "192.168.100.1",
- "192.168.100.2",
- "192.168.100.3",
- "192.168.100.4",
- "192.168.100.5",
- }
- http.HandleFunc("/nameserver/addrs", func(w http.ResponseWriter, r *http.Request) {
- fmt.Fprintf(w, strings.Join(srvs, ";"))
- })
- server := &http.Server{Addr: ":0", Handler: nil}
- listener, _ := net.Listen("tcp", ":0")
- go server.Serve(listener)
-
- port := listener.Addr().(*net.TCPAddr).Port
- nameServerDommain := fmt.Sprintf("http://127.0.0.1:%d/nameserver/addrs", port)
- rlog.Info("Temporary Nameserver", map[string]interface{}{
- "domain": nameServerDommain,
- })
-
- resolver := primitive.NewHttpResolver("DEFAULT", nameServerDommain)
- ns := &namesrvs{
- srvs: []string{},
- lock: new(sync.Mutex),
- resolver: resolver,
- }
-
- ns.UpdateNameServerAddress()
-
- index1 := ns.index
- IP1 := ns.getNameServerAddress()
-
- index2 := ns.index
- IP2 := ns.getNameServerAddress()
-
- So(index1+1, ShouldEqual, index2)
- So(IP1, ShouldEqual, srvs[index1])
- So(IP2, ShouldEqual, srvs[index2])
- })
-}
-
-func TestUpdateNameServerAddressUseEnv(t *testing.T) {
- Convey("Test UpdateNameServerAddress Use Env", t, func() {
- srvs := []string{
- "192.168.100.1",
- "192.168.100.2",
- "192.168.100.3",
- "192.168.100.4",
- "192.168.100.5",
- }
-
- resolver := primitive.NewEnvResolver()
- ns := &namesrvs{
- srvs: []string{},
- lock: new(sync.Mutex),
- resolver: resolver,
- }
- os.Setenv("NAMESRV_ADDR", strings.Join(srvs, ";"))
- ns.UpdateNameServerAddress()
-
- index1 := ns.index
- IP1 := ns.getNameServerAddress()
-
- index2 := ns.index
- IP2 := ns.getNameServerAddress()
-
- So(index1+1, ShouldEqual, index2)
- So(IP1, ShouldEqual, srvs[index1])
- So(IP2, ShouldEqual, srvs[index2])
- })
-}
diff --git a/internal/perm.go b/internal/perm.go
deleted file mode 100644
index 638b7a7..0000000
--- a/internal/perm.go
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package internal
-
-const (
- permPriority = 0x1 << 3
- permRead = 0x1 << 2
- permWrite = 0x1 << 1
- permInherit = 0x1 << 0
-)
-
-func queueIsReadable(perm int) bool {
- return (perm & permRead) == permRead
-}
-
-func queueIsWriteable(perm int) bool {
- return (perm & permWrite) == permWrite
-}
-
-func queueIsInherited(perm int) bool {
- return (perm & permInherit) == permInherit
-}
-
-func perm2string(perm int) string {
- bytes := make([]byte, 3)
- for i := 0; i < 3; i++ {
- bytes[i] = '-'
- }
-
- if queueIsReadable(perm) {
- bytes[0] = 'R'
- }
-
- if queueIsWriteable(perm) {
- bytes[1] = 'W'
- }
-
- if queueIsInherited(perm) {
- bytes[2] = 'X'
- }
-
- return string(bytes)
-}
diff --git a/internal/remote/codec.go b/internal/remote/codec.go
deleted file mode 100644
index 6b3d3d6..0000000
--- a/internal/remote/codec.go
+++ /dev/null
@@ -1,521 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package remote
-
-import (
- "bytes"
- "encoding/binary"
- "fmt"
- "io"
- "sync/atomic"
-
- jsoniter "github.com/json-iterator/go"
-)
-
-var opaque int32
-
-const (
- // 0, REQUEST_COMMAND
- RPCType = 0
- // 1, RPC
- RPCOneWay = 1
- //ResponseType for response
- ResponseType = 1
- _Flag = 0
- _Version = 317
-)
-
-type LanguageCode byte
-
-const (
- _Java = LanguageCode(0)
- _Go = LanguageCode(9)
- _Unknown = LanguageCode(127)
-)
-
-func (lc LanguageCode) MarshalJSON() ([]byte, error) {
- return []byte(`"GO"`), nil
-}
-
-func (lc *LanguageCode) UnmarshalJSON(b []byte) error {
- switch string(b) {
- case "JAVA":
- *lc = _Java
- case "GO", `"GO"`:
- *lc = _Go
- default:
- *lc = _Unknown
- }
- return nil
-}
-
-func (lc LanguageCode) String() string {
- switch lc {
- case _Java:
- return "JAVA"
- case _Go:
- return "GO"
- default:
- return "unknown"
- }
-}
-
-type RemotingCommand struct {
- Code int16 `json:"code"`
- Language LanguageCode `json:"language"`
- Version int16 `json:"version"`
- Opaque int32 `json:"opaque"`
- Flag int32 `json:"flag"`
- Remark string `json:"remark"`
- ExtFields map[string]string `json:"extFields"`
- Body []byte `json:"-"`
-}
-
-type CustomHeader interface {
- Encode() map[string]string
-}
-
-func NewRemotingCommand(code int16, header CustomHeader, body []byte) *RemotingCommand {
- cmd := &RemotingCommand{
- Code: code,
- Version: _Version,
- Opaque: atomic.AddInt32(&opaque, 1),
- Body: body,
- Language: _Go,
- ExtFields: make(map[string]string),
- }
-
- if header != nil {
- cmd.ExtFields = header.Encode()
- }
-
- return cmd
-}
-
-func (command *RemotingCommand) String() string {
- return fmt.Sprintf("Code: %d, opaque: %d, Remark: %s, ExtFields: %v",
- command.Code, command.Opaque, command.Remark, command.ExtFields)
-}
-
-func (command *RemotingCommand) isResponseType() bool {
- return command.Flag&(ResponseType) == ResponseType
-}
-
-func (command *RemotingCommand) markResponseType() {
- command.Flag = command.Flag | ResponseType
-}
-
-var (
- jsonSerializer = &jsonCodec{}
- rocketMqSerializer = &rmqCodec{}
- codecType byte
-)
-
-// encode RemotingCommand
-//
-// Frame format:
-// +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
-// + item | frame_size | header_length | header_body | body +
-// +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
-// + len | 4bytes | 4bytes | (21 + r_len + e_len) bytes | remain bytes +
-// +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
-func (command *RemotingCommand) WriteTo(w io.Writer) error {
- var (
- header []byte
- err error
- )
-
- switch codecType {
- case JsonCodecs:
- header, err = jsonSerializer.encodeHeader(command)
- case RocketMQCodecs:
- header, err = rocketMqSerializer.encodeHeader(command)
- }
-
- if err != nil {
- return err
- }
-
- frameSize := 4 + len(header) + len(command.Body)
- err = binary.Write(w, binary.BigEndian, int32(frameSize))
- if err != nil {
- return err
- }
-
- err = binary.Write(w, binary.BigEndian, markProtocolType(int32(len(header))))
- if err != nil {
- return err
- }
-
- _, err = w.Write(header)
- if err != nil {
- return err
- }
-
- _, err = w.Write(command.Body)
- return err
-}
-
-func encode(command *RemotingCommand) ([]byte, error) {
- var (
- header []byte
- err error
- )
-
- switch codecType {
- case JsonCodecs:
- header, err = jsonSerializer.encodeHeader(command)
- case RocketMQCodecs:
- header, err = rocketMqSerializer.encodeHeader(command)
- }
-
- if err != nil {
- return nil, err
- }
-
- frameSize := 4 + len(header) + len(command.Body)
- buf := bytes.NewBuffer(make([]byte, frameSize))
- buf.Reset()
-
- err = binary.Write(buf, binary.BigEndian, int32(frameSize))
- if err != nil {
- return nil, err
- }
-
- err = binary.Write(buf, binary.BigEndian, markProtocolType(int32(len(header))))
- if err != nil {
- return nil, err
- }
-
- err = binary.Write(buf, binary.BigEndian, header)
- if err != nil {
- return nil, err
- }
-
- err = binary.Write(buf, binary.BigEndian, command.Body)
- if err != nil {
- return nil, err
- }
-
- return buf.Bytes(), nil
-}
-
-func decode(data []byte) (*RemotingCommand, error) {
- buf := bytes.NewReader(data)
- length := int32(len(data))
- var oriHeaderLen int32
- err := binary.Read(buf, binary.BigEndian, &oriHeaderLen)
- if err != nil {
- return nil, err
- }
-
- headerLength := oriHeaderLen & 0xFFFFFF
- headerData := make([]byte, headerLength)
- if _, err = io.ReadFull(buf, headerData); err != nil {
- return nil, err
- }
-
- var command *RemotingCommand
- switch codeType := byte((oriHeaderLen >> 24) & 0xFF); codeType {
- case JsonCodecs:
- command, err = jsonSerializer.decodeHeader(headerData)
- case RocketMQCodecs:
- command, err = rocketMqSerializer.decodeHeader(headerData)
- default:
- err = fmt.Errorf("unknown codec type: %d", codeType)
- }
- if err != nil {
- return nil, err
- }
-
- bodyLength := length - 4 - headerLength
- if bodyLength > 0 {
- bodyData := make([]byte, bodyLength)
- if _, err = io.ReadFull(buf, bodyData); err != nil {
- return nil, err
- }
- command.Body = bodyData
- }
- return command, nil
-}
-
-func markProtocolType(source int32) []byte {
- result := make([]byte, 4)
- result[0] = codecType
- result[1] = byte((source >> 16) & 0xFF)
- result[2] = byte((source >> 8) & 0xFF)
- result[3] = byte(source & 0xFF)
- return result
-}
-
-const (
- JsonCodecs = byte(0)
- RocketMQCodecs = byte(1)
-)
-
-type serializer interface {
- encodeHeader(command *RemotingCommand) ([]byte, error)
- decodeHeader(data []byte) (*RemotingCommand, error)
-}
-
-// jsonCodec please refer to remoting/protocol/RemotingSerializable
-type jsonCodec struct{}
-
-func (c *jsonCodec) encodeHeader(command *RemotingCommand) ([]byte, error) {
- buf, err := jsoniter.Marshal(command)
- if err != nil {
- return nil, err
- }
- return buf, nil
-}
-
-func (c *jsonCodec) decodeHeader(header []byte) (*RemotingCommand, error) {
- command := &RemotingCommand{}
- command.ExtFields = make(map[string]string)
- command.Body = make([]byte, 0)
- err := jsoniter.Unmarshal(header, command)
- if err != nil {
- return nil, err
- }
- return command, nil
-}
-
-// rmqCodec implementation of RocketMQCodecs private protocol, please refer to remoting/protocol/RocketMQSerializable
-// RocketMQCodecs Private Protocol Header format:
-//
-// v_flag: version flag
-// r_len: length of remark body
-// r_body: data of remark body
-// e_len: length of extends fields body
-// e_body: data of extends fields
-//
-// +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
-// + item | request_code | l_flag | v_flag | opaque | request_flag | r_len | r_body | e_len | e_body +
-// +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
-// + len | 2bytes | 1byte | 2bytes | 4bytes | 4 bytes | 4 bytes | r_len bytes | 4 bytes | e_len bytes +
-// +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
-const (
- // header + body length
- headerFixedLength = 21
-)
-
-type rmqCodec struct{}
-
-// encodeHeader
-func (c *rmqCodec) encodeHeader(command *RemotingCommand) ([]byte, error) {
- extBytes, err := c.encodeMaps(command.ExtFields)
- if err != nil {
- return nil, err
- }
-
- buf := bytes.NewBuffer(make([]byte, headerFixedLength+len(command.Remark)+len(extBytes)))
- buf.Reset()
-
- // request code, length is 2 bytes
- err = binary.Write(buf, binary.BigEndian, int16(command.Code))
- if err != nil {
- return nil, err
- }
-
- // language flag, length is 1 byte
- err = binary.Write(buf, binary.BigEndian, _Go)
- if err != nil {
- return nil, err
- }
-
- // version flag, length is 2 bytes
- err = binary.Write(buf, binary.BigEndian, int16(command.Version))
- if err != nil {
- return nil, err
- }
-
- // opaque flag, opaque is request identifier, length is 4 bytes
- err = binary.Write(buf, binary.BigEndian, command.Opaque)
- if err != nil {
- return nil, err
- }
-
- // request flag, length is 4 bytes
- err = binary.Write(buf, binary.BigEndian, command.Flag)
- if err != nil {
- return nil, err
- }
-
- // remark length flag, length is 4 bytes
- err = binary.Write(buf, binary.BigEndian, int32(len(command.Remark)))
- if err != nil {
- return nil, err
- }
-
- // write remark, len(command.Remark) bytes
- if len(command.Remark) > 0 {
- err = binary.Write(buf, binary.BigEndian, []byte(command.Remark))
- if err != nil {
- return nil, err
- }
- }
-
- err = binary.Write(buf, binary.BigEndian, int32(len(extBytes)))
- if err != nil {
- return nil, err
- }
-
- if len(extBytes) > 0 {
- err = binary.Write(buf, binary.BigEndian, extBytes)
- if err != nil {
- return nil, err
- }
- }
-
- return buf.Bytes(), nil
-}
-
-func (c *rmqCodec) encodeMaps(maps map[string]string) ([]byte, error) {
- if maps == nil || len(maps) == 0 {
- return []byte{}, nil
- }
- extFieldsBuf := bytes.NewBuffer([]byte{})
- var err error
- for key, value := range maps {
- err = binary.Write(extFieldsBuf, binary.BigEndian, int16(len(key)))
- if err != nil {
- return nil, err
- }
- err = binary.Write(extFieldsBuf, binary.BigEndian, []byte(key))
- if err != nil {
- return nil, err
- }
-
- err = binary.Write(extFieldsBuf, binary.BigEndian, int32(len(value)))
- if err != nil {
- return nil, err
- }
- err = binary.Write(extFieldsBuf, binary.BigEndian, []byte(value))
- if err != nil {
- return nil, err
- }
- }
- return extFieldsBuf.Bytes(), nil
-}
-
-func (c *rmqCodec) decodeHeader(data []byte) (*RemotingCommand, error) {
- var err error
- command := &RemotingCommand{}
- buf := bytes.NewBuffer(data)
- var code int16
- err = binary.Read(buf, binary.BigEndian, &code)
- if err != nil {
- return nil, err
- }
- command.Code = code
-
- var (
- languageCode byte
- remarkLen int32
- extFieldsLen int32
- )
- err = binary.Read(buf, binary.BigEndian, &languageCode)
- if err != nil {
- return nil, err
- }
- command.Language = LanguageCode(languageCode)
-
- var version int16
- err = binary.Read(buf, binary.BigEndian, &version)
- if err != nil {
- return nil, err
- }
- command.Version = version
-
- // int opaque
- err = binary.Read(buf, binary.BigEndian, &command.Opaque)
- if err != nil {
- return nil, err
- }
-
- // int flag
- err = binary.Read(buf, binary.BigEndian, &command.Flag)
- if err != nil {
- return nil, err
- }
-
- // String remark
- err = binary.Read(buf, binary.BigEndian, &remarkLen)
- if err != nil {
- return nil, err
- }
-
- if remarkLen > 0 {
- var remarkData = make([]byte, remarkLen)
- if _, err = io.ReadFull(buf, remarkData); err != nil {
- return nil, err
- }
- command.Remark = string(remarkData)
- }
-
- err = binary.Read(buf, binary.BigEndian, &extFieldsLen)
- if err != nil {
- return nil, err
- }
-
- if extFieldsLen > 0 {
- extFieldsData := make([]byte, extFieldsLen)
- if _, err := io.ReadFull(buf, extFieldsData); err != nil {
- return nil, err
- }
-
- command.ExtFields = make(map[string]string)
- buf := bytes.NewBuffer(extFieldsData)
- var (
- kLen int16
- vLen int32
- )
- for buf.Len() > 0 {
- err = binary.Read(buf, binary.BigEndian, &kLen)
- if err != nil {
- return nil, err
- }
-
- key, err := getExtFieldsData(buf, int32(kLen))
- if err != nil {
- return nil, err
- }
-
- err = binary.Read(buf, binary.BigEndian, &vLen)
- if err != nil {
- return nil, err
- }
-
- value, err := getExtFieldsData(buf, vLen)
- if err != nil {
- return nil, err
- }
- command.ExtFields[key] = value
- }
- }
-
- return command, nil
-}
-
-func getExtFieldsData(buff io.Reader, length int32) (string, error) {
- var data = make([]byte, length)
- if _, err := io.ReadFull(buff, data); err != nil {
- return "", err
- }
-
- return string(data), nil
-}
diff --git a/internal/remote/codec_test.go b/internal/remote/codec_test.go
deleted file mode 100644
index 0717451..0000000
--- a/internal/remote/codec_test.go
+++ /dev/null
@@ -1,378 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package remote
-
-import (
- "encoding/json"
- "github.com/apache/rocketmq-client-go/v2/rlog"
- "math/rand"
- "reflect"
- "testing"
- "unsafe"
-
- jsoniter "github.com/json-iterator/go"
-
- "github.com/stretchr/testify/assert"
-)
-
-type testHeader struct {
-}
-
-func (t testHeader) Encode() map[string]string {
- properties := make(map[string]string)
- for i := 0; i < 10; i++ {
- properties[randomString(rand.Intn(20))] = randomString(rand.Intn(20))
- }
- return properties
-}
-
-func randomBytes(length int) []byte {
- bs := make([]byte, length)
- if _, err := rand.Read(bs); err != nil {
- panic("read random bytes fail")
- }
- return bs
-}
-
-func randomString(length int) string {
- bs := make([]byte, length)
- for i := 0; i < len(bs); i++ {
- bs[i] = byte(97 + rand.Intn(26))
- }
- return string(bs)
-}
-
-func randomNewRemotingCommand() *RemotingCommand {
- var h testHeader
- body := randomBytes(rand.Intn(100))
- return NewRemotingCommand(int16(rand.Intn(1000)), h, body)
-}
-
-func Test_encode(t *testing.T) {
- for i := 0; i < 1000; i++ {
- rc := randomNewRemotingCommand()
- if _, err := encode(rc); err != nil {
- t.Fatalf("encode RemotingCommand to bytes fail: %v", err)
- }
- }
-}
-
-func Benchmark_encode(b *testing.B) {
- rc := randomNewRemotingCommand()
- b.ResetTimer()
-
- for i := 0; i < b.N; i++ {
- if _, err := encode(rc); err != nil {
- b.Fatalf("encode RemotingCommand to bytes fail: %v", err)
- }
- }
-}
-
-func Test_decode(t *testing.T) {
- for i := 0; i < 1000; i++ {
- rc := randomNewRemotingCommand()
-
- bs, err := encode(rc)
- if err != nil {
- t.Fatalf("encode RemotingCommand to bytes fail: %v", err)
- }
- bs = bs[4:]
- decodedRc, err := decode(bs)
- if err != nil {
- t.Fatalf("decode bytes to RemotingCommand fail: %v", err)
- }
-
- if rc.Code != decodedRc.Code {
- t.Fatalf("wrong Code. want=%d, got=%d", rc.Code, decodedRc.Code)
- }
- if rc.Version != decodedRc.Version {
- t.Fatalf("wrong Version. want=%d, got=%d", rc.Version, decodedRc.Version)
- }
- if rc.Opaque != decodedRc.Opaque {
- t.Fatalf("wrong opaque. want=%d, got=%d", rc.Opaque, decodedRc.Opaque)
- }
- if rc.Remark != decodedRc.Remark {
- t.Fatalf("wrong remark. want=%s, got=%s", rc.Remark, decodedRc.Remark)
- }
- if rc.Flag != decodedRc.Flag {
- t.Fatalf("wrong flag. want=%d, got=%d", rc.Flag, decodedRc.Flag)
- }
- if !reflect.DeepEqual(rc.ExtFields, decodedRc.ExtFields) {
- t.Fatalf("wrong extFields, want=%v, got=%v", rc.ExtFields, decodedRc.ExtFields)
- }
- }
-}
-
-func Benchmark_decode(b *testing.B) {
- rc := randomNewRemotingCommand()
- bs, err := encode(rc)
- if err != nil {
- b.Fatalf("encode RemotingCommand to bytes fail: %v", err)
- }
- b.ResetTimer()
- bs = bs[4:]
- for i := 0; i < b.N; i++ {
- if _, err := decode(bs); err != nil {
- b.Fatalf("decode bytes to RemotingCommand fail: %v", err)
- }
- }
-}
-
-func Test_jsonCodec_encodeHeader(t *testing.T) {
- for i := 0; i < 1000; i++ {
- rc := randomNewRemotingCommand()
-
- if _, err := jsonSerializer.encodeHeader(rc); err != nil {
- t.Fatalf("encode header with jsonCodec fail: %v", err)
- }
- }
-}
-
-func Benchmark_jsonCodec_encodeHeader(b *testing.B) {
- rc := randomNewRemotingCommand()
- b.ResetTimer()
-
- for i := 0; i < b.N; i++ {
- if _, err := jsonSerializer.encodeHeader(rc); err != nil {
- b.Fatalf("encode header with jsonCodec fail: %v", err)
- }
- }
-}
-
-func Test_jsonCodec_decodeHeader(t *testing.T) {
- for i := 0; i < 1; i++ {
- rc := randomNewRemotingCommand()
-
- headers, err := jsonSerializer.encodeHeader(rc)
- if err != nil {
- t.Fatalf("encode header with jsonCodec fail: %v", err)
- }
-
- decodedRc, err := jsonSerializer.decodeHeader(headers)
- if err != nil {
- t.Fatalf("decode header with jsonCodec fail: %v", err)
- }
-
- if rc.Code != decodedRc.Code {
- t.Fatalf("wrong Code. want=%d, got=%d", rc.Code, decodedRc.Code)
- }
- if rc.Version != decodedRc.Version {
- t.Fatalf("wrong Version. want=%d, got=%d", rc.Version, decodedRc.Version)
- }
- if rc.Opaque != decodedRc.Opaque {
- t.Fatalf("wrong opaque. want=%d, got=%d", rc.Opaque, decodedRc.Opaque)
- }
- if rc.Remark != decodedRc.Remark {
- t.Fatalf("wrong remark. want=%s, got=%s", rc.Remark, decodedRc.Remark)
- }
- if rc.Flag != decodedRc.Flag {
- t.Fatalf("wrong flag. want=%d, got=%d", rc.Flag, decodedRc.Flag)
- }
- if !reflect.DeepEqual(rc.ExtFields, decodedRc.ExtFields) {
- t.Fatalf("wrong extFields, want=%v, got=%v", rc.ExtFields, decodedRc.ExtFields)
- }
- }
-}
-
-func Benchmark_jsonCodec_decodeHeader(b *testing.B) {
- rc := randomNewRemotingCommand()
- headers, err := jsonSerializer.encodeHeader(rc)
- if err != nil {
- b.Fatalf("encode header with jsonCodec fail: %v", err)
- }
- b.ResetTimer()
-
- for i := 0; i < b.N; i++ {
- if _, err := jsonSerializer.decodeHeader(headers); err != nil {
- b.Fatalf("decode header with jsonCodec fail: %v", err)
- }
- }
-}
-
-func Test_rmqCodec_encodeHeader(t *testing.T) {
- for i := 0; i < 1000; i++ {
- rc := randomNewRemotingCommand()
-
- if _, err := rocketMqSerializer.encodeHeader(rc); err != nil {
- t.Fatalf("encode header with rmqCodec fail: %v", err)
- }
- }
-}
-
-func Benchmark_rmqCodec_encodeHeader(b *testing.B) {
- rc := randomNewRemotingCommand()
- b.ResetTimer()
-
- for i := 0; i < b.N; i++ {
- if _, err := rocketMqSerializer.encodeHeader(rc); err != nil {
- b.Fatalf("encode header with rmqCodec fail: %v", err)
- }
- }
-}
-
-func Test_rmqCodec_decodeHeader(t *testing.T) {
- for i := 0; i < 1; i++ {
- rc := randomNewRemotingCommand()
-
- headers, err := rocketMqSerializer.encodeHeader(rc)
- if err != nil {
- t.Fatalf("encode header with rmqCodec fail: %v", err)
- }
-
- decodedRc, err := rocketMqSerializer.decodeHeader(headers)
- if err != nil {
- t.Fatalf("decode header with rmqCodec fail: %v", err)
- }
- if rc.Code != decodedRc.Code {
- t.Fatalf("wrong Code. want=%d, got=%d", rc.Code, decodedRc.Code)
- }
- if rc.Version != decodedRc.Version {
- t.Fatalf("wrong Version. want=%d, got=%d", rc.Version, decodedRc.Version)
- }
- if rc.Opaque != decodedRc.Opaque {
- t.Fatalf("wrong opaque. want=%d, got=%d", rc.Opaque, decodedRc.Opaque)
- }
- if rc.Remark != decodedRc.Remark {
- t.Fatalf("wrong remark. want=%s, got=%s", rc.Remark, decodedRc.Remark)
- }
- if rc.Flag != decodedRc.Flag {
- t.Fatalf("wrong flag. want=%d, got=%d", rc.Flag, decodedRc.Flag)
- }
- if !reflect.DeepEqual(rc.ExtFields, decodedRc.ExtFields) {
- t.Fatalf("wrong extFields, want=%v, got=%v", rc.ExtFields, decodedRc.ExtFields)
- }
-
- }
-}
-
-func Benchmark_rmqCodec_decodeHeader(b *testing.B) {
- rc := randomNewRemotingCommand()
- headers, err := rocketMqSerializer.encodeHeader(rc)
- if err != nil {
- b.Fatalf("encode header with rmqCodec fail: %v", err)
- }
- b.ResetTimer()
-
- for i := 0; i < b.N; i++ {
- if _, err := rocketMqSerializer.decodeHeader(headers); err != nil {
- b.Fatalf("decode header with rmqCodec fail: %v", err)
- }
- }
-}
-
-func TestCommandJsonEncodeDecode(t *testing.T) {
- var h testHeader
- cmd := NewRemotingCommand(192, h, []byte("Hello RocketMQCodecs"))
- codecType = JsonCodecs
- cmdData, err := encode(cmd)
- if err != nil {
- t.Errorf("failed to encode remotingCommand in JSON, %s", err)
- } else {
- if len(cmdData) == 0 {
- t.Errorf("failed to encode remotingCommand, result is empty.")
- }
- }
- cmdData = cmdData[4:]
- newCmd, err := decode(cmdData)
- if err != nil {
- t.Errorf("failed to decode remoting in JSON. %s", err)
- }
- if newCmd.Code != cmd.Code {
- t.Errorf("wrong command code. want=%d, got=%d", cmd.Code, newCmd.Code)
- }
- if newCmd.Version != cmd.Version {
- t.Errorf("wrong command version. want=%d, got=%d", cmd.Version, newCmd.Version)
- }
- if newCmd.Opaque != cmd.Opaque {
- t.Errorf("wrong command version. want=%d, got=%d", cmd.Opaque, newCmd.Opaque)
- }
- if newCmd.Flag != cmd.Flag {
- t.Errorf("wrong commad flag. want=%d, got=%d", cmd.Flag, newCmd.Flag)
- }
- if newCmd.Remark != cmd.Remark {
- t.Errorf("wrong command remakr. want=%s, got=%s", cmd.Remark, newCmd.Remark)
- }
-}
-
-func TestCommandRocketMQEncodeDecode(t *testing.T) {
- var h testHeader
- cmd := NewRemotingCommand(192, h, []byte("Hello RocketMQCodecs"))
- codecType = RocketMQCodecs
- cmdData, err := encode(cmd)
- if err != nil {
- t.Errorf("failed to encode remotingCommand in JSON, %s", err)
- } else {
- if len(cmdData) == 0 {
- t.Errorf("failed to encode remotingCommand, result is empty.")
- }
- }
- cmdData = cmdData[4:]
- newCmd, err := decode(cmdData)
- if err != nil {
- t.Errorf("failed to decode remoting in JSON. %s", err)
- }
- if newCmd.Code != cmd.Code {
- t.Errorf("wrong command code. want=%d, got=%d", cmd.Code, newCmd.Code)
- }
- if newCmd.Language != cmd.Language {
- t.Errorf("wrong command language. want=%d, got=%d", cmd.Language, newCmd.Language)
- }
- if newCmd.Version != cmd.Version {
- t.Errorf("wrong command version. want=%d, got=%d", cmd.Version, newCmd.Version)
- }
- if newCmd.Opaque != cmd.Opaque {
- t.Errorf("wrong command version. want=%d, got=%d", cmd.Opaque, newCmd.Opaque)
- }
- if newCmd.Flag != cmd.Flag {
- t.Errorf("wrong commad flag. want=%d, got=%d", cmd.Flag, newCmd.Flag)
- }
- if newCmd.Remark != cmd.Remark {
- t.Errorf("wrong command remakr. want=%s, got=%s", cmd.Remark, newCmd.Remark)
- }
-}
-
-func TestCommandJsonIter(t *testing.T) {
- var h testHeader
- cmd := NewRemotingCommand(192, h, []byte("Hello RocketMQCodecs"))
- cmdData, err := json.Marshal(cmd)
- assert.Nil(t, err)
- rlog.Info("Command Data From Json", map[string]interface{}{
- "data": *(*string)(unsafe.Pointer(&cmdData)),
- })
-
- data, err := jsoniter.Marshal(cmd)
- assert.Nil(t, err)
- rlog.Info("Command Data From Jsoniter", map[string]interface{}{
- "data": *(*string)(unsafe.Pointer(&data)),
- })
-
- var cmdResp RemotingCommand
- err = json.Unmarshal(cmdData, &cmdResp)
- assert.Nil(t, err)
- rlog.Info("Json Decode Success", map[string]interface{}{
- "cmd": cmdResp,
- "language": cmdResp.Language,
- })
-
- var cmdResp2 RemotingCommand
- err = json.Unmarshal(data, &cmdResp2)
- assert.Nil(t, err)
- rlog.Info("Json Decode Success", map[string]interface{}{
- "cmd": cmdResp2,
- "language": cmdResp2.Language,
- })
-}
diff --git a/internal/remote/future.go b/internal/remote/future.go
deleted file mode 100644
index a7d3268..0000000
--- a/internal/remote/future.go
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package remote
-
-import (
- "context"
- "github.com/apache/rocketmq-client-go/v2/errors"
- "sync"
-)
-
-// ResponseFuture
-type ResponseFuture struct {
- ResponseCommand *RemotingCommand
- Err error
- Opaque int32
- callback func(*ResponseFuture)
- Done chan bool
- callbackOnce sync.Once
- ctx context.Context
-}
-
-// NewResponseFuture create ResponseFuture with opaque, timeout and callback
-func NewResponseFuture(ctx context.Context, opaque int32, callback func(*ResponseFuture)) *ResponseFuture {
- return &ResponseFuture{
- Opaque: opaque,
- Done: make(chan bool),
- callback: callback,
- ctx: ctx,
- }
-}
-
-func (r *ResponseFuture) executeInvokeCallback() {
- r.callbackOnce.Do(func() {
- if r.callback != nil {
- r.callback(r)
- }
- })
-}
-
-func (r *ResponseFuture) waitResponse() (*RemotingCommand, error) {
- var (
- cmd *RemotingCommand
- err error
- )
- select {
- case <-r.Done:
- cmd, err = r.ResponseCommand, r.Err
- case <-r.ctx.Done():
- err = errors.ErrRequestTimeout
- r.Err = err
- }
- return cmd, err
-}
diff --git a/internal/remote/interceptor.go b/internal/remote/interceptor.go
deleted file mode 100644
index cf96717..0000000
--- a/internal/remote/interceptor.go
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package remote
-
-import (
- "context"
- "crypto/hmac"
- "crypto/sha1"
- "encoding/base64"
- "hash"
- "sort"
- "strings"
-
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-const (
- signature = "Signature"
- accessKey = "AccessKey"
- securityToken = "SecurityToken"
- keyFile = "KEY_FILE"
- // System.getProperty("rocketmq.client.keyFile", System.getProperty("user.home") + File.separator + "key");
-)
-
-func ACLInterceptor(credentials primitive.Credentials) primitive.Interceptor {
- return func(ctx context.Context, req, reply interface{}, next primitive.Invoker) error {
- cmd := req.(*RemotingCommand)
- m := make(map[string]string)
- order := make([]string, 1)
- m[accessKey] = credentials.AccessKey
- order[0] = accessKey
- if credentials.SecurityToken != "" {
- m[securityToken] = credentials.SecurityToken
- }
- for k, v := range cmd.ExtFields {
- m[k] = v
- order = append(order, k)
- }
- sort.Slice(order, func(i, j int) bool {
- return strings.Compare(order[i], order[j]) < 0
- })
- content := ""
- for idx := range order {
- content += m[order[idx]]
- }
- buf := make([]byte, len(content)+len(cmd.Body))
- copy(buf, []byte(content))
- copy(buf[len(content):], cmd.Body)
-
- cmd.ExtFields[signature] = calculateSignature(buf, []byte(credentials.SecretKey))
- cmd.ExtFields[accessKey] = credentials.AccessKey
-
- // The SecurityToken value is unnecessary, user can choose this one.
- if credentials.SecurityToken != "" {
- cmd.ExtFields[securityToken] = credentials.SecurityToken
- }
- err := next(ctx, req, reply)
- return err
- }
-}
-
-func calculateSignature(data, sk []byte) string {
- mac := hmac.New(func() hash.Hash {
- return sha1.New()
- }, sk)
- mac.Write(data)
- return base64.StdEncoding.EncodeToString(mac.Sum(nil))
-}
diff --git a/internal/remote/interceptor_test.go b/internal/remote/interceptor_test.go
deleted file mode 100644
index c2cc6ca..0000000
--- a/internal/remote/interceptor_test.go
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package remote
-
-import (
- "testing"
-
- "github.com/stretchr/testify/assert"
-)
-
-func Test_CalculateSignature(t *testing.T) {
- assert.Equal(t, "tAb/54Rwwcq+pbH8Loi7FWX4QSQ=",
- calculateSignature([]byte("Hello RocketMQ Client ACL Feature"), []byte("adiaushdiaushd")))
-}
diff --git a/internal/remote/mock_remote_client.go b/internal/remote/mock_remote_client.go
deleted file mode 100644
index 7d7b41c..0000000
--- a/internal/remote/mock_remote_client.go
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-// Code generated by MockGen. DO NOT EDIT.
-// Source: remote_client.go
-
-// Package remote is a generated GoMock package.
-package remote
-
-import (
- context "context"
- reflect "reflect"
-
- primitive "github.com/apache/rocketmq-client-go/v2/primitive"
- gomock "github.com/golang/mock/gomock"
-)
-
-// MockRemotingClient is a mock of RemotingClient interface
-type MockRemotingClient struct {
- ctrl *gomock.Controller
- recorder *MockRemotingClientMockRecorder
-}
-
-// MockRemotingClientMockRecorder is the mock recorder for MockRemotingClient
-type MockRemotingClientMockRecorder struct {
- mock *MockRemotingClient
-}
-
-// NewMockRemotingClient creates a new mock instance
-func NewMockRemotingClient(ctrl *gomock.Controller) *MockRemotingClient {
- mock := &MockRemotingClient{ctrl: ctrl}
- mock.recorder = &MockRemotingClientMockRecorder{mock}
- return mock
-}
-
-// EXPECT returns an object that allows the caller to indicate expected use
-func (m *MockRemotingClient) EXPECT() *MockRemotingClientMockRecorder {
- return m.recorder
-}
-
-// RegisterRequestFunc mocks base method
-func (m *MockRemotingClient) RegisterRequestFunc(code int16, f ClientRequestFunc) {
- m.ctrl.Call(m, "RegisterRequestFunc", code, f)
-}
-
-// RegisterRequestFunc indicates an expected call of RegisterRequestFunc
-func (mr *MockRemotingClientMockRecorder) RegisterRequestFunc(code, f interface{}) *gomock.Call {
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RegisterRequestFunc", reflect.TypeOf((*MockRemotingClient)(nil).RegisterRequestFunc), code, f)
-}
-
-// RegisterInterceptor mocks base method
-func (m *MockRemotingClient) RegisterInterceptor(interceptors ...primitive.Interceptor) {
- varargs := []interface{}{}
- for _, a := range interceptors {
- varargs = append(varargs, a)
- }
- m.ctrl.Call(m, "RegisterInterceptor", varargs...)
-}
-
-// RegisterInterceptor indicates an expected call of RegisterInterceptor
-func (mr *MockRemotingClientMockRecorder) RegisterInterceptor(interceptors ...interface{}) *gomock.Call {
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RegisterInterceptor", reflect.TypeOf((*MockRemotingClient)(nil).RegisterInterceptor), interceptors...)
-}
-
-// InvokeSync mocks base method
-func (m *MockRemotingClient) InvokeSync(ctx context.Context, addr string, request *RemotingCommand) (*RemotingCommand, error) {
- ret := m.ctrl.Call(m, "InvokeSync", ctx, addr, request)
- ret0, _ := ret[0].(*RemotingCommand)
- ret1, _ := ret[1].(error)
- return ret0, ret1
-}
-
-// InvokeSync indicates an expected call of InvokeSync
-func (mr *MockRemotingClientMockRecorder) InvokeSync(ctx, addr, request interface{}) *gomock.Call {
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InvokeSync", reflect.TypeOf((*MockRemotingClient)(nil).InvokeSync), ctx, addr, request)
-}
-
-// InvokeAsync mocks base method
-func (m *MockRemotingClient) InvokeAsync(ctx context.Context, addr string, request *RemotingCommand, callback func(*ResponseFuture)) error {
- ret := m.ctrl.Call(m, "InvokeAsync", ctx, addr, request, callback)
- ret0, _ := ret[0].(error)
- return ret0
-}
-
-// InvokeAsync indicates an expected call of InvokeAsync
-func (mr *MockRemotingClientMockRecorder) InvokeAsync(ctx, addr, request, callback interface{}) *gomock.Call {
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InvokeAsync", reflect.TypeOf((*MockRemotingClient)(nil).InvokeAsync), ctx, addr, request, callback)
-}
-
-// InvokeOneWay mocks base method
-func (m *MockRemotingClient) InvokeOneWay(ctx context.Context, addr string, request *RemotingCommand) error {
- ret := m.ctrl.Call(m, "InvokeOneWay", ctx, addr, request)
- ret0, _ := ret[0].(error)
- return ret0
-}
-
-// InvokeOneWay indicates an expected call of InvokeOneWay
-func (mr *MockRemotingClientMockRecorder) InvokeOneWay(ctx, addr, request interface{}) *gomock.Call {
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InvokeOneWay", reflect.TypeOf((*MockRemotingClient)(nil).InvokeOneWay), ctx, addr, request)
-}
-
-// ShutDown mocks base method
-func (m *MockRemotingClient) ShutDown() {
- m.ctrl.Call(m, "ShutDown")
-}
-
-// ShutDown indicates an expected call of ShutDown
-func (mr *MockRemotingClientMockRecorder) ShutDown() *gomock.Call {
- return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ShutDown", reflect.TypeOf((*MockRemotingClient)(nil).ShutDown))
-}
diff --git a/internal/remote/remote_client.go b/internal/remote/remote_client.go
deleted file mode 100644
index 33b4633..0000000
--- a/internal/remote/remote_client.go
+++ /dev/null
@@ -1,316 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package remote
-
-import (
- "bufio"
- "bytes"
- "context"
- "encoding/binary"
- "io"
- "net"
- "sync"
-
- "github.com/apache/rocketmq-client-go/v2/primitive"
-
- "github.com/apache/rocketmq-client-go/v2/rlog"
-)
-
-type ClientRequestFunc func(*RemotingCommand, net.Addr) *RemotingCommand
-
-type TcpOption struct {
- // TODO
-}
-
-//go:generate mockgen -source remote_client.go -destination mock_remote_client.go -self_package github.com/apache/rocketmq-client-go/v2/internal/remote --package remote RemotingClient
-type RemotingClient interface {
- RegisterRequestFunc(code int16, f ClientRequestFunc)
- RegisterInterceptor(interceptors ...primitive.Interceptor)
- InvokeSync(ctx context.Context, addr string, request *RemotingCommand) (*RemotingCommand, error)
- InvokeAsync(ctx context.Context, addr string, request *RemotingCommand, callback func(*ResponseFuture)) error
- InvokeOneWay(ctx context.Context, addr string, request *RemotingCommand) error
- ShutDown()
-}
-
-var _ RemotingClient = &remotingClient{}
-
-type remotingClient struct {
- responseTable sync.Map
- connectionTable sync.Map
- option TcpOption
- processors map[int16]ClientRequestFunc
- connectionLocker sync.Mutex
- interceptor primitive.Interceptor
-}
-
-func NewRemotingClient() *remotingClient {
- return &remotingClient{
- processors: make(map[int16]ClientRequestFunc),
- }
-}
-
-func (c *remotingClient) RegisterRequestFunc(code int16, f ClientRequestFunc) {
- c.processors[code] = f
-}
-
-// TODO: merge sync and async model. sync should run on async model by blocking on chan
-func (c *remotingClient) InvokeSync(ctx context.Context, addr string, request *RemotingCommand) (*RemotingCommand, error) {
- conn, err := c.connect(ctx, addr)
- if err != nil {
- return nil, err
- }
- resp := NewResponseFuture(ctx, request.Opaque, nil)
- c.responseTable.Store(resp.Opaque, resp)
- defer c.responseTable.Delete(request.Opaque)
- err = c.sendRequest(conn, request)
- if err != nil {
- return nil, err
- }
- return resp.waitResponse()
-}
-
-// InvokeAsync send request without blocking, just return immediately.
-func (c *remotingClient) InvokeAsync(ctx context.Context, addr string, request *RemotingCommand, callback func(*ResponseFuture)) error {
- conn, err := c.connect(ctx, addr)
- if err != nil {
- return err
- }
- resp := NewResponseFuture(ctx, request.Opaque, callback)
- c.responseTable.Store(resp.Opaque, resp)
- err = c.sendRequest(conn, request)
- if err != nil {
- return err
- }
- go primitive.WithRecover(func() {
- c.receiveAsync(resp)
- })
- return nil
-}
-
-func (c *remotingClient) receiveAsync(f *ResponseFuture) {
- _, err := f.waitResponse()
- if err != nil {
- f.executeInvokeCallback()
- }
-}
-
-func (c *remotingClient) InvokeOneWay(ctx context.Context, addr string, request *RemotingCommand) error {
- conn, err := c.connect(ctx, addr)
- if err != nil {
- return err
- }
- return c.sendRequest(conn, request)
-}
-
-func (c *remotingClient) connect(ctx context.Context, addr string) (*tcpConnWrapper, error) {
- //it needs additional locker.
- c.connectionLocker.Lock()
- defer c.connectionLocker.Unlock()
- conn, ok := c.connectionTable.Load(addr)
- if ok {
- return conn.(*tcpConnWrapper), nil
- }
- tcpConn, err := initConn(ctx, addr)
- if err != nil {
- return nil, err
- }
- c.connectionTable.Store(addr, tcpConn)
- go primitive.WithRecover(func() {
- c.receiveResponse(tcpConn)
- })
- return tcpConn, nil
-}
-
-func (c *remotingClient) receiveResponse(r *tcpConnWrapper) {
- var err error
- header := primitive.GetHeader()
- defer primitive.BackHeader(header)
- for {
- if err != nil {
- // conn has been closed actively
- if r.isClosed(err) {
- return
- }
- if err != io.EOF {
- rlog.Error("conn error, close connection", map[string]interface{}{
- rlog.LogKeyUnderlayError: err,
- })
- }
- c.closeConnection(r)
- r.destroy()
- break
- }
-
- _, err = io.ReadFull(r, *header)
- if err != nil {
- continue
- }
-
- var length int32
- err = binary.Read(bytes.NewReader(*header), binary.BigEndian, &length)
- if err != nil {
- continue
- }
-
- buf := make([]byte, length)
-
- _, err = io.ReadFull(r, buf)
- if err != nil {
- continue
- }
-
- cmd, err := decode(buf)
- if err != nil {
- rlog.Error("decode RemotingCommand error", map[string]interface{}{
- rlog.LogKeyUnderlayError: err,
- })
- continue
- }
- c.processCMD(cmd, r)
- }
-}
-
-func (c *remotingClient) processCMD(cmd *RemotingCommand, r *tcpConnWrapper) {
- if cmd.isResponseType() {
- resp, exist := c.responseTable.Load(cmd.Opaque)
- if exist {
- c.responseTable.Delete(cmd.Opaque)
- responseFuture := resp.(*ResponseFuture)
- go primitive.WithRecover(func() {
- responseFuture.ResponseCommand = cmd
- responseFuture.executeInvokeCallback()
- if responseFuture.Done != nil {
- close(responseFuture.Done)
- }
- })
- }
- } else {
- f := c.processors[cmd.Code]
- if f != nil {
- // single goroutine will be deadlock
- // TODO: optimize with goroutine pool, https://github.com/apache/rocketmq-client-go/v2/issues/307
- go primitive.WithRecover(func() {
- res := f(cmd, r.RemoteAddr())
- if res != nil {
- res.Opaque = cmd.Opaque
- res.Flag |= 1 << 0
- err := c.sendRequest(r, res)
- if err != nil {
- rlog.Warning("send response to broker error", map[string]interface{}{
- rlog.LogKeyUnderlayError: err,
- "responseCode": res.Code,
- })
- }
- }
- })
- } else {
- rlog.Warning("receive broker's requests, but no func to handle", map[string]interface{}{
- "responseCode": cmd.Code,
- })
- }
- }
-}
-
-func (c *remotingClient) createScanner(r io.Reader) *bufio.Scanner {
- scanner := bufio.NewScanner(r)
-
- // max batch size: 32, max message size: 4Mb
- scanner.Buffer(make([]byte, 1024*1024), 128*1024*1024)
- scanner.Split(func(data []byte, atEOF bool) (int, []byte, error) {
- defer func() {
- if err := recover(); err != nil {
- rlog.Error("scanner split panic", map[string]interface{}{
- "panic": err,
- })
- }
- }()
- if !atEOF {
- if len(data) >= 4 {
- var length int32
- err := binary.Read(bytes.NewReader(data[0:4]), binary.BigEndian, &length)
- if err != nil {
- rlog.Error("split data error", map[string]interface{}{
- rlog.LogKeyUnderlayError: err,
- })
- return 0, nil, err
- }
-
- if int(length)+4 <= len(data) {
- return int(length) + 4, data[4 : length+4], nil
- }
- }
- }
- return 0, nil, nil
- })
- return scanner
-}
-
-func (c *remotingClient) sendRequest(conn *tcpConnWrapper, request *RemotingCommand) error {
- var err error
- if c.interceptor != nil {
- err = c.interceptor(context.Background(), request, nil, func(ctx context.Context, req, reply interface{}) error {
- return c.doRequest(conn, request)
- })
- } else {
- err = c.doRequest(conn, request)
- }
- return err
-}
-
-func (c *remotingClient) doRequest(conn *tcpConnWrapper, request *RemotingCommand) error {
- conn.Lock()
- defer conn.Unlock()
- err := request.WriteTo(conn)
- if err != nil {
- c.closeConnection(conn)
- return err
- }
- return nil
-}
-
-func (c *remotingClient) closeConnection(toCloseConn *tcpConnWrapper) {
- c.connectionTable.Range(func(key, value interface{}) bool {
- if value == toCloseConn {
- c.connectionTable.Delete(key)
- return false
- } else {
- return true
- }
- })
-}
-
-func (c *remotingClient) ShutDown() {
- c.responseTable.Range(func(key, value interface{}) bool {
- c.responseTable.Delete(key)
- return true
- })
- c.connectionTable.Range(func(key, value interface{}) bool {
- conn := value.(*tcpConnWrapper)
- err := conn.destroy()
- if err != nil {
- rlog.Warning("close remoting conn error", map[string]interface{}{
- "remote": conn.RemoteAddr(),
- rlog.LogKeyUnderlayError: err,
- })
- }
- return true
- })
-}
-
-func (c *remotingClient) RegisterInterceptor(interceptors ...primitive.Interceptor) {
- c.interceptor = primitive.ChainInterceptors(interceptors...)
-}
diff --git a/internal/remote/remote_client_test.go b/internal/remote/remote_client_test.go
deleted file mode 100644
index fa33f4f..0000000
--- a/internal/remote/remote_client_test.go
+++ /dev/null
@@ -1,372 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package remote
-
-import (
- "bytes"
- "context"
- "github.com/apache/rocketmq-client-go/v2/errors"
- "math/rand"
- "net"
- "reflect"
- "sync"
- "testing"
- "time"
-
- "github.com/stretchr/testify/assert"
-)
-
-func TestNewResponseFuture(t *testing.T) {
- future := NewResponseFuture(context.Background(), 10, nil)
- if future.Opaque != 10 {
- t.Errorf("wrong ResponseFuture's opaque. want=%d, got=%d", 10, future.Opaque)
- }
- if future.Err != nil {
- t.Errorf("wrong RespnseFuture's Err. want=<nil>, got=%v", future.Err)
- }
- if future.callback != nil {
- t.Errorf("wrong ResponseFuture's callback. want=<nil>, got!=<nil>")
- }
- if future.Done == nil {
- t.Errorf("wrong ResponseFuture's done. want=<channel>, got=<nil>")
- }
-}
-
-func TestResponseFutureTimeout(t *testing.T) {
- callback := func(r *ResponseFuture) {
- if r.ResponseCommand.Remark == "" {
- r.ResponseCommand.Remark = "Hello RocketMQ."
- } else {
- r.ResponseCommand.Remark = r.ResponseCommand.Remark + "Go Client"
- }
- }
- future := NewResponseFuture(context.Background(), 10, callback)
- future.ResponseCommand = NewRemotingCommand(200,
- nil, nil)
-
- var wg sync.WaitGroup
- wg.Add(10)
- for i := 0; i < 10; i++ {
- go func() {
- future.executeInvokeCallback()
- wg.Done()
- }()
- }
- wg.Wait()
- if future.ResponseCommand.Remark != "Hello RocketMQ." {
- t.Errorf("wrong ResponseFuture.ResponseCommand.Remark. want=%s, got=%s",
- "Hello RocketMQ.", future.ResponseCommand.Remark)
- }
-
-}
-
-func TestResponseFutureWaitResponse(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Duration(1000))
- defer cancel()
- future := NewResponseFuture(ctx, 10, nil)
- if _, err := future.waitResponse(); err != errors.ErrRequestTimeout {
- t.Errorf("wrong ResponseFuture waitResponse. want=%v, got=%v",
- errors.ErrRequestTimeout, err)
- }
- future = NewResponseFuture(context.Background(), 10, nil)
- responseError := errors.ErrResponse
- go func() {
- time.Sleep(100 * time.Millisecond)
- future.Err = responseError
- future.Done <- true
- }()
- if _, err := future.waitResponse(); err != responseError {
- t.Errorf("wrong ResponseFuture waitResponse. want=%v. got=%v",
- responseError, err)
- }
- future = NewResponseFuture(context.Background(), 10, nil)
- responseRemotingCommand := NewRemotingCommand(202, nil, nil)
- go func() {
- time.Sleep(100 * time.Millisecond)
- future.ResponseCommand = responseRemotingCommand
- future.Done <- true
- }()
- if r, err := future.waitResponse(); err != nil {
- t.Errorf("wrong ResponseFuture waitResponse error: %v", err)
- } else {
- if r != responseRemotingCommand {
- t.Errorf("wrong ResponseFuture waitResposne result. want=%v, got=%v",
- responseRemotingCommand, r)
- }
- }
-}
-
-func TestCreateScanner(t *testing.T) {
- r := randomNewRemotingCommand()
- content, err := encode(r)
- if err != nil {
- t.Fatalf("failed to encode RemotingCommand. %s", err)
- }
- client := NewRemotingClient()
- reader := bytes.NewReader(content)
- scanner := client.createScanner(reader)
- for scanner.Scan() {
- rcr, err := decode(scanner.Bytes())
- if err != nil {
- t.Fatalf("failedd to decode RemotingCommand from scanner")
- }
- if r.Code != rcr.Code {
- t.Fatalf("wrong Code. want=%d, got=%d", r.Code, rcr.Code)
- }
- if r.Version != rcr.Version {
- t.Fatalf("wrong Version. want=%d, got=%d", r.Version, rcr.Version)
- }
- if r.Opaque != rcr.Opaque {
- t.Fatalf("wrong opaque. want=%d, got=%d", r.Opaque, rcr.Opaque)
- }
- if r.Flag != rcr.Flag {
- t.Fatalf("wrong flag. want=%d, got=%d", r.Opaque, rcr.Opaque)
- }
- if !reflect.DeepEqual(r.ExtFields, rcr.ExtFields) {
- t.Fatalf("wrong extFields. want=%v, got=%v", r.ExtFields, rcr.ExtFields)
- }
- }
-}
-
-func TestInvokeSync(t *testing.T) {
- addr := ":3004"
-
- clientSendRemtingCommand := NewRemotingCommand(10, nil, []byte("Hello RocketMQ"))
- serverSendRemotingCommand := NewRemotingCommand(20, nil, []byte("Welcome native"))
- serverSendRemotingCommand.Opaque = clientSendRemtingCommand.Opaque
- serverSendRemotingCommand.Flag = ResponseType
- var wg sync.WaitGroup
- wg.Add(1)
- client := NewRemotingClient()
-
- var clientSend sync.WaitGroup // blocking client send message until the server listen success.
- clientSend.Add(1)
-
- go func() {
- clientSend.Wait()
- receiveCommand, err := client.InvokeSync(context.Background(), addr,
- clientSendRemtingCommand)
- if err != nil {
- t.Fatalf("failed to invoke synchronous. %s", err)
- } else {
- assert.Equal(t, len(receiveCommand.ExtFields), 0)
- assert.Equal(t, len(serverSendRemotingCommand.ExtFields), 0)
- // in order to avoid the difference of ExtFields between the receiveCommand and serverSendRemotingCommand
- // the ExtFields in receiveCommand is map[string]string(nil), but serverSendRemotingCommand is map[string]string{}
- receiveCommand.ExtFields = nil
- serverSendRemotingCommand.ExtFields = nil
- assert.Equal(t, receiveCommand, serverSendRemotingCommand, "remotingCommand prased in client is different from server.")
- }
- wg.Done()
- }()
-
- l, err := net.Listen("tcp", addr)
- if err != nil {
- t.Fatal(err)
- }
- defer l.Close()
- clientSend.Done()
- for {
- conn, err := l.Accept()
- if err != nil {
- return
- }
- defer conn.Close()
- scanner := client.createScanner(conn)
- for scanner.Scan() {
- receivedRemotingCommand, err := decode(scanner.Bytes())
- if err != nil {
- t.Errorf("failed to decode RemotingCommnad. %s", err)
- }
- if clientSendRemtingCommand.Code != receivedRemotingCommand.Code {
- t.Errorf("wrong code. want=%d, got=%d", receivedRemotingCommand.Code,
- clientSendRemtingCommand.Code)
- }
- body, err := encode(serverSendRemotingCommand)
- if err != nil {
- t.Fatalf("failed to encode RemotingCommand")
- }
- _, err = conn.Write(body)
- if err != nil {
- t.Fatalf("failed to write body to conneciton.")
- }
- goto done
- }
- }
-done:
- wg.Wait()
-}
-
-func TestInvokeAsync(t *testing.T) {
- addr := ":3006"
- var wg sync.WaitGroup
- cnt := 50
- wg.Add(cnt)
- client := NewRemotingClient()
- for i := 0; i < cnt; i++ {
- go func(index int) {
- time.Sleep(time.Duration(rand.Intn(100)) * time.Millisecond)
- t.Logf("[Send: %d] asychronous message", index)
- sendRemotingCommand := randomNewRemotingCommand()
- err := client.InvokeAsync(context.Background(), addr, sendRemotingCommand, func(r *ResponseFuture) {
- t.Logf("[Receive: %d] asychronous message response", index)
- if string(sendRemotingCommand.Body) != string(r.ResponseCommand.Body) {
- t.Errorf("wrong response message. want=%s, got=%s", string(sendRemotingCommand.Body),
- string(r.ResponseCommand.Body))
- }
- wg.Done()
- })
- if err != nil {
- t.Errorf("failed to invokeAsync. %s", err)
- }
-
- }(i)
- }
- l, err := net.Listen("tcp", addr)
- if err != nil {
- t.Fatalf("failed to create tcp network. %s", err)
- }
- defer l.Close()
- count := 0
- for {
- conn, err := l.Accept()
- if err != nil {
- t.Fatalf("failed to create connection. %s", err)
- }
- defer conn.Close()
- scanner := client.createScanner(conn)
- for scanner.Scan() {
- t.Log("receive request")
- r, err := decode(scanner.Bytes())
- if err != nil {
- t.Errorf("failed to decode RemotingCommand %s", err)
- }
- r.markResponseType()
- body, _ := encode(r)
- _, err = conn.Write(body)
- if err != nil {
- t.Fatalf("failed to send response %s", err)
- }
- count++
- if count >= cnt {
- goto done
- }
- }
- }
-done:
-
- wg.Wait()
-}
-
-func TestInvokeAsyncTimeout(t *testing.T) {
- addr := ":3002"
-
- clientSendRemtingCommand := NewRemotingCommand(10, nil, []byte("Hello RocketMQ"))
- serverSendRemotingCommand := NewRemotingCommand(20, nil, []byte("Welcome native"))
- serverSendRemotingCommand.Opaque = clientSendRemtingCommand.Opaque
- serverSendRemotingCommand.Flag = ResponseType
-
- var wg sync.WaitGroup
- wg.Add(1)
- client := NewRemotingClient()
-
- var clientSend sync.WaitGroup // blocking client send message until the server listen success.
- clientSend.Add(1)
- go func() {
- clientSend.Wait()
- ctx, cancel := context.WithTimeout(context.Background(), time.Duration(10*time.Second))
- defer cancel()
- err := client.InvokeAsync(ctx, addr, clientSendRemtingCommand,
- func(r *ResponseFuture) {
- assert.NotNil(t, r.Err)
- assert.Equal(t, errors.ErrRequestTimeout, r.Err)
- wg.Done()
- })
- assert.Nil(t, err, "failed to invokeSync.")
- }()
-
- l, err := net.Listen("tcp", addr)
- assert.Nil(t, err)
- defer l.Close()
- clientSend.Done()
-
- for {
- conn, err := l.Accept()
- assert.Nil(t, err)
- defer conn.Close()
-
- scanner := client.createScanner(conn)
- for scanner.Scan() {
- t.Logf("receive request.")
- _, err := decode(scanner.Bytes())
- assert.Nil(t, err, "failed to decode RemotingCommnad.")
-
- time.Sleep(5 * time.Second) // force client timeout
- goto done
- }
- }
-done:
- wg.Wait()
-}
-
-func TestInvokeOneWay(t *testing.T) {
- addr := ":3008"
- clientSendRemtingCommand := NewRemotingCommand(10, nil, []byte("Hello RocketMQ"))
-
- var wg sync.WaitGroup
- wg.Add(1)
- client := NewRemotingClient()
-
- var clientSend sync.WaitGroup // blocking client send message until the server listen success.
- clientSend.Add(1)
- go func() {
- clientSend.Wait()
- err := client.InvokeOneWay(context.Background(), addr, clientSendRemtingCommand)
- if err != nil {
- t.Fatalf("failed to invoke synchronous. %s", err)
- }
- wg.Done()
- }()
-
- l, err := net.Listen("tcp", addr)
- if err != nil {
- t.Fatal(err)
- }
- defer l.Close()
- clientSend.Done()
- for {
- conn, err := l.Accept()
- if err != nil {
- return
- }
- defer conn.Close()
- scanner := client.createScanner(conn)
- for scanner.Scan() {
- receivedRemotingCommand, err := decode(scanner.Bytes())
- if err != nil {
- t.Errorf("failed to decode RemotingCommnad. %s", err)
- }
- if clientSendRemtingCommand.Code != receivedRemotingCommand.Code {
- t.Errorf("wrong code. want=%d, got=%d", receivedRemotingCommand.Code,
- clientSendRemtingCommand.Code)
- }
- goto done
- }
- }
-done:
- wg.Wait()
-}
diff --git a/internal/remote/rpchook.go b/internal/remote/rpchook.go
deleted file mode 100644
index a95391d..0000000
--- a/internal/remote/rpchook.go
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package remote
-
-type RPCHook interface {
- DoBeforeRequest(string, *RemotingCommand)
- DoAfterResponse(string, *RemotingCommand)
-}
diff --git a/internal/remote/tcp_conn.go b/internal/remote/tcp_conn.go
deleted file mode 100644
index ae340c6..0000000
--- a/internal/remote/tcp_conn.go
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package remote
-
-import (
- "context"
- "net"
- "sync"
-
- "go.uber.org/atomic"
-)
-
-// TODO: Adding TCP Connections Pool, https://github.com/apache/rocketmq-client-go/v2/issues/298
-type tcpConnWrapper struct {
- net.Conn
- sync.Mutex
- closed atomic.Bool
-}
-
-func initConn(ctx context.Context, addr string) (*tcpConnWrapper, error) {
- var d net.Dialer
- conn, err := d.DialContext(ctx, "tcp", addr)
- if err != nil {
- return nil, err
- }
- return &tcpConnWrapper{
- Conn: conn,
- }, nil
-}
-
-func (wrapper *tcpConnWrapper) destroy() error {
- wrapper.closed.Swap(true)
- return wrapper.Conn.Close()
-}
-
-func (wrapper *tcpConnWrapper) isClosed(err error) bool {
- if !wrapper.closed.Load() {
- return false
- }
-
- opErr, ok := err.(*net.OpError)
- if !ok {
- return false
- }
-
- return opErr.Err.Error() == "use of closed network connection"
-}
diff --git a/internal/request.go b/internal/request.go
deleted file mode 100644
index 0e3d8e1..0000000
--- a/internal/request.go
+++ /dev/null
@@ -1,480 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package internal
-
-import (
- "fmt"
- "strconv"
- "time"
-)
-
-const (
- ReqSendMessage = int16(10)
- ReqPullMessage = int16(11)
- ReqQueryMessage = int16(12)
- ReqQueryConsumerOffset = int16(14)
- ReqUpdateConsumerOffset = int16(15)
- ReqCreateTopic = int16(17)
- ReqSearchOffsetByTimestamp = int16(29)
- ReqGetMaxOffset = int16(30)
- ReqGetMinOffset = int16(31)
- ReqViewMessageByID = int16(33)
- ReqHeartBeat = int16(34)
- ReqConsumerSendMsgBack = int16(36)
- ReqENDTransaction = int16(37)
- ReqGetConsumerListByGroup = int16(38)
- ReqLockBatchMQ = int16(41)
- ReqUnlockBatchMQ = int16(42)
- ReqGetRouteInfoByTopic = int16(105)
- ReqGetBrokerClusterInfo = int16(106)
- ReqSendBatchMessage = int16(320)
- ReqCheckTransactionState = int16(39)
- ReqNotifyConsumerIdsChanged = int16(40)
- ReqGetAllTopicListFromNameServer = int16(206)
- ReqDeleteTopicInBroker = int16(215)
- ReqDeleteTopicInNameSrv = int16(216)
- ReqResetConsumerOffset = int16(220)
- ReqGetConsumerRunningInfo = int16(307)
- ReqConsumeMessageDirectly = int16(309)
-)
-
-type SendMessageRequestHeader struct {
- ProducerGroup string
- Topic string
- QueueId int
- SysFlag int
- BornTimestamp int64
- Flag int32
- Properties string
- ReconsumeTimes int
- UnitMode bool
- MaxReconsumeTimes int
- Batch bool
- DefaultTopic string
- DefaultTopicQueueNums int
-}
-
-func (request *SendMessageRequestHeader) Encode() map[string]string {
- maps := make(map[string]string)
- maps["producerGroup"] = request.ProducerGroup
- maps["topic"] = request.Topic
- maps["queueId"] = strconv.Itoa(request.QueueId)
- maps["sysFlag"] = fmt.Sprintf("%d", request.SysFlag)
- maps["bornTimestamp"] = strconv.FormatInt(request.BornTimestamp, 10)
- maps["flag"] = fmt.Sprintf("%d", request.Flag)
- maps["reconsumeTimes"] = strconv.Itoa(request.ReconsumeTimes)
- maps["unitMode"] = strconv.FormatBool(request.UnitMode)
- maps["maxReconsumeTimes"] = strconv.Itoa(request.MaxReconsumeTimes)
- maps["defaultTopic"] = "TBW102"
- maps["defaultTopicQueueNums"] = "4"
- maps["batch"] = strconv.FormatBool(request.Batch)
- maps["properties"] = request.Properties
-
- return maps
-}
-
-type EndTransactionRequestHeader struct {
- ProducerGroup string
- TranStateTableOffset int64
- CommitLogOffset int64
- CommitOrRollback int
- FromTransactionCheck bool
- MsgID string
- TransactionId string
-}
-
-type SendMessageRequestV2Header struct {
- *SendMessageRequestHeader
-}
-
-func (request *SendMessageRequestV2Header) Encode() map[string]string {
- maps := make(map[string]string)
- maps["a"] = request.ProducerGroup
- maps["b"] = request.Topic
- maps["c"] = request.DefaultTopic
- maps["d"] = strconv.Itoa(request.DefaultTopicQueueNums)
- maps["e"] = strconv.Itoa(request.QueueId)
- maps["f"] = fmt.Sprintf("%d", request.SysFlag)
- maps["g"] = strconv.FormatInt(request.BornTimestamp, 10)
- maps["h"] = fmt.Sprintf("%d", request.Flag)
- maps["i"] = request.Properties
- maps["j"] = strconv.Itoa(request.ReconsumeTimes)
- maps["k"] = strconv.FormatBool(request.UnitMode)
- maps["l"] = strconv.Itoa(request.MaxReconsumeTimes)
- maps["m"] = strconv.FormatBool(request.Batch)
- return maps
-}
-
-func (request *EndTransactionRequestHeader) Encode() map[string]string {
- maps := make(map[string]string)
- maps["producerGroup"] = request.ProducerGroup
- maps["tranStateTableOffset"] = strconv.FormatInt(request.TranStateTableOffset, 10)
- maps["commitLogOffset"] = strconv.Itoa(int(request.CommitLogOffset))
- maps["commitOrRollback"] = strconv.Itoa(request.CommitOrRollback)
- maps["fromTransactionCheck"] = strconv.FormatBool(request.FromTransactionCheck)
- maps["msgId"] = request.MsgID
- maps["transactionId"] = request.TransactionId
- return maps
-}
-
-type CheckTransactionStateRequestHeader struct {
- TranStateTableOffset int64
- CommitLogOffset int64
- MsgId string
- TransactionId string
- OffsetMsgId string
-}
-
-func (request *CheckTransactionStateRequestHeader) Encode() map[string]string {
- maps := make(map[string]string)
- maps["tranStateTableOffset"] = strconv.FormatInt(request.TranStateTableOffset, 10)
- maps["commitLogOffset"] = strconv.FormatInt(request.CommitLogOffset, 10)
- maps["msgId"] = request.MsgId
- maps["transactionId"] = request.TransactionId
- maps["offsetMsgId"] = request.OffsetMsgId
-
- return maps
-}
-
-func (request *CheckTransactionStateRequestHeader) Decode(properties map[string]string) {
- if len(properties) == 0 {
- return
- }
- if v, existed := properties["tranStateTableOffset"]; existed {
- request.TranStateTableOffset, _ = strconv.ParseInt(v, 10, 0)
- }
- if v, existed := properties["commitLogOffset"]; existed {
- request.CommitLogOffset, _ = strconv.ParseInt(v, 10, 0)
- }
- if v, existed := properties["msgId"]; existed {
- request.MsgId = v
- }
- if v, existed := properties["transactionId"]; existed {
- request.MsgId = v
- }
- if v, existed := properties["offsetMsgId"]; existed {
- request.MsgId = v
- }
-}
-
-type ConsumerSendMsgBackRequestHeader struct {
- Group string
- Offset int64
- DelayLevel int
- OriginMsgId string
- OriginTopic string
- UnitMode bool
- MaxReconsumeTimes int32
-}
-
-func (request *ConsumerSendMsgBackRequestHeader) Encode() map[string]string {
- maps := make(map[string]string)
- maps["group"] = request.Group
- maps["offset"] = strconv.FormatInt(request.Offset, 10)
- maps["delayLevel"] = strconv.Itoa(request.DelayLevel)
- maps["originMsgId"] = request.OriginMsgId
- maps["originTopic"] = request.OriginTopic
- maps["unitMode"] = strconv.FormatBool(request.UnitMode)
- maps["maxReconsumeTimes"] = strconv.Itoa(int(request.MaxReconsumeTimes))
-
- return maps
-}
-
-type PullMessageRequestHeader struct {
- ConsumerGroup string
- Topic string
- QueueId int32
- QueueOffset int64
- MaxMsgNums int32
- SysFlag int32
- CommitOffset int64
- SuspendTimeoutMillis time.Duration
- SubExpression string
- SubVersion int64
- ExpressionType string
-}
-
-func (request *PullMessageRequestHeader) Encode() map[string]string {
- maps := make(map[string]string)
- maps["consumerGroup"] = request.ConsumerGroup
- maps["topic"] = request.Topic
- maps["queueId"] = fmt.Sprintf("%d", request.QueueId)
- maps["queueOffset"] = fmt.Sprintf("%d", request.QueueOffset)
- maps["maxMsgNums"] = fmt.Sprintf("%d", request.MaxMsgNums)
- maps["sysFlag"] = fmt.Sprintf("%d", request.SysFlag)
- maps["commitOffset"] = fmt.Sprintf("%d", request.CommitOffset)
- maps["suspendTimeoutMillis"] = fmt.Sprintf("%d", request.SuspendTimeoutMillis/time.Millisecond)
- maps["subscription"] = request.SubExpression
- maps["subVersion"] = fmt.Sprintf("%d", request.SubVersion)
- maps["expressionType"] = request.ExpressionType
-
- return maps
-}
-
-type GetConsumerListRequestHeader struct {
- ConsumerGroup string `json:"consumerGroup"`
-}
-
-func (request *GetConsumerListRequestHeader) Encode() map[string]string {
- maps := make(map[string]string)
- maps["consumerGroup"] = request.ConsumerGroup
- return maps
-}
-
-type GetMaxOffsetRequestHeader struct {
- Topic string
- QueueId int
-}
-
-func (request *GetMaxOffsetRequestHeader) Encode() map[string]string {
- maps := make(map[string]string)
- maps["topic"] = request.Topic
- maps["queueId"] = strconv.Itoa(request.QueueId)
- return maps
-}
-
-type QueryConsumerOffsetRequestHeader struct {
- ConsumerGroup string
- Topic string
- QueueId int
-}
-
-func (request *QueryConsumerOffsetRequestHeader) Encode() map[string]string {
- maps := make(map[string]string)
- maps["consumerGroup"] = request.ConsumerGroup
- maps["topic"] = request.Topic
- maps["queueId"] = strconv.Itoa(request.QueueId)
- return maps
-}
-
-type SearchOffsetRequestHeader struct {
- Topic string
- QueueId int
- Timestamp int64
-}
-
-func (request *SearchOffsetRequestHeader) Encode() map[string]string {
- maps := make(map[string]string)
- maps["topic"] = request.Topic
- maps["queueId"] = strconv.Itoa(request.QueueId)
- maps["timestamp"] = strconv.FormatInt(request.Timestamp, 10)
- return maps
-}
-
-type UpdateConsumerOffsetRequestHeader struct {
- ConsumerGroup string
- Topic string
- QueueId int
- CommitOffset int64
-}
-
-func (request *UpdateConsumerOffsetRequestHeader) Encode() map[string]string {
- maps := make(map[string]string)
- maps["consumerGroup"] = request.ConsumerGroup
- maps["topic"] = request.Topic
- maps["queueId"] = strconv.Itoa(request.QueueId)
- maps["commitOffset"] = strconv.FormatInt(request.CommitOffset, 10)
- return maps
-}
-
-type GetRouteInfoRequestHeader struct {
- Topic string
-}
-
-func (request *GetRouteInfoRequestHeader) Encode() map[string]string {
- maps := make(map[string]string)
- maps["topic"] = request.Topic
- return maps
-}
-
-type GetConsumerRunningInfoHeader struct {
- consumerGroup string
- clientID string
-}
-
-func (request *GetConsumerRunningInfoHeader) Encode() map[string]string {
- maps := make(map[string]string)
- maps["consumerGroup"] = request.consumerGroup
- maps["clientId"] = request.clientID
- return maps
-}
-
-func (request *GetConsumerRunningInfoHeader) Decode(properties map[string]string) {
- if len(properties) == 0 {
- return
- }
- if v, existed := properties["consumerGroup"]; existed {
- request.consumerGroup = v
- }
-
- if v, existed := properties["clientId"]; existed {
- request.clientID = v
- }
-}
-
-type QueryMessageRequestHeader struct {
- Topic string
- Key string
- MaxNum int
- BeginTimestamp int64
- EndTimestamp int64
-}
-
-func (request *QueryMessageRequestHeader) Encode() map[string]string {
- maps := make(map[string]string)
- maps["topic"] = request.Topic
- maps["key"] = request.Key
- maps["maxNum"] = fmt.Sprintf("%d", request.MaxNum)
- maps["beginTimestamp"] = strconv.FormatInt(request.BeginTimestamp, 10)
- maps["endTimestamp"] = fmt.Sprintf("%d", request.EndTimestamp)
-
- return maps
-}
-
-func (request *QueryMessageRequestHeader) Decode(properties map[string]string) error {
- return nil
-}
-
-type ViewMessageRequestHeader struct {
- Offset int64
-}
-
-func (request *ViewMessageRequestHeader) Encode() map[string]string {
- maps := make(map[string]string)
- maps["offset"] = strconv.FormatInt(request.Offset, 10)
-
- return maps
-}
-
-type CreateTopicRequestHeader struct {
- Topic string
- DefaultTopic string
- ReadQueueNums int
- WriteQueueNums int
- Perm int
- TopicFilterType string
- TopicSysFlag int
- Order bool
-}
-
-func (request *CreateTopicRequestHeader) Encode() map[string]string {
- maps := make(map[string]string)
- maps["topic"] = request.Topic
- maps["defaultTopic"] = request.DefaultTopic
- maps["readQueueNums"] = fmt.Sprintf("%d", request.ReadQueueNums)
- maps["writeQueueNums"] = fmt.Sprintf("%d", request.WriteQueueNums)
- maps["perm"] = fmt.Sprintf("%d", request.Perm)
- maps["topicFilterType"] = request.TopicFilterType
- maps["topicSysFlag"] = fmt.Sprintf("%d", request.TopicSysFlag)
- maps["order"] = strconv.FormatBool(request.Order)
-
- return maps
-}
-
-type TopicListRequestHeader struct {
- Topic string
-}
-
-func (request *TopicListRequestHeader) Encode() map[string]string {
- maps := make(map[string]string)
- maps["topic"] = request.Topic
-
- return maps
-}
-
-type DeleteTopicRequestHeader struct {
- Topic string
-}
-
-func (request *DeleteTopicRequestHeader) Encode() map[string]string {
- maps := make(map[string]string)
- maps["topic"] = request.Topic
-
- return maps
-}
-
-type ResetOffsetHeader struct {
- topic string
- group string
- timestamp int64
- isForce bool
-}
-
-func (request *ResetOffsetHeader) Encode() map[string]string {
- maps := make(map[string]string)
- maps["topic"] = request.topic
- maps["group"] = request.group
- maps["timestamp"] = strconv.FormatInt(request.timestamp, 10)
- return maps
-}
-
-func (request *ResetOffsetHeader) Decode(properties map[string]string) {
- if len(properties) == 0 {
- return
- }
-
- if v, existed := properties["topic"]; existed {
- request.topic = v
- }
-
- if v, existed := properties["group"]; existed {
- request.group = v
- }
-
- if v, existed := properties["timestamp"]; existed {
- request.timestamp, _ = strconv.ParseInt(v, 10, 0)
- }
-}
-
-type ConsumeMessageDirectlyHeader struct {
- consumerGroup string
- clientID string
- msgId string
- brokerName string
-}
-
-func (request *ConsumeMessageDirectlyHeader) Encode() map[string]string {
- maps := make(map[string]string)
- maps["consumerGroup"] = request.consumerGroup
- maps["clientId"] = request.clientID
- maps["msgId"] = request.msgId
- maps["brokerName"] = request.brokerName
- return maps
-}
-
-func (request *ConsumeMessageDirectlyHeader) Decode(properties map[string]string) {
- if len(properties) == 0 {
- return
- }
-
- if v, existed := properties["consumerGroup"]; existed {
- request.consumerGroup = v
- }
-
- if v, existed := properties["clientId"]; existed {
- request.clientID = v
- }
-
- if v, existed := properties["msgId"]; existed {
- request.msgId = v
- }
-
- if v, existed := properties["brokerName"]; existed {
- request.brokerName = v
- }
-}
diff --git a/internal/response.go b/internal/response.go
deleted file mode 100644
index ae75b9c..0000000
--- a/internal/response.go
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package internal
-
-const (
- ResSuccess = int16(0)
- ResError = int16(1)
- ResFlushDiskTimeout = int16(10)
- ResSlaveNotAvailable = int16(11)
- ResFlushSlaveTimeout = int16(12)
- ResTopicNotExist = int16(17)
- ResPullNotFound = int16(19)
- ResPullRetryImmediately = int16(20)
- ResPullOffsetMoved = int16(21)
-)
-
-type SendMessageResponse struct {
- MsgId string
- QueueId int32
- QueueOffset int64
- TransactionId string
- MsgRegion string
-}
-
-func (response *SendMessageResponse) Decode(properties map[string]string) {
-
-}
-
-type PullMessageResponse struct {
- SuggestWhichBrokerId int64
- NextBeginOffset int64
- MinOffset int64
- MaxOffset int64
-}
diff --git a/internal/route.go b/internal/route.go
deleted file mode 100644
index 54dbbea..0000000
--- a/internal/route.go
+++ /dev/null
@@ -1,668 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package internal
-
-import (
- "context"
- "github.com/apache/rocketmq-client-go/v2/errors"
- "math/rand"
- "sort"
- "strconv"
- "strings"
- "sync"
- "sync/atomic"
- "time"
-
- jsoniter "github.com/json-iterator/go"
- "github.com/tidwall/gjson"
-
- "github.com/apache/rocketmq-client-go/v2/internal/remote"
- "github.com/apache/rocketmq-client-go/v2/internal/utils"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/rlog"
-)
-
-const (
- EnvNameServerAddr = "NAMESRV_ADDR"
-
- requestTimeout = 3 * time.Second
- defaultTopic = "TBW102"
- defaultQueueNums = 4
- MasterId = int64(0)
-)
-
-func (s *namesrvs) cleanOfflineBroker() {
- // TODO optimize
- s.lockNamesrv.Lock()
- s.brokerAddressesMap.Range(func(key, value interface{}) bool {
- brokerName := key.(string)
- bd := value.(*BrokerData)
- for k, v := range bd.BrokerAddresses {
- isBrokerAddrExistInTopicRoute := false
- s.routeDataMap.Range(func(key, value interface{}) bool {
- trd := value.(*TopicRouteData)
- for idx := range trd.BrokerDataList {
- for _, v1 := range trd.BrokerDataList[idx].BrokerAddresses {
- if v1 == v {
- isBrokerAddrExistInTopicRoute = true
- return false
- }
- }
- }
- return true
- })
- if !isBrokerAddrExistInTopicRoute {
- delete(bd.BrokerAddresses, k)
- rlog.Info("the broker: [name=%s, ID=%d, addr=%s,] is offline, remove it", map[string]interface{}{
- "brokerName": brokerName,
- "brokerID": k,
- "brokerAddr": v,
- })
- }
- }
- if len(bd.BrokerAddresses) == 0 {
- s.brokerAddressesMap.Delete(brokerName)
- rlog.Info("the broker name's host is offline, remove it", map[string]interface{}{
- "brokerName": brokerName,
- })
- }
- return true
- })
- s.lockNamesrv.Unlock()
-}
-
-// key is topic, value is TopicPublishInfo
-type TopicPublishInfo struct {
- OrderTopic bool
- HaveTopicRouterInfo bool
- MqList []*primitive.MessageQueue
- RouteData *TopicRouteData
- TopicQueueIndex int32
-}
-
-func (info *TopicPublishInfo) isOK() (bIsTopicOk bool) {
- return len(info.MqList) > 0
-}
-
-func (info *TopicPublishInfo) fetchQueueIndex() int {
- length := len(info.MqList)
- if length <= 0 {
- return -1
- }
- qIndex := atomic.AddInt32(&info.TopicQueueIndex, 1)
- return int(qIndex) % length
-}
-
-func (s *namesrvs) UpdateTopicRouteInfo(topic string) (*TopicRouteData, bool, error) {
- return s.UpdateTopicRouteInfoWithDefault(topic, "", 0)
-}
-
-func (s *namesrvs) UpdateTopicRouteInfoWithDefault(topic string, defaultTopic string, defaultQueueNum int) (*TopicRouteData, bool, error) {
- s.lockNamesrv.Lock()
- defer s.lockNamesrv.Unlock()
-
- var (
- routeData *TopicRouteData
- err error
- )
-
- t := topic
- if len(defaultTopic) > 0 {
- t = defaultTopic
- }
- routeData, err = s.queryTopicRouteInfoFromServer(t)
-
- if err != nil {
- rlog.Warning("query topic route from server error", map[string]interface{}{
- rlog.LogKeyUnderlayError: err,
- })
- }
-
- if routeData == nil {
- rlog.Warning("queryTopicRouteInfoFromServer return nil", map[string]interface{}{
- rlog.LogKeyTopic: topic,
- })
- return nil, false, err
- }
-
- if len(defaultTopic) > 0 {
- for _, q := range routeData.QueueDataList {
- if q.ReadQueueNums > defaultQueueNum {
- q.ReadQueueNums = defaultQueueNum
- q.WriteQueueNums = defaultQueueNum
- }
- }
- }
-
- oldRouteData, exist := s.routeDataMap.Load(topic)
-
- changed := true
- if exist {
- changed = s.topicRouteDataIsChange(oldRouteData.(*TopicRouteData), routeData)
- }
-
- if changed {
- if s.bundleClient != nil {
- s.bundleClient.producerMap.Range(func(key, value interface{}) bool {
- p := value.(InnerProducer)
- updated := changed
- if !updated {
- updated = p.IsPublishTopicNeedUpdate(topic)
- }
- if updated {
- publishInfo := s.bundleClient.GetNameSrv().(*namesrvs).routeData2PublishInfo(topic, routeData)
- publishInfo.HaveTopicRouterInfo = true
- p.UpdateTopicPublishInfo(topic, publishInfo)
- }
- return true
- })
- s.bundleClient.consumerMap.Range(func(key, value interface{}) bool {
- consumer := value.(InnerConsumer)
- updated := changed
- if !updated {
- updated = consumer.IsSubscribeTopicNeedUpdate(topic)
- }
- if updated {
- consumer.UpdateTopicSubscribeInfo(topic, routeData2SubscribeInfo(topic, routeData))
- }
-
- return true
- })
- rlog.Info("change the route for clients", nil)
- }
-
- s.routeDataMap.Store(topic, routeData)
- rlog.Info("the topic route info changed", map[string]interface{}{
- rlog.LogKeyTopic: topic,
- rlog.LogKeyValueChangedFrom: oldRouteData,
- rlog.LogKeyValueChangedTo: routeData.String(),
- })
- for _, brokerData := range routeData.BrokerDataList {
- s.brokerAddressesMap.Store(brokerData.BrokerName, brokerData)
- }
- }
-
- return routeData.clone(), changed, nil
-}
-
-func (s *namesrvs) AddBroker(routeData *TopicRouteData) {
- for _, brokerData := range routeData.BrokerDataList {
- s.brokerAddressesMap.Store(brokerData.BrokerName, brokerData)
- }
-}
-
-func (s *namesrvs) FindBrokerAddrByTopic(topic string) string {
- v, exist := s.routeDataMap.Load(topic)
- if !exist {
- return ""
- }
- routeData := v.(*TopicRouteData)
- if len(routeData.BrokerDataList) == 0 {
- return ""
- }
- i := utils.AbsInt(rand.Int())
- bd := routeData.BrokerDataList[i%len(routeData.BrokerDataList)]
- addr := bd.BrokerAddresses[MasterId]
- if addr == "" && len(bd.BrokerAddresses) > 0 {
- i = i % len(bd.BrokerAddresses)
- for _, v := range bd.BrokerAddresses {
- if i <= 0 {
- addr = v
- break
- }
- i--
- }
- }
- return addr
-}
-
-func (s *namesrvs) FindBrokerAddrByName(brokerName string) string {
- bd, exist := s.brokerAddressesMap.Load(brokerName)
-
- if !exist {
- return ""
- }
-
- return bd.(*BrokerData).BrokerAddresses[MasterId]
-}
-
-func (s *namesrvs) FindBrokerAddressInSubscribe(brokerName string, brokerId int64, onlyThisBroker bool) *FindBrokerResult {
- var (
- brokerAddr = ""
- slave = false
- found = false
- )
-
- rlog.Debug("broker id "+strconv.FormatInt(brokerId, 10), nil)
-
- v, exist := s.brokerAddressesMap.Load(brokerName)
-
- if !exist {
- return nil
- }
- data := v.(*BrokerData)
- if len(data.BrokerAddresses) == 0 {
- return nil
- }
-
- brokerAddr = data.BrokerAddresses[brokerId]
- slave = brokerId != MasterId
- if brokerAddr != "" {
- found = true
- }
-
- // not found && read from slave, try again use next brokerId
- if !found && slave {
- rlog.Debug("Not found broker addr and slave "+strconv.FormatBool(slave), nil)
- brokerAddr = data.BrokerAddresses[brokerId+1]
- found = brokerAddr != ""
- }
-
- // still not found && cloud use other broker addr, find anyone in BrokerAddresses
- if !found && !onlyThisBroker {
- rlog.Debug("STILL Not found broker addr", nil)
- for k, v := range data.BrokerAddresses {
- if v != "" {
- brokerAddr = v
- found = true
- slave = k != MasterId
- break
- }
- }
- }
-
- if found {
- rlog.Debug("Find broker addr "+brokerAddr, nil)
- }
-
- var result *FindBrokerResult
- if found {
- result = &FindBrokerResult{
- BrokerAddr: brokerAddr,
- Slave: slave,
- BrokerVersion: s.findBrokerVersion(brokerName, brokerAddr),
- }
- }
-
- return result
-}
-
-func (s *namesrvs) FetchSubscribeMessageQueues(topic string) ([]*primitive.MessageQueue, error) {
- routeData, err := s.queryTopicRouteInfoFromServer(topic)
-
- if err != nil {
- return nil, err
- }
-
- mqs := make([]*primitive.MessageQueue, 0)
-
- for _, qd := range routeData.QueueDataList {
- if queueIsReadable(qd.Perm) {
- for i := 0; i < qd.ReadQueueNums; i++ {
- mqs = append(mqs, &primitive.MessageQueue{Topic: topic, BrokerName: qd.BrokerName, QueueId: i})
- }
- }
- }
- return mqs, nil
-}
-
-func (s *namesrvs) FetchPublishMessageQueues(topic string) ([]*primitive.MessageQueue, error) {
- var (
- err error
- routeData *TopicRouteData
- )
-
- v, exist := s.routeDataMap.Load(topic)
- if !exist {
- routeData, err = s.queryTopicRouteInfoFromServer(topic)
- if err != nil {
- rlog.Error("queryTopicRouteInfoFromServer failed", map[string]interface{}{
- rlog.LogKeyTopic: topic,
- })
- return nil, err
- }
- s.routeDataMap.Store(topic, routeData)
- s.AddBroker(routeData)
- } else {
- routeData = v.(*TopicRouteData)
- }
-
- if err != nil {
- return nil, err
- }
- publishInfo := s.routeData2PublishInfo(topic, routeData)
-
- return publishInfo.MqList, nil
-}
-
-func (s *namesrvs) AddBrokerVersion(brokerName, brokerAddr string, version int32) {
- s.brokerLock.Lock()
- defer s.brokerLock.Unlock()
-
- m, exist := s.brokerVersionMap[brokerName]
- if !exist {
- m = make(map[string]int32, 4)
- s.brokerVersionMap[brokerName] = m
- }
- m[brokerAddr] = version
-}
-
-func (s *namesrvs) findBrokerVersion(brokerName, brokerAddr string) int32 {
- s.brokerLock.RLock()
- defer s.brokerLock.RUnlock()
-
- versions, exist := s.brokerVersionMap[brokerName]
-
- if !exist {
- return 0
- }
-
- return versions[brokerAddr]
-}
-
-func (s *namesrvs) queryTopicRouteInfoFromServer(topic string) (*TopicRouteData, error) {
- request := &GetRouteInfoRequestHeader{
- Topic: topic,
- }
-
- var (
- response *remote.RemotingCommand
- err error
- )
-
- //if s.Size() == 0, response will be nil, lead to panic below.
- if s.Size() == 0 {
- rlog.Error("namesrv list empty. UpdateNameServerAddress should be called first.", map[string]interface{}{
- "namesrv": s,
- "topic": topic,
- })
- return nil, primitive.NewRemotingErr("namesrv list empty")
- }
-
- for i := 0; i < s.Size(); i++ {
- rc := remote.NewRemotingCommand(ReqGetRouteInfoByTopic, request, nil)
- ctx, cancel := context.WithTimeout(context.Background(), requestTimeout)
- response, err = s.nameSrvClient.InvokeSync(ctx, s.getNameServerAddress(), rc)
-
- if err == nil {
- cancel()
- break
- }
- cancel()
- }
- if err != nil {
- rlog.Error("connect to namesrv failed.", map[string]interface{}{
- "namesrv": s,
- "topic": topic,
- })
- return nil, primitive.NewRemotingErr(err.Error())
- }
-
- switch response.Code {
- case ResSuccess:
- if response.Body == nil {
- return nil, primitive.NewMQClientErr(response.Code, response.Remark)
- }
- routeData := &TopicRouteData{}
-
- err = routeData.decode(string(response.Body))
- if err != nil {
- rlog.Warning("decode TopicRouteData error: %s", map[string]interface{}{
- rlog.LogKeyUnderlayError: err,
- "topic": topic,
- })
- return nil, err
- }
- return routeData, nil
- case ResTopicNotExist:
- return nil, errors.ErrTopicNotExist
- default:
- return nil, primitive.NewMQClientErr(response.Code, response.Remark)
- }
-}
-
-func (s *namesrvs) topicRouteDataIsChange(oldData *TopicRouteData, newData *TopicRouteData) bool {
- if oldData == nil || newData == nil {
- return true
- }
- oldDataCloned := oldData.clone()
- newDataCloned := newData.clone()
-
- sort.Slice(oldDataCloned.QueueDataList, func(i, j int) bool {
- return strings.Compare(oldDataCloned.QueueDataList[i].BrokerName, oldDataCloned.QueueDataList[j].BrokerName) > 0
- })
- sort.Slice(oldDataCloned.BrokerDataList, func(i, j int) bool {
- return strings.Compare(oldDataCloned.BrokerDataList[i].BrokerName, oldDataCloned.BrokerDataList[j].BrokerName) > 0
- })
- sort.Slice(newDataCloned.QueueDataList, func(i, j int) bool {
- return strings.Compare(newDataCloned.QueueDataList[i].BrokerName, newDataCloned.QueueDataList[j].BrokerName) > 0
- })
- sort.Slice(newDataCloned.BrokerDataList, func(i, j int) bool {
- return strings.Compare(newDataCloned.BrokerDataList[i].BrokerName, newDataCloned.BrokerDataList[j].BrokerName) > 0
- })
-
- return !oldDataCloned.equals(newDataCloned)
-}
-
-func (s *namesrvs) routeData2PublishInfo(topic string, data *TopicRouteData) *TopicPublishInfo {
- publishInfo := &TopicPublishInfo{
- RouteData: data,
- OrderTopic: false,
- }
-
- if data.OrderTopicConf != "" {
- brokers := strings.Split(data.OrderTopicConf, ";")
- for _, broker := range brokers {
- item := strings.Split(broker, ":")
- nums, _ := strconv.Atoi(item[1])
- for i := 0; i < nums; i++ {
- mq := &primitive.MessageQueue{
- Topic: topic,
- BrokerName: item[0],
- QueueId: i,
- }
- publishInfo.MqList = append(publishInfo.MqList, mq)
- }
- }
-
- publishInfo.OrderTopic = true
- return publishInfo
- }
-
- qds := data.QueueDataList
- sort.Slice(qds, func(i, j int) bool {
- return i-j >= 0
- })
-
- for _, qd := range qds {
- if !queueIsWriteable(qd.Perm) {
- continue
- }
-
- var bData *BrokerData
- for _, bd := range data.BrokerDataList {
- if bd.BrokerName == qd.BrokerName {
- bData = bd
- break
- }
- }
-
- if bData == nil || bData.BrokerAddresses[MasterId] == "" {
- continue
- }
-
- for i := 0; i < qd.WriteQueueNums; i++ {
- mq := &primitive.MessageQueue{
- Topic: topic,
- BrokerName: qd.BrokerName,
- QueueId: i,
- }
- publishInfo.MqList = append(publishInfo.MqList, mq)
- }
- }
-
- return publishInfo
-}
-
-// TopicRouteData TopicRouteData
-type TopicRouteData struct {
- OrderTopicConf string
- QueueDataList []*QueueData `json:"queueDatas"`
- BrokerDataList []*BrokerData `json:"brokerDatas"`
-}
-
-func (routeData *TopicRouteData) decode(data string) error {
- res := gjson.Parse(data)
- err := jsoniter.Unmarshal([]byte(res.Get("queueDatas").String()), &routeData.QueueDataList)
-
- if err != nil {
- return err
- }
-
- bds := res.Get("brokerDatas").Array()
- routeData.BrokerDataList = make([]*BrokerData, len(bds))
- for idx, v := range bds {
- bd := &BrokerData{
- BrokerName: v.Get("brokerName").String(),
- Cluster: v.Get("cluster").String(),
- BrokerAddresses: make(map[int64]string, 0),
- }
- addrs := v.Get("brokerAddrs").String()
- strs := strings.Split(addrs[1:len(addrs)-1], ",")
- if strs != nil {
- for _, str := range strs {
- i := strings.Index(str, ":")
- if i < 0 {
- continue
- }
- id, _ := strconv.ParseInt(str[0:i], 10, 64)
- bd.BrokerAddresses[id] = strings.Replace(str[i+1:], "\"", "", -1)
- }
- }
- routeData.BrokerDataList[idx] = bd
- }
- return nil
-}
-
-func (routeData *TopicRouteData) clone() *TopicRouteData {
- cloned := &TopicRouteData{
- OrderTopicConf: routeData.OrderTopicConf,
- QueueDataList: make([]*QueueData, len(routeData.QueueDataList)),
- BrokerDataList: make([]*BrokerData, len(routeData.BrokerDataList)),
- }
-
- for index, value := range routeData.QueueDataList {
- cloned.QueueDataList[index] = value
- }
-
- for index, value := range routeData.BrokerDataList {
- cloned.BrokerDataList[index] = value
- }
-
- return cloned
-}
-
-func (routeData *TopicRouteData) equals(data *TopicRouteData) bool {
- if len(routeData.BrokerDataList) != len(data.BrokerDataList) {
- return false
- }
- if len(routeData.QueueDataList) != len(data.QueueDataList) {
- return false
- }
-
- for idx := range routeData.BrokerDataList {
- if !routeData.BrokerDataList[idx].Equals(data.BrokerDataList[idx]) {
- return false
- }
- }
-
- for idx := range routeData.QueueDataList {
- if !routeData.QueueDataList[idx].Equals(data.QueueDataList[idx]) {
- return false
- }
- }
- return true
-}
-
-func (routeData *TopicRouteData) String() string {
- data, _ := jsoniter.Marshal(routeData)
- return string(data)
-}
-
-// QueueData QueueData
-type QueueData struct {
- BrokerName string `json:"brokerName"`
- ReadQueueNums int `json:"readQueueNums"`
- WriteQueueNums int `json:"writeQueueNums"`
- Perm int `json:"perm"`
- TopicSynFlag int `json:"topicSynFlag"`
-}
-
-func (q *QueueData) Equals(qd *QueueData) bool {
- if q.BrokerName != qd.BrokerName {
- return false
- }
-
- if q.ReadQueueNums != qd.ReadQueueNums {
- return false
- }
-
- if q.WriteQueueNums != qd.WriteQueueNums {
- return false
- }
-
- if q.Perm != qd.Perm {
- return false
- }
-
- if q.TopicSynFlag != qd.TopicSynFlag {
- return false
- }
-
- return true
-}
-
-// BrokerData BrokerData
-type BrokerData struct {
- Cluster string `json:"cluster"`
- BrokerName string `json:"brokerName"`
- BrokerAddresses map[int64]string `json:"brokerAddrs"`
- brokerAddressesLock sync.RWMutex
-}
-
-func (b *BrokerData) Equals(bd *BrokerData) bool {
- if b.Cluster != bd.Cluster {
- return false
- }
-
- if b.BrokerName != bd.BrokerName {
- return false
- }
-
- if len(b.BrokerAddresses) != len(bd.BrokerAddresses) {
- return false
- }
-
- for k, v := range b.BrokerAddresses {
- if bd.BrokerAddresses[k] != v {
- return false
- }
- }
-
- return true
-}
diff --git a/internal/route_test.go b/internal/route_test.go
deleted file mode 100644
index a1ebec4..0000000
--- a/internal/route_test.go
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package internal
-
-import (
- "context"
- "github.com/apache/rocketmq-client-go/v2/errors"
- "sync"
- "testing"
-
- "github.com/golang/mock/gomock"
- . "github.com/smartystreets/goconvey/convey"
- "github.com/stretchr/testify/assert"
-
- "github.com/apache/rocketmq-client-go/v2/internal/remote"
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-func TestQueryTopicRouteInfoFromServer(t *testing.T) {
- Convey("marshal of TraceContext", t, func() {
-
- ctrl := gomock.NewController(t)
- defer ctrl.Finish()
-
- remotingCli := remote.NewMockRemotingClient(ctrl)
-
- addr, err := primitive.NewNamesrvAddr("1.1.1.1:8880", "1.1.1.2:8880", "1.1.1.3:8880")
- assert.Nil(t, err)
-
- namesrv, err := NewNamesrv(primitive.NewPassthroughResolver(addr))
- assert.Nil(t, err)
- namesrv.nameSrvClient = remotingCli
-
- Convey("When marshal producer trace data", func() {
-
- count := 0
- remotingCli.EXPECT().InvokeSync(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(
- func(ctx context.Context, addr string, request *remote.RemotingCommand) (*remote.RemotingCommand, error) {
- count++
- if count < 3 {
- return nil, errors.ErrNotExisted
- }
- return &remote.RemotingCommand{
- Code: ResTopicNotExist,
- }, nil
- }).Times(3)
-
- data, err := namesrv.queryTopicRouteInfoFromServer("notexisted")
- assert.Nil(t, data)
- assert.Equal(t, errors.ErrTopicNotExist, err)
- })
- })
-}
-
-func TestAddBrokerVersion(t *testing.T) {
- s := &namesrvs{}
- s.brokerVersionMap = make(map[string]map[string]int32, 0)
- s.brokerLock = new(sync.RWMutex)
-
- v := s.findBrokerVersion("b1", "addr1")
- assert.Equal(t, v, int32(0))
-
- s.AddBrokerVersion("b1", "addr1", 1)
- v = s.findBrokerVersion("b1", "addr1")
- assert.Equal(t, v, int32(1))
-
- v = s.findBrokerVersion("b1", "addr2")
- assert.Equal(t, v, int32(0))
-}
-
-func TestFindBrokerAddressInSubscribe(t *testing.T) {
- s := &namesrvs{}
- s.brokerVersionMap = make(map[string]map[string]int32, 0)
- s.brokerLock = new(sync.RWMutex)
-
- brokerDataRaft1 := &BrokerData{
- Cluster: "cluster",
- BrokerName: "raft01",
- BrokerAddresses: map[int64]string{
- 0: "127.0.0.1:10911",
- 1: "127.0.0.1:10912",
- 2: "127.0.0.1:10913",
- },
- }
- s.brokerAddressesMap.Store(brokerDataRaft1.BrokerName, brokerDataRaft1)
- brokerDataRaft2 := &BrokerData{
- Cluster: "cluster",
- BrokerName: "raft02",
- BrokerAddresses: map[int64]string{
- 0: "127.0.0.1:10911",
- 2: "127.0.0.1:10912",
- 3: "127.0.0.1:10913",
- },
- }
- s.brokerAddressesMap.Store(brokerDataRaft2.BrokerName, brokerDataRaft2)
-
- Convey("Request master broker", t, func() {
- result := s.FindBrokerAddressInSubscribe(brokerDataRaft1.BrokerName, 0, false)
- assert.NotNil(t, result)
- assert.Equal(t, result.BrokerAddr, brokerDataRaft1.BrokerAddresses[0])
- assert.Equal(t, result.Slave, false)
- })
-
- Convey("Request slave broker from normal broker group", t, func() {
- result := s.FindBrokerAddressInSubscribe(brokerDataRaft1.BrokerName, 1, false)
- assert.NotNil(t, result)
- assert.Equal(t, result.BrokerAddr, brokerDataRaft1.BrokerAddresses[1])
- assert.Equal(t, result.Slave, true)
- })
-
- Convey("Request slave broker from non normal broker group", t, func() {
- result := s.FindBrokerAddressInSubscribe(brokerDataRaft2.BrokerName, 1, false)
- assert.NotNil(t, result)
- assert.Equal(t, result.BrokerAddr, brokerDataRaft2.BrokerAddresses[2])
- assert.Equal(t, result.Slave, true)
- })
-
- Convey("Request not exist broker", t, func() {
- result := s.FindBrokerAddressInSubscribe(brokerDataRaft1.BrokerName, 4, false)
- assert.NotNil(t, result)
- })
-}
diff --git a/internal/trace.go b/internal/trace.go
deleted file mode 100644
index 753a4d1..0000000
--- a/internal/trace.go
+++ /dev/null
@@ -1,526 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package internal
-
-import (
- "bytes"
- "context"
- "fmt"
- "runtime"
- "strconv"
- "strings"
- "sync/atomic"
- "time"
-
- "github.com/pkg/errors"
-
- "github.com/apache/rocketmq-client-go/v2/internal/remote"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/rlog"
-)
-
-type TraceBean struct {
- Topic string
- MsgId string
- OffsetMsgId string
- Tags string
- Keys string
- StoreHost string
- ClientHost string
- StoreTime int64
- RetryTimes int
- BodyLength int
- MsgType primitive.MessageType
-}
-
-type TraceTransferBean struct {
- transData string
- // not duplicate
- transKey []string
-}
-
-type TraceType string
-
-const (
- Pub TraceType = "Pub"
- SubBefore TraceType = "SubBefore"
- SubAfter TraceType = "SubAfter"
-
- contentSplitter = '\001'
- fieldSplitter = '\002'
-)
-
-type TraceContext struct {
- TraceType TraceType
- TimeStamp int64
- RegionId string
- RegionName string
- GroupName string
- CostTime int64
- IsSuccess bool
- RequestId string
- ContextCode int
- TraceBeans []TraceBean
-}
-
-func (ctx *TraceContext) marshal2Bean() *TraceTransferBean {
- buffer := bytes.NewBufferString("")
- switch ctx.TraceType {
- case Pub:
- bean := ctx.TraceBeans[0]
- buffer.WriteString(string(ctx.TraceType))
- buffer.WriteRune(contentSplitter)
- buffer.WriteString(strconv.FormatInt(ctx.TimeStamp, 10))
- buffer.WriteRune(contentSplitter)
- buffer.WriteString(ctx.RegionId)
- buffer.WriteRune(contentSplitter)
- ss := strings.Split(ctx.GroupName, "%")
- if len(ss) == 2 {
- buffer.WriteString(ss[1])
- } else {
- buffer.WriteString(ctx.GroupName)
- }
-
- buffer.WriteRune(contentSplitter)
- ssTopic := strings.Split(bean.Topic, "%")
- if len(ssTopic) == 2 {
- buffer.WriteString(ssTopic[1])
- } else {
- buffer.WriteString(bean.Topic)
- }
- //buffer.WriteString(bean.Topic)
- buffer.WriteRune(contentSplitter)
- buffer.WriteString(bean.MsgId)
- buffer.WriteRune(contentSplitter)
- buffer.WriteString(bean.Tags)
- buffer.WriteRune(contentSplitter)
- buffer.WriteString(bean.Keys)
- buffer.WriteRune(contentSplitter)
- buffer.WriteString(bean.StoreHost)
- buffer.WriteRune(contentSplitter)
- buffer.WriteString(strconv.Itoa(bean.BodyLength))
- buffer.WriteRune(contentSplitter)
- buffer.WriteString(strconv.FormatInt(ctx.CostTime, 10))
- buffer.WriteRune(contentSplitter)
- buffer.WriteString(strconv.Itoa(int(bean.MsgType)))
- buffer.WriteRune(contentSplitter)
- buffer.WriteString(bean.OffsetMsgId)
- buffer.WriteRune(contentSplitter)
- buffer.WriteString(strconv.FormatBool(ctx.IsSuccess))
- buffer.WriteRune(contentSplitter)
- buffer.WriteString(bean.ClientHost)
- buffer.WriteRune(fieldSplitter)
- case SubBefore:
- for _, bean := range ctx.TraceBeans {
- buffer.WriteString(string(ctx.TraceType))
- buffer.WriteRune(contentSplitter)
- buffer.WriteString(strconv.FormatInt(ctx.TimeStamp, 10))
- buffer.WriteRune(contentSplitter)
- buffer.WriteString(ctx.RegionId)
- buffer.WriteRune(contentSplitter)
- ss := strings.Split(ctx.GroupName, "%")
- if len(ss) == 2 {
- buffer.WriteString(ss[1])
- } else {
- buffer.WriteString(ctx.GroupName)
- }
- buffer.WriteRune(contentSplitter)
- buffer.WriteString(ctx.RequestId)
- buffer.WriteRune(contentSplitter)
- buffer.WriteString(bean.MsgId)
- buffer.WriteRune(contentSplitter)
- buffer.WriteString(strconv.Itoa(bean.RetryTimes))
- buffer.WriteRune(contentSplitter)
- buffer.WriteString(nullWrap(bean.Keys))
- buffer.WriteRune(contentSplitter)
- buffer.WriteString(bean.ClientHost)
- buffer.WriteRune(fieldSplitter)
- }
- case SubAfter:
- for _, bean := range ctx.TraceBeans {
- buffer.WriteString(string(ctx.TraceType))
- buffer.WriteRune(contentSplitter)
- buffer.WriteString(ctx.RequestId)
- buffer.WriteRune(contentSplitter)
- buffer.WriteString(bean.MsgId)
- buffer.WriteRune(contentSplitter)
- buffer.WriteString(strconv.FormatInt(ctx.CostTime, 10))
- buffer.WriteRune(contentSplitter)
- buffer.WriteString(strconv.FormatBool(ctx.IsSuccess))
- buffer.WriteRune(contentSplitter)
- buffer.WriteString(nullWrap(bean.Keys))
- buffer.WriteRune(contentSplitter)
- buffer.WriteString(strconv.Itoa(ctx.ContextCode))
- buffer.WriteRune(contentSplitter)
- buffer.WriteString(strconv.FormatInt(ctx.TimeStamp, 10))
- buffer.WriteRune(contentSplitter)
- buffer.WriteString(ctx.GroupName)
- buffer.WriteRune(fieldSplitter)
- }
- }
- transferBean := new(TraceTransferBean)
- transferBean.transData = buffer.String()
- for _, bean := range ctx.TraceBeans {
- transferBean.transKey = append(transferBean.transKey, bean.MsgId)
- if len(bean.Keys) > 0 {
- transferBean.transKey = append(transferBean.transKey, bean.Keys)
- }
- }
- return transferBean
-}
-
-// compatible with java console.
-func nullWrap(s string) string {
- if len(s) == 0 {
- return "null"
- }
- return s
-}
-
-type traceDispatcherType int
-
-const (
- RmqSysTraceTopic = "RMQ_SYS_TRACE_TOPIC"
-
- ProducerType traceDispatcherType = iota
- ConsumerType
-
- maxMsgSize = 128000 - 10*1000
- batchSize = 100
-
- TraceTopicPrefix = SystemTopicPrefix + "TRACE_DATA_"
- TraceGroupName = "_INNER_TRACE_PRODUCER"
-)
-
-type TraceDispatcher interface {
- GetTraceTopicName() string
-
- Start()
- Append(ctx TraceContext) bool
- Close()
-}
-
-type traceDispatcher struct {
- ctx context.Context
- cancel context.CancelFunc
- running bool
-
- traceTopic string
- access primitive.AccessChannel
-
- ticker *time.Ticker
- input chan TraceContext
- batchCh chan []*TraceContext
-
- discardCount int64
-
- // support deliver trace message to other cluster.
- namesrvs *namesrvs
- // round robin index
- rrindex int32
- cli RMQClient
-}
-
-func NewTraceDispatcher(traceCfg *primitive.TraceConfig) *traceDispatcher {
- ctx := context.Background()
- ctx, cancel := context.WithCancel(ctx)
-
- t := traceCfg.TraceTopic
- if len(t) == 0 {
- t = RmqSysTraceTopic
- }
-
- if traceCfg.Access == primitive.Cloud {
- t = TraceTopicPrefix + traceCfg.TraceTopic
- }
-
- if len(traceCfg.NamesrvAddrs) == 0 && traceCfg.Resolver == nil {
- panic("no NamesrvAddrs or Resolver configured")
- }
-
- var srvs *namesrvs
- var err error
- if len(traceCfg.NamesrvAddrs) > 0 {
- srvs, err = NewNamesrv(primitive.NewPassthroughResolver(traceCfg.NamesrvAddrs))
- } else {
- srvs, err = NewNamesrv(traceCfg.Resolver)
- }
-
- if err != nil {
- panic(errors.Wrap(err, "new Namesrv failed."))
- }
- if !traceCfg.Credentials.IsEmpty() {
- srvs.SetCredentials(traceCfg.Credentials)
- }
-
- cliOp := DefaultClientOptions()
- cliOp.GroupName = traceCfg.GroupName
- cliOp.NameServerAddrs = traceCfg.NamesrvAddrs
- cliOp.InstanceName = "INNER_TRACE_CLIENT_DEFAULT"
- cliOp.RetryTimes = 0
- cliOp.Namesrv = srvs
- cliOp.Credentials = traceCfg.Credentials
- cli := GetOrNewRocketMQClient(cliOp, nil)
- if cli == nil {
- return nil
- }
- cliOp.Namesrv = cli.GetNameSrv()
- return &traceDispatcher{
- ctx: ctx,
- cancel: cancel,
-
- traceTopic: t,
- access: traceCfg.Access,
- input: make(chan TraceContext, 1024),
- batchCh: make(chan []*TraceContext, 2048),
- cli: cli,
- namesrvs: srvs,
- }
-}
-
-func (td *traceDispatcher) GetTraceTopicName() string {
- return td.traceTopic
-}
-
-func (td *traceDispatcher) Start() {
- td.running = true
- td.cli.Start()
- go primitive.WithRecover(func() {
- td.process()
- })
-}
-
-func (td *traceDispatcher) Close() {
- td.running = false
- td.ticker.Stop()
- td.cancel()
-}
-
-func (td *traceDispatcher) Append(ctx TraceContext) bool {
- if !td.running {
- rlog.Error("traceDispatcher is closed.", nil)
- return false
- }
- select {
- case td.input <- ctx:
- return true
- default:
- rlog.Warning("buffer full", map[string]interface{}{
- "discardCount": atomic.AddInt64(&td.discardCount, 1),
- "TraceContext": ctx,
- })
- return false
- }
-}
-
-// process
-func (td *traceDispatcher) process() {
- var count int
- var batch []TraceContext
- maxWaitDuration := 5 * time.Millisecond
- maxWaitTime := maxWaitDuration.Nanoseconds()
- td.ticker = time.NewTicker(maxWaitDuration)
- lastput := time.Now()
- for {
- select {
- case ctx := <-td.input:
- count++
- lastput = time.Now()
- batch = append(batch, ctx)
- if count == batchSize {
- count = 0
- batchSend := batch
- go primitive.WithRecover(func() {
- td.batchCommit(batchSend)
- })
- batch = make([]TraceContext, 0)
- }
- case <-td.ticker.C:
- delta := time.Since(lastput).Nanoseconds()
- if delta > maxWaitTime {
- count++
- lastput = time.Now()
- if len(batch) > 0 {
- batchSend := batch
- go primitive.WithRecover(func() {
- td.batchCommit(batchSend)
- })
- batch = make([]TraceContext, 0)
- }
- }
- case <-td.ctx.Done():
- batchSend := batch
- go primitive.WithRecover(func() {
- td.batchCommit(batchSend)
- })
- batch = make([]TraceContext, 0)
-
- now := time.Now().UnixNano() / int64(time.Millisecond)
- end := now + 500
- for now < end {
- now = time.Now().UnixNano() / int64(time.Millisecond)
- runtime.Gosched()
- }
- rlog.Info(fmt.Sprintf("------end trace send %v %v", td.input, td.batchCh), nil)
- }
- }
-}
-
-// batchCommit commit slice of TraceContext. convert the ctxs to keyed pair(key is Topic + regionid).
-// flush according key one by one.
-func (td *traceDispatcher) batchCommit(ctxs []TraceContext) {
- keyedCtxs := make(map[string][]TraceTransferBean)
- for _, ctx := range ctxs {
- if len(ctx.TraceBeans) == 0 {
- return
- }
- topic := ctx.TraceBeans[0].Topic
- regionID := ctx.RegionId
- key := topic
- if len(regionID) > 0 {
- key = fmt.Sprintf("%s%c%s", topic, contentSplitter, regionID)
- }
- keyedCtxs[key] = append(keyedCtxs[key], *ctx.marshal2Bean())
- }
-
- for k, v := range keyedCtxs {
- arr := strings.Split(k, string([]byte{contentSplitter}))
- topic := k
- regionID := ""
- if len(arr) > 1 {
- topic = arr[0]
- regionID = arr[1]
- }
- td.flush(topic, regionID, v)
- }
-}
-
-type Keyset map[string]struct{}
-
-func (ks Keyset) slice() []string {
- slice := make([]string, len(ks))
- for k, _ := range ks {
- slice = append(slice, k)
- }
- return slice
-}
-
-// flush data in batch.
-func (td *traceDispatcher) flush(topic, regionID string, data []TraceTransferBean) {
- if len(data) == 0 {
- return
- }
-
- keyset := make(Keyset)
- var builder strings.Builder
- flushed := true
- for _, bean := range data {
- for _, k := range bean.transKey {
- keyset[k] = struct{}{}
- }
- builder.WriteString(bean.transData)
- flushed = false
-
- if builder.Len() > maxMsgSize {
- td.sendTraceDataByMQ(keyset, regionID, builder.String())
- builder.Reset()
- keyset = make(Keyset)
- flushed = true
- }
- }
- if !flushed {
- td.sendTraceDataByMQ(keyset, regionID, builder.String())
- }
-}
-
-func (td *traceDispatcher) sendTraceDataByMQ(keySet Keyset, regionID string, data string) {
- traceTopic := td.traceTopic
- if td.access == primitive.Cloud {
- traceTopic = td.traceTopic + regionID
- }
- msg := primitive.NewMessage(traceTopic, []byte(data))
- msg.WithKeys(keySet.slice())
-
- mq, addr := td.findMq(regionID)
- if mq == nil {
- return
- }
-
- var req = td.buildSendRequest(mq, msg)
- ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
- defer cancel()
- err := td.cli.InvokeAsync(ctx, addr, req, func(command *remote.RemotingCommand, e error) {
- resp := primitive.NewSendResult()
- if e != nil {
- rlog.Info("send trace data error.", map[string]interface{}{
- "traceData": data,
- })
- } else {
- td.cli.ProcessSendResponse(mq.BrokerName, command, resp, msg)
- rlog.Debug("send trace data success:", map[string]interface{}{
- "SendResult": resp,
- "traceData": data,
- })
- }
- })
- if err != nil {
- rlog.Info("send trace data error when invoke", map[string]interface{}{
- rlog.LogKeyUnderlayError: err,
- })
- }
-}
-
-func (td *traceDispatcher) findMq(regionID string) (*primitive.MessageQueue, string) {
- traceTopic := td.traceTopic
- if td.access == primitive.Cloud {
- traceTopic = td.traceTopic + regionID
- }
- mqs, err := td.namesrvs.FetchPublishMessageQueues(traceTopic)
- if err != nil {
- rlog.Error("fetch publish message queues failed", map[string]interface{}{
- rlog.LogKeyUnderlayError: err,
- })
- return nil, ""
- }
- i := atomic.AddInt32(&td.rrindex, 1)
- if i < 0 {
- i = 0
- atomic.StoreInt32(&td.rrindex, 0)
- }
- i %= int32(len(mqs))
- mq := mqs[i]
-
- brokerName := mq.BrokerName
- addr := td.namesrvs.FindBrokerAddrByName(brokerName)
-
- return mq, addr
-}
-
-func (td *traceDispatcher) buildSendRequest(mq *primitive.MessageQueue,
- msg *primitive.Message) *remote.RemotingCommand {
- req := &SendMessageRequestHeader{
- ProducerGroup: TraceGroupName,
- Topic: mq.Topic,
- QueueId: mq.QueueId,
- BornTimestamp: time.Now().UnixNano() / int64(time.Millisecond),
- Flag: msg.Flag,
- Properties: msg.MarshallProperties(),
- }
-
- return remote.NewRemotingCommand(ReqSendMessage, req, msg.Body)
-}
diff --git a/internal/trace_test.go b/internal/trace_test.go
deleted file mode 100644
index a636b38..0000000
--- a/internal/trace_test.go
+++ /dev/null
@@ -1,125 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package internal
-
-import (
- "testing"
-
- "github.com/apache/rocketmq-client-go/v2/primitive"
- . "github.com/smartystreets/goconvey/convey"
- "github.com/stretchr/testify/assert"
-)
-
-func TestMarshal2Bean(t *testing.T) {
-
- Convey("marshal of TraceContext", t, func() {
-
- Convey("When marshal producer trace data", func() {
- traceCtx := TraceContext{
- TraceType: Pub,
- TimeStamp: 1563780533299,
- RegionId: "DefaultRegion",
- GroupName: "ProducerGroupName",
- CostTime: 3572,
- IsSuccess: true,
- RequestId: "0A5DE93A815518B4AAC26F77F8330001",
- TraceBeans: []TraceBean{
- {
- Topic: "TopicTest",
- MsgId: "0A5DE93A833B18B4AAC26F842A2F0000",
- OffsetMsgId: "0A5DE93A00002A9F000000000042E322",
- Tags: "TagA",
- Keys: "OrderID1882",
- StoreHost: "10.93.233.58:10911",
- ClientHost: "10.93.233.58",
- StoreTime: 1563780535085,
- BodyLength: 11,
- MsgType: primitive.NormalMsg,
- },
- },
- }
- bean := traceCtx.marshal2Bean()
- assert.Equal(t, "Pub1563780533299DefaultRegionProducerGroupNameTopicTest0A5DE93A833B18B4AAC26F842A2F0000TagAOrderID188210.93.233.58:1091111357200A5DE93A00002A9F000000000042E322true10.93.233.58\x02",
- bean.transData)
- assert.Equal(t, []string{"0A5DE93A833B18B4AAC26F842A2F0000", "OrderID1882"}, bean.transKey)
-
- // consumer before test
- traceCtx = TraceContext{
- TraceType: SubBefore,
- TimeStamp: 1563789119096,
- GroupName: "CID_JODIE_1",
- IsSuccess: true,
- RequestId: "0A5DE93A96A818B4AAC26FFAFA780007",
- TraceBeans: []TraceBean{
- {
- Topic: "TopicTest",
- MsgId: "0A5DE93A973418B4AAC26FFAFA5A0000",
- Tags: "TagA",
- Keys: "OrderID1882",
- StoreHost: "10.93.233.58",
- ClientHost: "10.93.233.58",
- StoreTime: 1563789119092,
- BodyLength: 190,
- },
- },
- }
- bean = traceCtx.marshal2Bean()
-
- Convey("transData should equal to expected", func() {
- So(bean.transData, ShouldEqual, "SubBefore1563789119096CID_JODIE_10A5DE93A96A818B4AAC26FFAFA7800070A5DE93A973418B4AAC26FFAFA5A00000OrderID188210.93.233.58")
- })
-
- Convey("transkey should equal to expected", func() {
- expectedKey := []string{"0A5DE93A973418B4AAC26FFAFA5A0000", "OrderID1882"}
- So(bean.transKey[0], ShouldEqual, expectedKey[0])
- So(bean.transKey[1], ShouldEqual, expectedKey[1])
- })
- })
-
- Convey("When marshal consumer trace data", func() {
- traceCtx := TraceContext{
- TraceType: SubAfter,
- TimeStamp: 1563789119096,
- GroupName: "CID_JODIE_1",
- IsSuccess: true,
- RequestId: "0A5DE93A96A818B4AAC26FFAFA780007",
- TraceBeans: []TraceBean{
- {
- Topic: "TopicTest",
- MsgId: "0A5DE93A973418B4AAC26FFAFA5A0000",
- Tags: "TagA",
- Keys: "OrderID1882",
- StoreHost: "10.93.233.58",
- ClientHost: "10.93.233.58",
- StoreTime: 1563789119092,
- BodyLength: 190,
- },
- },
- }
- bean := traceCtx.marshal2Bean()
- Convey("transData should equal to expected", func() {
- So(bean.transData, ShouldEqual, "SubAfter0A5DE93A96A818B4AAC26FFAFA7800070A5DE93A973418B4AAC26FFAFA5A00000trueOrderID188201563789119096CID_JODIE_1")
- })
- Convey("transkey should equal to expected", func() {
- expectedKey := []string{"0A5DE93A973418B4AAC26FFAFA5A0000", "OrderID1882"}
- So(bean.transKey[0], ShouldEqual, expectedKey[0])
- So(bean.transKey[1], ShouldEqual, expectedKey[1])
- })
- })
- })
-}
diff --git a/internal/transaction.go b/internal/transaction.go
deleted file mode 100644
index 5cd9835..0000000
--- a/internal/transaction.go
+++ /dev/null
@@ -1,21 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package internal
-
-type TransactionListener interface {
-}
diff --git a/internal/utils/compression.go b/internal/utils/compression.go
deleted file mode 100644
index 162864f..0000000
--- a/internal/utils/compression.go
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package utils
-
-import (
- "bytes"
- "compress/zlib"
- "github.com/apache/rocketmq-client-go/v2/errors"
- "io/ioutil"
- "sync"
-)
-
-var zlibWriterPools []sync.Pool
-
-var bufPool = sync.Pool{
- New: func() interface{} {
- return &bytes.Buffer{}
- },
-}
-
-func init() {
- zlibWriterPools = make([]sync.Pool, zlib.BestCompression)
- for i := 0; i < zlib.BestCompression; i++ {
- compressLevel := i
- zlibWriterPools[i] = sync.Pool{
- New: func() interface{} {
- z, _ := zlib.NewWriterLevel(nil, compressLevel+1)
- return z
- },
- }
- }
-}
-
-func Compress(raw []byte, compressLevel int) ([]byte, error) {
- if compressLevel < zlib.BestSpeed || compressLevel > zlib.BestCompression {
- return nil, errors.ErrCompressLevel
- }
-
- buf := bufPool.Get().(*bytes.Buffer)
- defer bufPool.Put(buf)
- writerPool := &zlibWriterPools[compressLevel-1]
- writer := writerPool.Get().(*zlib.Writer)
- defer writerPool.Put(writer)
- buf.Reset()
- writer.Reset(buf)
- _, e := writer.Write(raw)
- if e != nil {
- return nil, e
- }
-
- e = writer.Close()
- if e != nil {
- return nil, e
- }
- result := make([]byte, buf.Len())
- buf.Read(result)
- return result, nil
-}
-
-func UnCompress(data []byte) []byte {
- rdata := bytes.NewReader(data)
- r, err := zlib.NewReader(rdata)
- if err != nil {
- return data
- }
- retData, err := ioutil.ReadAll(r)
- if err != nil {
- return data
- }
- return retData
-}
diff --git a/internal/utils/compression_test.go b/internal/utils/compression_test.go
deleted file mode 100644
index 4d10893..0000000
--- a/internal/utils/compression_test.go
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package utils
-
-import (
- "bytes"
- "compress/zlib"
- "encoding/json"
- "fmt"
- "math/rand"
- "testing"
-)
-
-func TestUnCompress(t *testing.T) {
- var b bytes.Buffer
- var oriStr string = "hello, go"
- zr := zlib.NewWriter(&b)
- zr.Write([]byte(oriStr))
- zr.Close()
-
- retBytes := UnCompress(b.Bytes())
- if string(retBytes) != oriStr {
- t.Errorf("UnCompress was incorrect, got %s, want: %s .", retBytes, []byte(oriStr))
- }
-}
-
-func TestCompress(t *testing.T) {
- raw := []byte("The quick brown fox jumps over the lazy dog")
- for i := zlib.BestSpeed; i <= zlib.BestCompression; i++ {
- compressed, e := Compress(raw, i)
- if e != nil {
- t.Errorf("Compress data:%s returns error: %v", string(raw), e)
- return
- }
- decompressed := UnCompress(compressed)
- if string(decompressed) != string(raw) {
- t.Errorf("data is corrupt, got: %s, want: %s", string(decompressed), string(raw))
- }
- }
-}
-
-func testCase(data []byte, level int, t *testing.T) {
- compressed, e := Compress(data, level)
- if e != nil {
- t.Errorf("Compress data:%v returns error: %v", data, e)
- }
- decompressed := UnCompress(compressed)
- if string(data) != string(decompressed) {
- t.Errorf("data is corrupt, got: %s, want: %s", string(decompressed), string(data))
- }
-}
-
-func generateRandTestData(n int) []byte {
- data := make([]byte, n)
- rand.Read(data)
- return data
-}
-
-func generateJsonString(n int) []byte {
- x := make(map[string]string)
- for i := 0; i < n; i++ {
- k := fmt.Sprintf("compression_key_%d", i)
- v := fmt.Sprintf("compression_value_%d", i)
- x[k] = v
- }
- data, _ := json.Marshal(x)
- return data
-}
-
-func TestCompressThreadSafe(t *testing.T) {
- for i := 0; i < 100; i++ {
- data := generateRandTestData(i * 100)
- level := i%zlib.BestCompression + 1
- go testCase(data, level, t)
- }
-
- for i := 0; i < 100; i++ {
- data := generateJsonString(i * 100)
- level := i%zlib.BestCompression + 1
- go testCase(data, level, t)
- }
-}
diff --git a/internal/utils/errors.go b/internal/utils/errors.go
deleted file mode 100644
index 0887a37..0000000
--- a/internal/utils/errors.go
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package utils
-
-import (
- "github.com/apache/rocketmq-client-go/v2/rlog"
-)
-
-func CheckError(action string, err error) {
- if err != nil {
- rlog.Error(action, map[string]interface{}{
- rlog.LogKeyUnderlayError: err.Error(),
- })
- }
-}
diff --git a/internal/utils/files.go b/internal/utils/files.go
deleted file mode 100644
index 2583d11..0000000
--- a/internal/utils/files.go
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package utils
-
-import (
- "errors"
- "fmt"
- "os"
- "path/filepath"
-)
-
-func FileReadAll(path string) ([]byte, error) {
- stat, err := os.Stat(path)
- if err != nil {
- return nil, err
- }
- file, err := os.Open(path)
- if err != nil {
- return nil, err
- }
- data := make([]byte, stat.Size())
- _, err = file.Read(data)
- if err != nil {
- return nil, err
- }
- return data, nil
-}
-
-func ensureDir(path string) error {
- info, err := os.Stat(path)
- if err != nil {
- if os.IsNotExist(err) {
- err = os.MkdirAll(path, 0755)
- }
- return err
- }
- if !info.IsDir() {
- return errors.New(path + " is a file")
- }
- return nil
-}
-
-func WriteToFile(path string, data []byte) error {
- if err := ensureDir(filepath.Dir(path)); err != nil {
- return err
- }
- tmpFile, err := os.Create(path + ".tmp")
- if err != nil {
- return err
- }
- _, err = tmpFile.Write(data)
- if err != nil {
- return err
- }
- CheckError(fmt.Sprintf("close %s", tmpFile.Name()), tmpFile.Close())
-
- prevContent, err := FileReadAll(path)
- if err == nil {
- bakFile, err := os.Create(path + ".bak")
- if err != nil {
- _, err = bakFile.Write(prevContent)
- }
- if err != nil {
- return err
- }
- CheckError(fmt.Sprintf("close %s", bakFile.Name()), bakFile.Close())
- }
-
- _, err = os.Stat(path)
- if err == nil {
- CheckError(fmt.Sprintf("remove %s", path), os.Remove(path))
- }
- return os.Rename(path+".tmp", path)
-}
diff --git a/internal/utils/math.go b/internal/utils/math.go
deleted file mode 100644
index 816631e..0000000
--- a/internal/utils/math.go
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package utils
-
-func AbsInt(i int) int {
- if i >= 0 {
- return i
- }
- return -i
-}
-
-func MinInt(a, b int) int {
- if a < b {
- return a
- }
- return b
-}
diff --git a/internal/utils/net.go b/internal/utils/net.go
deleted file mode 100644
index a4eeb56..0000000
--- a/internal/utils/net.go
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package utils
-
-import (
- "bytes"
- "fmt"
- "github.com/apache/rocketmq-client-go/v2/errors"
- "net"
- "strconv"
- "time"
-)
-
-var (
- LocalIP string
-)
-
-func init() {
- ip, err := ClientIP4()
- if err != nil {
- LocalIP = ""
- } else {
- LocalIP = fmt.Sprintf("%d.%d.%d.%d", ip[0], ip[1], ip[2], ip[3])
- }
-}
-
-func ClientIP4() ([]byte, error) {
- if ifaces, err := net.Interfaces(); err == nil && ifaces != nil {
- for _, iface := range ifaces {
- if iface.Flags&net.FlagLoopback != 0 || iface.Flags&net.FlagUp == 0 {
- continue
- }
- if addrs, err := iface.Addrs(); err == nil && addrs != nil {
- for _, addr := range addrs {
- if ipnet, ok := addr.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
- if ip4 := ipnet.IP.To4(); ip4 != nil {
- return ip4, nil
- }
- }
- }
- }
- }
- }
- return nil, errors.ErrUnknownIP
-}
-
-func FakeIP() []byte {
- buf := bytes.NewBufferString("")
- buf.WriteString(strconv.FormatInt(time.Now().UnixNano()/int64(time.Millisecond), 10))
- return buf.Bytes()[4:8]
-}
-
-func GetAddressByBytes(data []byte) string {
- return net.IPv4(data[0], data[1], data[2], data[3]).String()
-}
diff --git a/internal/utils/net_test.go b/internal/utils/net_test.go
deleted file mode 100644
index 3e65a75..0000000
--- a/internal/utils/net_test.go
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package utils
-
-import "testing"
-
-func TestLocalIP2(t *testing.T) {
- t.Log(LocalIP)
-}
diff --git a/internal/utils/set.go b/internal/utils/set.go
deleted file mode 100644
index e90fb36..0000000
--- a/internal/utils/set.go
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package utils
-
-import (
- "bytes"
- "encoding/json"
- "sort"
-)
-
-type UniqueItem interface {
- UniqueID() string
-}
-
-type StringUnique string
-
-func (str StringUnique) UniqueID() string {
- return string(str)
-}
-
-type Set struct {
- items map[string]UniqueItem
-}
-
-func NewSet() Set {
- return Set{
- items: make(map[string]UniqueItem, 0),
- }
-}
-
-func (s *Set) Add(v UniqueItem) {
- s.items[v.UniqueID()] = v
-}
-
-func (s *Set) AddKV(k, v string) {
- s.items[k] = StringUnique(v)
-}
-
-func (s *Set) Contains(k string) (UniqueItem, bool) {
- v, ok := s.items[k]
- return v, ok
-}
-
-func (s *Set) Len() int {
- return len(s.items)
-}
-
-var _ json.Marshaler = &Set{}
-
-func (s *Set) MarshalJSON() ([]byte, error) {
- if len(s.items) == 0 {
- return []byte("[]"), nil
- }
-
- buffer := new(bytes.Buffer)
- buffer.WriteByte('[')
- keys := make([]string, 0)
- for _, k := range s.items {
- var key string
- switch kval := k.(type) {
- case StringUnique:
- key = "\"" + string(kval) + "\""
- default:
- v, err := json.Marshal(k)
- if err != nil {
- return nil, err
- }
- key = string(v)
- }
- keys = append(keys, key)
- }
- sort.Slice(keys, func(i, j int) bool { return keys[i] < keys[j] })
-
- for i, key := range keys {
- if i > 0 {
- buffer.WriteByte(',')
- }
- buffer.WriteString(key)
- }
-
- buffer.WriteByte(']')
-
- return buffer.Bytes(), nil
-}
-
-func (s Set) UnmarshalJSON(data []byte) (err error) {
- return nil
-}
diff --git a/internal/utils/string.go b/internal/utils/string.go
deleted file mode 100644
index f347397..0000000
--- a/internal/utils/string.go
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package utils
-
-// HashString hashes a string to a unique hashcode.
-func HashString(s string) int {
- val := []byte(s)
- var h int32
-
- for idx := range val {
- h = 31*h + int32(val[idx])
- }
-
- return int(h)
-}
diff --git a/internal/validators.go b/internal/validators.go
deleted file mode 100644
index ac51db2..0000000
--- a/internal/validators.go
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package internal
-
-import (
- "regexp"
-
- "github.com/apache/rocketmq-client-go/v2/rlog"
-)
-
-const (
- _ValidPattern = "^[%|a-zA-Z0-9_-]+$"
- _CharacterMaxLength = 255
-)
-
-var (
- _Pattern, _ = regexp.Compile(_ValidPattern)
-)
-
-func ValidateGroup(group string) {
- if group == "" {
- rlog.Fatal("consumerGroup is empty", nil)
- }
-
- if len(group) > _CharacterMaxLength {
- rlog.Fatal("the specified group is longer than group max length 255.", nil)
- }
-}
diff --git a/primitive/auth.go b/primitive/auth.go
deleted file mode 100644
index 772bc4d..0000000
--- a/primitive/auth.go
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package primitive
-
-type Credentials struct {
- AccessKey string
- SecretKey string
- SecurityToken string
-}
-
-func (c Credentials) IsEmpty() bool {
- return c.AccessKey == "" || c.SecretKey == ""
-}
diff --git a/primitive/base.go b/primitive/base.go
deleted file mode 100644
index a45fbb9..0000000
--- a/primitive/base.go
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package primitive
-
-import (
- "github.com/apache/rocketmq-client-go/v2/errors"
- "regexp"
- "strings"
-)
-
-var (
- ipv4Regex, _ = regexp.Compile(`^((25[0-5]|2[0-4]\d|((1\d{2})|([1-9]?\d)))\.){3}(25[0-5]|2[0-4]\d|((1\d{2})|([1-9]?\d)))`)
- ipv6Regex, _ = regexp.Compile(`(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))`)
-)
-
-type NamesrvAddr []string
-
-func NewNamesrvAddr(s ...string) (NamesrvAddr, error) {
- if len(s) == 0 {
- return nil, errors.ErrNoNameserver
- }
-
- ss := s
- if len(ss) == 1 {
- // compatible with multi server env string: "a;b;c"
- ss = strings.Split(s[0], ";")
- }
-
- for _, srv := range ss {
- if err := verifyIP(srv); err != nil {
- return nil, err
- }
- }
-
- addrs := make(NamesrvAddr, 0)
- addrs = append(addrs, ss...)
- return addrs, nil
-}
-
-func (addr NamesrvAddr) Check() error {
- for _, srv := range addr {
- if err := verifyIP(srv); err != nil {
- return err
- }
- }
- return nil
-}
-
-var (
- httpPrefixRegex, _ = regexp.Compile("^(http|https)://")
-)
-
-func verifyIP(ip string) error {
- if httpPrefixRegex.MatchString(ip) {
- return nil
- }
- if strings.Contains(ip, ";") {
- return errors.ErrMultiIP
- }
- ipV4s := ipv4Regex.FindAllString(ip, -1)
- ipV6s := ipv6Regex.FindAllString(ip, -1)
-
- if len(ipV4s) == 0 && len(ipV6s) == 0 {
- return errors.ErrIllegalIP
- }
-
- if len(ipV4s) > 1 || len(ipV6s) > 1 {
- return errors.ErrMultiIP
- }
- return nil
-}
-
-var PanicHandler func(interface{})
-
-func WithRecover(fn func()) {
- defer func() {
- handler := PanicHandler
- if handler != nil {
- if err := recover(); err != nil {
- handler(err)
- }
- }
- }()
-
- fn()
-}
-
-func Diff(origin, latest []string) bool {
- if len(origin) != len(latest) {
- return true
- }
-
- // check added
- originFilter := make(map[string]struct{}, len(origin))
- for _, srv := range origin {
- originFilter[srv] = struct{}{}
- }
-
- latestFilter := make(map[string]struct{}, len(latest))
- for _, srv := range latest {
- if _, ok := originFilter[srv]; !ok {
- return true // added
- }
- latestFilter[srv] = struct{}{}
- }
-
- // check delete
- for _, srv := range origin {
- if _, ok := latestFilter[srv]; !ok {
- return true // deleted
- }
- }
- return false
-}
diff --git a/primitive/base_test.go b/primitive/base_test.go
deleted file mode 100644
index db947c4..0000000
--- a/primitive/base_test.go
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package primitive
-
-import (
- "testing"
-
- "github.com/stretchr/testify/assert"
-)
-
-func TestVerifyIP(t *testing.T) {
- IPs := "127.0.0.1:9876"
- err := verifyIP(IPs)
- assert.Nil(t, err)
-
- IPs = "12.24.123.243:10911"
- err = verifyIP(IPs)
- assert.Nil(t, err)
-
- IPs = "xa2.0.0.1:9876"
- err = verifyIP(IPs)
- assert.Equal(t, "IP addr error", err.Error())
-
- IPs = "333.0.0.1:9876"
- err = verifyIP(IPs)
- assert.Equal(t, "IP addr error", err.Error())
-
- IPs = "127.0.0.1:9876;12.24.123.243:10911"
- err = verifyIP(IPs)
- assert.Equal(t, "multiple IP addr does not support", err.Error())
-
- IPs = "bdbd:bdbd:ff:1:1:2:3:4:8888"
- err = verifyIP(IPs)
- assert.Nil(t, err)
-
- IPs = "[bdbd:bdbd:ff:1:1:2:3:4]:8888"
- err = verifyIP(IPs)
- assert.Nil(t, err)
-
- IPs = "[bdbd:bdbd:ff:1:1:2:3:4]:8888;[bdbd:bdbd:ff:1:1:2:3:4]:8889"
- err = verifyIP(IPs)
- assert.Equal(t, "multiple IP addr does not support", err.Error())
-}
-
-func TestBase(t *testing.T) {
- a := []string{}
- b := []string{}
- assert.False(t, Diff(a, b))
-
- a = []string{"a"}
- b = []string{"a", "b"}
- assert.True(t, Diff(a, b))
-
- a = []string{"a", "b", "c"}
- b = []string{"c", "a", "b"}
- assert.False(t, Diff(a, b))
-
- a = []string{"b", "a"}
- b = []string{"a", "c"}
- assert.True(t, Diff(a, b))
-}
diff --git a/primitive/ctx.go b/primitive/ctx.go
deleted file mode 100644
index 936b54c..0000000
--- a/primitive/ctx.go
+++ /dev/null
@@ -1,173 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-/*
- * Define the ctx key and value type.
- */
-package primitive
-
-import (
- "context"
- "math"
-
- "github.com/apache/rocketmq-client-go/v2/rlog"
-)
-
-type CtxKey int
-
-type CommunicationMode string
-
-type ConsumeReturnType string
-
-func (c ConsumeReturnType) Ordinal() int {
- switch c {
- case SuccessReturn:
- return 0
- case TimeoutReturn:
- return 1
- case ExceptionReturn:
- return 2
- case NullReturn:
- return 3
- case FailedReturn:
- return 4
- default:
- rlog.Error("Illegal Consumer Return Type", map[string]interface{}{
- "type": c,
- })
- return 0
- }
-}
-
-const (
- method CtxKey = iota
- msgCtx
- orderlyCtx
- concurrentlyCtx
- producerCtx
-
- // method name in producer
- SendSync CommunicationMode = "SendSync"
- SendOneway CommunicationMode = "SendOneway"
- SendAsync CommunicationMode = "SendAsync"
- // method name in consumer
- ConsumerPush = "ConsumerPush"
- ConsumerPull = "ConsumerPull"
-
- PropCtxType = "ConsumeContextType"
- SuccessReturn ConsumeReturnType = "SUCCESS"
- TimeoutReturn ConsumeReturnType = "TIMEOUT"
- ExceptionReturn ConsumeReturnType = "EXCEPTION"
- NullReturn ConsumeReturnType = "RETURNNULL"
- FailedReturn ConsumeReturnType = "FAILED"
-)
-
-type ConsumeMessageContext struct {
- ConsumerGroup string
- Msgs []*MessageExt
- MQ *MessageQueue
- Success bool
- Status string
- // mqTractContext
- Properties map[string]string
-}
-
-// WithMethod set call method name
-func WithMethod(ctx context.Context, m CommunicationMode) context.Context {
- return context.WithValue(ctx, method, m)
-}
-
-// GetMethod get call method name
-func GetMethod(ctx context.Context) CommunicationMode {
- return ctx.Value(method).(CommunicationMode)
-}
-
-// WithConsumerCtx set ConsumeMessageContext in PushConsumer
-func WithConsumerCtx(ctx context.Context, c *ConsumeMessageContext) context.Context {
- return context.WithValue(ctx, msgCtx, c)
-}
-
-// GetConsumerCtx get ConsumeMessageContext, only legal in PushConsumer. so should add bool return param indicate
-// whether exist.
-func GetConsumerCtx(ctx context.Context) (*ConsumeMessageContext, bool) {
- c, exist := ctx.Value(msgCtx).(*ConsumeMessageContext)
- return c, exist
-}
-
-type ConsumeOrderlyContext struct {
- MQ MessageQueue
- AutoCommit bool
- SuspendCurrentQueueTimeMillis int
-}
-
-func NewConsumeOrderlyContext() *ConsumeOrderlyContext {
- return &ConsumeOrderlyContext{
- AutoCommit: true,
- SuspendCurrentQueueTimeMillis: -1,
- }
-}
-
-func WithOrderlyCtx(ctx context.Context, c *ConsumeOrderlyContext) context.Context {
- return context.WithValue(ctx, orderlyCtx, c)
-}
-
-func GetOrderlyCtx(ctx context.Context) (*ConsumeOrderlyContext, bool) {
- c, exist := ctx.Value(orderlyCtx).(*ConsumeOrderlyContext)
- return c, exist
-}
-
-type ConsumeConcurrentlyContext struct {
- MQ MessageQueue
- DelayLevelWhenNextConsume int
- AckIndex int32
-}
-
-func NewConsumeConcurrentlyContext() *ConsumeConcurrentlyContext {
- return &ConsumeConcurrentlyContext{
- AckIndex: math.MaxInt32,
- }
-}
-
-func WithConcurrentlyCtx(ctx context.Context, c *ConsumeConcurrentlyContext) context.Context {
- return context.WithValue(ctx, concurrentlyCtx, c)
-}
-
-func GetConcurrentlyCtx(ctx context.Context) (*ConsumeConcurrentlyContext, bool) {
- c, exist := ctx.Value(concurrentlyCtx).(*ConsumeConcurrentlyContext)
- return c, exist
-}
-
-type ProducerCtx struct {
- ProducerGroup string
- Message Message
- MQ MessageQueue
- BrokerAddr string
- BornHost string
- CommunicationMode CommunicationMode
- SendResult *SendResult
- Props map[string]string
- MsgType MessageType
- Namespace string
-}
-
-func WithProducerCtx(ctx context.Context, c *ProducerCtx) context.Context {
- return context.WithValue(ctx, producerCtx, c)
-}
-
-func GetProducerCtx(ctx context.Context) *ProducerCtx {
- return ctx.Value(producerCtx).(*ProducerCtx)
-}
diff --git a/primitive/errors.go b/primitive/errors.go
deleted file mode 100644
index 0fc8a7f..0000000
--- a/primitive/errors.go
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package primitive
-
-import (
- "strconv"
-)
-
-type MQBrokerErr struct {
- ResponseCode int16
- ErrorMessage string
-}
-
-func (e MQBrokerErr) Error() string {
- return "CODE: " + strconv.Itoa(int(e.ResponseCode)) + " DESC: " + e.ErrorMessage
-}
-
-func NewRemotingErr(s string) error {
- return &RemotingErr{s: s}
-}
-
-type RemotingErr struct {
- s string
-}
-
-func (e *RemotingErr) Error() string {
- return e.s
-}
-
-func NewMQClientErr(code int16, msg string) error {
- return &MQClientErr{code: code, msg: msg}
-}
-
-type MQClientErr struct {
- code int16
- msg string
-}
-
-func (e MQClientErr) Error() string {
- return "CODE: " + strconv.Itoa(int(e.code)) + " DESC: " + e.msg
-}
-
-func IsRemotingErr(err error) bool {
- _, ok := err.(*RemotingErr)
- return ok
-}
diff --git a/primitive/interceptor.go b/primitive/interceptor.go
deleted file mode 100644
index 878aab5..0000000
--- a/primitive/interceptor.go
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package primitive
-
-import (
- "context"
-)
-
-// Invoker finish a message invoke on producer/consumer.
-type Invoker func(ctx context.Context, req, reply interface{}) error
-
-// Interceptor intercepts the invoke of a producer/consumer on messages.
-// In PushConsumer call, the req is []*MessageExt type and the reply is ConsumeResultHolder,
-// use type assert to get real type.
-type Interceptor func(ctx context.Context, req, reply interface{}, next Invoker) error
-
-func ChainInterceptors(interceptors ...Interceptor) Interceptor {
- if len(interceptors) == 0 {
- return nil
- }
- if len(interceptors) == 1 {
- return interceptors[0]
- }
- return func(ctx context.Context, req, reply interface{}, invoker Invoker) error {
- return interceptors[0](ctx, req, reply, getChainedInterceptor(interceptors, 0, invoker))
- }
-}
-
-func getChainedInterceptor(interceptors []Interceptor, cur int, finalInvoker Invoker) Invoker {
- if cur == len(interceptors)-1 {
- return finalInvoker
- }
- return func(ctx context.Context, req, reply interface{}) error {
- return interceptors[cur+1](ctx, req, reply, getChainedInterceptor(interceptors, cur+1, finalInvoker))
- }
-}
diff --git a/primitive/message.go b/primitive/message.go
deleted file mode 100644
index b8e8f83..0000000
--- a/primitive/message.go
+++ /dev/null
@@ -1,569 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package primitive
-
-import (
- "bytes"
- "encoding/binary"
- "encoding/hex"
- "fmt"
- "os"
- "strconv"
- "strings"
- "sync"
- "time"
-
- "github.com/apache/rocketmq-client-go/v2/internal/utils"
-)
-
-const (
- PropertyKeySeparator = " "
- PropertyKeys = "KEYS"
- PropertyTags = "TAGS"
- PropertyWaitStoreMsgOk = "WAIT"
- PropertyDelayTimeLevel = "DELAY"
- PropertyRetryTopic = "RETRY_TOPIC"
- PropertyRealTopic = "REAL_TOPIC"
- PropertyRealQueueId = "REAL_QID"
- PropertyTransactionPrepared = "TRAN_MSG"
- PropertyProducerGroup = "PGROUP"
- PropertyMinOffset = "MIN_OFFSET"
- PropertyMaxOffset = "MAX_OFFSET"
- PropertyBuyerId = "BUYER_ID"
- PropertyOriginMessageId = "ORIGIN_MESSAGE_ID"
- PropertyTransferFlag = "TRANSFER_FLAG"
- PropertyCorrectionFlag = "CORRECTION_FLAG"
- PropertyMQ2Flag = "MQ2_FLAG"
- PropertyReconsumeTime = "RECONSUME_TIME"
- PropertyMsgRegion = "MSG_REGION"
- PropertyTraceSwitch = "TRACE_ON"
- PropertyUniqueClientMessageIdKeyIndex = "UNIQ_KEY"
- PropertyMaxReconsumeTimes = "MAX_RECONSUME_TIMES"
- PropertyConsumeStartTime = "CONSUME_START_TIME"
- PropertyTranscationPreparedQueueOffset = "TRAN_PREPARED_QUEUE_OFFSET"
- PropertyTranscationCheckTimes = "TRANSACTION_CHECK_TIMES"
- PropertyCheckImmunityTimeInSeconds = "CHECK_IMMUNITY_TIME_IN_SECONDS"
- PropertyShardingKey = "SHARDING_KEY"
- PropertyTransactionID = "__transactionId__"
-)
-
-type Message struct {
- Topic string
- Body []byte
- Flag int32
- TransactionId string
- Batch bool
- Compress bool
- // Queue is the queue that messages will be sent to. the value must be set if want to custom the queue of message,
- // just ignore if not.
- Queue *MessageQueue
-
- properties map[string]string
- mutex sync.RWMutex
-}
-
-func (m *Message) WithProperties(p map[string]string) {
- m.mutex.Lock()
- m.properties = p
- m.mutex.Unlock()
-}
-
-func (m *Message) WithProperty(key, value string) {
- if key == "" || value == "" {
- return
- }
- m.mutex.Lock()
- if m.properties == nil {
- m.properties = make(map[string]string)
- }
- m.properties[key] = value
- m.mutex.Unlock()
-}
-
-func (m *Message) GetProperty(key string) string {
- m.mutex.RLock()
- v := m.properties[key]
- m.mutex.RUnlock()
- return v
-}
-
-func (m *Message) RemoveProperty(key string) string {
- m.mutex.Lock()
- defer m.mutex.Unlock()
- value, exist := m.properties[key]
- if !exist {
- return ""
- }
- delete(m.properties, key)
- return value
-}
-func (m *Message) MarshallProperties() string {
- m.mutex.RLock()
- defer m.mutex.RUnlock()
- buffer := bytes.NewBufferString("")
- for k, v := range m.properties {
- buffer.WriteString(k)
- buffer.WriteRune(nameValueSeparator)
- buffer.WriteString(v)
- buffer.WriteRune(propertySeparator)
- }
- return buffer.String()
-}
-
-// unmarshalProperties parse data into property kv pairs.
-func (m *Message) UnmarshalProperties(data []byte) {
- m.mutex.Lock()
- defer m.mutex.Unlock()
- if m.properties == nil {
- m.properties = make(map[string]string)
- }
- items := bytes.Split(data, []byte{propertySeparator})
- for _, item := range items {
- kv := bytes.Split(item, []byte{nameValueSeparator})
- if len(kv) == 2 {
- m.properties[string(kv[0])] = string(kv[1])
- }
- }
-}
-
-func (m *Message) GetProperties() map[string]string {
- m.mutex.Lock()
- defer m.mutex.Unlock()
- result := make(map[string]string, len(m.properties))
- for k, v := range m.properties {
- result[k] = v
- }
- return result
-}
-
-func NewMessage(topic string, body []byte) *Message {
- msg := &Message{
- Topic: topic,
- Body: body,
- properties: make(map[string]string),
- }
- //msg.properties[PropertyWaitStoreMsgOk] = strconv.FormatBool(true)
- return msg
-}
-
-// WithDelayTimeLevel set message delay time to consume.
-// reference delay level definition: 1s 5s 10s 30s 1m 2m 3m 4m 5m 6m 7m 8m 9m 10m 20m 30m 1h 2h
-// delay level starts from 1. for example, if we set param level=1, then the delay time is 1s.
-func (m *Message) WithDelayTimeLevel(level int) *Message {
- m.WithProperty(PropertyDelayTimeLevel, strconv.Itoa(level))
- return m
-}
-
-func (m *Message) WithTag(tags string) *Message {
- m.WithProperty(PropertyTags, tags)
- return m
-}
-
-func (m *Message) WithKeys(keys []string) *Message {
- var sb strings.Builder
- for _, k := range keys {
- sb.WriteString(k)
- sb.WriteString(PropertyKeySeparator)
- }
-
- m.WithProperty(PropertyKeys, sb.String())
- return m
-}
-
-func (m *Message) WithShardingKey(key string) *Message {
- m.WithProperty(PropertyShardingKey, key)
- return m
-}
-
-func (m *Message) GetTags() string {
- return m.GetProperty(PropertyTags)
-}
-
-func (m *Message) GetKeys() string {
- return m.GetProperty(PropertyKeys)
-}
-
-func (m *Message) GetShardingKey() string {
- return m.GetProperty(PropertyShardingKey)
-}
-
-func (m *Message) String() string {
- return fmt.Sprintf("[topic=%s, body=%s, Flag=%d, properties=%v, TransactionId=%s]",
- m.Topic, string(m.Body), m.Flag, m.properties, m.TransactionId)
-}
-
-func (m *Message) Marshal() []byte {
- // storeSize all size of message info.
- // TOTALSIZE MAGICCOD BODYCRC FLAG BODYSIZE BODY PROPERTYSIZE PROPERTY
- v := m.MarshallProperties()
- properties := []byte(v)
- storeSize := 4 + 4 + 4 + 4 + 4 + len(m.Body) + 2 + len(properties)
-
- buffer := make([]byte, storeSize)
- pos := 0
- binary.BigEndian.PutUint32(buffer[pos:], uint32(storeSize)) // 1. TOTALSIZE
- pos += 4
- binary.BigEndian.PutUint32(buffer[pos:], 0) // 2. MAGICCODE
- pos += 4
- binary.BigEndian.PutUint32(buffer[pos:], 0) // 3. BODYCRC
- pos += 4
- binary.BigEndian.PutUint32(buffer[pos:], uint32(m.Flag)) // 4. FLAG
- pos += 4
- binary.BigEndian.PutUint32(buffer[pos:], uint32(len(m.Body))) // 5. BODYSIZE
- pos += 4
- copy(buffer[pos:], m.Body)
- pos += len(m.Body)
-
- binary.BigEndian.PutUint16(buffer[pos:], uint16(len(properties))) // 7. PROPERTYSIZE
- pos += 2
- copy(buffer[pos:], properties)
-
- return buffer
-}
-
-type MessageExt struct {
- Message
- MsgId string
- OffsetMsgId string
- StoreSize int32
- QueueOffset int64
- SysFlag int32
- BornTimestamp int64
- BornHost string
- StoreTimestamp int64
- StoreHost string
- CommitLogOffset int64
- BodyCRC int32
- ReconsumeTimes int32
- PreparedTransactionOffset int64
-}
-
-func (msgExt *MessageExt) GetTags() string {
- return msgExt.GetProperty(PropertyTags)
-}
-
-func (msgExt *MessageExt) GetRegionID() string {
- return msgExt.GetProperty(PropertyMsgRegion)
-}
-
-func (msgExt *MessageExt) IsTraceOn() string {
- return msgExt.GetProperty(PropertyTraceSwitch)
-}
-
-func (msgExt *MessageExt) String() string {
- return fmt.Sprintf("[Message=%s, MsgId=%s, OffsetMsgId=%s,QueueId=%d, StoreSize=%d, QueueOffset=%d, SysFlag=%d, "+
- "BornTimestamp=%d, BornHost=%s, StoreTimestamp=%d, StoreHost=%s, CommitLogOffset=%d, BodyCRC=%d, "+
- "ReconsumeTimes=%d, PreparedTransactionOffset=%d]", msgExt.Message.String(), msgExt.MsgId, msgExt.OffsetMsgId, msgExt.Queue.QueueId,
- msgExt.StoreSize, msgExt.QueueOffset, msgExt.SysFlag, msgExt.BornTimestamp, msgExt.BornHost,
- msgExt.StoreTimestamp, msgExt.StoreHost, msgExt.CommitLogOffset, msgExt.BodyCRC, msgExt.ReconsumeTimes,
- msgExt.PreparedTransactionOffset)
-}
-
-func DecodeMessage(data []byte) []*MessageExt {
- msgs := make([]*MessageExt, 0)
- buf := bytes.NewBuffer(data)
- count := 0
- for count < len(data) {
- msg := &MessageExt{}
- msg.Queue = &MessageQueue{}
-
- // 1. total size
- binary.Read(buf, binary.BigEndian, &msg.StoreSize)
- count += 4
-
- // 2. magic code
- buf.Next(4)
- count += 4
-
- // 3. body CRC32
- binary.Read(buf, binary.BigEndian, &msg.BodyCRC)
- count += 4
-
- // 4. queueID
- var qId int32
- binary.Read(buf, binary.BigEndian, &qId)
- msg.Queue.QueueId = int(qId)
- count += 4
-
- // 5. Flag
- binary.Read(buf, binary.BigEndian, &msg.Flag)
- count += 4
-
- // 6. QueueOffset
- binary.Read(buf, binary.BigEndian, &msg.QueueOffset)
- count += 8
-
- // 7. physical offset
- binary.Read(buf, binary.BigEndian, &msg.CommitLogOffset)
- count += 8
-
- // 8. SysFlag
- binary.Read(buf, binary.BigEndian, &msg.SysFlag)
- count += 4
-
- // 9. BornTimestamp
- binary.Read(buf, binary.BigEndian, &msg.BornTimestamp)
- count += 8
-
- var (
- port int32
- hostBytes []byte
- )
- // 10. born host
- if msg.SysFlag&FlagBornHostV6 == FlagBornHostV6 {
- hostBytes = buf.Next(16)
- binary.Read(buf, binary.BigEndian, &port)
- msg.BornHost = fmt.Sprintf("%s:%d", utils.GetAddressByBytes(hostBytes), port)
- count += 20
- } else {
- hostBytes = buf.Next(4)
- binary.Read(buf, binary.BigEndian, &port)
- msg.BornHost = fmt.Sprintf("%s:%d", utils.GetAddressByBytes(hostBytes), port)
- count += 8
- }
-
- // 11. store timestamp
- binary.Read(buf, binary.BigEndian, &msg.StoreTimestamp)
- count += 8
-
- // 12. store host
- if msg.SysFlag&FlagStoreHostV6 == FlagStoreHostV6 {
- hostBytes = buf.Next(16)
- binary.Read(buf, binary.BigEndian, &port)
- msg.StoreHost = fmt.Sprintf("%s:%d", utils.GetAddressByBytes(hostBytes), port)
- count += 20
- } else {
- hostBytes = buf.Next(4)
- binary.Read(buf, binary.BigEndian, &port)
- msg.StoreHost = fmt.Sprintf("%s:%d", utils.GetAddressByBytes(hostBytes), port)
- count += 8
- }
-
- // 13. reconsume times
- binary.Read(buf, binary.BigEndian, &msg.ReconsumeTimes)
- count += 4
-
- // 14. prepared transaction offset
- binary.Read(buf, binary.BigEndian, &msg.PreparedTransactionOffset)
- count += 8
-
- // 15. body
- var length int32
- binary.Read(buf, binary.BigEndian, &length)
- msg.Body = buf.Next(int(length))
- if (msg.SysFlag & FlagCompressed) == FlagCompressed {
- msg.Body = utils.UnCompress(msg.Body)
- }
- count += 4 + int(length)
-
- // 16. topic
- _byte, _ := buf.ReadByte()
- msg.Topic = string(buf.Next(int(_byte)))
- count += 1 + int(_byte)
-
- // 17. properties
- var propertiesLength int16
- binary.Read(buf, binary.BigEndian, &propertiesLength)
- if propertiesLength > 0 {
- msg.UnmarshalProperties(buf.Next(int(propertiesLength)))
- }
- count += 2 + int(propertiesLength)
-
- msg.OffsetMsgId = CreateMessageId(hostBytes, port, msg.CommitLogOffset)
- //count += 16
- if msg.properties == nil {
- msg.properties = make(map[string]string, 0)
- }
- msgID := msg.GetProperty(PropertyUniqueClientMessageIdKeyIndex)
- if len(msgID) == 0 {
- msg.MsgId = msg.OffsetMsgId
- } else {
- msg.MsgId = msgID
- }
- msgs = append(msgs, msg)
- }
-
- return msgs
-}
-
-// MessageQueue message queue
-type MessageQueue struct {
- Topic string `json:"topic"`
- BrokerName string `json:"brokerName"`
- QueueId int `json:"queueId"`
-}
-
-func (mq *MessageQueue) String() string {
- return fmt.Sprintf("MessageQueue [topic=%s, brokerName=%s, queueId=%d]", mq.Topic, mq.BrokerName, mq.QueueId)
-}
-
-func (mq *MessageQueue) HashCode() int {
- result := 1
- result = 31*result + utils.HashString(mq.BrokerName)
- result = 31*result + mq.QueueId
- result = 31*result + utils.HashString(mq.Topic)
-
- return result
-}
-
-type AccessChannel int
-
-const (
- // connect to private IDC cluster.
- Local AccessChannel = iota
- // connect to Cloud service.
- Cloud
-)
-
-type MessageType int
-
-const (
- NormalMsg MessageType = iota
- TransMsgHalf
- TransMsgCommit
- DelayMsg
-)
-
-type LocalTransactionState int
-
-const (
- CommitMessageState LocalTransactionState = iota + 1
- RollbackMessageState
- UnknowState
-)
-
-type TransactionListener interface {
- // When send transactional prepare(half) message succeed, this method will be invoked to execute local transaction.
- ExecuteLocalTransaction(*Message) LocalTransactionState
-
- // When no response to prepare(half) message. broker will send check message to check the transaction status, and this
- // method will be invoked to get local transaction status.
- CheckLocalTransaction(*MessageExt) LocalTransactionState
-}
-
-type MessageID struct {
- Addr string
- Port int
- Offset int64
-}
-
-func CreateMessageId(addr []byte, port int32, offset int64) string {
- buffer := GetBuffer()
- defer BackBuffer(buffer)
- buffer.Write(addr)
- _ = binary.Write(buffer, binary.BigEndian, port)
- _ = binary.Write(buffer, binary.BigEndian, offset)
- return strings.ToUpper(hex.EncodeToString(buffer.Bytes()))
-}
-
-func UnmarshalMsgID(id []byte) (*MessageID, error) {
- if len(id) < 32 {
- return nil, fmt.Errorf("%s len < 32", string(id))
- }
- var (
- ipBytes = make([]byte, 4)
- portBytes = make([]byte, 4)
- offsetBytes = make([]byte, 8)
- )
- hex.Decode(ipBytes, id[0:8])
- hex.Decode(portBytes, id[8:16])
- hex.Decode(offsetBytes, id[16:32])
-
- return &MessageID{
- Addr: utils.GetAddressByBytes(ipBytes),
- Port: int(binary.BigEndian.Uint32(portBytes)),
- Offset: int64(binary.BigEndian.Uint64(offsetBytes)),
- }, nil
-}
-
-var (
- CompressedFlag = 0x1
-
- MultiTagsFlag = 0x1 << 1
-
- TransactionNotType = 0
-
- TransactionPreparedType = 0x1 << 2
-
- TransactionCommitType = 0x2 << 2
-
- TransactionRollbackType = 0x3 << 2
-)
-
-func GetTransactionValue(flag int) int {
- return flag & TransactionRollbackType
-}
-
-func ResetTransactionValue(flag int, typeFlag int) int {
- return (flag & (^TransactionRollbackType)) | typeFlag
-}
-
-func ClearCompressedFlag(flag int) int {
- return flag & (^CompressedFlag)
-}
-
-func SetCompressedFlag(flag int) int {
- return flag | CompressedFlag
-}
-
-var (
- counter int16 = 0
- startTimestamp int64 = 0
- nextTimestamp int64 = 0
- prefix string
- locker sync.Mutex
- classLoadId int32 = 0
-)
-
-func init() {
- buf := new(bytes.Buffer)
-
- ip, err := utils.ClientIP4()
- if err != nil {
- ip = utils.FakeIP()
- }
- _, _ = buf.Write(ip)
- _ = binary.Write(buf, binary.BigEndian, Pid())
- _ = binary.Write(buf, binary.BigEndian, classLoadId)
- prefix = strings.ToUpper(hex.EncodeToString(buf.Bytes()))
-}
-
-func CreateUniqID() string {
- locker.Lock()
- defer locker.Unlock()
-
- if time.Now().Unix() > nextTimestamp {
- updateTimestamp()
- }
- counter++
- buf := new(bytes.Buffer)
- _ = binary.Write(buf, binary.BigEndian, int32((time.Now().Unix()-startTimestamp)*1000))
- _ = binary.Write(buf, binary.BigEndian, counter)
-
- return prefix + hex.EncodeToString(buf.Bytes())
-}
-
-func updateTimestamp() {
- year, month := time.Now().Year(), time.Now().Month()
- startTimestamp = time.Date(year, month, 1, 0, 0, 0, 0, time.Local).Unix()
- nextTimestamp = time.Date(year, month, 1, 0, 0, 0, 0, time.Local).AddDate(0, 1, 0).Unix()
-}
-
-func Pid() int16 {
- return int16(os.Getpid())
-}
diff --git a/primitive/message_test.go b/primitive/message_test.go
deleted file mode 100644
index e47a298..0000000
--- a/primitive/message_test.go
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package primitive
-
-import "testing"
-
-func TestMessageID(t *testing.T) {
- id := []byte("0AAF0895000078BF000000000009BB4A")
- msgID, err := UnmarshalMsgID(id)
- if err != nil {
- t.Fatalf("unmarshal msg id error, ms is: %s", err.Error())
- }
- if msgID.Addr != "10.175.8.149" {
- t.Fatalf("parse messageID %s error", id)
- }
- if msgID.Port != 30911 {
- t.Fatalf("parse messageID %s error", id)
- }
- if msgID.Offset != 637770 {
- t.Fatalf("parse messageID %s error", id)
- }
- t.Log(msgID)
-}
diff --git a/primitive/nsresolver.go b/primitive/nsresolver.go
deleted file mode 100644
index 4e5917c..0000000
--- a/primitive/nsresolver.go
+++ /dev/null
@@ -1,223 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-package primitive
-
-import (
- "fmt"
- "io/ioutil"
- "net/http"
- "os"
- "os/user"
- "path"
- "strings"
- "time"
-
- "github.com/apache/rocketmq-client-go/v2/rlog"
-)
-
-// resolver for nameserver, monitor change of nameserver and notify client
-// consul or domain is common
-type NsResolver interface {
- Resolve() []string
- Description() string
-}
-
-type StaticResolver struct {
-}
-
-var _ NsResolver = (*EnvResolver)(nil)
-
-func NewEnvResolver() *EnvResolver {
- return &EnvResolver{}
-}
-
-type EnvResolver struct {
-}
-
-func (e *EnvResolver) Resolve() []string {
- if v := os.Getenv("NAMESRV_ADDR"); v != "" {
- return strings.Split(v, ";")
- }
- return nil
-}
-
-func (e *EnvResolver) Description() string {
- return "env resolver of var NAMESRV_ADDR"
-}
-
-type passthroughResolver struct {
- addr []string
- failback NsResolver
-}
-
-func NewPassthroughResolver(addr []string) *passthroughResolver {
- return &passthroughResolver{
- addr: addr,
- failback: NewEnvResolver(),
- }
-}
-
-func (p *passthroughResolver) Resolve() []string {
- if p.addr != nil {
- return p.addr
- }
- return p.failback.Resolve()
-}
-
-func (p *passthroughResolver) Description() string {
- return fmt.Sprintf("passthrough resolver of %v", p.addr)
-}
-
-const (
- DEFAULT_NAMESRV_ADDR = "http://jmenv.tbsite.net:8080/rocketmq/nsaddr"
-)
-
-var _ NsResolver = (*HttpResolver)(nil)
-
-type HttpResolver struct {
- domain string
- instance string
- cli http.Client
- failback NsResolver
-}
-
-func NewHttpResolver(instance string, domain ...string) *HttpResolver {
- d := DEFAULT_NAMESRV_ADDR
- if len(domain) > 0 {
- d = domain[0]
- }
- client := http.Client{Timeout: 10 * time.Second}
-
- h := &HttpResolver{
- domain: d,
- instance: instance,
- cli: client,
- failback: NewEnvResolver(),
- }
- return h
-}
-
-func (h *HttpResolver) Resolve() []string {
- addrs := h.get()
- if len(addrs) > 0 {
- return addrs
- }
-
- addrs = h.loadSnapshot()
- if len(addrs) > 0 {
- return addrs
- }
- return h.failback.Resolve()
-}
-
-func (h *HttpResolver) Description() string {
- return fmt.Sprintf("passthrough resolver of domain:%v instance:%v", h.domain, h.instance)
-}
-
-func (h *HttpResolver) get() []string {
- resp, err := h.cli.Get(h.domain)
- if err != nil || resp == nil || resp.StatusCode != 200 {
- data := map[string]interface{}{
- "NameServerDomain": h.domain,
- "err": err,
- }
- if resp != nil {
- data["StatusCode"] = resp.StatusCode
- }
- rlog.Error("name server http fetch failed", data)
- return nil
- }
-
- defer resp.Body.Close()
- body, err := ioutil.ReadAll(resp.Body)
- if err != nil {
- rlog.Error("name server read http response failed", map[string]interface{}{
- "NameServerDomain": h.domain,
- "err": err,
- })
- return nil
- }
-
- bodyStr := string(body)
- if bodyStr == "" {
- return nil
- }
-
- h.saveSnapshot(body)
-
- return strings.Split(string(body), ";")
-}
-
-func (h *HttpResolver) saveSnapshot(body []byte) error {
- filePath := h.getSnapshotFilePath(h.instance)
- err := ioutil.WriteFile(filePath, body, 0644)
- if err != nil {
- rlog.Error("name server snapshot save failed", map[string]interface{}{
- "filePath": filePath,
- "err": err,
- })
- return err
- }
-
- rlog.Info("name server snapshot save successfully", map[string]interface{}{
- "filePath": filePath,
- })
- return nil
-}
-
-func (h *HttpResolver) loadSnapshot() []string {
- filePath := h.getSnapshotFilePath(h.instance)
- _, err := os.Stat(filePath)
- if os.IsNotExist(err) {
- rlog.Warning("name server snapshot local file not exists", map[string]interface{}{
- "filePath": filePath,
- })
- return nil
- }
-
- bs, err := ioutil.ReadFile(filePath)
- if err != nil {
- return nil
- }
-
- rlog.Info("load the name server snapshot local file", map[string]interface{}{
- "filePath": filePath,
- })
- return strings.Split(string(bs), ";")
-}
-
-func (h *HttpResolver) getSnapshotFilePath(instanceName string) string {
- homeDir := ""
- if usr, err := user.Current(); err == nil {
- homeDir = usr.HomeDir
- } else {
- rlog.Error("name server domain, can't get user home directory", map[string]interface{}{
- "err": err,
- })
- }
- storePath := path.Join(homeDir, "/logs/rocketmq-go/snapshot")
- if _, err := os.Stat(storePath); os.IsNotExist(err) {
- if err = os.MkdirAll(storePath, 0755); err != nil {
- rlog.Fatal("can't create name server snapshot directory", map[string]interface{}{
- "path": storePath,
- "err": err,
- })
- }
- }
- filePath := path.Join(storePath, fmt.Sprintf("nameserver_addr-%s", instanceName))
- return filePath
-}
diff --git a/primitive/nsresolver_test.go b/primitive/nsresolver_test.go
deleted file mode 100644
index d42d2c6..0000000
--- a/primitive/nsresolver_test.go
+++ /dev/null
@@ -1,136 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-package primitive
-
-import (
- "fmt"
- "github.com/apache/rocketmq-client-go/v2/rlog"
- "io/ioutil"
- "net"
- "net/http"
- "os"
- "strings"
- "testing"
-
- . "github.com/smartystreets/goconvey/convey"
-)
-
-func TestEnvResolver(t *testing.T) {
- Convey("Test UpdateNameServerAddress Use Env", t, func() {
- srvs := []string{
- "192.168.100.1",
- "192.168.100.2",
- "192.168.100.3",
- "192.168.100.4",
- "192.168.100.5",
- }
-
- resolver := NewEnvResolver()
- os.Setenv("NAMESRV_ADDR", strings.Join(srvs, ";"))
-
- addrs := resolver.Resolve()
-
- So(Diff(srvs, addrs), ShouldBeFalse)
- })
-}
-
-func TestHttpResolverWithGet(t *testing.T) {
- Convey("Test UpdateNameServerAddress Save Local Snapshot", t, func() {
- srvs := []string{
- "192.168.100.1",
- "192.168.100.2",
- "192.168.100.3",
- "192.168.100.4",
- "192.168.100.5",
- }
- http.HandleFunc("/nameserver/addrs2", func(w http.ResponseWriter, r *http.Request) {
- fmt.Fprintf(w, strings.Join(srvs, ";"))
- })
- server := &http.Server{Addr: ":0", Handler: nil}
- listener, _ := net.Listen("tcp", ":0")
- go server.Serve(listener)
-
- port := listener.Addr().(*net.TCPAddr).Port
- nameServerDommain := fmt.Sprintf("http://127.0.0.1:%d/nameserver/addrs2", port)
- rlog.Info("Temporary Nameserver", map[string]interface{}{
- "domain": nameServerDommain,
- })
-
- resolver := NewHttpResolver("DEFAULT", nameServerDommain)
- resolver.Resolve()
-
- // check snapshot saved
- filePath := resolver.getSnapshotFilePath("DEFAULT")
- body := strings.Join(srvs, ";")
- bs, _ := ioutil.ReadFile(filePath)
- So(string(bs), ShouldEqual, body)
- })
-}
-
-func TestHttpResolverWithSnapshotFile(t *testing.T) {
- Convey("Test UpdateNameServerAddress Use Local Snapshot", t, func() {
- srvs := []string{
- "192.168.100.1",
- "192.168.100.2",
- "192.168.100.3",
- "192.168.100.4",
- "192.168.100.5",
- }
-
- resolver := NewHttpResolver("DEFAULT", "http://127.0.0.1:80/error/nsaddrs")
-
- os.Setenv("NAMESRV_ADDR", "") // clear env
- // setup local snapshot file
- filePath := resolver.getSnapshotFilePath("DEFAULT")
- body := strings.Join(srvs, ";")
- _ = ioutil.WriteFile(filePath, []byte(body), 0644)
-
- addrs := resolver.Resolve()
-
- So(Diff(addrs, srvs), ShouldBeFalse)
- })
-}
-
-func TesHttpReslverWithSnapshotFileOnce(t *testing.T) {
- Convey("Test UpdateNameServerAddress Load Local Snapshot Once", t, func() {
- srvs := []string{
- "192.168.100.1",
- "192.168.100.2",
- "192.168.100.3",
- "192.168.100.4",
- "192.168.100.5",
- }
-
- resolver := NewHttpResolver("DEFAULT", "http://127.0.0.1:80/error/nsaddrs")
-
- os.Setenv("NAMESRV_ADDR", "") // clear env
- // setup local snapshot file
- filePath := resolver.getSnapshotFilePath("DEFAULT")
- body := strings.Join(srvs, ";")
- _ = ioutil.WriteFile(filePath, []byte(body), 0644)
- // load local snapshot file first time
- addrs1 := resolver.Resolve()
-
- // change the local snapshot file to check load once
- _ = ioutil.WriteFile(filePath, []byte("127.0.0.1;127.0.0.2"), 0644)
-
- addrs2 := resolver.Resolve()
-
- So(Diff(addrs1, addrs2), ShouldBeFalse)
- So(Diff(addrs1, srvs), ShouldBeFalse)
- })
-}
diff --git a/primitive/pool.go b/primitive/pool.go
deleted file mode 100644
index 29cd834..0000000
--- a/primitive/pool.go
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package primitive
-
-import (
- "bytes"
- "sync"
-)
-
-var headerPool = sync.Pool{}
-var bufferPool = sync.Pool{}
-
-func init() {
- headerPool.New = func() interface{} {
- b := make([]byte, 4)
- return &b
- }
- bufferPool.New = func() interface{} {
- return new(bytes.Buffer)
- }
-}
-
-func GetHeader() *[]byte {
- d := headerPool.Get().(*[]byte)
- //d = (d)[:0]
- return d
-}
-
-func BackHeader(d *[]byte) {
- headerPool.Put(d)
-}
-
-func GetBuffer() *bytes.Buffer {
- b := bufferPool.Get().(*bytes.Buffer)
- b.Reset()
- return b
-}
-
-func BackBuffer(b *bytes.Buffer) {
- b.Reset()
- bufferPool.Put(b)
-}
diff --git a/primitive/result.go b/primitive/result.go
deleted file mode 100644
index 20d393a..0000000
--- a/primitive/result.go
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package primitive
-
-import (
- "fmt"
-)
-
-// SendStatus of message
-type SendStatus int
-
-const (
- SendOK SendStatus = iota
- SendFlushDiskTimeout
- SendFlushSlaveTimeout
- SendSlaveNotAvailable
- SendUnknownError
-
- FlagCompressed = 0x1
- FlagBornHostV6 = 0x1 << 4
- FlagStoreHostV6 = 0x1 << 5
- MsgIdLength = 8 + 8
-
- propertySeparator = '\002'
- nameValueSeparator = '\001'
-)
-
-// SendResult RocketMQ send result
-type SendResult struct {
- Status SendStatus
- MsgID string
- MessageQueue *MessageQueue
- QueueOffset int64
- TransactionID string
- OffsetMsgID string
- RegionID string
- TraceOn bool
-}
-
-func NewSendResult() *SendResult {
- return &SendResult{Status: SendUnknownError}
-}
-
-// SendResult send message result to string(detail result)
-func (result *SendResult) String() string {
- return fmt.Sprintf("SendResult [sendStatus=%d, msgIds=%s, offsetMsgId=%s, queueOffset=%d, messageQueue=%s]",
- result.Status, result.MsgID, result.OffsetMsgID, result.QueueOffset, result.MessageQueue.String())
-}
-
-// SendResult RocketMQ send result
-type TransactionSendResult struct {
- *SendResult
- State LocalTransactionState
-}
-
-// PullStatus pull Status
-type PullStatus int
-
-// predefined pull Status
-const (
- PullFound PullStatus = iota
- PullNoNewMsg
- PullNoMsgMatched
- PullOffsetIllegal
- PullBrokerTimeout
-)
-
-// PullResult the pull result
-type PullResult struct {
- NextBeginOffset int64
- MinOffset int64
- MaxOffset int64
- Status PullStatus
- SuggestWhichBrokerId int64
-
- // messageExts message info
- messageExts []*MessageExt
- //
- body []byte
-}
-
-func (result *PullResult) GetMessageExts() []*MessageExt {
- return result.messageExts
-}
-
-func (result *PullResult) SetMessageExts(msgExts []*MessageExt) {
- result.messageExts = msgExts
-}
-
-func (result *PullResult) GetMessages() []*Message {
- if len(result.messageExts) == 0 {
- return make([]*Message, 0)
- }
- return toMessages(result.messageExts)
-}
-
-func (result *PullResult) SetBody(data []byte) {
- result.body = data
-}
-
-func (result *PullResult) GetBody() []byte {
- return result.body
-}
-
-func (result *PullResult) String() string {
- return ""
-}
-
-func toMessages(messageExts []*MessageExt) []*Message {
- msgs := make([]*Message, 0)
-
- return msgs
-}
diff --git a/primitive/result_test.go b/primitive/result_test.go
deleted file mode 100644
index fcc03cf..0000000
--- a/primitive/result_test.go
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package primitive
-
-import (
- "testing"
-
- . "github.com/smartystreets/goconvey/convey"
- "github.com/stretchr/testify/assert"
-)
-
-func TestProperties(t *testing.T) {
- msg1 := NewMessage("test", nil)
- msg1.properties = map[string]string{
- "k1": "v1",
- "k2": "v2",
- }
- str := msg1.MarshallProperties()
- msg2 := NewMessage("test", nil)
- msg2.UnmarshalProperties([]byte(str))
- assert.Equal(t, msg1.properties, msg2.properties)
-}
-
-func TestCreateMessageId(t *testing.T) {
- Convey("MessageId gen", t, func() {
- b := []byte{10, 93, 233, 58}
- port := int32(10911)
- offset := int64(4391252)
- id := CreateMessageId(b, port, offset)
- Convey("generated messageId should be equal to expected", func() {
- assert.Equal(t, "0A5DE93A00002A9F0000000000430154", id)
- })
-
- b2 := []byte("127.0.0.1")
- port2 := int32(11)
- offset2 := int64(12)
- id2 := CreateMessageId(b2, port2, offset2)
- Convey("new generated messageId should be equal to expected", func() {
- assert.Equal(t, "3132372E302E302E310000000B000000000000000C", id2)
- })
-
- Convey("ex-generated messageId should not change", func() {
- assert.Equal(t, "0A5DE93A00002A9F0000000000430154", id)
- })
- })
-
-}
-
-func TestGetProperties(t *testing.T) {
- msg1 := NewMessage("test", nil)
- msg1.properties = map[string]string{
- "k1": "v1",
- "k2": "v2",
- }
- assert.Equal(t, msg1.GetProperties(), map[string]string{
- "k1": "v1",
- "k2": "v2",
- })
-}
diff --git a/primitive/trace.go b/primitive/trace.go
deleted file mode 100644
index 53a13f0..0000000
--- a/primitive/trace.go
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package primitive
-
-// config for message trace.
-type TraceConfig struct {
- TraceTopic string
- GroupName string
- Access AccessChannel
- NamesrvAddrs []string
- Resolver NsResolver
- Credentials // acl config for trace. omit if acl is closed on broker.
-}
diff --git a/producer/interceptor.go b/producer/interceptor.go
deleted file mode 100644
index 71eb8e7..0000000
--- a/producer/interceptor.go
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-/**
- * builtin interceptor
- */
-package producer
-
-import (
- "context"
- "fmt"
- "time"
-
- "github.com/apache/rocketmq-client-go/v2/internal"
- "github.com/apache/rocketmq-client-go/v2/internal/utils"
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-// WithTrace support rocketmq trace: https://github.com/apache/rocketmq/wiki/RIP-6-Message-Trace.
-func WithTrace(traceCfg *primitive.TraceConfig) Option {
- return func(options *producerOptions) {
-
- ori := options.Interceptors
- options.Interceptors = make([]primitive.Interceptor, 0)
- options.Interceptors = append(options.Interceptors, newTraceInterceptor(traceCfg))
- options.Interceptors = append(options.Interceptors, ori...)
- }
-}
-
-func newTraceInterceptor(traceCfg *primitive.TraceConfig) primitive.Interceptor {
- dispatcher := internal.NewTraceDispatcher(traceCfg)
- if dispatcher != nil {
- dispatcher.Start()
- }
-
- return func(ctx context.Context, req, reply interface{}, next primitive.Invoker) error {
- if dispatcher == nil {
- return fmt.Errorf("GetOrNewRocketMQClient faild")
- }
- beginT := time.Now()
- err := next(ctx, req, reply)
-
- producerCtx := primitive.GetProducerCtx(ctx)
- if producerCtx.Message.Topic == dispatcher.GetTraceTopicName() {
- return err
- }
-
- // SendOneway && SendAsync has no reply.
- if reply == nil {
- return err
- }
-
- result := reply.(*primitive.SendResult)
- if result.RegionID == "" || !result.TraceOn {
- return err
- }
-
- sendSuccess := result.Status == primitive.SendOK
- costT := time.Since(beginT).Nanoseconds() / int64(time.Millisecond)
- storeT := beginT.UnixNano()/int64(time.Millisecond) + costT/2
-
- traceBean := internal.TraceBean{
- Topic: producerCtx.Message.Topic,
- Tags: producerCtx.Message.GetTags(),
- Keys: producerCtx.Message.GetKeys(),
- StoreHost: producerCtx.BrokerAddr,
- ClientHost: utils.LocalIP,
- BodyLength: len(producerCtx.Message.Body),
- MsgType: producerCtx.MsgType,
- MsgId: result.MsgID,
- OffsetMsgId: result.OffsetMsgID,
- StoreTime: storeT,
- }
-
- traceCtx := internal.TraceContext{
- RequestId: primitive.CreateUniqID(), // set id
- TimeStamp: time.Now().UnixNano() / int64(time.Millisecond),
-
- TraceType: internal.Pub,
- GroupName: producerCtx.ProducerGroup,
- RegionId: result.RegionID,
- TraceBeans: []internal.TraceBean{traceBean},
- CostTime: costT,
- IsSuccess: sendSuccess,
- }
- dispatcher.Append(traceCtx)
- return err
- }
-}
diff --git a/producer/option.go b/producer/option.go
deleted file mode 100644
index ae76511..0000000
--- a/producer/option.go
+++ /dev/null
@@ -1,164 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package producer
-
-import (
- "time"
-
- "github.com/apache/rocketmq-client-go/v2/internal"
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-func defaultProducerOptions() producerOptions {
- opts := producerOptions{
- ClientOptions: internal.DefaultClientOptions(),
- Selector: NewRoundRobinQueueSelector(),
- SendMsgTimeout: 3 * time.Second,
- DefaultTopicQueueNums: 4,
- CreateTopicKey: "TBW102",
- Resolver: primitive.NewHttpResolver("DEFAULT"),
- CompressMsgBodyOverHowmuch: 4096,
- CompressLevel: 5,
- }
- opts.ClientOptions.GroupName = "DEFAULT_PRODUCER"
- return opts
-}
-
-type producerOptions struct {
- internal.ClientOptions
- Selector QueueSelector
- SendMsgTimeout time.Duration
- DefaultTopicQueueNums int
- CreateTopicKey string // "TBW102" Will be created at broker when isAutoCreateTopicEnable. when topic is not created,
- // and broker open isAutoCreateTopicEnable, topic will use "TBW102" config to create topic
- Resolver primitive.NsResolver
- CompressMsgBodyOverHowmuch int
- CompressLevel int
-}
-
-type Option func(*producerOptions)
-
-// WithGroupName set group name address
-func WithGroupName(group string) Option {
- return func(opts *producerOptions) {
- if group == "" {
- return
- }
- opts.GroupName = group
- }
-}
-
-func WithInstanceName(name string) Option {
- return func(opts *producerOptions) {
- opts.InstanceName = name
- }
-}
-
-// WithNamespace set the namespace of producer
-func WithNamespace(namespace string) Option {
- return func(opts *producerOptions) {
- opts.Namespace = namespace
- }
-}
-
-func WithSendMsgTimeout(duration time.Duration) Option {
- return func(opts *producerOptions) {
- opts.SendMsgTimeout = duration
- }
-}
-
-func WithVIPChannel(enable bool) Option {
- return func(opts *producerOptions) {
- opts.VIPChannelEnabled = enable
- }
-}
-
-// WithRetry return a Option that specifies the retry times when send failed.
-// TODO: use retry middleware instead
-func WithRetry(retries int) Option {
- return func(opts *producerOptions) {
- opts.RetryTimes = retries
- }
-}
-
-func WithInterceptor(f ...primitive.Interceptor) Option {
- return func(opts *producerOptions) {
- opts.Interceptors = append(opts.Interceptors, f...)
- }
-}
-
-func WithQueueSelector(s QueueSelector) Option {
- return func(options *producerOptions) {
- options.Selector = s
- }
-}
-
-func WithCredentials(c primitive.Credentials) Option {
- return func(options *producerOptions) {
- options.ClientOptions.Credentials = c
- }
-}
-
-func WithDefaultTopicQueueNums(queueNum int) Option {
- return func(options *producerOptions) {
- options.DefaultTopicQueueNums = queueNum
- }
-}
-
-func WithCreateTopicKey(topic string) Option {
- return func(options *producerOptions) {
- options.CreateTopicKey = topic
- }
-}
-
-// WithNsResolver set nameserver resolver to fetch nameserver addr
-func WithNsResolver(resolver primitive.NsResolver) Option {
- return func(options *producerOptions) {
- options.Resolver = resolver
- }
-}
-
-// WithNameServer set NameServer address, only support one NameServer cluster in alpha2
-func WithNameServer(nameServers primitive.NamesrvAddr) Option {
- return func(options *producerOptions) {
- options.Resolver = primitive.NewPassthroughResolver(nameServers)
- }
-}
-
-// WithNameServerDomain set NameServer domain
-func WithNameServerDomain(nameServerUrl string) Option {
- return func(opts *producerOptions) {
- opts.Resolver = primitive.NewHttpResolver("DEFAULT", nameServerUrl)
- }
-}
-
-// WithCompressMsgBodyOverHowmuch set compression threshold
-func WithCompressMsgBodyOverHowmuch(threshold int) Option {
- return func(opts *producerOptions) {
- opts.CompressMsgBodyOverHowmuch = threshold
- }
-}
-
-// WithCompressLevel set compress level (0~9)
-// 0 stands for best speed
-// 9 stands for best compression ratio
-func WithCompressLevel(level int) Option {
- return func(opts *producerOptions) {
- opts.CompressLevel = level
- }
-}
diff --git a/producer/producer.go b/producer/producer.go
deleted file mode 100644
index 3c875c6..0000000
--- a/producer/producer.go
+++ /dev/null
@@ -1,602 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package producer
-
-import (
- "bytes"
- "context"
- "fmt"
- errors2 "github.com/apache/rocketmq-client-go/v2/errors"
- "strconv"
- "sync"
- "sync/atomic"
- "time"
-
- "github.com/pkg/errors"
-
- "github.com/apache/rocketmq-client-go/v2/internal"
- "github.com/apache/rocketmq-client-go/v2/internal/remote"
- "github.com/apache/rocketmq-client-go/v2/internal/utils"
- "github.com/apache/rocketmq-client-go/v2/primitive"
- "github.com/apache/rocketmq-client-go/v2/rlog"
-)
-
-type defaultProducer struct {
- group string
- client internal.RMQClient
- state int32
- options producerOptions
- publishInfo sync.Map
- callbackCh chan interface{}
-
- interceptor primitive.Interceptor
-}
-
-func NewDefaultProducer(opts ...Option) (*defaultProducer, error) {
- defaultOpts := defaultProducerOptions()
- for _, apply := range opts {
- apply(&defaultOpts)
- }
- srvs, err := internal.NewNamesrv(defaultOpts.Resolver)
- if err != nil {
- return nil, errors.Wrap(err, "new Namesrv failed.")
- }
- if !defaultOpts.Credentials.IsEmpty() {
- srvs.SetCredentials(defaultOpts.Credentials)
- }
- defaultOpts.Namesrv = srvs
-
- producer := &defaultProducer{
- group: defaultOpts.GroupName,
- callbackCh: make(chan interface{}),
- options: defaultOpts,
- }
- producer.client = internal.GetOrNewRocketMQClient(defaultOpts.ClientOptions, producer.callbackCh)
- if producer.client == nil {
- return nil, fmt.Errorf("GetOrNewRocketMQClient faild")
- }
- defaultOpts.Namesrv = producer.client.GetNameSrv()
-
- producer.interceptor = primitive.ChainInterceptors(producer.options.Interceptors...)
-
- return producer, nil
-}
-
-func (p *defaultProducer) Start() error {
- if p == nil || p.client == nil {
- return fmt.Errorf("client instance is nil, can not start producer")
- }
- atomic.StoreInt32(&p.state, int32(internal.StateRunning))
- err := p.client.RegisterProducer(p.group, p)
- if err != nil {
- return err
- }
- p.client.Start()
- return nil
-}
-
-func (p *defaultProducer) Shutdown() error {
- atomic.StoreInt32(&p.state, int32(internal.StateShutdown))
- p.client.UnregisterProducer(p.group)
- p.client.Shutdown()
- return nil
-}
-
-func (p *defaultProducer) checkMsg(msgs ...*primitive.Message) error {
- if atomic.LoadInt32(&p.state) != int32(internal.StateRunning) {
- return errors2.ErrNotRunning
- }
-
- if len(msgs) == 0 {
- return errors2.ErrMessageEmpty
- }
-
- if len(msgs[0].Topic) == 0 {
- return errors2.ErrTopicEmpty
- }
-
- topic := msgs[0].Topic
- for _, msg := range msgs {
- if msg.Topic != topic {
- return errors2.ErrMultipleTopics
- }
- }
-
- return nil
-}
-
-func (p *defaultProducer) encodeBatch(msgs ...*primitive.Message) *primitive.Message {
- if len(msgs) == 1 {
- return msgs[0]
- }
-
- // encode batch
- batch := new(primitive.Message)
- batch.Topic = msgs[0].Topic
- batch.Queue = msgs[0].Queue
- if len(msgs) > 1 {
- batch.Body = MarshalMessageBatch(msgs...)
- batch.Batch = true
- } else {
- batch.Body = msgs[0].Body
- batch.Flag = msgs[0].Flag
- batch.WithProperties(msgs[0].GetProperties())
- batch.TransactionId = msgs[0].TransactionId
- }
- return batch
-}
-
-func MarshalMessageBatch(msgs ...*primitive.Message) []byte {
- buffer := bytes.NewBufferString("")
- for _, msg := range msgs {
- data := msg.Marshal()
- buffer.Write(data)
- }
- return buffer.Bytes()
-}
-
-func (p *defaultProducer) SendSync(ctx context.Context, msgs ...*primitive.Message) (*primitive.SendResult, error) {
- if err := p.checkMsg(msgs...); err != nil {
- return nil, err
- }
-
- p.messagesWithNamespace(msgs...)
-
- msg := p.encodeBatch(msgs...)
-
- resp := primitive.NewSendResult()
- if p.interceptor != nil {
- primitive.WithMethod(ctx, primitive.SendSync)
- producerCtx := &primitive.ProducerCtx{
- ProducerGroup: p.group,
- CommunicationMode: primitive.SendSync,
- BornHost: utils.LocalIP,
- Message: *msg,
- SendResult: resp,
- }
- ctx = primitive.WithProducerCtx(ctx, producerCtx)
-
- err := p.interceptor(ctx, msg, resp, func(ctx context.Context, req, reply interface{}) error {
- var err error
- realReq := req.(*primitive.Message)
- realReply := reply.(*primitive.SendResult)
- err = p.sendSync(ctx, realReq, realReply)
- return err
- })
- return resp, err
- }
-
- err := p.sendSync(ctx, msg, resp)
- return resp, err
-}
-
-func (p *defaultProducer) sendSync(ctx context.Context, msg *primitive.Message, resp *primitive.SendResult) error {
-
- retryTime := 1 + p.options.RetryTimes
-
- var (
- err error
- )
-
- var producerCtx *primitive.ProducerCtx
- for retryCount := 0; retryCount < retryTime; retryCount++ {
- mq := p.selectMessageQueue(msg)
- if mq == nil {
- err = fmt.Errorf("the topic=%s route info not found", msg.Topic)
- continue
- }
-
- addr := p.client.GetNameSrv().FindBrokerAddrByName(mq.BrokerName)
- if addr == "" {
- return fmt.Errorf("topic=%s route info not found", mq.Topic)
- }
-
- if p.interceptor != nil {
- producerCtx = primitive.GetProducerCtx(ctx)
- producerCtx.BrokerAddr = addr
- producerCtx.MQ = *mq
- }
-
- res, _err := p.client.InvokeSync(ctx, addr, p.buildSendRequest(mq, msg), 3*time.Second)
- if _err != nil {
- err = _err
- continue
- }
- return p.client.ProcessSendResponse(mq.BrokerName, res, resp, msg)
- }
- return err
-}
-
-func (p *defaultProducer) SendAsync(ctx context.Context, f func(context.Context, *primitive.SendResult, error), msgs ...*primitive.Message) error {
- if err := p.checkMsg(msgs...); err != nil {
- return err
- }
-
- p.messagesWithNamespace(msgs...)
-
- msg := p.encodeBatch(msgs...)
-
- if p.interceptor != nil {
- primitive.WithMethod(ctx, primitive.SendAsync)
-
- return p.interceptor(ctx, msg, nil, func(ctx context.Context, req, reply interface{}) error {
- return p.sendAsync(ctx, msg, f)
- })
- }
- return p.sendAsync(ctx, msg, f)
-}
-
-func (p *defaultProducer) sendAsync(ctx context.Context, msg *primitive.Message, h func(context.Context, *primitive.SendResult, error)) error {
-
- mq := p.selectMessageQueue(msg)
- if mq == nil {
- return errors.Errorf("the topic=%s route info not found", msg.Topic)
- }
-
- addr := p.client.GetNameSrv().FindBrokerAddrByName(mq.BrokerName)
- if addr == "" {
- return errors.Errorf("topic=%s route info not found", mq.Topic)
- }
-
- ctx, _ = context.WithTimeout(ctx, 3*time.Second)
- return p.client.InvokeAsync(ctx, addr, p.buildSendRequest(mq, msg), func(command *remote.RemotingCommand, err error) {
- resp := primitive.NewSendResult()
- if err != nil {
- h(ctx, nil, err)
- } else {
- p.client.ProcessSendResponse(mq.BrokerName, command, resp, msg)
- h(ctx, resp, nil)
- }
- })
-}
-
-func (p *defaultProducer) SendOneWay(ctx context.Context, msgs ...*primitive.Message) error {
- if err := p.checkMsg(msgs...); err != nil {
- return err
- }
-
- p.messagesWithNamespace(msgs...)
-
- msg := p.encodeBatch(msgs...)
-
- if p.interceptor != nil {
- primitive.WithMethod(ctx, primitive.SendOneway)
- return p.interceptor(ctx, msg, nil, func(ctx context.Context, req, reply interface{}) error {
- return p.SendOneWay(ctx, msg)
- })
- }
-
- return p.sendOneWay(ctx, msg)
-}
-
-func (p *defaultProducer) sendOneWay(ctx context.Context, msg *primitive.Message) error {
- retryTime := 1 + p.options.RetryTimes
-
- var err error
- for retryCount := 0; retryCount < retryTime; retryCount++ {
- mq := p.selectMessageQueue(msg)
- if mq == nil {
- err = fmt.Errorf("the topic=%s route info not found", msg.Topic)
- continue
- }
-
- addr := p.client.GetNameSrv().FindBrokerAddrByName(mq.BrokerName)
- if addr == "" {
- return fmt.Errorf("topic=%s route info not found", mq.Topic)
- }
-
- _err := p.client.InvokeOneWay(ctx, addr, p.buildSendRequest(mq, msg), 3*time.Second)
- if _err != nil {
- err = _err
- continue
- }
- return nil
- }
- return err
-}
-
-func (p *defaultProducer) messagesWithNamespace(msgs ...*primitive.Message) {
-
- if p.options.Namespace == "" {
- return
- }
-
- for _, msg := range msgs {
- msg.Topic = p.options.Namespace + "%" + msg.Topic
- }
-}
-
-func (p *defaultProducer) tryCompressMsg(msg *primitive.Message) bool {
- if msg.Compress {
- return true
- }
- if msg.Batch {
- return false
- }
- if len(msg.Body) < p.options.CompressMsgBodyOverHowmuch {
- return false
- }
- compressedBody, e := utils.Compress(msg.Body, p.options.CompressLevel)
- if e != nil {
- return false
- }
- msg.Body = compressedBody
- msg.Compress = true
- return true
-}
-
-func (p *defaultProducer) buildSendRequest(mq *primitive.MessageQueue,
- msg *primitive.Message) *remote.RemotingCommand {
- if !msg.Batch && msg.GetProperty(primitive.PropertyUniqueClientMessageIdKeyIndex) == "" {
- msg.WithProperty(primitive.PropertyUniqueClientMessageIdKeyIndex, primitive.CreateUniqID())
- }
- sysFlag := 0
- if p.tryCompressMsg(msg) {
- sysFlag = primitive.SetCompressedFlag(sysFlag)
- }
- v := msg.GetProperty(primitive.PropertyTransactionPrepared)
- if v != "" {
- tranMsg, err := strconv.ParseBool(v)
- if err == nil && tranMsg {
- sysFlag |= primitive.TransactionPreparedType
- }
- }
-
- req := &internal.SendMessageRequestHeader{
- ProducerGroup: p.group,
- Topic: mq.Topic,
- QueueId: mq.QueueId,
- SysFlag: sysFlag,
- BornTimestamp: time.Now().UnixNano() / int64(time.Millisecond),
- Flag: msg.Flag,
- Properties: msg.MarshallProperties(),
- ReconsumeTimes: 0,
- UnitMode: p.options.UnitMode,
- Batch: msg.Batch,
- }
- cmd := internal.ReqSendMessage
- if msg.Batch {
- cmd = internal.ReqSendBatchMessage
- reqv2 := &internal.SendMessageRequestV2Header{SendMessageRequestHeader: req}
- return remote.NewRemotingCommand(cmd, reqv2, msg.Body)
- }
-
- return remote.NewRemotingCommand(cmd, req, msg.Body)
-}
-
-func (p *defaultProducer) selectMessageQueue(msg *primitive.Message) *primitive.MessageQueue {
- topic := msg.Topic
-
- v, exist := p.publishInfo.Load(topic)
- if !exist {
- data, changed, err := p.client.GetNameSrv().UpdateTopicRouteInfo(topic)
- if err != nil && primitive.IsRemotingErr(err) {
- return nil
- }
- p.client.UpdatePublishInfo(topic, data, changed)
- v, exist = p.publishInfo.Load(topic)
- }
-
- if !exist {
- data, changed, _ := p.client.GetNameSrv().UpdateTopicRouteInfoWithDefault(topic, p.options.CreateTopicKey, p.options.DefaultTopicQueueNums)
- p.client.UpdatePublishInfo(topic, data, changed)
- v, exist = p.publishInfo.Load(topic)
- }
-
- if !exist {
- return nil
- }
-
- result := v.(*internal.TopicPublishInfo)
- if result == nil || !result.HaveTopicRouterInfo {
- return nil
- }
-
- if len(result.MqList) <= 0 {
- rlog.Error("can not find proper message queue", nil)
- return nil
- }
-
- return p.options.Selector.Select(msg, result.MqList)
-}
-
-func (p *defaultProducer) PublishTopicList() []string {
- topics := make([]string, 0)
- p.publishInfo.Range(func(key, value interface{}) bool {
- topics = append(topics, key.(string))
- return true
- })
- return topics
-}
-
-func (p *defaultProducer) UpdateTopicPublishInfo(topic string, info *internal.TopicPublishInfo) {
- if topic == "" || info == nil {
- return
- }
- p.publishInfo.Store(topic, info)
-}
-
-func (p *defaultProducer) IsPublishTopicNeedUpdate(topic string) bool {
- v, exist := p.publishInfo.Load(topic)
- if !exist {
- return true
- }
- info := v.(*internal.TopicPublishInfo)
- return info.MqList == nil || len(info.MqList) == 0
-}
-
-func (p *defaultProducer) IsUnitMode() bool {
- return false
-}
-
-type transactionProducer struct {
- producer *defaultProducer
- listener primitive.TransactionListener
-}
-
-// TODO: checkLocalTransaction
-func NewTransactionProducer(listener primitive.TransactionListener, opts ...Option) (*transactionProducer, error) {
- producer, err := NewDefaultProducer(opts...)
- if err != nil {
- return nil, errors.Wrap(err, "NewDefaultProducer failed.")
- }
- return &transactionProducer{
- producer: producer,
- listener: listener,
- }, nil
-}
-
-func (tp *transactionProducer) Start() error {
- go primitive.WithRecover(func() {
- tp.checkTransactionState()
- })
- return tp.producer.Start()
-}
-func (tp *transactionProducer) Shutdown() error {
- return tp.producer.Shutdown()
-}
-
-// TODO: check addr
-func (tp *transactionProducer) checkTransactionState() {
- for ch := range tp.producer.callbackCh {
- switch callback := ch.(type) {
- case *internal.CheckTransactionStateCallback:
- localTransactionState := tp.listener.CheckLocalTransaction(callback.Msg)
- uniqueKey := callback.Msg.GetProperty(primitive.PropertyUniqueClientMessageIdKeyIndex)
- if uniqueKey == "" {
- uniqueKey = callback.Msg.MsgId
- }
- transactionId := callback.Msg.GetProperty(primitive.PropertyTransactionID)
- if transactionId == "" {
- transactionId = callback.Header.TransactionId
- }
- if transactionId == "" {
- transactionId = callback.Msg.TransactionId
- }
- header := &internal.EndTransactionRequestHeader{
- CommitLogOffset: callback.Header.CommitLogOffset,
- ProducerGroup: tp.producer.group,
- TranStateTableOffset: callback.Header.TranStateTableOffset,
- FromTransactionCheck: true,
- MsgID: uniqueKey,
- TransactionId: transactionId,
- CommitOrRollback: tp.transactionState(localTransactionState),
- }
-
- req := remote.NewRemotingCommand(internal.ReqENDTransaction, header, nil)
- req.Remark = tp.errRemark(nil)
-
- err := tp.producer.client.InvokeOneWay(context.Background(), callback.Addr.String(), req,
- tp.producer.options.SendMsgTimeout)
- if err != nil {
- rlog.Error("send ReqENDTransaction to broker error", map[string]interface{}{
- "callback": callback.Addr.String(),
- "request": req.String(),
- rlog.LogKeyUnderlayError: err,
- })
- }
- default:
- rlog.Error(fmt.Sprintf("unknown type %v", ch), nil)
- }
- }
-}
-
-func (tp *transactionProducer) SendMessageInTransaction(ctx context.Context, msg *primitive.Message) (*primitive.TransactionSendResult, error) {
- msg.WithProperty(primitive.PropertyTransactionPrepared, "true")
- msg.WithProperty(primitive.PropertyProducerGroup, tp.producer.options.GroupName)
-
- rsp, err := tp.producer.SendSync(ctx, msg)
- if err != nil {
- return nil, err
- }
- localTransactionState := primitive.UnknowState
- switch rsp.Status {
- case primitive.SendOK:
- if len(rsp.TransactionID) > 0 {
- msg.WithProperty("__transactionId__", rsp.TransactionID)
- }
- transactionId := msg.GetProperty(primitive.PropertyUniqueClientMessageIdKeyIndex)
- if len(transactionId) > 0 {
- msg.TransactionId = transactionId
- }
- localTransactionState = tp.listener.ExecuteLocalTransaction(msg)
- if localTransactionState != primitive.CommitMessageState {
- rlog.Error("executeLocalTransaction but state unexpected", map[string]interface{}{
- "localState": localTransactionState,
- "message": msg,
- })
- }
-
- case primitive.SendFlushDiskTimeout, primitive.SendFlushSlaveTimeout, primitive.SendSlaveNotAvailable:
- localTransactionState = primitive.RollbackMessageState
- default:
- }
-
- tp.endTransaction(*rsp, err, localTransactionState)
-
- transactionSendResult := &primitive.TransactionSendResult{
- SendResult: rsp,
- State: localTransactionState,
- }
-
- return transactionSendResult, nil
-}
-
-func (tp *transactionProducer) endTransaction(result primitive.SendResult, err error, state primitive.LocalTransactionState) error {
- var msgID *primitive.MessageID
- if len(result.OffsetMsgID) > 0 {
- msgID, _ = primitive.UnmarshalMsgID([]byte(result.OffsetMsgID))
- } else {
- msgID, _ = primitive.UnmarshalMsgID([]byte(result.MsgID))
- }
- // 估计没有反序列化回来
- brokerAddr := tp.producer.client.GetNameSrv().FindBrokerAddrByName(result.MessageQueue.BrokerName)
- requestHeader := &internal.EndTransactionRequestHeader{
- TransactionId: result.TransactionID,
- CommitLogOffset: msgID.Offset,
- ProducerGroup: tp.producer.group,
- TranStateTableOffset: result.QueueOffset,
- MsgID: result.MsgID,
- CommitOrRollback: tp.transactionState(state),
- }
-
- req := remote.NewRemotingCommand(internal.ReqENDTransaction, requestHeader, nil)
- req.Remark = tp.errRemark(err)
-
- return tp.producer.client.InvokeOneWay(context.Background(), brokerAddr, req, tp.producer.options.SendMsgTimeout)
-}
-
-func (tp *transactionProducer) errRemark(err error) string {
- if err != nil {
- return "executeLocalTransactionBranch exception: " + err.Error()
- }
- return ""
-}
-
-func (tp *transactionProducer) transactionState(state primitive.LocalTransactionState) int {
- switch state {
- case primitive.CommitMessageState:
- return primitive.TransactionCommitType
- case primitive.RollbackMessageState:
- return primitive.TransactionRollbackType
- case primitive.UnknowState:
- return primitive.TransactionNotType
- default:
- return primitive.TransactionNotType
- }
-}
diff --git a/producer/producer_test.go b/producer/producer_test.go
deleted file mode 100644
index e1d72dd..0000000
--- a/producer/producer_test.go
+++ /dev/null
@@ -1,358 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package producer
-
-import (
- "context"
- "github.com/apache/rocketmq-client-go/v2/errors"
- "testing"
-
- "github.com/golang/mock/gomock"
- "github.com/stretchr/testify/assert"
-
- "github.com/apache/rocketmq-client-go/v2/internal"
- "github.com/apache/rocketmq-client-go/v2/internal/remote"
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-const (
- topic = "TopicTest"
- namespaceTopic = "Test%TopicTest"
-)
-
-func TestShutdown(t *testing.T) {
- p, _ := NewDefaultProducer(
- WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- WithRetry(2),
- WithQueueSelector(NewManualQueueSelector()),
- )
-
- ctrl := gomock.NewController(t)
- defer ctrl.Finish()
- client := internal.NewMockRMQClient(ctrl)
- p.client = client
-
- client.EXPECT().RegisterProducer(gomock.Any(), gomock.Any()).Return(nil)
- client.EXPECT().Start().Return()
- err := p.Start()
- assert.Nil(t, err)
-
- client.EXPECT().Shutdown().Return()
- client.EXPECT().UnregisterProducer(gomock.Any()).Return()
- err = p.Shutdown()
- assert.Nil(t, err)
-
- ctx := context.Background()
- msg := new(primitive.Message)
-
- r, err := p.SendSync(ctx, msg)
- assert.Equal(t, errors.ErrNotRunning, err)
- assert.Nil(t, r)
-
- err = p.SendOneWay(ctx, msg)
- assert.Equal(t, errors.ErrNotRunning, err)
-
- f := func(context.Context, *primitive.SendResult, error) {
- assert.False(t, true, "should not come in")
- }
- err = p.SendAsync(ctx, f, msg)
- assert.Equal(t, errors.ErrNotRunning, err)
-}
-
-func mockB4Send(p *defaultProducer) {
- p.publishInfo.Store(topic, &internal.TopicPublishInfo{
- HaveTopicRouterInfo: true,
- MqList: []*primitive.MessageQueue{
- {
- Topic: topic,
- BrokerName: "aa",
- QueueId: 0,
- },
- },
- })
- p.publishInfo.Store(namespaceTopic, &internal.TopicPublishInfo{
- HaveTopicRouterInfo: true,
- MqList: []*primitive.MessageQueue{
- {
- Topic: namespaceTopic,
- BrokerName: "aa",
- QueueId: 0,
- },
- },
- })
- p.options.Namesrv.AddBroker(&internal.TopicRouteData{
- BrokerDataList: []*internal.BrokerData{
- {
- Cluster: "cluster",
- BrokerName: "aa",
- BrokerAddresses: map[int64]string{
- 0: "1",
- },
- },
- },
- })
-}
-
-func TestSync(t *testing.T) {
- p, _ := NewDefaultProducer(
- WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- WithRetry(2),
- WithQueueSelector(NewManualQueueSelector()),
- )
-
- ctrl := gomock.NewController(t)
- defer ctrl.Finish()
- namesrvCli := internal.NewMockNamesrvs(ctrl)
- client := internal.NewMockRMQClient(ctrl)
- p.client = client
- client.SetNameSrv(namesrvCli)
- namesrvCli.EXPECT().FindBrokerAddrByName(gomock.Any()).Return("a")
-
- client.EXPECT().RegisterProducer(gomock.Any(), gomock.Any()).Return(nil)
- client.EXPECT().Start().Return()
- err := p.Start()
- assert.Nil(t, err)
-
- ctx := context.Background()
- msg := &primitive.Message{
- Topic: topic,
- Body: []byte("this is a message body"),
- Queue: &primitive.MessageQueue{
- Topic: topic,
- BrokerName: "aa",
- QueueId: 0,
- },
- }
- msg.WithProperty("key", "value")
-
- expectedResp := &primitive.SendResult{
- Status: primitive.SendOK,
- MsgID: "111",
- QueueOffset: 0,
- OffsetMsgID: "0",
- }
-
- mockB4Send(p)
-
- client.EXPECT().InvokeSync(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil)
- client.EXPECT().ProcessSendResponse(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Do(
- func(brokerName string, cmd *remote.RemotingCommand, resp *primitive.SendResult, msgs ...*primitive.Message) {
- resp.Status = expectedResp.Status
- resp.MsgID = expectedResp.MsgID
- resp.QueueOffset = expectedResp.QueueOffset
- resp.OffsetMsgID = expectedResp.OffsetMsgID
- })
- resp, err := p.SendSync(ctx, msg)
- assert.Nil(t, err)
- assert.Equal(t, expectedResp, resp)
-}
-
-func TestASync(t *testing.T) {
- p, _ := NewDefaultProducer(
- WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- WithRetry(2),
- WithQueueSelector(NewManualQueueSelector()),
- )
-
- ctrl := gomock.NewController(t)
- defer ctrl.Finish()
- namesrvCli := internal.NewMockNamesrvs(ctrl)
- client := internal.NewMockRMQClient(ctrl)
- p.client = client
- client.SetNameSrv(namesrvCli)
- namesrvCli.EXPECT().FindBrokerAddrByName(gomock.Any()).Return("a")
-
- client.EXPECT().RegisterProducer(gomock.Any(), gomock.Any()).Return(nil)
- client.EXPECT().Start().Return()
- err := p.Start()
- assert.Nil(t, err)
-
- ctx := context.Background()
- msg := &primitive.Message{
- Topic: topic,
- Body: []byte("this is a message body"),
- Queue: &primitive.MessageQueue{
- Topic: topic,
- BrokerName: "aa",
- QueueId: 0,
- },
- }
- msg.WithProperty("key", "value")
-
- expectedResp := &primitive.SendResult{
- Status: primitive.SendOK,
- MsgID: "111",
- QueueOffset: 0,
- OffsetMsgID: "0",
- }
-
- f := func(ctx context.Context, resp *primitive.SendResult, err error) {
- assert.Nil(t, err)
- assert.Equal(t, expectedResp, resp)
- }
-
- mockB4Send(p)
-
- client.EXPECT().InvokeAsync(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(
- func(ctx context.Context, addr string, request *remote.RemotingCommand,
- f func(*remote.RemotingCommand, error)) error {
- // mock invoke callback
- f(nil, nil)
- return nil
- })
- client.EXPECT().ProcessSendResponse(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Do(
- func(brokerName string, cmd *remote.RemotingCommand, resp *primitive.SendResult, msgs ...*primitive.Message) {
- resp.Status = expectedResp.Status
- resp.MsgID = expectedResp.MsgID
- resp.QueueOffset = expectedResp.QueueOffset
- resp.OffsetMsgID = expectedResp.OffsetMsgID
- })
-
- err = p.SendAsync(ctx, f, msg)
- assert.Nil(t, err)
-}
-
-func TestOneway(t *testing.T) {
- p, _ := NewDefaultProducer(
- WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- WithRetry(2),
- WithQueueSelector(NewManualQueueSelector()),
- )
-
- ctrl := gomock.NewController(t)
- defer ctrl.Finish()
- namesrvCli := internal.NewMockNamesrvs(ctrl)
- client := internal.NewMockRMQClient(ctrl)
- p.client = client
- client.SetNameSrv(namesrvCli)
- namesrvCli.EXPECT().FindBrokerAddrByName(gomock.Any()).Return("a")
-
- client.EXPECT().RegisterProducer(gomock.Any(), gomock.Any()).Return(nil)
- client.EXPECT().Start().Return()
- err := p.Start()
- assert.Nil(t, err)
-
- ctx := context.Background()
- msg := &primitive.Message{
- Topic: topic,
- Body: []byte("this is a message body"),
- Queue: &primitive.MessageQueue{
- Topic: topic,
- BrokerName: "aa",
- QueueId: 0,
- },
- }
- msg.WithProperty("key", "value")
-
- mockB4Send(p)
-
- client.EXPECT().InvokeOneWay(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
-
- err = p.SendOneWay(ctx, msg)
- assert.Nil(t, err)
-}
-
-func TestSyncWithNamespace(t *testing.T) {
- p, _ := NewDefaultProducer(
- WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- WithRetry(2),
- WithQueueSelector(NewManualQueueSelector()),
- WithNamespace("Test"),
- )
-
- ctrl := gomock.NewController(t)
- defer ctrl.Finish()
- namesrvCli := internal.NewMockNamesrvs(ctrl)
- client := internal.NewMockRMQClient(ctrl)
- p.client = client
- client.SetNameSrv(namesrvCli)
- namesrvCli.EXPECT().FindBrokerAddrByName(gomock.Any()).Return("a")
-
- client.EXPECT().RegisterProducer(gomock.Any(), gomock.Any()).Return(nil)
- client.EXPECT().Start().Return()
- err := p.Start()
- assert.Nil(t, err)
-
- ctx := context.Background()
- msg := &primitive.Message{
- Topic: topic,
- Body: []byte("this is a message body"),
- Queue: &primitive.MessageQueue{
- Topic: namespaceTopic,
- BrokerName: "aa",
- QueueId: 0,
- },
- }
- msg.WithProperty("key", "value")
-
- expectedResp := &primitive.SendResult{
- Status: primitive.SendOK,
- MsgID: "111",
- QueueOffset: 0,
- OffsetMsgID: "0",
- }
-
- mockB4Send(p)
-
- client.EXPECT().InvokeSync(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil)
- client.EXPECT().ProcessSendResponse(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Do(
- func(brokerName string, cmd *remote.RemotingCommand, resp *primitive.SendResult, msgs ...*primitive.Message) {
- resp.Status = expectedResp.Status
- resp.MsgID = expectedResp.MsgID
- resp.QueueOffset = expectedResp.QueueOffset
- resp.OffsetMsgID = expectedResp.OffsetMsgID
- })
- resp, err := p.SendSync(ctx, msg)
- assert.Nil(t, err)
- assert.Equal(t, expectedResp, resp)
- assert.Equal(t, namespaceTopic, msg.Topic)
-}
-
-func TestBatchSendDifferentTopics(t *testing.T) {
- p, _ := NewDefaultProducer(
- WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
- WithRetry(2),
- WithQueueSelector(NewManualQueueSelector()),
- )
-
- ctrl := gomock.NewController(t)
- defer ctrl.Finish()
- client := internal.NewMockRMQClient(ctrl)
- p.client = client
-
- client.EXPECT().RegisterProducer(gomock.Any(), gomock.Any()).Return(nil)
- client.EXPECT().Start().Return()
- err := p.Start()
- assert.Nil(t, err)
-
- ctx := context.Background()
- msgToA := &primitive.Message{
- Topic: "topic-A",
- Body: []byte("this is a message body"),
- }
-
- msgToB := &primitive.Message{
- Topic: "topic-B",
- Body: []byte("this is a message body"),
- }
-
- resp, err := p.SendSync(ctx, []*primitive.Message{msgToA, msgToB}...)
- assert.Nil(t, resp)
- assert.NotNil(t, err)
- assert.Equal(t, err, errors.ErrMultipleTopics)
-}
diff --git a/producer/selector.go b/producer/selector.go
deleted file mode 100644
index 74f5bad..0000000
--- a/producer/selector.go
+++ /dev/null
@@ -1,122 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package producer
-
-import (
- "hash/fnv"
- "math/rand"
- "sync"
- "time"
-
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-type QueueSelector interface {
- Select(*primitive.Message, []*primitive.MessageQueue) *primitive.MessageQueue
-}
-
-// manualQueueSelector use the queue manually set in the provided Message's QueueID field as the queue to send.
-type manualQueueSelector struct{}
-
-func NewManualQueueSelector() QueueSelector {
- return new(manualQueueSelector)
-}
-
-func (manualQueueSelector) Select(message *primitive.Message, queues []*primitive.MessageQueue) *primitive.MessageQueue {
- return message.Queue
-}
-
-// randomQueueSelector choose a random queue each time.
-type randomQueueSelector struct {
- mux sync.Mutex
- rander *rand.Rand
-}
-
-func NewRandomQueueSelector() QueueSelector {
- s := new(randomQueueSelector)
- s.rander = rand.New(rand.NewSource(time.Now().UTC().UnixNano()))
- return s
-}
-
-func (r *randomQueueSelector) Select(message *primitive.Message, queues []*primitive.MessageQueue) *primitive.MessageQueue {
- r.mux.Lock()
- i := r.rander.Intn(len(queues))
- r.mux.Unlock()
- return queues[i]
-}
-
-// roundRobinQueueSelector choose the queue by roundRobin.
-type roundRobinQueueSelector struct {
- sync.Locker
- indexer map[string]*uint32
-}
-
-func NewRoundRobinQueueSelector() QueueSelector {
- s := &roundRobinQueueSelector{
- Locker: new(sync.Mutex),
- indexer: map[string]*uint32{},
- }
- return s
-}
-
-func (r *roundRobinQueueSelector) Select(message *primitive.Message, queues []*primitive.MessageQueue) *primitive.MessageQueue {
- t := message.Topic
- var idx *uint32
-
- r.Lock()
- idx, exist := r.indexer[t]
- if !exist {
- var v uint32 = 0
- idx = &v
- r.indexer[t] = idx
- }
- *idx++
- r.Unlock()
-
- qIndex := *idx % uint32(len(queues))
- return queues[qIndex]
-}
-
-type hashQueueSelector struct {
- random QueueSelector
-}
-
-func NewHashQueueSelector() QueueSelector {
- return &hashQueueSelector{
- random: NewRandomQueueSelector(),
- }
-}
-
-// hashQueueSelector choose the queue by hash if message having sharding key, otherwise choose queue by random instead.
-func (h *hashQueueSelector) Select(message *primitive.Message, queues []*primitive.MessageQueue) *primitive.MessageQueue {
- key := message.GetShardingKey()
- if len(key) == 0 {
- return h.random.Select(message, queues)
- }
-
- hasher := fnv.New32a()
- _, err := hasher.Write([]byte(key))
- if err != nil {
- return nil
- }
- queueId := int(hasher.Sum32()) % len(queues)
- if queueId < 0 {
- queueId = -queueId
- }
- return queues[queueId]
-}
diff --git a/producer/selector_test.go b/producer/selector_test.go
deleted file mode 100644
index 72dc469..0000000
--- a/producer/selector_test.go
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package producer
-
-import (
- "testing"
-
- "github.com/stretchr/testify/assert"
-
- "github.com/apache/rocketmq-client-go/v2/primitive"
-)
-
-func TestRoundRobin(t *testing.T) {
- queues := make([]*primitive.MessageQueue, 10)
- for i := 0; i < 10; i++ {
- queues = append(queues, &primitive.MessageQueue{
- QueueId: i,
- })
- }
- s := NewRoundRobinQueueSelector()
-
- m := &primitive.Message{
- Topic: "test",
- }
- mrr := &primitive.Message{
- Topic: "rr",
- }
- for i := 0; i < 100; i++ {
- q := s.Select(m, queues)
- expected := (i + 1) % len(queues)
- assert.Equal(t, queues[expected], q, "i: %d", i)
-
- qrr := s.Select(mrr, queues)
- expected = (i + 1) % len(queues)
- assert.Equal(t, queues[expected], qrr, "i: %d", i)
- }
-}
-
-func TestHashQueueSelector(t *testing.T) {
- queues := make([]*primitive.MessageQueue, 10)
- for i := 0; i < 10; i++ {
- queues = append(queues, &primitive.MessageQueue{
- QueueId: i,
- })
- }
-
- s := NewHashQueueSelector()
-
- m1 := &primitive.Message{
- Topic: "test",
- Body: []byte("one message"),
- }
- m1.WithShardingKey("same_key")
- q1 := s.Select(m1, queues)
-
- m2 := &primitive.Message{
- Topic: "test",
- Body: []byte("another message"),
- }
- m2.WithShardingKey("same_key")
- q2 := s.Select(m2, queues)
- assert.Equal(t, *q1, *q2)
-}
diff --git a/rlog/log.go b/rlog/log.go
deleted file mode 100644
index a179a40..0000000
--- a/rlog/log.go
+++ /dev/null
@@ -1,180 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package rlog
-
-import (
- "os"
- "strings"
-
- "github.com/sirupsen/logrus"
-)
-
-const (
- LogKeyProducerGroup = "producerGroup"
- LogKeyConsumerGroup = "consumerGroup"
- LogKeyTopic = "topic"
- LogKeyMessageQueue = "MessageQueue"
- LogKeyUnderlayError = "underlayError"
- LogKeyBroker = "broker"
- LogKeyValueChangedFrom = "changedFrom"
- LogKeyValueChangedTo = "changeTo"
- LogKeyPullRequest = "PullRequest"
- LogKeyTimeStamp = "timestamp"
-)
-
-type Logger interface {
- Debug(msg string, fields map[string]interface{})
- Info(msg string, fields map[string]interface{})
- Warning(msg string, fields map[string]interface{})
- Error(msg string, fields map[string]interface{})
- Fatal(msg string, fields map[string]interface{})
- Level(level string)
- OutputPath(path string) (err error)
-}
-
-func init() {
- r := &defaultLogger{
- logger: logrus.New(),
- }
- level := os.Getenv("ROCKETMQ_GO_LOG_LEVEL")
- switch strings.ToLower(level) {
- case "debug":
- r.logger.SetLevel(logrus.DebugLevel)
- case "warn":
- r.logger.SetLevel(logrus.WarnLevel)
- case "error":
- r.logger.SetLevel(logrus.ErrorLevel)
- case "fatal":
- r.logger.SetLevel(logrus.FatalLevel)
- default:
- r.logger.SetLevel(logrus.InfoLevel)
- }
- rLog = r
-}
-
-var rLog Logger
-
-type defaultLogger struct {
- logger *logrus.Logger
-}
-
-func (l *defaultLogger) Debug(msg string, fields map[string]interface{}) {
- if msg == "" && len(fields) == 0 {
- return
- }
- l.logger.WithFields(fields).Debug(msg)
-}
-
-func (l *defaultLogger) Info(msg string, fields map[string]interface{}) {
- if msg == "" && len(fields) == 0 {
- return
- }
- l.logger.WithFields(fields).Info(msg)
-}
-
-func (l *defaultLogger) Warning(msg string, fields map[string]interface{}) {
- if msg == "" && len(fields) == 0 {
- return
- }
- l.logger.WithFields(fields).Warning(msg)
-}
-
-func (l *defaultLogger) Error(msg string, fields map[string]interface{}) {
- if msg == "" && len(fields) == 0 {
- return
- }
- l.logger.WithFields(fields).Error(msg)
-}
-
-func (l *defaultLogger) Fatal(msg string, fields map[string]interface{}) {
- if msg == "" && len(fields) == 0 {
- return
- }
- l.logger.WithFields(fields).Fatal(msg)
-}
-
-func (l *defaultLogger) Level(level string) {
- switch strings.ToLower(level) {
- case "debug":
- l.logger.SetLevel(logrus.DebugLevel)
- case "warn":
- l.logger.SetLevel(logrus.WarnLevel)
- case "error":
- l.logger.SetLevel(logrus.ErrorLevel)
- case "fatal":
- l.logger.SetLevel(logrus.FatalLevel)
- default:
- l.logger.SetLevel(logrus.InfoLevel)
- }
-}
-
-func (l *defaultLogger) OutputPath(path string) (err error) {
- var file *os.File
- file, err = os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
- if err != nil {
- return
- }
-
- l.logger.Out = file
- return
-}
-
-// SetLogger use specified logger user customized, in general, we suggest user to replace the default logger with specified
-func SetLogger(logger Logger) {
- rLog = logger
-}
-func SetLogLevel(level string) {
- if level == "" {
- return
- }
- rLog.Level(level)
-}
-
-func SetOutputPath(path string) (err error) {
- if "" == path {
- return
- }
-
- return rLog.OutputPath(path)
-}
-
-func Debug(msg string, fields map[string]interface{}) {
- rLog.Debug(msg, fields)
-}
-
-func Info(msg string, fields map[string]interface{}) {
- if msg == "" && len(fields) == 0 {
- return
- }
- rLog.Info(msg, fields)
-}
-
-func Warning(msg string, fields map[string]interface{}) {
- if msg == "" && len(fields) == 0 {
- return
- }
- rLog.Warning(msg, fields)
-}
-
-func Error(msg string, fields map[string]interface{}) {
- rLog.Error(msg, fields)
-}
-
-func Fatal(msg string, fields map[string]interface{}) {
- rLog.Fatal(msg, fields)
-}
